From 4815752ccc46ff742f67b369bc75ad3efcf11204 Mon Sep 17 00:00:00 2001 From: David Steele Date: Wed, 26 Jun 2019 08:24:58 -0400 Subject: [PATCH] Add Perl interface to C storage layer. Maintaining the storage layer/drivers in two languages is burdensome. Since the integration tests require the Perl storage layer/drivers we'll need them even after the core code is migrated to C. Create an interface layer so the Perl code can be removed and new storage drivers/features introduced without adding Perl equivalents. The goal is to move the integration tests to C so this interface will eventually be removed. That being the case, the interface was designed for maximum compatibility to ease the transition. The result looks a bit hacky but we'll improve it as needed until it can be retired. --- doc/xml/release.xml | 11 + doc/xml/user-guide.xml | 15 - lib/pgBackRest/Archive/Get/File.pm | 3 - lib/pgBackRest/Archive/Info.pm | 10 +- lib/pgBackRest/Backup/Backup.pm | 22 +- lib/pgBackRest/Backup/File.pm | 71 +- lib/pgBackRest/Backup/Filter/PageChecksum.pm | 168 - lib/pgBackRest/Common/Http/Client.pm | 364 -- lib/pgBackRest/Common/Http/Common.pm | 107 - lib/pgBackRest/Common/Xml.pm | 163 - lib/pgBackRest/LibCAuto.pm | 4 +- lib/pgBackRest/Protocol/Local/Minion.pm | 1 - lib/pgBackRest/Protocol/Remote/Minion.pm | 5 +- lib/pgBackRest/Protocol/Storage/Helper.pm | 135 +- lib/pgBackRest/Protocol/Storage/Remote.pm | 56 - lib/pgBackRest/Restore.pm | 5 +- lib/pgBackRest/RestoreFile.pm | 3 - lib/pgBackRest/Stanza.pm | 6 +- lib/pgBackRest/Storage/Base.pm | 91 +- lib/pgBackRest/Storage/Cifs/Driver.pm | 55 - lib/pgBackRest/Storage/Filter/CipherBlock.pm | 177 - lib/pgBackRest/Storage/Filter/Gzip.pm | 264 - lib/pgBackRest/Storage/Filter/Sha.pm | 124 - lib/pgBackRest/Storage/Helper.pm | 38 +- lib/pgBackRest/Storage/Posix/Driver.pm | 963 ---- lib/pgBackRest/Storage/Posix/FileRead.pm | 105 - lib/pgBackRest/Storage/Posix/FileWrite.pm | 209 - lib/pgBackRest/Storage/S3/Auth.pm | 283 - lib/pgBackRest/Storage/S3/Driver.pm | 507 -- lib/pgBackRest/Storage/S3/FileRead.pm | 68 - lib/pgBackRest/Storage/S3/FileWrite.pm | 205 - lib/pgBackRest/Storage/S3/Info.pm | 48 - lib/pgBackRest/Storage/S3/Request.pm | 264 - .../Storage/{Local.pm => Storage.pm} | 716 +-- lib/pgBackRest/Storage/StorageRead.pm | 190 + lib/pgBackRest/Storage/StorageWrite.pm | 163 + libc/LibC.h | 12 + libc/LibC.xs | 8 +- libc/Makefile.PL | 16 + libc/build/lib/pgBackRestLibC/Build.pm | 13 +- libc/typemap | 5 +- libc/xs/crypto/cipherBlock.xs | 114 - libc/xs/crypto/cipherBlock.xsh | 17 - libc/xs/crypto/hash.xs | 66 - libc/xs/crypto/hash.xsh | 8 - libc/xs/postgres/pageChecksum.xs | 39 - libc/xs/storage/storage.xs | 412 +- libc/xs/storage/storage.xsh | 222 + libc/xs/storage/storageRead.xs | 159 + libc/xs/storage/storageRead.xsh | 4 + libc/xs/storage/storageWrite.xs | 146 + libc/xs/storage/storageWrite.xsh | 4 + src/Makefile.in | 3 +- src/command/restore/file.c | 3 +- src/common/error.c | 2 +- src/perl/embed.auto.c | 4595 +++-------------- src/perl/libc.auto.c | 1676 ++++-- src/storage/storage.c | 27 +- src/storage/storage.intern.h | 6 +- test/define.yaml | 85 +- test/expect/mock-all-001.log | 4 +- test/expect/mock-all-002.log | 6 +- test/expect/mock-archive-002.log | 2 + test/expect/mock-archive-stop-002.log | 2 + test/expect/mock-stanza-001.log | 2 +- test/expect/mock-stanza-002.log | 26 + test/lib/pgBackRestTest/Common/DefineTest.pm | 2 +- test/lib/pgBackRestTest/Common/FileTest.pm | 25 +- test/lib/pgBackRestTest/Common/RunTest.pm | 7 +- .../pgBackRestTest/Env/Host/HostBackupTest.pm | 108 +- .../Env/Host/HostDbSyntheticTest.pm | 12 +- .../Module/Backup/BackupFileUnitPerlTest.pm | 27 +- .../Module/Common/CommonHttpClientPerlTest.pm | 173 - .../Module/Common/CommonIniPerlTest.pm | 36 - .../Module/Manifest/ManifestAllPerlTest.pm | 31 +- .../pgBackRestTest/Module/Mock/MockAllTest.pm | 2 +- .../Module/Mock/MockArchiveStopTest.pm | 12 +- .../Module/Mock/MockStanzaTest.pm | 3 +- .../Module/Performance/PerformanceIoTest.pm | 127 - .../pgBackRestTest/Module/Real/RealAllTest.pm | 20 +- .../Module/Stanza/StanzaAllPerlTest.pm | 17 +- .../StorageFilterCipherBlockPerlTest.pm | 285 - .../Storage/StorageFilterGzipPerlTest.pm | 222 - .../Storage/StorageFilterShaPerlTest.pm | 107 - .../Module/Storage/StorageHelperPerlTest.pm | 47 +- .../Module/Storage/StorageLocalPerlTest.pm | 475 -- .../Module/Storage/StoragePerlTest.pm | 341 ++ .../Module/Storage/StoragePosixPerlTest.pm | 444 -- .../Module/Storage/StorageS3AuthPerlTest.pm | 167 - .../Module/Storage/StorageS3CertPerlTest.pm | 113 - .../Module/Storage/StorageS3PerlTest.pm | 219 - .../Storage/StorageS3RequestPerlTest.pm | 184 - test/src/module/storage/posixTest.c | 5 + 93 files changed, 4412 insertions(+), 12102 deletions(-) delete mode 100644 lib/pgBackRest/Backup/Filter/PageChecksum.pm delete mode 100644 lib/pgBackRest/Common/Http/Client.pm delete mode 100644 lib/pgBackRest/Common/Http/Common.pm delete mode 100644 lib/pgBackRest/Common/Xml.pm delete mode 100644 lib/pgBackRest/Storage/Cifs/Driver.pm delete mode 100644 lib/pgBackRest/Storage/Filter/CipherBlock.pm delete mode 100644 lib/pgBackRest/Storage/Filter/Gzip.pm delete mode 100644 lib/pgBackRest/Storage/Filter/Sha.pm delete mode 100644 lib/pgBackRest/Storage/Posix/Driver.pm delete mode 100644 lib/pgBackRest/Storage/Posix/FileRead.pm delete mode 100644 lib/pgBackRest/Storage/Posix/FileWrite.pm delete mode 100644 lib/pgBackRest/Storage/S3/Auth.pm delete mode 100644 lib/pgBackRest/Storage/S3/Driver.pm delete mode 100644 lib/pgBackRest/Storage/S3/FileRead.pm delete mode 100644 lib/pgBackRest/Storage/S3/FileWrite.pm delete mode 100644 lib/pgBackRest/Storage/S3/Info.pm delete mode 100644 lib/pgBackRest/Storage/S3/Request.pm rename lib/pgBackRest/Storage/{Local.pm => Storage.pm} (55%) create mode 100644 lib/pgBackRest/Storage/StorageRead.pm create mode 100644 lib/pgBackRest/Storage/StorageWrite.pm delete mode 100644 libc/xs/crypto/cipherBlock.xs delete mode 100644 libc/xs/crypto/cipherBlock.xsh create mode 100644 libc/xs/storage/storage.xsh create mode 100644 libc/xs/storage/storageRead.xs create mode 100644 libc/xs/storage/storageRead.xsh create mode 100644 libc/xs/storage/storageWrite.xs create mode 100644 libc/xs/storage/storageWrite.xsh delete mode 100644 test/lib/pgBackRestTest/Module/Common/CommonHttpClientPerlTest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Performance/PerformanceIoTest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockPerlTest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipPerlTest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Storage/StorageFilterShaPerlTest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Storage/StorageLocalPerlTest.pm create mode 100644 test/lib/pgBackRestTest/Module/Storage/StoragePerlTest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Storage/StoragePosixPerlTest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Storage/StorageS3AuthPerlTest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Storage/StorageS3CertPerlTest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Storage/StorageS3PerlTest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Storage/StorageS3RequestPerlTest.pm diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 7de847581..ab5748547 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -13,6 +13,17 @@ + + + + + + + +

Add Perl interface to C storage layer.

+ + + diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index 67b6cc4b3..54b4addbf 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -584,20 +584,6 @@ - - Install packages required for S3-compatible object store support - - - apt-get install libio-socket-ssl-perl libxml-libxml-perl - -y 2>&1 - - - - yum install perl-XML-LibXML perl-IO-Socket-SSL - -y 2>&1 - - -

supports locating repositories in S3-compatible object stores. The bucket used to store the repository must be created in advance — will not do it automatically. The repository can be located in the bucket root (/) but it's usually best to place it in a subpath so object store logs or other data can also be stored in the bucket without conflicts.

@@ -627,7 +613,6 @@ {[s3-bucket]} {[s3-endpoint]} {[s3-region]} - /etc/pki/tls/certs/ca-bundle.crt 4 diff --git a/lib/pgBackRest/Archive/Get/File.pm b/lib/pgBackRest/Archive/Get/File.pm index e2de52414..e76314a1b 100644 --- a/lib/pgBackRest/Archive/Get/File.pm +++ b/lib/pgBackRest/Archive/Get/File.pm @@ -21,9 +21,6 @@ use pgBackRest::Common::Log; use pgBackRest::Config::Config; use pgBackRest::Protocol::Helper; use pgBackRest::Protocol::Storage::Helper; -use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::Gzip; -use pgBackRest::Storage::Filter::Sha; use pgBackRest::Storage::Helper; #################################################################################################################################### diff --git a/lib/pgBackRest/Archive/Info.pm b/lib/pgBackRest/Archive/Info.pm index 04bd7700d..fa97ebede 100644 --- a/lib/pgBackRest/Archive/Info.pm +++ b/lib/pgBackRest/Archive/Info.pm @@ -27,7 +27,6 @@ use pgBackRest::InfoCommon; use pgBackRest::Manifest; use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::Gzip; use pgBackRest::Storage::Helper; #################################################################################################################################### @@ -409,9 +408,6 @@ sub reconstruct # Get the db-system-id from the WAL file depending on the version of postgres my $iSysIdOffset = $strDbVersion >= PG_VERSION_93 ? PG_WAL_SYSTEM_ID_OFFSET_GTE_93 : PG_WAL_SYSTEM_ID_OFFSET_LT_93; - # Read first 8k of WAL segment - my $tBlock; - # Error if the file encryption setting is not valid for the repo if (!storageRepo()->encryptionValid(storageRepo()->encrypted($strArchiveFilePath))) { @@ -423,10 +419,12 @@ sub reconstruct my $oFileIo = storageRepo()->openRead( $strArchiveFilePath, {rhyFilter => $strArchiveFile =~ ('\.' . COMPRESS_EXT . '$') ? - [{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef, + [{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]}] : undef, strCipherPass => $self->cipherPassSub()}); + $oFileIo->open(); - $oFileIo->read(\$tBlock, 512, true); + my $tBlock; + $oFileIo->read(\$tBlock, 512); $oFileIo->close(); # Get the required data from the file that was pulled into scalar $tBlock diff --git a/lib/pgBackRest/Backup/Backup.pm b/lib/pgBackRest/Backup/Backup.pm index dad38579b..d4ebac46c 100644 --- a/lib/pgBackRest/Backup/Backup.pm +++ b/lib/pgBackRest/Backup/Backup.pm @@ -30,8 +30,6 @@ use pgBackRest::Protocol::Helper; use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Common::Io::Handle; use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::Gzip; -use pgBackRest::Storage::Filter::Sha; use pgBackRest::Storage::Helper; use pgBackRest::Version; @@ -229,7 +227,7 @@ sub resumeClean if ($cType eq 'd') { logDebugMisc($strOperation, "remove path ${strName}"); - $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strName}", {bRecurse => true}); + $oStorageRepo->pathRemove(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strName}", {bRecurse => true}); } # Else add the file/link to be deleted later else @@ -323,7 +321,7 @@ sub processManifest storageRepo()->pathCreate(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strPath}", {bIgnoreExists => true}); } - if (storageRepo()->driver()->capability(STORAGE_CAPABILITY_LINK)) + if (storageRepo()->capability(STORAGE_CAPABILITY_LINK)) { for my $strTarget ($oBackupManifest->keys(MANIFEST_SECTION_BACKUP_TARGET)) { @@ -745,7 +743,7 @@ sub process &log(WARN, "aborted backup ${strAbortedBackup} cannot be resumed: ${strReason}"); &log(TEST, TEST_BACKUP_NORESUME); - $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strAbortedBackup}", {bRecurse => true}); + $oStorageRepo->pathRemove(STORAGE_REPO_BACKUP . "/${strAbortedBackup}", {bRecurse => true}); undef($oAbortedManifest); } @@ -979,7 +977,9 @@ sub process # Add compression filter if ($bCompress) { - push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP}); + push( + @{$rhyFilter}, + {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, cfgOption(CFGOPT_COMPRESS_LEVEL)]}); } # If the backups are encrypted, then the passphrase for the backup set from the manifest file is required to access @@ -1064,7 +1064,7 @@ sub process $oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, $strBackupLabel); # Sync backup path if supported - if ($oStorageRepo->driver()->capability(STORAGE_CAPABILITY_PATH_SYNC)) + if ($oStorageRepo->capability(STORAGE_CAPABILITY_PATH_SYNC)) { # Sync all paths in the backup $oStorageRepo->pathSync(STORAGE_REPO_BACKUP . "/${strBackupLabel}"); @@ -1096,12 +1096,12 @@ sub process {'strCipherPass' => $strCipherPassManifest}), $oStorageRepo->openWrite( "${strHistoryPath}/${strBackupLabel}.manifest." . COMPRESS_EXT, - {rhyFilter => [{strClass => STORAGE_FILTER_GZIP}], + {rhyFilter => [{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, 9]}], bPathCreate => true, bAtomic => true, strCipherPass => defined($strCipherPassManifest) ? $strCipherPassManifest : undef})); # Sync history path if supported - if ($oStorageRepo->driver()->capability(STORAGE_CAPABILITY_PATH_SYNC)) + if ($oStorageRepo->capability(STORAGE_CAPABILITY_PATH_SYNC)) { $oStorageRepo->pathSync(STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY); $oStorageRepo->pathSync($strHistoryPath); @@ -1110,7 +1110,7 @@ sub process # Create a link to the most recent backup $oStorageRepo->remove(STORAGE_REPO_BACKUP . qw(/) . LINK_LATEST); - if (storageRepo()->driver()->capability(STORAGE_CAPABILITY_LINK)) + if (storageRepo()->capability(STORAGE_CAPABILITY_LINK)) { $oStorageRepo->linkCreate( STORAGE_REPO_BACKUP . "/${strBackupLabel}", STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST, {bRelative => true}); @@ -1120,7 +1120,7 @@ sub process $oBackupInfo->add($oBackupManifest); # Sync backup root path if supported - if ($oStorageRepo->driver()->capability(STORAGE_CAPABILITY_PATH_SYNC)) + if ($oStorageRepo->capability(STORAGE_CAPABILITY_PATH_SYNC)) { $oStorageRepo->pathSync(STORAGE_REPO_BACKUP); } diff --git a/lib/pgBackRest/Backup/File.pm b/lib/pgBackRest/Backup/File.pm index 6b02f17da..87167b3aa 100644 --- a/lib/pgBackRest/Backup/File.pm +++ b/lib/pgBackRest/Backup/File.pm @@ -12,17 +12,15 @@ use Exporter qw(import); use File::Basename qw(dirname); use Storable qw(dclone); -use pgBackRest::Backup::Filter::PageChecksum; use pgBackRest::Common::Exception; use pgBackRest::Common::Io::Handle; use pgBackRest::Common::Log; use pgBackRest::Common::String; +use pgBackRest::Config::Config; use pgBackRest::DbVersion; use pgBackRest::Manifest; use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::Gzip; -use pgBackRest::Storage::Filter::Sha; use pgBackRest::Storage::Helper; #################################################################################################################################### @@ -143,7 +141,7 @@ sub backupFile if ($bCompress) { - push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}); + push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]}); } # Get the checksum @@ -163,8 +161,8 @@ sub backupFile # Copy the file if ($bCopy) { - # Add sha filter - my $rhyFilter = [{strClass => STORAGE_FILTER_SHA}]; + # Add size and sha filters + my $rhyFilter = [{strClass => COMMON_IO_HANDLE}, {strClass => STORAGE_FILTER_SHA}]; # Add page checksum filter if ($bChecksumPage) @@ -174,29 +172,44 @@ sub backupFile push( @{$rhyFilter}, - {strClass => BACKUP_FILTER_PAGECHECKSUM, + {strClass => "pgBackRest::Backup::Filter::PageChecksum", rxyParam => [$iSegmentNo, $hExtraParam->{iWalId}, $hExtraParam->{iWalOffset}]}); }; # Add compression if ($bCompress) { - push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{iLevel => $iCompressLevel}]}); + push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, $iCompressLevel]}); + } + # Else add protocol compression if the destination is not compressed and there is no encryption + elsif (!defined($strCipherPass)) + { + push( + @{$rhyFilter}, + {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, true, cfgOption(CFGOPT_COMPRESS_LEVEL)]}); } - # Open the file + # Open the source file my $oSourceFileIo = storageDb()->openRead($strDbFile, {rhyFilter => $rhyFilter, bIgnoreMissing => $bIgnoreMissing}); - # If source file exists - if (defined($oSourceFileIo)) + # Open the destination file + $rhyFilter = undef; + + # Add protocol decompression if the destination is not compressed and there is no encryption + if (!$bCompress && !defined($strCipherPass)) { - my $oDestinationFileIo = $oStorageRepo->openWrite( - STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}", - {bPathCreate => true, bProtocolCompress => !$bCompress, strCipherPass => $strCipherPass}); + push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, true]}); + } - # Copy the file - $oStorageRepo->copy($oSourceFileIo, $oDestinationFileIo); + my $oDestinationFileIo = $oStorageRepo->openWrite( + STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}", + {bPathCreate => true, rhyFilter => $rhyFilter, strCipherPass => $strCipherPass}); + $oDestinationFileIo->{oStorageCWrite}->filterAdd(COMMON_IO_HANDLE, undef); + + # Copy the file + if ($oStorageRepo->copy($oSourceFileIo, $oDestinationFileIo)) + { # Get sha checksum and size $strCopyChecksum = $oSourceFileIo->result(STORAGE_FILTER_SHA); $lCopySize = $oSourceFileIo->result(COMMON_IO_HANDLE); @@ -208,7 +221,17 @@ sub backupFile } # Get results of page checksum validation - $rExtra = $bChecksumPage ? $oSourceFileIo->result(BACKUP_FILTER_PAGECHECKSUM) : undef; + if ($bChecksumPage) + { + my $rExtraRaw = $oSourceFileIo->result("pgBackRest::Backup::Filter::PageChecksum"); + + $rExtra = + { + bValid => $rExtraRaw->{valid} ? true : false, + bAlign => $rExtraRaw->{align} ? true : false, + iyPageError => $rExtraRaw->{error}, + }; + } } # Else if source file is missing the database removed it else @@ -223,21 +246,21 @@ sub backupFile # # If the file was checksummed then get the size in all cases since we don't already have it. if ((($iCopyResult == BACKUP_FILE_COPY || $iCopyResult == BACKUP_FILE_RECOPY) && - $oStorageRepo->driver()->capability(STORAGE_CAPABILITY_SIZE_DIFF)) || + $oStorageRepo->capability(STORAGE_CAPABILITY_SIZE_DIFF)) || $iCopyResult == BACKUP_FILE_CHECKSUM) { - $lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}"))->size(); + $lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . "/${strBackupLabel}/${strFileOp}"))->{size}; } # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'iCopyResult', value => $iCopyResult, trace => true}, - {name => 'lCopySize', value => $lCopySize, trace => true}, - {name => 'lRepoSize', value => $lRepoSize, trace => true}, - {name => 'strCopyChecksum', value => $strCopyChecksum, trace => true}, - {name => 'rExtra', value => $rExtra, trace => true}, + {name => 'iCopyResult', value => $iCopyResult}, + {name => 'lCopySize', value => $lCopySize}, + {name => 'lRepoSize', value => $lRepoSize}, + {name => 'strCopyChecksum', value => $strCopyChecksum}, + {name => 'rExtra', value => $rExtra}, ); } diff --git a/lib/pgBackRest/Backup/Filter/PageChecksum.pm b/lib/pgBackRest/Backup/Filter/PageChecksum.pm deleted file mode 100644 index 96467c55c..000000000 --- a/lib/pgBackRest/Backup/Filter/PageChecksum.pm +++ /dev/null @@ -1,168 +0,0 @@ -#################################################################################################################################### -# Backup Page Checksum Filter -#################################################################################################################################### -package pgBackRest::Backup::Filter::PageChecksum; -use parent 'pgBackRest::Common::Io::Filter'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::DbVersion qw(PG_PAGE_SIZE); -use pgBackRest::LibC qw(:checksum); - -#################################################################################################################################### -# Package name constant -#################################################################################################################################### -use constant BACKUP_FILTER_PAGECHECKSUM => __PACKAGE__; - push @EXPORT, qw(BACKUP_FILTER_PAGECHECKSUM); - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParent, - $iSegmentNo, - $iWalId, - $iWalOffset, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParent', trace => true}, - {name => 'iSegmentNo', trace => true}, - {name => 'iWalId', trace => true}, - {name => 'iWalOffset', trace => true}, - ); - - # Bless with new class - my $self = $class->SUPER::new($oParent); - bless $self, $class; - - # Set variables - $self->{iSegmentNo} = $iSegmentNo; - $self->{iWalId} = $iWalId; - $self->{iWalOffset} = $iWalOffset; - - # Create the result object - $self->{hResult}{bValid} = true; - $self->{hResult}{bAlign} = true; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# read - validate page checksums -#################################################################################################################################### -sub read -{ - my $self = shift; - my $rtBuffer = shift; - my $iSize = shift; - - # Call the io method - my $iActualSize = $self->parent()->read($rtBuffer, $iSize); - - # Validate page checksums for the read block - if ($iActualSize > 0) - { - # If the buffer is not divisible by 0 then it's not valid - if (!$self->{hResult}{bAlign} || ($iActualSize % PG_PAGE_SIZE != 0)) - { - if (!$self->{hResult}{bAlign}) - { - confess &log(ASSERT, "should not be possible to see two misaligned blocks in a row"); - } - - $self->{hResult}{bValid} = false; - $self->{hResult}{bAlign} = false; - delete($self->{hResult}{iyPageError}); - } - else - { - # Calculate offset to the first block in the buffer - my $iBlockOffset = int(($self->size() - $iActualSize) / PG_PAGE_SIZE) + ($self->{iSegmentNo} * 131072); - - if (!pageChecksumBufferTest( - $$rtBuffer, $iActualSize, $iBlockOffset, PG_PAGE_SIZE, $self->{iWalId}, - $self->{iWalOffset})) - { - $self->{hResult}{bValid} = false; - - # Now figure out exactly where the errors occurred. It would be more efficient if the checksum function returned an - # array, but we're hoping there won't be that many errors to scan so this should work fine. - for (my $iBlockNo = 0; $iBlockNo < int($iActualSize / PG_PAGE_SIZE); $iBlockNo++) - { - my $iBlockNoStart = $iBlockOffset + $iBlockNo; - - if (!pageChecksumTest( - substr($$rtBuffer, $iBlockNo * PG_PAGE_SIZE, PG_PAGE_SIZE), $iBlockNoStart, PG_PAGE_SIZE, - $self->{iWalId}, $self->{iWalOffset})) - { - my $iLastIdx = defined($self->{hResult}{iyPageError}) ? @{$self->{hResult}{iyPageError}} - 1 : 0; - my $iyLast = defined($self->{hResult}{iyPageError}) ? $self->{hResult}{iyPageError}[$iLastIdx] : undef; - - if (!defined($iyLast) || (!ref($iyLast) && $iyLast != $iBlockNoStart - 1) || - (ref($iyLast) && $iyLast->[1] != $iBlockNoStart - 1)) - { - push(@{$self->{hResult}{iyPageError}}, $iBlockNoStart); - } - elsif (!ref($iyLast)) - { - $self->{hResult}{iyPageError}[$iLastIdx] = undef; - push(@{$self->{hResult}{iyPageError}[$iLastIdx]}, $iyLast); - push(@{$self->{hResult}{iyPageError}[$iLastIdx]}, $iBlockNoStart); - } - else - { - $self->{hResult}{iyPageError}[$iLastIdx][1] = $iBlockNoStart; - } - } - } - } - } - } - - # Return the actual size read - return $iActualSize; -} - -#################################################################################################################################### -# close - close and set the result -#################################################################################################################################### -sub close -{ - my $self = shift; - - if (defined($self->{hResult})) - { - # Set result - $self->resultSet(BACKUP_FILTER_PAGECHECKSUM, $self->{hResult}); - - # Delete the sha object - undef($self->{hResult}); - - # Close io - return $self->parent()->close(); - } -} - -1; diff --git a/lib/pgBackRest/Common/Http/Client.pm b/lib/pgBackRest/Common/Http/Client.pm deleted file mode 100644 index d6a406e02..000000000 --- a/lib/pgBackRest/Common/Http/Client.pm +++ /dev/null @@ -1,364 +0,0 @@ -#################################################################################################################################### -# HTTP Client -#################################################################################################################################### -package pgBackRest::Common::Http::Client; -use parent 'pgBackRest::Common::Io::Buffered'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use IO::Socket::SSL; -use Socket qw(SOL_SOCKET SO_KEEPALIVE); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Io::Buffered; -use pgBackRest::Common::Log; -use pgBackRest::Common::String; -use pgBackRest::Common::Xml; -use pgBackRest::Common::Http::Common; - -#################################################################################################################################### -# Constants -#################################################################################################################################### -use constant HTTP_VERB_GET => 'GET'; - push @EXPORT, qw(HTTP_VERB_GET); -use constant HTTP_VERB_POST => 'POST'; - push @EXPORT, qw(HTTP_VERB_POST); -use constant HTTP_VERB_PUT => 'PUT'; - push @EXPORT, qw(HTTP_VERB_PUT); - -use constant HTTP_HEADER_CONTENT_LENGTH => 'content-length'; - push @EXPORT, qw(HTTP_HEADER_CONTENT_LENGTH); -use constant HTTP_HEADER_TRANSFER_ENCODING => 'transfer-encoding'; - push @EXPORT, qw(HTTP_HEADER_TRANSFER_ENCODING); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strHost, - $strVerb, - $iPort, - $strUri, - $hQuery, - $hRequestHeader, - $rstrRequestBody, - $bResponseBodyPrefetch, - $iProtocolTimeout, - $iTryTotal, - $lBufferMax, - $bVerifySsl, - $strCaPath, - $strCaFile, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strHost', trace => true}, - {name => 'strVerb', trace => true}, - {name => 'iPort', optional => true, default => 443, trace => true}, - {name => 'strUri', optional => true, default => qw(/), trace => true}, - {name => 'hQuery', optional => true, trace => true}, - {name => 'hRequestHeader', optional => true, trace => true}, - {name => 'rstrRequestBody', optional => true, trace => true}, - {name => 'bResponseBodyPrefetch', optional => true, default => false, trace => true}, - {name => 'iProtocolTimeout', optional => true, default => 300, trace => true}, - {name => 'iTryTotal', optional => true, default => 3, trace => true}, - {name => 'lBufferMax', optional => true, default => 32768, trace => true}, - {name => 'bVerifySsl', optional => true, default => true, trace => true}, - {name => 'strCaPath', optional => true, trace => true}, - {name => 'strCaFile', optional => true, trace => true}, - ); - - # Retry as many times as requested - my $self; - my $iTry = 1; - my $bRetry; - - do - { - # Disable logging if a failure will be retried - logDisable() if $iTry < $iTryTotal; - $bRetry = false; - - eval - { - # Connect to the server - my $oSocket; - - if (eval{require IO::Socket::IP}) - { - $oSocket = IO::Socket::IP->new(PeerHost => $strHost, PeerPort => $iPort) - or confess &log(ERROR, "unable to create socket: $@", ERROR_HOST_CONNECT); - } - else - { - require IO::Socket::INET; - - $oSocket = IO::Socket::INET->new(PeerHost => $strHost, PeerPort => $iPort) - or confess &log(ERROR, "unable to create socket: $@", ERROR_HOST_CONNECT); - } - - setsockopt($oSocket, SOL_SOCKET,SO_KEEPALIVE, 1) - or confess &log(ERROR, "unable to set socket keepalive: $@", ERROR_HOST_CONNECT); - - eval - { - IO::Socket::SSL->start_SSL( - $oSocket, SSL_verify_mode => $bVerifySsl ? SSL_VERIFY_PEER : SSL_VERIFY_NONE, SSL_ca_path => $strCaPath, - SSL_ca_file => $strCaFile); - } - or do - { - logErrorResult( - ERROR_HOST_CONNECT, coalesce(length($!) == 0 ? undef : $!, $SSL_ERROR), length($!) > 0 ? $SSL_ERROR : undef); - }; - - # Bless with new class - $self = $class->SUPER::new( - new pgBackRest::Common::Io::Handle('httpClient', $oSocket, $oSocket), $iProtocolTimeout, $lBufferMax); - bless $self, $class; - - # Store socket - $self->{oSocket} = $oSocket; - - # Generate the query string - my $strQuery = httpQuery($hQuery); - - # Construct the request headers - $self->{strRequestHeader} = "${strVerb} " . httpUriEncode($strUri, true) . "?${strQuery} HTTP/1.1" . "\r\n"; - - foreach my $strHeader (sort(keys(%{$hRequestHeader}))) - { - $self->{strRequestHeader} .= "${strHeader}: $hRequestHeader->{$strHeader}\r\n"; - } - - $self->{strRequestHeader} .= "\r\n"; - - # Write request headers - $self->write(\$self->{strRequestHeader}); - - # Write content - if (defined($rstrRequestBody)) - { - my $iTotalSize = length($$rstrRequestBody); - my $iTotalSent = 0; - - # Write the request body in buffer-sized chunks - do - { - my $strBufferWrite = substr($$rstrRequestBody, $iTotalSent, $lBufferMax); - $iTotalSent += $self->write(\$strBufferWrite); - } while ($iTotalSent < $iTotalSize); - } - - # Read response code - ($self->{strResponseProtocol}, $self->{iResponseCode}, $self->{strResponseMessage}) = - split(' ', trim($self->readLine())); - - # Read the response headers - $self->{iContentLength} = 0; - $self->{strResponseHeader} = ''; - my $strHeader = trim($self->readLine()); - - while ($strHeader ne '') - { - # Validate header - $self->{strResponseHeader} .= "${strHeader}\n"; - - my $iColonPos = index($strHeader, ':'); - - if ($iColonPos == -1) - { - confess &log(ERROR, "http header '${strHeader}' requires colon separator", ERROR_PROTOCOL); - } - - # Parse header - my $strHeaderKey = lc(substr($strHeader, 0, $iColonPos)); - my $strHeaderValue = trim(substr($strHeader, $iColonPos + 1)); - - # Store the header - $self->{hResponseHeader}{$strHeaderKey} = $strHeaderValue; - - # Process content length - if ($strHeaderKey eq HTTP_HEADER_CONTENT_LENGTH) - { - $self->{iContentLength} = $strHeaderValue + 0; - $self->{iContentRemaining} = $self->{iContentLength}; - } - # Process transfer encoding (only chunked is supported) - elsif ($strHeaderKey eq HTTP_HEADER_TRANSFER_ENCODING) - { - if ($strHeaderValue eq 'chunked') - { - $self->{iContentLength} = -1; - } - else - { - confess &log(ERROR, "invalid value '${strHeaderValue} for http header '${strHeaderKey}'", ERROR_PROTOCOL); - } - } - - # Read next header - $strHeader = trim($self->readLine()); - } - - # Prefetch response - mostly useful when the response is known to be short - if ($bResponseBodyPrefetch) - { - $self->{strResponseBody} = $self->responseBody(); - } - - # Enable logging if a failure will be retried - logEnable() if $iTry < $iTryTotal; - return 1; - } - or do - { - # Enable logging if a failure will be retried - logEnable() if $iTry < $iTryTotal; - - # If tries reaches total allowed then error - if ($iTry == $iTryTotal) - { - confess $EVAL_ERROR; - } - - # Try again - $iTry++; - $bRetry = true; - }; - } - while ($bRetry); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# read - read content from http stream -#################################################################################################################################### -sub read -{ - my $self = shift; - my $rtBuffer = shift; - my $iRequestSize = shift; - - # Make sure request size is not larger than what remains to be read - $iRequestSize = $iRequestSize < $self->{iContentRemaining} ? $iRequestSize : $self->{iContentRemaining}; - $self->{iContentRemaining} -= $iRequestSize; - - my $iActualSize = $self->SUPER::read($rtBuffer, $iRequestSize, true); - - # Set eof if there is nothing left to read - if ($self->{iContentRemaining} == 0) - { - $self->SUPER::eofSet(true); - } - - return $iActualSize; -} - -#################################################################################################################################### -# close/DESTROY - close the HTTP connection -#################################################################################################################################### -sub close -{ - my $self = shift; - - # Only close if the socket is open - if (defined($self->{oSocket})) - { - $self->{oSocket}->close(); - undef($self->{oSocket}); - } -} - -sub DESTROY {shift->close()} - -#################################################################################################################################### -# responseBody - return the entire body of the response in a buffer -#################################################################################################################################### -sub responseBody -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->responseBody' - ); - - # Return prefetched response body if it exists - return $self->{strResponseBody} if exists($self->{strResponseBody}); - - # Fetch response body if content length is not 0 - my $strResponseBody = undef; - - if ($self->{iContentLength} != 0) - { - # Transfer encoding is chunked - if ($self->{iContentLength} == -1) - { - while (1) - { - # Read chunk length - my $strChunkLength = trim($self->readLine()); - my $iChunkLength = hex($strChunkLength); - - # Exit if chunk length is 0 - last if ($iChunkLength == 0); - - # Read the chunk and consume the terminating LF - $self->SUPER::read(\$strResponseBody, $iChunkLength, true); - $self->readLine(); - }; - } - # Else content length is known - else - { - $self->SUPER::read(\$strResponseBody, $self->{iContentLength}, true); - } - - $self->close(); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rstrResponseBody', value => \$strResponseBody, trace => true} - ); -} - -#################################################################################################################################### -# Properties. -#################################################################################################################################### -sub contentLength {shift->{iContentLength}} # Content length if available (-1 means not known yet) -sub requestHeaderText {trim(shift->{strRequestHeader})} -sub responseCode {shift->{iResponseCode}} -sub responseHeader {shift->{hResponseHeader}} -sub responseHeaderText {trim(shift->{strResponseHeader})} -sub responseMessage {shift->{strResponseMessage}} -sub responseProtocol {shift->{strResponseProtocol}} - -1; diff --git a/lib/pgBackRest/Common/Http/Common.pm b/lib/pgBackRest/Common/Http/Common.pm deleted file mode 100644 index eec9694e6..000000000 --- a/lib/pgBackRest/Common/Http/Common.pm +++ /dev/null @@ -1,107 +0,0 @@ -#################################################################################################################################### -# HTTP Common -#################################################################################################################################### -package pgBackRest::Common::Http::Common; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; - -#################################################################################################################################### -# httpQuery - encode an HTTP query from a hash -#################################################################################################################################### -sub httpQuery -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $hQuery, - ) = - logDebugParam - ( - __PACKAGE__ . '::httpQuery', \@_, - {name => 'hQuery', required => false, trace => true}, - ); - - # Generate the query string - my $strQuery = ''; - - # If a hash (the normal case) - if (ref($hQuery)) - { - foreach my $strParam (sort(keys(%{$hQuery}))) - { - # Parameters may not be defined - this is OK - if (defined($hQuery->{$strParam})) - { - $strQuery .= ($strQuery eq '' ? '' : '&') . $strParam . '=' . httpUriEncode($hQuery->{$strParam}); - } - } - } - # Else query string was passed directly as a scalar - elsif (defined($hQuery)) - { - $strQuery = $hQuery; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strQuery', value => $strQuery, trace => true} - ); -} - -push @EXPORT, qw(httpQuery); - -#################################################################################################################################### -# httpUriEncode - encode query values to conform with URI specs -#################################################################################################################################### -sub httpUriEncode -{ - my $strString = shift; - my $bPath = shift; - - # Only encode if source string is defined - my $strEncodedString; - - if (defined($strString)) - { - # Iterate all characters in the string - for (my $iIndex = 0; $iIndex < length($strString); $iIndex++) - { - my $cChar = substr($strString, $iIndex, 1); - - # These characters are reproduced verbatim - if (($cChar ge 'A' && $cChar le 'Z') || ($cChar ge 'a' && $cChar le 'z') || ($cChar ge '0' && $cChar le '9') || - $cChar eq '_' || $cChar eq '-' || $cChar eq '~' || $cChar eq '.' || ($bPath && $cChar eq '/')) - { - $strEncodedString .= $cChar; - } - # Forward slash is encoded - elsif ($cChar eq '/') - { - $strEncodedString .= '%2F'; - } - # All other characters are hex-encoded - else - { - $strEncodedString .= sprintf('%%%02X', ord($cChar)); - } - } - } - - return $strEncodedString; -} - -push @EXPORT, qw(httpUriEncode); - -1; diff --git a/lib/pgBackRest/Common/Xml.pm b/lib/pgBackRest/Common/Xml.pm deleted file mode 100644 index bb0f9dd9a..000000000 --- a/lib/pgBackRest/Common/Xml.pm +++ /dev/null @@ -1,163 +0,0 @@ -#################################################################################################################################### -# XML Helper Functions -#################################################################################################################################### -package pgBackRest::Common::Xml; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use XML::LibXML; - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; - -#################################################################################################################################### -# xmlParse - parse a string into an xml document and return the root node -#################################################################################################################################### -use constant XML_HEADER => ''; - push @EXPORT, qw(XML_HEADER); - -#################################################################################################################################### -# Convert a string to xml so that it is suitable to be appended into xml -#################################################################################################################################### -sub xmlFromText -{ - my $strText = shift; - - return XML::LibXML::Text->new($strText)->toString(); -} - -push @EXPORT, qw(xmlFromText); - -#################################################################################################################################### -# xmlParse - parse a string into an xml document and return the root node -#################################################################################################################################### -sub xmlParse -{ - my $rstrXml = shift; - - my $oXml = XML::LibXML->load_xml(string => $rstrXml)->documentElement(); - - return $oXml; -} - -push @EXPORT, qw(xmlParse); - -#################################################################################################################################### -# xmlTagChildren - get all children that match the tag -#################################################################################################################################### -sub xmlTagChildren -{ - my $oXml = shift; - my $strTag = shift; - - return $oXml->getChildrenByTagName($strTag); -} - -push @EXPORT, qw(xmlTagChildren); - -#################################################################################################################################### -# xmlTagText - get the text content for a tag, error if the tag is required and does not exist -#################################################################################################################################### -sub xmlTagText -{ - my $oXml = shift; - my $strTag = shift; - my $bRequired = shift; - # my $strDefault = shift; - - # Get the tag or tags - my @oyTag = $oXml->getElementsByTagName($strTag); - - # Error if the tag does not exist and is required - if (@oyTag > 1) - { - confess &log(ERROR, @oyTag . " '${strTag}' tag(s) exist, but only one was expected", ERROR_FORMAT); - } - elsif (@oyTag == 0) - { - if (!defined($bRequired) || $bRequired) - { - confess &log(ERROR, "tag '${strTag}' does not exist", ERROR_FORMAT); - } - } - else - { - return $oyTag[0]->textContent(); - } - - return; -} - -push @EXPORT, qw(xmlTagText); - -#################################################################################################################################### -# xmlTagBool - get the boolean content for a tag, error if the tag is required and does not exist or is not boolean -#################################################################################################################################### -sub xmlTagBool -{ - my $oXml = shift; - my $strTag = shift; - my $bRequired = shift; - # my $strDefault = shift; - - # Test content for boolean value - my $strContent = xmlTagText($oXml, $strTag, $bRequired); - - if (defined($strContent)) - { - if ($strContent eq 'true') - { - return true; - } - elsif ($strContent eq 'false') - { - return false; - } - else - { - confess &log(ERROR, "invalid boolean value '${strContent}' for tag '${strTag}'", ERROR_FORMAT); - } - } - - return; -} - -push @EXPORT, qw(xmlTagBool); - -#################################################################################################################################### -# xmlTagInt - get the integer content for a tag, error if the tag is required and does not exist or is not an integer -#################################################################################################################################### -sub xmlTagInt -{ - my $oXml = shift; - my $strTag = shift; - my $bRequired = shift; - # my $strDefault = shift; - - # Test content for boolean value - my $iContent = xmlTagText($oXml, $strTag, $bRequired); - - if (defined($iContent)) - { - eval - { - $iContent = $iContent + 0; - return 1; - } - or do - { - confess &log(ERROR, "invalid integer value '${iContent}' for tag '${strTag}'", ERROR_FORMAT); - } - } - - return $iContent; -} - -push @EXPORT, qw(xmlTagInt); - -1; diff --git a/lib/pgBackRest/LibCAuto.pm b/lib/pgBackRest/LibCAuto.pm index db7a0742b..ca089ffda 100644 --- a/lib/pgBackRest/LibCAuto.pm +++ b/lib/pgBackRest/LibCAuto.pm @@ -74,8 +74,6 @@ sub libcAutoExportTag checksum => [ 'pageChecksum', - 'pageChecksumBufferTest', - 'pageChecksumTest', ], config => @@ -352,7 +350,7 @@ sub libcAutoExportTag storage => [ - 'storagePosixPathRemove', + 'storageRepoFree', ], test => diff --git a/lib/pgBackRest/Protocol/Local/Minion.pm b/lib/pgBackRest/Protocol/Local/Minion.pm index 83237451b..596336b51 100644 --- a/lib/pgBackRest/Protocol/Local/Minion.pm +++ b/lib/pgBackRest/Protocol/Local/Minion.pm @@ -11,7 +11,6 @@ use Carp qw(confess); use pgBackRest::Backup::File; use pgBackRest::Common::Log; use pgBackRest::Config::Config; -use pgBackRest::Storage::Local; use pgBackRest::Protocol::Base::Master; use pgBackRest::Protocol::Base::Minion; use pgBackRest::Protocol::Command::Minion; diff --git a/lib/pgBackRest/Protocol/Remote/Minion.pm b/lib/pgBackRest/Protocol/Remote/Minion.pm index acdc6352d..db9106bad 100644 --- a/lib/pgBackRest/Protocol/Remote/Minion.pm +++ b/lib/pgBackRest/Protocol/Remote/Minion.pm @@ -91,11 +91,12 @@ sub init my $oSourceFileIo = $oStorage->openRead(@{shift()}); # If the source file exists - if (defined($oSourceFileIo)) + if (defined($oSourceFileIo) && (!defined($oSourceFileIo->{oStorageCRead}) || $oSourceFileIo->open())) { $self->outputWrite(true); - $oStorage->copy($oSourceFileIo, new pgBackRest::Protocol::Storage::File($self, $oSourceFileIo)); + $oStorage->copy( + $oSourceFileIo, new pgBackRest::Protocol::Storage::File($self, $oSourceFileIo), {bSourceOpen => true}); return true; } diff --git a/lib/pgBackRest/Protocol/Storage/Helper.pm b/lib/pgBackRest/Protocol/Storage/Helper.pm index 288d4efd1..64c4cea62 100644 --- a/lib/pgBackRest/Protocol/Storage/Helper.pm +++ b/lib/pgBackRest/Protocol/Storage/Helper.pm @@ -13,11 +13,10 @@ use File::Basename qw(basename); use pgBackRest::Common::Log; use pgBackRest::Config::Config; +use pgBackRest::LibC qw(:storage); use pgBackRest::Protocol::Helper; use pgBackRest::Protocol::Storage::Remote; -use pgBackRest::Storage::Base; use pgBackRest::Storage::Helper; -use pgBackRest::Storage::Local; #################################################################################################################################### # Storage constants @@ -60,9 +59,8 @@ sub storageDb { if (isDbLocal({iRemoteIdx => $iRemoteIdx})) { - $hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Storage::Local( - cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iRemoteIdx)), new pgBackRest::Storage::Posix::Driver(), - {strTempExtension => STORAGE_TEMP_EXT, lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)}); + $hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Storage::Storage( + STORAGE_DB, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)}); } else { @@ -81,54 +79,6 @@ sub storageDb push @EXPORT, qw(storageDb); -#################################################################################################################################### -# storageRepoRule - rules for paths in the repository -#################################################################################################################################### -sub storageRepoRule -{ - my $strRule = shift; - my $strFile = shift; - my $strStanza = shift; - - # Result path and file - my $strResultFile; - - # Return archive path - if ($strRule eq STORAGE_REPO_ARCHIVE) - { - $strResultFile = "archive" . (defined($strStanza) ? "/${strStanza}" : ''); - - # If file is not defined nothing further to do - if (defined($strFile)) - { - my ($strArchiveId, $strWalFile) = split('/', $strFile); - - # If a WAL file (including .backup) - if (defined($strWalFile) && $strWalFile =~ /^[0-F]{24}/) - { - $strResultFile .= "/${strArchiveId}/" . substr($strWalFile, 0, 16) . "/${strWalFile}"; - } - # Else other files - else - { - $strResultFile .= "/${strFile}"; - } - } - } - # Return backup path - elsif ($strRule eq STORAGE_REPO_BACKUP) - { - $strResultFile = "backup" . (defined($strStanza) ? "/${strStanza}" : '') . (defined($strFile) ? "/${strFile}" : ''); - } - # Else error - else - { - confess &log(ASSERT, "invalid " . STORAGE_REPO . " storage rule ${strRule}"); - } - - return $strResultFile; -} - #################################################################################################################################### # storageRepo - get repository storage #################################################################################################################################### @@ -146,85 +96,18 @@ sub storageRepo {name => 'strStanza', optional => true, trace => true}, ); - if (!defined($strStanza)) - { - if (cfgOptionValid(CFGOPT_STANZA) && cfgOptionTest(CFGOPT_STANZA)) - { - $strStanza = cfgOption(CFGOPT_STANZA); - } - else - { - $strStanza = STORAGE_REPO; - } - } - # Create storage if not defined - if (!defined($hStorage->{&STORAGE_REPO}{$strStanza})) + if (!defined($hStorage->{&STORAGE_REPO})) { if (isRepoLocal()) { - # Path rules - my $hRule = - { - &STORAGE_REPO_ARCHIVE => - { - fnRule => \&storageRepoRule, - xData => $strStanza eq STORAGE_REPO ? undef : $strStanza, - }, - &STORAGE_REPO_BACKUP => - { - fnRule => \&storageRepoRule, - xData => $strStanza eq STORAGE_REPO ? undef : $strStanza, - }, - }; - - # Create the driver - my $oDriver; - - if (cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3)) - { - require pgBackRest::Storage::S3::Driver; - - $oDriver = new pgBackRest::Storage::S3::Driver( - cfgOption(CFGOPT_REPO_S3_BUCKET), cfgOption(CFGOPT_REPO_S3_ENDPOINT), cfgOption(CFGOPT_REPO_S3_REGION), - cfgOption(CFGOPT_REPO_S3_KEY), cfgOption(CFGOPT_REPO_S3_KEY_SECRET), - {strHost => cfgOption(CFGOPT_REPO_S3_HOST, false), bVerifySsl => cfgOption(CFGOPT_REPO_S3_VERIFY_TLS, false), - strCaPath => cfgOption(CFGOPT_REPO_S3_CA_PATH, false), - strCaFile => cfgOption(CFGOPT_REPO_S3_CA_FILE, false), lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE), - strSecurityToken => cfgOption(CFGOPT_REPO_S3_TOKEN, false)}); - } - elsif (cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_CIFS)) - { - require pgBackRest::Storage::Cifs::Driver; - - $oDriver = new pgBackRest::Storage::Cifs::Driver(); - } - else - { - $oDriver = new pgBackRest::Storage::Posix::Driver(); - } - - # Set the encryption for the repo - my $strCipherType; - my $strCipherPass; - - # If the encryption is not the default (none) then set the user-defined passphrase and magic based on the type - if (cfgOption(CFGOPT_REPO_CIPHER_TYPE) ne CFGOPTVAL_REPO_CIPHER_TYPE_NONE) - { - $strCipherType = cfgOption(CFGOPT_REPO_CIPHER_TYPE); - $strCipherPass = cfgOption(CFGOPT_REPO_CIPHER_PASS); - } - - # Create local storage - $hStorage->{&STORAGE_REPO}{$strStanza} = new pgBackRest::Storage::Local( - cfgOption(CFGOPT_REPO_PATH), $oDriver, - {strTempExtension => STORAGE_TEMP_EXT, hRule => $hRule, lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE), - strCipherType => $strCipherType, strCipherPassUser => $strCipherPass}); + $hStorage->{&STORAGE_REPO} = new pgBackRest::Storage::Storage( + STORAGE_REPO, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)}); } else { # Create remote storage - $hStorage->{&STORAGE_REPO}{$strStanza} = new pgBackRest::Protocol::Storage::Remote( + $hStorage->{&STORAGE_REPO} = new pgBackRest::Protocol::Storage::Remote( protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP)); } } @@ -233,7 +116,7 @@ sub storageRepo return logDebugReturn ( $strOperation, - {name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}{$strStanza}, trace => true}, + {name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}, trace => true}, ); } @@ -249,6 +132,8 @@ sub storageRepoCacheClear delete($hStorage->{&STORAGE_REPO}); + storageRepoFree(); + # Return from function and log return values if any return logDebugReturn($strOperation); } diff --git a/lib/pgBackRest/Protocol/Storage/Remote.pm b/lib/pgBackRest/Protocol/Storage/Remote.pm index 64a3bffad..0eb626001 100644 --- a/lib/pgBackRest/Protocol/Storage/Remote.pm +++ b/lib/pgBackRest/Protocol/Storage/Remote.pm @@ -15,7 +15,6 @@ use pgBackRest::Config::Config; use pgBackRest::Protocol::Helper; use pgBackRest::Protocol::Storage::File; use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::Gzip; #################################################################################################################################### # new @@ -195,29 +194,10 @@ sub openRead {name => 'rhParam', required => false}, ); - # Determine whether protocol compress will be used - my $bProtocolCompress = protocolCompress($rhParam); - - # Compress on the remote side - if ($bProtocolCompress) - { - push( - @{$rhParam->{rhyFilter}}, - {strClass => STORAGE_FILTER_GZIP, - rxyParam => [{iLevel => cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK), bWantGzip => false}]}); - } - my $oSourceFileIo = $self->{oProtocol}->cmdExecute(OP_STORAGE_OPEN_READ, [$strFileExp, $rhParam]) ? new pgBackRest::Protocol::Storage::File($self->{oProtocol}) : undef; - # Decompress on the local side - if ($bProtocolCompress) - { - $oSourceFileIo = new pgBackRest::Storage::Filter::Gzip( - $oSourceFileIo, {strCompressType => STORAGE_DECOMPRESS, bWantGzip => false}); - } - # Return from function and log return values if any return logDebugReturn ( @@ -247,28 +227,10 @@ sub openWrite {name => 'rhParam', required => false}, ); - # Determine whether protocol compress will be used - my $bProtocolCompress = protocolCompress($rhParam); - - # Decompress on the remote side - if ($bProtocolCompress) - { - push( - @{$rhParam->{rhyFilter}}, - {strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS, bWantGzip => false}]}); - } - # Open the remote file $self->{oProtocol}->cmdWrite(OP_STORAGE_OPEN_WRITE, [$strFileExp, $rhParam]); my $oDestinationFileIo = new pgBackRest::Protocol::Storage::File($self->{oProtocol}); - # Compress on local side - if ($bProtocolCompress) - { - $oDestinationFileIo = new pgBackRest::Storage::Filter::Gzip( - $oDestinationFileIo, {iLevel => cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK), bWantGzip => false}); - } - # Return from function and log return values if any return logDebugReturn ( @@ -367,24 +329,6 @@ sub cipherPassUser ); } -#################################################################################################################################### -# Used internally to determine if protocol compression should be enabled -#################################################################################################################################### -sub protocolCompress -{ - my $rhParam = shift; - - my $bProtocolCompress = false; - - if (defined($rhParam->{bProtocolCompress})) - { - $bProtocolCompress = $rhParam->{bProtocolCompress} && cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK) > 0 ? true : false; - delete($rhParam->{bProtocolCompress}); - } - - return $bProtocolCompress; -} - #################################################################################################################################### # getters #################################################################################################################################### diff --git a/lib/pgBackRest/Restore.pm b/lib/pgBackRest/Restore.pm index c01a60dfb..693e0023c 100644 --- a/lib/pgBackRest/Restore.pm +++ b/lib/pgBackRest/Restore.pm @@ -227,8 +227,7 @@ sub manifestLoad # Copy the backup manifest to the db cluster path storageDb()->copy( - storageRepo()->openRead(STORAGE_REPO_BACKUP . "/$self->{strBackupSet}/" . FILE_MANIFEST, {bProtocolCompress => true, - strCipherPass => $strCipherPass}), + storageRepo()->openRead(STORAGE_REPO_BACKUP . "/$self->{strBackupSet}/" . FILE_MANIFEST, {strCipherPass => $strCipherPass}), $self->{strDbClusterPath} . '/' . FILE_MANIFEST); # Load the manifest into a hash @@ -1089,7 +1088,7 @@ sub process # Copy backup info, load it, then delete $oStorageDb->copy( - storageRepo()->openRead(STORAGE_REPO_BACKUP . qw(/) . FILE_BACKUP_INFO, {bProtocolCompress => true}), + storageRepo()->openRead(STORAGE_REPO_BACKUP . qw(/) . FILE_BACKUP_INFO), $self->{strDbClusterPath} . '/' . FILE_BACKUP_INFO); my $oBackupInfo = new pgBackRest::Backup::Info($self->{strDbClusterPath}, false, undef, {oStorage => storageDb()}); diff --git a/lib/pgBackRest/RestoreFile.pm b/lib/pgBackRest/RestoreFile.pm index 420cd605f..949528492 100644 --- a/lib/pgBackRest/RestoreFile.pm +++ b/lib/pgBackRest/RestoreFile.pm @@ -20,9 +20,6 @@ use pgBackRest::Common::String; use pgBackRest::Config::Config; use pgBackRest::Manifest; use pgBackRest::Protocol::Storage::Helper; -use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::Gzip; -use pgBackRest::Storage::Filter::Sha; use pgBackRest::Storage::Helper; #################################################################################################################################### diff --git a/lib/pgBackRest/Stanza.pm b/lib/pgBackRest/Stanza.pm index 460e1d409..d43e911d6 100644 --- a/lib/pgBackRest/Stanza.pm +++ b/lib/pgBackRest/Stanza.pm @@ -294,7 +294,7 @@ sub stanzaDelete my ($strOperation) = logDebugParam(__PACKAGE__ . '->stanzaDelete'); my $strStanza = cfgOption(CFGOPT_STANZA); - my $oStorageRepo = storageRepo({strStanza => $strStanza}); + my $oStorageRepo = storageRepo(); # If at least an archive or backup directory exists for the stanza, then continue, else nothing to do if ($oStorageRepo->pathExists(STORAGE_REPO_ARCHIVE) || $oStorageRepo->pathExists(STORAGE_REPO_BACKUP)) @@ -342,8 +342,8 @@ sub stanzaDelete } # Recursively remove the stanza archive and backup directories - $oStorageRepo->remove(STORAGE_REPO_ARCHIVE, {bRecurse => true, bIgnoreMissing => true}); - $oStorageRepo->remove(STORAGE_REPO_BACKUP, {bRecurse => true, bIgnoreMissing => true}); + $oStorageRepo->pathRemove(STORAGE_REPO_ARCHIVE, {bRecurse => true, bIgnoreMissing => true}); + $oStorageRepo->pathRemove(STORAGE_REPO_BACKUP, {bRecurse => true, bIgnoreMissing => true}); # Remove the stop file so processes can run. lockStart(); diff --git a/lib/pgBackRest/Storage/Base.pm b/lib/pgBackRest/Storage/Base.pm index 58f862a4c..5d6dec4d5 100644 --- a/lib/pgBackRest/Storage/Base.pm +++ b/lib/pgBackRest/Storage/Base.pm @@ -16,6 +16,17 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Io::Base; use pgBackRest::Common::Log; +#################################################################################################################################### +# Storage constants +#################################################################################################################################### +use constant STORAGE_LOCAL => ''; + push @EXPORT, qw(STORAGE_LOCAL); + +use constant STORAGE_S3 => 's3'; + push @EXPORT, qw(STORAGE_S3); +use constant STORAGE_POSIX => 'posix'; + push @EXPORT, qw(STORAGE_POSIX); + #################################################################################################################################### # Compress constants #################################################################################################################################### @@ -34,6 +45,16 @@ use constant STORAGE_DECRYPT => 'decrypt' use constant CIPHER_MAGIC => 'Salted__'; push @EXPORT, qw(CIPHER_MAGIC); +#################################################################################################################################### +# Filter constants +#################################################################################################################################### +use constant STORAGE_FILTER_CIPHER_BLOCK => 'pgBackRest::Storage::Filter::CipherBlock'; + push @EXPORT, qw(STORAGE_FILTER_CIPHER_BLOCK); +use constant STORAGE_FILTER_GZIP => 'pgBackRest::Storage::Filter::Gzip'; + push @EXPORT, qw(STORAGE_FILTER_GZIP); +use constant STORAGE_FILTER_SHA => 'pgBackRest::Storage::Filter::Sha'; + push @EXPORT, qw(STORAGE_FILTER_SHA); + #################################################################################################################################### # Capability constants #################################################################################################################################### @@ -78,9 +99,10 @@ sub new ); } + #################################################################################################################################### -# copy - copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed -# instead of file names. +# Copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed instead of +# file names. #################################################################################################################################### sub copy { @@ -92,47 +114,62 @@ sub copy $strOperation, $xSourceFile, $xDestinationFile, + $bSourceOpen, ) = logDebugParam ( __PACKAGE__ . '->copy', \@_, {name => 'xSourceFile', required => false}, - {name => 'xDestinationFile', required => false}, + {name => 'xDestinationFile'}, + {name => 'bSourceOpen', optional => true, default => false}, ); - # Was the file copied? + # Is source/destination an IO object or a file expression? + my $oSourceFileIo = defined($xSourceFile) ? (ref($xSourceFile) ? $xSourceFile : $self->openRead($xSourceFile)) : undef; + + # Does the source file exist? my $bResult = false; - # Is source an IO object or a file expression? - my $oSourceFileIo = - defined($xSourceFile) ? - (ref($xSourceFile) ? $xSourceFile : $self->openRead($self->pathGet($xSourceFile))) : undef; - - # Proceed if source file exists + # Copy if the source file exists if (defined($oSourceFileIo)) { - # Is destination an IO object or a file expression? - my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($self->pathGet($xDestinationFile)); + my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($xDestinationFile); - # Copy the data - my $lSizeRead; - - do + # Use C copy if source and destination are C objects + if (defined($oSourceFileIo->{oStorageCRead}) && defined($oDestinationFileIo->{oStorageCWrite})) { - # Read data - my $tBuffer = ''; - - $lSizeRead = $oSourceFileIo->read(\$tBuffer, $self->{lBufferMax}); - $oDestinationFileIo->write(\$tBuffer); + $bResult = $self->{oStorageC}->copy( + $oSourceFileIo->{oStorageCRead}, $oDestinationFileIo->{oStorageCWrite}) ? true : false; } - while ($lSizeRead != 0); + else + { + # Open the source file if it is a C object + $bResult = defined($oSourceFileIo->{oStorageCRead}) ? ($bSourceOpen || $oSourceFileIo->open()) : true; - # Close files - $oSourceFileIo->close(); - $oDestinationFileIo->close(); + if ($bResult) + { + # Open the destination file if it is a C object + if (defined($oDestinationFileIo->{oStorageCWrite})) + { + $oDestinationFileIo->open(); + } - # File was copied - $bResult = true; + # Copy the data + do + { + # Read data + my $tBuffer = ''; + + $oSourceFileIo->read(\$tBuffer, $self->{lBufferMax}); + $oDestinationFileIo->write(\$tBuffer); + } + while (!$oSourceFileIo->eof()); + + # Close files + $oSourceFileIo->close(); + $oDestinationFileIo->close(); + } + } } return logDebugReturn diff --git a/lib/pgBackRest/Storage/Cifs/Driver.pm b/lib/pgBackRest/Storage/Cifs/Driver.pm deleted file mode 100644 index 2e987193d..000000000 --- a/lib/pgBackRest/Storage/Cifs/Driver.pm +++ /dev/null @@ -1,55 +0,0 @@ -#################################################################################################################################### -# CIFS Storage -# -# Implements storage functions for Posix-compliant file systems. -#################################################################################################################################### -package pgBackRest::Storage::Cifs::Driver; -use parent 'pgBackRest::Storage::Posix::Driver'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); - -use pgBackRest::Common::Log; -use pgBackRest::Storage::Base; - -#################################################################################################################################### -# Package name constant -#################################################################################################################################### -use constant STORAGE_CIFS_DRIVER => __PACKAGE__; - push @EXPORT, qw(STORAGE_CIFS_DRIVER); - -#################################################################################################################################### -# pathSync - CIFS does not support path sync so this is a noop -#################################################################################################################################### -sub pathSync -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathSync', \@_, - {name => 'strPath', trace => true}, - ); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Getters/Setters -#################################################################################################################################### -sub capability {shift eq STORAGE_CAPABILITY_SIZE_DIFF ? true : false} -sub className {STORAGE_CIFS_DRIVER} - -1; diff --git a/lib/pgBackRest/Storage/Filter/CipherBlock.pm b/lib/pgBackRest/Storage/Filter/CipherBlock.pm deleted file mode 100644 index 9ef07f21d..000000000 --- a/lib/pgBackRest/Storage/Filter/CipherBlock.pm +++ /dev/null @@ -1,177 +0,0 @@ -#################################################################################################################################### -# Block Cipher Filter -#################################################################################################################################### -package pgBackRest::Storage::Filter::CipherBlock; -use parent 'pgBackRest::Common::Io::Filter'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Io::Base; -use pgBackRest::Common::Log; -use pgBackRest::LibC qw(:crypto); -use pgBackRest::Storage::Base; - -#################################################################################################################################### -# Package name constant -#################################################################################################################################### -use constant STORAGE_FILTER_CIPHER_BLOCK => __PACKAGE__; - push @EXPORT, qw(STORAGE_FILTER_CIPHER_BLOCK); - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParent, - $strCipherType, - $tCipherPass, - $strMode, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParent', trace => true}, - {name => 'strCipherType', trace => true}, - {name => 'tCipherPass', trace => true}, - {name => 'strMode', optional => true, default => STORAGE_ENCRYPT, trace => true}, - ); - - # Bless with new class - my $self = $class->SUPER::new($oParent); - bless $self, $class; - - # Check mode is valid - $self->{strMode} = $strMode; - - if (!($self->{strMode} eq STORAGE_ENCRYPT || $self->{strMode} eq STORAGE_DECRYPT)) - { - confess &log(ASSERT, "unknown cipher mode: $self->{strMode}"); - } - - # Set read/write - $self->{bWrite} = false; - - # Create cipher object - $self->{oCipher} = new pgBackRest::LibC::Cipher::Block( - $self->{strMode} eq STORAGE_ENCRYPT ? CIPHER_MODE_ENCRYPT : CIPHER_MODE_DECRYPT, $strCipherType, $tCipherPass, - length($tCipherPass)); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# read - encrypt/decrypt data -#################################################################################################################################### -sub read -{ - my $self = shift; - my $rtBuffer = shift; - my $iSize = shift; - - # Return 0 if all data has been read - return 0 if $self->eof(); - - # Loop until required bytes have been read - my $tBufferRead = ''; - my $iBufferReadSize = 0; - - do - { - # Read data - my $tCipherBuffer; - my $iActualSize = $self->SUPER::read(\$tCipherBuffer, $iSize); - - # If something was read, then process it - if ($iActualSize > 0) - { - $tBufferRead .= $self->{oCipher}->process($tCipherBuffer); - } - - # If eof then flush the remaining data - if ($self->eof()) - { - $tBufferRead .= $self->{oCipher}->flush(); - } - - # Get the current size of the read buffer - $iBufferReadSize = length($tBufferRead); - } - while ($iBufferReadSize < $iSize && !$self->eof()); - - # Append to the read buffer - $$rtBuffer .= $tBufferRead; - - # Return the actual size read - return $iBufferReadSize; -} - -#################################################################################################################################### -# write - encrypt/decrypt data -#################################################################################################################################### -sub write -{ - my $self = shift; - my $rtBuffer = shift; - - # Set write flag so close will flush buffer - $self->{bWrite} = true; - - # Write the buffer if defined - my $tCipherBuffer; - - if (defined($$rtBuffer)) - { - $tCipherBuffer = $self->{oCipher}->process($$rtBuffer); - } - - # Call the io method. If $rtBuffer is undefined, then this is expected to error. - $self->SUPER::write(\$tCipherBuffer); - - return length($$rtBuffer); -} - -#################################################################################################################################### -# close - close the file -#################################################################################################################################### -sub close -{ - my $self = shift; - - # Only close the object if not already closed - if ($self->{oCipher}) - { - # Flush the write buffer - if ($self->{bWrite}) - { - my $tCipherBuffer = $self->{oCipher}->flush(); - $self->SUPER::write(\$tCipherBuffer); - } - - undef($self->{oCipher}); - - # Close io - return $self->SUPER::close(); - } - - return false; -} - -1; diff --git a/lib/pgBackRest/Storage/Filter/Gzip.pm b/lib/pgBackRest/Storage/Filter/Gzip.pm deleted file mode 100644 index e5a1cd3b7..000000000 --- a/lib/pgBackRest/Storage/Filter/Gzip.pm +++ /dev/null @@ -1,264 +0,0 @@ -#################################################################################################################################### -# GZIP Filter -#################################################################################################################################### -package pgBackRest::Storage::Filter::Gzip; -use parent 'pgBackRest::Common::Io::Filter'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Compress::Raw::Zlib qw(WANT_GZIP MAX_WBITS Z_OK Z_BUF_ERROR Z_DATA_ERROR Z_STREAM_END); -use Exporter qw(import); - our @EXPORT = qw(); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Io::Base; -use pgBackRest::Common::Log; -use pgBackRest::Storage::Base; - -#################################################################################################################################### -# Package name constant -#################################################################################################################################### -use constant STORAGE_FILTER_GZIP => __PACKAGE__; - push @EXPORT, qw(STORAGE_FILTER_GZIP); - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParent, - $bWantGzip, - $strCompressType, - $iLevel, - $lCompressBufferMax, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParent', trace => true}, - {name => 'bWantGzip', optional => true, default => true, trace => true}, - {name => 'strCompressType', optional => true, default => STORAGE_COMPRESS, trace => true}, - {name => 'iLevel', optional => true, default => 6, trace => true}, - {name => 'lCompressBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX, trace => true}, - ); - - # Bless with new class - my $self = $class->SUPER::new($oParent); - bless $self, $class; - - # Set variables - $self->{bWantGzip} = $bWantGzip; - $self->{iLevel} = $iLevel; - $self->{lCompressBufferMax} = $lCompressBufferMax; - $self->{strCompressType} = $strCompressType; - - # Set read/write - $self->{bWrite} = false; - - # Create the zlib object - my $iZLibStatus; - - if ($self->{strCompressType} eq STORAGE_COMPRESS) - { - ($self->{oZLib}, $iZLibStatus) = new Compress::Raw::Zlib::Deflate( - WindowBits => $self->{bWantGzip} ? WANT_GZIP : MAX_WBITS, Level => $self->{iLevel}, - Bufsize => $self->{lCompressBufferMax}, AppendOutput => 1); - - $self->{tCompressedBuffer} = undef; - } - else - { - ($self->{oZLib}, $iZLibStatus) = new Compress::Raw::Zlib::Inflate( - WindowBits => $self->{bWantGzip} ? WANT_GZIP : MAX_WBITS, Bufsize => $self->{lCompressBufferMax}, - LimitOutput => 1, AppendOutput => 1); - - $self->{tUncompressedBuffer} = undef; - $self->{lUncompressedBufferSize} = 0; - } - - $self->errorCheck($iZLibStatus); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# errorCheck - check status code for errors -#################################################################################################################################### -sub errorCheck -{ - my $self = shift; - my $iZLibStatus = shift; - - if (!($iZLibStatus == Z_OK || $iZLibStatus == Z_BUF_ERROR)) - { - logErrorResult( - $self->{bWrite} ? ERROR_FILE_WRITE : ERROR_FILE_READ, - 'unable to ' . ($self->{strCompressType} eq STORAGE_COMPRESS ? 'deflate' : 'inflate') . " '" . - $self->parent()->name() . "'", - $self->{oZLib}->msg()); - } - - return Z_OK; -} - -#################################################################################################################################### -# read - compress/decompress data -#################################################################################################################################### -sub read -{ - my $self = shift; - my $rtBuffer = shift; - my $iSize = shift; - - if ($self->{strCompressType} eq STORAGE_COMPRESS) - { - return 0 if $self->eof(); - - my $lSizeBegin = defined($$rtBuffer) ? length($$rtBuffer) : 0; - my $lUncompressedSize; - my $lCompressedSize; - - do - { - my $tUncompressedBuffer; - $lUncompressedSize = $self->parent()->read(\$tUncompressedBuffer, $iSize); - - if ($lUncompressedSize > 0) - { - $self->errorCheck($self->{oZLib}->deflate($tUncompressedBuffer, $$rtBuffer)); - } - else - { - $self->errorCheck($self->{oZLib}->flush($$rtBuffer)); - } - - $lCompressedSize = length($$rtBuffer) - $lSizeBegin; - } - while ($lUncompressedSize > 0 && $lCompressedSize < $iSize); - - # Return the actual size read - return $lCompressedSize; - } - else - { - # If the local buffer size is not large enough to satisfy the request and there is still data to decompress - while ($self->{lUncompressedBufferSize} < $iSize && !$self->parent()->eof()) - { - if (!defined($self->{tCompressedBuffer}) || length($self->{tCompressedBuffer}) == 0) - { - $self->parent()->read(\$self->{tCompressedBuffer}, $self->{lCompressBufferMax}); - } - - my $iZLibStatus = $self->{oZLib}->inflate($self->{tCompressedBuffer}, $self->{tUncompressedBuffer}); - $self->{lUncompressedBufferSize} = length($self->{tUncompressedBuffer}); - - last if $iZLibStatus == Z_STREAM_END; - - $self->errorCheck($iZLibStatus); - } - - # Actual size is the lesser of the local buffer size or requested size - if the local buffer is smaller than the requested - # size it means that there was nothing more to be read - my $iActualSize = $self->{lUncompressedBufferSize} < $iSize ? $self->{lUncompressedBufferSize} : $iSize; - - # Append to the request buffer - $$rtBuffer .= substr($self->{tUncompressedBuffer}, 0, $iActualSize); - - # Truncate local buffer - $self->{tUncompressedBuffer} = substr($self->{tUncompressedBuffer}, $iActualSize); - $self->{lUncompressedBufferSize} -= $iActualSize; - - # Return the actual size read - return $iActualSize; - } -} - -#################################################################################################################################### -# write - compress/decompress data -#################################################################################################################################### -sub write -{ - my $self = shift; - my $rtBuffer = shift; - - $self->{bWrite} = true; - - if ($self->{strCompressType} eq STORAGE_COMPRESS) - { - # Compress the data - $self->errorCheck($self->{oZLib}->deflate($$rtBuffer, $self->{tCompressedBuffer})); - - # Only write when buffer is full - if (length($self->{tCompressedBuffer}) > $self->{lCompressBufferMax}) - { - $self->parent()->write(\$self->{tCompressedBuffer}); - $self->{tCompressedBuffer} = undef; - } - } - else - { - my $tCompressedBuffer = $$rtBuffer; - - while (length($tCompressedBuffer) > 0) - { - my $tUncompressedBuffer; - - my $iZLibStatus = $self->{oZLib}->inflate($tCompressedBuffer, $tUncompressedBuffer); - $self->parent()->write(\$tUncompressedBuffer); - - last if $iZLibStatus == Z_STREAM_END; - - $self->errorCheck($iZLibStatus); - } - } - - # Return bytes written - return length($$rtBuffer); -} - -#################################################################################################################################### -# close - close the file -#################################################################################################################################### -sub close -{ - my $self = shift; - - if (defined($self->{oZLib})) - { - # Flush the write buffer - if ($self->{bWrite}) - { - if ($self->{strCompressType} eq STORAGE_COMPRESS) - { - # Flush out last compressed bytes - $self->errorCheck($self->{oZLib}->flush($self->{tCompressedBuffer})); - - # Write last compressed bytes - $self->parent()->write(\$self->{tCompressedBuffer}); - } - } - - undef($self->{oZLib}); - - # Close io - return $self->parent()->close(); - } - - return false; -} - -1; diff --git a/lib/pgBackRest/Storage/Filter/Sha.pm b/lib/pgBackRest/Storage/Filter/Sha.pm deleted file mode 100644 index 2ddbc3e78..000000000 --- a/lib/pgBackRest/Storage/Filter/Sha.pm +++ /dev/null @@ -1,124 +0,0 @@ -#################################################################################################################################### -# SHA Filter -#################################################################################################################################### -package pgBackRest::Storage::Filter::Sha; -use parent 'pgBackRest::Common::Io::Filter'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; - -#################################################################################################################################### -# Package name constant -#################################################################################################################################### -use constant STORAGE_FILTER_SHA => __PACKAGE__; - push @EXPORT, qw(STORAGE_FILTER_SHA); - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParent, - $strAlgorithm, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParent', trace => true}, - {name => 'strAlgorithm', optional => true, default => 'sha1', trace => true}, - ); - - # Bless with new class - my $self = $class->SUPER::new($oParent); - bless $self, $class; - - # Set variables - $self->{strAlgorithm} = $strAlgorithm; - - # Create SHA object - $self->{oSha} = new pgBackRest::LibC::Crypto::Hash($self->{strAlgorithm}); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# read - calculate sha digest -#################################################################################################################################### -sub read -{ - my $self = shift; - my $rtBuffer = shift; - my $iSize = shift; - - # Call the io method - my $tShaBuffer; - my $iActualSize = $self->parent()->read(\$tShaBuffer, $iSize); - - # Calculate sha for the returned buffer - if ($iActualSize > 0) - { - $self->{oSha}->process($tShaBuffer); - $$rtBuffer .= $tShaBuffer; - } - - # Return the actual size read - return $iActualSize; -} - -#################################################################################################################################### -# write - calculate sha digest -#################################################################################################################################### -sub write -{ - my $self = shift; - my $rtBuffer = shift; - - # Calculate sha for the buffer - $self->{oSha}->process($$rtBuffer); - - # Call the io method - return $self->parent()->write($rtBuffer); -} - -#################################################################################################################################### -# close - close the file -#################################################################################################################################### -sub close -{ - my $self = shift; - - if (defined($self->{oSha})) - { - # Set result - $self->resultSet(STORAGE_FILTER_SHA, $self->{oSha}->result()); - - # Delete the sha object - delete($self->{oSha}); - - # Close io - return $self->parent->close(); - } - - return false; -} - -1; diff --git a/lib/pgBackRest/Storage/Helper.pm b/lib/pgBackRest/Storage/Helper.pm index 7ae2fcf92..05c083339 100644 --- a/lib/pgBackRest/Storage/Helper.pm +++ b/lib/pgBackRest/Storage/Helper.pm @@ -13,16 +13,10 @@ use File::Basename qw(basename); use pgBackRest::Common::Log; use pgBackRest::Config::Config; -use pgBackRest::Storage::Posix::Driver; -use pgBackRest::Storage::Local; +use pgBackRest::Storage::Base; +use pgBackRest::Storage::Storage; use pgBackRest::Version; -#################################################################################################################################### -# Storage constants -#################################################################################################################################### -use constant STORAGE_LOCAL => ''; - push @EXPORT, qw(STORAGE_LOCAL); - #################################################################################################################################### # Compression extension #################################################################################################################################### @@ -35,11 +29,6 @@ use constant COMPRESS_EXT => 'gz'; use constant STORAGE_TEMP_EXT => PROJECT_EXE . '.tmp'; push @EXPORT, qw(STORAGE_TEMP_EXT); -#################################################################################################################################### -# Cache storage so it can be retrieved quickly -#################################################################################################################################### -my $hStorage; - #################################################################################################################################### # storageLocal - get local storage # @@ -49,32 +38,13 @@ my $hStorage; sub storageLocal { # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - ) = - logDebugParam - ( - __PACKAGE__ . '::storageLocal', \@_, - {name => 'strPath', default => '/', trace => true}, - ); - - # Create storage if not defined - if (!defined($hStorage->{&STORAGE_LOCAL}{$strPath})) - { - # Create local storage - $hStorage->{&STORAGE_LOCAL}{$strPath} = new pgBackRest::Storage::Local( - $strPath, new pgBackRest::Storage::Posix::Driver(), - {strTempExtension => STORAGE_TEMP_EXT, - lBufferMax => cfgOptionValid(CFGOPT_BUFFER_SIZE, false) ? cfgOption(CFGOPT_BUFFER_SIZE, false) : undef}); - } + my ($strOperation) = logDebugParam(__PACKAGE__ . '::storageLocal'); # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'oStorageLocal', value => $hStorage->{&STORAGE_LOCAL}{$strPath}, trace => true}, + {name => 'oStorageLocal', value => new pgBackRest::Storage::Storage(STORAGE_LOCAL), trace => true}, ); } diff --git a/lib/pgBackRest/Storage/Posix/Driver.pm b/lib/pgBackRest/Storage/Posix/Driver.pm deleted file mode 100644 index 040c8aace..000000000 --- a/lib/pgBackRest/Storage/Posix/Driver.pm +++ /dev/null @@ -1,963 +0,0 @@ -#################################################################################################################################### -# Posix Storage -# -# Implements storage functions for Posix-compliant file systems. -#################################################################################################################################### -package pgBackRest::Storage::Posix::Driver; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(basename dirname); -use Fcntl qw(:mode); -use File::stat qw{lstat}; - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::Storage::Base; -use pgBackRest::Storage::Posix::FileRead; -use pgBackRest::Storage::Posix::FileWrite; - -#################################################################################################################################### -# Package name constant -#################################################################################################################################### -use constant STORAGE_POSIX_DRIVER => __PACKAGE__; - push @EXPORT, qw(STORAGE_POSIX_DRIVER); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{bFileSync}, - $self->{bPathSync}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'bFileSync', optional => true, default => true}, - {name => 'bPathSync', optional => true, default => true}, - ); - - # Set default temp extension - $self->{strTempExtension} = 'tmp'; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# exists - check if a path or file exists -#################################################################################################################################### -sub exists -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - ) = - logDebugParam - ( - __PACKAGE__ . '->exists', \@_, - {name => 'strFile', trace => true}, - ); - - # Does the path/file exist? - my $bExists = true; - my $oStat = lstat($strFile); - - # Use stat to test if file exists - if (defined($oStat)) - { - # Check that it is actually a file - $bExists = !S_ISDIR($oStat->mode) ? true : false; - } - else - { - # If the error is not entry missing, then throw error - if (!$OS_ERROR{ENOENT}) - { - logErrorResult(ERROR_FILE_EXISTS, "unable to test if file '${strFile}' exists", $OS_ERROR); - } - - $bExists = false; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bExists', value => $bExists, trace => true} - ); -} - -#################################################################################################################################### -# info - get information for path/file -#################################################################################################################################### -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathFile, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strFile', trace => true}, - {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, - ); - - # Stat the path/file - my $oInfo = lstat($strPathFile); - - # Check for errors - if (!defined($oInfo)) - { - if (!($OS_ERROR{ENOENT} && $bIgnoreMissing)) - { - logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to stat '${strPathFile}'", $OS_ERROR); - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oInfo', value => $oInfo, trace => true} - ); -} - -#################################################################################################################################### -# linkCreate -#################################################################################################################################### -sub linkCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strSourcePathFile, - $strDestinationLink, - $bHard, - $bPathCreate, - $bIgnoreExists, - ) = - logDebugParam - ( - __PACKAGE__ . '->linkCreate', \@_, - {name => 'strSourcePathFile', trace => true}, - {name => 'strDestinationLink', trace => true}, - {name => 'bHard', optional=> true, default => false, trace => true}, - {name => 'bPathCreate', optional=> true, default => true, trace => true}, - {name => 'bIgnoreExists', optional => true, default => false, trace => true}, - ); - - if (!($bHard ? link($strSourcePathFile, $strDestinationLink) : symlink($strSourcePathFile, $strDestinationLink))) - { - my $strMessage = "unable to create link '${strDestinationLink}'"; - - # If parent path or source is missing - if ($OS_ERROR{ENOENT}) - { - # Check if source is missing - if (!$self->exists($strSourcePathFile)) - { - confess &log(ERROR, "${strMessage} because source '${strSourcePathFile}' does not exist", ERROR_FILE_MISSING); - } - - if (!$bPathCreate) - { - confess &log(ERROR, "${strMessage} because parent does not exist", ERROR_PATH_MISSING); - } - - # Create parent path - $self->pathCreate(dirname($strDestinationLink), {bIgnoreExists => true, bCreateParent => true}); - - # Create link - $self->linkCreate($strSourcePathFile, $strDestinationLink, {bHard => $bHard}); - } - # Else if link already exists - elsif ($OS_ERROR{EEXIST}) - { - if (!$bIgnoreExists) - { - confess &log(ERROR, "${strMessage} because it already exists", ERROR_PATH_EXISTS); - } - } - else - { - logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR); - } - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# linkDestination - get destination of symlink -#################################################################################################################################### -sub linkDestination -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strLink, - ) = - logDebugParam - ( - __PACKAGE__ . '->linkDestination', \@_, - {name => 'strLink', trace => true}, - ); - - # Get link destination - my $strLinkDestination = readlink($strLink); - - # Check for errors - if (!defined($strLinkDestination)) - { - logErrorResult( - $OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to get destination for link ${strLink}", $OS_ERROR); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strLinkDestination', value => $strLinkDestination, trace => true} - ); -} - -#################################################################################################################################### -# list - list all files/paths in path -#################################################################################################################################### -sub list -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->list', \@_, - {name => 'strPath', trace => true}, - {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, - ); - - # Working variables - my @stryFileList; - my $hPath; - - # Attempt to open the path - if (opendir($hPath, $strPath)) - { - @stryFileList = grep(!/^(\.)|(\.\.)$/i, readdir($hPath)); - close($hPath); - } - # Else process errors - else - { - # Ignore the error if the file is missing and missing files should be ignored - if (!($OS_ERROR{ENOENT} && $bIgnoreMissing)) - { - logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to read path '${strPath}'", $OS_ERROR); - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryFileList', value => \@stryFileList, ref => true, trace => true} - ); -} - -#################################################################################################################################### -# manifest - build path/file/link manifest starting with base path and including all subpaths -#################################################################################################################################### -sub manifest -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - $bIgnoreMissing, - $strFilter, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifest', \@_, - {name => 'strPath', trace => true}, - {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, - {name => 'strFilter', optional => true, trace => true}, - ); - - # Generate the manifest - my $hManifest = {}; - $self->manifestRecurse($strPath, undef, 0, $hManifest, $bIgnoreMissing, $strFilter); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hManifest', value => $hManifest, trace => true} - ); -} - -sub manifestRecurse -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - $strSubPath, - $iDepth, - $hManifest, - $bIgnoreMissing, - $strFilter, - ) = - logDebugParam - ( - __PACKAGE__ . '::manifestRecurse', \@_, - {name => 'strPath', trace => true}, - {name => 'strSubPath', required => false, trace => true}, - {name => 'iDepth', default => 0, trace => true}, - {name => 'hManifest', required => false, trace => true}, - {name => 'bIgnoreMissing', required => false, default => false, trace => true}, - {name => 'strFilter', required => false, trace => true}, - ); - - # Set operation and debug strings - my $strPathRead = $strPath . (defined($strSubPath) ? "/${strSubPath}" : ''); - my $hPath; - - # If this is the top level stat the path to discover if it is actually a file - my $oPathInfo = $self->info($strPathRead, {bIgnoreMissing => $bIgnoreMissing}); - - if (defined($oPathInfo)) - { - # If the initial path passed is a file then generate the manifest for just that file - if ($iDepth == 0 && !S_ISDIR($oPathInfo->mode())) - { - $hManifest->{basename($strPathRead)} = $self->manifestStat($strPathRead); - } - # Else read as a normal directory - else - { - # Get a list of all files in the path (including .) - my @stryFileList = @{$self->list($strPathRead, {bIgnoreMissing => $iDepth != 0})}; - unshift(@stryFileList, '.'); - my $hFileStat = $self->manifestList($strPathRead, \@stryFileList, $strFilter); - - # Loop through all subpaths/files in the path - foreach my $strFile (keys(%{$hFileStat})) - { - my $strManifestFile = $iDepth == 0 ? $strFile : ($strSubPath . ($strFile eq qw(.) ? '' : "/${strFile}")); - $hManifest->{$strManifestFile} = $hFileStat->{$strFile}; - - # Recurse into directories - if ($hManifest->{$strManifestFile}{type} eq 'd' && $strFile ne qw(.)) - { - $self->manifestRecurse($strPath, $strManifestFile, $iDepth + 1, $hManifest); - } - } - } - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -sub manifestList -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - $stryFile, - $strFilter, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifestList', \@_, - {name => 'strPath', trace => true}, - {name => 'stryFile', trace => true}, - {name => 'strFilter', required => false, trace => true}, - ); - - my $hFileStat = {}; - - foreach my $strFile (@{$stryFile}) - { - if ($strFile ne '.' && defined($strFilter) && $strFilter ne $strFile) - { - next; - } - - $hFileStat->{$strFile} = $self->manifestStat("${strPath}" . ($strFile eq qw(.) ? '' : "/${strFile}")); - - if (!defined($hFileStat->{$strFile})) - { - delete($hFileStat->{$strFile}); - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hFileStat', value => $hFileStat, trace => true} - ); -} - -sub manifestStat -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifestStat', \@_, - {name => 'strFile', trace => true}, - ); - - # Stat the path/file, ignoring any that are missing - my $oStat = $self->info($strFile, {bIgnoreMissing => true}); - - # Generate file data if stat succeeded (i.e. file exists) - my $hFile; - - if (defined($oStat)) - { - # Check for regular file - if (S_ISREG($oStat->mode)) - { - $hFile->{type} = 'f'; - - # Get size - $hFile->{size} = $oStat->size; - - # Get modification time - $hFile->{modification_time} = $oStat->mtime; - } - # Check for directory - elsif (S_ISDIR($oStat->mode)) - { - $hFile->{type} = 'd'; - } - # Check for link - elsif (S_ISLNK($oStat->mode)) - { - $hFile->{type} = 'l'; - $hFile->{link_destination} = $self->linkDestination($strFile); - } - # Not a recognized type - else - { - confess &log(ERROR, "${strFile} is not of type directory, file, or link", ERROR_FILE_INVALID); - } - - # Get user name - $hFile->{user} = getpwuid($oStat->uid); - - # Get group name - $hFile->{group} = getgrgid($oStat->gid); - - # Get mode - if ($hFile->{type} ne 'l') - { - $hFile->{mode} = sprintf('%04o', S_IMODE($oStat->mode)); - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hFile', value => $hFile, trace => true} - ); -} - -#################################################################################################################################### -# move - move path/file -#################################################################################################################################### -sub move -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strSourceFile, - $strDestinationFile, - $bPathCreate, - ) = - logDebugParam - ( - __PACKAGE__ . '->move', \@_, - {name => 'strSourceFile', trace => true}, - {name => 'strDestinationFile', trace => true}, - {name => 'bPathCreate', default => false, trace => true}, - ); - - # Get source and destination paths - my $strSourcePathFile = dirname($strSourceFile); - my $strDestinationPathFile = dirname($strDestinationFile); - - # Move the file - if (!rename($strSourceFile, $strDestinationFile)) - { - my $strMessage = "unable to move '${strSourceFile}'"; - - # If something is missing determine if it is the source or destination - if ($OS_ERROR{ENOENT}) - { - if (!$self->exists($strSourceFile)) - { - logErrorResult(ERROR_FILE_MISSING, "${strMessage} because it is missing"); - } - - if ($bPathCreate) - { - # Attempt to create the path - ignore exists here in case another process creates it first - $self->pathCreate($strDestinationPathFile, {bCreateParent => true, bIgnoreExists => true}); - - # Try move again - $self->move($strSourceFile, $strDestinationFile); - } - else - { - logErrorResult(ERROR_PATH_MISSING, "${strMessage} to missing path '${strDestinationPathFile}'"); - } - } - # Else raise the error - else - { - logErrorResult(ERROR_FILE_MOVE, "${strMessage} to '${strDestinationFile}'", $OS_ERROR); - } - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# openRead - open file for reading -#################################################################################################################################### -sub openRead -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->openRead', \@_, - {name => 'strFile', trace => true}, - {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, - ); - - my $oFileIO = new pgBackRest::Storage::Posix::FileRead($self, $strFile, {bIgnoreMissing => $bIgnoreMissing}); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oFileIO', value => $oFileIO, trace => true}, - ); -} - -#################################################################################################################################### -# openWrite - open file for writing -#################################################################################################################################### -sub openWrite -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - $strMode, - $strUser, - $strGroup, - $lTimestamp, - $bPathCreate, - $bAtomic, - ) = - logDebugParam - ( - __PACKAGE__ . '->openWrite', \@_, - {name => 'strFile', trace => true}, - {name => 'strMode', optional => true, trace => true}, - {name => 'strUser', optional => true, trace => true}, - {name => 'strGroup', optional => true, trace => true}, - {name => 'lTimestamp', optional => true, trace => true}, - {name => 'bPathCreate', optional => true, trace => true}, - {name => 'bAtomic', optional => true, trace => true}, - ); - - my $oFileIO = new pgBackRest::Storage::Posix::FileWrite( - $self, $strFile, - {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lTimestamp, bPathCreate => $bPathCreate, - bAtomic => $bAtomic, bSync => $self->{bFileSync}}); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oFileIO', value => $oFileIO, trace => true}, - ); -} - -#################################################################################################################################### -# owner - change ownership of path/file -#################################################################################################################################### -sub owner -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFilePath, - $strUser, - $strGroup, - ) = - logDebugParam - ( - __PACKAGE__ . '->owner', \@_, - {name => 'strFilePath', trace => true}, - {name => 'strUser', optional => true, trace => true}, - {name => 'strGroup', optional => true, trace => true}, - ); - - # Only proceed if user or group was specified - if (defined($strUser) || defined($strGroup)) - { - my $strMessage = "unable to set ownership for '${strFilePath}'"; - my $iUserId; - my $iGroupId; - - # If the user or group is not defined then get it by stat'ing the file. This is because the chown function requires that - # both user and group be set. - my $oStat = $self->info($strFilePath); - - if (!defined($strUser)) - { - $iUserId = $oStat->uid; - } - - if (!defined($strGroup)) - { - $iGroupId = $oStat->gid; - } - - # Lookup user if specified - if (defined($strUser)) - { - $iUserId = getpwnam($strUser); - - if (!defined($iUserId)) - { - logErrorResult(ERROR_FILE_OWNER, "${strMessage} because user '${strUser}' does not exist"); - } - } - - # Lookup group if specified - if (defined($strGroup)) - { - $iGroupId = getgrnam($strGroup); - - if (!defined($iGroupId)) - { - logErrorResult(ERROR_FILE_OWNER, "${strMessage} because group '${strGroup}' does not exist"); - } - } - - # Set ownership on the file if the user or group would be changed - if ($iUserId != $oStat->uid || $iGroupId != $oStat->gid) - { - if (!chown($iUserId, $iGroupId, $strFilePath)) - { - logErrorResult(ERROR_FILE_OWNER, "${strMessage}", $OS_ERROR); - } - } - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# pathCreate - create path -#################################################################################################################################### -sub pathCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - $strMode, - $bIgnoreExists, - $bCreateParent, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathCreate', \@_, - {name => 'strPath', trace => true}, - {name => 'strMode', optional => true, default => '0750', trace => true}, - {name => 'bIgnoreExists', optional => true, default => false, trace => true}, - {name => 'bCreateParent', optional => true, default => false, trace => true}, - ); - - # Attempt to create the directory - if (!mkdir($strPath, oct($strMode))) - { - my $strMessage = "unable to create path '${strPath}'"; - - # If parent path is missing - if ($OS_ERROR{ENOENT}) - { - if (!$bCreateParent) - { - confess &log(ERROR, "${strMessage} because parent does not exist", ERROR_PATH_MISSING); - } - - # Create parent path - $self->pathCreate(dirname($strPath), {strMode => $strMode, bIgnoreExists => true, bCreateParent => $bCreateParent}); - - # Create path - $self->pathCreate($strPath, {strMode => $strMode, bIgnoreExists => true}); - } - # Else if path already exists - elsif ($OS_ERROR{EEXIST}) - { - if (!$bIgnoreExists) - { - confess &log(ERROR, "${strMessage} because it already exists", ERROR_PATH_EXISTS); - } - } - else - { - logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR); - } - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# pathExists - check if path exists -#################################################################################################################################### -sub pathExists -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathExists', \@_, - {name => 'strPath', trace => true}, - ); - - # Does the path/file exist? - my $bExists = true; - my $oStat = lstat($strPath); - - # Use stat to test if path exists - if (defined($oStat)) - { - # Check that it is actually a path - $bExists = S_ISDIR($oStat->mode) ? true : false; - } - else - { - # If the error is not entry missing, then throw error - if (!$OS_ERROR{ENOENT}) - { - logErrorResult(ERROR_FILE_EXISTS, "unable to test if path '${strPath}' exists", $OS_ERROR); - } - - $bExists = false; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bExists', value => $bExists, trace => true} - ); -} - -#################################################################################################################################### -# pathSync - perform fsync on path -#################################################################################################################################### -sub pathSync -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathSync', \@_, - {name => 'strPath', trace => true}, - ); - - open(my $hPath, "<", $strPath) - or confess &log(ERROR, "unable to open '${strPath}' for sync", ERROR_PATH_OPEN); - open(my $hPathDup, ">&", $hPath) - or confess &log(ERROR, "unable to duplicate '${strPath}' handle for sync", ERROR_PATH_OPEN); - - $hPathDup->sync() - or confess &log(ERROR, "unable to sync path '${strPath}'", ERROR_PATH_SYNC); - - close($hPathDup); - close($hPath); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# remove - remove path/file -#################################################################################################################################### -sub remove -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathFile, - $bIgnoreMissing, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->remove', \@_, - {name => 'strPathFile', trace => true}, - {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, - {name => 'bRecurse', optional => true, default => false, trace => true}, - ); - - # Working variables - my $bRemoved = true; - - # Remove a tree - if ($bRecurse) - { - # Dynamically load the driver - require pgBackRest::LibC; - pgBackRest::LibC->import(qw(:storage)); - - storagePosixPathRemove($strPathFile, !$bIgnoreMissing, $bRecurse) - } - # Only remove the specified file - else - { - foreach my $strFile (ref($strPathFile) ? @{$strPathFile} : ($strPathFile)) - { - if (unlink($strFile) != 1) - { - $bRemoved = false; - - # Throw error if this is not an ignored missing file - if (!($OS_ERROR{ENOENT} && $bIgnoreMissing)) - { - logErrorResult( - $OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to remove file '${strFile}'", $OS_ERROR); - } - } - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bRemoved', value => $bRemoved, trace => true} - ); -} - -#################################################################################################################################### -# Getters/Setters -#################################################################################################################################### -sub capability {true} -sub className {STORAGE_POSIX_DRIVER} -sub tempExtension {shift->{strTempExtension}} -sub tempExtensionSet {my $self = shift; $self->{strTempExtension} = shift} - -1; diff --git a/lib/pgBackRest/Storage/Posix/FileRead.pm b/lib/pgBackRest/Storage/Posix/FileRead.pm deleted file mode 100644 index 2aaa907d8..000000000 --- a/lib/pgBackRest/Storage/Posix/FileRead.pm +++ /dev/null @@ -1,105 +0,0 @@ -#################################################################################################################################### -# Posix File Read -#################################################################################################################################### -package pgBackRest::Storage::Posix::FileRead; -use parent 'pgBackRest::Common::Io::Handle'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Fcntl qw(O_RDONLY); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oDriver, - $strName, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oDriver', trace => true}, - {name => 'strName', trace => true}, - {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, - ); - - # Open the file - my $fhFile; - - if (!sysopen($fhFile, $strName, O_RDONLY)) - { - if (!($OS_ERROR{ENOENT} && $bIgnoreMissing)) - { - logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, "unable to open '${strName}'", $OS_ERROR); - } - - undef($fhFile); - } - - # Create IO object if open succeeded - my $self; - - if (defined($fhFile)) - { - # Set file mode to binary - binmode($fhFile); - - # Create the class hash - $self = $class->SUPER::new("'${strName}'", $fhFile); - bless $self, $class; - - # Set variables - $self->{oDriver} = $oDriver; - $self->{strName} = $strName; - $self->{fhFile} = $fhFile; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# close - close the file -#################################################################################################################################### -sub close -{ - my $self = shift; - - if (defined($self->handle())) - { - # Close the file - close($self->handle()); - undef($self->{fhFile}); - - # Close parent - $self->SUPER::close(); - } - - return true; -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub handle {shift->{fhFile}} -sub name {shift->{strName}} - -1; diff --git a/lib/pgBackRest/Storage/Posix/FileWrite.pm b/lib/pgBackRest/Storage/Posix/FileWrite.pm deleted file mode 100644 index 79ddb52a0..000000000 --- a/lib/pgBackRest/Storage/Posix/FileWrite.pm +++ /dev/null @@ -1,209 +0,0 @@ -#################################################################################################################################### -# Posix File Write -#################################################################################################################################### -package pgBackRest::Storage::Posix::FileWrite; -use parent 'pgBackRest::Common::Io::Handle'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC); -use File::Basename qw(dirname); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; - -use pgBackRest::Common::Io::Handle; -use pgBackRest::Storage::Base; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oDriver, - $strName, - $strMode, - $strUser, - $strGroup, - $lTimestamp, - $bPathCreate, - $bAtomic, - $bSync, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oDriver', trace => true}, - {name => 'strName', trace => true}, - {name => 'strMode', optional => true, trace => true}, - {name => 'strUser', optional => true, trace => true}, - {name => 'strGroup', optional => true, trace => true}, - {name => 'lTimestamp', optional => true, trace => true}, - {name => 'bPathCreate', optional => true, default => false, trace => true}, - {name => 'bAtomic', optional => true, default => false, trace => true}, - {name => 'bSync', optional => true, default => true, trace => true}, - ); - - # Create the class hash - my $self = $class->SUPER::new("'${strName}'"); - bless $self, $class; - - # Set variables - $self->{oDriver} = $oDriver; - $self->{strName} = $strName; - $self->{strMode} = $strMode; - $self->{strUser} = $strUser; - $self->{strGroup} = $strGroup; - $self->{lTimestamp} = $lTimestamp; - $self->{bPathCreate} = $bPathCreate; - $self->{bAtomic} = $bAtomic; - $self->{bSync} = $bSync; - - # If atomic create temp filename - if ($self->{bAtomic}) - { - # Create temp file name - $self->{strNameTmp} = "$self->{strName}." . $self->{oDriver}->tempExtension(); - } - - # Open file on first write to avoid creating extraneous files on error - $self->{bOpened} = false; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# open - open the file -#################################################################################################################################### -sub open -{ - my $self = shift; - - # Get the file name - my $strFile = $self->{bAtomic} ? $self->{strNameTmp} : $self->{strName}; - - # Open the file - if (!sysopen( - $self->{fhFile}, $strFile, O_WRONLY | O_CREAT | O_TRUNC, oct(defined($self->{strMode}) ? $self->{strMode} : '0666'))) - { - # If the path does not exist create it if requested - if ($OS_ERROR{ENOENT} && $self->{bPathCreate}) - { - $self->{oDriver}->pathCreate(dirname($strFile), {bIgnoreExists => true, bCreateParent => true}); - $self->{bPathCreate} = false; - return $self->open(); - } - - logErrorResult($OS_ERROR{ENOENT} ? ERROR_PATH_MISSING : ERROR_FILE_OPEN, "unable to open '${strFile}'", $OS_ERROR); - } - - # Set file mode to binary - binmode($self->{fhFile}); - - # Set the owner - $self->{oDriver}->owner($strFile, {strUser => $self->{strUser}, strGroup => $self->{strGroup}}); - - # Set handle - $self->handleWriteSet($self->{fhFile}); - - # Mark file as opened - $self->{bOpened} = true; - - return true; -} - -#################################################################################################################################### -# write - write data to a file -#################################################################################################################################### -sub write -{ - my $self = shift; - my $rtBuffer = shift; - - # Open file if it is not open already - $self->open() if !$self->opened(); - - return $self->SUPER::write($rtBuffer); -} - -#################################################################################################################################### -# close - close the file -#################################################################################################################################### -sub close -{ - my $self = shift; - - if (defined($self->handle())) - { - # Sync the file - if ($self->{bSync}) - { - $self->handle()->sync(); - } - - # Close the file - close($self->handle()); - undef($self->{fhFile}); - - # Get current filename - my $strCurrentName = $self->{bAtomic} ? $self->{strNameTmp} : $self->{strName}; - - # Set the modification time - if (defined($self->{lTimestamp})) - { - utime(time(), $self->{lTimestamp}, $strCurrentName) - or logErrorResult(ERROR_FILE_WRITE, "unable to set time for '${strCurrentName}'", $OS_ERROR); - } - - # Move the file from temp to final if atomic - if ($self->{bAtomic}) - { - $self->{oDriver}->move($strCurrentName, $self->{strName}); - } - - # Set result - $self->resultSet(COMMON_IO_HANDLE, $self->{lSize}); - - # Close parent - $self->SUPER::close(); - } - - return true; -} - -#################################################################################################################################### -# Close the handle if it is open (in case close() was never called) -#################################################################################################################################### -sub DESTROY -{ - my $self = shift; - - if (defined($self->handle())) - { - CORE::close($self->handle()); - undef($self->{fhFile}); - } -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub handle {shift->{fhFile}} -sub opened {shift->{bOpened}} -sub name {shift->{strName}} - -1; diff --git a/lib/pgBackRest/Storage/S3/Auth.pm b/lib/pgBackRest/Storage/S3/Auth.pm deleted file mode 100644 index 1262d5203..000000000 --- a/lib/pgBackRest/Storage/S3/Auth.pm +++ /dev/null @@ -1,283 +0,0 @@ -#################################################################################################################################### -# S3 Authentication -# -# Contains the functions required to do S3 authentication. It's a complicated topic and too much to cover here, but there is -# excellent documentation at http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html. -#################################################################################################################################### -package pgBackRest::Storage::S3::Auth; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Digest::SHA qw(hmac_sha256 hmac_sha256_hex); -use Exporter qw(import); - our @EXPORT = qw(); -use POSIX qw(strftime); - -use pgBackRest::Common::Http::Common; -use pgBackRest::Common::Log; -use pgBackRest::LibC qw(:crypto); - -#################################################################################################################################### -# Constants -#################################################################################################################################### -use constant S3 => 's3'; -use constant AWS4 => 'AWS4'; -use constant AWS4_REQUEST => 'aws4_request'; -use constant AWS4_HMAC_SHA256 => 'AWS4-HMAC-SHA256'; - -use constant S3_HEADER_AUTHORIZATION => 'authorization'; - push @EXPORT, qw(S3_HEADER_AUTHORIZATION); -use constant S3_HEADER_DATE => 'x-amz-date'; - push @EXPORT, qw(S3_HEADER_DATE); -use constant S3_HEADER_CONTENT_SHA256 => 'x-amz-content-sha256'; - push @EXPORT, qw(S3_HEADER_CONTENT_SHA256); -use constant S3_HEADER_HOST => 'host'; - push @EXPORT, qw(S3_HEADER_HOST); -use constant S3_HEADER_TOKEN => 'x-amz-security-token'; - push @EXPORT, qw(S3_HEADER_TOKEN); - -use constant PAYLOAD_DEFAULT_HASH => cryptoHashOne('sha256', ''); - push @EXPORT, qw(PAYLOAD_DEFAULT_HASH); - -#################################################################################################################################### -# s3DateTime - format date/time for authentication -#################################################################################################################################### -sub s3DateTime -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $lTime, - ) = - logDebugParam - ( - __PACKAGE__ . '::s3DateTime', \@_, - {name => 'lTime', default => time(), trace => true}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strDateTime', value => strftime("%Y%m%dT%H%M%SZ", gmtime($lTime)), trace => true} - ); -} - -push @EXPORT, qw(s3DateTime); - -#################################################################################################################################### -# s3CanonicalRequest - strictly formatted version of the HTTP request used for signing -#################################################################################################################################### -sub s3CanonicalRequest -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strVerb, - $strUri, - $strQuery, - $hHeader, - $strPayloadHash, - ) = - logDebugParam - ( - __PACKAGE__ . '::s3CanonicalRequest', \@_, - {name => 'strVerb', trace => true}, - {name => 'strUri', trace => true}, - {name => 'strQuery', trace => true}, - {name => 'hHeader', trace => true}, - {name => 'strPayloadHash', trace => true}, - ); - - # Create the canonical request - my $strCanonicalRequest = - "${strVerb}\n${strUri}\n${strQuery}\n"; - my $strSignedHeaders; - - foreach my $strHeader (sort(keys(%{$hHeader}))) - { - if (lc($strHeader) ne $strHeader) - { - confess &log(ASSERT, "header '${strHeader}' must be lower case"); - } - - $strCanonicalRequest .= $strHeader . ":$hHeader->{$strHeader}\n"; - $strSignedHeaders .= (defined($strSignedHeaders) ? qw(;) : '') . lc($strHeader); - } - - $strCanonicalRequest .= "\n${strSignedHeaders}\n${strPayloadHash}"; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strCanonicalRequest', value => $strCanonicalRequest, trace => true}, - {name => 'strSignedHeaders', value => $strSignedHeaders, trace => true}, - ); -} - -push @EXPORT, qw(s3CanonicalRequest); - -#################################################################################################################################### -# s3SigningKey - signing keys last for seven days, but we'll regenerate every day because it doesn't seem too burdensome -#################################################################################################################################### -my $hSigningKeyCache; # Cache signing keys rather than regenerating them every time - -sub s3SigningKey -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDate, - $strRegion, - $strSecretAccessKey, - ) = - logDebugParam - ( - __PACKAGE__ . '::s3SigningKey', \@_, - {name => 'strDate', trace => true}, - {name => 'strRegion', trace => true}, - {name => 'strSecretAccessKey', redact => true, trace => true}, - ); - - # Check for signing key in cache - my $strSigningKey = $hSigningKeyCache->{$strDate}{$strRegion}{$strSecretAccessKey}; - - # If not found then generate it - if (!defined($strSigningKey)) - { - my $strDateKey = hmac_sha256($strDate, AWS4 . $strSecretAccessKey); - my $strRegionKey = hmac_sha256($strRegion, $strDateKey); - my $strServiceKey = hmac_sha256(S3, $strRegionKey); - $strSigningKey = hmac_sha256(AWS4_REQUEST, $strServiceKey); - - # Cache the signing key - $hSigningKeyCache->{$strDate}{$strRegion}{$strSecretAccessKey} = $strSigningKey; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strSigningKey', value => $strSigningKey, redact => true, trace => true} - ); -} - -push @EXPORT, qw(s3SigningKey); - -#################################################################################################################################### -# s3StringToSign - string that will be signed by the signing key for authentication -#################################################################################################################################### -sub s3StringToSign -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDateTime, - $strRegion, - $strCanonicalRequestHash, - ) = - logDebugParam - ( - __PACKAGE__ . '::s3StringToSign', \@_, - {name => 'strDateTime', trace => true}, - {name => 'strRegion', trace => true}, - {name => 'strCanonicalRequestHash', trace => true}, - ); - - my $strStringToSign = - AWS4_HMAC_SHA256 . "\n${strDateTime}\n" . substr($strDateTime, 0, 8) . "/${strRegion}/" . S3 . '/' . AWS4_REQUEST . "\n" . - $strCanonicalRequestHash; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strStringToSign', value => $strStringToSign, trace => true} - ); -} - -push @EXPORT, qw(s3StringToSign); - -#################################################################################################################################### -# s3AuthorizationHeader - authorization string that will be used in the HTTP "authorization" header -#################################################################################################################################### -sub s3AuthorizationHeader -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strRegion, - $strHost, - $strVerb, - $strUri, - $strQuery, - $strDateTime, - $hHeader, - $strAccessKeyId, - $strSecretAccessKey, - $strSecurityToken, - $strPayloadHash, - ) = - logDebugParam - ( - __PACKAGE__ . '::s3AuthorizationHeader', \@_, - {name => 'strRegion', trace => true}, - {name => 'strHost', trace => true}, - {name => 'strVerb', trace => true}, - {name => 'strUri', trace => true}, - {name => 'strQuery', trace => true}, - {name => 'strDateTime', trace => true}, - {name => 'hHeader', required => false, trace => true}, - {name => 'strAccessKeyId', redact => true, trace => true}, - {name => 'strSecretAccessKey', redact => true, trace => true}, - {name => 'strSecurityToken', required => false, redact => true, trace => true}, - {name => 'strPayloadHash', trace => true}, - ); - - # Delete the authorization header if it already exists. This could happen on a retry. - delete($hHeader->{&S3_HEADER_AUTHORIZATION}); - - # Add s3 required headers - $hHeader->{&S3_HEADER_HOST} = $strHost; - $hHeader->{&S3_HEADER_CONTENT_SHA256} = $strPayloadHash; - $hHeader->{&S3_HEADER_DATE} = $strDateTime; - - # Add security token if defined - if (defined($strSecurityToken)) - { - $hHeader->{&S3_HEADER_TOKEN} = $strSecurityToken; - } - - # Create authorization string - my ($strCanonicalRequest, $strSignedHeaders) = s3CanonicalRequest( - $strVerb, httpUriEncode($strUri, true), $strQuery, $hHeader, $strPayloadHash); - my $strStringToSign = s3StringToSign($strDateTime, $strRegion, cryptoHashOne('sha256', $strCanonicalRequest)); - - $hHeader->{&S3_HEADER_AUTHORIZATION} = - AWS4_HMAC_SHA256 . " Credential=${strAccessKeyId}/" . substr($strDateTime, 0, 8) . "/${strRegion}/" . S3 . qw(/) . - AWS4_REQUEST . ",SignedHeaders=${strSignedHeaders},Signature=" . hmac_sha256_hex($strStringToSign, - s3SigningKey(substr($strDateTime, 0, 8), $strRegion, $strSecretAccessKey)); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hHeader', value => $hHeader, trace => true}, - {name => 'strCanonicalRequest', value => $strCanonicalRequest, trace => true}, - {name => 'strSignedHeaders', value => $strSignedHeaders, trace => true}, - {name => 'strStringToSign', value => $strStringToSign, trace => true}, - ); -} - -push @EXPORT, qw(s3AuthorizationHeader); - -1; diff --git a/lib/pgBackRest/Storage/S3/Driver.pm b/lib/pgBackRest/Storage/S3/Driver.pm deleted file mode 100644 index f99e69dc9..000000000 --- a/lib/pgBackRest/Storage/S3/Driver.pm +++ /dev/null @@ -1,507 +0,0 @@ -#################################################################################################################################### -# S3 Storage -#################################################################################################################################### -package pgBackRest::Storage::S3::Driver; -use parent 'pgBackRest::Storage::S3::Request'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use Digest::MD5 qw(md5_base64); -use File::Basename qw(basename dirname); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::Common::String; -use pgBackRest::Common::Xml; -use pgBackRest::Storage::S3::FileRead; -use pgBackRest::Storage::S3::FileWrite; -use pgBackRest::Storage::S3::Request; -use pgBackRest::Storage::S3::Info; - -#################################################################################################################################### -# Package name constant -#################################################################################################################################### -use constant STORAGE_S3_DRIVER => __PACKAGE__; - push @EXPORT, qw(STORAGE_S3_DRIVER); - -#################################################################################################################################### -# Query constants -#################################################################################################################################### -use constant S3_QUERY_CONTINUATION_TOKEN => 'continuation-token'; -use constant S3_QUERY_DELIMITER => 'delimiter'; -use constant S3_QUERY_LIST_TYPE => 'list-type'; -use constant S3_QUERY_PREFIX => 'prefix'; - -#################################################################################################################################### -# Batch maximum size -#################################################################################################################################### -use constant S3_BATCH_MAX => 1000; - -#################################################################################################################################### -# openWrite - open a file for write -#################################################################################################################################### -sub openWrite -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - ) = - logDebugParam - ( - __PACKAGE__ . '->openWrite', \@_, - {name => 'strFile', trace => true}, - ); - - my $oFileIO = new pgBackRest::Storage::S3::FileWrite($self, $strFile); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oFileIO', value => $oFileIO, trace => true}, - ); -} - -#################################################################################################################################### -# openRead -#################################################################################################################################### -sub openRead -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->openRead', \@_, - {name => 'strFile', trace => true}, - {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, - ); - - my $oFileIO = new pgBackRest::Storage::S3::FileRead($self, $strFile, {bIgnoreMissing => $bIgnoreMissing}); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oFileIO', value => $oFileIO, trace => true}, - ); -} - -#################################################################################################################################### -# manifest -#################################################################################################################################### -sub manifest -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - $bRecurse, - $bPath, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifest', \@_, - {name => 'strPath', trace => true}, - # Optional parameters not part of the driver spec - {name => 'bRecurse', optional => true, default => true, trace => true}, - {name => 'bPath', optional => true, default => true, trace => true}, - ); - - # Determine the prefix (this is the search path within the bucket) - my $strPrefix = $strPath eq qw{/} ? undef : substr($strPath, 1) . ($bPath ? qw{/} : ''); - - # A delimiter must be used if recursion is not desired - my $strDelimiter = $bRecurse ? undef : '/'; - - # Hash to hold the manifest - my $hManifest = {}; - - # Continuation token - returned from requests where there is more data to be fetched - my $strContinuationToken; - - do - { - # Get the file list - my $oResponse = $self->request( - HTTP_VERB_GET, {hQuery => - {&S3_QUERY_LIST_TYPE => 2, &S3_QUERY_PREFIX => $strPrefix, &S3_QUERY_DELIMITER => $strDelimiter, - &S3_QUERY_CONTINUATION_TOKEN => $strContinuationToken}, strResponseType => S3_RESPONSE_TYPE_XML}); - - # Modify the prefix for file searches so the filename is not stripped off - if (defined($strPrefix) && !$bPath) - { - # If there are no paths in the prefix then undef it - if (index($strPrefix, qw{/}) == -1) - { - undef($strPrefix); - } - else - { - $strPrefix = dirname($strPrefix) . qw{/}; - } - } - - # Store files - foreach my $oFile (xmlTagChildren($oResponse, "Contents")) - { - my $strName = xmlTagText($oFile, "Key"); - - # Strip off prefix - if (defined($strPrefix)) - { - $strName = substr($strName, length($strPrefix)); - } - - $hManifest->{$strName}->{type} = 'f'; - $hManifest->{$strName}->{size} = xmlTagText($oFile, 'Size') + 0; - - # Generate paths from the name if recursing - if ($bRecurse) - { - my @stryName = split(qw{/}, $strName); - - if (@stryName > 1) - { - $strName = undef; - - for (my $iIndex = 0; $iIndex < @stryName - 1; $iIndex++) - { - $strName .= (defined($strName) ? qw{/} : '') . $stryName[$iIndex]; - $hManifest->{$strName}->{type} = 'd'; - } - } - } - } - - # Store directories - if ($bPath && !$bRecurse) - { - foreach my $oPath (xmlTagChildren($oResponse, "CommonPrefixes")) - { - my $strName = xmlTagText($oPath, "Prefix"); - - # Strip off prefix - if (defined($strPrefix)) - { - $strName = substr($strName, length($strPrefix)); - } - - # Strip off final / - $strName = substr($strName, 0, length($strName) - 1); - - $hManifest->{$strName}->{type} = 'd'; - } - } - - $strContinuationToken = xmlTagText($oResponse, "NextContinuationToken", false); - } - while (defined($strContinuationToken)); - - # Add . for the initial path (this is just for compatibility with filesystems that have directories) - if ($bPath) - { - $hManifest->{qw{.}}->{type} = 'd'; - } - - # use Data::Dumper; &log(WARN, 'MANIFEST' . Dumper($hManifest)); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hManifest', value => $hManifest, trace => true} - ); -} - -#################################################################################################################################### -# list - list a directory -#################################################################################################################################### -sub list -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - $strExpression - ) = - logDebugParam - ( - __PACKAGE__ . '->list', \@_, - {name => 'strPath', trace => true}, - {name => 'strExpression', optional => true, trace => true}, - ); - - # Use the regexp to build a prefix to shorten searches - my $strPrefix = regexPrefix($strExpression); - - # Get list using manifest function - my @stryFileList = grep( - !/^\.$/i, keys(%{$self->manifest( - $strPath . (defined($strPrefix) ? "/${strPrefix}" : ''), {bRecurse => false, bPath => !defined($strPrefix)})})); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryFileList', value => \@stryFileList, ref => true, trace => true} - ); -} - -#################################################################################################################################### -# pathCreate - directories do no exist in s3 so this is a noop -#################################################################################################################################### -sub pathCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathCreate', \@_, - {name => 'strPath', trace => true}, - ); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# pathSync - directories do not exist in s3 so this is a noop -#################################################################################################################################### -sub pathSync -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathSync', \@_, - {name => 'strPath', trace => true}, - ); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# exists - check if a file exists -#################################################################################################################################### -sub exists -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - ) = - logDebugParam - ( - __PACKAGE__ . '->exists', \@_, - {name => 'strFile', trace => true}, - ); - - # Does the path/file exist? - my $bExists = defined($self->manifest($strFile, {bRecurse => false, bPath => false})->{basename($strFile)}) ? true : false; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bExists', value => $bExists, trace => true} - ); -} - -#################################################################################################################################### -# pathExists -#################################################################################################################################### -sub pathExists -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathExists', \@_, - {name => 'strPath', trace => true}, - ); - - my $bExists = true; - - # Only check if path <> / - if ($strPath ne qw{/}) - { - # Does the path exist? - my $rhInfo = $self->manifest(dirname($strPath), {bRecurse => false, bPath => true})->{basename($strPath)}; - $bExists = defined($rhInfo) && $rhInfo->{type} eq 'd' ? true : false; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bExists', value => $bExists, trace => true} - ); -} - -#################################################################################################################################### -# info - get information about a file -#################################################################################################################################### -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strFile', trace => true}, - ); - - # Get the file - my $rhFile = $self->manifest($strFile, {bRecurse => false, bPath => false})->{basename($strFile)}; - - if (!defined($rhFile)) - { - confess &log(ERROR, "unable to get info for missing file ${strFile}", ERROR_FILE_MISSING); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oInfo', value => new pgBackRest::Storage::S3::Info($rhFile->{size}), trace => true} - ); -} - -#################################################################################################################################### -# remove -#################################################################################################################################### -sub remove -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $rstryFile, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->remove', \@_, - {name => 'rstryFile', trace => true}, - {name => 'bRecurse', optional => true, default => false, trace => true}, - ); - - # Remove a tree - if ($bRecurse) - { - my $rhManifest = $self->manifest($rstryFile); - my @stryRemoveFile; - - # Iterate all files in the manifest - foreach my $strFile (sort({$b cmp $a} keys(%{$rhManifest}))) - { - next if $rhManifest->{$strFile}->{type} eq 'd'; - push(@stryRemoveFile, "${rstryFile}/${strFile}"); - } - - # Remove files - if (@stryRemoveFile > 0) - { - $self->remove(\@stryRemoveFile); - } - } - # Only remove the specified file - else - { - # If stryFile is a scalar, convert to an array - my $rstryFileAll = ref($rstryFile) ? $rstryFile : [$rstryFile]; - - do - { - my $strFile = shift(@{$rstryFileAll}); - my $iTotal = 0; - my $strXml = XML_HEADER . 'true'; - - while (defined($strFile)) - { - $iTotal++; - $strXml .= '' . xmlFromText(substr($strFile, 1)) . ''; - - $strFile = $iTotal < S3_BATCH_MAX ? shift(@{$rstryFileAll}) : undef; - } - - $strXml .= ''; - - my $hHeader = {'content-md5' => md5_base64($strXml) . '=='}; - - # Delete a file - my $oResponse = $self->request( - HTTP_VERB_POST, - {hQuery => 'delete=', rstrBody => \$strXml, hHeader => $hHeader, strResponseType => S3_RESPONSE_TYPE_XML}); - } - while (@{$rstryFileAll} > 0); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bResult', value => true, trace => true} - ); -} - - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub capability {false} -sub className {STORAGE_S3_DRIVER} - -1; diff --git a/lib/pgBackRest/Storage/S3/FileRead.pm b/lib/pgBackRest/Storage/S3/FileRead.pm deleted file mode 100644 index d490c1ff3..000000000 --- a/lib/pgBackRest/Storage/S3/FileRead.pm +++ /dev/null @@ -1,68 +0,0 @@ -#################################################################################################################################### -# S3 File Read -#################################################################################################################################### -package pgBackRest::Storage::S3::FileRead; -use parent 'pgBackRest::Common::Http::Client'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Digest::MD5 qw(md5_base64); -use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC); -use File::Basename qw(dirname); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::Common::Xml; -use pgBackRest::Storage::Base; -use pgBackRest::Storage::S3::Request; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oDriver, - $strName, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oDriver', trace => true}, - {name => 'strName', trace => true}, - {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, - ); - - # Open file - my $self = $oDriver->request( - HTTP_VERB_GET, {strUri => $strName, strResponseType => S3_RESPONSE_TYPE_IO, bIgnoreMissing => $bIgnoreMissing}); - - # Bless with new class if file exists - if (defined($self)) - { - bless $self, $class; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub name {shift->{strName}} - -1; diff --git a/lib/pgBackRest/Storage/S3/FileWrite.pm b/lib/pgBackRest/Storage/S3/FileWrite.pm deleted file mode 100644 index 222b65be5..000000000 --- a/lib/pgBackRest/Storage/S3/FileWrite.pm +++ /dev/null @@ -1,205 +0,0 @@ -#################################################################################################################################### -# S3 File Write -#################################################################################################################################### -package pgBackRest::Storage::S3::FileWrite; -use parent 'pgBackRest::Common::Io::Base'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Digest::MD5 qw(md5_base64); -use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC); -use File::Basename qw(dirname); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Io::Handle; -use pgBackRest::Common::Log; -use pgBackRest::Common::Xml; -use pgBackRest::Storage::Base; -use pgBackRest::Storage::S3::Request; - -#################################################################################################################################### -# Constants -#################################################################################################################################### -use constant S3_BUFFER_MAX => 16777216; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oDriver, - $strName, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oDriver', trace => true}, - {name => 'strName', trace => true}, - ); - - # Create the class hash - my $self = $class->SUPER::new("'${strName}'"); - bless $self, $class; - - # Set variables - $self->{oDriver} = $oDriver; - $self->{strName} = $strName; - - # Start with an empty buffer - $self->{rtBuffer} = ''; - - # Has anything been written? - $self->{bWritten} = false; - - # How much has been written? - $self->{lSize} = 0; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# open - open the file -#################################################################################################################################### -sub open -{ - my $self = shift; - - # Request an upload id - my $oResponse = $self->{oDriver}->request( - HTTP_VERB_POST, {strUri => $self->{strName}, hQuery => 'uploads=', strResponseType => S3_RESPONSE_TYPE_XML}); - - $self->{strUploadId} = xmlTagText($oResponse, 'UploadId'); - - # Intialize the multi-part array - $self->{rstryMultiPart} = []; -} - -#################################################################################################################################### -# write - write data to a file -#################################################################################################################################### -sub write -{ - my $self = shift; - my $rtBuffer = shift; - - # Note that write has been called - $self->{bWritten} = true; - - if (defined($rtBuffer)) - { - $self->{rtBuffer} .= $$rtBuffer; - $self->{lSize} += length($$rtBuffer); - - # Wait until buffer is at least max before writing to avoid writing smaller files multi-part - if (length($self->{rtBuffer}) >= S3_BUFFER_MAX) - { - $self->flush(); - } - - return length($$rtBuffer); - } - - return 0; -} - -#################################################################################################################################### -# flush - flush whatever is in the buffer out -#################################################################################################################################### -sub flush -{ - my $self = shift; - - # Open file if it is not open already - $self->open() if !$self->opened(); - - # Put a file - $self->{oDriver}->request( - HTTP_VERB_PUT, - {strUri => $self->{strName}, - hQuery => {'partNumber' => @{$self->{rstryMultiPart}} + 1, 'uploadId' => $self->{strUploadId}}, - rstrBody => \$self->{rtBuffer}, hHeader => {'content-md5' => md5_base64($self->{rtBuffer}) . '=='}}); - - # Store the returned etag - push(@{$self->{rstryMultiPart}}, $self->{oDriver}->{hResponseHeader}{&S3_HEADER_ETAG}); - - # Clear the buffer - $self->{rtBuffer} = ''; -} - -#################################################################################################################################### -# close - close the file -#################################################################################################################################### -sub close -{ - my $self = shift; - - # Only close if something was written - if ($self->{bWritten}) - { - # Make sure close does not run again - $self->{bWritten} = false; - - # If the file is open then multipart transfer has already started and must be completed - if ($self->opened()) - { - # flush out whatever is in the buffer - $self->flush(); - - my $strXml = XML_HEADER . ''; - my $iPartNo = 0; - - foreach my $strETag (@{$self->{rstryMultiPart}}) - { - $iPartNo++; - - $strXml .= "${iPartNo}${strETag}"; - } - - $strXml .= ''; - - # Finalize file - my $oResponse = $self->{oDriver}->request( - HTTP_VERB_POST, - {strUri => $self->{strName}, hQuery => {'uploadId' => $self->{strUploadId}}, - rstrBody => \$strXml, hHeader => {'content-md5' => md5_base64($strXml) . '=='}, - strResponseType => S3_RESPONSE_TYPE_XML}); - } - # Else the file can be transmitted in one block - else - { - $self->{oDriver}->request( - HTTP_VERB_PUT, - {strUri => $self->{strName}, rstrBody => \$self->{rtBuffer}, - hHeader => {'content-md5' => md5_base64($self->{rtBuffer}) . '=='}}); - } - } - - # This is how we report the write size back to the caller. It's a bit hokey -- results are based on the object name and in all - # other cases I/O passes through Io::Handle. It doesn't seem worth fixing since it could break things and this code is being - # migrated to C anyway. - $self->resultSet(COMMON_IO_HANDLE, $self->{lSize}); - - return true; -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub opened {defined(shift->{strUploadId})} -sub name {shift->{strName}} - -1; diff --git a/lib/pgBackRest/Storage/S3/Info.pm b/lib/pgBackRest/Storage/S3/Info.pm deleted file mode 100644 index 865d1cfbb..000000000 --- a/lib/pgBackRest/Storage/S3/Info.pm +++ /dev/null @@ -1,48 +0,0 @@ -#################################################################################################################################### -# S3 File Info -#################################################################################################################################### -package pgBackRest::Storage::S3::Info; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use pgBackRest::Common::Log; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{lSize}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'lSize'}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub size {shift->{lSize}} - -1; diff --git a/lib/pgBackRest/Storage/S3/Request.pm b/lib/pgBackRest/Storage/S3/Request.pm deleted file mode 100644 index 2f97b8972..000000000 --- a/lib/pgBackRest/Storage/S3/Request.pm +++ /dev/null @@ -1,264 +0,0 @@ -#################################################################################################################################### -# S3 Request -#################################################################################################################################### -package pgBackRest::Storage::S3::Request; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use IO::Socket::SSL; - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Http::Client; -use pgBackRest::Common::Http::Common; -use pgBackRest::Common::Io::Base; -use pgBackRest::Common::Log; -use pgBackRest::Common::String; -use pgBackRest::Common::Xml; -use pgBackRest::LibC qw(:crypto); -use pgBackRest::Storage::S3::Auth; - -#################################################################################################################################### -# Constants -#################################################################################################################################### -use constant HTTP_VERB_GET => 'GET'; - push @EXPORT, qw(HTTP_VERB_GET); -use constant HTTP_VERB_POST => 'POST'; - push @EXPORT, qw(HTTP_VERB_POST); -use constant HTTP_VERB_PUT => 'PUT'; - push @EXPORT, qw(HTTP_VERB_PUT); - -use constant S3_HEADER_CONTENT_LENGTH => 'content-length'; - push @EXPORT, qw(S3_HEADER_CONTENT_LENGTH); -use constant S3_HEADER_TRANSFER_ENCODING => 'transfer-encoding'; - push @EXPORT, qw(S3_HEADER_TRANSFER_ENCODING); -use constant S3_HEADER_ETAG => 'etag'; - push @EXPORT, qw(S3_HEADER_ETAG); - -use constant S3_RESPONSE_TYPE_IO => 'io'; - push @EXPORT, qw(S3_RESPONSE_TYPE_IO); -use constant S3_RESPONSE_TYPE_NONE => 'none'; - push @EXPORT, qw(S3_RESPONSE_TYPE_NONE); -use constant S3_RESPONSE_TYPE_XML => 'xml'; - push @EXPORT, qw(S3_RESPONSE_TYPE_XML); - -use constant S3_RESPONSE_CODE_SUCCESS => 200; -use constant S3_RESPONSE_CODE_ERROR_AUTH => 403; -use constant S3_RESPONSE_CODE_ERROR_NOT_FOUND => 404; -use constant S3_RESPONSE_CODE_ERROR_RETRY_CLASS => 5; - -use constant S3_RETRY_MAX => 4; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{strBucket}, - $self->{strEndPoint}, - $self->{strRegion}, - $self->{strAccessKeyId}, - $self->{strSecretAccessKey}, - $self->{strSecurityToken}, - $self->{strHost}, - $self->{iPort}, - $self->{bVerifySsl}, - $self->{strCaPath}, - $self->{strCaFile}, - $self->{lBufferMax}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strBucket'}, - {name => 'strEndPoint'}, - {name => 'strRegion'}, - {name => 'strAccessKeyId', redact => true}, - {name => 'strSecretAccessKey', redact => true}, - {name => 'strSecurityToken', optional => true, redact => true}, - {name => 'strHost', optional => true}, - {name => 'iPort', optional => true}, - {name => 'bVerifySsl', optional => true, default => true}, - {name => 'strCaPath', optional => true}, - {name => 'strCaFile', optional => true}, - {name => 'lBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX}, - ); - - # If host is not set then it will be bucket + endpoint - $self->{strHost} = defined($self->{strHost}) ? $self->{strHost} : "$self->{strBucket}.$self->{strEndPoint}"; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# request - send a request to S3 -#################################################################################################################################### -sub request -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strVerb, - $strUri, - $hQuery, - $hHeader, - $rstrBody, - $strResponseType, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->request', \@_, - {name => 'strVerb', trace => true}, - {name => 'strUri', optional => true, default => '/', trace => true}, - {name => 'hQuery', optional => true, trace => true}, - {name => 'hHeader', optional => true, trace => true}, - {name => 'rstrBody', optional => true, trace => true}, - {name => 'strResponseType', optional => true, default => S3_RESPONSE_TYPE_NONE, trace => true}, - {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, - ); - - # Server response - my $oResponse; - - # Allow retries on S3 internal failures - my $bRetry; - my $iRetryTotal = 0; - - do - { - # Assume that a retry will not be attempted which is true in most cases - $bRetry = false; - - # Set content length and hash - $hHeader->{&S3_HEADER_CONTENT_SHA256} = defined($rstrBody) ? cryptoHashOne('sha256', $$rstrBody) : PAYLOAD_DEFAULT_HASH; - $hHeader->{&S3_HEADER_CONTENT_LENGTH} = defined($rstrBody) ? length($$rstrBody) : 0; - - # Generate authorization header - ($hHeader, my $strCanonicalRequest, my $strSignedHeaders, my $strStringToSign) = s3AuthorizationHeader( - $self->{strRegion}, "$self->{strBucket}.$self->{strEndPoint}", $strVerb, $strUri, httpQuery($hQuery), s3DateTime(), - $hHeader, $self->{strAccessKeyId}, $self->{strSecretAccessKey}, $self->{strSecurityToken}, - $hHeader->{&S3_HEADER_CONTENT_SHA256}); - - # Send the request - my $oHttpClient = new pgBackRest::Common::Http::Client( - $self->{strHost}, $strVerb, - {iPort => $self->{iPort}, strUri => $strUri, hQuery => $hQuery, hRequestHeader => $hHeader, - rstrRequestBody => $rstrBody, bVerifySsl => $self->{bVerifySsl}, strCaPath => $self->{strCaPath}, - strCaFile => $self->{strCaFile}, bResponseBodyPrefetch => $strResponseType eq S3_RESPONSE_TYPE_XML, - lBufferMax => $self->{lBufferMax}}); - - # Check response code - my $iResponseCode = $oHttpClient->responseCode(); - - if ($iResponseCode == S3_RESPONSE_CODE_SUCCESS) - { - # Save the response headers locally - $self->{hResponseHeader} = $oHttpClient->responseHeader(); - - # XML response is expected - if ($strResponseType eq S3_RESPONSE_TYPE_XML) - { - my $rtResponseBody = $oHttpClient->responseBody(); - - if ($oHttpClient->contentLength() == 0 || !defined($$rtResponseBody)) - { - confess &log(ERROR, - "response type '${strResponseType}' was requested but content length is zero or content is missing", - ERROR_PROTOCOL); - } - - $oResponse = xmlParse($$rtResponseBody); - } - # An IO object is expected for file responses - elsif ($strResponseType eq S3_RESPONSE_TYPE_IO) - { - $oResponse = $oHttpClient; - } - } - else - { - # If file was not found - if ($iResponseCode == S3_RESPONSE_CODE_ERROR_NOT_FOUND) - { - # If missing files should not be ignored then error - if (!$bIgnoreMissing) - { - confess &log(ERROR, "unable to open '${strUri}': No such file or directory", ERROR_FILE_MISSING); - } - - $bRetry = false; - } - # Else a more serious error - else - { - # Retry for S3 internal or rate-limiting errors (any 5xx error should be retried) - if (int($iResponseCode / 100) == S3_RESPONSE_CODE_ERROR_RETRY_CLASS) - { - # Increment retry total and check if retry should be attempted - $iRetryTotal++; - $bRetry = $iRetryTotal <= S3_RETRY_MAX; - - # Sleep after first retry just in case data needs to stabilize - if ($iRetryTotal > 1) - { - sleep(5); - } - } - - # If no retry then throw the error - if (!$bRetry) - { - my $rstrResponseBody = $oHttpClient->responseBody(); - - # Redact authorization header because it contains the access key - my $strRequestHeader = $oHttpClient->requestHeaderText(); - $strRequestHeader =~ s/^${\S3_HEADER_AUTHORIZATION}:.*$/${\S3_HEADER_AUTHORIZATION}: /mg; - - confess &log(ERROR, - 'S3 request error' . ($iRetryTotal > 0 ? " after " . (S3_RETRY_MAX + 1) . " tries" : '') . - " [$iResponseCode] " . $oHttpClient->responseMessage() . - "\n*** request header ***\n${strRequestHeader}" . - ($iResponseCode == S3_RESPONSE_CODE_ERROR_AUTH ? - "\n*** canonical request ***\n" . $strCanonicalRequest . - "\n*** signed headers ***\n" . $strSignedHeaders . - "\n*** string to sign ***\n" . $strStringToSign : '') . - "\n*** response header ***\n" . $oHttpClient->responseHeaderText() . - (defined($$rstrResponseBody) ? "\n*** response body ***\n${$rstrResponseBody}" : ''), - ERROR_PROTOCOL); - } - } - } - } - while ($bRetry); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oResponse', value => $oResponse, trace => true, ref => true} - ); -} - -1; diff --git a/lib/pgBackRest/Storage/Local.pm b/lib/pgBackRest/Storage/Storage.pm similarity index 55% rename from lib/pgBackRest/Storage/Local.pm rename to lib/pgBackRest/Storage/Storage.pm index 588bbbe44..7a04a6133 100644 --- a/lib/pgBackRest/Storage/Local.pm +++ b/lib/pgBackRest/Storage/Storage.pm @@ -1,9 +1,7 @@ #################################################################################################################################### -# Local Storage -# -# Implements storage functionality using drivers. +# C Storage Interface #################################################################################################################################### -package pgBackRest::Storage::Local; +package pgBackRest::Storage::Storage; use parent 'pgBackRest::Storage::Base'; use strict; @@ -12,12 +10,16 @@ use Carp qw(confess); use English '-no_match_vars'; use File::Basename qw(dirname); +use Fcntl qw(:mode); +use File::stat qw{lstat}; +use JSON::PP; use pgBackRest::Common::Exception; +use pgBackRest::Common::Io::Handle; use pgBackRest::Common::Log; -use pgBackRest::Common::String; use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::Sha; +use pgBackRest::Storage::StorageRead; +use pgBackRest::Storage::StorageWrite; #################################################################################################################################### # new @@ -26,58 +28,41 @@ sub new { my $class = shift; + # Create the class hash + my $self = {}; + bless $self, $class; + # Assign function parameters, defaults, and log debug info - my ( - $strOperation, - $strPathBase, - $oDriver, - $hRule, - $bAllowTemp, - $strTempExtension, - $strDefaultPathMode, - $strDefaultFileMode, - $lBufferMax, - $strCipherType, - $strCipherPassUser, + my $strOperation, + $self->{strType}, + $self->{strPath}, + $self->{lBufferMax}, + $self->{strDefaultPathMode}, + $self->{strDefaultFileMode}, ) = logDebugParam ( __PACKAGE__ . '->new', \@_, - {name => 'strPathBase'}, - {name => 'oDriver'}, - {name => 'hRule', optional => true}, - {name => 'bAllowTemp', optional => true, default => true}, - {name => 'strTempExtension', optional => true, default => 'tmp'}, + {name => 'strType'}, + {name => 'strPath', optional => true}, + {name => 'lBufferMax', optional => true, default => 65536}, {name => 'strDefaultPathMode', optional => true, default => '0750'}, {name => 'strDefaultFileMode', optional => true, default => '0640'}, - {name => 'lBufferMax', optional => true}, - {name => 'strCipherType', optional => true}, - {name => 'strCipherPassUser', optional => true, redact => true}, ); - # Create class - my $self = $class->SUPER::new({lBufferMax => $lBufferMax}); - bless $self, $class; + # Create C storage object + $self->{oStorageC} = pgBackRest::LibC::Storage->new($self->{strType}, $self->{strPath}); - $self->{strPathBase} = $strPathBase; - $self->{oDriver} = $oDriver; - $self->{hRule} = $hRule; - $self->{bAllowTemp} = $bAllowTemp; - $self->{strTempExtension} = $strTempExtension; - $self->{strDefaultPathMode} = $strDefaultPathMode; - $self->{strDefaultFileMode} = $strDefaultFileMode; - $self->{strCipherType} = $strCipherType; - $self->{strCipherPassUser} = $strCipherPassUser; - - if (defined($self->{strCipherType})) + # Get encryption settings + if ($self->{strType} eq '') { - require pgBackRest::Storage::Filter::CipherBlock; - pgBackRest::Storage::Filter::CipherBlock->import(); + $self->{strCipherType} = $self->{oStorageC}->cipherType(); + $self->{strCipherPass} = $self->{oStorageC}->cipherPass(); } - # Set temp extension in driver - $self->driver()->tempExtensionSet($self->{strTempExtension}) if $self->driver()->can('tempExtensionSet'); + # Create JSON object + $self->{oJSON} = JSON::PP->new()->allow_nonref(); # Return from function and log return values if any return logDebugReturn @@ -88,7 +73,7 @@ sub new } #################################################################################################################################### -# exists - check if file exists +# Check if file exists (not a path) #################################################################################################################################### sub exists { @@ -107,19 +92,62 @@ sub exists ); # Check exists - my $bExists = $self->driver()->exists($self->pathGet($strFileExp)); + my $bExists = $self->{oStorageC}->exists($strFileExp); # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'bExists', value => $bExists} + {name => 'bExists', value => $bExists ? true : false} ); } #################################################################################################################################### -# hashSize - calculate sha1 hash and size of file. If special encryption settings are required, then the file objects from -# openRead/openWrite must be passed instead of file names. +# Read a buffer from storage all at once +#################################################################################################################################### +sub get +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $xFile, + $strCipherPass, + ) = + logDebugParam + ( + __PACKAGE__ . '->get', \@_, + {name => 'xFile', required => false, trace => true}, + {name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true}, + ); + + # Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or + # if the repo has a user passphrase defined - else pass undef + my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead($xFile, {strCipherPass => $strCipherPass})) : undef; + + # Get the file contents + my $bEmpty = false; + my $tContent = $self->{oStorageC}->get($oFileIo->{oStorageCRead}); + + if (defined($tContent) && length($tContent) == 0) + { + $tContent = undef; + $bEmpty = true; + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'rtContent', value => defined($tContent) || $bEmpty ? \$tContent : undef, trace => true}, + ); +} + +#################################################################################################################################### +# Calculate sha1 hash and size of file. If special encryption settings are required, then the file objects from openRead/openWrite +# must be passed instead of file names. #################################################################################################################################### sub hashSize { @@ -144,29 +172,17 @@ sub hashSize my $lSize; # Is this an IO object or a file expression? - my $oFileIo = - defined($xFileExp) ? (ref($xFileExp) ? $xFileExp : - $self->openRead($self->pathGet($xFileExp), {bIgnoreMissing => $bIgnoreMissing})) : undef; + my $oFileIo = ref($xFileExp) ? $xFileExp : $self->openRead($xFileExp, {bIgnoreMissing => $bIgnoreMissing}); - if (defined($oFileIo)) + # Add size and sha filters + $oFileIo->{oStorageCRead}->filterAdd(COMMON_IO_HANDLE, undef); + $oFileIo->{oStorageCRead}->filterAdd(STORAGE_FILTER_SHA, undef); + + # Read the file and set results if it exists + if ($self->{oStorageC}->readDrain($oFileIo->{oStorageCRead})) { - $lSize = 0; - my $oShaIo = new pgBackRest::Storage::Filter::Sha($oFileIo); - my $lSizeRead; - - do - { - my $tContent; - $lSizeRead = $oShaIo->read(\$tContent, $self->{lBufferMax}); - $lSize += $lSizeRead; - } - while ($lSizeRead != 0); - - # Close the file - $oShaIo->close(); - - # Get the hash - $strHash = $oShaIo->result(STORAGE_FILTER_SHA); + $strHash = $oFileIo->result(STORAGE_FILTER_SHA); + $lSize = $oFileIo->result(COMMON_IO_HANDLE); } # Return from function and log return values if any @@ -179,7 +195,7 @@ sub hashSize } #################################################################################################################################### -# info - get information for path/file +# Get information for path/file #################################################################################################################################### sub info { @@ -194,19 +210,24 @@ sub info ) = logDebugParam ( - __PACKAGE__ . '::fileStat', \@_, + __PACKAGE__ . '->info', \@_, {name => 'strPathFileExp'}, {name => 'bIgnoreMissing', optional => true, default => false}, ); - # Stat the path/file - my $oInfo = $self->driver()->info($self->pathGet($strPathFileExp), {bIgnoreMissing => $bIgnoreMissing}); + my $rhInfo; + my $strJson = $self->{oStorageC}->info($strPathFileExp, $bIgnoreMissing); + + if (defined($strJson)) + { + $rhInfo = $self->{oJSON}->decode($strJson); + } # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'oInfo', value => $oInfo, trace => true} + {name => 'rhInfo', value => $rhInfo, trace => true} ); } @@ -274,16 +295,50 @@ sub linkCreate ); } - # Create the link - $self->driver()->linkCreate( - $strSourcePathFile, $strDestinationLink, {bHard => $bHard, bPathCreate => $bPathCreate, bIgnoreExists => $bIgnoreExists}); + if (!($bHard ? link($strSourcePathFile, $strDestinationLink) : symlink($strSourcePathFile, $strDestinationLink))) + { + my $strMessage = "unable to create link '${strDestinationLink}'"; + + # If parent path or source is missing + if ($OS_ERROR{ENOENT}) + { + # Check if source is missing + if (!$self->exists($strSourcePathFile)) + { + confess &log(ERROR, "${strMessage} because source '${strSourcePathFile}' does not exist", ERROR_FILE_MISSING); + } + + if (!$bPathCreate) + { + confess &log(ERROR, "${strMessage} because parent does not exist", ERROR_PATH_MISSING); + } + + # Create parent path + $self->pathCreate(dirname($strDestinationLink), {bIgnoreExists => true, bCreateParent => true}); + + # Create link + $self->linkCreate($strSourcePathFile, $strDestinationLink, {bHard => $bHard}); + } + # Else if link already exists + elsif ($OS_ERROR{EEXIST}) + { + if (!$bIgnoreExists) + { + confess &log(ERROR, "${strMessage} because it already exists", ERROR_PATH_EXISTS); + } + } + else + { + logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR); + } + } # Return from function and log return values if any return logDebugReturn($strOperation); } #################################################################################################################################### -# list - list all files/paths in path +# List all files/paths in path #################################################################################################################################### sub list { @@ -308,24 +363,12 @@ sub list ); # Get file list - my $rstryFileList = $self->driver()->list( - $self->pathGet($strPathExp), {strExpression => $strExpression, bIgnoreMissing => $bIgnoreMissing}); + my $rstryFileList = []; + my $strFileList = $self->{oStorageC}->list($strPathExp, $bIgnoreMissing, $strSortOrder eq 'forward', $strExpression); - # Apply expression if defined - if (defined($strExpression)) + if (defined($strFileList) && $strFileList ne '[]') { - @{$rstryFileList} = grep(/$strExpression/i, @{$rstryFileList}); - } - - # Reverse sort - if ($strSortOrder eq 'reverse') - { - @{$rstryFileList} = sort {$b cmp $a} @{$rstryFileList}; - } - # Normal sort - else - { - @{$rstryFileList} = sort @{$rstryFileList}; + $rstryFileList = $self->{oJSON}->decode($strFileList); } # Return from function and log return values if any @@ -337,7 +380,7 @@ sub list } #################################################################################################################################### -# manifest - build path/file/link manifest starting with base path and including all subpaths +# Build path/file/link manifest starting with base path and including all subpaths #################################################################################################################################### sub manifest { @@ -357,7 +400,7 @@ sub manifest {name => 'strFilter', optional => true, trace => true}, ); - my $hManifest = $self->driver()->manifest($self->pathGet($strPathExp), {strFilter => $strFilter}); + my $hManifest = $self->{oJSON}->decode($self->{oStorageC}->manifest($strPathExp, $strFilter)); # Return from function and log return values if any return logDebugReturn @@ -378,31 +421,33 @@ sub move my ( $strOperation, - $strSourcePathFileExp, - $strDestinationPathFileExp, + $strSourceFileExp, + $strDestinationFileExp, $bPathCreate, ) = logDebugParam ( __PACKAGE__ . '->move', \@_, - {name => 'strSourcePathExp'}, - {name => 'strDestinationPathExp'}, - {name => 'bPathCreate', optional => true, default => false, trace => true}, + {name => 'strSourceFileExp'}, + {name => 'strDestinationFileExp'}, ); - # Set operation variables - $self->driver()->move( - $self->pathGet($strSourcePathFileExp), $self->pathGet($strDestinationPathFileExp), {bPathCreate => $bPathCreate}); + # Get source and destination paths + my $strSourceFile = $self->pathGet($strSourceFileExp); + my $strDestinationFile = $self->pathGet($strDestinationFileExp); + + # Move the file + if (!rename($strSourceFile, $strDestinationFile)) + { + logErrorResult(ERROR_FILE_MOVE, "unable to move '${strSourceFile}' to '${strDestinationFile}'", $OS_ERROR); + } # Return from function and log return values if any - return logDebugReturn - ( - $strOperation - ); + return logDebugReturn($strOperation); } #################################################################################################################################### -# openRead - open file for reading +# Open file for reading #################################################################################################################################### sub openRead { @@ -423,31 +468,25 @@ sub openRead {name => 'xFileExp'}, {name => 'bIgnoreMissing', optional => true, default => false}, {name => 'rhyFilter', optional => true}, - {name => 'strCipherPass', optional => true, redact => true}, + {name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true}, ); # Open the file - my $oFileIo = $self->driver()->openRead($self->pathGet($xFileExp), {bIgnoreMissing => $bIgnoreMissing}); + my $oFileIo = pgBackRest::LibC::StorageRead->new($self->{oStorageC}, $xFileExp, $bIgnoreMissing); - # Apply filters if file is defined - if (defined($oFileIo)) + # If cipher is set then decryption is the first filter applied to the read + if (defined($self->cipherType())) { - # If cipher is set then add the filter so that decryption is the first filter applied to the data read before any of the - # other filters - if (defined($self->cipherType())) - { - $oFileIo = &STORAGE_FILTER_CIPHER_BLOCK->new( - $oFileIo, $self->cipherType(), defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser(), - {strMode => STORAGE_DECRYPT}); - } + $oFileIo->filterAdd(STORAGE_FILTER_CIPHER_BLOCK, $self->{oJSON}->encode([false, $self->cipherType(), $strCipherPass])); + } - # Apply any other filters - if (defined($rhyFilter)) + # Apply any other filters + if (defined($rhyFilter)) + { + foreach my $rhFilter (@{$rhyFilter}) { - foreach my $rhFilter (@{$rhyFilter}) - { - $oFileIo = $rhFilter->{strClass}->new($oFileIo, @{$rhFilter->{rxyParam}}); - } + $oFileIo->filterAdd( + $rhFilter->{strClass}, defined($rhFilter->{rxyParam}) ? $self->{oJSON}->encode($rhFilter->{rxyParam}) : undef); } } @@ -455,12 +494,12 @@ sub openRead return logDebugReturn ( $strOperation, - {name => 'oFileIo', value => $oFileIo, trace => true}, + {name => 'oFileIo', value => new pgBackRest::Storage::StorageRead($self, $oFileIo), trace => true}, ); } #################################################################################################################################### -# openWrite - open file for writing +# Open file for writing #################################################################################################################################### sub openWrite { @@ -487,44 +526,43 @@ sub openWrite {name => 'strMode', optional => true, default => $self->{strDefaultFileMode}}, {name => 'strUser', optional => true}, {name => 'strGroup', optional => true}, - {name => 'lTimestamp', optional => true}, + {name => 'lTimestamp', optional => true, default => '0'}, {name => 'bAtomic', optional => true, default => false}, {name => 'bPathCreate', optional => true, default => false}, {name => 'rhyFilter', optional => true}, - {name => 'strCipherPass', optional => true, redact => true}, + {name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true}, ); # Open the file - my $oFileIo = $self->driver()->openWrite($self->pathGet($xFileExp), - {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lTimestamp, bPathCreate => $bPathCreate, - bAtomic => $bAtomic}); - - # If cipher is set then add filter so that encryption is performed just before the data is actually written - if (defined($self->cipherType())) - { - $oFileIo = &STORAGE_FILTER_CIPHER_BLOCK->new( - $oFileIo, $self->cipherType(), defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser()); - } + my $oFileIo = pgBackRest::LibC::StorageWrite->new( + $self->{oStorageC}, $xFileExp, oct($strMode), $strUser, $strGroup, $lTimestamp, $bAtomic, $bPathCreate); # Apply any other filters if (defined($rhyFilter)) { - foreach my $rhFilter (reverse(@{$rhyFilter})) + foreach my $rhFilter (@{$rhyFilter}) { - $oFileIo = $rhFilter->{strClass}->new($oFileIo, @{$rhFilter->{rxyParam}}); + $oFileIo->filterAdd( + $rhFilter->{strClass}, defined($rhFilter->{rxyParam}) ? $self->{oJSON}->encode($rhFilter->{rxyParam}) : undef); } } + # If cipher is set then encryption is the last filter applied to the write + if (defined($self->cipherType())) + { + $oFileIo->filterAdd(STORAGE_FILTER_CIPHER_BLOCK, $self->{oJSON}->encode([true, $self->cipherType(), $strCipherPass])); + } + # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'oFileIo', value => $oFileIo, trace => true}, + {name => 'oFileIo', value => new pgBackRest::Storage::StorageWrite($self, $oFileIo), trace => true}, ); } #################################################################################################################################### -# owner - change ownership of path/file +# Change ownership of path/file #################################################################################################################################### sub owner { @@ -546,8 +584,64 @@ sub owner {name => 'strGroup', required => false} ); - # Set ownership - $self->driver()->owner($self->pathGet($strPathFileExp), {strUser => $strUser, strGroup => $strGroup}); + # Only proceed if user or group was specified + if (defined($strUser) || defined($strGroup)) + { + my $strPathFile = $self->pathGet($strPathFileExp); + my $strMessage = "unable to set ownership for '${strPathFile}'"; + my $iUserId; + my $iGroupId; + + # If the user or group is not defined then get it by stat'ing the file. This is because the chown function requires that + # both user and group be set. + my $oStat = lstat($strPathFile); + + if (!defined($oStat)) + { + confess &log(ERROR, "unable to stat '${strPathFile}': No such file or directory", ERROR_FILE_MISSING); + } + + if (!defined($strUser)) + { + $iUserId = $oStat->uid; + } + + if (!defined($strGroup)) + { + $iGroupId = $oStat->gid; + } + + # Lookup user if specified + if (defined($strUser)) + { + $iUserId = getpwnam($strUser); + + if (!defined($iUserId)) + { + logErrorResult(ERROR_FILE_OWNER, "${strMessage} because user '${strUser}' does not exist"); + } + } + + # Lookup group if specified + if (defined($strGroup)) + { + $iGroupId = getgrnam($strGroup); + + if (!defined($iGroupId)) + { + logErrorResult(ERROR_FILE_OWNER, "${strMessage} because group '${strGroup}' does not exist"); + } + } + + # Set ownership on the file if the user or group would be changed + if ($iUserId != $oStat->uid || $iGroupId != $oStat->gid) + { + if (!chown($iUserId, $iGroupId, $strPathFile)) + { + logErrorResult(ERROR_FILE_OWNER, "${strMessage}", $OS_ERROR); + } + } + } # Return from function and log return values if any return logDebugReturn @@ -557,7 +651,73 @@ sub owner } #################################################################################################################################### -# pathCreate - create path +# Generate an absolute path from an absolute base path and a relative path +#################################################################################################################################### +sub pathAbsolute +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strBasePath, + $strPath + ) = + logDebugParam + ( + __PACKAGE__ . '->pathAbsolute', \@_, + {name => 'strBasePath', trace => true}, + {name => 'strPath', trace => true} + ); + + # Working variables + my $strAbsolutePath; + + # If the path is already absolute + if (index($strPath, '/') == 0) + { + $strAbsolutePath = $strPath; + } + # Else make it absolute using the base path + else + { + # Make sure the absolute path is really absolute + if (index($strBasePath, '/') != 0 || index($strBasePath, '/..') != -1) + { + confess &log(ERROR, "${strBasePath} is not an absolute path", ERROR_PATH_TYPE); + } + + while (index($strPath, '..') == 0) + { + $strBasePath = dirname($strBasePath); + $strPath = substr($strPath, 2); + + if (index($strPath, '/') == 0) + { + $strPath = substr($strPath, 1); + } + } + + $strAbsolutePath = "${strBasePath}/${strPath}"; + } + + # Make sure the result is really an absolute path + if (index($strAbsolutePath, '/') != 0 || index($strAbsolutePath, '/..') != -1) + { + confess &log(ERROR, "result ${strAbsolutePath} was not an absolute path", ERROR_PATH_TYPE); + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'strAbsolutePath', value => $strAbsolutePath, trace => true} + ); +} + +#################################################################################################################################### +# Create a path #################################################################################################################################### sub pathCreate { @@ -576,14 +736,13 @@ sub pathCreate ( __PACKAGE__ . '->pathCreate', \@_, {name => 'strPathExp'}, - {name => 'strMode', optional => true, default => $self->{strDefaultPathMode}}, + {name => 'strMode', optional => true}, {name => 'bIgnoreExists', optional => true, default => false}, {name => 'bCreateParent', optional => true, default => false}, ); # Create path - $self->driver()->pathCreate( - $self->pathGet($strPathExp), {strMode => $strMode, bIgnoreExists => $bIgnoreExists, bCreateParent => $bCreateParent}); + $self->{oStorageC}->pathCreate($strPathExp, $strMode, $bIgnoreExists, $bCreateParent); # Return from function and log return values if any return logDebugReturn @@ -593,7 +752,7 @@ sub pathCreate } #################################################################################################################################### -# pathExists - check if path exists +# Check if path exists #################################################################################################################################### sub pathExists { @@ -612,18 +771,18 @@ sub pathExists ); # Check exists - my $bExists = $self->driver()->pathExists($self->pathGet($strPathExp)); + my $bExists = $self->{oStorageC}->pathExists($strPathExp); # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'bExists', value => $bExists} + {name => 'bExists', value => $bExists ? true : false} ); } #################################################################################################################################### -# pathGet - resolve a path expression into an absolute path +# Resolve a path expression into an absolute path #################################################################################################################################### sub pathGet { @@ -633,99 +792,16 @@ sub pathGet my ( $strOperation, - $strPathExp, # File that that needs to be translated to a path - $bTemp, # Return the temp file name + $strPathExp, ) = logDebugParam - ( - __PACKAGE__ . '->pathGet', \@_, - {name => 'strPathExp', required => false, trace => true}, - {name => 'bTemp', optional => true, default => false, trace => true}, - ); + ( + __PACKAGE__ . '->pathGet', \@_, + {name => 'strPathExp'}, + ); - # Path and file to be returned - my $strPath; - my $strFile; - - # Is this an absolute path type? - my $bAbsolute = false; - - if (defined($strPathExp) && index($strPathExp, qw(/)) == 0) - { - $bAbsolute = true; - $strPath = $strPathExp; - } - else - { - # Is it a rule type - if (defined($strPathExp) && index($strPathExp, qw(<)) == 0) - { - # Extract the rule type - my $iPos = index($strPathExp, qw(>)); - - if ($iPos == -1) - { - confess &log(ASSERT, "found < but not > in '${strPathExp}'"); - } - - my $strType = substr($strPathExp, 0, $iPos + 1); - - # Extract the filename - if ($iPos < length($strPathExp) - 1) - { - $strFile = substr($strPathExp, $iPos + 2); - } - - # Lookup the rule - if (!defined($self->{hRule}->{$strType})) - { - confess &log(ASSERT, "storage rule '${strType}' does not exist"); - } - - # If rule is a ref then call the function - if (ref($self->{hRule}->{$strType})) - { - $strPath = $self->pathBase(); - $strFile = $self->{hRule}{$strType}{fnRule}->($strType, $strFile, $self->{hRule}{$strType}{xData}); - } - # Else get the path - else - { - $strPath = $self->pathBase() . ($self->pathBase() =~ /\/$/ ? '' : qw{/}) . $self->{hRule}->{$strType}; - } - } - # Else it must be relative - else - { - $strPath = $self->pathBase(); - $strFile = $strPathExp; - } - } - - # Make sure a temp file is valid for this type and file - if ($bTemp) - { - # Error when temp files are not allowed - if (!$self->{bAllowTemp}) - { - confess &log(ASSERT, "temp file not supported for storage '" . $self->pathBase() . "'"); - } - - # The file must be defined - if (!$bAbsolute) - { - if (!defined($strFile)) - { - confess &log(ASSERT, 'file part must be defined when temp file specified'); - } - } - } - - # Combine path and file - $strPath .= defined($strFile) ? ($strPath =~ /\/$/ ? '' : qw{/}) . "${strFile}" : ''; - - # Add temp extension - $strPath .= $bTemp ? ".$self->{strTempExtension}" : ''; + # Check exists + my $strPath = $self->{oStorageC}->pathGet($strPathExp); # Return from function and log return values if any return logDebugReturn @@ -735,6 +811,35 @@ sub pathGet ); } +#################################################################################################################################### +# Remove path and all files below it +#################################################################################################################################### +sub pathRemove +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPathExp, + $bIgnoreMissing, + $bRecurse, + ) = + logDebugParam + ( + __PACKAGE__ . '->pathRemove', \@_, + {name => 'strPathExp'}, + {name => 'bIgnoreMissing', optional => true, default => true}, + {name => 'bRecurse', optional => true, default => false}, + ); + + $self->{oStorageC}->pathRemove($strPathExp, $bIgnoreMissing, $bRecurse); + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + #################################################################################################################################### # Sync path so newly added file entries are not lost #################################################################################################################################### @@ -754,14 +859,52 @@ sub pathSync {name => 'strPathExp'}, ); - $self->driver()->pathSync($self->pathGet($strPathExp)); + $self->{oStorageC}->pathSync($strPathExp); # Return from function and log return values if any return logDebugReturn($strOperation); } #################################################################################################################################### -# remove - remove path/file +# put - writes a buffer out to storage all at once +#################################################################################################################################### +sub put +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $xFile, + $xContent, + $strCipherPass, + ) = + logDebugParam + ( + __PACKAGE__ . '->put', \@_, + {name => 'xFile', trace => true}, + {name => 'xContent', required => false, trace => true}, + {name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), trace => true, redact => true}, + ); + + # Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or if + # the repo has a user passphrase defined - else pass undef + my $oFileIo = ref($xFile) ? $xFile : $self->openWrite($xFile, {strCipherPass => $strCipherPass}); + + # Write the content + my $lSize = $self->{oStorageC}->put($oFileIo->{oStorageCWrite}, ref($xContent) ? $$xContent : $xContent); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'lSize', value => $lSize, trace => true}, + ); +} + +#################################################################################################################################### +# Remove file #################################################################################################################################### sub remove { @@ -771,40 +914,23 @@ sub remove my ( $strOperation, - $xstryPathFileExp, + $xFileExp, $bIgnoreMissing, - $bRecurse, ) = logDebugParam ( __PACKAGE__ . '->remove', \@_, - {name => 'xstryPathFileExp'}, + {name => 'xFileExp'}, {name => 'bIgnoreMissing', optional => true, default => true}, - {name => 'bRecurse', optional => true, default => false, trace => true}, ); - # Evaluate expressions for all files - my @stryPathFileExp; - - if (ref($xstryPathFileExp)) + foreach my $strFileExp (ref($xFileExp) ? @{$xFileExp} : ($xFileExp)) { - foreach my $strPathFileExp (@{$xstryPathFileExp}) - { - push(@stryPathFileExp, $self->pathGet($strPathFileExp)); - } + $self->{oStorageC}->remove($strFileExp, $bIgnoreMissing); } - # Remove path(s)/file(s) - my $bRemoved = $self->driver()->remove( - ref($xstryPathFileExp) ? \@stryPathFileExp : $self->pathGet($xstryPathFileExp), - {bIgnoreMissing => $bIgnoreMissing, bRecurse => $bRecurse}); - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bRemoved', value => $bRemoved} - ); + return logDebugReturn($strOperation); } #################################################################################################################################### @@ -818,27 +944,27 @@ sub encrypted my ( $strOperation, - $strFileName, + $strFileExp, $bIgnoreMissing, ) = logDebugParam ( __PACKAGE__ . '->encrypted', \@_, - {name => 'strFileName'}, + {name => 'strFileExp'}, {name => 'bIgnoreMissing', optional => true, default => false}, ); - my $tMagicSignature; my $bEncrypted = false; # Open the file via the driver - my $oFile = $self->driver()->openRead($self->pathGet($strFileName), {bIgnoreMissing => $bIgnoreMissing}); + my $oFileIo = new pgBackRest::Storage::StorageRead( + $self, pgBackRest::LibC::StorageRead->new($self->{oStorageC}, $strFileExp, $bIgnoreMissing)); # If the file does not exist because we're ignoring missing (else it would error before this is executed) then determine if it # should be encrypted based on the repo - if (!defined($oFile)) + if (!$oFileIo->open()) { - if (defined($self->{strCipherType})) + if (defined($self->cipherType())) { $bEncrypted = true; } @@ -846,18 +972,14 @@ sub encrypted else { # If the file does exist, then read the magic signature - my $lSizeRead = $oFile->read(\$tMagicSignature, length(CIPHER_MAGIC)); + my $tMagicSignature = ''; + my $lSizeRead = $oFileIo->read(\$tMagicSignature, length(CIPHER_MAGIC)); + $oFileIo->close(); - # Close the file handle - $oFile->close(); - - # If the file is able to be read, then if it is encrypted it must at least have the magic signature, even if it were - # originally a 0 byte file - if (($lSizeRead > 0) && substr($tMagicSignature, 0, length(CIPHER_MAGIC)) eq CIPHER_MAGIC) + if (substr($tMagicSignature, 0, length(CIPHER_MAGIC)) eq CIPHER_MAGIC) { $bEncrypted = true; } - } # Return from function and log return values if any @@ -887,38 +1009,22 @@ sub encryptionValid {name => 'bEncrypted'}, ); - my $bValid = true; - - # If encryption is set on the file then make sure the repo is encrypted and visa-versa - if ($bEncrypted) - { - if (!defined($self->{strCipherType})) - { - $bValid = false; - } - } - else - { - if (defined($self->{strCipherType})) - { - $bValid = false; - } - } + my $bValid = ($bEncrypted && defined($self->cipherType())) || (!$bEncrypted && !defined($self->cipherType())); # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'bValid', value => $bValid} + {name => 'bValid', value => $bValid ? true : false} ); } #################################################################################################################################### # Getters #################################################################################################################################### -sub pathBase {shift->{strPathBase}} -sub driver {shift->{oDriver}} +sub capability {shift->type() eq STORAGE_POSIX} +sub type {shift->{oStorageC}->type()} sub cipherType {shift->{strCipherType}} -sub cipherPassUser {shift->{strCipherPassUser}} +sub cipherPassUser {shift->{strCipherPass}} 1; diff --git a/lib/pgBackRest/Storage/StorageRead.pm b/lib/pgBackRest/Storage/StorageRead.pm new file mode 100644 index 000000000..e8a9ac3d0 --- /dev/null +++ b/lib/pgBackRest/Storage/StorageRead.pm @@ -0,0 +1,190 @@ +#################################################################################################################################### +# C Storage Read Interface +#################################################################################################################################### +package pgBackRest::Storage::StorageRead; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use File::Basename qw(dirname); +use Fcntl qw(:mode); +use File::stat qw{lstat}; +use JSON::PP; + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; +use pgBackRest::Storage::Base; + +#################################################################################################################################### +# new +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Create the class hash + my $self = {}; + bless $self, $class; + + # Assign function parameters, defaults, and log debug info + ( + my $strOperation, + $self->{oStorage}, + $self->{oStorageCRead}, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'oStorage'}, + {name => 'oStorageCRead'}, + ); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self} + ); +} + +#################################################################################################################################### +# Open the file +#################################################################################################################################### +sub open +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my ($strOperation) = logDebugParam(__PACKAGE__ . '->open'); + + return logDebugReturn + ( + $strOperation, + {name => 'bResult', value => $self->{oStorageCRead}->open() ? true : false, trace => true}, + ); +} + +#################################################################################################################################### +# Read data +#################################################################################################################################### +sub read +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my ( + $strOperation, + $rtBuffer, + $iSize, + ) = + logDebugParam + ( + __PACKAGE__ . '->read', \@_, + {name => 'rtBuffer'}, + {name => 'iSize'}, + ); + + # Read if not eof + my $iActualSize = 0; + + if (!$self->eof()) + { + my $tBuffer = $self->{oStorageCRead}->read($iSize); + $iActualSize = length($tBuffer); + $$rtBuffer .= $tBuffer; + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'iActualSize', value => $iActualSize} + ); +} + +#################################################################################################################################### +# Is the file at eof? +#################################################################################################################################### +sub eof +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my ($strOperation) = logDebugParam(__PACKAGE__ . '->eof'); + + return logDebugReturn + ( + $strOperation, + {name => 'bResult', value => $self->{oStorageCRead}->eof() ? true : false, trace => true}, + ); +} + +#################################################################################################################################### +# Close the file +#################################################################################################################################### +sub close +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my ($strOperation) = logDebugParam(__PACKAGE__ . '->close'); + + $self->{oStorageCRead}->close(); + + return logDebugReturn + ( + $strOperation, + {name => 'bResult', value => true, trace => true}, + ); +} + +#################################################################################################################################### +# Get a filter result +#################################################################################################################################### +sub result +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strClass, + ) = + logDebugParam + ( + __PACKAGE__ . '->result', \@_, + {name => 'strClass'}, + ); + + my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCRead}->result($strClass)); + + return logDebugReturn + ( + $strOperation, + {name => 'xResult', value => $xResult, trace => true}, + ); +} + +#################################################################################################################################### +# Get all filter results +#################################################################################################################################### +sub resultAll +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my ($strOperation) = logDebugParam(__PACKAGE__ . '->resultAll'); + + my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCRead}->resultAll()); + + return logDebugReturn + ( + $strOperation, + {name => 'xResultAll', value => $xResult, trace => true}, + ); +} + +1; diff --git a/lib/pgBackRest/Storage/StorageWrite.pm b/lib/pgBackRest/Storage/StorageWrite.pm new file mode 100644 index 000000000..fc39b2eee --- /dev/null +++ b/lib/pgBackRest/Storage/StorageWrite.pm @@ -0,0 +1,163 @@ +#################################################################################################################################### +# C Storage Write Interface +#################################################################################################################################### +package pgBackRest::Storage::StorageWrite; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use File::Basename qw(dirname); +use Fcntl qw(:mode); +use File::stat qw{lstat}; +use JSON::PP; + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; +use pgBackRest::Storage::Base; + +#################################################################################################################################### +# new +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Create the class hash + my $self = {}; + bless $self, $class; + + # Assign function parameters, defaults, and log debug info + ( + my $strOperation, + $self->{oStorage}, + $self->{oStorageCWrite}, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'oStorage'}, + {name => 'oStorageCWrite'}, + ); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self} + ); +} + +#################################################################################################################################### +# Open the file +#################################################################################################################################### +sub open +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my ($strOperation) = logDebugParam(__PACKAGE__ . '->open'); + + $self->{oStorageCWrite}->open(); + + return logDebugReturn + ( + $strOperation, + {name => 'bResult', value => true, trace => true}, + ); +} + +#################################################################################################################################### +# Write data +#################################################################################################################################### +sub write +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my ( + $strOperation, + $rtBuffer, + ) = + logDebugParam + ( + __PACKAGE__ . '->write', \@_, + {name => 'rtBuffer'}, + ); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'iActualSize', value => $self->{oStorageCWrite}->write($$rtBuffer)} + ); +} + +#################################################################################################################################### +# Close the file +#################################################################################################################################### +sub close +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my ($strOperation) = logDebugParam(__PACKAGE__ . '->close'); + + $self->{oStorageCWrite}->close(); + + return logDebugReturn + ( + $strOperation, + {name => 'bResult', value => true, trace => true}, + ); +} + +#################################################################################################################################### +# Get a filter result +#################################################################################################################################### +sub result +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strClass, + ) = + logDebugParam + ( + __PACKAGE__ . '->result', \@_, + {name => 'strClass'}, + ); + + my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCWrite}->result($strClass)); + + return logDebugReturn + ( + $strOperation, + {name => 'xResult', value => $xResult, trace => true}, + ); +} + +#################################################################################################################################### +# Get all filter results +#################################################################################################################################### +sub resultAll +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my ($strOperation) = logDebugParam(__PACKAGE__ . '->resultAll'); + + my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCWrite}->resultAll()); + + return logDebugReturn + ( + $strOperation, + {name => 'xResultAll', value => $xResult, trace => true}, + ); +} + +1; diff --git a/libc/LibC.h b/libc/LibC.h index dc67404bb..1ce623db6 100644 --- a/libc/LibC.h +++ b/libc/LibC.h @@ -181,3 +181,15 @@ Free memory context in destructors ***********************************************************************************************************************************/ #define MEM_CONTEXT_XS_DESTROY(memContext) \ memContextFree(memContext) + +/*********************************************************************************************************************************** +Create new string from an SV +***********************************************************************************************************************************/ +#define STR_NEW_SV(param) \ + (SvOK(param) ? strNewN(SvPV_nolen(param), SvCUR(param)) : NULL) + +/*********************************************************************************************************************************** +Create const buffer from an SV +***********************************************************************************************************************************/ +#define BUF_CONST_SV(param) \ + (SvOK(param) ? BUF(SvPV_nolen(param), SvCUR(param)) : NULL) diff --git a/libc/LibC.xs b/libc/LibC.xs index 377dfbcda..42f9617e4 100644 --- a/libc/LibC.xs +++ b/libc/LibC.xs @@ -50,6 +50,7 @@ These includes are from the src directory. There is no Perl-specific code in th ***********************************************************************************************************************************/ #include "common/crypto/common.h" #include "common/error.h" +#include "common/io/io.h" #include "common/lock.h" #include "config/config.h" #include "config/define.h" @@ -69,9 +70,11 @@ XSH includes These includes define data structures that are required for the C to Perl interface but are not part of the regular C source. ***********************************************************************************************************************************/ -#include "xs/crypto/cipherBlock.xsh" #include "xs/crypto/hash.xsh" #include "xs/common/encode.xsh" +#include "xs/storage/storage.xsh" +#include "xs/storage/storageRead.xsh" +#include "xs/storage/storageWrite.xsh" /*********************************************************************************************************************************** Module definition @@ -97,8 +100,9 @@ INCLUDE: xs/common/lock.xs INCLUDE: xs/config/config.xs INCLUDE: xs/config/configTest.xs INCLUDE: xs/config/define.xs -INCLUDE: xs/crypto/cipherBlock.xs INCLUDE: xs/crypto/hash.xs INCLUDE: xs/crypto/random.xs INCLUDE: xs/postgres/pageChecksum.xs INCLUDE: xs/storage/storage.xs +INCLUDE: xs/storage/storageRead.xs +INCLUDE: xs/storage/storageWrite.xs diff --git a/libc/Makefile.PL b/libc/Makefile.PL index 0632ea186..c3eb150e7 100644 --- a/libc/Makefile.PL +++ b/libc/Makefile.PL @@ -42,6 +42,7 @@ my @stryCFile = ( 'LibC.c', + 'command/backup/pageChecksum.c', 'command/command.c', 'common/compress/gzip/common.c', 'common/compress/gzip/compress.c', @@ -61,8 +62,14 @@ my @stryCFile = 'common/io/filter/group.c', 'common/io/filter/size.c', 'common/io/handleWrite.c', + 'common/io/http/cache.c', + 'common/io/http/client.c', + 'common/io/http/common.c', + 'common/io/http/header.c', + 'common/io/http/query.c', 'common/io/io.c', 'common/io/read.c', + 'common/io/tls/client.c', 'common/io/write.c', 'common/lock.c', 'common/log.c', @@ -86,10 +93,19 @@ my @stryCFile = 'config/load.c', 'config/parse.c', 'perl/config.c', + 'protocol/client.c', + 'protocol/command.c', + 'protocol/helper.c', + 'protocol/parallel.c', + 'protocol/parallelJob.c', + 'protocol/server.c', 'postgres/pageChecksum.c', 'storage/posix/read.c', 'storage/posix/storage.c', 'storage/posix/write.c', + 'storage/s3/read.c', + 'storage/s3/storage.c', + 'storage/s3/write.c', 'storage/helper.c', 'storage/read.c', 'storage/storage.c', diff --git a/libc/build/lib/pgBackRestLibC/Build.pm b/libc/build/lib/pgBackRestLibC/Build.pm index a3409d69c..a6140bf23 100644 --- a/libc/build/lib/pgBackRestLibC/Build.pm +++ b/libc/build/lib/pgBackRestLibC/Build.pm @@ -19,8 +19,6 @@ use lib dirname($0) . '/../lib'; use pgBackRest::Common::Log; use pgBackRest::Common::String; -use pgBackRest::Storage::Local; -use pgBackRest::Storage::Posix::Driver; use pgBackRest::Version; use pgBackRestBuild::Build; @@ -28,6 +26,9 @@ use pgBackRestBuild::Build::Common; use pgBackRestBuild::Config::Data; use pgBackRestBuild::Error::Data; +use pgBackRestTest::Common::Storage; +use pgBackRestTest::Common::StoragePosix; + #################################################################################################################################### # Perl function and constant exports #################################################################################################################################### @@ -74,8 +75,6 @@ my $rhExport = { &BLD_EXPORTTYPE_SUB => [qw( pageChecksum - pageChecksumBufferTest - pageChecksumTest )], }, @@ -149,7 +148,7 @@ my $rhExport = 'storage' => { &BLD_EXPORTTYPE_SUB => [qw( - storagePosixPathRemove + storageRepoFree )], }, @@ -172,8 +171,8 @@ sub buildXsAll my @stryBuilt; # Storage - my $oStorage = new pgBackRest::Storage::Local( - $strBuildPath, new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false})); + my $oStorage = new pgBackRestTest::Common::Storage( + $strBuildPath, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); # Build interface file my $strContent = diff --git a/libc/typemap b/libc/typemap index b96e4b2c9..14e50c447 100644 --- a/libc/typemap +++ b/libc/typemap @@ -1,2 +1,3 @@ -pgBackRest::LibC::Cipher::Block T_PTROBJ -pgBackRest::LibC::Crypto::Hash T_PTROBJ +pgBackRest::LibC::Storage T_PTROBJ +pgBackRest::LibC::StorageRead T_PTROBJ +pgBackRest::LibC::StorageWrite T_PTROBJ diff --git a/libc/xs/crypto/cipherBlock.xs b/libc/xs/crypto/cipherBlock.xs deleted file mode 100644 index 174252bf9..000000000 --- a/libc/xs/crypto/cipherBlock.xs +++ /dev/null @@ -1,114 +0,0 @@ -#################################################################################################################################### -# Block Cipher Perl Exports -# -# XS wrapper for functions in cipher/block.c. -#################################################################################################################################### - -MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::Cipher::Block - -#################################################################################################################################### -pgBackRest::LibC::Cipher::Block -new(class, mode, type, key, keySize, digest = NULL) - const char *class - U32 mode - const char *type - unsigned char *key - I32 keySize - const char *digest -CODE: - RETVAL = NULL; - - CHECK(type != NULL); - CHECK(key != NULL); - CHECK(keySize != 0); - - // Not much point to this but it keeps the var from being unused - if (strcmp(class, PACKAGE_NAME_LIBC "::Cipher::Block") != 0) - croak("unexpected class name '%s'", class); - - MEM_CONTEXT_XS_NEW_BEGIN("cipherBlockXs") - { - RETVAL = memNew(sizeof(CipherBlockXs)); - RETVAL->memContext = MEM_CONTEXT_XS(); - - RETVAL->pxPayload = cipherBlockNew(mode, cipherType(STR(type)), BUF(key, keySize), digest == NULL ? NULL : STR(digest)); - } - MEM_CONTEXT_XS_NEW_END(); -OUTPUT: - RETVAL - -#################################################################################################################################### -SV * -process(self, source) - pgBackRest::LibC::Cipher::Block self - SV *source -CODE: - RETVAL = NULL; - - MEM_CONTEXT_XS_BEGIN(self->memContext) - { - STRLEN tSize; - const unsigned char *sourcePtr = (const unsigned char *)SvPV(source, tSize); - - RETVAL = NEWSV(0, ioBufferSize()); - SvPOK_only(RETVAL); - - if (tSize > 0) - { - size_t outBufferUsed = 0; - - do - { - SvGROW(RETVAL, outBufferUsed + ioBufferSize()); - Buffer *outBuffer = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL) + outBufferUsed, ioBufferSize()); - - ioFilterProcessInOut(self->pxPayload, BUF(sourcePtr, tSize), outBuffer); - outBufferUsed += bufUsed(outBuffer); - } - while (ioFilterInputSame(self->pxPayload)); - - SvCUR_set(RETVAL, outBufferUsed); - } - else - SvCUR_set(RETVAL, 0); - } - MEM_CONTEXT_XS_END(); -OUTPUT: - RETVAL - -#################################################################################################################################### -SV * -flush(self) - pgBackRest::LibC::Cipher::Block self -CODE: - RETVAL = NULL; - - MEM_CONTEXT_XS_BEGIN(self->memContext) - { - RETVAL = NEWSV(0, ioBufferSize()); - SvPOK_only(RETVAL); - - size_t outBufferUsed = 0; - - do - { - SvGROW(RETVAL, outBufferUsed + ioBufferSize()); - Buffer *outBuffer = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL) + outBufferUsed, ioBufferSize()); - - ioFilterProcessInOut(self->pxPayload, NULL, outBuffer); - outBufferUsed += bufUsed(outBuffer); - } - while (!ioFilterDone(self->pxPayload)); - - SvCUR_set(RETVAL, outBufferUsed); - } - MEM_CONTEXT_XS_END(); -OUTPUT: - RETVAL - -#################################################################################################################################### -void -DESTROY(self) - pgBackRest::LibC::Cipher::Block self -CODE: - MEM_CONTEXT_XS_DESTROY(self->memContext); diff --git a/libc/xs/crypto/cipherBlock.xsh b/libc/xs/crypto/cipherBlock.xsh deleted file mode 100644 index 33b9aae0a..000000000 --- a/libc/xs/crypto/cipherBlock.xsh +++ /dev/null @@ -1,17 +0,0 @@ -/*********************************************************************************************************************************** -Block Cipher XS Header -***********************************************************************************************************************************/ -#include "common/assert.h" -#include "common/crypto/cipherBlock.h" -#include "common/io/io.h" -#include "common/memContext.h" - -// Encrypt/decrypt modes -#define CIPHER_MODE_ENCRYPT ((int)cipherModeEncrypt) -#define CIPHER_MODE_DECRYPT ((int)cipherModeDecrypt) - -typedef struct CipherBlockXs -{ - MemContext *memContext; - IoFilter *pxPayload; -} CipherBlockXs, *pgBackRest__LibC__Cipher__Block; diff --git a/libc/xs/crypto/hash.xs b/libc/xs/crypto/hash.xs index df82037dd..171735ab7 100644 --- a/libc/xs/crypto/hash.xs +++ b/libc/xs/crypto/hash.xs @@ -4,72 +4,6 @@ # XS wrapper for functions in cipher/hash.c. #################################################################################################################################### -MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::Crypto::Hash - -#################################################################################################################################### -pgBackRest::LibC::Crypto::Hash -new(class, type) - const char *class - const char *type -CODE: - RETVAL = NULL; - - // Don't warn when class param is used - (void)class; - - MEM_CONTEXT_XS_NEW_BEGIN("cryptoHashXs") - { - RETVAL = memNew(sizeof(CryptoHashXs)); - RETVAL->memContext = MEM_CONTEXT_XS(); - RETVAL->pxPayload = cryptoHashNew(strNew(type)); - } - MEM_CONTEXT_XS_NEW_END(); -OUTPUT: - RETVAL - -#################################################################################################################################### -void -process(self, message) - pgBackRest::LibC::Crypto::Hash self - SV *message -CODE: - MEM_CONTEXT_XS_TEMP_BEGIN() - { - STRLEN messageSize; - const void *messagePtr = SvPV(message, messageSize); - - if (messageSize > 0) - ioFilterProcessIn(self->pxPayload, BUF(messagePtr, messageSize)); - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -SV * -result(self) - pgBackRest::LibC::Crypto::Hash self -CODE: - RETVAL = NULL; - - MEM_CONTEXT_XS_TEMP_BEGIN() - { - const String *hash = varStr(ioFilterResult(self->pxPayload)); - - RETVAL = newSV(strSize(hash)); - SvPOK_only(RETVAL); - strcpy((char *)SvPV_nolen(RETVAL), strPtr(hash)); - SvCUR_set(RETVAL, strSize(hash)); - } - MEM_CONTEXT_XS_TEMP_END(); -OUTPUT: - RETVAL - -#################################################################################################################################### -void -DESTROY(self) - pgBackRest::LibC::Crypto::Hash self -CODE: - MEM_CONTEXT_XS_DESTROY(self->memContext); - MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC #################################################################################################################################### diff --git a/libc/xs/crypto/hash.xsh b/libc/xs/crypto/hash.xsh index d1cb474e5..98b367e48 100644 --- a/libc/xs/crypto/hash.xsh +++ b/libc/xs/crypto/hash.xsh @@ -2,11 +2,3 @@ Cryptographic Hashes XS Header ***********************************************************************************************************************************/ #include "common/crypto/hash.h" -#include "common/io/filter/filter.intern.h" -#include "common/memContext.h" - -typedef struct CryptoHashXs -{ - MemContext *memContext; - IoFilter *pxPayload; -} CryptoHashXs, *pgBackRest__LibC__Crypto__Hash; diff --git a/libc/xs/postgres/pageChecksum.xs b/libc/xs/postgres/pageChecksum.xs index 74eb02e86..b4c7c05a6 100644 --- a/libc/xs/postgres/pageChecksum.xs +++ b/libc/xs/postgres/pageChecksum.xs @@ -20,42 +20,3 @@ CODE: ERROR_XS_END(); OUTPUT: RETVAL - -bool -pageChecksumTest(page, blockNo, pageSize, ignoreWalId, ignoreWalOffset) - const char *page - U32 blockNo - U32 pageSize - U32 ignoreWalId - U32 ignoreWalOffset -CODE: - RETVAL = false; - - ERROR_XS_BEGIN() - { - RETVAL = pageChecksumTest( - (const unsigned char *)page, blockNo, pageSize, ignoreWalId, ignoreWalOffset); - } - ERROR_XS_END(); -OUTPUT: - RETVAL - -bool -pageChecksumBufferTest(pageBuffer, pageBufferSize, blockNoBegin, pageSize, ignoreWalId, ignoreWalOffset) - const char *pageBuffer - U32 pageBufferSize - U32 blockNoBegin - U32 pageSize - U32 ignoreWalId - U32 ignoreWalOffset -CODE: - RETVAL = false; - - ERROR_XS_BEGIN() - { - RETVAL = pageChecksumBufferTest( - (const unsigned char *)pageBuffer, pageBufferSize, blockNoBegin, pageSize, ignoreWalId, ignoreWalOffset); - } - ERROR_XS_END(); -OUTPUT: - RETVAL diff --git a/libc/xs/storage/storage.xs b/libc/xs/storage/storage.xs index 120c153b5..546a203a0 100644 --- a/libc/xs/storage/storage.xs +++ b/libc/xs/storage/storage.xs @@ -2,19 +2,409 @@ # Storage Exports # ---------------------------------------------------------------------------------------------------------------------------------- +MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::Storage + +#################################################################################################################################### +pgBackRest::LibC::Storage +new(class, type, path) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + const String *class = STR_NEW_SV($arg); + const String *type = STR_NEW_SV($arg); + const String *path = STR_NEW_SV($arg); +CODE: + CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::Storage")); + + if (strEqZ(type, "")) + { + memContextSwitch(MEM_CONTEXT_XS_OLD()); + RETVAL = storagePosixNew( + path == NULL ? STRDEF("/") : path, STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); + storagePathEnforceSet((Storage *)RETVAL, false); + memContextSwitch(MEM_CONTEXT_XS_TEMP()); + } + else if (strEqZ(type, "")) + { + CHECK(path == NULL); + RETVAL = (Storage *)storageRepoWrite(); + } + else if (strEqZ(type, "")) + { + CHECK(path == NULL); + + memContextSwitch(MEM_CONTEXT_XS_OLD()); + RETVAL = storagePosixNew(cfgOptionStr(cfgOptPgPath), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); + storagePathEnforceSet((Storage *)RETVAL, false); + memContextSwitch(MEM_CONTEXT_XS_TEMP()); + } + else + THROW_FMT(AssertError, "unexpected storage type '%s'", strPtr(type)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +bool +copy(self, source, destination) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead source + pgBackRest::LibC::StorageWrite destination +CODE: + RETVAL = storageCopyNP(source, destination); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +bool +exists(self, fileExp) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self + const String *fileExp = STR_NEW_SV($arg); +CODE: + RETVAL = storageExistsNP(self, fileExp); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +SV * +get(self, read) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead read +CODE: + RETVAL = NULL; + Buffer *buffer = storageGetNP(read); + + if (buffer != NULL) + { + if (bufUsed(buffer) == 0) + RETVAL = newSVpv("", 0); + else + RETVAL = newSVpv((char *)bufPtr(buffer), bufUsed(buffer)); + } +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +SV * +info(self, pathExp, ignoreMissing) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self + const String *pathExp = STR_NEW_SV($arg); + bool ignoreMissing +CODE: + RETVAL = NULL; + + StorageInfo info = storageInfoP(self, pathExp, .ignoreMissing = ignoreMissing); + + if (info.exists) + { + String *json = storageManifestXsInfo(NULL, &info); + RETVAL = newSVpv((char *)strPtr(json), strSize(json)); + } +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +SV * +list(self, pathExp, ignoreMissing, sortAsc, expression) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self + const String *pathExp = STR_NEW_SV($arg); + bool ignoreMissing + bool sortAsc + const String *expression = STR_NEW_SV($arg); +CODE: + StringList *fileList = strLstSort( + storageListP(self, pathExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false, + .expression = expression), sortAsc ? sortOrderAsc : sortOrderDesc); + + const String *fileListJson = jsonFromVar(varNewVarLst(varLstNewStrLst(fileList)), 0); + + RETVAL = newSVpv(strPtr(fileListJson), strSize(fileListJson)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +SV * +manifest(self, pathExp, filter=NULL) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self + const String *pathExp = STR_NEW_SV($arg); + const String *filter = STR_NEW_SV($arg); +CODE: + StorageManifestXsCallbackData data = {.storage = self, .json = strNew("{"), .pathRoot = pathExp, .filter = filter}; + + // If a path is specified + StorageInfo info = storageInfoP(self, pathExp, .ignoreMissing = true); + + if (!info.exists || info.type == storageTypePath) + { + storageInfoListP( + self, data.pathRoot, storageManifestXsCallback, &data, + .errorOnMissing = storageFeature(self, storageFeaturePath) ? true : false); + } + // Else a file is specified + else + { + info.name = strBase(storagePath(self, pathExp)); + strCat(data.json, strPtr(storageManifestXsInfo(NULL, &info))); + } + + strCat(data.json, "}"); + + RETVAL = newSVpv((char *)strPtr(data.json), strSize(data.json)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +pathCreate(self, pathExp, mode, ignoreExists, createParent) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self + const String *pathExp = STR_NEW_SV($arg); + const String *mode = STR_NEW_SV($arg); + bool ignoreExists + bool createParent +CODE: + if (storageFeature(self, storageFeaturePath)) + storagePathCreateP( + self, pathExp, .mode = mode ? cvtZToIntBase(strPtr(mode), 8) : 0, .errorOnExists = !ignoreExists, + .noParentCreate = !createParent); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +bool +pathExists(self, pathExp) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self + const String *pathExp = STR_NEW_SV($arg); +CODE: + RETVAL = true; + + if (storageFeature(self, storageFeaturePath)) + RETVAL = storagePathExistsNP(self, pathExp); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +SV * +pathGet(self, pathExp) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self + const String *pathExp = STR_NEW_SV($arg); +CODE: + String *path = storagePathNP(self, pathExp); + RETVAL = newSVpv((char *)strPtr(path), strSize(path)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +pathRemove(self, pathExp, ignoreMissing, recurse) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self + const String *pathExp = STR_NEW_SV($arg); + bool ignoreMissing + bool recurse +CODE: + storagePathRemoveP( + self, pathExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false, .recurse = recurse); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +pathSync(self, pathExp) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self + const String *pathExp = STR_NEW_SV($arg); +CODE: + storagePathSyncNP(self, pathExp); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +UV +put(self, write, buffer) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageWrite write + const Buffer *buffer = BUF_CONST_SV($arg); +CODE: + storagePutNP(write, buffer); + RETVAL = buffer ? bufUsed(buffer) : 0; +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +bool +readDrain(self, read) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead read +CODE: + RETVAL = false; + + // Read and discard all IO (this is useful for processing filters) + if (ioReadOpen(storageReadIo(read))) + { + Buffer *buffer = bufNew(ioBufferSize()); + + do + { + ioRead(storageReadIo(read), buffer); + bufUsedZero(buffer); + } + while (!ioReadEof(storageReadIo(read))); + + ioReadClose(storageReadIo(read)); + RETVAL = true; + } +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +remove(self, fileExp, ignoreMissing) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self + const String *fileExp = STR_NEW_SV($arg); + bool ignoreMissing +CODE: + storageRemoveP(self, fileExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +const char * +cipherType(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +CODE: + if (cfgOptionStr(cfgOptRepoCipherType) == NULL || cipherType(cfgOptionStr(cfgOptRepoCipherType)) == cipherTypeNone) + RETVAL = NULL; + else + RETVAL = strPtr(cfgOptionStr(cfgOptRepoCipherType)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +const char * +cipherPass(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +CODE: + RETVAL = strPtr(cfgOptionStr(cfgOptRepoCipherPass)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +const char * +type(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self +CODE: + RETVAL = strPtr(storageType(self)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC #################################################################################################################################### void -storagePosixPathRemove(path, errorOnMissing, recurse) - const char *path - bool errorOnMissing - bool recurse +storageRepoFree() CODE: - MEM_CONTEXT_XS_TEMP_BEGIN() - { - storagePathRemoveP( - storagePosixNew(strNew("/"), 0640, 750, true, NULL), strNew(path), .errorOnMissing = errorOnMissing, - .recurse = recurse); - } - MEM_CONTEXT_XS_TEMP_END(); + storageHelperFree(); diff --git a/libc/xs/storage/storage.xsh b/libc/xs/storage/storage.xsh new file mode 100644 index 000000000..bc8835fdc --- /dev/null +++ b/libc/xs/storage/storage.xsh @@ -0,0 +1,222 @@ +/*********************************************************************************************************************************** +Storage XS Header +***********************************************************************************************************************************/ +#include "command/backup/pageChecksum.h" +#include "common/assert.h" +#include "common/compress/gzip/compress.h" +#include "common/compress/gzip/decompress.h" +#include "common/crypto/cipherBlock.h" +#include "common/io/filter/size.h" +#include "common/memContext.h" +#include "common/type/convert.h" +#include "common/type/json.h" +#include "postgres/interface.h" +#include "storage/helper.h" +#include "storage/storage.intern.h" + +typedef Storage *pgBackRest__LibC__Storage; + +/*********************************************************************************************************************************** +Manifest callback +***********************************************************************************************************************************/ +typedef struct StorageManifestXsCallbackData +{ + const Storage *storage; + const String *pathRoot; + const String *path; + String *json; + const String *filter; +} StorageManifestXsCallbackData; + +String * +storageManifestXsInfo(const String *path, const StorageInfo *info) +{ + String *json = strNew(""); + + if (info->name != NULL) + { + strCatFmt( + json, "%s:", strPtr(jsonFromStr(path == NULL ? info->name : strNewFmt("%s/%s", strPtr(path), strPtr(info->name))))); + } + + strCatFmt(json, "{\"group\":%s,\"user\":%s,\"type\":\"", strPtr(jsonFromStr(info->group)), strPtr(jsonFromStr(info->user))); + + switch (info->type) + { + case storageTypeFile: + { + strCatFmt( + json, "f\",\"mode\":\"%04o\",\"modification_time\":%" PRId64 ",\"size\":%" PRIu64 "}", info->mode, + (int64_t)info->timeModified, info->size); + break; + } + + case storageTypeLink: + { + strCatFmt(json, "l\",\"link_destination\":%s}", strPtr(jsonFromStr(info->linkDestination))); + break; + } + + case storageTypePath: + { + strCatFmt(json, "d\",\"mode\":\"%04o\"}", info->mode); + break; + } + } + + return json; +} + +void +storageManifestXsCallback(void *callbackData, const StorageInfo *info) +{ + StorageManifestXsCallbackData *data = (StorageManifestXsCallbackData *)callbackData; + + if (data->path == NULL || !strEqZ(info->name, ".")) + { + if (!strEqZ(info->name, ".") && data->filter && !strEq(data->filter, info->name)) + return; + + if (strSize(data->json) != 1) + strCat(data->json, ","); + + strCat(data->json, strPtr(storageManifestXsInfo(data->path, info))); + + if (info->type == storageTypePath) + { + if (!strEqZ(info->name, ".")) + { + StorageManifestXsCallbackData dataSub = + { + .storage = data->storage, + .json = data->json, + .pathRoot = data->pathRoot, + .path = data->path == NULL ? info->name : strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)), + }; + + storageInfoListNP( + dataSub.storage, strNewFmt("%s/%s", strPtr(dataSub.pathRoot), strPtr(dataSub.path)), storageManifestXsCallback, + &dataSub); + } + } + } +} + +/*********************************************************************************************************************************** +Add IO filter +***********************************************************************************************************************************/ +void +storageFilterXsAdd(IoFilterGroup *filterGroup, const String *filter, const String *paramJson) +{ + VariantList *paramList = paramJson ? jsonToVarLst(paramJson) : NULL; + + if (strEqZ(filter, "pgBackRest::Storage::Filter::CipherBlock")) + { + ioFilterGroupAdd( + filterGroup, + cipherBlockNew( + varUInt64Force(varLstGet(paramList, 0)) ? cipherModeEncrypt : cipherModeDecrypt, + cipherType(varStr(varLstGet(paramList, 1))), BUFSTR(varStr(varLstGet(paramList, 2))), NULL)); + } + else if (strEqZ(filter, "pgBackRest::Storage::Filter::Sha")) + { + ioFilterGroupAdd(filterGroup, cryptoHashNew(HASH_TYPE_SHA1_STR)); + } + else if (strEqZ(filter, "pgBackRest::Common::Io::Handle")) + { + ioFilterGroupAdd(filterGroup, ioSizeNew()); + } + else if (strEqZ(filter, "pgBackRest::Backup::Filter::PageChecksum")) + { + ioFilterGroupAdd( + filterGroup, + pageChecksumNew( + varUInt64Force(varLstGet(paramList, 0)), PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, + (uint64_t)varUIntForce(varLstGet(paramList, 1)) << 32 | varUIntForce(varLstGet(paramList, 2)))); + } + else if (strEqZ(filter, "pgBackRest::Storage::Filter::Gzip")) + { + if (strEqZ(varStr(varLstGet(paramList, 0)), "compress")) + { + ioFilterGroupAdd( + filterGroup, gzipCompressNew(varUIntForce(varLstGet(paramList, 2)), varBoolForce(varLstGet(paramList, 1)))); + } + else + { + ioFilterGroupAdd(filterGroup, gzipDecompressNew(varBoolForce(varLstGet(paramList, 1)))); + } + } + else + THROW_FMT(AssertError, "unable to add invalid filter '%s'", strPtr(filter)); +} + +/*********************************************************************************************************************************** +Get result from IO filter +***********************************************************************************************************************************/ +String * +storageFilterXsResult(const IoFilterGroup *filterGroup, const String *filter) +{ + const Variant *result; + + if (strEqZ(filter, "pgBackRest::Storage::Filter::Sha")) + { + result = ioFilterGroupResult(filterGroup, CRYPTO_HASH_FILTER_TYPE_STR); + } + else if (strEqZ(filter, "pgBackRest::Common::Io::Handle")) + { + result = ioFilterGroupResult(filterGroup, SIZE_FILTER_TYPE_STR); + } + else if (strEqZ(filter, "pgBackRest::Backup::Filter::PageChecksum")) + { + result = ioFilterGroupResult(filterGroup, PAGE_CHECKSUM_FILTER_TYPE_STR); + } + else + THROW_FMT(AssertError, "unable to get result for invalid filter '%s'", strPtr(filter)); + + if (result == NULL) + THROW_FMT(AssertError, "unable to find result for filter '%s'", strPtr(filter)); + + return jsonFromVar(result, 0); +} + +/*********************************************************************************************************************************** +Get results from all IO filters +***********************************************************************************************************************************/ +String * +storageFilterXsResultAll(const IoFilterGroup *filterGroup) +{ + const VariantList *filterList = kvKeyList(varKv(ioFilterGroupResultAll(filterGroup))); + String *result = strNew("{"); + + for (unsigned int filterIdx = 0; filterIdx < varLstSize(filterList); filterIdx++) + { + const String *filter = varStr(varLstGet(filterList, filterIdx)); + const String *filterPerl = NULL; + + if (strEq(filter, CRYPTO_HASH_FILTER_TYPE_STR)) + { + filterPerl = STRDEF("pgBackRest::Storage::Filter::Sha"); + } + else if (strEq(filter, SIZE_FILTER_TYPE_STR)) + { + filterPerl = STRDEF("pgBackRest::Common::Io::Handle"); + } + else if (strEq(filter, PAGE_CHECKSUM_FILTER_TYPE_STR)) + { + filterPerl = STRDEF("pgBackRest::Backup::Filter::PageChecksum"); + } + + if (filterPerl != NULL) + { + if (strSize(result) > 1) + strCat(result, ","); + + strCatFmt( + result, "%s:%s", strPtr(jsonFromStr(filterPerl)), strPtr(storageFilterXsResult(filterGroup, filterPerl))); + } + } + + strCat(result, "}"); + + return result; +} diff --git a/libc/xs/storage/storageRead.xs b/libc/xs/storage/storageRead.xs new file mode 100644 index 000000000..29738dbd7 --- /dev/null +++ b/libc/xs/storage/storageRead.xs @@ -0,0 +1,159 @@ +# ---------------------------------------------------------------------------------------------------------------------------------- +# Storage Read Exports +# ---------------------------------------------------------------------------------------------------------------------------------- + +MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::StorageRead + +#################################################################################################################################### +pgBackRest::LibC::StorageRead +new(class, storage, file, ignoreMissing) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + const String *class = STR_NEW_SV($arg); + pgBackRest::LibC::Storage storage + const String *file = STR_NEW_SV($arg); + bool ignoreMissing +CODE: + CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::StorageRead")); + + RETVAL = storageReadMove(storageNewReadP(storage, file, .ignoreMissing = ignoreMissing), MEM_CONTEXT_XS_OLD()); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +filterAdd(self, filter, param) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead self + const String *filter = STR_NEW_SV($arg); + const String *param = STR_NEW_SV($arg); +CODE: + IoFilterGroup *filterGroup = ioReadFilterGroup(storageReadIo(self)); + storageFilterXsAdd(filterGroup, filter, param); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +bool +open(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead self +CODE: + RETVAL = ioReadOpen(storageReadIo(self)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +SV * +read(self, bufferSize) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead self + U32 bufferSize +CODE: + RETVAL = NEWSV(0, bufferSize); + SvPOK_only(RETVAL); + + Buffer *bufferRead = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL), bufferSize); + ioRead(storageReadIo(self), bufferRead); + + SvCUR_set(RETVAL, bufUsed(bufferRead)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +bool +eof(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead self +CODE: + RETVAL = ioReadEof(storageReadIo(self)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +close(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead self +CODE: + ioReadClose(storageReadIo(self)); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +const char * +result(self, filter) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead self + const String *filter = STR_NEW_SV($arg); +CODE: + RETVAL = strPtr(storageFilterXsResult(ioReadFilterGroup(storageReadIo(self)), filter)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +const char * +resultAll(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead self +CODE: + RETVAL = strPtr(storageFilterXsResultAll(ioReadFilterGroup(storageReadIo(self)))); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +DESTROY(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageRead self +CODE: + storageReadFree(self); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); diff --git a/libc/xs/storage/storageRead.xsh b/libc/xs/storage/storageRead.xsh new file mode 100644 index 000000000..c2112cbc9 --- /dev/null +++ b/libc/xs/storage/storageRead.xsh @@ -0,0 +1,4 @@ +/*********************************************************************************************************************************** +Storage Read XS Header +***********************************************************************************************************************************/ +typedef StorageRead *pgBackRest__LibC__StorageRead; diff --git a/libc/xs/storage/storageWrite.xs b/libc/xs/storage/storageWrite.xs new file mode 100644 index 000000000..4632599b3 --- /dev/null +++ b/libc/xs/storage/storageWrite.xs @@ -0,0 +1,146 @@ +# ---------------------------------------------------------------------------------------------------------------------------------- +# Storage Write Exports +# ---------------------------------------------------------------------------------------------------------------------------------- + +MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::StorageWrite + +#################################################################################################################################### +pgBackRest::LibC::StorageWrite +new(class, storage, file, mode, user, group, timeModified, atomic, pathCreate) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + const String *class = STR_NEW_SV($arg); + pgBackRest::LibC::Storage storage + const String *file = STR_NEW_SV($arg); + U32 mode + const String *user = STR_NEW_SV($arg); + const String *group = STR_NEW_SV($arg); + IV timeModified + bool atomic + bool pathCreate +CODE: + CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::StorageWrite")); + + RETVAL = storageWriteMove( + storageNewWriteP( + storage, file, .modeFile = mode, .user = user, .group = group, .timeModified = (time_t)timeModified, + .noCreatePath = storageFeature(storage, storageFeaturePath) ? !pathCreate : false, .noSyncPath = !atomic, + .noAtomic = !atomic), + MEM_CONTEXT_XS_OLD()); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +filterAdd(self, filter, param) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageWrite self + const String *filter = STR_NEW_SV($arg); + const String *param = STR_NEW_SV($arg); +CODE: + IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(self)); + storageFilterXsAdd(filterGroup, filter, param); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +open(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageWrite self +CODE: + ioWriteOpen(storageWriteIo(self)); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +UV +write(self, buffer) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageWrite self + const Buffer *buffer = BUF_CONST_SV($arg); +CODE: + ioWrite(storageWriteIo(self), buffer); + RETVAL = bufUsed(buffer); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +close(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageWrite self +CODE: + ioWriteClose(storageWriteIo(self)); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +const char * +result(self, filter) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageWrite self + const String *filter = STR_NEW_SV($arg); +CODE: + RETVAL = strPtr(storageFilterXsResult(ioWriteFilterGroup(storageWriteIo(self)), filter)); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +const char * +resultAll(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageWrite self +CODE: + RETVAL = strPtr(storageFilterXsResultAll(ioWriteFilterGroup(storageWriteIo(self)))); +OUTPUT: + RETVAL +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +void +DESTROY(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::StorageWrite self +CODE: + storageWriteFree(self); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); diff --git a/libc/xs/storage/storageWrite.xsh b/libc/xs/storage/storageWrite.xsh new file mode 100644 index 000000000..5d4a3fd29 --- /dev/null +++ b/libc/xs/storage/storageWrite.xsh @@ -0,0 +1,4 @@ +/*********************************************************************************************************************************** +Storage Write XS Header +***********************************************************************************************************************************/ +typedef StorageWrite *pgBackRest__LibC__StorageWrite; diff --git a/src/Makefile.in b/src/Makefile.in index f2cc83dc7..c192be074 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -52,6 +52,7 @@ SRCS = \ command/archive/push/protocol.c \ command/archive/push/push.c \ command/backup/common.c \ + command/backup/pageChecksum.c \ command/expire/expire.c \ command/help/help.c \ command/info/info.c \ @@ -433,7 +434,7 @@ main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push. perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/config.c -o perl/config.o -perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/pageChecksum.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/cipherBlock.xsh ../libc/xs/crypto/hash.xsh +perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h command/backup/pageChecksum.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/exec.c -o perl/exec.o postgres/interface.o: postgres/interface.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h postgres/interface.h postgres/interface/version.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h diff --git a/src/command/restore/file.c b/src/command/restore/file.c index cd1fb547c..626c8741e 100644 --- a/src/command/restore/file.c +++ b/src/command/restore/file.c @@ -90,7 +90,8 @@ restoreFile( ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR)); Buffer *buffer = bufNew(ioBufferSize()); - ioReadOpen(read); + + CHECK(ioReadOpen(read)); do { diff --git a/src/common/error.c b/src/common/error.c index b87fc893d..3109b7232 100644 --- a/src/common/error.c +++ b/src/common/error.c @@ -31,7 +31,7 @@ struct ErrorType /*********************************************************************************************************************************** Maximum allowed number of nested try blocks ***********************************************************************************************************************************/ -#define ERROR_TRY_MAX 32 +#define ERROR_TRY_MAX 64 /*********************************************************************************************************************************** States for each try diff --git a/src/perl/embed.auto.c b/src/perl/embed.auto.c index 0573dea42..63b516986 100644 --- a/src/perl/embed.auto.c +++ b/src/perl/embed.auto.c @@ -269,9 +269,6 @@ static const EmbeddedModule embeddedModule[] = "use pgBackRest::Config::Config;\n" "use pgBackRest::Protocol::Helper;\n" "use pgBackRest::Protocol::Storage::Helper;\n" - "use pgBackRest::Storage::Base;\n" - "use pgBackRest::Storage::Filter::Gzip;\n" - "use pgBackRest::Storage::Filter::Sha;\n" "use pgBackRest::Storage::Helper;\n" "\n\n\n\n\n" "sub archiveGetCheck\n" @@ -391,7 +388,6 @@ static const EmbeddedModule embeddedModule[] = "use pgBackRest::Manifest;\n" "use pgBackRest::Protocol::Storage::Helper;\n" "use pgBackRest::Storage::Base;\n" - "use pgBackRest::Storage::Filter::Gzip;\n" "use pgBackRest::Storage::Helper;\n" "\n\n\n\n" "use constant ARCHIVE_INFO_FILE => 'archive.info';\n" @@ -708,8 +704,6 @@ static const EmbeddedModule embeddedModule[] = "\n\n" "my $iSysIdOffset = $strDbVersion >= PG_VERSION_93 ? PG_WAL_SYSTEM_ID_OFFSET_GTE_93 : PG_WAL_SYSTEM_ID_OFFSET_LT_93;\n" "\n\n" - "my $tBlock;\n" - "\n\n" "if (!storageRepo()->encryptionValid(storageRepo()->encrypted($strArchiveFilePath)))\n" "{\n" "confess &log(ERROR, \"encryption incompatible for '$strArchiveFilePath'\" .\n" @@ -719,10 +713,12 @@ static const EmbeddedModule embeddedModule[] = "my $oFileIo = storageRepo()->openRead(\n" "$strArchiveFilePath,\n" "{rhyFilter => $strArchiveFile =~ ('\\.' . COMPRESS_EXT . '$') ?\n" - "[{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef,\n" + "[{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]}] : undef,\n" "strCipherPass => $self->cipherPassSub()});\n" + "$oFileIo->open();\n" "\n" - "$oFileIo->read(\\$tBlock, 512, true);\n" + "my $tBlock;\n" + "$oFileIo->read(\\$tBlock, 512);\n" "$oFileIo->close();\n" "\n\n" "my ($iMagic, $iFlag, $junk, $ullDbSysId) = unpack('SSa' . $iSysIdOffset . 'Q', $tBlock);\n" @@ -901,8 +897,6 @@ static const EmbeddedModule embeddedModule[] = "use pgBackRest::Protocol::Storage::Helper;\n" "use pgBackRest::Common::Io::Handle;\n" "use pgBackRest::Storage::Base;\n" - "use pgBackRest::Storage::Filter::Gzip;\n" - "use pgBackRest::Storage::Filter::Sha;\n" "use pgBackRest::Storage::Helper;\n" "use pgBackRest::Version;\n" "\n\n\n\n" @@ -1074,7 +1068,7 @@ static const EmbeddedModule embeddedModule[] = "if ($cType eq 'd')\n" "{\n" "logDebugMisc($strOperation, \"remove path ${strName}\");\n" - "$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strName}\", {bRecurse => true});\n" + "$oStorageRepo->pathRemove(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strName}\", {bRecurse => true});\n" "}\n" "\n" "else\n" @@ -1154,7 +1148,7 @@ static const EmbeddedModule embeddedModule[] = "storageRepo()->pathCreate(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strPath}\", {bIgnoreExists => true});\n" "}\n" "\n" - "if (storageRepo()->driver()->capability(STORAGE_CAPABILITY_LINK))\n" + "if (storageRepo()->capability(STORAGE_CAPABILITY_LINK))\n" "{\n" "for my $strTarget ($oBackupManifest->keys(MANIFEST_SECTION_BACKUP_TARGET))\n" "{\n" @@ -1516,7 +1510,7 @@ static const EmbeddedModule embeddedModule[] = "&log(WARN, \"aborted backup ${strAbortedBackup} cannot be resumed: ${strReason}\");\n" "&log(TEST, TEST_BACKUP_NORESUME);\n" "\n" - "$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strAbortedBackup}\", {bRecurse => true});\n" + "$oStorageRepo->pathRemove(STORAGE_REPO_BACKUP . \"/${strAbortedBackup}\", {bRecurse => true});\n" "undef($oAbortedManifest);\n" "}\n" "\n" @@ -1716,7 +1710,9 @@ static const EmbeddedModule embeddedModule[] = "\n\n" "if ($bCompress)\n" "{\n" - "push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP});\n" + "push(\n" + "@{$rhyFilter},\n" + "{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, cfgOption(CFGOPT_COMPRESS_LEVEL)]});\n" "}\n" "\n\n\n" "my $oDestinationFileIo = $oStorageRepo->openWrite(\n" @@ -1786,7 +1782,7 @@ static const EmbeddedModule embeddedModule[] = "$oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef, $lTimestampStop + 0);\n" "$oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, $strBackupLabel);\n" "\n\n" - "if ($oStorageRepo->driver()->capability(STORAGE_CAPABILITY_PATH_SYNC))\n" + "if ($oStorageRepo->capability(STORAGE_CAPABILITY_PATH_SYNC))\n" "{\n" "\n" "$oStorageRepo->pathSync(STORAGE_REPO_BACKUP . \"/${strBackupLabel}\");\n" @@ -1814,11 +1810,11 @@ static const EmbeddedModule embeddedModule[] = "{'strCipherPass' => $strCipherPassManifest}),\n" "$oStorageRepo->openWrite(\n" "\"${strHistoryPath}/${strBackupLabel}.manifest.\" . COMPRESS_EXT,\n" - "{rhyFilter => [{strClass => STORAGE_FILTER_GZIP}],\n" + "{rhyFilter => [{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, 9]}],\n" "bPathCreate => true, bAtomic => true,\n" "strCipherPass => defined($strCipherPassManifest) ? $strCipherPassManifest : undef}));\n" "\n\n" - "if ($oStorageRepo->driver()->capability(STORAGE_CAPABILITY_PATH_SYNC))\n" + "if ($oStorageRepo->capability(STORAGE_CAPABILITY_PATH_SYNC))\n" "{\n" "$oStorageRepo->pathSync(STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY);\n" "$oStorageRepo->pathSync($strHistoryPath);\n" @@ -1826,7 +1822,7 @@ static const EmbeddedModule embeddedModule[] = "\n\n" "$oStorageRepo->remove(STORAGE_REPO_BACKUP . qw(/) . LINK_LATEST);\n" "\n" - "if (storageRepo()->driver()->capability(STORAGE_CAPABILITY_LINK))\n" + "if (storageRepo()->capability(STORAGE_CAPABILITY_LINK))\n" "{\n" "$oStorageRepo->linkCreate(\n" "STORAGE_REPO_BACKUP . \"/${strBackupLabel}\", STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST, {bRelative => true});\n" @@ -1834,7 +1830,7 @@ static const EmbeddedModule embeddedModule[] = "\n\n" "$oBackupInfo->add($oBackupManifest);\n" "\n\n" - "if ($oStorageRepo->driver()->capability(STORAGE_CAPABILITY_PATH_SYNC))\n" + "if ($oStorageRepo->capability(STORAGE_CAPABILITY_PATH_SYNC))\n" "{\n" "$oStorageRepo->pathSync(STORAGE_REPO_BACKUP);\n" "}\n" @@ -2072,17 +2068,15 @@ static const EmbeddedModule embeddedModule[] = "use File::Basename qw(dirname);\n" "use Storable qw(dclone);\n" "\n" - "use pgBackRest::Backup::Filter::PageChecksum;\n" "use pgBackRest::Common::Exception;\n" "use pgBackRest::Common::Io::Handle;\n" "use pgBackRest::Common::Log;\n" "use pgBackRest::Common::String;\n" + "use pgBackRest::Config::Config;\n" "use pgBackRest::DbVersion;\n" "use pgBackRest::Manifest;\n" "use pgBackRest::Protocol::Storage::Helper;\n" "use pgBackRest::Storage::Base;\n" - "use pgBackRest::Storage::Filter::Gzip;\n" - "use pgBackRest::Storage::Filter::Sha;\n" "use pgBackRest::Storage::Helper;\n" "\n\n\n\n" "use constant BACKUP_FILE_CHECKSUM => 0;\n" @@ -2186,7 +2180,7 @@ static const EmbeddedModule embeddedModule[] = "\n" "if ($bCompress)\n" "{\n" - "push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]});\n" + "push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]});\n" "}\n" "\n\n" "($strCopyChecksum, $lCopySize) = $oStorageRepo->hashSize(\n" @@ -2203,7 +2197,7 @@ static const EmbeddedModule embeddedModule[] = "if ($bCopy)\n" "{\n" "\n" - "my $rhyFilter = [{strClass => STORAGE_FILTER_SHA}];\n" + "my $rhyFilter = [{strClass => COMMON_IO_HANDLE}, {strClass => STORAGE_FILTER_SHA}];\n" "\n\n" "if ($bChecksumPage)\n" "{\n" @@ -2212,25 +2206,40 @@ static const EmbeddedModule embeddedModule[] = "\n" "push(\n" "@{$rhyFilter},\n" - "{strClass => BACKUP_FILTER_PAGECHECKSUM,\n" + "{strClass => \"pgBackRest::Backup::Filter::PageChecksum\",\n" "rxyParam => [$iSegmentNo, $hExtraParam->{iWalId}, $hExtraParam->{iWalOffset}]});\n" "};\n" "\n\n" "if ($bCompress)\n" "{\n" - "push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{iLevel => $iCompressLevel}]});\n" + "push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, false, $iCompressLevel]});\n" + "}\n" + "\n" + "elsif (!defined($strCipherPass))\n" + "{\n" + "push(\n" + "@{$rhyFilter},\n" + "{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_COMPRESS, true, cfgOption(CFGOPT_COMPRESS_LEVEL)]});\n" "}\n" "\n\n" "my $oSourceFileIo = storageDb()->openRead($strDbFile, {rhyFilter => $rhyFilter, bIgnoreMissing => $bIgnoreMissing});\n" "\n\n" - "if (defined($oSourceFileIo))\n" + "$rhyFilter = undef;\n" + "\n\n" + "if (!$bCompress && !defined($strCipherPass))\n" "{\n" + "push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, true]});\n" + "}\n" + "\n" "my $oDestinationFileIo = $oStorageRepo->openWrite(\n" "STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\",\n" - "{bPathCreate => true, bProtocolCompress => !$bCompress, strCipherPass => $strCipherPass});\n" - "\n\n" - "$oStorageRepo->copy($oSourceFileIo, $oDestinationFileIo);\n" + "{bPathCreate => true, rhyFilter => $rhyFilter, strCipherPass => $strCipherPass});\n" + "\n" + "$oDestinationFileIo->{oStorageCWrite}->filterAdd(COMMON_IO_HANDLE, undef);\n" "\n\n" + "if ($oStorageRepo->copy($oSourceFileIo, $oDestinationFileIo))\n" + "{\n" + "\n" "$strCopyChecksum = $oSourceFileIo->result(STORAGE_FILTER_SHA);\n" "$lCopySize = $oSourceFileIo->result(COMMON_IO_HANDLE);\n" "$lRepoSize = $oDestinationFileIo->result(COMMON_IO_HANDLE);\n" @@ -2240,7 +2249,17 @@ static const EmbeddedModule embeddedModule[] = "confess &log(ERROR, \"REPO_SIZE IS NOT SET\");\n" "}\n" "\n\n" - "$rExtra = $bChecksumPage ? $oSourceFileIo->result(BACKUP_FILTER_PAGECHECKSUM) : undef;\n" + "if ($bChecksumPage)\n" + "{\n" + "my $rExtraRaw = $oSourceFileIo->result(\"pgBackRest::Backup::Filter::PageChecksum\");\n" + "\n" + "$rExtra =\n" + "{\n" + "bValid => $rExtraRaw->{valid} ? true : false,\n" + "bAlign => $rExtraRaw->{align} ? true : false,\n" + "iyPageError => $rExtraRaw->{error},\n" + "};\n" + "}\n" "}\n" "\n" "else\n" @@ -2250,20 +2269,20 @@ static const EmbeddedModule embeddedModule[] = "}\n" "\n\n\n\n\n\n" "if ((($iCopyResult == BACKUP_FILE_COPY || $iCopyResult == BACKUP_FILE_RECOPY) &&\n" - "$oStorageRepo->driver()->capability(STORAGE_CAPABILITY_SIZE_DIFF)) ||\n" + "$oStorageRepo->capability(STORAGE_CAPABILITY_SIZE_DIFF)) ||\n" "$iCopyResult == BACKUP_FILE_CHECKSUM)\n" "{\n" - "$lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\"))->size();\n" + "$lRepoSize = ($oStorageRepo->info(STORAGE_REPO_BACKUP . \"/${strBackupLabel}/${strFileOp}\"))->{size};\n" "}\n" "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'iCopyResult', value => $iCopyResult, trace => true},\n" - "{name => 'lCopySize', value => $lCopySize, trace => true},\n" - "{name => 'lRepoSize', value => $lRepoSize, trace => true},\n" - "{name => 'strCopyChecksum', value => $strCopyChecksum, trace => true},\n" - "{name => 'rExtra', value => $rExtra, trace => true},\n" + "{name => 'iCopyResult', value => $iCopyResult},\n" + "{name => 'lCopySize', value => $lCopySize},\n" + "{name => 'lRepoSize', value => $lRepoSize},\n" + "{name => 'strCopyChecksum', value => $strCopyChecksum},\n" + "{name => 'rExtra', value => $rExtra},\n" ");\n" "}\n" "\n" @@ -2478,152 +2497,6 @@ static const EmbeddedModule embeddedModule[] = "\n" "1;\n" }, - { - .name = "pgBackRest/Backup/Filter/PageChecksum.pm", - .data = - "\n\n\n" - "package pgBackRest::Backup::Filter::PageChecksum;\n" - "use parent 'pgBackRest::Common::Io::Filter';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::DbVersion qw(PG_PAGE_SIZE);\n" - "use pgBackRest::LibC qw(:checksum);\n" - "\n\n\n\n" - "use constant BACKUP_FILTER_PAGECHECKSUM => __PACKAGE__;\n" - "push @EXPORT, qw(BACKUP_FILTER_PAGECHECKSUM);\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$oParent,\n" - "$iSegmentNo,\n" - "$iWalId,\n" - "$iWalOffset,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'oParent', trace => true},\n" - "{name => 'iSegmentNo', trace => true},\n" - "{name => 'iWalId', trace => true},\n" - "{name => 'iWalOffset', trace => true},\n" - ");\n" - "\n\n" - "my $self = $class->SUPER::new($oParent);\n" - "bless $self, $class;\n" - "\n\n" - "$self->{iSegmentNo} = $iSegmentNo;\n" - "$self->{iWalId} = $iWalId;\n" - "$self->{iWalOffset} = $iWalOffset;\n" - "\n\n" - "$self->{hResult}{bValid} = true;\n" - "$self->{hResult}{bAlign} = true;\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub read\n" - "{\n" - "my $self = shift;\n" - "my $rtBuffer = shift;\n" - "my $iSize = shift;\n" - "\n\n" - "my $iActualSize = $self->parent()->read($rtBuffer, $iSize);\n" - "\n\n" - "if ($iActualSize > 0)\n" - "{\n" - "\n" - "if (!$self->{hResult}{bAlign} || ($iActualSize % PG_PAGE_SIZE != 0))\n" - "{\n" - "if (!$self->{hResult}{bAlign})\n" - "{\n" - "confess &log(ASSERT, \"should not be possible to see two misaligned blocks in a row\");\n" - "}\n" - "\n" - "$self->{hResult}{bValid} = false;\n" - "$self->{hResult}{bAlign} = false;\n" - "delete($self->{hResult}{iyPageError});\n" - "}\n" - "else\n" - "{\n" - "\n" - "my $iBlockOffset = int(($self->size() - $iActualSize) / PG_PAGE_SIZE) + ($self->{iSegmentNo} * 131072);\n" - "\n" - "if (!pageChecksumBufferTest(\n" - "$$rtBuffer, $iActualSize, $iBlockOffset, PG_PAGE_SIZE, $self->{iWalId},\n" - "$self->{iWalOffset}))\n" - "{\n" - "$self->{hResult}{bValid} = false;\n" - "\n\n\n" - "for (my $iBlockNo = 0; $iBlockNo < int($iActualSize / PG_PAGE_SIZE); $iBlockNo++)\n" - "{\n" - "my $iBlockNoStart = $iBlockOffset + $iBlockNo;\n" - "\n" - "if (!pageChecksumTest(\n" - "substr($$rtBuffer, $iBlockNo * PG_PAGE_SIZE, PG_PAGE_SIZE), $iBlockNoStart, PG_PAGE_SIZE,\n" - "$self->{iWalId}, $self->{iWalOffset}))\n" - "{\n" - "my $iLastIdx = defined($self->{hResult}{iyPageError}) ? @{$self->{hResult}{iyPageError}} - 1 : 0;\n" - "my $iyLast = defined($self->{hResult}{iyPageError}) ? $self->{hResult}{iyPageError}[$iLastIdx] : undef;\n" - "\n" - "if (!defined($iyLast) || (!ref($iyLast) && $iyLast != $iBlockNoStart - 1) ||\n" - "(ref($iyLast) && $iyLast->[1] != $iBlockNoStart - 1))\n" - "{\n" - "push(@{$self->{hResult}{iyPageError}}, $iBlockNoStart);\n" - "}\n" - "elsif (!ref($iyLast))\n" - "{\n" - "$self->{hResult}{iyPageError}[$iLastIdx] = undef;\n" - "push(@{$self->{hResult}{iyPageError}[$iLastIdx]}, $iyLast);\n" - "push(@{$self->{hResult}{iyPageError}[$iLastIdx]}, $iBlockNoStart);\n" - "}\n" - "else\n" - "{\n" - "$self->{hResult}{iyPageError}[$iLastIdx][1] = $iBlockNoStart;\n" - "}\n" - "}\n" - "}\n" - "}\n" - "}\n" - "}\n" - "\n\n" - "return $iActualSize;\n" - "}\n" - "\n\n\n\n" - "sub close\n" - "{\n" - "my $self = shift;\n" - "\n" - "if (defined($self->{hResult}))\n" - "{\n" - "\n" - "$self->resultSet(BACKUP_FILTER_PAGECHECKSUM, $self->{hResult});\n" - "\n\n" - "undef($self->{hResult});\n" - "\n\n" - "return $self->parent()->close();\n" - "}\n" - "}\n" - "\n" - "1;\n" - }, { .name = "pgBackRest/Backup/Info.pm", .data = @@ -4125,423 +3998,6 @@ static const EmbeddedModule embeddedModule[] = "\n" "1;\n" }, - { - .name = "pgBackRest/Common/Http/Client.pm", - .data = - "\n\n\n" - "package pgBackRest::Common::Http::Client;\n" - "use parent 'pgBackRest::Common::Io::Buffered';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "use IO::Socket::SSL;\n" - "use Socket qw(SOL_SOCKET SO_KEEPALIVE);\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Io::Buffered;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::Common::String;\n" - "use pgBackRest::Common::Xml;\n" - "use pgBackRest::Common::Http::Common;\n" - "\n\n\n\n" - "use constant HTTP_VERB_GET => 'GET';\n" - "push @EXPORT, qw(HTTP_VERB_GET);\n" - "use constant HTTP_VERB_POST => 'POST';\n" - "push @EXPORT, qw(HTTP_VERB_POST);\n" - "use constant HTTP_VERB_PUT => 'PUT';\n" - "push @EXPORT, qw(HTTP_VERB_PUT);\n" - "\n" - "use constant HTTP_HEADER_CONTENT_LENGTH => 'content-length';\n" - "push @EXPORT, qw(HTTP_HEADER_CONTENT_LENGTH);\n" - "use constant HTTP_HEADER_TRANSFER_ENCODING => 'transfer-encoding';\n" - "push @EXPORT, qw(HTTP_HEADER_TRANSFER_ENCODING);\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strHost,\n" - "$strVerb,\n" - "$iPort,\n" - "$strUri,\n" - "$hQuery,\n" - "$hRequestHeader,\n" - "$rstrRequestBody,\n" - "$bResponseBodyPrefetch,\n" - "$iProtocolTimeout,\n" - "$iTryTotal,\n" - "$lBufferMax,\n" - "$bVerifySsl,\n" - "$strCaPath,\n" - "$strCaFile,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'strHost', trace => true},\n" - "{name => 'strVerb', trace => true},\n" - "{name => 'iPort', optional => true, default => 443, trace => true},\n" - "{name => 'strUri', optional => true, default => qw(/), trace => true},\n" - "{name => 'hQuery', optional => true, trace => true},\n" - "{name => 'hRequestHeader', optional => true, trace => true},\n" - "{name => 'rstrRequestBody', optional => true, trace => true},\n" - "{name => 'bResponseBodyPrefetch', optional => true, default => false, trace => true},\n" - "{name => 'iProtocolTimeout', optional => true, default => 300, trace => true},\n" - "{name => 'iTryTotal', optional => true, default => 3, trace => true},\n" - "{name => 'lBufferMax', optional => true, default => 32768, trace => true},\n" - "{name => 'bVerifySsl', optional => true, default => true, trace => true},\n" - "{name => 'strCaPath', optional => true, trace => true},\n" - "{name => 'strCaFile', optional => true, trace => true},\n" - ");\n" - "\n\n" - "my $self;\n" - "my $iTry = 1;\n" - "my $bRetry;\n" - "\n" - "do\n" - "{\n" - "\n" - "logDisable() if $iTry < $iTryTotal;\n" - "$bRetry = false;\n" - "\n" - "eval\n" - "{\n" - "\n" - "my $oSocket;\n" - "\n" - "if (eval{require IO::Socket::IP})\n" - "{\n" - "$oSocket = IO::Socket::IP->new(PeerHost => $strHost, PeerPort => $iPort)\n" - "or confess &log(ERROR, \"unable to create socket: $@\", ERROR_HOST_CONNECT);\n" - "}\n" - "else\n" - "{\n" - "require IO::Socket::INET;\n" - "\n" - "$oSocket = IO::Socket::INET->new(PeerHost => $strHost, PeerPort => $iPort)\n" - "or confess &log(ERROR, \"unable to create socket: $@\", ERROR_HOST_CONNECT);\n" - "}\n" - "\n" - "setsockopt($oSocket, SOL_SOCKET,SO_KEEPALIVE, 1)\n" - "or confess &log(ERROR, \"unable to set socket keepalive: $@\", ERROR_HOST_CONNECT);\n" - "\n" - "eval\n" - "{\n" - "IO::Socket::SSL->start_SSL(\n" - "$oSocket, SSL_verify_mode => $bVerifySsl ? SSL_VERIFY_PEER : SSL_VERIFY_NONE, SSL_ca_path => $strCaPath,\n" - "SSL_ca_file => $strCaFile);\n" - "}\n" - "or do\n" - "{\n" - "logErrorResult(\n" - "ERROR_HOST_CONNECT, coalesce(length($!) == 0 ? undef : $!, $SSL_ERROR), length($!) > 0 ? $SSL_ERROR : undef);\n" - "};\n" - "\n\n" - "$self = $class->SUPER::new(\n" - "new pgBackRest::Common::Io::Handle('httpClient', $oSocket, $oSocket), $iProtocolTimeout, $lBufferMax);\n" - "bless $self, $class;\n" - "\n\n" - "$self->{oSocket} = $oSocket;\n" - "\n\n" - "my $strQuery = httpQuery($hQuery);\n" - "\n\n" - "$self->{strRequestHeader} = \"${strVerb} \" . httpUriEncode($strUri, true) . \"?${strQuery} HTTP/1.1\" . \"\\r\\n\";\n" - "\n" - "foreach my $strHeader (sort(keys(%{$hRequestHeader})))\n" - "{\n" - "$self->{strRequestHeader} .= \"${strHeader}: $hRequestHeader->{$strHeader}\\r\\n\";\n" - "}\n" - "\n" - "$self->{strRequestHeader} .= \"\\r\\n\";\n" - "\n\n" - "$self->write(\\$self->{strRequestHeader});\n" - "\n\n" - "if (defined($rstrRequestBody))\n" - "{\n" - "my $iTotalSize = length($$rstrRequestBody);\n" - "my $iTotalSent = 0;\n" - "\n\n" - "do\n" - "{\n" - "my $strBufferWrite = substr($$rstrRequestBody, $iTotalSent, $lBufferMax);\n" - "$iTotalSent += $self->write(\\$strBufferWrite);\n" - "} while ($iTotalSent < $iTotalSize);\n" - "}\n" - "\n\n" - "($self->{strResponseProtocol}, $self->{iResponseCode}, $self->{strResponseMessage}) =\n" - "split(' ', trim($self->readLine()));\n" - "\n\n" - "$self->{iContentLength} = 0;\n" - "$self->{strResponseHeader} = '';\n" - "my $strHeader = trim($self->readLine());\n" - "\n" - "while ($strHeader ne '')\n" - "{\n" - "\n" - "$self->{strResponseHeader} .= \"${strHeader}\\n\";\n" - "\n" - "my $iColonPos = index($strHeader, ':');\n" - "\n" - "if ($iColonPos == -1)\n" - "{\n" - "confess &log(ERROR, \"http header '${strHeader}' requires colon separator\", ERROR_PROTOCOL);\n" - "}\n" - "\n\n" - "my $strHeaderKey = lc(substr($strHeader, 0, $iColonPos));\n" - "my $strHeaderValue = trim(substr($strHeader, $iColonPos + 1));\n" - "\n\n" - "$self->{hResponseHeader}{$strHeaderKey} = $strHeaderValue;\n" - "\n\n" - "if ($strHeaderKey eq HTTP_HEADER_CONTENT_LENGTH)\n" - "{\n" - "$self->{iContentLength} = $strHeaderValue + 0;\n" - "$self->{iContentRemaining} = $self->{iContentLength};\n" - "}\n" - "\n" - "elsif ($strHeaderKey eq HTTP_HEADER_TRANSFER_ENCODING)\n" - "{\n" - "if ($strHeaderValue eq 'chunked')\n" - "{\n" - "$self->{iContentLength} = -1;\n" - "}\n" - "else\n" - "{\n" - "confess &log(ERROR, \"invalid value '${strHeaderValue} for http header '${strHeaderKey}'\", ERROR_PROTOCOL);\n" - "}\n" - "}\n" - "\n\n" - "$strHeader = trim($self->readLine());\n" - "}\n" - "\n\n" - "if ($bResponseBodyPrefetch)\n" - "{\n" - "$self->{strResponseBody} = $self->responseBody();\n" - "}\n" - "\n\n" - "logEnable() if $iTry < $iTryTotal;\n" - "return 1;\n" - "}\n" - "or do\n" - "{\n" - "\n" - "logEnable() if $iTry < $iTryTotal;\n" - "\n\n" - "if ($iTry == $iTryTotal)\n" - "{\n" - "confess $EVAL_ERROR;\n" - "}\n" - "\n\n" - "$iTry++;\n" - "$bRetry = true;\n" - "};\n" - "}\n" - "while ($bRetry);\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub read\n" - "{\n" - "my $self = shift;\n" - "my $rtBuffer = shift;\n" - "my $iRequestSize = shift;\n" - "\n\n" - "$iRequestSize = $iRequestSize < $self->{iContentRemaining} ? $iRequestSize : $self->{iContentRemaining};\n" - "$self->{iContentRemaining} -= $iRequestSize;\n" - "\n" - "my $iActualSize = $self->SUPER::read($rtBuffer, $iRequestSize, true);\n" - "\n\n" - "if ($self->{iContentRemaining} == 0)\n" - "{\n" - "$self->SUPER::eofSet(true);\n" - "}\n" - "\n" - "return $iActualSize;\n" - "}\n" - "\n\n\n\n" - "sub close\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "if (defined($self->{oSocket}))\n" - "{\n" - "$self->{oSocket}->close();\n" - "undef($self->{oSocket});\n" - "}\n" - "}\n" - "\n" - "sub DESTROY {shift->close()}\n" - "\n\n\n\n" - "sub responseBody\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->responseBody'\n" - ");\n" - "\n\n" - "return $self->{strResponseBody} if exists($self->{strResponseBody});\n" - "\n\n" - "my $strResponseBody = undef;\n" - "\n" - "if ($self->{iContentLength} != 0)\n" - "{\n" - "\n" - "if ($self->{iContentLength} == -1)\n" - "{\n" - "while (1)\n" - "{\n" - "\n" - "my $strChunkLength = trim($self->readLine());\n" - "my $iChunkLength = hex($strChunkLength);\n" - "\n\n" - "last if ($iChunkLength == 0);\n" - "\n\n" - "$self->SUPER::read(\\$strResponseBody, $iChunkLength, true);\n" - "$self->readLine();\n" - "};\n" - "}\n" - "\n" - "else\n" - "{\n" - "$self->SUPER::read(\\$strResponseBody, $self->{iContentLength}, true);\n" - "}\n" - "\n" - "$self->close();\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'rstrResponseBody', value => \\$strResponseBody, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub contentLength {shift->{iContentLength}}\n" - "sub requestHeaderText {trim(shift->{strRequestHeader})}\n" - "sub responseCode {shift->{iResponseCode}}\n" - "sub responseHeader {shift->{hResponseHeader}}\n" - "sub responseHeaderText {trim(shift->{strResponseHeader})}\n" - "sub responseMessage {shift->{strResponseMessage}}\n" - "sub responseProtocol {shift->{strResponseProtocol}}\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Common/Http/Common.pm", - .data = - "\n\n\n" - "package pgBackRest::Common::Http::Common;\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Log;\n" - "\n\n\n\n" - "sub httpQuery\n" - "{\n" - "\n" - "my\n" - "(\n" - "$strOperation,\n" - "$hQuery,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '::httpQuery', \\@_,\n" - "{name => 'hQuery', required => false, trace => true},\n" - ");\n" - "\n\n" - "my $strQuery = '';\n" - "\n\n" - "if (ref($hQuery))\n" - "{\n" - "foreach my $strParam (sort(keys(%{$hQuery})))\n" - "{\n" - "\n" - "if (defined($hQuery->{$strParam}))\n" - "{\n" - "$strQuery .= ($strQuery eq '' ? '' : '&') . $strParam . '=' . httpUriEncode($hQuery->{$strParam});\n" - "}\n" - "}\n" - "}\n" - "\n" - "elsif (defined($hQuery))\n" - "{\n" - "$strQuery = $hQuery;\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'strQuery', value => $strQuery, trace => true}\n" - ");\n" - "}\n" - "\n" - "push @EXPORT, qw(httpQuery);\n" - "\n\n\n\n" - "sub httpUriEncode\n" - "{\n" - "my $strString = shift;\n" - "my $bPath = shift;\n" - "\n\n" - "my $strEncodedString;\n" - "\n" - "if (defined($strString))\n" - "{\n" - "\n" - "for (my $iIndex = 0; $iIndex < length($strString); $iIndex++)\n" - "{\n" - "my $cChar = substr($strString, $iIndex, 1);\n" - "\n\n" - "if (($cChar ge 'A' && $cChar le 'Z') || ($cChar ge 'a' && $cChar le 'z') || ($cChar ge '0' && $cChar le '9') ||\n" - "$cChar eq '_' || $cChar eq '-' || $cChar eq '~' || $cChar eq '.' || ($bPath && $cChar eq '/'))\n" - "{\n" - "$strEncodedString .= $cChar;\n" - "}\n" - "\n" - "elsif ($cChar eq '/')\n" - "{\n" - "$strEncodedString .= '%2F';\n" - "}\n" - "\n" - "else\n" - "{\n" - "$strEncodedString .= sprintf('%%%02X', ord($cChar));\n" - "}\n" - "}\n" - "}\n" - "\n" - "return $strEncodedString;\n" - "}\n" - "\n" - "push @EXPORT, qw(httpUriEncode);\n" - "\n" - "1;\n" - }, { .name = "pgBackRest/Common/Ini.pm", .data = @@ -7241,143 +6697,6 @@ static const EmbeddedModule embeddedModule[] = "\n" "1;\n" }, - { - .name = "pgBackRest/Common/Xml.pm", - .data = - "\n\n\n" - "package pgBackRest::Common::Xml;\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "use XML::LibXML;\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Log;\n" - "\n\n\n\n" - "use constant XML_HEADER => '';\n" - "push @EXPORT, qw(XML_HEADER);\n" - "\n\n\n\n" - "sub xmlFromText\n" - "{\n" - "my $strText = shift;\n" - "\n" - "return XML::LibXML::Text->new($strText)->toString();\n" - "}\n" - "\n" - "push @EXPORT, qw(xmlFromText);\n" - "\n\n\n\n" - "sub xmlParse\n" - "{\n" - "my $rstrXml = shift;\n" - "\n" - "my $oXml = XML::LibXML->load_xml(string => $rstrXml)->documentElement();\n" - "\n" - "return $oXml;\n" - "}\n" - "\n" - "push @EXPORT, qw(xmlParse);\n" - "\n\n\n\n" - "sub xmlTagChildren\n" - "{\n" - "my $oXml = shift;\n" - "my $strTag = shift;\n" - "\n" - "return $oXml->getChildrenByTagName($strTag);\n" - "}\n" - "\n" - "push @EXPORT, qw(xmlTagChildren);\n" - "\n\n\n\n" - "sub xmlTagText\n" - "{\n" - "my $oXml = shift;\n" - "my $strTag = shift;\n" - "my $bRequired = shift;\n" - "\n\n\n" - "my @oyTag = $oXml->getElementsByTagName($strTag);\n" - "\n\n" - "if (@oyTag > 1)\n" - "{\n" - "confess &log(ERROR, @oyTag . \" '${strTag}' tag(s) exist, but only one was expected\", ERROR_FORMAT);\n" - "}\n" - "elsif (@oyTag == 0)\n" - "{\n" - "if (!defined($bRequired) || $bRequired)\n" - "{\n" - "confess &log(ERROR, \"tag '${strTag}' does not exist\", ERROR_FORMAT);\n" - "}\n" - "}\n" - "else\n" - "{\n" - "return $oyTag[0]->textContent();\n" - "}\n" - "\n" - "return;\n" - "}\n" - "\n" - "push @EXPORT, qw(xmlTagText);\n" - "\n\n\n\n" - "sub xmlTagBool\n" - "{\n" - "my $oXml = shift;\n" - "my $strTag = shift;\n" - "my $bRequired = shift;\n" - "\n\n\n" - "my $strContent = xmlTagText($oXml, $strTag, $bRequired);\n" - "\n" - "if (defined($strContent))\n" - "{\n" - "if ($strContent eq 'true')\n" - "{\n" - "return true;\n" - "}\n" - "elsif ($strContent eq 'false')\n" - "{\n" - "return false;\n" - "}\n" - "else\n" - "{\n" - "confess &log(ERROR, \"invalid boolean value '${strContent}' for tag '${strTag}'\", ERROR_FORMAT);\n" - "}\n" - "}\n" - "\n" - "return;\n" - "}\n" - "\n" - "push @EXPORT, qw(xmlTagBool);\n" - "\n\n\n\n" - "sub xmlTagInt\n" - "{\n" - "my $oXml = shift;\n" - "my $strTag = shift;\n" - "my $bRequired = shift;\n" - "\n\n\n" - "my $iContent = xmlTagText($oXml, $strTag, $bRequired);\n" - "\n" - "if (defined($iContent))\n" - "{\n" - "eval\n" - "{\n" - "$iContent = $iContent + 0;\n" - "return 1;\n" - "}\n" - "or do\n" - "{\n" - "confess &log(ERROR, \"invalid integer value '${iContent}' for tag '${strTag}'\", ERROR_FORMAT);\n" - "}\n" - "}\n" - "\n" - "return $iContent;\n" - "}\n" - "\n" - "push @EXPORT, qw(xmlTagInt);\n" - "\n" - "1;\n" - }, { .name = "pgBackRest/Config/Config.pm", .data = @@ -8979,8 +8298,6 @@ static const EmbeddedModule embeddedModule[] = "checksum =>\n" "[\n" "'pageChecksum',\n" - "'pageChecksumBufferTest',\n" - "'pageChecksumTest',\n" "],\n" "\n" "config =>\n" @@ -9257,7 +8574,7 @@ static const EmbeddedModule embeddedModule[] = "\n" "storage =>\n" "[\n" - "'storagePosixPathRemove',\n" + "'storageRepoFree',\n" "],\n" "\n" "test =>\n" @@ -11956,7 +11273,6 @@ static const EmbeddedModule embeddedModule[] = "use pgBackRest::Backup::File;\n" "use pgBackRest::Common::Log;\n" "use pgBackRest::Config::Config;\n" - "use pgBackRest::Storage::Local;\n" "use pgBackRest::Protocol::Base::Master;\n" "use pgBackRest::Protocol::Base::Minion;\n" "use pgBackRest::Protocol::Command::Minion;\n" @@ -12705,11 +12021,12 @@ static const EmbeddedModule embeddedModule[] = "{\n" "my $oSourceFileIo = $oStorage->openRead(@{shift()});\n" "\n\n" - "if (defined($oSourceFileIo))\n" + "if (defined($oSourceFileIo) && (!defined($oSourceFileIo->{oStorageCRead}) || $oSourceFileIo->open()))\n" "{\n" "$self->outputWrite(true);\n" "\n" - "$oStorage->copy($oSourceFileIo, new pgBackRest::Protocol::Storage::File($self, $oSourceFileIo));\n" + "$oStorage->copy(\n" + "$oSourceFileIo, new pgBackRest::Protocol::Storage::File($self, $oSourceFileIo), {bSourceOpen => true});\n" "\n" "return true;\n" "}\n" @@ -12903,11 +12220,10 @@ static const EmbeddedModule embeddedModule[] = "\n" "use pgBackRest::Common::Log;\n" "use pgBackRest::Config::Config;\n" + "use pgBackRest::LibC qw(:storage);\n" "use pgBackRest::Protocol::Helper;\n" "use pgBackRest::Protocol::Storage::Remote;\n" - "use pgBackRest::Storage::Base;\n" "use pgBackRest::Storage::Helper;\n" - "use pgBackRest::Storage::Local;\n" "\n\n\n\n" "use constant STORAGE_DB => '';\n" "push @EXPORT, qw(STORAGE_DB);\n" @@ -12940,9 +12256,8 @@ static const EmbeddedModule embeddedModule[] = "{\n" "if (isDbLocal({iRemoteIdx => $iRemoteIdx}))\n" "{\n" - "$hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Storage::Local(\n" - "cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iRemoteIdx)), new pgBackRest::Storage::Posix::Driver(),\n" - "{strTempExtension => STORAGE_TEMP_EXT, lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});\n" + "$hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Storage::Storage(\n" + "STORAGE_DB, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});\n" "}\n" "else\n" "{\n" @@ -12960,47 +12275,6 @@ static const EmbeddedModule embeddedModule[] = "\n" "push @EXPORT, qw(storageDb);\n" "\n\n\n\n" - "sub storageRepoRule\n" - "{\n" - "my $strRule = shift;\n" - "my $strFile = shift;\n" - "my $strStanza = shift;\n" - "\n\n" - "my $strResultFile;\n" - "\n\n" - "if ($strRule eq STORAGE_REPO_ARCHIVE)\n" - "{\n" - "$strResultFile = \"archive\" . (defined($strStanza) ? \"/${strStanza}\" : '');\n" - "\n\n" - "if (defined($strFile))\n" - "{\n" - "my ($strArchiveId, $strWalFile) = split('/', $strFile);\n" - "\n\n" - "if (defined($strWalFile) && $strWalFile =~ /^[0-F]{24}/)\n" - "{\n" - "$strResultFile .= \"/${strArchiveId}/\" . substr($strWalFile, 0, 16) . \"/${strWalFile}\";\n" - "}\n" - "\n" - "else\n" - "{\n" - "$strResultFile .= \"/${strFile}\";\n" - "}\n" - "}\n" - "}\n" - "\n" - "elsif ($strRule eq STORAGE_REPO_BACKUP)\n" - "{\n" - "$strResultFile = \"backup\" . (defined($strStanza) ? \"/${strStanza}\" : '') . (defined($strFile) ? \"/${strFile}\" : '');\n" - "}\n" - "\n" - "else\n" - "{\n" - "confess &log(ASSERT, \"invalid \" . STORAGE_REPO . \" storage rule ${strRule}\");\n" - "}\n" - "\n" - "return $strResultFile;\n" - "}\n" - "\n\n\n\n" "sub storageRepo\n" "{\n" "\n" @@ -13014,81 +12288,18 @@ static const EmbeddedModule embeddedModule[] = "__PACKAGE__ . '::storageRepo', \\@_,\n" "{name => 'strStanza', optional => true, trace => true},\n" ");\n" - "\n" - "if (!defined($strStanza))\n" - "{\n" - "if (cfgOptionValid(CFGOPT_STANZA) && cfgOptionTest(CFGOPT_STANZA))\n" - "{\n" - "$strStanza = cfgOption(CFGOPT_STANZA);\n" - "}\n" - "else\n" - "{\n" - "$strStanza = STORAGE_REPO;\n" - "}\n" - "}\n" "\n\n" - "if (!defined($hStorage->{&STORAGE_REPO}{$strStanza}))\n" + "if (!defined($hStorage->{&STORAGE_REPO}))\n" "{\n" "if (isRepoLocal())\n" "{\n" - "\n" - "my $hRule =\n" - "{\n" - "&STORAGE_REPO_ARCHIVE =>\n" - "{\n" - "fnRule => \\&storageRepoRule,\n" - "xData => $strStanza eq STORAGE_REPO ? undef : $strStanza,\n" - "},\n" - "&STORAGE_REPO_BACKUP =>\n" - "{\n" - "fnRule => \\&storageRepoRule,\n" - "xData => $strStanza eq STORAGE_REPO ? undef : $strStanza,\n" - "},\n" - "};\n" - "\n\n" - "my $oDriver;\n" - "\n" - "if (cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3))\n" - "{\n" - "require pgBackRest::Storage::S3::Driver;\n" - "\n" - "$oDriver = new pgBackRest::Storage::S3::Driver(\n" - "cfgOption(CFGOPT_REPO_S3_BUCKET), cfgOption(CFGOPT_REPO_S3_ENDPOINT), cfgOption(CFGOPT_REPO_S3_REGION),\n" - "cfgOption(CFGOPT_REPO_S3_KEY), cfgOption(CFGOPT_REPO_S3_KEY_SECRET),\n" - "{strHost => cfgOption(CFGOPT_REPO_S3_HOST, false), bVerifySsl => cfgOption(CFGOPT_REPO_S3_VERIFY_TLS, false),\n" - "strCaPath => cfgOption(CFGOPT_REPO_S3_CA_PATH, false),\n" - "strCaFile => cfgOption(CFGOPT_REPO_S3_CA_FILE, false), lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE),\n" - "strSecurityToken => cfgOption(CFGOPT_REPO_S3_TOKEN, false)});\n" - "}\n" - "elsif (cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_CIFS))\n" - "{\n" - "require pgBackRest::Storage::Cifs::Driver;\n" - "\n" - "$oDriver = new pgBackRest::Storage::Cifs::Driver();\n" - "}\n" - "else\n" - "{\n" - "$oDriver = new pgBackRest::Storage::Posix::Driver();\n" - "}\n" - "\n\n" - "my $strCipherType;\n" - "my $strCipherPass;\n" - "\n\n" - "if (cfgOption(CFGOPT_REPO_CIPHER_TYPE) ne CFGOPTVAL_REPO_CIPHER_TYPE_NONE)\n" - "{\n" - "$strCipherType = cfgOption(CFGOPT_REPO_CIPHER_TYPE);\n" - "$strCipherPass = cfgOption(CFGOPT_REPO_CIPHER_PASS);\n" - "}\n" - "\n\n" - "$hStorage->{&STORAGE_REPO}{$strStanza} = new pgBackRest::Storage::Local(\n" - "cfgOption(CFGOPT_REPO_PATH), $oDriver,\n" - "{strTempExtension => STORAGE_TEMP_EXT, hRule => $hRule, lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE),\n" - "strCipherType => $strCipherType, strCipherPassUser => $strCipherPass});\n" + "$hStorage->{&STORAGE_REPO} = new pgBackRest::Storage::Storage(\n" + "STORAGE_REPO, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});\n" "}\n" "else\n" "{\n" "\n" - "$hStorage->{&STORAGE_REPO}{$strStanza} = new pgBackRest::Protocol::Storage::Remote(\n" + "$hStorage->{&STORAGE_REPO} = new pgBackRest::Protocol::Storage::Remote(\n" "protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP));\n" "}\n" "}\n" @@ -13096,7 +12307,7 @@ static const EmbeddedModule embeddedModule[] = "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}{$strStanza}, trace => true},\n" + "{name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}, trace => true},\n" ");\n" "}\n" "\n" @@ -13108,6 +12319,8 @@ static const EmbeddedModule embeddedModule[] = "my ($strOperation) = logDebugParam(__PACKAGE__ . '::storageRepoCacheClear');\n" "\n" "delete($hStorage->{&STORAGE_REPO});\n" + "\n" + "storageRepoFree();\n" "\n\n" "return logDebugReturn($strOperation);\n" "}\n" @@ -13134,7 +12347,6 @@ static const EmbeddedModule embeddedModule[] = "use pgBackRest::Protocol::Helper;\n" "use pgBackRest::Protocol::Storage::File;\n" "use pgBackRest::Storage::Base;\n" - "use pgBackRest::Storage::Filter::Gzip;\n" "\n\n\n\n" "sub new\n" "{\n" @@ -13282,27 +12494,11 @@ static const EmbeddedModule embeddedModule[] = "{name => 'strFileExp'},\n" "{name => 'rhParam', required => false},\n" ");\n" - "\n\n" - "my $bProtocolCompress = protocolCompress($rhParam);\n" - "\n\n" - "if ($bProtocolCompress)\n" - "{\n" - "push(\n" - "@{$rhParam->{rhyFilter}},\n" - "{strClass => STORAGE_FILTER_GZIP,\n" - "rxyParam => [{iLevel => cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK), bWantGzip => false}]});\n" - "}\n" "\n" "my $oSourceFileIo =\n" "$self->{oProtocol}->cmdExecute(OP_STORAGE_OPEN_READ, [$strFileExp, $rhParam]) ?\n" "new pgBackRest::Protocol::Storage::File($self->{oProtocol}) : undef;\n" "\n\n" - "if ($bProtocolCompress)\n" - "{\n" - "$oSourceFileIo = new pgBackRest::Storage::Filter::Gzip(\n" - "$oSourceFileIo, {strCompressType => STORAGE_DECOMPRESS, bWantGzip => false});\n" - "}\n" - "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" @@ -13327,24 +12523,9 @@ static const EmbeddedModule embeddedModule[] = "{name => 'rhParam', required => false},\n" ");\n" "\n\n" - "my $bProtocolCompress = protocolCompress($rhParam);\n" - "\n\n" - "if ($bProtocolCompress)\n" - "{\n" - "push(\n" - "@{$rhParam->{rhyFilter}},\n" - "{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS, bWantGzip => false}]});\n" - "}\n" - "\n\n" "$self->{oProtocol}->cmdWrite(OP_STORAGE_OPEN_WRITE, [$strFileExp, $rhParam]);\n" "my $oDestinationFileIo = new pgBackRest::Protocol::Storage::File($self->{oProtocol});\n" "\n\n" - "if ($bProtocolCompress)\n" - "{\n" - "$oDestinationFileIo = new pgBackRest::Storage::Filter::Gzip(\n" - "$oDestinationFileIo, {iLevel => cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK), bWantGzip => false});\n" - "}\n" - "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" @@ -13427,21 +12608,6 @@ static const EmbeddedModule embeddedModule[] = ");\n" "}\n" "\n\n\n\n" - "sub protocolCompress\n" - "{\n" - "my $rhParam = shift;\n" - "\n" - "my $bProtocolCompress = false;\n" - "\n" - "if (defined($rhParam->{bProtocolCompress}))\n" - "{\n" - "$bProtocolCompress = $rhParam->{bProtocolCompress} && cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK) > 0 ? true : false;\n" - "delete($rhParam->{bProtocolCompress});\n" - "}\n" - "\n" - "return $bProtocolCompress;\n" - "}\n" - "\n\n\n\n" "sub protocol {shift->{oProtocol}};\n" "\n" "1;\n" @@ -13643,8 +12809,7 @@ static const EmbeddedModule embeddedModule[] = "}\n" "\n\n" "storageDb()->copy(\n" - "storageRepo()->openRead(STORAGE_REPO_BACKUP . \"/$self->{strBackupSet}/\" . FILE_MANIFEST, {bProtocolCompress => true,\n" - "strCipherPass => $strCipherPass}),\n" + "storageRepo()->openRead(STORAGE_REPO_BACKUP . \"/$self->{strBackupSet}/\" . FILE_MANIFEST, {strCipherPass => $strCipherPass}),\n" "$self->{strDbClusterPath} . '/' . FILE_MANIFEST);\n" "\n\n" "my $oManifest = new pgBackRest::Manifest(\n" @@ -14400,7 +13565,7 @@ static const EmbeddedModule embeddedModule[] = "}\n" "\n\n" "$oStorageDb->copy(\n" - "storageRepo()->openRead(STORAGE_REPO_BACKUP . qw(/) . FILE_BACKUP_INFO, {bProtocolCompress => true}),\n" + "storageRepo()->openRead(STORAGE_REPO_BACKUP . qw(/) . FILE_BACKUP_INFO),\n" "$self->{strDbClusterPath} . '/' . FILE_BACKUP_INFO);\n" "\n" "my $oBackupInfo = new pgBackRest::Backup::Info($self->{strDbClusterPath}, false, undef, {oStorage => storageDb()});\n" @@ -14659,9 +13824,6 @@ static const EmbeddedModule embeddedModule[] = "use pgBackRest::Config::Config;\n" "use pgBackRest::Manifest;\n" "use pgBackRest::Protocol::Storage::Helper;\n" - "use pgBackRest::Storage::Base;\n" - "use pgBackRest::Storage::Filter::Gzip;\n" - "use pgBackRest::Storage::Filter::Sha;\n" "use pgBackRest::Storage::Helper;\n" "\n\n\n\n\n\n" "sub restoreLog\n" @@ -14966,7 +14128,7 @@ static const EmbeddedModule embeddedModule[] = "my ($strOperation) = logDebugParam(__PACKAGE__ . '->stanzaDelete');\n" "\n" "my $strStanza = cfgOption(CFGOPT_STANZA);\n" - "my $oStorageRepo = storageRepo({strStanza => $strStanza});\n" + "my $oStorageRepo = storageRepo();\n" "\n\n" "if ($oStorageRepo->pathExists(STORAGE_REPO_ARCHIVE) || $oStorageRepo->pathExists(STORAGE_REPO_BACKUP))\n" "{\n" @@ -15006,8 +14168,8 @@ static const EmbeddedModule embeddedModule[] = "$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strBackup}/\" . FILE_MANIFEST_COPY, {bIgnoreMissing => true});\n" "}\n" "\n\n" - "$oStorageRepo->remove(STORAGE_REPO_ARCHIVE, {bRecurse => true, bIgnoreMissing => true});\n" - "$oStorageRepo->remove(STORAGE_REPO_BACKUP, {bRecurse => true, bIgnoreMissing => true});\n" + "$oStorageRepo->pathRemove(STORAGE_REPO_ARCHIVE, {bRecurse => true, bIgnoreMissing => true});\n" + "$oStorageRepo->pathRemove(STORAGE_REPO_BACKUP, {bRecurse => true, bIgnoreMissing => true});\n" "\n\n" "lockStart();\n" "}\n" @@ -15400,6 +14562,14 @@ static const EmbeddedModule embeddedModule[] = "use pgBackRest::Common::Io::Base;\n" "use pgBackRest::Common::Log;\n" "\n\n\n\n" + "use constant STORAGE_LOCAL => '';\n" + "push @EXPORT, qw(STORAGE_LOCAL);\n" + "\n" + "use constant STORAGE_S3 => 's3';\n" + "push @EXPORT, qw(STORAGE_S3);\n" + "use constant STORAGE_POSIX => 'posix';\n" + "push @EXPORT, qw(STORAGE_POSIX);\n" + "\n\n\n\n" "use constant STORAGE_COMPRESS => 'compress';\n" "push @EXPORT, qw(STORAGE_COMPRESS);\n" "use constant STORAGE_DECOMPRESS => 'decompress';\n" @@ -15411,6 +14581,13 @@ static const EmbeddedModule embeddedModule[] = "push @EXPORT, qw(STORAGE_DECRYPT);\n" "use constant CIPHER_MAGIC => 'Salted__';\n" "push @EXPORT, qw(CIPHER_MAGIC);\n" + "\n\n\n\n" + "use constant STORAGE_FILTER_CIPHER_BLOCK => 'pgBackRest::Storage::Filter::CipherBlock';\n" + "push @EXPORT, qw(STORAGE_FILTER_CIPHER_BLOCK);\n" + "use constant STORAGE_FILTER_GZIP => 'pgBackRest::Storage::Filter::Gzip';\n" + "push @EXPORT, qw(STORAGE_FILTER_GZIP);\n" + "use constant STORAGE_FILTER_SHA => 'pgBackRest::Storage::Filter::Sha';\n" + "push @EXPORT, qw(STORAGE_FILTER_SHA);\n" "\n\n\n\n\n\n\n" "use constant STORAGE_CAPABILITY_SIZE_DIFF => 'size-diff';\n" "push @EXPORT, qw(STORAGE_CAPABILITY_SIZE_DIFF);\n" @@ -15443,7 +14620,7 @@ static const EmbeddedModule embeddedModule[] = "{name => 'self', value => $self}\n" ");\n" "}\n" - "\n\n\n\n\n" + "\n\n\n\n\n\n" "sub copy\n" "{\n" "my $self = shift;\n" @@ -15453,41 +14630,56 @@ static const EmbeddedModule embeddedModule[] = "$strOperation,\n" "$xSourceFile,\n" "$xDestinationFile,\n" + "$bSourceOpen,\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->copy', \\@_,\n" "{name => 'xSourceFile', required => false},\n" - "{name => 'xDestinationFile', required => false},\n" + "{name => 'xDestinationFile'},\n" + "{name => 'bSourceOpen', optional => true, default => false},\n" ");\n" "\n\n" + "my $oSourceFileIo = defined($xSourceFile) ? (ref($xSourceFile) ? $xSourceFile : $self->openRead($xSourceFile)) : undef;\n" + "\n\n" "my $bResult = false;\n" "\n\n" - "my $oSourceFileIo =\n" - "defined($xSourceFile) ?\n" - "(ref($xSourceFile) ? $xSourceFile : $self->openRead($self->pathGet($xSourceFile))) : undef;\n" - "\n\n" "if (defined($oSourceFileIo))\n" "{\n" - "\n" - "my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($self->pathGet($xDestinationFile));\n" + "my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($xDestinationFile);\n" "\n\n" - "my $lSizeRead;\n" + "if (defined($oSourceFileIo->{oStorageCRead}) && defined($oDestinationFileIo->{oStorageCWrite}))\n" + "{\n" + "$bResult = $self->{oStorageC}->copy(\n" + "$oSourceFileIo->{oStorageCRead}, $oDestinationFileIo->{oStorageCWrite}) ? true : false;\n" + "}\n" + "else\n" + "{\n" "\n" + "$bResult = defined($oSourceFileIo->{oStorageCRead}) ? ($bSourceOpen || $oSourceFileIo->open()) : true;\n" + "\n" + "if ($bResult)\n" + "{\n" + "\n" + "if (defined($oDestinationFileIo->{oStorageCWrite}))\n" + "{\n" + "$oDestinationFileIo->open();\n" + "}\n" + "\n\n" "do\n" "{\n" "\n" "my $tBuffer = '';\n" "\n" - "$lSizeRead = $oSourceFileIo->read(\\$tBuffer, $self->{lBufferMax});\n" + "$oSourceFileIo->read(\\$tBuffer, $self->{lBufferMax});\n" "$oDestinationFileIo->write(\\$tBuffer);\n" "}\n" - "while ($lSizeRead != 0);\n" + "while (!$oSourceFileIo->eof());\n" "\n\n" "$oSourceFileIo->close();\n" "$oDestinationFileIo->close();\n" - "\n\n" - "$bResult = true;\n" + "}\n" + "}\n" "}\n" "\n" "return logDebugReturn\n" @@ -15649,527 +14841,6 @@ static const EmbeddedModule embeddedModule[] = "\n" "1;\n" }, - { - .name = "pgBackRest/Storage/Cifs/Driver.pm", - .data = - "\n\n\n\n\n" - "package pgBackRest::Storage::Cifs::Driver;\n" - "use parent 'pgBackRest::Storage::Posix::Driver';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::Storage::Base;\n" - "\n\n\n\n" - "use constant STORAGE_CIFS_DRIVER => __PACKAGE__;\n" - "push @EXPORT, qw(STORAGE_CIFS_DRIVER);\n" - "\n\n\n\n" - "sub pathSync\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPath,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->pathSync', \\@_,\n" - "{name => 'strPath', trace => true},\n" - ");\n" - "\n\n" - "return logDebugReturn($strOperation);\n" - "}\n" - "\n\n\n\n" - "sub capability {shift eq STORAGE_CAPABILITY_SIZE_DIFF ? true : false}\n" - "sub className {STORAGE_CIFS_DRIVER}\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Storage/Filter/CipherBlock.pm", - .data = - "\n\n\n" - "package pgBackRest::Storage::Filter::CipherBlock;\n" - "use parent 'pgBackRest::Common::Io::Filter';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Io::Base;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::LibC qw(:crypto);\n" - "use pgBackRest::Storage::Base;\n" - "\n\n\n\n" - "use constant STORAGE_FILTER_CIPHER_BLOCK => __PACKAGE__;\n" - "push @EXPORT, qw(STORAGE_FILTER_CIPHER_BLOCK);\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$oParent,\n" - "$strCipherType,\n" - "$tCipherPass,\n" - "$strMode,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'oParent', trace => true},\n" - "{name => 'strCipherType', trace => true},\n" - "{name => 'tCipherPass', trace => true},\n" - "{name => 'strMode', optional => true, default => STORAGE_ENCRYPT, trace => true},\n" - ");\n" - "\n\n" - "my $self = $class->SUPER::new($oParent);\n" - "bless $self, $class;\n" - "\n\n" - "$self->{strMode} = $strMode;\n" - "\n" - "if (!($self->{strMode} eq STORAGE_ENCRYPT || $self->{strMode} eq STORAGE_DECRYPT))\n" - "{\n" - "confess &log(ASSERT, \"unknown cipher mode: $self->{strMode}\");\n" - "}\n" - "\n\n" - "$self->{bWrite} = false;\n" - "\n\n" - "$self->{oCipher} = new pgBackRest::LibC::Cipher::Block(\n" - "$self->{strMode} eq STORAGE_ENCRYPT ? CIPHER_MODE_ENCRYPT : CIPHER_MODE_DECRYPT, $strCipherType, $tCipherPass,\n" - "length($tCipherPass));\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub read\n" - "{\n" - "my $self = shift;\n" - "my $rtBuffer = shift;\n" - "my $iSize = shift;\n" - "\n\n" - "return 0 if $self->eof();\n" - "\n\n" - "my $tBufferRead = '';\n" - "my $iBufferReadSize = 0;\n" - "\n" - "do\n" - "{\n" - "\n" - "my $tCipherBuffer;\n" - "my $iActualSize = $self->SUPER::read(\\$tCipherBuffer, $iSize);\n" - "\n\n" - "if ($iActualSize > 0)\n" - "{\n" - "$tBufferRead .= $self->{oCipher}->process($tCipherBuffer);\n" - "}\n" - "\n\n" - "if ($self->eof())\n" - "{\n" - "$tBufferRead .= $self->{oCipher}->flush();\n" - "}\n" - "\n\n" - "$iBufferReadSize = length($tBufferRead);\n" - "}\n" - "while ($iBufferReadSize < $iSize && !$self->eof());\n" - "\n\n" - "$$rtBuffer .= $tBufferRead;\n" - "\n\n" - "return $iBufferReadSize;\n" - "}\n" - "\n\n\n\n" - "sub write\n" - "{\n" - "my $self = shift;\n" - "my $rtBuffer = shift;\n" - "\n\n" - "$self->{bWrite} = true;\n" - "\n\n" - "my $tCipherBuffer;\n" - "\n" - "if (defined($$rtBuffer))\n" - "{\n" - "$tCipherBuffer = $self->{oCipher}->process($$rtBuffer);\n" - "}\n" - "\n\n" - "$self->SUPER::write(\\$tCipherBuffer);\n" - "\n" - "return length($$rtBuffer);\n" - "}\n" - "\n\n\n\n" - "sub close\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "if ($self->{oCipher})\n" - "{\n" - "\n" - "if ($self->{bWrite})\n" - "{\n" - "my $tCipherBuffer = $self->{oCipher}->flush();\n" - "$self->SUPER::write(\\$tCipherBuffer);\n" - "}\n" - "\n" - "undef($self->{oCipher});\n" - "\n\n" - "return $self->SUPER::close();\n" - "}\n" - "\n" - "return false;\n" - "}\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Storage/Filter/Gzip.pm", - .data = - "\n\n\n" - "package pgBackRest::Storage::Filter::Gzip;\n" - "use parent 'pgBackRest::Common::Io::Filter';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Compress::Raw::Zlib qw(WANT_GZIP MAX_WBITS Z_OK Z_BUF_ERROR Z_DATA_ERROR Z_STREAM_END);\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Io::Base;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::Storage::Base;\n" - "\n\n\n\n" - "use constant STORAGE_FILTER_GZIP => __PACKAGE__;\n" - "push @EXPORT, qw(STORAGE_FILTER_GZIP);\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$oParent,\n" - "$bWantGzip,\n" - "$strCompressType,\n" - "$iLevel,\n" - "$lCompressBufferMax,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'oParent', trace => true},\n" - "{name => 'bWantGzip', optional => true, default => true, trace => true},\n" - "{name => 'strCompressType', optional => true, default => STORAGE_COMPRESS, trace => true},\n" - "{name => 'iLevel', optional => true, default => 6, trace => true},\n" - "{name => 'lCompressBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX, trace => true},\n" - ");\n" - "\n\n" - "my $self = $class->SUPER::new($oParent);\n" - "bless $self, $class;\n" - "\n\n" - "$self->{bWantGzip} = $bWantGzip;\n" - "$self->{iLevel} = $iLevel;\n" - "$self->{lCompressBufferMax} = $lCompressBufferMax;\n" - "$self->{strCompressType} = $strCompressType;\n" - "\n\n" - "$self->{bWrite} = false;\n" - "\n\n" - "my $iZLibStatus;\n" - "\n" - "if ($self->{strCompressType} eq STORAGE_COMPRESS)\n" - "{\n" - "($self->{oZLib}, $iZLibStatus) = new Compress::Raw::Zlib::Deflate(\n" - "WindowBits => $self->{bWantGzip} ? WANT_GZIP : MAX_WBITS, Level => $self->{iLevel},\n" - "Bufsize => $self->{lCompressBufferMax}, AppendOutput => 1);\n" - "\n" - "$self->{tCompressedBuffer} = undef;\n" - "}\n" - "else\n" - "{\n" - "($self->{oZLib}, $iZLibStatus) = new Compress::Raw::Zlib::Inflate(\n" - "WindowBits => $self->{bWantGzip} ? WANT_GZIP : MAX_WBITS, Bufsize => $self->{lCompressBufferMax},\n" - "LimitOutput => 1, AppendOutput => 1);\n" - "\n" - "$self->{tUncompressedBuffer} = undef;\n" - "$self->{lUncompressedBufferSize} = 0;\n" - "}\n" - "\n" - "$self->errorCheck($iZLibStatus);\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub errorCheck\n" - "{\n" - "my $self = shift;\n" - "my $iZLibStatus = shift;\n" - "\n" - "if (!($iZLibStatus == Z_OK || $iZLibStatus == Z_BUF_ERROR))\n" - "{\n" - "logErrorResult(\n" - "$self->{bWrite} ? ERROR_FILE_WRITE : ERROR_FILE_READ,\n" - "'unable to ' . ($self->{strCompressType} eq STORAGE_COMPRESS ? 'deflate' : 'inflate') . \" '\" .\n" - "$self->parent()->name() . \"'\",\n" - "$self->{oZLib}->msg());\n" - "}\n" - "\n" - "return Z_OK;\n" - "}\n" - "\n\n\n\n" - "sub read\n" - "{\n" - "my $self = shift;\n" - "my $rtBuffer = shift;\n" - "my $iSize = shift;\n" - "\n" - "if ($self->{strCompressType} eq STORAGE_COMPRESS)\n" - "{\n" - "return 0 if $self->eof();\n" - "\n" - "my $lSizeBegin = defined($$rtBuffer) ? length($$rtBuffer) : 0;\n" - "my $lUncompressedSize;\n" - "my $lCompressedSize;\n" - "\n" - "do\n" - "{\n" - "my $tUncompressedBuffer;\n" - "$lUncompressedSize = $self->parent()->read(\\$tUncompressedBuffer, $iSize);\n" - "\n" - "if ($lUncompressedSize > 0)\n" - "{\n" - "$self->errorCheck($self->{oZLib}->deflate($tUncompressedBuffer, $$rtBuffer));\n" - "}\n" - "else\n" - "{\n" - "$self->errorCheck($self->{oZLib}->flush($$rtBuffer));\n" - "}\n" - "\n" - "$lCompressedSize = length($$rtBuffer) - $lSizeBegin;\n" - "}\n" - "while ($lUncompressedSize > 0 && $lCompressedSize < $iSize);\n" - "\n\n" - "return $lCompressedSize;\n" - "}\n" - "else\n" - "{\n" - "\n" - "while ($self->{lUncompressedBufferSize} < $iSize && !$self->parent()->eof())\n" - "{\n" - "if (!defined($self->{tCompressedBuffer}) || length($self->{tCompressedBuffer}) == 0)\n" - "{\n" - "$self->parent()->read(\\$self->{tCompressedBuffer}, $self->{lCompressBufferMax});\n" - "}\n" - "\n" - "my $iZLibStatus = $self->{oZLib}->inflate($self->{tCompressedBuffer}, $self->{tUncompressedBuffer});\n" - "$self->{lUncompressedBufferSize} = length($self->{tUncompressedBuffer});\n" - "\n" - "last if $iZLibStatus == Z_STREAM_END;\n" - "\n" - "$self->errorCheck($iZLibStatus);\n" - "}\n" - "\n\n\n" - "my $iActualSize = $self->{lUncompressedBufferSize} < $iSize ? $self->{lUncompressedBufferSize} : $iSize;\n" - "\n\n" - "$$rtBuffer .= substr($self->{tUncompressedBuffer}, 0, $iActualSize);\n" - "\n\n" - "$self->{tUncompressedBuffer} = substr($self->{tUncompressedBuffer}, $iActualSize);\n" - "$self->{lUncompressedBufferSize} -= $iActualSize;\n" - "\n\n" - "return $iActualSize;\n" - "}\n" - "}\n" - "\n\n\n\n" - "sub write\n" - "{\n" - "my $self = shift;\n" - "my $rtBuffer = shift;\n" - "\n" - "$self->{bWrite} = true;\n" - "\n" - "if ($self->{strCompressType} eq STORAGE_COMPRESS)\n" - "{\n" - "\n" - "$self->errorCheck($self->{oZLib}->deflate($$rtBuffer, $self->{tCompressedBuffer}));\n" - "\n\n" - "if (length($self->{tCompressedBuffer}) > $self->{lCompressBufferMax})\n" - "{\n" - "$self->parent()->write(\\$self->{tCompressedBuffer});\n" - "$self->{tCompressedBuffer} = undef;\n" - "}\n" - "}\n" - "else\n" - "{\n" - "my $tCompressedBuffer = $$rtBuffer;\n" - "\n" - "while (length($tCompressedBuffer) > 0)\n" - "{\n" - "my $tUncompressedBuffer;\n" - "\n" - "my $iZLibStatus = $self->{oZLib}->inflate($tCompressedBuffer, $tUncompressedBuffer);\n" - "$self->parent()->write(\\$tUncompressedBuffer);\n" - "\n" - "last if $iZLibStatus == Z_STREAM_END;\n" - "\n" - "$self->errorCheck($iZLibStatus);\n" - "}\n" - "}\n" - "\n\n" - "return length($$rtBuffer);\n" - "}\n" - "\n\n\n\n" - "sub close\n" - "{\n" - "my $self = shift;\n" - "\n" - "if (defined($self->{oZLib}))\n" - "{\n" - "\n" - "if ($self->{bWrite})\n" - "{\n" - "if ($self->{strCompressType} eq STORAGE_COMPRESS)\n" - "{\n" - "\n" - "$self->errorCheck($self->{oZLib}->flush($self->{tCompressedBuffer}));\n" - "\n\n" - "$self->parent()->write(\\$self->{tCompressedBuffer});\n" - "}\n" - "}\n" - "\n" - "undef($self->{oZLib});\n" - "\n\n" - "return $self->parent()->close();\n" - "}\n" - "\n" - "return false;\n" - "}\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Storage/Filter/Sha.pm", - .data = - "\n\n\n" - "package pgBackRest::Storage::Filter::Sha;\n" - "use parent 'pgBackRest::Common::Io::Filter';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Log;\n" - "\n\n\n\n" - "use constant STORAGE_FILTER_SHA => __PACKAGE__;\n" - "push @EXPORT, qw(STORAGE_FILTER_SHA);\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$oParent,\n" - "$strAlgorithm,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'oParent', trace => true},\n" - "{name => 'strAlgorithm', optional => true, default => 'sha1', trace => true},\n" - ");\n" - "\n\n" - "my $self = $class->SUPER::new($oParent);\n" - "bless $self, $class;\n" - "\n\n" - "$self->{strAlgorithm} = $strAlgorithm;\n" - "\n\n" - "$self->{oSha} = new pgBackRest::LibC::Crypto::Hash($self->{strAlgorithm});\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub read\n" - "{\n" - "my $self = shift;\n" - "my $rtBuffer = shift;\n" - "my $iSize = shift;\n" - "\n\n" - "my $tShaBuffer;\n" - "my $iActualSize = $self->parent()->read(\\$tShaBuffer, $iSize);\n" - "\n\n" - "if ($iActualSize > 0)\n" - "{\n" - "$self->{oSha}->process($tShaBuffer);\n" - "$$rtBuffer .= $tShaBuffer;\n" - "}\n" - "\n\n" - "return $iActualSize;\n" - "}\n" - "\n\n\n\n" - "sub write\n" - "{\n" - "my $self = shift;\n" - "my $rtBuffer = shift;\n" - "\n\n" - "$self->{oSha}->process($$rtBuffer);\n" - "\n\n" - "return $self->parent()->write($rtBuffer);\n" - "}\n" - "\n\n\n\n" - "sub close\n" - "{\n" - "my $self = shift;\n" - "\n" - "if (defined($self->{oSha}))\n" - "{\n" - "\n" - "$self->resultSet(STORAGE_FILTER_SHA, $self->{oSha}->result());\n" - "\n\n" - "delete($self->{oSha});\n" - "\n\n" - "return $self->parent->close();\n" - "}\n" - "\n" - "return false;\n" - "}\n" - "\n" - "1;\n" - }, { .name = "pgBackRest/Storage/Helper.pm", .data = @@ -16186,48 +14857,25 @@ static const EmbeddedModule embeddedModule[] = "\n" "use pgBackRest::Common::Log;\n" "use pgBackRest::Config::Config;\n" - "use pgBackRest::Storage::Posix::Driver;\n" - "use pgBackRest::Storage::Local;\n" + "use pgBackRest::Storage::Base;\n" + "use pgBackRest::Storage::Storage;\n" "use pgBackRest::Version;\n" "\n\n\n\n" - "use constant STORAGE_LOCAL => '';\n" - "push @EXPORT, qw(STORAGE_LOCAL);\n" - "\n\n\n\n" "use constant COMPRESS_EXT => 'gz';\n" "push @EXPORT, qw(COMPRESS_EXT);\n" "\n\n\n\n" "use constant STORAGE_TEMP_EXT => PROJECT_EXE . '.tmp';\n" "push @EXPORT, qw(STORAGE_TEMP_EXT);\n" - "\n\n\n\n" - "my $hStorage;\n" "\n\n\n\n\n\n\n" "sub storageLocal\n" "{\n" "\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPath,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '::storageLocal', \\@_,\n" - "{name => 'strPath', default => '/', trace => true},\n" - ");\n" - "\n\n" - "if (!defined($hStorage->{&STORAGE_LOCAL}{$strPath}))\n" - "{\n" - "\n" - "$hStorage->{&STORAGE_LOCAL}{$strPath} = new pgBackRest::Storage::Local(\n" - "$strPath, new pgBackRest::Storage::Posix::Driver(),\n" - "{strTempExtension => STORAGE_TEMP_EXT,\n" - "lBufferMax => cfgOptionValid(CFGOPT_BUFFER_SIZE, false) ? cfgOption(CFGOPT_BUFFER_SIZE, false) : undef});\n" - "}\n" + "my ($strOperation) = logDebugParam(__PACKAGE__ . '::storageLocal');\n" "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'oStorageLocal', value => $hStorage->{&STORAGE_LOCAL}{$strPath}, trace => true},\n" + "{name => 'oStorageLocal', value => new pgBackRest::Storage::Storage(STORAGE_LOCAL), trace => true},\n" ");\n" "}\n" "\n" @@ -16236,10 +14884,10 @@ static const EmbeddedModule embeddedModule[] = "1;\n" }, { - .name = "pgBackRest/Storage/Local.pm", + .name = "pgBackRest/Storage/Storage.pm", .data = - "\n\n\n\n\n" - "package pgBackRest::Storage::Local;\n" + "\n\n\n" + "package pgBackRest::Storage::Storage;\n" "use parent 'pgBackRest::Storage::Base';\n" "\n" "use strict;\n" @@ -16248,66 +14896,51 @@ static const EmbeddedModule embeddedModule[] = "use English '-no_match_vars';\n" "\n" "use File::Basename qw(dirname);\n" + "use Fcntl qw(:mode);\n" + "use File::stat qw{lstat};\n" + "use JSON::PP;\n" "\n" "use pgBackRest::Common::Exception;\n" + "use pgBackRest::Common::Io::Handle;\n" "use pgBackRest::Common::Log;\n" - "use pgBackRest::Common::String;\n" "use pgBackRest::Storage::Base;\n" - "use pgBackRest::Storage::Filter::Sha;\n" + "use pgBackRest::Storage::StorageRead;\n" + "use pgBackRest::Storage::StorageWrite;\n" "\n\n\n\n" "sub new\n" "{\n" "my $class = shift;\n" "\n\n" - "my\n" + "my $self = {};\n" + "bless $self, $class;\n" + "\n\n" "(\n" - "$strOperation,\n" - "$strPathBase,\n" - "$oDriver,\n" - "$hRule,\n" - "$bAllowTemp,\n" - "$strTempExtension,\n" - "$strDefaultPathMode,\n" - "$strDefaultFileMode,\n" - "$lBufferMax,\n" - "$strCipherType,\n" - "$strCipherPassUser,\n" + "my $strOperation,\n" + "$self->{strType},\n" + "$self->{strPath},\n" + "$self->{lBufferMax},\n" + "$self->{strDefaultPathMode},\n" + "$self->{strDefaultFileMode},\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'strPathBase'},\n" - "{name => 'oDriver'},\n" - "{name => 'hRule', optional => true},\n" - "{name => 'bAllowTemp', optional => true, default => true},\n" - "{name => 'strTempExtension', optional => true, default => 'tmp'},\n" + "{name => 'strType'},\n" + "{name => 'strPath', optional => true},\n" + "{name => 'lBufferMax', optional => true, default => 65536},\n" "{name => 'strDefaultPathMode', optional => true, default => '0750'},\n" "{name => 'strDefaultFileMode', optional => true, default => '0640'},\n" - "{name => 'lBufferMax', optional => true},\n" - "{name => 'strCipherType', optional => true},\n" - "{name => 'strCipherPassUser', optional => true, redact => true},\n" ");\n" "\n\n" - "my $self = $class->SUPER::new({lBufferMax => $lBufferMax});\n" - "bless $self, $class;\n" - "\n" - "$self->{strPathBase} = $strPathBase;\n" - "$self->{oDriver} = $oDriver;\n" - "$self->{hRule} = $hRule;\n" - "$self->{bAllowTemp} = $bAllowTemp;\n" - "$self->{strTempExtension} = $strTempExtension;\n" - "$self->{strDefaultPathMode} = $strDefaultPathMode;\n" - "$self->{strDefaultFileMode} = $strDefaultFileMode;\n" - "$self->{strCipherType} = $strCipherType;\n" - "$self->{strCipherPassUser} = $strCipherPassUser;\n" - "\n" - "if (defined($self->{strCipherType}))\n" + "$self->{oStorageC} = pgBackRest::LibC::Storage->new($self->{strType}, $self->{strPath});\n" + "\n\n" + "if ($self->{strType} eq '')\n" "{\n" - "require pgBackRest::Storage::Filter::CipherBlock;\n" - "pgBackRest::Storage::Filter::CipherBlock->import();\n" + "$self->{strCipherType} = $self->{oStorageC}->cipherType();\n" + "$self->{strCipherPass} = $self->{oStorageC}->cipherPass();\n" "}\n" "\n\n" - "$self->driver()->tempExtensionSet($self->{strTempExtension}) if $self->driver()->can('tempExtensionSet');\n" + "$self->{oJSON} = JSON::PP->new()->allow_nonref();\n" "\n\n" "return logDebugReturn\n" "(\n" @@ -16331,12 +14964,47 @@ static const EmbeddedModule embeddedModule[] = "{name => 'strFileExp'},\n" ");\n" "\n\n" - "my $bExists = $self->driver()->exists($self->pathGet($strFileExp));\n" + "my $bExists = $self->{oStorageC}->exists($strFileExp);\n" "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'bExists', value => $bExists}\n" + "{name => 'bExists', value => $bExists ? true : false}\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub get\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my\n" + "(\n" + "$strOperation,\n" + "$xFile,\n" + "$strCipherPass,\n" + ") =\n" + "logDebugParam\n" + "(\n" + "__PACKAGE__ . '->get', \\@_,\n" + "{name => 'xFile', required => false, trace => true},\n" + "{name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true},\n" + ");\n" + "\n\n\n" + "my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead($xFile, {strCipherPass => $strCipherPass})) : undef;\n" + "\n\n" + "my $bEmpty = false;\n" + "my $tContent = $self->{oStorageC}->get($oFileIo->{oStorageCRead});\n" + "\n" + "if (defined($tContent) && length($tContent) == 0)\n" + "{\n" + "$tContent = undef;\n" + "$bEmpty = true;\n" + "}\n" + "\n\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'rtContent', value => defined($tContent) || $bEmpty ? \\$tContent : undef, trace => true},\n" ");\n" "}\n" "\n\n\n\n\n" @@ -16360,27 +15028,15 @@ static const EmbeddedModule embeddedModule[] = "my $strHash;\n" "my $lSize;\n" "\n\n" - "my $oFileIo =\n" - "defined($xFileExp) ? (ref($xFileExp) ? $xFileExp :\n" - "$self->openRead($self->pathGet($xFileExp), {bIgnoreMissing => $bIgnoreMissing})) : undef;\n" - "\n" - "if (defined($oFileIo))\n" - "{\n" - "$lSize = 0;\n" - "my $oShaIo = new pgBackRest::Storage::Filter::Sha($oFileIo);\n" - "my $lSizeRead;\n" - "\n" - "do\n" - "{\n" - "my $tContent;\n" - "$lSizeRead = $oShaIo->read(\\$tContent, $self->{lBufferMax});\n" - "$lSize += $lSizeRead;\n" - "}\n" - "while ($lSizeRead != 0);\n" + "my $oFileIo = ref($xFileExp) ? $xFileExp : $self->openRead($xFileExp, {bIgnoreMissing => $bIgnoreMissing});\n" "\n\n" - "$oShaIo->close();\n" + "$oFileIo->{oStorageCRead}->filterAdd(COMMON_IO_HANDLE, undef);\n" + "$oFileIo->{oStorageCRead}->filterAdd(STORAGE_FILTER_SHA, undef);\n" "\n\n" - "$strHash = $oShaIo->result(STORAGE_FILTER_SHA);\n" + "if ($self->{oStorageC}->readDrain($oFileIo->{oStorageCRead}))\n" + "{\n" + "$strHash = $oFileIo->result(STORAGE_FILTER_SHA);\n" + "$lSize = $oFileIo->result(COMMON_IO_HANDLE);\n" "}\n" "\n\n" "return logDebugReturn\n" @@ -16403,17 +15059,23 @@ static const EmbeddedModule embeddedModule[] = ") =\n" "logDebugParam\n" "(\n" - "__PACKAGE__ . '::fileStat', \\@_,\n" + "__PACKAGE__ . '->info', \\@_,\n" "{name => 'strPathFileExp'},\n" "{name => 'bIgnoreMissing', optional => true, default => false},\n" ");\n" - "\n\n" - "my $oInfo = $self->driver()->info($self->pathGet($strPathFileExp), {bIgnoreMissing => $bIgnoreMissing});\n" + "\n" + "my $rhInfo;\n" + "my $strJson = $self->{oStorageC}->info($strPathFileExp, $bIgnoreMissing);\n" + "\n" + "if (defined($strJson))\n" + "{\n" + "$rhInfo = $self->{oJSON}->decode($strJson);\n" + "}\n" "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'oInfo', value => $oInfo, trace => true}\n" + "{name => 'rhInfo', value => $rhInfo, trace => true}\n" ");\n" "}\n" "\n\n\n\n" @@ -16472,705 +15134,6 @@ static const EmbeddedModule embeddedModule[] = "{name => 'strSourcePathFile', value => $strSourcePathFile, trace => true}\n" ");\n" "}\n" - "\n\n" - "$self->driver()->linkCreate(\n" - "$strSourcePathFile, $strDestinationLink, {bHard => $bHard, bPathCreate => $bPathCreate, bIgnoreExists => $bIgnoreExists});\n" - "\n\n" - "return logDebugReturn($strOperation);\n" - "}\n" - "\n\n\n\n" - "sub list\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPathExp,\n" - "$strExpression,\n" - "$strSortOrder,\n" - "$bIgnoreMissing,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->list', \\@_,\n" - "{name => 'strPathExp', required => false},\n" - "{name => 'strExpression', optional => true},\n" - "{name => 'strSortOrder', optional => true, default => 'forward'},\n" - "{name => 'bIgnoreMissing', optional => true, default => false},\n" - ");\n" - "\n\n" - "my $rstryFileList = $self->driver()->list(\n" - "$self->pathGet($strPathExp), {strExpression => $strExpression, bIgnoreMissing => $bIgnoreMissing});\n" - "\n\n" - "if (defined($strExpression))\n" - "{\n" - "@{$rstryFileList} = grep(/$strExpression/i, @{$rstryFileList});\n" - "}\n" - "\n\n" - "if ($strSortOrder eq 'reverse')\n" - "{\n" - "@{$rstryFileList} = sort {$b cmp $a} @{$rstryFileList};\n" - "}\n" - "\n" - "else\n" - "{\n" - "@{$rstryFileList} = sort @{$rstryFileList};\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'stryFileList', value => $rstryFileList}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub manifest\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPathExp,\n" - "$strFilter,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->manifest', \\@_,\n" - "{name => 'strPathExp'},\n" - "{name => 'strFilter', optional => true, trace => true},\n" - ");\n" - "\n" - "my $hManifest = $self->driver()->manifest($self->pathGet($strPathExp), {strFilter => $strFilter});\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'hManifest', value => $hManifest, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub move\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strSourcePathFileExp,\n" - "$strDestinationPathFileExp,\n" - "$bPathCreate,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->move', \\@_,\n" - "{name => 'strSourcePathExp'},\n" - "{name => 'strDestinationPathExp'},\n" - "{name => 'bPathCreate', optional => true, default => false, trace => true},\n" - ");\n" - "\n\n" - "$self->driver()->move(\n" - "$self->pathGet($strSourcePathFileExp), $self->pathGet($strDestinationPathFileExp), {bPathCreate => $bPathCreate});\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub openRead\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$xFileExp,\n" - "$bIgnoreMissing,\n" - "$rhyFilter,\n" - "$strCipherPass,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->openRead', \\@_,\n" - "{name => 'xFileExp'},\n" - "{name => 'bIgnoreMissing', optional => true, default => false},\n" - "{name => 'rhyFilter', optional => true},\n" - "{name => 'strCipherPass', optional => true, redact => true},\n" - ");\n" - "\n\n" - "my $oFileIo = $self->driver()->openRead($self->pathGet($xFileExp), {bIgnoreMissing => $bIgnoreMissing});\n" - "\n\n" - "if (defined($oFileIo))\n" - "{\n" - "\n\n" - "if (defined($self->cipherType()))\n" - "{\n" - "$oFileIo = &STORAGE_FILTER_CIPHER_BLOCK->new(\n" - "$oFileIo, $self->cipherType(), defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser(),\n" - "{strMode => STORAGE_DECRYPT});\n" - "}\n" - "\n\n" - "if (defined($rhyFilter))\n" - "{\n" - "foreach my $rhFilter (@{$rhyFilter})\n" - "{\n" - "$oFileIo = $rhFilter->{strClass}->new($oFileIo, @{$rhFilter->{rxyParam}});\n" - "}\n" - "}\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'oFileIo', value => $oFileIo, trace => true},\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub openWrite\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$xFileExp,\n" - "$strMode,\n" - "$strUser,\n" - "$strGroup,\n" - "$lTimestamp,\n" - "$bAtomic,\n" - "$bPathCreate,\n" - "$rhyFilter,\n" - "$strCipherPass,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->openWrite', \\@_,\n" - "{name => 'xFileExp'},\n" - "{name => 'strMode', optional => true, default => $self->{strDefaultFileMode}},\n" - "{name => 'strUser', optional => true},\n" - "{name => 'strGroup', optional => true},\n" - "{name => 'lTimestamp', optional => true},\n" - "{name => 'bAtomic', optional => true, default => false},\n" - "{name => 'bPathCreate', optional => true, default => false},\n" - "{name => 'rhyFilter', optional => true},\n" - "{name => 'strCipherPass', optional => true, redact => true},\n" - ");\n" - "\n\n" - "my $oFileIo = $self->driver()->openWrite($self->pathGet($xFileExp),\n" - "{strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lTimestamp, bPathCreate => $bPathCreate,\n" - "bAtomic => $bAtomic});\n" - "\n\n" - "if (defined($self->cipherType()))\n" - "{\n" - "$oFileIo = &STORAGE_FILTER_CIPHER_BLOCK->new(\n" - "$oFileIo, $self->cipherType(), defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser());\n" - "}\n" - "\n\n" - "if (defined($rhyFilter))\n" - "{\n" - "foreach my $rhFilter (reverse(@{$rhyFilter}))\n" - "{\n" - "$oFileIo = $rhFilter->{strClass}->new($oFileIo, @{$rhFilter->{rxyParam}});\n" - "}\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'oFileIo', value => $oFileIo, trace => true},\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub owner\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPathFileExp,\n" - "$strUser,\n" - "$strGroup\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->owner', \\@_,\n" - "{name => 'strPathFileExp'},\n" - "{name => 'strUser', required => false},\n" - "{name => 'strGroup', required => false}\n" - ");\n" - "\n\n" - "$self->driver()->owner($self->pathGet($strPathFileExp), {strUser => $strUser, strGroup => $strGroup});\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub pathCreate\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPathExp,\n" - "$strMode,\n" - "$bIgnoreExists,\n" - "$bCreateParent,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->pathCreate', \\@_,\n" - "{name => 'strPathExp'},\n" - "{name => 'strMode', optional => true, default => $self->{strDefaultPathMode}},\n" - "{name => 'bIgnoreExists', optional => true, default => false},\n" - "{name => 'bCreateParent', optional => true, default => false},\n" - ");\n" - "\n\n" - "$self->driver()->pathCreate(\n" - "$self->pathGet($strPathExp), {strMode => $strMode, bIgnoreExists => $bIgnoreExists, bCreateParent => $bCreateParent});\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub pathExists\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPathExp,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->pathExists', \\@_,\n" - "{name => 'strPathExp'},\n" - ");\n" - "\n\n" - "my $bExists = $self->driver()->pathExists($self->pathGet($strPathExp));\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'bExists', value => $bExists}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub pathGet\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPathExp,\n" - "$bTemp,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->pathGet', \\@_,\n" - "{name => 'strPathExp', required => false, trace => true},\n" - "{name => 'bTemp', optional => true, default => false, trace => true},\n" - ");\n" - "\n\n" - "my $strPath;\n" - "my $strFile;\n" - "\n\n" - "my $bAbsolute = false;\n" - "\n" - "if (defined($strPathExp) && index($strPathExp, qw(/)) == 0)\n" - "{\n" - "$bAbsolute = true;\n" - "$strPath = $strPathExp;\n" - "}\n" - "else\n" - "{\n" - "\n" - "if (defined($strPathExp) && index($strPathExp, qw(<)) == 0)\n" - "{\n" - "\n" - "my $iPos = index($strPathExp, qw(>));\n" - "\n" - "if ($iPos == -1)\n" - "{\n" - "confess &log(ASSERT, \"found < but not > in '${strPathExp}'\");\n" - "}\n" - "\n" - "my $strType = substr($strPathExp, 0, $iPos + 1);\n" - "\n\n" - "if ($iPos < length($strPathExp) - 1)\n" - "{\n" - "$strFile = substr($strPathExp, $iPos + 2);\n" - "}\n" - "\n\n" - "if (!defined($self->{hRule}->{$strType}))\n" - "{\n" - "confess &log(ASSERT, \"storage rule '${strType}' does not exist\");\n" - "}\n" - "\n\n" - "if (ref($self->{hRule}->{$strType}))\n" - "{\n" - "$strPath = $self->pathBase();\n" - "$strFile = $self->{hRule}{$strType}{fnRule}->($strType, $strFile, $self->{hRule}{$strType}{xData});\n" - "}\n" - "\n" - "else\n" - "{\n" - "$strPath = $self->pathBase() . ($self->pathBase() =~ /\\/$/ ? '' : qw{/}) . $self->{hRule}->{$strType};\n" - "}\n" - "}\n" - "\n" - "else\n" - "{\n" - "$strPath = $self->pathBase();\n" - "$strFile = $strPathExp;\n" - "}\n" - "}\n" - "\n\n" - "if ($bTemp)\n" - "{\n" - "\n" - "if (!$self->{bAllowTemp})\n" - "{\n" - "confess &log(ASSERT, \"temp file not supported for storage '\" . $self->pathBase() . \"'\");\n" - "}\n" - "\n\n" - "if (!$bAbsolute)\n" - "{\n" - "if (!defined($strFile))\n" - "{\n" - "confess &log(ASSERT, 'file part must be defined when temp file specified');\n" - "}\n" - "}\n" - "}\n" - "\n\n" - "$strPath .= defined($strFile) ? ($strPath =~ /\\/$/ ? '' : qw{/}) . \"${strFile}\" : '';\n" - "\n\n" - "$strPath .= $bTemp ? \".$self->{strTempExtension}\" : '';\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'strPath', value => $strPath, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub pathSync\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPathExp,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->pathSync', \\@_,\n" - "{name => 'strPathExp'},\n" - ");\n" - "\n" - "$self->driver()->pathSync($self->pathGet($strPathExp));\n" - "\n\n" - "return logDebugReturn($strOperation);\n" - "}\n" - "\n\n\n\n" - "sub remove\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$xstryPathFileExp,\n" - "$bIgnoreMissing,\n" - "$bRecurse,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->remove', \\@_,\n" - "{name => 'xstryPathFileExp'},\n" - "{name => 'bIgnoreMissing', optional => true, default => true},\n" - "{name => 'bRecurse', optional => true, default => false, trace => true},\n" - ");\n" - "\n\n" - "my @stryPathFileExp;\n" - "\n" - "if (ref($xstryPathFileExp))\n" - "{\n" - "foreach my $strPathFileExp (@{$xstryPathFileExp})\n" - "{\n" - "push(@stryPathFileExp, $self->pathGet($strPathFileExp));\n" - "}\n" - "}\n" - "\n\n" - "my $bRemoved = $self->driver()->remove(\n" - "ref($xstryPathFileExp) ? \\@stryPathFileExp : $self->pathGet($xstryPathFileExp),\n" - "{bIgnoreMissing => $bIgnoreMissing, bRecurse => $bRecurse});\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'bRemoved', value => $bRemoved}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub encrypted\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strFileName,\n" - "$bIgnoreMissing,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->encrypted', \\@_,\n" - "{name => 'strFileName'},\n" - "{name => 'bIgnoreMissing', optional => true, default => false},\n" - ");\n" - "\n" - "my $tMagicSignature;\n" - "my $bEncrypted = false;\n" - "\n\n" - "my $oFile = $self->driver()->openRead($self->pathGet($strFileName), {bIgnoreMissing => $bIgnoreMissing});\n" - "\n\n\n" - "if (!defined($oFile))\n" - "{\n" - "if (defined($self->{strCipherType}))\n" - "{\n" - "$bEncrypted = true;\n" - "}\n" - "}\n" - "else\n" - "{\n" - "\n" - "my $lSizeRead = $oFile->read(\\$tMagicSignature, length(CIPHER_MAGIC));\n" - "\n\n" - "$oFile->close();\n" - "\n\n\n" - "if (($lSizeRead > 0) && substr($tMagicSignature, 0, length(CIPHER_MAGIC)) eq CIPHER_MAGIC)\n" - "{\n" - "$bEncrypted = true;\n" - "}\n" - "\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'bEncrypted', value => $bEncrypted}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub encryptionValid\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$bEncrypted,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->encryptionValid', \\@_,\n" - "{name => 'bEncrypted'},\n" - ");\n" - "\n" - "my $bValid = true;\n" - "\n\n" - "if ($bEncrypted)\n" - "{\n" - "if (!defined($self->{strCipherType}))\n" - "{\n" - "$bValid = false;\n" - "}\n" - "}\n" - "else\n" - "{\n" - "if (defined($self->{strCipherType}))\n" - "{\n" - "$bValid = false;\n" - "}\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'bValid', value => $bValid}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub pathBase {shift->{strPathBase}}\n" - "sub driver {shift->{oDriver}}\n" - "sub cipherType {shift->{strCipherType}}\n" - "sub cipherPassUser {shift->{strCipherPassUser}}\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Storage/Posix/Driver.pm", - .data = - "\n\n\n\n\n" - "package pgBackRest::Storage::Posix::Driver;\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "use File::Basename qw(basename dirname);\n" - "use Fcntl qw(:mode);\n" - "use File::stat qw{lstat};\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::Storage::Base;\n" - "use pgBackRest::Storage::Posix::FileRead;\n" - "use pgBackRest::Storage::Posix::FileWrite;\n" - "\n\n\n\n" - "use constant STORAGE_POSIX_DRIVER => __PACKAGE__;\n" - "push @EXPORT, qw(STORAGE_POSIX_DRIVER);\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my $self = {};\n" - "bless $self, $class;\n" - "\n\n" - "(\n" - "my $strOperation,\n" - "$self->{bFileSync},\n" - "$self->{bPathSync},\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'bFileSync', optional => true, default => true},\n" - "{name => 'bPathSync', optional => true, default => true},\n" - ");\n" - "\n\n" - "$self->{strTempExtension} = 'tmp';\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub exists\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strFile,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->exists', \\@_,\n" - "{name => 'strFile', trace => true},\n" - ");\n" - "\n\n" - "my $bExists = true;\n" - "my $oStat = lstat($strFile);\n" - "\n\n" - "if (defined($oStat))\n" - "{\n" - "\n" - "$bExists = !S_ISDIR($oStat->mode) ? true : false;\n" - "}\n" - "else\n" - "{\n" - "\n" - "if (!$OS_ERROR{ENOENT})\n" - "{\n" - "logErrorResult(ERROR_FILE_EXISTS, \"unable to test if file '${strFile}' exists\", $OS_ERROR);\n" - "}\n" - "\n" - "$bExists = false;\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'bExists', value => $bExists, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub info\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPathFile,\n" - "$bIgnoreMissing,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->info', \\@_,\n" - "{name => 'strFile', trace => true},\n" - "{name => 'bIgnoreMissing', optional => true, default => false, trace => true},\n" - ");\n" - "\n\n" - "my $oInfo = lstat($strPathFile);\n" - "\n\n" - "if (!defined($oInfo))\n" - "{\n" - "if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))\n" - "{\n" - "logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, \"unable to stat '${strPathFile}'\", $OS_ERROR);\n" - "}\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'oInfo', value => $oInfo, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub linkCreate\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strSourcePathFile,\n" - "$strDestinationLink,\n" - "$bHard,\n" - "$bPathCreate,\n" - "$bIgnoreExists,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->linkCreate', \\@_,\n" - "{name => 'strSourcePathFile', trace => true},\n" - "{name => 'strDestinationLink', trace => true},\n" - "{name => 'bHard', optional=> true, default => false, trace => true},\n" - "{name => 'bPathCreate', optional=> true, default => true, trace => true},\n" - "{name => 'bIgnoreExists', optional => true, default => false, trace => true},\n" - ");\n" "\n" "if (!($bHard ? link($strSourcePathFile, $strDestinationLink) : symlink($strSourcePathFile, $strDestinationLink)))\n" "{\n" @@ -17210,36 +15173,6 @@ static const EmbeddedModule embeddedModule[] = "return logDebugReturn($strOperation);\n" "}\n" "\n\n\n\n" - "sub linkDestination\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strLink,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->linkDestination', \\@_,\n" - "{name => 'strLink', trace => true},\n" - ");\n" - "\n\n" - "my $strLinkDestination = readlink($strLink);\n" - "\n\n" - "if (!defined($strLinkDestination))\n" - "{\n" - "logErrorResult(\n" - "$OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, \"unable to get destination for link ${strLink}\", $OS_ERROR);\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'strLinkDestination', value => $strLinkDestination, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" "sub list\n" "{\n" "my $self = shift;\n" @@ -17247,38 +15180,32 @@ static const EmbeddedModule embeddedModule[] = "my\n" "(\n" "$strOperation,\n" - "$strPath,\n" + "$strPathExp,\n" + "$strExpression,\n" + "$strSortOrder,\n" "$bIgnoreMissing,\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->list', \\@_,\n" - "{name => 'strPath', trace => true},\n" - "{name => 'bIgnoreMissing', optional => true, default => false, trace => true},\n" + "{name => 'strPathExp', required => false},\n" + "{name => 'strExpression', optional => true},\n" + "{name => 'strSortOrder', optional => true, default => 'forward'},\n" + "{name => 'bIgnoreMissing', optional => true, default => false},\n" ");\n" "\n\n" - "my @stryFileList;\n" - "my $hPath;\n" - "\n\n" - "if (opendir($hPath, $strPath))\n" - "{\n" - "@stryFileList = grep(!/^(\\.)|(\\.\\.)$/i, readdir($hPath));\n" - "close($hPath);\n" - "}\n" + "my $rstryFileList = [];\n" + "my $strFileList = $self->{oStorageC}->list($strPathExp, $bIgnoreMissing, $strSortOrder eq 'forward', $strExpression);\n" "\n" - "else\n" + "if (defined($strFileList) && $strFileList ne '[]')\n" "{\n" - "\n" - "if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))\n" - "{\n" - "logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, \"unable to read path '${strPath}'\", $OS_ERROR);\n" - "}\n" + "$rstryFileList = $self->{oJSON}->decode($strFileList);\n" "}\n" "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'stryFileList', value => \\@stryFileList, ref => true, trace => true}\n" + "{name => 'stryFileList', value => $rstryFileList}\n" ");\n" "}\n" "\n\n\n\n" @@ -17289,20 +15216,17 @@ static const EmbeddedModule embeddedModule[] = "my\n" "(\n" "$strOperation,\n" - "$strPath,\n" - "$bIgnoreMissing,\n" + "$strPathExp,\n" "$strFilter,\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->manifest', \\@_,\n" - "{name => 'strPath', trace => true},\n" - "{name => 'bIgnoreMissing', optional => true, default => false, trace => true},\n" + "{name => 'strPathExp'},\n" "{name => 'strFilter', optional => true, trace => true},\n" ");\n" - "\n\n" - "my $hManifest = {};\n" - "$self->manifestRecurse($strPath, undef, 0, $hManifest, $bIgnoreMissing, $strFilter);\n" + "\n" + "my $hManifest = $self->{oJSON}->decode($self->{oStorageC}->manifest($strPathExp, $strFilter));\n" "\n\n" "return logDebugReturn\n" "(\n" @@ -17310,174 +15234,6 @@ static const EmbeddedModule embeddedModule[] = "{name => 'hManifest', value => $hManifest, trace => true}\n" ");\n" "}\n" - "\n" - "sub manifestRecurse\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPath,\n" - "$strSubPath,\n" - "$iDepth,\n" - "$hManifest,\n" - "$bIgnoreMissing,\n" - "$strFilter,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '::manifestRecurse', \\@_,\n" - "{name => 'strPath', trace => true},\n" - "{name => 'strSubPath', required => false, trace => true},\n" - "{name => 'iDepth', default => 0, trace => true},\n" - "{name => 'hManifest', required => false, trace => true},\n" - "{name => 'bIgnoreMissing', required => false, default => false, trace => true},\n" - "{name => 'strFilter', required => false, trace => true},\n" - ");\n" - "\n\n" - "my $strPathRead = $strPath . (defined($strSubPath) ? \"/${strSubPath}\" : '');\n" - "my $hPath;\n" - "\n\n" - "my $oPathInfo = $self->info($strPathRead, {bIgnoreMissing => $bIgnoreMissing});\n" - "\n" - "if (defined($oPathInfo))\n" - "{\n" - "\n" - "if ($iDepth == 0 && !S_ISDIR($oPathInfo->mode()))\n" - "{\n" - "$hManifest->{basename($strPathRead)} = $self->manifestStat($strPathRead);\n" - "}\n" - "\n" - "else\n" - "{\n" - "\n" - "my @stryFileList = @{$self->list($strPathRead, {bIgnoreMissing => $iDepth != 0})};\n" - "unshift(@stryFileList, '.');\n" - "my $hFileStat = $self->manifestList($strPathRead, \\@stryFileList, $strFilter);\n" - "\n\n" - "foreach my $strFile (keys(%{$hFileStat}))\n" - "{\n" - "my $strManifestFile = $iDepth == 0 ? $strFile : ($strSubPath . ($strFile eq qw(.) ? '' : \"/${strFile}\"));\n" - "$hManifest->{$strManifestFile} = $hFileStat->{$strFile};\n" - "\n\n" - "if ($hManifest->{$strManifestFile}{type} eq 'd' && $strFile ne qw(.))\n" - "{\n" - "$self->manifestRecurse($strPath, $strManifestFile, $iDepth + 1, $hManifest);\n" - "}\n" - "}\n" - "}\n" - "}\n" - "\n\n" - "return logDebugReturn($strOperation);\n" - "}\n" - "\n" - "sub manifestList\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPath,\n" - "$stryFile,\n" - "$strFilter,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->manifestList', \\@_,\n" - "{name => 'strPath', trace => true},\n" - "{name => 'stryFile', trace => true},\n" - "{name => 'strFilter', required => false, trace => true},\n" - ");\n" - "\n" - "my $hFileStat = {};\n" - "\n" - "foreach my $strFile (@{$stryFile})\n" - "{\n" - "if ($strFile ne '.' && defined($strFilter) && $strFilter ne $strFile)\n" - "{\n" - "next;\n" - "}\n" - "\n" - "$hFileStat->{$strFile} = $self->manifestStat(\"${strPath}\" . ($strFile eq qw(.) ? '' : \"/${strFile}\"));\n" - "\n" - "if (!defined($hFileStat->{$strFile}))\n" - "{\n" - "delete($hFileStat->{$strFile});\n" - "}\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'hFileStat', value => $hFileStat, trace => true}\n" - ");\n" - "}\n" - "\n" - "sub manifestStat\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strFile,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->manifestStat', \\@_,\n" - "{name => 'strFile', trace => true},\n" - ");\n" - "\n\n" - "my $oStat = $self->info($strFile, {bIgnoreMissing => true});\n" - "\n\n" - "my $hFile;\n" - "\n" - "if (defined($oStat))\n" - "{\n" - "\n" - "if (S_ISREG($oStat->mode))\n" - "{\n" - "$hFile->{type} = 'f';\n" - "\n\n" - "$hFile->{size} = $oStat->size;\n" - "\n\n" - "$hFile->{modification_time} = $oStat->mtime;\n" - "}\n" - "\n" - "elsif (S_ISDIR($oStat->mode))\n" - "{\n" - "$hFile->{type} = 'd';\n" - "}\n" - "\n" - "elsif (S_ISLNK($oStat->mode))\n" - "{\n" - "$hFile->{type} = 'l';\n" - "$hFile->{link_destination} = $self->linkDestination($strFile);\n" - "}\n" - "\n" - "else\n" - "{\n" - "confess &log(ERROR, \"${strFile} is not of type directory, file, or link\", ERROR_FILE_INVALID);\n" - "}\n" - "\n\n" - "$hFile->{user} = getpwuid($oStat->uid);\n" - "\n\n" - "$hFile->{group} = getgrgid($oStat->gid);\n" - "\n\n" - "if ($hFile->{type} ne 'l')\n" - "{\n" - "$hFile->{mode} = sprintf('%04o', S_IMODE($oStat->mode));\n" - "}\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'hFile', value => $hFile, trace => true}\n" - ");\n" - "}\n" "\n\n\n\n" "sub move\n" "{\n" @@ -17486,49 +15242,23 @@ static const EmbeddedModule embeddedModule[] = "my\n" "(\n" "$strOperation,\n" - "$strSourceFile,\n" - "$strDestinationFile,\n" + "$strSourceFileExp,\n" + "$strDestinationFileExp,\n" "$bPathCreate,\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->move', \\@_,\n" - "{name => 'strSourceFile', trace => true},\n" - "{name => 'strDestinationFile', trace => true},\n" - "{name => 'bPathCreate', default => false, trace => true},\n" + "{name => 'strSourceFileExp'},\n" + "{name => 'strDestinationFileExp'},\n" ");\n" "\n\n" - "my $strSourcePathFile = dirname($strSourceFile);\n" - "my $strDestinationPathFile = dirname($strDestinationFile);\n" + "my $strSourceFile = $self->pathGet($strSourceFileExp);\n" + "my $strDestinationFile = $self->pathGet($strDestinationFileExp);\n" "\n\n" "if (!rename($strSourceFile, $strDestinationFile))\n" "{\n" - "my $strMessage = \"unable to move '${strSourceFile}'\";\n" - "\n\n" - "if ($OS_ERROR{ENOENT})\n" - "{\n" - "if (!$self->exists($strSourceFile))\n" - "{\n" - "logErrorResult(ERROR_FILE_MISSING, \"${strMessage} because it is missing\");\n" - "}\n" - "\n" - "if ($bPathCreate)\n" - "{\n" - "\n" - "$self->pathCreate($strDestinationPathFile, {bCreateParent => true, bIgnoreExists => true});\n" - "\n\n" - "$self->move($strSourceFile, $strDestinationFile);\n" - "}\n" - "else\n" - "{\n" - "logErrorResult(ERROR_PATH_MISSING, \"${strMessage} to missing path '${strDestinationPathFile}'\");\n" - "}\n" - "}\n" - "\n" - "else\n" - "{\n" - "logErrorResult(ERROR_FILE_MOVE, \"${strMessage} to '${strDestinationFile}'\", $OS_ERROR);\n" - "}\n" + "logErrorResult(ERROR_FILE_MOVE, \"unable to move '${strSourceFile}' to '${strDestinationFile}'\", $OS_ERROR);\n" "}\n" "\n\n" "return logDebugReturn($strOperation);\n" @@ -17541,22 +15271,40 @@ static const EmbeddedModule embeddedModule[] = "my\n" "(\n" "$strOperation,\n" - "$strFile,\n" + "$xFileExp,\n" "$bIgnoreMissing,\n" + "$rhyFilter,\n" + "$strCipherPass,\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->openRead', \\@_,\n" - "{name => 'strFile', trace => true},\n" - "{name => 'bIgnoreMissing', optional => true, default => false, trace => true},\n" + "{name => 'xFileExp'},\n" + "{name => 'bIgnoreMissing', optional => true, default => false},\n" + "{name => 'rhyFilter', optional => true},\n" + "{name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true},\n" ");\n" - "\n" - "my $oFileIO = new pgBackRest::Storage::Posix::FileRead($self, $strFile, {bIgnoreMissing => $bIgnoreMissing});\n" + "\n\n" + "my $oFileIo = pgBackRest::LibC::StorageRead->new($self->{oStorageC}, $xFileExp, $bIgnoreMissing);\n" + "\n\n" + "if (defined($self->cipherType()))\n" + "{\n" + "$oFileIo->filterAdd(STORAGE_FILTER_CIPHER_BLOCK, $self->{oJSON}->encode([false, $self->cipherType(), $strCipherPass]));\n" + "}\n" + "\n\n" + "if (defined($rhyFilter))\n" + "{\n" + "foreach my $rhFilter (@{$rhyFilter})\n" + "{\n" + "$oFileIo->filterAdd(\n" + "$rhFilter->{strClass}, defined($rhFilter->{rxyParam}) ? $self->{oJSON}->encode($rhFilter->{rxyParam}) : undef);\n" + "}\n" + "}\n" "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'oFileIO', value => $oFileIO, trace => true},\n" + "{name => 'oFileIo', value => new pgBackRest::Storage::StorageRead($self, $oFileIo), trace => true},\n" ");\n" "}\n" "\n\n\n\n" @@ -17567,35 +15315,51 @@ static const EmbeddedModule embeddedModule[] = "my\n" "(\n" "$strOperation,\n" - "$strFile,\n" + "$xFileExp,\n" "$strMode,\n" "$strUser,\n" "$strGroup,\n" "$lTimestamp,\n" - "$bPathCreate,\n" "$bAtomic,\n" + "$bPathCreate,\n" + "$rhyFilter,\n" + "$strCipherPass,\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->openWrite', \\@_,\n" - "{name => 'strFile', trace => true},\n" - "{name => 'strMode', optional => true, trace => true},\n" - "{name => 'strUser', optional => true, trace => true},\n" - "{name => 'strGroup', optional => true, trace => true},\n" - "{name => 'lTimestamp', optional => true, trace => true},\n" - "{name => 'bPathCreate', optional => true, trace => true},\n" - "{name => 'bAtomic', optional => true, trace => true},\n" + "{name => 'xFileExp'},\n" + "{name => 'strMode', optional => true, default => $self->{strDefaultFileMode}},\n" + "{name => 'strUser', optional => true},\n" + "{name => 'strGroup', optional => true},\n" + "{name => 'lTimestamp', optional => true, default => '0'},\n" + "{name => 'bAtomic', optional => true, default => false},\n" + "{name => 'bPathCreate', optional => true, default => false},\n" + "{name => 'rhyFilter', optional => true},\n" + "{name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true},\n" ");\n" - "\n" - "my $oFileIO = new pgBackRest::Storage::Posix::FileWrite(\n" - "$self, $strFile,\n" - "{strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lTimestamp, bPathCreate => $bPathCreate,\n" - "bAtomic => $bAtomic, bSync => $self->{bFileSync}});\n" + "\n\n" + "my $oFileIo = pgBackRest::LibC::StorageWrite->new(\n" + "$self->{oStorageC}, $xFileExp, oct($strMode), $strUser, $strGroup, $lTimestamp, $bAtomic, $bPathCreate);\n" + "\n\n" + "if (defined($rhyFilter))\n" + "{\n" + "foreach my $rhFilter (@{$rhyFilter})\n" + "{\n" + "$oFileIo->filterAdd(\n" + "$rhFilter->{strClass}, defined($rhFilter->{rxyParam}) ? $self->{oJSON}->encode($rhFilter->{rxyParam}) : undef);\n" + "}\n" + "}\n" + "\n\n" + "if (defined($self->cipherType()))\n" + "{\n" + "$oFileIo->filterAdd(STORAGE_FILTER_CIPHER_BLOCK, $self->{oJSON}->encode([true, $self->cipherType(), $strCipherPass]));\n" + "}\n" "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'oFileIO', value => $oFileIO, trace => true},\n" + "{name => 'oFileIo', value => new pgBackRest::Storage::StorageWrite($self, $oFileIo), trace => true},\n" ");\n" "}\n" "\n\n\n\n" @@ -17606,25 +15370,31 @@ static const EmbeddedModule embeddedModule[] = "my\n" "(\n" "$strOperation,\n" - "$strFilePath,\n" + "$strPathFileExp,\n" "$strUser,\n" - "$strGroup,\n" + "$strGroup\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->owner', \\@_,\n" - "{name => 'strFilePath', trace => true},\n" - "{name => 'strUser', optional => true, trace => true},\n" - "{name => 'strGroup', optional => true, trace => true},\n" + "{name => 'strPathFileExp'},\n" + "{name => 'strUser', required => false},\n" + "{name => 'strGroup', required => false}\n" ");\n" "\n\n" "if (defined($strUser) || defined($strGroup))\n" "{\n" - "my $strMessage = \"unable to set ownership for '${strFilePath}'\";\n" + "my $strPathFile = $self->pathGet($strPathFileExp);\n" + "my $strMessage = \"unable to set ownership for '${strPathFile}'\";\n" "my $iUserId;\n" "my $iGroupId;\n" "\n\n\n" - "my $oStat = $self->info($strFilePath);\n" + "my $oStat = lstat($strPathFile);\n" + "\n" + "if (!defined($oStat))\n" + "{\n" + "confess &log(ERROR, \"unable to stat '${strPathFile}': No such file or directory\", ERROR_FILE_MISSING);\n" + "}\n" "\n" "if (!defined($strUser))\n" "{\n" @@ -17658,14 +15428,75 @@ static const EmbeddedModule embeddedModule[] = "\n\n" "if ($iUserId != $oStat->uid || $iGroupId != $oStat->gid)\n" "{\n" - "if (!chown($iUserId, $iGroupId, $strFilePath))\n" + "if (!chown($iUserId, $iGroupId, $strPathFile))\n" "{\n" "logErrorResult(ERROR_FILE_OWNER, \"${strMessage}\", $OS_ERROR);\n" "}\n" "}\n" "}\n" "\n\n" - "return logDebugReturn($strOperation);\n" + "return logDebugReturn\n" + "(\n" + "$strOperation\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub pathAbsolute\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my\n" + "(\n" + "$strOperation,\n" + "$strBasePath,\n" + "$strPath\n" + ") =\n" + "logDebugParam\n" + "(\n" + "__PACKAGE__ . '->pathAbsolute', \\@_,\n" + "{name => 'strBasePath', trace => true},\n" + "{name => 'strPath', trace => true}\n" + ");\n" + "\n\n" + "my $strAbsolutePath;\n" + "\n\n" + "if (index($strPath, '/') == 0)\n" + "{\n" + "$strAbsolutePath = $strPath;\n" + "}\n" + "\n" + "else\n" + "{\n" + "\n" + "if (index($strBasePath, '/') != 0 || index($strBasePath, '/..') != -1)\n" + "{\n" + "confess &log(ERROR, \"${strBasePath} is not an absolute path\", ERROR_PATH_TYPE);\n" + "}\n" + "\n" + "while (index($strPath, '..') == 0)\n" + "{\n" + "$strBasePath = dirname($strBasePath);\n" + "$strPath = substr($strPath, 2);\n" + "\n" + "if (index($strPath, '/') == 0)\n" + "{\n" + "$strPath = substr($strPath, 1);\n" + "}\n" + "}\n" + "\n" + "$strAbsolutePath = \"${strBasePath}/${strPath}\";\n" + "}\n" + "\n\n" + "if (index($strAbsolutePath, '/') != 0 || index($strAbsolutePath, '/..') != -1)\n" + "{\n" + "confess &log(ERROR, \"result ${strAbsolutePath} was not an absolute path\", ERROR_PATH_TYPE);\n" + "}\n" + "\n\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'strAbsolutePath', value => $strAbsolutePath, trace => true}\n" + ");\n" "}\n" "\n\n\n\n" "sub pathCreate\n" @@ -17675,7 +15506,7 @@ static const EmbeddedModule embeddedModule[] = "my\n" "(\n" "$strOperation,\n" - "$strPath,\n" + "$strPathExp,\n" "$strMode,\n" "$bIgnoreExists,\n" "$bCreateParent,\n" @@ -17683,42 +15514,18 @@ static const EmbeddedModule embeddedModule[] = "logDebugParam\n" "(\n" "__PACKAGE__ . '->pathCreate', \\@_,\n" - "{name => 'strPath', trace => true},\n" - "{name => 'strMode', optional => true, default => '0750', trace => true},\n" - "{name => 'bIgnoreExists', optional => true, default => false, trace => true},\n" - "{name => 'bCreateParent', optional => true, default => false, trace => true},\n" + "{name => 'strPathExp'},\n" + "{name => 'strMode', optional => true},\n" + "{name => 'bIgnoreExists', optional => true, default => false},\n" + "{name => 'bCreateParent', optional => true, default => false},\n" ");\n" "\n\n" - "if (!mkdir($strPath, oct($strMode)))\n" - "{\n" - "my $strMessage = \"unable to create path '${strPath}'\";\n" + "$self->{oStorageC}->pathCreate($strPathExp, $strMode, $bIgnoreExists, $bCreateParent);\n" "\n\n" - "if ($OS_ERROR{ENOENT})\n" - "{\n" - "if (!$bCreateParent)\n" - "{\n" - "confess &log(ERROR, \"${strMessage} because parent does not exist\", ERROR_PATH_MISSING);\n" - "}\n" - "\n\n" - "$self->pathCreate(dirname($strPath), {strMode => $strMode, bIgnoreExists => true, bCreateParent => $bCreateParent});\n" - "\n\n" - "$self->pathCreate($strPath, {strMode => $strMode, bIgnoreExists => true});\n" - "}\n" - "\n" - "elsif ($OS_ERROR{EEXIST})\n" - "{\n" - "if (!$bIgnoreExists)\n" - "{\n" - "confess &log(ERROR, \"${strMessage} because it already exists\", ERROR_PATH_EXISTS);\n" - "}\n" - "}\n" - "else\n" - "{\n" - "logErrorResult(ERROR_PATH_CREATE, ${strMessage}, $OS_ERROR);\n" - "}\n" - "}\n" - "\n\n" - "return logDebugReturn($strOperation);\n" + "return logDebugReturn\n" + "(\n" + "$strOperation\n" + ");\n" "}\n" "\n\n\n\n" "sub pathExists\n" @@ -17728,887 +15535,67 @@ static const EmbeddedModule embeddedModule[] = "my\n" "(\n" "$strOperation,\n" - "$strPath,\n" + "$strPathExp,\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->pathExists', \\@_,\n" - "{name => 'strPath', trace => true},\n" + "{name => 'strPathExp'},\n" ");\n" "\n\n" - "my $bExists = true;\n" - "my $oStat = lstat($strPath);\n" - "\n\n" - "if (defined($oStat))\n" - "{\n" - "\n" - "$bExists = S_ISDIR($oStat->mode) ? true : false;\n" - "}\n" - "else\n" - "{\n" - "\n" - "if (!$OS_ERROR{ENOENT})\n" - "{\n" - "logErrorResult(ERROR_FILE_EXISTS, \"unable to test if path '${strPath}' exists\", $OS_ERROR);\n" - "}\n" - "\n" - "$bExists = false;\n" - "}\n" + "my $bExists = $self->{oStorageC}->pathExists($strPathExp);\n" "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'bExists', value => $bExists, trace => true}\n" + "{name => 'bExists', value => $bExists ? true : false}\n" ");\n" "}\n" "\n\n\n\n" - "sub pathSync\n" + "sub pathGet\n" "{\n" "my $self = shift;\n" "\n\n" "my\n" "(\n" "$strOperation,\n" - "$strPath,\n" + "$strPathExp,\n" ") =\n" "logDebugParam\n" "(\n" - "__PACKAGE__ . '->pathSync', \\@_,\n" - "{name => 'strPath', trace => true},\n" + "__PACKAGE__ . '->pathGet', \\@_,\n" + "{name => 'strPathExp'},\n" ");\n" - "\n" - "open(my $hPath, \"<\", $strPath)\n" - "or confess &log(ERROR, \"unable to open '${strPath}' for sync\", ERROR_PATH_OPEN);\n" - "open(my $hPathDup, \">&\", $hPath)\n" - "or confess &log(ERROR, \"unable to duplicate '${strPath}' handle for sync\", ERROR_PATH_OPEN);\n" - "\n" - "$hPathDup->sync()\n" - "or confess &log(ERROR, \"unable to sync path '${strPath}'\", ERROR_PATH_SYNC);\n" - "\n" - "close($hPathDup);\n" - "close($hPath);\n" "\n\n" - "return logDebugReturn($strOperation);\n" + "my $strPath = $self->{oStorageC}->pathGet($strPathExp);\n" + "\n\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'strPath', value => $strPath, trace => true}\n" + ");\n" "}\n" "\n\n\n\n" - "sub remove\n" + "sub pathRemove\n" "{\n" "my $self = shift;\n" "\n\n" "my\n" "(\n" "$strOperation,\n" - "$strPathFile,\n" + "$strPathExp,\n" "$bIgnoreMissing,\n" "$bRecurse,\n" ") =\n" "logDebugParam\n" "(\n" - "__PACKAGE__ . '->remove', \\@_,\n" - "{name => 'strPathFile', trace => true},\n" - "{name => 'bIgnoreMissing', optional => true, default => false, trace => true},\n" - "{name => 'bRecurse', optional => true, default => false, trace => true},\n" - ");\n" - "\n\n" - "my $bRemoved = true;\n" - "\n\n" - "if ($bRecurse)\n" - "{\n" - "\n" - "require pgBackRest::LibC;\n" - "pgBackRest::LibC->import(qw(:storage));\n" - "\n" - "storagePosixPathRemove($strPathFile, !$bIgnoreMissing, $bRecurse)\n" - "}\n" - "\n" - "else\n" - "{\n" - "foreach my $strFile (ref($strPathFile) ? @{$strPathFile} : ($strPathFile))\n" - "{\n" - "if (unlink($strFile) != 1)\n" - "{\n" - "$bRemoved = false;\n" - "\n\n" - "if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))\n" - "{\n" - "logErrorResult(\n" - "$OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, \"unable to remove file '${strFile}'\", $OS_ERROR);\n" - "}\n" - "}\n" - "}\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'bRemoved', value => $bRemoved, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub capability {true}\n" - "sub className {STORAGE_POSIX_DRIVER}\n" - "sub tempExtension {shift->{strTempExtension}}\n" - "sub tempExtensionSet {my $self = shift; $self->{strTempExtension} = shift}\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Storage/Posix/FileRead.pm", - .data = - "\n\n\n" - "package pgBackRest::Storage::Posix::FileRead;\n" - "use parent 'pgBackRest::Common::Io::Handle';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Fcntl qw(O_RDONLY);\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Log;\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$oDriver,\n" - "$strName,\n" - "$bIgnoreMissing,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'oDriver', trace => true},\n" - "{name => 'strName', trace => true},\n" - "{name => 'bIgnoreMissing', optional => true, default => false, trace => true},\n" - ");\n" - "\n\n" - "my $fhFile;\n" - "\n" - "if (!sysopen($fhFile, $strName, O_RDONLY))\n" - "{\n" - "if (!($OS_ERROR{ENOENT} && $bIgnoreMissing))\n" - "{\n" - "logErrorResult($OS_ERROR{ENOENT} ? ERROR_FILE_MISSING : ERROR_FILE_OPEN, \"unable to open '${strName}'\", $OS_ERROR);\n" - "}\n" - "\n" - "undef($fhFile);\n" - "}\n" - "\n\n" - "my $self;\n" - "\n" - "if (defined($fhFile))\n" - "{\n" - "\n" - "binmode($fhFile);\n" - "\n\n" - "$self = $class->SUPER::new(\"'${strName}'\", $fhFile);\n" - "bless $self, $class;\n" - "\n\n" - "$self->{oDriver} = $oDriver;\n" - "$self->{strName} = $strName;\n" - "$self->{fhFile} = $fhFile;\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub close\n" - "{\n" - "my $self = shift;\n" - "\n" - "if (defined($self->handle()))\n" - "{\n" - "\n" - "close($self->handle());\n" - "undef($self->{fhFile});\n" - "\n\n" - "$self->SUPER::close();\n" - "}\n" - "\n" - "return true;\n" - "}\n" - "\n\n\n\n" - "sub handle {shift->{fhFile}}\n" - "sub name {shift->{strName}}\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Storage/Posix/FileWrite.pm", - .data = - "\n\n\n" - "package pgBackRest::Storage::Posix::FileWrite;\n" - "use parent 'pgBackRest::Common::Io::Handle';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);\n" - "use File::Basename qw(dirname);\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Log;\n" - "\n" - "use pgBackRest::Common::Io::Handle;\n" - "use pgBackRest::Storage::Base;\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$oDriver,\n" - "$strName,\n" - "$strMode,\n" - "$strUser,\n" - "$strGroup,\n" - "$lTimestamp,\n" - "$bPathCreate,\n" - "$bAtomic,\n" - "$bSync,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'oDriver', trace => true},\n" - "{name => 'strName', trace => true},\n" - "{name => 'strMode', optional => true, trace => true},\n" - "{name => 'strUser', optional => true, trace => true},\n" - "{name => 'strGroup', optional => true, trace => true},\n" - "{name => 'lTimestamp', optional => true, trace => true},\n" - "{name => 'bPathCreate', optional => true, default => false, trace => true},\n" - "{name => 'bAtomic', optional => true, default => false, trace => true},\n" - "{name => 'bSync', optional => true, default => true, trace => true},\n" - ");\n" - "\n\n" - "my $self = $class->SUPER::new(\"'${strName}'\");\n" - "bless $self, $class;\n" - "\n\n" - "$self->{oDriver} = $oDriver;\n" - "$self->{strName} = $strName;\n" - "$self->{strMode} = $strMode;\n" - "$self->{strUser} = $strUser;\n" - "$self->{strGroup} = $strGroup;\n" - "$self->{lTimestamp} = $lTimestamp;\n" - "$self->{bPathCreate} = $bPathCreate;\n" - "$self->{bAtomic} = $bAtomic;\n" - "$self->{bSync} = $bSync;\n" - "\n\n" - "if ($self->{bAtomic})\n" - "{\n" - "\n" - "$self->{strNameTmp} = \"$self->{strName}.\" . $self->{oDriver}->tempExtension();\n" - "}\n" - "\n\n" - "$self->{bOpened} = false;\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub open\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my $strFile = $self->{bAtomic} ? $self->{strNameTmp} : $self->{strName};\n" - "\n\n" - "if (!sysopen(\n" - "$self->{fhFile}, $strFile, O_WRONLY | O_CREAT | O_TRUNC, oct(defined($self->{strMode}) ? $self->{strMode} : '0666')))\n" - "{\n" - "\n" - "if ($OS_ERROR{ENOENT} && $self->{bPathCreate})\n" - "{\n" - "$self->{oDriver}->pathCreate(dirname($strFile), {bIgnoreExists => true, bCreateParent => true});\n" - "$self->{bPathCreate} = false;\n" - "return $self->open();\n" - "}\n" - "\n" - "logErrorResult($OS_ERROR{ENOENT} ? ERROR_PATH_MISSING : ERROR_FILE_OPEN, \"unable to open '${strFile}'\", $OS_ERROR);\n" - "}\n" - "\n\n" - "binmode($self->{fhFile});\n" - "\n\n" - "$self->{oDriver}->owner($strFile, {strUser => $self->{strUser}, strGroup => $self->{strGroup}});\n" - "\n\n" - "$self->handleWriteSet($self->{fhFile});\n" - "\n\n" - "$self->{bOpened} = true;\n" - "\n" - "return true;\n" - "}\n" - "\n\n\n\n" - "sub write\n" - "{\n" - "my $self = shift;\n" - "my $rtBuffer = shift;\n" - "\n\n" - "$self->open() if !$self->opened();\n" - "\n" - "return $self->SUPER::write($rtBuffer);\n" - "}\n" - "\n\n\n\n" - "sub close\n" - "{\n" - "my $self = shift;\n" - "\n" - "if (defined($self->handle()))\n" - "{\n" - "\n" - "if ($self->{bSync})\n" - "{\n" - "$self->handle()->sync();\n" - "}\n" - "\n\n" - "close($self->handle());\n" - "undef($self->{fhFile});\n" - "\n\n" - "my $strCurrentName = $self->{bAtomic} ? $self->{strNameTmp} : $self->{strName};\n" - "\n\n" - "if (defined($self->{lTimestamp}))\n" - "{\n" - "utime(time(), $self->{lTimestamp}, $strCurrentName)\n" - "or logErrorResult(ERROR_FILE_WRITE, \"unable to set time for '${strCurrentName}'\", $OS_ERROR);\n" - "}\n" - "\n\n" - "if ($self->{bAtomic})\n" - "{\n" - "$self->{oDriver}->move($strCurrentName, $self->{strName});\n" - "}\n" - "\n\n" - "$self->resultSet(COMMON_IO_HANDLE, $self->{lSize});\n" - "\n\n" - "$self->SUPER::close();\n" - "}\n" - "\n" - "return true;\n" - "}\n" - "\n\n\n\n" - "sub DESTROY\n" - "{\n" - "my $self = shift;\n" - "\n" - "if (defined($self->handle()))\n" - "{\n" - "CORE::close($self->handle());\n" - "undef($self->{fhFile});\n" - "}\n" - "}\n" - "\n\n\n\n" - "sub handle {shift->{fhFile}}\n" - "sub opened {shift->{bOpened}}\n" - "sub name {shift->{strName}}\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Storage/S3/Auth.pm", - .data = - "\n\n\n\n\n\n" - "package pgBackRest::Storage::S3::Auth;\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Digest::SHA qw(hmac_sha256 hmac_sha256_hex);\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "use POSIX qw(strftime);\n" - "\n" - "use pgBackRest::Common::Http::Common;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::LibC qw(:crypto);\n" - "\n\n\n\n" - "use constant S3 => 's3';\n" - "use constant AWS4 => 'AWS4';\n" - "use constant AWS4_REQUEST => 'aws4_request';\n" - "use constant AWS4_HMAC_SHA256 => 'AWS4-HMAC-SHA256';\n" - "\n" - "use constant S3_HEADER_AUTHORIZATION => 'authorization';\n" - "push @EXPORT, qw(S3_HEADER_AUTHORIZATION);\n" - "use constant S3_HEADER_DATE => 'x-amz-date';\n" - "push @EXPORT, qw(S3_HEADER_DATE);\n" - "use constant S3_HEADER_CONTENT_SHA256 => 'x-amz-content-sha256';\n" - "push @EXPORT, qw(S3_HEADER_CONTENT_SHA256);\n" - "use constant S3_HEADER_HOST => 'host';\n" - "push @EXPORT, qw(S3_HEADER_HOST);\n" - "use constant S3_HEADER_TOKEN => 'x-amz-security-token';\n" - "push @EXPORT, qw(S3_HEADER_TOKEN);\n" - "\n" - "use constant PAYLOAD_DEFAULT_HASH => cryptoHashOne('sha256', '');\n" - "push @EXPORT, qw(PAYLOAD_DEFAULT_HASH);\n" - "\n\n\n\n" - "sub s3DateTime\n" - "{\n" - "\n" - "my\n" - "(\n" - "$strOperation,\n" - "$lTime,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '::s3DateTime', \\@_,\n" - "{name => 'lTime', default => time(), trace => true},\n" - ");\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'strDateTime', value => strftime(\"%Y%m%dT%H%M%SZ\", gmtime($lTime)), trace => true}\n" - ");\n" - "}\n" - "\n" - "push @EXPORT, qw(s3DateTime);\n" - "\n\n\n\n" - "sub s3CanonicalRequest\n" - "{\n" - "\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strVerb,\n" - "$strUri,\n" - "$strQuery,\n" - "$hHeader,\n" - "$strPayloadHash,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '::s3CanonicalRequest', \\@_,\n" - "{name => 'strVerb', trace => true},\n" - "{name => 'strUri', trace => true},\n" - "{name => 'strQuery', trace => true},\n" - "{name => 'hHeader', trace => true},\n" - "{name => 'strPayloadHash', trace => true},\n" - ");\n" - "\n\n" - "my $strCanonicalRequest =\n" - "\"${strVerb}\\n${strUri}\\n${strQuery}\\n\";\n" - "my $strSignedHeaders;\n" - "\n" - "foreach my $strHeader (sort(keys(%{$hHeader})))\n" - "{\n" - "if (lc($strHeader) ne $strHeader)\n" - "{\n" - "confess &log(ASSERT, \"header '${strHeader}' must be lower case\");\n" - "}\n" - "\n" - "$strCanonicalRequest .= $strHeader . \":$hHeader->{$strHeader}\\n\";\n" - "$strSignedHeaders .= (defined($strSignedHeaders) ? qw(;) : '') . lc($strHeader);\n" - "}\n" - "\n" - "$strCanonicalRequest .= \"\\n${strSignedHeaders}\\n${strPayloadHash}\";\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'strCanonicalRequest', value => $strCanonicalRequest, trace => true},\n" - "{name => 'strSignedHeaders', value => $strSignedHeaders, trace => true},\n" - ");\n" - "}\n" - "\n" - "push @EXPORT, qw(s3CanonicalRequest);\n" - "\n\n\n\n" - "my $hSigningKeyCache;\n" - "\n" - "sub s3SigningKey\n" - "{\n" - "\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strDate,\n" - "$strRegion,\n" - "$strSecretAccessKey,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '::s3SigningKey', \\@_,\n" - "{name => 'strDate', trace => true},\n" - "{name => 'strRegion', trace => true},\n" - "{name => 'strSecretAccessKey', redact => true, trace => true},\n" - ");\n" - "\n\n" - "my $strSigningKey = $hSigningKeyCache->{$strDate}{$strRegion}{$strSecretAccessKey};\n" - "\n\n" - "if (!defined($strSigningKey))\n" - "{\n" - "my $strDateKey = hmac_sha256($strDate, AWS4 . $strSecretAccessKey);\n" - "my $strRegionKey = hmac_sha256($strRegion, $strDateKey);\n" - "my $strServiceKey = hmac_sha256(S3, $strRegionKey);\n" - "$strSigningKey = hmac_sha256(AWS4_REQUEST, $strServiceKey);\n" - "\n\n" - "$hSigningKeyCache->{$strDate}{$strRegion}{$strSecretAccessKey} = $strSigningKey;\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'strSigningKey', value => $strSigningKey, redact => true, trace => true}\n" - ");\n" - "}\n" - "\n" - "push @EXPORT, qw(s3SigningKey);\n" - "\n\n\n\n" - "sub s3StringToSign\n" - "{\n" - "\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strDateTime,\n" - "$strRegion,\n" - "$strCanonicalRequestHash,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '::s3StringToSign', \\@_,\n" - "{name => 'strDateTime', trace => true},\n" - "{name => 'strRegion', trace => true},\n" - "{name => 'strCanonicalRequestHash', trace => true},\n" + "__PACKAGE__ . '->pathRemove', \\@_,\n" + "{name => 'strPathExp'},\n" + "{name => 'bIgnoreMissing', optional => true, default => true},\n" + "{name => 'bRecurse', optional => true, default => false},\n" ");\n" "\n" - "my $strStringToSign =\n" - "AWS4_HMAC_SHA256 . \"\\n${strDateTime}\\n\" . substr($strDateTime, 0, 8) . \"/${strRegion}/\" . S3 . '/' . AWS4_REQUEST . \"\\n\" .\n" - "$strCanonicalRequestHash;\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'strStringToSign', value => $strStringToSign, trace => true}\n" - ");\n" - "}\n" - "\n" - "push @EXPORT, qw(s3StringToSign);\n" - "\n\n\n\n" - "sub s3AuthorizationHeader\n" - "{\n" - "\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strRegion,\n" - "$strHost,\n" - "$strVerb,\n" - "$strUri,\n" - "$strQuery,\n" - "$strDateTime,\n" - "$hHeader,\n" - "$strAccessKeyId,\n" - "$strSecretAccessKey,\n" - "$strSecurityToken,\n" - "$strPayloadHash,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '::s3AuthorizationHeader', \\@_,\n" - "{name => 'strRegion', trace => true},\n" - "{name => 'strHost', trace => true},\n" - "{name => 'strVerb', trace => true},\n" - "{name => 'strUri', trace => true},\n" - "{name => 'strQuery', trace => true},\n" - "{name => 'strDateTime', trace => true},\n" - "{name => 'hHeader', required => false, trace => true},\n" - "{name => 'strAccessKeyId', redact => true, trace => true},\n" - "{name => 'strSecretAccessKey', redact => true, trace => true},\n" - "{name => 'strSecurityToken', required => false, redact => true, trace => true},\n" - "{name => 'strPayloadHash', trace => true},\n" - ");\n" - "\n\n" - "delete($hHeader->{&S3_HEADER_AUTHORIZATION});\n" - "\n\n" - "$hHeader->{&S3_HEADER_HOST} = $strHost;\n" - "$hHeader->{&S3_HEADER_CONTENT_SHA256} = $strPayloadHash;\n" - "$hHeader->{&S3_HEADER_DATE} = $strDateTime;\n" - "\n\n" - "if (defined($strSecurityToken))\n" - "{\n" - "$hHeader->{&S3_HEADER_TOKEN} = $strSecurityToken;\n" - "}\n" - "\n\n" - "my ($strCanonicalRequest, $strSignedHeaders) = s3CanonicalRequest(\n" - "$strVerb, httpUriEncode($strUri, true), $strQuery, $hHeader, $strPayloadHash);\n" - "my $strStringToSign = s3StringToSign($strDateTime, $strRegion, cryptoHashOne('sha256', $strCanonicalRequest));\n" - "\n" - "$hHeader->{&S3_HEADER_AUTHORIZATION} =\n" - "AWS4_HMAC_SHA256 . \" Credential=${strAccessKeyId}/\" . substr($strDateTime, 0, 8) . \"/${strRegion}/\" . S3 . qw(/) .\n" - "AWS4_REQUEST . \",SignedHeaders=${strSignedHeaders},Signature=\" . hmac_sha256_hex($strStringToSign,\n" - "s3SigningKey(substr($strDateTime, 0, 8), $strRegion, $strSecretAccessKey));\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'hHeader', value => $hHeader, trace => true},\n" - "{name => 'strCanonicalRequest', value => $strCanonicalRequest, trace => true},\n" - "{name => 'strSignedHeaders', value => $strSignedHeaders, trace => true},\n" - "{name => 'strStringToSign', value => $strStringToSign, trace => true},\n" - ");\n" - "}\n" - "\n" - "push @EXPORT, qw(s3AuthorizationHeader);\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Storage/S3/Driver.pm", - .data = - "\n\n\n" - "package pgBackRest::Storage::S3::Driver;\n" - "use parent 'pgBackRest::Storage::S3::Request';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "use Digest::MD5 qw(md5_base64);\n" - "use File::Basename qw(basename dirname);\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::Common::String;\n" - "use pgBackRest::Common::Xml;\n" - "use pgBackRest::Storage::S3::FileRead;\n" - "use pgBackRest::Storage::S3::FileWrite;\n" - "use pgBackRest::Storage::S3::Request;\n" - "use pgBackRest::Storage::S3::Info;\n" - "\n\n\n\n" - "use constant STORAGE_S3_DRIVER => __PACKAGE__;\n" - "push @EXPORT, qw(STORAGE_S3_DRIVER);\n" - "\n\n\n\n" - "use constant S3_QUERY_CONTINUATION_TOKEN => 'continuation-token';\n" - "use constant S3_QUERY_DELIMITER => 'delimiter';\n" - "use constant S3_QUERY_LIST_TYPE => 'list-type';\n" - "use constant S3_QUERY_PREFIX => 'prefix';\n" - "\n\n\n\n" - "use constant S3_BATCH_MAX => 1000;\n" - "\n\n\n\n" - "sub openWrite\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strFile,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->openWrite', \\@_,\n" - "{name => 'strFile', trace => true},\n" - ");\n" - "\n" - "my $oFileIO = new pgBackRest::Storage::S3::FileWrite($self, $strFile);\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'oFileIO', value => $oFileIO, trace => true},\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub openRead\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strFile,\n" - "$bIgnoreMissing,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->openRead', \\@_,\n" - "{name => 'strFile', trace => true},\n" - "{name => 'bIgnoreMissing', optional => true, default => false, trace => true},\n" - ");\n" - "\n" - "my $oFileIO = new pgBackRest::Storage::S3::FileRead($self, $strFile, {bIgnoreMissing => $bIgnoreMissing});\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'oFileIO', value => $oFileIO, trace => true},\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub manifest\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPath,\n" - "$bRecurse,\n" - "$bPath,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->manifest', \\@_,\n" - "{name => 'strPath', trace => true},\n" - "\n" - "{name => 'bRecurse', optional => true, default => true, trace => true},\n" - "{name => 'bPath', optional => true, default => true, trace => true},\n" - ");\n" - "\n\n" - "my $strPrefix = $strPath eq qw{/} ? undef : substr($strPath, 1) . ($bPath ? qw{/} : '');\n" - "\n\n" - "my $strDelimiter = $bRecurse ? undef : '/';\n" - "\n\n" - "my $hManifest = {};\n" - "\n\n" - "my $strContinuationToken;\n" - "\n" - "do\n" - "{\n" - "\n" - "my $oResponse = $self->request(\n" - "HTTP_VERB_GET, {hQuery =>\n" - "{&S3_QUERY_LIST_TYPE => 2, &S3_QUERY_PREFIX => $strPrefix, &S3_QUERY_DELIMITER => $strDelimiter,\n" - "&S3_QUERY_CONTINUATION_TOKEN => $strContinuationToken}, strResponseType => S3_RESPONSE_TYPE_XML});\n" - "\n\n" - "if (defined($strPrefix) && !$bPath)\n" - "{\n" - "\n" - "if (index($strPrefix, qw{/}) == -1)\n" - "{\n" - "undef($strPrefix);\n" - "}\n" - "else\n" - "{\n" - "$strPrefix = dirname($strPrefix) . qw{/};\n" - "}\n" - "}\n" - "\n\n" - "foreach my $oFile (xmlTagChildren($oResponse, \"Contents\"))\n" - "{\n" - "my $strName = xmlTagText($oFile, \"Key\");\n" - "\n\n" - "if (defined($strPrefix))\n" - "{\n" - "$strName = substr($strName, length($strPrefix));\n" - "}\n" - "\n" - "$hManifest->{$strName}->{type} = 'f';\n" - "$hManifest->{$strName}->{size} = xmlTagText($oFile, 'Size') + 0;\n" - "\n\n" - "if ($bRecurse)\n" - "{\n" - "my @stryName = split(qw{/}, $strName);\n" - "\n" - "if (@stryName > 1)\n" - "{\n" - "$strName = undef;\n" - "\n" - "for (my $iIndex = 0; $iIndex < @stryName - 1; $iIndex++)\n" - "{\n" - "$strName .= (defined($strName) ? qw{/} : '') . $stryName[$iIndex];\n" - "$hManifest->{$strName}->{type} = 'd';\n" - "}\n" - "}\n" - "}\n" - "}\n" - "\n\n" - "if ($bPath && !$bRecurse)\n" - "{\n" - "foreach my $oPath (xmlTagChildren($oResponse, \"CommonPrefixes\"))\n" - "{\n" - "my $strName = xmlTagText($oPath, \"Prefix\");\n" - "\n\n" - "if (defined($strPrefix))\n" - "{\n" - "$strName = substr($strName, length($strPrefix));\n" - "}\n" - "\n\n" - "$strName = substr($strName, 0, length($strName) - 1);\n" - "\n" - "$hManifest->{$strName}->{type} = 'd';\n" - "}\n" - "}\n" - "\n" - "$strContinuationToken = xmlTagText($oResponse, \"NextContinuationToken\", false);\n" - "}\n" - "while (defined($strContinuationToken));\n" - "\n\n" - "if ($bPath)\n" - "{\n" - "$hManifest->{qw{.}}->{type} = 'd';\n" - "}\n" - "\n\n\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'hManifest', value => $hManifest, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub list\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPath,\n" - "$strExpression\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->list', \\@_,\n" - "{name => 'strPath', trace => true},\n" - "{name => 'strExpression', optional => true, trace => true},\n" - ");\n" - "\n\n" - "my $strPrefix = regexPrefix($strExpression);\n" - "\n\n" - "my @stryFileList = grep(\n" - "!/^\\.$/i, keys(%{$self->manifest(\n" - "$strPath . (defined($strPrefix) ? \"/${strPrefix}\" : ''), {bRecurse => false, bPath => !defined($strPrefix)})}));\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'stryFileList', value => \\@stryFileList, ref => true, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub pathCreate\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPath,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->pathCreate', \\@_,\n" - "{name => 'strPath', trace => true},\n" - ");\n" + "$self->{oStorageC}->pathRemove($strPathExp, $bIgnoreMissing, $bRecurse);\n" "\n\n" "return logDebugReturn($strOperation);\n" "}\n" @@ -18620,98 +15607,46 @@ static const EmbeddedModule embeddedModule[] = "my\n" "(\n" "$strOperation,\n" - "$strPath,\n" + "$strPathExp,\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->pathSync', \\@_,\n" - "{name => 'strPath', trace => true},\n" + "{name => 'strPathExp'},\n" ");\n" + "\n" + "$self->{oStorageC}->pathSync($strPathExp);\n" "\n\n" "return logDebugReturn($strOperation);\n" "}\n" "\n\n\n\n" - "sub exists\n" + "sub put\n" "{\n" "my $self = shift;\n" "\n\n" "my\n" "(\n" "$strOperation,\n" - "$strFile,\n" + "$xFile,\n" + "$xContent,\n" + "$strCipherPass,\n" ") =\n" "logDebugParam\n" "(\n" - "__PACKAGE__ . '->exists', \\@_,\n" - "{name => 'strFile', trace => true},\n" + "__PACKAGE__ . '->put', \\@_,\n" + "{name => 'xFile', trace => true},\n" + "{name => 'xContent', required => false, trace => true},\n" + "{name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), trace => true, redact => true},\n" ");\n" + "\n\n\n" + "my $oFileIo = ref($xFile) ? $xFile : $self->openWrite($xFile, {strCipherPass => $strCipherPass});\n" "\n\n" - "my $bExists = defined($self->manifest($strFile, {bRecurse => false, bPath => false})->{basename($strFile)}) ? true : false;\n" + "my $lSize = $self->{oStorageC}->put($oFileIo->{oStorageCWrite}, ref($xContent) ? $$xContent : $xContent);\n" "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'bExists', value => $bExists, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub pathExists\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strPath,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->pathExists', \\@_,\n" - "{name => 'strPath', trace => true},\n" - ");\n" - "\n" - "my $bExists = true;\n" - "\n\n" - "if ($strPath ne qw{/})\n" - "{\n" - "\n" - "my $rhInfo = $self->manifest(dirname($strPath), {bRecurse => false, bPath => true})->{basename($strPath)};\n" - "$bExists = defined($rhInfo) && $rhInfo->{type} eq 'd' ? true : false;\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'bExists', value => $bExists, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n" - "sub info\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$strFile,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->info', \\@_,\n" - "{name => 'strFile', trace => true},\n" - ");\n" - "\n\n" - "my $rhFile = $self->manifest($strFile, {bRecurse => false, bPath => false})->{basename($strFile)};\n" - "\n" - "if (!defined($rhFile))\n" - "{\n" - "confess &log(ERROR, \"unable to get info for missing file ${strFile}\", ERROR_FILE_MISSING);\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'oInfo', value => new pgBackRest::Storage::S3::Info($rhFile->{size}), trace => true}\n" + "{name => 'lSize', value => $lSize, trace => true},\n" ");\n" "}\n" "\n\n\n\n" @@ -18722,312 +15657,123 @@ static const EmbeddedModule embeddedModule[] = "my\n" "(\n" "$strOperation,\n" - "$rstryFile,\n" - "$bRecurse,\n" - ") =\n" - "logDebugParam\n" - "(\n" - "__PACKAGE__ . '->remove', \\@_,\n" - "{name => 'rstryFile', trace => true},\n" - "{name => 'bRecurse', optional => true, default => false, trace => true},\n" - ");\n" - "\n\n" - "if ($bRecurse)\n" - "{\n" - "my $rhManifest = $self->manifest($rstryFile);\n" - "my @stryRemoveFile;\n" - "\n\n" - "foreach my $strFile (sort({$b cmp $a} keys(%{$rhManifest})))\n" - "{\n" - "next if $rhManifest->{$strFile}->{type} eq 'd';\n" - "push(@stryRemoveFile, \"${rstryFile}/${strFile}\");\n" - "}\n" - "\n\n" - "if (@stryRemoveFile > 0)\n" - "{\n" - "$self->remove(\\@stryRemoveFile);\n" - "}\n" - "}\n" - "\n" - "else\n" - "{\n" - "\n" - "my $rstryFileAll = ref($rstryFile) ? $rstryFile : [$rstryFile];\n" - "\n" - "do\n" - "{\n" - "my $strFile = shift(@{$rstryFileAll});\n" - "my $iTotal = 0;\n" - "my $strXml = XML_HEADER . 'true';\n" - "\n" - "while (defined($strFile))\n" - "{\n" - "$iTotal++;\n" - "$strXml .= '' . xmlFromText(substr($strFile, 1)) . '';\n" - "\n" - "$strFile = $iTotal < S3_BATCH_MAX ? shift(@{$rstryFileAll}) : undef;\n" - "}\n" - "\n" - "$strXml .= '';\n" - "\n" - "my $hHeader = {'content-md5' => md5_base64($strXml) . '=='};\n" - "\n\n" - "my $oResponse = $self->request(\n" - "HTTP_VERB_POST,\n" - "{hQuery => 'delete=', rstrBody => \\$strXml, hHeader => $hHeader, strResponseType => S3_RESPONSE_TYPE_XML});\n" - "}\n" - "while (@{$rstryFileAll} > 0);\n" - "}\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'bResult', value => true, trace => true}\n" - ");\n" - "}\n" - "\n\n\n\n\n" - "sub capability {false}\n" - "sub className {STORAGE_S3_DRIVER}\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Storage/S3/FileRead.pm", - .data = - "\n\n\n" - "package pgBackRest::Storage::S3::FileRead;\n" - "use parent 'pgBackRest::Common::Http::Client';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Digest::MD5 qw(md5_base64);\n" - "use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);\n" - "use File::Basename qw(dirname);\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::Common::Xml;\n" - "use pgBackRest::Storage::Base;\n" - "use pgBackRest::Storage::S3::Request;\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my\n" - "(\n" - "$strOperation,\n" - "$oDriver,\n" - "$strName,\n" + "$xFileExp,\n" "$bIgnoreMissing,\n" ") =\n" "logDebugParam\n" "(\n" - "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'oDriver', trace => true},\n" - "{name => 'strName', trace => true},\n" - "{name => 'bIgnoreMissing', optional => true, default => false, trace => true},\n" + "__PACKAGE__ . '->remove', \\@_,\n" + "{name => 'xFileExp'},\n" + "{name => 'bIgnoreMissing', optional => true, default => true},\n" ");\n" - "\n\n" - "my $self = $oDriver->request(\n" - "HTTP_VERB_GET, {strUri => $strName, strResponseType => S3_RESPONSE_TYPE_IO, bIgnoreMissing => $bIgnoreMissing});\n" - "\n\n" - "if (defined($self))\n" + "\n" + "foreach my $strFileExp (ref($xFileExp) ? @{$xFileExp} : ($xFileExp))\n" "{\n" - "bless $self, $class;\n" + "$self->{oStorageC}->remove($strFileExp, $bIgnoreMissing);\n" "}\n" "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self, trace => true}\n" - ");\n" + "return logDebugReturn($strOperation);\n" "}\n" "\n\n\n\n" - "sub name {shift->{strName}}\n" - "\n" - "1;\n" - }, - { - .name = "pgBackRest/Storage/S3/FileWrite.pm", - .data = - "\n\n\n" - "package pgBackRest::Storage::S3::FileWrite;\n" - "use parent 'pgBackRest::Common::Io::Base';\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "use English '-no_match_vars';\n" - "\n" - "use Digest::MD5 qw(md5_base64);\n" - "use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);\n" - "use File::Basename qw(dirname);\n" - "\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Io::Handle;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::Common::Xml;\n" - "use pgBackRest::Storage::Base;\n" - "use pgBackRest::Storage::S3::Request;\n" - "\n\n\n\n" - "use constant S3_BUFFER_MAX => 16777216;\n" - "\n\n\n\n" - "sub new\n" + "sub encrypted\n" "{\n" - "my $class = shift;\n" + "my $self = shift;\n" "\n\n" "my\n" "(\n" "$strOperation,\n" - "$oDriver,\n" - "$strName,\n" + "$strFileExp,\n" + "$bIgnoreMissing,\n" ") =\n" "logDebugParam\n" "(\n" - "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'oDriver', trace => true},\n" - "{name => 'strName', trace => true},\n" + "__PACKAGE__ . '->encrypted', \\@_,\n" + "{name => 'strFileExp'},\n" + "{name => 'bIgnoreMissing', optional => true, default => false},\n" ");\n" + "\n" + "my $bEncrypted = false;\n" "\n\n" - "my $self = $class->SUPER::new(\"'${strName}'\");\n" - "bless $self, $class;\n" - "\n\n" - "$self->{oDriver} = $oDriver;\n" - "$self->{strName} = $strName;\n" - "\n\n" - "$self->{rtBuffer} = '';\n" - "\n\n" - "$self->{bWritten} = false;\n" - "\n\n" - "$self->{lSize} = 0;\n" + "my $oFileIo = new pgBackRest::Storage::StorageRead(\n" + "$self, pgBackRest::LibC::StorageRead->new($self->{oStorageC}, $strFileExp, $bIgnoreMissing));\n" + "\n\n\n" + "if (!$oFileIo->open())\n" + "{\n" + "if (defined($self->cipherType()))\n" + "{\n" + "$bEncrypted = true;\n" + "}\n" + "}\n" + "else\n" + "{\n" + "\n" + "my $tMagicSignature = '';\n" + "my $lSizeRead = $oFileIo->read(\\$tMagicSignature, length(CIPHER_MAGIC));\n" + "$oFileIo->close();\n" + "\n" + "if (substr($tMagicSignature, 0, length(CIPHER_MAGIC)) eq CIPHER_MAGIC)\n" + "{\n" + "$bEncrypted = true;\n" + "}\n" + "}\n" "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'self', value => $self, trace => true}\n" + "{name => 'bEncrypted', value => $bEncrypted}\n" ");\n" "}\n" "\n\n\n\n" - "sub open\n" + "sub encryptionValid\n" "{\n" "my $self = shift;\n" "\n\n" - "my $oResponse = $self->{oDriver}->request(\n" - "HTTP_VERB_POST, {strUri => $self->{strName}, hQuery => 'uploads=', strResponseType => S3_RESPONSE_TYPE_XML});\n" + "my\n" + "(\n" + "$strOperation,\n" + "$bEncrypted,\n" + ") =\n" + "logDebugParam\n" + "(\n" + "__PACKAGE__ . '->encryptionValid', \\@_,\n" + "{name => 'bEncrypted'},\n" + ");\n" "\n" - "$self->{strUploadId} = xmlTagText($oResponse, 'UploadId');\n" + "my $bValid = ($bEncrypted && defined($self->cipherType())) || (!$bEncrypted && !defined($self->cipherType()));\n" "\n\n" - "$self->{rstryMultiPart} = [];\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'bValid', value => $bValid ? true : false}\n" + ");\n" "}\n" "\n\n\n\n" - "sub write\n" - "{\n" - "my $self = shift;\n" - "my $rtBuffer = shift;\n" - "\n\n" - "$self->{bWritten} = true;\n" - "\n" - "if (defined($rtBuffer))\n" - "{\n" - "$self->{rtBuffer} .= $$rtBuffer;\n" - "$self->{lSize} += length($$rtBuffer);\n" - "\n\n" - "if (length($self->{rtBuffer}) >= S3_BUFFER_MAX)\n" - "{\n" - "$self->flush();\n" - "}\n" - "\n" - "return length($$rtBuffer);\n" - "}\n" - "\n" - "return 0;\n" - "}\n" - "\n\n\n\n" - "sub flush\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "$self->open() if !$self->opened();\n" - "\n\n" - "$self->{oDriver}->request(\n" - "HTTP_VERB_PUT,\n" - "{strUri => $self->{strName},\n" - "hQuery => {'partNumber' => @{$self->{rstryMultiPart}} + 1, 'uploadId' => $self->{strUploadId}},\n" - "rstrBody => \\$self->{rtBuffer}, hHeader => {'content-md5' => md5_base64($self->{rtBuffer}) . '=='}});\n" - "\n\n" - "push(@{$self->{rstryMultiPart}}, $self->{oDriver}->{hResponseHeader}{&S3_HEADER_ETAG});\n" - "\n\n" - "$self->{rtBuffer} = '';\n" - "}\n" - "\n\n\n\n" - "sub close\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "if ($self->{bWritten})\n" - "{\n" - "\n" - "$self->{bWritten} = false;\n" - "\n\n" - "if ($self->opened())\n" - "{\n" - "\n" - "$self->flush();\n" - "\n" - "my $strXml = XML_HEADER . '';\n" - "my $iPartNo = 0;\n" - "\n" - "foreach my $strETag (@{$self->{rstryMultiPart}})\n" - "{\n" - "$iPartNo++;\n" - "\n" - "$strXml .= \"${iPartNo}${strETag}\";\n" - "}\n" - "\n" - "$strXml .= '';\n" - "\n\n" - "my $oResponse = $self->{oDriver}->request(\n" - "HTTP_VERB_POST,\n" - "{strUri => $self->{strName}, hQuery => {'uploadId' => $self->{strUploadId}},\n" - "rstrBody => \\$strXml, hHeader => {'content-md5' => md5_base64($strXml) . '=='},\n" - "strResponseType => S3_RESPONSE_TYPE_XML});\n" - "}\n" - "\n" - "else\n" - "{\n" - "$self->{oDriver}->request(\n" - "HTTP_VERB_PUT,\n" - "{strUri => $self->{strName}, rstrBody => \\$self->{rtBuffer},\n" - "hHeader => {'content-md5' => md5_base64($self->{rtBuffer}) . '=='}});\n" - "}\n" - "}\n" - "\n\n\n\n" - "$self->resultSet(COMMON_IO_HANDLE, $self->{lSize});\n" - "\n" - "return true;\n" - "}\n" - "\n\n\n\n" - "sub opened {defined(shift->{strUploadId})}\n" - "sub name {shift->{strName}}\n" + "sub capability {shift->type() eq STORAGE_POSIX}\n" + "sub type {shift->{oStorageC}->type()}\n" + "sub cipherType {shift->{strCipherType}}\n" + "sub cipherPassUser {shift->{strCipherPass}}\n" "\n" "1;\n" }, { - .name = "pgBackRest/Storage/S3/Info.pm", + .name = "pgBackRest/Storage/StorageRead.pm", .data = "\n\n\n" - "package pgBackRest::Storage::S3::Info;\n" + "package pgBackRest::Storage::StorageRead;\n" "\n" "use strict;\n" "use warnings FATAL => qw(all);\n" "use Carp qw(confess);\n" "use English '-no_match_vars';\n" "\n" + "use File::Basename qw(dirname);\n" + "use Fcntl qw(:mode);\n" + "use File::stat qw{lstat};\n" + "use JSON::PP;\n" + "\n" + "use pgBackRest::Common::Exception;\n" "use pgBackRest::Common::Log;\n" + "use pgBackRest::Storage::Base;\n" "\n\n\n\n" "sub new\n" "{\n" @@ -19038,12 +15784,14 @@ static const EmbeddedModule embeddedModule[] = "\n\n" "(\n" "my $strOperation,\n" - "$self->{lSize},\n" + "$self->{oStorage},\n" + "$self->{oStorageCRead},\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'lSize'},\n" + "{name => 'oStorage'},\n" + "{name => 'oStorageCRead'},\n" ");\n" "\n\n" "return logDebugReturn\n" @@ -19053,62 +15801,139 @@ static const EmbeddedModule embeddedModule[] = ");\n" "}\n" "\n\n\n\n" - "sub size {shift->{lSize}}\n" + "sub open\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my ($strOperation) = logDebugParam(__PACKAGE__ . '->open');\n" + "\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'bResult', value => $self->{oStorageCRead}->open() ? true : false, trace => true},\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub read\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my (\n" + "$strOperation,\n" + "$rtBuffer,\n" + "$iSize,\n" + ") =\n" + "logDebugParam\n" + "(\n" + "__PACKAGE__ . '->read', \\@_,\n" + "{name => 'rtBuffer'},\n" + "{name => 'iSize'},\n" + ");\n" + "\n\n" + "my $iActualSize = 0;\n" + "\n" + "if (!$self->eof())\n" + "{\n" + "my $tBuffer = $self->{oStorageCRead}->read($iSize);\n" + "$iActualSize = length($tBuffer);\n" + "$$rtBuffer .= $tBuffer;\n" + "}\n" + "\n\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'iActualSize', value => $iActualSize}\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub eof\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my ($strOperation) = logDebugParam(__PACKAGE__ . '->eof');\n" + "\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'bResult', value => $self->{oStorageCRead}->eof() ? true : false, trace => true},\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub close\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my ($strOperation) = logDebugParam(__PACKAGE__ . '->close');\n" + "\n" + "$self->{oStorageCRead}->close();\n" + "\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'bResult', value => true, trace => true},\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub result\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my\n" + "(\n" + "$strOperation,\n" + "$strClass,\n" + ") =\n" + "logDebugParam\n" + "(\n" + "__PACKAGE__ . '->result', \\@_,\n" + "{name => 'strClass'},\n" + ");\n" + "\n" + "my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCRead}->result($strClass));\n" + "\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'xResult', value => $xResult, trace => true},\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub resultAll\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my ($strOperation) = logDebugParam(__PACKAGE__ . '->resultAll');\n" + "\n" + "my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCRead}->resultAll());\n" + "\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'xResultAll', value => $xResult, trace => true},\n" + ");\n" + "}\n" "\n" "1;\n" }, { - .name = "pgBackRest/Storage/S3/Request.pm", + .name = "pgBackRest/Storage/StorageWrite.pm", .data = "\n\n\n" - "package pgBackRest::Storage::S3::Request;\n" + "package pgBackRest::Storage::StorageWrite;\n" "\n" "use strict;\n" "use warnings FATAL => qw(all);\n" "use Carp qw(confess);\n" "use English '-no_match_vars';\n" "\n" - "use Exporter qw(import);\n" - "our @EXPORT = qw();\n" - "use IO::Socket::SSL;\n" + "use File::Basename qw(dirname);\n" + "use Fcntl qw(:mode);\n" + "use File::stat qw{lstat};\n" + "use JSON::PP;\n" "\n" "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Http::Client;\n" - "use pgBackRest::Common::Http::Common;\n" - "use pgBackRest::Common::Io::Base;\n" "use pgBackRest::Common::Log;\n" - "use pgBackRest::Common::String;\n" - "use pgBackRest::Common::Xml;\n" - "use pgBackRest::LibC qw(:crypto);\n" - "use pgBackRest::Storage::S3::Auth;\n" - "\n\n\n\n" - "use constant HTTP_VERB_GET => 'GET';\n" - "push @EXPORT, qw(HTTP_VERB_GET);\n" - "use constant HTTP_VERB_POST => 'POST';\n" - "push @EXPORT, qw(HTTP_VERB_POST);\n" - "use constant HTTP_VERB_PUT => 'PUT';\n" - "push @EXPORT, qw(HTTP_VERB_PUT);\n" - "\n" - "use constant S3_HEADER_CONTENT_LENGTH => 'content-length';\n" - "push @EXPORT, qw(S3_HEADER_CONTENT_LENGTH);\n" - "use constant S3_HEADER_TRANSFER_ENCODING => 'transfer-encoding';\n" - "push @EXPORT, qw(S3_HEADER_TRANSFER_ENCODING);\n" - "use constant S3_HEADER_ETAG => 'etag';\n" - "push @EXPORT, qw(S3_HEADER_ETAG);\n" - "\n" - "use constant S3_RESPONSE_TYPE_IO => 'io';\n" - "push @EXPORT, qw(S3_RESPONSE_TYPE_IO);\n" - "use constant S3_RESPONSE_TYPE_NONE => 'none';\n" - "push @EXPORT, qw(S3_RESPONSE_TYPE_NONE);\n" - "use constant S3_RESPONSE_TYPE_XML => 'xml';\n" - "push @EXPORT, qw(S3_RESPONSE_TYPE_XML);\n" - "\n" - "use constant S3_RESPONSE_CODE_SUCCESS => 200;\n" - "use constant S3_RESPONSE_CODE_ERROR_AUTH => 403;\n" - "use constant S3_RESPONSE_CODE_ERROR_NOT_FOUND => 404;\n" - "use constant S3_RESPONSE_CODE_ERROR_RETRY_CLASS => 5;\n" - "\n" - "use constant S3_RETRY_MAX => 4;\n" + "use pgBackRest::Storage::Base;\n" "\n\n\n\n" "sub new\n" "{\n" @@ -19119,180 +15944,110 @@ static const EmbeddedModule embeddedModule[] = "\n\n" "(\n" "my $strOperation,\n" - "$self->{strBucket},\n" - "$self->{strEndPoint},\n" - "$self->{strRegion},\n" - "$self->{strAccessKeyId},\n" - "$self->{strSecretAccessKey},\n" - "$self->{strSecurityToken},\n" - "$self->{strHost},\n" - "$self->{iPort},\n" - "$self->{bVerifySsl},\n" - "$self->{strCaPath},\n" - "$self->{strCaFile},\n" - "$self->{lBufferMax},\n" + "$self->{oStorage},\n" + "$self->{oStorageCWrite},\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->new', \\@_,\n" - "{name => 'strBucket'},\n" - "{name => 'strEndPoint'},\n" - "{name => 'strRegion'},\n" - "{name => 'strAccessKeyId', redact => true},\n" - "{name => 'strSecretAccessKey', redact => true},\n" - "{name => 'strSecurityToken', optional => true, redact => true},\n" - "{name => 'strHost', optional => true},\n" - "{name => 'iPort', optional => true},\n" - "{name => 'bVerifySsl', optional => true, default => true},\n" - "{name => 'strCaPath', optional => true},\n" - "{name => 'strCaFile', optional => true},\n" - "{name => 'lBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX},\n" + "{name => 'oStorage'},\n" + "{name => 'oStorageCWrite'},\n" ");\n" "\n\n" - "$self->{strHost} = defined($self->{strHost}) ? $self->{strHost} : \"$self->{strBucket}.$self->{strEndPoint}\";\n" - "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'self', value => $self, trace => true}\n" + "{name => 'self', value => $self}\n" ");\n" "}\n" "\n\n\n\n" - "sub request\n" + "sub open\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my ($strOperation) = logDebugParam(__PACKAGE__ . '->open');\n" + "\n" + "$self->{oStorageCWrite}->open();\n" + "\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'bResult', value => true, trace => true},\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub write\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my (\n" + "$strOperation,\n" + "$rtBuffer,\n" + ") =\n" + "logDebugParam\n" + "(\n" + "__PACKAGE__ . '->write', \\@_,\n" + "{name => 'rtBuffer'},\n" + ");\n" + "\n\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'iActualSize', value => $self->{oStorageCWrite}->write($$rtBuffer)}\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub close\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my ($strOperation) = logDebugParam(__PACKAGE__ . '->close');\n" + "\n" + "$self->{oStorageCWrite}->close();\n" + "\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'bResult', value => true, trace => true},\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub result\n" "{\n" "my $self = shift;\n" "\n\n" "my\n" "(\n" "$strOperation,\n" - "$strVerb,\n" - "$strUri,\n" - "$hQuery,\n" - "$hHeader,\n" - "$rstrBody,\n" - "$strResponseType,\n" - "$bIgnoreMissing,\n" + "$strClass,\n" ") =\n" "logDebugParam\n" "(\n" - "__PACKAGE__ . '->request', \\@_,\n" - "{name => 'strVerb', trace => true},\n" - "{name => 'strUri', optional => true, default => '/', trace => true},\n" - "{name => 'hQuery', optional => true, trace => true},\n" - "{name => 'hHeader', optional => true, trace => true},\n" - "{name => 'rstrBody', optional => true, trace => true},\n" - "{name => 'strResponseType', optional => true, default => S3_RESPONSE_TYPE_NONE, trace => true},\n" - "{name => 'bIgnoreMissing', optional => true, default => false, trace => true},\n" + "__PACKAGE__ . '->result', \\@_,\n" + "{name => 'strClass'},\n" ");\n" - "\n\n" - "my $oResponse;\n" - "\n\n" - "my $bRetry;\n" - "my $iRetryTotal = 0;\n" "\n" - "do\n" - "{\n" + "my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCWrite}->result($strClass));\n" "\n" - "$bRetry = false;\n" - "\n\n" - "$hHeader->{&S3_HEADER_CONTENT_SHA256} = defined($rstrBody) ? cryptoHashOne('sha256', $$rstrBody) : PAYLOAD_DEFAULT_HASH;\n" - "$hHeader->{&S3_HEADER_CONTENT_LENGTH} = defined($rstrBody) ? length($$rstrBody) : 0;\n" - "\n\n" - "($hHeader, my $strCanonicalRequest, my $strSignedHeaders, my $strStringToSign) = s3AuthorizationHeader(\n" - "$self->{strRegion}, \"$self->{strBucket}.$self->{strEndPoint}\", $strVerb, $strUri, httpQuery($hQuery), s3DateTime(),\n" - "$hHeader, $self->{strAccessKeyId}, $self->{strSecretAccessKey}, $self->{strSecurityToken},\n" - "$hHeader->{&S3_HEADER_CONTENT_SHA256});\n" - "\n\n" - "my $oHttpClient = new pgBackRest::Common::Http::Client(\n" - "$self->{strHost}, $strVerb,\n" - "{iPort => $self->{iPort}, strUri => $strUri, hQuery => $hQuery, hRequestHeader => $hHeader,\n" - "rstrRequestBody => $rstrBody, bVerifySsl => $self->{bVerifySsl}, strCaPath => $self->{strCaPath},\n" - "strCaFile => $self->{strCaFile}, bResponseBodyPrefetch => $strResponseType eq S3_RESPONSE_TYPE_XML,\n" - "lBufferMax => $self->{lBufferMax}});\n" - "\n\n" - "my $iResponseCode = $oHttpClient->responseCode();\n" - "\n" - "if ($iResponseCode == S3_RESPONSE_CODE_SUCCESS)\n" - "{\n" - "\n" - "$self->{hResponseHeader} = $oHttpClient->responseHeader();\n" - "\n\n" - "if ($strResponseType eq S3_RESPONSE_TYPE_XML)\n" - "{\n" - "my $rtResponseBody = $oHttpClient->responseBody();\n" - "\n" - "if ($oHttpClient->contentLength() == 0 || !defined($$rtResponseBody))\n" - "{\n" - "confess &log(ERROR,\n" - "\"response type '${strResponseType}' was requested but content length is zero or content is missing\",\n" - "ERROR_PROTOCOL);\n" - "}\n" - "\n" - "$oResponse = xmlParse($$rtResponseBody);\n" - "}\n" - "\n" - "elsif ($strResponseType eq S3_RESPONSE_TYPE_IO)\n" - "{\n" - "$oResponse = $oHttpClient;\n" - "}\n" - "}\n" - "else\n" - "{\n" - "\n" - "if ($iResponseCode == S3_RESPONSE_CODE_ERROR_NOT_FOUND)\n" - "{\n" - "\n" - "if (!$bIgnoreMissing)\n" - "{\n" - "confess &log(ERROR, \"unable to open '${strUri}': No such file or directory\", ERROR_FILE_MISSING);\n" - "}\n" - "\n" - "$bRetry = false;\n" - "}\n" - "\n" - "else\n" - "{\n" - "\n" - "if (int($iResponseCode / 100) == S3_RESPONSE_CODE_ERROR_RETRY_CLASS)\n" - "{\n" - "\n" - "$iRetryTotal++;\n" - "$bRetry = $iRetryTotal <= S3_RETRY_MAX;\n" - "\n\n" - "if ($iRetryTotal > 1)\n" - "{\n" - "sleep(5);\n" - "}\n" - "}\n" - "\n\n" - "if (!$bRetry)\n" - "{\n" - "my $rstrResponseBody = $oHttpClient->responseBody();\n" - "\n\n" - "my $strRequestHeader = $oHttpClient->requestHeaderText();\n" - "$strRequestHeader =~ s/^${\\S3_HEADER_AUTHORIZATION}:.*$/${\\S3_HEADER_AUTHORIZATION}: /mg;\n" - "\n" - "confess &log(ERROR,\n" - "'S3 request error' . ($iRetryTotal > 0 ? \" after \" . (S3_RETRY_MAX + 1) . \" tries\" : '') .\n" - "\" [$iResponseCode] \" . $oHttpClient->responseMessage() .\n" - "\"\\n*** request header ***\\n${strRequestHeader}\" .\n" - "($iResponseCode == S3_RESPONSE_CODE_ERROR_AUTH ?\n" - "\"\\n*** canonical request ***\\n\" . $strCanonicalRequest .\n" - "\"\\n*** signed headers ***\\n\" . $strSignedHeaders .\n" - "\"\\n*** string to sign ***\\n\" . $strStringToSign : '') .\n" - "\"\\n*** response header ***\\n\" . $oHttpClient->responseHeaderText() .\n" - "(defined($$rstrResponseBody) ? \"\\n*** response body ***\\n${$rstrResponseBody}\" : ''),\n" - "ERROR_PROTOCOL);\n" - "}\n" - "}\n" - "}\n" - "}\n" - "while ($bRetry);\n" - "\n\n" "return logDebugReturn\n" "(\n" "$strOperation,\n" - "{name => 'oResponse', value => $oResponse, trace => true, ref => true}\n" + "{name => 'xResult', value => $xResult, trace => true},\n" + ");\n" + "}\n" + "\n\n\n\n" + "sub resultAll\n" + "{\n" + "my $self = shift;\n" + "\n\n" + "my ($strOperation) = logDebugParam(__PACKAGE__ . '->resultAll');\n" + "\n" + "my $xResult = $self->{oStorage}->{oJSON}->decode($self->{oStorageCWrite}->resultAll());\n" + "\n" + "return logDebugReturn\n" + "(\n" + "$strOperation,\n" + "{name => 'xResultAll', value => $xResult, trace => true},\n" ");\n" "}\n" "\n" diff --git a/src/perl/libc.auto.c b/src/perl/libc.auto.c index 75e097a97..069d6bcf9 100644 --- a/src/perl/libc.auto.c +++ b/src/perl/libc.auto.c @@ -58,6 +58,7 @@ These includes are from the src directory. There is no Perl-specific code in th ***********************************************************************************************************************************/ #include "common/crypto/common.h" #include "common/error.h" +#include "common/io/io.h" #include "common/lock.h" #include "config/config.h" #include "config/define.h" @@ -77,9 +78,11 @@ XSH includes These includes define data structures that are required for the C to Perl interface but are not part of the regular C source. ***********************************************************************************************************************************/ -#include "xs/crypto/cipherBlock.xsh" #include "xs/crypto/hash.xsh" #include "xs/common/encode.xsh" +#include "xs/storage/storage.xsh" +#include "xs/storage/storageRead.xsh" +#include "xs/storage/storageWrite.xsh" /*********************************************************************************************************************************** Module definition @@ -259,10 +262,7 @@ XS_EUPXS(XS_pgBackRest__LibC_libcUvSize) /* INCLUDE: Including 'xs/config/define.xs' from 'xs/config/configTest.xs' */ -/* INCLUDE: Including 'xs/crypto/cipherBlock.xs' from 'xs/config/define.xs' */ - - -/* INCLUDE: Including 'xs/crypto/hash.xs' from 'xs/crypto/cipherBlock.xs' */ +/* INCLUDE: Including 'xs/crypto/hash.xs' from 'xs/config/define.xs' */ /* INCLUDE: Including 'xs/crypto/random.xs' from 'xs/crypto/hash.xs' */ @@ -274,24 +274,90 @@ XS_EUPXS(XS_pgBackRest__LibC_libcUvSize) /* INCLUDE: Including 'xs/storage/storage.xs' from 'xs/postgres/pageChecksum.xs' */ -XS_EUPXS(XS_pgBackRest__LibC_storagePosixPathRemove); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC_storagePosixPathRemove) +/* INCLUDE: Including 'xs/storage/storageRead.xs' from 'xs/storage/storage.xs' */ + + +/* INCLUDE: Including 'xs/storage/storageWrite.xs' from 'xs/storage/storageRead.xs' */ + + +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_new); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_new) +{ + dVAR; dXSARGS; + if (items != 9) + croak_xs_usage(cv, "class, storage, file, mode, user, group, timeModified, atomic, pathCreate"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + const String * class = STR_NEW_SV(ST(0)); + pgBackRest__LibC__Storage storage; + const String * file = STR_NEW_SV(ST(2)); + U32 mode = (unsigned long)SvUV(ST(3)) +; + const String * user = STR_NEW_SV(ST(4)); + const String * group = STR_NEW_SV(ST(5)); + IV timeModified = (IV)SvIV(ST(6)) +; + bool atomic = (bool)SvTRUE(ST(7)) +; + bool pathCreate = (bool)SvTRUE(ST(8)) +; + pgBackRest__LibC__StorageWrite RETVAL; + + if (SvROK(ST(1)) && sv_derived_from(ST(1), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(1))); + storage = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageWrite::new", + "storage", "pgBackRest::LibC::Storage") +; + CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::StorageWrite")); + + RETVAL = storageWriteMove( + storageNewWriteP( + storage, file, .modeFile = mode, .user = user, .group = group, .timeModified = (time_t)timeModified, + .noCreatePath = storageFeature(storage, storageFeaturePath) ? !pathCreate : false, .noSyncPath = !atomic, + .noAtomic = !atomic), + MEM_CONTEXT_XS_OLD()); + { + SV * RETVALSV; + RETVALSV = sv_newmortal(); + sv_setref_pv(RETVALSV, "pgBackRest::LibC::StorageWrite", (void*)RETVAL); + ST(0) = RETVALSV; + } + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_filterAdd); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_filterAdd) { dVAR; dXSARGS; if (items != 3) - croak_xs_usage(cv, "path, errorOnMissing, recurse"); + croak_xs_usage(cv, "self, filter, param"); { - const char * path = (const char *)SvPV_nolen(ST(0)) -; - bool errorOnMissing = (bool)SvTRUE(ST(1)) -; - bool recurse = (bool)SvTRUE(ST(2)) -; MEM_CONTEXT_XS_TEMP_BEGIN() { - storagePathRemoveP( - storagePosixNew(strNew("/"), 0640, 750, true, NULL), strNew(path), .errorOnMissing = errorOnMissing, - .recurse = recurse); + pgBackRest__LibC__StorageWrite self; + const String * filter = STR_NEW_SV(ST(1)); + const String * param = STR_NEW_SV(ST(2)); + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageWrite")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageWrite,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageWrite::filterAdd", + "self", "pgBackRest::LibC::StorageWrite") +; + IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(self)); + storageFilterXsAdd(filterGroup, filter, param); } MEM_CONTEXT_XS_TEMP_END(); } @@ -299,6 +365,1148 @@ XS_EUPXS(XS_pgBackRest__LibC_storagePosixPathRemove) } +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_open); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_open) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageWrite self; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageWrite")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageWrite,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageWrite::open", + "self", "pgBackRest::LibC::StorageWrite") +; + ioWriteOpen(storageWriteIo(self)); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_write); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_write) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, buffer"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageWrite self; + const Buffer * buffer = BUF_CONST_SV(ST(1)); + UV RETVAL; + dXSTARG; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageWrite")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageWrite,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageWrite::write", + "self", "pgBackRest::LibC::StorageWrite") +; + ioWrite(storageWriteIo(self), buffer); + RETVAL = bufUsed(buffer); + XSprePUSH; PUSHu((UV)RETVAL); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_close); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_close) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageWrite self; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageWrite")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageWrite,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageWrite::close", + "self", "pgBackRest::LibC::StorageWrite") +; + ioWriteClose(storageWriteIo(self)); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_result); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_result) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, filter"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageWrite self; + const String * filter = STR_NEW_SV(ST(1)); + const char * RETVAL; + dXSTARG; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageWrite")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageWrite,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageWrite::result", + "self", "pgBackRest::LibC::StorageWrite") +; + RETVAL = strPtr(storageFilterXsResult(ioWriteFilterGroup(storageWriteIo(self)), filter)); + sv_setpv(TARG, RETVAL); XSprePUSH; PUSHTARG; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_resultAll); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_resultAll) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageWrite self; + const char * RETVAL; + dXSTARG; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageWrite")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageWrite,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageWrite::resultAll", + "self", "pgBackRest::LibC::StorageWrite") +; + RETVAL = strPtr(storageFilterXsResultAll(ioWriteFilterGroup(storageWriteIo(self)))); + sv_setpv(TARG, RETVAL); XSprePUSH; PUSHTARG; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_DESTROY); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageWrite_DESTROY) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageWrite self; + + if (SvROK(ST(0))) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageWrite,tmp); + } + else + Perl_croak_nocontext("%s: %s is not a reference", + "pgBackRest::LibC::StorageWrite::DESTROY", + "self") +; + storageWriteFree(self); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +/* INCLUDE: Returning to 'xs/storage/storageRead.xs' from 'xs/storage/storageWrite.xs' */ + + +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_new); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_new) +{ + dVAR; dXSARGS; + if (items != 4) + croak_xs_usage(cv, "class, storage, file, ignoreMissing"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + const String * class = STR_NEW_SV(ST(0)); + pgBackRest__LibC__Storage storage; + const String * file = STR_NEW_SV(ST(2)); + bool ignoreMissing = (bool)SvTRUE(ST(3)) +; + pgBackRest__LibC__StorageRead RETVAL; + + if (SvROK(ST(1)) && sv_derived_from(ST(1), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(1))); + storage = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageRead::new", + "storage", "pgBackRest::LibC::Storage") +; + CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::StorageRead")); + + RETVAL = storageReadMove(storageNewReadP(storage, file, .ignoreMissing = ignoreMissing), MEM_CONTEXT_XS_OLD()); + { + SV * RETVALSV; + RETVALSV = sv_newmortal(); + sv_setref_pv(RETVALSV, "pgBackRest::LibC::StorageRead", (void*)RETVAL); + ST(0) = RETVALSV; + } + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_filterAdd); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_filterAdd) +{ + dVAR; dXSARGS; + if (items != 3) + croak_xs_usage(cv, "self, filter, param"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead self; + const String * filter = STR_NEW_SV(ST(1)); + const String * param = STR_NEW_SV(ST(2)); + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageRead")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageRead::filterAdd", + "self", "pgBackRest::LibC::StorageRead") +; + IoFilterGroup *filterGroup = ioReadFilterGroup(storageReadIo(self)); + storageFilterXsAdd(filterGroup, filter, param); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_open); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_open) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead self; + bool RETVAL; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageRead")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageRead::open", + "self", "pgBackRest::LibC::StorageRead") +; + RETVAL = ioReadOpen(storageReadIo(self)); + ST(0) = boolSV(RETVAL); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_read); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_read) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, bufferSize"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead self; + U32 bufferSize = (unsigned long)SvUV(ST(1)) +; + SV * RETVAL; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageRead")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageRead::read", + "self", "pgBackRest::LibC::StorageRead") +; + RETVAL = NEWSV(0, bufferSize); + SvPOK_only(RETVAL); + + Buffer *bufferRead = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL), bufferSize); + ioRead(storageReadIo(self), bufferRead); + + SvCUR_set(RETVAL, bufUsed(bufferRead)); + RETVAL = sv_2mortal(RETVAL); + ST(0) = RETVAL; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_eof); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_eof) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead self; + bool RETVAL; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageRead")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageRead::eof", + "self", "pgBackRest::LibC::StorageRead") +; + RETVAL = ioReadEof(storageReadIo(self)); + ST(0) = boolSV(RETVAL); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_close); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_close) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead self; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageRead")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageRead::close", + "self", "pgBackRest::LibC::StorageRead") +; + ioReadClose(storageReadIo(self)); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_result); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_result) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, filter"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead self; + const String * filter = STR_NEW_SV(ST(1)); + const char * RETVAL; + dXSTARG; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageRead")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageRead::result", + "self", "pgBackRest::LibC::StorageRead") +; + RETVAL = strPtr(storageFilterXsResult(ioReadFilterGroup(storageReadIo(self)), filter)); + sv_setpv(TARG, RETVAL); XSprePUSH; PUSHTARG; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_resultAll); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_resultAll) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead self; + const char * RETVAL; + dXSTARG; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::StorageRead")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::StorageRead::resultAll", + "self", "pgBackRest::LibC::StorageRead") +; + RETVAL = strPtr(storageFilterXsResultAll(ioReadFilterGroup(storageReadIo(self)))); + sv_setpv(TARG, RETVAL); XSprePUSH; PUSHTARG; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_DESTROY); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__StorageRead_DESTROY) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead self; + + if (SvROK(ST(0))) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not a reference", + "pgBackRest::LibC::StorageRead::DESTROY", + "self") +; + storageReadFree(self); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +/* INCLUDE: Returning to 'xs/storage/storage.xs' from 'xs/storage/storageRead.xs' */ + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_new); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_new) +{ + dVAR; dXSARGS; + if (items != 3) + croak_xs_usage(cv, "class, type, path"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + const String * class = STR_NEW_SV(ST(0)); + const String * type = STR_NEW_SV(ST(1)); + const String * path = STR_NEW_SV(ST(2)); + pgBackRest__LibC__Storage RETVAL; + CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::Storage")); + + if (strEqZ(type, "")) + { + memContextSwitch(MEM_CONTEXT_XS_OLD()); + RETVAL = storagePosixNew( + path == NULL ? STRDEF("/") : path, STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); + storagePathEnforceSet((Storage *)RETVAL, false); + memContextSwitch(MEM_CONTEXT_XS_TEMP()); + } + else if (strEqZ(type, "")) + { + CHECK(path == NULL); + RETVAL = (Storage *)storageRepoWrite(); + } + else if (strEqZ(type, "")) + { + CHECK(path == NULL); + + memContextSwitch(MEM_CONTEXT_XS_OLD()); + RETVAL = storagePosixNew(cfgOptionStr(cfgOptPgPath), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); + storagePathEnforceSet((Storage *)RETVAL, false); + memContextSwitch(MEM_CONTEXT_XS_TEMP()); + } + else + THROW_FMT(AssertError, "unexpected storage type '%s'", strPtr(type)); + { + SV * RETVALSV; + RETVALSV = sv_newmortal(); + sv_setref_pv(RETVALSV, "pgBackRest::LibC::Storage", (void*)RETVAL); + ST(0) = RETVALSV; + } + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_copy); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_copy) +{ + dVAR; dXSARGS; + if (items != 3) + croak_xs_usage(cv, "self, source, destination"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead source; + pgBackRest__LibC__StorageWrite destination; + bool RETVAL; + + if (SvROK(ST(1)) && sv_derived_from(ST(1), "pgBackRest::LibC::StorageRead")) { + IV tmp = SvIV((SV*)SvRV(ST(1))); + source = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::copy", + "source", "pgBackRest::LibC::StorageRead") +; + + if (SvROK(ST(2)) && sv_derived_from(ST(2), "pgBackRest::LibC::StorageWrite")) { + IV tmp = SvIV((SV*)SvRV(ST(2))); + destination = INT2PTR(pgBackRest__LibC__StorageWrite,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::copy", + "destination", "pgBackRest::LibC::StorageWrite") +; + RETVAL = storageCopyNP(source, destination); + ST(0) = boolSV(RETVAL); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_exists); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_exists) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, fileExp"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const String * fileExp = STR_NEW_SV(ST(1)); + bool RETVAL; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::exists", + "self", "pgBackRest::LibC::Storage") +; + RETVAL = storageExistsNP(self, fileExp); + ST(0) = boolSV(RETVAL); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_get); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_get) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, read"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead read; + SV * RETVAL; + + if (SvROK(ST(1)) && sv_derived_from(ST(1), "pgBackRest::LibC::StorageRead")) { + IV tmp = SvIV((SV*)SvRV(ST(1))); + read = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::get", + "read", "pgBackRest::LibC::StorageRead") +; + RETVAL = NULL; + Buffer *buffer = storageGetNP(read); + + if (buffer != NULL) + { + if (bufUsed(buffer) == 0) + RETVAL = newSVpv("", 0); + else + RETVAL = newSVpv((char *)bufPtr(buffer), bufUsed(buffer)); + } + RETVAL = sv_2mortal(RETVAL); + ST(0) = RETVAL; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_info); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_info) +{ + dVAR; dXSARGS; + if (items != 3) + croak_xs_usage(cv, "self, pathExp, ignoreMissing"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const String * pathExp = STR_NEW_SV(ST(1)); + bool ignoreMissing = (bool)SvTRUE(ST(2)) +; + SV * RETVAL; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::info", + "self", "pgBackRest::LibC::Storage") +; + RETVAL = NULL; + + StorageInfo info = storageInfoP(self, pathExp, .ignoreMissing = ignoreMissing); + + if (info.exists) + { + String *json = storageManifestXsInfo(NULL, &info); + RETVAL = newSVpv((char *)strPtr(json), strSize(json)); + } + RETVAL = sv_2mortal(RETVAL); + ST(0) = RETVAL; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_list); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_list) +{ + dVAR; dXSARGS; + if (items != 5) + croak_xs_usage(cv, "self, pathExp, ignoreMissing, sortAsc, expression"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const String * pathExp = STR_NEW_SV(ST(1)); + bool ignoreMissing = (bool)SvTRUE(ST(2)) +; + bool sortAsc = (bool)SvTRUE(ST(3)) +; + const String * expression = STR_NEW_SV(ST(4)); + SV * RETVAL; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::list", + "self", "pgBackRest::LibC::Storage") +; + StringList *fileList = strLstSort( + storageListP(self, pathExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false, + .expression = expression), sortAsc ? sortOrderAsc : sortOrderDesc); + + const String *fileListJson = jsonFromVar(varNewVarLst(varLstNewStrLst(fileList)), 0); + + RETVAL = newSVpv(strPtr(fileListJson), strSize(fileListJson)); + RETVAL = sv_2mortal(RETVAL); + ST(0) = RETVAL; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_manifest); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_manifest) +{ + dVAR; dXSARGS; + if (items < 2 || items > 3) + croak_xs_usage(cv, "self, pathExp, filter=NULL"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const String * pathExp = STR_NEW_SV(ST(1)); + const String * filter = STR_NEW_SV(ST(2)); + SV * RETVAL; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::manifest", + "self", "pgBackRest::LibC::Storage") +; + StorageManifestXsCallbackData data = {.storage = self, .json = strNew("{"), .pathRoot = pathExp, .filter = filter}; + + // If a path is specified + StorageInfo info = storageInfoP(self, pathExp, .ignoreMissing = true); + + if (!info.exists || info.type == storageTypePath) + { + storageInfoListP( + self, data.pathRoot, storageManifestXsCallback, &data, + .errorOnMissing = storageFeature(self, storageFeaturePath) ? true : false); + } + // Else a file is specified + else + { + info.name = strBase(storagePath(self, pathExp)); + strCat(data.json, strPtr(storageManifestXsInfo(NULL, &info))); + } + + strCat(data.json, "}"); + + RETVAL = newSVpv((char *)strPtr(data.json), strSize(data.json)); + RETVAL = sv_2mortal(RETVAL); + ST(0) = RETVAL; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_pathCreate); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_pathCreate) +{ + dVAR; dXSARGS; + if (items != 5) + croak_xs_usage(cv, "self, pathExp, mode, ignoreExists, createParent"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const String * pathExp = STR_NEW_SV(ST(1)); + const String * mode = STR_NEW_SV(ST(2)); + bool ignoreExists = (bool)SvTRUE(ST(3)) +; + bool createParent = (bool)SvTRUE(ST(4)) +; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::pathCreate", + "self", "pgBackRest::LibC::Storage") +; + if (storageFeature(self, storageFeaturePath)) + storagePathCreateP( + self, pathExp, .mode = mode ? cvtZToIntBase(strPtr(mode), 8) : 0, .errorOnExists = !ignoreExists, + .noParentCreate = !createParent); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_pathExists); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_pathExists) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, pathExp"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const String * pathExp = STR_NEW_SV(ST(1)); + bool RETVAL; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::pathExists", + "self", "pgBackRest::LibC::Storage") +; + RETVAL = true; + + if (storageFeature(self, storageFeaturePath)) + RETVAL = storagePathExistsNP(self, pathExp); + ST(0) = boolSV(RETVAL); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_pathGet); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_pathGet) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, pathExp"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const String * pathExp = STR_NEW_SV(ST(1)); + SV * RETVAL; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::pathGet", + "self", "pgBackRest::LibC::Storage") +; + String *path = storagePathNP(self, pathExp); + RETVAL = newSVpv((char *)strPtr(path), strSize(path)); + RETVAL = sv_2mortal(RETVAL); + ST(0) = RETVAL; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_pathRemove); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_pathRemove) +{ + dVAR; dXSARGS; + if (items != 4) + croak_xs_usage(cv, "self, pathExp, ignoreMissing, recurse"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const String * pathExp = STR_NEW_SV(ST(1)); + bool ignoreMissing = (bool)SvTRUE(ST(2)) +; + bool recurse = (bool)SvTRUE(ST(3)) +; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::pathRemove", + "self", "pgBackRest::LibC::Storage") +; + storagePathRemoveP( + self, pathExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false, .recurse = recurse); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_pathSync); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_pathSync) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, pathExp"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const String * pathExp = STR_NEW_SV(ST(1)); + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::pathSync", + "self", "pgBackRest::LibC::Storage") +; + storagePathSyncNP(self, pathExp); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_put); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_put) +{ + dVAR; dXSARGS; + if (items != 3) + croak_xs_usage(cv, "self, write, buffer"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageWrite write; + const Buffer * buffer = BUF_CONST_SV(ST(2)); + UV RETVAL; + dXSTARG; + + if (SvROK(ST(1)) && sv_derived_from(ST(1), "pgBackRest::LibC::StorageWrite")) { + IV tmp = SvIV((SV*)SvRV(ST(1))); + write = INT2PTR(pgBackRest__LibC__StorageWrite,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::put", + "write", "pgBackRest::LibC::StorageWrite") +; + storagePutNP(write, buffer); + RETVAL = buffer ? bufUsed(buffer) : 0; + XSprePUSH; PUSHu((UV)RETVAL); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_readDrain); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_readDrain) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, read"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__StorageRead read; + bool RETVAL; + + if (SvROK(ST(1)) && sv_derived_from(ST(1), "pgBackRest::LibC::StorageRead")) { + IV tmp = SvIV((SV*)SvRV(ST(1))); + read = INT2PTR(pgBackRest__LibC__StorageRead,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::readDrain", + "read", "pgBackRest::LibC::StorageRead") +; + RETVAL = false; + + // Read and discard all IO (this is useful for processing filters) + if (ioReadOpen(storageReadIo(read))) + { + Buffer *buffer = bufNew(ioBufferSize()); + + do + { + ioRead(storageReadIo(read), buffer); + bufUsedZero(buffer); + } + while (!ioReadEof(storageReadIo(read))); + + ioReadClose(storageReadIo(read)); + RETVAL = true; + } + ST(0) = boolSV(RETVAL); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_remove); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_remove) +{ + dVAR; dXSARGS; + if (items != 3) + croak_xs_usage(cv, "self, fileExp, ignoreMissing"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const String * fileExp = STR_NEW_SV(ST(1)); + bool ignoreMissing = (bool)SvTRUE(ST(2)) +; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::remove", + "self", "pgBackRest::LibC::Storage") +; + storageRemoveP(self, fileExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_cipherType); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_cipherType) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + const char * RETVAL; + dXSTARG; + if (cfgOptionStr(cfgOptRepoCipherType) == NULL || cipherType(cfgOptionStr(cfgOptRepoCipherType)) == cipherTypeNone) + RETVAL = NULL; + else + RETVAL = strPtr(cfgOptionStr(cfgOptRepoCipherType)); + sv_setpv(TARG, RETVAL); XSprePUSH; PUSHTARG; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_cipherPass); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_cipherPass) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + const char * RETVAL; + dXSTARG; + RETVAL = strPtr(cfgOptionStr(cfgOptRepoCipherPass)); + sv_setpv(TARG, RETVAL); XSprePUSH; PUSHTARG; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Storage_type); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_type) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + const char * RETVAL; + dXSTARG; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::type", + "self", "pgBackRest::LibC::Storage") +; + RETVAL = strPtr(storageType(self)); + sv_setpv(TARG, RETVAL); XSprePUSH; PUSHTARG; + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC_storageRepoFree); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC_storageRepoFree) +{ + dVAR; dXSARGS; + if (items != 0) + croak_xs_usage(cv, ""); + { + storageHelperFree(); + } + XSRETURN_EMPTY; +} + + /* INCLUDE: Returning to 'xs/postgres/pageChecksum.xs' from 'xs/storage/storage.xs' */ @@ -331,72 +1539,6 @@ XS_EUPXS(XS_pgBackRest__LibC_pageChecksum) } -XS_EUPXS(XS_pgBackRest__LibC_pageChecksumTest); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC_pageChecksumTest) -{ - dVAR; dXSARGS; - if (items != 5) - croak_xs_usage(cv, "page, blockNo, pageSize, ignoreWalId, ignoreWalOffset"); - { - const char * page = (const char *)SvPV_nolen(ST(0)) -; - U32 blockNo = (unsigned long)SvUV(ST(1)) -; - U32 pageSize = (unsigned long)SvUV(ST(2)) -; - U32 ignoreWalId = (unsigned long)SvUV(ST(3)) -; - U32 ignoreWalOffset = (unsigned long)SvUV(ST(4)) -; - bool RETVAL; - RETVAL = false; - - ERROR_XS_BEGIN() - { - RETVAL = pageChecksumTest( - (const unsigned char *)page, blockNo, pageSize, ignoreWalId, ignoreWalOffset); - } - ERROR_XS_END(); - ST(0) = boolSV(RETVAL); - } - XSRETURN(1); -} - - -XS_EUPXS(XS_pgBackRest__LibC_pageChecksumBufferTest); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC_pageChecksumBufferTest) -{ - dVAR; dXSARGS; - if (items != 6) - croak_xs_usage(cv, "pageBuffer, pageBufferSize, blockNoBegin, pageSize, ignoreWalId, ignoreWalOffset"); - { - const char * pageBuffer = (const char *)SvPV_nolen(ST(0)) -; - U32 pageBufferSize = (unsigned long)SvUV(ST(1)) -; - U32 blockNoBegin = (unsigned long)SvUV(ST(2)) -; - U32 pageSize = (unsigned long)SvUV(ST(3)) -; - U32 ignoreWalId = (unsigned long)SvUV(ST(4)) -; - U32 ignoreWalOffset = (unsigned long)SvUV(ST(5)) -; - bool RETVAL; - RETVAL = false; - - ERROR_XS_BEGIN() - { - RETVAL = pageChecksumBufferTest( - (const unsigned char *)pageBuffer, pageBufferSize, blockNoBegin, pageSize, ignoreWalId, ignoreWalOffset); - } - ERROR_XS_END(); - ST(0) = boolSV(RETVAL); - } - XSRETURN(1); -} - - /* INCLUDE: Returning to 'xs/crypto/random.xs' from 'xs/postgres/pageChecksum.xs' */ @@ -426,137 +1568,6 @@ XS_EUPXS(XS_pgBackRest__LibC_cryptoRandomBytes) /* INCLUDE: Returning to 'xs/crypto/hash.xs' from 'xs/crypto/random.xs' */ -XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_new); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_new) -{ - dVAR; dXSARGS; - if (items != 2) - croak_xs_usage(cv, "class, type"); - { - const char * class = (const char *)SvPV_nolen(ST(0)) -; - const char * type = (const char *)SvPV_nolen(ST(1)) -; - pgBackRest__LibC__Crypto__Hash RETVAL; - RETVAL = NULL; - - // Don't warn when class param is used - (void)class; - - MEM_CONTEXT_XS_NEW_BEGIN("cryptoHashXs") - { - RETVAL = memNew(sizeof(CryptoHashXs)); - RETVAL->memContext = MEM_CONTEXT_XS(); - RETVAL->pxPayload = cryptoHashNew(strNew(type)); - } - MEM_CONTEXT_XS_NEW_END(); - { - SV * RETVALSV; - RETVALSV = sv_newmortal(); - sv_setref_pv(RETVALSV, "pgBackRest::LibC::Crypto::Hash", (void*)RETVAL); - ST(0) = RETVALSV; - } - } - XSRETURN(1); -} - - -XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_process); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_process) -{ - dVAR; dXSARGS; - if (items != 2) - croak_xs_usage(cv, "self, message"); - { - pgBackRest__LibC__Crypto__Hash self; - SV * message = ST(1) -; - - if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Crypto::Hash")) { - IV tmp = SvIV((SV*)SvRV(ST(0))); - self = INT2PTR(pgBackRest__LibC__Crypto__Hash,tmp); - } - else - Perl_croak_nocontext("%s: %s is not of type %s", - "pgBackRest::LibC::Crypto::Hash::process", - "self", "pgBackRest::LibC::Crypto::Hash") -; - MEM_CONTEXT_XS_TEMP_BEGIN() - { - STRLEN messageSize; - const void *messagePtr = SvPV(message, messageSize); - - if (messageSize > 0) - ioFilterProcessIn(self->pxPayload, BUF(messagePtr, messageSize)); - } - MEM_CONTEXT_XS_TEMP_END(); - } - XSRETURN_EMPTY; -} - - -XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_result); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_result) -{ - dVAR; dXSARGS; - if (items != 1) - croak_xs_usage(cv, "self"); - { - pgBackRest__LibC__Crypto__Hash self; - SV * RETVAL; - - if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Crypto::Hash")) { - IV tmp = SvIV((SV*)SvRV(ST(0))); - self = INT2PTR(pgBackRest__LibC__Crypto__Hash,tmp); - } - else - Perl_croak_nocontext("%s: %s is not of type %s", - "pgBackRest::LibC::Crypto::Hash::result", - "self", "pgBackRest::LibC::Crypto::Hash") -; - RETVAL = NULL; - - MEM_CONTEXT_XS_TEMP_BEGIN() - { - const String *hash = varStr(ioFilterResult(self->pxPayload)); - - RETVAL = newSV(strSize(hash)); - SvPOK_only(RETVAL); - strcpy((char *)SvPV_nolen(RETVAL), strPtr(hash)); - SvCUR_set(RETVAL, strSize(hash)); - } - MEM_CONTEXT_XS_TEMP_END(); - RETVAL = sv_2mortal(RETVAL); - ST(0) = RETVAL; - } - XSRETURN(1); -} - - -XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_DESTROY); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_DESTROY) -{ - dVAR; dXSARGS; - if (items != 1) - croak_xs_usage(cv, "self"); - { - pgBackRest__LibC__Crypto__Hash self; - - if (SvROK(ST(0))) { - IV tmp = SvIV((SV*)SvRV(ST(0))); - self = INT2PTR(pgBackRest__LibC__Crypto__Hash,tmp); - } - else - Perl_croak_nocontext("%s: %s is not a reference", - "pgBackRest::LibC::Crypto::Hash::DESTROY", - "self") -; - MEM_CONTEXT_XS_DESTROY(self->memContext); - } - XSRETURN_EMPTY; -} - - XS_EUPXS(XS_pgBackRest__LibC_cryptoHashOne); /* prototype to pass -Wmissing-prototypes */ XS_EUPXS(XS_pgBackRest__LibC_cryptoHashOne) { @@ -591,195 +1602,7 @@ XS_EUPXS(XS_pgBackRest__LibC_cryptoHashOne) } -/* INCLUDE: Returning to 'xs/crypto/cipherBlock.xs' from 'xs/crypto/hash.xs' */ - - -XS_EUPXS(XS_pgBackRest__LibC__Cipher__Block_new); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC__Cipher__Block_new) -{ - dVAR; dXSARGS; - if (items < 5 || items > 6) - croak_xs_usage(cv, "class, mode, type, key, keySize, digest = NULL"); - { - const char * class = (const char *)SvPV_nolen(ST(0)) -; - U32 mode = (unsigned long)SvUV(ST(1)) -; - const char * type = (const char *)SvPV_nolen(ST(2)) -; - unsigned char * key = (unsigned char *)SvPV_nolen(ST(3)) -; - I32 keySize = (I32)SvIV(ST(4)) -; - const char * digest; - pgBackRest__LibC__Cipher__Block RETVAL; - - if (items < 6) - digest = NULL; - else { - digest = (const char *)SvPV_nolen(ST(5)) -; - } - RETVAL = NULL; - - CHECK(type != NULL); - CHECK(key != NULL); - CHECK(keySize != 0); - - // Not much point to this but it keeps the var from being unused - if (strcmp(class, PACKAGE_NAME_LIBC "::Cipher::Block") != 0) - croak("unexpected class name '%s'", class); - - MEM_CONTEXT_XS_NEW_BEGIN("cipherBlockXs") - { - RETVAL = memNew(sizeof(CipherBlockXs)); - RETVAL->memContext = MEM_CONTEXT_XS(); - - RETVAL->pxPayload = cipherBlockNew(mode, cipherType(STR(type)), BUF(key, keySize), digest == NULL ? NULL : STR(digest)); - } - MEM_CONTEXT_XS_NEW_END(); - { - SV * RETVALSV; - RETVALSV = sv_newmortal(); - sv_setref_pv(RETVALSV, "pgBackRest::LibC::Cipher::Block", (void*)RETVAL); - ST(0) = RETVALSV; - } - } - XSRETURN(1); -} - - -XS_EUPXS(XS_pgBackRest__LibC__Cipher__Block_process); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC__Cipher__Block_process) -{ - dVAR; dXSARGS; - if (items != 2) - croak_xs_usage(cv, "self, source"); - { - pgBackRest__LibC__Cipher__Block self; - SV * source = ST(1) -; - SV * RETVAL; - - if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Cipher::Block")) { - IV tmp = SvIV((SV*)SvRV(ST(0))); - self = INT2PTR(pgBackRest__LibC__Cipher__Block,tmp); - } - else - Perl_croak_nocontext("%s: %s is not of type %s", - "pgBackRest::LibC::Cipher::Block::process", - "self", "pgBackRest::LibC::Cipher::Block") -; - RETVAL = NULL; - - MEM_CONTEXT_XS_BEGIN(self->memContext) - { - STRLEN tSize; - const unsigned char *sourcePtr = (const unsigned char *)SvPV(source, tSize); - - RETVAL = NEWSV(0, ioBufferSize()); - SvPOK_only(RETVAL); - - if (tSize > 0) - { - size_t outBufferUsed = 0; - - do - { - SvGROW(RETVAL, outBufferUsed + ioBufferSize()); - Buffer *outBuffer = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL) + outBufferUsed, ioBufferSize()); - - ioFilterProcessInOut(self->pxPayload, BUF(sourcePtr, tSize), outBuffer); - outBufferUsed += bufUsed(outBuffer); - } - while (ioFilterInputSame(self->pxPayload)); - - SvCUR_set(RETVAL, outBufferUsed); - } - else - SvCUR_set(RETVAL, 0); - } - MEM_CONTEXT_XS_END(); - RETVAL = sv_2mortal(RETVAL); - ST(0) = RETVAL; - } - XSRETURN(1); -} - - -XS_EUPXS(XS_pgBackRest__LibC__Cipher__Block_flush); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC__Cipher__Block_flush) -{ - dVAR; dXSARGS; - if (items != 1) - croak_xs_usage(cv, "self"); - { - pgBackRest__LibC__Cipher__Block self; - SV * RETVAL; - - if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Cipher::Block")) { - IV tmp = SvIV((SV*)SvRV(ST(0))); - self = INT2PTR(pgBackRest__LibC__Cipher__Block,tmp); - } - else - Perl_croak_nocontext("%s: %s is not of type %s", - "pgBackRest::LibC::Cipher::Block::flush", - "self", "pgBackRest::LibC::Cipher::Block") -; - RETVAL = NULL; - - MEM_CONTEXT_XS_BEGIN(self->memContext) - { - RETVAL = NEWSV(0, ioBufferSize()); - SvPOK_only(RETVAL); - - size_t outBufferUsed = 0; - - do - { - SvGROW(RETVAL, outBufferUsed + ioBufferSize()); - Buffer *outBuffer = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL) + outBufferUsed, ioBufferSize()); - - ioFilterProcessInOut(self->pxPayload, NULL, outBuffer); - outBufferUsed += bufUsed(outBuffer); - } - while (!ioFilterDone(self->pxPayload)); - - SvCUR_set(RETVAL, outBufferUsed); - } - MEM_CONTEXT_XS_END(); - RETVAL = sv_2mortal(RETVAL); - ST(0) = RETVAL; - } - XSRETURN(1); -} - - -XS_EUPXS(XS_pgBackRest__LibC__Cipher__Block_DESTROY); /* prototype to pass -Wmissing-prototypes */ -XS_EUPXS(XS_pgBackRest__LibC__Cipher__Block_DESTROY) -{ - dVAR; dXSARGS; - if (items != 1) - croak_xs_usage(cv, "self"); - { - pgBackRest__LibC__Cipher__Block self; - - if (SvROK(ST(0))) { - IV tmp = SvIV((SV*)SvRV(ST(0))); - self = INT2PTR(pgBackRest__LibC__Cipher__Block,tmp); - } - else - Perl_croak_nocontext("%s: %s is not a reference", - "pgBackRest::LibC::Cipher::Block::DESTROY", - "self") -; - MEM_CONTEXT_XS_DESTROY(self->memContext); - } - XSRETURN_EMPTY; -} - - -/* INCLUDE: Returning to 'xs/config/define.xs' from 'xs/crypto/cipherBlock.xs' */ +/* INCLUDE: Returning to 'xs/config/define.xs' from 'xs/crypto/hash.xs' */ XS_EUPXS(XS_pgBackRest__LibC_cfgCommandId); /* prototype to pass -Wmissing-prototypes */ @@ -1250,20 +2073,45 @@ XS_EXTERNAL(boot_pgBackRest__LibC) #endif newXS_deffile("pgBackRest::LibC::libcUvSize", XS_pgBackRest__LibC_libcUvSize); - newXS_deffile("pgBackRest::LibC::storagePosixPathRemove", XS_pgBackRest__LibC_storagePosixPathRemove); + newXS_deffile("pgBackRest::LibC::StorageWrite::new", XS_pgBackRest__LibC__StorageWrite_new); + newXS_deffile("pgBackRest::LibC::StorageWrite::filterAdd", XS_pgBackRest__LibC__StorageWrite_filterAdd); + newXS_deffile("pgBackRest::LibC::StorageWrite::open", XS_pgBackRest__LibC__StorageWrite_open); + newXS_deffile("pgBackRest::LibC::StorageWrite::write", XS_pgBackRest__LibC__StorageWrite_write); + newXS_deffile("pgBackRest::LibC::StorageWrite::close", XS_pgBackRest__LibC__StorageWrite_close); + newXS_deffile("pgBackRest::LibC::StorageWrite::result", XS_pgBackRest__LibC__StorageWrite_result); + newXS_deffile("pgBackRest::LibC::StorageWrite::resultAll", XS_pgBackRest__LibC__StorageWrite_resultAll); + newXS_deffile("pgBackRest::LibC::StorageWrite::DESTROY", XS_pgBackRest__LibC__StorageWrite_DESTROY); + newXS_deffile("pgBackRest::LibC::StorageRead::new", XS_pgBackRest__LibC__StorageRead_new); + newXS_deffile("pgBackRest::LibC::StorageRead::filterAdd", XS_pgBackRest__LibC__StorageRead_filterAdd); + newXS_deffile("pgBackRest::LibC::StorageRead::open", XS_pgBackRest__LibC__StorageRead_open); + newXS_deffile("pgBackRest::LibC::StorageRead::read", XS_pgBackRest__LibC__StorageRead_read); + newXS_deffile("pgBackRest::LibC::StorageRead::eof", XS_pgBackRest__LibC__StorageRead_eof); + newXS_deffile("pgBackRest::LibC::StorageRead::close", XS_pgBackRest__LibC__StorageRead_close); + newXS_deffile("pgBackRest::LibC::StorageRead::result", XS_pgBackRest__LibC__StorageRead_result); + newXS_deffile("pgBackRest::LibC::StorageRead::resultAll", XS_pgBackRest__LibC__StorageRead_resultAll); + newXS_deffile("pgBackRest::LibC::StorageRead::DESTROY", XS_pgBackRest__LibC__StorageRead_DESTROY); + newXS_deffile("pgBackRest::LibC::Storage::new", XS_pgBackRest__LibC__Storage_new); + newXS_deffile("pgBackRest::LibC::Storage::copy", XS_pgBackRest__LibC__Storage_copy); + newXS_deffile("pgBackRest::LibC::Storage::exists", XS_pgBackRest__LibC__Storage_exists); + newXS_deffile("pgBackRest::LibC::Storage::get", XS_pgBackRest__LibC__Storage_get); + newXS_deffile("pgBackRest::LibC::Storage::info", XS_pgBackRest__LibC__Storage_info); + newXS_deffile("pgBackRest::LibC::Storage::list", XS_pgBackRest__LibC__Storage_list); + newXS_deffile("pgBackRest::LibC::Storage::manifest", XS_pgBackRest__LibC__Storage_manifest); + newXS_deffile("pgBackRest::LibC::Storage::pathCreate", XS_pgBackRest__LibC__Storage_pathCreate); + newXS_deffile("pgBackRest::LibC::Storage::pathExists", XS_pgBackRest__LibC__Storage_pathExists); + newXS_deffile("pgBackRest::LibC::Storage::pathGet", XS_pgBackRest__LibC__Storage_pathGet); + newXS_deffile("pgBackRest::LibC::Storage::pathRemove", XS_pgBackRest__LibC__Storage_pathRemove); + newXS_deffile("pgBackRest::LibC::Storage::pathSync", XS_pgBackRest__LibC__Storage_pathSync); + newXS_deffile("pgBackRest::LibC::Storage::put", XS_pgBackRest__LibC__Storage_put); + newXS_deffile("pgBackRest::LibC::Storage::readDrain", XS_pgBackRest__LibC__Storage_readDrain); + newXS_deffile("pgBackRest::LibC::Storage::remove", XS_pgBackRest__LibC__Storage_remove); + newXS_deffile("pgBackRest::LibC::Storage::cipherType", XS_pgBackRest__LibC__Storage_cipherType); + newXS_deffile("pgBackRest::LibC::Storage::cipherPass", XS_pgBackRest__LibC__Storage_cipherPass); + newXS_deffile("pgBackRest::LibC::Storage::type", XS_pgBackRest__LibC__Storage_type); + newXS_deffile("pgBackRest::LibC::storageRepoFree", XS_pgBackRest__LibC_storageRepoFree); newXS_deffile("pgBackRest::LibC::pageChecksum", XS_pgBackRest__LibC_pageChecksum); - newXS_deffile("pgBackRest::LibC::pageChecksumTest", XS_pgBackRest__LibC_pageChecksumTest); - newXS_deffile("pgBackRest::LibC::pageChecksumBufferTest", XS_pgBackRest__LibC_pageChecksumBufferTest); newXS_deffile("pgBackRest::LibC::cryptoRandomBytes", XS_pgBackRest__LibC_cryptoRandomBytes); - newXS_deffile("pgBackRest::LibC::Crypto::Hash::new", XS_pgBackRest__LibC__Crypto__Hash_new); - newXS_deffile("pgBackRest::LibC::Crypto::Hash::process", XS_pgBackRest__LibC__Crypto__Hash_process); - newXS_deffile("pgBackRest::LibC::Crypto::Hash::result", XS_pgBackRest__LibC__Crypto__Hash_result); - newXS_deffile("pgBackRest::LibC::Crypto::Hash::DESTROY", XS_pgBackRest__LibC__Crypto__Hash_DESTROY); newXS_deffile("pgBackRest::LibC::cryptoHashOne", XS_pgBackRest__LibC_cryptoHashOne); - newXS_deffile("pgBackRest::LibC::Cipher::Block::new", XS_pgBackRest__LibC__Cipher__Block_new); - newXS_deffile("pgBackRest::LibC::Cipher::Block::process", XS_pgBackRest__LibC__Cipher__Block_process); - newXS_deffile("pgBackRest::LibC::Cipher::Block::flush", XS_pgBackRest__LibC__Cipher__Block_flush); - newXS_deffile("pgBackRest::LibC::Cipher::Block::DESTROY", XS_pgBackRest__LibC__Cipher__Block_DESTROY); newXS_deffile("pgBackRest::LibC::cfgCommandId", XS_pgBackRest__LibC_cfgCommandId); newXS_deffile("pgBackRest::LibC::cfgOptionId", XS_pgBackRest__LibC_cfgOptionId); newXS_deffile("pgBackRest::LibC::cfgDefOptionDefault", XS_pgBackRest__LibC_cfgDefOptionDefault); diff --git a/src/storage/storage.c b/src/storage/storage.c index ec82d96f0..ce7a9f09a 100644 --- a/src/storage/storage.c +++ b/src/storage/storage.c @@ -28,7 +28,7 @@ struct Storage mode_t modeFile; mode_t modePath; bool write; - bool pathResolve; + bool pathEnforce; StoragePathExpressionCallback pathExpressionFunction; }; @@ -73,6 +73,7 @@ storageNew( this->path = strDup(path); this->modeFile = modeFile; this->modePath = modePath; + this->pathEnforce = true; this->write = write; this->pathExpressionFunction = pathExpressionFunction; @@ -493,8 +494,8 @@ storagePath(const Storage *this, const String *pathExp) // Make sure the base storage path is contained within the path expression if (this->path != NULL && !strEqZ(this->path, "/")) { - if (!strBeginsWith(pathExp, this->path) || - !(strSize(pathExp) == strSize(this->path) || *(strPtr(pathExp) + strSize(this->path)) == '/')) + if (this->pathEnforce && (!strBeginsWith(pathExp, this->path) || + !(strSize(pathExp) == strSize(this->path) || *(strPtr(pathExp) + strSize(this->path)) == '/'))) { THROW_FMT(AssertError, "absolute path '%s' is not in base path '%s'", strPtr(pathExp), strPtr(this->path)); } @@ -586,10 +587,6 @@ storagePathCreate(const Storage *this, const String *pathExp, StoragePathCreateP ASSERT(this->interface.pathCreate != NULL && storageFeature(this, storageFeaturePath)); ASSERT(this->write); - // It doesn't make sense to combine these parameters because if we are creating missing parent paths why error when they exist? - // If this somehow wasn't caught in testing, the worst case is that the path would not be created and an error would be thrown. - ASSERT(!(param.noParentCreate && param.errorOnExists)); - MEM_CONTEXT_TEMP_BEGIN() { // Build the path @@ -781,6 +778,22 @@ storageInterface(const Storage *this) FUNCTION_TEST_RETURN(this->interface); } +/*********************************************************************************************************************************** +Set whether absolute paths are required to be in the base path +***********************************************************************************************************************************/ +void +storagePathEnforceSet(Storage *this, bool enforce) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STORAGE, this); + FUNCTION_TEST_PARAM(BOOL, enforce); + FUNCTION_TEST_END(); + + this->pathEnforce = enforce; + + FUNCTION_TEST_RETURN_VOID(); +} + /*********************************************************************************************************************************** Get the storage type (posix, cifs, etc.) ***********************************************************************************************************************************/ diff --git a/src/storage/storage.intern.h b/src/storage/storage.intern.h index aab77cf96..3b508574f 100644 --- a/src/storage/storage.intern.h +++ b/src/storage/storage.intern.h @@ -82,11 +82,15 @@ Storage *storageNew( StoragePathExpressionCallback pathExpressionFunction, void *driver, StorageInterface interface); /*********************************************************************************************************************************** -Getters +Getters/Setters ***********************************************************************************************************************************/ void *storageDriver(const Storage *this); StorageInterface storageInterface(const Storage *this); +// The option is intended to be used only with the Perl interface since Perl is not tidy about where it reads. It should be +// removed when the Perl interface is removed. +void storagePathEnforceSet(Storage *this, bool enforce); + /*********************************************************************************************************************************** Macros for function logging ***********************************************************************************************************************************/ diff --git a/test/define.yaml b/test/define.yaml index 721614ef8..011a591ad 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -276,13 +276,6 @@ unit: - name: encode-perl total: 1 - # ---------------------------------------------------------------------------------------------------------------------------- - - name: http-client-perl - total: 2 - - coverage: - Common/Http/Client: partial - # ---------------------------------------------------------------------------------------------------------------------------- - name: reg-exp total: 3 @@ -423,82 +416,18 @@ unit: test: # ---------------------------------------------------------------------------------------------------------------------------- - - name: filter-cipher-block-perl - total: 2 + - name: perl + total: 13 coverage: - Storage/Filter/CipherBlock: full - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: filter-gzip-perl - total: 3 - - coverage: - Storage/Filter/Gzip: full - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: filter-sha-perl - total: 2 - - coverage: - Storage/Filter/Sha: full - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: posix-perl - total: 10 - - coverage: - Storage/Posix/Driver: partial - Storage/Posix/FileRead: partial - Storage/Posix/FileWrite: partial - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: s3-auth-perl - total: 5 - - coverage: - Storage/S3/Auth: full - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: s3-cert-perl - total: 1 - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: s3-request-perl - total: 2 - - coverage: - Storage/S3/Request: partial - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: s3-perl - total: 7 - - coverage: - Storage/S3/Driver: partial - Storage/S3/FileRead: partial - Storage/S3/FileWrite: full - - vm: - - co7 - - u14 - - u16 - - u18 - - d8 - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: local-perl - total: 10 - - coverage: - Storage/Local: partial + Storage/Storage: partial # ---------------------------------------------------------------------------------------------------------------------------- - name: helper-perl - total: 4 + total: 3 coverage: - Storage/Helper: partial + Storage/Helper: full # ---------------------------------------------------------------------------------------------------------------------------- - name: cifs @@ -840,7 +769,3 @@ performance: # ---------------------------------------------------------------------------------------------------------------------------- - name: archive total: 1 - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: io - total: 1 diff --git a/test/expect/mock-all-001.log b/test/expect/mock-all-001.log index 3886f70b5..8721cc615 100644 --- a/test/expect/mock-all-001.log +++ b/test/expect/mock-all-001.log @@ -994,8 +994,8 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] -P00 ERROR: [055]: unable to stat '[TEST_PATH]/db-master/db/base_tbs': No such file or directory -P00 INFO: backup command end: aborted with exception [055] +P00 ERROR: [073]: unable to list file info for missing path '[TEST_PATH]/db-master/db/base_tbs' +P00 INFO: backup command end: aborted with exception [073] incr backup - invalid tablespace in $PGDATA (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --stanza=db backup diff --git a/test/expect/mock-all-002.log b/test/expect/mock-all-002.log index 0d510ee98..fdb1aa88f 100644 --- a/test/expect/mock-all-002.log +++ b/test/expect/mock-all-002.log @@ -257,7 +257,7 @@ full backup - resume (backup host) ------------------------------------------------------------------------------------------------------------------------------------ P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. -P00 WARN: backup [BACKUP-FULL-1] missing in repository removed from backup.info +P00 WARN: backup [BACKUP-FULL-1] missing manifest removed from backup.info P00 WARN: --no-online passed and postmaster.pid exists but --force was passed so backup will continue though it looks like the postmaster is running and the backup will probably not be consistent P00 WARN: aborted backup [BACKUP-FULL-2] of same type exists, will be cleaned to remove invalid files and resumed P00 TEST: PgBaCkReStTeSt-BACKUP-RESUME-PgBaCkReStTeSt @@ -743,7 +743,7 @@ incr backup - resume and add tablespace 2 (backup host) ------------------------------------------------------------------------------------------------------------------------------------ P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. -P00 WARN: backup [BACKUP-INCR-1] missing in repository removed from backup.info +P00 WARN: backup [BACKUP-INCR-1] missing manifest removed from backup.info P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] P00 WARN: file pg_data/changetime.txt timestamp in the past or size changed but timestamp did not, enabling delta checksum P00 WARN: aborted backup [BACKUP-INCR-2] of same type exists, will be cleaned to remove invalid files and resumed @@ -3350,6 +3350,7 @@ P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1] P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/base/base2.txt (9B, 100%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0 P00 INFO: diff backup size = 9B P00 INFO: new backup label = [BACKUP-DIFF-6] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: backup command end: completed successfully P00 INFO: expire command begin P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired @@ -3558,6 +3559,7 @@ P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1] P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/base/base2.txt (9B, 100%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0 P00 INFO: diff backup size = 9B P00 INFO: new backup label = [BACKUP-DIFF-7] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: backup command end: completed successfully P00 INFO: expire command begin P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired diff --git a/test/expect/mock-archive-002.log b/test/expect/mock-archive-002.log index 20fba910c..fb5644484 100644 --- a/test/expect/mock-archive-002.log +++ b/test/expect/mock-archive-002.log @@ -29,6 +29,8 @@ stanza-create db - stanza create (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online --force stanza-create ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-2] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-create command end: completed successfully + supplemental file: /backup/db/backup.info diff --git a/test/expect/mock-archive-stop-002.log b/test/expect/mock-archive-stop-002.log index dbc46d4f3..cfd9e3896 100644 --- a/test/expect/mock-archive-stop-002.log +++ b/test/expect/mock-archive-stop-002.log @@ -5,6 +5,8 @@ stanza-create db - create required data for stanza (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-create ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-create command end: completed successfully + supplemental file: /backup/db/backup.info diff --git a/test/expect/mock-stanza-001.log b/test/expect/mock-stanza-001.log index 9522c7e60..e20f169a2 100644 --- a/test/expect/mock-stanza-001.log +++ b/test/expect/mock-stanza-001.log @@ -209,7 +209,7 @@ stanza-create db - gunzip fail on forced stanza-create (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --no-online --force stanza-create ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db -P00 ERROR: [041]: unable to open '[TEST_PATH]/db-master/repo/archive/db/9.3-1/0000000100000001/000000010000000100000001-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz': Permission denied +P00 ERROR: [041]: unable to open file '[TEST_PATH]/db-master/repo/archive/db/9.3-1/0000000100000001/000000010000000100000001-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz' for read: [13] Permission denied P00 INFO: stanza-create command end: aborted with exception [041] + supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info diff --git a/test/expect/mock-stanza-002.log b/test/expect/mock-stanza-002.log index a16af65dc..71ad99a7b 100644 --- a/test/expect/mock-stanza-002.log +++ b/test/expect/mock-stanza-002.log @@ -16,12 +16,16 @@ P00 ERROR: [055]: archive.info does not exist but is required to push/get WAL s HINT: is archive_command configured in postgresql.conf? HINT: has a stanza-create been performed? HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme. +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-upgrade command end: aborted with exception [055] stanza-create db - successfully create the stanza (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-create ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-create command end: completed successfully + supplemental file: /backup/db/backup.info @@ -67,6 +71,8 @@ stanza-create db - do not fail on rerun of stanza-create - info files exist and ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db P00 INFO: stanza-create was already performed +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-create command end: completed successfully + supplemental file: /backup/db/backup.info @@ -113,6 +119,8 @@ stanza-create db - fail on database mismatch without force option (backup host) P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db P00 ERROR: [028]: backup info file or archive info file invalid HINT: use stanza-upgrade if the database has been upgraded or use --force +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-create command end: aborted with exception [028] + supplemental file: /backup/db/backup.info @@ -158,6 +166,8 @@ stanza-upgrade db - already up to date (backup host) ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db P00 INFO: the stanza data is already up to date +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-upgrade command end: completed successfully + supplemental file: /backup/db/backup.info @@ -209,6 +219,8 @@ stanza-create db - force create archive.info from gz file (backup host) ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db P00 ERROR: [055]: archive information missing and repo is encrypted and info file(s) are missing, --force cannot be used +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-create command end: aborted with exception [055] + supplemental file: /backup/db/backup.info @@ -246,6 +258,8 @@ stanza-upgrade db - successful upgrade creates additional history (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-upgrade ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-upgrade command end: completed successfully + supplemental file: /backup/db/backup.info @@ -309,6 +323,8 @@ P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_ P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%) P00 INFO: full backup size = 48MB P00 INFO: new backup label = [BACKUP-FULL-1] +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: backup command end: completed successfully P00 INFO: expire command begin P00 INFO: remove archive path: /archive/db/9.3-1 @@ -381,6 +397,8 @@ stanza-create db - fail no force to recreate the stanza from backups (backup hos ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db P00 ERROR: [055]: backup information missing and repo is encrypted and info file(s) are missing, --force cannot be used +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-create command end: aborted with exception [055] + supplemental file: /archive/db/archive.info @@ -407,6 +425,8 @@ stanza-create db - use force to recreate the stanza from backups (backup host) ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db P00 ERROR: [055]: backup information missing and repo is encrypted and info file(s) are missing, --force cannot be used +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-create command end: aborted with exception [055] + supplemental file: /archive/db/archive.info @@ -432,6 +452,8 @@ stanza-upgrade db - successfully upgrade - no info file mismatch (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-upgrade ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-upgrade command end: completed successfully + supplemental file: /backup/db/backup.info @@ -495,6 +517,8 @@ P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_ P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%) P00 INFO: full backup size = 48MB P00 INFO: new backup label = [BACKUP-FULL-2] +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: backup command end: completed successfully P00 INFO: expire command begin P00 DETAIL: tls statistics:[TLS-STATISTICS] @@ -695,6 +719,8 @@ stanza-delete db - successfully delete the stanza (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db stanza-delete ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-delete command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-delete command end: completed successfully db must not exist for successful delete diff --git a/test/lib/pgBackRestTest/Common/DefineTest.pm b/test/lib/pgBackRestTest/Common/DefineTest.pm index 17011b82e..f7ec75b5d 100644 --- a/test/lib/pgBackRestTest/Common/DefineTest.pm +++ b/test/lib/pgBackRestTest/Common/DefineTest.pm @@ -136,7 +136,7 @@ sub testDefLoad # Set module type variables $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_C} = - $strModuleType eq TESTDEF_UNIT && $strTest !~ /\-perl$/ ? true : false; + $strModuleType eq TESTDEF_UNIT && $strTest !~ /perl$/ ? true : false; $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_INTEGRATION} = $strModuleType eq TESTDEF_INTEGRATION ? true : false; $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_EXPECT} = $bExpect; $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_CONTAINER} = $bContainer; diff --git a/test/lib/pgBackRestTest/Common/FileTest.pm b/test/lib/pgBackRestTest/Common/FileTest.pm index ae2f58da8..e2a67a3ce 100644 --- a/test/lib/pgBackRestTest/Common/FileTest.pm +++ b/test/lib/pgBackRestTest/Common/FileTest.pm @@ -27,8 +27,7 @@ use pgBackRest::Common::String; use pgBackRest::Common::Wait; use pgBackRest::Config::Config; use pgBackRest::Manifest; -use pgBackRest::Storage::Local; -use pgBackRest::Storage::S3::Driver; +use pgBackRest::Storage::Base; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::HostGroupTest; @@ -168,7 +167,7 @@ sub forceStorageMode ); # Mode commands are ignored on S3 - if ($oStorage->driver()->className() ne STORAGE_S3_DRIVER) + if ($oStorage->type() ne STORAGE_S3) { executeTest('sudo chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp)); } @@ -203,7 +202,7 @@ sub forceStorageMove ); # If S3 then use storage commands to remove - if ($oStorage->driver()->className() eq STORAGE_S3_DRIVER) + if ($oStorage->type() eq STORAGE_S3) { hostGroupGet()->hostGet(HOST_S3)->executeS3( 'mv' . ($bRecurse ? ' --recursive' : '') . ' s3://' . HOST_S3_BUCKET . $oStorage->pathGet($strSourcePathExp) . @@ -244,8 +243,8 @@ sub forceStorageOwner {name => 'bRecurse', optional => true, default => false}, ); - # Mode commands are ignored on S3 - if ($oStorage->driver()->className() ne STORAGE_S3_DRIVER) + # Owner commands are ignored on S3 + if ($oStorage->type() ne STORAGE_S3) { executeTest('sudo chown ' . ($bRecurse ? '-R ' : '') . "${strOwner} " . $oStorage->pathGet($strPathExp)); } @@ -278,11 +277,19 @@ sub forceStorageRemove ); # If S3 then use storage commands to remove - if ($oStorage->driver()->className() eq STORAGE_S3_DRIVER) + if ($oStorage->type() eq STORAGE_S3) { - $oStorage->remove($strPathExp, {bRecurse => $bRecurse}); + my $oInfo = $oStorage->info($strPathExp, {bIgnoreMissing => true}); + + if (defined($oInfo) && $oInfo->{type} eq 'f') + { + $oStorage->remove($strPathExp); + } + else + { + $oStorage->pathRemove($strPathExp, {bRecurse => true}); + } } - # Else remove using filesystem commands else { executeTest('sudo rm -f' . ($bRecurse ? 'r ' : ' ') . $oStorage->pathGet($strPathExp)); diff --git a/test/lib/pgBackRestTest/Common/RunTest.pm b/test/lib/pgBackRestTest/Common/RunTest.pm index 9f8637335..88f9a495d 100644 --- a/test/lib/pgBackRestTest/Common/RunTest.pm +++ b/test/lib/pgBackRestTest/Common/RunTest.pm @@ -19,10 +19,11 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Log; use pgBackRest::Common::String; use pgBackRest::Common::Wait; -use pgBackRest::Storage::Posix::Driver; -use pgBackRest::Storage::Local; +use pgBackRest::Storage::Base; +use pgBackRest::Storage::Storage; use pgBackRest::Version; +use pgBackRestTest::Common::BuildTest; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::LogTest; @@ -155,7 +156,7 @@ sub process $self->{bFirstTest} = true; # Initialize test storage - $oStorage = new pgBackRest::Storage::Local($self->testPath(), new pgBackRest::Storage::Posix::Driver()); + $oStorage = new pgBackRest::Storage::Storage(STORAGE_LOCAL, {strPath => $self->testPath()}); # Generate backrest exe $self->{strBackRestExe} = testRunExe( diff --git a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm index 647939dcb..7f1e039ae 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm @@ -15,6 +15,7 @@ use Exporter qw(import); our @EXPORT = qw(); use Fcntl ':mode'; use File::Basename qw(dirname); +use File::stat qw{lstat}; use Storable qw(dclone); use pgBackRest::Archive::Info; @@ -27,9 +28,8 @@ use pgBackRest::Config::Config; use pgBackRest::DbVersion; use pgBackRest::Manifest; use pgBackRest::Protocol::Storage::Helper; -use pgBackRest::Storage::Posix::Driver; -use pgBackRest::Storage::S3::Driver; use pgBackRest::Version; +use pgBackRest::Storage::Base; use pgBackRestTest::Env::Host::HostBaseTest; use pgBackRestTest::Env::Host::HostS3Test; @@ -262,65 +262,68 @@ sub backupEnd } # Make sure tablespace links are correct - if (($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink()) && $self->hasLink()) + if ($self->hasLink()) { - my $hTablespaceManifest = storageRepo()->manifest( - STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC); - - # Remove . and .. - delete($hTablespaceManifest->{'.'}); - delete($hTablespaceManifest->{'..'}); - - # Iterate file links - for my $strFile (sort(keys(%{$hTablespaceManifest}))) + if ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink()) { - # Make sure the link is in the expected manifest - my $hManifestTarget = - $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"}; + my $hTablespaceManifest = storageRepo()->manifest( + STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC); - if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK || - $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile) + # Remove . and .. + delete($hTablespaceManifest->{'.'}); + delete($hTablespaceManifest->{'..'}); + + # Iterate file links + for my $strFile (sort(keys(%{$hTablespaceManifest}))) { - confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id"); + # Make sure the link is in the expected manifest + my $hManifestTarget = + $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"}; + + if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK || + $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile) + { + confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id"); + } + + # Make sure the link really is a link + if ($hTablespaceManifest->{$strFile}{type} ne 'l') + { + confess &log(ERROR, "'${strFile}' in tablespace directory is not a link"); + } + + # Make sure the link destination is correct + my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}"; + + if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination) + { + confess &log(ERROR, + "'${strFile}' link should reference '${strLinkDestination}' but actually references " . + "'$hTablespaceManifest->{$strFile}{link_destination}'"); + } } - # Make sure the link really is a link - if ($hTablespaceManifest->{$strFile}{type} ne 'l') + # Iterate manifest targets + for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}}))) { - confess &log(ERROR, "'${strFile}' in tablespace directory is not a link"); - } + my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}; + my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID}; - # Make sure the link destination is correct - my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}"; - - if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination) - { - confess &log(ERROR, - "'${strFile}' link should reference '${strLinkDestination}' but actually references " . - "'$hTablespaceManifest->{$strFile}{link_destination}'"); + # Make sure the target exists as a link on disk + if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) && + !defined($hTablespaceManifest->{$strTablespaceId})) + { + confess &log(ERROR, + "target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'"); + } } } - - # Iterate manifest targets - for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}}))) + # Else there should not be a tablespace directory at all + elsif (storageRepo()->pathExists(STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)) { - my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}; - my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID}; - - # Make sure the target exists as a link on disk - if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) && - !defined($hTablespaceManifest->{$strTablespaceId})) - { - confess &log(ERROR, - "target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'"); - } + confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory'); } } - # Else there should not be a tablespace directory at all - elsif (storageRepo()->pathExists(STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)) - { - confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory'); - } # Check that latest link exists unless repo links are disabled my $strLatestLink = storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST); @@ -495,7 +498,8 @@ sub backupCompare my $lRepoSize = $oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ? $oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) : - (storageRepo()->info(STORAGE_REPO_BACKUP . "/${strBackup}/${strFileKey}" . ($bCompressed ? '.gz' : '')))->size; + (storageRepo()->info(STORAGE_REPO_BACKUP . + "/${strBackup}/${strFileKey}" . ($bCompressed ? '.gz' : '')))->{size}; if (defined($lRepoSize) && $lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE}) @@ -1882,7 +1886,7 @@ sub restoreCompare if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0) { - my $oStat = storageTest()->info($oActualManifest->dbPathGet($strSectionPath, $strName)); + my $oStat = lstat($oActualManifest->dbPathGet($strSectionPath, $strName)); # When performing a selective restore, the files for the database(s) that are not restored are still copied but as empty # sparse files (blocks == 0). If the file is not a sparse file or is a link, then get the actual checksum for comparison @@ -2049,8 +2053,8 @@ sub backupDestination {return shift->{strBackupDestination}} sub backrestExe {return testRunGet()->backrestExe()} sub bogusHost {return shift->{bBogusHost}} sub hardLink {return shift->{bHardLink}} -sub hasLink {storageRepo()->driver()->className() eq STORAGE_POSIX_DRIVER} -sub isFS {storageRepo()->driver()->className() ne STORAGE_S3_DRIVER} +sub hasLink {storageRepo()->capability(STORAGE_CAPABILITY_LINK)} +sub isFS {storageRepo()->type() ne STORAGE_S3} sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()} sub isHostDbMaster {return shift->nameGet() eq HOST_DB_MASTER} sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY} diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm index af49c3ba0..6bc8f0d57 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm @@ -224,7 +224,7 @@ sub manifestFileCreate my $strPathFile = $self->dbFileCreate($oManifestRef, $strTarget, $strFile, $strContent, $lTime, $strMode); # Stat the file - my $oStat = storageTest()->info($strPathFile); + my $oStat = lstat($strPathFile); ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); @@ -338,7 +338,7 @@ sub manifestLinkCreate my $strDbFile = $self->dbLinkCreate($oManifestRef, $strPath, $strFile, $strDestination); # Stat the link - my $oStat = storageTest()->info($strDbFile); + my $oStat = lstat($strDbFile); # Check for errors in stat if (!defined($oStat)) @@ -360,7 +360,7 @@ sub manifestLinkCreate (defined(dirname($strPath)) ? dirname($strPath) : '') . "/${strDestination}"; } - $oStat = storageTest()->info($strDestinationFile); + $oStat = lstat($strDestinationFile); my $strSection = MANIFEST_SECTION_TARGET_PATH; @@ -556,7 +556,7 @@ sub manifestTablespaceCreate # Load linked path into manifest my $strLinkPath = $self->tablespacePath($iOid); my $strTarget = MANIFEST_TARGET_PGTBLSPC . "/${iOid}"; - my $oStat = storageTest()->info($strLinkPath); + my $oStat = lstat($strLinkPath); ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); @@ -582,7 +582,7 @@ sub manifestTablespaceCreate } # Load tablespace path into manifest - $oStat = storageTest()->info($strTablespacePath); + $oStat = lstat($strTablespacePath); ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGTBLSPC} = ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}; @@ -599,7 +599,7 @@ sub manifestTablespaceCreate or confess "unable to link ${strLink} to ${strLinkPath}"; # Load link into the manifest - $oStat = storageTest()->info($strLink); + $oStat = lstat($strLink); my $strLinkTarget = MANIFEST_TARGET_PGDATA . "/${strTarget}"; ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); diff --git a/test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm b/test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm index da5908ca8..9c6015d5d 100644 --- a/test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm @@ -163,10 +163,9 @@ sub run MANIFEST_SUBKEY_CHECKSUM, $strPgControlHash)}, true, "manifest updated for pg_control"); # Neither backup.manifest nor backup.manifest.copy written because size threshold not met - $self->testException(sub {storageRepo()->openRead("$strBackupPath/" . FILE_MANIFEST . INI_COPY_EXT)}, ERROR_FILE_MISSING, - "unable to open '$strBackupPath/" . FILE_MANIFEST . INI_COPY_EXT . "': No such file or directory"); - $self->testException(sub {storageRepo()->openRead("$strBackupPath/" . FILE_MANIFEST)}, ERROR_FILE_MISSING, - "unable to open '$strBackupPath/" . FILE_MANIFEST . "': No such file or directory"); + $self->testResult(sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST)}, false, "backup.manifest missing"); + $self->testResult( + sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST . INI_COPY_EXT)}, false, "backup.manifest.copy missing"); #--------------------------------------------------------------------------------------------------------------------------- # No prior checksum, no compression, no page checksum, no extra, no delta, no hasReference @@ -180,8 +179,7 @@ sub run $lResultCopySize == $lFileSize && $lResultRepoSize == $lFileSize), true, 'file copied to repo successfully'); - $self->testException(sub {storageRepo()->openRead("$strFileRepo.gz")}, ERROR_FILE_MISSING, - "unable to open '$strFileRepo.gz': No such file or directory"); + $self->testResult(sub {storageRepo()->exists("${strFileRepo}.gz")}, false, "${strFileRepo}.gz missing"); ($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate( $oBackupManifest, @@ -212,18 +210,13 @@ sub run # Backup.manifest not written but backup.manifest.copy written because size threshold met $self->testResult(sub {storageTest()->exists("$strBackupPath/" . FILE_MANIFEST . INI_COPY_EXT)}, true, 'backup.manifest.copy exists in repo'); - $self->testException(sub {storageRepo()->openRead("$strBackupPath/" . FILE_MANIFEST)}, ERROR_FILE_MISSING, - "unable to open '$strBackupPath/" . FILE_MANIFEST . "': No such file or directory"); + $self->testResult( + sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST)}, false, 'backup.manifest.copy missing in repo'); storageTest()->remove($strFileRepo); storageTest()->remove($strPgControlRepo); #--------------------------------------------------------------------------------------------------------------------------- - # No prior checksum, yes compression, yes page checksum, no extra, no delta, no hasReference - $self->testException(sub {backupFile($strFileDb, $strRepoFile, $lFileSize, undef, true, - $strBackupLabel, true, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, false, false, undef)}, ERROR_ASSERT, - "iWalId is required in Backup::Filter::PageChecksum->new"); - # Build the lsn start parameter to pass to the extra function my $hStartLsnParam = { @@ -231,7 +224,6 @@ sub run iWalOffset => 0xFFFF, }; - #--------------------------------------------------------------------------------------------------------------------------- # No prior checksum, yes compression, yes page checksum, yes extra, no delta, no hasReference ($iResultCopyResult, $lResultCopySize, $lResultRepoSize, $strResultCopyChecksum, $rResultExtra) = backupFile($strFileDb, $strRepoFile, $lFileSize, undef, true, $strBackupLabel, true, @@ -243,8 +235,7 @@ sub run $lResultRepoSize == $lRepoFileCompressSize && $rResultExtra->{bValid}), true, 'file copied to repo successfully'); # Only the compressed version of the file exists - $self->testException(sub {storageRepo()->openRead("$strFileRepo")}, ERROR_FILE_MISSING, - "unable to open '$strFileRepo': No such file or directory"); + $self->testResult(sub {storageRepo()->exists("$strFileRepo")}, false, "only compressed version exists"); ($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate( $oBackupManifest, @@ -368,7 +359,7 @@ sub run # do not ignoreMissing $self->testException(sub {backupFile("$strFileDb.1", "$strRepoFile.1", $lFileSize, $strFileHash, false, $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, false, undef, true, false, undef)}, - ERROR_FILE_MISSING, "unable to open '$strFileDb.1': No such file or directory"); + ERROR_FILE_MISSING, "unable to open missing file '${strFileDb}.1' for read"); #--------------------------------------------------------------------------------------------------------------------------- # Restore the compressed file @@ -518,7 +509,7 @@ sub run $self->testException(sub {backupFile($strFileDb, $strRepoFile, $lFileSize, $strFileHash, false, $strBackupLabel, false, cfgOption(CFGOPT_COMPRESS_LEVEL), $lFileTime, true, undef, true, false, undef)}, - ERROR_FILE_MISSING, "unable to open '$strFileRepo': No such file or directory"); + ERROR_FILE_MISSING, "unable to open missing file '${strFileRepo}' for read"); } ################################################################################################################################ diff --git a/test/lib/pgBackRestTest/Module/Common/CommonHttpClientPerlTest.pm b/test/lib/pgBackRestTest/Module/Common/CommonHttpClientPerlTest.pm deleted file mode 100644 index a77b30649..000000000 --- a/test/lib/pgBackRestTest/Module/Common/CommonHttpClientPerlTest.pm +++ /dev/null @@ -1,173 +0,0 @@ -#################################################################################################################################### -# S3 Request Tests -#################################################################################################################################### -package pgBackRestTest::Module::Common::CommonHttpClientPerlTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - - use IO::Socket::SSL; -use POSIX qw(strftime); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Http::Client; -use pgBackRest::Common::Log; -use pgBackRest::Common::Wait; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# Port to use for testing -#################################################################################################################################### -use constant HTTPS_TEST_PORT => 9443; - -#################################################################################################################################### -# httpsServerResponse -#################################################################################################################################### -sub httpsServerResponse -{ - my $self = shift; - my $iResponseCode = shift; - my $strContent = shift; - - # Write header - $self->{oConnection}->write("HTTP/1.1 ${iResponseCode} GenericMessage\r\n"); - $self->{oConnection}->write(HTTP_HEADER_CONTENT_LENGTH . ': ' . (defined($strContent) ? length($strContent) : 0) . "\r\n"); - - # Write new line before content (even if there isn't any) - $self->{oConnection}->write("\r\n"); - - # Write content - if (defined($strContent)) - { - $self->{oConnection}->write($strContent); - } - - # This will block until the connection is closed by the client - $self->{oConnection}->read(); -} - -#################################################################################################################################### -# httpsServerAccept -#################################################################################################################################### -sub httpsServerAccept -{ - my $self = shift; - - # Wait for a connection - $self->{oConnection} = $self->{oSocketServer}->accept() - or confess "failed to accept or handshake $!, $SSL_ERROR"; - &log(INFO, " * socket server connected"); -} - -#################################################################################################################################### -# httpsServer -#################################################################################################################################### -sub httpsServer -{ - my $self = shift; - my $fnServer = shift; - - # Fork off the server - if (fork() == 0) - { - # Run server function - $fnServer->(); - - exit 0; - } -} - -#################################################################################################################################### -# Start the https testing server -#################################################################################################################################### -sub initModule -{ - my $self = shift; - - # Open the domain socket - $self->{oSocketServer} = IO::Socket::SSL->new( - LocalAddr => '127.0.0.1', LocalPort => HTTPS_TEST_PORT, Listen => 1, SSL_cert_file => CERT_FAKE_SERVER, - SSL_key_file => CERT_FAKE_SERVER_KEY) - or confess "unable to open https server for testing: $!"; - &log(INFO, " * socket server open"); -} - -#################################################################################################################################### -# Stop the https testing server -#################################################################################################################################### -sub cleanModule -{ - my $self = shift; - - # Shutdown server - $self->{oSocketServer}->close(); - &log(INFO, " * socket server closed"); -} - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - # Test variables - my $strTestHost = '127.0.0.1'; - my $strTestData = 'TESTDATA'; - - ################################################################################################################################ - if ($self->begin('content-length defined')) - { - $self->httpsServer(sub - { - $self->httpsServerAccept(); - $self->httpsServerResponse(200, $strTestData); - }); - - #--------------------------------------------------------------------------------------------------------------------------- - my $oHttpClient = $self->testResult( - sub {new pgBackRest::Common::Http::Client( - $strTestHost, HTTP_VERB_GET, {iPort => HTTPS_TEST_PORT, bVerifySsl => false})}, - '[object]', 'new http client'); - - $self->testResult(sub {${$oHttpClient->responseBody()}}, $strTestData, 'response body read'); - } - - ################################################################################################################################ - if ($self->begin('retry')) - { - $self->httpsServer(sub - { - $self->httpsServerAccept(); - $self->{oConnection}->write("HTTP/1.1 404 Error\r\nBogus-Header\r\n\r\n"); - - $self->httpsServerAccept(); - $self->{oConnection}->write("HTTP/1.1 404 Error\r\nBogus-Header\r\n\r\n"); - - $self->httpsServerAccept(); - $self->httpsServerResponse(200, $strTestData); - }); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {new pgBackRest::Common::Http::Client( - $strTestHost, HTTP_VERB_GET, {iPort => HTTPS_TEST_PORT, bVerifySsl => false, iTryTotal => 1})}, - ERROR_PROTOCOL, "http header 'Bogus-Header' requires colon separator"); - - $self->testResult( - sub {new pgBackRest::Common::Http::Client( - $strTestHost, HTTP_VERB_GET, {iPort => HTTPS_TEST_PORT, bVerifySsl => false, iTryTotal => 2})}, - '[object]', 'successful retries'); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Common/CommonIniPerlTest.pm b/test/lib/pgBackRestTest/Module/Common/CommonIniPerlTest.pm index b14a0f80f..f5e5a5fc3 100644 --- a/test/lib/pgBackRestTest/Module/Common/CommonIniPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Common/CommonIniPerlTest.pm @@ -297,42 +297,6 @@ sub run $self->testException(sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_CRYPTO, "unable to parse '$strTestFile'" . "\nHINT: Is or was the repo encrypted?"); - - # Encryption - #--------------------------------------------------------------------------------------------------------------------------- - executeTest("rm -rf ${strTestFile}*"); - - my $strCipherPass = 'x'; - my $strCipherPassSub = 'y'; - - # Unencrypted storage but a passphrase passed - $self->testException(sub {new pgBackRest::Common::Ini($strTestFile, {bLoad => false, - strCipherPass => $strCipherPass})}, ERROR_ASSERT, - "a user passphrase and sub passphrase are both required when encrypting"); - - # Unencrypted storage but a sub passphrase passed - $self->testException(sub {new pgBackRest::Common::Ini($strTestFile, {bLoad => false, - strCipherPassSub => $strCipherPassSub})}, ERROR_ASSERT, - "a user passphrase and sub passphrase are both required when encrypting"); - - # Create Encrypted storage - my $oStorage = new pgBackRest::Storage::Local($self->testPath(), new pgBackRest::Storage::Posix::Driver(), - {strCipherType => CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC, strCipherPassUser => $strCipherPass}); - - $self->testException(sub {new pgBackRest::Common::Ini($strTestFile, {oStorage => $oStorage})}, ERROR_CRYPTO, - "passphrase is required when storage is encrypted"); - - $self->testException(sub {new pgBackRest::Common::Ini($strTestFile, {bLoad => false, oStorage => $oStorage, - strCipherPass => $strCipherPass})}, ERROR_ASSERT, - "a user passphrase and sub passphrase are both required when encrypting"); - - $oIni = $self->testResult(sub { - new pgBackRest::Common::Ini( - $strTestFile, - {bLoad => false, oStorage => $oStorage, strCipherPass => $strCipherPass, strCipherPassSub => $strCipherPassSub})}, - '[object]', 'create new ini with encryption passphrases'); - $self->testResult(sub {($oIni->cipherPassSub() eq $strCipherPassSub) && - ($oIni->cipherPass() eq $strCipherPass)}, true, ' new ini has encryption passphrases'); } ################################################################################################################################ diff --git a/test/lib/pgBackRestTest/Module/Manifest/ManifestAllPerlTest.pm b/test/lib/pgBackRestTest/Module/Manifest/ManifestAllPerlTest.pm index 6d0a0b816..01effdc75 100644 --- a/test/lib/pgBackRestTest/Module/Manifest/ManifestAllPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Manifest/ManifestAllPerlTest.pm @@ -198,8 +198,8 @@ sub run # Attempt to save without proper path #--------------------------------------------------------------------------------------------------------------------------- - $self->testException(sub {$oManifest->save()}, ERROR_PATH_MISSING, - "unable to open '" . $strBackupManifestFile . "': No such file or directory"); + $self->testException(sub {$oManifest->save()}, ERROR_FILE_MISSING, + "unable to open file '${strBackupManifestFile}' for write in missing path"); # Create path and save #--------------------------------------------------------------------------------------------------------------------------- @@ -226,8 +226,8 @@ sub run # Build error if offline = true and no tablespace path #--------------------------------------------------------------------------------------------------------------------------- - $self->testException(sub {$oManifest->build(storageDb(), $self->{strDbPath}, undef, false, false)}, ERROR_FILE_MISSING, - "unable to stat '" . $self->{strDbPath} . "/" . MANIFEST_TARGET_PGTBLSPC . "': No such file or directory"); + $self->testException(sub {$oManifest->build(storageDb(), $self->{strDbPath}, undef, false, false)}, ERROR_PATH_MISSING, + "unable to list file info for missing path '" . $self->{strDbPath} . "/" . MANIFEST_TARGET_PGTBLSPC . "'"); # bOnline = true tests #--------------------------------------------------------------------------------------------------------------------------- @@ -236,7 +236,11 @@ sub run {bLoad => false, strDbVersion => PG_VERSION_94, iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94)}); # Create expected manifest from base + my $oStorageTemp = $oManifestBase->{oStorage}; + $oManifestBase->{oStorage} = undef; my $oManifestExpected = dclone($oManifestBase); + $oManifestBase->{oStorage} = $oStorageTemp; + $oManifestExpected->{oStorage} = $oStorageTemp; # Add global/pg_control file and PG_VERSION file and create a directory with a different modes than default storageDb()->put(storageDb()->openWrite($self->{strDbPath} . '/' . DB_FILE_PGCONTROL, @@ -523,7 +527,11 @@ sub run # Unskip code path coverage #--------------------------------------------------------------------------------------------------------------------------- + $oStorageTemp = $oManifestExpected->{oStorage}; + $oManifestExpected->{oStorage} = undef; my $oManifestExpectedUnskip = dclone($oManifestExpected); + $oManifestExpected->{oStorage} = $oStorageTemp; + $oManifestExpectedUnskip->{oStorage} = $oStorageTemp; # Change DB version to 93 $oManifest = new pgBackRest::Manifest( @@ -954,7 +962,7 @@ sub run storageDb()->remove('postgresql.auto.conf'); storageDb()->remove('hosts'); storageDb()->remove('pg_log/logfile'); - storageDb()->remove('global/exclude', {bRecurse => true}); + storageDb()->pathRemove('global/exclude', {bRecurse => true}); # Reload the manifest with version < 9.0 #--------------------------------------------------------------------------------------------------------------------------- @@ -1229,7 +1237,11 @@ sub run {bLoad => false, strDbVersion => PG_VERSION_94, iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94)}); # Create expected manifest from base + my $oStorageTemp = $oManifestBase->{oStorage}; + $oManifestBase->{oStorage} = undef; my $oManifestExpected = dclone($oManifestBase); + $oManifestBase->{oStorage} = $oStorageTemp; + $oManifestExpected->{oStorage} = $oStorageTemp; # Future timestamp on file #--------------------------------------------------------------------------------------------------------------------------- @@ -1255,7 +1267,11 @@ sub run # Future timestamp in last manifest #--------------------------------------------------------------------------------------------------------------------------- + $oStorageTemp = $oManifestExpected->{oStorage}; + $oManifestExpected->{oStorage} = undef; my $oLastManifest = dclone($oManifestExpected); + $oManifestExpected->{oStorage} = $oStorageTemp; + $oLastManifest->{oStorage} = $oStorageTemp; # Set a backup label $oLastManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, BOGUS); @@ -1507,7 +1523,12 @@ sub run { my $oManifest = new pgBackRest::Manifest($strBackupManifestFile, {bLoad => false, strDbVersion => PG_VERSION_94, iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94)}); + + my $oStorageTemp = $oManifestBase->{oStorage}; + $oManifestBase->{oStorage} = undef; my $oManifestExpected = dclone($oManifestBase); + $oManifestBase->{oStorage} = $oStorageTemp; + $oManifestExpected->{oStorage} = $oStorageTemp; # Add a bogus file - all traces to be removed after the manifest has been built to simulate an inital manifest and avoid # missing files error diff --git a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm index 62674732e..255500e8e 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm @@ -878,7 +878,7 @@ sub run $oHostBackup->backup( $strType, '$PGDATA is a substring of valid tblspc excluding / (file missing err expected)', - {oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_FILE_MISSING}); + {oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_PATH_MISSING}); testFileRemove("${strTblSpcPath}/99999"); } diff --git a/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm index f954c5161..5f5b906df 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm @@ -73,14 +73,12 @@ sub run true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true, bS3 => $bS3, bRepoEncrypt => $bEncrypt}); - my $oStorage = storageRepo(); - # Create compression extension my $strCompressExt = $bCompress ? qw{.} . COMPRESS_EXT : ''; # Create the wal path my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog'; - $oStorage->pathCreate($strWalPath, {bCreateParent => true}); + storageTest()->pathCreate($strWalPath, {bCreateParent => true}); # Create the test path for pg_control and generate pg_control for stanza-create storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true}); @@ -97,7 +95,7 @@ sub run if ($iError == 0) { $oHostBackup->infoMunge( - $oStorage->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), + storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), {&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}, &INFO_ARCHIVE_SECTION_DB_HISTORY => {1 => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}}); } @@ -121,12 +119,12 @@ sub run # Fix the database version if ($iError == 0) { - $oHostBackup->infoRestore($oStorage->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)); + $oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)); } #--------------------------------------------------------------------------------------------------------------------------- $self->testResult( - sub {$oStorage->list( + sub {storageRepo()->list( STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')}, "000000010000000100000001-${strWalHash}${strCompressExt}", 'segment 2-4 not pushed', {iWaitSeconds => 5}); @@ -135,7 +133,7 @@ sub run $oHostDbMaster->archivePush($strWalPath, $strWalTestFile, 5); $self->testResult( - sub {$oStorage->list( + sub {storageRepo()->list( STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')}, "(000000010000000100000001-${strWalHash}${strCompressExt}, " . "000000010000000100000005-${strWalHash}${strCompressExt})", diff --git a/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm index ae950a2f1..79a3fa1a4 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm @@ -25,7 +25,6 @@ use pgBackRest::InfoCommon; use pgBackRest::Manifest; use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::Gzip; use pgBackRest::Storage::Helper; use pgBackRestTest::Env::HostEnvTest; @@ -198,7 +197,7 @@ sub run storageRepo()->copy( storageRepo()->openRead( STORAGE_REPO_ARCHIVE . "/${strArchiveTest}.gz", - {rhyFilter => [{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}]}), + {rhyFilter => [{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]}]}), STORAGE_REPO_ARCHIVE . "/${strArchiveTest}"); $oHostBackup->stanzaCreate('force create archive.info from uncompressed file', diff --git a/test/lib/pgBackRestTest/Module/Performance/PerformanceIoTest.pm b/test/lib/pgBackRestTest/Module/Performance/PerformanceIoTest.pm deleted file mode 100644 index 65b8d4145..000000000 --- a/test/lib/pgBackRestTest/Module/Performance/PerformanceIoTest.pm +++ /dev/null @@ -1,127 +0,0 @@ -#################################################################################################################################### -# I/O Performance Tests -#################################################################################################################################### -package pgBackRestTest::Module::Performance::PerformanceIoTest; - use parent 'pgBackRestTest::Env::ConfigEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Storable qw(dclone); -use Time::HiRes qw(gettimeofday); - -use pgBackRest::Common::Log; -use pgBackRest::Config::Config; -use pgBackRest::Protocol::Helper; -use pgBackRest::Protocol::Storage::Helper; -use pgBackRest::Storage::Filter::Gzip; -use pgBackRest::Storage::Filter::Sha; -use pgBackRest::Storage::Helper; - -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# initModule -#################################################################################################################################### -sub initModule -{ - my $self = shift; - - # Load reference page data - my $tPageBin = ${storageTest()->get($self->dataPath() . '/filecopy.table.bin')}; - - # Create large test file - $self->{iTableLargeSize} = 32; - $self->{strTableLargeFile} = 'table-large.bin'; - - my $oFileWrite = storageTest()->openWrite($self->{strTableLargeFile}); - - for (my $iIndex = 0; $iIndex < $self->{iTableLargeSize}; $iIndex++) - { - $oFileWrite->write(\$tPageBin); - } - - $oFileWrite->close(); -} - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - ################################################################################################################################ - if ($self->begin("copy")) - { - # Setup the remote for testing remote storage - $self->optionTestSet(CFGOPT_STANZA, $self->stanza()); - $self->optionTestSet(CFGOPT_PG_PATH, $self->testPath()); - $self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath()); - $self->optionTestSet(CFGOPT_LOG_PATH, $self->testPath()); - $self->optionTestSet(CFGOPT_REPO_HOST, 'localhost'); - $self->optionTestSet(CFGOPT_REPO_HOST_USER, $self->backrestUser()); - $self->configTestLoad(CFGCMD_RESTORE); - - protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP, undef, {strBackRestBin => $self->backrestExe()}); - storageRepo(); - - # Setup file info - my $strFile = $self->{strTableLargeFile}; - my $strFileCopy = "${strFile}.copy"; - - my $iRunTotal = 1; - &log(INFO, "time is average of ${iRunTotal} run(s)"); - - foreach my $bGzip (false, true) - { - foreach my $bRemote (false, true) - { - my $rhyFilter; - - push(@{$rhyFilter}, {strClass => STORAGE_FILTER_SHA}); - push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{iLevel => 6}]}) if ($bGzip); - - my $lTimeTotal = 0; - my $lTimeBegin; - - for (my $iIndex = 0; $iIndex < $iRunTotal; $iIndex++) - { - # Get the remote or local for writing - my $oStorageWrite = $bRemote ? storageRepo() : storageTest(); - - # Start the timer - $lTimeBegin = gettimeofday(); - - # Copy the file - storageTest()->copy( - storageTest()->openRead($strFile, {rhyFilter => $rhyFilter}), - $oStorageWrite->openWrite($strFileCopy)); - - # Record time - $lTimeTotal += gettimeofday() - $lTimeBegin; - - # Remove file so it can be copied again - executeTest("sudo rm " . $oStorageWrite->pathGet($strFileCopy)); - } - - # Calculate out output metrics - my $fExecutionTime = int($lTimeTotal * 1000 / $iRunTotal) / 1000; - my $fGbPerHour = int((60 * 60) * 1000 / ((1024 / $self->{iTableLargeSize}) * $fExecutionTime)) / 1000; - - &log(INFO, "sha1 1, gz ${bGzip}, rmt ${bRemote}: ${fExecutionTime}s, ${fGbPerHour} GB/hr"); - } - } - - # Destroy protocol object - protocolDestroy(); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm index 1756372dd..57602e1c2 100644 --- a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm @@ -36,6 +36,8 @@ use pgBackRestTest::Env::Host::HostBaseTest; use pgBackRestTest::Env::Host::HostBackupTest; use pgBackRestTest::Env::Host::HostDbTest; use pgBackRestTest::Env::HostEnvTest; +use pgBackRestTest::Common::Storage; +use pgBackRestTest::Common::StoragePosix; #################################################################################################################################### # run @@ -270,7 +272,7 @@ sub run executeTest("sudo chmod 400 ${strDir}"); $strComment = 'confirm master manifest->build executed'; - $oHostDbMaster->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_OPEN}); + $oHostDbMaster->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN}); executeTest("sudo rmdir ${strDir}"); # Providing a sufficient archive-timeout, verify that the check command runs successfully now with valid @@ -503,10 +505,10 @@ sub run my $strComment = 'confirm standby manifest->build executed'; # If there is an invalid host, the final error returned from check will be the inability to resolve the name which is - # a read error instead of an open error + # an open error instead of a read error if (!$oHostDbStandby->bogusHost()) { - $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_OPEN}); + $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN}); } else { @@ -746,13 +748,19 @@ sub run { my ($strSHA1, $lSize) = storageTest()->hashSize($strDb1TablePath); - # Create a zeroed sparse file in the test directory that is the same size as the filenode.map + # Create a zeroed sparse file in the test directory that is the same size as the filenode.map. We need to use the + # posix driver directly to do this because handles cannot be passed back from the C code. + my $oStorageTrunc = new pgBackRestTest::Common::Storage($self->testPath(), new pgBackRestTest::Common::StoragePosix()); + my $strTestTable = $self->testPath() . "/testtable"; - my $oDestinationFileIo = storageTest()->openWrite($strTestTable); + my $oDestinationFileIo = $oStorageTrunc->openWrite($strTestTable); $oDestinationFileIo->open(); # Truncate to the original size which will create a sparse file. - truncate($oDestinationFileIo->handle(), $lSize); + if (!truncate($oDestinationFileIo->handle(), $lSize)) + { + confess "unable to truncate '$strTestTable' with handle " . $oDestinationFileIo->handle(); + } $oDestinationFileIo->close(); # Confirm the test filenode.map and the database test1 filenode.map are zeroed diff --git a/test/lib/pgBackRestTest/Module/Stanza/StanzaAllPerlTest.pm b/test/lib/pgBackRestTest/Module/Stanza/StanzaAllPerlTest.pm index b17e035de..2a64b7226 100644 --- a/test/lib/pgBackRestTest/Module/Stanza/StanzaAllPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Stanza/StanzaAllPerlTest.pm @@ -280,7 +280,7 @@ sub run # Change the permissions on the archived file so reconstruction fails executeTest('sudo chmod 220 ' . $strArchivedFile); $self->testException(sub {(new pgBackRest::Stanza())->stanzaCreate()}, ERROR_FILE_OPEN, - "unable to open '" . $strArchivedFile . "': Permission denied"); + "unable to open file '${strArchivedFile}' for read"); executeTest('sudo chmod 644 ' . $strArchivedFile); # Clear the cached repo settings and change repo settings to encrypted @@ -312,7 +312,10 @@ sub run #--------------------------------------------------------------------------------------------------------------------------- storageRepo()->pathCreate(STORAGE_REPO_ARCHIVE . "/" . PG_VERSION_94 . "-1"); storageRepo()->pathCreate(STORAGE_REPO_ARCHIVE . "/" . PG_VERSION_94 . "-1/0000000100000001"); - storageRepo()->put($strArchivedFile, $tUnencryptedArchiveContent); + storageRepo()->put( + storageRepo()->openWrite( + $strArchivedFile, {strCipherPass => new pgBackRest::Archive::Info($self->{strArchivePath})->cipherPassSub()}), + $tUnencryptedArchiveContent); storageRepo()->pathCreate($strBackupPath); # Empty backup path - no backup in progress # Confirm encrypted and create the stanza with force @@ -477,7 +480,7 @@ sub run #--------------------------------------------------------------------------------------------------------------------------- executeTest('sudo chmod 220 ' . $self->{strArchivePath}); $self->testException(sub {$oStanza->infoObject(STORAGE_REPO_ARCHIVE, $self->{strArchivePath})}, ERROR_FILE_OPEN, - "unable to open '" . $self->{strArchivePath} . "/archive.info': Permission denied"); + "unable to open file '" . $self->{strArchivePath} . "/archive.info' for read"); executeTest('sudo chmod 640 ' . $self->{strArchivePath}); # Reset force option -------- @@ -492,8 +495,8 @@ sub run forceStorageRemove(storageRepo(), storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT)); executeTest('sudo chmod 220 ' . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)); $self->testException(sub {$oStanza->infoObject(STORAGE_REPO_BACKUP, $self->{strBackupPath})}, ERROR_FILE_OPEN, - "unable to open '" . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO) . - "': Permission denied"); + "unable to open file '" . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO) . + "' for read"); executeTest('sudo chmod 640 ' . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)); } @@ -762,8 +765,8 @@ sub run executeTest("sudo chown 7777 " . $self->{strArchivePath}); lockStop(); - $self->testException(sub {$oStanza->stanzaDelete()}, ERROR_FILE_OPEN, - "unable to remove file '" . $self->{strArchivePath} . "/" . ARCHIVE_INFO_FILE . "': Permission denied"); + $self->testException(sub {$oStanza->stanzaDelete()}, ERROR_FILE_REMOVE, + "unable to remove '" . $self->{strArchivePath} . "/" . ARCHIVE_INFO_FILE . "'"); # Remove the repo executeTest("sudo rm -rf " . $self->{strArchivePath}); diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockPerlTest.pm deleted file mode 100644 index 169589c98..000000000 --- a/test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockPerlTest.pm +++ /dev/null @@ -1,285 +0,0 @@ -#################################################################################################################################### -# Tests for Block Cipher -#################################################################################################################################### -package pgBackRestTest::Module::Storage::StorageFilterCipherBlockPerlTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Fcntl qw(O_RDONLY); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::LibC qw(:random :crypto); -use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::CipherBlock; -use pgBackRest::Storage::Posix::Driver; - -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - # Test data - my $strFile = $self->testPath() . qw{/} . 'file.txt'; - my $strFileEncrypt = $self->testPath() . qw{/} . 'file.enc.txt'; - my $strFileDecrypt = $self->testPath() . qw{/} . 'file.dcr.txt'; - my $strFileBin = $self->testPath() . qw{/} . 'file.bin'; - my $strFileBinEncrypt = $self->testPath() . qw{/} . 'file.enc.bin'; - my $strFileContent = 'TESTDATA'; - my $iFileLength = length($strFileContent); - my $oDriver = new pgBackRest::Storage::Posix::Driver(); - my $tCipherPass = 'areallybadkey'; - my $strCipherType = 'aes-256-cbc'; - my $tContent; - - ################################################################################################################################ - if ($self->begin('new()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - # Create an unencrypted file - executeTest("echo -n '${strFileContent}' | tee ${strFile}"); - - $self->testException( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openRead($strFile), $strCipherType, $tCipherPass, {strMode => BOGUS})}, - ERROR_ASSERT, 'unknown cipher mode: ' . BOGUS); - - $self->testException( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openRead($strFile), BOGUS, $tCipherPass)}, - ERROR_ASSERT, "invalid cipher name '" . BOGUS . "'"); - - $self->testException( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openWrite($strFile), $strCipherType, $tCipherPass, {strMode => BOGUS})}, - ERROR_ASSERT, 'unknown cipher mode: ' . BOGUS); - - $self->testException( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openWrite($strFile), BOGUS, $tCipherPass)}, - ERROR_ASSERT, "invalid cipher name '" . BOGUS . "'"); - } - - ################################################################################################################################ - if ($self->begin('read() and write()')) - { - my $tBuffer; - - #--------------------------------------------------------------------------------------------------------------------------- - # Create an plaintext file - executeTest("echo -n '${strFileContent}' | tee ${strFile}"); - - # Instantiate the cipher object - default action encrypt - my $oEncryptIo = $self->testResult(sub {new pgBackRest::Storage::Filter::CipherBlock($oDriver->openRead($strFile), - $strCipherType, $tCipherPass)}, '[object]', 'new encrypt file'); - - $self->testResult(sub {$oEncryptIo->read(\$tBuffer, 2)}, 16, ' read 16 bytes (header)'); - $self->testResult(sub {$oEncryptIo->read(\$tBuffer, 2)}, 16, ' read 16 bytes (data)'); - $self->testResult(sub {$oEncryptIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes'); - - $self->testResult(sub {$tBuffer ne $strFileContent}, true, ' data read is encrypted'); - - $self->testResult(sub {$oEncryptIo->close()}, true, ' close'); - $self->testResult(sub {$oEncryptIo->close()}, false, ' close again'); - - # tBuffer is now encrypted - test write decrypts correctly - my $oDecryptFileIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock($oDriver->openWrite($strFileDecrypt), - $strCipherType, $tCipherPass, {strMode => STORAGE_DECRYPT})}, - '[object]', ' new decrypt file'); - - $self->testResult(sub {$oDecryptFileIo->write(\$tBuffer)}, 32, ' write decrypted'); - $self->testResult(sub {$oDecryptFileIo->close()}, true, ' close'); - - $self->testResult(sub {${$self->storageTest()->get($strFileDecrypt)}}, $strFileContent, ' data written is decrypted'); - - #--------------------------------------------------------------------------------------------------------------------------- - $tBuffer = $strFileContent; - my $oEncryptFileIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock($oDriver->openWrite($strFileEncrypt), - $strCipherType, $tCipherPass)}, - '[object]', 'new write encrypt'); - - $tContent = ''; - $self->testResult(sub {$oEncryptFileIo->write(\$tContent)}, 0, ' attempt empty buffer write'); - - undef($tContent); - $self->testException( - sub {$oEncryptFileIo->write(\$tContent)}, ERROR_FILE_WRITE, - "unable to write to '${strFileEncrypt}': Use of uninitialized value"); - - # Encrypted length is not known so use tBuffer then test that tBuffer was encrypted - my $iWritten = $self->testResult(sub {$oEncryptFileIo->write(\$tBuffer)}, length($tBuffer), ' write encrypted'); - $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); - - $tContent = $self->storageTest()->get($strFileDecrypt); - $self->testResult(sub {defined($tContent) && $tContent ne $strFileContent}, true, ' data written is encrypted'); - - #--------------------------------------------------------------------------------------------------------------------------- - undef($tBuffer); - # Open encrypted file for decrypting - $oEncryptFileIo = - $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass, - {strMode => STORAGE_DECRYPT})}, - '[object]', 'new read encrypted file, decrypt'); - - # Try to read more than the length of the data expected to be output from the decrypt and confirm the decrypted length is - # the same as the original decrypted content. - $self->testResult(sub {$oEncryptFileIo->read(\$tBuffer, $iFileLength+4)}, $iFileLength, ' read all bytes'); - - # Just because length is the same does not mean content is so confirm - $self->testResult($tBuffer, $strFileContent, ' data read is decrypted'); - $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); - - #--------------------------------------------------------------------------------------------------------------------------- - undef($tContent); - undef($tBuffer); - my $strFileBinHash = '1c7e00fd09b9dd11fc2966590b3e3274645dd031'; - - executeTest('cp ' . $self->dataPath() . "/filecopy.archive2.bin ${strFileBin}"); - $self->testResult( - sub {cryptoHashOne('sha1', ${storageTest()->get($strFileBin)})}, $strFileBinHash, 'bin test - check sha1'); - - $tContent = ${storageTest()->get($strFileBin)}; - - $oEncryptFileIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openWrite($strFileBinEncrypt), $strCipherType, $tCipherPass)}, - '[object]', ' new write encrypt'); - - $self->testResult(sub {$oEncryptFileIo->write(\$tContent)}, length($tContent), ' write encrypted'); - $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); - $self->testResult( - sub {cryptoHashOne('sha1', ${storageTest()->get($strFileBinEncrypt)}) ne $strFileBinHash}, true, - ' check sha1 different'); - - my $oEncryptBinFileIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openRead($strFileBinEncrypt), $strCipherType, $tCipherPass, - {strMode => STORAGE_DECRYPT})}, - '[object]', 'new read encrypted bin file'); - - $self->testResult(sub {$oEncryptBinFileIo->read(\$tBuffer, 16777216)}, 16777216, ' read 16777216 bytes'); - $self->testResult(sub {cryptoHashOne('sha1', $tBuffer)}, $strFileBinHash, ' check sha1 same as original'); - $self->testResult(sub {$oEncryptBinFileIo->close()}, true, ' close'); - - # Try to read the file with the wrong passphrase - undef($tBuffer); - undef($oEncryptBinFileIo); - - $oEncryptBinFileIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openRead($strFileBinEncrypt), $strCipherType, BOGUS, - {strMode => STORAGE_DECRYPT})}, - '[object]', 'new read Encrypted bin file with wrong passphrase'); - - $self->testResult(sub {$oEncryptBinFileIo->read(\$tBuffer, 16777216)}, 16777216, ' read all bytes'); - $self->testResult(sub {cryptoHashOne('sha1', $tBuffer) ne $strFileBinHash}, true, ' check sha1 NOT same as original'); - - # Test file against openssl to make sure they are compatible - #--------------------------------------------------------------------------------------------------------------------------- - undef($tBuffer); - - $self->storageTest()->put($strFile, $strFileContent); - - executeTest( - "openssl enc -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFile} -out ${strFileEncrypt}"); - - $oEncryptFileIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass, - {strMode => STORAGE_DECRYPT})}, - '[object]', 'read file encrypted by openssl'); - - $self->testResult(sub {$oEncryptFileIo->read(\$tBuffer, 16)}, 8, ' read 8 bytes'); - $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); - $self->testResult(sub {$tBuffer}, $strFileContent, ' check content same as original'); - - $self->storageTest()->remove($strFile); - $self->storageTest()->remove($strFileEncrypt); - - $oEncryptFileIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openWrite($strFileEncrypt), $strCipherType, $tCipherPass)}, - '[object]', 'write file to be read by openssl'); - - $self->testResult(sub {$oEncryptFileIo->write(\$tBuffer)}, 8, ' write 8 bytes'); - $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); - - executeTest( - "openssl enc -d -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFileEncrypt} -out ${strFile}"); - - $self->testResult(sub {${$self->storageTest()->get($strFile)}}, $strFileContent, ' check content same as original'); - - # Test empty file against openssl to make sure they are compatible - #--------------------------------------------------------------------------------------------------------------------------- - $tBuffer = ''; - - $self->storageTest()->put($strFile); - - executeTest( - "openssl enc -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFile} -out ${strFileEncrypt}"); - - $oEncryptFileIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass, - {strMode => STORAGE_DECRYPT})}, - '[object]', 'read empty file encrypted by openssl'); - - $self->testResult(sub {$oEncryptFileIo->read(\$tBuffer, 16)}, 0, ' read 0 bytes'); - $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); - $self->testResult(sub {$tBuffer}, '', ' check content same as original'); - - $self->storageTest()->remove($strFile); - $self->storageTest()->remove($strFileEncrypt); - - $oEncryptFileIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openWrite($strFileEncrypt), $strCipherType, $tCipherPass)}, - '[object]', 'write file to be read by openssl'); - - $self->testResult(sub {$oEncryptFileIo->write(\$tBuffer)}, 0, ' write 0 bytes'); - $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); - - executeTest( - "openssl enc -d -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFileEncrypt} -out ${strFile}"); - - $self->testResult(sub {${$self->storageTest()->get($strFile)}}, undef, ' check content same as original'); - - # Error on empty file decrypt - an empty file that has been encrypted will be 32 bytes - #--------------------------------------------------------------------------------------------------------------------------- - undef($tBuffer); - $self->storageTest()->put($strFileEncrypt); - - $oEncryptFileIo = - $self->testResult( - sub {new pgBackRest::Storage::Filter::CipherBlock( - $oDriver->openRead($strFileEncrypt), $strCipherType, $tCipherPass, - {strMode => STORAGE_DECRYPT})}, - '[object]', 'new read empty attempt decrypt'); - - $self->testException(sub {$oEncryptFileIo->read(\$tBuffer, 16)}, ERROR_CRYPTO, 'cipher header missing'); - $self->testResult(sub {$oEncryptFileIo->close()}, true, 'close'); - - # OpenSSL should error on the empty file - executeTest( - "openssl enc -d -k ${tCipherPass} -md sha1 -aes-256-cbc -in ${strFileEncrypt} -out ${strFile}", - {iExpectedExitStatus => 1}); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipPerlTest.pm deleted file mode 100644 index ab1711110..000000000 --- a/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipPerlTest.pm +++ /dev/null @@ -1,222 +0,0 @@ -#################################################################################################################################### -# Tests for Storage::Filter::Gzip module -#################################################################################################################################### -package pgBackRestTest::Module::Storage::StorageFilterGzipPerlTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Compress::Raw::Zlib qw(Z_OK Z_BUF_ERROR Z_DATA_ERROR); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::LibC qw(:crypto); -use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::Gzip; -use pgBackRest::Storage::Posix::Driver; - -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - # Test data - my $strFile = $self->testPath() . qw{/} . 'file.txt'; - my $strFileGz = "${strFile}.gz"; - my $strFileContent = 'TESTDATA'; - my $iFileLength = length($strFileContent); - my $oDriver = new pgBackRest::Storage::Posix::Driver(); - - ################################################################################################################################ - if ($self->begin('errorCheck()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - my $oGzipIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openWrite($strFileGz))}, '[object]', 'new write'); - - $oGzipIo->{bWrite} = true; - $self->testException(sub {$oGzipIo->errorCheck(Z_DATA_ERROR)}, ERROR_FILE_WRITE, "unable to deflate '${strFileGz}'"); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {$oGzipIo->errorCheck(Z_OK)}, Z_OK, 'Z_OK'); - $self->testResult(sub {$oGzipIo->errorCheck(Z_BUF_ERROR)}, Z_OK, 'Z_BUF_ERROR'); - - #--------------------------------------------------------------------------------------------------------------------------- - $oGzipIo->{bWrite} = false; - $oGzipIo->{strCompressType} = STORAGE_DECOMPRESS; - $self->testException(sub {$oGzipIo->errorCheck(Z_DATA_ERROR)}, ERROR_FILE_READ, "unable to inflate '${strFileGz}'"); - } - - ################################################################################################################################ - if ($self->begin('write()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - my $oGzipIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openWrite($strFileGz), {lCompressBufferMax => 4})}, - '[object]', 'new write compress'); - - my $tBuffer = substr($strFileContent, 0, 2); - $self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes'); - $tBuffer = substr($strFileContent, 2, 2); - $self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes'); - $tBuffer = substr($strFileContent, 4, 2); - $self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes'); - $tBuffer = substr($strFileContent, 6, 2); - $self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes'); - $tBuffer = ''; - $self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 0, ' write 0 bytes'); - - $self->testResult(sub {$oGzipIo->close()}, true, ' close'); - - executeTest("gzip -d ${strFileGz}"); - $self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content'); - - #--------------------------------------------------------------------------------------------------------------------------- - executeTest("gzip ${strFile}"); - my $tFile = ${storageTest()->get($strFileGz)}; - - $oGzipIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Gzip( - $oDriver->openWrite($strFile), {strCompressType => STORAGE_DECOMPRESS})}, '[object]', 'new write decompress'); - - $tBuffer = substr($tFile, 0, 10); - $self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 10, ' write bytes'); - $tBuffer = substr($tFile, 10); - $self->testResult(sub {$oGzipIo->write(\$tBuffer)}, length($tFile) - 10, ' write bytes'); - $self->testResult(sub {$oGzipIo->close()}, true, ' close'); - - $self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content'); - - } - - ################################################################################################################################ - if ($self->begin('read()')) - { - my $tBuffer; - - #--------------------------------------------------------------------------------------------------------------------------- - my $oGzipIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openWrite($strFileGz), {bWantGzip => false, iLevel => 3})}, - '[object]', 'new write compress'); - $self->testResult($oGzipIo->{iLevel}, 3, ' check level'); - $self->testResult(sub {$oGzipIo->write(\$strFileContent, $iFileLength)}, $iFileLength, ' write'); - $self->testResult(sub {$oGzipIo->close()}, true, ' close'); - - #--------------------------------------------------------------------------------------------------------------------------- - $oGzipIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Gzip( - $oDriver->openRead($strFileGz), {bWantGzip => false, strCompressType => STORAGE_DECOMPRESS})}, - '[object]', 'new read decompress'); - - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 4)}, 4, ' read 4 bytes'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 2, ' read 2 bytes'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 2, ' read 2 bytes'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes'); - - $self->testResult(sub {$oGzipIo->close()}, true, ' close'); - $self->testResult(sub {$oGzipIo->close()}, false, ' close again'); - $self->testResult($tBuffer, $strFileContent, ' check content'); - - storageTest()->remove($strFileGz); - - #--------------------------------------------------------------------------------------------------------------------------- - $tBuffer = 'AA'; - storageTest()->put($strFile, $strFileContent); - - $oGzipIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFile))}, - '[object]', 'new read compress'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 10, ' read 10 bytes (request 2)'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 18, ' read 18 bytes (request 2)'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes (request 2)'); - $self->testResult(sub {$oGzipIo->close()}, true, ' close'); - - $self->testResult(sub {storageTest()->put($strFileGz, substr($tBuffer, 2))}, 28, ' put content'); - executeTest("gzip -df ${strFileGz}"); - $self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content'); - - #--------------------------------------------------------------------------------------------------------------------------- - $tBuffer = undef; - - executeTest('cat ' . $self->dataPath() . "/filecopy.archive2.bin | gzip -c > ${strFileGz}"); - - $oGzipIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Gzip( - $oDriver->openRead($strFileGz), {lCompressBufferMax => 4096, strCompressType => STORAGE_DECOMPRESS})}, - '[object]', 'new read decompress'); - - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 8388608)}, 8388608, ' read 8388608 bytes'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 1)}, 0, ' read 0 bytes'); - - $self->testResult(sub {$oGzipIo->close()}, true, ' close'); - $self->testResult(cryptoHashOne('sha1', $tBuffer), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content'); - - storageTest()->remove($strFileGz); - - #--------------------------------------------------------------------------------------------------------------------------- - $tBuffer = undef; - - executeTest('cp ' . $self->dataPath() . "/filecopy.archive2.bin ${strFile}"); - - $oGzipIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFile), {strCompressType => STORAGE_COMPRESS})}, - '[object]', 'new read compress'); - - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000) > 0}, true, ' read bytes'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000) > 0}, true, ' read bytes'); - $self->testResult(sub {$oGzipIo->close()}, true, ' close'); - - $self->testResult(sub {storageTest()->put($strFileGz, $tBuffer) > 0}, true, ' put content'); - executeTest("gzip -df ${strFileGz}"); - $self->testResult( - sub {cryptoHashOne('sha1', ${storageTest()->get($strFile)})}, '1c7e00fd09b9dd11fc2966590b3e3274645dd031', - ' check content'); - - #--------------------------------------------------------------------------------------------------------------------------- - $tBuffer = undef; - - my $oFile = $self->testResult( - sub {storageTest()->openWrite($strFile)}, '[object]', 'open file to extend during compression'); - - $oGzipIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFile), {lCompressBufferMax => 4194304})}, - '[object]', ' new read compress'); - - $self->testResult(sub {$oFile->write(\$strFileContent)}, length($strFileContent), ' write first block'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000) > 0}, true, ' read compressed first block (compression done)'); - - $self->testResult(sub {$oFile->write(\$strFileContent)}, length($strFileContent), ' write second block'); - $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000)}, 0, ' read compressed = 0'); - - $self->testResult(sub {storageTest()->put($strFileGz, $tBuffer) > 0}, true, ' put content'); - executeTest("gzip -df ${strFileGz}"); - $self->testResult( - sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content'); - - #--------------------------------------------------------------------------------------------------------------------------- - storageTest()->put($strFileGz, $strFileContent); - - $oGzipIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFileGz), {strCompressType => STORAGE_DECOMPRESS})}, - '[object]', 'new read decompress'); - - $self->testException( - sub {$oGzipIo->read(\$tBuffer, 1)}, ERROR_FILE_READ, "unable to inflate '${strFileGz}': incorrect header check"); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageFilterShaPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageFilterShaPerlTest.pm deleted file mode 100644 index c4a07aee5..000000000 --- a/test/lib/pgBackRestTest/Module/Storage/StorageFilterShaPerlTest.pm +++ /dev/null @@ -1,107 +0,0 @@ -#################################################################################################################################### -# Tests for StorageFilterSha module -#################################################################################################################################### -package pgBackRestTest::Module::Storage::StorageFilterShaPerlTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::LibC qw(:crypto); -use pgBackRest::Storage::Base; -use pgBackRest::Storage::Filter::Sha; -use pgBackRest::Storage::Posix::Driver; - -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - # Test data - my $strFile = $self->testPath() . qw{/} . 'file.txt'; - my $strFileContent = 'TESTDATA'; - my $iFileLength = length($strFileContent); - my $oDriver = new pgBackRest::Storage::Posix::Driver(); - - ################################################################################################################################ - if ($self->begin('read()')) - { - my $tBuffer; - - #--------------------------------------------------------------------------------------------------------------------------- - executeTest("echo -n '${strFileContent}' | tee ${strFile}"); - - my $oFileIo = $self->testResult(sub {$oDriver->openRead($strFile)}, '[object]', 'open read'); - my $oShaIo = $self->testResult(sub {new pgBackRest::Storage::Filter::Sha($oFileIo)}, '[object]', 'new read'); - - $self->testResult(sub {$oShaIo->read(\$tBuffer, 2, undef)}, 2, 'read 2 bytes'); - $self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 2)}, 2, 'read 2 bytes'); - $self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 4)}, 2, 'read 2 bytes'); - $self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 6)}, 2, 'read 2 bytes'); - $self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 8)}, 0, 'read 0 bytes'); - - $self->testResult(sub {$oShaIo->close()}, true, 'close'); - my $strSha = $self->testResult( - sub {$oShaIo->result(STORAGE_FILTER_SHA)}, cryptoHashOne('sha1', $strFileContent), - 'check hash against original content'); - $self->testResult($strSha, cryptoHashOne('sha1', $tBuffer), 'check hash against buffer'); - $self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, 'check content'); - - #--------------------------------------------------------------------------------------------------------------------------- - $tBuffer = undef; - - $oFileIo = $self->testResult( - sub {$oDriver->openRead($self->dataPath() . '/filecopy.archive2.bin')}, '[object]', 'open read'); - $oShaIo = $self->testResult(sub {new pgBackRest::Storage::Filter::Sha($oFileIo)}, '[object]', 'new read'); - - $self->testResult(sub {$oShaIo->read(\$tBuffer, 8388608)}, 8388608, ' read 8388608 bytes'); - $self->testResult(sub {$oShaIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes'); - $self->testResult(sub {$oShaIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes'); - $self->testResult(sub {$oShaIo->read(\$tBuffer, 1)}, 0, ' read 0 bytes'); - - $self->testResult(sub {$oShaIo->close()}, true, ' close'); - $self->testResult(sub {$oShaIo->close()}, false, ' close again to make sure nothing bad happens'); - $self->testResult($oShaIo->result(STORAGE_FILTER_SHA), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check hash'); - $self->testResult(cryptoHashOne('sha1', $tBuffer), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content'); - } - - ################################################################################################################################ - if ($self->begin('write()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - my $oFileIo = $self->testResult(sub {$oDriver->openWrite($strFile, {bAtomic => true})}, '[object]', 'open write'); - my $oShaIo = $self->testResult( - sub {new pgBackRest::Storage::Filter::Sha($oFileIo)}, '[object]', 'new'); - - my $tBuffer = substr($strFileContent, 0, 2); - $self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes'); - $tBuffer = substr($strFileContent, 2, 2); - $self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes'); - $tBuffer = substr($strFileContent, 4, 2); - $self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes'); - $tBuffer = substr($strFileContent, 6, 2); - $self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes'); - $tBuffer = ''; - $self->testResult(sub {$oShaIo->write(\$tBuffer)}, 0, 'write 0 bytes'); - - $self->testResult(sub {$oShaIo->close()}, true, 'close'); - my $strSha = $self->testResult( - sub {$oShaIo->result(STORAGE_FILTER_SHA)}, cryptoHashOne('sha1', $strFileContent), - 'check hash against original content'); - $self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, 'check content'); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageHelperPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageHelperPerlTest.pm index a144660fb..d3e43cdb8 100644 --- a/test/lib/pgBackRestTest/Module/Storage/StorageHelperPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Storage/StorageHelperPerlTest.pm @@ -61,11 +61,12 @@ sub run #------------------------------------------------------------------------------------------------------------------------------- if ($self->begin("storageLocal()")) { - $self->testResult(sub {storageLocal($self->testPath())->put($strFile, $strFileContent)}, $iFileSize, 'put'); - $self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check put'); + $self->testResult(sub {storageLocal($self->testPath())->put("/tmp/${strFile}", $strFileContent)}, $iFileSize, 'put'); + $self->testResult(sub {${storageTest()->get("/tmp/${strFile}")}}, $strFileContent, ' check put'); - $self->testResult(sub {storageLocal($self->testPath())->put($strFile, $strFileContent)}, $iFileSize, 'put cache storage'); - $self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check put'); + $self->testResult( + sub {storageLocal($self->testPath())->put("/tmp/${strFile}", $strFileContent)}, $iFileSize, 'put cache storage'); + $self->testResult(sub {${storageTest()->get("/tmp/${strFile}")}}, $strFileContent, ' check put'); } #------------------------------------------------------------------------------------------------------------------------------- @@ -109,44 +110,6 @@ sub run $self->testResult( sub {storageRepo()->pathGet(STORAGE_REPO_BACKUP . '/file')}, $self->testPath() . '/repo/backup/db/file', 'check backup file'); - - #--------------------------------------------------------------------------------------------------------------------------- - # Insert a bogus rule to generate an error - storageRepo()->{hRule}{''} = - { - fnRule => storageRepo()->{hRule}{&STORAGE_REPO_ARCHIVE}{fnRule}, - }; - - $self->testException(sub {storageRepo()->pathGet('')}, ERROR_ASSERT, 'invalid storage rule '); - } - - #------------------------------------------------------------------------------------------------------------------------------- - if ($self->begin("storageRepo() encryption")) - { - my $strStanzaEncrypt = 'test-encrypt'; - $self->optionTestSet(CFGOPT_REPO_CIPHER_TYPE, CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC); - $self->testException( - sub {$self->configTestLoad(CFGCMD_ARCHIVE_PUSH)}, ERROR_OPTION_REQUIRED, - 'archive-push command requires option: repo1-cipher-pass'); - - # Set the encryption passphrase and confirm passphrase and type have been set in the storage object - $self->optionTestSet(CFGOPT_REPO_CIPHER_PASS, 'x'); - $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); - - $self->testResult(sub {storageRepo({strStanza => $strStanzaEncrypt})->cipherType() eq - CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC}, true, 'encryption type set'); - $self->testResult(sub {storageRepo({strStanza => $strStanzaEncrypt})->cipherPassUser() eq 'x'}, true, - 'encryption passphrase set'); - - # Cannot change encryption after it has been set (cached values not reset) - $self->optionTestClear(CFGOPT_REPO_CIPHER_TYPE); - $self->optionTestClear(CFGOPT_REPO_CIPHER_PASS); - $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); - - $self->testResult(sub {storageRepo({strStanza => $strStanzaEncrypt})->cipherType() eq - CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC}, true, 'encryption type not reset'); - $self->testResult(sub {storageRepo({strStanza => $strStanzaEncrypt})->cipherPassUser() eq 'x'}, true, - 'encryption passphrase not reset'); } } diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageLocalPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageLocalPerlTest.pm deleted file mode 100644 index c8f8a93e3..000000000 --- a/test/lib/pgBackRestTest/Module/Storage/StorageLocalPerlTest.pm +++ /dev/null @@ -1,475 +0,0 @@ -#################################################################################################################################### -# Tests for Storage::Local module -#################################################################################################################################### -package pgBackRestTest::Module::Storage::StorageLocalPerlTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use pgBackRest::Config::Config; -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::LibC qw(:crypto); -use pgBackRest::Storage::Filter::Sha; -use pgBackRest::Storage::Base; -use pgBackRest::Storage::Local; - -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# initModule - common objects and variables used by all tests. -#################################################################################################################################### -sub initModule -{ - my $self = shift; - - # Local path - $self->{strPathLocal} = $self->testPath() . '/local'; - - # Create the dynamic rule - my $fnRule = sub - { - my $strRule = shift; - my $strFile = shift; - my $xData = shift; - - if ($strRule eq '') - { - return "fn-rule-1/${xData}" . (defined($strFile) ? "/${strFile}" : ''); - } - else - { - return 'fn-rule-2/' . (defined($strFile) ? "${strFile}/${strFile}" : 'no-file'); - } - }; - - # Create the rule hash - my $hRule = - { - '' => 'static-rule-path', - '' => - { - fnRule => $fnRule, - xData => 'test', - }, - '' => - { - fnRule => $fnRule, - }, - }; - - # Create local storage - $self->{oStorageLocal} = new pgBackRest::Storage::Local( - $self->pathLocal(), new pgBackRest::Storage::Posix::Driver(), {hRule => $hRule, bAllowTemp => false}); - - # Create encrypted storage - $self->{oStorageEncrypt} = new pgBackRest::Storage::Local( - $self->testPath(), new pgBackRest::Storage::Posix::Driver(), - {hRule => $hRule, bAllowTemp => false, strCipherType => CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC}); - - # Remote path - $self->{strPathRemote} = $self->testPath() . '/remote'; - - # Create the repo path so the remote won't complain that it's missing - mkdir($self->pathRemote()) - or confess &log(ERROR, "unable to create repo directory '" . $self->pathRemote() . qw{'}); - - # Remove repo path now that the remote is created - rmdir($self->{strPathRemote}) - or confess &log(ERROR, "unable to remove repo directory '" . $self->pathRemote() . qw{'}); - - # Create remote storage - $self->{oStorageRemote} = new pgBackRest::Storage::Local( - $self->pathRemote(), new pgBackRest::Storage::Posix::Driver(), {hRule => $hRule}); -} - -#################################################################################################################################### -# initTest - initialization before each test -#################################################################################################################################### -sub initTest -{ - my $self = shift; - - executeTest( - 'ssh ' . $self->backrestUser() . '\@' . $self->host() . ' mkdir -m 700 ' . $self->pathRemote(), {bSuppressStdErr => true}); - - executeTest('mkdir -m 700 ' . $self->pathLocal()); -} - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - # Define test file - my $strFile = 'file.txt'; - my $strFileCopy = 'file.txt.copy'; - my $strFileHash = 'bbbcf2c59433f68f22376cd2439d6cd309378df6'; - my $strFileContent = 'TESTDATA'; - my $iFileSize = length($strFileContent); - - ################################################################################################################################ - if ($self->begin("pathGet()")) - { - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {$self->storageLocal()->pathGet('/test', {bTemp => true})}, - ERROR_ASSERT, "temp file not supported for storage '" . $self->storageLocal()->pathBase() . "'"); - $self->testException( - sub {$self->storageRemote()->pathGet('', {bTemp => true})}, - ERROR_ASSERT, 'file part must be defined when temp file specified'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageRemote()->pathGet('/file', {bTemp => true})}, "/file.tmp", 'absolute path temp'); - $self->testResult(sub {$self->storageRemote()->pathGet('/file')}, "/file", 'absolute path file'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->pathGet('file')}, $self->storageLocal()->pathBase() . '/file', 'relative path'); - $self->testResult( - sub {$self->storageRemote()->pathGet('file', {bTemp => true})}, - $self->storageRemote()->pathBase() . '/file.tmp', 'relative path temp'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {$self->storageLocal()->pathGet(' in 'testException( - sub {$self->storageLocal()->pathGet('')}, ERROR_ASSERT, "storage rule '' does not exist"); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->pathGet('/file')}, - $self->storageLocal()->pathBase() . '/static-rule-path/file', 'static rule file'); - $self->testResult( - sub {$self->storageLocal()->pathGet('')}, - $self->storageLocal()->pathBase() . '/static-rule-path', 'static rule path'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->pathGet('/file')}, - $self->storageLocal()->pathBase() . '/fn-rule-1/test/file', 'function rule 1 file'); - $self->testResult( - sub {$self->storageLocal()->pathGet('/file')}, - $self->storageLocal()->pathBase() . '/fn-rule-2/file/file', 'function rule 2 file'); - $self->testResult( - sub {$self->storageLocal()->pathGet('')}, - $self->storageLocal()->pathBase() . '/fn-rule-1/test', 'function rule 1 path'); - $self->testResult( - sub {$self->storageLocal()->pathGet('')}, - $self->storageLocal()->pathBase() . '/fn-rule-2/no-file', 'function rule 2 no file'); - } - - ################################################################################################################################ - if ($self->begin('openWrite()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - my $oFileIo = $self->testResult(sub {$self->storageLocal()->openWrite($strFile)}, '[object]', 'open write'); - - $self->testResult(sub {$oFileIo->write(\$strFileContent)}, $iFileSize, "write $iFileSize bytes"); - $self->testResult(sub {$oFileIo->close()}, true, 'close'); - - # Check that it is not encrypted - $self->testResult(sub {$self->storageLocal()->encrypted($strFile)}, false, 'test storage not encrypted'); - } - - ################################################################################################################################ - if ($self->begin('put()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile))}, 0, 'put empty'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->put($strFile)}, 0, 'put empty (all defaults)'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), $strFileContent)}, $iFileSize, 'put'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), \$strFileContent)}, $iFileSize, - 'put reference'); - } - - ################################################################################################################################ - if ($self->begin('openRead()')) - { - my $tContent; - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->openRead($strFile, {bIgnoreMissing => true})}, undef, 'ignore missing'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {$self->storageLocal()->openRead($strFile)}, ERROR_FILE_MISSING, - "unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': No such file or directory"); - - #--------------------------------------------------------------------------------------------------------------------------- - executeTest('sudo touch ' . $self->pathLocal() . "/${strFile} && sudo chmod 700 " . $self->pathLocal() . "/${strFile}"); - - $self->testException( - sub {$self->storageLocal()->openRead($strFile)}, ERROR_FILE_OPEN, - "unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': Permission denied"); - - executeTest('sudo rm ' . $self->pathLocal() . "/${strFile}"); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->storageLocal()->put($self->storageLocal()->openWrite($strFile), $strFileContent); - - my $oFileIo = $self->testResult(sub {$self->storageLocal()->openRead($strFile)}, '[object]', 'open read'); - - $self->testResult(sub {$oFileIo->read(\$tContent, $iFileSize)}, $iFileSize, "read $iFileSize bytes"); - $self->testResult($tContent, $strFileContent, ' check read'); - - #--------------------------------------------------------------------------------------------------------------------------- - $oFileIo = $self->testResult( - sub {$self->storageLocal()->openRead($strFile, {rhyFilter => [{strClass => STORAGE_FILTER_SHA}]})}, '[object]', - 'open read + checksum'); - - undef($tContent); - $self->testResult(sub {$oFileIo->read(\$tContent, $iFileSize)}, $iFileSize, "read $iFileSize bytes"); - $self->testResult(sub {$oFileIo->close()}, true, 'close'); - $self->testResult($tContent, $strFileContent, ' check read'); - $self->testResult($oFileIo->result(STORAGE_FILTER_SHA), cryptoHashOne('sha1', $strFileContent), ' check hash'); - } - - ################################################################################################################################ - if ($self->begin('get()')) - { - my $tBuffer; - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->get($self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}))}, undef, - 'get missing'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->storageLocal()->put($strFile); - $self->testResult(sub {${$self->storageLocal()->get($strFile)}}, undef, 'get empty'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->storageLocal()->put($strFile, $strFileContent); - $self->testResult(sub {${$self->storageLocal()->get($strFile)}}, $strFileContent, 'get'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {${$self->storageLocal()->get($self->storageLocal()->openRead($strFile))}}, $strFileContent, 'get from io'); - } - - ################################################################################################################################ - if ($self->begin('hashSize()')) - { - my $tBuffer; - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->put($strFile, $strFileContent)}, 8, 'put'); - - $self->testResult( - sub {$self->storageLocal()->hashSize($strFile)}, - qw{(} . cryptoHashOne('sha1', $strFileContent) . ', ' . $iFileSize . qw{)}, ' check hash/size'); - $self->testResult( - sub {$self->storageLocal()->hashSize(BOGUS, {bIgnoreMissing => true})}, "([undef], [undef])", - ' check missing hash/size'); - } - - ################################################################################################################################ - if ($self->begin('copy()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING, - "unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': No such file or directory"); - $self->testResult( - sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageLocal()->copy( - $self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}), - $self->storageLocal()->openWrite($strFileCopy))}, - false, 'missing source io'); - $self->testResult( - sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING, - "unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': No such file or directory"); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->storageLocal()->put($strFile, $strFileContent); - - $self->testResult(sub {$self->storageLocal()->copy($strFile, $strFileCopy)}, true, 'copy filename->filename'); - $self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->storageLocal()->remove($strFileCopy); - - $self->testResult( - sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, true, 'copy io->filename'); - $self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->storageLocal()->remove($strFileCopy); - - $self->testResult( - sub {$self->storageLocal()->copy( - $self->storageLocal()->openRead($strFile), $self->storageLocal()->openWrite($strFileCopy))}, - true, 'copy io->io'); - $self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy'); - } - - ################################################################################################################################ - if ($self->begin('info()')) - { - $self->testResult(sub {$self->storageLocal()->info($self->{strPathLocal})}, "[object]", 'stat dir successfully'); - - $self->testException(sub {$self->storageLocal()->info($strFile)}, ERROR_FILE_MISSING, - "unable to stat '". $self->{strPathLocal} . "/" . $strFile ."': No such file or directory"); - } - - ################################################################################################################################ - if ($self->begin('pathCreate()')) - { - my $strTestPath = $self->{strPathLocal} . "/" . BOGUS; - - $self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath)}, "[undef]", - "test creation of path " . $strTestPath); - - $self->testException(sub {$self->storageLocal()->pathCreate($strTestPath)}, ERROR_PATH_EXISTS, - "unable to create path '". $strTestPath. "' because it already exists"); - - $self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath, {bIgnoreExists => true})}, "[undef]", - "ignore path exists"); - } - - ################################################################################################################################ - if ($self->begin('encryption')) - { - my $strCipherPass = 'x'; - $self->testResult(sub {cryptoHashOne('sha1', $strFileContent)}, $strFileHash, 'hash check contents to be written'); - - # Error when passphrase not passed - #--------------------------------------------------------------------------------------------------------------------------- - my $oFileIo = $self->testException(sub {$self->storageEncrypt()->openWrite($strFile)}, - ERROR_ASSERT, 'tCipherPass is required in Storage::Filter::CipherBlock->new'); - - # Write an encrypted file - #--------------------------------------------------------------------------------------------------------------------------- - $oFileIo = $self->testResult(sub {$self->storageEncrypt()->openWrite($strFile, {strCipherPass => $strCipherPass})}, - '[object]', 'open write'); - - my $iWritten = $oFileIo->write(\$strFileContent); - $self->testResult(sub {$oFileIo->close()}, true, ' close'); - - # Check that it is encrypted and valid for the repo encryption type - $self->testResult(sub {$self->storageEncrypt()->encryptionValid($self->storageEncrypt()->encrypted($strFile))}, true, - ' test storage encrypted and valid'); - - $self->testResult( - sub {cryptoHashOne('sha1', ${storageTest()->get($strFile)}) ne $strFileHash}, true, ' check written sha1 different'); - - # Error when passphrase not passed - #--------------------------------------------------------------------------------------------------------------------------- - $oFileIo = $self->testException(sub {$self->storageEncrypt()->openRead($strFile)}, - ERROR_ASSERT, 'tCipherPass is required in Storage::Filter::CipherBlock->new'); - - # Read it and confirm it decrypts and is same as original content - #--------------------------------------------------------------------------------------------------------------------------- - $oFileIo = $self->testResult(sub {$self->storageEncrypt()->openRead($strFile, {strCipherPass => $strCipherPass})}, - '[object]', 'open read and decrypt'); - my $strContent; - $oFileIo->read(\$strContent, $iWritten); - $self->testResult(sub {$oFileIo->close()}, true, ' close'); - $self->testResult($strContent, $strFileContent, ' decrypt read equal orginal contents'); - - # Copy - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$self->storageEncrypt()->copy( - $self->storageEncrypt()->openRead($strFile, {strCipherPass => $strCipherPass}), - $self->storageEncrypt()->openWrite($strFileCopy, {strCipherPass => $strCipherPass}))}, - true, 'copy - decrypt/encrypt'); - - $self->testResult( - sub {cryptoHashOne('sha1', ${$self->storageEncrypt()->get($strFileCopy, {strCipherPass => $strCipherPass})})}, - $strFileHash, ' check decrypted copy file sha1 same as original plaintext file'); - - # Write an empty encrypted file - #--------------------------------------------------------------------------------------------------------------------------- - my $strFileZero = 'file-0.txt'; - my $strZeroContent = ''; - $oFileIo = $self->testResult( - sub {$self->storageEncrypt()->openWrite($strFileZero, {strCipherPass => $strCipherPass})}, '[object]', - 'open write for zero'); - - $self->testResult(sub {$oFileIo->write(\$strZeroContent)}, 0, ' zero written'); - $self->testResult(sub {$oFileIo->close()}, true, ' close'); - - $self->testResult(sub {$self->storageEncrypt()->encrypted($strFile)}, true, ' test empty file encrypted'); - - # Write an unencrypted file to the encrypted storage and check if the file is valid for that storage - #--------------------------------------------------------------------------------------------------------------------------- - my $strFileTest = $self->testPath() . qw{/} . 'test.file.txt'; - - # Create empty file - executeTest("touch ${strFileTest}"); - $self->testResult(sub {$self->storageEncrypt()->encrypted($strFileTest)}, false, 'empty file so not encrypted'); - - # Add unencrypted content to the file - executeTest("echo -n '${strFileContent}' | tee ${strFileTest}"); - $self->testResult(sub {$self->storageEncrypt()->encryptionValid($self->storageEncrypt()->encrypted($strFileTest))}, false, - 'storage encryption and unencrypted file format do not match'); - - # Unencrypted file valid in unencrypted storage - $self->testResult(sub {$self->storageLocal()->encryptionValid($self->storageLocal()->encrypted($strFileTest))}, true, - 'unencrypted file valid in unencrypted storage'); - - # Prepend encryption Magic Signature and test encrypted file in unencrypted storage not valid - executeTest('echo "' . CIPHER_MAGIC . '$(cat ' . $strFileTest . ')" > ' . $strFileTest); - $self->testResult(sub {$self->storageLocal()->encryptionValid($self->storageLocal()->encrypted($strFileTest))}, false, - 'storage unencrypted and encrypted file format do not match'); - - # Test a file that does not exist - #--------------------------------------------------------------------------------------------------------------------------- - $strFileTest = $self->testPath() . qw{/} . 'testfile'; - $self->testException(sub {$self->storageEncrypt()->encrypted($strFileTest)}, ERROR_FILE_MISSING, - "unable to open '" . $strFileTest . "': No such file or directory"); - - $self->testResult(sub {$self->storageEncrypt()->encrypted($strFileTest, {bIgnoreMissing => true})}, true, - 'encryption for ignore missing file returns encrypted for encrypted storage'); - - $self->testResult(sub {$self->storageLocal()->encrypted($strFileTest, {bIgnoreMissing => true})}, false, - 'encryption for ignore missing file returns unencrypted for unencrypted storage'); - } -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub host {return '127.0.0.1'} -sub pathLocal {return shift->{strPathLocal}}; -sub pathRemote {return shift->{strPathRemote}}; -sub protocolLocal {return shift->{oProtocolLocal}}; -sub protocolRemote {return shift->{oProtocolRemote}}; -sub storageLocal {return shift->{oStorageLocal}}; -sub storageEncrypt {return shift->{oStorageEncrypt}}; -sub storageRemote {return shift->{oStorageRemote}}; - -1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StoragePerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StoragePerlTest.pm new file mode 100644 index 000000000..cae5cd34f --- /dev/null +++ b/test/lib/pgBackRestTest/Module/Storage/StoragePerlTest.pm @@ -0,0 +1,341 @@ +#################################################################################################################################### +# Tests for Storage::Local module +#################################################################################################################################### +package pgBackRestTest::Module::Storage::StoragePerlTest; +use parent 'pgBackRestTest::Common::RunTest'; + +#################################################################################################################################### +# Perl includes +#################################################################################################################################### +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use pgBackRest::Config::Config; +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; +use pgBackRest::LibC qw(:crypto); +use pgBackRest::Storage::Base; + +use pgBackRestTest::Common::ContainerTest; +use pgBackRestTest::Common::ExecuteTest; +use pgBackRestTest::Common::RunTest; +use pgBackRestTest::Env::Host::HostBackupTest; + +#################################################################################################################################### +# run +#################################################################################################################################### +sub run +{ + my $self = shift; + + # Define test file + my $strFile = $self->testPath() . '/file.txt'; + my $strFileCopy = $self->testPath() . '/file.txt.copy'; + my $strFileHash = 'bbbcf2c59433f68f22376cd2439d6cd309378df6'; + my $strFileContent = 'TESTDATA'; + my $iFileSize = length($strFileContent); + + # Create local storage + $self->{oStorageLocal} = new pgBackRest::Storage::Storage(''); + + ################################################################################################################################ + if ($self->begin("pathGet()")) + { + $self->testResult(sub {$self->storageLocal()->pathGet('file')}, '/file', 'relative path'); + $self->testResult(sub {$self->storageLocal()->pathGet('/file2')}, '/file2', 'absolute path'); + } + + ################################################################################################################################ + if ($self->begin('put()')) + { + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult( + sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile))}, 0, 'put empty'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult( + sub {$self->storageLocal()->put($strFile)}, 0, 'put empty (all defaults)'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult( + sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), $strFileContent)}, $iFileSize, 'put'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult( + sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), \$strFileContent)}, $iFileSize, + 'put reference'); + } + + ################################################################################################################################ + if ($self->begin('get()')) + { + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult( + sub {$self->storageLocal()->get($self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}))}, undef, + 'get missing'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->storageLocal()->put($strFile); + $self->testResult(sub {${$self->storageLocal()->get($strFile)}}, undef, 'get empty'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->storageLocal()->put($strFile, $strFileContent); + $self->testResult(sub {${$self->storageLocal()->get($strFile)}}, $strFileContent, 'get'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult( + sub {${$self->storageLocal()->get($self->storageLocal()->openRead($strFile))}}, $strFileContent, 'get from io'); + } + + ################################################################################################################################ + if ($self->begin('hashSize()')) + { + $self->testResult( + sub {$self->storageLocal()->put($strFile, $strFileContent)}, 8, 'put'); + + $self->testResult( + sub {$self->storageLocal()->hashSize($strFile)}, + qw{(} . cryptoHashOne('sha1', $strFileContent) . ', ' . $iFileSize . qw{)}, ' check hash/size'); + $self->testResult( + sub {$self->storageLocal()->hashSize(BOGUS, {bIgnoreMissing => true})}, "([undef], [undef])", + ' check missing hash/size'); + } + + ################################################################################################################################ + if ($self->begin('copy()')) + { + #--------------------------------------------------------------------------------------------------------------------------- + $self->testException( + sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING, + "unable to open missing file '${strFile}' for read"); + $self->testResult( + sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult( + sub {$self->storageLocal()->copy( + $self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}), + $self->storageLocal()->openWrite($strFileCopy))}, + false, 'missing source io'); + $self->testResult( + sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testException( + sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING, + "unable to open missing file '${strFile}' for read"); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->storageLocal()->put($strFile, $strFileContent); + + $self->testResult(sub {$self->storageLocal()->copy($strFile, $strFileCopy)}, true, 'copy filename->filename'); + $self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->storageLocal()->remove($strFileCopy); + + $self->testResult( + sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, true, 'copy io->filename'); + $self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->storageLocal()->remove($strFileCopy); + + $self->testResult( + sub {$self->storageLocal()->copy( + $self->storageLocal()->openRead($strFile), $self->storageLocal()->openWrite($strFileCopy))}, + true, 'copy io->io'); + $self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy'); + } + + ################################################################################################################################ + if ($self->begin('exists()')) + { + $self->storageLocal()->put($self->testPath() . "/test.file"); + + $self->testResult(sub {$self->storageLocal()->exists($self->testPath() . "/test.file")}, true, 'existing file'); + $self->testResult(sub {$self->storageLocal()->exists($self->testPath() . "/test.missing")}, false, 'missing file'); + $self->testResult(sub {$self->storageLocal()->exists($self->testPath())}, false, 'path'); + } + + ################################################################################################################################ + if ($self->begin('info()')) + { + $self->testResult( + sub {$self->storageLocal()->info($self->testPath())}, + "{group => " . $self->group() . ", mode => 0770, type => d, user => " . $self->pgUser() . "}", + 'stat dir successfully'); + + $self->testException(sub {$self->storageLocal()->info(BOGUS)}, ERROR_FILE_OPEN, + "unable to get info for missing path/file '/bogus'"); + } + + ################################################################################################################################ + if ($self->begin("manifest() and list()")) + { + #--------------------------------------------------------------------------------------------------------------------------- + $self->testException( + sub {$self->storageLocal()->manifest($self->testPath() . '/missing')}, + ERROR_PATH_MISSING, "unable to list file info for missing path '" . $self->testPath() . "/missing'"); + + #--------------------------------------------------------------------------------------------------------------------------- + # Setup test data + executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1'); + executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1/sub2'); + executeTest('mkdir -m 750 ' . $self->testPath() . '/sub2'); + + executeTest("echo 'TESTDATA' > " . $self->testPath() . '/test.txt'); + utime(1111111111, 1111111111, $self->testPath() . '/test.txt'); + executeTest('chmod 1640 ' . $self->testPath() . '/test.txt'); + + executeTest("echo 'TESTDATA_' > ". $self->testPath() . '/sub1/test-sub1.txt'); + utime(1111111112, 1111111112, $self->testPath() . '/sub1/test-sub1.txt'); + executeTest('chmod 0640 ' . $self->testPath() . '/sub1/test-sub1.txt'); + + executeTest("echo 'TESTDATA__' > " . $self->testPath() . '/sub1/sub2/test-sub2.txt'); + utime(1111111113, 1111111113, $self->testPath() . '/sub1/sub2/test-sub2.txt'); + executeTest('chmod 0646 ' . $self->testPath() . '/sub1/test-sub1.txt'); + + executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/test-hardlink.txt'); + executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/sub2/test-hardlink.txt'); + + executeTest('ln -s .. ' . $self->testPath() . '/sub1/test'); + executeTest('chmod 0700 ' . $self->testPath() . '/sub1/test'); + executeTest('ln -s ../.. ' . $self->testPath() . '/sub1/sub2/test'); + executeTest('chmod 0750 ' . $self->testPath() . '/sub1/sub2/test'); + + executeTest('chmod 0770 ' . $self->testPath()); + + $self->testResult( + sub {$self->storageLocal()->manifest($self->testPath())}, + '{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}, ' . + 'sub1 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' . + 'sub1/sub2 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' . + 'sub1/sub2/test => {group => ' . $self->group() . ', link_destination => ../.., type => l, user => ' . + $self->pgUser() . '}, ' . + 'sub1/sub2/test-hardlink.txt => ' . + '{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111111, size => 9, type => f, user => ' . + $self->pgUser() . '}, ' . + 'sub1/sub2/test-sub2.txt => ' . + '{group => ' . $self->group() . ', mode => 0666, modification_time => 1111111113, size => 11, type => f, user => ' . + $self->pgUser() . '}, ' . + 'sub1/test => {group => ' . $self->group() . ', link_destination => .., type => l, user => ' . $self->pgUser() . '}, ' . + 'sub1/test-hardlink.txt => ' . + '{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111111, size => 9, type => f, user => ' . + $self->pgUser() . '}, ' . + 'sub1/test-sub1.txt => ' . + '{group => ' . $self->group() . ', mode => 0646, modification_time => 1111111112, size => 10, type => f, user => ' . + $self->pgUser() . '}, ' . + 'sub2 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' . + 'test.txt => ' . + '{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111111, size => 9, type => f, user => ' . + $self->pgUser() . '}}', + 'complete manifest'); + + $self->testResult(sub {$self->storageLocal()->list($self->testPath())}, "(sub1, sub2, test.txt)", "list"); + $self->testResult(sub {$self->storageLocal()->list($self->testPath(), {strExpression => "2\$"})}, "sub2", "list"); + $self->testResult( + sub {$self->storageLocal()->list($self->testPath(), {strSortOrder => 'reverse'})}, "(test.txt, sub2, sub1)", + "list reverse"); + $self->testResult(sub {$self->storageLocal()->list($self->testPath() . "/sub2")}, "[undef]", "list empty"); + $self->testResult( + sub {$self->storageLocal()->list($self->testPath() . "/sub99", {bIgnoreMissing => true})}, "[undef]", "list missing"); + $self->testException( + sub {$self->storageLocal()->list($self->testPath() . "/sub99")}, ERROR_PATH_MISSING, + "unable to list files for missing path '" . $self->testPath() . "/sub99'"); + } + + ################################################################################################################################ + if ($self->begin('move()')) + { + my $strFileCopy = "${strFile}.copy"; + my $strFileSub = $self->testPath() . '/sub/file.txt'; + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testException( + sub {$self->storageLocal()->move($strFile, $strFileCopy)}, ERROR_FILE_MOVE, + "unable to move '${strFile}' to '${strFile}.copy': No such file or directory"); + } + + ################################################################################################################################ + if ($self->begin('owner()')) + { + my $strFile = $self->testPath() . "/test.txt"; + + $self->testException( + sub {$self->storageLocal()->owner($strFile, 'root')}, ERROR_FILE_MISSING, + "unable to stat '${strFile}': No such file or directory"); + + executeTest("touch ${strFile}"); + + $self->testException( + sub {$self->storageLocal()->owner($strFile, BOGUS)}, ERROR_FILE_OWNER, + "unable to set ownership for '${strFile}' because user 'bogus' does not exist"); + $self->testException( + sub {$self->storageLocal()->owner($strFile, undef, BOGUS)}, ERROR_FILE_OWNER, + "unable to set ownership for '${strFile}' because group 'bogus' does not exist"); + + $self->testResult(sub {$self->storageLocal()->owner($strFile)}, undef, "no ownership changes"); + $self->testResult(sub {$self->storageLocal()->owner($strFile, TEST_USER)}, undef, "same user"); + $self->testResult(sub {$self->storageLocal()->owner($strFile, undef, TEST_GROUP)}, undef, "same group"); + $self->testResult( + sub {$self->storageLocal()->owner($strFile, TEST_USER, TEST_GROUP)}, undef, + "same user, group"); + + $self->testException( + sub {$self->storageLocal()->owner($strFile, 'root', undef)}, ERROR_FILE_OWNER, + "unable to set ownership for '${strFile}': Operation not permitted"); + $self->testException( + sub {$self->storageLocal()->owner($strFile, undef, 'root')}, ERROR_FILE_OWNER, + "unable to set ownership for '${strFile}': Operation not permitted"); + + executeTest("sudo chown :root ${strFile}"); + $self->testResult( + sub {$self->storageLocal()->owner($strFile, undef, TEST_GROUP)}, undef, "change group back from root"); + } + + ################################################################################################################################ + if ($self->begin('pathCreate()')) + { + my $strTestPath = $self->testPath() . "/" . BOGUS; + + $self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath)}, "[undef]", + "test creation of path " . $strTestPath); + + $self->testException(sub {$self->storageLocal()->pathCreate($strTestPath)}, ERROR_PATH_CREATE, + "unable to create path '". $strTestPath. "'"); + + $self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath, {bIgnoreExists => true})}, "[undef]", + "ignore path exists"); + } + + ################################################################################################################################ + if ($self->begin('pathExists()')) + { + $self->storageLocal()->put($self->testPath() . "/test.file"); + + $self->testResult(sub {$self->storageLocal()->pathExists($self->testPath() . "/test.file")}, false, 'existing file'); + $self->testResult(sub {$self->storageLocal()->pathExists($self->testPath() . "/test.missing")}, false, 'missing file'); + $self->testResult(sub {$self->storageLocal()->pathExists($self->testPath())}, true, 'path'); + } + + ################################################################################################################################ + if ($self->begin('pathSync()')) + { + $self->testResult(sub {$self->storageLocal()->pathSync($self->testPath())}, "[undef]", "test path sync"); + } +} + +#################################################################################################################################### +# Getters +#################################################################################################################################### +# sub host {return '127.0.0.1'} +# sub pathLocal {return shift->{strPathLocal}}; +# sub pathRemote {return shift->{strPathRemote}}; +sub storageLocal {return shift->{oStorageLocal}}; +# sub storageEncrypt {return shift->{oStorageEncrypt}}; +# sub storageRemote {return shift->{oStorageRemote}}; + +1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StoragePosixPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StoragePosixPerlTest.pm deleted file mode 100644 index efc2600f8..000000000 --- a/test/lib/pgBackRestTest/Module/Storage/StoragePosixPerlTest.pm +++ /dev/null @@ -1,444 +0,0 @@ -#################################################################################################################################### -# Posix Driver Tests -#################################################################################################################################### -package pgBackRestTest::Module::Storage::StoragePosixPerlTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use File::Basename qw(basename dirname); -use IO::Socket::UNIX; - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::Storage::Posix::Driver; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - # Test data - my $strFile = $self->testPath() . '/file.txt'; - my $strFileContent = 'TESTDATA'; - my $iFileLength = length($strFileContent); - my $iFileLengthHalf = int($iFileLength / 2); - - # Test driver - my $oPosix = new pgBackRest::Storage::Posix::Driver(); - - ################################################################################################################################ - if ($self->begin('exists()')) - { - my $strPathSub = $self->testPath() . '/sub'; - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$oPosix->exists($strFile)}, false, 'file'); - - #--------------------------------------------------------------------------------------------------------------------------- - executeTest("sudo mkdir ${strPathSub} && sudo chmod 700 ${strPathSub}"); - - $self->testResult( - sub {$oPosix->pathExists($strPathSub)}, true, 'path'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {$oPosix->exists("${strPathSub}/file")}, ERROR_FILE_EXISTS, - "unable to test if file '${strPathSub}/file' exists: Permission denied"); - } - - ################################################################################################################################ - if ($self->begin("manifestList()")) - { - #--------------------------------------------------------------------------------------------------------------------------- - my @stryFile = ('.', 'test.txt'); - - $self->testResult( - sub {$oPosix->manifestList($self->testPath(), \@stryFile)}, - '{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}}', - 'skip missing file'); - } - - ################################################################################################################################ - if ($self->begin("manifestStat()")) - { - #--------------------------------------------------------------------------------------------------------------------------- - my $strFile = $self->testPath() . '/test.txt'; - - $self->testResult(sub {$oPosix->manifestStat($strFile)}, '[undef]', 'ignore missing file'); - - #--------------------------------------------------------------------------------------------------------------------------- - storageTest()->put($strFile, "TEST"); - utime(1111111111, 1111111111, $strFile); - executeTest('chmod 1640 ' . $strFile); - - $self->testResult( - sub {$oPosix->manifestStat($strFile)}, - '{group => ' . $self->group() . - ', mode => 1640, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}', - 'stat file'); - - #--------------------------------------------------------------------------------------------------------------------------- - my $strSocketFile = $self->testPath() . '/test.socket'; - - # Create a socket to test invalid files - my $oSocket = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Local => $strSocketFile, Listen => 1); - - $self->testException( - sub {$oPosix->manifestStat($strSocketFile)}, ERROR_FILE_INVALID, - "${strSocketFile} is not of type directory, file, or link"); - - # Cleanup socket - $oSocket->close(); - storageTest()->remove($strSocketFile); - - #--------------------------------------------------------------------------------------------------------------------------- - my $strTestPath = $self->testPath() . '/public_dir'; - storageTest()->pathCreate($strTestPath, {strMode => '0750'}); - - $self->testResult( - sub {$oPosix->manifestStat($strTestPath)}, - '{group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}', - 'stat directory'); - - #--------------------------------------------------------------------------------------------------------------------------- - my $strTestLink = $self->testPath() . '/public_dir_link'; - - symlink($strTestPath, $strTestLink) - or confess &log(ERROR, "unable to create symlink from ${strTestPath} to ${strTestLink}"); - - $self->testResult( - sub {$oPosix->manifestStat($strTestLink)}, - '{group => ' . $self->group() . ", link_destination => ${strTestPath}, type => l, user => " . $self->pgUser() . '}', - 'stat link'); - } - - ################################################################################################################################ - if ($self->begin("manifestRecurse()")) - { - #--------------------------------------------------------------------------------------------------------------------------- - my $strTestPath = $self->testPath() . '/public_dir'; - my $strTestFile = "${strTestPath}/test.txt"; - - $self->testException( - sub {my $hManifest = {}; $oPosix->manifestRecurse($strTestFile, undef, 0, $hManifest); $hManifest}, - ERROR_FILE_MISSING, "unable to stat '${strTestFile}': No such file or directory"); - - #--------------------------------------------------------------------------------------------------------------------------- - storageTest()->pathCreate($strTestPath, {strMode => '0750'}); - - $self->testResult( - sub {my $hManifest = {}; $oPosix->manifestRecurse($strTestPath, undef, 0, $hManifest); $hManifest}, - '{. => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}}', - 'empty directory manifest'); - - #--------------------------------------------------------------------------------------------------------------------------- - storageTest()->put($strTestFile, "TEST"); - utime(1111111111, 1111111111, $strTestFile); - executeTest('chmod 0750 ' . $strTestFile); - - storageTest()->pathCreate("${strTestPath}/sub", {strMode => '0750'}); - - $self->testResult( - sub {my $hManifest = {}; $oPosix->manifestRecurse( - $self->testPath(), basename($strTestPath), 1, $hManifest); $hManifest}, - '{public_dir => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' . - 'public_dir/sub => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' . - 'public_dir/' . basename($strTestFile) . ' => {group => ' . $self->group() . - ', mode => 0750, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}}', - 'directory and file manifest'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {my $hManifest = {}; $oPosix->manifestRecurse($strTestFile, undef, 0, $hManifest); $hManifest}, - '{' . basename($strTestFile) . ' => {group => ' . $self->group() . - ', mode => 0750, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}}', - 'single file manifest'); - } - - ################################################################################################################################ - if ($self->begin("manifest()")) - { - #--------------------------------------------------------------------------------------------------------------------------- - my $strMissingFile = $self->testPath() . '/missing'; - - $self->testException( - sub {$oPosix->manifest($strMissingFile)}, - ERROR_FILE_MISSING, "unable to stat '${strMissingFile}': No such file or directory"); - - #--------------------------------------------------------------------------------------------------------------------------- - # Setup test data - executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1'); - executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1/sub2'); - - executeTest("echo 'TESTDATA' > " . $self->testPath() . '/test.txt'); - utime(1111111111, 1111111111, $self->testPath() . '/test.txt'); - executeTest('chmod 1640 ' . $self->testPath() . '/test.txt'); - - executeTest("echo 'TESTDATA_' > ". $self->testPath() . '/sub1/test-sub1.txt'); - utime(1111111112, 1111111112, $self->testPath() . '/sub1/test-sub1.txt'); - executeTest('chmod 0640 ' . $self->testPath() . '/sub1/test-sub1.txt'); - - executeTest("echo 'TESTDATA__' > " . $self->testPath() . '/sub1/sub2/test-sub2.txt'); - utime(1111111113, 1111111113, $self->testPath() . '/sub1/sub2/test-sub2.txt'); - executeTest('chmod 0646 ' . $self->testPath() . '/sub1/test-sub1.txt'); - - executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/test-hardlink.txt'); - executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/sub2/test-hardlink.txt'); - - executeTest('ln -s .. ' . $self->testPath() . '/sub1/test'); - executeTest('chmod 0700 ' . $self->testPath() . '/sub1/test'); - executeTest('ln -s ../.. ' . $self->testPath() . '/sub1/sub2/test'); - executeTest('chmod 0750 ' . $self->testPath() . '/sub1/sub2/test'); - - executeTest('chmod 0770 ' . $self->testPath()); - - $self->testResult( - sub {$oPosix->manifest($self->testPath())}, - '{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}, ' . - 'sub1 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' . - 'sub1/sub2 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' . - 'sub1/sub2/test => {group => ' . $self->group() . ', link_destination => ../.., type => l, user => ' . - $self->pgUser() . '}, ' . - 'sub1/sub2/test-hardlink.txt => ' . - '{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' . - $self->pgUser() . '}, ' . - 'sub1/sub2/test-sub2.txt => ' . - '{group => ' . $self->group() . ', mode => 0666, modification_time => 1111111113, size => 11, type => f, user => ' . - $self->pgUser() . '}, ' . - 'sub1/test => {group => ' . $self->group() . ', link_destination => .., type => l, user => ' . $self->pgUser() . '}, ' . - 'sub1/test-hardlink.txt => ' . - '{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' . - $self->pgUser() . '}, ' . - 'sub1/test-sub1.txt => ' . - '{group => ' . $self->group() . ', mode => 0646, modification_time => 1111111112, size => 10, type => f, user => ' . - $self->pgUser() . '}, ' . - 'test.txt => ' . - '{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' . - $self->pgUser() . '}}', - 'complete manifest'); - } - - ################################################################################################################################ - if ($self->begin('openRead() & Posix::FileRead')) - { - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {$oPosix->openRead($strFile)}, ERROR_FILE_MISSING, "unable to open '${strFile}': No such file or directory"); - - #--------------------------------------------------------------------------------------------------------------------------- - executeTest("echo -n '${strFileContent}' | tee ${strFile}"); - - $self->testResult( - sub {$oPosix->openRead($strFile)}, '[object]', 'open read'); - } - - ################################################################################################################################ - if ($self->begin('openWrite() & Posix::FileWrite')) - { - my $tContent = $strFileContent; - - #--------------------------------------------------------------------------------------------------------------------------- - executeTest("echo -n '${strFileContent}' | tee ${strFile}"); - executeTest("chmod 600 ${strFile} && sudo chown root:root ${strFile}"); - - $self->testException( - sub {new pgBackRest::Storage::Posix::FileRead($oPosix, $strFile)}, ERROR_FILE_OPEN, - "unable to open '${strFile}': Permission denied"); - - executeTest("sudo rm -rf ${strFile}"); - - #--------------------------------------------------------------------------------------------------------------------------- - my $oPosixIo = $self->testResult( - sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFile)}, '[object]', 'open'); - - $tContent = undef; - $self->testException( - sub {$oPosixIo->write(\$tContent)}, ERROR_FILE_WRITE, "unable to write to '${strFile}': Use of uninitialized value"); - - $tContent = substr($strFileContent, 0, $iFileLengthHalf); - $self->testResult( - sub {$oPosixIo->write(\$tContent)}, $iFileLengthHalf, 'write part 1'); - - $tContent = substr($strFileContent, $iFileLengthHalf); - $self->testResult( - sub {$oPosixIo->write(\$tContent)}, $iFileLength - $iFileLengthHalf, - 'write part 2'); - $oPosixIo->close(); - - $tContent = undef; - $self->testResult( - sub {(new pgBackRest::Storage::Posix::FileRead($oPosix, $strFile))->read(\$tContent, $iFileLength)}, - $iFileLength, 'check write content length'); - $self->testResult($tContent, $strFileContent, 'check write content'); - - #--------------------------------------------------------------------------------------------------------------------------- - $oPosixIo = $self->testResult( - sub {new pgBackRest::Storage::Posix::FileWrite( - $oPosix, "${strFile}.atomic", {bAtomic => true, strMode => '0666', lTimestamp => time(), bSync => false})}, - '[object]', 'open'); - - $self->testResult(sub {$oPosixIo->write(\$tContent, $iFileLength)}, $iFileLength, 'write'); - $self->testResult(sub {$oPosixIo->close()}, true, 'close'); - - $self->testResult(sub {${storageTest()->get("${strFile}.atomic")}}, $strFileContent, 'check content'); - - #--------------------------------------------------------------------------------------------------------------------------- - $oPosixIo = $self->testResult( - sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFile)}, '[object]', 'open'); - - $self->testResult(sub {$oPosixIo->close()}, true, 'close'); - - undef($oPosixIo); - - # Test that a premature destroy (from error or otherwise) does not rename the file - #--------------------------------------------------------------------------------------------------------------------------- - my $strFileAbort = $self->testPath() . '/file-abort.txt'; - my $strFileAbortTmp = "${strFileAbort}.tmp"; - - $oPosixIo = $self->testResult( - sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFileAbort, {bAtomic => true})}, '[object]', 'open'); - - $oPosixIo->write(\$strFileContent); - undef($oPosixIo); - - $self->testResult(sub {$oPosix->exists($strFileAbort)}, false, 'destination file does not exist'); - $self->testResult(sub {$oPosix->exists($strFileAbortTmp)}, true, 'destination file tmp exists'); - - #--------------------------------------------------------------------------------------------------------------------------- - $oPosixIo = $self->testResult( - sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFile, {lTimestamp => time()})}, '[object]', 'open'); - $self->testResult(sub {$oPosixIo->write(\$strFileContent, $iFileLength)}, $iFileLength, 'write'); - executeTest("rm -f $strFile"); - - $self->testException( - sub {$oPosixIo->close()}, ERROR_FILE_WRITE, "unable to set time for '${strFile}': No such file or directory"); - } - - ################################################################################################################################ - if ($self->begin('owner()')) - { - my $strFile = $self->testPath() . "/test.txt"; - - $self->testException( - sub {$oPosix->owner($strFile, {strUser => 'root'})}, ERROR_FILE_MISSING, - "unable to stat '${strFile}': No such file or directory"); - - executeTest("touch ${strFile}"); - - $self->testException( - sub {$oPosix->owner($strFile, {strUser => BOGUS})}, ERROR_FILE_OWNER, - "unable to set ownership for '${strFile}' because user 'bogus' does not exist"); - $self->testException( - sub {$oPosix->owner($strFile, {strGroup => BOGUS})}, ERROR_FILE_OWNER, - "unable to set ownership for '${strFile}' because group 'bogus' does not exist"); - - $self->testResult(sub {$oPosix->owner($strFile)}, undef, "no ownership changes"); - $self->testResult(sub {$oPosix->owner($strFile, {strUser => TEST_USER})}, undef, "same user"); - $self->testResult(sub {$oPosix->owner($strFile, {strGroup => TEST_GROUP})}, undef, "same group"); - $self->testResult( - sub {$oPosix->owner($strFile, {strUser => TEST_USER, strGroup => TEST_GROUP})}, undef, "same user, group"); - - $self->testException( - sub {$oPosix->owner($strFile, {strUser => 'root'})}, ERROR_FILE_OWNER, - "unable to set ownership for '${strFile}': Operation not permitted"); - $self->testException( - sub {$oPosix->owner($strFile, {strGroup => 'root'})}, ERROR_FILE_OWNER, - "unable to set ownership for '${strFile}': Operation not permitted"); - - executeTest("sudo chown :root ${strFile}"); - $self->testResult( - sub {$oPosix->owner($strFile, {strGroup => TEST_GROUP})}, undef, "change group back from root"); - } - - ################################################################################################################################ - if ($self->begin('pathCreate()')) - { - my $strPathParent = $self->testPath() . '/parent'; - my $strPathSub = "${strPathParent}/sub1/sub2"; - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {$oPosix->pathCreate($strPathParent)}, undef, 'parent path'); - $self->testResult( - sub {$oPosix->pathExists($strPathParent)}, true, ' check path'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {$oPosix->pathCreate($strPathParent)}, ERROR_PATH_EXISTS, - "unable to create path '${strPathParent}' because it already exists"); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$oPosix->pathCreate($strPathParent, {bIgnoreExists => true})}, undef, 'path already exists'); - - #--------------------------------------------------------------------------------------------------------------------------- - executeTest("sudo chown root:root ${strPathParent} && sudo chmod 700 ${strPathParent}"); - - $self->testException( - sub {$oPosix->pathCreate($strPathSub)}, ERROR_PATH_CREATE, - "unable to create path '${strPathSub}': Permission denied"); - - #--------------------------------------------------------------------------------------------------------------------------- - executeTest("rmdir ${strPathParent}"); - - $self->testException( - sub {$oPosix->pathCreate($strPathSub)}, ERROR_PATH_MISSING, - "unable to create path '${strPathSub}' because parent does not exist"); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$oPosix->pathCreate($strPathSub, {bCreateParent => true})}, undef, 'path with parents'); - $self->testResult( - sub {$oPosix->pathExists($strPathSub)}, true, ' check path'); - } - - ################################################################################################################################ - if ($self->begin('move()')) - { - my $strFileCopy = "${strFile}.copy"; - my $strFileSub = $self->testPath() . '/sub/file.txt'; - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {$oPosix->move($strFile, $strFileCopy)}, ERROR_FILE_MISSING, - "unable to move '${strFile}' because it is missing"); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {storageTest()->put($strFile, $strFileContent)}, $iFileLength, 'put'); - $self->testResult( - sub {$oPosix->move($strFile, $strFileCopy)}, undef, 'simple move'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {$oPosix->move($strFileCopy, $strFileSub)}, ERROR_PATH_MISSING, - "unable to move '${strFileCopy}' to missing path '" . dirname($strFileSub) . "'"); - - #--------------------------------------------------------------------------------------------------------------------------- - executeTest('sudo mkdir ' . dirname($strFileSub) . ' && sudo chmod 700 ' . dirname($strFileSub)); - - $self->testException( - sub {$oPosix->move($strFileCopy, $strFileSub)}, ERROR_FILE_MOVE, - "unable to move '${strFileCopy}' to '${strFileSub}': Permission denied"); - - executeTest('sudo rmdir ' . dirname($strFileSub)); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$oPosix->move($strFileCopy, $strFileSub, {bCreatePath => true})}, undef, 'create parent path'); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageS3AuthPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageS3AuthPerlTest.pm deleted file mode 100644 index 23254bfe4..000000000 --- a/test/lib/pgBackRestTest/Module/Storage/StorageS3AuthPerlTest.pm +++ /dev/null @@ -1,167 +0,0 @@ -#################################################################################################################################### -# S3 Authentication Tests -#################################################################################################################################### -package pgBackRestTest::Module::Storage::StorageS3AuthPerlTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use POSIX qw(strftime); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::Common::Wait; -use pgBackRest::Storage::S3::Auth; - -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - ################################################################################################################################ - if ($self->begin('s3DateTime')) - { - $self->testResult(sub {s3DateTime(1491267845)}, '20170404T010405Z', 'format date/time'); - - #--------------------------------------------------------------------------------------------------------------------------- - waitRemainder(); - $self->testResult(sub {s3DateTime()}, strftime("%Y%m%dT%H%M%SZ", gmtime()), 'format current date/time'); - } - - ################################################################################################################################ - if ($self->begin('s3CanonicalRequest')) - { - $self->testResult( - sub {s3CanonicalRequest( - 'GET', qw(/), 'list-type=2', - {'host' => 'bucket.s3.amazonaws.com', 'x-amz-date' => '20170606T121212Z', - 'x-amz-content-sha256' => '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9'}, - '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')}, - "(GET\n/\nlist-type=2\nhost:bucket.s3.amazonaws.com\n" . - "x-amz-content-sha256:705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9\n" . - "x-amz-date:20170606T121212Z\n\nhost;x-amz-content-sha256;x-amz-date\n" . - '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9' . - ', host;x-amz-content-sha256;x-amz-date)', - 'canonical request'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testException( - sub {s3CanonicalRequest( - 'GET', qw(/), 'list-type=2', {'Host' => 'bucket.s3.amazonaws.com'}, - '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')}, - ERROR_ASSERT, "header 'Host' must be lower case"); - } - - ################################################################################################################################ - if ($self->begin('s3SigningKey')) - { - $self->testResult( - sub {unpack('H*', s3SigningKey('20170412', 'us-east-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))}, - '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9', 'signing key'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {unpack('H*', s3SigningKey('20170412', 'us-east-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))}, - '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9', 'same signing key from cache'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {unpack('H*', s3SigningKey('20170505', 'us-west-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))}, - 'c1a1cb590bbc38ba789c8e5695a1ec0cd7fd44c6949f922e149005a221524c09', 'new signing key'); - } - - ################################################################################################################################ - if ($self->begin('s3StringToSign')) - { - $self->testResult( - sub {s3StringToSign( - '20170412T141414Z', 'us-east-1', '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')}, - "AWS4-HMAC-SHA256\n20170412T141414Z\n20170412/us-east-1/s3/aws4_request\n" . - "705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9", - 'string to sign'); - } - - ################################################################################################################################ - if ($self->begin('s3AuthorizationHeader')) - { - $self->testResult( - sub {s3AuthorizationHeader( - 'us-east-1', 'bucket.s3.amazonaws.com', 'GET', qw(/), 'list-type=2', '20170606T121212Z', - {'authorization' => BOGUS, 'host' => 'bucket.s3.amazonaws.com', 'x-amz-date' => '20170606T121212Z'}, - 'AKIAIOSFODNN7EXAMPLE', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', undef, - 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')}, - '({authorization => AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,' . - 'SignedHeaders=host;x-amz-content-sha256;x-amz-date,' . - 'Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3,' . - ' host => bucket.s3.amazonaws.com,' . - ' x-amz-content-sha256 => e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,' . - ' x-amz-date => 20170606T121212Z}, ' . - "GET\n" . - "/\n" . - "list-type=2\n" . - "host:bucket.s3.amazonaws.com\n" . - "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" . - "x-amz-date:20170606T121212Z\n" . - "\n" . - "host;x-amz-content-sha256;x-amz-date\n" . - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855, " . - "host;x-amz-content-sha256;x-amz-date, " . - "AWS4-HMAC-SHA256\n" . - "20170606T121212Z\n" . - "20170606/us-east-1/s3/aws4_request\n" . - "4f2d4ee971f579e60ba6b3895e87434e17b1260f04392f02b512c1e8bada72dd)", - 'authorization header request'); - - $self->testResult( - sub {s3AuthorizationHeader( - 'us-east-1', 'bucket.s3.amazonaws.com', 'GET', qw(/), 'list-type=2', '20170606T121212Z', - {'authorization' => BOGUS, 'host' => 'bucket.s3.amazonaws.com', 'x-amz-date' => '20170606T121212Z'}, - 'AKIAIOSFODNN7EXAMPLE', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', - 'AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQW' . - 'LWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGd' . - 'QrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU' . - '9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz' . - '+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==', - 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')}, - '({authorization => AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,' . - 'SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,' . - 'Signature=c12565bf5d7e0ef623f76d66e09e5431aebef803f6a25a01c586525f17e474a3,' . - ' host => bucket.s3.amazonaws.com,' . - ' x-amz-content-sha256 => e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,' . - ' x-amz-date => 20170606T121212Z, x-amz-security-token => AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4H' . - 'IZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZ' . - 'ampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+sc' . - 'qKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==}, ' . - "GET\n" . - "/\n" . - "list-type=2\n" . - "host:bucket.s3.amazonaws.com\n" . - "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" . - "x-amz-date:20170606T121212Z\n" . - "x-amz-security-token:AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIe" . - "oIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlR" . - "d8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJab" . - "IQwj2ICCR/oLxBA==\n" . - "\n" . - "host;x-amz-content-sha256;x-amz-date;x-amz-security-token\n" . - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855, " . - "host;x-amz-content-sha256;x-amz-date;x-amz-security-token, " . - "AWS4-HMAC-SHA256\n" . - "20170606T121212Z\n" . - "20170606/us-east-1/s3/aws4_request\n" . - "c171e7a68355ef4e0e6e1003d2d4a79a7b06e7424e3000ba619f5f7882a3251e)", - 'authorization header request with token'); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageS3CertPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageS3CertPerlTest.pm deleted file mode 100644 index aa49d38c3..000000000 --- a/test/lib/pgBackRestTest/Module/Storage/StorageS3CertPerlTest.pm +++ /dev/null @@ -1,113 +0,0 @@ -#################################################################################################################################### -# S3 SSL Certificate Tests -# -# Verify that SSL certificate validation works on live S3 servers. -#################################################################################################################################### -package pgBackRestTest::Module::Storage::StorageS3CertPerlTest; -use parent 'pgBackRestTest::Env::ConfigEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Storable qw(dclone); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; -use pgBackRest::Protocol::Storage::Helper; - -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::VmTest; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - # Use long random string so bucket lookups will fail and expose access errors - my $strBucket = 'bnBfyKpXR8ZqQY5RXszxemRgvtmjXd4tf5HkFYhTpT9BndUCYMDy5NCCyRz'; - my $strEndpoint = 's3-us-west-2.amazonaws.com'; - my $strRegion = 'us-west-2'; - - # Options - $self->optionTestSet(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3); - $self->optionTestSet(CFGOPT_REPO_S3_KEY, BOGUS); - $self->optionTestSet(CFGOPT_REPO_S3_KEY_SECRET, BOGUS); - $self->optionTestSet(CFGOPT_REPO_S3_TOKEN, BOGUS); - $self->optionTestSet(CFGOPT_REPO_S3_BUCKET, $strBucket); - $self->optionTestSet(CFGOPT_REPO_S3_ENDPOINT, $strEndpoint); - $self->optionTestSet(CFGOPT_REPO_S3_REGION, $strRegion); - $self->optionTestSet(CFGOPT_STANZA, $self->stanza()); - - $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); - - ################################################################################################################################ - if ($self->begin('validation')) - { - if ($self->vm eq VM_U12) - { - &log(INFO, 'cannot test - certificates are no longer maintained for ' . $self->vm()); - } - else - { - #----------------------------------------------------------------------------------------------------------------------- - if ($self->vm() eq VM_CO7) - { - # Tests fails on co7 because by default certs cannot be located. This logic may need to be changed in the future if - # this bug gets fixed by Red Hat. UPDATE: The behavior changed here but it does not seems to be fixed. - $self->testException( - sub {storageRepo({strStanza => 'test1'})->list('/')}, ERROR_HOST_CONNECT, - 'SSL connect attempt failed with unknown error error.*certificate verify failed', - 'cert verify fails on ' . VM_CO7); - - # It should work when verification is disabled - $self->optionTestSetBool(CFGOPT_REPO_S3_VERIFY_TLS, false); - $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); - - $self->testException( - sub {storageRepo({strStanza => 'test2'})->list('/')}, ERROR_PROTOCOL, 'S3 request error \[403\] Forbidden.*', - 'connection succeeds with verification disabled, (expected) error on invalid access key'); - - $self->optionTestClear(CFGOPT_REPO_S3_VERIFY_TLS); - $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); - } - - #----------------------------------------------------------------------------------------------------------------------- - # CO7 doesn't locate certs automatically so specify the path - if ($self->vm() eq VM_CO7) - { - $self->optionTestSet(CFGOPT_REPO_S3_CA_FILE, '/etc/pki/tls/certs/ca-bundle.crt'); - $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); - } - - $self->testException( - sub {storageRepo({strStanza => 'test3'})->list('/')}, ERROR_PROTOCOL, 'S3 request error \[403\] Forbidden.*', - 'connection succeeds, (expected) error on invalid access key'); - - if ($self->vm() eq VM_CO7) - { - $self->optionTestClear(CFGOPT_REPO_S3_CA_FILE); - $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); - } - - #----------------------------------------------------------------------------------------------------------------------- - $self->optionTestSet(CFGOPT_REPO_S3_CA_PATH, '/bogus'); - $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); - - $self->testException( - sub {storageRepo({strStanza => 'test4'})->list('/')}, ERROR_HOST_CONNECT, - $self->vm() eq VM_CO6 ? 'SSL connect attempt failed with unknown error.*certificate verify failed' : 'No such file or directory', - 'invalid ca path'); - } - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageS3PerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageS3PerlTest.pm deleted file mode 100644 index 25bf32013..000000000 --- a/test/lib/pgBackRestTest/Module/Storage/StorageS3PerlTest.pm +++ /dev/null @@ -1,219 +0,0 @@ -#################################################################################################################################### -# S3 Storage Tests -#################################################################################################################################### -package pgBackRestTest::Module::Storage::StorageS3PerlTest; -use parent 'pgBackRestTest::Env::S3EnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use pgBackRest::Common::Log; -use pgBackRest::Common::String; -use pgBackRest::LibC qw(:crypto); -use pgBackRest::Storage::S3::Driver; - -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# initTest -#################################################################################################################################### -sub initTest -{ - my $self = shift; - - executeTest("$self->{strS3Command} rm --recursive s3://pgbackrest-dev"); -} - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - # Initialize the driver - my $oS3 = $self->initS3(); - my $oStorage = new pgBackRest::Storage::Local('', $oS3); - - # Test variables - my $strFile = 'file.txt'; - my $strFileContent = 'TESTDATA'; - my $iFileLength = length($strFileContent); - - ################################################################################################################################ - if ($self->begin('exists()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {$oStorage->exists($strFile)}, false, 'root file does not exist'); - - #--------------------------------------------------------------------------------------------------------------------------- - storageTest()->put($strFile, $strFileContent); - executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev"); - - $self->testResult(sub {$oStorage->exists($strFile)}, true, 'root file exists'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {$oStorage->pathExists('/path/to')}, false, 'sub path does not exist'); - $self->testResult(sub {$oStorage->exists("/path/to/${strFile}")}, false, 'sub file does not exist'); - - #--------------------------------------------------------------------------------------------------------------------------- - executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev/path/to/${strFile}"); - - $self->testResult(sub {$oStorage->pathExists('/path/to')}, true, 'sub path exists'); - # $oStorage->pathExists('/path/to'); - $self->testResult(sub {$oStorage->exists("/path/to/${strFile}")}, true, 'sub file exists'); - } - - ################################################################################################################################ - if ($self->begin('manifest()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {$oStorage->manifest('')}, '{. => {type => d}}', 'no files'); - - #--------------------------------------------------------------------------------------------------------------------------- - storageTest()->put($strFile, $strFileContent); - storageTest()->put("${strFile}2", $strFileContent . '2'); - - executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev"); - executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2"); - - $self->testResult( - sub {$oStorage->manifest('')}, - '{. => {type => d}, file.txt => {size => 8, type => f}, path => {type => d}, path/to => {type => d},' . - ' path/to/file.txt2 => {size => 9, type => f}}', - 'root path'); - $self->testResult( - sub {$oStorage->manifest('/path/to')}, '{. => {type => d}, file.txt2 => {size => 9, type => f}}', 'sub path'); - } - - ################################################################################################################################ - if ($self->begin('list()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {$oStorage->list('')}, '[undef]', 'no files'); - - #--------------------------------------------------------------------------------------------------------------------------- - storageTest()->put($strFile, $strFileContent); - storageTest()->put("${strFile}2", $strFileContent . '2'); - - executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev"); - executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2"); - - $self->testResult(sub {$oStorage->list('')}, '(file.txt, path)', 'root path'); - $self->testResult(sub {$oStorage->list('/path/to')}, 'file.txt2', 'sub path'); - } - - ################################################################################################################################ - if ($self->begin('remove()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - $oStorage->put($strFile, $strFileContent); - $oStorage->put("/path/to/${strFile}2", $strFileContent); - $oStorage->put("/path/to/${strFile}3", $strFileContent); - $oStorage->put("/path/to/${strFile}4 \@+", $strFileContent); - - $self->testResult( - sub {$oStorage->manifest('/')}, - '{. => {type => d}, file.txt => {size => 8, type => f}, path => {type => d}, path/to => {type => d},' . - ' path/to/file.txt2 => {size => 8, type => f}, path/to/file.txt3 => {size => 8, type => f},' . - ' path/to/file.txt4 @+ => {size => 8, type => f}}', - 'check manifest'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {$oStorage->remove('/path/to', {bRecurse => true})}, true, 'remove subpath'); - - $self->testResult( - sub {$oStorage->manifest('/')}, - '{. => {type => d}, file.txt => {size => 8, type => f}}', 'check manifest'); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {$oStorage->remove($strFile)}, true, 'remove file'); - - $self->testResult(sub {$oStorage->manifest('/')}, '{. => {type => d}}', 'check manifest'); - } - - ################################################################################################################################ - if ($self->begin('info()')) - { - #--------------------------------------------------------------------------------------------------------------------------- - storageTest()->put($strFile, $strFileContent); - storageTest()->put("${strFile}2", $strFileContent . '2'); - - executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev"); - executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev"); - executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2"); - - $self->testResult(sub {$oStorage->info($strFile)->size()}, 8, 'file size'); - $self->testResult(sub {$oStorage->info("/path/to/${strFile}2")->size()}, 9, 'file 2 size'); - } - - ################################################################################################################################ - if ($self->begin('openRead() && S3::FileRead')) - { - # Create a random 1mb file - my $strRandomFile = $self->testPath() . '/random@1mb.bin'; - executeTest("dd if=/dev/urandom of=${strRandomFile} bs=1024k count=1", {bSuppressStdErr => true}); - my $strRandom = ${storageTest()->get($strRandomFile)}; - - executeTest("$self->{strS3Command} cp ${strRandomFile} s3://pgbackrest-dev/path/to/${strFile}"); - - #--------------------------------------------------------------------------------------------------------------------------- - my $tBuffer; - my $oFileRead = $self->testResult(sub {$oS3->openRead("/path/to/${strFile}")}, '[object]', 'open read'); - $self->testResult(sub {$oFileRead->read(\$tBuffer, 524288)}, 524288, ' read half'); - $self->testResult(sub {$oFileRead->read(\$tBuffer, 524288)}, 524288, ' read half'); - $self->testResult(sub {$oFileRead->read(\$tBuffer, 512)}, 0, ' read 0'); - $self->testResult(length($tBuffer), 1048576, ' check length'); - $self->testResult(cryptoHashOne('sha1', $tBuffer), cryptoHashOne('sha1', $strRandom), ' check hash'); - } - - ################################################################################################################################ - if ($self->begin('openWrite() && S3::FileWrite')) - { - # Create a random 1mb file - my $strRandomFile = $self->testPath() . '/random1mb.bin'; - executeTest("dd if=/dev/urandom of=${strRandomFile} bs=1024k count=1", {bSuppressStdErr => true}); - my $strRandom = ${storageTest()->get($strRandomFile)}; - - #--------------------------------------------------------------------------------------------------------------------------- - my $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}")}, '[object]', 'open write'); - $self->testResult(sub {$oFileWrite->name()}, "/path/to/${strFile}", ' check filename'); - $self->testResult(sub {$oFileWrite->close()}, true, ' close without writing'); - - #--------------------------------------------------------------------------------------------------------------------------- - $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}" . '.@')}, '[object]', 'open write'); - $self->testResult(sub {$oFileWrite->write()}, 0, ' write undef'); - $self->testResult(sub {$oFileWrite->write(\$strFileContent)}, $iFileLength, ' write'); - $oFileWrite->close(); - - $self->testResult(sub {$oS3->exists("/path/to/${strFile}" . '.@')}, true, 'destination file exists'); - - # Test that a premature destroy (from error or otherwise) does not rename the file - #--------------------------------------------------------------------------------------------------------------------------- - $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/abort.file" . '.@')}, '[object]', 'open write'); - $self->testResult(sub {$oFileWrite->write()}, 0, ' write undef'); - $self->testResult(sub {$oFileWrite->write(\$strFileContent)}, $iFileLength, ' write'); - - undef($oFileWrite); - $self->testResult(sub {$oS3->exists("/path/to/abort.file")}, false, 'destination file does not exist'); - - #--------------------------------------------------------------------------------------------------------------------------- - $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}")}, '[object]', 'open write'); - - for (my $iIndex = 1; $iIndex <= 17; $iIndex++) - { - $self->testResult(sub {$oFileWrite->write(\$strRandom)}, 1024 * 1024, ' write 1mb'); - } - - $self->testResult(sub {$oFileWrite->close()}, true, ' close'); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageS3RequestPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageS3RequestPerlTest.pm deleted file mode 100644 index 7573be1e4..000000000 --- a/test/lib/pgBackRestTest/Module/Storage/StorageS3RequestPerlTest.pm +++ /dev/null @@ -1,184 +0,0 @@ -#################################################################################################################################### -# S3 Request Tests -#################################################################################################################################### -package pgBackRestTest::Module::Storage::StorageS3RequestPerlTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - - use IO::Socket::SSL; -use POSIX qw(strftime); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Http::Client; -use pgBackRest::Common::Log; -use pgBackRest::Common::Wait; -use pgBackRest::Storage::S3::Request; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# Port to use for testing -#################################################################################################################################### -use constant HTTPS_TEST_PORT => 9443; - -#################################################################################################################################### -# httpsServerResponse -#################################################################################################################################### -sub httpsServerResponse -{ - my $self = shift; - my $iResponseCode = shift; - my $strContent = shift; - - # Write header - $self->{oConnection}->write("HTTP/1.1 ${iResponseCode} GenericMessage\r\n"); - $self->{oConnection}->write(HTTP_HEADER_CONTENT_LENGTH . ': ' . (defined($strContent) ? length($strContent) : 0) . "\r\n"); - - # Write new line before content (even if there isn't any) - $self->{oConnection}->write("\r\n"); - - # Write content - if (defined($strContent)) - { - $self->{oConnection}->write($strContent); - } - - # This will block until the connection is closed by the client - $self->{oConnection}->read(); -} - -#################################################################################################################################### -# httpsServerAccept -#################################################################################################################################### -sub httpsServerAccept -{ - my $self = shift; - - # Wait for a connection - $self->{oConnection} = $self->{oSocketServer}->accept() - or confess "failed to accept or handshake $!, $SSL_ERROR"; - &log(INFO, " * socket server connected"); -} - -#################################################################################################################################### -# httpsServer -#################################################################################################################################### -sub httpsServer -{ - my $self = shift; - my $fnServer = shift; - - # Fork off the server - if (fork() == 0) - { - # Run server function - $fnServer->(); - - exit 0; - } -} - -#################################################################################################################################### -# Start the https testing server -#################################################################################################################################### -sub initModule -{ - my $self = shift; - - # Open the domain socket - $self->{oSocketServer} = IO::Socket::SSL->new( - LocalAddr => '127.0.0.1', LocalPort => HTTPS_TEST_PORT, Listen => 1, SSL_cert_file => CERT_FAKE_SERVER, - SSL_key_file => CERT_FAKE_SERVER_KEY) - or confess "unable to open https server for testing: $!"; - &log(INFO, " * socket server open"); -} - -#################################################################################################################################### -# Stop the https testing server -#################################################################################################################################### -sub cleanModule -{ - my $self = shift; - - # Shutdown server - $self->{oSocketServer}->close(); - &log(INFO, " * socket server closed"); -} - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - # Initialize request object - my $oS3Request = new pgBackRest::Storage::S3::Request( - BOGUS, BOGUS, BOGUS, BOGUS, BOGUS, {strHost => '127.0.0.1', iPort => HTTPS_TEST_PORT, bVerifySsl => false}); - - ################################################################################################################################ - if ($self->begin('success')) - { - $self->httpsServer(sub - { - $self->httpsServerAccept(); - $self->httpsServerResponse(200); - - #----------------------------------------------------------------------------------------------------------------------- - $self->httpsServerAccept(); - $self->httpsServerResponse(200); - }); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {$oS3Request->request(HTTP_VERB_GET)}, undef, 'successful request'); - $self->testResult(sub {$oS3Request->request(HTTP_VERB_GET)}, undef, 'successful request'); - } - - ################################################################################################################################ - if ($self->begin('retry')) - { - $self->httpsServer(sub - { - $self->httpsServerAccept(); - $self->httpsServerResponse(500); - - $self->httpsServerAccept(); - $self->httpsServerResponse(500); - - $self->httpsServerAccept(); - $self->httpsServerResponse(200); - - #----------------------------------------------------------------------------------------------------------------------- - $self->httpsServerAccept(); - $self->httpsServerResponse(500); - - $self->httpsServerAccept(); - $self->httpsServerResponse(500); - - $self->httpsServerAccept(); - $self->httpsServerResponse(500); - - $self->httpsServerAccept(); - $self->httpsServerResponse(500); - - $self->httpsServerAccept(); - $self->httpsServerResponse(500); - }); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {$oS3Request->request(HTTP_VERB_GET)}, undef, 'successful request after retries'); - $self->testException( - sub {$oS3Request->request(HTTP_VERB_GET)}, ERROR_PROTOCOL, 'S3 request error after 5 tries \[500\] GenericMessage.*'); - } -} - -1; diff --git a/test/src/module/storage/posixTest.c b/test/src/module/storage/posixTest.c index e82c88ece..e61cc8f6f 100644 --- a/test/src/module/storage/posixTest.c +++ b/test/src/module/storage/posixTest.c @@ -465,6 +465,11 @@ testRun(void) storagePathNP(storageTest, strNew("/path/toot")), AssertError, "absolute path '/path/toot' is not in base path '/path/to'"); + // Path enforcement disabled + storagePathEnforceSet(storageTest, false); + TEST_RESULT_STR(strPtr(storagePathNP(storageTest, strNew("/bogus"))), "/bogus", "path enforce disabled"); + storagePathEnforceSet(storageTest, true); + TEST_ERROR(storagePathNP(storageTest, strNew(" not found in path expression '" BOGUS_STR)), AssertError,