From 206415d4c754053275666bfcc0eb138f39884c21 Mon Sep 17 00:00:00 2001 From: David Steele Date: Wed, 30 Aug 2017 16:34:05 -0400 Subject: [PATCH] Fixed an issue that could cause compression to abort on growing files. Reported by Jesper St John, Aleksandr Rogozin. --- doc/xml/release.xml | 19 +++++++++++++++++ lib/pgBackRest/Common/Io/Handle.pm | 17 +++++++++++++++ lib/pgBackRest/Storage/Filter/Gzip.pm | 6 ++++-- .../Module/Common/CommonIoHandleTest.pm | 5 ++++- .../Module/Storage/StorageFilterGzipTest.pm | 21 +++++++++++++++++++ 5 files changed, 65 insertions(+), 3 deletions(-) diff --git a/doc/xml/release.xml b/doc/xml/release.xml index a723585fa..6fc46a41a 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -13,6 +13,15 @@ + + + + + + +

Fixed an issue that could cause compression to abort on growing files.

+ + @@ -3286,6 +3295,11 @@ eradman + + Aleksandr Rogozin + arogozin + + Cynthia Shang cmwshang @@ -3301,6 +3315,11 @@ gregscds + + Jesper St John + Underhunden + + Todd Vernick gintoddic diff --git a/lib/pgBackRest/Common/Io/Handle.pm b/lib/pgBackRest/Common/Io/Handle.pm index d82e67a17..91344626f 100644 --- a/lib/pgBackRest/Common/Io/Handle.pm +++ b/lib/pgBackRest/Common/Io/Handle.pm @@ -55,6 +55,9 @@ sub new # Size tracks number of bytes read and written $self->{lSize} = 0; + # Initialize EOF to false + $self->{bEOF} = false; + # Return from function and log return values if any return logDebugReturn ( @@ -63,6 +66,17 @@ sub new ); } +#################################################################################################################################### +# eof - have reads reached eof? +# +# Note that there may be more bytes to be read but this is set to true whenever there is a zero read and back to false on a +# non-zero read. +#################################################################################################################################### +sub eof +{ + return shift->{bEOF}; +} + #################################################################################################################################### # read - read data from handle #################################################################################################################################### @@ -93,6 +107,9 @@ sub read # Update size $self->{lSize} += $iActualSize; + # Update EOF + $self->{bEOF} = $iActualSize == 0 ? true : false; + return $iActualSize; } diff --git a/lib/pgBackRest/Storage/Filter/Gzip.pm b/lib/pgBackRest/Storage/Filter/Gzip.pm index fe574a45f..2da2ad8bf 100644 --- a/lib/pgBackRest/Storage/Filter/Gzip.pm +++ b/lib/pgBackRest/Storage/Filter/Gzip.pm @@ -126,6 +126,8 @@ sub read if ($self->{strCompressType} eq STORAGE_COMPRESS) { + return 0 if $self->eof(); + my $lSizeBegin = defined($$rtBuffer) ? length($$rtBuffer) : 0; my $lUncompressedSize; my $lCompressedSize; @@ -172,8 +174,8 @@ sub read } } - # Actual size is the lesser of the local buffer size or requested size - if the local buffer is smaller than the requested size - # it means that there was nothing more to be read. + # Actual size is the lesser of the local buffer size or requested size - if the local buffer is smaller than the requested + # size it means that there was nothing more to be read my $iActualSize = $self->{lUncompressedBufferSize} < $iSize ? $self->{lUncompressedBufferSize} : $iSize; # Append the to the request buffer diff --git a/test/lib/pgBackRestTest/Module/Common/CommonIoHandleTest.pm b/test/lib/pgBackRestTest/Module/Common/CommonIoHandleTest.pm index de4c94737..31a6c5c86 100644 --- a/test/lib/pgBackRestTest/Module/Common/CommonIoHandleTest.pm +++ b/test/lib/pgBackRestTest/Module/Common/CommonIoHandleTest.pm @@ -105,12 +105,15 @@ sub run $self->testResult(sub {$oIoHandle->read(\$tContent, $iFileLengthHalf)}, $iFileLengthHalf, ' read part 1'); $self->testResult($tContent, substr($strFileContent, 0, $iFileLengthHalf), ' check read'); + $self->testResult(sub {$oIoHandle->eof()}, false, ' not eof'); + $self->testResult( sub {$oIoHandle->read( \$tContent, $iFileLength - $iFileLengthHalf)}, $iFileLength - $iFileLengthHalf, ' read part 2'); $self->testResult($tContent, $strFileContent, ' check read'); - $self->testResult(sub {$oIoHandle->read(\$tContent, 1)}, 0, ' eof'); + $self->testResult(sub {$oIoHandle->read(\$tContent, 1)}, 0, ' zero read'); + $self->testResult(sub {$oIoHandle->eof()}, true, ' eof'); } ################################################################################################################################ diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipTest.pm index d503c81b7..53539b168 100644 --- a/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipTest.pm +++ b/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipTest.pm @@ -182,6 +182,27 @@ sub run $self->testResult( sub {sha1_hex(${storageTest()->get($strFile)})}, '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content'); + #--------------------------------------------------------------------------------------------------------------------------- + $tBuffer = undef; + + my $oFile = $self->testResult( + sub {storageTest()->openWrite($strFile)}, '[object]', 'open file to extend during compression'); + + $oGzipIo = $self->testResult( + sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFile), {lCompressBufferMax => 4194304})}, + '[object]', ' new read compress'); + + $self->testResult(sub {$oFile->write(\$strFileContent)}, length($strFileContent), ' write first block'); + $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000) > 0}, true, ' read compressed first block (compression done)'); + + $self->testResult(sub {$oFile->write(\$strFileContent)}, length($strFileContent), ' write second block'); + $self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000)}, 0, ' read compressed = 0'); + + $self->testResult(sub {storageTest()->put($strFileGz, $tBuffer) > 0}, true, ' put content'); + executeTest("gzip -df ${strFileGz}"); + $self->testResult( + sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content'); + #--------------------------------------------------------------------------------------------------------------------------- storageTest()->put($strFileGz, $strFileContent);