diff --git a/doc/xml/release.xml b/doc/xml/release.xml index c55602223..a9f7f9b6d 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -21,6 +21,10 @@ + +

Move cryptographic hash functions to C using OpenSSL.

+ +

Split log levels into separate header file. Many modules that use debug.h do not need to do logging so this reduces dependencies for those modules.

 diff --git a/lib/pgBackRest/Common/Ini.pm b/lib/pgBackRest/Common/Ini.pm index 4f5cb9ce7..82b584791 100644 --- a/lib/pgBackRest/Common/Ini.pm +++ b/lib/pgBackRest/Common/Ini.pm @@ -8,18 +8,16 @@ use warnings FATAL => qw(all); use Carp qw(confess); use English '-no_match_vars'; -use Digest::SHA; use Exporter qw(import); our @EXPORT = qw(); -use Fcntl qw(:mode O_WRONLY O_CREAT O_TRUNC); -use File::Basename qw(dirname basename); -use IO::Handle; +use File::Basename qw(dirname); use JSON::PP; use Storable qw(dclone); use pgBackRest::Common::Exception; use pgBackRest::Common::Log; use pgBackRest::Common::String; +use pgBackRest::LibC qw(:crypto); use pgBackRest::Version; #################################################################################################################################### @@ -550,13 +548,9 @@ sub hash # Remove the old checksum delete($self->{oContent}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM}); - # Calculate the checksum - my $oSHA = Digest::SHA->new('sha1'); - my $oJSON = JSON::PP->new()->canonical()->allow_nonref(); - $oSHA->add($oJSON->encode($self->{oContent})); - # Set the new checksum - $self->{oContent}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = $oSHA->hexdigest(); + $self->{oContent}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = + cryptoHashOne('sha1', JSON::PP->new()->canonical()->allow_nonref()->encode($self->{oContent})); return $self->{oContent}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM}; } diff --git a/lib/pgBackRest/LibCAuto.pm b/lib/pgBackRest/LibCAuto.pm index e5f76c686..0746703bd 100644 --- a/lib/pgBackRest/LibCAuto.pm +++ b/lib/pgBackRest/LibCAuto.pm @@ -183,12 +183,6 @@ sub libcAutoExportTag 'pageChecksumTest', ], - cipher => - [ - 'CIPHER_MODE_ENCRYPT', - 'CIPHER_MODE_DECRYPT', - ], - config => [ 'CFGOPTVAL_INFO_OUTPUT_TEXT', @@ -350,6 +344,13 @@ sub libcAutoExportTag 'cfgOptionTotal', ], + crypto => + [ + 'CIPHER_MODE_ENCRYPT', + 'CIPHER_MODE_DECRYPT', + 'cryptoHashOne', + ], + debug => [ 'libcUvSize', diff --git a/lib/pgBackRest/Manifest.pm b/lib/pgBackRest/Manifest.pm index 336e895c6..8447b1928 100644 --- a/lib/pgBackRest/Manifest.pm +++ b/lib/pgBackRest/Manifest.pm @@ -11,7 +11,6 @@ use Carp qw(confess); use Exporter qw(import); our @EXPORT = qw(); use File::Basename qw(dirname basename); -use Digest::SHA; use Time::Local qw(timelocal); use pgBackRest::DbVersion; diff --git a/lib/pgBackRest/Storage/Filter/CipherBlock.pm b/lib/pgBackRest/Storage/Filter/CipherBlock.pm index b828e17da..9ef07f21d 100644 --- a/lib/pgBackRest/Storage/Filter/CipherBlock.pm +++ b/lib/pgBackRest/Storage/Filter/CipherBlock.pm @@ -15,7 +15,7 @@ use Exporter qw(import); use pgBackRest::Common::Exception; use pgBackRest::Common::Io::Base; use pgBackRest::Common::Log; -use pgBackRest::LibC qw(:cipher); +use pgBackRest::LibC qw(:crypto); use pgBackRest::Storage::Base; #################################################################################################################################### diff --git a/lib/pgBackRest/Storage/Filter/Sha.pm b/lib/pgBackRest/Storage/Filter/Sha.pm index a73387da5..2ddbc3e78 100644 --- a/lib/pgBackRest/Storage/Filter/Sha.pm +++ b/lib/pgBackRest/Storage/Filter/Sha.pm @@ -50,7 +50,7 @@ sub new $self->{strAlgorithm} = $strAlgorithm; # Create SHA object - $self->{oSha} = Digest::SHA->new($self->{strAlgorithm}); + $self->{oSha} = new pgBackRest::LibC::Crypto::Hash($self->{strAlgorithm}); # Return from function and log return values if any return logDebugReturn @@ -76,7 +76,7 @@ sub read # Calculate sha for the returned buffer if ($iActualSize > 0) { - $self->{oSha}->add($tShaBuffer); + $self->{oSha}->process($tShaBuffer); $$rtBuffer .= $tShaBuffer; } @@ -93,7 +93,7 @@ sub write my $rtBuffer = shift; # Calculate sha for the buffer - $self->{oSha}->add($$rtBuffer); + $self->{oSha}->process($$rtBuffer); # Call the io method return $self->parent()->write($rtBuffer); @@ -109,7 +109,7 @@ sub close if (defined($self->{oSha})) { # Set result - $self->resultSet(STORAGE_FILTER_SHA, $self->{oSha}->hexdigest()); + $self->resultSet(STORAGE_FILTER_SHA, $self->{oSha}->result()); # Delete the sha object delete($self->{oSha}); diff --git a/lib/pgBackRest/Storage/S3/Auth.pm b/lib/pgBackRest/Storage/S3/Auth.pm index ca08f9205..7744cc2a2 100644 --- a/lib/pgBackRest/Storage/S3/Auth.pm +++ b/lib/pgBackRest/Storage/S3/Auth.pm @@ -11,12 +11,13 @@ use warnings FATAL => qw(all); use Carp qw(confess); use English '-no_match_vars'; -use Digest::SHA qw(hmac_sha256 hmac_sha256_hex sha256_hex); +use Digest::SHA qw(hmac_sha256 hmac_sha256_hex); use Exporter qw(import); our @EXPORT = qw(); use POSIX qw(strftime); use pgBackRest::Common::Log; +use pgBackRest::LibC qw(:crypto); #################################################################################################################################### # Constants @@ -37,7 +38,7 @@ use constant S3_HEADER_HOST => 'host'; use constant S3_HEADER_TOKEN => 'x-amz-security-token'; push @EXPORT, qw(S3_HEADER_TOKEN); -use constant PAYLOAD_DEFAULT_HASH => sha256_hex(''); +use constant PAYLOAD_DEFAULT_HASH => cryptoHashOne('sha256', ''); push @EXPORT, qw(PAYLOAD_DEFAULT_HASH); #################################################################################################################################### @@ -257,7 +258,7 @@ sub s3AuthorizationHeader # Create authorization string my ($strCanonicalRequest, $strSignedHeaders) = s3CanonicalRequest($strVerb, $strUri, $strQuery, $hHeader, $strPayloadHash); - my $strStringToSign = s3StringToSign($strDateTime, $strRegion, sha256_hex($strCanonicalRequest)); + my $strStringToSign = s3StringToSign($strDateTime, $strRegion, cryptoHashOne('sha256', $strCanonicalRequest)); $hHeader->{&S3_HEADER_AUTHORIZATION} = AWS4_HMAC_SHA256 . " Credential=${strAccessKeyId}/" . substr($strDateTime, 0, 8) . "/${strRegion}/" . S3 . qw(/) . diff --git a/lib/pgBackRest/Storage/S3/Request.pm b/lib/pgBackRest/Storage/S3/Request.pm index 759e98526..5fc085067 100644 --- a/lib/pgBackRest/Storage/S3/Request.pm +++ b/lib/pgBackRest/Storage/S3/Request.pm @@ -8,7 +8,6 @@ use warnings FATAL => qw(all); use Carp qw(confess); use English '-no_match_vars'; -use Digest::SHA qw(hmac_sha256 hmac_sha256_hex sha256_hex); use Exporter qw(import); our @EXPORT = qw(); use IO::Socket::SSL; @@ -20,6 +19,7 @@ use pgBackRest::Common::Io::Base; use pgBackRest::Common::Log; use pgBackRest::Common::String; use pgBackRest::Common::Xml; +use pgBackRest::LibC qw(:crypto); use pgBackRest::Storage::S3::Auth; #################################################################################################################################### @@ -152,7 +152,7 @@ sub request $bRetry = false; # Set content length and hash - $hHeader->{&S3_HEADER_CONTENT_SHA256} = defined($rstrBody) ? sha256_hex($$rstrBody) : PAYLOAD_DEFAULT_HASH; + $hHeader->{&S3_HEADER_CONTENT_SHA256} = defined($rstrBody) ? cryptoHashOne('sha256', $$rstrBody) : PAYLOAD_DEFAULT_HASH; $hHeader->{&S3_HEADER_CONTENT_LENGTH} = defined($rstrBody) ? length($$rstrBody) : 0; # Generate authorization header diff --git a/libc/LibC.xs b/libc/LibC.xs index cabf2335a..ca2e034c6 100644 --- a/libc/LibC.xs +++ b/libc/LibC.xs @@ -68,6 +68,7 @@ XSH includes These includes define data structures that are required for the C to Perl interface but are not part of the regular C source. ***********************************************************************************************************************************/ #include "xs/crypto/cipherBlock.xsh" +#include "xs/crypto/hash.xsh" #include "xs/common/encode.xsh" /*********************************************************************************************************************************** @@ -95,6 +96,7 @@ INCLUDE: xs/config/config.xs INCLUDE: xs/config/configTest.xs INCLUDE: xs/config/define.xs INCLUDE: xs/crypto/cipherBlock.xs +INCLUDE: xs/crypto/hash.xs INCLUDE: xs/crypto/random.xs INCLUDE: xs/postgres/pageChecksum.xs INCLUDE: xs/storage/storage.xs diff --git a/libc/Makefile.PL b/libc/Makefile.PL index 80a1ba942..3e87450f5 100644 --- a/libc/Makefile.PL +++ b/libc/Makefile.PL @@ -59,6 +59,7 @@ my @stryCFile = 'config/parse.c', 'crypto/cipherBlock.c', 'crypto/crypto.c', + 'crypto/hash.c', 'crypto/random.c', 'perl/config.c', 'postgres/pageChecksum.c', diff --git a/libc/build/lib/pgBackRestLibC/Build.pm b/libc/build/lib/pgBackRestLibC/Build.pm index 6458da2bc..4dcd91259 100644 --- a/libc/build/lib/pgBackRestLibC/Build.pm +++ b/libc/build/lib/pgBackRestLibC/Build.pm @@ -50,14 +50,6 @@ my $rhExport = )], }, - 'cipher' => - { - &BLD_EXPORTTYPE_SUB => [qw( - CIPHER_MODE_ENCRYPT - CIPHER_MODE_DECRYPT - )], - }, - 'config' => { &BLD_EXPORTTYPE_SUB => [qw( @@ -82,6 +74,15 @@ my $rhExport = )], }, + 'crypto' => + { + &BLD_EXPORTTYPE_SUB => [qw( + CIPHER_MODE_ENCRYPT + CIPHER_MODE_DECRYPT + cryptoHashOne + )], + }, + 'debug' => { &BLD_EXPORTTYPE_SUB => [qw( diff --git a/libc/typemap b/libc/typemap index 06352cd25..b96e4b2c9 100644 --- a/libc/typemap +++ b/libc/typemap @@ -1 +1,2 @@ pgBackRest::LibC::Cipher::Block T_PTROBJ +pgBackRest::LibC::Crypto::Hash T_PTROBJ diff --git a/libc/xs/crypto/hash.xs b/libc/xs/crypto/hash.xs new file mode 100644 index 000000000..49f75f0a3 --- /dev/null +++ b/libc/xs/crypto/hash.xs @@ -0,0 +1,96 @@ +#################################################################################################################################### +# Cryptographic Hashes Perl Exports +# +# XS wrapper for functions in cipher/hash.c. +#################################################################################################################################### + +MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::Crypto::Hash + +#################################################################################################################################### +pgBackRest::LibC::Crypto::Hash +new(class, type) + const char *class + const char *type +CODE: + RETVAL = NULL; + + // Don't warn when class param is used + (void)class; + + MEM_CONTEXT_XS_NEW_BEGIN("cryptoHashXs") + { + RETVAL = memNew(sizeof(CryptoHashXs)); + RETVAL->memContext = MEM_COMTEXT_XS(); + RETVAL->pxPayload = cryptoHashNew(strNew(type)); + } + MEM_CONTEXT_XS_NEW_END(); +OUTPUT: + RETVAL + +#################################################################################################################################### +void +process(self, message) + pgBackRest::LibC::Crypto::Hash self + SV *message +CODE: + MEM_CONTEXT_XS_TEMP_BEGIN() + { + STRLEN messageSize; + const unsigned char *messagePtr = (const unsigned char *)SvPV(message, messageSize); + + cryptoHashProcessC(self->pxPayload, messagePtr, messageSize); + } + MEM_CONTEXT_XS_TEMP_END(); + +#################################################################################################################################### +SV * +result(self) + pgBackRest::LibC::Crypto::Hash self +CODE: + RETVAL = NULL; + + MEM_CONTEXT_XS_TEMP_BEGIN() + { + String *hash = cryptoHashHex(self->pxPayload); + + RETVAL = newSV(strSize(hash)); + SvPOK_only(RETVAL); + strcpy((char *)SvPV_nolen(RETVAL), strPtr(hash)); + SvCUR_set(RETVAL, strSize(hash)); + } + MEM_CONTEXT_XS_TEMP_END(); +OUTPUT: + RETVAL + +#################################################################################################################################### +void +DESTROY(self) + pgBackRest::LibC::Crypto::Hash self +CODE: + MEM_CONTEXT_XS_DESTROY(self->memContext); + +MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC + +#################################################################################################################################### +SV * +cryptoHashOne(type, message) + const char *type + SV *message +CODE: + RETVAL = NULL; + + MEM_CONTEXT_XS_TEMP_BEGIN() + { + STRLEN messageSize; + const unsigned char *messagePtr = (const unsigned char *)SvPV(message, messageSize); + + String *hash = cryptoHashOneC(strNew(type), messagePtr, messageSize); + + RETVAL = newSV(strSize(hash)); + SvPOK_only(RETVAL); + strcpy((char *)SvPV_nolen(RETVAL), strPtr(hash)); + SvCUR_set(RETVAL, strSize(hash)); + } + MEM_CONTEXT_XS_TEMP_END(); +OUTPUT: + RETVAL diff --git a/libc/xs/crypto/hash.xsh b/libc/xs/crypto/hash.xsh new file mode 100644 index 000000000..8e30ae7f6 --- /dev/null +++ b/libc/xs/crypto/hash.xsh @@ -0,0 +1,11 @@ +/*********************************************************************************************************************************** +Cryptographic Hashes XS Header +***********************************************************************************************************************************/ +#include "common/memContext.h" +#include "crypto/hash.h" + +typedef struct CryptoHashXs +{ + MemContext *memContext; + CryptoHash *pxPayload; +} CryptoHashXs, *pgBackRest__LibC__Crypto__Hash; diff --git a/src/Makefile b/src/Makefile index 733a2f15c..69b51465a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -87,6 +87,7 @@ SRCS = \ config/load.c \ config/parse.c \ crypto/cipherBlock.c \ + crypto/hash.c \ crypto/crypto.c \ crypto/random.c \ perl/config.c \ @@ -224,6 +225,9 @@ crypto/cipherBlock.o: crypto/cipherBlock.c common/debug.h common/error.auto.h co crypto/crypto.o: crypto/crypto.c common/debug.h common/log.h common/logLevel.h common/stackTrace.h common/type/convert.h crypto/crypto.h $(CC) $(CFLAGS) -c crypto/crypto.c -o crypto/crypto.o +crypto/hash.o: crypto/hash.c common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/string.h crypto/crypto.h crypto/hash.h + $(CC) $(CFLAGS) -c crypto/hash.c -o crypto/hash.o + crypto/random.o: crypto/random.c common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/stackTrace.h common/type/convert.h crypto/random.h $(CC) $(CFLAGS) -c crypto/random.c -o crypto/random.o @@ -233,7 +237,7 @@ main.o: main.c command/archive/get/get.h command/archive/push/push.h command/com perl/config.o: perl/config.c common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h $(CC) $(CFLAGS) -c perl/config.c -o perl/config.o -perl/exec.o: perl/exec.c ../libc/LibC.h common/debug.h common/encode.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h crypto/cipher.h crypto/cipherBlock.h crypto/random.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/pageChecksum.h storage/driver/posix/driver.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/info.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/cipherBlock.xsh +perl/exec.o: perl/exec.c ../libc/LibC.h common/debug.h common/encode.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h crypto/cipher.h crypto/cipherBlock.h crypto/hash.h crypto/random.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/pageChecksum.h storage/driver/posix/driver.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/info.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/cipherBlock.xsh ../libc/xs/crypto/hash.xsh $(CC) $(CFLAGS) -c perl/exec.c -o perl/exec.o postgres/info.o: postgres/info.c common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h postgres/info.h postgres/type.h postgres/version.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h diff --git a/src/crypto/hash.c b/src/crypto/hash.c new file mode 100644 index 000000000..edf1843d9 --- /dev/null +++ b/src/crypto/hash.c @@ -0,0 +1,269 @@ +/*********************************************************************************************************************************** +Cryptographic Hashes +***********************************************************************************************************************************/ +#include + +#include +#include + +#include "common/debug.h" +#include "common/log.h" +#include "common/memContext.h" +#include "crypto/crypto.h" +#include "crypto/hash.h" + +/*********************************************************************************************************************************** +Track state during block encrypt/decrypt +***********************************************************************************************************************************/ +struct CryptoHash +{ + MemContext *memContext; // Context to store data + const EVP_MD *hashType; // Hash type + EVP_MD_CTX *hashContext; // Message hash context + Buffer *hash; // Hash in binary form +}; + +/*********************************************************************************************************************************** +New digest object +***********************************************************************************************************************************/ +CryptoHash * +cryptoHashNew(const String *type) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(STRING, type); + + FUNCTION_DEBUG_ASSERT(type != NULL); + FUNCTION_DEBUG_END(); + + // Only need to init once + if (!cryptoIsInit()) + cryptoInit(); + + // Allocate memory to hold process state + CryptoHash *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("CryptoHash") + { + // Allocate state and set context + this = memNew(sizeof(CryptoHash)); + this->memContext = MEM_CONTEXT_NEW(); + + // Lookup digest + if ((this->hashType = EVP_get_digestbyname(strPtr(type))) == NULL) + THROW_FMT(AssertError, "unable to load hash '%s'", strPtr(type)); + + // Create and initialize the context + if ((this->hashContext = EVP_MD_CTX_create()) == NULL) // {uncoverable - no failure condition known} + THROW(MemoryError, "unable to create hash context"); // {+uncoverable} + + if (EVP_DigestInit_ex(this->hashContext, this->hashType, NULL) != 1) // {uncoverable - no failure condition known} + THROW(MemoryError, "unable to initialize hash context"); // {+uncoverable} + + // Set free callback to ensure hash context is freed + memContextCallback(this->memContext, (MemContextCallback)cryptoHashFree, this); + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_DEBUG_RESULT(CRYPTO_HASH, this); +} + +/*********************************************************************************************************************************** +Add message data to the hash +***********************************************************************************************************************************/ +void cryptoHashProcessC(CryptoHash *this, const unsigned char *message, size_t messageSize) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(CRYPTO_HASH, this); + FUNCTION_DEBUG_PARAM(UCHARP, message); + FUNCTION_DEBUG_PARAM(SIZE, messageSize); + + FUNCTION_DEBUG_ASSERT(this != NULL); + FUNCTION_DEBUG_ASSERT(this->hashContext != NULL); + FUNCTION_DEBUG_ASSERT(message != NULL); + FUNCTION_DEBUG_END(); + + if (EVP_DigestUpdate( // {uncoverable - no failure condition known} + this->hashContext, message, messageSize) != 1) + { + THROW(MemoryError, "unable to process message hash"); // {+uncoverable} + } + + FUNCTION_DEBUG_RESULT_VOID(); +} + +/*********************************************************************************************************************************** +Add message data to the hash from a Buffer +***********************************************************************************************************************************/ +void cryptoHashProcess(CryptoHash *this, Buffer *message) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CRYPTO_HASH, this); + FUNCTION_TEST_PARAM(BUFFER, message); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_ASSERT(message != NULL); + FUNCTION_TEST_END(); + + cryptoHashProcessC(this, bufPtr(message), bufSize(message)); + + FUNCTION_TEST_RESULT_VOID(); +} + +/*********************************************************************************************************************************** +Add message data to the hash from a String +***********************************************************************************************************************************/ +void cryptoHashProcessStr(CryptoHash *this, String *message) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CRYPTO_HASH, this); + FUNCTION_TEST_PARAM(STRING, message); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_ASSERT(message != NULL); + FUNCTION_TEST_END(); + + cryptoHashProcessC(this, (const unsigned char *)strPtr(message), strSize(message)); + + FUNCTION_TEST_RESULT_VOID(); +} + +/*********************************************************************************************************************************** +Get binary representation of the hash +***********************************************************************************************************************************/ +const Buffer *cryptoHash(CryptoHash *this) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(CRYPTO_HASH, this); + + FUNCTION_DEBUG_ASSERT(this != NULL); + FUNCTION_DEBUG_END(); + + if (this->hash == NULL) + { + MEM_CONTEXT_BEGIN(this->memContext) + { + this->hash = bufNew((size_t)EVP_MD_size(this->hashType)); + unsigned int hashSize; + + if (EVP_DigestFinal_ex( // {uncoverable - no failure condition known} + this->hashContext, bufPtr(this->hash), &hashSize) != 1) + { + THROW(MemoryError, "unable to finalize message hash"); // {+uncoverable} + } + + // Free the context + EVP_MD_CTX_destroy(this->hashContext); + this->hashContext = NULL; + } + MEM_CONTEXT_END(); + } + + FUNCTION_DEBUG_RESULT(BUFFER, this->hash); +} + +/*********************************************************************************************************************************** +Get string representation of the hash +***********************************************************************************************************************************/ +String *cryptoHashHex(CryptoHash *this) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(CRYPTO_HASH, this); + + FUNCTION_DEBUG_ASSERT(this != NULL); + FUNCTION_DEBUG_END(); + + const Buffer *hash = cryptoHash(this); + String *hashStr = strNew(""); + + for (unsigned int hashIdx = 0; hashIdx < bufSize(hash); hashIdx++) + strCatFmt(hashStr, "%02x", bufPtr(hash)[hashIdx]); + + FUNCTION_DEBUG_RESULT(STRING, hashStr); +} + +/*********************************************************************************************************************************** +Free memory +***********************************************************************************************************************************/ +void cryptoHashFree(CryptoHash *this) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(CRYPTO_HASH, this); + FUNCTION_DEBUG_END(); + + if (this != NULL) + { + if (this->hashContext != NULL) + { + EVP_MD_CTX_destroy(this->hashContext); + this->hashContext = NULL; + } + + memContextFree(this->memContext); + } + + FUNCTION_DEBUG_RESULT_VOID(); +} + +/*********************************************************************************************************************************** +Get hash for one C buffer +***********************************************************************************************************************************/ +String * +cryptoHashOneC(const String *type, const unsigned char *message, size_t messageSize) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(STRING, type); + FUNCTION_DEBUG_PARAM(UCHARP, message); + + FUNCTION_DEBUG_ASSERT(type != NULL); + FUNCTION_DEBUG_ASSERT(message != NULL); + FUNCTION_DEBUG_END(); + + String *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + CryptoHash *hash = cryptoHashNew(type); + cryptoHashProcessC(hash, message, messageSize); + + memContextSwitch(MEM_CONTEXT_OLD()); + result = cryptoHashHex(hash); + memContextSwitch(MEM_CONTEXT_TEMP()); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_DEBUG_RESULT(STRING, result); +} + +/*********************************************************************************************************************************** +Get hash for one Buffer +***********************************************************************************************************************************/ +String * +cryptoHashOne(const String *type, Buffer *message) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, type); + FUNCTION_TEST_PARAM(BUFFER, message); + + FUNCTION_TEST_ASSERT(type != NULL); + FUNCTION_TEST_ASSERT(message != NULL); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RESULT(STRING, cryptoHashOneC(type, bufPtr(message), bufSize(message))); +} + +/*********************************************************************************************************************************** +Get hash for one String +***********************************************************************************************************************************/ +String * +cryptoHashOneStr(const String *type, String *message) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, type); + FUNCTION_TEST_PARAM(STRING, message); + + FUNCTION_TEST_ASSERT(type != NULL); + FUNCTION_TEST_ASSERT(message != NULL); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RESULT(STRING, cryptoHashOneC(type, (const unsigned char *)strPtr(message), strSize(message))); +} diff --git a/src/crypto/hash.h b/src/crypto/hash.h new file mode 100644 index 000000000..4683d420c --- /dev/null +++ b/src/crypto/hash.h @@ -0,0 +1,44 @@ +/*********************************************************************************************************************************** +Cryptographic Hashes +***********************************************************************************************************************************/ +#ifndef CRYPTO_HASH_H +#define CRYPTO_HASH_H + +/*********************************************************************************************************************************** +Hash object +***********************************************************************************************************************************/ +typedef struct CryptoHash CryptoHash; + +#include "common/type/string.h" + +/*********************************************************************************************************************************** +Hash types +***********************************************************************************************************************************/ +#define HASH_TYPE_MD5 "md5" +#define HASH_TYPE_SHA1 "sha1" +#define HASH_TYPE_SHA256 "sha256" + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +CryptoHash *cryptoHashNew(const String *type); +void cryptoHashProcess(CryptoHash *this, Buffer *message); +void cryptoHashProcessC(CryptoHash *this, const unsigned char *message, size_t messageSize); +void cryptoHashProcessStr(CryptoHash *this, String *message); +const Buffer *cryptoHash(CryptoHash *this); +String *cryptoHashHex(CryptoHash *this); +void cryptoHashFree(CryptoHash *this); + +String *cryptoHashOne(const String *type, Buffer *message); +String *cryptoHashOneC(const String *type, const unsigned char *message, size_t messageSize); +String *cryptoHashOneStr(const String *type, String *message); + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +#define FUNCTION_DEBUG_CRYPTO_HASH_TYPE \ + CryptoHash * +#define FUNCTION_DEBUG_CRYPTO_HASH_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "CryptoHash", buffer, bufferSize) + +#endif diff --git a/src/perl/embed.auto.c b/src/perl/embed.auto.c index 08660b6c9..5a0b13d92 100644 --- a/src/perl/embed.auto.c +++ b/src/perl/embed.auto.c @@ -6444,18 +6444,16 @@ static const EmbeddedModule embeddedModule[] = "use Carp qw(confess);\n" "use English '-no_match_vars';\n" "\n" - "use Digest::SHA;\n" "use Exporter qw(import);\n" "our @EXPORT = qw();\n" - "use Fcntl qw(:mode O_WRONLY O_CREAT O_TRUNC);\n" - "use File::Basename qw(dirname basename);\n" - "use IO::Handle;\n" + "use File::Basename qw(dirname);\n" "use JSON::PP;\n" "use Storable qw(dclone);\n" "\n" "use pgBackRest::Common::Exception;\n" "use pgBackRest::Common::Log;\n" "use pgBackRest::Common::String;\n" + "use pgBackRest::LibC qw(:crypto);\n" "use pgBackRest::Version;\n" "\n" "\n" @@ -6987,12 +6985,8 @@ static const EmbeddedModule embeddedModule[] = "delete($self->{oContent}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM});\n" "\n" "\n" - "my $oSHA = Digest::SHA->new('sha1');\n" - "my $oJSON = JSON::PP->new()->canonical()->allow_nonref();\n" - "$oSHA->add($oJSON->encode($self->{oContent}));\n" - "\n" - "\n" - "$self->{oContent}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = $oSHA->hexdigest();\n" + "$self->{oContent}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} =\n" + "cryptoHashOne('sha1', JSON::PP->new()->canonical()->allow_nonref()->encode($self->{oContent}));\n" "\n" "return $self->{oContent}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM};\n" "}\n" @@ -13055,12 +13049,6 @@ static const EmbeddedModule embeddedModule[] = "'pageChecksumTest',\n" "],\n" "\n" - "cipher =>\n" - "[\n" - "'CIPHER_MODE_ENCRYPT',\n" - "'CIPHER_MODE_DECRYPT',\n" - "],\n" - "\n" "config =>\n" "[\n" "'CFGOPTVAL_INFO_OUTPUT_TEXT',\n" @@ -13222,6 +13210,13 @@ static const EmbeddedModule embeddedModule[] = "'cfgOptionTotal',\n" "],\n" "\n" + "crypto =>\n" + "[\n" + "'CIPHER_MODE_ENCRYPT',\n" + "'CIPHER_MODE_DECRYPT',\n" + "'cryptoHashOne',\n" + "],\n" + "\n" "debug =>\n" "[\n" "'libcUvSize',\n" @@ -13625,7 +13620,6 @@ static const EmbeddedModule embeddedModule[] = "use Exporter qw(import);\n" "our @EXPORT = qw();\n" "use File::Basename qw(dirname basename);\n" - "use Digest::SHA;\n" "use Time::Local qw(timelocal);\n" "\n" "use pgBackRest::DbVersion;\n" @@ -20790,7 +20784,7 @@ static const EmbeddedModule embeddedModule[] = "use pgBackRest::Common::Exception;\n" "use pgBackRest::Common::Io::Base;\n" "use pgBackRest::Common::Log;\n" - "use pgBackRest::LibC qw(:cipher);\n" + "use pgBackRest::LibC qw(:crypto);\n" "use pgBackRest::Storage::Base;\n" "\n" "\n" @@ -21274,7 +21268,7 @@ static const EmbeddedModule embeddedModule[] = "$self->{strAlgorithm} = $strAlgorithm;\n" "\n" "\n" - "$self->{oSha} = Digest::SHA->new($self->{strAlgorithm});\n" + "$self->{oSha} = new pgBackRest::LibC::Crypto::Hash($self->{strAlgorithm});\n" "\n" "\n" "return logDebugReturn\n" @@ -21300,7 +21294,7 @@ static const EmbeddedModule embeddedModule[] = "\n" "if ($iActualSize > 0)\n" "{\n" - "$self->{oSha}->add($tShaBuffer);\n" + "$self->{oSha}->process($tShaBuffer);\n" "$$rtBuffer .= $tShaBuffer;\n" "}\n" "\n" @@ -21317,7 +21311,7 @@ static const EmbeddedModule embeddedModule[] = "my $rtBuffer = shift;\n" "\n" "\n" - "$self->{oSha}->add($$rtBuffer);\n" + "$self->{oSha}->process($$rtBuffer);\n" "\n" "\n" "return $self->parent()->write($rtBuffer);\n" @@ -21333,7 +21327,7 @@ static const EmbeddedModule embeddedModule[] = "if (defined($self->{oSha}))\n" "{\n" "\n" - "$self->resultSet(STORAGE_FILTER_SHA, $self->{oSha}->hexdigest());\n" + "$self->resultSet(STORAGE_FILTER_SHA, $self->{oSha}->result());\n" "\n" "\n" "delete($self->{oSha});\n" @@ -23695,12 +23689,13 @@ static const EmbeddedModule embeddedModule[] = "use Carp qw(confess);\n" "use English '-no_match_vars';\n" "\n" - "use Digest::SHA qw(hmac_sha256 hmac_sha256_hex sha256_hex);\n" + "use Digest::SHA qw(hmac_sha256 hmac_sha256_hex);\n" "use Exporter qw(import);\n" "our @EXPORT = qw();\n" "use POSIX qw(strftime);\n" "\n" "use pgBackRest::Common::Log;\n" + "use pgBackRest::LibC qw(:crypto);\n" "\n" "\n" "\n" @@ -23721,7 +23716,7 @@ static const EmbeddedModule embeddedModule[] = "use constant S3_HEADER_TOKEN => 'x-amz-security-token';\n" "push @EXPORT, qw(S3_HEADER_TOKEN);\n" "\n" - "use constant PAYLOAD_DEFAULT_HASH => sha256_hex('');\n" + "use constant PAYLOAD_DEFAULT_HASH => cryptoHashOne('sha256', '');\n" "push @EXPORT, qw(PAYLOAD_DEFAULT_HASH);\n" "\n" "\n" @@ -23941,7 +23936,7 @@ static const EmbeddedModule embeddedModule[] = "\n" "\n" "my ($strCanonicalRequest, $strSignedHeaders) = s3CanonicalRequest($strVerb, $strUri, $strQuery, $hHeader, $strPayloadHash);\n" - "my $strStringToSign = s3StringToSign($strDateTime, $strRegion, sha256_hex($strCanonicalRequest));\n" + "my $strStringToSign = s3StringToSign($strDateTime, $strRegion, cryptoHashOne('sha256', $strCanonicalRequest));\n" "\n" "$hHeader->{&S3_HEADER_AUTHORIZATION} =\n" "AWS4_HMAC_SHA256 . \" Credential=${strAccessKeyId}/\" . substr($strDateTime, 0, 8) . \"/${strRegion}/\" . S3 . qw(/) .\n" @@ -24810,7 +24805,6 @@ static const EmbeddedModule embeddedModule[] = "use Carp qw(confess);\n" "use English '-no_match_vars';\n" "\n" - "use Digest::SHA qw(hmac_sha256 hmac_sha256_hex sha256_hex);\n" "use Exporter qw(import);\n" "our @EXPORT = qw();\n" "use IO::Socket::SSL;\n" @@ -24822,6 +24816,7 @@ static const EmbeddedModule embeddedModule[] = "use pgBackRest::Common::Log;\n" "use pgBackRest::Common::String;\n" "use pgBackRest::Common::Xml;\n" + "use pgBackRest::LibC qw(:crypto);\n" "use pgBackRest::Storage::S3::Auth;\n" "\n" "\n" @@ -24954,7 +24949,7 @@ static const EmbeddedModule embeddedModule[] = "$bRetry = false;\n" "\n" "\n" - "$hHeader->{&S3_HEADER_CONTENT_SHA256} = defined($rstrBody) ? sha256_hex($$rstrBody) : PAYLOAD_DEFAULT_HASH;\n" + "$hHeader->{&S3_HEADER_CONTENT_SHA256} = defined($rstrBody) ? cryptoHashOne('sha256', $$rstrBody) : PAYLOAD_DEFAULT_HASH;\n" "$hHeader->{&S3_HEADER_CONTENT_LENGTH} = defined($rstrBody) ? length($$rstrBody) : 0;\n" "\n" "\n" diff --git a/src/perl/libc.auto.c b/src/perl/libc.auto.c index 9d8c700bf..0bb711632 100644 --- a/src/perl/libc.auto.c +++ b/src/perl/libc.auto.c @@ -76,6 +76,7 @@ XSH includes These includes define data structures that are required for the C to Perl interface but are not part of the regular C source. ***********************************************************************************************************************************/ #include "xs/crypto/cipherBlock.xsh" +#include "xs/crypto/hash.xsh" #include "xs/common/encode.xsh" /*********************************************************************************************************************************** @@ -259,7 +260,10 @@ XS_EUPXS(XS_pgBackRest__LibC_libcUvSize) /* INCLUDE: Including 'xs/crypto/cipherBlock.xs' from 'xs/config/define.xs' */ -/* INCLUDE: Including 'xs/crypto/random.xs' from 'xs/crypto/cipherBlock.xs' */ +/* INCLUDE: Including 'xs/crypto/hash.xs' from 'xs/crypto/cipherBlock.xs' */ + + +/* INCLUDE: Including 'xs/crypto/random.xs' from 'xs/crypto/hash.xs' */ /* INCLUDE: Including 'xs/postgres/pageChecksum.xs' from 'xs/crypto/random.xs' */ @@ -415,7 +419,174 @@ XS_EUPXS(XS_pgBackRest__LibC_randomBytes) } -/* INCLUDE: Returning to 'xs/crypto/cipherBlock.xs' from 'xs/crypto/random.xs' */ +/* INCLUDE: Returning to 'xs/crypto/hash.xs' from 'xs/crypto/random.xs' */ + + +XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_new); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_new) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "class, type"); + { + const char * class = (const char *)SvPV_nolen(ST(0)) +; + const char * type = (const char *)SvPV_nolen(ST(1)) +; + pgBackRest__LibC__Crypto__Hash RETVAL; + RETVAL = NULL; + + // Don't warn when class param is used + (void)class; + + MEM_CONTEXT_XS_NEW_BEGIN("cryptoHashXs") + { + RETVAL = memNew(sizeof(CryptoHashXs)); + RETVAL->memContext = MEM_COMTEXT_XS(); + RETVAL->pxPayload = cryptoHashNew(strNew(type)); + } + MEM_CONTEXT_XS_NEW_END(); + { + SV * RETVALSV; + RETVALSV = sv_newmortal(); + sv_setref_pv(RETVALSV, "pgBackRest::LibC::Crypto::Hash", (void*)RETVAL); + ST(0) = RETVALSV; + } + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_process); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_process) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "self, message"); + { + pgBackRest__LibC__Crypto__Hash self; + SV * message = ST(1) +; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Crypto::Hash")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Crypto__Hash,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Crypto::Hash::process", + "self", "pgBackRest::LibC::Crypto::Hash") +; + MEM_CONTEXT_XS_TEMP_BEGIN() + { + STRLEN messageSize; + const unsigned char *messagePtr = (const unsigned char *)SvPV(message, messageSize); + + cryptoHashProcessC(self->pxPayload, messagePtr, messageSize); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + +XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_result); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_result) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + pgBackRest__LibC__Crypto__Hash self; + SV * RETVAL; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Crypto::Hash")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Crypto__Hash,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Crypto::Hash::result", + "self", "pgBackRest::LibC::Crypto::Hash") +; + RETVAL = NULL; + + MEM_CONTEXT_XS_TEMP_BEGIN() + { + String *hash = cryptoHashHex(self->pxPayload); + + RETVAL = newSV(strSize(hash)); + SvPOK_only(RETVAL); + strcpy((char *)SvPV_nolen(RETVAL), strPtr(hash)); + SvCUR_set(RETVAL, strSize(hash)); + } + MEM_CONTEXT_XS_TEMP_END(); + RETVAL = sv_2mortal(RETVAL); + ST(0) = RETVAL; + } + XSRETURN(1); +} + + +XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_DESTROY); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Crypto__Hash_DESTROY) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + pgBackRest__LibC__Crypto__Hash self; + + if (SvROK(ST(0))) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Crypto__Hash,tmp); + } + else + Perl_croak_nocontext("%s: %s is not a reference", + "pgBackRest::LibC::Crypto::Hash::DESTROY", + "self") +; + MEM_CONTEXT_XS_DESTROY(self->memContext); + } + XSRETURN_EMPTY; +} + + +XS_EUPXS(XS_pgBackRest__LibC_cryptoHashOne); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC_cryptoHashOne) +{ + dVAR; dXSARGS; + if (items != 2) + croak_xs_usage(cv, "type, message"); + { + const char * type = (const char *)SvPV_nolen(ST(0)) +; + SV * message = ST(1) +; + SV * RETVAL; + RETVAL = NULL; + + MEM_CONTEXT_XS_TEMP_BEGIN() + { + STRLEN messageSize; + const unsigned char *messagePtr = (const unsigned char *)SvPV(message, messageSize); + + String *hash = cryptoHashOneC(strNew(type), messagePtr, messageSize); + + RETVAL = newSV(strSize(hash)); + SvPOK_only(RETVAL); + strcpy((char *)SvPV_nolen(RETVAL), strPtr(hash)); + SvCUR_set(RETVAL, strSize(hash)); + } + MEM_CONTEXT_XS_TEMP_END(); + RETVAL = sv_2mortal(RETVAL); + ST(0) = RETVAL; + } + XSRETURN(1); +} + + +/* INCLUDE: Returning to 'xs/crypto/cipherBlock.xs' from 'xs/crypto/hash.xs' */ XS_EUPXS(XS_pgBackRest__LibC__Cipher__Block_new); /* prototype to pass -Wmissing-prototypes */ @@ -1047,6 +1218,11 @@ XS_EXTERNAL(boot_pgBackRest__LibC) newXS_deffile("pgBackRest::LibC::pageChecksumTest", XS_pgBackRest__LibC_pageChecksumTest); newXS_deffile("pgBackRest::LibC::pageChecksumBufferTest", XS_pgBackRest__LibC_pageChecksumBufferTest); newXS_deffile("pgBackRest::LibC::randomBytes", XS_pgBackRest__LibC_randomBytes); + newXS_deffile("pgBackRest::LibC::Crypto::Hash::new", XS_pgBackRest__LibC__Crypto__Hash_new); + newXS_deffile("pgBackRest::LibC::Crypto::Hash::process", XS_pgBackRest__LibC__Crypto__Hash_process); + newXS_deffile("pgBackRest::LibC::Crypto::Hash::result", XS_pgBackRest__LibC__Crypto__Hash_result); + newXS_deffile("pgBackRest::LibC::Crypto::Hash::DESTROY", XS_pgBackRest__LibC__Crypto__Hash_DESTROY); + newXS_deffile("pgBackRest::LibC::cryptoHashOne", XS_pgBackRest__LibC_cryptoHashOne); newXS_deffile("pgBackRest::LibC::Cipher::Block::new", XS_pgBackRest__LibC__Cipher__Block_new); newXS_deffile("pgBackRest::LibC::Cipher::Block::process", XS_pgBackRest__LibC__Cipher__Block_process); newXS_deffile("pgBackRest::LibC::Cipher::Block::flush", XS_pgBackRest__LibC__Cipher__Block_flush); diff --git a/test/define.yaml b/test/define.yaml index 76c34f0e8..4828ab299 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -285,6 +285,13 @@ unit: coverage: crypto/random: full + # ---------------------------------------------------------------------------------------------------------------------------- + - name: hash + total: 2 + + coverage: + crypto/hash: full + # ---------------------------------------------------------------------------------------------------------------------------- - name: cipherBlock total: 2 diff --git a/test/lib/pgBackRestTest/Env/HostEnvTest.pm b/test/lib/pgBackRestTest/Env/HostEnvTest.pm index 4cce5576f..d098a49c1 100644 --- a/test/lib/pgBackRestTest/Env/HostEnvTest.pm +++ b/test/lib/pgBackRestTest/Env/HostEnvTest.pm @@ -11,7 +11,6 @@ use strict; use warnings FATAL => qw(all); use Carp qw(confess); -use Digest::SHA qw(sha1_hex); use Exporter qw(import); our @EXPORT = qw(); use Storable qw(dclone); @@ -20,6 +19,7 @@ use pgBackRest::Archive::Common; use pgBackRest::Common::Log; use pgBackRest::Config::Config; use pgBackRest::DbVersion; +use pgBackRest::LibC qw(:crypto); use pgBackRest::Protocol::Storage::Helper; use pgBackRestTest::Env::Host::HostBackupTest; @@ -441,7 +441,7 @@ sub walGenerateContentChecksum {name => 'hParam', required => false, trace => true}, ); - return sha1_hex(${$self->walGenerateContent($strPgVersion, $hParam)}); + return cryptoHashOne('sha1', ${$self->walGenerateContent($strPgVersion, $hParam)}); } #################################################################################################################################### @@ -462,7 +462,7 @@ sub walGenerate my $rtWalContent = $self->walGenerateContent($strPgVersion, {iSourceNo => $iSourceNo}); my $strWalFile = - "${strWalPath}/${strWalSegment}" . ($bChecksum ? '-' . sha1_hex($rtWalContent) : '') . + "${strWalPath}/${strWalSegment}" . ($bChecksum ? '-' . cryptoHashOne('sha1', $rtWalContent) : '') . (defined($bPartial) && $bPartial ? '.partial' : ''); # Put the WAL segment and the ready file diff --git a/test/lib/pgBackRestTest/Module/Archive/ArchiveGetPerlTest.pm b/test/lib/pgBackRestTest/Module/Archive/ArchiveGetPerlTest.pm index 4a291a180..2354da185 100644 --- a/test/lib/pgBackRestTest/Module/Archive/ArchiveGetPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Archive/ArchiveGetPerlTest.pm @@ -12,7 +12,6 @@ use warnings FATAL => qw(all); use Carp qw(confess); use Storable qw(dclone); -use Digest::SHA qw(sha1_hex); use pgBackRest::Archive::Common; use pgBackRest::Archive::Get::Async; @@ -24,6 +23,7 @@ use pgBackRest::Common::Log; use pgBackRest::Config::Config; use pgBackRest::DbVersion; use pgBackRest::Manifest; +use pgBackRest::LibC qw(:crypto); use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; @@ -90,7 +90,7 @@ sub run # Define test file my $strFileContent = 'TESTDATA'; - my $strFileHash = sha1_hex($strFileContent); + my $strFileHash = cryptoHashOne('sha1', $strFileContent); my $iFileSize = length($strFileContent); my $strDestinationPath = $self->{strDbPath} . "/pg_xlog"; @@ -187,7 +187,7 @@ sub run archiveGetCheck(PG_VERSION_92, $self->dbSysId(PG_VERSION_92), $strWalSegment, false); # Using the returned values, confirm the correct file is read - $self->testResult(sub {sha1_hex(${storageRepo()->get($self->{strArchivePath} . "/" . $strArchiveId . "/" . + $self->testResult(sub {cryptoHashOne('sha1', ${storageRepo()->get($self->{strArchivePath} . "/" . $strArchiveId . "/" . substr($strWalSegment, 0, 16) . "/" . $strArchiveFile)})}, $strFileHash, 'check correct WAL archiveID when in multiple locations'); } @@ -223,7 +223,7 @@ sub run storageRepo()->pathCreate($strArchivePath); storageRepo()->put($strArchivePath . BOGUS, BOGUS); - my $strBogusHash = sha1_hex(BOGUS); + my $strBogusHash = cryptoHashOne('sha1', BOGUS); # Create path to copy file storageRepo()->pathCreate($strDestinationPath); @@ -232,7 +232,7 @@ sub run "non-WAL segment copied"); # Confirm the correct file is copied - $self->testResult(sub {sha1_hex(${storageRepo()->get($strDestinationFile)})}, $strBogusHash, + $self->testResult(sub {cryptoHashOne('sha1', ${storageRepo()->get($strDestinationFile)})}, $strBogusHash, ' check correct non-WAL copied from older archiveId'); # create same WAL segment in same DB but different archives and different hash values. Confirm latest one copied. @@ -257,7 +257,7 @@ sub run "WAL segment copied"); # Confirm the correct file is copied - $self->testResult(sub {sha1_hex(${storageRepo()->get($strDestinationFile)})}, $strFileHash, + $self->testResult(sub {cryptoHashOne('sha1', ${storageRepo()->get($strDestinationFile)})}, $strFileHash, ' check correct WAL copied when in multiple locations'); # Get files from an older DB version to simulate restoring from an old backup set to a database that is of that same version @@ -268,7 +268,7 @@ sub run $strWalSegmentName = "${strWalSegment}-${strFileHash}"; my $strWalContent = 'WALTESTDATA'; - my $strWalHash = sha1_hex($strWalContent); + my $strWalHash = cryptoHashOne('sha1', $strWalContent); # Store with actual data that will match the hash check storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true}); @@ -284,7 +284,7 @@ sub run "WAL segment copied from older db backupset to same version older db"); # Confirm the correct file is copied - $self->testResult(sub {sha1_hex(${storageRepo()->get($strDestinationFile)})}, $strWalHash, + $self->testResult(sub {cryptoHashOne('sha1', ${storageRepo()->get($strDestinationFile)})}, $strWalHash, ' check correct WAL copied from older db'); } diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockPerlTest.pm index dad5fa9a6..e90c5d085 100644 --- a/test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Storage/StorageFilterCipherBlockPerlTest.pm @@ -13,11 +13,10 @@ use Carp qw(confess); use English '-no_match_vars'; use Fcntl qw(O_RDONLY); -use Digest::SHA qw(sha1_hex); use pgBackRest::Common::Exception; use pgBackRest::Common::Log; -use pgBackRest::LibC qw(:random); +use pgBackRest::LibC qw(:random :crypto); use pgBackRest::Storage::Base; use pgBackRest::Storage::Filter::CipherBlock; use pgBackRest::Storage::Posix::Driver; @@ -153,7 +152,7 @@ sub run executeTest('cp ' . $self->dataPath() . "/filecopy.archive2.bin ${strFileBin}"); $self->testResult( - sub {sha1_hex(${storageTest()->get($strFileBin)})}, $strFileBinHash, 'bin test - check sha1'); + sub {cryptoHashOne('sha1', ${storageTest()->get($strFileBin)})}, $strFileBinHash, 'bin test - check sha1'); $tContent = ${storageTest()->get($strFileBin)}; @@ -165,7 +164,8 @@ sub run $self->testResult(sub {$oEncryptFileIo->write(\$tContent)}, length($tContent), ' write encrypted'); $self->testResult(sub {$oEncryptFileIo->close()}, true, ' close'); $self->testResult( - sub {sha1_hex(${storageTest()->get($strFileBinEncrypt)}) ne $strFileBinHash}, true, ' check sha1 different'); + sub {cryptoHashOne('sha1', ${storageTest()->get($strFileBinEncrypt)}) ne $strFileBinHash}, true, + ' check sha1 different'); my $oEncryptBinFileIo = $self->testResult( sub {new pgBackRest::Storage::Filter::CipherBlock( @@ -174,7 +174,7 @@ sub run '[object]', 'new read encrypted bin file'); $self->testResult(sub {$oEncryptBinFileIo->read(\$tBuffer, 16777216)}, 16777216, ' read 16777216 bytes'); - $self->testResult(sub {sha1_hex($tBuffer)}, $strFileBinHash, ' check sha1 same as original'); + $self->testResult(sub {cryptoHashOne('sha1', $tBuffer)}, $strFileBinHash, ' check sha1 same as original'); $self->testResult(sub {$oEncryptBinFileIo->close()}, true, ' close'); # Try to read the file with the wrong passphrase @@ -188,7 +188,7 @@ sub run '[object]', 'new read Encrypted bin file with wrong passphrase'); $self->testResult(sub {$oEncryptBinFileIo->read(\$tBuffer, 16777216)}, 16777216, ' read all bytes'); - $self->testResult(sub {sha1_hex($tBuffer) ne $strFileBinHash}, true, ' check sha1 NOT same as original'); + $self->testResult(sub {cryptoHashOne('sha1', $tBuffer) ne $strFileBinHash}, true, ' check sha1 NOT same as original'); # Test file against openssl to make sure they are compatible #--------------------------------------------------------------------------------------------------------------------------- diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipPerlTest.pm index 6671937c9..ab1711110 100644 --- a/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipPerlTest.pm @@ -13,10 +13,10 @@ use Carp qw(confess); use English '-no_match_vars'; use Compress::Raw::Zlib qw(Z_OK Z_BUF_ERROR Z_DATA_ERROR); -use Digest::SHA qw(sha1_hex); use pgBackRest::Common::Exception; use pgBackRest::Common::Log; +use pgBackRest::LibC qw(:crypto); use pgBackRest::Storage::Base; use pgBackRest::Storage::Filter::Gzip; use pgBackRest::Storage::Posix::Driver; @@ -163,7 +163,7 @@ sub run $self->testResult(sub {$oGzipIo->read(\$tBuffer, 1)}, 0, ' read 0 bytes'); $self->testResult(sub {$oGzipIo->close()}, true, ' close'); - $self->testResult(sha1_hex($tBuffer), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content'); + $self->testResult(cryptoHashOne('sha1', $tBuffer), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content'); storageTest()->remove($strFileGz); @@ -183,7 +183,8 @@ sub run $self->testResult(sub {storageTest()->put($strFileGz, $tBuffer) > 0}, true, ' put content'); executeTest("gzip -df ${strFileGz}"); $self->testResult( - sub {sha1_hex(${storageTest()->get($strFile)})}, '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content'); + sub {cryptoHashOne('sha1', ${storageTest()->get($strFile)})}, '1c7e00fd09b9dd11fc2966590b3e3274645dd031', + ' check content'); #--------------------------------------------------------------------------------------------------------------------------- $tBuffer = undef; diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageFilterShaPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageFilterShaPerlTest.pm index e9929160f..c4a07aee5 100644 --- a/test/lib/pgBackRestTest/Module/Storage/StorageFilterShaPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Storage/StorageFilterShaPerlTest.pm @@ -12,10 +12,9 @@ use warnings FATAL => qw(all); use Carp qw(confess); use English '-no_match_vars'; -use Digest::SHA qw(sha1_hex); - use pgBackRest::Common::Exception; use pgBackRest::Common::Log; +use pgBackRest::LibC qw(:crypto); use pgBackRest::Storage::Base; use pgBackRest::Storage::Filter::Sha; use pgBackRest::Storage::Posix::Driver; @@ -55,8 +54,9 @@ sub run $self->testResult(sub {$oShaIo->close()}, true, 'close'); my $strSha = $self->testResult( - sub {$oShaIo->result(STORAGE_FILTER_SHA)}, sha1_hex($strFileContent), 'check hash against original content'); - $self->testResult($strSha, sha1_hex($tBuffer), 'check hash against buffer'); + sub {$oShaIo->result(STORAGE_FILTER_SHA)}, cryptoHashOne('sha1', $strFileContent), + 'check hash against original content'); + $self->testResult($strSha, cryptoHashOne('sha1', $tBuffer), 'check hash against buffer'); $self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, 'check content'); #--------------------------------------------------------------------------------------------------------------------------- @@ -74,7 +74,7 @@ sub run $self->testResult(sub {$oShaIo->close()}, true, ' close'); $self->testResult(sub {$oShaIo->close()}, false, ' close again to make sure nothing bad happens'); $self->testResult($oShaIo->result(STORAGE_FILTER_SHA), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check hash'); - $self->testResult(sha1_hex($tBuffer), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content'); + $self->testResult(cryptoHashOne('sha1', $tBuffer), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content'); } ################################################################################################################################ @@ -98,7 +98,8 @@ sub run $self->testResult(sub {$oShaIo->close()}, true, 'close'); my $strSha = $self->testResult( - sub {$oShaIo->result(STORAGE_FILTER_SHA)}, sha1_hex($strFileContent), 'check hash against original content'); + sub {$oShaIo->result(STORAGE_FILTER_SHA)}, cryptoHashOne('sha1', $strFileContent), + 'check hash against original content'); $self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, 'check content'); } } diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageLocalPerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageLocalPerlTest.pm index 659d1999e..a524fc179 100644 --- a/test/lib/pgBackRestTest/Module/Storage/StorageLocalPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Storage/StorageLocalPerlTest.pm @@ -12,11 +12,10 @@ use warnings FATAL => qw(all); use Carp qw(confess); use English '-no_match_vars'; -use Digest::SHA qw(sha1_hex); - use pgBackRest::Config::Config; use pgBackRest::Common::Exception; use pgBackRest::Common::Log; +use pgBackRest::LibC qw(:crypto); use pgBackRest::Storage::Filter::Sha; use pgBackRest::Storage::Base; use pgBackRest::Storage::Local; @@ -246,7 +245,7 @@ sub run $self->testResult(sub {$oFileIo->read(\$tContent, $iFileSize)}, $iFileSize, "read $iFileSize bytes"); $self->testResult(sub {$oFileIo->close()}, true, 'close'); $self->testResult($tContent, $strFileContent, ' check read'); - $self->testResult($oFileIo->result(STORAGE_FILTER_SHA), sha1_hex($strFileContent), ' check hash'); + $self->testResult($oFileIo->result(STORAGE_FILTER_SHA), cryptoHashOne('sha1', $strFileContent), ' check hash'); } ################################################################################################################################ @@ -283,7 +282,7 @@ sub run $self->testResult( sub {$self->storageLocal()->hashSize($strFile)}, - qw{(} . sha1_hex($strFileContent) . ', ' . $iFileSize . qw{)}, ' check hash/size'); + qw{(} . cryptoHashOne('sha1', $strFileContent) . ', ' . $iFileSize . qw{)}, ' check hash/size'); } ################################################################################################################################ @@ -361,7 +360,7 @@ sub run if ($self->begin('encryption')) { my $strCipherPass = 'x'; - $self->testResult(sub {sha1_hex($strFileContent)}, $strFileHash, 'hash check contents to be written'); + $self->testResult(sub {cryptoHashOne('sha1', $strFileContent)}, $strFileHash, 'hash check contents to be written'); # Error when passphrase not passed #--------------------------------------------------------------------------------------------------------------------------- @@ -381,7 +380,7 @@ sub run ' test storage encrypted and valid'); $self->testResult( - sub {sha1_hex(${storageTest()->get($strFile)}) ne $strFileHash}, true, ' check written sha1 different'); + sub {cryptoHashOne('sha1', ${storageTest()->get($strFile)}) ne $strFileHash}, true, ' check written sha1 different'); # Error when passphrase not passed #--------------------------------------------------------------------------------------------------------------------------- @@ -406,8 +405,8 @@ sub run true, 'copy - decrypt/encrypt'); $self->testResult( - sub {sha1_hex(${$self->storageEncrypt()->get($strFileCopy, {strCipherPass => $strCipherPass})})}, $strFileHash, - ' check decrypted copy file sha1 same as original plaintext file'); + sub {cryptoHashOne('sha1', ${$self->storageEncrypt()->get($strFileCopy, {strCipherPass => $strCipherPass})})}, + $strFileHash, ' check decrypted copy file sha1 same as original plaintext file'); # Write an empty encrypted file #--------------------------------------------------------------------------------------------------------------------------- diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageS3PerlTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageS3PerlTest.pm index e1af2f62b..b25a61374 100644 --- a/test/lib/pgBackRestTest/Module/Storage/StorageS3PerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Storage/StorageS3PerlTest.pm @@ -12,10 +12,9 @@ use warnings FATAL => qw(all); use Carp qw(confess); use English '-no_match_vars'; -use Digest::SHA qw(sha1_hex); - use pgBackRest::Common::Log; use pgBackRest::Common::String; +use pgBackRest::LibC qw(:crypto); use pgBackRest::Storage::S3::Driver; use pgBackRestTest::Common::ExecuteTest; @@ -172,7 +171,7 @@ sub run $self->testResult(sub {$oFileRead->read(\$tBuffer, 524288)}, 524288, ' read half'); $self->testResult(sub {$oFileRead->read(\$tBuffer, 512)}, 0, ' read 0'); $self->testResult(length($tBuffer), 1048576, ' check length'); - $self->testResult(sha1_hex($tBuffer), sha1_hex($strRandom), ' check hash'); + $self->testResult(cryptoHashOne('sha1', $tBuffer), cryptoHashOne('sha1', $strRandom), ' check hash'); } ################################################################################################################################ diff --git a/test/src/module/crypto/hashTest.c b/test/src/module/crypto/hashTest.c new file mode 100644 index 000000000..c36568404 --- /dev/null +++ b/test/src/module/crypto/hashTest.c @@ -0,0 +1,65 @@ +/*********************************************************************************************************************************** +Test Cryptographic Hashes +***********************************************************************************************************************************/ + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void +testRun() +{ + FUNCTION_HARNESS_VOID(); + + // ***************************************************************************************************************************** + if (testBegin("cryptoHashNew(), cryptoHashProcess*(), cryptoHashStr(), and cryptoHashFree()")) + { + CryptoHash *hash = NULL; + + TEST_ERROR(cryptoHashNew(strNew(BOGUS_STR)), AssertError, "unable to load hash 'BOGUS'"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ASSIGN(hash, cryptoHashNew(strNew(HASH_TYPE_SHA1)), "create sha1 hash"); + TEST_RESULT_VOID(cryptoHashFree(hash), " free hash"); + TEST_RESULT_VOID(cryptoHashFree(NULL), " free null hash"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ASSIGN(hash, cryptoHashNew(strNew(HASH_TYPE_SHA1)), "create sha1 hash"); + TEST_RESULT_STR(strPtr(cryptoHashHex(hash)), "da39a3ee5e6b4b0d3255bfef95601890afd80709", " check empty hash"); + TEST_RESULT_STR(strPtr(cryptoHashHex(hash)), "da39a3ee5e6b4b0d3255bfef95601890afd80709", " check empty hash again"); + TEST_RESULT_VOID(cryptoHashFree(hash), " free hash"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ASSIGN(hash, cryptoHashNew(strNew(HASH_TYPE_SHA1)), "create sha1 hash"); + TEST_RESULT_VOID(cryptoHashProcessC(hash, (const unsigned char *)"1", 1), " add 1"); + TEST_RESULT_VOID(cryptoHashProcessStr(hash, strNew("2")), " add 2"); + TEST_RESULT_VOID(cryptoHashProcess(hash, bufNewStr(strNew("3"))), " add 3"); + TEST_RESULT_VOID(cryptoHashProcess(hash, bufNewStr(strNew("4"))), " add 4"); + TEST_RESULT_VOID(cryptoHashProcess(hash, bufNewStr(strNew("5"))), " add 5"); + + TEST_RESULT_STR(strPtr(cryptoHashHex(hash)), "8cb2237d0679ca88db6464eac60da96345513964", " check small hash"); + TEST_RESULT_VOID(cryptoHashFree(hash), " free hash"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ASSIGN(hash, cryptoHashNew(strNew(HASH_TYPE_MD5)), "create md5 hash"); + TEST_RESULT_STR(strPtr(cryptoHashHex(hash)), "d41d8cd98f00b204e9800998ecf8427e", " check empty hash"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ASSIGN(hash, cryptoHashNew(strNew(HASH_TYPE_SHA256)), "create sha256 hash"); + TEST_RESULT_STR( + strPtr(cryptoHashHex(hash)), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + " check empty hash"); + } + + // ***************************************************************************************************************************** + if (testBegin("cryptoHashOne*()")) + { + TEST_RESULT_STR( + strPtr(cryptoHashOne(strNew(HASH_TYPE_SHA1), bufNewStr(strNew("12345")))), "8cb2237d0679ca88db6464eac60da96345513964", + " check small hash"); + TEST_RESULT_STR( + strPtr(cryptoHashOneStr(strNew(HASH_TYPE_SHA1), strNew("12345"))), "8cb2237d0679ca88db6464eac60da96345513964", + " check small hash"); + } + + FUNCTION_HARNESS_RESULT_VOID(); +} diff --git a/test/test.pl b/test/test.pl index 6fc1aa597..48eb1d9f3 100755 --- a/test/test.pl +++ b/test/test.pl @@ -773,7 +773,7 @@ eval if (!$oStorageBackRest->exists($strLibCSmart) || $oStorageBackRest->info($strLibCSmart)->mtime < $lTimestampLast) { - &log(INFO, ' libc dependencies have changed for ${strBuildVM}, rebuilding...'); + &log(INFO, " libc dependencies have changed for ${strBuildVM}, rebuilding..."); $bRebuild = true; }