From 700722b8b577a69f46d2bf023b0e2b8ff71d88ea Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 11 Sep 2026 18:25:19 +0300 Subject: [PATCH] Derive the cipher key with SHA-256 at repository format 6. Format 6 derives encryption keys from the passphrase with SHA-256 rather than SHA-1. Older repositories keep SHA-1, since the migration does not rewrite the files they already encrypted. The digest is picked when a passphrase is generated rather than at each use, so it is stored with the passphrase in CipherSpec and defaults to SHA-256. A passphrase outlives the format of the file it is stored in, so the format cannot decide how the key is derived. Info files at format 6 store cipher-digest beside cipher-pass, and a passphrase stored without one derives with SHA-1. stanza-upgrade writes back the digest the passphrase already had, so archives and backups from before the migration still load. Only a new passphrase gets SHA-256: the one for a stanza created at format 6, or for the manifest of a full backup written at it. The info file has the same problem in reverse. Its format sits inside the content that the passphrase decrypts, so a reader would have to decrypt the file to find out how to decrypt it. Format 6 puts a plaintext header in front instead: PGBR, the three-digit format, and a spare byte for later. It goes where the OpenSSL Salted__ magic used to go and is the same eight bytes, so the salt still starts in the same place. A file that starts with the magic came from format 5, the only format older than the header. The header format is checked before anything is decrypted, then compared against the format found in the content, so a file stitched together from two others is rejected rather than half read. The format constants move out of version.h into a new common/format, along with repoFormatValidate(), repoFormatDigest(), and CipherBlockFormat. That filter reads the header, builds a block cipher with the digest the format calls for, and streams the rest of the file through it. The block cipher no longer knows anything about repositories: it takes the digest from the spec and takes what sits in front of the content as a parameter. Verify checksums info files as stored rather than as decrypted, since the header has to come off before decryption can start. A file and its copy are written from the same bytes, so comparing the two still works. In an encrypted repository created by a Perl version the file and its copy are separate encryptions, so their checksums differ and verify reports it at detail level. A log line is better than a separate code path for repositories with seven year old backups. --- build/config.yaml | 2 +- doc/xml/reference.xml | 6 +- doc/xml/release/2020s/2026/2.60.0.xml | 11 + src/command/backup/backup.c | 2 +- src/command/backup/file.c.inc | 6 +- src/command/manifest/manifest.c | 2 +- src/command/remote/remote.c | 2 + src/command/repo/get.c | 18 +- src/command/restore/blockDelta.c | 3 +- src/command/restore/file.c.inc | 9 +- src/command/stanza/common.c | 12 +- src/command/stanza/common.h | 4 +- src/command/stanza/create.c | 5 +- src/command/verify/verify.c | 67 ++-- src/common/crypto/cipherBlock.c | 38 +-- src/common/crypto/cipherBlock.h | 18 +- src/common/crypto/common.c | 20 ++ src/common/crypto/common.intern.h | 21 ++ src/common/crypto/hash.c | 15 +- src/common/crypto/spec.c | 24 +- src/common/crypto/spec.h | 39 ++- src/common/format/cipherBlockFormat.c | 432 +++++++++++++++++++++++++ src/common/format/cipherBlockFormat.h | 55 ++++ src/common/format/format.c | 51 +++ src/common/format/format.h | 37 +++ src/config/config.c | 8 +- src/info/info.c | 82 +++-- src/info/info.h | 5 +- src/info/infoArchive.c | 7 +- src/info/infoBackup.c | 7 +- src/info/manifest/serialize.c.inc | 2 +- src/meson.build | 2 + src/version.h | 14 - test/define.yaml | 8 + test/src/harness/config.h | 2 +- test/src/harness/host.c | 2 +- test/src/harness/info.c | 88 +++++ test/src/harness/info.h | 30 +- test/src/harness/storage.c | 25 +- test/src/module/command/backupTest.c | 9 +- test/src/module/command/restoreTest.c | 13 +- test/src/module/command/stanzaTest.c | 73 ++++- test/src/module/command/verifyTest.c | 57 +++- test/src/module/common/cryptoTest.c | 62 ++-- test/src/module/common/formatTest.c | 272 ++++++++++++++++ test/src/module/info/infoArchiveTest.c | 20 +- test/src/module/info/infoBackupTest.c | 21 +- test/src/module/info/infoPgTest.c | 4 +- test/src/module/info/infoTest.c | 168 +++++++++- test/src/module/info/manifestTest.c | 12 +- test/src/module/storage/remoteTest.c | 8 +- 51 files changed, 1664 insertions(+), 236 deletions(-) create mode 100644 src/common/crypto/common.intern.h create mode 100644 src/common/format/cipherBlockFormat.c create mode 100644 src/common/format/cipherBlockFormat.h create mode 100644 src/common/format/format.c create mode 100644 src/common/format/format.h create mode 100644 test/src/module/common/formatTest.c diff --git a/build/config.yaml b/build/config.yaml index 7986b8df3..687281c65 100644 --- a/build/config.yaml +++ b/build/config.yaml @@ -1691,7 +1691,7 @@ option: # Command-line only so that a format left in a configuration file cannot upgrade a repository as a side effect of an unrelated # stanza-upgrade. This does not cover the environment, which can set any option that is valid for the command, so a format left # there does migrate on the next stanza-upgrade and the option reference documents it. The allow list must be kept in sync with - # REPOSITORY_FORMAT_MIN/MAX in version.h + # REPOSITORY_FORMAT_MIN/MAX in common/format/format.h repo-format: group: repo type: integer diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index e6b8228f4..e19090601 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -2929,7 +2929,7 @@ Format supported by all versions since 1.00 - Adds no features yet and is reserved for features in development + Derives new repository encryption keys with SHA-256 rather than SHA-1

An existing stanza is migrated to a newer format with the stanza-upgrade command rather than with this command, which sets the format only for a stanza it creates.

@@ -2968,10 +2968,14 @@

The first backup after a migration is a full backup even when a differential or incremental backup is requested, since no backup in the repository was written at the new format.

+

A passphrase already stored in the repository keeps deriving its key with SHA-1, since the files it encrypted are not rewritten by the migration. Only a new passphrase derives with SHA-256, i.e. the passphrase generated for each full backup written at the new format, or the passphrases of a stanza created at that format.

+

Once the info files have been migrated, a version of that does not support the new format is no longer able to read the stanza at all, including the parts of it that are still at the older format. The format cannot be downgraded.

+

On an encrypted repository such a version reports an error about the cipher rather than the format, since the header written from format 6 takes the place of the magic that an older version expects to find.

+

This option cannot be set in a configuration file but it can be set in the environment, where it applies to every stanza-upgrade run until it is removed.

Set this option for the run that migrates the stanza. A format left in the environment migrates every stanza that is upgraded from that environment. diff --git a/doc/xml/release/2020s/2026/2.60.0.xml b/doc/xml/release/2020s/2026/2.60.0.xml index 8ae5ff834..22510234c 100644 --- a/doc/xml/release/2020s/2026/2.60.0.xml +++ b/doc/xml/release/2020s/2026/2.60.0.xml @@ -138,6 +138,17 @@

General framework for new repository formats.

+ + + + + + + + +

Derive the cipher key with SHA-256 at repository format 6

+ + diff --git a/src/command/backup/backup.c b/src/command/backup/backup.c index a4e564be3..42b2f9f14 100644 --- a/src/command/backup/backup.c +++ b/src/command/backup/backup.c @@ -241,7 +241,7 @@ cmdBackup(void) // Build an incremental backup if type is not full (manifestPrior will be freed in this call) if (!backupBuildIncr(manifest, manifestPrior, backupStartResult.walSegmentName)) - manifestCipherSpecSet(manifest, cipherSpecGen(cfgOptionStrId(cfgOptRepoCipherType))); + manifestCipherSpecSet(manifest, cipherSpecGen(cfgOptionStrId(cfgOptRepoCipherType), manifestFormat(manifest))); // Set delta if it is not already set and the manifest requires it if (!cfgOptionBool(cfgOptDelta) && varBool(manifestData(manifest)->backupOptionDelta)) diff --git a/src/command/backup/file.c.inc b/src/command/backup/file.c.inc index 5501c2aeb..ec1fe887d 100644 --- a/src/command/backup/file.c.inc +++ b/src/command/backup/file.c.inc @@ -103,7 +103,8 @@ backupFileBlockIncr( if (cipherSpecType(cipherSpecBackup) != cipherTypeNone) { ioFilterGroupAdd( - ioWriteFilterGroup(blockMapWrite), cipherBlockNewP(cipherModeDecrypt, cipherSpecBackup, .raw = true)); + ioWriteFilterGroup(blockMapWrite), cipherBlockNewP( + cipherModeDecrypt, cipherSpecBackup, .header = cipherBlockHeaderNone)); } ioWriteOpen(blockMapWrite); @@ -370,7 +371,8 @@ backupFile( IoFilter *const encrypt = cipherSpecType(cipherSpecBackup) != cipherTypeNone ? cipherBlockNewP( - cipherModeEncrypt, cipherSpecBackup, .raw = bundleRaw || file->blockIncrSize != 0) : + cipherModeEncrypt, cipherSpecBackup, + .header = bundleRaw || file->blockIncrSize != 0 ? cipherBlockHeaderNone : cipherBlockHeaderMagic) : NULL; // If block incremental then add the filter and pass compress/encrypt filters to it since each block is diff --git a/src/command/manifest/manifest.c b/src/command/manifest/manifest.c index 20550db7e..ee937841b 100644 --- a/src/command/manifest/manifest.c +++ b/src/command/manifest/manifest.c @@ -235,7 +235,7 @@ cmdManifestBlockDeltaRender(const Manifest *const manifest, const ManifestFile * { ioFilterGroupAdd( ioReadFilterGroup(storageReadIo(read)), - cipherBlockNewP(cipherModeDecrypt, manifestCipherSpec(manifest), .raw = true)); + cipherBlockNewP(cipherModeDecrypt, manifestCipherSpec(manifest), .header = cipherBlockHeaderNone)); } ioReadOpen(storageReadIo(read)); diff --git a/src/command/remote/remote.c b/src/command/remote/remote.c index 5a8ab64a9..6c0d0b8bc 100644 --- a/src/command/remote/remote.c +++ b/src/command/remote/remote.c @@ -14,6 +14,7 @@ Remote Command #include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/debug.h" +#include "common/format/cipherBlockFormat.h" #include "common/io/filter/sink.h" #include "common/io/filter/size.h" #include "common/log.h" @@ -44,6 +45,7 @@ static const StorageRemoteFilterHandler storageRemoteFilterHandler[] = {.type = BLOCK_CHECKSUM_FILTER_TYPE, .handlerParam = blockChecksumNewPack}, {.type = BLOCK_INCR_FILTER_TYPE, .handlerParam = blockIncrNewPack}, {.type = CIPHER_BLOCK_FILTER_TYPE, .handlerParam = cipherBlockNewPack}, + {.type = CIPHER_BLOCK_FORMAT_FILTER_TYPE, .handlerParam = cipherBlockFormatNewPack}, {.type = CRYPTO_HASH_FILTER_TYPE, .handlerParam = cryptoHashNewPack}, {.type = PAGE_CHECKSUM_FILTER_TYPE, .handlerParam = pageChecksumNewPack}, {.type = SINK_FILTER_TYPE, .handlerNoParam = ioSinkNew}, diff --git a/src/command/repo/get.c b/src/command/repo/get.c index 81a42e28a..4d4b68677 100644 --- a/src/command/repo/get.c +++ b/src/command/repo/get.c @@ -9,6 +9,7 @@ Repository Get Command #include "command/repo/get.h" #include "common/crypto/cipherBlock.h" #include "common/debug.h" +#include "common/format/cipherBlockFormat.h" #include "common/io/fdWrite.h" #include "common/io/io.h" #include "common/log.h" @@ -16,6 +17,7 @@ Repository Get Command #include "config/config.h" #include "storage/helper.h" +#include "info/info.h" #include "info/infoArchive.h" #include "info/infoBackup.h" @@ -47,6 +49,9 @@ storageGetProcess(IoWrite *const destination) IoRead *const source = storageReadIo( storageNewReadP(storageRepo(), file, .ignoreMissing = cfgOptionBool(cfgOptIgnoreMissing))); + // Is the file an info file, i.e. one that has a header in front of its content? + bool fileIsInfo = false; + // Add decryption if needed if (!cfgOptionBool(cfgOptRaw)) { @@ -96,6 +101,9 @@ storageGetProcess(IoWrite *const destination) cfgCipherSpecMain()); cipherSpec = infoArchiveCipherSpec(info); } + // Else the file is the archive info, which the repo passphrase opens + else + fileIsInfo = true; } // Backup path @@ -128,6 +136,9 @@ storageGetProcess(IoWrite *const destination) else cipherSpec = cipherSpecManifest; } + // Else the file is the backup info, which the repo passphrase opens + else + fileIsInfo = true; } } @@ -137,8 +148,11 @@ storageGetProcess(IoWrite *const destination) ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone); - // Add encryption filter - cipherBlockFilterGroupAdd(ioReadFilterGroup(source), cipherModeDecrypt, cipherSpec); + // Add the decryption filter. An info file is read through the filter that reads its header. + if (fileIsInfo) + cipherBlockFormatFilterGroupReadAdd(ioReadFilterGroup(source), cipherSpec); + else + cipherBlockFilterGroupAdd(ioReadFilterGroup(source), cipherModeDecrypt, cipherSpec); } } diff --git a/src/command/restore/blockDelta.c b/src/command/restore/blockDelta.c index ff83b9993..bf63009f3 100644 --- a/src/command/restore/blockDelta.c +++ b/src/command/restore/blockDelta.c @@ -237,7 +237,8 @@ blockDeltaNext(BlockDelta *const this, const BlockDeltaRead *const readDelta, Io if (cipherSpecType(this->cipherSpecBackup) != cipherTypeNone) { ioFilterGroupAdd( - ioReadFilterGroup(this->limitRead), cipherBlockNewP(cipherModeDecrypt, this->cipherSpecBackup, .raw = true)); + ioReadFilterGroup(this->limitRead), cipherBlockNewP( + cipherModeDecrypt, this->cipherSpecBackup, .header = cipherBlockHeaderNone)); } if (this->compressType != compressTypeNone) diff --git a/src/command/restore/file.c.inc b/src/command/restore/file.c.inc index fd98327ea..a44663865 100644 --- a/src/command/restore/file.c.inc +++ b/src/command/restore/file.c.inc @@ -264,7 +264,8 @@ restoreFile( if (cipherSpecType(cipherSpecBackup) != cipherTypeNone) { ioFilterGroupAdd( - ioReadFilterGroup(blockMapRead), cipherBlockNewP(cipherModeDecrypt, cipherSpecBackup, .raw = true)); + ioReadFilterGroup(blockMapRead), cipherBlockNewP( + cipherModeDecrypt, cipherSpecBackup, .header = cipherBlockHeaderNone)); } ioReadOpen(blockMapRead); @@ -336,7 +337,11 @@ restoreFile( // Add decryption filter if (cipherSpecType(cipherSpecBackup) != cipherTypeNone) { - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecBackup, .raw = bundleRaw)); + ioFilterGroupAdd( + filterGroup, + cipherBlockNewP( + cipherModeDecrypt, cipherSpecBackup, + .header = bundleRaw ? cipherBlockHeaderNone : cipherBlockHeaderMagic)); } // Add decompression filter diff --git a/src/command/stanza/common.c b/src/command/stanza/common.c index cdb7feb38..30b2939f3 100644 --- a/src/command/stanza/common.c +++ b/src/command/stanza/common.c @@ -6,6 +6,7 @@ Stanza Commands Handler #include "command/check/common.h" #include "command/stanza/common.h" #include "common/debug.h" +#include "common/format/format.h" #include "common/log.h" #include "config/config.h" #include "db/helper.h" @@ -16,10 +17,11 @@ Stanza Commands Handler /**********************************************************************************************************************************/ FN_EXTERN CipherSpec * -cipherSpecGen(const CipherType cipherType) +cipherSpecGen(const CipherType cipherType, const unsigned int format) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING_ID, cipherType); + FUNCTION_TEST_PARAM(UINT, format); FUNCTION_TEST_END(); CipherSpec *result; @@ -33,8 +35,12 @@ cipherSpecGen(const CipherType cipherType) uint8_t buffer[48]; // 48 is the amount of entropy needed to get a 64 base key cryptoRandomBytes(buffer, sizeof(buffer)); - // The pass is the encoded text rather than the bytes it encodes, so it is stored and derived from as that text - result = cipherSpecNew(cipherType, BUFSTR(strNewEncode(encodingBase64, BUF(buffer, sizeof(buffer))))); + // The pass is the base64 text rather than the bytes it encodes, since the text is what gets stored and what the key + // derives from. A new pass derives with the digest its format calls for and that digest is stored with it, so a + // reader derives the same key. + result = cipherSpecNewP( + cipherType, BUFSTR(strNewEncode(encodingBase64, BUF(buffer, sizeof(buffer)))), + .digest = repoFormatDigest(format)); cipherSpecMove(result, memContextPrior()); } MEM_CONTEXT_TEMP_END(); diff --git a/src/command/stanza/common.h b/src/command/stanza/common.h index 225ca5ee3..14886f327 100644 --- a/src/command/stanza/common.h +++ b/src/command/stanza/common.h @@ -10,8 +10,8 @@ Stanza Commands Handler /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ -// Generate cipher spec with a new pass, none when the type is none -FN_EXTERN CipherSpec *cipherSpecGen(CipherType cipherType); +// Generate a sub pass for a file at this format, which sets the digest since that is what will derive it when it is read back +FN_EXTERN CipherSpec *cipherSpecGen(CipherType cipherType, unsigned int format); // Validate and return database information FN_EXTERN PgControl pgValidate(void); diff --git a/src/command/stanza/create.c b/src/command/stanza/create.c index 856ffe5f6..61d158571 100644 --- a/src/command/stanza/create.c +++ b/src/command/stanza/create.c @@ -78,7 +78,7 @@ cmdStanzaCreate(void) const unsigned int format = cfgOptionIdxUInt(cfgOptRepoFormat, repoIdx); // If the repo is encrypted, generate a cipher passphrase for encrypting subsequent archive files - const CipherSpec *const cipherSpecArchive = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx)); + const CipherSpec *const cipherSpecArchive = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), format); // Create and save archive info infoArchive = infoArchiveNew(pgControl.version, pgControl.systemId, format, cipherSpecArchive); @@ -86,7 +86,8 @@ cmdStanzaCreate(void) infoArchiveSaveFile(infoArchive, storageRepoWriteStanza, INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMainIdx(repoIdx)); // If the repo is encrypted, generate a cipher passphrase for encrypting subsequent manifests - const CipherSpec *const cipherSpecManifest = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx)); + const CipherSpec *const cipherSpecManifest = cipherSpecGen( + cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), format); // Create and save backup info infoBackup = infoBackupNew( diff --git a/src/command/verify/verify.c b/src/command/verify/verify.c index f30d40fe3..0bbb186dc 100644 --- a/src/command/verify/verify.c +++ b/src/command/verify/verify.c @@ -17,6 +17,7 @@ Verify contents of the repository. #include "common/compress/helper.h" #include "common/crypto/cipherBlock.h" #include "common/debug.h" +#include "common/format/cipherBlockFormat.h" #include "common/io/fdWrite.h" #include "common/io/io.h" #include "common/log.h" @@ -163,35 +164,6 @@ verifyInvalidFileAdd(List *const invalidFileList, const VerifyResult reason, con FUNCTION_TEST_RETURN_VOID(); } -/*********************************************************************************************************************************** -Load a file into memory -***********************************************************************************************************************************/ -static StorageRead * -verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherSpec) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRING, pathFileName); // Fully qualified path/file name - FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); // Cipher spec to open file if encrypted - FUNCTION_TEST_END(); - - ASSERT(pathFileName != NULL); - - // Read the file and error if missing - StorageRead *const result = storageNewReadP(storageRepo(), pathFileName); - - // *read points to a location within result so update result with contents based on necessary filters - IoRead *const read = storageReadIo(result); - - cipherBlockFilterGroupAdd(ioReadFilterGroup(read), cipherModeDecrypt, cipherSpec); - ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1)); - - // If the file is compressed, add a decompression filter - if (compressTypeFromName(pathFileName) != compressTypeNone) - ioFilterGroupAdd(ioReadFilterGroup(read), decompressFilterP(compressTypeFromName(pathFileName))); - - FUNCTION_TEST_RETURN(STORAGE_READ, result); -} - /*********************************************************************************************************************************** Get status of info files in the repository ***********************************************************************************************************************************/ @@ -214,18 +186,32 @@ verifyInfoFile(const String *const pathFileName, const bool keepFile, const Ciph { TRY_BEGIN() { - IoRead *const infoRead = storageReadIo(verifyFileLoad(pathFileName, cipherSpec)); + StorageRead *const infoStorageRead = storageNewReadP(storageRepo(), pathFileName); + IoRead *const infoRead = storageReadIo(infoStorageRead); + + ioFilterGroupAdd(ioReadFilterGroup(infoRead), cryptoHashNew(hashTypeSha1)); + + const bool isBackup = strBeginsWith(pathFileName, INFO_BACKUP_PATH_FILE_STR); + const bool isArchive = strBeginsWith(pathFileName, INFO_ARCHIVE_PATH_FILE_STR); + + // Add decryption after the hash so the checksum is over the file as stored. An info file is read through the filter + // that reads its header, a manifest with the spec as it was given since it has no header of its own. + if (isBackup || isArchive) + cipherBlockFormatFilterGroupReadAdd(ioReadFilterGroup(infoRead), cipherSpec); + else + cipherBlockFilterGroupAdd(ioReadFilterGroup(infoRead), cipherModeDecrypt, cipherSpec); // If directed to keep the loaded file in memory, then move the file into the result, else drain the io and close it if (keepFile) { - if (strBeginsWith(pathFileName, INFO_BACKUP_PATH_FILE_STR)) + if (isBackup) result.backup = infoBackupMove(infoBackupNewLoad(infoRead, cipherSpec), memContextPrior()); - else if (strBeginsWith(pathFileName, INFO_ARCHIVE_PATH_FILE_STR)) + else if (isArchive) result.archive = infoArchiveMove(infoArchiveNewLoad(infoRead, cipherSpec), memContextPrior()); else result.manifest = manifestMove(manifestNewLoad(infoRead, cipherSpec), memContextPrior()); } + // Else nothing needs the file, so the drain runs the filters else ioReadDrain(infoRead); @@ -823,11 +809,18 @@ verifyArchive(VerifyJobData *const jobData) if (archiveResult->pgWalInfo.size == 0) { // Initialize the WAL segment size from the first WAL - StorageRead *const walRead = verifyFileLoad( - strNewFmt( - STORAGE_REPO_ARCHIVE "/%s/%s/%s", strZ(archiveResult->archiveId), strZ(walPath), - strZ(strLstGet(jobData->walFileList, 0))), - jobData->cipherSpecArchive); + const String *const walFile = strNewFmt( + STORAGE_REPO_ARCHIVE "/%s/%s/%s", strZ(archiveResult->archiveId), strZ(walPath), + strZ(strLstGet(jobData->walFileList, 0))); + StorageRead *const walRead = storageNewReadP(storageRepo(), walFile); + IoFilterGroup *const walFilterGroup = ioReadFilterGroup(storageReadIo(walRead)); + + // Add decryption filter when required + cipherBlockFilterGroupAdd(walFilterGroup, cipherModeDecrypt, jobData->cipherSpecArchive); + + // If the file is compressed, add a decompression filter + if (compressTypeFromName(walFile) != compressTypeNone) + ioFilterGroupAdd(walFilterGroup, decompressFilterP(compressTypeFromName(walFile))); const PgWal walInfo = pgWalFromBuffer( storageGetP(walRead, .exactSize = PG_WAL_HEADER_SIZE), cfgOptionStrNull(cfgOptPgVersionForce)); diff --git a/src/common/crypto/cipherBlock.c b/src/common/crypto/cipherBlock.c index bd3ff3557..3699df82a 100644 --- a/src/common/crypto/cipherBlock.c +++ b/src/common/crypto/cipherBlock.c @@ -10,22 +10,13 @@ Block Cipher #include "common/crypto/cipherBlock.h" #include "common/crypto/common.h" +#include "common/crypto/common.intern.h" #include "common/debug.h" #include "common/io/filter/filter.h" #include "common/log.h" +#include "common/type/convert.h" #include "common/type/object.h" -/*********************************************************************************************************************************** -Header constants and sizes -***********************************************************************************************************************************/ -// Magic constant for salted encrypt. Only salted encrypt is done here, but this constant is required for compatibility with the -// openssl command-line tool. -#define CIPHER_BLOCK_MAGIC "Salted__" -#define CIPHER_BLOCK_MAGIC_SIZE (sizeof(CIPHER_BLOCK_MAGIC) - 1) - -// Total length of cipher header -#define CIPHER_BLOCK_HEADER_SIZE (CIPHER_BLOCK_MAGIC_SIZE + PKCS5_SALT_LEN) - /*********************************************************************************************************************************** Object type ***********************************************************************************************************************************/ @@ -37,7 +28,7 @@ typedef struct CipherBlock bool processDone; // Has any data been processed? const Buffer *pass; // Passphrase used to generate encryption key size_t headerSize; // Size of header read during decrypt - uint8_t header[CIPHER_BLOCK_HEADER_SIZE]; // Buffer to hold partial header during decrypt + uint8_t header[CIPHER_BLOCK_MAGIC_SIZE + PKCS5_SALT_LEN]; // Buffer to hold partial header during decrypt const EVP_CIPHER *cipher; // Cipher object const EVP_MD *digest; // Message digest object EVP_CIPHER_CTX *cipherContext; // Encrypt/decrypt context @@ -149,7 +140,7 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s else if (sourceSize > 0) { // Check if the entire header has been read - const size_t headerExpected = this->raw ? PKCS5_SALT_LEN : CIPHER_BLOCK_HEADER_SIZE; + const size_t headerExpected = this->raw ? PKCS5_SALT_LEN : CIPHER_BLOCK_MAGIC_SIZE + PKCS5_SALT_LEN; if (this->headerSize + sourceSize >= headerExpected) { @@ -161,8 +152,8 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s source += headerExpected - this->headerSize; sourceSize -= headerExpected - this->headerSize; - // The first bytes of the file to decrypt should be equal to the magic. If not then this is not an encrypted file, - // or at least not in a format we recognize. + // The first bytes of the file to decrypt should be equal to the magic. If not then this is not an encrypted + // file, or at least not in a format we recognize. if (!this->raw && memcmp(this->header, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) != 0) THROW(CryptoError, "cipher header invalid"); } @@ -389,7 +380,7 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(STRING_ID, mode); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); - FUNCTION_LOG_PARAM(BOOL, param.raw); + FUNCTION_LOG_PARAM(ENUM, param.header); FUNCTION_LOG_END(); ASSERT(cipherSpec != NULL); @@ -409,16 +400,17 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const zFree(cipherTypeZ); + // Lookup digest, which the spec supplies since the pass and the digest are chosen together + const EVP_MD *const digest = cryptoDigest(cipherSpecDigest(cipherSpec)); + OBJ_NEW_BEGIN(CipherBlock, .childQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1) { *this = (CipherBlock) { .mode = mode, - .raw = param.raw, + .raw = param.header == cipherBlockHeaderNone, .cipher = cipher, - - // The key is always derived with SHA-1. Deriving it with anything else is not supported yet. - .digest = EVP_sha1(), + .digest = digest, .pass = bufDup(cipherSpecPass(cipherSpec)), }; } @@ -433,7 +425,7 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const pckWriteU64P(packWrite, mode); cipherSpecPack(packWrite, cipherSpec); - pckWriteBoolP(packWrite, param.raw); + pckWriteU32P(packWrite, param.header); pckWriteEndP(packWrite); paramList = pckMove(pckWriteResult(packWrite), memContextPrior()); @@ -457,9 +449,9 @@ cipherBlockNewPack(const Pack *const paramList) PackRead *const paramListPack = pckReadNew(paramList); const CipherMode cipherMode = (CipherMode)pckReadU64P(paramListPack); const CipherSpec *const cipherSpec = cipherSpecNewPack(paramListPack); - const bool raw = pckReadBoolP(paramListPack); + const CipherBlockHeader header = (CipherBlockHeader)pckReadU32P(paramListPack); - result = ioFilterMove(cipherBlockNewP(cipherMode, cipherSpec, .raw = raw), memContextPrior()); + result = ioFilterMove(cipherBlockNewP(cipherMode, cipherSpec, .header = header), memContextPrior()); } MEM_CONTEXT_TEMP_END(); diff --git a/src/common/crypto/cipherBlock.h b/src/common/crypto/cipherBlock.h index ad28a2614..80453b2bb 100644 --- a/src/common/crypto/cipherBlock.h +++ b/src/common/crypto/cipherBlock.h @@ -7,6 +7,13 @@ Block Cipher Header #include "common/crypto/spec.h" #include "common/io/filter/group.h" +/*********************************************************************************************************************************** +Magic constant for salted encrypt, written before the salt unless the header is none. Anything written in place of the magic must be +the same size so that what follows still begins with the salt. +***********************************************************************************************************************************/ +#define CIPHER_BLOCK_MAGIC "Salted__" +#define CIPHER_BLOCK_MAGIC_SIZE (sizeof(CIPHER_BLOCK_MAGIC) - 1) + /*********************************************************************************************************************************** Filter type constant ***********************************************************************************************************************************/ @@ -15,10 +22,19 @@ Filter type constant /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ +// What precedes the content of a file. The magic is what openssl writes and is the default. None saves the eight bytes the magic +// takes for a file nothing but this version reads, e.g. a file bundled into a backup, and for a file that something else has +// written in place of the magic, e.g. the format header. +typedef enum +{ + cipherBlockHeaderMagic = 0, // Salted magic openssl writes + cipherBlockHeaderNone, // Nothing, to save space +} CipherBlockHeader; + typedef struct CipherBlockNewParam { VAR_PARAM_HEADER; - bool raw; // Omit header magic to save space + CipherBlockHeader header; // What precedes the content, the magic by default } CipherBlockNewParam; #define cipherBlockNewP(mode, cipherSpec, ...) \ diff --git a/src/common/crypto/common.c b/src/common/crypto/common.c index 17d73b444..f426793c0 100644 --- a/src/common/crypto/common.c +++ b/src/common/crypto/common.c @@ -10,6 +10,7 @@ Crypto Common #include #include "common/crypto/common.h" +#include "common/crypto/common.intern.h" #include "common/debug.h" #include "common/log.h" @@ -85,3 +86,22 @@ cryptoRandomBytes(uint8_t *const buffer, const size_t size) FUNCTION_LOG_RETURN_VOID(); } + +/**********************************************************************************************************************************/ +FN_EXTERN const EVP_MD * +cryptoDigest(const HashType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING_ID, type); + FUNCTION_TEST_END(); + + char typeZ[STRID_MAX + 1]; + strIdToZ(type, typeZ); + + const EVP_MD *const result = EVP_get_digestbyname(typeZ); + + if (result == NULL) + THROW_FMT(AssertError, "unable to load digest '%s'", typeZ); + + FUNCTION_TEST_RETURN_TYPE_CONST_P(EVP_MD, result); +} diff --git a/src/common/crypto/common.intern.h b/src/common/crypto/common.intern.h new file mode 100644 index 000000000..6c9ee3aed --- /dev/null +++ b/src/common/crypto/common.intern.h @@ -0,0 +1,21 @@ +/*********************************************************************************************************************************** +Crypto Common Internal + +Openssl types are used here, so this is included only by the crypto modules that work with them rather than by everything that +needs a hash type. +***********************************************************************************************************************************/ +#ifndef COMMON_CRYPTO_COMMON_INTERN_H +#define COMMON_CRYPTO_COMMON_INTERN_H + +#include + +#include "common/crypto/common.h" + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Digest openssl knows by the name of the hash type. Error when there is no such digest, which is a coding error since a hash type +// is not read from user input. +FN_EXTERN const EVP_MD *cryptoDigest(HashType type); + +#endif diff --git a/src/common/crypto/hash.c b/src/common/crypto/hash.c index a0f537032..b63974852 100644 --- a/src/common/crypto/hash.c +++ b/src/common/crypto/hash.c @@ -10,6 +10,7 @@ Cryptographic Hash #include #include "common/crypto/common.h" +#include "common/crypto/common.intern.h" #include "common/crypto/hash.h" #include "common/debug.h" #include "common/io/filter/filter.h" @@ -193,12 +194,7 @@ cryptoHashNew(const HashType type) // Else use the standard OpenSSL implementation else { - // Lookup digest - char typeZ[STRID_MAX + 1]; - strIdToZ(type, typeZ); - - if ((this->hashType = EVP_get_digestbyname(typeZ)) == NULL) - THROW_FMT(AssertError, "unable to load hash '%s'", typeZ); + this->hashType = cryptoDigest(type); // Create context cryptoError((this->hashContext = EVP_MD_CTX_new()) == NULL, "unable to create hash context"); @@ -299,12 +295,7 @@ cryptoHmacOne(const HashType type, const Buffer *const key, const Buffer *const // Init crypto subsystem cryptoInit(); - // Lookup digest - char typeZ[STRID_MAX + 1]; - strIdToZ(type, typeZ); - - const EVP_MD *const hashType = EVP_get_digestbyname(typeZ); - ASSERT(hashType != NULL); + const EVP_MD *const hashType = cryptoDigest(type); // Allocate a buffer to hold the hmac Buffer *const result = bufNew((size_t)EVP_MD_size(hashType)); diff --git a/src/common/crypto/spec.c b/src/common/crypto/spec.c index 76627b8af..5dc6d9b49 100644 --- a/src/common/crypto/spec.c +++ b/src/common/crypto/spec.c @@ -17,11 +17,12 @@ struct CipherSpec /**********************************************************************************************************************************/ FN_EXTERN CipherSpec * -cipherSpecNew(const CipherType type, const Buffer *const pass) +cipherSpecNew(const CipherType type, const Buffer *const pass, const CipherSpecNewParam param) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING_ID, type); FUNCTION_TEST_PARAM(BUFFER, pass); + FUNCTION_TEST_PARAM(STRING_ID, param.digest); FUNCTION_TEST_END(); ASSERT((type == cipherTypeNone) == (pass == NULL)); @@ -32,7 +33,10 @@ cipherSpecNew(const CipherType type, const Buffer *const pass) *this = (CipherSpec){.pub = {.type = type}}; if (this->pub.type != cipherTypeNone) + { + this->pub.digest = param.digest == 0 ? hashTypeSha256 : param.digest; this->pub.pass = bufDup(pass); + } } OBJ_NEW_END(); @@ -56,7 +60,10 @@ cipherSpecNewPack(PackRead *const packRead) // Nothing else was written when there is no cipher. The pass is read here rather than copied in since this context is the // one it belongs in. if (this->pub.type != cipherTypeNone) + { + this->pub.digest = (HashType)pckReadStrIdP(packRead); this->pub.pass = pckReadBinP(packRead); + } } OBJ_NEW_END(); @@ -79,7 +86,10 @@ cipherSpecPack(PackWrite *const packWrite, const CipherSpec *const this) // Nothing else is needed when there is no cipher if (cipherSpecType(this) != cipherTypeNone) + { + pckWriteStrIdP(packWrite, cipherSpecDigest(this)); pckWriteBinP(packWrite, cipherSpecPass(this)); + } FUNCTION_TEST_RETURN_VOID(); } @@ -91,6 +101,14 @@ cipherSpecToLog(const CipherSpec *const this, StringStatic *const debugLog) char typeZ[STRID_MAX + 1]; strIdToZ(cipherSpecType(this), typeZ); - // The pass is never logged - strStcFmt(debugLog, "{type: %s}", typeZ); + // There is no digest when there is no cipher. The pass is never logged. + if (cipherSpecType(this) == cipherTypeNone) + strStcFmt(debugLog, "{type: %s}", typeZ); + else + { + char digestZ[STRID_MAX + 1]; + strIdToZ(cipherSpecDigest(this), digestZ); + + strStcFmt(debugLog, "{type: %s, digest: %s}", typeZ, digestZ); + } } diff --git a/src/common/crypto/spec.h b/src/common/crypto/spec.h index 00a61a808..919503b8d 100644 --- a/src/common/crypto/spec.h +++ b/src/common/crypto/spec.h @@ -4,8 +4,16 @@ Cipher Spec Everything needed to encrypt or decrypt, kept together so that adding to it does not mean changing every function and protocol message that contains it. -The passphrase (pass) contains the bytes the key is derived from. There is no pass when the type is none, and the pass is never -logged. +The pass holds the bytes used to derive the key. The digest travels with the pass because the two are chosen together and deriving +with the wrong digest gives a wrong key instead of an error. + +The pass is a buffer rather than a string because it may be binary or it may be text and nothing here needs to know which. It is +copied into the object, so the caller can release whatever it read the pass from. + +The digest defaults to SHA-256, so a caller with no preference gets the digest new work should use. Repositories before format 6 +derived with SHA-1, so a caller that works with that format must pass SHA-1 rather than taking the default. + +There is no digest or pass when the type is none, and the pass is never logged. ***********************************************************************************************************************************/ #ifndef COMMON_CRYPTO_SPEC_H #define COMMON_CRYPTO_SPEC_H @@ -23,14 +31,23 @@ typedef struct CipherSpec CipherSpec; /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ -// Create from a pass -FN_EXTERN CipherSpec *cipherSpecNew(CipherType type, const Buffer *pass); +// Create from a pass, which is the key bytes or the passphrase text +typedef struct CipherSpecNewParam +{ + VAR_PARAM_HEADER; + HashType digest; // Digest to derive the key with instead of SHA-256 +} CipherSpecNewParam; -// Create for no encryption +#define cipherSpecNewP(type, pass, ...) \ + cipherSpecNew(type, pass, (CipherSpecNewParam){VAR_PARAM_INIT, __VA_ARGS__}) + +FN_EXTERN CipherSpec *cipherSpecNew(CipherType type, const Buffer *pass, CipherSpecNewParam param); + +// Create for data that is not encrypted FN_INLINE_ALWAYS CipherSpec * cipherSpecNewNone(void) { - return cipherSpecNew(cipherTypeNone, NULL); + return cipherSpecNewP(cipherTypeNone, NULL); } // Create from a pack written by cipherSpecPack() @@ -42,6 +59,7 @@ Getters/Setters typedef struct CipherSpecPub { CipherType type; // Cipher type, none when not encrypted + HashType digest; // Digest the pass derives the key with const Buffer *pass; // Passphrase text or key bytes } CipherSpecPub; @@ -52,6 +70,13 @@ cipherSpecType(const CipherSpec *const this) return THIS_PUB(CipherSpec)->type; } +// Digest the pass derives the key with +FN_INLINE_ALWAYS HashType +cipherSpecDigest(const CipherSpec *const this) +{ + return THIS_PUB(CipherSpec)->digest; +} + // Passphrase text or key bytes FN_INLINE_ALWAYS const Buffer * cipherSpecPass(const CipherSpec *const this) @@ -66,7 +91,7 @@ Functions FN_INLINE_ALWAYS CipherSpec * cipherSpecDup(const CipherSpec *const this) { - return cipherSpecNew(cipherSpecType(this), cipherSpecPass(this)); + return cipherSpecNewP(cipherSpecType(this), cipherSpecPass(this), .digest = cipherSpecDigest(this)); } // Write to a pack so it can be passed over a protocol diff --git a/src/common/format/cipherBlockFormat.c b/src/common/format/cipherBlockFormat.c new file mode 100644 index 000000000..75fbee4af --- /dev/null +++ b/src/common/format/cipherBlockFormat.c @@ -0,0 +1,432 @@ +/*********************************************************************************************************************************** +Cipher Block Format +***********************************************************************************************************************************/ +#include + +#include +#include + +#include "common/crypto/cipherBlock.h" +#include "common/debug.h" +#include "common/format/cipherBlockFormat.h" +#include "common/format/format.h" +#include "common/io/filter/filter.h" +#include "common/log.h" +#include "common/type/convert.h" +#include "common/type/object.h" +#include "common/type/pack.h" + +/*********************************************************************************************************************************** +Format header + +A file written at a format that has one begins with fixed-size plaintext naming the format it was written with. The header exists +because the digest the pass derives with follows the format, and the format is recorded inside the file that the pass encrypts. A +reader that could not see the format in advance would have to decrypt to learn what it should have decrypted with. + +The header takes the place of the salted magic, which is why the content behind it is encrypted raw. Both are eight bytes followed +by the salt, so the eight bytes are consumed either way and what follows begins with the salt no matter which was there. It also +means a file of either kind is opened with the openssl command-line tool the same way: replace the first eight bytes with the magic +that tool expects. + +A file that begins with the magic rather than the header was written at format 5, the only format there was before the header, so +that is not an error when a header was expected. + +For the four bytes after the header magic, the first three are the format and the last is reserved for future use, e.g. naming +which key the file was encrypted with once a repository can hold more than one. The format comes first so that it is always at the +same place, which is what lets a version work out whether it can read the file at all. Once the format is identified as compatible +with this version, the spare byte is examined, and it must only be the underscore this version writes. +***********************************************************************************************************************************/ +#define CIPHER_BLOCK_FORMAT_MAGIC "PGBR" +#define CIPHER_BLOCK_FORMAT_MAGIC_SIZE (sizeof(CIPHER_BLOCK_FORMAT_MAGIC) - 1) +#define CIPHER_BLOCK_FORMAT_RESERVED '_' +#define CIPHER_BLOCK_FORMAT_SIZE 3 + +// Total length of the header, which is the magic, the format, and the reserved byte +#define CIPHER_BLOCK_FORMAT_HEADER_SIZE \ + (CIPHER_BLOCK_FORMAT_MAGIC_SIZE + CIPHER_BLOCK_FORMAT_SIZE + 1) + +// The header is written in place of the magic, so the two must add up to exactly the same size or the salt would no longer begin at +// the same place +static_assert( + CIPHER_BLOCK_FORMAT_HEADER_SIZE == CIPHER_BLOCK_MAGIC_SIZE, "cipher header must be the size of the magic it replaces"); + +// A format too large for the digits it is written in would be truncated to a different format +static_assert(REPOSITORY_FORMAT_MAX < 1000, "repository format must fit in the header digits"); +static_assert(CIPHER_BLOCK_FORMAT_SIZE == 3, "format digits must match the digits the header is written with"); + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +typedef struct CipherBlockFormat +{ + const CipherSpec *cipherSpec; // Cipher spec the content is decrypted with + unsigned int formatExpected; // Format the caller expects, zero when any will do + unsigned int format; // Format the header gave + + uint8_t header[CIPHER_BLOCK_FORMAT_HEADER_SIZE]; // Header bytes held until there are enough to read + size_t headerSize; // Header bytes held so far + IoFilter *cipherBlock; // Block cipher the content behind the header is decrypted by + size_t sourceOffset; // Bytes of the current source taken for the header +} CipherBlockFormat; + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +static void +cipherBlockFormatToLog(const CipherBlockFormat *const this, StringStatic *const debugLog) +{ + strStcFmt(debugLog, "{format: %u}", this->format); +} + +#define FUNCTION_LOG_CIPHER_BLOCK_FORMAT_TYPE \ + CipherBlockFormat * +#define FUNCTION_LOG_CIPHER_BLOCK_FORMAT_FORMAT(value, buffer, bufferSize) \ + FUNCTION_LOG_OBJECT_FORMAT(value, cipherBlockFormatToLog, buffer, bufferSize) + +/*********************************************************************************************************************************** +Read the header and return the format it gives. A file that begins with the magic instead was written at format 5, the only format +there was before the header, so that start is not an error here. +***********************************************************************************************************************************/ +static unsigned int +cipherBlockFormatHeaderRead(const uint8_t *const header) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM_P(BYTEDATA, header); + FUNCTION_TEST_END(); + + ASSERT(header != NULL); + + const char *const headerZ = (const char *)header; + unsigned int result = REPOSITORY_FORMAT_5; + + if (memcmp(headerZ, CIPHER_BLOCK_FORMAT_MAGIC, CIPHER_BLOCK_FORMAT_MAGIC_SIZE) == 0) + { + for (unsigned int digitIdx = 0; digitIdx < CIPHER_BLOCK_FORMAT_SIZE; digitIdx++) + { + if (!isdigit((unsigned char)headerZ[CIPHER_BLOCK_FORMAT_MAGIC_SIZE + digitIdx])) + THROW(FormatError, "invalid cipher header"); + } + + result = cvtZSubNToUInt(headerZ, CIPHER_BLOCK_FORMAT_MAGIC_SIZE, CIPHER_BLOCK_FORMAT_SIZE); + + // Error on a format this version cannot read before anything is decrypted + repoFormatValidate(result); + + // The format is one this version can read, so the reserved byte must be the value this version writes + if (headerZ[CIPHER_BLOCK_FORMAT_HEADER_SIZE - 1] != CIPHER_BLOCK_FORMAT_RESERVED) + THROW(FormatError, "invalid cipher header"); + } + // Else the bytes must be the magic, since that is all a file with no header can begin with + else if (memcmp(headerZ, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) != 0) + THROW(CryptoError, "cipher header invalid"); + + FUNCTION_TEST_RETURN(UINT, result); +} + +/*********************************************************************************************************************************** +Build the block cipher the content behind the header is decrypted by, which is only possible once the header has been read since the +header is what gives the format and the format is what gives the digest +***********************************************************************************************************************************/ +static void +cipherBlockFormatCipherNew(CipherBlockFormat *const this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CIPHER_BLOCK_FORMAT, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(this->cipherBlock == NULL); + + this->format = cipherBlockFormatHeaderRead(this->header); + + // Error when the format the caller expected is not the one the file was written with + if (this->formatExpected != 0 && this->formatExpected != this->format) + THROW_FMT(FormatError, "expected repository format %u but found %u", this->formatExpected, this->format); + + // The eight bytes at the front have been read and are not passed on, whichever of the two they were, so what reaches the block + // cipher begins with the salt either way and it is told to expect no header of its own + MEM_CONTEXT_OBJ_BEGIN(this) + { + this->cipherBlock = cipherBlockNewP( + cipherModeDecrypt, + cipherSpecNewP( + cipherSpecType(this->cipherSpec), cipherSpecPass(this->cipherSpec), .digest = repoFormatDigest(this->format)), + .header = cipherBlockHeaderNone); + } + MEM_CONTEXT_OBJ_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Read the header and pass everything behind it through the block cipher the header selected +***********************************************************************************************************************************/ +static void +cipherBlockFormatProcess(THIS_VOID, const Buffer *const source, Buffer *const destination) +{ + THIS(CipherBlockFormat); + + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(CIPHER_BLOCK_FORMAT, this); + FUNCTION_LOG_PARAM(BUFFER, source); + FUNCTION_LOG_PARAM(BUFFER, destination); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(destination != NULL); + + if (source != NULL) + { + // Hold back the header until there is enough of it to read + if (this->cipherBlock == NULL) + { + this->sourceOffset = CIPHER_BLOCK_FORMAT_HEADER_SIZE - this->headerSize; + + if (this->sourceOffset > bufUsed(source)) + this->sourceOffset = bufUsed(source); + + memcpy(this->header + this->headerSize, bufPtrConst(source), this->sourceOffset); + this->headerSize += this->sourceOffset; + + // Nothing can be decrypted until the header is complete + if (this->headerSize < CIPHER_BLOCK_FORMAT_HEADER_SIZE) + FUNCTION_LOG_RETURN_VOID(); + + cipherBlockFormatCipherNew(this); + } + + // Decrypt whatever of the source the header did not take + if (this->sourceOffset < bufUsed(source)) + { + ioFilterProcessInOut( + this->cipherBlock, BUF(bufPtrConst(source) + this->sourceOffset, bufUsed(source) - this->sourceOffset), + destination); + } + + // The header is stripped from the source only once + if (!ioFilterInputSame(this->cipherBlock)) + this->sourceOffset = 0; + } + // Else all the input has been seen, so the block cipher is flushed + else + { + // A file too short to hold a header cannot have one + if (this->cipherBlock == NULL) + THROW(CryptoError, "cipher header missing"); + + ioFilterProcessInOut(this->cipherBlock, NULL, destination); + } + + FUNCTION_LOG_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Is the filter done? +***********************************************************************************************************************************/ +static bool +cipherBlockFormatDone(const THIS_VOID) +{ + THIS(const CipherBlockFormat); + + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CIPHER_BLOCK_FORMAT, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(BOOL, this->cipherBlock != NULL && ioFilterDone(this->cipherBlock)); +} + +/*********************************************************************************************************************************** +Should the same input be provided again? +***********************************************************************************************************************************/ +static bool +cipherBlockFormatInputSame(const THIS_VOID) +{ + THIS(const CipherBlockFormat); + + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CIPHER_BLOCK_FORMAT, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(BOOL, this->cipherBlock != NULL && ioFilterInputSame(this->cipherBlock)); +} + +/*********************************************************************************************************************************** +Report the format the header gave +***********************************************************************************************************************************/ +static Pack * +cipherBlockFormatResultPack(THIS_VOID) +{ + THIS(CipherBlockFormat); + + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CIPHER_BLOCK_FORMAT, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + Pack *result; + + MEM_CONTEXT_TEMP_BEGIN() + { + PackWrite *const packWrite = pckWriteNewP(); + + pckWriteU32P(packWrite, this->format); + pckWriteEndP(packWrite); + + result = pckMove(pckWriteResult(packWrite), memContextPrior()); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_TEST_RETURN(PACK, result); +} + +/**********************************************************************************************************************************/ +FN_EXTERN IoFilter * +cipherBlockFormatNew(const CipherSpec *const cipherSpec, const CipherBlockFormatNewParam param) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_LOG_PARAM(UINT, param.format); + FUNCTION_LOG_END(); + + FUNCTION_AUDIT_HELPER(); + + ASSERT(cipherSpec != NULL); + ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone); + + OBJ_NEW_BEGIN(CipherBlockFormat, .childQty = MEM_CONTEXT_QTY_MAX) + { + *this = (CipherBlockFormat) + { + .cipherSpec = cipherSpecDup(cipherSpec), + .formatExpected = param.format, + }; + } + OBJ_NEW_END(); + + // Create param list + Pack *paramList; + + MEM_CONTEXT_TEMP_BEGIN() + { + PackWrite *const packWrite = pckWriteNewP(); + + cipherSpecPack(packWrite, cipherSpec); + pckWriteU32P(packWrite, param.format); + pckWriteEndP(packWrite); + + paramList = pckMove(pckWriteResult(packWrite), memContextPrior()); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN( + IO_FILTER, + ioFilterNewP( + CIPHER_BLOCK_FORMAT_FILTER_TYPE, this, paramList, .done = cipherBlockFormatDone, + .inOut = cipherBlockFormatProcess, .inputSame = cipherBlockFormatInputSame, + .result = cipherBlockFormatResultPack)); +} + +FN_EXTERN IoFilter * +cipherBlockFormatNewPack(const Pack *const paramList) +{ + IoFilter *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + PackRead *const paramListPack = pckReadNew(paramList); + const CipherSpec *const cipherSpec = cipherSpecNewPack(paramListPack); + const unsigned int format = pckReadU32P(paramListPack); + + result = ioFilterMove(cipherBlockFormatNewP(cipherSpec, .format = format), memContextPrior()); + } + MEM_CONTEXT_TEMP_END(); + + return result; +} + +/**********************************************************************************************************************************/ +FN_EXTERN unsigned int +cipherBlockFormatResult(PackRead *const packRead) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, packRead); + FUNCTION_TEST_END(); + + ASSERT(packRead != NULL); + + FUNCTION_TEST_RETURN(UINT, pckReadU32P(packRead)); +} + +/**********************************************************************************************************************************/ +FN_EXTERN void +cipherBlockFormatFilterGroupWriteAdd( + Buffer *const buffer, IoFilterGroup *const filterGroup, const CipherSpec *const cipherSpec, const unsigned int format) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(BUFFER, buffer); + FUNCTION_LOG_PARAM(IO_FILTER_GROUP, filterGroup); + FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_LOG_PARAM(UINT, format); + FUNCTION_LOG_END(); + + FUNCTION_AUDIT_HELPER(); + + ASSERT(buffer != NULL); + ASSERT(filterGroup != NULL); + ASSERT(cipherSpec != NULL); + ASSERT(format >= REPOSITORY_FORMAT_MIN && format <= REPOSITORY_FORMAT_MAX); + + if (cipherSpecType(cipherSpec) != cipherTypeNone) + { + // A format from 6 writes the header in place of the magic, so the content behind it is encrypted raw + const bool header = format >= REPOSITORY_FORMAT_6; + + if (header) + { + char headerZ[CIPHER_BLOCK_FORMAT_HEADER_SIZE + 1]; + + // Write the format zero-padded so it is always the same size, taking the digits it is written in from the value so + // that a format too large to fit could not run past them. The terminator lands on the byte after the header, which is + // not written to the buffer. + snprintf( + headerZ, sizeof(headerZ), CIPHER_BLOCK_FORMAT_MAGIC "%03u%c", format % 1000, CIPHER_BLOCK_FORMAT_RESERVED); + + bufCatC(buffer, (const uint8_t *)headerZ, 0, CIPHER_BLOCK_FORMAT_HEADER_SIZE); + } + + ioFilterGroupAdd( + filterGroup, + cipherBlockNewP( + cipherModeEncrypt, + cipherSpecNewP( + cipherSpecType(cipherSpec), cipherSpecPass(cipherSpec), .digest = repoFormatDigest(format)), + .header = header ? cipherBlockHeaderNone : cipherBlockHeaderMagic)); + } + + FUNCTION_LOG_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +FN_EXTERN IoFilterGroup * +cipherBlockFormatFilterGroupReadAdd(IoFilterGroup *const filterGroup, const CipherSpec *const cipherSpec) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(IO_FILTER_GROUP, filterGroup); + FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_LOG_END(); + + FUNCTION_AUDIT_HELPER(); + + ASSERT(filterGroup != NULL); + ASSERT(cipherSpec != NULL); + + if (cipherSpecType(cipherSpec) != cipherTypeNone) + ioFilterGroupAdd(filterGroup, cipherBlockFormatNewP(cipherSpec)); + + FUNCTION_LOG_RETURN(IO_FILTER_GROUP, filterGroup); +} diff --git a/src/common/format/cipherBlockFormat.h b/src/common/format/cipherBlockFormat.h new file mode 100644 index 000000000..1e54423ff --- /dev/null +++ b/src/common/format/cipherBlockFormat.h @@ -0,0 +1,55 @@ +/*********************************************************************************************************************************** +Cipher Block Format + +Encryption of a file written at a repository format, which is the format header in front of the content and the digest the format +derives the key with. Everything the format defines is here so that the block cipher itself has no knowledge of the repository. + +A writer knows the format, so it writes the header itself and encrypts the content raw behind it. A reader cannot know the format +until it has read the header, and a filter cannot be added to a read once it is open, so the read side is a filter that parses +the header, works out from it what the content is encrypted with, and passes everything after it through a block cipher of its own. +***********************************************************************************************************************************/ +#ifndef COMMON_FORMAT_CIPHERBLOCKFORMAT_H +#define COMMON_FORMAT_CIPHERBLOCKFORMAT_H + +#include "common/crypto/spec.h" +#include "common/io/filter/group.h" + +/*********************************************************************************************************************************** +Filter type constant +***********************************************************************************************************************************/ +#define CIPHER_BLOCK_FORMAT_FILTER_TYPE STRID5("cipher-fmt", 0x28d36e45441230) + +/*********************************************************************************************************************************** +Constructors +***********************************************************************************************************************************/ +typedef struct CipherBlockFormatNewParam +{ + VAR_PARAM_HEADER; + unsigned int format; // Format the file is expected at, any when zero +} CipherBlockFormatNewParam; + +#define cipherBlockFormatNewP(cipherSpec, ...) \ + cipherBlockFormatNew(cipherSpec, (CipherBlockFormatNewParam){VAR_PARAM_INIT, __VA_ARGS__}) + +FN_EXTERN IoFilter *cipherBlockFormatNew(const CipherSpec *cipherSpec, CipherBlockFormatNewParam param); +FN_EXTERN IoFilter *cipherBlockFormatNewPack(const Pack *paramList); + +/*********************************************************************************************************************************** +Getters/Setters +***********************************************************************************************************************************/ +// The format the header gave, which is what the header was read for +FN_EXTERN unsigned int cipherBlockFormatResult(PackRead *packRead); + +/*********************************************************************************************************************************** +Helper functions +***********************************************************************************************************************************/ +// Write the header a format calls for into the buffer the content will be written into and add encryption to the filter group. +// Nothing is written and nothing is added when the repository is not encrypted. +FN_EXTERN void cipherBlockFormatFilterGroupWriteAdd( + Buffer *buffer, IoFilterGroup *filterGroup, const CipherSpec *cipherSpec, unsigned int format); + +// Add decryption to the filter group for a file at a format that has yet to be read. Nothing is added when the repository is not +// encrypted. +FN_EXTERN IoFilterGroup *cipherBlockFormatFilterGroupReadAdd(IoFilterGroup *filterGroup, const CipherSpec *cipherSpec); + +#endif diff --git a/src/common/format/format.c b/src/common/format/format.c new file mode 100644 index 000000000..a837a6ac3 --- /dev/null +++ b/src/common/format/format.c @@ -0,0 +1,51 @@ +/*********************************************************************************************************************************** +Repository Format +***********************************************************************************************************************************/ +#include + +#include "common/debug.h" +#include "common/format/format.h" +#include "version.h" + +/**********************************************************************************************************************************/ +FN_EXTERN void +repoFormatValidate(const unsigned int format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_END(); + + // A format newer than this version can read requires an upgrade. Do not suggest a version since this version cannot know which + // version added the format. + if (format > REPOSITORY_FORMAT_MAX) + { + THROW_FMT( + FormatError, + "repository format %u requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + // A format older than this version can read requires an older version to migrate the repository + if (format < REPOSITORY_FORMAT_MIN) + { + THROW_FMT( + FormatError, + "repository format %u is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +FN_EXTERN HashType +repoFormatDigest(const unsigned int format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RETURN(STRING_ID, format >= REPOSITORY_FORMAT_6 ? hashTypeSha256 : hashTypeSha1); +} diff --git a/src/common/format/format.h b/src/common/format/format.h new file mode 100644 index 000000000..309112e97 --- /dev/null +++ b/src/common/format/format.h @@ -0,0 +1,37 @@ +/*********************************************************************************************************************************** +Repository Format + +Defines the format for info and manifest files as well as on-disk structure. Each info file and manifest stores the format it was +written with, so a repository may contain more than one format while older backups and archives expire. + +A constant is defined for each format so that code which varies by format can be explicit about the format it applies to. +REPOSITORY_FORMAT_MIN/MAX are the range that can be read. The allow list for repo-format in build/config.yaml must be kept in sync +with MIN/MAX and its default is the format used for new repositories. +***********************************************************************************************************************************/ +#ifndef COMMON_FORMAT_FORMAT_H +#define COMMON_FORMAT_FORMAT_H + +#include "common/crypto/common.h" + +/*********************************************************************************************************************************** +Format numbers +***********************************************************************************************************************************/ +#define REPOSITORY_FORMAT_5 5 +#define REPOSITORY_FORMAT_6 6 + +#define REPOSITORY_FORMAT_MIN REPOSITORY_FORMAT_5 +#define REPOSITORY_FORMAT_MAX REPOSITORY_FORMAT_6 + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Error when a format cannot be read by this version. The format in a cipher header is checked before anything is decrypted, since +// decrypting requires knowing what the format expects and this version does not know what a newer format expects. +FN_EXTERN void repoFormatValidate(unsigned int format); + +// Digest a pass stored in a file at this format derives the key with. SHA-1 is what every repository used before format 6 and is +// kept for those, so a repository that has not been migrated is read and written exactly as it was. A pass is generated with the +// digest of the file it will be stored in, since that is what a reader will derive it with. +FN_EXTERN HashType repoFormatDigest(unsigned int format); + +#endif diff --git a/src/config/config.c b/src/config/config.c index de49e46c3..f16c48a93 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1117,8 +1117,12 @@ cfgCipherSpecMainIdx(const unsigned int repoIdx) { const CipherType cipherType = cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx); - configLocal->cipherSpecMain[repoIdx] = cipherSpecNew( - cipherType, cipherType == cipherTypeNone ? NULL : BUFSTR(cfgOptionIdxStr(cfgOptRepoCipherPass, repoIdx))); + // The digest this pass derives with is decided by the format of the file it opens rather than chosen with the pass, + // so every caller rebuilds the spec from that format. SHA-1 is named here for a caller that does not, since a file + // that can be opened without reading a header first is one written before the format that added the header. + configLocal->cipherSpecMain[repoIdx] = cipherSpecNewP( + cipherType, cipherType == cipherTypeNone ? NULL : BUFSTR(cfgOptionIdxStr(cfgOptRepoCipherPass, repoIdx)), + .digest = hashTypeSha1); } MEM_CONTEXT_END(); } diff --git a/src/info/info.c b/src/info/info.c index d31941f1b..8f6d1a94e 100644 --- a/src/info/info.c +++ b/src/info/info.c @@ -9,8 +9,11 @@ Info Handler #include "common/crypto/hash.h" #include "common/debug.h" +#include "common/format/cipherBlockFormat.h" +#include "common/format/format.h" #include "common/ini.h" #include "common/io/filter/filter.h" +#include "common/io/io.h" #include "common/log.h" #include "common/type/convert.h" #include "common/type/json.h" @@ -121,6 +124,7 @@ infoNew(const unsigned int format, const CipherSpec *const cipherSpecSub) #define INFO_SECTION_BACKREST "backrest" #define INFO_KEY_CHECKSUM "backrest-checksum" #define INFO_SECTION_CIPHER "cipher" +#define INFO_KEY_CIPHER_DIGEST "cipher-digest" #define INFO_KEY_CIPHER_PASS "cipher-pass" FN_EXTERN Info * @@ -150,6 +154,7 @@ infoNewLoad( String *const sectionLast = strNew(); // The last section seen during load IoFilter *const checksumActualFilter = cryptoHashNew(hashTypeSha1); // Checksum calculated from the file const String *checksumExpected = NULL; // Checksum found in ini file + HashType cipherDigest = hashTypeSha1; // Digest the stored pass derives with INFO_CHECKSUM_BEGIN(checksumActualFilter); @@ -186,30 +191,10 @@ infoNewLoad( // Validate and store format if (strEqZ(value->key, INFO_KEY_FORMAT)) { - const uint64_t format = varUInt64(jsonToVar(value->value)); + const unsigned int format = jsonReadUInt(jsonReadNew(value->value)); + repoFormatValidate(format); - // A format newer than this version can read requires an upgrade. Do not suggest a version since - // this version cannot know which version added the format. - if (format > REPOSITORY_FORMAT_MAX) - { - THROW_FMT( - FormatError, - "repository format %" PRIu64 " requires a newer version of " PROJECT_NAME "\n" - "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", - format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); - } - - // A format older than this version can read requires an older version to migrate the repository - if (format < REPOSITORY_FORMAT_MIN) - { - THROW_FMT( - FormatError, - "repository format %" PRIu64 " is no longer supported by " PROJECT_NAME "\n" - "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", - format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); - } - - this->pub.format = (unsigned int)format; + this->pub.format = format; } // Store pgBackRest version else if (strEqZ(value->key, INFO_KEY_VERSION)) @@ -233,14 +218,23 @@ infoNewLoad( // Process cipher section else if (strEqZ(value->section, INFO_SECTION_CIPHER)) { + // Store the digest the pass derives with. A file written before the digest was stored has none, so the + // default is what every repository derived with then. + if (strEqZ(value->key, INFO_KEY_CIPHER_DIGEST)) + { + cipherDigest = jsonReadStrId(jsonReadNew(value->value)); + } // No validation needed for cipher-pass, just store it - if (strEqZ(value->key, INFO_KEY_CIPHER_PASS)) + else if (strEqZ(value->key, INFO_KEY_CIPHER_PASS)) { MEM_CONTEXT_OBJ_BEGIN(this) { - // The dependent files are encrypted with the same cipher type as this one - this->pub.cipherSpec = cipherSpecNew( - cipherSpecType(cipherSpec), BUFSTR(varStr(jsonToVar(value->value)))); + // The dependent files are encrypted with the same cipher type as this one and derive with the + // digest stored with the pass. The digest is read before this since the keys come out in order + // and digest sorts before pass. + this->pub.cipherSpec = cipherSpecNewP( + cipherSpecType(cipherSpec), BUFSTR(varStr(jsonToVar(value->value))), + .digest = cipherDigest); } MEM_CONTEXT_OBJ_END(); } @@ -281,6 +275,22 @@ infoNewLoad( // format is zero until the key is found and the value stored, so if we got here then the key was not found. if (infoFormat(this) == 0) THROW(FormatError, "repository format not found\nHINT: is this a valid " PROJECT_NAME " info file?"); + + // Only a read that looked for a header reports a format, so a result here means the file had one. The header is written + // from the same format as the content, so a file where they disagree has been damaged or put together from parts of two + // files. + PackRead *const cipherResult = ioFilterGroupResultP(ioReadFilterGroup(read), CIPHER_BLOCK_FORMAT_FILTER_TYPE); + + if (cipherResult != NULL) + { + const unsigned int formatHeader = cipherBlockFormatResult(cipherResult); + + if (this->pub.format != formatHeader) + { + THROW_FMT( + FormatError, "repository format %u does not match header format %u", this->pub.format, formatHeader); + } + } } MEM_CONTEXT_TEMP_END(); @@ -406,6 +416,24 @@ infoSave(Info *const this, IoWrite *const write, InfoSaveCallback *const callbac if (cipherSpecType(infoCipherSpec(this)) != cipherTypeNone) { callbackFunction(callbackData, STRDEF(INFO_SECTION_CIPHER), &data); + + // A format before the digest could be stored has nowhere to put it and a reader assumes SHA-1, so a pass stored at + // one of those formats must derive with SHA-1 or the reader will derive a different key. This is checked rather than + // asserted because the file is written either way and nothing reports it, leaving a pass that no reader can derive. + CHECK( + AssertError, infoFormat(this) >= REPOSITORY_FORMAT_6 || cipherSpecDigest(infoCipherSpec(this)) == hashTypeSha1, + "pass must derive with sha1 before the format that stores the digest"); + + // Store the digest the pass derives with so that a pass outlives the format of the file it is stored in. A pass in a + // file written before this could be stored derives with SHA-1, which is what a reader assumes when it finds no digest. + if (infoFormat(this) >= REPOSITORY_FORMAT_6) + { + char digestZ[STRID_MAX + 1]; + strIdToZ(cipherSpecDigest(infoCipherSpec(this)), digestZ); + + infoSaveValue(&data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_DIGEST, jsonFromVar(VARSTRZ(digestZ))); + } + infoSaveValue( &data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_PASS, jsonFromVar(VARSTR(strNewBuf(cipherSpecPass(infoCipherSpec(this)))))); diff --git a/src/info/info.h b/src/info/info.h index 3af5818af..88a4fb843 100644 --- a/src/info/info.h +++ b/src/info/info.h @@ -41,8 +41,9 @@ Constructors ***********************************************************************************************************************************/ FN_EXTERN Info *infoNew(unsigned int format, const CipherSpec *cipherSpecSub); -// Create new object and load contents from a file. The cipher spec the file is read with supplies the type for the cipher spec -// built from the pass stored in it. +// Create new object and load contents from a file. Decryption is added by the caller since which filter a file needs depends on +// whether it begins with a header. The cipher spec the file is read with supplies the type for the cipher spec built from the pass +// stored in it. FN_EXTERN Info *infoNewLoad( IoRead *read, const CipherSpec *cipherSpec, InfoLoadNewCallback *callbackFunction, void *callbackData); diff --git a/src/info/infoArchive.c b/src/info/infoArchive.c index 4e38a6003..824d38582 100644 --- a/src/info/infoArchive.c +++ b/src/info/infoArchive.c @@ -8,8 +8,8 @@ Archive Info Handler #include #include -#include "common/crypto/cipherBlock.h" #include "common/debug.h" +#include "common/format/cipherBlockFormat.h" #include "common/ini.h" #include "common/io/bufferWrite.h" #include "common/io/io.h" @@ -252,8 +252,7 @@ infoArchiveLoadFileCallback(void *const data, const unsigned int try) // Attempt to load the file IoRead *const read = storageReadIo(storageNewReadP(loadData->storage, fileName)); - cipherBlockFilterGroupAdd( - ioReadFilterGroup(read), cipherModeDecrypt, loadData->cipherSpec); + cipherBlockFormatFilterGroupReadAdd(ioReadFilterGroup(read), loadData->cipherSpec); MEM_CONTEXT_BEGIN(loadData->memContext) { @@ -340,7 +339,7 @@ infoArchiveSaveFile( // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); IoWrite *const write = ioBufferWriteNew(buffer); - cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec); + cipherBlockFormatFilterGroupWriteAdd(buffer, ioWriteFilterGroup(write), cipherSpec, infoArchiveFormat(infoArchive)); infoArchiveSave(infoArchive, write); // Save the file and make a copy diff --git a/src/info/infoBackup.c b/src/info/infoBackup.c index 5385097a2..5bdf6853e 100644 --- a/src/info/infoBackup.c +++ b/src/info/infoBackup.c @@ -9,8 +9,8 @@ Backup Info Handler #include #include "command/backup/common.h" -#include "common/crypto/cipherBlock.h" #include "common/debug.h" +#include "common/format/cipherBlockFormat.h" #include "common/ini.h" #include "common/io/bufferWrite.h" #include "common/io/io.h" @@ -716,8 +716,7 @@ infoBackupLoadFileCallback(void *const data, const unsigned int try) // Attempt to load the file IoRead *const read = storageReadIo(storageNewReadP(loadData->storage, fileName)); - cipherBlockFilterGroupAdd( - ioReadFilterGroup(read), cipherModeDecrypt, loadData->cipherSpec); + cipherBlockFormatFilterGroupReadAdd(ioReadFilterGroup(read), loadData->cipherSpec); MEM_CONTEXT_BEGIN(loadData->memContext) { @@ -939,7 +938,7 @@ infoBackupSaveFile( // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); IoWrite *const write = ioBufferWriteNew(buffer); - cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec); + cipherBlockFormatFilterGroupWriteAdd(buffer, ioWriteFilterGroup(write), cipherSpec, infoBackupFormat(infoBackup)); infoBackupSave(infoBackup, write); // Save the file and make a copy diff --git a/src/info/manifest/serialize.c.inc b/src/info/manifest/serialize.c.inc index 64bf889b7..d3867bb98 100644 --- a/src/info/manifest/serialize.c.inc +++ b/src/info/manifest/serialize.c.inc @@ -1037,7 +1037,7 @@ manifestLoadFileCallback(void *const data, const unsigned int try) // Construct filename based on try const String *const fileName = try == 0 ? loadData->fileName : strNewFmt("%s" INFO_COPY_EXT, strZ(loadData->fileName)); - // Attempt to load the file + // Attempt to load the file. A manifest has no header of its own, so it is decrypted with the spec as it was given. IoRead *const read = storageReadIo(storageNewReadP(loadData->storage, fileName)); cipherBlockFilterGroupAdd(ioReadFilterGroup(read), cipherModeDecrypt, loadData->cipherSpec); diff --git a/src/meson.build b/src/meson.build index 67793b0d8..4063c28db 100644 --- a/src/meson.build +++ b/src/meson.build @@ -126,6 +126,8 @@ src_pgbackrest = [ 'common/error/retry.c', 'common/exec.c', 'common/fork.c', + 'common/format/cipherBlockFormat.c', + 'common/format/format.c', 'common/ini.c', 'common/io/bufferRead.c', 'common/io/bufferWrite.c', diff --git a/src/version.h b/src/version.h index d1e078206..eaa579b5f 100644 --- a/src/version.h +++ b/src/version.h @@ -24,20 +24,6 @@ Config include path name. The parent path will vary based on configuration. ***********************************************************************************************************************************/ #define PROJECT_CONFIG_INCLUDE_PATH "conf.d" -/*********************************************************************************************************************************** -Format Number -- defines format for info and manifest files as well as on-disk structure. Each info file and manifest stores the -format it was written with, so a repository may contain more than one format while older backups and archives expire. - -A constant is defined for each format so that code which varies by format can be explicit about the format it applies to. -REPOSITORY_FORMAT_MIN/MAX are the range that can be read. The allow list for repo-format in build/config.yaml must be kept in sync -with MIN/MAX and its default is the format used for new repositories. -***********************************************************************************************************************************/ -#define REPOSITORY_FORMAT_5 5 -#define REPOSITORY_FORMAT_6 6 - -#define REPOSITORY_FORMAT_MIN REPOSITORY_FORMAT_5 -#define REPOSITORY_FORMAT_MAX REPOSITORY_FORMAT_6 - /*********************************************************************************************************************************** Project version components. PROJECT_VERSION and PROJECT_VERSION_NUM are automatically generated from the component parts. ***********************************************************************************************************************************/ diff --git a/test/define.yaml b/test/define.yaml index 36ea93f94..a74427a5b 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -375,6 +375,14 @@ unit: - common/crypto/spec - common/crypto/xxhash + # -------------------------------------------------------------------------------------------------------------------------------- + - name: common/format + total: 3 + + coverage: + - common/format/cipherBlockFormat + - common/format/format + # -------------------------------------------------------------------------------------------------------------------------------- - name: common/io/tls total: 6 diff --git a/test/src/harness/config.h b/test/src/harness/config.h index c12931645..ea2504123 100644 --- a/test/src/harness/config.h +++ b/test/src/harness/config.h @@ -18,7 +18,7 @@ Config option constants #define TEST_CIPHER_PASS_ARCHIVE "xarchivex" // Cipher spec for a test pass, defaulting to the main pass, since most tests encrypt with that -#define TEST_CIPHER_SPEC_PASS(pass) cipherSpecNew(cipherTypeAes256Cbc, BUFSTRZ(pass)) +#define TEST_CIPHER_SPEC_PASS(pass) cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRZ(pass)) #define TEST_CIPHER_SPEC TEST_CIPHER_SPEC_PASS(TEST_CIPHER_PASS) /*********************************************************************************************************************************** diff --git a/test/src/harness/host.c b/test/src/harness/host.c index d82554a0d..6492c946d 100644 --- a/test/src/harness/host.c +++ b/test/src/harness/host.c @@ -1232,7 +1232,7 @@ hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const si hrnHostLocal.storage = strIdFromZ(testDef->stg); hrnHostLocal.compressType = compressTypeEnum(strIdFromZ(testDef->cmp)); hrnHostLocal.cipherSpecMain = - testDef->enc ? cipherSpecNew(cipherTypeAes256Cbc, BUFSTRZ(HRN_CIPHER_PASSPHRASE)) : cipherSpecNewNone(); + testDef->enc ? cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRZ(HRN_CIPHER_PASSPHRASE)) : cipherSpecNewNone(); hrnHostLocal.repoTotal = testDef->rt; hrnHostLocal.tls = testDef->tls; hrnHostLocal.bundle = testDef->bnd; diff --git a/test/src/harness/info.c b/test/src/harness/info.c index d99f178e4..4a120dcc1 100644 --- a/test/src/harness/info.c +++ b/test/src/harness/info.c @@ -6,8 +6,12 @@ Harness for Loading Test Configurations #include #include "common/assert.h" +#include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" +#include "common/format/cipherBlockFormat.h" +#include "common/format/format.h" #include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" #include "common/io/filter/filter.h" #include "common/type/json.h" #include "info/info.h" @@ -116,6 +120,90 @@ harnessInfoChecksum(const String *const info) FUNCTION_HARNESS_RETURN(BUFFER, harnessInfoChecksumFormat(REPOSITORY_FORMAT_DEFAULT, info)); } +/**********************************************************************************************************************************/ +void +hrnInfoPut(const Storage *const storage, const char *const file, const char *const info, HrnInfoPutParam param) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(STORAGE, storage); + FUNCTION_HARNESS_PARAM(STRINGZ, file); + FUNCTION_HARNESS_PARAM(STRINGZ, info); + FUNCTION_HARNESS_PARAM(UINT, param.format); + FUNCTION_HARNESS_PARAM(BOOL, param.header); + FUNCTION_HARNESS_PARAM(CIPHER_SPEC, param.cipherSpec); + FUNCTION_HARNESS_PARAM(STRINGZ, param.comment); + FUNCTION_HARNESS_END(); + + ASSERT(info != NULL); + + // Default to the format a new repository is created at + if (param.format == 0) + param.format = REPOSITORY_FORMAT_DEFAULT; + + const Buffer *content = harnessInfoChecksumFormat(param.format, STR(info)); + + // Encrypt the way the format stores the file. A file that contains a header gets it in place of the magic the cipher writes, + // and the pass derives with the digest the format calls for. + if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) + { + Buffer *const encrypted = bufNew(0); + IoWrite *const write = ioBufferWriteNew(encrypted); + + // If a file with a header + if (param.header) + { + cipherBlockFormatFilterGroupWriteAdd(encrypted, ioWriteFilterGroup(write), param.cipherSpec, param.format); + } + // Else a file with no header of its own, e.g. a manifest, which is still stored with the digest of its format + else + { + cipherBlockFilterGroupAdd( + ioWriteFilterGroup(write), cipherModeEncrypt, + cipherSpecNewP( + cipherSpecType(param.cipherSpec), cipherSpecPass(param.cipherSpec), + .digest = repoFormatDigest(param.format))); + } + + ioWriteOpen(write); + ioWrite(write, content); + ioWriteClose(write); + + content = encrypted; + } + + hrnStoragePut(storage, file, content, "put info", (HrnStoragePutParam){VAR_PARAM_INIT, .comment = param.comment}); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +Buffer * +harnessInfoEncrypt(const Buffer *const content, const CipherSpec *const cipherSpec, const HarnessInfoEncryptParam param) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(BUFFER, content); + FUNCTION_HARNESS_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_HARNESS_PARAM(UINT, param.format); + FUNCTION_HARNESS_END(); + + ASSERT(content != NULL); + ASSERT(cipherSpec != NULL); + + Buffer *const result = bufNew(0); + IoWrite *const write = ioBufferWriteNew(result); + + if (param.format == 0) + cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec); + else + cipherBlockFormatFilterGroupWriteAdd(result, ioWriteFilterGroup(write), cipherSpec, param.format); + + ioWriteOpen(write); + ioWrite(write, content); + ioWriteClose(write); + + FUNCTION_HARNESS_RETURN(BUFFER, result); +} + /*********************************************************************************************************************************** Test callback that logs the results to a string ***********************************************************************************************************************************/ diff --git a/test/src/harness/info.h b/test/src/harness/info.h index e1457ee7d..733bb8144 100644 --- a/test/src/harness/info.h +++ b/test/src/harness/info.h @@ -1,6 +1,7 @@ /*********************************************************************************************************************************** Harness for Generating Test Info Files ***********************************************************************************************************************************/ +#include "common/format/format.h" #include "common/type/buffer.h" #include "info/info.h" @@ -13,17 +14,27 @@ know this since the option supplies it everywhere else. #define REPOSITORY_FORMAT_DEFAULT REPOSITORY_FORMAT_5 /*********************************************************************************************************************************** -Write info to a file and add the checksum +Write info to a file and add the checksum, storing it the way the format stores it when it is encrypted ***********************************************************************************************************************************/ +typedef struct HrnInfoPutParam +{ + VAR_PARAM_HEADER; + unsigned int format; // Repository format, default format when zero + bool header; // Does the file contain a header, i.e. is it an info file? + const CipherSpec *cipherSpec; // Cipher spec when the file is encrypted, digest set by format + const char *comment; // Comment +} HrnInfoPutParam; + #define HRN_INFO_PUT(storage, file, info, ...) \ do \ { \ hrnTestLogPrefix(__LINE__); \ - hrnStoragePut( \ - storage, file, harnessInfoChecksumZ(info), "put info", (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__}); \ + hrnInfoPut(storage, file, info, (HrnInfoPutParam){VAR_PARAM_INIT, __VA_ARGS__}); \ } \ while (0) +void hrnInfoPut(const Storage *storage, const char *file, const char *info, HrnInfoPutParam param); + /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ @@ -31,4 +42,17 @@ Buffer *harnessInfoChecksum(const String *info); Buffer *harnessInfoChecksumFormat(unsigned int format, const String *info); Buffer *harnessInfoChecksumZ(const char *info); +// Encrypt content the way a file at a format is stored, i.e. with the header and digest that go with the format. No format means +// the file has none of its own, e.g. a manifest, so it is stored with no header and the digest the spec was given. +typedef struct HarnessInfoEncryptParam +{ + VAR_PARAM_HEADER; + unsigned int format; // Repository format the file is stored at +} HarnessInfoEncryptParam; + +#define harnessInfoEncryptP(content, cipherSpec, ...) \ + harnessInfoEncrypt(content, cipherSpec, (HarnessInfoEncryptParam){VAR_PARAM_INIT, __VA_ARGS__}) + +Buffer *harnessInfoEncrypt(const Buffer *content, const CipherSpec *cipherSpec, HarnessInfoEncryptParam param); + void harnessInfoLoadNewCallback(void *callbackData, const String *section, const String *key, JsonRead *json); diff --git a/test/src/harness/storage.c b/test/src/harness/storage.c index 89a5dbdd7..8b8901f21 100644 --- a/test/src/harness/storage.c +++ b/test/src/harness/storage.c @@ -69,7 +69,17 @@ testStorageGet(const Storage *const storage, const char *const file, const char // Add decrypt filter if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) { - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, param.cipherSpec)); + // Derive with SHA-1 since the harness reads and writes files the way a repository at the format these tests build stores + // them, which is the format that had no header to define anything else. A caller that asked for a digest would not get + // it, so only a spec at the default is accepted. + ASSERT(cipherSpecDigest(param.cipherSpec) == hashTypeSha256); + + ioFilterGroupAdd( + filterGroup, + cipherBlockNewP( + cipherModeDecrypt, + cipherSpecNewP( + cipherSpecType(param.cipherSpec), cipherSpecPass(param.cipherSpec), .digest = hashTypeSha1))); strCatFmt( filter, "enc[%s,%s] ", zNewStrId(cipherSpecType(param.cipherSpec)), @@ -393,7 +403,18 @@ hrnStoragePut( // Add encrypted filter if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, param.cipherSpec)); + { + // Derive with SHA-1 to match how the harness reads these files back. A caller that asked for a digest would not get it, + // so only a spec at the default is accepted. + ASSERT(cipherSpecDigest(param.cipherSpec) == hashTypeSha256); + + ioFilterGroupAdd( + filterGroup, + cipherBlockNewP( + cipherModeEncrypt, + cipherSpecNewP( + cipherSpecType(param.cipherSpec), cipherSpecPass(param.cipherSpec), .digest = hashTypeSha1))); + } // Add file name printf( diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index 0981445a0..380e1a0b2 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -93,7 +93,8 @@ testBackupValidateFile( if (cipherSpecType(cipherSpecBackup) != cipherTypeNone) { ioFilterGroupAdd( - ioReadFilterGroup(storageReadIo(read)), cipherBlockNewP(cipherModeDecrypt, cipherSpecBackup, .raw = true)); + ioReadFilterGroup(storageReadIo(read)), cipherBlockNewP( + cipherModeDecrypt, cipherSpecBackup, .header = cipherBlockHeaderNone)); } ioReadOpen(storageReadIo(read)); @@ -173,7 +174,8 @@ testBackupValidateFile( if (cipherSpecType(cipherSpecBackup) != cipherTypeNone) { ioFilterGroupAdd( - ioReadFilterGroup(storageReadIo(read)), cipherBlockNewP(cipherModeDecrypt, cipherSpecBackup, .raw = raw)); + ioReadFilterGroup(storageReadIo(read)), cipherBlockNewP( + cipherModeDecrypt, cipherSpecBackup, .header = raw ? cipherBlockHeaderNone : cipherBlockHeaderMagic)); } if (manifestData->backupOptionCompressType != compressTypeNone) @@ -1331,7 +1333,8 @@ testRun(void) blockIncrNew( 3, 3, 8, 2, 4, 5, NULL, compressFilterP(compressTypeGz, 1, .raw = true), cipherBlockNewP( - cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS)), .raw = true)))), + cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS)), + .header = cipherBlockHeaderNone)))), "block incr pack"); } diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index 53d9fe0d3..45ea6cbf7 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -311,7 +311,8 @@ testRun(void) TEST_ERROR( restoreFile( strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strZ(repoFileReferenceFull), strZ(repoFile1)), repoIdx, compressTypeGz, - 0, false, false, false, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("badpass")), NULL, fileList), + 0, false, false, false, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("badpass"), .digest = hashTypeSha1), NULL, + fileList), ChecksumError, "error restoring 'normal': actual checksum 'd1cd8a7d11daa26814b93eb604e1d49ab4b43770' does not match expected checksum" " 'ffffffffffffffffffffffffffffffffffffffff'"); @@ -2309,7 +2310,8 @@ testRun(void) Manifest *manifestEncrypted = manifestLoadFile( storageRepoIdxWrite(0), STRDEF(STORAGE_REPO_BACKUP "/" TEST_LABEL "/" BACKUP_MANIFEST_FILE), cipherSpecNewNone()); - manifestCipherSpecSet(manifestEncrypted, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); + manifestCipherSpecSet( + manifestEncrypted, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE), .digest = hashTypeSha1)); // Open file for write IoWrite *write = storageWriteIo( @@ -2321,7 +2323,8 @@ testRun(void) #define TEST_CIPHER_PASS_MANIFEST "backpass" cipherBlockFilterGroupAdd( ioWriteFilterGroup(write), cipherModeEncrypt, - cipherSpecNew(cfgOptionIdxStrId(cfgOptRepoCipherType, 1), BUFSTRDEF(TEST_CIPHER_PASS_MANIFEST))); + cipherSpecNewP( + cfgOptionIdxStrId(cfgOptRepoCipherType, 1), BUFSTRDEF(TEST_CIPHER_PASS_MANIFEST), .digest = hashTypeSha1)); manifestSave(manifestEncrypted, write); // Write backup.info to the encrypted repo @@ -2332,10 +2335,10 @@ testRun(void) // Write archive.info to the encrypted repo InfoArchive *infoArchive = infoArchiveNew( PG_VERSION_11, 6569239123849665679, REPOSITORY_FORMAT_DEFAULT, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE), .digest = hashTypeSha1)); infoArchiveSaveFile( infoArchive, storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE_STR, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS))); + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS))); TEST_RESULT_VOID(hrnCmdRestore(), "successful restore"); diff --git a/test/src/module/command/stanzaTest.c b/test/src/module/command/stanzaTest.c index b29ffa9bf..bfa2a287a 100644 --- a/test/src/module/command/stanzaTest.c +++ b/test/src/module/command/stanzaTest.c @@ -1,6 +1,8 @@ /*********************************************************************************************************************************** Test Stanza Commands ***********************************************************************************************************************************/ +#include "common/crypto/cipherBlock.h" +#include "common/io/bufferWrite.h" #include "postgres/interface.h" #include "postgres/version.h" #include "storage/posix/storage.h" @@ -23,6 +25,7 @@ testRun(void) Storage *storageHrn = storagePosixNewP(HRN_PATH_STR, .write = true); #define TEST_STANZA "db" + #define TEST_WAL_MIGRATE "archive/db/15-1/000000010000000000000001" #define TEST_STANZA_OTHER "otherstanza" StringList *argListBase = strLstNew(); @@ -137,7 +140,7 @@ testRun(void) TEST_ASSIGN( infoArchive, infoArchiveLoadFile( - storageRepoIdx(1), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), + storageRepoIdx(1), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), "load archive info from encrypted repo2"); TEST_RESULT_UINT(cipherSpecType(infoArchiveCipherSpec(infoArchive)), cipherTypeAes256Cbc, "cipher sub set"); @@ -145,7 +148,7 @@ testRun(void) TEST_ASSIGN( infoBackup, infoBackupLoadFile( - storageRepoIdx(1), INFO_BACKUP_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), + storageRepoIdx(1), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), "load backup info from encrypted repo2"); TEST_RESULT_UINT(cipherSpecType(infoBackupCipherSpec(infoBackup)), cipherTypeAes256Cbc, "cipher sub set"); @@ -185,14 +188,14 @@ testRun(void) TEST_ASSIGN( infoArchive, infoArchiveLoadFile( - storageRepoIdx(3), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), + storageRepoIdx(3), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), "load archive info from encrypted repo4"); TEST_RESULT_UINT(cipherSpecType(infoArchiveCipherSpec(infoArchive)), cipherTypeAes256Cbc, "cipher sub set"); TEST_ASSIGN( infoBackup, infoBackupLoadFile( - storageRepoIdx(3), INFO_BACKUP_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), + storageRepoIdx(3), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), "load backup info from encrypted repo4"); TEST_RESULT_UINT(cipherSpecType(infoBackupCipherSpec(infoBackup)), cipherTypeAes256Cbc, "cipher sub set"); @@ -1190,6 +1193,68 @@ testRun(void) hrnCfgEnvKeyRemoveRaw(cfgOptRepoFormat, 1); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("stanza-upgrade - a file encrypted before a format migration is readable after it"); + + // A stanza on an encrypted repository at the format before the digest could be stored + argList = strLstNew(); + hrnCfgArgRawBool(argList, cfgOptOnline, false); + hrnCfgArgRawZ(argList, cfgOptStanza, TEST_STANZA); + hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, TEST_PATH "/repo-migrate"); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + hrnCfgArgKeyRawStrId(argList, cfgOptRepoCipherType, 1, cipherTypeAes256Cbc); + hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 1, "12345678"); + HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); + + TEST_RESULT_VOID(cmdStanzaCreate(), "stanza create on encrypted repo"); + TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'db' on repo1"); + + const CipherSpec *const cipherSpecMain = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("12345678")); + const InfoArchive *infoArchiveMigrate = NULL; + + TEST_ASSIGN( + infoArchiveMigrate, infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecMain), + "load archive info before migration"); + + // Write a file with the archive sub-pass, which is how everything below the info files is encrypted + Buffer *const walBuffer = bufNew(0); + IoWrite *const walWrite = ioBufferWriteNew(walBuffer); + + cipherBlockFilterGroupAdd( + ioWriteFilterGroup(walWrite), cipherModeEncrypt, infoArchiveCipherSpec(infoArchiveMigrate)); + ioWriteOpen(walWrite); + ioWrite(walWrite, BUFSTRDEF("WAL BEFORE MIGRATION")); + ioWriteClose(walWrite); + + HRN_STORAGE_PUT(storageRepoIdxWrite(0), TEST_WAL_MIGRATE, walBuffer, .comment = "wal written at format 5"); + + // Migrate the repository + hrnCfgArgKeyRawZ(argList, cfgOptRepoFormat, 1, "6"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - format 6 on encrypted repo"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" + "P00 INFO: upgrade repository format from 5 to 6"); + + // The pass is unchanged by the migration, so a file it encrypted before the migration must still be readable. The digest + // the pass derives with cannot follow the format of the info file storing it or this read would fail. + TEST_ASSIGN( + infoArchiveMigrate, infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecMain), + "load archive info after migration"); + TEST_RESULT_UINT(infoArchiveFormat(infoArchiveMigrate), REPOSITORY_FORMAT_6, "archive info at format 6"); + + StorageRead *const walRead = storageNewReadP(storageRepoIdx(0), STRDEF(TEST_WAL_MIGRATE)); + + cipherBlockFilterGroupAdd( + ioReadFilterGroup(storageReadIo(walRead)), cipherModeDecrypt, infoArchiveCipherSpec(infoArchiveMigrate)); + + TEST_RESULT_STR_Z( + strNewBuf(storageGetP(walRead)), "WAL BEFORE MIGRATION", "wal from before the migration is still readable"); + + hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 1); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("stanza-upgrade - every format that can be read can be requested"); diff --git a/test/src/module/command/verifyTest.c b/test/src/module/command/verifyTest.c index 92740965a..352567fc6 100644 --- a/test/src/module/command/verifyTest.c +++ b/test/src/module/command/verifyTest.c @@ -876,22 +876,57 @@ testRun(void) hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 1, TEST_CIPHER_PASS); HRN_CFG_LOAD(cfgCmdVerify, argList); + // The copy is a copy of the file rather than a second encryption of the same content, which is how the info files are + // saved, i.e. the content is built once in a buffer and that buffer is written twice HRN_INFO_PUT( - storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .cipherSpec = TEST_CIPHER_SPEC, - .comment = "encrypted backup.info"); + storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .header = true, + .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted backup.info"); + HRN_STORAGE_COPY( + storageRepo(), INFO_BACKUP_PATH_FILE, storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted backup.info.copy"); HRN_INFO_PUT( - storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, TEST_NO_CURRENT_BACKUP, - .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted backup.info.copy"); - HRN_INFO_PUT( - storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, .header = true, .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted archive.info"); - HRN_INFO_PUT( - storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, - .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted archive.info.copy"); + HRN_STORAGE_COPY( + storageRepo(), INFO_ARCHIVE_PATH_FILE, storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted archive.info.copy"); TEST_RESULT_VOID(cmdVerify(), "usable encrypted backup and archive info files"); TEST_RESULT_LOG(""); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("encrypted info files at format 6"); + + HRN_INFO_PUT( + storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .format = REPOSITORY_FORMAT_6, .header = true, + .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted backup.info at format 6"); + HRN_STORAGE_COPY( + storageRepo(), INFO_BACKUP_PATH_FILE, storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted backup.info.copy at format 6"); + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, .format = REPOSITORY_FORMAT_6, + .header = true, .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted archive.info at format 6"); + HRN_STORAGE_COPY( + storageRepo(), INFO_ARCHIVE_PATH_FILE, storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted archive.info.copy at format 6"); + + TEST_RESULT_VOID(cmdVerify(), "usable encrypted backup and archive info files at format 6"); + TEST_RESULT_LOG(""); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a copy that cannot be decrypted is reported as such rather than as a checksum that does not match"); + + // Damage the content of the copy, which nothing but the drain reads, so that it no longer decrypts + Buffer *const archiveInfoCopy = bufDup( + storageGetP(storageNewReadP(storageRepo(), STRDEF(INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT)))); + + bufPtr(archiveInfoCopy)[bufUsed(archiveInfoCopy) - 1] ^= 0xff; + HRN_STORAGE_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, archiveInfoCopy, .comment = "damaged archive.info.copy"); + + TEST_RESULT_VOID(cmdVerify(), "encrypted archive.info.copy that cannot be decrypted"); + TEST_RESULT_LOG("P00 DETAIL: unable to flush"); + hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 1); harnessLogLevelReset(); } @@ -941,12 +976,12 @@ testRun(void) TEST_RESULT_UINT( verifyFile( filePathName, 0, NULL, compressTypeGz, fileChecksum, fileSize, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("pass"))), + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("pass"), .digest = hashTypeSha1)), verifyOk, "file encrypted compressed ok"); TEST_RESULT_UINT( verifyFile( filePathName, 0, NULL, compressTypeGz, bufNewDecode(encodingHex, STRDEF("aa")), fileSize, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("pass"))), + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("pass"), .digest = hashTypeSha1)), verifyChecksumMismatch, "file encrypted compressed checksum mismatch"); } diff --git a/test/src/module/common/cryptoTest.c b/test/src/module/common/cryptoTest.c index 93ac692cd..6336faf38 100644 --- a/test/src/module/common/cryptoTest.c +++ b/test/src/module/common/cryptoTest.c @@ -2,9 +2,11 @@ Test Block Cipher ***********************************************************************************************************************************/ #include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" #include "common/io/filter/filter.h" #include "common/io/io.h" #include "common/type/json.h" +#include "version.h" /*********************************************************************************************************************************** Data for testing @@ -14,6 +16,9 @@ Data for testing #define TEST_PLAINTEXT "plaintext" #define TEST_BUFFER_SIZE 256 +// What a file that is not raw begins with, i.e. the magic and the salt behind it +#define TEST_HEADER_SIZE (CIPHER_BLOCK_MAGIC_SIZE + PKCS5_SALT_LEN) + /*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ @@ -72,21 +77,26 @@ testRun(void) // ***************************************************************************************************************************** if (testBegin("CipherBlock")) { - // Cipher error + // Cipher and digest errors // ------------------------------------------------------------------------------------------------------------------------- TEST_ERROR( - cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(strIdFromZ(BOGUS_STR), testPass)), AssertError, + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(strIdFromZ(BOGUS_STR), testPass)), AssertError, "unable to load cipher 'BOGUS'"); + TEST_ERROR( + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass, .digest = strIdFromZ(BOGUS_STR))), + AssertError, "unable to load digest 'BOGUS'"); // Initialization of object // ------------------------------------------------------------------------------------------------------------------------- - // Build from a duplicate to show the copy contains the type and pass of the original + // Build from a duplicate to show the copy contains the type, digest, and pass of the original TEST_RESULT_UINT(cipherSpecType(cipherSpecDup(cipherSpecNewNone())), cipherTypeNone, "dup of none"); - const CipherSpec *const cipherSpec = cipherSpecDup(cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS))); + const CipherSpec *const cipherSpec = cipherSpecDup(cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS))); - TEST_RESULT_UINT(cipherSpecType(cipherSpec), cipherTypeAes256Cbc, "dup type"); - TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(cipherSpec)), TEST_PASS, "dup pass"); + TEST_RESULT_UINT(cipherSpecDigest(cipherSpec), hashTypeSha256, "dup default digest"); + TEST_RESULT_UINT( + cipherSpecDigest(cipherSpecDup(cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS), .digest = hashTypeSha1))), + hashTypeSha1, "dup digest"); // A pack contains nothing but the type when there is no cipher PackWrite *packWrite = pckWriteNewP(); @@ -97,15 +107,17 @@ testRun(void) TEST_RESULT_UINT( cipherSpecType(cipherSpecNewPack(pckReadNew(pckWriteResult(packWrite)))), cipherTypeNone, "unpack none"); - // Else it contains the type and pass + // Else it contains the type, digest, and pass. Pack a digest that is not the default so that a pack which loses the digest + // cannot pass by falling back to the default. packWrite = pckWriteNewP(); - cipherSpecPack(packWrite, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + cipherSpecPack(packWrite, cipherSpecNewP(cipherTypeAes256Cbc, testPass, .digest = hashTypeSha1)); pckWriteEndP(packWrite); const CipherSpec *const cipherSpecUnpack = cipherSpecNewPack(pckReadNew(pckWriteResult(packWrite))); TEST_RESULT_UINT(cipherSpecType(cipherSpecUnpack), cipherTypeAes256Cbc, "unpack type"); + TEST_RESULT_UINT(cipherSpecDigest(cipherSpecUnpack), hashTypeSha1, "unpack digest"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(cipherSpecUnpack)), TEST_PASS, "unpack pass"); CipherBlock *cipherBlock = (CipherBlock *)ioFilterDriver(cipherBlockNewP(cipherModeEncrypt, cipherSpec)); @@ -123,7 +135,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- Buffer *encryptBuffer = bufNew(TEST_BUFFER_SIZE); - IoFilter *blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + IoFilter *blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockEncryptFilter = cipherBlockNewPack(ioFilterParamList(blockEncryptFilter)); CipherBlock *blockEncrypt = (CipherBlock *)ioFilterDriver(blockEncryptFilter); @@ -143,7 +155,7 @@ testRun(void) TEST_RESULT_BOOL(blockEncrypt->saltDone, true, "salt done is true"); TEST_RESULT_BOOL(blockEncrypt->processDone, true, "process done is true"); TEST_RESULT_UINT(blockEncrypt->headerSize, 0, "header size is 0"); - TEST_RESULT_UINT(bufUsed(encryptBuffer), CIPHER_BLOCK_HEADER_SIZE, "cipher size is header len"); + TEST_RESULT_UINT(bufUsed(encryptBuffer), TEST_HEADER_SIZE, "cipher size is header len"); TEST_RESULT_UINT( cipherBlockProcessSize(blockEncrypt, strlen(TEST_PLAINTEXT)), @@ -158,13 +170,13 @@ testRun(void) bufLimitClear(encryptBuffer); TEST_RESULT_UINT( - bufUsed(encryptBuffer), CIPHER_BLOCK_HEADER_SIZE + (size_t)EVP_CIPHER_block_size(blockEncrypt->cipher), + bufUsed(encryptBuffer), TEST_HEADER_SIZE + (size_t)EVP_CIPHER_block_size(blockEncrypt->cipher), "cipher size increases by one block"); TEST_RESULT_BOOL(ioFilterDone(blockEncryptFilter), false, "filter is not done"); ioFilterProcessInOut(blockEncryptFilter, NULL, encryptBuffer); TEST_RESULT_UINT( - bufUsed(encryptBuffer), CIPHER_BLOCK_HEADER_SIZE + (size_t)(EVP_CIPHER_block_size(blockEncrypt->cipher) * 2), + bufUsed(encryptBuffer), TEST_HEADER_SIZE + (size_t)(EVP_CIPHER_block_size(blockEncrypt->cipher) * 2), "cipher size increases by one block on flush"); TEST_RESULT_BOOL(ioFilterDone(blockEncryptFilter), true, "filter is done"); @@ -174,7 +186,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- Buffer *decryptBuffer = bufNew(TEST_BUFFER_SIZE); - IoFilter *blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + IoFilter *blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecryptFilter = cipherBlockNewPack(ioFilterParamList(blockDecryptFilter)); CipherBlock *blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); @@ -194,7 +206,7 @@ testRun(void) // Decrypt in small chunks to test buffering // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -221,7 +233,7 @@ testRun(void) ioFilterProcessInOut( blockDecryptFilter, - bufNewC(bufPtr(encryptBuffer) + CIPHER_BLOCK_HEADER_SIZE, bufUsed(encryptBuffer) - CIPHER_BLOCK_HEADER_SIZE), + bufNewC(bufPtr(encryptBuffer) + TEST_HEADER_SIZE, bufUsed(encryptBuffer) - TEST_HEADER_SIZE), decryptBuffer); TEST_RESULT_UINT_INT(bufUsed(decryptBuffer), EVP_CIPHER_block_size(blockDecrypt->cipher), "decrypt size is one block"); @@ -235,7 +247,8 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("encrypt zero byte file with no magic"); - blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass), .raw = true); + blockEncryptFilter = cipherBlockNewP( + cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = cipherBlockHeaderNone); blockEncrypt = (CipherBlock *)ioFilterDriver(blockEncryptFilter); bufUsedZero(encryptBuffer); @@ -248,13 +261,14 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("error on decrypt expecting magic"); - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); TEST_ERROR(ioFilterProcessInOut(blockDecryptFilter, encryptBuffer, decryptBuffer), CryptoError, "cipher header invalid"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("decrypt zero byte file with no magic"); - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass), .raw = true); + blockDecryptFilter = cipherBlockNewP( + cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = cipherBlockHeaderNone); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -268,7 +282,7 @@ testRun(void) // Invalid cipher header // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); TEST_ERROR( @@ -279,7 +293,7 @@ testRun(void) // Invalid encrypted data cannot be flushed // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -293,7 +307,7 @@ testRun(void) // File with no header should not flush // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -304,7 +318,7 @@ testRun(void) // File with header only should error // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -325,7 +339,7 @@ testRun(void) TEST_RESULT_VOID( cipherBlockFilterGroupAdd( - filterGroup, cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("X"))), " filter add"); + filterGroup, cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("X"))), " filter add"); TEST_RESULT_UINT(ioFilterGroupSize(filterGroup), 1, " check filter add"); } @@ -334,7 +348,7 @@ testRun(void) { IoFilter *hash = NULL; - TEST_ERROR(cryptoHashNew(STRID5("bogus", 0x13a9de20)), AssertError, "unable to load hash 'bogus'"); + TEST_ERROR(cryptoHashNew(STRID5("bogus", 0x13a9de20)), AssertError, "unable to load digest 'bogus'"); // ------------------------------------------------------------------------------------------------------------------------- TEST_ASSIGN(hash, cryptoHashNew(hashTypeSha1), "create sha1 hash"); diff --git a/test/src/module/common/formatTest.c b/test/src/module/common/formatTest.c new file mode 100644 index 000000000..4a603e1d8 --- /dev/null +++ b/test/src/module/common/formatTest.c @@ -0,0 +1,272 @@ +/*********************************************************************************************************************************** +Test Repository Format +***********************************************************************************************************************************/ +#include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" +#include "common/io/io.h" +#include "version.h" + +/*********************************************************************************************************************************** +Data for testing +***********************************************************************************************************************************/ +#define TEST_PASS "areallybadpassphrase" +#define TEST_PLAINTEXT "plaintext" +#define TEST_BUFFER_SIZE 256 + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +static void +testRun(void) +{ + FUNCTION_HARNESS_VOID(); + + // ***************************************************************************************************************************** + if (testBegin("repoFormatValidate()")) + { + TEST_ERROR( + repoFormatValidate(REPOSITORY_FORMAT_MIN - 1), FormatError, + "repository format 4 is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + TEST_ERROR( + repoFormatValidate(REPOSITORY_FORMAT_MAX + 1), FormatError, + "repository format 7 requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + + TEST_RESULT_VOID(repoFormatValidate(REPOSITORY_FORMAT_5), "format 5 is readable"); + TEST_RESULT_VOID(repoFormatValidate(REPOSITORY_FORMAT_6), "format 6 is readable"); + } + + // ***************************************************************************************************************************** + if (testBegin("repoFormatDigest()")) + { + TEST_RESULT_UINT(repoFormatDigest(REPOSITORY_FORMAT_5), hashTypeSha1, "format 5 derives with sha1"); + TEST_RESULT_UINT(repoFormatDigest(REPOSITORY_FORMAT_6), hashTypeSha256, "format 6 derives with sha256"); + } + + // ***************************************************************************************************************************** + if (testBegin("cipherBlockFormatNew()")) + { + const Buffer *const testPlainText = BUFSTRDEF(TEST_PLAINTEXT); + const Buffer *const testPass = BUFSTRDEF(TEST_PASS); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, testPass); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("nothing is added when the repository is not encrypted"); + + Buffer *plain = bufNew(0); + IoWrite *plainWrite = ioBufferWriteNew(plain); + + TEST_RESULT_VOID( + cipherBlockFormatFilterGroupWriteAdd(plain, ioWriteFilterGroup(plainWrite), cipherSpecNewNone(), REPOSITORY_FORMAT_6), + "add write filter for no cipher"); + ioWriteOpen(plainWrite); + ioWrite(plainWrite, testPlainText); + ioWriteClose(plainWrite); + + TEST_RESULT_STR_Z(strNewBuf(plain), TEST_PLAINTEXT, "content is stored as it is"); + + IoWrite *plainRead = ioBufferWriteNew(bufNew(0)); + TEST_RESULT_PTR_NE( + cipherBlockFormatFilterGroupReadAdd(ioWriteFilterGroup(plainRead), cipherSpecNewNone()), NULL, + "add read filter for no cipher"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("write a header and read the format back from it"); + + Buffer *headerBuffer = bufNew(0); + IoWrite *headerWrite = ioBufferWriteNew(headerBuffer); + + cipherBlockFormatFilterGroupWriteAdd(headerBuffer, ioWriteFilterGroup(headerWrite), cipherSpec, REPOSITORY_FORMAT_6); + ioWriteOpen(headerWrite); + ioWrite(headerWrite, testPlainText); + ioWriteClose(headerWrite); + + TEST_RESULT_BOOL( + memcmp(bufPtrConst(headerBuffer), CIPHER_BLOCK_FORMAT_MAGIC "006_", CIPHER_BLOCK_FORMAT_HEADER_SIZE) == 0, true, + "header names the format"); + + // The format is not given on decrypt, so it comes from the header and is what the pass derives with + Buffer *headerResult = bufNew(0); + IoWrite *headerRead = ioBufferWriteNew(headerResult); + IoFilterGroup *headerFilterGroup = ioWriteFilterGroup(headerRead); + + cipherBlockFormatFilterGroupReadAdd(headerFilterGroup, cipherSpec); + ioWriteOpen(headerRead); + ioWrite(headerRead, headerBuffer); + ioWriteClose(headerRead); + + TEST_RESULT_STR_Z(strNewBuf(headerResult), TEST_PLAINTEXT, "content decrypted with the digest the header called for"); + TEST_RESULT_UINT( + cipherBlockFormatResult(ioFilterGroupResultP(headerFilterGroup, CIPHER_BLOCK_FORMAT_FILTER_TYPE)), + REPOSITORY_FORMAT_6, "filter reports the format"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("content is handed on in pieces when the destination cannot take it whole"); + + // Content long enough that decryption has output to hand on while input is still arriving, so the destination fills before + // the source has been consumed + Buffer *const piecesPlainText = bufNew(TEST_BUFFER_SIZE); + memset(bufPtr(piecesPlainText), 'x', bufSize(piecesPlainText)); + bufUsedSet(piecesPlainText, bufSize(piecesPlainText)); + + Buffer *const piecesBuffer = bufNew(0); + IoWrite *const piecesWrite = ioBufferWriteNew(piecesBuffer); + + cipherBlockFormatFilterGroupWriteAdd(piecesBuffer, ioWriteFilterGroup(piecesWrite), cipherSpec, REPOSITORY_FORMAT_6); + ioWriteOpen(piecesWrite); + ioWrite(piecesWrite, piecesPlainText); + ioWriteClose(piecesWrite); + + IoRead *const readPieces = ioBufferReadNew(piecesBuffer); + Buffer *const piecesResult = bufNew(0); + Buffer *const piece = bufNew(4); + + cipherBlockFormatFilterGroupReadAdd(ioReadFilterGroup(readPieces), cipherSpec); + + ioBufferSizeSet(4); + ioReadOpen(readPieces); + + while (!ioReadEof(readPieces)) + { + bufUsedZero(piece); + ioRead(readPieces, piece); + bufCat(piecesResult, piece); + } + + ioReadClose(readPieces); + ioBufferSizeSet(TEST_BUFFER_SIZE); + + TEST_RESULT_BOOL(bufEq(piecesResult, piecesPlainText), true, "content decrypted in pieces"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("the header is read no matter how the input is split"); + + Buffer *const splitResult = bufNew(0); + IoWrite *const splitWrite = ioBufferWriteNew(splitResult); + + cipherBlockFormatFilterGroupReadAdd(ioWriteFilterGroup(splitWrite), cipherSpec); + + // Write in pieces smaller than the header so it arrives in more than one part, then a piece that completes it exactly, so + // that what follows arrives with the header already read + ioBufferSizeSet(4); + ioWriteOpen(splitWrite); + ioWrite(splitWrite, BUF(bufPtrConst(headerBuffer), 4)); + ioWrite(splitWrite, BUF(bufPtrConst(headerBuffer) + 4, 4)); + ioWrite(splitWrite, BUF(bufPtrConst(headerBuffer) + 8, bufUsed(headerBuffer) - 8)); + ioWriteClose(splitWrite); + ioBufferSizeSet(TEST_BUFFER_SIZE); + + TEST_RESULT_STR_Z(strNewBuf(splitResult), TEST_PLAINTEXT, "content decrypted from split input"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a file that begins with the magic was written before there was a header"); + + Buffer *magicBuffer = bufNew(0); + IoWrite *magicWrite = ioBufferWriteNew(magicBuffer); + + cipherBlockFormatFilterGroupWriteAdd(magicBuffer, ioWriteFilterGroup(magicWrite), cipherSpec, REPOSITORY_FORMAT_5); + ioWriteOpen(magicWrite); + ioWrite(magicWrite, testPlainText); + ioWriteClose(magicWrite); + + TEST_RESULT_BOOL( + memcmp(bufPtrConst(magicBuffer), CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) == 0, true, + "a format before the header writes the magic"); + + headerResult = bufNew(0); + headerRead = ioBufferWriteNew(headerResult); + headerFilterGroup = ioWriteFilterGroup(headerRead); + + cipherBlockFormatFilterGroupReadAdd(headerFilterGroup, cipherSpec); + ioWriteOpen(headerRead); + ioWrite(headerRead, magicBuffer); + ioWriteClose(headerRead); + + TEST_RESULT_STR_Z(strNewBuf(headerResult), TEST_PLAINTEXT, "content decrypted"); + TEST_RESULT_UINT( + cipherBlockFormatResult(ioFilterGroupResultP(headerFilterGroup, CIPHER_BLOCK_FORMAT_FILTER_TYPE)), + REPOSITORY_FORMAT_5, "filter reports the format before the header"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a format given on decrypt must be the one the header names"); + + IoWrite *const headerMismatch = ioBufferWriteNew(bufNew(0)); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerMismatch), cipherBlockFormatNewP(cipherSpec, .format = REPOSITORY_FORMAT_5)); + ioWriteOpen(headerMismatch); + + TEST_ERROR(ioWrite(headerMismatch, headerBuffer), FormatError, "expected repository format 5 but found 6"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a format given on decrypt that the header agrees with is accepted, here from a pack"); + + headerResult = bufNew(0); + headerRead = ioBufferWriteNew(headerResult); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerRead), + cipherBlockFormatNewPack( + ioFilterParamList(cipherBlockFormatNewP(cipherSpec, .format = REPOSITORY_FORMAT_6)))); + ioWriteOpen(headerRead); + ioWrite(headerRead, headerBuffer); + ioWriteClose(headerRead); + + TEST_RESULT_STR_Z(strNewBuf(headerResult), TEST_PLAINTEXT, "content decrypted"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a file too short to hold a header cannot have one"); + + IoWrite *const headerShort = ioBufferWriteNew(bufNew(0)); + + cipherBlockFormatFilterGroupReadAdd(ioWriteFilterGroup(headerShort), cipherSpec); + ioWriteOpen(headerShort); + ioWrite(headerShort, BUFSTRDEF("PGBR")); + + TEST_ERROR(ioWriteClose(headerShort), CryptoError, "cipher header missing"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("damaged headers"); + + // Decrypt the buffer above after damaging a byte of the header, which is the same buffer for each case + #define TEST_HEADER_DAMAGE(damageIdx, damageChar, errorType, errorMessage) \ + do \ + { \ + Buffer *const damaged = bufDup(headerBuffer); \ + bufPtr(damaged)[damageIdx] = damageChar; \ + \ + IoWrite *const write = ioBufferWriteNew(bufNew(0)); \ + cipherBlockFormatFilterGroupReadAdd(ioWriteFilterGroup(write), cipherSpec); \ + ioWriteOpen(write); \ + \ + TEST_ERROR(ioWrite(write, damaged), errorType, errorMessage); \ + } \ + while (0) + + // Where the format should be, so this version cannot tell what it is looking at + TEST_HEADER_DAMAGE(CIPHER_BLOCK_FORMAT_MAGIC_SIZE, 'X', FormatError, "invalid cipher header"); + + // The byte held back for later, which must be the one this version writes + TEST_HEADER_DAMAGE(CIPHER_BLOCK_FORMAT_HEADER_SIZE - 1, 'X', FormatError, "invalid cipher header"); + + // A format newer than this version can read, reported before anything is decrypted + TEST_HEADER_DAMAGE( + CIPHER_BLOCK_FORMAT_HEADER_SIZE - 2, '7', FormatError, + "repository format 7 requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + + // A format older than this version can read + TEST_HEADER_DAMAGE( + CIPHER_BLOCK_FORMAT_HEADER_SIZE - 2, '4', FormatError, + "repository format 4 is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + + // Neither a header nor the magic, which is what a file that was never encrypted looks like from here + TEST_HEADER_DAMAGE(0, 'X', CryptoError, "cipher header invalid"); + + #undef TEST_HEADER_DAMAGE + } + + FUNCTION_HARNESS_RETURN_VOID(); +} diff --git a/test/src/module/info/infoArchiveTest.c b/test/src/module/info/infoArchiveTest.c index 5a33e9a51..713ea73be 100644 --- a/test/src/module/info/infoArchiveTest.c +++ b/test/src/module/info/infoArchiveTest.c @@ -87,16 +87,26 @@ testRun(void) info, infoArchiveNew( PG_VERSION_10, 6569239123849665999, REPOSITORY_FORMAT_DEFAULT, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), + cipherSpecNewP( + cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"), + .digest = hashTypeSha1)), "infoArchiveNew() - cipher sub"); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); + contentSave = bufNew(0); - TEST_RESULT_VOID(infoArchiveSave(info, ioBufferWriteNew(contentSave)), "save new with cipher"); + IoWrite *write = ioBufferWriteNew(contentSave); + cipherBlockFormatFilterGroupWriteAdd(contentSave, ioWriteFilterGroup(write), cipherSpec, REPOSITORY_FORMAT_DEFAULT); - TEST_ASSIGN( - info, infoArchiveNewLoad(ioBufferReadNew(contentSave), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x"))), - "load encrypted archive info"); + TEST_RESULT_VOID(infoArchiveSave(info, write), "save new with cipher"); + TEST_RESULT_BOOL( + strBeginsWithZ(strNewBuf(contentSave), "PGBR"), false, "no header before the format that added it"); + + IoRead *const read = ioBufferReadNew(contentSave); + cipherBlockFormatFilterGroupReadAdd(ioReadFilterGroup(read), cipherSpec); + + TEST_ASSIGN(info, infoArchiveNewLoad(read, cipherSpec), "load encrypted archive info"); TEST_RESULT_STR_Z(infoArchiveId(info), "10-1", "archiveId set"); TEST_RESULT_PTR(infoArchivePg(info), infoArchivePg(info), "infoPg set"); TEST_RESULT_STR_Z( diff --git a/test/src/module/info/infoBackupTest.c b/test/src/module/info/infoBackupTest.c index ab1e8a5ce..1719dee70 100644 --- a/test/src/module/info/infoBackupTest.c +++ b/test/src/module/info/infoBackupTest.c @@ -87,18 +87,27 @@ testRun(void) TEST_ASSIGN( infoBackup, infoBackupNew( - PG_VERSION_10, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_10), REPOSITORY_FORMAT_DEFAULT, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), + PG_VERSION_10, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_10), REPOSITORY_FORMAT_6, + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), "infoBackupNew() - cipher sub"); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); + contentSave = bufNew(0); - TEST_RESULT_VOID(infoBackupSave(infoBackup, ioBufferWriteNew(contentSave)), "save new with cipher sub"); + IoWrite *write = ioBufferWriteNew(contentSave); + cipherBlockFormatFilterGroupWriteAdd(contentSave, ioWriteFilterGroup(write), cipherSpec, REPOSITORY_FORMAT_6); + + TEST_RESULT_VOID(infoBackupSave(infoBackup, write), "save new with cipher sub"); + TEST_RESULT_STR_Z( + strNewZN((const char *)bufPtrConst(contentSave), 8), "PGBR006_", "header names the format"); infoBackup = NULL; - TEST_ASSIGN( - infoBackup, infoBackupNewLoad(ioBufferReadNew(contentSave), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x"))), - "load backup info with cipher sub"); + + IoRead *const read = ioBufferReadNew(contentSave); + cipherBlockFormatFilterGroupReadAdd(ioReadFilterGroup(read), cipherSpec); + + TEST_ASSIGN(infoBackup, infoBackupNewLoad(read, cipherSpec), "load backup info with cipher sub"); TEST_RESULT_PTR(infoBackupPg(infoBackup), infoBackupPg(infoBackup), "infoPg set"); TEST_RESULT_STR_Z( strNewBuf(cipherSpecPass(infoBackupCipherSpec(infoBackup))), diff --git a/test/src/module/info/infoPgTest.c b/test/src/module/info/infoPgTest.c index 13a04301b..18b7578ad 100644 --- a/test/src/module/info/infoPgTest.c +++ b/test/src/module/info/infoPgTest.c @@ -43,7 +43,7 @@ testRun(void) TEST_ASSIGN( infoPg, - infoPgNew(infoPgArchive, REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + infoPgNew(infoPgArchive, REPOSITORY_FORMAT_DEFAULT, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), "infoPgNew(cipherTypeAes256Cbc, 123xyz)"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 0, " 0 history"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(infoPgInfo(infoPg)))), "123xyz", " cipherPass set"); @@ -80,7 +80,7 @@ testRun(void) TEST_ASSIGN( infoPg, infoPgSet( - infoPgNew(infoPgBackup, REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + infoPgNew(infoPgBackup, REPOSITORY_FORMAT_DEFAULT, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), PG_VERSION_18, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_18)), "infoPgSet - infoPgBackup"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 1, " 1 history"); diff --git a/test/src/module/info/infoTest.c b/test/src/module/info/infoTest.c index 2bd306e5d..356de56b6 100644 --- a/test/src/module/info/infoTest.c +++ b/test/src/module/info/infoTest.c @@ -2,12 +2,25 @@ Test Info Handler ***********************************************************************************************************************************/ #include "common/crypto/cipherBlock.h" +#include "common/format/cipherBlockFormat.h" #include "common/io/bufferRead.h" #include "common/io/bufferWrite.h" #include "storage/posix/storage.h" #include "harness/info.h" +/*********************************************************************************************************************************** +Read a buffer through the filter that reads the header, as a caller loading a file that has one does +***********************************************************************************************************************************/ +static IoRead * +testInfoReadHeader(const Buffer *const buffer, const CipherSpec *const cipherSpec) +{ + IoRead *const result = ioBufferReadNew(buffer); + cipherBlockFormatFilterGroupReadAdd(ioReadFilterGroup(result), cipherSpec); + + return result; +} + /*********************************************************************************************************************************** Test load callback ***********************************************************************************************************************************/ @@ -79,7 +92,7 @@ testRun(void) Info *info = NULL; TEST_ASSIGN( - info, infoNew(REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + info, infoNew(REPOSITORY_FORMAT_DEFAULT, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), "infoNew(cipher)"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(info))), "123xyz", " cipherPass is set"); @@ -167,7 +180,7 @@ testRun(void) IoRead *read = ioBufferReadNew(contentLoad); ioFilterGroupAdd( ioReadFilterGroup(read), - cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("X")))); + cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("X")))); TEST_ERROR( infoNewLoad(read, cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), CryptoError, @@ -255,20 +268,163 @@ testRun(void) callbackContent = strNew(); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); + + // A file with no header, e.g. a manifest, is decrypted with the spec as it was given since nothing defines the digest + // as anything else + IoRead *const readNoHeader = ioBufferReadNew(harnessInfoEncryptP(contentLoad, cipherSpec)); + cipherBlockFilterGroupAdd(ioReadFilterGroup(readNoHeader), cipherModeDecrypt, cipherSpec); + TEST_ASSIGN( - info, - infoNewLoad( - ioBufferReadNew(contentLoad), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x")), harnessInfoLoadNewCallback, - callbackContent), + info, infoNewLoad(readNoHeader, cipherSpec, harnessInfoLoadNewCallback, callbackContent), "info with content and cipher"); TEST_RESULT_STR_Z(callbackContent, "[c] key=1\n[d] key=1\n", " check callback content"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(info))), "somepass", " check cipher pass set"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest"); TEST_RESULT_STR_Z(infoBackrestVersion(info), PROJECT_VERSION, " check backrest version"); contentSave = bufNew(0); TEST_RESULT_VOID(infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "info save"); TEST_RESULT_STR(strNewBuf(contentSave), strNewBuf(contentLoad), " check save"); + + // Migrating to a format that stores the digest does not change the digest the pass derives with, since the files the pass + // encrypted before the migration are not rewritten + contentSave = bufNew(0); + + TEST_RESULT_VOID(infoFormatSet(info, REPOSITORY_FORMAT_6), "migrate to format 6"); + TEST_RESULT_VOID( + infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "save migrated info"); + TEST_RESULT_BOOL( + strstr(strZ(strNewBuf(contentSave)), "cipher-digest=\"sha1\"") != NULL, true, " check digest stored with pass"); + + TEST_ASSIGN( + info, + infoNewLoad( + testInfoReadHeader(harnessInfoEncryptP(contentSave, cipherSpec, .format = REPOSITORY_FORMAT_6), cipherSpec), + cipherSpec, harnessInfoLoadNewCallback, strNew()), + "load migrated info"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest unchanged"); + + // Header + // ------------------------------------------------------------------------------------------------------------------------- + // An unencrypted file has no header no matter the format, since the format is read from the content + contentSave = bufNew(0); + + IoWrite *const writeNone = ioBufferWriteNew(contentSave); + cipherBlockFormatFilterGroupWriteAdd(contentSave, ioWriteFilterGroup(writeNone), cipherSpecNewNone(), REPOSITORY_FORMAT_6); + + TEST_RESULT_VOID(infoSave(info, writeNone, testInfoSaveCallback, strNewZ("1")), "info save"); + TEST_RESULT_BOOL(strBeginsWithZ(strNewBuf(contentSave), "PGBR"), false, " check no header"); + + contentLoad = harnessInfoChecksumFormat( + REPOSITORY_FORMAT_6, + STRDEF( + "[cipher]\n" + "cipher-digest=\"sha256\"\n" + "cipher-pass=\"somepass\"\n")); + + callbackContent = strNew(); + + TEST_ASSIGN( + info, + infoNewLoad( + testInfoReadHeader(harnessInfoEncryptP(contentLoad, cipherSpec, .format = REPOSITORY_FORMAT_6), cipherSpec), + cipherSpec, harnessInfoLoadNewCallback, callbackContent), + "info with header"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha256, " check cipher sub digest"); + + // A pass migrated from a format that could not store the digest keeps deriving with SHA-1, so the files it encrypted + // before the migration are still readable + const Buffer *const contentMigrated = harnessInfoChecksumFormat( + REPOSITORY_FORMAT_6, + STRDEF( + "[cipher]\n" + "cipher-digest=\"sha1\"\n" + "cipher-pass=\"somepass\"\n")); + + TEST_ASSIGN( + info, + infoNewLoad( + testInfoReadHeader(harnessInfoEncryptP(contentMigrated, cipherSpec, .format = REPOSITORY_FORMAT_6), cipherSpec), + cipherSpec, harnessInfoLoadNewCallback, callbackContent), + "info migrated to the format that stores the digest"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest"); + + // The content on its own, which is how a caller that wants the file rather than the values in it reads an info file + IoRead *const infoRead = ioBufferReadNew(harnessInfoEncryptP(contentLoad, cipherSpec, .format = REPOSITORY_FORMAT_6)); + + ioFilterGroupAdd( + ioReadFilterGroup(infoRead), cipherBlockFormatNewP(cipherSpec)); + ioReadOpen(infoRead); + + TEST_RESULT_STR(strNewBuf(ioReadBuf(infoRead)), strNewBuf(contentLoad), "info content read"); + + // A file written before the header existed is read as the format that had none + contentLoad = harnessInfoChecksumZ("[c]\nkey=1\n"); + + TEST_ASSIGN( + info, + infoNewLoad( + testInfoReadHeader(harnessInfoEncryptP(contentLoad, cipherSpec, .format = REPOSITORY_FORMAT_5), cipherSpec), + cipherSpec, harnessInfoLoadNewCallback, callbackContent), + "info with no header"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_5, " check format"); + + // A file too short to hold a header cannot have one + TEST_ERROR( + infoNewLoad( + testInfoReadHeader(BUFSTRDEF("PGBR"), cipherSpec), cipherSpec, harnessInfoLoadNewCallback, callbackContent), + CryptoError, + "cipher header missing\n" + "HINT: is or was the repo encrypted?"); + + // A file with neither header was never encrypted, which is reported the way the cipher reports it + TEST_ERROR( + infoNewLoad( + testInfoReadHeader(BUFSTRDEF("[backrest]\nbackrest-format=5\n"), cipherSpec), cipherSpec, + harnessInfoLoadNewCallback, callbackContent), + CryptoError, + "cipher header invalid\n" + "HINT: is or was the repo encrypted?"); + + // Header uses the byte held back for later, so this version does not know what it is looking at + Buffer *contentHeader = harnessInfoEncryptP(contentLoad, cipherSpec, .format = REPOSITORY_FORMAT_6); + bufPtr(contentHeader)[7] = 'X'; + + TEST_ERROR( + infoNewLoad( + testInfoReadHeader(contentHeader, cipherSpec), cipherSpec, harnessInfoLoadNewCallback, callbackContent), + FormatError, "invalid cipher header"); + + // Header damaged where the format should be + contentHeader = harnessInfoEncryptP(contentLoad, cipherSpec, .format = REPOSITORY_FORMAT_6); + bufPtr(contentHeader)[5] = 'X'; + + TEST_ERROR( + infoNewLoad( + testInfoReadHeader(contentHeader, cipherSpec), cipherSpec, harnessInfoLoadNewCallback, callbackContent), + FormatError, "invalid cipher header"); + + // Header names a format this version cannot read, which is reported before anything is decrypted + contentHeader = harnessInfoEncryptP(contentLoad, cipherSpec, .format = REPOSITORY_FORMAT_6); + bufPtr(contentHeader)[6] = '7'; + + TEST_ERROR( + infoNewLoad( + testInfoReadHeader(contentHeader, cipherSpec), cipherSpec, harnessInfoLoadNewCallback, callbackContent), + FormatError, + "repository format 7 requires a newer version of pgBackRest\n" + "HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6."); + + // Header and content disagree about the format + TEST_ERROR( + infoNewLoad( + testInfoReadHeader(harnessInfoEncryptP(contentLoad, cipherSpec, .format = REPOSITORY_FORMAT_6), cipherSpec), + cipherSpec, harnessInfoLoadNewCallback, callbackContent), + FormatError, "repository format 5 does not match header format 6"); } // ***************************************************************************************************************************** diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c index 1d97cb238..c1b16766d 100644 --- a/test/src/module/info/manifestTest.c +++ b/test/src/module/info/manifestTest.c @@ -1453,6 +1453,7 @@ testRun(void) "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n" "\n" "[cipher]\n" + "cipher-digest=\"sha256\"\n" "cipher-pass=\"somepass\"\n" "\n" "[target:file]\n" @@ -1477,9 +1478,11 @@ testRun(void) MEM_CONTEXT_TEMP_BEGIN() { - TEST_ASSIGN( - manifest, manifestNewLoad(ioBufferReadNew(contentLoad), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x"))), - "load manifest"); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); + IoRead *const read = ioBufferReadNew(harnessInfoEncryptP(contentLoad, cipherSpec)); + cipherBlockFilterGroupAdd(ioReadFilterGroup(read), cipherModeDecrypt, cipherSpec); + + TEST_ASSIGN(manifest, manifestNewLoad(read, cipherSpec), "load manifest"); TEST_RESULT_VOID(manifestMove(manifest, memContextPrior()), "move manifest"); } MEM_CONTEXT_TEMP_END(); @@ -1781,7 +1784,8 @@ testRun(void) TEST_RESULT_UINT(cipherSpecType(manifestCipherSpec(manifest)), cipherTypeNone, "check cipher subpass"); TEST_RESULT_VOID( - manifestCipherSpecSet(manifest, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("supersecret"))), + manifestCipherSpecSet( + manifest, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("supersecret"), .digest = hashTypeSha1)), "cipher subpass set"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(manifestCipherSpec(manifest))), "supersecret", "check cipher subpass"); diff --git a/test/src/module/storage/remoteTest.c b/test/src/module/storage/remoteTest.c index 960550c29..89fc9c0b1 100644 --- a/test/src/module/storage/remoteTest.c +++ b/test/src/module/storage/remoteTest.c @@ -381,8 +381,8 @@ testRun(void) IoFilterGroup *filterGroup = ioReadFilterGroup(storageReadIo(fileRead)); ioFilterGroupAdd(filterGroup, ioSizeNew()); ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1)); - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x")))); - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x")))); + ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")))); + ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")))); ioFilterGroupAdd(filterGroup, compressFilterP(compressTypeGz, 3)); ioFilterGroupAdd(filterGroup, decompressFilterP(compressTypeGz)); @@ -422,7 +422,7 @@ testRun(void) // Write an encrypted file larger than the buffer size so the error will happen after the first chunk has been read HRN_STORAGE_PUT( storageTest, TEST_PATH "/repo128/test.bin", contentBuf, - .cipherSpec = cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x"))); + .cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x"))); // Enable read retry to show that it is disabled when the filters are passed to the remote uint64_t featureOld = storageRepo->pub.interface.feature; @@ -435,7 +435,7 @@ testRun(void) ioFilterGroupAdd( ioReadFilterGroup(storageReadIo(fileRead)), - cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x")))); + cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")))); TEST_ERROR(storageGetP(fileRead), CryptoError, "raised from remote-0 shim protocol: unable to flush");