diff --git a/build/config.yaml b/build/config.yaml index a253dfa9d..a8db514d9 100644 --- a/build/config.yaml +++ b/build/config.yaml @@ -1743,6 +1743,22 @@ option: deprecate: repo-cipher-pass: {} + repo-cipher-rotate: + section: global + group: repo + type: time + default: 90d + allow-range: [15d, 9999999d] + command: + backup: {} + expire: {} + command-role: + main: {} + depend: + option: repo-cipher-type + list: + - aes-256-cbc + repo-hardlink: section: global group: repo diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index 14ef1516f..b71f956a5 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -1456,6 +1456,18 @@ zWaf6XtpjIVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO + + + Archive key rotation period. + + +

When the key that encrypts archived WAL is older than this period, expire generates a new key and WAL archived after that is encrypted with it. WAL encrypted with an earlier key remains readable. If rotation is not desired then set the option to the max value.

+ +

Archive key rotation requires repo-format 6 or higher.

+ + + 30d + diff --git a/doc/xml/release/2020s/2026/2.60.0.xml b/doc/xml/release/2020s/2026/2.60.0.xml index df5f68464..01ec81012 100644 --- a/doc/xml/release/2020s/2026/2.60.0.xml +++ b/doc/xml/release/2020s/2026/2.60.0.xml @@ -126,13 +126,16 @@ + + + -

Pass a cipher spec rather than a type and passphrase.

+

Add archive key rotation for repository format 6.

diff --git a/src/command/archive/get/file.c b/src/command/archive/get/file.c index 990c56346..723e38982 100644 --- a/src/command/archive/get/file.c +++ b/src/command/archive/get/file.c @@ -7,8 +7,8 @@ Archive Get File #include "command/archive/get/file.h" #include "command/control/common.h" #include "common/compress/helper.h" -#include "common/crypto/cipherBlock.h" #include "common/debug.h" +#include "common/format/cipherBlockFormat.h" #include "common/io/filter/group.h" #include "common/log.h" #include "config/config.h" @@ -53,12 +53,11 @@ archiveGetFile( StorageWrite *const destination = storageNewWriteP( storage, walDestination, .noCreatePath = true, .noSyncFile = true, .noSyncPath = true, .noAtomic = true); - // If there is a cipher then add the decrypt filter - if (cipherSpecType(actual->cipherSpecArchive) != cipherTypeNone) + // Add decrypt filter to read the format header + if (cipherSpecMapSize(actual->cipherSpecArchive) != 0) { - ioFilterGroupAdd( - ioWriteFilterGroup(storageWriteIo(destination)), - cipherBlockNewP(cipherModeDecrypt, actual->cipherSpecArchive)); + cipherBlockFormatFilterGroupReadAddMap( + ioWriteFilterGroup(storageWriteIo(destination)), actual->cipherSpecArchive); compressible = false; } diff --git a/src/command/archive/get/file.h b/src/command/archive/get/file.h index f8045641b..275d21fa7 100644 --- a/src/command/archive/get/file.h +++ b/src/command/archive/get/file.h @@ -4,7 +4,7 @@ Archive Get File #ifndef COMMAND_ARCHIVE_GET_FILE_H #define COMMAND_ARCHIVE_GET_FILE_H -#include "common/crypto/spec.h" +#include "common/format/cipherSpecMap.h" #include "common/type/string.h" #include "storage/storage.h" @@ -17,7 +17,7 @@ typedef struct ArchiveGetFile const String *file; // File in the repo (with path, checksum, ext, etc.) unsigned int repoIdx; // Repo idx const String *archiveId; // Repo archive id - const CipherSpec *cipherSpecArchive; // Repo archive cipher spec + const CipherSpecMap *cipherSpecArchive; // Repo archive cipher keys } ArchiveGetFile; typedef struct ArchiveGetFileResult diff --git a/src/command/archive/get/get.c b/src/command/archive/get/get.c index d94afe587..ee9f00a70 100644 --- a/src/command/archive/get/get.c +++ b/src/command/archive/get/get.c @@ -94,7 +94,7 @@ typedef struct ArchiveGetFindCacheArchive typedef struct ArchiveGetFindCacheRepo { unsigned int repoIdx; - const CipherSpec *cipherSpecArchive; // Repo archive cipher spec + const CipherSpecMap *cipherSpecArchive; // Repo archive cipher keys List *archiveList; // Cached list of archiveIds and associated paths StringList *warnList; // Track repo warnings so each is only reported once } ArchiveGetFindCacheRepo; @@ -416,7 +416,7 @@ archiveGetCheck(const StringList *const archiveRequestList) // Build cipher spec in the result list context once rather than rebuilding it per candidate file later MEM_CONTEXT_BEGIN(lstMemContext(result.archiveFileMapList)) { - cacheRepo.cipherSpecArchive = cipherSpecDupP(infoArchiveCipherSpec(info)); + cacheRepo.cipherSpecArchive = cipherSpecMapDup(infoArchiveCipherSpecMap(info)); } MEM_CONTEXT_END(); @@ -924,7 +924,7 @@ archiveGetAsyncCallback(void *const data, const unsigned int clientIdx) pckWriteStrP(param, actual->file); pckWriteU32P(param, actual->repoIdx); pckWriteStrP(param, actual->archiveId); - cipherSpecPack(param, actual->cipherSpecArchive); + cipherSpecMapPack(param, actual->cipherSpecArchive); } MEM_CONTEXT_PRIOR_BEGIN() diff --git a/src/command/archive/get/protocol.c b/src/command/archive/get/protocol.c index 9def22757..0a06d3e20 100644 --- a/src/command/archive/get/protocol.c +++ b/src/command/archive/get/protocol.c @@ -38,7 +38,7 @@ archiveGetFileProtocol(PackRead *const param) ArchiveGetFile actual = {.file = pckReadStrP(param)}; actual.repoIdx = pckReadU32P(param); actual.archiveId = pckReadStrP(param); - actual.cipherSpecArchive = cipherSpecNewPack(param); + actual.cipherSpecArchive = cipherSpecMapNewPack(param); lstAdd(actualList, &actual); } diff --git a/src/command/archive/push/file.c b/src/command/archive/push/file.c index bf745df6b..104b12e6a 100644 --- a/src/command/archive/push/file.c +++ b/src/command/archive/push/file.c @@ -7,9 +7,9 @@ Archive Push File #include "command/archive/find.h" #include "command/archive/push/file.h" #include "command/control/common.h" -#include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/debug.h" +#include "common/format/cipherBlockFormat.h" #include "common/io/filter/group.h" #include "common/io/io.h" #include "common/log.h" @@ -265,13 +265,10 @@ archivePushFile( strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strZ(repoData->archiveId), strZ(archiveDestination)), .compressible = compressible); - // If there is a cipher then add the encrypt filter - if (cipherSpecType(repoData->cipherSpecArchive) != cipherTypeNone) - { - ioFilterGroupAdd( - ioWriteFilterGroup(storageWriteIo(destination[repoListIdx])), - cipherBlockNewP(cipherModeEncrypt, repoData->cipherSpecArchive)); - } + // Add the encrypt filter to write the format header when required + cipherBlockFormatFilterGroupWriteAddP( + ioWriteFilterGroup(storageWriteIo(destination[repoListIdx])), repoData->cipherSpecArchive, + repoData->format, .keyId = repoData->cipherIdArchive); } } diff --git a/src/command/archive/push/file.h b/src/command/archive/push/file.h index c6635ac06..d1ee1f454 100644 --- a/src/command/archive/push/file.h +++ b/src/command/archive/push/file.h @@ -17,7 +17,9 @@ typedef struct ArchivePushFileRepoData { unsigned int repoIdx; const String *archiveId; + unsigned int format; // Repo format const CipherSpec *cipherSpecArchive; // Repo archive cipher spec + const String *cipherIdArchive; // Archive key id, NULL when no id } ArchivePushFileRepoData; /*********************************************************************************************************************************** diff --git a/src/command/archive/push/protocol.c b/src/command/archive/push/protocol.c index c564d13b4..4a149d9b4 100644 --- a/src/command/archive/push/protocol.c +++ b/src/command/archive/push/protocol.c @@ -48,7 +48,9 @@ archivePushFileProtocol(PackRead *const param) ArchivePushFileRepoData repo = {.repoIdx = pckReadU32P(param)}; repo.archiveId = pckReadStrP(param); + repo.format = pckReadU32P(param); repo.cipherSpecArchive = cipherSpecNewPack(param); + repo.cipherIdArchive = pckReadStrP(param); pckReadObjEndP(param); lstAdd(repoList, &repo); diff --git a/src/command/archive/push/push.c b/src/command/archive/push/push.c index 318588000..488a0abb7 100644 --- a/src/command/archive/push/push.c +++ b/src/command/archive/push/push.c @@ -255,6 +255,13 @@ archivePushCheck(const bool pgPathSet) } } + // Get the current archive key. Format >= 6 also requires the key id. + const CipherSpecMap *const archiveCipherSpecMap = infoArchiveCipherSpecMap(info); + const String *const archiveCipherId = cipherSpecMapIdCurrent(archiveCipherSpecMap); + const CipherSpec *const archiveCipherSpec = + archiveCipherId == NULL ? + infoArchiveCipherSpec(info) : cipherSpecMapGet(archiveCipherSpecMap, archiveCipherId); + MEM_CONTEXT_PRIOR_BEGIN() { result.pgVersion = archiveInfo.version; @@ -264,7 +271,9 @@ archivePushCheck(const bool pgPathSet) { .repoIdx = repoIdx, .archiveId = strDup(archiveId), - .cipherSpecArchive = cipherSpecDupP(infoArchiveCipherSpec(info)), + .format = infoArchiveFormat(info), + .cipherSpecArchive = cipherSpecDupP(archiveCipherSpec), + .cipherIdArchive = strDup(archiveCipherId), }; lstAdd(result.repoList, &archivePushFileRepoData); @@ -488,7 +497,9 @@ archivePushAsyncCallback(void *const data, const unsigned int clientIdx) pckWriteObjBeginP(param); pckWriteU32P(param, data->repoIdx); pckWriteStrP(param, data->archiveId); + pckWriteU32P(param, data->format); cipherSpecPack(param, data->cipherSpecArchive); + pckWriteStrP(param, data->cipherIdArchive); pckWriteObjEndP(param); } diff --git a/src/command/backup/backup.c b/src/command/backup/backup.c index 5e73c84d2..b697af85a 100644 --- a/src/command/backup/backup.c +++ b/src/command/backup/backup.c @@ -19,6 +19,7 @@ Backup Command #include "common/compress/helper.h" #include "common/crypto/cipherBlock.h" #include "common/debug.h" +#include "common/format/cipherBlockFormat.h" #include "common/io/filter/size.h" #include "common/log.h" #include "common/regExp.h" diff --git a/src/command/backup/complete.c.inc b/src/command/backup/complete.c.inc index 777f7e26c..ad9dd0208 100644 --- a/src/command/backup/complete.c.inc +++ b/src/command/backup/complete.c.inc @@ -101,8 +101,8 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif IoFilterGroup *const filterGroup = ioReadFilterGroup(storageReadIo(read)); // Decrypt with archive key if encrypted - cipherBlockFilterGroupAdd( - filterGroup, cipherModeDecrypt, infoArchiveCipherSpec(backupData->archiveInfo)); + cipherBlockFormatFilterGroupReadAddMap( + filterGroup, infoArchiveCipherSpecMap(backupData->archiveInfo)); // Compress/decompress if archive and backup do not have the same compression settings if (archiveCompressType != backupCompressType) @@ -230,12 +230,6 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest) // ------------------------------------------------------------------------------------------------------------------------- infoBackupDataAdd(infoBackup, manifest); infoBackupSaveFile(infoBackup, storageRepoWrite(), INFO_BACKUP_PATH_FILE_STR, cfgCipherSpecMain()); - - // Save archive.info/copy so the timestamps will be updated to prevent lifecycle settings from removing the files early - // ------------------------------------------------------------------------------------------------------------------------- - infoArchiveSaveFile( - infoArchiveLoadFile(storageRepo(), INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMain()), storageRepoWrite(), - INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMain()); } MEM_CONTEXT_TEMP_END(); diff --git a/src/command/expire/expire.c b/src/command/expire/expire.c index 55b189c12..7f4dfe93b 100644 --- a/src/command/expire/expire.c +++ b/src/command/expire/expire.c @@ -7,7 +7,9 @@ Expire Command #include "command/backup/common.h" #include "command/control/common.h" #include "command/expire/expire.h" +#include "command/stanza/common.h" #include "common/debug.h" +#include "common/format/format.h" #include "common/regExp.h" #include "common/time.h" #include "common/type/list.h" @@ -1081,6 +1083,56 @@ removeExpiredHistory(const InfoBackup *const infoBackup, const unsigned int repo FUNCTION_LOG_RETURN_VOID(); } +/*********************************************************************************************************************************** +Rotate the archive key when it is older than the rotation period +***********************************************************************************************************************************/ +static void +expireArchiveCipherRotate(InfoArchive *const infoArchive, const unsigned int repoIdx) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(INFO_ARCHIVE, infoArchive); + FUNCTION_LOG_PARAM(UINT, repoIdx); + FUNCTION_LOG_END(); + + ASSERT(infoArchive != NULL); + + if (cfgOptionIdxTest(cfgOptRepoCipherRotate, repoIdx)) + { + // Format < 6 has a single archive key that cannot be rotated. Warn only when the option is not the default. + if (infoArchiveFormat(infoArchive) < REPOSITORY_FORMAT_6) + { + if (cfgOptionIdxSource(cfgOptRepoCipherRotate, repoIdx) != cfgSourceDefault) + { + LOG_WARN_FMT( + "option '%s' requires repository format %d or higher", cfgOptionIdxName(cfgOptRepoCipherRotate, repoIdx), + REPOSITORY_FORMAT_6); + } + } + else + { + const time_t timeNow = time(NULL); + + if (infoArchiveCipherRotateTime(infoArchive) < + timeNow - (time_t)(cfgOptionIdxUInt64(cfgOptRepoCipherRotate, repoIdx) / MSEC_PER_SEC)) + { + MEM_CONTEXT_TEMP_BEGIN() + { + infoArchiveCipherRotate( + infoArchive, + cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), infoArchiveFormat(infoArchive)), timeNow); + } + MEM_CONTEXT_TEMP_END(); + + LOG_INFO_FMT( + "%s: rotate archive key to id %s", cfgOptionGroupName(cfgOptGrpRepo, repoIdx), + strZ(cipherSpecMapIdCurrent(infoArchiveCipherSpecMap(infoArchive)))); + } + } + } + + FUNCTION_LOG_RETURN_VOID(); +} + /**********************************************************************************************************************************/ FN_EXTERN void cmdExpire(void) @@ -1240,6 +1292,19 @@ cmdExpire(void) removeExpiredArchive(infoBackup, timeBasedFullRetention, repoIdx); removeExpiredHistory(infoBackup, repoIdx); + // Load archive info and rotate the archive key when it is due + InfoArchive *const infoArchive = infoArchiveLoadFile( + storageRepo, INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMainIdx(repoIdx)); + + expireArchiveCipherRotate(infoArchive, repoIdx); + + // Save archive.info/copy to update the timestamps and prevent lifecycle settings from removing the files early + if (!cfgOptionValid(cfgOptDryRun) || !cfgOptionBool(cfgOptDryRun)) + { + infoArchiveSaveFile( + infoArchive, storageRepoIdxWrite(repoIdx), INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMainIdx(repoIdx)); + } + // Check the oldest retained backup for page checksum errors and issue a warning if any if (infoBackupDataTotal(infoBackup) > 0) expireChecksumErrorCheck(infoBackup, infoBackupData(infoBackup, 0).backupLabel, repoIdx, false); diff --git a/src/command/remote/remote.c b/src/command/remote/remote.c index 6c0d0b8bc..fb933c632 100644 --- a/src/command/remote/remote.c +++ b/src/command/remote/remote.c @@ -46,6 +46,7 @@ static const StorageRemoteFilterHandler storageRemoteFilterHandler[] = {.type = BLOCK_INCR_FILTER_TYPE, .handlerParam = blockIncrNewPack}, {.type = CIPHER_BLOCK_FILTER_TYPE, .handlerParam = cipherBlockNewPack}, {.type = CIPHER_BLOCK_FORMAT_FILTER_TYPE, .handlerParam = cipherBlockFormatNewPack}, + {.type = CIPHER_BLOCK_FORMAT_HEADER_FILTER_TYPE, .handlerParam = cipherBlockFormatHeaderNewPack}, {.type = CRYPTO_HASH_FILTER_TYPE, .handlerParam = cryptoHashNewPack}, {.type = PAGE_CHECKSUM_FILTER_TYPE, .handlerParam = pageChecksumNewPack}, {.type = SINK_FILTER_TYPE, .handlerNoParam = ioSinkNew}, diff --git a/src/command/repo/get.c b/src/command/repo/get.c index 4d4b68677..ec20a79af 100644 --- a/src/command/repo/get.c +++ b/src/command/repo/get.c @@ -73,6 +73,7 @@ storageGetProcess(IoWrite *const destination) // should contain only stanza paths. // ----------------------------------------------------------------------------------------------------------------- const CipherSpec *cipherSpec = NULL; + const CipherSpecMap *cipherSpecMap = NULL; const StringList *const filePathSplitLst = strLstNewSplit(file, FSLASH_STR); // At a minimum the path must contain archive/backup, a stanza, and a file @@ -91,19 +92,20 @@ storageGetProcess(IoWrite *const destination) // Archive path if (strEq(strLstGet(filePathSplitLst, 0), STORAGE_PATH_ARCHIVE_STR)) { - cipherSpec = cfgCipherSpecMain(); - - // Find the archive passphrase + // Find the archive keys. WAL stores the id of the key it is encrypted with in the header. if (!strEndsWithZ(file, INFO_ARCHIVE_FILE) && !strEndsWithZ(file, INFO_ARCHIVE_FILE INFO_COPY_EXT)) { const InfoArchive *const info = infoArchiveLoadFile( storageRepo(), strNewFmt(STORAGE_PATH_ARCHIVE "/%s/%s", strZ(stanza), INFO_ARCHIVE_FILE), cfgCipherSpecMain()); - cipherSpec = infoArchiveCipherSpec(info); + cipherSpecMap = infoArchiveCipherSpecMap(info); } // Else the file is the archive info, which the repo passphrase opens else + { + cipherSpec = cfgCipherSpecMain(); fileIsInfo = true; + } } // Backup path @@ -143,16 +145,20 @@ storageGetProcess(IoWrite *const destination) } // Error when unable to determine cipher passphrase - if (cipherSpec == NULL) + if (cipherSpec == NULL && cipherSpecMap == NULL) THROW_FMT(OptionInvalidValueError, "unable to determine cipher passphrase for '%s'", strZ(file)); - ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone); - - // Add the decryption filter. An info file is read through the filter that reads its header. + // Add the decryption filter. An info file is read through the filter that reads its header and WAL through the + // filter that selects the key by the id in the header. if (fileIsInfo) cipherBlockFormatFilterGroupReadAdd(ioReadFilterGroup(source), cipherSpec); + else if (cipherSpecMap != NULL) + cipherBlockFormatFilterGroupReadAddMap(ioReadFilterGroup(source), cipherSpecMap); else + { + ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone); cipherBlockFilterGroupAdd(ioReadFilterGroup(source), cipherModeDecrypt, cipherSpec); + } } } diff --git a/src/command/restore/restore.c b/src/command/restore/restore.c index 99d2ad6d5..9c7982c72 100644 --- a/src/command/restore/restore.c +++ b/src/command/restore/restore.c @@ -73,7 +73,7 @@ cmdRestore(void) strNewFmt("%s-%u", strZ(pgVersionToStr(data->pgVersion)), data->pgId), data->pgVersion, cvtZToUIntBase(strZ(strSubN(data->archiveStart, 0, 8)), 16), pgLsnFromStr(data->lsnStart), cfgOptionStrNull(cfgOptTargetTimeline), cfgOptionSeq(cfgOptType), - infoArchiveCipherSpec(archiveInfo)); + infoArchiveCipherSpecMap(archiveInfo)); } // Remotes (if any) are no longer needed since the rest of the repository reads will be done by the local processes diff --git a/src/command/restore/timeline.c b/src/command/restore/timeline.c index dd7f71289..b4e10a772 100644 --- a/src/command/restore/timeline.c +++ b/src/command/restore/timeline.c @@ -9,7 +9,7 @@ Timeline Management #include #include "command/restore/timeline.h" -#include "common/crypto/cipherBlock.h" +#include "common/format/cipherBlockFormat.h" #include "common/log.h" #include "config/config.h" #include "postgres/interface.h" @@ -79,13 +79,13 @@ Load history file static List * historyLoad( const Storage *const storageRepo, const String *const archiveId, const unsigned int timeline, - const CipherSpec *const cipherSpecArchive) + const CipherSpecMap *const cipherSpecArchive) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STORAGE, storageRepo); FUNCTION_LOG_PARAM(STRING, archiveId); FUNCTION_LOG_PARAM(UINT, timeline); - FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpecArchive); + FUNCTION_LOG_PARAM(CIPHER_SPEC_MAP, cipherSpecArchive); FUNCTION_LOG_END(); List *result; @@ -94,7 +94,7 @@ historyLoad( { const String *const historyFile = strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%08X.history", strZ(archiveId), timeline); StorageRead *const storageRead = storageNewReadP(storageRepo, historyFile); - cipherBlockFilterGroupAdd(ioReadFilterGroup(storageReadIo(storageRead)), cipherModeDecrypt, cipherSpecArchive); + cipherBlockFormatFilterGroupReadAddMap(ioReadFilterGroup(storageReadIo(storageRead)), cipherSpecArchive); const Buffer *const history = storageGetP(storageRead); TRY_BEGIN() @@ -158,7 +158,7 @@ FN_EXTERN void timelineVerify( const Storage *const storageRepo, const String *const archiveId, const unsigned int pgVersion, const unsigned int timelineBackup, const uint64_t lsnBackup, const String *timelineTargetStr, const unsigned int recoveryType, - const CipherSpec *const cipherSpecArchive) + const CipherSpecMap *const cipherSpecArchive) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STORAGE, storageRepo); @@ -168,7 +168,7 @@ timelineVerify( FUNCTION_LOG_PARAM(UINT64, lsnBackup); FUNCTION_LOG_PARAM(STRING, timelineTargetStr); FUNCTION_LOG_PARAM(UINT, recoveryType); - FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpecArchive); + FUNCTION_LOG_PARAM(CIPHER_SPEC_MAP, cipherSpecArchive); FUNCTION_LOG_END(); ASSERT(storageRepo != NULL); diff --git a/src/command/restore/timeline.h b/src/command/restore/timeline.h index 8b8fc4528..992655b35 100644 --- a/src/command/restore/timeline.h +++ b/src/command/restore/timeline.h @@ -4,7 +4,7 @@ Timeline Management #ifndef COMMAND_RESTORE_TIMELINE_H #define COMMAND_RESTORE_TIMELINE_H -#include "common/crypto/spec.h" +#include "common/format/cipherSpecMap.h" #include "storage/storage.h" /*********************************************************************************************************************************** @@ -13,6 +13,6 @@ Functions // Verify that target timeline is valid for a backup FN_EXTERN void timelineVerify( const Storage *storageRepo, const String *archiveId, unsigned int pgVersion, unsigned int timelineBackup, uint64_t lsnBackup, - const String *timelineTargetStr, unsigned int recoveryType, const CipherSpec *cipherSpecArchive); + const String *timelineTargetStr, unsigned int recoveryType, const CipherSpecMap *cipherSpecArchive); #endif diff --git a/src/command/stanza/create.c b/src/command/stanza/create.c index 61d158571..2883a7cdb 100644 --- a/src/command/stanza/create.c +++ b/src/command/stanza/create.c @@ -6,12 +6,14 @@ Stanza Create Command #include #include #include +#include #include "command/check/common.h" #include "command/control/common.h" #include "command/stanza/common.h" #include "command/stanza/create.h" #include "common/debug.h" +#include "common/format/format.h" #include "common/log.h" #include "common/memContext.h" #include "config/config.h" @@ -77,11 +79,19 @@ cmdStanzaCreate(void) // Format for the new stanza const unsigned int format = cfgOptionIdxUInt(cfgOptRepoFormat, repoIdx); - // If the repo is encrypted, generate a cipher passphrase for encrypting subsequent archive files + // If the repo is encrypted, generate a cipher passphrase for encrypting archive files. Format >= 6 adds it by + // rotating to the first key id and earlier formats store it as the default key with no id. const CipherSpec *const cipherSpecArchive = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), format); + CipherSpecMap *const cipherSpecMapArchive = cipherSpecMapNew(); + + if (cipherSpecType(cipherSpecArchive) != cipherTypeNone && format < REPOSITORY_FORMAT_6) + cipherSpecMapAdd(cipherSpecMapArchive, CIPHER_SPEC_MAP_ID_DEFAULT_STR, cipherSpecArchive); // Create and save archive info - infoArchive = infoArchiveNew(pgControl.version, pgControl.systemId, format, cipherSpecArchive); + infoArchive = infoArchiveNew(pgControl.version, pgControl.systemId, format, cipherSpecMapArchive); + + if (cipherSpecType(cipherSpecArchive) != cipherTypeNone && format >= REPOSITORY_FORMAT_6) + infoArchiveCipherRotate(infoArchive, cipherSpecArchive, time(NULL)); infoArchiveSaveFile(infoArchive, storageRepoWriteStanza, INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMainIdx(repoIdx)); diff --git a/src/command/stanza/upgrade.c b/src/command/stanza/upgrade.c index 21d7a81d3..90631bf87 100644 --- a/src/command/stanza/upgrade.c +++ b/src/command/stanza/upgrade.c @@ -6,12 +6,14 @@ Stanza Upgrade Command #include #include #include +#include #include "command/check/common.h" #include "command/control/common.h" #include "command/stanza/common.h" #include "command/stanza/upgrade.h" #include "common/debug.h" +#include "common/format/format.h" #include "common/log.h" #include "common/memContext.h" #include "config/config.h" @@ -117,6 +119,15 @@ cmdStanzaUpgrade(void) infoArchiveFormatSet(infoArchive, format); infoBackupFormatSet(infoBackup, format); + // A stanza migrated to format 6 keeps the key it already has as id 0 and adds a key for new WAL using rotation + if (formatArchive < REPOSITORY_FORMAT_6) + { + const CipherType cipherType = cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx); + + if (cipherType != cipherTypeNone) + infoArchiveCipherRotate(infoArchive, cipherSpecGen(cipherType, format), time(NULL)); + } + infoArchiveUpgrade = true; infoBackupUpgrade = true; } diff --git a/src/command/verify/file.c b/src/command/verify/file.c index 3566a7e4c..4ca4b3f45 100644 --- a/src/command/verify/file.c +++ b/src/command/verify/file.c @@ -7,6 +7,7 @@ Verify File #include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/debug.h" +#include "common/format/cipherBlockFormat.h" #include "common/io/filter/group.h" #include "common/io/filter/sink.h" #include "common/io/filter/size.h" @@ -18,7 +19,7 @@ Verify File FN_EXTERN VerifyResult verifyFile( const String *const filePathName, const uint64_t offset, const Variant *const limit, const CompressType compressType, - const Buffer *const fileChecksum, const uint64_t fileSize, const CipherSpec *const cipherSpec) + const Buffer *const fileChecksum, const uint64_t fileSize, const CipherSpecMap *const cipherSpecMap, const bool formatHeader) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, filePathName); // Fully qualified file name @@ -27,7 +28,8 @@ verifyFile( FUNCTION_LOG_PARAM(ENUM, compressType); // Compression type FUNCTION_LOG_PARAM(BUFFER, fileChecksum); // Checksum for the file FUNCTION_LOG_PARAM(UINT64, fileSize); // Size of file - FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); // Cipher spec to access the repo file if encrypted + FUNCTION_LOG_PARAM(CIPHER_SPEC_MAP, cipherSpecMap); // Cipher keys to access the repo file if encrypted + FUNCTION_LOG_PARAM(BOOL, formatHeader); // May the file begin with a format header? FUNCTION_LOG_END(); ASSERT(filePathName != NULL); @@ -45,8 +47,17 @@ verifyFile( IoFilterGroup *const filterGroup = ioReadFilterGroup(read); // Add decryption filter - if (cipherSpecType(cipherSpec) != cipherTypeNone) - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpec)); + if (cipherSpecMapSize(cipherSpecMap) != 0) + { + // WAL contains a format header with the key id. Backup files contain no header so their key is under the default id. + if (formatHeader) + cipherBlockFormatFilterGroupReadAddMap(filterGroup, cipherSpecMap); + else + { + cipherBlockFilterGroupAdd( + filterGroup, cipherModeDecrypt, cipherSpecMapGet(cipherSpecMap, CIPHER_SPEC_MAP_ID_DEFAULT_STR)); + } + } // Add decompression filter if (compressType != compressTypeNone) diff --git a/src/command/verify/file.h b/src/command/verify/file.h index 0d5def6d3..ea82ea3c6 100644 --- a/src/command/verify/file.h +++ b/src/command/verify/file.h @@ -5,7 +5,7 @@ Verify File #define COMMAND_VERIFY_FILE_H #include "common/compress/helper.h" -#include "common/crypto/spec.h" +#include "common/format/cipherSpecMap.h" /*********************************************************************************************************************************** File result @@ -25,6 +25,6 @@ Functions // Verify a file in the pgBackRest repository FN_EXTERN VerifyResult verifyFile( const String *filePathName, uint64_t offset, const Variant *limit, CompressType compressType, const Buffer *fileChecksum, - uint64_t fileSize, const CipherSpec *cipherSpec); + uint64_t fileSize, const CipherSpecMap *cipherSpecMap, bool formatHeader); #endif diff --git a/src/command/verify/protocol.c b/src/command/verify/protocol.c index e63149111..c9578a94d 100644 --- a/src/command/verify/protocol.c +++ b/src/command/verify/protocol.c @@ -41,12 +41,13 @@ verifyFileProtocol(PackRead *const param) const CompressType compressType = (CompressType)pckReadU32P(param); const Buffer *const fileChecksum = pckReadBinP(param); const uint64_t fileSize = pckReadU64P(param); - const CipherSpec *const cipherSpec = cipherSpecNewPack(param); + const CipherSpecMap *const cipherSpecMap = cipherSpecMapNewPack(param); + const bool formatHeader = pckReadBoolP(param); // Return result pckWriteU32P( protocolServerResultData(result), - verifyFile(filePathName, offset, limit, compressType, fileChecksum, fileSize, cipherSpec)); + verifyFile(filePathName, offset, limit, compressType, fileChecksum, fileSize, cipherSpecMap, formatHeader)); } MEM_CONTEXT_TEMP_END(); diff --git a/src/command/verify/verify.c b/src/command/verify/verify.c index 0bbb186dc..5e38e6114 100644 --- a/src/command/verify/verify.c +++ b/src/command/verify/verify.c @@ -125,7 +125,7 @@ typedef struct VerifyJobData const InfoPg *pgHistory; // Database history list bool backupProcessing; // Are we processing WAL or are we processing backups const CipherSpec *cipherSpecManifest; // Cipher spec for reading backup manifests - const CipherSpec *cipherSpecArchive; // Cipher spec for reading WAL files + const CipherSpecMap *cipherSpecArchive; // Cipher keys for reading WAL files unsigned int jobErrorTotal; // Total errors that occurred during the job execution List *archiveIdResultList; // Archive results List *backupResultList; // Backup results @@ -816,7 +816,7 @@ verifyArchive(VerifyJobData *const jobData) IoFilterGroup *const walFilterGroup = ioReadFilterGroup(storageReadIo(walRead)); // Add decryption filter when required - cipherBlockFilterGroupAdd(walFilterGroup, cipherModeDecrypt, jobData->cipherSpecArchive); + cipherBlockFormatFilterGroupReadAddMap(walFilterGroup, jobData->cipherSpecArchive); // If the file is compressed, add a decompression filter if (compressTypeFromName(walFile) != compressTypeNone) @@ -856,7 +856,8 @@ verifyArchive(VerifyJobData *const jobData) pckWriteU32P(param, compressTypeFromName(filePathName)); pckWriteBinP(param, checksum); pckWriteU64P(param, archiveResult->pgWalInfo.size); - cipherSpecPack(param, jobData->cipherSpecArchive); + cipherSpecMapPack(param, jobData->cipherSpecArchive); + pckWriteBoolP(param, true); // Assign job to result, prepending the archiveId to the key for consistency with backup processing const String *const jobKey = strNewFmt("%s/%s", strZ(archiveResult->archiveId), strZ(filePathName)); @@ -1090,7 +1091,8 @@ verifyBackup(VerifyJobData *const jobData) pckWriteU32P(param, compressTypeNone); pckWriteBinP(param, BUF(fileData.checksumRepoSha1, HASH_TYPE_SHA1_SIZE)); pckWriteU64P(param, fileData.sizeRepo); - cipherSpecPack(param, cipherSpecNewNone()); + cipherSpecMapPack(param, cipherSpecMapNew()); + pckWriteBoolP(param, false); } // Else use the file checksum, which may require additional filters, e.g. decompression else @@ -1098,7 +1100,13 @@ verifyBackup(VerifyJobData *const jobData) pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType); pckWriteBinP(param, BUF(fileData.checksumSha1, HASH_TYPE_SHA1_SIZE)); pckWriteU64P(param, fileData.size); - cipherSpecPack(param, manifestCipherSpec(jobData->manifest)); + // Backup files contain no key id so their key goes under the default id + CipherSpecMap *const cipherSpecMap = cipherSpecMapNew(); + cipherSpecMapAdd( + cipherSpecMap, CIPHER_SPEC_MAP_ID_DEFAULT_STR, manifestCipherSpec(jobData->manifest)); + + cipherSpecMapPack(param, cipherSpecMap); + pckWriteBoolP(param, false); } // Assign job to result (prepend backup label being processed to the key since some files are in a prior @@ -1616,7 +1624,7 @@ verifyProcess(const bool verboseText) .walFileList = strLstNew(), .pgHistory = infoArchivePg(archiveInfo), .cipherSpecManifest = infoBackupCipherSpec(backupInfo), - .cipherSpecArchive = infoArchiveCipherSpec(archiveInfo), + .cipherSpecArchive = infoArchiveCipherSpecMap(archiveInfo), .archiveIdResultList = lstNewP(sizeof(VerifyArchiveResult), .comparator = archiveIdComparator), .backupResultList = lstNewP(sizeof(VerifyBackupResult), .comparator = lstComparatorStr), }; diff --git a/src/common/format/cipherBlockFormat.c b/src/common/format/cipherBlockFormat.c index a3c89becf..00ffbfb61 100644 --- a/src/common/format/cipherBlockFormat.c +++ b/src/common/format/cipherBlockFormat.c @@ -31,19 +31,23 @@ that tool expects. A file that begins with the magic rather than the header was written at format 5, the only format there was before the header, so that is not an error when a header was expected. -For the four bytes after the header magic, the first three are the format and the last is reserved for future use, e.g. naming -which key the file was encrypted with once a repository can hold more than one. The format comes first so that it is always at the -same place, which is what lets a version work out whether it can read the file at all. Once the format is identified as compatible -with this version, the spare byte is examined, and it must only be the underscore this version writes. +The first three bytes of the header magic are the format and the last is a marker indicating whether a key id follows. The format +comes first so it is always in the same place, which is how a version decides whether it can read the file. The marker is only +checked once the format turns out to be readable, and it must be valid for this version. + +If a key id is indicated by the marker, there is first a byte to indicate the length of the key id and then the key id as a string. ***********************************************************************************************************************************/ #define CIPHER_BLOCK_FORMAT_MAGIC "PGBR" #define CIPHER_BLOCK_FORMAT_MAGIC_SIZE (sizeof(CIPHER_BLOCK_FORMAT_MAGIC) - 1) -#define CIPHER_BLOCK_FORMAT_RESERVED '_' +#define CIPHER_BLOCK_FORMAT_MARKER_NONE '_' +#define CIPHER_BLOCK_FORMAT_MARKER_KEY 'K' #define CIPHER_BLOCK_FORMAT_SIZE 3 -// Total length of the header, which is the magic, the format, and the reserved byte -#define CIPHER_BLOCK_FORMAT_HEADER_SIZE \ - (CIPHER_BLOCK_FORMAT_MAGIC_SIZE + CIPHER_BLOCK_FORMAT_SIZE + 1) +// The key id length is encoded as a single byte with a max of 255 characters +#define CIPHER_BLOCK_FORMAT_KEY_SIZE_MAX 255 + +// Total header length: magic, format, and marker +#define CIPHER_BLOCK_FORMAT_HEADER_SIZE (CIPHER_BLOCK_FORMAT_MAGIC_SIZE + CIPHER_BLOCK_FORMAT_SIZE + 1) // The header is written in place of the magic, so the two must add up to exactly the same size or the salt would no longer begin at // the same place @@ -54,17 +58,22 @@ static_assert( static_assert(REPOSITORY_FORMAT_MAX < 1000, "repository format must fit in the header digits"); static_assert(CIPHER_BLOCK_FORMAT_SIZE == 3, "format digits must match the digits the header is written with"); +// Max possible size of the header, the fixed part plus a length byte and the longest key id it can contain +#define CIPHER_BLOCK_FORMAT_HEADER_SIZE_MAX \ + (CIPHER_BLOCK_FORMAT_HEADER_SIZE + 1 + CIPHER_BLOCK_FORMAT_KEY_SIZE_MAX) + /*********************************************************************************************************************************** Object type ***********************************************************************************************************************************/ typedef struct CipherBlockFormat { - const CipherSpec *cipherSpec; // Cipher spec the content is decrypted with + const CipherSpecMap *cipherSpecMap; // Keys to choose from unsigned int formatExpected; // Format the caller expects, zero when any will do unsigned int format; // Format the header gave - uint8_t header[CIPHER_BLOCK_FORMAT_HEADER_SIZE]; // Header bytes held until there are enough to read + uint8_t header[CIPHER_BLOCK_FORMAT_HEADER_SIZE_MAX]; // Header bytes held until there are enough to read size_t headerSize; // Header bytes held so far + size_t headerSizeExpected; // Total header size, which grows as the header is read IoFilter *cipherBlock; // Block cipher the content behind the header is decrypted by size_t sourceOffset; // Bytes of the current source taken for the header } CipherBlockFormat; @@ -112,9 +121,12 @@ cipherBlockFormatHeaderRead(const uint8_t *const header) // Error on a format this version cannot read before anything is decrypted repoFormatValidate(result); - // The format is one this version can read, so the reserved byte must be the value this version writes - if (headerZ[CIPHER_BLOCK_FORMAT_HEADER_SIZE - 1] != CIPHER_BLOCK_FORMAT_RESERVED) + // Check that the marker is valid for this version + if (headerZ[CIPHER_BLOCK_FORMAT_HEADER_SIZE - 1] != CIPHER_BLOCK_FORMAT_MARKER_NONE && + headerZ[CIPHER_BLOCK_FORMAT_HEADER_SIZE - 1] != CIPHER_BLOCK_FORMAT_MARKER_KEY) + { THROW(FormatError, "invalid cipher header"); + } } // Else the bytes must be the magic, since that is all a file with no header can begin with else if (memcmp(headerZ, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) != 0) @@ -124,8 +136,8 @@ cipherBlockFormatHeaderRead(const uint8_t *const header) } /*********************************************************************************************************************************** -Build the block cipher the content behind the header is decrypted by, which is only possible once the header has been read since the -header is what gives the format and the format is what gives the digest +Build the block cipher that decrypts the content, which is only possible once the header has been read since the header defines the +format and the key id ***********************************************************************************************************************************/ static void cipherBlockFormatCipherNew(CipherBlockFormat *const this) @@ -137,18 +149,33 @@ cipherBlockFormatCipherNew(CipherBlockFormat *const this) ASSERT(this != NULL); ASSERT(this->cipherBlock == NULL); - this->format = cipherBlockFormatHeaderRead(this->header); - // Error when the format the caller expected is not the one the file was written with if (this->formatExpected != 0 && this->formatExpected != this->format) THROW_FMT(FormatError, "expected repository format %u but found %u", this->formatExpected, this->format); - // The eight bytes at the front have been read and are not passed on, whichever of the two they were, so what reaches the block - // cipher begins with the salt either way and it is told to expect no header of its own + // Get the key specified in the header or the default key when none is specified + const CipherSpec *cipherSpec; + + MEM_CONTEXT_TEMP_BEGIN() + { + const String *keyId = CIPHER_SPEC_MAP_ID_DEFAULT_STR; + + if (this->headerSize > CIPHER_BLOCK_FORMAT_HEADER_SIZE) + { + keyId = strNewZN( + (const char *)this->header + CIPHER_BLOCK_FORMAT_HEADER_SIZE + 1, + this->headerSize - CIPHER_BLOCK_FORMAT_HEADER_SIZE - 1); + } + + cipherSpec = cipherSpecMapGet(this->cipherSpecMap, keyId); + } + MEM_CONTEXT_TEMP_END(); + + // Decrypt raw since the format header has already been parsed MEM_CONTEXT_OBJ_BEGIN(this) { this->cipherBlock = cipherBlockNewP( - cipherModeDecrypt, cipherSpecDupP(this->cipherSpec, .digestDefault = repoFormatDigest(this->format)), + cipherModeDecrypt, cipherSpecDupP(cipherSpec, .digestDefault = repoFormatDigest(this->format)), .header = cipherBlockHeaderNone); } MEM_CONTEXT_OBJ_END(); @@ -175,20 +202,47 @@ cipherBlockFormatProcess(THIS_VOID, const Buffer *const source, Buffer *const de if (source != NULL) { - // Hold back the header until there is enough of it to read + // Read until the header is complete. The total size is not known until the marker and key id length have been read. if (this->cipherBlock == NULL) { - this->sourceOffset = CIPHER_BLOCK_FORMAT_HEADER_SIZE - this->headerSize; + this->sourceOffset = 0; - if (this->sourceOffset > bufUsed(source)) - this->sourceOffset = bufUsed(source); + do + { + size_t copySize = this->headerSizeExpected - this->headerSize; - memcpy(this->header + this->headerSize, bufPtrConst(source), this->sourceOffset); - this->headerSize += this->sourceOffset; + if (copySize > bufUsed(source) - this->sourceOffset) + copySize = bufUsed(source) - this->sourceOffset; - // Nothing can be decrypted until the header is complete - if (this->headerSize < CIPHER_BLOCK_FORMAT_HEADER_SIZE) - FUNCTION_LOG_RETURN_VOID(); + memcpy(this->header + this->headerSize, bufPtrConst(source) + this->sourceOffset, copySize); + this->headerSize += copySize; + this->sourceOffset += copySize; + + // Nothing can be decrypted until the header is complete + if (this->headerSize < this->headerSizeExpected) + FUNCTION_LOG_RETURN_VOID(); + + // Parse the fixed part first to reject an invalid header before using the marker + if (this->headerSizeExpected == CIPHER_BLOCK_FORMAT_HEADER_SIZE) + { + this->format = cipherBlockFormatHeaderRead(this->header); + + if (this->header[CIPHER_BLOCK_FORMAT_HEADER_SIZE - 1] == CIPHER_BLOCK_FORMAT_MARKER_KEY) + this->headerSizeExpected++; + } + // Else the length is complete so wait for the key id + else if (this->headerSizeExpected == CIPHER_BLOCK_FORMAT_HEADER_SIZE + 1) + { + const size_t keySize = this->header[CIPHER_BLOCK_FORMAT_HEADER_SIZE]; + + // Marker indicates a key id but the length is zero + if (keySize == 0) + THROW(FormatError, "invalid cipher header"); + + this->headerSizeExpected += keySize; + } + } + while (this->headerSize < this->headerSizeExpected); cipherBlockFormatCipherNew(this); } @@ -284,24 +338,25 @@ cipherBlockFormatResultPack(THIS_VOID) /**********************************************************************************************************************************/ FN_EXTERN IoFilter * -cipherBlockFormatNew(const CipherSpec *const cipherSpec, const CipherBlockFormatNewParam param) +cipherBlockFormatNew(const CipherSpecMap *const cipherSpecMap, const CipherBlockFormatNewParam param) { FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_LOG_PARAM(CIPHER_SPEC_MAP, cipherSpecMap); FUNCTION_LOG_PARAM(UINT, param.format); FUNCTION_LOG_END(); FUNCTION_AUDIT_HELPER(); - ASSERT(cipherSpec != NULL); - ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone); + ASSERT(cipherSpecMap != NULL); + ASSERT(cipherSpecMapSize(cipherSpecMap) != 0); OBJ_NEW_BEGIN(CipherBlockFormat, .childQty = MEM_CONTEXT_QTY_MAX) { *this = (CipherBlockFormat) { - .cipherSpec = cipherSpecDupP(cipherSpec), + .cipherSpecMap = cipherSpecMapDup(cipherSpecMap), .formatExpected = param.format, + .headerSizeExpected = CIPHER_BLOCK_FORMAT_HEADER_SIZE, }; } OBJ_NEW_END(); @@ -313,7 +368,7 @@ cipherBlockFormatNew(const CipherSpec *const cipherSpec, const CipherBlockFormat { PackWrite *const packWrite = pckWriteNewP(); - cipherSpecPack(packWrite, cipherSpec); + cipherSpecMapPack(packWrite, cipherSpecMap); pckWriteU32P(packWrite, param.format); pckWriteEndP(packWrite); @@ -337,10 +392,10 @@ cipherBlockFormatNewPack(const Pack *const paramList) MEM_CONTEXT_TEMP_BEGIN() { PackRead *const paramListPack = pckReadNew(paramList); - const CipherSpec *const cipherSpec = cipherSpecNewPack(paramListPack); + const CipherSpecMap *const cipherSpecMap = cipherSpecMapNewPack(paramListPack); const unsigned int format = pckReadU32P(paramListPack); - result = ioFilterMove(cipherBlockFormatNewP(cipherSpec, .format = format), memContextPrior()); + result = ioFilterMove(cipherBlockFormatNewP(cipherSpecMap, .format = format), memContextPrior()); } MEM_CONTEXT_TEMP_END(); @@ -360,48 +415,195 @@ cipherBlockFormatResult(PackRead *const packRead) FUNCTION_TEST_RETURN(UINT, pckReadU32P(packRead)); } +/*********************************************************************************************************************************** +Filter to write the header before the content. The header must be plaintext so this filter runs after the block cipher. +***********************************************************************************************************************************/ +typedef struct CipherBlockFormatHeader +{ + const Buffer *header; // Header to write before any content + size_t headerOffset; // Header bytes written so far + size_t sourceOffset; // Content bytes written from the current source + bool inputSame; // Is the same input required on the next process call? +} CipherBlockFormatHeader; + +static void +cipherBlockFormatHeaderProcess(THIS_VOID, const Buffer *const source, Buffer *const destination) +{ + THIS(CipherBlockFormatHeader); + + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM_P(VOID, this); + FUNCTION_LOG_PARAM(BUFFER, source); + FUNCTION_LOG_PARAM(BUFFER, destination); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(source != NULL); + ASSERT(destination != NULL); + + // Write the header before any content + if (this->headerOffset < bufUsed(this->header)) + { + size_t copySize = bufUsed(this->header) - this->headerOffset; + + if (copySize > bufRemains(destination)) + copySize = bufRemains(destination); + + bufCatC(destination, bufPtrConst(this->header), this->headerOffset, copySize); + this->headerOffset += copySize; + } + + // Write the content once the header is complete + if (this->headerOffset == bufUsed(this->header)) + { + size_t copySize = bufUsed(source) - this->sourceOffset; + + if (copySize > bufRemains(destination)) + copySize = bufRemains(destination); + + bufCatC(destination, bufPtrConst(source), this->sourceOffset, copySize); + this->sourceOffset += copySize; + } + + this->inputSame = this->headerOffset < bufUsed(this->header) || this->sourceOffset < bufUsed(source); + + // Reset source offset when source has been completely processed + if (!this->inputSame) + this->sourceOffset = 0; + + FUNCTION_LOG_RETURN_VOID(); +} + +static bool +cipherBlockFormatHeaderInputSame(const THIS_VOID) +{ + THIS(const CipherBlockFormatHeader); + + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM_P(VOID, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(BOOL, this->inputSame); +} + +static IoFilter * +cipherBlockFormatHeaderNew(const Buffer *const header) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(BUFFER, header); + FUNCTION_TEST_END(); + + FUNCTION_AUDIT_HELPER(); + + ASSERT(header != NULL); + ASSERT(!bufEmpty(header)); + + OBJ_NEW_BEGIN(CipherBlockFormatHeader, .childQty = MEM_CONTEXT_QTY_MAX) + { + *this = (CipherBlockFormatHeader){.header = bufDup(header)}; + } + OBJ_NEW_END(); + + // Create param list + Pack *paramList; + + MEM_CONTEXT_TEMP_BEGIN() + { + PackWrite *const packWrite = pckWriteNewP(); + + pckWriteBinP(packWrite, header); + pckWriteEndP(packWrite); + + paramList = pckMove(pckWriteResult(packWrite), memContextPrior()); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_TEST_RETURN( + IO_FILTER, + ioFilterNewP( + CIPHER_BLOCK_FORMAT_HEADER_FILTER_TYPE, this, paramList, .inOut = cipherBlockFormatHeaderProcess, + .inputSame = cipherBlockFormatHeaderInputSame)); +} + +FN_EXTERN IoFilter * +cipherBlockFormatHeaderNewPack(const Pack *const paramList) +{ + IoFilter *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + result = ioFilterMove(cipherBlockFormatHeaderNew(pckReadBinP(pckReadNew(paramList))), memContextPrior()); + } + MEM_CONTEXT_TEMP_END(); + + return result; +} + /**********************************************************************************************************************************/ FN_EXTERN void cipherBlockFormatFilterGroupWriteAdd( - Buffer *const buffer, IoFilterGroup *const filterGroup, const CipherSpec *const cipherSpec, const unsigned int format) + IoFilterGroup *const filterGroup, const CipherSpec *const cipherSpec, const unsigned int format, + const CipherBlockFormatFilterGroupWriteAddParam param) { FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(BUFFER, buffer); FUNCTION_LOG_PARAM(IO_FILTER_GROUP, filterGroup); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); FUNCTION_LOG_PARAM(UINT, format); + FUNCTION_LOG_PARAM(STRING, param.keyId); FUNCTION_LOG_END(); FUNCTION_AUDIT_HELPER(); - ASSERT(buffer != NULL); ASSERT(filterGroup != NULL); ASSERT(cipherSpec != NULL); ASSERT(format >= REPOSITORY_FORMAT_MIN && format <= REPOSITORY_FORMAT_MAX); + ASSERT(param.keyId == NULL || !strEmpty(param.keyId)); + ASSERT(param.keyId == NULL || strSize(param.keyId) <= CIPHER_BLOCK_FORMAT_KEY_SIZE_MAX); + + // Key id is only valid for format >= 6 + ASSERT(param.keyId == NULL || format >= REPOSITORY_FORMAT_6); if (cipherSpecType(cipherSpec) != cipherTypeNone) { - // A format from 6 writes the header in place of the magic, so the content behind it is encrypted raw + // Format >= 6 writes the header in place of the magic and the content behind it is encrypted raw const bool header = format >= REPOSITORY_FORMAT_6; - if (header) - { - char headerZ[CIPHER_BLOCK_FORMAT_HEADER_SIZE + 1]; - - // Write the format zero-padded so it is always the same size, taking the digits it is written in from the value so - // that a format too large to fit could not run past them. The terminator lands on the byte after the header, which is - // not written to the buffer. - snprintf( - headerZ, sizeof(headerZ), CIPHER_BLOCK_FORMAT_MAGIC "%03u%c", format % 1000, CIPHER_BLOCK_FORMAT_RESERVED); - - bufCatC(buffer, (const uint8_t *)headerZ, 0, CIPHER_BLOCK_FORMAT_HEADER_SIZE); - } - ioFilterGroupAdd( filterGroup, cipherBlockNewP( cipherModeEncrypt, cipherSpecDupP(cipherSpec, .digestDefault = repoFormatDigest(format)), .header = header ? cipherBlockHeaderNone : cipherBlockHeaderMagic)); + + // The filter is added after the block cipher so the plaintext header is placed before the cipher output + if (header) + { + MEM_CONTEXT_TEMP_BEGIN() + { + char headerZ[CIPHER_BLOCK_FORMAT_HEADER_SIZE + 1]; + + // Zero-pad the format and add the marker, depending on whether or not there is a key + snprintf( + headerZ, sizeof(headerZ), CIPHER_BLOCK_FORMAT_MAGIC "%03u%c", format % 1000, + param.keyId == NULL ? CIPHER_BLOCK_FORMAT_MARKER_NONE : CIPHER_BLOCK_FORMAT_MARKER_KEY); + + Buffer *const headerBuffer = bufNew(CIPHER_BLOCK_FORMAT_HEADER_SIZE_MAX); + bufCatC(headerBuffer, (const uint8_t *)headerZ, 0, CIPHER_BLOCK_FORMAT_HEADER_SIZE); + + // Write the key size and id when present + if (param.keyId != NULL) + { + const uint8_t keySize = (uint8_t)strSize(param.keyId); + + bufCatC(headerBuffer, &keySize, 0, sizeof(keySize)); + bufCatC(headerBuffer, (const uint8_t *)strZ(param.keyId), 0, strSize(param.keyId)); + } + + ioFilterGroupAdd(filterGroup, cipherBlockFormatHeaderNew(headerBuffer)); + } + MEM_CONTEXT_TEMP_END(); + } } FUNCTION_LOG_RETURN_VOID(); @@ -422,7 +624,37 @@ cipherBlockFormatFilterGroupReadAdd(IoFilterGroup *const filterGroup, const Ciph ASSERT(cipherSpec != NULL); if (cipherSpecType(cipherSpec) != cipherTypeNone) - ioFilterGroupAdd(filterGroup, cipherBlockFormatNewP(cipherSpec)); + { + MEM_CONTEXT_TEMP_BEGIN() + { + CipherSpecMap *const cipherSpecMap = cipherSpecMapNew(); + cipherSpecMapAdd(cipherSpecMap, CIPHER_SPEC_MAP_ID_DEFAULT_STR, cipherSpec); + + ioFilterGroupAdd(filterGroup, cipherBlockFormatNewP(cipherSpecMap)); + } + MEM_CONTEXT_TEMP_END(); + } + + FUNCTION_LOG_RETURN(IO_FILTER_GROUP, filterGroup); +} + +/**********************************************************************************************************************************/ +FN_EXTERN IoFilterGroup * +cipherBlockFormatFilterGroupReadAddMap(IoFilterGroup *const filterGroup, const CipherSpecMap *const cipherSpecMap) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(IO_FILTER_GROUP, filterGroup); + FUNCTION_LOG_PARAM(CIPHER_SPEC_MAP, cipherSpecMap); + FUNCTION_LOG_END(); + + FUNCTION_AUDIT_HELPER(); + + ASSERT(filterGroup != NULL); + ASSERT(cipherSpecMap != NULL); + + // An unencrypted repository has no keys, so there is nothing to decrypt + if (cipherSpecMapSize(cipherSpecMap) != 0) + ioFilterGroupAdd(filterGroup, cipherBlockFormatNewP(cipherSpecMap)); FUNCTION_LOG_RETURN(IO_FILTER_GROUP, filterGroup); } diff --git a/src/common/format/cipherBlockFormat.h b/src/common/format/cipherBlockFormat.h index 1e54423ff..a9bf5ae4a 100644 --- a/src/common/format/cipherBlockFormat.h +++ b/src/common/format/cipherBlockFormat.h @@ -7,18 +7,24 @@ derives the key with. Everything the format defines is here so that the block ci A writer knows the format, so it writes the header itself and encrypts the content raw behind it. A reader cannot know the format until it has read the header, and a filter cannot be added to a read once it is open, so the read side is a filter that parses the header, works out from it what the content is encrypted with, and passes everything after it through a block cipher of its own. + +A file whose key cannot be determined from where the file is located stores the key in its header. ***********************************************************************************************************************************/ #ifndef COMMON_FORMAT_CIPHERBLOCKFORMAT_H #define COMMON_FORMAT_CIPHERBLOCKFORMAT_H #include "common/crypto/spec.h" +#include "common/format/cipherSpecMap.h" #include "common/io/filter/group.h" /*********************************************************************************************************************************** -Filter type constant +Filter type constants ***********************************************************************************************************************************/ #define CIPHER_BLOCK_FORMAT_FILTER_TYPE STRID5("cipher-fmt", 0x28d36e45441230) +// Filter that writes the format header +#define CIPHER_BLOCK_FORMAT_HEADER_FILTER_TYPE STRID5("cipher-hdr", 0x24446e45441230) + /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ @@ -28,12 +34,16 @@ typedef struct CipherBlockFormatNewParam unsigned int format; // Format the file is expected at, any when zero } CipherBlockFormatNewParam; -#define cipherBlockFormatNewP(cipherSpec, ...) \ - cipherBlockFormatNew(cipherSpec, (CipherBlockFormatNewParam){VAR_PARAM_INIT, __VA_ARGS__}) +// Decrypt with the stored key id or with CIPHER_SPEC_MAP_ID_DEFAULT when no key is stored +#define cipherBlockFormatNewP(cipherSpecMap, ...) \ + cipherBlockFormatNew(cipherSpecMap, (CipherBlockFormatNewParam){VAR_PARAM_INIT, __VA_ARGS__}) -FN_EXTERN IoFilter *cipherBlockFormatNew(const CipherSpec *cipherSpec, CipherBlockFormatNewParam param); +FN_EXTERN IoFilter *cipherBlockFormatNew(const CipherSpecMap *cipherSpecMap, CipherBlockFormatNewParam param); FN_EXTERN IoFilter *cipherBlockFormatNewPack(const Pack *paramList); +// Create filter that writes the format header from its param list +FN_EXTERN IoFilter *cipherBlockFormatHeaderNewPack(const Pack *paramList); + /*********************************************************************************************************************************** Getters/Setters ***********************************************************************************************************************************/ @@ -43,13 +53,28 @@ FN_EXTERN unsigned int cipherBlockFormatResult(PackRead *packRead); /*********************************************************************************************************************************** Helper functions ***********************************************************************************************************************************/ -// Write the header a format calls for into the buffer the content will be written into and add encryption to the filter group. -// Nothing is written and nothing is added when the repository is not encrypted. -FN_EXTERN void cipherBlockFormatFilterGroupWriteAdd( - Buffer *buffer, IoFilterGroup *filterGroup, const CipherSpec *cipherSpec, unsigned int format); +// Write the header the format requires into the buffer the content will be written into and add encryption to the filter group. A +// key id is stored in the header when one is specified. No header is added when the repository is not encrypted. +typedef struct CipherBlockFormatFilterGroupWriteAddParam +{ + VAR_PARAM_HEADER; + const String *keyId; // Key id to store in the header, NULL to store no id +} CipherBlockFormatFilterGroupWriteAddParam; -// Add decryption to the filter group for a file at a format that has yet to be read. Nothing is added when the repository is not -// encrypted. +#define cipherBlockFormatFilterGroupWriteAddP(filterGroup, cipherSpec, format, ...) \ + cipherBlockFormatFilterGroupWriteAdd( \ + filterGroup, cipherSpec, format, (CipherBlockFormatFilterGroupWriteAddParam){VAR_PARAM_INIT, __VA_ARGS__}) + +FN_EXTERN void cipherBlockFormatFilterGroupWriteAdd( + IoFilterGroup *filterGroup, const CipherSpec *cipherSpec, unsigned int format, + CipherBlockFormatFilterGroupWriteAddParam param); + +// Add decryption to the filter group for a file with format yet to be determined. The key is stored under +// CIPHER_SPEC_MAP_ID_DEFAULT, so the file must not contain a key id. Nothing is added when the repository is not encrypted. FN_EXTERN IoFilterGroup *cipherBlockFormatFilterGroupReadAdd(IoFilterGroup *filterGroup, const CipherSpec *cipherSpec); +// Add decryption to the filter group for a file that may contain a key id. An empty map means the repository is not encrypted and +// nothing is added. +FN_EXTERN IoFilterGroup *cipherBlockFormatFilterGroupReadAddMap(IoFilterGroup *filterGroup, const CipherSpecMap *cipherSpecMap); + #endif diff --git a/src/common/format/cipherSpecMap.c b/src/common/format/cipherSpecMap.c new file mode 100644 index 000000000..801c50cba --- /dev/null +++ b/src/common/format/cipherSpecMap.c @@ -0,0 +1,223 @@ +/*********************************************************************************************************************************** +Cipher Spec Map +***********************************************************************************************************************************/ +#include + +#include "common/debug.h" +#include "common/format/cipherSpecMap.h" +#include "common/log.h" + +STRING_EXTERN(CIPHER_SPEC_MAP_ID_DEFAULT_STR, CIPHER_SPEC_MAP_ID_DEFAULT); + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +struct CipherSpecMap +{ + CipherSpecMapPub pub; // Publicly accessible variables +}; + +/**********************************************************************************************************************************/ +FN_EXTERN CipherSpecMap * +cipherSpecMapNew(void) +{ + FUNCTION_TEST_VOID(); + + OBJ_NEW_BEGIN(CipherSpecMap, .childQty = MEM_CONTEXT_QTY_MAX) + { + *this = (CipherSpecMap) + { + .pub = {.list = lstNewP(sizeof(CipherSpecMapItem), .comparator = lstComparatorStr)}, + }; + } + OBJ_NEW_END(); + + FUNCTION_TEST_RETURN(CIPHER_SPEC_MAP, this); +} + +/**********************************************************************************************************************************/ +FN_EXTERN CipherSpecMap * +cipherSpecMapNewPack(PackRead *const packRead) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, packRead); + FUNCTION_TEST_END(); + + ASSERT(packRead != NULL); + + CipherSpecMap *const this = cipherSpecMapNew(); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + const unsigned int total = pckReadU32P(packRead); + + for (unsigned int keyIdx = 0; keyIdx < total; keyIdx++) + { + const CipherSpecMapItem item = + { + .id = pckReadStrP(packRead), + .cipherSpec = cipherSpecNewPack(packRead), + }; + + lstAdd(this->pub.list, &item); + } + + lstSort(this->pub.list, sortOrderAsc); + + this->pub.idCurrent = pckReadStrP(packRead); + } + MEM_CONTEXT_OBJ_END(); + + // The key with no id is never current + ASSERT(this->pub.idCurrent == NULL || !strEqZ(this->pub.idCurrent, CIPHER_SPEC_MAP_ID_DEFAULT)); + + FUNCTION_TEST_RETURN(CIPHER_SPEC_MAP, this); +} + +/**********************************************************************************************************************************/ +FN_EXTERN const CipherSpec * +cipherSpecMapGet(const CipherSpecMap *const this, const String *const id) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CIPHER_SPEC_MAP, this); + FUNCTION_TEST_PARAM(STRING, id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(id != NULL); + + const CipherSpecMapItem *const item = lstFind(this->pub.list, &id); + + // Error when request key was not found + if (item == NULL) + THROW_FMT(CryptoError, "unable to find cipher key '%s'", strZ(id)); + + FUNCTION_TEST_RETURN_CONST(CIPHER_SPEC, item->cipherSpec); +} + +/**********************************************************************************************************************************/ +FN_EXTERN void +cipherSpecMapAdd(CipherSpecMap *const this, const String *const id, const CipherSpec *const cipherSpec) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CIPHER_SPEC_MAP, this); + FUNCTION_TEST_PARAM(STRING, id); + FUNCTION_TEST_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(id != NULL); + ASSERT(!strEmpty(id)); + ASSERT(cipherSpec != NULL); + ASSERT(lstFind(this->pub.list, &id) == NULL); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + const CipherSpecMapItem item = {.id = strDup(id), .cipherSpec = cipherSpecDupP(cipherSpec)}; + + lstAdd(this->pub.list, &item); + lstSort(this->pub.list, sortOrderAsc); + + // The default key id is for files that contain no id so it is never current + if (!strEqZ(id, CIPHER_SPEC_MAP_ID_DEFAULT)) + this->pub.idCurrent = item.id; + } + MEM_CONTEXT_OBJ_END(); + + // The key with no id is never current and every other key is current once added + ASSERT(strEqZ(id, CIPHER_SPEC_MAP_ID_DEFAULT) || strEq(cipherSpecMapIdCurrent(this), id)); + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +FN_EXTERN void +cipherSpecMapIdCurrentSet(CipherSpecMap *const this, const String *const id) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CIPHER_SPEC_MAP, this); + FUNCTION_TEST_PARAM(STRING, id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(id != NULL); + ASSERT(!strEqZ(id, CIPHER_SPEC_MAP_ID_DEFAULT)); + + const CipherSpecMapItem *const item = lstFind(this->pub.list, &id); + ASSERT(item != NULL); + + // Point at the id stored with the key rather than duplicating it + this->pub.idCurrent = item->id; + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +FN_EXTERN CipherSpecMap * +cipherSpecMapDup(const CipherSpecMap *const this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CIPHER_SPEC_MAP, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + CipherSpecMap *const result = cipherSpecMapNew(); + + for (unsigned int keyIdx = 0; keyIdx < cipherSpecMapSize(this); keyIdx++) + { + const CipherSpecMapItem *const item = cipherSpecMapGetIdx(this, keyIdx); + + cipherSpecMapAdd(result, item->id, item->cipherSpec); + } + + if (cipherSpecMapIdCurrent(this) != NULL) + cipherSpecMapIdCurrentSet(result, cipherSpecMapIdCurrent(this)); + + FUNCTION_TEST_RETURN(CIPHER_SPEC_MAP, result); +} + +/**********************************************************************************************************************************/ +FN_EXTERN void +cipherSpecMapPack(PackWrite *const packWrite, const CipherSpecMap *const this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, packWrite); + FUNCTION_TEST_PARAM(CIPHER_SPEC_MAP, this); + FUNCTION_TEST_END(); + + ASSERT(packWrite != NULL); + ASSERT(this != NULL); + + pckWriteU32P(packWrite, cipherSpecMapSize(this)); + + for (unsigned int keyIdx = 0; keyIdx < cipherSpecMapSize(this); keyIdx++) + { + const CipherSpecMapItem *const item = cipherSpecMapGetIdx(this, keyIdx); + + pckWriteStrP(packWrite, item->id); + cipherSpecPack(packWrite, item->cipherSpec); + } + + pckWriteStrP(packWrite, cipherSpecMapIdCurrent(this)); + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +FN_EXTERN void +cipherSpecMapToLog(const CipherSpecMap *const this, StringStatic *const debugLog) +{ + // List only the ids (never the keys themselves) + strStcCat(debugLog, "{ids: ["); + + for (unsigned int keyIdx = 0; keyIdx < cipherSpecMapSize(this); keyIdx++) + { + if (keyIdx != 0) + strStcCat(debugLog, ", "); + + strStcFmt(debugLog, "%s", strZ(cipherSpecMapGetIdx(this, keyIdx)->id)); + } + + strStcCat(debugLog, "]}"); +} diff --git a/src/common/format/cipherSpecMap.h b/src/common/format/cipherSpecMap.h new file mode 100644 index 000000000..75de0021c --- /dev/null +++ b/src/common/format/cipherSpecMap.h @@ -0,0 +1,118 @@ +/*********************************************************************************************************************************** +Cipher Spec Map + +The keys used to encrypt a repository, stored by id. A file identifies its key by id, either in the header or by where the file is +stored. The meaning of an id is set by the caller. archive.info numbers its keys and backup.info uses the backup set label. + +One key is current and new files are encrypted with it. Adding a key makes it current, except for the key stored under +CIPHER_SPEC_MAP_ID_DEFAULT. +***********************************************************************************************************************************/ +#ifndef COMMON_FORMAT_CIPHERSPECMAP_H +#define COMMON_FORMAT_CIPHERSPECMAP_H + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +typedef struct CipherSpecMap CipherSpecMap; + +#include "common/crypto/spec.h" +#include "common/type/list.h" +#include "common/type/object.h" +#include "common/type/pack.h" + +/*********************************************************************************************************************************** +Id for the key required by a file that contains no key id. In archive.info this is the key preserved by migration, which is what +files written before key ids existed were encrypted with. +***********************************************************************************************************************************/ +#define CIPHER_SPEC_MAP_ID_DEFAULT "0" +STRING_DECLARE(CIPHER_SPEC_MAP_ID_DEFAULT_STR); + +/*********************************************************************************************************************************** +Constructors +***********************************************************************************************************************************/ +FN_EXTERN CipherSpecMap *cipherSpecMapNew(void); + +// Create from a pack written by cipherSpecMapPack() +FN_EXTERN CipherSpecMap *cipherSpecMapNewPack(PackRead *packRead); + +/*********************************************************************************************************************************** +Getters/Setters +***********************************************************************************************************************************/ +typedef struct CipherSpecMapItem +{ + const String *id; // Id associated with the key + const CipherSpec *cipherSpec; // Key and the digest it derives with +} CipherSpecMapItem; + +typedef struct CipherSpecMapPub +{ + List *list; // Keys sorted by id + const String *idCurrent; // Current key id +} CipherSpecMapPub; + +// Total keys +FN_INLINE_ALWAYS unsigned int +cipherSpecMapSize(const CipherSpecMap *const this) +{ + return lstSize(THIS_PUB(CipherSpecMap)->list); +} + +// Key by index +FN_INLINE_ALWAYS const CipherSpecMapItem * +cipherSpecMapGetIdx(const CipherSpecMap *const this, const unsigned int idx) +{ + return lstGet(THIS_PUB(CipherSpecMap)->list, idx); +} + +// Key by id. Errors when the id is not found. +FN_EXTERN const CipherSpec *cipherSpecMapGet(const CipherSpecMap *this, const String *id); + +// Current key id, NULL when the map has only the default key with no id +FN_INLINE_ALWAYS const String * +cipherSpecMapIdCurrent(const CipherSpecMap *const this) +{ + return THIS_PUB(CipherSpecMap)->idCurrent; +} + +// Set the current key id. The id must already be in the map. +FN_EXTERN void cipherSpecMapIdCurrentSet(CipherSpecMap *this, const String *id); + +// Duplicate +FN_EXTERN CipherSpecMap *cipherSpecMapDup(const CipherSpecMap *this); + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Add a key and make it current, except for a key stored under CIPHER_SPEC_MAP_ID_DEFAULT. The id must be unique. +FN_EXTERN void cipherSpecMapAdd(CipherSpecMap *this, const String *id, const CipherSpec *cipherSpec); + +// Write to a pack so it can be passed over a protocol +FN_EXTERN void cipherSpecMapPack(PackWrite *packWrite, const CipherSpecMap *this); + +// Move to a new parent mem context +FN_INLINE_ALWAYS CipherSpecMap * +cipherSpecMapMove(CipherSpecMap *const this, MemContext *const parentNew) +{ + return objMove(this, parentNew); +} + +/*********************************************************************************************************************************** +Destructor +***********************************************************************************************************************************/ +FN_INLINE_ALWAYS void +cipherSpecMapFree(CipherSpecMap *const this) +{ + objFree(this); +} + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +FN_EXTERN void cipherSpecMapToLog(const CipherSpecMap *this, StringStatic *debugLog); + +#define FUNCTION_LOG_CIPHER_SPEC_MAP_TYPE \ + CipherSpecMap * +#define FUNCTION_LOG_CIPHER_SPEC_MAP_FORMAT(value, buffer, bufferSize) \ + FUNCTION_LOG_OBJECT_FORMAT(value, cipherSpecMapToLog, buffer, bufferSize) + +#endif diff --git a/src/config/config.auto.h b/src/config/config.auto.h index fbf388ca0..5890e3e96 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -146,7 +146,7 @@ Option constants #define CFGOPT_VERBOSE "verbose" #define CFGOPT_VERSION "version" -#define CFG_OPTION_TOTAL 201 +#define CFG_OPTION_TOTAL 202 /*********************************************************************************************************************************** Option value constants @@ -587,6 +587,7 @@ typedef enum cfgOptRepoBundleLimit, cfgOptRepoBundleSize, cfgOptRepoCipherPass, + cfgOptRepoCipherRotate, cfgOptRepoCipherType, cfgOptRepoFormat, cfgOptRepoGcsBucket, diff --git a/src/config/parse.auto.c.inc b/src/config/parse.auto.c.inc index 6222f8c98..d8c7f0fee 100644 --- a/src/config/parse.auto.c.inc +++ b/src/config/parse.auto.c.inc @@ -29,6 +29,7 @@ static const StringPubConst parseRuleValueStr[] = PARSE_RULE_STRPUB("12"), // val/str PARSE_RULE_STRPUB("128KiB"), // val/str PARSE_RULE_STRPUB("128MiB"), // val/str + PARSE_RULE_STRPUB("15d"), // val/str PARSE_RULE_STRPUB("15m"), // val/str PARSE_RULE_STRPUB("15s"), // val/str PARSE_RULE_STRPUB("16GiB"), // val/str @@ -72,8 +73,10 @@ static const StringPubConst parseRuleValueStr[] = PARSE_RULE_STRPUB("8MiB"), // val/str PARSE_RULE_STRPUB("9"), // val/str PARSE_RULE_STRPUB("900"), // val/str + PARSE_RULE_STRPUB("90d"), // val/str PARSE_RULE_STRPUB("999"), // val/str PARSE_RULE_STRPUB("9999999"), // val/str + PARSE_RULE_STRPUB("9999999d"), // val/str PARSE_RULE_STRPUB("accept-new"), // val/str PARSE_RULE_STRPUB("aes-256-cbc"), // val/str PARSE_RULE_STRPUB("asc"), // val/str @@ -168,6 +171,7 @@ typedef enum parseRuleValStrQT_12_QT, // val/str/enum parseRuleValStrQT_128KiB_QT, // val/str/enum parseRuleValStrQT_128MiB_QT, // val/str/enum + parseRuleValStrQT_15d_QT, // val/str/enum parseRuleValStrQT_15m_QT, // val/str/enum parseRuleValStrQT_15s_QT, // val/str/enum parseRuleValStrQT_16GiB_QT, // val/str/enum @@ -211,8 +215,10 @@ typedef enum parseRuleValStrQT_8MiB_QT, // val/str/enum parseRuleValStrQT_9_QT, // val/str/enum parseRuleValStrQT_900_QT, // val/str/enum + parseRuleValStrQT_90d_QT, // val/str/enum parseRuleValStrQT_999_QT, // val/str/enum parseRuleValStrQT_9999999_QT, // val/str/enum + parseRuleValStrQT_9999999d_QT, // val/str/enum parseRuleValStrQT_accept_DS_new_QT, // val/str/enum parseRuleValStrQT_aes_DS_256_DS_cbc_QT, // val/str/enum parseRuleValStrQT_asc_QT, // val/str/enum @@ -483,6 +489,9 @@ static const int64_t parseRuleValueTime[] = 3600000, // val/time 86400000, // val/time 604800000, // val/time + 1296000000, // val/time + 7776000000, // val/time + 863999913600000, // val/time }; static const uint8_t parseRuleValueTimeStrMap[] = @@ -497,6 +506,9 @@ static const uint8_t parseRuleValueTimeStrMap[] = parseRuleValStrQT_1h_QT, // val/time/strmap parseRuleValStrQT_1d_QT, // val/time/strmap parseRuleValStrQT_7d_QT, // val/time/strmap + parseRuleValStrQT_15d_QT, // val/time/strmap + parseRuleValStrQT_90d_QT, // val/time/strmap + parseRuleValStrQT_9999999d_QT, // val/time/strmap }; typedef enum @@ -511,6 +523,9 @@ typedef enum parseRuleValTime1h, // val/time/enum parseRuleValTime1d, // val/time/enum parseRuleValTime7d, // val/time/enum + parseRuleValTime15d, // val/time/enum + parseRuleValTime90d, // val/time/enum + parseRuleValTime9999999d, // val/time/enum } ParseRuleValueTime; /*********************************************************************************************************************************** @@ -5983,6 +5998,45 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] = ), // opt/repo-cipher-pass ), // opt/repo-cipher-pass // ----------------------------------------------------------------------------------------------------------------------------- + PARSE_RULE_OPTION // opt/repo-cipher-rotate + ( // opt/repo-cipher-rotate + PARSE_RULE_OPTION_NAME("repo-cipher-rotate"), // opt/repo-cipher-rotate + PARSE_RULE_OPTION_TYPE(Time), // opt/repo-cipher-rotate + PARSE_RULE_OPTION_RESET(true), // opt/repo-cipher-rotate + PARSE_RULE_OPTION_REQUIRED(true), // opt/repo-cipher-rotate + PARSE_RULE_OPTION_SECTION(Global), // opt/repo-cipher-rotate + PARSE_RULE_OPTION_GROUP_ID(Repo), // opt/repo-cipher-rotate + // opt/repo-cipher-rotate + PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/repo-cipher-rotate + ( // opt/repo-cipher-rotate + PARSE_RULE_OPTION_COMMAND(Backup) // opt/repo-cipher-rotate + PARSE_RULE_OPTION_COMMAND(Expire) // opt/repo-cipher-rotate + ), // opt/repo-cipher-rotate + // opt/repo-cipher-rotate + PARSE_RULE_OPTIONAL // opt/repo-cipher-rotate + ( // opt/repo-cipher-rotate + PARSE_RULE_OPTIONAL_GROUP // opt/repo-cipher-rotate + ( // opt/repo-cipher-rotate + PARSE_RULE_OPTIONAL_DEPEND // opt/repo-cipher-rotate + ( // opt/repo-cipher-rotate + PARSE_RULE_VAL_OPT(RepoCipherType), // opt/repo-cipher-rotate + PARSE_RULE_VAL_STRID(aes_DS_256_DS_cbc), // opt/repo-cipher-rotate + ), // opt/repo-cipher-rotate + // opt/repo-cipher-rotate + PARSE_RULE_OPTIONAL_ALLOW_RANGE // opt/repo-cipher-rotate + ( // opt/repo-cipher-rotate + PARSE_RULE_VAL_TIME(15d), // opt/repo-cipher-rotate + PARSE_RULE_VAL_TIME(9999999d), // opt/repo-cipher-rotate + ), // opt/repo-cipher-rotate + // opt/repo-cipher-rotate + PARSE_RULE_OPTIONAL_DEFAULT // opt/repo-cipher-rotate + ( // opt/repo-cipher-rotate + PARSE_RULE_VAL_TIME(90d), // opt/repo-cipher-rotate + ), // opt/repo-cipher-rotate + ), // opt/repo-cipher-rotate + ), // opt/repo-cipher-rotate + ), // opt/repo-cipher-rotate + // ----------------------------------------------------------------------------------------------------------------------------- PARSE_RULE_OPTION // opt/repo-cipher-type ( // opt/repo-cipher-type PARSE_RULE_OPTION_NAME("repo-cipher-type"), // opt/repo-cipher-type @@ -12218,6 +12272,7 @@ static const uint8_t optionResolveOrder[] = cfgOptRepoBlockSizeSuper, // opt-resolve-order cfgOptRepoBlockSizeSuperFull, // opt-resolve-order cfgOptRepoCipherPass, // opt-resolve-order + cfgOptRepoCipherRotate, // opt-resolve-order cfgOptRepoGcsKeyType, // opt-resolve-order cfgOptRepoHost, // opt-resolve-order cfgOptRepoHostConfig, // opt-resolve-order diff --git a/src/info/info.c b/src/info/info.c index 6281a011b..99d8c8024 100644 --- a/src/info/info.c +++ b/src/info/info.c @@ -28,6 +28,7 @@ Object types struct Info { InfoPub pub; // Publicly accessible variables + const CipherSpec *cipherSpecNone; // Returned by infoCipherSpec() when there are no keys }; struct InfoSave @@ -124,8 +125,11 @@ infoNew(const unsigned int format, const CipherSpec *const cipherSpecSub) #define INFO_SECTION_BACKREST "backrest" #define INFO_KEY_CHECKSUM "backrest-checksum" #define INFO_SECTION_CIPHER "cipher" -#define INFO_KEY_CIPHER_DIGEST "cipher-digest" #define INFO_KEY_CIPHER_PASS "cipher-pass" +#define INFO_KEY_CIPHER_PASS_CURRENT "cipher-pass-current" +#define INFO_KEY_CIPHER_PASS_ROTATE "cipher-pass-rotate" +#define INFO_KEY_CIPHER_DIGEST "digest" +#define INFO_KEY_CIPHER_KEY "key" FN_EXTERN Info * infoNewLoad( @@ -154,7 +158,6 @@ infoNewLoad( String *const sectionLast = strNew(); // The last section seen during load IoFilter *const checksumActualFilter = cryptoHashNew(hashTypeSha1); // Checksum calculated from the file const String *checksumExpected = NULL; // Checksum found in ini file - HashType cipherDigest = hashTypeSha1; // Digest the stored pass derives with INFO_CHECKSUM_BEGIN(checksumActualFilter); @@ -218,26 +221,61 @@ infoNewLoad( // Process cipher section else if (strEqZ(value->section, INFO_SECTION_CIPHER)) { - // Store the digest the pass derives with. A file written before the digest was stored has none, so the - // default is what every repository derived with then. - if (strEqZ(value->key, INFO_KEY_CIPHER_DIGEST)) - { - cipherDigest = jsonReadStrId(jsonReadNew(value->value)); - } // No validation needed for cipher-pass, just store it - else if (strEqZ(value->key, INFO_KEY_CIPHER_PASS)) + if (strEqZ(value->key, INFO_KEY_CIPHER_PASS)) { - MEM_CONTEXT_OBJ_BEGIN(this) + MEM_CONTEXT_TEMP_BEGIN() { - // The dependent files are encrypted with the same cipher type as this one and derive with the - // digest stored with the pass. The digest is read before this since the keys come out in order - // and digest sorts before pass. - this->pub.cipherSpec = cipherSpecNewP( - cipherSpecType(cipherSpec), BUFSTR(varStr(jsonToVar(value->value))), - .digest = cipherDigest); + // Dependent files are encrypted with the same cipher type as this one + const CipherType cipherType = cipherSpecType(cipherSpec); + CipherSpecMap *const cipherSpecMap = cipherSpecMapNew(); + const Variant *const cipherPass = jsonToVar(value->value); + + // Format >= 6 stores every key by id along with the digest it derives with + if (infoFormat(this) >= REPOSITORY_FORMAT_6) + { + const KeyValue *const keyKv = varKv(cipherPass); + const VariantList *const idList = kvKeyList(keyKv); + + for (unsigned int idIdx = 0; idIdx < varLstSize(idList); idIdx++) + { + const Variant *const id = varLstGet(idList, idIdx); + const KeyValue *const keyData = varKv(kvGet(keyKv, id)); + + cipherSpecMapAdd( + cipherSpecMap, varStr(id), + cipherSpecNewP( + cipherType, + BUFSTR(varStr(kvGet(keyData, VARSTRDEF(INFO_KEY_CIPHER_KEY)))), + .digest = strIdFromZ( + strZ(varStr(kvGet(keyData, VARSTRDEF(INFO_KEY_CIPHER_DIGEST))))))); + } + } + // Else there is one key with no id and no digest, which every repository derives with SHA-1 + else + { + cipherSpecMapAdd( + cipherSpecMap, CIPHER_SPEC_MAP_ID_DEFAULT_STR, + cipherSpecNewP(cipherType, BUFSTR(varStr(cipherPass)), .digest = hashTypeSha1)); + } + + infoCipherSpecMapSet(this, cipherSpecMap); } - MEM_CONTEXT_OBJ_END(); + MEM_CONTEXT_TEMP_END(); } + // Current key id + else if (strEqZ(value->key, INFO_KEY_CIPHER_PASS_CURRENT)) + { + MEM_CONTEXT_TEMP_BEGIN() + { + cipherSpecMapIdCurrentSet( + this->pub.cipherSpecMap, jsonReadStr(jsonReadNew(value->value))); + } + MEM_CONTEXT_TEMP_END(); + } + // Time of the last key rotation + else if (strEqZ(value->key, INFO_KEY_CIPHER_PASS_ROTATE)) + this->pub.cipherRotateTime = (time_t)jsonReadInt64(jsonReadNew(value->value)); } // Else pass to callback for processing else @@ -295,8 +333,8 @@ infoNewLoad( MEM_CONTEXT_TEMP_END(); // A file with no cipher section has no encrypted dependent files - if (this->pub.cipherSpec == NULL) - infoCipherSpecSet(this, NULL); + if (this->pub.cipherSpecMap == NULL) + infoCipherSpecMapSet(this, NULL); } OBJ_NEW_END(); @@ -412,33 +450,70 @@ infoSave(Info *const this, IoWrite *const write, InfoSaveCallback *const callbac infoSaveValue(&data, INFO_SECTION_BACKREST, INFO_KEY_FORMAT, jsonFromVar(VARUINT(infoFormat(this)))); infoSaveValue(&data, INFO_SECTION_BACKREST, INFO_KEY_VERSION, jsonFromVar(VARSTRDEF(PROJECT_VERSION))); - // Add cipher passphrase if defined - if (cipherSpecType(infoCipherSpec(this)) != cipherTypeNone) + // Add cipher keys if defined + const CipherSpecMap *const cipherSpecMap = infoCipherSpecMap(this); + + if (cipherSpecMapSize(cipherSpecMap) != 0) { callbackFunction(callbackData, STRDEF(INFO_SECTION_CIPHER), &data); - // A format before the digest could be stored has nowhere to put it and a reader assumes SHA-1, so a pass stored at - // one of those formats must derive with SHA-1 or the reader will derive a different key. Format >= 6 must have a - // digest to store. Release builds check this too, instead of silently writing a pass that no reader can derive. - CHECK( - AssertError, - infoFormat(this) >= REPOSITORY_FORMAT_6 ? - cipherSpecDigest(infoCipherSpec(this)) != 0 : cipherSpecDigest(infoCipherSpec(this)) == hashTypeSha1, - "pass must derive with sha1 before format 6 and must have a digest from format 6"); - - // Store the digest the pass derives with so that a pass outlives the format of the file it is stored in. A pass in a - // file written before this could be stored derives with SHA-1, which is what a reader assumes when it finds no digest. - if (infoFormat(this) >= REPOSITORY_FORMAT_6) + MEM_CONTEXT_TEMP_BEGIN() { - char digestZ[STRID_MAX + 1]; - strIdToZ(cipherSpecDigest(infoCipherSpec(this)), digestZ); + JsonWrite *const json = jsonWriteNewP(); - infoSaveValue(&data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_DIGEST, jsonFromVar(VARSTRZ(digestZ))); + // Format >= 6 stores every key by id along with the digest it derives with + if (infoFormat(this) >= REPOSITORY_FORMAT_6) + { + jsonWriteObjectBegin(json); + + for (unsigned int keyIdx = 0; keyIdx < cipherSpecMapSize(cipherSpecMap); keyIdx++) + { + const CipherSpecMapItem *const item = cipherSpecMapGetIdx(cipherSpecMap, keyIdx); + + // A key must have a digest to store. Release builds check this too, instead of silently writing a key + // that no reader can derive. + CHECK(AssertError, cipherSpecDigest(item->cipherSpec) != 0, "cipher key must have a digest from format 6"); + + jsonWriteObjectBegin(jsonWriteKey(json, item->id)); + jsonWriteStrId(jsonWriteKeyZ(json, INFO_KEY_CIPHER_DIGEST), cipherSpecDigest(item->cipherSpec)); + jsonWriteStr( + jsonWriteKeyZ(json, INFO_KEY_CIPHER_KEY), strNewBuf(cipherSpecPass(item->cipherSpec))); + jsonWriteObjectEnd(json); + } + + jsonWriteObjectEnd(json); + } + // Else store only the key + else + { + const CipherSpec *const cipherSpec = infoCipherSpec(this); + + CHECK(AssertError, cipherSpecMapSize(cipherSpecMap) == 1, "only one cipher key can be stored before format 6"); + CHECK( + AssertError, cipherSpecDigest(cipherSpec) == hashTypeSha1, + "cipher key must derive with sha1 before format 6"); + + jsonWriteStr(json, strNewBuf(cipherSpecPass(cipherSpec))); + } + + infoSaveValue(&data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_PASS, jsonWriteResult(json)); + + // Record the current key id, which only exists at format >= 6 + if (cipherSpecMapIdCurrent(cipherSpecMap) != NULL) + { + infoSaveValue( + &data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_PASS_CURRENT, + jsonFromVar(VARSTR(cipherSpecMapIdCurrent(cipherSpecMap)))); + } + + // Record the time of the last key rotation + if (infoCipherRotateTime(this) != 0) + { + infoSaveValue( + &data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_PASS_ROTATE, jsonFromVar(VARINT64(infoCipherRotateTime(this)))); + } } - - infoSaveValue( - &data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_PASS, - jsonFromVar(VARSTR(strNewBuf(cipherSpecPass(infoCipherSpec(this)))))); + MEM_CONTEXT_TEMP_END(); } // Flush out any additional sections @@ -479,6 +554,95 @@ infoFormatSet(Info *const this, const unsigned int format) FUNCTION_TEST_RETURN_VOID(); } +FN_EXTERN const CipherSpec * +infoCipherSpec(const Info *const this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(INFO, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (this->cipherSpecNone != NULL) + FUNCTION_TEST_RETURN_CONST(CIPHER_SPEC, this->cipherSpecNone); + + FUNCTION_TEST_RETURN_CONST(CIPHER_SPEC, cipherSpecMapGet(infoCipherSpecMap(this), CIPHER_SPEC_MAP_ID_DEFAULT_STR)); +} + +/**********************************************************************************************************************************/ +FN_EXTERN void +infoCipherSpecMapSet(Info *const this, const CipherSpecMap *const cipherSpecMap) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(INFO, this); + FUNCTION_TEST_PARAM(CIPHER_SPEC_MAP, cipherSpecMap); + FUNCTION_TEST_END(); + + FUNCTION_AUDIT_IF(memContextCurrent() != objMemContext(this)); // Do not audit calls from within the object + + ASSERT(this != NULL); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + // NULL becomes an empty map so the getter never returns NULL + this->pub.cipherSpecMap = cipherSpecMap == NULL ? cipherSpecMapNew() : cipherSpecMapDup(cipherSpecMap); + + // The default key is looked up on demand, so only a caller that asks for one key errors when there is no default + this->cipherSpecNone = cipherSpecMapSize(this->pub.cipherSpecMap) == 0 ? cipherSpecNewNone() : NULL; + } + MEM_CONTEXT_OBJ_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +FN_EXTERN void +infoCipherSpecAdd(Info *const this, const String *const id, const CipherSpec *const cipherSpec) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(INFO, this); + FUNCTION_TEST_PARAM(STRING, id); + FUNCTION_TEST_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_TEST_END(); + + FUNCTION_AUDIT_IF(memContextCurrent() != objMemContext(this)); // Do not audit calls from within the object + + ASSERT(this != NULL); + ASSERT(id != NULL); + ASSERT(cipherSpec != NULL && cipherSpecType(cipherSpec) != cipherTypeNone); + ASSERT(cipherSpecDigest(cipherSpec) != 0); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + CipherSpecMap *const cipherSpecMap = cipherSpecMapDup(this->pub.cipherSpecMap); + cipherSpecMapAdd(cipherSpecMap, id, cipherSpec); + + this->pub.cipherSpecMap = cipherSpecMap; + this->cipherSpecNone = NULL; + } + MEM_CONTEXT_OBJ_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +FN_EXTERN void +infoCipherRotateTimeSet(Info *const this, const time_t cipherRotateTime) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(INFO, this); + FUNCTION_TEST_PARAM(TIME, cipherRotateTime); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(cipherRotateTime > 0); + + this->pub.cipherRotateTime = cipherRotateTime; + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ FN_EXTERN void infoCipherSpecSet(Info *const this, const CipherSpec *const cipherSpec) { @@ -490,14 +654,26 @@ infoCipherSpecSet(Info *const this, const CipherSpec *const cipherSpec) FUNCTION_AUDIT_IF(memContextCurrent() != objMemContext(this)); // Do not audit calls from within the object ASSERT(this != NULL); - ASSERT(cipherSpec == NULL || cipherSpecType(cipherSpec) == cipherTypeNone || cipherSpecDigest(cipherSpec) != 0); - MEM_CONTEXT_OBJ_BEGIN(this) + // No keys needed when there is nothing to encrypt + if (cipherSpec == NULL || cipherSpecType(cipherSpec) == cipherTypeNone) { - // Copy so the caller is free to release what was passed in, and so the getter never returns NULL - this->pub.cipherSpec = cipherSpec == NULL ? cipherSpecNewNone() : cipherSpecDupP(cipherSpec); + infoCipherSpecMapSet(this, NULL); + } + // Else the one key, stored under the default id + else + { + ASSERT(cipherSpecDigest(cipherSpec) != 0); + + MEM_CONTEXT_TEMP_BEGIN() + { + CipherSpecMap *const cipherSpecMap = cipherSpecMapNew(); + cipherSpecMapAdd(cipherSpecMap, CIPHER_SPEC_MAP_ID_DEFAULT_STR, cipherSpec); + + infoCipherSpecMapSet(this, cipherSpecMap); + } + MEM_CONTEXT_TEMP_END(); } - MEM_CONTEXT_OBJ_END(); FUNCTION_TEST_RETURN_VOID(); } diff --git a/src/info/info.h b/src/info/info.h index 88a4fb843..9a13db12a 100644 --- a/src/info/info.h +++ b/src/info/info.h @@ -11,6 +11,7 @@ typedef struct Info Info; typedef struct InfoSave InfoSave; #include "common/crypto/spec.h" +#include "common/format/cipherSpecMap.h" #include "common/ini.h" #include "common/type/json.h" #include "storage/storage.h" @@ -54,7 +55,8 @@ typedef struct InfoPub { unsigned int format; // Repository format the file was written with const String *backrestVersion; // pgBackRest version - const CipherSpec *cipherSpec; // Cipher spec for dependent files + CipherSpecMap *cipherSpecMap; // Cipher keys for dependent files + time_t cipherRotateTime; // Time of the last cipher key rotation } InfoPub; // Repository format @@ -66,17 +68,34 @@ infoFormat(const Info *const this) FN_EXTERN void infoFormatSet(Info *this, unsigned int format); -// Cipher spec for the files that depend on this one, e.g. the manifest for backup.info. Never NULL, so it can be handed on -// without a check, and none when there is no pass. -FN_INLINE_ALWAYS const CipherSpec * -infoCipherSpec(const Info *const this) +// Cipher keys for the files that require multiple keys, e.g. manifests. Never NULL, empty when there are no keys. +FN_INLINE_ALWAYS const CipherSpecMap * +infoCipherSpecMap(const Info *const this) { - return THIS_PUB(Info)->cipherSpec; + return THIS_PUB(Info)->cipherSpecMap; } -// Set cipher spec for dependent files. NULL means they are not encrypted. +// Default cipher spec or none when there are no keys +FN_EXTERN const CipherSpec *infoCipherSpec(const Info *this); + +// Set cipher keys for dependent files. NULL means they are not encrypted. +FN_EXTERN void infoCipherSpecMapSet(Info *this, const CipherSpecMap *cipherSpecMap); + +// Set a single cipher spec for dependent files under the default id. NULL means they are not encrypted. FN_EXTERN void infoCipherSpecSet(Info *this, const CipherSpec *cipherSpec); +// Add a cipher key for dependent files under an id and make it current +FN_EXTERN void infoCipherSpecAdd(Info *this, const String *id, const CipherSpec *cipherSpec); + +// Time of the last cipher key rotation, zero when none +FN_INLINE_ALWAYS time_t +infoCipherRotateTime(const Info *const this) +{ + return THIS_PUB(Info)->cipherRotateTime; +} + +FN_EXTERN void infoCipherRotateTimeSet(Info *this, time_t cipherRotateTime); + // pgBackRest version FN_INLINE_ALWAYS const String * infoBackrestVersion(const Info *const this) diff --git a/src/info/infoArchive.c b/src/info/infoArchive.c index 824d38582..4396626fb 100644 --- a/src/info/infoArchive.c +++ b/src/info/infoArchive.c @@ -10,10 +10,12 @@ Archive Info Handler #include "common/debug.h" #include "common/format/cipherBlockFormat.h" +#include "common/format/format.h" #include "common/ini.h" #include "common/io/bufferWrite.h" #include "common/io/io.h" #include "common/log.h" +#include "common/type/convert.h" #include "info/infoArchive.h" #include "info/infoPg.h" #include "postgres/interface.h" @@ -58,13 +60,14 @@ infoArchiveNewInternal(void) /**********************************************************************************************************************************/ FN_EXTERN InfoArchive * infoArchiveNew( - const unsigned int pgVersion, const uint64_t pgSystemId, const unsigned int format, const CipherSpec *const cipherSpecSub) + const unsigned int pgVersion, const uint64_t pgSystemId, const unsigned int format, + const CipherSpecMap *const cipherSpecMapSub) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(UINT, pgVersion); FUNCTION_LOG_PARAM(UINT64, pgSystemId); FUNCTION_LOG_PARAM(UINT, format); - FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpecSub); + FUNCTION_LOG_PARAM(CIPHER_SPEC_MAP, cipherSpecMapSub); FUNCTION_LOG_END(); ASSERT(pgVersion > 0 && pgSystemId > 0); @@ -76,7 +79,8 @@ infoArchiveNew( this = infoArchiveNewInternal(); // Initialize the pg data - this->pub.infoPg = infoPgNew(infoPgArchive, format, cipherSpecSub); + this->pub.infoPg = infoPgNew(infoPgArchive, format, NULL); + infoCipherSpecMapSet(infoPgInfo(this->pub.infoPg), cipherSpecMapSub); infoArchivePgSet(this, pgVersion, pgSystemId); } OBJ_NEW_END(); @@ -107,6 +111,35 @@ infoArchiveNewLoad(IoRead *const read, const CipherSpec *const cipherSpec) FUNCTION_LOG_RETURN(INFO_ARCHIVE, this); } +/**********************************************************************************************************************************/ +FN_EXTERN void +infoArchiveCipherRotate(InfoArchive *const this, const CipherSpec *const cipherSpec, const time_t rotateTime) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(INFO_ARCHIVE, this); + FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_LOG_PARAM(TIME, rotateTime); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(infoArchiveFormat(this) >= REPOSITORY_FORMAT_6); + + MEM_CONTEXT_TEMP_BEGIN() + { + // The next key id follows the current key id, or is the first key id when there is no current key + const String *const idCurrent = cipherSpecMapIdCurrent(infoArchiveCipherSpecMap(this)); + const String *const id = strNewFmt("%u", idCurrent == NULL ? 1 : cvtZToUInt(strZ(idCurrent)) + 1); + + Info *const info = infoPgInfo(infoArchivePg(this)); + + infoCipherSpecAdd(info, id, cipherSpec); + infoCipherRotateTimeSet(info, rotateTime); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN_VOID(); +} + /**********************************************************************************************************************************/ const String * infoArchiveIdHistoryMatch( @@ -339,7 +372,7 @@ infoArchiveSaveFile( // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); IoWrite *const write = ioBufferWriteNew(buffer); - cipherBlockFormatFilterGroupWriteAdd(buffer, ioWriteFilterGroup(write), cipherSpec, infoArchiveFormat(infoArchive)); + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(write), cipherSpec, infoArchiveFormat(infoArchive)); infoArchiveSave(infoArchive, write); // Save the file and make a copy diff --git a/src/info/infoArchive.h b/src/info/infoArchive.h index d8c861cd7..ac14e1a67 100644 --- a/src/info/infoArchive.h +++ b/src/info/infoArchive.h @@ -30,7 +30,7 @@ STRING_DECLARE(INFO_ARCHIVE_PATH_FILE_COPY_STR); Constructors ***********************************************************************************************************************************/ FN_EXTERN InfoArchive *infoArchiveNew( - const unsigned int pgVersion, const uint64_t pgSystemId, unsigned int format, const CipherSpec *cipherSpecSub); + const unsigned int pgVersion, const uint64_t pgSystemId, unsigned int format, const CipherSpecMap *cipherSpecMapSub); // Create new object and load contents from IoRead FN_EXTERN InfoArchive *infoArchiveNewLoad(IoRead *read, const CipherSpec *cipherSpec); @@ -67,6 +67,20 @@ infoArchiveCipherSpec(const InfoArchive *const this) return infoPgCipherSpec(infoArchivePg(this)); } +// Cipher keys for dependent files +FN_INLINE_ALWAYS const CipherSpecMap * +infoArchiveCipherSpecMap(const InfoArchive *const this) +{ + return infoCipherSpecMap(infoPgInfo(infoArchivePg(this))); +} + +// Time of the last archive key rotation, zero when none +FN_INLINE_ALWAYS time_t +infoArchiveCipherRotateTime(const InfoArchive *const this) +{ + return infoCipherRotateTime(infoPgInfo(infoArchivePg(this))); +} + // Repository format FN_INLINE_ALWAYS unsigned int infoArchiveFormat(const InfoArchive *const this) @@ -79,6 +93,10 @@ FN_EXTERN void infoArchiveFormatSet(InfoArchive *this, unsigned int format); /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ +// Add a cipher key for WAL at the next key id, make it current, and record the time of the rotation. Key ids are sequential from +// one and are never reused. Id zero is reserved for a key migrated from format 5. +FN_EXTERN void infoArchiveCipherRotate(InfoArchive *this, const CipherSpec *cipherSpec, time_t rotateTime); + // Given a backrest history id and postgres systemId and version, return the archiveId of the best match FN_EXTERN const String *infoArchiveIdHistoryMatch( const InfoArchive *this, const unsigned int historyId, const unsigned int pgVersion, const uint64_t pgSystemId); diff --git a/src/info/infoBackup.c b/src/info/infoBackup.c index 5bdf6853e..8594845c2 100644 --- a/src/info/infoBackup.c +++ b/src/info/infoBackup.c @@ -938,7 +938,7 @@ infoBackupSaveFile( // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); IoWrite *const write = ioBufferWriteNew(buffer); - cipherBlockFormatFilterGroupWriteAdd(buffer, ioWriteFilterGroup(write), cipherSpec, infoBackupFormat(infoBackup)); + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(write), cipherSpec, infoBackupFormat(infoBackup)); infoBackupSave(infoBackup, write); // Save the file and make a copy diff --git a/src/meson.build b/src/meson.build index 4063c28db..434f7290a 100644 --- a/src/meson.build +++ b/src/meson.build @@ -127,6 +127,7 @@ src_pgbackrest = [ 'common/exec.c', 'common/fork.c', 'common/format/cipherBlockFormat.c', + 'common/format/cipherSpecMap.c', 'common/format/format.c', 'common/ini.c', 'common/io/bufferRead.c', diff --git a/test/define.yaml b/test/define.yaml index 85c3edb1b..e2c0ae31c 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -377,10 +377,11 @@ unit: # -------------------------------------------------------------------------------------------------------------------------------- - name: common/format - total: 3 + total: 4 coverage: - common/format/cipherBlockFormat + - common/format/cipherSpecMap - common/format/format # -------------------------------------------------------------------------------------------------------------------------------- diff --git a/test/spec-rekey.md b/test/spec-rekey.md new file mode 100644 index 000000000..b1affbf80 --- /dev/null +++ b/test/spec-rekey.md @@ -0,0 +1,108 @@ +# Key Rotation + +**Working Document!!!** This spec guides development and will be removed before the final commit. Anything durable (user-facing behavior, the format feature table, migration guidance) moves to the user documentation before this file is deleted. The repository format 6 infrastructure this builds on is merged; format 6 carries the SHA-256 key derivation and nothing else, and this branch is what fills it out. + +## Goals + +- Rotate the archive key, which is the only bulk-data key that never rotates today. +- Give each backup set one key covering both its manifests and its data files, rotating with every full backup. +- Rotate the user passphrase without touching bulk data, when the user asks for it. Later commit. + +## Current State (Format 5) + +- `repo-cipher-pass` (user-supplied) encrypts `backup.info` and `archive.info`. +- `backup.info` holds one sub-passphrase that encrypts every `backup.manifest` in the stanza. +- Each manifest holds a sub-passphrase that encrypts the data files of its own backup set. A new one is generated per full backup (`src/command/backup/backup.c:244`) and inherited by diff and incr (`src/command/backup/incr.c.inc:149`), so this level already rotates per set and expires with it. +- `archive.info` holds one sub-passphrase that encrypts all WAL across all archive-ids for the life of the stanza. This is the only bulk-data key that never rotates. +- Sub-passphrases are 48 random bytes, base64-encoded to 64 characters (`cipherSpecGen()`, `src/command/stanza/common.c:20`). + +## What Format 6 Already Provides + +- Each info file and manifest stores the format it was written with, and readers trust the stored value, so a repository holds more than one format at once while older backups and archives expire. +- A stanza is created at a format with `stanza-create --repoN-format` and migrated with `stanza-upgrade --repoN-format`. Once the info files are at format 6 a version that does not support it cannot read the stanza at all, which is the gate that makes the changes below safe. +- Encrypted info files begin with an eight byte plaintext header, `PGBR` plus three format digits plus one more byte, written in place of the OpenSSL `Salted__` magic. `CipherBlockFormat` reads the header and streams the rest through a block cipher built from what the header said. +- A key stores the digest it derives with, so a key outlives the format of the file it is stored in. Keys carried over by migration keep SHA-1; only new keys get SHA-256. +- The eighth byte of the header is reserved, and that byte is where the archive key id goes. + +## Key Storage + +- At format 6 the `cipher-pass` value in both info files holds every key the file is responsible for rather than the single key it holds today, as an object keyed by the key id. + +``` +cipher-pass={"1":{"key":"afg...","digest":"sha256"}, ...} +``` + +- The value is an object rather than the key on its own because the digest has to travel with each key. A flat `{"1":"afg..."}` would mean inferring the digest from the id, which is the inference the stored digest exists to avoid. +- In `archive.info` the id is the sequential key id that the file header names. In `backup.info` it is the full backup label of the set the key belongs to. +- A key inherited from a format 5 migration goes in id `0`: the stanza-wide manifest key in `backup.info`, the stanza-wide archive key in `archive.info`. Format 6 never writes id 0 itself and a backup label cannot collide with it. +- `stanza-create` at format 6 writes a single archive key at id 1. `stanza-upgrade` from format 5 keeps the key it already has at id 0 and adds a new key at id 1, which becomes current, so WAL written before the migration stays readable under the key that wrote it. +- There is no scalar `cipher-pass` at format 6. A stanza created at format 6 has no id 0 entry at all. +- The digest belongs to the entry rather than to the file. An id 0 key derives with SHA-1 and every format 6 key derives with SHA-256, so the scalar `cipher-digest` written by the format 6 infrastructure commit moves into the entries. This also survives a future format that changes the digest again, which a scalar would not. +- Keys are held in memory as a new `CipherSpecMap` in `common/format`, keyed by the id, which is what `CipherBlockFormat` selects from and what the read path packs for the protocol. It lives beside the format rather than in `common/crypto` so the block cipher stays free of repository concepts. +- `Info` holds a `CipherSpecMap` at every format. A format 5 file loads its single scalar key as the one entry, so only the loader and the saver branch on format and nothing above them does. +- Neither info file carries a key id of its own, since both are encrypted with `repo-cipher-pass` and the map holds only the sub-keys for dependent files. Their header stays eight bytes with the `_` marker, so replacing those bytes with `Salted__` still hands the file to `openssl enc` for a user who wants to read it by hand. A format 6 info file needs `-md sha256` where format 5 needed `-md sha1`. WAL is not worth the same treatment: between the id in front and the key map there is no realistic manual path, and there never was much of one. + +## Backup Set Key + +- Format 6 stores one key per backup set in `backup.info`, keyed by the full backup label, and it encrypts both the set's manifests and its data files. Every full backup generates a fresh entry, so rotation is automatic and needs no option, marker, or age tracking. +- This merges two format 5 levels. The split had no security value, since the data key lives inside a manifest encrypted with the manifest key, so holding the manifest key already unlocked the data transitively. Once the manifest key is per set they are the same key. +- Format 6 manifests drop the `[cipher]` sub-passphrase. Format 5 manifests keep theirs and are read according to their stored format. +- Lookup is exact. A diff or incr label carries its full's label as the prefix before `_`, so the key for any manifest or data file, current or under `backup.history`, comes from parsing the label. No boundary comparison exists anywhere. The prior-manifest inheritance at `src/command/backup/incr.c.inc:149` is no longer needed for format 6 sets, though the format inheritance beside it stays. +- Keying by label rather than by a sequential id is deliberate. `repo-retention-history` keeps manifests under `backup.history` after their entry leaves `[backup:current]`, so an id recorded against the backup entry would disappear while the manifest it unlocks survives. The label is in the path either way. +- The entry is recorded when the label is assigned, before the first manifest copy is saved, so a resumed backup finds its key by the normal lookup with no special case. An entry left by an aborted full is inert and is pruned once no manifest references it. +- Every data key consumer already holds the manifest and reaches the key through `manifestCipherSpec()`. At format 6 that getter is filled in from the `backup.info` entry once the manifest loads, so verify, restore, expire, `repo get`, resume, incr, and `backup/db` are unchanged. +- A migrated stanza keeps its format 5 stanza-wide manifest key as id 0 until the last format 5 manifest expires. Format 5 sets keep reading their data keys from their manifests as always. + +## Archive Key + +- `archive.info` holds a map of keys. One is current and encrypts new WAL; the rest are retained only so WAL already written can still be read. +- Key ids are sequential from 1 and are never reused. Which one is current is recorded in `cipher-pass-current` beside the map rather than inferred, since the map is keyed by an opaque id and sorts as text, so the highest id is not the last entry once there are ten of them. +- Only the most recent rotation is dated, in a `cipher-pass-rotate` key beside the map, as seconds since the epoch. Rotation compares the current key's age and nothing looks at when a retired key was made, so dating every entry would store what nothing reads. +- Rotation appends a key and makes it current. No existing file is rewritten and nothing else in the repository moves. + +### Key id in the file + +- The eighth byte of the format header says whether a key id follows. `_` means nothing follows, which is what the info files write since they are encrypted with the user passphrase and there is only ever one of those. `K` means a key id follows immediately: one byte of length, then that many bytes of id, then the salt. +- The id is stored as bytes rather than as a number, so it is whatever the map is keyed by and needs no conversion at either end. Archive ids happen to be decimal text today, but an id that is not a number, e.g. a backup label, costs nothing to support later. +- A single length byte allows 255 bytes of id, which is ample: a backup label is under 40 and a decimal id would have to pass 10^255 to overflow. +- The reader therefore holds back a variable number of bytes rather than a fixed eight. `cipherBlockFormatProcess()` reads the fixed header, then the length byte when the marker is `K`, then that many bytes, growing its held-back size as each part arrives. +- WAL already carries the `Salted__` magic, since `src/command/archive/push/file.c:273` encrypts with the default header, so the fixed part of the header replaces it for free and only the length byte and the id are new. The files that omit the magic to save space are bundled and block-incremental backup files, and those are keyed by label, so they keep no header at all. +- The key id in the file is what makes rotation need no coordination with running archivers. A pusher still holding a previous `archive.info` keeps writing under the previous key, and the file records which key that was, so a reader finds it without guessing. There is no boundary to compare against and no trial decryption anywhere in the design. + +### Reading + +- archive-get caches one cipher spec per repo (`src/command/archive/get/get.c:419`) and packs it into the per-file protocol payload (`src/command/archive/get/get.c:927`). The id is not known until the file is open, so the key map goes down to the local process instead and `CipherBlockFormat` selects from it. verify (`src/command/verify/verify.c:859`) and restore's timeline read (`src/command/restore/timeline.c:97`) change the same way. +- The payload is per file, so the whole map repeats for every segment requested. That is accepted: there is no easy way to move the payload to job level today, and a healthy repository has only a few active keys. Trimming the map by some heuristic before packing it is a later option if it ever matters. +- Pruning at expire is what keeps the map small. An unpruned entry is only wasted bytes, so pruning is never required for correctness. +- Archive-id handling is unaffected. archive-push writes only to the current archive-id and archive-get already searches every archive-id matching the current version and system-id (`src/command/archive/get/get.c:430`), and neither cares which key a file used. + +### Rotation schedule + +- A new option, `repo-cipher-rotate`, gives the rotation period as a time and defaults to 90 days, with a minimum of 15 days. Setting it to the max value effectively disables rotation. It is repo indexed and applies only to the archive key: the backup set key rotates with every full backup, and the user passphrase is user-supplied so it cannot rotate on its own. +- Expire performs the rotation, since it writes `archive.info` on every run and runs regularly, typically daily through backup's expire phase. When the current key is older than the period, expire appends a new key and makes it current. Rotation latency is bounded by expire cadence, which is negligible against a period measured in weeks. If expire never runs then neither does rotation. +- A dry run logs the rotation but does not save it. At format 5 the option is ignored, with a warning when it is not the default, since there is only the one key. +- `stanza-create` at format 6 and a migration to format 6 both add their key through rotation, which assigns the next id and sets `cipher-pass-rotate`, so the clock starts there. The next id is the current id plus one, or 1 when there is no current key. + +## Key Pruning at Expire + +- Archive: a key is dropped once no WAL that references it remains. Expire already removes WAL, so it knows when the last file for a key is gone. The current key is never dropped. +- Backup set: an entry serves exactly one set, so it is dropped when the last manifest of that set is removed, current or under `backup.history`. Expire performs both removals. The entry for the newest set is never dropped. +- Both happen in expire's existing info file rewrites under the backup lock, so no additional locking is required. + +## Info File Write Coordination + +- Every info file writer, backup and expire for `backup.info`, expire for `archive.info`, and the stanza commands for both, requires the backup lock and sets `lock-remote-required` (`build/config.yaml`), so the repository host's lock directory is the single serialization point regardless of which host each command runs on. +- archive-push and archive-get never write `archive.info`, so rotation needs no coordination with them beyond the atomic info save. A reader sees the old file or the new one and both are valid, per the running-archiver rule above. +- Backup used to resave `archive.info` at completion only to freshen its timestamp so object-store lifecycle settings do not remove it early. That resave moved to expire, which saves `archive.info` on every run that is not a dry run, so backup no longer writes `archive.info`. With `expire-auto` disabled the timestamp is freshened only when expire is run separately. +- With directly-reached storage (object stores, shared NFS) and writers invoked from several hosts there is no common lock point. This predates this work and applies to backup and expire today. The documented practice remains to run backup and expire against a given repo from one place. + +## Later Commits + +- User passphrase rotation. `repo-cipher-pass` wraps only the info files, so rotating it rewrites four small objects (`backup.info`, `archive.info`, and their copies) and never touches bulk data. The plan is a config-first flow: set `repo-cipher-pass` to the new passphrase and `repo-cipher-pass-old` to the old one, then run a command that converges each info file. Every command retries an info file load with the old passphrase and warns when the fallback is used, so config can be updated across hosts non-atomically. +- A `stanza-rekey` command to drive that convergence and to force an archive key rotation on demand. It is repo-only by design, with no cluster access, so keys can be rotated during incident response with the cluster down or unreachable. That is why it is not a flag on `stanza-upgrade`, which requires cluster access for `pgValidate()`. + +## Documentation Plan + +- The features per format version table in the `repo-format` option reference is a stub today. This work fills in the format 6 entries. +- The `repo-cipher-rotate` option reference, and the `stanza-rekey` command reference when that lands. +- Release notes announce archive key rotation and the per backup set key. diff --git a/test/src/harness/info.c b/test/src/harness/info.c index 4a120dcc1..a11f65756 100644 --- a/test/src/harness/info.c +++ b/test/src/harness/info.c @@ -152,7 +152,7 @@ hrnInfoPut(const Storage *const storage, const char *const file, const char *con // If a file with a header if (param.header) { - cipherBlockFormatFilterGroupWriteAdd(encrypted, ioWriteFilterGroup(write), param.cipherSpec, param.format); + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(write), param.cipherSpec, param.format); } // Else a file with no header of its own, e.g. a manifest, which is still stored with the digest of its format else @@ -195,7 +195,7 @@ harnessInfoEncrypt(const Buffer *const content, const CipherSpec *const cipherSp if (param.format == 0) cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec); else - cipherBlockFormatFilterGroupWriteAdd(result, ioWriteFilterGroup(write), cipherSpec, param.format); + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(write), cipherSpec, param.format); ioWriteOpen(write); ioWrite(write, content); diff --git a/test/src/module/command/archive/getTest.c b/test/src/module/command/archive/getTest.c index 2abc1cc56..56d649a6f 100644 --- a/test/src/module/command/archive/getTest.c +++ b/test/src/module/command/archive/getTest.c @@ -1,6 +1,7 @@ /*********************************************************************************************************************************** Test Archive Get Command ***********************************************************************************************************************************/ +#include "common/io/bufferWrite.h" #include "common/io/fdRead.h" #include "common/io/fdWrite.h" @@ -1095,6 +1096,60 @@ testRun(void) TEST_RESULT_UINT(storageInfoP(storagePg(), STRDEF("pg_wal/RECOVERYXLOG")).size, 16 * 1024 * 1024, "check size"); TEST_STORAGE_LIST(storagePgWrite(), "pg_wal", "RECOVERYXLOG\n", .remove = true); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("get encrypted format 6 WAL segment"); + + HRN_INFO_PUT( + storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE, + "[cipher]\n" + "cipher-pass={\"1\":{\"digest\":\"sha256\",\"key\":\"" TEST_CIPHER_PASS_ARCHIVE "\"}}\n" + "cipher-pass-current=\"1\"\n" + "\n" + "[db]\n" + "db-id=1\n" + "\n" + "[db:history]\n" + "1={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}", + .format = REPOSITORY_FORMAT_6, .header = true, .cipherSpec = TEST_CIPHER_SPEC); + + // Store the WAL as format 6 with the key id in the header + Buffer *const walFormat6 = bufNew(0); + IoWrite *const walFormat6Write = ioBufferWriteNew(walFormat6); + + ioFilterGroupAdd(ioWriteFilterGroup(walFormat6Write), compressFilterP(compressTypeGz, 1)); + cipherBlockFormatFilterGroupWriteAddP( + ioWriteFilterGroup(walFormat6Write), TEST_CIPHER_SPEC_PASS(TEST_CIPHER_PASS_ARCHIVE), REPOSITORY_FORMAT_6, + .keyId = STRDEF("1")); + + ioWriteOpen(walFormat6Write); + ioWrite(walFormat6Write, buffer); + ioWriteClose(walFormat6Write); + + TEST_RESULT_STR_Z(strNewZN((const char *)bufPtrConst(walFormat6), 8), "PGBR006K", "header contains a key id"); + + HRN_STORAGE_PUT( + storageRepoIdxWrite(1), + STORAGE_REPO_ARCHIVE "/10-1/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz", walFormat6); + + TEST_RESULT_INT(cmdArchiveGet(), 0, "get"); + + TEST_RESULT_LOG( + "P00 WARN: repo1: [FileMissingError] unable to load info file '" TEST_PATH "/repo-bogus/archive/test1/archive.info'" + " or '" TEST_PATH "/repo-bogus/archive/test1/archive.info.copy':\n" + " FileMissingError: unable to open missing file '" TEST_PATH "/repo-bogus/archive/test1/archive.info'" + " for read\n" + " FileMissingError: unable to open missing file '" TEST_PATH "/repo-bogus/archive/test1/archive.info.copy'" + " for read\n" + " HINT: archive.info cannot be opened but is required to push/get WAL segments.\n" + " HINT: is archive_command configured correctly in postgresql.conf?\n" + " HINT: has a stanza-create been performed?\n" + " HINT: use --no-archive-check to disable archive checks during backup if you have an alternate" + " archiving scheme.\n" + "P00 INFO: found 01ABCDEF01ABCDEF01ABCDEF in the repo2: 10-1 archive"); + + TEST_RESULT_UINT(storageInfoP(storagePg(), STRDEF("pg_wal/RECOVERYXLOG")).size, 16 * 1024 * 1024, "check size"); + TEST_STORAGE_LIST(storagePgWrite(), "pg_wal", "RECOVERYXLOG\n", .remove = true); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("repo1 has info but bad permissions"); diff --git a/test/src/module/command/archive/pushTest.c b/test/src/module/command/archive/pushTest.c index d3b1a5d16..4fb394e39 100644 --- a/test/src/module/command/archive/pushTest.c +++ b/test/src/module/command/archive/pushTest.c @@ -567,6 +567,39 @@ testRun(void) storageTest, zNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1), .remove = true, .comment = "check repo3 for WAL file then remove"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("encrypted format 6 WAL segment begins with the format header"); + + HRN_INFO_PUT( + storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE, + "[cipher]\n" + "cipher-pass={\"1\":{\"digest\":\"sha256\",\"key\":\"badsubpassphrase\"}}\n" + "cipher-pass-current=\"1\"\n" + "\n" + "[db]\n" + "db-id=1\n" + "\n" + "[db:history]\n" + "1={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}", + .format = REPOSITORY_FORMAT_6, .header = true, .cipherSpec = TEST_CIPHER_SPEC_PASS("badpassphrase")); + + TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment"); + TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000002' to the archive"); + + const Buffer *const walFormat6 = storageGetP( + storageNewReadP( + storageTest, + strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1))); + + TEST_RESULT_STR_Z(strNewZN((const char *)bufPtrConst(walFormat6), 8), "PGBR006K", "header contains the format"); + TEST_RESULT_UINT(bufPtrConst(walFormat6)[8], 1, "key id is one byte"); + TEST_RESULT_STR_Z(strNewZN((const char *)bufPtrConst(walFormat6) + 9, 1), "1", "header contains the key id"); + + HRN_STORAGE_REMOVE( + storageTest, zNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)); + HRN_STORAGE_REMOVE( + storageTest, zNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("write error on one repo but other repo succeeds"); diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index 57bcf24b7..ed5fb9402 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -2798,27 +2798,11 @@ testRun(void) // Remove halted backup so there's no resume HRN_STORAGE_PATH_REMOVE(storageRepoWrite(), STORAGE_REPO_BACKUP "/20191016-042640F", .recurse = true); - // Set archive.info/copy to an older timestamp so we can be sure it was updated as part of backup - time_t archiveInfoOldTimestamp = 967746268; - HRN_STORAGE_TIME(storageRepo(), INFO_ARCHIVE_PATH_FILE, archiveInfoOldTimestamp); - HRN_STORAGE_TIME(storageRepo(), INFO_ARCHIVE_PATH_FILE_COPY, archiveInfoOldTimestamp); - - // Get a copy of archive.info - const String *archiveInfoContent = strNewBuf(storageGetP(storageNewReadP(storageRepo(), INFO_ARCHIVE_PATH_FILE_STR))); - // Run backup hrnBackupPqScriptP( PG_VERSION_96, backupTimeStart, .backupStandby = true, .walCompressType = compressTypeGz, .startFast = true); TEST_RESULT_VOID(hrnCmdBackup(), "backup"); - // Check archive.info/copy timestamp was updated but contents were not - TEST_RESULT_INT_NE( - storageInfoP(storageRepo(), INFO_ARCHIVE_PATH_FILE_STR).timeModified, archiveInfoOldTimestamp, "time updated"); - TEST_STORAGE_GET(storageRepo(), INFO_ARCHIVE_PATH_FILE, strZ(archiveInfoContent)); - TEST_RESULT_INT_NE( - storageInfoP(storageRepo(), INFO_ARCHIVE_PATH_FILE_COPY_STR).timeModified, archiveInfoOldTimestamp, "time updated"); - TEST_STORAGE_GET(storageRepo(), INFO_ARCHIVE_PATH_FILE_COPY, strZ(archiveInfoContent)); - // Set log level back to detail harnessLogLevelSet(logLevelDetail); diff --git a/test/src/module/command/expireTest.c b/test/src/module/command/expireTest.c index 2926ccf27..98e3b7330 100644 --- a/test/src/module/command/expireTest.c +++ b/test/src/module/command/expireTest.c @@ -1110,11 +1110,19 @@ testRun(void) hrnCfgArgRawZ(argList, cfgOptRepo, "1"); HRN_CFG_LOAD(cfgCmdExpire, argList); + // Set archive.info to an older timestamp so we can be sure it was updated as part of expire + const time_t archiveInfoOldTimestamp = 967746268; + HRN_STORAGE_TIME(storageRepo(), INFO_ARCHIVE_PATH_FILE, archiveInfoOldTimestamp); + TEST_RESULT_VOID(cmdExpire(), "expire last backup in archive sub path and remove sub path"); TEST_RESULT_BOOL( storagePathExistsP(storageRepo(), STRDEF(STORAGE_REPO_ARCHIVE "/9.4-1/0000000100000000")), false, "archive sub path removed repo1"); + // Check archive.info timestamp was updated + TEST_RESULT_INT_NE( + storageInfoP(storageRepo(), INFO_ARCHIVE_PATH_FILE_STR).timeModified, archiveInfoOldTimestamp, "time updated"); + TEST_RESULT_LOG( "P00 INFO: repo1: expire full backup 20181119-152138F\n" "P00 INFO: repo1: remove expired backup 20181119-152138F\n" @@ -1145,10 +1153,13 @@ testRun(void) storageTest, TEST_PATH "/repo/backup/db/" INFO_BACKUP_FILE INFO_COPY_EXT, TEST_PATH "/repo/backup/db/" INFO_BACKUP_FILE INFO_COPY_EXT ".save"); - // Rename archive.info file on repo2 to cause error + // Rename archive.info files on repo2 to cause error HRN_STORAGE_MOVE( storageTest, TEST_PATH "/repo2/archive/db/" INFO_ARCHIVE_FILE, TEST_PATH "/repo2/archive/db/" INFO_ARCHIVE_FILE ".save"); + HRN_STORAGE_MOVE( + storageTest, TEST_PATH "/repo2/archive/db/" INFO_ARCHIVE_FILE INFO_COPY_EXT, + TEST_PATH "/repo2/archive/db/" INFO_ARCHIVE_FILE INFO_COPY_EXT ".save"); // Configure dry-run argList2 = strLstDup(argList); @@ -1179,8 +1190,11 @@ testRun(void) " HINT: use --no-archive-check to disable archive checks during backup if you have an alternate" " archiving scheme."); - // Restore saved archive.info file + // Restore saved archive.info files HRN_STORAGE_MOVE(storageTest, "repo2/archive/db/" INFO_ARCHIVE_FILE ".save", "repo2/archive/db/" INFO_ARCHIVE_FILE); + HRN_STORAGE_MOVE( + storageTest, "repo2/archive/db/" INFO_ARCHIVE_FILE INFO_COPY_EXT ".save", + "repo2/archive/db/" INFO_ARCHIVE_FILE INFO_COPY_EXT); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("expire command - multi-repo, continue to next repo after error"); @@ -1220,10 +1234,10 @@ testRun(void) TEST_RESULT_VOID(cmdExpire(), "expire (dry-run) - log expired backups and archive path to remove"); TEST_STORAGE_LIST( - storageRepo(), STORAGE_REPO_ARCHIVE, "10-2/\n9.4-1/\narchive.info\n", .noRecurse = true, + storageRepo(), STORAGE_REPO_ARCHIVE, "10-2/\n9.4-1/\narchive.info\narchive.info.copy\n", .noRecurse = true, .comment = "repo1: 9.4-1 archive path not removed"); TEST_STORAGE_LIST( - storageRepoIdx(1), STORAGE_REPO_ARCHIVE, "10-2/\n9.4-1/\narchive.info\n", .noRecurse = true, + storageRepoIdx(1), STORAGE_REPO_ARCHIVE, "10-2/\n9.4-1/\narchive.info\narchive.info.copy\n", .noRecurse = true, .comment = "repo2: 9.4-1 archive path not removed"); TEST_STORAGE_LIST( storageRepoIdx(1), STORAGE_REPO_ARCHIVE "/9.4-1/", @@ -1289,10 +1303,10 @@ testRun(void) TEST_RESULT_VOID(cmdExpire(), "expire backups and remove archive path"); TEST_STORAGE_LIST( - storageRepo(), STORAGE_REPO_ARCHIVE, "10-2/\narchive.info\n", .noRecurse = true, + storageRepo(), STORAGE_REPO_ARCHIVE, "10-2/\narchive.info\narchive.info.copy\n", .noRecurse = true, .comment = "repo1: retention-archive-type=full so 9.4-1 archive path removed"); TEST_STORAGE_LIST( - storageRepoIdx(1), STORAGE_REPO_ARCHIVE, "10-2/\n9.4-1/\narchive.info\n", .noRecurse = true, + storageRepoIdx(1), STORAGE_REPO_ARCHIVE, "10-2/\n9.4-1/\narchive.info\narchive.info.copy\n", .noRecurse = true, .comment = "repo2: retention-archive-type=diff so 9.4-1 archive path not removed"); TEST_STORAGE_LIST( storageRepoIdx(1), STORAGE_REPO_ARCHIVE "/9.4-1/", @@ -2710,6 +2724,104 @@ testRun(void) storageInfoP(storageRepoIdx(1), STRDEF(STORAGE_REPO_BACKUP "/latest")).linkDestination, STRDEF("20181119-152850F"), "latest link updated, repo2"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("archive key rotation set at format 5"); + + argList = strLstDup(argListAvoidWarn); + hrnCfgArgRawZ(argList, cfgOptRepo, "2"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 2, TEST_PATH "/repo2"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 2, "1"); + hrnCfgArgKeyRawStrId(argList, cfgOptRepoCipherType, 2, cipherTypeAes256Cbc); + hrnCfgArgKeyRawZ(argList, cfgOptRepoCipherRotate, 2, "30d"); + HRN_CFG_LOAD(cfgCmdExpire, argList); + + TEST_RESULT_VOID(cmdExpire(), "expire"); + TEST_RESULT_LOG( + "P00 DETAIL: repo2: 12-2 archive retention on backup 20181119-152850F, start = 000000010000000000000002\n" + "P00 INFO: repo2: 12-2 no archive to remove\n" + "P00 WARN: option 'repo2-cipher-rotate' requires repository format 6 or higher"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("archive key not due for rotation"); + + const time_t timeRotate = time(NULL); + + #define TEST_ARCHIVE_INFO_FORMAT_6(rotateTime) \ + zNewFmt( \ + "[cipher]\n" \ + "cipher-pass={\"1\":{\"digest\":\"sha256\",\"key\":\"" TEST_CIPHER_PASS_ARCHIVE "\"}}\n" \ + "cipher-pass-current=\"1\"\n" \ + "cipher-pass-rotate=%" PRId64 "\n" \ + "\n" \ + "[db]\n" \ + "db-id=2\n" \ + "db-system-id=6626363367545678089\n" \ + "db-version=\"12\"\n" \ + "\n" \ + "[db:history]\n" \ + "1={\"db-id\":6625592122879095702,\"db-version\":\"9.4\"}\n" \ + "2={\"db-id\":6626363367545678089,\"db-version\":\"12\"}", \ + (int64_t)(rotateTime)) + + HRN_INFO_PUT( + storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_FORMAT_6(timeRotate - 29 * SEC_PER_DAY), + .format = REPOSITORY_FORMAT_6, .header = true, .cipherSpec = TEST_CIPHER_SPEC); + + TEST_RESULT_VOID(cmdExpire(), "expire"); + TEST_RESULT_LOG( + "P00 DETAIL: repo2: 12-2 archive retention on backup 20181119-152850F, start = 000000010000000000000002\n" + "P00 INFO: repo2: 12-2 no archive to remove"); + + InfoArchive *infoArchive = NULL; + + TEST_ASSIGN( + infoArchive, infoArchiveLoadFile(storageRepoIdx(1), INFO_ARCHIVE_PATH_FILE_STR, TEST_CIPHER_SPEC), "load archive info"); + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(infoArchiveCipherSpecMap(infoArchive)), "1", "current key not rotated"); + TEST_RESULT_INT(infoArchiveCipherRotateTime(infoArchive), timeRotate - 29 * SEC_PER_DAY, "rotation time unchanged"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("archive key due for rotation, dry run"); + + HRN_INFO_PUT( + storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_FORMAT_6(timeRotate - 31 * SEC_PER_DAY), + .format = REPOSITORY_FORMAT_6, .header = true, .cipherSpec = TEST_CIPHER_SPEC); + + StringList *argListDryRun = strLstDup(argList); + hrnCfgArgRawBool(argListDryRun, cfgOptDryRun, true); + HRN_CFG_LOAD(cfgCmdExpire, argListDryRun); + + TEST_RESULT_VOID(cmdExpire(), "expire"); + TEST_RESULT_LOG( + "P00 DETAIL: [DRY-RUN] repo2: 12-2 archive retention on backup 20181119-152850F, start = 000000010000000000000002\n" + "P00 INFO: [DRY-RUN] repo2: 12-2 no archive to remove\n" + "P00 INFO: [DRY-RUN] repo2: rotate archive key to id 2"); + + TEST_ASSIGN( + infoArchive, infoArchiveLoadFile(storageRepoIdx(1), INFO_ARCHIVE_PATH_FILE_STR, TEST_CIPHER_SPEC), "load archive info"); + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(infoArchiveCipherSpecMap(infoArchive)), "1", "current key not rotated"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("archive key due for rotation"); + + HRN_CFG_LOAD(cfgCmdExpire, argList); + + TEST_RESULT_VOID(cmdExpire(), "expire"); + TEST_RESULT_LOG( + "P00 DETAIL: repo2: 12-2 archive retention on backup 20181119-152850F, start = 000000010000000000000002\n" + "P00 INFO: repo2: 12-2 no archive to remove\n" + "P00 INFO: repo2: rotate archive key to id 2"); + + TEST_ASSIGN( + infoArchive, infoArchiveLoadFile(storageRepoIdx(1), INFO_ARCHIVE_PATH_FILE_STR, TEST_CIPHER_SPEC), "load archive info"); + + const CipherSpecMap *const cipherSpecMapRotate = infoArchiveCipherSpecMap(infoArchive); + + TEST_RESULT_UINT(cipherSpecMapSize(cipherSpecMapRotate), 2, "prior key kept"); + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(cipherSpecMapRotate), "2", "new key is current"); + TEST_RESULT_UINT( + cipherSpecDigest(cipherSpecMapGet(cipherSpecMapRotate, STRDEF("2"))), hashTypeSha256, "new key derives with sha256"); + TEST_RESULT_BOOL(infoArchiveCipherRotateTime(infoArchive) >= timeRotate, true, "rotation time updated"); + // Cleanup hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 2); harnessLogLevelReset(); @@ -3061,6 +3173,17 @@ testRun(void) "1={\"db-catalog-version\":202506291,\"db-control-version\":1800,\"db-system-id\":7577015877005525116" ",\"db-version\":\"18\"}\n"); + // Create archive info + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, + "[db]\n" + "db-id=1\n" + "db-system-id=7577015877005525116\n" + "db-version=\"18\"\n" + "\n" + "[db:history]\n" + "1={\"db-id\":7577015877005525116,\"db-version\":\"18\"}"); + TEST_RESULT_VOID(cmdExpire(), "no backups to expire using --oldest"); TEST_RESULT_LOG( "P00 WARN: repo1: expire oldest requested but no eligible full backup to expire"); @@ -3183,17 +3306,6 @@ testRun(void) HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), STORAGE_REPO_BACKUP "/20251127-101503F/" BACKUP_MANIFEST_FILE); HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), STORAGE_REPO_BACKUP "/20251127-101503F_20251127-101508D/" BACKUP_MANIFEST_FILE); - // Create archive info - HRN_INFO_PUT( - storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, - "[db]\n" - "db-id=1\n" - "db-system-id=7577015877005525116\n" - "db-version=\"18\"\n" - "\n" - "[db:history]\n" - "1={\"db-id\":7577015877005525116,\"db-version\":\"18\"}"); - // Create archive directories and generate archive archiveGenerate(storageRepoWrite(), STORAGE_REPO_ARCHIVE, 1, 32, "18-1", "0000000100000000"); diff --git a/test/src/module/command/repoTest.c b/test/src/module/command/repoTest.c index a71d7e5d1..c4dd35423 100644 --- a/test/src/module/command/repoTest.c +++ b/test/src/module/command/repoTest.c @@ -1,6 +1,7 @@ /*********************************************************************************************************************************** Test Repo Commands ***********************************************************************************************************************************/ +#include "common/format/cipherBlockFormat.h" #include "common/io/bufferRead.h" #include "common/io/bufferWrite.h" #include "storage/posix/storage.h" @@ -620,6 +621,51 @@ testRun(void) TEST_RESULT_INT(storageGetProcess(ioBufferWriteNew(writeBuffer)), 0, "get"); TEST_RESULT_BOOL(bufEq(writeBuffer, archiveFileBuffer), true, "get matches put"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("get encrypted format 6 WAL archive file"); + + // Format 6 archive.info with the migrated key at id 0 and the current key at id 1 + HRN_INFO_PUT( + storageFixture, STORAGE_PATH_ARCHIVE "/test/" INFO_ARCHIVE_FILE, + "[cipher]\n" + "cipher-pass={\"0\":{\"digest\":\"sha1\",\"key\":\"custom\"},\"1\":{\"digest\":\"sha256\",\"key\":\"custom6\"}}\n" + "cipher-pass-current=\"1\"\n" + "\n" + "[db]\n" + "db-id=1\n" + "db-system-id=6846378200844646865\n" + "db-version=\"12\"\n" + "\n" + "[db:history]\n" + "1={\"db-id\":6846378200844646865,\"db-version\":\"12\"}", + .format = REPOSITORY_FORMAT_6, .header = true, .cipherSpec = TEST_CIPHER_SPEC); + + // Store the WAL as format 6 with the key id in the header + Buffer *const archiveFileBuffer6 = bufNew(0); + IoWrite *const archiveFileWrite6 = ioBufferWriteNew(archiveFileBuffer6); + + cipherBlockFormatFilterGroupWriteAddP( + ioWriteFilterGroup(archiveFileWrite6), TEST_CIPHER_SPEC_PASS("custom6"), REPOSITORY_FORMAT_6, .keyId = STRDEF("1")); + ioWriteOpen(archiveFileWrite6); + ioWrite(archiveFileWrite6, archiveFileBuffer); + ioWriteClose(archiveFileWrite6); + + HRN_STORAGE_PUT( + storageFixture, STORAGE_PATH_ARCHIVE "/test/12-1/000000010000000100000002-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + archiveFileBuffer6); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo"); + hrnCfgArgRawStrId(argList, cfgOptRepoCipherType, cipherTypeAes256Cbc); + strLstAddFmt( + argList, "%s/repo/" STORAGE_PATH_ARCHIVE "/test/12-1/000000010000000100000002-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + TEST_PATH); + HRN_CFG_LOAD(cfgCmdRepoGet, argList); + + writeBuffer = bufNew(0); + TEST_RESULT_INT(storageGetProcess(ioBufferWriteNew(writeBuffer)), 0, "get"); + TEST_RESULT_BOOL(bufEq(writeBuffer, archiveFileBuffer), true, "get matches put"); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("get encrypted backup.manifest"); diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index 7eb292318..7c39d1cbd 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -2072,44 +2072,44 @@ testRun(void) TEST_RESULT_VOID( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_11, 1, 0xA1, NULL, CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), "follow current timeline because of version"); TEST_RESULT_VOID( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_11, 1, 0xA1, STRDEF("latest"), CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), "follow latest timeline as requested"); TEST_RESULT_VOID( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 1, 0xA1, NULL, CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), "follow latest timeline because of version"); TEST_RESULT_VOID( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 1, 0xA1, STRDEF("current"), CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), "follow current timeline as requested"); TEST_RESULT_VOID( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 1, 0xA1, STRDEF("1"), CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), "follow requested timeline (same as current)"); TEST_RESULT_VOID( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 0x10, 0xA1, STRDEF("0x10"), CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), "follow requested hex timeline (same as current)"); TEST_ERROR( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 0x10, 0xA1, STRDEF("bogus"), CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), DbMismatchError, "invalid target timeline 'bogus'"); HRN_STORAGE_PUT_Z(storageTest, "repo/archive/test1/17-1/00000009.history", "8"); TEST_ERROR( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 8, 0xA1, STRDEF("9"), CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), FormatError, "invalid timeline '9' at '" TEST_PATH "/repo/archive/test1/17-1/00000009.history':" " invalid history line format '8'"); @@ -2122,27 +2122,27 @@ testRun(void) TEST_RESULT_VOID( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 10, 0x4FFFFFF, NULL, CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), "follow current timeline"); TEST_RESULT_VOID( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 9, 0x4FFFFFF, NULL, CFGOPTVAL_RESTORE_TYPE_IMMEDIATE, - cipherSpecNewNone()), + cipherSpecMapNew()), "follow current timeline (based on type immediate)"); TEST_RESULT_VOID( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 9, 0x4FFFFFF, NULL, CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), "follow latest timeline"); TEST_RESULT_VOID( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 9, 0x4FFFFFF, STRDEF("10"), CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), "target timeline found"); TEST_ERROR( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 9, 0x6000000, STRDEF("10"), CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), DbMismatchError, "target timeline A forked from backup timeline 9 at 0/5000000 which is before backup lsn of 0/6000000\n" "HINT: was the target timeline created by accidentally promoting a standby?\n" @@ -2156,7 +2156,7 @@ testRun(void) TEST_ERROR( timelineVerify( storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 6, 0x4FFFFFF, STRDEF("11"), CFGOPTVAL_RESTORE_TYPE_DEFAULT, - cipherSpecNewNone()), + cipherSpecMapNew()), DbMismatchError, "backup timeline 6, lsn 0/4ffffff is not in the history of target timeline B\n" "HINT: was the target timeline created by promoting from a timeline < latest?"); } @@ -2334,9 +2334,13 @@ testRun(void) TEST_CIPHER_PASS_MANIFEST "\"\n\n" TEST_RESTORE_BACKUP_INFO_DB, .cipherSpec = TEST_CIPHER_SPEC); // Write archive.info to the encrypted repo - InfoArchive *infoArchive = infoArchiveNew( - PG_VERSION_11, 6569239123849665679, REPOSITORY_FORMAT_DEFAULT, + CipherSpecMap *const cipherSpecMapArchive = cipherSpecMapNew(); + cipherSpecMapAdd( + cipherSpecMapArchive, CIPHER_SPEC_MAP_ID_DEFAULT_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE), .digest = hashTypeSha1)); + + InfoArchive *infoArchive = infoArchiveNew( + PG_VERSION_11, 6569239123849665679, REPOSITORY_FORMAT_DEFAULT, cipherSpecMapArchive); infoArchiveSaveFile( infoArchive, storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS))); diff --git a/test/src/module/command/stanzaTest.c b/test/src/module/command/stanzaTest.c index bfa2a287a..602f85d4c 100644 --- a/test/src/module/command/stanzaTest.c +++ b/test/src/module/command/stanzaTest.c @@ -125,9 +125,12 @@ testRun(void) hrnCfgArgKeyRawZ(argList, cfgOptRepoFormat, 1, "6"); hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 4, TEST_PATH "/repo4"); hrnCfgArgKeyRawStrId(argList, cfgOptRepoCipherType, 4, cipherTypeAes256Cbc); + hrnCfgArgKeyRawZ(argList, cfgOptRepoFormat, 4, "6"); hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 4, "87654321"); HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); + const time_t timeCreate = time(NULL); + TEST_RESULT_VOID(cmdStanzaCreate(), "stanza create - files already exist on repo1 and both are valid"); TEST_RESULT_LOG( "P00 INFO: stanza-create for stanza 'db' on repo1\n" @@ -184,13 +187,27 @@ testRun(void) TEST_RESULT_UINT( infoArchiveFormat(infoArchive), REPOSITORY_FORMAT_DEFAULT, "archive info on existing stanza still at default format"); - // Confirm other repo encrypted with different password + // Confirm other repo encrypted with different password. Format 6 stores the archive key at the first key id. TEST_ASSIGN( infoArchive, infoArchiveLoadFile( storageRepoIdx(3), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), "load archive info from encrypted repo4"); - TEST_RESULT_UINT(cipherSpecType(infoArchiveCipherSpec(infoArchive)), cipherTypeAes256Cbc, "cipher sub set"); + + const CipherSpecMap *const cipherSpecMapCreate = infoArchiveCipherSpecMap(infoArchive); + + TEST_RESULT_UINT(cipherSpecMapSize(cipherSpecMapCreate), 1, "one archive key"); + TEST_RESULT_STR_Z(cipherSpecMapGetIdx(cipherSpecMapCreate, 0)->id, "1", "archive key stored at id 1"); + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(cipherSpecMapCreate), "1", "archive key at id 1 is current"); + TEST_RESULT_BOOL( + infoArchiveCipherRotateTime(infoArchive) >= timeCreate && + infoArchiveCipherRotateTime(infoArchive) <= time(NULL), + true, "rotation time is the create time"); + TEST_RESULT_UINT( + cipherSpecDigest(cipherSpecMapGet(cipherSpecMapCreate, STRDEF("1"))), hashTypeSha256, + "archive key derives with sha256"); + TEST_RESULT_UINT( + bufSize(cipherSpecPass(cipherSpecMapGet(cipherSpecMapCreate, STRDEF("1")))), 64, "archive key is 64 characters"); TEST_ASSIGN( infoBackup, @@ -1233,6 +1250,8 @@ testRun(void) hrnCfgArgKeyRawZ(argList, cfgOptRepoFormat, 1, "6"); HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + const time_t timeMigrate = time(NULL); + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - format 6 on encrypted repo"); TEST_RESULT_LOG( "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" @@ -1245,6 +1264,22 @@ testRun(void) "load archive info after migration"); TEST_RESULT_UINT(infoArchiveFormat(infoArchiveMigrate), REPOSITORY_FORMAT_6, "archive info at format 6"); + // The migrated key stays at id 0 and a key for new WAL is added at id 1 + const CipherSpecMap *const cipherSpecMapMigrate = infoArchiveCipherSpecMap(infoArchiveMigrate); + + TEST_RESULT_UINT(cipherSpecMapSize(cipherSpecMapMigrate), 2, "migrated and new archive key"); + TEST_RESULT_UINT( + cipherSpecDigest(cipherSpecMapGet(cipherSpecMapMigrate, CIPHER_SPEC_MAP_ID_DEFAULT_STR)), hashTypeSha1, + "migrated key derives with sha1"); + TEST_RESULT_UINT( + cipherSpecDigest(cipherSpecMapGet(cipherSpecMapMigrate, STRDEF("1"))), hashTypeSha256, + "new key derives with sha256"); + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(cipherSpecMapMigrate), "1", "new key is current"); + TEST_RESULT_BOOL( + infoArchiveCipherRotateTime(infoArchiveMigrate) >= timeMigrate && + infoArchiveCipherRotateTime(infoArchiveMigrate) <= time(NULL), + true, "rotation time is the migration time"); + StorageRead *const walRead = storageNewReadP(storageRepoIdx(0), STRDEF(TEST_WAL_MIGRATE)); cipherBlockFilterGroupAdd( diff --git a/test/src/module/command/verifyTest.c b/test/src/module/command/verifyTest.c index 352567fc6..a43e803a5 100644 --- a/test/src/module/command/verifyTest.c +++ b/test/src/module/command/verifyTest.c @@ -895,7 +895,7 @@ testRun(void) TEST_RESULT_LOG(""); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("encrypted info files at format 6"); + TEST_TITLE("encrypted format 6 info files"); HRN_INFO_PUT( storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .format = REPOSITORY_FORMAT_6, .header = true, @@ -934,6 +934,13 @@ testRun(void) // ***************************************************************************************************************************** if (testBegin("verifyFile()")) { + // Backup files contain no key id, so the key goes under the default id + CipherSpecMap *const cipherSpecMapNone = cipherSpecMapNew(); + CipherSpecMap *const cipherSpecMapPass = cipherSpecMapNew(); + cipherSpecMapAdd( + cipherSpecMapPass, CIPHER_SPEC_MAP_ID_DEFAULT_STR, + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("pass"), .digest = hashTypeSha1)); + // Load Parameters StringList *argList = strLstDup(argListBase); HRN_CFG_LOAD(cfgCmdVerify, argList); @@ -944,7 +951,7 @@ testRun(void) String *filePathName = strNewZ(STORAGE_REPO_ARCHIVE "/testfile"); HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), strZ(filePathName)); TEST_RESULT_UINT( - verifyFile(filePathName, 0, NULL, compressTypeNone, HASH_TYPE_SHA1_ZERO_BUF, 0, cipherSpecNewNone()), + verifyFile(filePathName, 0, NULL, compressTypeNone, HASH_TYPE_SHA1_ZERO_BUF, 0, cipherSpecMapNone, false), verifyOk, "file ok"); // ------------------------------------------------------------------------------------------------------------------------- @@ -952,7 +959,7 @@ testRun(void) HRN_STORAGE_PUT_Z(storageRepoWrite(), strZ(filePathName), fileContents); TEST_RESULT_UINT( - verifyFile(filePathName, 0, NULL, compressTypeNone, fileChecksum, 0, cipherSpecNewNone()), + verifyFile(filePathName, 0, NULL, compressTypeNone, fileChecksum, 0, cipherSpecMapNone, false), verifySizeInvalid, "file size invalid"); // ------------------------------------------------------------------------------------------------------------------------- @@ -960,7 +967,8 @@ testRun(void) TEST_RESULT_UINT( verifyFile( - strNewFmt(STORAGE_REPO_ARCHIVE "/missingFile"), 0, NULL, compressTypeNone, fileChecksum, 0, cipherSpecNewNone()), + strNewFmt(STORAGE_REPO_ARCHIVE "/missingFile"), 0, NULL, compressTypeNone, fileChecksum, 0, cipherSpecMapNone, + false), verifyFileMissing, "file missing"); // ------------------------------------------------------------------------------------------------------------------------- @@ -976,12 +984,12 @@ testRun(void) TEST_RESULT_UINT( verifyFile( filePathName, 0, NULL, compressTypeGz, fileChecksum, fileSize, - cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("pass"), .digest = hashTypeSha1)), + cipherSpecMapPass, false), verifyOk, "file encrypted compressed ok"); TEST_RESULT_UINT( verifyFile( filePathName, 0, NULL, compressTypeGz, bufNewDecode(encodingHex, STRDEF("aa")), fileSize, - cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("pass"), .digest = hashTypeSha1)), + cipherSpecMapPass, false), verifyChecksumMismatch, "file encrypted compressed checksum mismatch"); } diff --git a/test/src/module/common/formatTest.c b/test/src/module/common/formatTest.c index 4a603e1d8..6f02ca5fb 100644 --- a/test/src/module/common/formatTest.c +++ b/test/src/module/common/formatTest.c @@ -51,6 +51,9 @@ testRun(void) const Buffer *const testPass = BUFSTRDEF(TEST_PASS); const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, testPass); + CipherSpecMap *const keyMapDefault = cipherSpecMapNew(); + cipherSpecMapAdd(keyMapDefault, CIPHER_SPEC_MAP_ID_DEFAULT_STR, cipherSpec); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("nothing is added when the repository is not encrypted"); @@ -58,7 +61,7 @@ testRun(void) IoWrite *plainWrite = ioBufferWriteNew(plain); TEST_RESULT_VOID( - cipherBlockFormatFilterGroupWriteAdd(plain, ioWriteFilterGroup(plainWrite), cipherSpecNewNone(), REPOSITORY_FORMAT_6), + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(plainWrite), cipherSpecNewNone(), REPOSITORY_FORMAT_6), "add write filter for no cipher"); ioWriteOpen(plainWrite); ioWrite(plainWrite, testPlainText); @@ -77,14 +80,14 @@ testRun(void) Buffer *headerBuffer = bufNew(0); IoWrite *headerWrite = ioBufferWriteNew(headerBuffer); - cipherBlockFormatFilterGroupWriteAdd(headerBuffer, ioWriteFilterGroup(headerWrite), cipherSpec, REPOSITORY_FORMAT_6); + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(headerWrite), cipherSpec, REPOSITORY_FORMAT_6); ioWriteOpen(headerWrite); ioWrite(headerWrite, testPlainText); ioWriteClose(headerWrite); TEST_RESULT_BOOL( memcmp(bufPtrConst(headerBuffer), CIPHER_BLOCK_FORMAT_MAGIC "006_", CIPHER_BLOCK_FORMAT_HEADER_SIZE) == 0, true, - "header names the format"); + "header contains the format"); // The format is not given on decrypt, so it comes from the header and is what the pass derives with Buffer *headerResult = bufNew(0); @@ -113,7 +116,7 @@ testRun(void) Buffer *const piecesBuffer = bufNew(0); IoWrite *const piecesWrite = ioBufferWriteNew(piecesBuffer); - cipherBlockFormatFilterGroupWriteAdd(piecesBuffer, ioWriteFilterGroup(piecesWrite), cipherSpec, REPOSITORY_FORMAT_6); + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(piecesWrite), cipherSpec, REPOSITORY_FORMAT_6); ioWriteOpen(piecesWrite); ioWrite(piecesWrite, piecesPlainText); ioWriteClose(piecesWrite); @@ -165,7 +168,7 @@ testRun(void) Buffer *magicBuffer = bufNew(0); IoWrite *magicWrite = ioBufferWriteNew(magicBuffer); - cipherBlockFormatFilterGroupWriteAdd(magicBuffer, ioWriteFilterGroup(magicWrite), cipherSpec, REPOSITORY_FORMAT_5); + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(magicWrite), cipherSpec, REPOSITORY_FORMAT_5); ioWriteOpen(magicWrite); ioWrite(magicWrite, testPlainText); ioWriteClose(magicWrite); @@ -189,12 +192,12 @@ testRun(void) REPOSITORY_FORMAT_5, "filter reports the format before the header"); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("a format given on decrypt must be the one the header names"); + TEST_TITLE("a format given on decrypt must be the one the header contains"); IoWrite *const headerMismatch = ioBufferWriteNew(bufNew(0)); ioFilterGroupAdd( - ioWriteFilterGroup(headerMismatch), cipherBlockFormatNewP(cipherSpec, .format = REPOSITORY_FORMAT_5)); + ioWriteFilterGroup(headerMismatch), cipherBlockFormatNewP(keyMapDefault, .format = REPOSITORY_FORMAT_5)); ioWriteOpen(headerMismatch); TEST_ERROR(ioWrite(headerMismatch, headerBuffer), FormatError, "expected repository format 5 but found 6"); @@ -208,7 +211,7 @@ testRun(void) ioFilterGroupAdd( ioWriteFilterGroup(headerRead), cipherBlockFormatNewPack( - ioFilterParamList(cipherBlockFormatNewP(cipherSpec, .format = REPOSITORY_FORMAT_6)))); + ioFilterParamList(cipherBlockFormatNewP(keyMapDefault, .format = REPOSITORY_FORMAT_6)))); ioWriteOpen(headerRead); ioWrite(headerRead, headerBuffer); ioWriteClose(headerRead); @@ -266,6 +269,247 @@ testRun(void) TEST_HEADER_DAMAGE(0, 'X', CryptoError, "cipher header invalid"); #undef TEST_HEADER_DAMAGE + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("read with the key id stored in the header"); + + const CipherSpec *const keySpecOld = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("oldkey"), .digest = hashTypeSha1); + const CipherSpec *const keySpecNew = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("newkey")); + + CipherSpecMap *const keyMap = cipherSpecMapNew(); + cipherSpecMapAdd(keyMap, STRDEF(CIPHER_SPEC_MAP_ID_DEFAULT), keySpecOld); + cipherSpecMapAdd(keyMap, STRDEF("7"), keySpecNew); + + Buffer *const keyBuffer = bufNew(0); + IoRead *const keyWrite = ioBufferReadNew(testPlainText); + Buffer *const keyPiece = bufNew(4); + + // Build the write filters from their param lists, as the remote protocol does + IoFilterGroup *const keyWriteGroup = ioFilterGroupNew(); + cipherBlockFormatFilterGroupWriteAddP(keyWriteGroup, keySpecNew, REPOSITORY_FORMAT_6, .keyId = STRDEF("7")); + + PackRead *const keyWriteParam = pckReadNew(ioFilterGroupParamAll(keyWriteGroup)); + + TEST_RESULT_UINT(pckReadStrIdP(keyWriteParam), CIPHER_BLOCK_FILTER_TYPE, "block cipher filter"); + ioFilterGroupAdd(ioReadFilterGroup(keyWrite), cipherBlockNewPack(pckReadPackP(keyWriteParam))); + TEST_RESULT_UINT(pckReadStrIdP(keyWriteParam), CIPHER_BLOCK_FORMAT_HEADER_FILTER_TYPE, "header filter"); + ioFilterGroupAdd(ioReadFilterGroup(keyWrite), cipherBlockFormatHeaderNewPack(pckReadPackP(keyWriteParam))); + + // Take the result in pieces smaller than the header so the header is written in more than one part + ioBufferSizeSet(4); + ioReadOpen(keyWrite); + + while (!ioReadEof(keyWrite)) + { + bufUsedZero(keyPiece); + ioRead(keyWrite, keyPiece); + bufCat(keyBuffer, keyPiece); + } + + ioReadClose(keyWrite); + ioBufferSizeSet(TEST_BUFFER_SIZE); + + TEST_RESULT_BOOL( + memcmp(bufPtrConst(keyBuffer), CIPHER_BLOCK_FORMAT_MAGIC "006K\0017", CIPHER_BLOCK_FORMAT_HEADER_SIZE + 2) == 0, true, + "header contains the key after its length"); + + Buffer *keyResult = bufNew(0); + IoWrite *keyRead = ioBufferWriteNew(keyResult); + + cipherBlockFormatFilterGroupReadAddMap(ioWriteFilterGroup(keyRead), keyMap); + ioWriteOpen(keyRead); + ioWrite(keyRead, keyBuffer); + ioWriteClose(keyRead); + + TEST_RESULT_STR_Z(strNewBuf(keyResult), TEST_PLAINTEXT, "content decrypted with the key the header contains"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("key id split across input, from a pack"); + + keyResult = bufNew(0); + keyRead = ioBufferWriteNew(keyResult); + + // Build from a pack, as the remote protocol does + ioFilterGroupAdd( + ioWriteFilterGroup(keyRead), cipherBlockFormatNewPack(ioFilterParamList(cipherBlockFormatNewP(keyMap)))); + + // Split so the length byte and the key id each arrive separately + ioBufferSizeSet(4); + ioWriteOpen(keyRead); + ioWrite(keyRead, BUF(bufPtrConst(keyBuffer), CIPHER_BLOCK_FORMAT_HEADER_SIZE)); + ioWrite(keyRead, BUF(bufPtrConst(keyBuffer) + CIPHER_BLOCK_FORMAT_HEADER_SIZE, 1)); + ioWrite(keyRead, BUF(bufPtrConst(keyBuffer) + CIPHER_BLOCK_FORMAT_HEADER_SIZE + 1, 1)); + ioWrite( + keyRead, + BUF( + bufPtrConst(keyBuffer) + CIPHER_BLOCK_FORMAT_HEADER_SIZE + 2, + bufUsed(keyBuffer) - CIPHER_BLOCK_FORMAT_HEADER_SIZE - 2)); + ioWriteClose(keyRead); + ioBufferSizeSet(TEST_BUFFER_SIZE); + + TEST_RESULT_STR_Z(strNewBuf(keyResult), TEST_PLAINTEXT, "content decrypted from split input"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("key digest is used, not the digest of the format"); + + // A key kept by migration derives with SHA-1 even in a file written at a format that derives with SHA-256 + Buffer *const digestBuffer = bufNew(0); + IoWrite *const digestWrite = ioBufferWriteNew(digestBuffer); + + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(digestWrite), keySpecOld, REPOSITORY_FORMAT_5); + ioWriteOpen(digestWrite); + ioWrite(digestWrite, testPlainText); + ioWriteClose(digestWrite); + + // Replace the magic with a format 6 header, which leaves content that only the SHA-1 key can decrypt + memcpy(bufPtr(digestBuffer), CIPHER_BLOCK_FORMAT_MAGIC "006_", CIPHER_BLOCK_FORMAT_HEADER_SIZE); + + Buffer *const digestResult = bufNew(0); + IoWrite *const digestRead = ioBufferWriteNew(digestResult); + + cipherBlockFormatFilterGroupReadAddMap(ioWriteFilterGroup(digestRead), keyMap); + ioWriteOpen(digestRead); + ioWrite(digestRead, digestBuffer); + ioWriteClose(digestRead); + + TEST_RESULT_STR_Z(strNewBuf(digestResult), TEST_PLAINTEXT, "content decrypted with the digest of the key"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("file with no key id uses the migrated key"); + + Buffer *const migratedBuffer = bufNew(0); + IoWrite *const migratedWrite = ioBufferWriteNew(migratedBuffer); + + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(migratedWrite), keySpecOld, REPOSITORY_FORMAT_5); + ioWriteOpen(migratedWrite); + ioWrite(migratedWrite, testPlainText); + ioWriteClose(migratedWrite); + + Buffer *const migratedResult = bufNew(0); + IoWrite *const migratedRead = ioBufferWriteNew(migratedResult); + + cipherBlockFormatFilterGroupReadAddMap(ioWriteFilterGroup(migratedRead), keyMap); + ioWriteOpen(migratedRead); + ioWrite(migratedRead, migratedBuffer); + ioWriteClose(migratedRead); + + TEST_RESULT_STR_Z(strNewBuf(migratedResult), TEST_PLAINTEXT, "content decrypted with the migrated key"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("key id not in the map"); + + CipherSpecMap *const keyMapShort = cipherSpecMapNew(); + cipherSpecMapAdd(keyMapShort, STRDEF(CIPHER_SPEC_MAP_ID_DEFAULT), keySpecOld); + + IoWrite *const keyMissing = ioBufferWriteNew(bufNew(0)); + cipherBlockFormatFilterGroupReadAddMap(ioWriteFilterGroup(keyMissing), keyMapShort); + ioWriteOpen(keyMissing); + + TEST_ERROR(ioWrite(keyMissing, keyBuffer), CryptoError, "unable to find cipher key '7'"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("key id in a file read with a single key"); + + IoWrite *const keyUnexpected = ioBufferWriteNew(bufNew(0)); + cipherBlockFormatFilterGroupReadAdd(ioWriteFilterGroup(keyUnexpected), keySpecNew); + ioWriteOpen(keyUnexpected); + + TEST_ERROR(ioWrite(keyUnexpected, keyBuffer), CryptoError, "unable to find cipher key '7'"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("key id with zero length"); + + Buffer *const keyEmpty = bufDup(keyBuffer); + bufPtr(keyEmpty)[CIPHER_BLOCK_FORMAT_HEADER_SIZE] = 0; + + IoWrite *const keyEmptyRead = ioBufferWriteNew(bufNew(0)); + cipherBlockFormatFilterGroupReadAddMap(ioWriteFilterGroup(keyEmptyRead), keyMap); + ioWriteOpen(keyEmptyRead); + + TEST_ERROR(ioWrite(keyEmptyRead, keyEmpty), FormatError, "invalid cipher header"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("no filter added when the map is empty"); + + IoWrite *const keyNone = ioBufferWriteNew(bufNew(0)); + TEST_RESULT_PTR_NE( + cipherBlockFormatFilterGroupReadAddMap(ioWriteFilterGroup(keyNone), cipherSpecMapNew()), NULL, + "add read filter for no keys"); + TEST_RESULT_UINT(ioFilterGroupSize(ioWriteFilterGroup(keyNone)), 0, "no filter added"); + } + + // ***************************************************************************************************************************** + if (testBegin("cipherSpecMapNew()")) + { + const CipherSpec *const cipherSpec1 = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("key1"), .digest = hashTypeSha256); + const CipherSpec *const cipherSpec2 = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("key2"), .digest = hashTypeSha1); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("add and get keys"); + + CipherSpecMap *map = NULL; + TEST_ASSIGN(map, cipherSpecMapNew(), "new map"); + TEST_RESULT_UINT(cipherSpecMapSize(map), 0, "no keys"); + + // Add out of order to check that the map sorts + TEST_RESULT_VOID(cipherSpecMapAdd(map, STRDEF("2"), cipherSpec2), "add 2"); + TEST_RESULT_VOID(cipherSpecMapAdd(map, STRDEF("0"), cipherSpec1), "add 0"); + + TEST_RESULT_UINT(cipherSpecMapSize(map), 2, "two keys"); + TEST_RESULT_STR_Z(cipherSpecMapGetIdx(map, 0)->id, "0", "first id"); + TEST_RESULT_STR_Z(cipherSpecMapGetIdx(map, 1)->id, "2", "second id"); + + // Each key keeps its own digest + TEST_RESULT_UINT(cipherSpecDigest(cipherSpecMapGet(map, STRDEF("0"))), hashTypeSha256, "key 0 digest"); + TEST_RESULT_UINT(cipherSpecDigest(cipherSpecMapGet(map, STRDEF("2"))), hashTypeSha1, "key 2 digest"); + TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(cipherSpecMapGet(map, STRDEF("2")))), "key2", "key 2 pass"); + + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(map), "2", "adding the key with no id did not make it current"); + + // A map with only the key that has no id has no current key + CipherSpecMap *const mapDefault = cipherSpecMapNew(); + TEST_RESULT_VOID(cipherSpecMapAdd(mapDefault, CIPHER_SPEC_MAP_ID_DEFAULT_STR, cipherSpec1), "add key with no id"); + TEST_RESULT_PTR(cipherSpecMapIdCurrent(mapDefault), NULL, "no current key"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("duplicate keeps the current key"); + + TEST_RESULT_PTR(cipherSpecMapIdCurrent(cipherSpecMapDup(cipherSpecMapNew())), NULL, "dup empty map"); + + CipherSpecMap *mapDup = NULL; + TEST_ASSIGN(mapDup, cipherSpecMapDup(map), "dup map"); + TEST_RESULT_UINT(cipherSpecMapSize(mapDup), 2, "two keys"); + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(mapDup), "2", "current key id"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("key id not found"); + + TEST_ERROR(cipherSpecMapGet(map, STRDEF("1")), CryptoError, "unable to find cipher key '1'"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("pack round trip"); + + PackWrite *const packWrite = pckWriteNewP(); + TEST_RESULT_VOID(cipherSpecMapPack(packWrite, map), "pack map"); + pckWriteEndP(packWrite); + + CipherSpecMap *mapPack = NULL; + TEST_ASSIGN(mapPack, cipherSpecMapNewPack(pckReadNew(pckWriteResult(packWrite))), "unpack map"); + + TEST_RESULT_UINT(cipherSpecMapSize(mapPack), 2, "two keys"); + TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(cipherSpecMapGet(mapPack, STRDEF("0")))), "key1", "key 0 pass"); + TEST_RESULT_UINT(cipherSpecDigest(cipherSpecMapGet(mapPack, STRDEF("2"))), hashTypeSha1, "key 2 digest"); + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(mapPack), "2", "current key id"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("log ids but not keys"); + + char logBuf[STACK_TRACE_PARAM_MAX]; + + TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(map, cipherSpecMapToLog, logBuf, sizeof(logBuf)), "cipherSpecMapToLog"); + TEST_RESULT_Z(logBuf, "{ids: [0, 2]}", "check log"); + + TEST_RESULT_VOID(cipherSpecMapFree(map), "free map"); } FUNCTION_HARNESS_RETURN_VOID(); diff --git a/test/src/module/info/infoArchiveTest.c b/test/src/module/info/infoArchiveTest.c index 713ea73be..e880dcbab 100644 --- a/test/src/module/info/infoArchiveTest.c +++ b/test/src/module/info/infoArchiveTest.c @@ -83,13 +83,16 @@ testRun(void) TEST_TITLE("create with cipher"); // Recreate from scratch with cipher + CipherSpecMap *const cipherSpecMapSub = cipherSpecMapNew(); + cipherSpecMapAdd( + cipherSpecMapSub, CIPHER_SPEC_MAP_ID_DEFAULT_STR, + cipherSpecNewP( + cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"), + .digest = hashTypeSha1)); + TEST_ASSIGN( info, - infoArchiveNew( - PG_VERSION_10, 6569239123849665999, REPOSITORY_FORMAT_DEFAULT, - cipherSpecNewP( - cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"), - .digest = hashTypeSha1)), + infoArchiveNew(PG_VERSION_10, 6569239123849665999, REPOSITORY_FORMAT_DEFAULT, cipherSpecMapSub), "infoArchiveNew() - cipher sub"); const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); @@ -97,7 +100,7 @@ testRun(void) contentSave = bufNew(0); IoWrite *write = ioBufferWriteNew(contentSave); - cipherBlockFormatFilterGroupWriteAdd(contentSave, ioWriteFilterGroup(write), cipherSpec, REPOSITORY_FORMAT_DEFAULT); + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(write), cipherSpec, REPOSITORY_FORMAT_DEFAULT); TEST_RESULT_VOID(infoArchiveSave(info, write), "save new with cipher"); TEST_RESULT_BOOL( @@ -124,6 +127,26 @@ testRun(void) TEST_RESULT_INT(infoPgData.version, PG_VERSION_96, "version set"); TEST_RESULT_UINT(infoPgData.systemId, 6569239123849665679, "systemId set"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("rotate cipher key"); + + const CipherSpec *const cipherSpecRotate = cipherSpecNewP( + cipherTypeAes256Cbc, BUFSTRDEF("rotate"), .digest = hashTypeSha256); + + TEST_RESULT_VOID(infoArchiveFormatSet(info, REPOSITORY_FORMAT_6), "set format 6"); + TEST_RESULT_INT(infoArchiveCipherRotateTime(info), 0, "no rotation time"); + + TEST_RESULT_VOID(infoArchiveCipherRotate(info, cipherSpecRotate, 1700000000), "rotate after migrated key"); + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(infoArchiveCipherSpecMap(info)), "1", "first key id"); + TEST_RESULT_INT(infoArchiveCipherRotateTime(info), 1700000000, "rotation time"); + + // Ids increment as numbers rather than as text + TEST_RESULT_VOID(infoCipherSpecAdd(infoPgInfo(infoArchivePg(info)), STRDEF("9"), cipherSpecRotate), "add key 9"); + TEST_RESULT_VOID(infoArchiveCipherRotate(info, cipherSpecRotate, 1700086400), "rotate after key 9"); + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(infoArchiveCipherSpecMap(info)), "10", "key id after 9"); + TEST_RESULT_UINT(cipherSpecMapSize(infoArchiveCipherSpecMap(info)), 4, "all keys kept"); + TEST_RESULT_INT(infoArchiveCipherRotateTime(info), 1700086400, "rotation time"); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("object free"); diff --git a/test/src/module/info/infoBackupTest.c b/test/src/module/info/infoBackupTest.c index 9b02f00e5..a905d3ee5 100644 --- a/test/src/module/info/infoBackupTest.c +++ b/test/src/module/info/infoBackupTest.c @@ -98,7 +98,7 @@ testRun(void) contentSave = bufNew(0); IoWrite *write = ioBufferWriteNew(contentSave); - cipherBlockFormatFilterGroupWriteAdd(contentSave, ioWriteFilterGroup(write), cipherSpec, REPOSITORY_FORMAT_6); + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(write), cipherSpec, REPOSITORY_FORMAT_6); TEST_RESULT_VOID(infoBackupSave(infoBackup, write), "save new with cipher sub"); TEST_RESULT_STR_Z( diff --git a/test/src/module/info/infoPgTest.c b/test/src/module/info/infoPgTest.c index d692325c0..d39f66a76 100644 --- a/test/src/module/info/infoPgTest.c +++ b/test/src/module/info/infoPgTest.c @@ -38,7 +38,7 @@ testRun(void) TEST_ASSIGN(infoPg, infoPgNew(infoPgBackup, REPOSITORY_FORMAT_DEFAULT, NULL), "infoPgNew(cipherTypeNone, NULL)"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 0, " 0 history"); - TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(infoPgInfo(infoPg))), cipherTypeNone, " cipher spec none"); + TEST_RESULT_UINT(cipherSpecType(infoPgCipherSpec(infoPg)), cipherTypeNone, " cipher spec none"); TEST_RESULT_INT(infoPgDataCurrentId(infoPg), 0, " 0 historyCurrent"); TEST_ASSIGN( @@ -48,7 +48,7 @@ testRun(void) cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"), .digest = hashTypeSha1)), "infoPgNew(cipherTypeAes256Cbc, 123xyz)"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 0, " 0 history"); - TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(infoPgInfo(infoPg)))), "123xyz", " cipherPass set"); + TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoPgCipherSpec(infoPg))), "123xyz", " cipherPass set"); TEST_RESULT_INT(infoPgDataCurrentId(infoPg), 0, " 0 historyCurrent"); // ------------------------------------------------------------------------------------------------------------------------- @@ -76,7 +76,7 @@ testRun(void) TEST_RESULT_UINT(pgData.systemId, 6569239123849665999, " system-id updated"); TEST_RESULT_UINT(pgData.version, PG_VERSION_18, " version updated"); TEST_RESULT_UINT(pgData.catalogVersion, 0, " catalog version not set for archive"); - TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(infoPgInfo(infoPg))), cipherTypeNone, " cipher spec none"); + TEST_RESULT_UINT(cipherSpecType(infoPgCipherSpec(infoPg)), cipherTypeNone, " cipher spec none"); // ------------------------------------------------------------------------------------------------------------------------- TEST_ASSIGN( @@ -94,7 +94,7 @@ testRun(void) TEST_RESULT_UINT(pgData.systemId, 6569239123849665679, " system-id set"); TEST_RESULT_UINT(pgData.version, PG_VERSION_18, " version set"); TEST_RESULT_UINT(pgData.catalogVersion, 202506291, " catalog version updated"); - TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(infoPgInfo(infoPg)))), "123xyz", " cipherPass set"); + TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoPgCipherSpec(infoPg))), "123xyz", " cipherPass set"); } // ***************************************************************************************************************************** diff --git a/test/src/module/info/infoTest.c b/test/src/module/info/infoTest.c index 19643c20a..f130ca39e 100644 --- a/test/src/module/info/infoTest.c +++ b/test/src/module/info/infoTest.c @@ -21,6 +21,15 @@ testInfoReadHeader(const Buffer *const buffer, const CipherSpec *const cipherSpe return result; } +/*********************************************************************************************************************************** +The key an info file holds under the default id +***********************************************************************************************************************************/ +static const CipherSpec * +testInfoCipherSpec(const Info *const info) +{ + return infoCipherSpec(info); +} + /*********************************************************************************************************************************** Test load callback ***********************************************************************************************************************************/ @@ -96,10 +105,13 @@ testRun(void) infoNew( REPOSITORY_FORMAT_DEFAULT, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"), .digest = hashTypeSha1)), "infoNew(cipher)"); - TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(info))), "123xyz", " cipherPass is set"); + TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(testInfoCipherSpec(info))), "123xyz", " cipherPass is set"); TEST_ASSIGN(info, infoNew(REPOSITORY_FORMAT_DEFAULT, NULL), "infoNew(NULL)"); - TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(info)), cipherTypeNone, " cipher spec is none"); + TEST_RESULT_UINT(cipherSpecType(testInfoCipherSpec(info)), cipherTypeNone, " cipher spec is none"); + + TEST_ASSIGN(info, infoNew(REPOSITORY_FORMAT_DEFAULT, cipherSpecNewNone()), "infoNew(none)"); + TEST_RESULT_UINT(cipherSpecMapSize(infoCipherSpecMap(info)), 0, " no cipher keys"); } // ***************************************************************************************************************************** @@ -203,7 +215,7 @@ testRun(void) info, infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), "info with other cipher"); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); - TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(info)), cipherTypeNone, " check cipher pass not set"); + TEST_RESULT_UINT(cipherSpecType(testInfoCipherSpec(info)), cipherTypeNone, " check cipher pass not set"); // Base file with content // ------------------------------------------------------------------------------------------------------------------------- @@ -220,7 +232,7 @@ testRun(void) info, infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), "info with content"); TEST_RESULT_STR_Z(callbackContent, "[c] key=1\n[d] key=1\n", " check callback content"); - TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(info)), cipherTypeNone, " check cipher pass not set"); + TEST_RESULT_UINT(cipherSpecType(testInfoCipherSpec(info)), cipherTypeNone, " check cipher pass not set"); Buffer *contentSave = bufNew(0); @@ -281,8 +293,8 @@ testRun(void) info, infoNewLoad(readNoHeader, cipherSpecNoHeader, harnessInfoLoadNewCallback, callbackContent), "info with content and cipher"); TEST_RESULT_STR_Z(callbackContent, "[c] key=1\n[d] key=1\n", " check callback content"); - TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(info))), "somepass", " check cipher pass set"); - TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest"); + TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(testInfoCipherSpec(info))), "somepass", " check cipher pass set"); + TEST_RESULT_UINT(cipherSpecDigest(testInfoCipherSpec(info)), hashTypeSha1, " check cipher sub digest"); TEST_RESULT_STR_Z(infoBackrestVersion(info), PROJECT_VERSION, " check backrest version"); contentSave = bufNew(0); @@ -298,7 +310,8 @@ testRun(void) TEST_RESULT_VOID( infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "save migrated info"); TEST_RESULT_BOOL( - strstr(strZ(strNewBuf(contentSave)), "cipher-digest=\"sha1\"") != NULL, true, " check digest stored with pass"); + strstr(strZ(strNewBuf(contentSave)), "cipher-pass={\"0\":{\"digest\":\"sha1\",\"key\":\"somepass\"}}") != NULL, true, + " check key stored by id with its digest"); TEST_ASSIGN( info, @@ -307,7 +320,7 @@ testRun(void) cipherSpec, harnessInfoLoadNewCallback, strNew()), "load migrated info"); TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); - TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest unchanged"); + TEST_RESULT_UINT(cipherSpecDigest(testInfoCipherSpec(info)), hashTypeSha1, " check cipher sub digest unchanged"); // Header // ------------------------------------------------------------------------------------------------------------------------- @@ -315,7 +328,7 @@ testRun(void) contentSave = bufNew(0); IoWrite *const writeNone = ioBufferWriteNew(contentSave); - cipherBlockFormatFilterGroupWriteAdd(contentSave, ioWriteFilterGroup(writeNone), cipherSpecNewNone(), REPOSITORY_FORMAT_6); + cipherBlockFormatFilterGroupWriteAddP(ioWriteFilterGroup(writeNone), cipherSpecNewNone(), REPOSITORY_FORMAT_6); TEST_RESULT_VOID(infoSave(info, writeNone, testInfoSaveCallback, strNewZ("1")), "info save"); TEST_RESULT_BOOL(strBeginsWithZ(strNewBuf(contentSave), "PGBR"), false, " check no header"); @@ -324,8 +337,7 @@ testRun(void) REPOSITORY_FORMAT_6, STRDEF( "[cipher]\n" - "cipher-digest=\"sha256\"\n" - "cipher-pass=\"somepass\"\n")); + "cipher-pass={\"0\":{\"digest\":\"sha256\",\"key\":\"somepass\"}}\n")); callbackContent = strNew(); @@ -336,7 +348,7 @@ testRun(void) cipherSpec, harnessInfoLoadNewCallback, callbackContent), "info with header"); TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); - TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha256, " check cipher sub digest"); + TEST_RESULT_UINT(cipherSpecDigest(testInfoCipherSpec(info)), hashTypeSha256, " check cipher sub digest"); // A pass migrated from a format that could not store the digest keeps deriving with SHA-1, so the files it encrypted // before the migration are still readable @@ -344,8 +356,7 @@ testRun(void) REPOSITORY_FORMAT_6, STRDEF( "[cipher]\n" - "cipher-digest=\"sha1\"\n" - "cipher-pass=\"somepass\"\n")); + "cipher-pass={\"0\":{\"digest\":\"sha1\",\"key\":\"somepass\"}}\n")); TEST_ASSIGN( info, @@ -353,13 +364,56 @@ testRun(void) testInfoReadHeader(harnessInfoEncryptP(contentMigrated, cipherSpec, .format = REPOSITORY_FORMAT_6), cipherSpec), cipherSpec, harnessInfoLoadNewCallback, callbackContent), "info migrated to the format that stores the digest"); - TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest"); + TEST_RESULT_UINT(cipherSpecDigest(testInfoCipherSpec(info)), hashTypeSha1, " check cipher sub digest"); + + // Two keys, as rotation stores them. The last key added is current. + CipherSpecMap *const cipherSpecMapMulti = cipherSpecMapNew(); + cipherSpecMapAdd( + cipherSpecMapMulti, CIPHER_SPEC_MAP_ID_DEFAULT_STR, + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("oldpass"), .digest = hashTypeSha1)); + + TEST_RESULT_VOID(infoCipherSpecMapSet(info, cipherSpecMapMulti), "set migrated key"); + TEST_RESULT_VOID( + infoCipherSpecAdd( + info, STRDEF("9"), cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("newpass"), .digest = hashTypeSha256)), + "add second key"); + TEST_RESULT_VOID(infoCipherRotateTimeSet(info, 1700000000), "set rotation time"); + + contentSave = bufNew(0); + + TEST_RESULT_VOID(infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "save two keys"); + TEST_RESULT_BOOL( + strstr( + strZ(strNewBuf(contentSave)), + "cipher-pass={\"0\":{\"digest\":\"sha1\",\"key\":\"oldpass\"}" + ",\"9\":{\"digest\":\"sha256\",\"key\":\"newpass\"}}\n" + "cipher-pass-current=\"9\"\n" + "cipher-pass-rotate=1700000000\n") != NULL, + true, " check both keys stored by id"); + + TEST_ASSIGN( + info, + infoNewLoad( + testInfoReadHeader(harnessInfoEncryptP(contentSave, cipherSpec, .format = REPOSITORY_FORMAT_6), cipherSpec), + cipherSpec, harnessInfoLoadNewCallback, strNew()), + "load two keys"); + + const CipherSpecMap *const cipherSpecMapLoad = infoCipherSpecMap(info); + + TEST_RESULT_UINT(cipherSpecMapSize(cipherSpecMapLoad), 2, " check two keys"); + TEST_RESULT_STR_Z(cipherSpecMapIdCurrent(cipherSpecMapLoad), "9", " check current key id"); + TEST_RESULT_INT(infoCipherRotateTime(info), 1700000000, " check rotation time"); + TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(testInfoCipherSpec(info))), "oldpass", " check default key"); + TEST_RESULT_UINT(cipherSpecDigest(testInfoCipherSpec(info)), hashTypeSha1, " check default key digest"); + TEST_RESULT_STR_Z( + strNewBuf(cipherSpecPass(cipherSpecMapGet(cipherSpecMapLoad, STRDEF("9")))), "newpass", " check key 9"); + TEST_RESULT_UINT( + cipherSpecDigest(cipherSpecMapGet(cipherSpecMapLoad, STRDEF("9"))), hashTypeSha256, " check key 9 digest"); // The content on its own, which is how a caller that wants the file rather than the values in it reads an info file IoRead *const infoRead = ioBufferReadNew(harnessInfoEncryptP(contentLoad, cipherSpec, .format = REPOSITORY_FORMAT_6)); - ioFilterGroupAdd( - ioReadFilterGroup(infoRead), cipherBlockFormatNewP(cipherSpec)); + cipherBlockFormatFilterGroupReadAdd(ioReadFilterGroup(infoRead), cipherSpec); ioReadOpen(infoRead); TEST_RESULT_STR(strNewBuf(ioReadBuf(infoRead)), strNewBuf(contentLoad), "info content read"); diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c index 33281a43f..3cea21a7d 100644 --- a/test/src/module/info/manifestTest.c +++ b/test/src/module/info/manifestTest.c @@ -1453,8 +1453,7 @@ testRun(void) "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n" "\n" "[cipher]\n" - "cipher-digest=\"sha256\"\n" - "cipher-pass=\"somepass\"\n" + "cipher-pass={\"0\":{\"digest\":\"sha256\",\"key\":\"somepass\"}}\n" "\n" "[target:file]\n" "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"reference\":\"20190808-163540F\""