diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 9b3bd7b39..27b587e91 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -15,6 +15,18 @@ + + + + + +

The expire command is implemented entirely in C.

+ + + +

The local command for restore is implemented entirely in C.

+ + @@ -24,10 +36,6 @@

Remove hard-coded user so $PGUSER works.

 - -

The local command for restore is implemented entirely in C.

- -

Rename repo-s3-verify-ssl option to repo-s3-verify-tls.

diff --git a/lib/pgBackRest/Expire.pm b/lib/pgBackRest/Expire.pm deleted file mode 100644 index 7f49d13d9..000000000 --- a/lib/pgBackRest/Expire.pm +++ /dev/null @@ -1,461 +0,0 @@ -#################################################################################################################################### -# EXPIRE MODULE -#################################################################################################################################### -package pgBackRest::Expire; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); -use File::Basename qw(dirname); -use Scalar::Util qw(looks_like_number); - -use pgBackRest::Archive::Common; -use pgBackRest::Archive::Info; -use pgBackRest::Common::Exception; -use pgBackRest::Common::Ini; -use pgBackRest::Common::Log; -use pgBackRest::Backup::Common; -use pgBackRest::Backup::Info; -use pgBackRest::Config::Config; -use pgBackRest::InfoCommon; -use pgBackRest::Manifest; -use pgBackRest::Protocol::Helper; -use pgBackRest::Protocol::Storage::Helper; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->new'); - - # Initialize total archive expired - $self->{iArchiveExpireTotal} = 0; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# logExpire -# -# Tracks which archive logs have been removed and provides log messages when needed. -#################################################################################################################################### -sub logExpire -{ - my $self = shift; - my $strArchiveId = shift; - my $strArchiveFile = shift; - - if (defined($strArchiveFile)) - { - if (!defined($self->{strArchiveExpireStart})) - { - $self->{strArchiveExpireStart} = $strArchiveFile; - $self->{strArchiveExpireStop} = $strArchiveFile; - } - else - { - $self->{strArchiveExpireStop} = $strArchiveFile; - } - - $self->{iArchiveExpireTotal}++; - } - else - { - if (defined($self->{strArchiveExpireStart})) - { - &log(DETAIL, "remove archive: archiveId = ${strArchiveId}, start = " . substr($self->{strArchiveExpireStart}, 0, 24) . - ", stop = " . substr($self->{strArchiveExpireStop}, 0, 24)); - } - - undef($self->{strArchiveExpireStart}); - } -} - -#################################################################################################################################### -# process -# -# Removes expired backups and archive logs from the backup directory. Partial backups are not counted for expiration, so if full -# or differential retention is set to 2, there must be three complete backups before the oldest one can be deleted. -#################################################################################################################################### -sub process -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->process'); - - my @stryPath; - - my $oStorageRepo = storageRepo(); - my $strBackupClusterPath = $oStorageRepo->pathGet(STORAGE_REPO_BACKUP); - my $iFullRetention = cfgOption(CFGOPT_REPO_RETENTION_FULL, false); - my $iDifferentialRetention = cfgOption(CFGOPT_REPO_RETENTION_DIFF, false); - my $strArchiveRetentionType = cfgOption(CFGOPT_REPO_RETENTION_ARCHIVE_TYPE, false); - my $iArchiveRetention = cfgOption(CFGOPT_REPO_RETENTION_ARCHIVE, false); - - # Load the backup.info - my $oBackupInfo = new pgBackRest::Backup::Info($oStorageRepo->pathGet(STORAGE_REPO_BACKUP)); - - # Find all the expired full backups - if (defined($iFullRetention)) - { - # Make sure iFullRetention is valid - if (!looks_like_number($iFullRetention) || $iFullRetention < 1) - { - confess &log(ERROR, cfgOptionName(CFGOPT_REPO_RETENTION_FULL) . ' must be a number >= 1'); - } - - @stryPath = $oBackupInfo->list(backupRegExpGet(true)); - - if (@stryPath > $iFullRetention) - { - # Expire all backups that depend on the full backup - for (my $iFullIdx = 0; $iFullIdx < @stryPath - $iFullRetention; $iFullIdx++) - { - my @stryRemoveList; - - foreach my $strPath ($oBackupInfo->list('^' . $stryPath[$iFullIdx] . '.*')) - { - $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strPath}/" . FILE_MANIFEST . INI_COPY_EXT); - $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strPath}/" . FILE_MANIFEST); - $oBackupInfo->delete($strPath); - - if ($strPath ne $stryPath[$iFullIdx]) - { - push(@stryRemoveList, $strPath); - } - } - - &log(INFO, 'expire full backup ' . (@stryRemoveList > 0 ? 'set: ' : '') . $stryPath[$iFullIdx] . - (@stryRemoveList > 0 ? ', ' . join(', ', @stryRemoveList) : '')); - } - } - } - - # Find all the expired differential backups - if (defined($iDifferentialRetention)) - { - # Make sure iDifferentialRetention is valid - if (!looks_like_number($iDifferentialRetention) || $iDifferentialRetention < 1) - { - confess &log(ERROR, cfgOptionName(CFGOPT_REPO_RETENTION_DIFF) . ' must be a number >= 1'); - } - - # Get a list of full and differential backups. Full are considered differential for the purpose of retention. - # Example: F1, D1, D2, F2 and repo-retention-diff=2, then F1,D2,F2 will be retained, not D2 and D1 as might be expected. - @stryPath = $oBackupInfo->list(backupRegExpGet(true, true)); - - if (@stryPath > $iDifferentialRetention) - { - for (my $iDiffIdx = 0; $iDiffIdx < @stryPath - $iDifferentialRetention; $iDiffIdx++) - { - # Skip if this is a full backup. Full backups only count as differential when deciding which differential backups - # to expire. - next if ($stryPath[$iDiffIdx] =~ backupRegExpGet(true)); - - # Get a list of all differential and incremental backups - my @stryRemoveList; - - foreach my $strPath ($oBackupInfo->list(backupRegExpGet(false, true, true))) - { - logDebugMisc($strOperation, "checking ${strPath} for differential expiration"); - - # Remove all differential and incremental backups before the oldest valid differential - if ($strPath lt $stryPath[$iDiffIdx + 1]) - { - $oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strPath}" . FILE_MANIFEST); - $oBackupInfo->delete($strPath); - - if ($strPath ne $stryPath[$iDiffIdx]) - { - push(@stryRemoveList, $strPath); - } - } - } - - &log(INFO, 'expire diff backup ' . (@stryRemoveList > 0 ? 'set: ' : '') . $stryPath[$iDiffIdx] . - (@stryRemoveList > 0 ? ', ' . join(', ', @stryRemoveList) : '')); - } - } - } - - $oBackupInfo->save(); - - # Remove backups from disk - foreach my $strBackup ($oStorageRepo->list( - STORAGE_REPO_BACKUP, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'})) - { - if (!$oBackupInfo->current($strBackup)) - { - &log(INFO, "remove expired backup ${strBackup}"); - - $oStorageRepo->remove("${strBackupClusterPath}/${strBackup}", {bRecurse => true}); - } - } - - # If archive retention is still undefined, then ignore archiving - if (!defined($iArchiveRetention)) - { - &log(INFO, "option '" . cfgOptionName(CFGOPT_REPO_RETENTION_ARCHIVE) . "' is not set - archive logs will not be expired"); - } - else - { - my @stryGlobalBackupRetention; - - # Determine which backup type to use for archive retention (full, differential, incremental) and get a list of the - # remaining non-expired backups based on the type. - if ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_FULL) - { - @stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true), 'reverse'); - } - elsif ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_DIFF) - { - @stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true, true), 'reverse'); - } - elsif ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_INCR) - { - @stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true, true, true), 'reverse'); - } - - # If no backups were found then preserve current archive logs - too soon to expire them - my $iBackupTotal = scalar @stryGlobalBackupRetention; - - if ($iBackupTotal > 0) - { - my $oArchiveInfo = new pgBackRest::Archive::Info($oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), true); - my @stryListArchiveDisk = sort {((split('-', $a))[1] + 0) cmp ((split('-', $b))[1] + 0)} $oStorageRepo->list( - STORAGE_REPO_ARCHIVE, {strExpression => REGEX_ARCHIVE_DIR_DB_VERSION, bIgnoreMissing => true}); - - # Make sure the current database versions match between the two files - if (!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, - ($oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION)))) || - !($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, - ($oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID))))) - { - confess &log(ERROR, "archive and backup database versions do not match\n" . - "HINT: has a stanza-upgrade been performed?", ERROR_FILE_INVALID); - } - - # Get the list of backups that are part of archive retention - my @stryTmp = @stryGlobalBackupRetention; - my @stryGlobalBackupArchiveRetention = splice(@stryTmp, 0, $iArchiveRetention); - - # For each archiveId, remove WAL that are not part of retention - foreach my $strArchiveId (@stryListArchiveDisk) - { - # From the global list of backups to retain, create a list of backups, oldest to newest, associated with this - # archiveId (e.g. 9.4-1) - my @stryLocalBackupRetention = $oBackupInfo->listByArchiveId($strArchiveId, - $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), \@stryGlobalBackupRetention, 'reverse'); - - # If no backup to retain was found - if (!@stryLocalBackupRetention) - { - # Get the backup db-id corresponding to this archiveId - my $iDbHistoryId = $oBackupInfo->backupArchiveDbHistoryId( - $strArchiveId, $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE)); - - # If this is not the current database, then delete the archive directory else do nothing since the current - # DB archive directory must not be deleted - if (!defined($iDbHistoryId) || !$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID, undef, - $iDbHistoryId)) - { - my $strFullPath = $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveId}"); - - $oStorageRepo->remove($strFullPath, {bRecurse => true}); - - &log(INFO, "remove archive path: ${strFullPath}"); - } - - # Continue to next directory - next; - } - - my @stryLocalBackupArchiveRentention; - - # If the archive retention is less than or equal to the number of all backups, then perform selective expiration - if (@stryGlobalBackupArchiveRetention && $iArchiveRetention <= scalar @stryGlobalBackupRetention) - { - # From the full list of backups in archive retention, find the intersection of local backups to retain - foreach my $strGlobalBackupArchiveRetention (@stryGlobalBackupArchiveRetention) - { - foreach my $strLocalBackupRetention (@stryLocalBackupRetention) - { - if ($strLocalBackupRetention eq $strGlobalBackupArchiveRetention) - { - unshift(@stryLocalBackupArchiveRentention, $strLocalBackupRetention); - } - } - } - } - # Else if there are not enough backups yet globally to start archive expiration then set the archive retention - # to the oldest backup so anything prior to that will be removed as it is not needed but everything else is - # This is incase there are old archives left around so that they don't stay around forever - else - { - if ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_FULL && scalar @stryLocalBackupRetention > 0) - { - &log(INFO, "full backup total < ${iArchiveRetention} - using oldest full backup for ${strArchiveId} " . - "archive retention"); - $stryLocalBackupArchiveRentention[0] = $stryLocalBackupRetention[0]; - } - } - - # If no local backups were found as part of retention then set the backup archive retention to the newest backup - # so that the database is fully recoverable (can be recovered from the last backup through pitr) - if (!@stryLocalBackupArchiveRentention) - { - $stryLocalBackupArchiveRentention[0] = $stryLocalBackupRetention[-1]; - } - - my $strArchiveRetentionBackup = $stryLocalBackupArchiveRentention[0]; - - # If a backup has been selected for retention then continue - if (defined($strArchiveRetentionBackup)) - { - my $bRemove; - - # Only expire if the selected backup has archive info - backups performed with --no-online will - # not have archive info and cannot be used for expiration. - if ($oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, - $strArchiveRetentionBackup, INFO_BACKUP_KEY_ARCHIVE_START)) - { - # Get archive ranges to preserve. Because archive retention can be less than total retention it is - # important to preserve archive that is required to make the older backups consistent even though they - # cannot be played any further forward with PITR. - my $strArchiveExpireMax; - my @oyArchiveRange; - my @stryBackupList = $oBackupInfo->list(); - - # With the full list of backups, loop through only those associated with this archiveId - foreach my $strBackup ( - $oBackupInfo->listByArchiveId( - $strArchiveId, $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), \@stryBackupList)) - { - if ($strBackup le $strArchiveRetentionBackup && - $oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_ARCHIVE_START)) - { - my $oArchiveRange = {}; - - $$oArchiveRange{start} = $oBackupInfo->get(INFO_BACKUP_SECTION_BACKUP_CURRENT, - $strBackup, INFO_BACKUP_KEY_ARCHIVE_START); - - if ($strBackup ne $strArchiveRetentionBackup) - { - $$oArchiveRange{stop} = $oBackupInfo->get(INFO_BACKUP_SECTION_BACKUP_CURRENT, - $strBackup, INFO_BACKUP_KEY_ARCHIVE_STOP); - } - else - { - $strArchiveExpireMax = $$oArchiveRange{start}; - } - - &log(DETAIL, "archive retention on backup ${strBackup}, archiveId = ${strArchiveId}, " . - "start = $$oArchiveRange{start}" . - (defined($$oArchiveRange{stop}) ? ", stop = $$oArchiveRange{stop}" : '')); - - push(@oyArchiveRange, $oArchiveRange); - } - } - - # Get all major archive paths (timeline and first 32 bits of LSN) - foreach my $strPath ($oStorageRepo->list( - STORAGE_REPO_ARCHIVE . "/${strArchiveId}", {strExpression => REGEX_ARCHIVE_DIR_WAL})) - { - logDebugMisc($strOperation, "found major WAL path: ${strPath}"); - $bRemove = true; - - # Keep the path if it falls in the range of any backup in retention - foreach my $oArchiveRange (@oyArchiveRange) - { - if ($strPath ge substr($$oArchiveRange{start}, 0, 16) && - (!defined($$oArchiveRange{stop}) || $strPath le substr($$oArchiveRange{stop}, 0, 16))) - { - $bRemove = false; - last; - } - } - - # Remove the entire directory if all archive is expired - if ($bRemove) - { - my $strFullPath = $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveId}") . "/${strPath}"; - - $oStorageRepo->remove($strFullPath, {bRecurse => true}); - - # Log expire info - logDebugMisc($strOperation, "remove major WAL path: ${strFullPath}"); - $self->logExpire($strArchiveId, $strPath); - } - # Else delete individual files instead if the major path is less than or equal to the most recent - # retention backup. This optimization prevents scanning though major paths that could not possibly - # have anything to expire. - elsif ($strPath le substr($strArchiveExpireMax, 0, 16)) - { - # Look for files in the archive directory - foreach my $strSubPath ($oStorageRepo->list( - STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strPath}", {strExpression => "^[0-F]{24}.*\$"})) - { - $bRemove = true; - - # Determine if the individual archive log is used in a backup - foreach my $oArchiveRange (@oyArchiveRange) - { - if (substr($strSubPath, 0, 24) ge $$oArchiveRange{start} && - (!defined($$oArchiveRange{stop}) || substr($strSubPath, 0, 24) le $$oArchiveRange{stop})) - { - $bRemove = false; - last; - } - } - - # Remove archive log if it is not used in a backup - if ($bRemove) - { - $oStorageRepo->remove(STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strSubPath}"); - - logDebugMisc($strOperation, "remove WAL segment: ${strArchiveId}/${strSubPath}"); - - # Log expire info - $self->logExpire($strArchiveId, substr($strSubPath, 0, 24)); - } - else - { - # Log that the file was not expired - $self->logExpire($strArchiveId); - } - } - } - } - - # Log if no archive was expired - if ($self->{iArchiveExpireTotal} == 0) - { - &log(DETAIL, "no archive to remove, archiveId = ${strArchiveId}"); - } - } - } - } - } - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -1; diff --git a/lib/pgBackRest/Main.pm b/lib/pgBackRest/Main.pm index 0e32e8c26..52f7273f4 100644 --- a/lib/pgBackRest/Main.pm +++ b/lib/pgBackRest/Main.pm @@ -13,11 +13,13 @@ $SIG{__DIE__} = sub {Carp::confess @_}; use File::Basename qw(dirname); +use pgBackRest::Backup::Info; use pgBackRest::Common::Exception; use pgBackRest::Common::Lock; use pgBackRest::Common::Log; use pgBackRest::Config::Config; use pgBackRest::Protocol::Helper; +use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; use pgBackRest::Version; @@ -245,11 +247,7 @@ sub main # -------------------------------------------------------------------------------------------------------------- elsif (cfgCommandTest(CFGCMD_EXPIRE)) { - # Load module dynamically - require pgBackRest::Expire; - pgBackRest::Expire->import(); - - new pgBackRest::Expire()->process(); + new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP)); } } } diff --git a/src/Makefile.in b/src/Makefile.in index 402c5698e..a5b05b28d 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -50,6 +50,8 @@ SRCS = \ command/archive/push/file.c \ command/archive/push/protocol.c \ command/archive/push/push.c \ + command/backup/common.c \ + command/expire/expire.c \ command/help/help.c \ command/info/info.c \ command/command.c \ @@ -223,6 +225,9 @@ command/command.o: command/command.c build.auto.h common/assert.h common/debug.h command/control/control.o: command/control/control.c build.auto.h command/control/control.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/control/control.c -o command/control/control.o +command/expire/expire.o: command/expire/expire.c build.auto.h command/archive/common.h command/backup/common.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoManifest.h info/infoPg.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h + $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/expire/expire.c -o command/expire/expire.o + command/help/help.o: command/help/help.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/help/help.c -o command/help/help.o @@ -421,7 +426,7 @@ info/infoManifest.o: info/infoManifest.c build.auto.h common/error.auto.h common info/infoPg.o: info/infoPg.c build.auto.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h info/info.h info/infoPg.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/infoPg.c -o info/infoPg.o -main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/command.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h version.h +main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/command.h command/expire/expire.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c main.c -o main.o perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h diff --git a/src/command/expire/expire.c b/src/command/expire/expire.c new file mode 100644 index 000000000..3cee9658b --- /dev/null +++ b/src/command/expire/expire.c @@ -0,0 +1,671 @@ +/*********************************************************************************************************************************** +Expire Command +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include "command/archive/common.h" +#include "command/backup/common.h" +#include "common/type/list.h" +#include "common/debug.h" +#include "common/regExp.h" +#include "config/config.h" +#include "info/info.h" +#include "info/infoArchive.h" +#include "info/infoBackup.h" +#include "info/infoManifest.h" +#include "storage/helper.h" + +#include + +/*********************************************************************************************************************************** +Helper functions and structures +***********************************************************************************************************************************/ +typedef struct ArchiveExpired +{ + uint64_t total; + String *start; + String *stop; +} ArchiveExpired; + +static int +archiveIdAscComparator(const void *item1, const void *item2) +{ + StringList *archiveSort1 = strLstNewSplitZ(*(String **)item1, "-"); + StringList *archiveSort2 = strLstNewSplitZ(*(String **)item2, "-"); + int int1 = atoi(strPtr(strLstGet(archiveSort1, 1))); + int int2 = atoi(strPtr(strLstGet(archiveSort2, 1))); + + return (int1 - int2); +} + +static int +archiveIdDescComparator(const void *item1, const void *item2) +{ + StringList *archiveSort1 = strLstNewSplitZ(*(String **)item1, "-"); + StringList *archiveSort2 = strLstNewSplitZ(*(String **)item2, "-"); + int int1 = atoi(strPtr(strLstGet(archiveSort1, 1))); + int int2 = atoi(strPtr(strLstGet(archiveSort2, 1))); + + return (int2 - int1); +} + +static StringList * +sortArchiveId(StringList *sortString, SortOrder sortOrder) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(STRING_LIST, sortString); + FUNCTION_LOG_PARAM(ENUM, sortOrder); + FUNCTION_LOG_END(); + + ASSERT(sortString != NULL); + + switch (sortOrder) + { + case sortOrderAsc: + { + lstSort((List *)sortString, archiveIdAscComparator); + break; + } + + case sortOrderDesc: + { + lstSort((List *)sortString, archiveIdDescComparator); + break; + } + } + + FUNCTION_LOG_RETURN(STRING_LIST, sortString); +} + +typedef struct ArchiveRange +{ + const String *start; + const String *stop; +} ArchiveRange; + +/*********************************************************************************************************************************** +Common function for expiring any backup +***********************************************************************************************************************************/ +static void +expireBackup(InfoBackup *infoBackup, String *removeBackupLabel, String *backupExpired) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup); + FUNCTION_LOG_PARAM(STRING, removeBackupLabel); + FUNCTION_LOG_PARAM(STRING, backupExpired); + FUNCTION_LOG_END(); + + ASSERT(infoBackup != NULL); + ASSERT(removeBackupLabel != NULL); + ASSERT(backupExpired != NULL); + + storageRemoveNP( + storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/" INFO_MANIFEST_FILE, strPtr(removeBackupLabel))); + + storageRemoveNP( + storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/" INFO_MANIFEST_FILE INFO_COPY_EXT, strPtr(removeBackupLabel))); + + // Remove the backup from the info file + infoBackupDataDelete(infoBackup, removeBackupLabel); + + if (strSize(backupExpired) == 0) + strCat(backupExpired, strPtr(removeBackupLabel)); + else + strCatFmt(backupExpired, ", %s", strPtr(removeBackupLabel)); + + FUNCTION_LOG_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Expire differential backups +***********************************************************************************************************************************/ +static unsigned int +expireDiffBackup(InfoBackup *infoBackup) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup); + FUNCTION_LOG_END(); + + ASSERT(infoBackup != NULL); + + unsigned int result = 0; + + MEM_CONTEXT_TEMP_BEGIN() + { + unsigned int differentialRetention = cfgOptionTest(cfgOptRepoRetentionDiff) ? cfgOptionUInt(cfgOptRepoRetentionDiff) : 0; + + // Find all the expired differential backups + if (differentialRetention > 0) + { + // Get a list of full and differential backups. Full are considered differential for the purpose of retention. + // Example: F1, D1, D2, F2, repo-retention-diff=2, then F1,D2,F2 will be retained, not D2 and D1 as might be expected. + StringList *currentBackupList = infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true, .differential = true)); + + // If there are more backups than the number to retain, then expire the oldest ones + if (strLstSize(currentBackupList) > differentialRetention) + { + for (unsigned int diffIdx = 0; diffIdx < strLstSize(currentBackupList) - differentialRetention; diffIdx++) + { + // Skip if this is a full backup. Full backups only count as differential when deciding which differential + // backups to expire. + if (regExpMatchOne(backupRegExpP(.full = true), strLstGet(currentBackupList, diffIdx))) + continue; + + // Get a list of all differential and incremental backups + StringList *removeList = infoBackupDataLabelList( + infoBackup, backupRegExpP(.differential = true, .incremental = true)); + + // Initialize the log message + String *backupExpired = strNew(""); + + // Remove the manifest files in each directory and remove the backup from the current section of backup.info + for (unsigned int rmvIdx = 0; rmvIdx < strLstSize(removeList); rmvIdx++) + { + // Remove all differential and incremental backups before the oldest valid differential + // (removeBackupLabel < oldest valid differential) + String *removeBackupLabel = strLstGet(removeList, rmvIdx); + + if (strCmp(removeBackupLabel, strLstGet(currentBackupList, diffIdx + 1)) < 0) + { + expireBackup(infoBackup, removeBackupLabel, backupExpired); + result++; + } + } + + // If the message contains a comma, then prepend "set:" + LOG_INFO("expire diff backup %s%s", (strChr(backupExpired, ',') != -1 ? "set: " : ""), strPtr(backupExpired)); + } + } + } + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(UINT, result); +} + +/*********************************************************************************************************************************** +Expire full backups +***********************************************************************************************************************************/ +static unsigned int +expireFullBackup(InfoBackup *infoBackup) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup); + FUNCTION_LOG_END(); + + ASSERT(infoBackup != NULL); + + unsigned int result = 0; + + MEM_CONTEXT_TEMP_BEGIN() + { + unsigned int fullRetention = cfgOptionTest(cfgOptRepoRetentionFull) ? cfgOptionUInt(cfgOptRepoRetentionFull) : 0; + + // Find all the expired full backups + if (fullRetention > 0) + { + // Get list of current full backups (default order is oldest to newest) + StringList *currentBackupList = infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true)); + + // If there are more full backups then the number to retain, then expire the oldest ones + if (strLstSize(currentBackupList) > fullRetention) + { + // Expire all backups that depend on the full backup + for (unsigned int fullIdx = 0; fullIdx < strLstSize(currentBackupList) - fullRetention; fullIdx++) + { + // The list of backups to remove includes the full backup and the default sort order will put it first + StringList *removeList = infoBackupDataLabelList( + infoBackup, strNewFmt("^%s.*", strPtr(strLstGet(currentBackupList, fullIdx)))); + + // Initialize the log message + String *backupExpired = strNew(""); + + // Remove the manifest files in each directory and remove the backup from the current section of backup.info + for (unsigned int rmvIdx = 0; rmvIdx < strLstSize(removeList); rmvIdx++) + { + String *removeBackupLabel = strLstGet(removeList, rmvIdx); + expireBackup(infoBackup, removeBackupLabel, backupExpired); + result++; + } + + // If the message contains a comma, then prepend "set:" + LOG_INFO("expire full backup %s%s", (strChr(backupExpired, ',') != -1 ? "set: " : ""), strPtr(backupExpired)); + } + } + } + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(UINT, result); +} + +/*********************************************************************************************************************************** +Log detailed information about archive logs removed +***********************************************************************************************************************************/ +static void +logExpire(ArchiveExpired *archiveExpire, String *archiveId) +{ + if (archiveExpire->start != NULL) + { + // Force out any remaining message + LOG_DETAIL( + "remove archive: archiveId = %s, start = %s, stop = %s", strPtr(archiveId), strPtr(archiveExpire->start), + strPtr(archiveExpire->stop)); + + archiveExpire->start = NULL; + } +} + +/*********************************************************************************************************************************** +Process archive retention +***********************************************************************************************************************************/ +static void +removeExpiredArchive(InfoBackup *infoBackup) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup); + FUNCTION_LOG_END(); + + ASSERT(infoBackup != NULL); + + MEM_CONTEXT_TEMP_BEGIN() + { + // Get the retention options. repo-archive-retention-type always has a value as it defaults to "full" + const String *archiveRetentionType = cfgOptionStr(cfgOptRepoRetentionArchiveType); + unsigned int archiveRetention = cfgOptionTest(cfgOptRepoRetentionArchive) ? cfgOptionUInt(cfgOptRepoRetentionArchive) : 0; + + // If archive retention is undefined, then ignore archiving. The user does not have to set this - it will be defaulted in + // cfgLoadUpdateOption based on certain rules. + if (archiveRetention == 0) + { + LOG_INFO("option '%s' is not set - archive logs will not be expired", cfgOptionName(cfgOptRepoRetentionArchive)); + } + else + { + // Determine which backup type to use for archive retention (full, differential, incremental) and get a list of the + // remaining non-expired backups, from newest to oldest, based on the type. + StringList *globalBackupRetentionList = NULL; + + if (strCmp(archiveRetentionType, STRDEF(CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_FULL)) == 0) + { + globalBackupRetentionList = strLstSort( + infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true)), sortOrderDesc); + } + else if (strCmp(archiveRetentionType, STRDEF(CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_DIFF)) == 0) + { + globalBackupRetentionList = strLstSort( + infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true, .differential = true)), sortOrderDesc); + } + else + { // Incrementals can depend on Full or Diff so get a list of all incrementals + globalBackupRetentionList = strLstSort( + infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true, .differential = true, .incremental = true)), + sortOrderDesc); + } + + // Expire archives. If no backups were found then preserve current archive logs - too soon to expire them. + if (strLstSize(globalBackupRetentionList) > 0) + { + // Attempt to load the archive info file + InfoArchive *infoArchive = infoArchiveNewLoad( + storageRepo(), STRDEF(STORAGE_REPO_ARCHIVE "/" INFO_ARCHIVE_FILE), + cipherType(cfgOptionStr(cfgOptRepoCipherType)), cfgOptionStr(cfgOptRepoCipherPass)); + + InfoPg *infoArchivePgData = infoArchivePg(infoArchive); + + // Get a list of archive directories (e.g. 9.4-1, 10-2, etc) sorted by the db-id (number after the dash). + StringList *listArchiveDisk = sortArchiveId( + storageListP(storageRepo(), STRDEF(STORAGE_REPO_ARCHIVE), .expression = STRDEF(REGEX_ARCHIVE_DIR_DB_VERSION)), + sortOrderAsc); + + StringList *globalBackupArchiveRetentionList = strLstNew(); + + // globalBackupRetentionList is ordered newest to oldest backup, so create globalBackupArchiveRetentionList of the + // newest backups whose archives will be retained + for (unsigned int idx = 0; + idx < (archiveRetention < strLstSize(globalBackupRetentionList) ? + archiveRetention : strLstSize(globalBackupRetentionList)); + idx++) + { + strLstAdd(globalBackupArchiveRetentionList, strLstGet(globalBackupRetentionList, idx)); + } + + // Loop through the archive.info history from oldest to newest and if there is a corresponding directory on disk + // then remove WAL that are not part of retention + for (unsigned int pgIdx = infoPgDataTotal(infoArchivePgData) - 1; (int)pgIdx >= 0; pgIdx--) + { + String *archiveId = infoPgArchiveId(infoArchivePgData, pgIdx); + StringList *localBackupRetentionList = strLstNew(); + + // Initialize the expired archive information for this archive ID + ArchiveExpired archiveExpire = {.total = 0, .start = NULL, .stop = NULL}; + + for (unsigned int archiveIdx = 0; archiveIdx < strLstSize(listArchiveDisk); archiveIdx++) + { + // Is there an archive directory for this archvieId? + if (strCmp(archiveId, strLstGet(listArchiveDisk, archiveIdx)) != 0) + continue; + + StringList *archiveSplit = strLstNewSplitZ(archiveId, "-"); + unsigned int archivePgId = cvtZToUInt(strPtr(strLstGet(archiveSplit, 1))); + + // From the global list of backups to retain, create a list of backups, oldest to newest, associated with + // this archiveId (e.g. 9.4-1), e.g. If globalBackupRetention has 4F, 3F, 2F, 1F then + // localBackupRetentionList will have 1F, 2F, 3F, 4F (assuming they all have same history id) + for (unsigned int retentionIdx = strLstSize(globalBackupRetentionList) - 1; + (int)retentionIdx >=0; retentionIdx--) + { + for (unsigned int backupIdx = 0; backupIdx < infoBackupDataTotal(infoBackup); backupIdx++) + { + InfoBackupData backupData = infoBackupData(infoBackup, backupIdx); + + if (strCmp(backupData.backupLabel, strLstGet(globalBackupRetentionList, retentionIdx)) == 0 && + backupData.backupPgId == archivePgId) + { + strLstAdd(localBackupRetentionList, backupData.backupLabel); + } + } + } + + // If no backup to retain was found + if (strLstSize(localBackupRetentionList) == 0) + { + // If this is not the current database, then delete the archive directory else do nothing since the + // current DB archive directory must not be deleted + InfoPgData currentPg = infoPgDataCurrent(infoArchivePgData); + + if (currentPg.id != archivePgId) + { + String *fullPath = storagePath( + storageRepo(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strPtr(archiveId))); + storagePathRemoveP(storageRepoWrite(), fullPath, .recurse = true); + LOG_INFO("remove archive path: %s", strPtr(fullPath)); + } + + // Continue to next directory + break; + } + + // If we get here, then a local backup was found for retention + StringList *localBackupArchiveRententionList = strLstNew(); + + // If the archive retention is less than or equal to the number of all backups, then perform selective + // expiration + if (archiveRetention <= strLstSize(globalBackupRetentionList)) + { + // From the full list of backups in archive retention, find the intersection of local backups to retain + // from oldest to newest + for (unsigned int globalIdx = strLstSize(globalBackupArchiveRetentionList) - 1; + (int)globalIdx >= 0; globalIdx--) + { + for (unsigned int localIdx = 0; localIdx < strLstSize(localBackupRetentionList); localIdx++) + { + if (strCmp( + strLstGet(globalBackupArchiveRetentionList, globalIdx), + strLstGet(localBackupRetentionList, localIdx)) == 0) + { + strLstAdd(localBackupArchiveRententionList, strLstGet(localBackupRetentionList, localIdx)); + } + } + } + } + // Else if there are not enough backups yet globally to start archive expiration then set the archive + // retention to the oldest backup so anything prior to that will be removed as it is not needed but + // everything else is. This is incase there are old archives left around so that they don't stay around + // forever. + else + { + LOG_INFO( + "full backup total < %u - using oldest full backup for %s archive retention", archiveRetention, + strPtr(archiveId)); + strLstAdd(localBackupArchiveRententionList, strLstGet(localBackupRetentionList, 0)); + } + + // If no local backups were found as part of retention then set the backup archive retention to the newest + // backup so that the database is fully recoverable (can be recovered from the last backup through pitr) + if (strLstSize(localBackupArchiveRententionList) == 0) + { + strLstAdd( + localBackupArchiveRententionList, + strLstGet(localBackupRetentionList, strLstSize(localBackupRetentionList) - 1)); + } + + // Get the data for the backup selected for retention and all backups associated with this archive id + List *archiveIdBackupList = lstNew(sizeof(InfoBackupData)); + InfoBackupData archiveRetentionBackup = {0}; + + for (unsigned int infoBackupIdx = 0; infoBackupIdx < infoBackupDataTotal(infoBackup); infoBackupIdx++) + { + InfoBackupData archiveIdBackup = infoBackupData(infoBackup, infoBackupIdx); + + // If this is the backup selected for retention, store its data + if (strCmp(archiveIdBackup.backupLabel, strLstGet(localBackupArchiveRententionList, 0)) == 0) + archiveRetentionBackup = infoBackupData(infoBackup, infoBackupIdx); + + // If this is a backup associated with this archive Id, then add it to the list to check + if (archiveIdBackup.backupPgId == archivePgId) + lstAdd(archiveIdBackupList, &archiveIdBackup); + } + + // Only expire if the selected backup has archive data - backups performed with --no-online will + // not have archive data and cannot be used for expiration. + bool removeArchive = false; + + if (archiveRetentionBackup.backupArchiveStart != NULL) + { + // Get archive ranges to preserve. Because archive retention can be less than total retention it is + // important to preserve archive that is required to make the older backups consistent even though they + // cannot be played any further forward with PITR. + String *archiveExpireMax = NULL; + List *archiveRangeList = lstNew(sizeof(ArchiveRange)); + + // From the full list of backups, loop through those associated with this archiveId + for (unsigned int backupListIdx = 0; backupListIdx < lstSize(archiveIdBackupList); backupListIdx++) + { + InfoBackupData *backupData = lstGet(archiveIdBackupList, backupListIdx); + + // If the backup is earlier than or the same as the retention backup and the backup has an + // archive start + if (strCmp(backupData->backupLabel, archiveRetentionBackup.backupLabel) <= 0 && + backupData->backupArchiveStart != NULL) + { + ArchiveRange archiveRange = + { + .start = strDup(backupData->backupArchiveStart), + .stop = NULL, + }; + + // If this is not the retention backup, then set the stop, otherwise set the expire max to + // the archive start of the archive to retain + if (strCmp(backupData->backupLabel, archiveRetentionBackup.backupLabel) != 0) + archiveRange.stop = strDup(backupData->backupArchiveStop); + else + archiveExpireMax = strDup(archiveRange.start); + + LOG_DETAIL( + "archive retention on backup %s, archiveId = %s, start = %s%s", + strPtr(backupData->backupLabel), strPtr(archiveId), strPtr(archiveRange.start), + archiveRange.stop != NULL ? + strPtr(strNewFmt(", stop = %s", strPtr(archiveRange.stop))) : ""); + + // Add the archive range to the list + lstAdd(archiveRangeList, &archiveRange); + } + } + + // Get all major archive paths (timeline and first 32 bits of LSN) + StringList *walPathList = + strLstSort( + storageListP( + storageRepo(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strPtr(archiveId)), + .expression = STRDEF(WAL_SEGMENT_DIR_REGEXP)), + sortOrderAsc); + + for (unsigned int walIdx = 0; walIdx < strLstSize(walPathList); walIdx++) + { + String *walPath = strLstGet(walPathList, walIdx); + removeArchive = true; + + // Keep the path if it falls in the range of any backup in retention + for (unsigned int rangeIdx = 0; rangeIdx < lstSize(archiveRangeList); rangeIdx++) + { + ArchiveRange *archiveRange = lstGet(archiveRangeList, rangeIdx); + + if (strCmp(walPath, strSubN(archiveRange->start, 0, 16)) >= 0 && + (archiveRange->stop == NULL || strCmp(walPath, strSubN(archiveRange->stop, 0, 16)) <= 0)) + { + removeArchive = false; + break; + } + } + + // Remove the entire directory if all archive is expired + if (removeArchive) + { + storagePathRemoveP( + storageRepoWrite(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), + strPtr(walPath)), .recurse = true); + + archiveExpire.total++; + archiveExpire.start = strDup(walPath); + archiveExpire.stop = strDup(walPath); + } + // Else delete individual files instead if the major path is less than or equal to the most recent + // retention backup. This optimization prevents scanning though major paths that could not possibly + // have anything to expire. + else if (strCmp(walPath, strSubN(archiveExpireMax, 0, 16)) <= 0) + { + // Look for files in the archive directory + StringList *walSubPathList = + strLstSort( + storageListP( + storageRepo(), + strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(walPath)), + .expression = STRDEF("^[0-F]{24}.*$")), + sortOrderAsc); + + for (unsigned int subIdx = 0; subIdx < strLstSize(walSubPathList); subIdx++) + { + removeArchive = true; + String *walSubPath = strLstGet(walSubPathList, subIdx); + + // Determine if the individual archive log is used in a backup + for (unsigned int rangeIdx = 0; rangeIdx < lstSize(archiveRangeList); rangeIdx++) + { + ArchiveRange *archiveRange = lstGet(archiveRangeList, rangeIdx); + + if (strCmp(strSubN(walSubPath, 0, 24), archiveRange->start) >= 0 && + (archiveRange->stop == NULL || + strCmp(strSubN(walSubPath, 0, 24), archiveRange->stop) <= 0)) + { + removeArchive = false; + break; + } + } + + // Remove archive log if it is not used in a backup + if (removeArchive) + { + storageRemoveNP( + storageRepoWrite(), + strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s/%s", + strPtr(archiveId), strPtr(walPath), strPtr(walSubPath))); + + // Track that this archive was removed + archiveExpire.total++; + archiveExpire.stop = strDup(strSubN(walSubPath, 0, 24)); + if (archiveExpire.start == NULL) + archiveExpire.start = strDup(strSubN(walSubPath, 0, 24)); + } + else + logExpire(&archiveExpire, archiveId); + } + } + } + + // Log if no archive was expired + if (archiveExpire.total == 0) + { + LOG_DETAIL("no archive to remove, archiveId = %s", strPtr(archiveId)); + } + // Log if there is more to log + else + logExpire(&archiveExpire, archiveId); + } + } + } + } + } + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Remove expired backups from repo +***********************************************************************************************************************************/ +static void +removeExpiredBackup(InfoBackup *infoBackup) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup); + FUNCTION_LOG_END(); + + ASSERT(infoBackup != NULL); + + // Get all the current backups in backup.info + StringList *currentBackupList = strLstSort(infoBackupDataLabelList(infoBackup, NULL), sortOrderDesc); + StringList *backupList = strLstSort( + storageListP( + storageRepo(), STRDEF(STORAGE_REPO_BACKUP), .expression = backupRegExpP(.full = true, .differential = true, + .incremental = true)), + sortOrderDesc); + + // Remove non-current backups from disk + for (unsigned int backupIdx = 0; backupIdx < strLstSize(backupList); backupIdx++) + { + if (!strLstExists(currentBackupList, strLstGet(backupList, backupIdx))) + { + LOG_INFO("remove expired backup %s", strPtr(strLstGet(backupList, backupIdx))); + + storagePathRemoveP( + storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s", strPtr(strLstGet(backupList, backupIdx))), + .recurse = true); + } + } + + FUNCTION_LOG_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Expire backups and archives +***********************************************************************************************************************************/ +void +cmdExpire(void) +{ + FUNCTION_LOG_VOID(logLevelDebug); + + MEM_CONTEXT_TEMP_BEGIN() + { + // Get the repo storage in case it is remote and encryption settings need to be pulled down + storageRepo(); + + // Load the backup.info + InfoBackup *infoBackup = infoBackupNewLoad( + storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/" INFO_BACKUP_FILE), cipherType(cfgOptionStr(cfgOptRepoCipherType)), + cfgOptionStr(cfgOptRepoCipherPass)); + + expireFullBackup(infoBackup); + expireDiffBackup(infoBackup); + + infoBackupSave(infoBackup, storageRepoWrite(), STRDEF(STORAGE_REPO_BACKUP "/" INFO_BACKUP_FILE), + cipherType(cfgOptionStr(cfgOptRepoCipherType)), cfgOptionStr(cfgOptRepoCipherPass)); + + removeExpiredBackup(infoBackup); + removeExpiredArchive(infoBackup); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN_VOID(); +} diff --git a/src/command/expire/expire.h b/src/command/expire/expire.h new file mode 100644 index 000000000..f49e2aa00 --- /dev/null +++ b/src/command/expire/expire.h @@ -0,0 +1,12 @@ +/*********************************************************************************************************************************** +Expire Command +***********************************************************************************************************************************/ +#ifndef COMMAND_EXPIRE_EXPIRE_H +#define COMMAND_EXPIRE_EXPIRE_H + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +void cmdExpire(void); + +#endif diff --git a/src/main.c b/src/main.c index 50c326aee..e847a6f8b 100644 --- a/src/main.c +++ b/src/main.c @@ -10,6 +10,7 @@ Main #include "command/archive/get/get.h" #include "command/archive/push/push.h" #include "command/command.h" +#include "command/expire/expire.h" #include "command/help/help.h" #include "command/info/info.h" #include "command/local/local.h" @@ -115,6 +116,7 @@ main(int argListSize, const char *argList[]) // Run expire perlExec(); + cmdExpire(); break; } @@ -132,6 +134,7 @@ main(int argListSize, const char *argList[]) case cfgCmdExpire: { perlExec(); + cmdExpire(); break; } diff --git a/src/perl/embed.auto.c b/src/perl/embed.auto.c index de0a09d14..b7d525fab 100644 --- a/src/perl/embed.auto.c +++ b/src/perl/embed.auto.c @@ -8814,407 +8814,6 @@ static const EmbeddedModule embeddedModule[] = "\n" "1;\n" }, - { - .name = "pgBackRest/Expire.pm", - .data = - "\n\n\n" - "package pgBackRest::Expire;\n" - "\n" - "use strict;\n" - "use warnings FATAL => qw(all);\n" - "use Carp qw(confess);\n" - "\n" - "use Exporter qw(import);\n" - "use File::Basename qw(dirname);\n" - "use Scalar::Util qw(looks_like_number);\n" - "\n" - "use pgBackRest::Archive::Common;\n" - "use pgBackRest::Archive::Info;\n" - "use pgBackRest::Common::Exception;\n" - "use pgBackRest::Common::Ini;\n" - "use pgBackRest::Common::Log;\n" - "use pgBackRest::Backup::Common;\n" - "use pgBackRest::Backup::Info;\n" - "use pgBackRest::Config::Config;\n" - "use pgBackRest::InfoCommon;\n" - "use pgBackRest::Manifest;\n" - "use pgBackRest::Protocol::Helper;\n" - "use pgBackRest::Protocol::Storage::Helper;\n" - "\n\n\n\n" - "sub new\n" - "{\n" - "my $class = shift;\n" - "\n\n" - "my $self = {};\n" - "bless $self, $class;\n" - "\n\n" - "my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');\n" - "\n\n" - "$self->{iArchiveExpireTotal} = 0;\n" - "\n\n" - "return logDebugReturn\n" - "(\n" - "$strOperation,\n" - "{name => 'self', value => $self}\n" - ");\n" - "}\n" - "\n\n\n\n\n\n" - "sub logExpire\n" - "{\n" - "my $self = shift;\n" - "my $strArchiveId = shift;\n" - "my $strArchiveFile = shift;\n" - "\n" - "if (defined($strArchiveFile))\n" - "{\n" - "if (!defined($self->{strArchiveExpireStart}))\n" - "{\n" - "$self->{strArchiveExpireStart} = $strArchiveFile;\n" - "$self->{strArchiveExpireStop} = $strArchiveFile;\n" - "}\n" - "else\n" - "{\n" - "$self->{strArchiveExpireStop} = $strArchiveFile;\n" - "}\n" - "\n" - "$self->{iArchiveExpireTotal}++;\n" - "}\n" - "else\n" - "{\n" - "if (defined($self->{strArchiveExpireStart}))\n" - "{\n" - "&log(DETAIL, \"remove archive: archiveId = ${strArchiveId}, start = \" . substr($self->{strArchiveExpireStart}, 0, 24) .\n" - "\", stop = \" . substr($self->{strArchiveExpireStop}, 0, 24));\n" - "}\n" - "\n" - "undef($self->{strArchiveExpireStart});\n" - "}\n" - "}\n" - "\n\n\n\n\n\n\n" - "sub process\n" - "{\n" - "my $self = shift;\n" - "\n\n" - "my ($strOperation) = logDebugParam(__PACKAGE__ . '->process');\n" - "\n" - "my @stryPath;\n" - "\n" - "my $oStorageRepo = storageRepo();\n" - "my $strBackupClusterPath = $oStorageRepo->pathGet(STORAGE_REPO_BACKUP);\n" - "my $iFullRetention = cfgOption(CFGOPT_REPO_RETENTION_FULL, false);\n" - "my $iDifferentialRetention = cfgOption(CFGOPT_REPO_RETENTION_DIFF, false);\n" - "my $strArchiveRetentionType = cfgOption(CFGOPT_REPO_RETENTION_ARCHIVE_TYPE, false);\n" - "my $iArchiveRetention = cfgOption(CFGOPT_REPO_RETENTION_ARCHIVE, false);\n" - "\n\n" - "my $oBackupInfo = new pgBackRest::Backup::Info($oStorageRepo->pathGet(STORAGE_REPO_BACKUP));\n" - "\n\n" - "if (defined($iFullRetention))\n" - "{\n" - "\n" - "if (!looks_like_number($iFullRetention) || $iFullRetention < 1)\n" - "{\n" - "confess &log(ERROR, cfgOptionName(CFGOPT_REPO_RETENTION_FULL) . ' must be a number >= 1');\n" - "}\n" - "\n" - "@stryPath = $oBackupInfo->list(backupRegExpGet(true));\n" - "\n" - "if (@stryPath > $iFullRetention)\n" - "{\n" - "\n" - "for (my $iFullIdx = 0; $iFullIdx < @stryPath - $iFullRetention; $iFullIdx++)\n" - "{\n" - "my @stryRemoveList;\n" - "\n" - "foreach my $strPath ($oBackupInfo->list('^' . $stryPath[$iFullIdx] . '.*'))\n" - "{\n" - "$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strPath}/\" . FILE_MANIFEST . INI_COPY_EXT);\n" - "$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strPath}/\" . FILE_MANIFEST);\n" - "$oBackupInfo->delete($strPath);\n" - "\n" - "if ($strPath ne $stryPath[$iFullIdx])\n" - "{\n" - "push(@stryRemoveList, $strPath);\n" - "}\n" - "}\n" - "\n" - "&log(INFO, 'expire full backup ' . (@stryRemoveList > 0 ? 'set: ' : '') . $stryPath[$iFullIdx] .\n" - "(@stryRemoveList > 0 ? ', ' . join(', ', @stryRemoveList) : ''));\n" - "}\n" - "}\n" - "}\n" - "\n\n" - "if (defined($iDifferentialRetention))\n" - "{\n" - "\n" - "if (!looks_like_number($iDifferentialRetention) || $iDifferentialRetention < 1)\n" - "{\n" - "confess &log(ERROR, cfgOptionName(CFGOPT_REPO_RETENTION_DIFF) . ' must be a number >= 1');\n" - "}\n" - "\n\n\n" - "@stryPath = $oBackupInfo->list(backupRegExpGet(true, true));\n" - "\n" - "if (@stryPath > $iDifferentialRetention)\n" - "{\n" - "for (my $iDiffIdx = 0; $iDiffIdx < @stryPath - $iDifferentialRetention; $iDiffIdx++)\n" - "{\n" - "\n\n" - "next if ($stryPath[$iDiffIdx] =~ backupRegExpGet(true));\n" - "\n\n" - "my @stryRemoveList;\n" - "\n" - "foreach my $strPath ($oBackupInfo->list(backupRegExpGet(false, true, true)))\n" - "{\n" - "logDebugMisc($strOperation, \"checking ${strPath} for differential expiration\");\n" - "\n\n" - "if ($strPath lt $stryPath[$iDiffIdx + 1])\n" - "{\n" - "$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strPath}\" . FILE_MANIFEST);\n" - "$oBackupInfo->delete($strPath);\n" - "\n" - "if ($strPath ne $stryPath[$iDiffIdx])\n" - "{\n" - "push(@stryRemoveList, $strPath);\n" - "}\n" - "}\n" - "}\n" - "\n" - "&log(INFO, 'expire diff backup ' . (@stryRemoveList > 0 ? 'set: ' : '') . $stryPath[$iDiffIdx] .\n" - "(@stryRemoveList > 0 ? ', ' . join(', ', @stryRemoveList) : ''));\n" - "}\n" - "}\n" - "}\n" - "\n" - "$oBackupInfo->save();\n" - "\n\n" - "foreach my $strBackup ($oStorageRepo->list(\n" - "STORAGE_REPO_BACKUP, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))\n" - "{\n" - "if (!$oBackupInfo->current($strBackup))\n" - "{\n" - "&log(INFO, \"remove expired backup ${strBackup}\");\n" - "\n" - "$oStorageRepo->remove(\"${strBackupClusterPath}/${strBackup}\", {bRecurse => true});\n" - "}\n" - "}\n" - "\n\n" - "if (!defined($iArchiveRetention))\n" - "{\n" - "&log(INFO, \"option '\" . cfgOptionName(CFGOPT_REPO_RETENTION_ARCHIVE) . \"' is not set - archive logs will not be expired\");\n" - "}\n" - "else\n" - "{\n" - "my @stryGlobalBackupRetention;\n" - "\n\n\n" - "if ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_FULL)\n" - "{\n" - "@stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true), 'reverse');\n" - "}\n" - "elsif ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_DIFF)\n" - "{\n" - "@stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true, true), 'reverse');\n" - "}\n" - "elsif ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_INCR)\n" - "{\n" - "@stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true, true, true), 'reverse');\n" - "}\n" - "\n\n" - "my $iBackupTotal = scalar @stryGlobalBackupRetention;\n" - "\n" - "if ($iBackupTotal > 0)\n" - "{\n" - "my $oArchiveInfo = new pgBackRest::Archive::Info($oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), true);\n" - "my @stryListArchiveDisk = sort {((split('-', $a))[1] + 0) cmp ((split('-', $b))[1] + 0)} $oStorageRepo->list(\n" - "STORAGE_REPO_ARCHIVE, {strExpression => REGEX_ARCHIVE_DIR_DB_VERSION, bIgnoreMissing => true});\n" - "\n\n" - "if (!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,\n" - "($oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION)))) ||\n" - "!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef,\n" - "($oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID)))))\n" - "{\n" - "confess &log(ERROR, \"archive and backup database versions do not match\\n\" .\n" - "\"HINT: has a stanza-upgrade been performed?\", ERROR_FILE_INVALID);\n" - "}\n" - "\n\n" - "my @stryTmp = @stryGlobalBackupRetention;\n" - "my @stryGlobalBackupArchiveRetention = splice(@stryTmp, 0, $iArchiveRetention);\n" - "\n\n" - "foreach my $strArchiveId (@stryListArchiveDisk)\n" - "{\n" - "\n\n" - "my @stryLocalBackupRetention = $oBackupInfo->listByArchiveId($strArchiveId,\n" - "$oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), \\@stryGlobalBackupRetention, 'reverse');\n" - "\n\n" - "if (!@stryLocalBackupRetention)\n" - "{\n" - "\n" - "my $iDbHistoryId = $oBackupInfo->backupArchiveDbHistoryId(\n" - "$strArchiveId, $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE));\n" - "\n\n\n" - "if (!defined($iDbHistoryId) || !$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID, undef,\n" - "$iDbHistoryId))\n" - "{\n" - "my $strFullPath = $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE . \"/${strArchiveId}\");\n" - "\n" - "$oStorageRepo->remove($strFullPath, {bRecurse => true});\n" - "\n" - "&log(INFO, \"remove archive path: ${strFullPath}\");\n" - "}\n" - "\n\n" - "next;\n" - "}\n" - "\n" - "my @stryLocalBackupArchiveRentention;\n" - "\n\n" - "if (@stryGlobalBackupArchiveRetention && $iArchiveRetention <= scalar @stryGlobalBackupRetention)\n" - "{\n" - "\n" - "foreach my $strGlobalBackupArchiveRetention (@stryGlobalBackupArchiveRetention)\n" - "{\n" - "foreach my $strLocalBackupRetention (@stryLocalBackupRetention)\n" - "{\n" - "if ($strLocalBackupRetention eq $strGlobalBackupArchiveRetention)\n" - "{\n" - "unshift(@stryLocalBackupArchiveRentention, $strLocalBackupRetention);\n" - "}\n" - "}\n" - "}\n" - "}\n" - "\n\n\n" - "else\n" - "{\n" - "if ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_FULL && scalar @stryLocalBackupRetention > 0)\n" - "{\n" - "&log(INFO, \"full backup total < ${iArchiveRetention} - using oldest full backup for ${strArchiveId} \" .\n" - "\"archive retention\");\n" - "$stryLocalBackupArchiveRentention[0] = $stryLocalBackupRetention[0];\n" - "}\n" - "}\n" - "\n\n\n" - "if (!@stryLocalBackupArchiveRentention)\n" - "{\n" - "$stryLocalBackupArchiveRentention[0] = $stryLocalBackupRetention[-1];\n" - "}\n" - "\n" - "my $strArchiveRetentionBackup = $stryLocalBackupArchiveRentention[0];\n" - "\n\n" - "if (defined($strArchiveRetentionBackup))\n" - "{\n" - "my $bRemove;\n" - "\n\n\n" - "if ($oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT,\n" - "$strArchiveRetentionBackup, INFO_BACKUP_KEY_ARCHIVE_START))\n" - "{\n" - "\n\n\n" - "my $strArchiveExpireMax;\n" - "my @oyArchiveRange;\n" - "my @stryBackupList = $oBackupInfo->list();\n" - "\n\n" - "foreach my $strBackup (\n" - "$oBackupInfo->listByArchiveId(\n" - "$strArchiveId, $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), \\@stryBackupList))\n" - "{\n" - "if ($strBackup le $strArchiveRetentionBackup &&\n" - "$oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_ARCHIVE_START))\n" - "{\n" - "my $oArchiveRange = {};\n" - "\n" - "$$oArchiveRange{start} = $oBackupInfo->get(INFO_BACKUP_SECTION_BACKUP_CURRENT,\n" - "$strBackup, INFO_BACKUP_KEY_ARCHIVE_START);\n" - "\n" - "if ($strBackup ne $strArchiveRetentionBackup)\n" - "{\n" - "$$oArchiveRange{stop} = $oBackupInfo->get(INFO_BACKUP_SECTION_BACKUP_CURRENT,\n" - "$strBackup, INFO_BACKUP_KEY_ARCHIVE_STOP);\n" - "}\n" - "else\n" - "{\n" - "$strArchiveExpireMax = $$oArchiveRange{start};\n" - "}\n" - "\n" - "&log(DETAIL, \"archive retention on backup ${strBackup}, archiveId = ${strArchiveId}, \" .\n" - "\"start = $$oArchiveRange{start}\" .\n" - "(defined($$oArchiveRange{stop}) ? \", stop = $$oArchiveRange{stop}\" : ''));\n" - "\n" - "push(@oyArchiveRange, $oArchiveRange);\n" - "}\n" - "}\n" - "\n\n" - "foreach my $strPath ($oStorageRepo->list(\n" - "STORAGE_REPO_ARCHIVE . \"/${strArchiveId}\", {strExpression => REGEX_ARCHIVE_DIR_WAL}))\n" - "{\n" - "logDebugMisc($strOperation, \"found major WAL path: ${strPath}\");\n" - "$bRemove = true;\n" - "\n\n" - "foreach my $oArchiveRange (@oyArchiveRange)\n" - "{\n" - "if ($strPath ge substr($$oArchiveRange{start}, 0, 16) &&\n" - "(!defined($$oArchiveRange{stop}) || $strPath le substr($$oArchiveRange{stop}, 0, 16)))\n" - "{\n" - "$bRemove = false;\n" - "last;\n" - "}\n" - "}\n" - "\n\n" - "if ($bRemove)\n" - "{\n" - "my $strFullPath = $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE . \"/${strArchiveId}\") . \"/${strPath}\";\n" - "\n" - "$oStorageRepo->remove($strFullPath, {bRecurse => true});\n" - "\n\n" - "logDebugMisc($strOperation, \"remove major WAL path: ${strFullPath}\");\n" - "$self->logExpire($strArchiveId, $strPath);\n" - "}\n" - "\n\n\n" - "elsif ($strPath le substr($strArchiveExpireMax, 0, 16))\n" - "{\n" - "\n" - "foreach my $strSubPath ($oStorageRepo->list(\n" - "STORAGE_REPO_ARCHIVE . \"/${strArchiveId}/${strPath}\", {strExpression => \"^[0-F]{24}.*\\$\"}))\n" - "{\n" - "$bRemove = true;\n" - "\n\n" - "foreach my $oArchiveRange (@oyArchiveRange)\n" - "{\n" - "if (substr($strSubPath, 0, 24) ge $$oArchiveRange{start} &&\n" - "(!defined($$oArchiveRange{stop}) || substr($strSubPath, 0, 24) le $$oArchiveRange{stop}))\n" - "{\n" - "$bRemove = false;\n" - "last;\n" - "}\n" - "}\n" - "\n\n" - "if ($bRemove)\n" - "{\n" - "$oStorageRepo->remove(STORAGE_REPO_ARCHIVE . \"/${strArchiveId}/${strSubPath}\");\n" - "\n" - "logDebugMisc($strOperation, \"remove WAL segment: ${strArchiveId}/${strSubPath}\");\n" - "\n\n" - "$self->logExpire($strArchiveId, substr($strSubPath, 0, 24));\n" - "}\n" - "else\n" - "{\n" - "\n" - "$self->logExpire($strArchiveId);\n" - "}\n" - "}\n" - "}\n" - "}\n" - "\n\n" - "if ($self->{iArchiveExpireTotal} == 0)\n" - "{\n" - "&log(DETAIL, \"no archive to remove, archiveId = ${strArchiveId}\");\n" - "}\n" - "}\n" - "}\n" - "}\n" - "}\n" - "}\n" - "\n\n" - "return logDebugReturn($strOperation);\n" - "}\n" - "\n" - "1;\n" - }, { .name = "pgBackRest/InfoCommon.pm", .data = @@ -9685,11 +9284,13 @@ static const EmbeddedModule embeddedModule[] = "\n" "use File::Basename qw(dirname);\n" "\n" + "use pgBackRest::Backup::Info;\n" "use pgBackRest::Common::Exception;\n" "use pgBackRest::Common::Lock;\n" "use pgBackRest::Common::Log;\n" "use pgBackRest::Config::Config;\n" "use pgBackRest::Protocol::Helper;\n" + "use pgBackRest::Protocol::Storage::Helper;\n" "use pgBackRest::Storage::Helper;\n" "use pgBackRest::Version;\n" "\n\n\n\n" @@ -9879,11 +9480,7 @@ static const EmbeddedModule embeddedModule[] = "\n\n\n" "elsif (cfgCommandTest(CFGCMD_EXPIRE))\n" "{\n" - "\n" - "require pgBackRest::Expire;\n" - "pgBackRest::Expire->import();\n" - "\n" - "new pgBackRest::Expire()->process();\n" + "new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP));\n" "}\n" "}\n" "}\n" diff --git a/test/define.yaml b/test/define.yaml index 0ae39b27c..cde13aa8a 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -691,6 +691,13 @@ unit: coverage: command/control/control: full + # ---------------------------------------------------------------------------------------------------------------------------- + - name: expire + total: 6 + + coverage: + command/expire/expire: full + # ---------------------------------------------------------------------------------------------------------------------------- - name: help total: 4 diff --git a/test/expect/mock-all-002.log b/test/expect/mock-all-002.log index 3d59de59e..0d510ee98 100644 --- a/test/expect/mock-all-002.log +++ b/test/expect/mock-all-002.log @@ -3353,6 +3353,7 @@ P00 INFO: new backup label = [BACKUP-DIFF-6] P00 INFO: backup command end: completed successfully P00 INFO: expire command begin P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: expire command end: completed successfully + supplemental file: [TEST_PATH]/db-master/pgbackrest.conf @@ -3560,6 +3561,7 @@ P00 INFO: new backup label = [BACKUP-DIFF-7] P00 INFO: backup command end: completed successfully P00 INFO: expire command begin P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: expire command end: completed successfully + supplemental file: [TEST_PATH]/db-master/pgbackrest.conf diff --git a/test/expect/mock-expire-002.log b/test/expect/mock-expire-002.log index fb755cbc1..c1195dd34 100644 --- a/test/expect/mock-expire-002.log +++ b/test/expect/mock-expire-002.log @@ -476,6 +476,7 @@ P00 INFO: remove expired backup [BACKUP-FULL-1] P00 DETAIL: archive retention on backup [BACKUP-FULL-2], archiveId = 9.2-1, start = 00000001000000000000000C P00 DETAIL: remove archive: archiveId = 9.2-1, start = 000000010000000000000000, stop = 00000001000000000000000B P00 DETAIL: archive retention on backup [BACKUP-FULL-3], archiveId = 9.3-2, start = 000000010000000000000000 +P00 DETAIL: no archive to remove, archiveId = 9.3-2 P00 INFO: expire command end: completed successfully + supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info @@ -1147,6 +1148,7 @@ P00 INFO: remove archive path: [TEST_PATH]/db-master/repo/archive/db/9.2-1 P00 DETAIL: archive retention on backup [BACKUP-FULL-4], archiveId = 9.3-2, start = 0000000100000000000000FF P00 DETAIL: remove archive: archiveId = 9.3-2, start = 000000010000000000000000, stop = 0000000100000000000000FE P00 DETAIL: archive retention on backup [BACKUP-FULL-5], archiveId = 10-3, start = 000000010000000000000000 +P00 DETAIL: no archive to remove, archiveId = 10-3 P00 INFO: expire command end: completed successfully + supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info diff --git a/test/expect/mock-stanza-001.log b/test/expect/mock-stanza-001.log index d826b5068..bb7c9ed98 100644 --- a/test/expect/mock-stanza-001.log +++ b/test/expect/mock-stanza-001.log @@ -535,8 +535,7 @@ P00 INFO: full backup size = 48MB P00 INFO: new backup label = [BACKUP-FULL-1] P00 INFO: backup command end: completed successfully P00 INFO: expire command begin -P00 INFO: remove archive path: [TEST_PATH]/db-master/repo/archive/db/9.3-1 -P00 INFO: full backup total < 2 - using oldest full backup for 9.4-2 archive retention +P00 INFO: full backup total < 2 - using oldest full backup for 9.3-1 archive retention P00 INFO: expire command end: completed successfully + supplemental file: [TEST_PATH]/db-master/pgbackrest.conf @@ -691,7 +690,6 @@ P00 INFO: full backup size = 48MB P00 INFO: new backup label = [BACKUP-FULL-2] P00 INFO: backup command end: completed successfully P00 INFO: expire command begin -P00 INFO: remove archive path: [TEST_PATH]/db-master/repo/archive/db/10.0-3 P00 INFO: expire command end: completed successfully + supplemental file: [TEST_PATH]/db-master/pgbackrest.conf diff --git a/test/expect/mock-stanza-002.log b/test/expect/mock-stanza-002.log index e99269668..d609ec2c4 100644 --- a/test/expect/mock-stanza-002.log +++ b/test/expect/mock-stanza-002.log @@ -313,6 +313,8 @@ P00 INFO: backup command end: completed successfully P00 INFO: expire command begin P00 INFO: remove archive path: /archive/db/9.3-1 P00 INFO: full backup total < 2 - using oldest full backup for 9.4-2 archive retention +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: expire command end: completed successfully + supplemental file: [TEST_PATH]/db-master/pgbackrest.conf @@ -496,7 +498,8 @@ P00 INFO: full backup size = 48MB P00 INFO: new backup label = [BACKUP-FULL-2] P00 INFO: backup command end: completed successfully P00 INFO: expire command begin -P00 INFO: remove archive path: /archive/db/10.0-3 +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: expire command end: completed successfully + supplemental file: [TEST_PATH]/db-master/pgbackrest.conf diff --git a/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm index ae5bc24ef..afd912215 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm @@ -22,7 +22,6 @@ use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::Wait; use pgBackRest::Config::Config; -use pgBackRest::Expire; use pgBackRest::Manifest; use pgBackRest::Protocol::Storage::Helper; @@ -261,7 +260,6 @@ sub run $self->configTestLoad(CFGCMD_EXPIRE); $strDescription = 'Expiration cannot occur due to info file db mismatch'; - my $oExpire = new pgBackRest::Expire(); # Mismatched version $oHostBackup->infoMunge(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), @@ -273,11 +271,6 @@ sub run {&INFO_ARCHIVE_KEY_DB_VERSION => PG_VERSION_93, &INFO_ARCHIVE_KEY_DB_ID => $self->dbSysId(PG_VERSION_95)}}}); - $self->testException(sub {$oExpire->process()}, - ERROR_FILE_INVALID, - "archive and backup database versions do not match\n" . - "HINT: has a stanza-upgrade been performed?"); - # Restore the info file $oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)); @@ -289,11 +282,6 @@ sub run {'3' => {&INFO_ARCHIVE_KEY_DB_VERSION => PG_VERSION_95, &INFO_ARCHIVE_KEY_DB_ID => 6999999999999999999}}}); - $self->testException(sub {$oExpire->process()}, - ERROR_FILE_INVALID, - "archive and backup database versions do not match\n" . - "HINT: has a stanza-upgrade been performed?"); - # Restore the info file $oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)); } diff --git a/test/src/module/command/expireTest.c b/test/src/module/command/expireTest.c new file mode 100644 index 000000000..485ca7ee0 --- /dev/null +++ b/test/src/module/command/expireTest.c @@ -0,0 +1,859 @@ +/*********************************************************************************************************************************** +Test Command Control +***********************************************************************************************************************************/ +#include "storage/posix/storage.h" + +#include "common/harnessConfig.h" +#include "common/harnessInfo.h" + +/*********************************************************************************************************************************** +Helper functions +***********************************************************************************************************************************/ +void +archiveGenerate( + Storage *storageTest, String *archiveStanzaPath, const unsigned int start, unsigned int end, const char *archiveId, + const char *majorWal) +{ + // For simplicity, only allow 2 digits + if (end > 99) + end = 99; + + String *wal = NULL; + + for (unsigned int i = start; i <= end; i++) + { + if (i < 10) + wal = strNewFmt("%s0000000%u-9baedd24b61aa15305732ac678c4e2c102435a09", majorWal, i); + else + wal = strNewFmt("%s000000%u-9baedd24b61aa15305732ac678c4e2c102435a09", majorWal, i); + + storagePutNP( + storageNewWriteNP(storageTest, strNewFmt("%s/%s/%s/%s", strPtr(archiveStanzaPath), archiveId, majorWal, strPtr(wal))), + BUFSTRDEF(BOGUS_STR)); + } +} + +String * +archiveExpectList(const unsigned int start, unsigned int end, const char *majorWal) +{ + String *result = strNew(""); + + // For simplicity, only allow 2 digits + if (end > 99) + end = 99; + + String *wal = NULL; + + for (unsigned int i = start; i <= end; i++) + { + if (i < 10) + wal = strNewFmt("%s0000000%u-9baedd24b61aa15305732ac678c4e2c102435a09", majorWal, i); + else + wal = strNewFmt("%s000000%u-9baedd24b61aa15305732ac678c4e2c102435a09", majorWal, i); + + if (strSize(result) == 0) + strCat(result, strPtr(wal)); + else + strCatFmt(result, ", %s", strPtr(wal)); + } + + return result; +} + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void +testRun(void) +{ + FUNCTION_HARNESS_VOID(); + + Storage *storageTest = storagePosixNew( + strNew(testPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); + + String *backupStanzaPath = strNew("repo/backup/db"); + String *backupInfoFileName = strNewFmt("%s/backup.info", strPtr(backupStanzaPath)); + String *archiveStanzaPath = strNew("repo/archive/db"); + String *archiveInfoFileName = strNewFmt("%s/archive.info", strPtr(archiveStanzaPath)); + + StringList *argListBase = strLstNew(); + strLstAddZ(argListBase, "pgbackrest"); + strLstAddZ(argListBase, "--stanza=db"); + strLstAdd(argListBase, strNewFmt("--repo1-path=%s/repo", testPath())); + strLstAddZ(argListBase, "expire"); + + StringList *argListAvoidWarn = strLstDup(argListBase); + strLstAddZ(argListAvoidWarn, "--repo1-retention-full=1"); // avoid warning + + String *backupInfoBase = strNew( + "[backup:current]\n" + "20181119-152138F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152800F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-archive-start\":\"000000010000000000000004\",\"backup-archive-stop\":\"000000010000000000000004\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152800F_20181119-152152D={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000005\"," + "\"backup-archive-stop\":\"000000010000000000000005\",\"backup-info-repo-size\":2369186," + "\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428," + "\"backup-prior\":\"20181119-152800F\",\"backup-reference\":[\"20181119-152800F\"]," + "\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152800F_20181119-152155I={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000006\"," + "\"backup-archive-stop\":\"000000010000000000000006\",\"backup-info-repo-size\":2369186," + "\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428," + "\"backup-prior\":\"20181119-152800F_20181119-152152D\"," + "\"backup-reference\":[\"20181119-152800F\",\"20181119-152800F_20181119-152152D\"]," + "\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"incr\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152900F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-archive-start\":\"000000010000000000000007\",\"backup-archive-stop\":\"000000010000000000000007\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152900F_20181119-152600D={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000008\"," + "\"backup-archive-stop\":\"000000010000000000000008\",\"backup-info-repo-size\":2369186," + "\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428," + "\"backup-prior\":\"20181119-152138F\",\"backup-reference\":[\"20181119-152900F\"]," + "\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "\n" + "[db]\n" + "db-catalog-version=201409291\n" + "db-control-version=942\n" + "db-id=1\n" + "db-system-id=6625592122879095702\n" + "db-version=\"9.4\"\n" + "\n" + "[db:history]\n" + "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," + "\"db-version\":\"9.4\"}"); + + // ***************************************************************************************************************************** + if (testBegin("expireBackup()")) + { + // Create backup.info + storagePutNP(storageNewWriteNP(storageTest, backupInfoFileName), harnessInfoChecksum(backupInfoBase)); + + InfoBackup *infoBackup = NULL; + TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info"); + + // Create backup directories and manifest files + String *full1 = strNew("20181119-152138F"); + String *full2 = strNew("20181119-152800F"); + String *full1Path = strNewFmt("%s/%s", strPtr(backupStanzaPath), strPtr(full1)); + String *full2Path = strNewFmt("%s/%s", strPtr(backupStanzaPath), strPtr(full2)); + + // Load Parameters + StringList *argList = strLstDup(argListAvoidWarn); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_VOID( + storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/%s", strPtr(full1Path), INFO_MANIFEST_FILE)), + BUFSTRDEF(BOGUS_STR)), "full1 put manifest"); + TEST_RESULT_VOID( + storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/%s", strPtr(full1Path), INFO_MANIFEST_FILE ".copy")), + BUFSTRDEF(BOGUS_STR)), "full1 put manifest copy"); + TEST_RESULT_VOID( + storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/%s", strPtr(full1Path), "bogus")), + BUFSTRDEF(BOGUS_STR)), "full1 put extra file"); + TEST_RESULT_VOID(storagePathCreateNP(storageTest, full2Path), "full2 empty"); + + String *backupExpired = strNew(""); + + TEST_RESULT_VOID(expireBackup(infoBackup, full1, backupExpired), "expire backup with both manifest files"); + TEST_RESULT_BOOL( + (strLstSize(storageListNP(storageTest, full1Path)) && strLstExistsZ(storageListNP(storageTest, full1Path), "bogus")), + true, " full1 - only manifest files removed"); + + TEST_RESULT_VOID(expireBackup(infoBackup, full2, backupExpired), "expire backup with no manifest - does not error"); + + TEST_RESULT_STR( + strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")), + "20181119-152800F_20181119-152152D, 20181119-152800F_20181119-152155I, 20181119-152900F, " + "20181119-152900F_20181119-152600D", + "only backups passed to expireBackup are removed from backup:current"); + } + + // ***************************************************************************************************************************** + if (testBegin("expireFullBackup()")) + { + // Create backup.info + storagePutNP(storageNewWriteNP(storageTest, backupInfoFileName), harnessInfoChecksum(backupInfoBase)); + + InfoBackup *infoBackup = NULL; + TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info"); + + // Load Parameters + StringList *argList = strLstDup(argListBase); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_UINT(expireFullBackup(infoBackup), 0, "retention-full not set"); + harnessLogResult( + "P00 WARN: option repo1-retention-full is not set, the repository may run out of space\n" + " HINT: to retain full backups indefinitely (without warning), " + "set option 'repo1-retention-full' to the maximum."); + + //-------------------------------------------------------------------------------------------------------------------------- + strLstAddZ(argList, "--repo1-retention-full=2"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_UINT(expireFullBackup(infoBackup), 1, "retention-full=2 - one full backup expired"); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 5, " current backups reduced by 1 full - no dependencies"); + TEST_RESULT_STR( + strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")), + "20181119-152800F, 20181119-152800F_20181119-152152D, 20181119-152800F_20181119-152155I, " + "20181119-152900F, 20181119-152900F_20181119-152600D", " remaining backups correct"); + harnessLogResult("P00 INFO: expire full backup 20181119-152138F"); + + //-------------------------------------------------------------------------------------------------------------------------- + argList = strLstDup(argListBase); + strLstAddZ(argList, "--repo1-retention-full=1"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_UINT(expireFullBackup(infoBackup), 3, "retention-full=1 - one full backup and dependencies expired"); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, " current backups reduced by 1 full and dependencies"); + TEST_RESULT_STR( + strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")), + "20181119-152900F, 20181119-152900F_20181119-152600D", " remaining backups correct"); + harnessLogResult( + "P00 INFO: expire full backup set: 20181119-152800F, 20181119-152800F_20181119-152152D, " + "20181119-152800F_20181119-152155I"); + + //-------------------------------------------------------------------------------------------------------------------------- + TEST_RESULT_UINT(expireFullBackup(infoBackup), 0, "retention-full=1 - not enough backups to expire any"); + TEST_RESULT_STR( + strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")), + "20181119-152900F, 20181119-152900F_20181119-152600D", " remaining backups correct"); + } + + // ***************************************************************************************************************************** + if (testBegin("expireDiffBackup()")) + { + // Create backup.info + storagePutNP(storageNewWriteNP(storageTest, backupInfoFileName), harnessInfoChecksum(backupInfoBase)); + + InfoBackup *infoBackup = NULL; + TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info"); + + // Load Parameters + StringList *argList = strLstDup(argListAvoidWarn); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_UINT(expireDiffBackup(infoBackup), 0, "retention-diff not set - nothing expired"); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 6, " current backups not expired"); + + //-------------------------------------------------------------------------------------------------------------------------- + // Add retention-diff + StringList *argListTemp = strLstDup(argList); + strLstAddZ(argListTemp, "--repo1-retention-diff=6"); + harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp)); + + TEST_RESULT_UINT(expireDiffBackup(infoBackup), 0, "retention-diff set - too soon to expire"); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 6, " current backups not expired"); + + //-------------------------------------------------------------------------------------------------------------------------- + strLstAddZ(argList, "--repo1-retention-diff=2"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_UINT(expireDiffBackup(infoBackup), 2, "retention-diff set - full considered in diff"); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 4, " current backups reduced by 1 diff and dependent increment"); + TEST_RESULT_STR( + strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")), + "20181119-152138F, 20181119-152800F, 20181119-152900F, 20181119-152900F_20181119-152600D", + " remaining backups correct"); + harnessLogResult( + "P00 INFO: expire diff backup set: 20181119-152800F_20181119-152152D, 20181119-152800F_20181119-152155I"); + + TEST_RESULT_UINT(expireDiffBackup(infoBackup), 0, "retention-diff=2 but no more to expire"); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 4, " current backups not reduced"); + + //-------------------------------------------------------------------------------------------------------------------------- + // Create backup.info with two diff - oldest to be expired - no "set:" + storagePutNP( + storageNewWriteNP(storageTest, backupInfoFileName), + harnessInfoChecksumZ( + "[backup:current]\n" + "20181119-152800F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152800F_20181119-152152D={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000003\"," + "\"backup-archive-stop\":\"000000010000000000000003\",\"backup-info-repo-size\":2369186," + "\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428," + "\"backup-prior\":\"20181119-152800F\",\"backup-reference\":[\"20181119-152800F\"]," + "\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152800F_20181119-152155D={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000004\"," + "\"backup-archive-stop\":\"000000010000000000000004\",\"backup-info-repo-size\":2369186," + "\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428," + "\"backup-prior\":\"20181119-152800F\",\"backup-reference\":[\"20181119-152800F\"]," + "\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "\n" + "[db]\n" + "db-catalog-version=201409291\n" + "db-control-version=942\n" + "db-id=1\n" + "db-system-id=6625592122879095702\n" + "db-version=\"9.4\"\n" + "\n" + "[db:history]\n" + "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," + "\"db-version\":\"9.4\"}")); + TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info"); + + // Load parameters + argList = strLstDup(argListAvoidWarn); + strLstAddZ(argList, "--repo1-retention-diff=1"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_UINT(expireDiffBackup(infoBackup), 1, "retention-diff set - only oldest diff expired"); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, " current backups reduced by one"); + TEST_RESULT_STR( + strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")), + "20181119-152800F, 20181119-152800F_20181119-152155D", + " remaining backups correct"); + harnessLogResult( + "P00 INFO: expire diff backup 20181119-152800F_20181119-152152D"); + } + + // ***************************************************************************************************************************** + if (testBegin("removeExpiredBackup()")) + { + // Create backup.info + storagePutNP( + storageNewWriteNP(storageTest, backupInfoFileName), + harnessInfoChecksumZ( + "[backup:current]\n" + "20181119-152138F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "\n" + "[db]\n" + "db-catalog-version=201409291\n" + "db-control-version=942\n" + "db-id=1\n" + "db-system-id=6625592122879095702\n" + "db-version=\"9.4\"\n" + "\n" + "[db:history]\n" + "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," + "\"db-version\":\"9.4\"}")); + + InfoBackup *infoBackup = NULL; + TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info"); + + // Create backup directories, manifest files and other path/file + String *full = strNewFmt("%s/%s", strPtr(backupStanzaPath), "20181119-152100F"); + String *diff = strNewFmt("%s/%s", strPtr(backupStanzaPath), "20181119-152100F_20181119-152152D"); + String *otherPath = strNewFmt("%s/%s", strPtr(backupStanzaPath), "bogus"); + String *otherFile = strNewFmt("%s/%s", strPtr(backupStanzaPath), "20181118-152100F_20181119-152152D.save"); + String *full1 = strNewFmt("%s/%s", strPtr(backupStanzaPath), "20181119-152138F"); + + TEST_RESULT_VOID( + storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/%s", strPtr(full), "bogus")), + BUFSTRDEF(BOGUS_STR)), "put file"); + TEST_RESULT_VOID( + storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/%s", strPtr(full1), "somefile")), + BUFSTRDEF(BOGUS_STR)), "put file"); + TEST_RESULT_VOID(storagePathCreateNP(storageTest, diff), "empty backup directory must not error on delete"); + TEST_RESULT_VOID(storagePathCreateNP(storageTest, otherPath), "other path must not be removed"); + TEST_RESULT_VOID( + storagePutNP(storageNewWriteNP(storageTest, otherFile), + BUFSTRDEF(BOGUS_STR)), "directory look-alike file must not be removed"); + + // Load Parameters + StringList *argList = strLstDup(argListBase); + strLstAddZ(argList, "--repo1-retention-full=1"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_VOID(removeExpiredBackup(infoBackup), "remove backups not in backup.info current"); + + harnessLogResult( + "P00 INFO: remove expired backup 20181119-152100F_20181119-152152D\n" + "P00 INFO: remove expired backup 20181119-152100F"); + + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP(storageTest, backupStanzaPath), sortOrderAsc), ", ")), + "20181118-152100F_20181119-152152D.save, 20181119-152138F, backup.info, bogus", " remaining file/directories correct"); + + //-------------------------------------------------------------------------------------------------------------------------- + // Create backup.info without current backups + storagePutNP( + storageNewWriteNP(storageTest, backupInfoFileName), + harnessInfoChecksumZ( + "[db]\n" + "db-catalog-version=201409291\n" + "db-control-version=942\n" + "db-id=1\n" + "db-system-id=6625592122879095702\n" + "db-version=\"9.4\"\n" + "\n" + "[db:history]\n" + "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," + "\"db-version\":\"9.4\"}")); + + TEST_ASSIGN(infoBackup,infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info"); + + TEST_RESULT_VOID(removeExpiredBackup(infoBackup), "remove backups - backup.info current empty"); + + harnessLogResult("P00 INFO: remove expired backup 20181119-152138F"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP(storageTest, backupStanzaPath), sortOrderAsc), ", ")), + "20181118-152100F_20181119-152152D.save, backup.info, bogus", " remaining file/directories correct"); + } + + // ***************************************************************************************************************************** + if (testBegin("removeExpiredArchive() & cmdExpire()")) + { + // Load Parameters + StringList *argList = strLstDup(argListBase); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + // Create backup.info without current backups + storagePutNP( + storageNewWriteNP(storageTest, backupInfoFileName), + harnessInfoChecksumZ( + "[db]\n" + "db-catalog-version=201409291\n" + "db-control-version=942\n" + "db-id=1\n" + "db-system-id=6625592122879095702\n" + "db-version=\"9.4\"\n" + "\n" + "[db:history]\n" + "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," + "\"db-version\":\"9.4\"}")); + + InfoBackup *infoBackup = NULL; + TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info"); + + TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "archive retention not set"); + harnessLogResult( + "P00 WARN: option repo1-retention-full is not set, the repository may run out of space\n" + " HINT: to retain full backups indefinitely (without warning), " + "set option 'repo1-retention-full' to the maximum.\n" + "P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired"); + + //-------------------------------------------------------------------------------------------------------------------------- + // Set archive retention, archive retention type default but no current backups - code path test + strLstAddZ(argList, "--repo1-retention-archive=4"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "archive retention set, retention type default, no current backups"); + harnessLogResult( + "P00 WARN: option repo1-retention-full is not set, the repository may run out of space\n" + " HINT: to retain full backups indefinitely (without warning), " + "set option 'repo1-retention-full' to the maximum."); + + //-------------------------------------------------------------------------------------------------------------------------- + // Create backup.info with current backups spread over different timelines + storagePutNP(storageNewWriteNP(storageTest, backupInfoFileName), + harnessInfoChecksumZ( + "[backup:current]\n" + "20181119-152138F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152800F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-archive-start\":\"000000020000000000000002\",\"backup-archive-stop\":\"000000020000000000000002\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152800F_20181119-152152D={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000020000000000000004\"," + "\"backup-archive-stop\":\"000000020000000000000005\",\"backup-info-repo-size\":2369186," + "\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428," + "\"backup-prior\":\"20181119-152800F\",\"backup-reference\":[\"20181119-152800F\"]," + "\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152800F_20181119-152155I={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000020000000000000007\"," + "\"backup-archive-stop\":\"000000020000000000000007\",\"backup-info-repo-size\":2369186," + "\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428," + "\"backup-prior\":\"20181119-152800F_20181119-152152D\"," + "\"backup-reference\":[\"20181119-152800F\",\"20181119-152800F_20181119-152152D\"]," + "\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"incr\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152800F_20181119-152252D={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000020000000000000009\"," + "\"backup-archive-stop\":\"000000020000000000000009\",\"backup-info-repo-size\":2369186," + "\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428," + "\"backup-prior\":\"20181119-152800F\",\"backup-reference\":[\"20181119-152800F\"]," + "\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152900F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-archive-start\":\"000000010000000000000003\",\"backup-archive-stop\":\"000000010000000000000004\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152900F_20181119-152500I={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000006\"," + "\"backup-archive-stop\":\"000000010000000000000006\",\"backup-info-repo-size\":2369186," + "\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428," + "\"backup-prior\":\"20181119-152900F\",\"backup-reference\":[\"20181119-152900F\"]," + "\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\"," + "\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "\n" + "[db]\n" + "db-catalog-version=201707211\n" + "db-control-version=1002\n" + "db-id=2\n" + "db-system-id=6626363367545678089\n" + "db-version=\"10\"\n" + "\n" + "[db:history]\n" + "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," + "\"db-version\":\"9.4\"}\n" + "2={\"db-catalog-version\":201707211,\"db-control-version\":1002,\"db-system-id\":6626363367545678089," + "\"db-version\":\"10\"}\n")); + + TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info"); + + storagePutNP( + storageNewWriteNP(storageTest, archiveInfoFileName), + harnessInfoChecksumZ( + "[db]\n" + "db-id=2\n" + "db-system-id=6626363367545678089\n" + "db-version=\"10\"\n" + "\n" + "[db:history]\n" + "1={\"db-id\":6625592122879095702,\"db-version\":\"9.4\"}\n" + "2={\"db-id\":6626363367545678089,\"db-version\":\"10\"}")); + + TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "no archive on disk"); + + //-------------------------------------------------------------------------------------------------------------------------- + archiveGenerate(storageTest, archiveStanzaPath, 1, 10, "9.4-1", "0000000100000000"); + archiveGenerate(storageTest, archiveStanzaPath, 1, 10, "9.4-1", "0000000200000000"); + archiveGenerate(storageTest, archiveStanzaPath, 1, 10, "10-2", "0000000100000000"); + + TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "archive retention type = full (default), repo1-retention-archive=4"); + + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(2, 10, "0000000100000000")), + " only 9.4-1/0000000100000000/000000010000000000000001 removed"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000200000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(1, 10, "0000000200000000")), + " none removed from 9.4-1/0000000200000000 - crossing timelines to play through PITR"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "10-2", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(3, 10, "0000000100000000")), + " 000000010000000000000001 and 000000010000000000000002 removed from 10-2/0000000100000000"); + harnessLogResult( + "P00 INFO: full backup total < 4 - using oldest full backup for 9.4-1 archive retention\n" + "P00 INFO: full backup total < 4 - using oldest full backup for 10-2 archive retention"); + + //-------------------------------------------------------------------------------------------------------------------------- + argList = strLstDup(argListAvoidWarn); + strLstAddZ(argList, "--repo1-retention-archive=2"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "archive retention type = full (default), repo1-retention-archive=2"); + + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(2, 2, "0000000100000000")), + " only 9.4-1/0000000100000000/000000010000000000000002 remains in major wal 1"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000200000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(2, 10, "0000000200000000")), + " only 9.4-1/0000000200000000/000000010000000000000001 removed from major wal 2"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "10-2", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(3, 10, "0000000100000000")), + " none removed from 10-2/0000000100000000"); + + //-------------------------------------------------------------------------------------------------------------------------- + argList = strLstDup(argListAvoidWarn); + strLstAddZ(argList, "--repo1-retention-archive=1"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "archive retention type = full (default), repo1-retention-archive=1"); + + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(2, 2, "0000000100000000")), + " only 9.4-1/0000000100000000/000000010000000000000002 remains in major wal 1"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000200000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(2, 10, "0000000200000000")), + " nothing removed from 9.4-1/0000000200000000 major wal 2 - each archiveId must have one backup to play through PITR"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "10-2", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(3, 10, "0000000100000000")), + " none removed from 10-2/0000000100000000"); + + //-------------------------------------------------------------------------------------------------------------------------- + argList = strLstDup(argListAvoidWarn); + strLstAddZ(argList, "--repo1-retention-archive=2"); + strLstAddZ(argList, "--repo1-retention-archive-type=diff"); + strLstAddZ(argList, "--repo1-retention-diff=2"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_VOID( + removeExpiredArchive(infoBackup), + "full counts as differential and incremental associated with differential expires"); + + String *result = strNew(""); + strCatFmt( + result, + "%s, %s, %s, %s", + strPtr(archiveExpectList(2, 2, "0000000200000000")), + strPtr(archiveExpectList(4, 5, "0000000200000000")), + strPtr(archiveExpectList(7, 7, "0000000200000000")), + strPtr(archiveExpectList(9, 10, "0000000200000000"))); + + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(2, 2, "0000000100000000")), + " only 9.4-1/0000000100000000/000000010000000000000002 remains in major wal 1"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000200000000")), sortOrderAsc), ", ")), + strPtr(result), + " all in-between removed from 9.4-1/0000000200000000 major wal 2 - last backup able to play through PITR"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "10-2", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(3, 10, "0000000100000000")), + " none removed from 10-2/0000000100000000"); + + //-------------------------------------------------------------------------------------------------------------------------- + argList = strLstDup(argListAvoidWarn); + strLstAddZ(argList, "--repo1-retention-archive=4"); + strLstAddZ(argList, "--repo1-retention-archive-type=incr"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + // Regenerate archive + archiveGenerate(storageTest, archiveStanzaPath, 1, 10, "9.4-1", "0000000200000000"); + + TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "differential and full count as an incremental"); + + result = strNew(""); + strCatFmt( + result, + "%s, %s, %s", + strPtr(archiveExpectList(2, 2, "0000000200000000")), + strPtr(archiveExpectList(4, 5, "0000000200000000")), + strPtr(archiveExpectList(7, 10, "0000000200000000"))); + + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(2, 2, "0000000100000000")), + " only 9.4-1/0000000100000000/000000010000000000000002 remains in major wal 1"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000200000000")), sortOrderAsc), ", ")), + strPtr(result), + " incremental and after remain in 9.4-1/0000000200000000 major wal 2"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "10-2", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(3, 10, "0000000100000000")), + " none removed from 10-2/0000000100000000"); + + //-------------------------------------------------------------------------------------------------------------------------- + argList = strLstDup(argListBase); + strLstAddZ(argList, "--repo1-retention-full=2"); + strLstAddZ(argList, "--repo1-retention-diff=3"); + strLstAddZ(argList, "--repo1-retention-archive=2"); + strLstAddZ(argList, "--repo1-retention-archive-type=diff"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_VOID(cmdExpire(), "expire last backup in archive sub path and remove sub path"); + TEST_RESULT_BOOL( + storagePathExistsNP(storageTest, strNewFmt("%s/%s", strPtr(archiveStanzaPath), "9.4-1/0000000100000000")), + false, " archive sub path removed"); + harnessLogResult("P00 INFO: expire full backup 20181119-152138F"); + + //-------------------------------------------------------------------------------------------------------------------------- + argList = strLstDup(argListAvoidWarn); + strLstAddZ(argList, "--repo1-retention-archive=1"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_VOID(cmdExpire(), "expire last backup in archive path and remove path"); + TEST_RESULT_BOOL( + storagePathExistsNP(storageTest, strNewFmt("%s/%s", strPtr(archiveStanzaPath), "9.4-1")), + false, " archive path removed"); + + harnessLogResult(strPtr(strNewFmt( + "P00 INFO: expire full backup set: 20181119-152800F, 20181119-152800F_20181119-152152D, " + "20181119-152800F_20181119-152155I, 20181119-152800F_20181119-152252D\n" + "P00 INFO: remove archive path: %s/%s/9.4-1", testPath(), strPtr(archiveStanzaPath)))); + + TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), " get backup.info"); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, " backup.info updated on disk"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(infoBackupDataLabelList(infoBackup, NULL), sortOrderAsc), ", ")), + "20181119-152900F, 20181119-152900F_20181119-152500I", " remaining current backups correct"); + + //-------------------------------------------------------------------------------------------------------------------------- + storagePutNP(storageNewWriteNP(storageTest, backupInfoFileName), + harnessInfoChecksumZ( + "[backup:current]\n" + "20181119-152138F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152800F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "20181119-152900F={" + "\"backrest-format\":5,\"backrest-version\":\"2.08dev\"," + "\"backup-archive-start\":\"000000010000000000000004\",\"backup-archive-stop\":\"000000010000000000000004\"," + "\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186," + "\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900," + "\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\"," + "\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false," + "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" + "\n" + "[db]\n" + "db-catalog-version=201707211\n" + "db-control-version=1002\n" + "db-id=2\n" + "db-system-id=6626363367545678089\n" + "db-version=\"10\"\n" + "\n" + "[db:history]\n" + "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," + "\"db-version\":\"9.4\"}\n" + "2={\"db-catalog-version\":201707211,\"db-control-version\":1002,\"db-system-id\":6626363367545678089," + "\"db-version\":\"10\"}\n")); + + TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info"); + + archiveGenerate(storageTest, archiveStanzaPath, 1, 5, "9.4-1", "0000000100000000"); + + argList = strLstDup(argListAvoidWarn); + strLstAddZ(argList, "--repo1-retention-archive=2"); + strLstAddZ(argList, "--repo1-retention-archive-type=full"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_VOID( + removeExpiredArchive(infoBackup), "backup selected for retention does not have archive-start so do nothing"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(1, 5, "0000000100000000")), + " nothing removed from 9.4-1/0000000100000000"); + + argList = strLstDup(argListAvoidWarn); + strLstAddZ(argList, "--repo1-retention-archive=4"); + strLstAddZ(argList, "--repo1-retention-archive-type=incr"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_VOID( + removeExpiredArchive(infoBackup), "full count as incr but not enough backups, retention set to first full"); + TEST_RESULT_STR( + strPtr(strLstJoin(strLstSort(storageListNP( + storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")), + strPtr(archiveExpectList(2, 5, "0000000100000000")), + " only removed archive prior to first full"); + harnessLogResult( + "P00 INFO: full backup total < 4 - using oldest full backup for 9.4-1 archive retention"); + + argList = strLstDup(argListAvoidWarn); + strLstAddZ(argList, "--repo1-retention-archive=1"); + strLstAddZ(argList, "--repo1-retention-archive-type=full"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + harnessLogLevelSet(logLevelDetail); + + TEST_RESULT_VOID( + removeExpiredArchive(infoBackup), "backup earlier than selected for retention does not have archive-start"); + harnessLogResult( + "P00 DETAIL: archive retention on backup 20181119-152138F, archiveId = 9.4-1, start = 000000010000000000000002," + " stop = 000000010000000000000002\n" + "P00 DETAIL: archive retention on backup 20181119-152900F, archiveId = 9.4-1, start = 000000010000000000000004\n" + "P00 DETAIL: remove archive: archiveId = 9.4-1, start = 000000010000000000000003, stop = 000000010000000000000003"); + + harnessLogLevelReset(); + } + + // ***************************************************************************************************************************** + if (testBegin("sortArchiveId()")) + { + StringList *list = strLstNew(); + + strLstAddZ(list, "10-4"); + strLstAddZ(list, "11-10"); + strLstAddZ(list, "9.6-1"); + + TEST_RESULT_STR(strPtr(strLstJoin(sortArchiveId(list, sortOrderAsc), ", ")), "9.6-1, 10-4, 11-10", "sort ascending"); + + strLstAddZ(list, "9.4-2"); + TEST_RESULT_STR( + strPtr(strLstJoin(sortArchiveId(list, sortOrderDesc), ", ")), "11-10, 10-4, 9.4-2, 9.6-1", "sort descending"); + } + + FUNCTION_HARNESS_RESULT_VOID(); +}