From 73be64ce49aaff9d94f70c47590476c404365ce4 Mon Sep 17 00:00:00 2001 From: David Steele Date: Wed, 20 Feb 2019 15:52:07 +0200 Subject: [PATCH] Add separate archive-get-async command. This command was previously forked off from the archive-get command which required a bit of artificial option and log manipulation. A separate command is easier to test and will work on platforms that don't have fork(), e.g. Windows. --- build/lib/pgBackRestBuild/Config/Data.pm | 64 +++++++++++++++-- doc/lib/BackRestDoc/Common/DocConfig.pm | 2 +- doc/xml/release.xml | 4 ++ lib/pgBackRest/Archive/Get/Get.pm | 4 +- lib/pgBackRest/LibCAuto.pm | 1 + lib/pgBackRest/Main.pm | 13 +++- src/Makefile | 2 +- src/command/archive/get/get.c | 91 ++++++++++-------------- src/command/help/help.c | 6 +- src/config/config.auto.c | 11 +++ src/config/config.auto.h | 3 +- src/config/define.auto.c | 89 +++++++++++++++++++++-- src/config/define.auto.h | 1 + src/config/define.c | 2 +- src/perl/embed.auto.c | 16 ++++- test/expect/mock-archive-002.log | 4 +- test/expect/mock-stanza-002.log | 2 +- test/expect/mock-stanza-003.log | 2 +- test/src/module/archive/getTest.c | 6 +- 19 files changed, 243 insertions(+), 80 deletions(-) diff --git a/build/lib/pgBackRestBuild/Config/Data.pm b/build/lib/pgBackRestBuild/Config/Data.pm index eb21edf54..c348c118b 100644 --- a/build/lib/pgBackRestBuild/Config/Data.pm +++ b/build/lib/pgBackRestBuild/Config/Data.pm @@ -76,6 +76,8 @@ use pgBackRest::Version; #################################################################################################################################### use constant CFGCMD_ARCHIVE_GET => 'archive-get'; push @EXPORT, qw(CFGCMD_ARCHIVE_GET); +use constant CFGCMD_ARCHIVE_GET_ASYNC => 'archive-get-async'; + push @EXPORT, qw(CFGCMD_ARCHIVE_GET_ASYNC); use constant CFGCMD_ARCHIVE_PUSH => 'archive-push'; push @EXPORT, qw(CFGCMD_ARCHIVE_PUSH); use constant CFGCMD_BACKUP => 'backup'; @@ -567,6 +569,13 @@ my $rhCommandDefine = &CFGDEF_LOCK_TYPE => CFGDEF_LOCK_TYPE_ARCHIVE, }, + &CFGCMD_ARCHIVE_GET_ASYNC => + { + &CFGDEF_LOG_FILE => true, + &CFGDEF_LOCK_REQUIRED => true, + &CFGDEF_LOCK_TYPE => CFGDEF_LOCK_TYPE_ARCHIVE, + }, + &CFGCMD_ARCHIVE_PUSH => { &CFGDEF_LOG_FILE => false, @@ -664,6 +673,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -762,6 +772,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1069,6 +1080,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1092,6 +1104,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1123,6 +1136,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_RESTORE => {}, @@ -1138,6 +1152,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1160,6 +1175,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1181,6 +1197,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1204,6 +1221,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1227,6 +1245,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_EXPIRE => {}, @@ -1250,6 +1269,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1275,6 +1295,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1299,6 +1320,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1384,6 +1406,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => { @@ -1428,6 +1451,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_CHECK => {}, &CFGCMD_INFO => {}, @@ -1661,6 +1685,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1767,6 +1792,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1790,13 +1816,23 @@ my %hConfigDefine = &CFGDEF_DEFAULT => '/var/spool/' . PROJECT_EXE, &CFGDEF_COMMAND => { - &CFGCMD_ARCHIVE_GET => {}, - &CFGCMD_ARCHIVE_PUSH => {}, - }, - &CFGDEF_DEPEND => - { - &CFGDEF_DEPEND_OPTION => CFGOPT_ARCHIVE_ASYNC, - &CFGDEF_DEPEND_LIST => [true], + &CFGCMD_ARCHIVE_GET => + { + &CFGDEF_DEPEND => + { + &CFGDEF_DEPEND_OPTION => CFGOPT_ARCHIVE_ASYNC, + &CFGDEF_DEPEND_LIST => [true], + }, + }, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, + &CFGCMD_ARCHIVE_PUSH => + { + &CFGDEF_DEPEND => + { + &CFGDEF_DEPEND_OPTION => CFGOPT_ARCHIVE_ASYNC, + &CFGDEF_DEPEND_LIST => [true], + }, + }, }, }, @@ -1809,6 +1845,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_RESTORE => {}, @@ -1835,6 +1872,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1858,6 +1896,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1889,6 +1928,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1913,6 +1953,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, &CFGCMD_ARCHIVE_PUSH => {}, &CFGCMD_BACKUP => {}, &CFGCMD_CHECK => {}, @@ -1976,6 +2017,7 @@ my %hConfigDefine = &CFGDEF_COMMAND => { &CFGCMD_ARCHIVE_GET => {}, + &CFGCMD_ARCHIVE_GET_ASYNC => {}, }, }, @@ -2199,6 +2241,10 @@ my %hConfigDefine = { &CFGDEF_INTERNAL => true, }, + &CFGCMD_ARCHIVE_GET_ASYNC => + { + &CFGDEF_INTERNAL => true, + }, &CFGCMD_ARCHIVE_PUSH => { &CFGDEF_INTERNAL => true, @@ -2316,6 +2362,10 @@ my %hConfigDefine = { &CFGDEF_REQUIRED => false }, + &CFGCMD_ARCHIVE_GET_ASYNC => + { + &CFGDEF_REQUIRED => false + }, &CFGCMD_ARCHIVE_PUSH => { &CFGDEF_REQUIRED => false diff --git a/doc/lib/BackRestDoc/Common/DocConfig.pm b/doc/lib/BackRestDoc/Common/DocConfig.pm index 73458b672..1219484b5 100644 --- a/doc/lib/BackRestDoc/Common/DocConfig.pm +++ b/doc/lib/BackRestDoc/Common/DocConfig.pm @@ -195,7 +195,7 @@ sub process foreach my $strCommand (cfgDefineCommandList()) { - if ($strCommand eq CFGCMD_REMOTE || $strCommand eq CFGCMD_LOCAL) + if ($strCommand eq CFGCMD_REMOTE || $strCommand eq CFGCMD_LOCAL || $strCommand eq CFGCMD_ARCHIVE_GET_ASYNC) { next; } diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 2129cfde6..bfde115a0 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -29,6 +29,10 @@

Migrate remote command to C.

+ +

Add separate archive-get-async command.

+ + diff --git a/lib/pgBackRest/Archive/Get/Get.pm b/lib/pgBackRest/Archive/Get/Get.pm index 7c5f31b3b..037ae9eef 100644 --- a/lib/pgBackRest/Archive/Get/Get.pm +++ b/lib/pgBackRest/Archive/Get/Get.pm @@ -44,11 +44,13 @@ sub process ( $strOperation, $rstryCommandArg, + $bAsync, ) = logDebugParam ( __PACKAGE__ . '->process', \@_, {name => 'rstryCommandArg'}, + {name => 'bAsync'}, ); my $iResult = 0; @@ -60,7 +62,7 @@ sub process } # Start the async process and wait for WAL to complete - if (cfgOption(CFGOPT_ARCHIVE_ASYNC)) + if ($bAsync) { # Load module dynamically require pgBackRest::Archive::Get::Async; diff --git a/lib/pgBackRest/LibCAuto.pm b/lib/pgBackRest/LibCAuto.pm index 951039c02..264729b46 100644 --- a/lib/pgBackRest/LibCAuto.pm +++ b/lib/pgBackRest/LibCAuto.pm @@ -104,6 +104,7 @@ sub libcAutoExportTag 'CFGOPTVAL_RESTORE_TYPE_IMMEDIATE', 'CFGOPTVAL_RESTORE_TYPE_DEFAULT', 'CFGCMD_ARCHIVE_GET', + 'CFGCMD_ARCHIVE_GET_ASYNC', 'CFGCMD_ARCHIVE_PUSH', 'CFGCMD_BACKUP', 'CFGCMD_CHECK', diff --git a/lib/pgBackRest/Main.pm b/lib/pgBackRest/Main.pm index a02d33ade..1bf1cd41f 100644 --- a/lib/pgBackRest/Main.pm +++ b/lib/pgBackRest/Main.pm @@ -88,7 +88,18 @@ sub main require pgBackRest::Archive::Get::Get; pgBackRest::Archive::Get::Get->import(); - $iResult = new pgBackRest::Archive::Get::Get()->process(\@stryCommandArg); + $iResult = new pgBackRest::Archive::Get::Get()->process(\@stryCommandArg, false); + } + + # Process archive-get-async command + # -------------------------------------------------------------------------------------------------------------------------- + elsif (cfgCommandTest(CFGCMD_ARCHIVE_GET_ASYNC)) + { + # Load module dynamically + require pgBackRest::Archive::Get::Get; + pgBackRest::Archive::Get::Get->import(); + + $iResult = new pgBackRest::Archive::Get::Get()->process(\@stryCommandArg, true); } # Process remote command diff --git a/src/Makefile b/src/Makefile index 1deb073bf..e32c316fc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -182,7 +182,7 @@ command/archive/common.o: command/archive/common.c command/archive/common.h comm command/archive/get/file.o: command/archive/get/file.c command/archive/common.h command/archive/get/file.h command/control/control.h common/assert.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h compress/gzip.h compress/gzipDecompress.h config/config.auto.h config/config.h config/define.auto.h config/define.h crypto/cipherBlock.h crypto/crypto.h info/infoArchive.h info/infoPg.h postgres/interface.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h $(CC) $(CFLAGS) -c command/archive/get/file.c -o command/archive/get/file.o -command/archive/get/get.o: command/archive/get/get.c command/archive/common.h command/archive/get/file.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h crypto/crypto.h perl/exec.h postgres/interface.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h +command/archive/get/get.o: command/archive/get/get.c command/archive/common.h command/archive/get/file.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h crypto/crypto.h perl/exec.h postgres/interface.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h $(CC) $(CFLAGS) -c command/archive/get/get.c -o command/archive/get/get.o command/archive/push/push.o: command/archive/push/push.c command/archive/common.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h diff --git a/src/command/archive/get/get.c b/src/command/archive/get/get.c index e75ae142d..84fb4fb24 100644 --- a/src/command/archive/get/get.c +++ b/src/command/archive/get/get.c @@ -1,6 +1,9 @@ /*********************************************************************************************************************************** Archive Get Command ***********************************************************************************************************************************/ +#include +#include +#include #include #include @@ -14,7 +17,7 @@ Archive Get Command #include "common/regExp.h" #include "common/wait.h" #include "config/config.h" -#include "config/load.h" +#include "config/exec.h" #include "perl/exec.h" #include "postgres/interface.h" #include "storage/helper.h" @@ -130,8 +133,6 @@ cmdArchiveGet(void) walDestination = strNewFmt("%s/%s", strPtr(cfgOptionStr(cfgOptPgPath)), strPtr(walDestination)); // Async get can only be performed on WAL segments, history or other files must use synchronous mode - bool asyncServer = false; - if (cfgOptionBool(cfgOptArchiveAsync) && walIsSegment(walSegment)) { bool found = false; // Has the WAL segment been found yet? @@ -202,6 +203,31 @@ cmdArchiveGet(void) if (!forked && (!found || !queueFull) && lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 0, false)) { + // Get control info + PgControl pgControl = pgControlFromFile(cfgOptionStr(cfgOptPgPath)); + + // Create the queue + storagePathCreateNP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN_STR); + + // The async process should not output on the console at all + KeyValue *optionReplace = kvNew(); + + kvPut(optionReplace, varNewStr(strNew(cfgOptionName(cfgOptLogLevelConsole))), varNewStrZ("off")); + kvPut(optionReplace, varNewStr(strNew(cfgOptionName(cfgOptLogLevelStderr))), varNewStrZ("off")); + + // Generate command options + StringList *commandExec = cfgExecParam(cfgCmdArchiveGetAsync, optionReplace); + strLstInsert(commandExec, 0, cfgExe()); + + // Clean the current queue using the list of WAL that we ideally want in the queue. queueNeed() + // will return the list of WAL needed to fill the queue and this will be passed to the async process. + const StringList *queue = queueNeed( + walSegment, found, (size_t)cfgOptionInt64(cfgOptArchiveGetQueueMax), pgControl.walSegmentSize, + pgControl.version); + + for (unsigned int queueIdx = 0; queueIdx < strLstSize(queue); queueIdx++) + strLstAdd(commandExec, strLstGet(queue, queueIdx)); + // Release the lock and mark the async process as forked lockRelease(true); forked = true; @@ -209,49 +235,13 @@ cmdArchiveGet(void) // Fork off the async process if (fork() == 0) { - // In the async server - asyncServer = true; - result = 0; + // Detach from parent process + forkDetach(); - // Only run async if the lock can be reacquired. We just held it so this should not be an issue unless - // another process sneaks in. In general there should be only one archive-get process running but in - // theory there could be more than one. - if (lockAcquire( // {uncoverable - almost impossible to make this lock fail} - cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 0, false)) - { - // Get control info - PgControl pgControl = pgControlFromFile(cfgOptionStr(cfgOptPgPath)); - - // Create the queue - storagePathCreateNP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN_STR); - - // Clean the current queue using the list of WAL that we ideally want in the queue. queueNeed() - // will return the list of WAL needed to fill the queue and this will be passed to the async process. - cfgCommandParamSet( - queueNeed( - walSegment, found, (size_t)cfgOptionInt64(cfgOptArchiveGetQueueMax), pgControl.walSegmentSize, - pgControl.version)); - - // The async process should not output on the console at all - cfgOptionSet(cfgOptLogLevelConsole, cfgSourceParam, varNewStrZ("off")); - cfgOptionSet(cfgOptLogLevelStderr, cfgSourceParam, varNewStrZ("off")); - cfgLoadLogSetting(); - - // Open the log file - cfgLoadLogFile( - strNewFmt("%s/%s-%s-async.log", strPtr(cfgOptionStr(cfgOptLogPath)), - strPtr(cfgOptionStr(cfgOptStanza)), cfgCommandName(cfgCommand()))); - - // Log command info since we are starting a new log - cmdBegin(true); - - // Detach from parent process - forkDetach(); - - perlExec(); - } - - break; // {uncovered - async calls always return errors for now} + // Execute the binary. This statement will not return if it is successful. + THROW_ON_SYS_ERROR_FMT( + execvp(strPtr(cfgExe()), (char ** const)strLstPtr(commandExec)) == -1, + ExecuteError, "unable to execute '%s'", cfgCommandName(cfgCmdArchiveGetAsync)); } } @@ -282,13 +272,10 @@ cmdArchiveGet(void) } // Log whether or not the file was found - if (!asyncServer) // {uncovered - async calls always return errors for now} - { - if (result == 0) - LOG_INFO("found %s in the archive", strPtr(walSegment)); - else - LOG_INFO("unable to find %s in the archive", strPtr(walSegment)); - } + if (result == 0) + LOG_INFO("found %s in the archive", strPtr(walSegment)); + else + LOG_INFO("unable to find %s in the archive", strPtr(walSegment)); } MEM_CONTEXT_TEMP_END(); diff --git a/src/command/help/help.c b/src/command/help/help.c index 4097a25e0..4423cb62c 100644 --- a/src/command/help/help.c +++ b/src/command/help/help.c @@ -166,8 +166,12 @@ helpRender(void) if (commandId == cfgCmdNone) continue; - if (strlen(cfgCommandName(commandId)) > commandSizeMax) + // Only check size if the command has a help summary so we know it is a documented command + if (cfgDefCommandHelpSummary(cfgCommandDefIdFromId(commandId)) != NULL && + strlen(cfgCommandName(commandId)) > commandSizeMax) + { commandSizeMax = strlen(cfgCommandName(commandId)); + } } // Output help for each command diff --git a/src/config/config.auto.c b/src/config/config.auto.c index 24fe9439d..2dc3e4e41 100644 --- a/src/config/config.auto.c +++ b/src/config/config.auto.c @@ -20,6 +20,17 @@ static ConfigCommandData configCommandData[CFG_COMMAND_TOTAL] = CONFIG_COMMAND_L CONFIG_COMMAND_LOCK_TYPE(lockTypeArchive) ) + CONFIG_COMMAND + ( + CONFIG_COMMAND_NAME("archive-get-async") + + CONFIG_COMMAND_LOG_FILE(true) + CONFIG_COMMAND_LOG_LEVEL_DEFAULT(logLevelInfo) + CONFIG_COMMAND_LOG_LEVEL_STDERR_MAX(logLevelTrace) + CONFIG_COMMAND_LOCK_REQUIRED(true) + CONFIG_COMMAND_LOCK_TYPE(lockTypeArchive) + ) + CONFIG_COMMAND ( CONFIG_COMMAND_NAME("archive-push") diff --git a/src/config/config.auto.h b/src/config/config.auto.h index 2d6c42225..1e07740a9 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -9,7 +9,7 @@ Automatically generated by Build.pm -- do not modify directly. /*********************************************************************************************************************************** Command constants ***********************************************************************************************************************************/ -#define CFG_COMMAND_TOTAL 17 +#define CFG_COMMAND_TOTAL 18 /*********************************************************************************************************************************** Option constants @@ -22,6 +22,7 @@ Command enum typedef enum { cfgCmdArchiveGet, + cfgCmdArchiveGetAsync, cfgCmdArchivePush, cfgCmdBackup, cfgCmdCheck, diff --git a/src/config/define.auto.c b/src/config/define.auto.c index 86984e84b..b7f9ee634 100644 --- a/src/config/define.auto.c +++ b/src/config/define.auto.c @@ -20,6 +20,11 @@ static ConfigDefineCommandData configDefineCommandData[] = CFGDEFDATA_COMMAND_LI ) ) + CFGDEFDATA_COMMAND + ( + CFGDEFDATA_COMMAND_NAME("archive-get-async") + ) + CFGDEFDATA_COMMAND ( CFGDEFDATA_COMMAND_NAME("archive-push") @@ -369,6 +374,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) ) CFGDEFDATA_OPTION_OPTIONAL_LIST @@ -522,6 +528,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -607,6 +614,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -668,6 +676,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore) @@ -701,6 +710,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -745,6 +755,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -786,6 +797,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -830,6 +842,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -877,6 +890,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -953,6 +967,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -1265,6 +1280,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdExpire) @@ -1315,6 +1331,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -1375,6 +1392,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -1454,6 +1472,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -1509,6 +1528,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -1552,6 +1572,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -1596,6 +1617,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -1677,6 +1699,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -1818,6 +1841,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -1855,6 +1879,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -1880,6 +1905,13 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_OPTIONAL_INTERNAL(true) ) + CFGDEFDATA_OPTION_OPTIONAL_COMMAND_OVERRRIDE + ( + CFGDEFDATA_OPTION_OPTIONAL_COMMAND(cfgDefCmdArchiveGetAsync) + + CFGDEFDATA_OPTION_OPTIONAL_INTERNAL(true) + ) + CFGDEFDATA_OPTION_OPTIONAL_COMMAND_OVERRRIDE ( CFGDEFDATA_OPTION_OPTIONAL_COMMAND(cfgDefCmdArchivePush) @@ -2171,6 +2203,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -2194,6 +2227,13 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_OPTIONAL_REQUIRED(false) ) + CFGDEFDATA_OPTION_OPTIONAL_COMMAND_OVERRRIDE + ( + CFGDEFDATA_OPTION_OPTIONAL_COMMAND(cfgDefCmdArchiveGetAsync) + + CFGDEFDATA_OPTION_OPTIONAL_REQUIRED(false) + ) + CFGDEFDATA_OPTION_OPTIONAL_COMMAND_OVERRRIDE ( CFGDEFDATA_OPTION_OPTIONAL_COMMAND(cfgDefCmdArchivePush) @@ -2360,6 +2400,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore) @@ -2396,6 +2437,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -2483,6 +2525,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -2537,6 +2580,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -2623,6 +2667,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -2702,6 +2747,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo) @@ -2742,6 +2788,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo) @@ -2783,6 +2830,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo) @@ -2823,6 +2871,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo) @@ -2862,6 +2911,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo) @@ -2905,6 +2955,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo) @@ -2950,6 +3001,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3155,6 +3207,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3204,6 +3257,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3253,6 +3307,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3302,6 +3357,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3351,6 +3407,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3400,6 +3457,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3463,6 +3521,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3526,6 +3585,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3575,6 +3635,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3625,6 +3686,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3679,6 +3741,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) @@ -3812,18 +3875,35 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) ) CFGDEFDATA_OPTION_OPTIONAL_LIST ( - CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST + CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("/var/spool/pgbackrest") + + CFGDEFDATA_OPTION_OPTIONAL_COMMAND_OVERRRIDE ( - cfgDefOptArchiveAsync, - "1" + CFGDEFDATA_OPTION_OPTIONAL_COMMAND(cfgDefCmdArchiveGet) + + CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST + ( + cfgDefOptArchiveAsync, + "1" + ) ) - CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("/var/spool/pgbackrest") + CFGDEFDATA_OPTION_OPTIONAL_COMMAND_OVERRRIDE + ( + CFGDEFDATA_OPTION_OPTIONAL_COMMAND(cfgDefCmdArchivePush) + + CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST + ( + cfgDefOptArchiveAsync, + "1" + ) + ) ) ) @@ -3856,6 +3936,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_COMMAND_LIST ( CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) diff --git a/src/config/define.auto.h b/src/config/define.auto.h index cc2ea2fbf..db0806d78 100644 --- a/src/config/define.auto.h +++ b/src/config/define.auto.h @@ -12,6 +12,7 @@ Command define enum typedef enum { cfgDefCmdArchiveGet, + cfgDefCmdArchiveGetAsync, cfgDefCmdArchivePush, cfgDefCmdBackup, cfgDefCmdCheck, diff --git a/src/config/define.c b/src/config/define.c index f8e3d577c..2d01dff6d 100644 --- a/src/config/define.c +++ b/src/config/define.c @@ -53,7 +53,7 @@ typedef struct ConfigDefineOptionData unsigned int section:2; // Config section (e.g. global, stanza, cmd-line) bool required:1; // Is the option required? bool secure:1; // Does the option need to be redacted on logs and cmd-line? - unsigned int commandValid:15; // Bitmap for commands that the option is valid for + unsigned int commandValid:17; // Bitmap for commands that the option is valid for const char *helpSection; // Classify the option const char *helpSummary; // Brief summary of the option diff --git a/src/perl/embed.auto.c b/src/perl/embed.auto.c index 7644b127b..485c57418 100644 --- a/src/perl/embed.auto.c +++ b/src/perl/embed.auto.c @@ -872,11 +872,13 @@ static const EmbeddedModule embeddedModule[] = "(\n" "$strOperation,\n" "$rstryCommandArg,\n" + "$bAsync,\n" ") =\n" "logDebugParam\n" "(\n" "__PACKAGE__ . '->process', \\@_,\n" "{name => 'rstryCommandArg'},\n" + "{name => 'bAsync'},\n" ");\n" "\n" "my $iResult = 0;\n" @@ -886,7 +888,7 @@ static const EmbeddedModule embeddedModule[] = "confess &log(ERROR, cfgCommandName(CFGCMD_ARCHIVE_GET) . ' operation must run on db host', ERROR_HOST_INVALID);\n" "}\n" "\n\n" - "if (cfgOption(CFGOPT_ARCHIVE_ASYNC))\n" + "if ($bAsync)\n" "{\n" "\n" "require pgBackRest::Archive::Get::Async;\n" @@ -10521,6 +10523,7 @@ static const EmbeddedModule embeddedModule[] = "'CFGOPTVAL_RESTORE_TYPE_IMMEDIATE',\n" "'CFGOPTVAL_RESTORE_TYPE_DEFAULT',\n" "'CFGCMD_ARCHIVE_GET',\n" + "'CFGCMD_ARCHIVE_GET_ASYNC',\n" "'CFGCMD_ARCHIVE_PUSH',\n" "'CFGCMD_BACKUP',\n" "'CFGCMD_CHECK',\n" @@ -10844,7 +10847,16 @@ static const EmbeddedModule embeddedModule[] = "require pgBackRest::Archive::Get::Get;\n" "pgBackRest::Archive::Get::Get->import();\n" "\n" - "$iResult = new pgBackRest::Archive::Get::Get()->process(\\@stryCommandArg);\n" + "$iResult = new pgBackRest::Archive::Get::Get()->process(\\@stryCommandArg, false);\n" + "}\n" + "\n\n\n" + "elsif (cfgCommandTest(CFGCMD_ARCHIVE_GET_ASYNC))\n" + "{\n" + "\n" + "require pgBackRest::Archive::Get::Get;\n" + "pgBackRest::Archive::Get::Get->import();\n" + "\n" + "$iResult = new pgBackRest::Archive::Get::Get()->process(\\@stryCommandArg, true);\n" "}\n" "\n\n\n" "elsif (cfgCommandTest(CFGCMD_REMOTE))\n" diff --git a/test/expect/mock-archive-002.log b/test/expect/mock-archive-002.log index a0a1f711f..2f82f98c4 100644 --- a/test/expect/mock-archive-002.log +++ b/test/expect/mock-archive-002.log @@ -108,7 +108,7 @@ P00 INFO: archive-get command begin [BACKREST-VERSION]: [700000007000000070000 P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void) P00 DEBUG: perl/exec::perlExec: (void) -P00 DEBUG: Archive::Get::Get->process(): rstryCommandArg = (700000007000000070000000, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) +P00 DEBUG: Archive::Get::Get->process(): bAsync = false, rstryCommandArg = (700000007000000070000000, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) P00 DEBUG: Archive::Get::File::archiveGetFile(): bAtomic = false, strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 700000007000000070000000 P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false @@ -150,7 +150,7 @@ P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000 P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void) P00 DEBUG: perl/exec::perlExec: (void) -P00 DEBUG: Archive::Get::Get->process(): rstryCommandArg = (000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) +P00 DEBUG: Archive::Get::Get->process(): bAsync = false, rstryCommandArg = (000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) P00 DEBUG: Archive::Get::File::archiveGetFile(): bAtomic = false, strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001 P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false diff --git a/test/expect/mock-stanza-002.log b/test/expect/mock-stanza-002.log index a00897922..7ced6ca8d 100644 --- a/test/expect/mock-stanza-002.log +++ b/test/expect/mock-stanza-002.log @@ -401,7 +401,7 @@ P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000 P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void) P00 DEBUG: perl/exec::perlExec: (void) -P00 DEBUG: Archive::Get::Get->process(): rstryCommandArg = (000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) +P00 DEBUG: Archive::Get::Get->process(): bAsync = false, rstryCommandArg = (000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) P00 DEBUG: Archive::Get::File::archiveGetFile(): bAtomic = false, strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false diff --git a/test/expect/mock-stanza-003.log b/test/expect/mock-stanza-003.log index c853f5c33..6e3873567 100644 --- a/test/expect/mock-stanza-003.log +++ b/test/expect/mock-stanza-003.log @@ -494,7 +494,7 @@ P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000 P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void) P00 DEBUG: perl/exec::perlExec: (void) -P00 DEBUG: Archive::Get::Get->process(): rstryCommandArg = (000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) +P00 DEBUG: Archive::Get::Get->process(): bAsync = false, rstryCommandArg = (000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) P00 DEBUG: Archive::Get::File::archiveGetFile(): bAtomic = false, strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false diff --git a/test/src/module/archive/getTest.c b/test/src/module/archive/getTest.c index f93b53ca9..6d96b3360 100644 --- a/test/src/module/archive/getTest.c +++ b/test/src/module/archive/getTest.c @@ -294,7 +294,7 @@ testRun(void) if (testBegin("cmdArchiveGet()")) { StringList *argList = strLstNew(); - strLstAddZ(argList, "pgbackrest"); + strLstAddZ(argList, "pgbackrest-bogus"); // Break this until async tests are setup correctly strLstAddZ(argList, "--archive-timeout=1"); strLstAdd(argList, strNewFmt("--log-path=%s", testPath())); strLstAdd(argList, strNewFmt("--log-level-file=debug")); @@ -397,6 +397,7 @@ testRun(void) strLstAddZ(argList, "--archive-async"); strLstAdd(argList, walSegment); strLstAddZ(argList, "pg_wal/RECOVERYXLOG"); + strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath())); harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); HARNESS_FORK_BEGIN() @@ -412,9 +413,6 @@ testRun(void) // Check for missing WAL // ------------------------------------------------------------------------------------------------------------------------- - strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath())); - harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); - storagePutNP( storageNewWriteNP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s.ok", strPtr(walSegment))), NULL);