From 4bffa0c5bb7d551c60b8d8bcff9a3bbe6e02eddd Mon Sep 17 00:00:00 2001 From: David Steele Date: Wed, 26 Jun 2019 15:02:30 -0400 Subject: [PATCH] Add test function to create the S3 bucket instead of using aws cli. Eventually the idea is to remove the dependency on aws cli since Python is a big install. --- libc/xs/storage/storage.xs | 17 ++++++++++++ libc/xs/storage/storage.xsh | 1 + src/Makefile.in | 2 +- src/perl/libc.auto.c | 32 ++++++++++++++++++++++ test/lib/pgBackRestTest/Env/HostEnvTest.pm | 9 ++++-- 5 files changed, 57 insertions(+), 4 deletions(-) diff --git a/libc/xs/storage/storage.xs b/libc/xs/storage/storage.xs index 546a203a0..1248ad17a 100644 --- a/libc/xs/storage/storage.xs +++ b/libc/xs/storage/storage.xs @@ -47,6 +47,23 @@ CLEANUP: } MEM_CONTEXT_XS_TEMP_END(); +#################################################################################################################################### +void +bucketCreate(self) +PREINIT: + MEM_CONTEXT_XS_TEMP_BEGIN() + { +INPUT: + pgBackRest::LibC::Storage self +CODE: + if (strEq(storageType(self), STORAGE_S3_TYPE_STR)) + storageS3Request((StorageS3 *)storageDriver(self), HTTP_VERB_PUT_STR, FSLASH_STR, NULL, NULL, true, false); + else + THROW_FMT(AssertError, "unable to create bucket on '%s' storage", strPtr(storageType(self))); +CLEANUP: + } + MEM_CONTEXT_XS_TEMP_END(); + #################################################################################################################################### bool copy(self, source, destination) diff --git a/libc/xs/storage/storage.xsh b/libc/xs/storage/storage.xsh index bc8835fdc..fef3a89a8 100644 --- a/libc/xs/storage/storage.xsh +++ b/libc/xs/storage/storage.xsh @@ -12,6 +12,7 @@ Storage XS Header #include "common/type/json.h" #include "postgres/interface.h" #include "storage/helper.h" +#include "storage/s3/storage.intern.h" #include "storage/storage.intern.h" typedef Storage *pgBackRest__LibC__Storage; diff --git a/src/Makefile.in b/src/Makefile.in index c192be074..5134d2cbd 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -434,7 +434,7 @@ main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push. perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/config.c -o perl/config.o -perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h command/backup/pageChecksum.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh +perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h command/backup/pageChecksum.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/exec.c -o perl/exec.o postgres/interface.o: postgres/interface.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h postgres/interface.h postgres/interface/version.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h diff --git a/src/perl/libc.auto.c b/src/perl/libc.auto.c index 069d6bcf9..5fbfdeb59 100644 --- a/src/perl/libc.auto.c +++ b/src/perl/libc.auto.c @@ -893,6 +893,37 @@ XS_EUPXS(XS_pgBackRest__LibC__Storage_new) } +XS_EUPXS(XS_pgBackRest__LibC__Storage_bucketCreate); /* prototype to pass -Wmissing-prototypes */ +XS_EUPXS(XS_pgBackRest__LibC__Storage_bucketCreate) +{ + dVAR; dXSARGS; + if (items != 1) + croak_xs_usage(cv, "self"); + { + MEM_CONTEXT_XS_TEMP_BEGIN() + { + pgBackRest__LibC__Storage self; + + if (SvROK(ST(0)) && sv_derived_from(ST(0), "pgBackRest::LibC::Storage")) { + IV tmp = SvIV((SV*)SvRV(ST(0))); + self = INT2PTR(pgBackRest__LibC__Storage,tmp); + } + else + Perl_croak_nocontext("%s: %s is not of type %s", + "pgBackRest::LibC::Storage::bucketCreate", + "self", "pgBackRest::LibC::Storage") +; + if (strEq(storageType(self), STORAGE_S3_TYPE_STR)) + storageS3Request((StorageS3 *)storageDriver(self), HTTP_VERB_PUT_STR, FSLASH_STR, NULL, NULL, true, false); + else + THROW_FMT(AssertError, "unable to create bucket on '%s' storage", strPtr(storageType(self))); + } + MEM_CONTEXT_XS_TEMP_END(); + } + XSRETURN_EMPTY; +} + + XS_EUPXS(XS_pgBackRest__LibC__Storage_copy); /* prototype to pass -Wmissing-prototypes */ XS_EUPXS(XS_pgBackRest__LibC__Storage_copy) { @@ -2091,6 +2122,7 @@ XS_EXTERNAL(boot_pgBackRest__LibC) newXS_deffile("pgBackRest::LibC::StorageRead::resultAll", XS_pgBackRest__LibC__StorageRead_resultAll); newXS_deffile("pgBackRest::LibC::StorageRead::DESTROY", XS_pgBackRest__LibC__StorageRead_DESTROY); newXS_deffile("pgBackRest::LibC::Storage::new", XS_pgBackRest__LibC__Storage_new); + newXS_deffile("pgBackRest::LibC::Storage::bucketCreate", XS_pgBackRest__LibC__Storage_bucketCreate); newXS_deffile("pgBackRest::LibC::Storage::copy", XS_pgBackRest__LibC__Storage_copy); newXS_deffile("pgBackRest::LibC::Storage::exists", XS_pgBackRest__LibC__Storage_exists); newXS_deffile("pgBackRest::LibC::Storage::get", XS_pgBackRest__LibC__Storage_get); diff --git a/test/lib/pgBackRestTest/Env/HostEnvTest.pm b/test/lib/pgBackRestTest/Env/HostEnvTest.pm index 8c21c88ff..88f087fa0 100644 --- a/test/lib/pgBackRestTest/Env/HostEnvTest.pm +++ b/test/lib/pgBackRestTest/Env/HostEnvTest.pm @@ -122,9 +122,6 @@ sub setup if (defined($oHostS3)) { $oHostGroup->hostAdd($oHostS3, {rstryHostName => ['pgbackrest-dev.s3.amazonaws.com', 's3.amazonaws.com']}); - - # Wait for server to start - $oHostS3->executeS3('mb s3://' . HOST_S3_BUCKET); } # Create db master config @@ -186,6 +183,12 @@ sub setup $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); + # Create S3 bucket + if (defined($oHostS3)) + { + storageRepo()->{oStorageC}->bucketCreate(); + } + return $oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3; }