From 692b1e274adb9f1f23e3d29d127af521a53a27e9 Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 19 Jun 2017 18:55:00 -0400 Subject: [PATCH] Split full module into mock and real to allow better test combinations and save time in CI. --- .travis.yml | 36 +++++++---------- doc/xml/release.xml | 4 ++ ...ull-synthetic-001.log => mock-all-001.log} | 0 ...ull-synthetic-002.log => mock-all-002.log} | 0 ...ull-synthetic-003.log => mock-all-003.log} | 0 .../{full-real-001.log => real-all-001.log} | 0 .../{full-real-002.log => real-all-002.log} | 0 .../{full-real-003.log => real-all-003.log} | 0 .../{full-real-004.log => real-all-004.log} | 0 .../{full-real-005.log => real-all-005.log} | 0 .../{full-real-006.log => real-all-006.log} | 0 .../{full-real-007.log => real-all-007.log} | 0 .../{full-real-008.log => real-all-008.log} | 0 .../{full-real-009.log => real-all-009.log} | 0 .../{full-real-010.log => real-all-010.log} | 0 .../{full-real-011.log => real-all-011.log} | 0 .../{full-real-012.log => real-all-012.log} | 0 test/lib/pgBackRestTest/Common/CiTest.pm | 40 +++++-------------- test/lib/pgBackRestTest/Common/DefineTest.pm | 19 +++++++-- .../MockAllTest.pm} | 4 +- .../FullRealTest.pm => Real/RealAllTest.pm} | 4 +- 21 files changed, 49 insertions(+), 58 deletions(-) rename test/expect/{full-synthetic-001.log => mock-all-001.log} (100%) rename test/expect/{full-synthetic-002.log => mock-all-002.log} (100%) rename test/expect/{full-synthetic-003.log => mock-all-003.log} (100%) rename test/expect/{full-real-001.log => real-all-001.log} (100%) rename test/expect/{full-real-002.log => real-all-002.log} (100%) rename test/expect/{full-real-003.log => real-all-003.log} (100%) rename test/expect/{full-real-004.log => real-all-004.log} (100%) rename test/expect/{full-real-005.log => real-all-005.log} (100%) rename test/expect/{full-real-006.log => real-all-006.log} (100%) rename test/expect/{full-real-007.log => real-all-007.log} (100%) rename test/expect/{full-real-008.log => real-all-008.log} (100%) rename test/expect/{full-real-009.log => real-all-009.log} (100%) rename test/expect/{full-real-010.log => real-all-010.log} (100%) rename test/expect/{full-real-011.log => real-all-011.log} (100%) rename test/expect/{full-real-012.log => real-all-012.log} (100%) rename test/lib/pgBackRestTest/Module/{Full/FullSyntheticTest.pm => Mock/MockAllTest.pm} (99%) rename test/lib/pgBackRestTest/Module/{Full/FullRealTest.pm => Real/RealAllTest.pm} (99%) diff --git a/.travis.yml b/.travis.yml index 88ccb23ad..39ad6cd67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,27 +13,21 @@ services: - docker env: - - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza" - - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=full --test=synthetic --no-lint --no-package" - - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=9.1" PGB_TEST_PARAM="--module=full --test=real --db=9.1 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=9.0" PGB_TEST_PARAM="--module=full --test=real --db=9.0 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="u16" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --no-lint" - - PGB_TEST_VM="u16" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=full --test=synthetic --no-lint --no-package" - - PGB_TEST_VM="u16" PGB_BUILD_PARAM="--db=9.4" PGB_TEST_PARAM="--module=full --test=real --db=9.4 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --no-lint" - - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=full --test=synthetic --no-lint --no-package" - - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=9.3" PGB_TEST_PARAM="--module=full --test=real --db=9.3 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=8.4" PGB_TEST_PARAM="--module=full --test=real --db=8.4 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --no-lint" - - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=full --test=synthetic --no-lint --no-package" - - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=9.6" PGB_TEST_PARAM="--module=full --test=real --db=9.6 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=9.5" PGB_TEST_PARAM="--module=full --test=real --db=9.5 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="u14" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --no-lint" - - PGB_TEST_VM="u14" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=full --test=synthetic --no-lint --no-package" - - PGB_TEST_VM="u14" PGB_BUILD_PARAM="--db=9.2" PGB_TEST_PARAM="--module=full --test=real --db=9.2 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="u12" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --no-lint" - - PGB_TEST_VM="u12" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=full --test=synthetic --no-lint --no-package" - - PGB_TEST_VM="u12" PGB_BUILD_PARAM="--db=8.3" PGB_TEST_PARAM="--module=full --test=real --db=8.3 --process-max=2 --no-lint --no-package" + - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --module=mock" + - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=9.1" PGB_TEST_PARAM="--module=real --db=9.1 --process-max=2 --no-lint --no-package" + - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=9.0" PGB_TEST_PARAM="--module=real --db=9.0 --process-max=2 --no-lint --no-package" + - PGB_TEST_VM="u16" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --module=mock --no-lint" + - PGB_TEST_VM="u16" PGB_BUILD_PARAM="--db=9.4" PGB_TEST_PARAM="--module=real --db=9.4 --process-max=2 --no-lint --no-package" + - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --module=mock --no-lint" + - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=9.3" PGB_TEST_PARAM="--module=real --db=9.3 --process-max=2 --no-lint --no-package" + - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=8.4" PGB_TEST_PARAM="--module=real --db=8.4 --process-max=2 --no-lint --no-package" + - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --module=mock --no-lint" + - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=9.6" PGB_TEST_PARAM="--module=real --db=9.6 --process-max=2 --no-lint --no-package" + - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=9.5" PGB_TEST_PARAM="--module=real --db=9.5 --process-max=2 --no-lint --no-package" + - PGB_TEST_VM="u14" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --module=mock --no-lint" + - PGB_TEST_VM="u14" PGB_BUILD_PARAM="--db=9.2" PGB_TEST_PARAM="--module=real --db=9.2 --process-max=2 --no-lint --no-package" + - PGB_TEST_VM="u12" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--vm-max=2 --module=common --module=help --module=config --module=storage --module=protocol --module=info --module=archive --module=backup --module=expire --module=stanza --module=mock --no-lint" + - PGB_TEST_VM="u12" PGB_BUILD_PARAM="--db=8.3" PGB_TEST_PARAM="--module=real --db=8.3 --process-max=2 --no-lint --no-package" before_install: - sudo apt-get -qq update && sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl python-pip diff --git a/doc/xml/release.xml b/doc/xml/release.xml index d8bcd0ece..aac45f735 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -186,6 +186,10 @@

Combine hardlink and non/compressed in synthetic tests to reduce test time and improve coverage.

 + + +

Split full module into mock and real to allow better test combinations and save time in CI.

+ diff --git a/test/expect/full-synthetic-001.log b/test/expect/mock-all-001.log similarity index 100% rename from test/expect/full-synthetic-001.log rename to test/expect/mock-all-001.log diff --git a/test/expect/full-synthetic-002.log b/test/expect/mock-all-002.log similarity index 100% rename from test/expect/full-synthetic-002.log rename to test/expect/mock-all-002.log diff --git a/test/expect/full-synthetic-003.log b/test/expect/mock-all-003.log similarity index 100% rename from test/expect/full-synthetic-003.log rename to test/expect/mock-all-003.log diff --git a/test/expect/full-real-001.log b/test/expect/real-all-001.log similarity index 100% rename from test/expect/full-real-001.log rename to test/expect/real-all-001.log diff --git a/test/expect/full-real-002.log b/test/expect/real-all-002.log similarity index 100% rename from test/expect/full-real-002.log rename to test/expect/real-all-002.log diff --git a/test/expect/full-real-003.log b/test/expect/real-all-003.log similarity index 100% rename from test/expect/full-real-003.log rename to test/expect/real-all-003.log diff --git a/test/expect/full-real-004.log b/test/expect/real-all-004.log similarity index 100% rename from test/expect/full-real-004.log rename to test/expect/real-all-004.log diff --git a/test/expect/full-real-005.log b/test/expect/real-all-005.log similarity index 100% rename from test/expect/full-real-005.log rename to test/expect/real-all-005.log diff --git a/test/expect/full-real-006.log b/test/expect/real-all-006.log similarity index 100% rename from test/expect/full-real-006.log rename to test/expect/real-all-006.log diff --git a/test/expect/full-real-007.log b/test/expect/real-all-007.log similarity index 100% rename from test/expect/full-real-007.log rename to test/expect/real-all-007.log diff --git a/test/expect/full-real-008.log b/test/expect/real-all-008.log similarity index 100% rename from test/expect/full-real-008.log rename to test/expect/real-all-008.log diff --git a/test/expect/full-real-009.log b/test/expect/real-all-009.log similarity index 100% rename from test/expect/full-real-009.log rename to test/expect/real-all-009.log diff --git a/test/expect/full-real-010.log b/test/expect/real-all-010.log similarity index 100% rename from test/expect/full-real-010.log rename to test/expect/real-all-010.log diff --git a/test/expect/full-real-011.log b/test/expect/real-all-011.log similarity index 100% rename from test/expect/full-real-011.log rename to test/expect/real-all-011.log diff --git a/test/expect/full-real-012.log b/test/expect/real-all-012.log similarity index 100% rename from test/expect/full-real-012.log rename to test/expect/real-all-012.log diff --git a/test/lib/pgBackRestTest/Common/CiTest.pm b/test/lib/pgBackRestTest/Common/CiTest.pm index 6de461228..b1c06253e 100644 --- a/test/lib/pgBackRestTest/Common/CiTest.pm +++ b/test/lib/pgBackRestTest/Common/CiTest.pm @@ -95,18 +95,18 @@ sub process { my $hVm = vmGet(); my @stryModule; - my $strFullModule = undef; + my $strRealModule = undef; # Get all modules but full to break up the tests foreach my $strModule (testDefModuleList()) { - if ($strModule ne 'full') + if ($strModule ne 'real') { push(@stryModule, $strModule); } else { - $strFullModule = $strModule; + $strRealModule = $strModule; } } @@ -119,40 +119,22 @@ sub process join(' --module=', @stryModule) . ($bFirst ? '' : " ${strConfigNotFirst}") . "\"\n"; $bFirst = false; - # Now generate full tests - my $strRealTest = undef; - - if (!defined($strFullModule)) + # Now generate real tests + if (!defined($strRealModule)) { - confess "${strFullModule} module not found, has the name changed?"; + confess "${strRealModule} module not found, has the name changed?"; } - foreach my $strTest (testDefModuleTestList($strFullModule)) + foreach my $strTest (testDefModuleTestList($strRealModule)) { - if ($strTest eq 'real') - { - $strRealTest = $strTest; - - foreach my $strDbVersion (sort {$b cmp $a} @{$hVm->{$strVm}{&VM_DB_MINIMAL}}) - { - $strConfig .= - " - PGB_TEST_VM=\"${strVm}\" PGB_BUILD_PARAM=\"--db=${strDbVersion}\"" . - " PGB_TEST_PARAM=\"--module=full --test=real --db=${strDbVersion}" . - " --process-max=2 ${strConfigNotFirst} ${strConfigNotFirstOS}\"\n"; - } - } - else + foreach my $strDbVersion (sort {$b cmp $a} @{$hVm->{$strVm}{&VM_DB_MINIMAL}}) { $strConfig .= - " - PGB_TEST_VM=\"${strVm}\" PGB_BUILD_PARAM=\"--db=none\"" . - " PGB_TEST_PARAM=\"--vm-max=2 --module=full --test=${strTest} ${strConfigNotFirst} ${strConfigNotFirstOS}\"\n"; + " - PGB_TEST_VM=\"${strVm}\" PGB_BUILD_PARAM=\"--db=${strDbVersion}\"" . + " PGB_TEST_PARAM=\"--module=${strRealModule} --db=${strDbVersion}" . + " --process-max=2 ${strConfigNotFirst} ${strConfigNotFirstOS}\"\n"; } } - - if (!defined($strRealTest)) - { - confess "${strRealTest} test not found in ${strFullModule} module, has the name changed?"; - } } # Configure install and script diff --git a/test/lib/pgBackRestTest/Common/DefineTest.pm b/test/lib/pgBackRestTest/Common/DefineTest.pm index b83177d55..55f329afe 100644 --- a/test/lib/pgBackRestTest/Common/DefineTest.pm +++ b/test/lib/pgBackRestTest/Common/DefineTest.pm @@ -408,9 +408,9 @@ my $oTestDef = }, ] }, - # Full tests + # Mock tests { - &TESTDEF_NAME => 'full', + &TESTDEF_NAME => 'mock', &TESTDEF_EXPECT => true, &TESTDEF_INDIVIDUAL => true, &TESTDEF_PROCESS => true, @@ -418,11 +418,22 @@ my $oTestDef = &TESTDEF_TEST => [ { - &TESTDEF_NAME => 'synthetic', + &TESTDEF_NAME => 'all', &TESTDEF_TOTAL => 3, }, + ] + }, + # Real tests + { + &TESTDEF_NAME => 'real', + &TESTDEF_EXPECT => true, + &TESTDEF_INDIVIDUAL => true, + &TESTDEF_PROCESS => true, + + &TESTDEF_TEST => + [ { - &TESTDEF_NAME => 'real', + &TESTDEF_NAME => 'all', &TESTDEF_TOTAL => 12, &TESTDEF_DB => true, } diff --git a/test/lib/pgBackRestTest/Module/Full/FullSyntheticTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm similarity index 99% rename from test/lib/pgBackRestTest/Module/Full/FullSyntheticTest.pm rename to test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm index 43e75764c..11aad1f10 100644 --- a/test/lib/pgBackRestTest/Module/Full/FullSyntheticTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm @@ -1,7 +1,7 @@ #################################################################################################################################### -# FullSyntheticTest.pm - Tests for all commands that can be run against synthetic data +# Test All Commands on Mock Data #################################################################################################################################### -package pgBackRestTest::Module::Full::FullSyntheticTest; +package pgBackRestTest::Module::Mock::MockAllTest; use parent 'pgBackRestTest::Env::HostEnvTest'; #################################################################################################################################### diff --git a/test/lib/pgBackRestTest/Module/Full/FullRealTest.pm b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm similarity index 99% rename from test/lib/pgBackRestTest/Module/Full/FullRealTest.pm rename to test/lib/pgBackRestTest/Module/Real/RealAllTest.pm index 3caaa1017..57d945ab6 100644 --- a/test/lib/pgBackRestTest/Module/Full/FullRealTest.pm +++ b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm @@ -1,7 +1,7 @@ #################################################################################################################################### -# FullRealTest.pm - Tests for all commands against a real database +# Test All Commands On PostgreSQL Clusters #################################################################################################################################### -package pgBackRestTest::Module::Full::FullRealTest; +package pgBackRestTest::Module::Real::RealAllTest; use parent 'pgBackRestTest::Env::HostEnvTest'; ####################################################################################################################################