From 4f7608e759d3e2510fb0e984107b5ba7ea00554f Mon Sep 17 00:00:00 2001 From: Daniel Fischer Date: Tue, 24 Feb 2009 11:42:11 +0100 Subject: [PATCH 1/3] bug#42888: Add collections of test runs to make it both configurable and transparent what kinds of tests we run during integration testing. --- mysql-test/collections/README | 30 +++++++++++++++++++++++++++ mysql-test/collections/default.daily | 0 mysql-test/collections/default.push | 2 ++ mysql-test/collections/default.weekly | 0 4 files changed, 32 insertions(+) create mode 100644 mysql-test/collections/README create mode 100644 mysql-test/collections/default.daily create mode 100644 mysql-test/collections/default.push create mode 100644 mysql-test/collections/default.weekly diff --git a/mysql-test/collections/README b/mysql-test/collections/README new file mode 100644 index 00000000000..9af84646a40 --- /dev/null +++ b/mysql-test/collections/README @@ -0,0 +1,30 @@ +This directory contains collections of test runs that we run during our +integration and release testing. Each file contains zero or more lines, +with one invocation of mysql-test-run.pl on each. These invocations are +written so that, with the assumption that perl is in your search path, +any collection can run as a shell script or a batch file, with the parent +mysql-test directory being the current working directory. + +During integration testing, we choose the collection to run by following +these steps: + +1) We choose the extension to look for, based on these rules: + - If we're running a per-push test, we choose ".push" as the extension. + - If we're running a daily test, we choose ".daily" as the extension. + - If we're running a weekly test, we choose ".weekly" as the extension. + +2) If there is a collection that has the same name as the branch we're + testing plus the extension as determined in step 1, we choose that + collection. + +3) If the branch is unknown or we have removed all characters from it + and still not found a matching collection, we choose the name "default" + plus the extension determined in step 1. If there is no such file, + we give up and don't test anything at all. + +4) If we haven't found a collection yet, we remove the last character from + the branch name and go back to step 2. + +5) The commands from the collection are run line by line via execv() or + similar system calls. They are not run as a shell script. Shell + expansions are not guaranteed to work and most likely won't. diff --git a/mysql-test/collections/default.daily b/mysql-test/collections/default.daily new file mode 100644 index 00000000000..e69de29bb2d diff --git a/mysql-test/collections/default.push b/mysql-test/collections/default.push new file mode 100644 index 00000000000..0f4115c8565 --- /dev/null +++ b/mysql-test/collections/default.push @@ -0,0 +1,2 @@ +perl mysql-test-run.pl --timer --force --comment=n_stm +perl mysql-test-run.pl --timer --force --comment=ps_stm --ps-protocol diff --git a/mysql-test/collections/default.weekly b/mysql-test/collections/default.weekly new file mode 100644 index 00000000000..e69de29bb2d From f0b3c8914fa4fa2b01fb123ba7a8240a73c4548e Mon Sep 17 00:00:00 2001 From: Daniel Fischer Date: Tue, 24 Feb 2009 14:54:04 +0100 Subject: [PATCH 2/3] include collections in dist --- mysql-test/Makefile.am | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/Makefile.am b/mysql-test/Makefile.am index 0c4a708dd14..cdb2d9fd598 100644 --- a/mysql-test/Makefile.am +++ b/mysql-test/Makefile.am @@ -33,7 +33,7 @@ endif benchdir_root= $(prefix) testdir = $(benchdir_root)/mysql-test EXTRA_SCRIPTS = mysql-test-run-shell.sh install_test_db.sh valgrind.supp $(PRESCRIPTS) -EXTRA_DIST = $(EXTRA_SCRIPTS) suite +EXTRA_DIST = $(EXTRA_SCRIPTS) suite collections GENSCRIPTS = mysql-test-run-shell mysql-test-run install_test_db mtr PRESCRIPTS = mysql-test-run.pl mysql-stress-test.pl test_SCRIPTS = $(GENSCRIPTS) $(PRESCRIPTS) @@ -80,6 +80,7 @@ install-data-local: $(DESTDIR)$(testdir)/std_data/ndb_backup50_data_be \ $(DESTDIR)$(testdir)/std_data/ndb_backup50_data_le \ $(DESTDIR)$(testdir)/lib \ + $(DESTDIR)$(testdir)/collections \ $(DESTDIR)$(testdir)/std_data/funcs_1 $(INSTALL_DATA) $(srcdir)/README $(DESTDIR)$(testdir) -$(INSTALL_DATA) $(srcdir)/t/*.def $(DESTDIR)$(testdir)/t From 0d097d80e2174f7051aadd12f38519693cdb9341 Mon Sep 17 00:00:00 2001 From: Daniel Fischer Date: Wed, 25 Feb 2009 15:00:17 +0100 Subject: [PATCH 3/3] address review comments --- mysql-test/collections/README.experimental | 2 +- mysql-test/lib/mtr_report.pm | 16 ++++++--- mysql-test/mysql-test-run.pl | 38 +++++++++++++--------- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/mysql-test/collections/README.experimental b/mysql-test/collections/README.experimental index d06e55f1246..9eee2394423 100644 --- a/mysql-test/collections/README.experimental +++ b/mysql-test/collections/README.experimental @@ -11,7 +11,7 @@ The syntax is as follows: 2) Empty lines and lines starting with a hash (#) are ignored. -3) If any other line contains a black followed by a hash (#), the hash +3) If any other line contains a blank followed by a hash (#), the hash and any subsequent characters are ignored. 4) The full test case name including the suite and execution mode diff --git a/mysql-test/lib/mtr_report.pm b/mysql-test/lib/mtr_report.pm index d1cae6324cb..73401dc4b33 100644 --- a/mysql-test/lib/mtr_report.pm +++ b/mysql-test/lib/mtr_report.pm @@ -109,10 +109,10 @@ sub mtr_report_test ($) { my ($tinfo)= @_; my $test_name = _mtr_report_test_name($tinfo); - my $comment= $tinfo->{'comment'}; - my $logfile= $tinfo->{'logfile'}; - my $warnings= $tinfo->{'warnings'}; - my $result= $tinfo->{'result'}; + my $comment= $tinfo->{'comment'}; + my $logfile= $tinfo->{'logfile'}; + my $warnings= $tinfo->{'warnings'}; + my $result= $tinfo->{'result'}; if ($result eq 'MTR_RES_FAILED'){ @@ -123,14 +123,20 @@ sub mtr_report_test ($) { { # Find out if this test case is an experimental one, so we can treat # the failure as an expected failure instead of a regression. - for my $exp ( @$::opt_experimental ) { + for my $exp ( @$::experimental_test_cases ) { if ( $exp ne $test_name ) { + # if the expression is not the name of this test case, but has + # an asterisk at the end, determine if the characters up to + # but excluding the asterisk are the same if ( $exp ne "" && substr($exp, -1, 1) eq "*" ) { $exp = substr($exp, 0, length($exp) - 1); if ( substr($test_name, 0, length($exp)) ne $exp ) { + # no match, try next entry next; } + # if yes, fall through to set the exp-fail status } else { + # no match, try next entry next; } } diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 8e772279aa3..89a6d89687a 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -170,6 +170,7 @@ my $config; # The currently running config my $current_config_name; # The currently running config file template our $opt_experimental; +our $experimental_test_cases; my $baseport; my $opt_build_thread= $ENV{'MTR_BUILD_THREAD'} || "auto"; @@ -963,24 +964,29 @@ sub command_line_setup { if ( $opt_experimental ) { - if ( open(FILE, "<", $opt_experimental) ) { - mtr_report("Using experimental file: $opt_experimental"); - $opt_experimental = []; - while() { - chomp; - s/( +|^)#.*$//; - s/^ +//; - s/ +$//; - if ( $_ eq "" ) { - next; - } - print " - $_\n"; - push @$opt_experimental, $_; + # read the list of experimental test cases from the file specified on + # the command line + open(FILE, "<", $opt_experimental) or mtr_error("Can't read experimental file: $opt_experimental"); + mtr_report("Using experimental file: $opt_experimental"); + $experimental_test_cases = []; + while() { + chomp; + # remove comments (# foo) at the beginning of the line, or after a + # blank at the end of the line + s/( +|^)#.*$//; + # remove whitespace + s/^ +//; + s/ +$//; + # if nothing left, don't need to remember this line + if ( $_ eq "" ) { + next; } - close FILE; - } else { - mtr_error("Can't read experimental file: $opt_experimental"); + # remember what is left as the name of another test case that should be + # treated as experimental + print " - $_\n"; + push @$experimental_test_cases, $_; } + close FILE; } foreach my $arg ( @ARGV )