database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-05 07:21:24 +03:00
Code Activity

Improve speed of make check-world

Browse Source 
Before, make check-world would create a new temporary installation for
each test suite, which is slow and wasteful.  Instead, we now create one
test installation that is used by all test suites that are part of a
make run.

The management of the temporary installation is removed from pg_regress
and handled in the makefiles.  This allows for better control, and
unifies the code with that of test suites not run through pg_regress.

review and msvc support by Michael Paquier <michael.paquier@gmail.com>

more review by Fabien Coelho <coelho@cri.ensmp.fr>
This commit is contained in:
Peter Eisentraut
2015-04-23 08:59:52 -04:00
parent 50a16e30eb
commit dcae5facca
17 changed files with 145 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/test/isolation/Makefile
View File

@ -9,9 +9,6 @@ subdir = src/test/isolation
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
# where to find psql for testing an existing installation
PSQLDIR = $(bindir)
override CPPFLAGS := -I$(srcdir) -I$(libpq_srcdir) -I$(srcdir)/../regress $(CPPFLAGS)
OBJS = specparse.o isolationtester.o $(WIN32RES)
@ -55,17 +52,17 @@ maintainer-clean: distclean
rm -f specparse.c specscanner.c
installcheck: all
./pg_isolation_regress --psqldir='$(PSQLDIR)' $(EXTRA_REGRESS_OPTS) --inputdir=$(srcdir) --schedule=$(srcdir)/isolation_schedule
./pg_isolation_regress --bindir='$(bindir)' $(EXTRA_REGRESS_OPTS) --inputdir=$(srcdir) --schedule=$(srcdir)/isolation_schedule
check: all
./pg_isolation_regress --temp-install=./tmp_check --inputdir=$(srcdir) --top-builddir=$(top_builddir) $(EXTRA_REGRESS_OPTS) --schedule=$(srcdir)/isolation_schedule
$(with_temp_install) ./pg_isolation_regress --temp-instance=./tmp_check --inputdir=$(srcdir) --bindir= $(EXTRA_REGRESS_OPTS) --schedule=$(srcdir)/isolation_schedule
# Versions of the check tests that include the prepared_transactions test
# It only makes sense to run these if set up to use prepared transactions,
# via TEMP_CONFIG for the check case, or via the postgresql.conf for the
# installcheck case.
installcheck-prepared-txns: all
./pg_isolation_regress --psqldir='$(PSQLDIR)' $(EXTRA_REGRESS_OPTS) --inputdir=$(srcdir) --schedule=$(srcdir)/isolation_schedule prepared-transactions
installcheck-prepared-txns: all temp-install
./pg_isolation_regress --bindir='$(bindir)' $(EXTRA_REGRESS_OPTS) --inputdir=$(srcdir) --schedule=$(srcdir)/isolation_schedule prepared-transactions
check-prepared-txns: all
./pg_isolation_regress --temp-install=./tmp_check $(EXTRA_REGRESS_OPTS) --inputdir=$(srcdir) --top-builddir=$(top_builddir) --schedule=$(srcdir)/isolation_schedule prepared-transactions
check-prepared-txns: all temp-install
./pg_isolation_regress --temp-instance=./tmp_check $(EXTRA_REGRESS_OPTS) --inputdir=$(srcdir) --schedule=$(srcdir)/isolation_schedule prepared-transactions

4
src/test/regress/GNUmakefile
View File

@ -23,9 +23,6 @@ ifdef TEMP_CONFIG
TEMP_CONF += --temp-config=$(TEMP_CONFIG)
endif
# where to find psql for testing an existing installation
PSQLDIR = $(bindir)
# maximum simultaneous connections for parallel tests
MAXCONNOPT =
ifdef MAX_CONNECTIONS
@ -34,7 +31,6 @@ endif
# stuff to pass into build of pg_regress
EXTRADEFS = '-DHOST_TUPLE="$(host_tuple)"' \
'-DMAKEPROG="$(MAKE)"' \
'-DSHELLPROG="$(SHELL)"' \
'-DDLSUFFIX="$(DLSUFFIX)"'

234
src/test/regress/pg_regress.c
View File

@ -45,25 +45,10 @@ typedef struct _resultmap
} _resultmap;
/*
* Values obtained from pg_config_paths.h and Makefile. The PG installation
* paths are only used in temp_install mode: we use these strings to find
* out where "make install" will put stuff under the temp_install directory.
* In non-temp_install mode, the only thing we need is the location of psql,
* which we expect to find in psqldir, or in the PATH if psqldir isn't given.
*
* XXX Because pg_regress is not installed in bindir, we can't support
* this for relocatable trees as it is. --psqldir would need to be
* specified in those cases.
* Values obtained from Makefile.
*/
char *bindir = PGBINDIR;
char *libdir = LIBDIR;
char *datadir = PGSHAREDIR;
char *host_platform = HOST_TUPLE;
#ifndef WIN32_ONLY_COMPILER
static char *makeprog = MAKEPROG;
#endif
#ifndef WIN32 /* not used in WIN32 case */
static char *shellprog = SHELLPROG;
#endif
@ -86,7 +71,7 @@ _stringlist *dblist = NULL;
bool debug = false;
char *inputdir = ".";
char *outputdir = ".";
char *psqldir = PGBINDIR;
char *bindir = PGBINDIR;
char *launcher = NULL;
static _stringlist *loadlanguage = NULL;
static _stringlist *loadextension = NULL;
@ -94,9 +79,8 @@ static int max_connections = 0;
static char *encoding = NULL;
static _stringlist *schedulelist = NULL;
static _stringlist *extra_tests = NULL;
static char *temp_install = NULL;
static char *temp_instance = NULL;
static char *temp_config = NULL;
static char *top_builddir = NULL;
static bool nolocale = false;
static bool use_existing = false;
static char *hostname = NULL;
@ -105,7 +89,6 @@ static bool port_specified_by_user = false;
static char *dlpath = PKGLIBDIR;
static char *user = NULL;
static _stringlist *extraroles = NULL;
static _stringlist *extra_install = NULL;
static char *config_auth_datadir = NULL;
/* internal variables */
@ -282,8 +265,10 @@ stop_postmaster(void)
fflush(stderr);
snprintf(buf, sizeof(buf),
"\"%s/pg_ctl\" stop -D \"%s/data\" -s -m fast",
bindir, temp_install);
"\"%s%spg_ctl\" stop -D \"%s/data\" -s -m fast",
bindir ? bindir : "",
bindir ? "/" : "",
temp_instance);
r = system(buf);
if (r != 0)
{
@ -730,27 +715,6 @@ doputenv(const char *var, const char *val)
putenv(s);
}
/*
* Set the environment variable "pathname", prepending "addval" to its
* old value (if any).
*/
static void
add_to_path(const char *pathname, char separator, const char *addval)
{
char *oldval = getenv(pathname);
char *newval;
if (!oldval || !oldval[0])
{
/* no previous value */
newval = psprintf("%s=%s", pathname, addval);
}
else
newval = psprintf("%s=%s%c%s", pathname, addval, separator, oldval);
putenv(newval);
}
/*
* Prepare environment variables for running regression tests
*/
@ -825,7 +789,7 @@ initialize_environment(void)
putenv(new_pgoptions);
}
if (temp_install)
if (temp_instance)
{
/*
* Clear out any environment vars that might cause psql to connect to
@ -863,49 +827,6 @@ initialize_environment(void)
sprintf(s, "%d", port);
doputenv("PGPORT", s);
}
/*
* GNU make stores some flags in the MAKEFLAGS environment variable to
* pass arguments to its own children. If we are invoked by make,
* that causes the make invoked by us to think its part of the make
* task invoking us, and so it tries to communicate with the toplevel
* make. Which fails.
*
* Unset the variable to protect against such problems. We also reset
* MAKELEVEL to be certain the child doesn't notice the make above us.
*/
unsetenv("MAKEFLAGS");
unsetenv("MAKELEVEL");
/*
* Adjust path variables to point into the temp-install tree
*/
bindir = psprintf("%s/install/%s", temp_install, bindir);
libdir = psprintf("%s/install/%s", temp_install, libdir);
datadir = psprintf("%s/install/%s", temp_install, datadir);
/* psql will be installed into temp-install bindir */
psqldir = bindir;
/*
* Set up shared library paths to include the temp install.
*
* LD_LIBRARY_PATH covers many platforms. DYLD_LIBRARY_PATH works on
* Darwin, and maybe other Mach-based systems. LIBPATH is for AIX.
* Windows needs shared libraries in PATH (only those linked into
* executables, not dlopen'ed ones). Feel free to account for others
* as well.
*/
add_to_path("LD_LIBRARY_PATH", ':', libdir);
add_to_path("DYLD_LIBRARY_PATH", ':', libdir);
add_to_path("LIBPATH", ':', libdir);
#if defined(WIN32)
add_to_path("PATH", ';', libdir);
#elif defined(__CYGWIN__)
add_to_path("PATH", ':', libdir);
#endif
}
else
{
@ -1158,8 +1079,8 @@ psql_command(const char *database, const char *query,...)
/* And now we can build and execute the shell command */
snprintf(psql_cmd, sizeof(psql_cmd),
"\"%s%spsql\" -X -c \"%s\" \"%s\"",
psqldir ? psqldir : "",
psqldir ? "/" : "",
bindir ? bindir : "",
bindir ? "/" : "",
query_escaped,
database);
@ -2051,21 +1972,18 @@ help(void)
printf(_(" --outputdir=DIR place output files in DIR (default \".\")\n"));
printf(_(" --schedule=FILE use test ordering schedule from FILE\n"));
printf(_(" (can be used multiple times to concatenate)\n"));
printf(_(" --temp-install=DIR create a temporary installation in DIR\n"));
printf(_(" --use-existing use an existing installation\n"));
printf(_(" --temp-instance=DIR create a temporary instance in DIR\n"));
printf(_(" --use-existing use an existing installation\n")); // XXX
printf(_("\n"));
printf(_("Options for \"temp-install\" mode:\n"));
printf(_(" --extra-install=DIR additional directory to install (e.g., contrib)\n"));
printf(_("Options for \"temp-instance\" mode:\n"));
printf(_(" --no-locale use C locale\n"));
printf(_(" --port=PORT start postmaster on PORT\n"));
printf(_(" --temp-config=FILE append contents of FILE to temporary config\n"));
printf(_(" --top-builddir=DIR (relative) path to top level build directory\n"));
printf(_("\n"));
printf(_("Options for using an existing installation:\n"));
printf(_(" --host=HOST use postmaster running on HOST\n"));
printf(_(" --port=PORT use postmaster running at PORT\n"));
printf(_(" --user=USER connect as USER\n"));
printf(_(" --psqldir=DIR use psql in DIR (default: configured bindir)\n"));
printf(_("\n"));
printf(_("The exit status is 0 if all tests passed, 1 if some tests failed, and 2\n"));
printf(_("if the tests could not be run for some reason.\n"));
@ -2087,20 +2005,18 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
{"encoding", required_argument, NULL, 6},
{"outputdir", required_argument, NULL, 7},
{"schedule", required_argument, NULL, 8},
{"temp-install", required_argument, NULL, 9},
{"temp-instance", required_argument, NULL, 9},
{"no-locale", no_argument, NULL, 10},
{"top-builddir", required_argument, NULL, 11},
{"host", required_argument, NULL, 13},
{"port", required_argument, NULL, 14},
{"user", required_argument, NULL, 15},
{"psqldir", required_argument, NULL, 16},
{"bindir", required_argument, NULL, 16},
{"dlpath", required_argument, NULL, 17},
{"create-role", required_argument, NULL, 18},
{"temp-config", required_argument, NULL, 19},
{"use-existing", no_argument, NULL, 20},
{"launcher", required_argument, NULL, 21},
{"load-extension", required_argument, NULL, 22},
{"extra-install", required_argument, NULL, 23},
{"config-auth", required_argument, NULL, 24},
{NULL, 0, NULL, 0}
};
@ -2172,14 +2088,11 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
add_stringlist_item(&schedulelist, optarg);
break;
case 9:
temp_install = make_absolute_path(optarg);
temp_instance = make_absolute_path(optarg);
break;
case 10:
nolocale = true;
break;
case 11:
top_builddir = strdup(optarg);
break;
case 13:
hostname = strdup(optarg);
break;
@ -2191,9 +2104,11 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
user = strdup(optarg);
break;
case 16:
/* "--psqldir=" should mean to use PATH */
/* "--bindir=" means to use PATH */
if (strlen(optarg))
psqldir = strdup(optarg);
bindir = strdup(optarg);
else
bindir = NULL;
break;
case 17:
dlpath = strdup(optarg);
@ -2213,9 +2128,6 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
case 22:
add_stringlist_item(&loadextension, optarg);
break;
case 23:
add_stringlist_item(&extra_install, optarg);
break;
case 24:
config_auth_datadir = pstrdup(optarg);
break;
@ -2244,7 +2156,7 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
exit(0);
}
if (temp_install && !port_specified_by_user)
if (temp_instance && !port_specified_by_user)
/*
* To reduce chances of interference with parallel installations, use
@ -2270,83 +2182,45 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
unlimit_core_size();
#endif
if (temp_install)
if (temp_instance)
{
FILE *pg_conf;
_stringlist *sl;
/*
* Prepare the temp installation
* Prepare the temp instance
*/
if (!top_builddir)
{
fprintf(stderr, _("--top-builddir must be specified when using --temp-install\n"));
exit(2);
}
if (directory_exists(temp_install))
if (directory_exists(temp_instance))
{
header(_("removing existing temp installation"));
if (!rmtree(temp_install, true))
header(_("removing existing temp instance"));
if (!rmtree(temp_instance, true))
{
fprintf(stderr, _("\n%s: could not remove temp installation \"%s\"\n"),
progname, temp_install);
fprintf(stderr, _("\n%s: could not remove temp instance \"%s\"\n"),
progname, temp_instance);
exit(2);
}
}
header(_("creating temporary installation"));
header(_("creating temporary instance"));
/* make the temp install top directory */
make_directory(temp_install);
/* make the temp instance top directory */
make_directory(temp_instance);
/* and a directory for log files */
snprintf(buf, sizeof(buf), "%s/log", outputdir);
snprintf(buf, sizeof(buf), "%s/log", temp_instance);
if (!directory_exists(buf))
make_directory(buf);
/* "make install" */
#ifndef WIN32_ONLY_COMPILER
snprintf(buf, sizeof(buf),
"\"%s\" -C \"%s\" DESTDIR=\"%s/install\" install > \"%s/log/install.log\" 2>&1",
makeprog, top_builddir, temp_install, outputdir);
#else
snprintf(buf, sizeof(buf),
"perl \"%s/src/tools/msvc/install.pl\" \"%s/install\" >\"%s/log/install.log\" 2>&1",
top_builddir, temp_install, outputdir);
#endif
if (system(buf))
{
fprintf(stderr, _("\n%s: installation failed\nExamine %s/log/install.log for the reason.\nCommand was: %s\n"), progname, outputdir, buf);
exit(2);
}
for (sl = extra_install; sl != NULL; sl = sl->next)
{
#ifndef WIN32_ONLY_COMPILER
snprintf(buf, sizeof(buf),
"\"%s\" -C \"%s/%s\" DESTDIR=\"%s/install\" install >> \"%s/log/install.log\" 2>&1",
makeprog, top_builddir, sl->str, temp_install, outputdir);
#else
fprintf(stderr, _("\n%s: --extra-install option not supported on this platform\n"), progname);
exit(2);
#endif
if (system(buf))
{
fprintf(stderr, _("\n%s: installation failed\nExamine %s/log/install.log for the reason.\nCommand was: %s\n"), progname, outputdir, buf);
exit(2);
}
}
/* initdb */
header(_("initializing database system"));
snprintf(buf, sizeof(buf),
"\"%s/initdb\" -D \"%s/data\" -L \"%s\" --noclean --nosync%s%s > \"%s/log/initdb.log\" 2>&1",
bindir, temp_install, datadir,
"\"%s%sinitdb\" -D \"%s/data\" --noclean --nosync%s%s > \"%s/log/initdb.log\" 2>&1",
bindir ? bindir : "",
bindir ? "/" : "",
temp_instance,
debug ? " --debug" : "",
nolocale ? " --no-locale" : "",
outputdir);
temp_instance);
if (system(buf))
{
fprintf(stderr, _("\n%s: initdb failed\nExamine %s/log/initdb.log for the reason.\nCommand was: %s\n"), progname, outputdir, buf);
@ -2361,7 +2235,7 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
* failures, don't set max_prepared_transactions any higher than
* actually needed by the prepared_xacts regression test.)
*/
snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_install);
snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
pg_conf = fopen(buf, "a");
if (pg_conf == NULL)
{
@ -2399,7 +2273,7 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
* Since we successfully used the same buffer for the much-longer
* "initdb" command, this can't truncate.
*/
snprintf(buf, sizeof(buf), "%s/data", temp_install);
snprintf(buf, sizeof(buf), "%s/data", temp_instance);
config_sspi_auth(buf);
#elif !defined(HAVE_UNIX_SOCKETS)
#error Platform has no means to secure the test installation.
@ -2409,8 +2283,10 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
* Check if there is a postmaster running already.
*/
snprintf(buf2, sizeof(buf2),
"\"%s/psql\" -X postgres <%s 2>%s",
bindir, DEVNULL, DEVNULL);
"\"%s%spsql\" -X postgres <%s 2>%s",
bindir ? bindir : "",
bindir ? "/" : "",
DEVNULL, DEVNULL);
for (i = 0; i < 16; i++)
{
@ -2441,12 +2317,14 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
*/
header(_("starting postmaster"));
snprintf(buf, sizeof(buf),
"\"%s/postgres\" -D \"%s/data\" -F%s "
"\"%s%spostgres\" -D \"%s/data\" -F%s "
"-c \"listen_addresses=%s\" -k \"%s\" "
"> \"%s/log/postmaster.log\" 2>&1",
bindir, temp_install, debug ? " -d 5" : "",
bindir ? bindir : "",
bindir ? "/" : "",
temp_instance, debug ? " -d 5" : "",
hostname ? hostname : "", sockdir ? sockdir : "",
outputdir);
temp_instance);
postmaster_pid = spawn_process(buf);
if (postmaster_pid == INVALID_PID)
{
@ -2560,23 +2438,23 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
/*
* Shut down temp installation's postmaster
*/
if (temp_install)
if (temp_instance)
{
header(_("shutting down postmaster"));
stop_postmaster();
}
/*
* If there were no errors, remove the temp installation immediately to
* conserve disk space. (If there were errors, we leave the installation
* If there were no errors, remove the temp instance immediately to
* conserve disk space. (If there were errors, we leave the instance
* in place for possible manual investigation.)
*/
if (temp_install && fail_count == 0 && fail_ignore_count == 0)
if (temp_instance && fail_count == 0 && fail_ignore_count == 0)
{
header(_("removing temporary installation"));
if (!rmtree(temp_install, true))
fprintf(stderr, _("\n%s: could not remove temp installation \"%s\"\n"),
progname, temp_install);
header(_("removing temporary instance"));
if (!rmtree(temp_instance, true))
fprintf(stderr, _("\n%s: could not remove temp instance \"%s\"\n"),
progname, temp_instance);
}
fclose(logfile);

6
src/test/regress/pg_regress.h
View File

@ -43,12 +43,6 @@ extern char *inputdir;
extern char *outputdir;
extern char *launcher;
/*
* This should not be global but every module should be able to read command
* line parameters.
*/
extern char *psqldir;
extern const char *basic_diff_opts;
extern const char *pretty_diff_opts;

4
src/test/regress/pg_regress_main.c
View File

@ -65,8 +65,8 @@ psql_start_test(const char *testname,
snprintf(psql_cmd + offset, sizeof(psql_cmd) - offset,
"\"%s%spsql\" -X -a -q -d \"%s\" < \"%s\" > \"%s\" 2>&1",
psqldir ? psqldir : "",
psqldir ? "/" : "",
bindir ? bindir : "",
bindir ? "/" : "",
dblist->str,
infile,
outfile);