mirror of
https://github.com/postgres/postgres.git
synced 2025-06-11 20:28:21 +03:00
Add support for asynchronous execution.
This implements asynchronous execution, which runs multiple parts of a non-parallel-aware Append concurrently rather than serially to improve performance when possible. Currently, the only node type that can be run concurrently is a ForeignScan that is an immediate child of such an Append. In the case where such ForeignScans access data on different remote servers, this would run those ForeignScans concurrently, and overlap the remote operations to be performed simultaneously, so it'll improve the performance especially when the operations involve time-consuming ones such as remote join and remote aggregation. We may extend this to other node types such as joins or aggregates over ForeignScans in the future. This also adds the support for postgres_fdw, which is enabled by the table-level/server-level option "async_capable". The default is false. Robert Haas, Kyotaro Horiguchi, Thomas Munro, and myself. This commit is mostly based on the patch proposed by Robert Haas, but also uses stuff from the patch proposed by Kyotaro Horiguchi and from the patch proposed by Thomas Munro. Reviewed by Kyotaro Horiguchi, Konstantin Knizhnik, Andrey Lepikhov, Movead Li, Thomas Munro, Justin Pryzby, and others. Discussion: https://postgr.es/m/CA%2BTgmoaXQEt4tZ03FtQhnzeDEMzBck%2BLrni0UWHVVgOTnA6C1w%40mail.gmail.com Discussion: https://postgr.es/m/CA%2BhUKGLBRyu0rHrDCMC4%3DRn3252gogyp1SjOgG8SEKKZv%3DFwfQ%40mail.gmail.com Discussion: https://postgr.es/m/20200228.170650.667613673625155850.horikyota.ntt%40gmail.com
This commit is contained in:
Etsuro Fujita
@ -2928,3 +2928,198 @@ SELECT tableoid::regclass, * FROM batch_cp_upd_test;
|
||||
|
||||
-- Clean up
|
||||
DROP TABLE batch_table, batch_cp_upd_test CASCADE;
|
||||
|
||||
-- ===================================================================
|
||||
-- test asynchronous execution
|
||||
-- ===================================================================
|
||||
|
||||
ALTER SERVER loopback OPTIONS (DROP extensions);
|
||||
ALTER SERVER loopback OPTIONS (ADD async_capable 'true');
|
||||
ALTER SERVER loopback2 OPTIONS (ADD async_capable 'true');
|
||||
|
||||
CREATE TABLE async_pt (a int, b int, c text) PARTITION BY RANGE (a);
|
||||
CREATE TABLE base_tbl1 (a int, b int, c text);
|
||||
CREATE TABLE base_tbl2 (a int, b int, c text);
|
||||
CREATE FOREIGN TABLE async_p1 PARTITION OF async_pt FOR VALUES FROM (1000) TO (2000)
|
||||
SERVER loopback OPTIONS (table_name 'base_tbl1');
|
||||
CREATE FOREIGN TABLE async_p2 PARTITION OF async_pt FOR VALUES FROM (2000) TO (3000)
|
||||
SERVER loopback2 OPTIONS (table_name 'base_tbl2');
|
||||
INSERT INTO async_p1 SELECT 1000 + i, i, to_char(i, 'FM0000') FROM generate_series(0, 999, 5) i;
|
||||
INSERT INTO async_p2 SELECT 2000 + i, i, to_char(i, 'FM0000') FROM generate_series(0, 999, 5) i;
|
||||
ANALYZE async_pt;
|
||||
|
||||
-- simple queries
|
||||
CREATE TABLE result_tbl (a int, b int, c text);
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
INSERT INTO result_tbl SELECT * FROM async_pt WHERE b % 100 = 0;
|
||||
INSERT INTO result_tbl SELECT * FROM async_pt WHERE b % 100 = 0;
|
||||
|
||||
SELECT * FROM result_tbl ORDER BY a;
|
||||
DELETE FROM result_tbl;
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
INSERT INTO result_tbl SELECT * FROM async_pt WHERE b === 505;
|
||||
INSERT INTO result_tbl SELECT * FROM async_pt WHERE b === 505;
|
||||
|
||||
SELECT * FROM result_tbl ORDER BY a;
|
||||
DELETE FROM result_tbl;
|
||||
|
||||
-- Check case where multiple partitions use the same connection
|
||||
CREATE TABLE base_tbl3 (a int, b int, c text);
|
||||
CREATE FOREIGN TABLE async_p3 PARTITION OF async_pt FOR VALUES FROM (3000) TO (4000)
|
||||
SERVER loopback2 OPTIONS (table_name 'base_tbl3');
|
||||
INSERT INTO async_p3 SELECT 3000 + i, i, to_char(i, 'FM0000') FROM generate_series(0, 999, 5) i;
|
||||
ANALYZE async_pt;
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
INSERT INTO result_tbl SELECT * FROM async_pt WHERE b === 505;
|
||||
INSERT INTO result_tbl SELECT * FROM async_pt WHERE b === 505;
|
||||
|
||||
SELECT * FROM result_tbl ORDER BY a;
|
||||
DELETE FROM result_tbl;
|
||||
|
||||
DROP FOREIGN TABLE async_p3;
|
||||
DROP TABLE base_tbl3;
|
||||
|
||||
-- Check case where the partitioned table has local/remote partitions
|
||||
CREATE TABLE async_p3 PARTITION OF async_pt FOR VALUES FROM (3000) TO (4000);
|
||||
INSERT INTO async_p3 SELECT 3000 + i, i, to_char(i, 'FM0000') FROM generate_series(0, 999, 5) i;
|
||||
ANALYZE async_pt;
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
INSERT INTO result_tbl SELECT * FROM async_pt WHERE b === 505;
|
||||
INSERT INTO result_tbl SELECT * FROM async_pt WHERE b === 505;
|
||||
|
||||
SELECT * FROM result_tbl ORDER BY a;
|
||||
DELETE FROM result_tbl;
|
||||
|
||||
-- partitionwise joins
|
||||
SET enable_partitionwise_join TO true;
|
||||
|
||||
CREATE TABLE join_tbl (a1 int, b1 int, c1 text, a2 int, b2 int, c2 text);
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
INSERT INTO join_tbl SELECT * FROM async_pt t1, async_pt t2 WHERE t1.a = t2.a AND t1.b = t2.b AND t1.b % 100 = 0;
|
||||
INSERT INTO join_tbl SELECT * FROM async_pt t1, async_pt t2 WHERE t1.a = t2.a AND t1.b = t2.b AND t1.b % 100 = 0;
|
||||
|
||||
SELECT * FROM join_tbl ORDER BY a1;
|
||||
DELETE FROM join_tbl;
|
||||
|
||||
RESET enable_partitionwise_join;
|
||||
|
||||
-- Test interaction of async execution with plan-time partition pruning
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
SELECT * FROM async_pt WHERE a < 3000;
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
SELECT * FROM async_pt WHERE a < 2000;
|
||||
|
||||
-- Test interaction of async execution with run-time partition pruning
|
||||
SET plan_cache_mode TO force_generic_plan;
|
||||
|
||||
PREPARE async_pt_query (int, int) AS
|
||||
INSERT INTO result_tbl SELECT * FROM async_pt WHERE a < $1 AND b === $2;
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
EXECUTE async_pt_query (3000, 505);
|
||||
EXECUTE async_pt_query (3000, 505);
|
||||
|
||||
SELECT * FROM result_tbl ORDER BY a;
|
||||
DELETE FROM result_tbl;
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
EXECUTE async_pt_query (2000, 505);
|
||||
EXECUTE async_pt_query (2000, 505);
|
||||
|
||||
SELECT * FROM result_tbl ORDER BY a;
|
||||
DELETE FROM result_tbl;
|
||||
|
||||
RESET plan_cache_mode;
|
||||
|
||||
CREATE TABLE local_tbl(a int, b int, c text);
|
||||
INSERT INTO local_tbl VALUES (1505, 505, 'foo'), (2505, 505, 'bar');
|
||||
ANALYZE local_tbl;
|
||||
|
||||
CREATE INDEX base_tbl1_idx ON base_tbl1 (a);
|
||||
CREATE INDEX base_tbl2_idx ON base_tbl2 (a);
|
||||
CREATE INDEX async_p3_idx ON async_p3 (a);
|
||||
ANALYZE base_tbl1;
|
||||
ANALYZE base_tbl2;
|
||||
ANALYZE async_p3;
|
||||
|
||||
ALTER FOREIGN TABLE async_p1 OPTIONS (use_remote_estimate 'true');
|
||||
ALTER FOREIGN TABLE async_p2 OPTIONS (use_remote_estimate 'true');
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
SELECT * FROM local_tbl, async_pt WHERE local_tbl.a = async_pt.a AND local_tbl.c = 'bar';
|
||||
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
|
||||
SELECT * FROM local_tbl, async_pt WHERE local_tbl.a = async_pt.a AND local_tbl.c = 'bar';
|
||||
SELECT * FROM local_tbl, async_pt WHERE local_tbl.a = async_pt.a AND local_tbl.c = 'bar';
|
||||
|
||||
ALTER FOREIGN TABLE async_p1 OPTIONS (DROP use_remote_estimate);
|
||||
ALTER FOREIGN TABLE async_p2 OPTIONS (DROP use_remote_estimate);
|
||||
|
||||
DROP TABLE local_tbl;
|
||||
DROP INDEX base_tbl1_idx;
|
||||
DROP INDEX base_tbl2_idx;
|
||||
DROP INDEX async_p3_idx;
|
||||
|
||||
-- Test that pending requests are processed properly
|
||||
SET enable_mergejoin TO false;
|
||||
SET enable_hashjoin TO false;
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
SELECT * FROM async_pt t1, async_p2 t2 WHERE t1.a = t2.a AND t1.b === 505;
|
||||
SELECT * FROM async_pt t1, async_p2 t2 WHERE t1.a = t2.a AND t1.b === 505;
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
SELECT * FROM async_pt t1 WHERE t1.b === 505 LIMIT 1;
|
||||
SELECT * FROM async_pt t1 WHERE t1.b === 505 LIMIT 1;
|
||||
|
||||
-- Check with foreign modify
|
||||
CREATE TABLE local_tbl (a int, b int, c text);
|
||||
INSERT INTO local_tbl VALUES (1505, 505, 'foo');
|
||||
|
||||
CREATE TABLE base_tbl3 (a int, b int, c text);
|
||||
CREATE FOREIGN TABLE remote_tbl (a int, b int, c text)
|
||||
SERVER loopback OPTIONS (table_name 'base_tbl3');
|
||||
INSERT INTO remote_tbl VALUES (2505, 505, 'bar');
|
||||
|
||||
CREATE TABLE base_tbl4 (a int, b int, c text);
|
||||
CREATE FOREIGN TABLE insert_tbl (a int, b int, c text)
|
||||
SERVER loopback OPTIONS (table_name 'base_tbl4');
|
||||
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
INSERT INTO insert_tbl (SELECT * FROM local_tbl UNION ALL SELECT * FROM remote_tbl);
|
||||
INSERT INTO insert_tbl (SELECT * FROM local_tbl UNION ALL SELECT * FROM remote_tbl);
|
||||
|
||||
SELECT * FROM insert_tbl ORDER BY a;
|
||||
|
||||
-- Check with direct modify
|
||||
EXPLAIN (VERBOSE, COSTS OFF)
|
||||
WITH t AS (UPDATE remote_tbl SET c = c || c RETURNING *)
|
||||
INSERT INTO join_tbl SELECT * FROM async_pt LEFT JOIN t ON (async_pt.a = t.a AND async_pt.b = t.b) WHERE async_pt.b === 505;
|
||||
WITH t AS (UPDATE remote_tbl SET c = c || c RETURNING *)
|
||||
INSERT INTO join_tbl SELECT * FROM async_pt LEFT JOIN t ON (async_pt.a = t.a AND async_pt.b = t.b) WHERE async_pt.b === 505;
|
||||
|
||||
SELECT * FROM join_tbl ORDER BY a1;
|
||||
DELETE FROM join_tbl;
|
||||
|
||||
RESET enable_mergejoin;
|
||||
RESET enable_hashjoin;
|
||||
|
||||
-- Clean up
|
||||
DROP TABLE async_pt;
|
||||
DROP TABLE base_tbl1;
|
||||
DROP TABLE base_tbl2;
|
||||
DROP TABLE result_tbl;
|
||||
DROP TABLE local_tbl;
|
||||
DROP FOREIGN TABLE remote_tbl;
|
||||
DROP FOREIGN TABLE insert_tbl;
|
||||
DROP TABLE base_tbl3;
|
||||
DROP TABLE base_tbl4;
|
||||
DROP TABLE join_tbl;
|
||||
|
||||
ALTER SERVER loopback OPTIONS (DROP async_capable);
|
||||
ALTER SERVER loopback2 OPTIONS (DROP async_capable);
|
||||
|
||||
Reference in New Issue
Block a user