diff --git a/mysql-test/suite/heap/tmp_table_convert_dedup.result b/mysql-test/suite/heap/tmp_table_convert_dedup.result new file mode 100644 index 00000000000..966f3b88eea --- /dev/null +++ b/mysql-test/suite/heap/tmp_table_convert_dedup.result @@ -0,0 +1,141 @@ +# +# create_internal_tmp_table_from_heap() must check the row that +# overflowed the in-memory table against the unique constraint of the +# table it converts to. +# +# That row has not been stored anywhere yet, so the conversion writes +# it to the new table itself. If it is written while the new handler +# still holds its indexes disabled for the bulk copy, its duplicate is +# not detected, and a duplicate row survives in a table whose only +# purpose is to remove duplicates. +# +# The on-disk temporary engine only disables its indexes for tables of +# at least a hundred rows, so the in-memory table has to hold that many +# rows when it overflows. blob_big.inc uses fifty distinct values and +# therefore never reaches that path. +# +# +# Setup: 1000 distinct pairs of ~1 KB values, each pair present twice, +# with the two copies of a pair adjacent, so that the write which +# overflows a deduplicating temporary table is always a duplicate of a +# row the conversion has already copied. +# +# The columns are wide enough that the deduplicating key does not fit +# into a key of the on-disk temporary engine, so deduplication goes +# through a unique constraint rather than through a unique index. +# +set @save_tmp_memory_table_size=@@tmp_memory_table_size; +set @save_max_heap_table_size=@@max_heap_table_size; +CREATE TABLE t1 (id INT, v VARCHAR(1024), w VARCHAR(1024)) ENGINE=MyISAM; +INSERT INTO t1 +SELECT (seq+1) DIV 2, +CONCAT('v', LPAD((seq+1) DIV 2, 6, '0')), +CONCAT('w', LPAD((seq+1) DIV 2, 6, '0')) +FROM seq_1_to_2000; +SELECT COUNT(*), COUNT(DISTINCT v, w) FROM t1; +COUNT(*) COUNT(DISTINCT v, w) +2000 1000 +# +# ================================================================ +# Run 1: the temporary tables overflow and are converted +# ================================================================ +# +set @@tmp_memory_table_size=1024*1024*2; +set @@max_heap_table_size=1024*1024*2; +# --- SELECT DISTINCT (end_write path) --- +FLUSH STATUS; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT DISTINCT v, w FROM t1) dt; +rows_returned distinct_rows +1000 1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# --- SELECT DISTINCT with ORDER BY --- +FLUSH STATUS; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT DISTINCT v, w FROM t1 ORDER BY v, w) dt; +rows_returned distinct_rows +1000 1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# --- GROUP BY (end_update path) --- +FLUSH STATUS; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT v, w FROM t1 GROUP BY v, w) dt; +rows_returned distinct_rows +1000 1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# --- UNION DISTINCT (select_unit::write_record path) --- +FLUSH STATUS; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM ((SELECT v, w FROM t1) UNION (SELECT v, w FROM t1)) dt; +rows_returned distinct_rows +1000 1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# --- INSERT SELECT DISTINCT: the duplicate reaches a user table --- +FLUSH STATUS; +CREATE TABLE t2 (v VARCHAR(1024), w VARCHAR(1024)) ENGINE=MyISAM; +INSERT INTO t2 SELECT DISTINCT v, w FROM t1; +SELECT COUNT(*) AS rows_stored, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM t2; +rows_stored distinct_rows +1000 1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +DROP TABLE t2; +# +# ================================================================ +# Run 2: the same statements with enough memory to stay in HEAP +# ================================================================ +# +set @@tmp_memory_table_size=1024*1024*512; +set @@max_heap_table_size=1024*1024*512; +FLUSH STATUS; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT DISTINCT v, w FROM t1) dt; +rows_returned distinct_rows +1000 1000 +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT DISTINCT v, w FROM t1 ORDER BY v, w) dt; +rows_returned distinct_rows +1000 1000 +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT v, w FROM t1 GROUP BY v, w) dt; +rows_returned distinct_rows +1000 1000 +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM ((SELECT v, w FROM t1) UNION (SELECT v, w FROM t1)) dt; +rows_returned distinct_rows +1000 1000 +CREATE TABLE t2 (v VARCHAR(1024), w VARCHAR(1024)) ENGINE=MyISAM; +INSERT INTO t2 SELECT DISTINCT v, w FROM t1; +SELECT COUNT(*) AS rows_stored, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM t2; +rows_stored distinct_rows +1000 1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +OFF +# Clean up +set @@tmp_memory_table_size=@save_tmp_memory_table_size; +set @@max_heap_table_size=@save_max_heap_table_size; +DROP TABLE t1, t2; diff --git a/mysql-test/suite/heap/tmp_table_convert_dedup.test b/mysql-test/suite/heap/tmp_table_convert_dedup.test new file mode 100644 index 00000000000..f3993c14891 --- /dev/null +++ b/mysql-test/suite/heap/tmp_table_convert_dedup.test @@ -0,0 +1,134 @@ +--source include/have_sequence.inc + +--echo # +--echo # create_internal_tmp_table_from_heap() must check the row that +--echo # overflowed the in-memory table against the unique constraint of the +--echo # table it converts to. +--echo # +--echo # That row has not been stored anywhere yet, so the conversion writes +--echo # it to the new table itself. If it is written while the new handler +--echo # still holds its indexes disabled for the bulk copy, its duplicate is +--echo # not detected, and a duplicate row survives in a table whose only +--echo # purpose is to remove duplicates. +--echo # +--echo # The on-disk temporary engine only disables its indexes for tables of +--echo # at least a hundred rows, so the in-memory table has to hold that many +--echo # rows when it overflows. blob_big.inc uses fifty distinct values and +--echo # therefore never reaches that path. +--echo # + +# SHOW STATUS output differs across protocol variants; disable them globally. +--disable_cursor_protocol +--disable_ps_protocol +--disable_ps2_protocol + +--echo # +--echo # Setup: 1000 distinct pairs of ~1 KB values, each pair present twice, +--echo # with the two copies of a pair adjacent, so that the write which +--echo # overflows a deduplicating temporary table is always a duplicate of a +--echo # row the conversion has already copied. +--echo # +--echo # The columns are wide enough that the deduplicating key does not fit +--echo # into a key of the on-disk temporary engine, so deduplication goes +--echo # through a unique constraint rather than through a unique index. +--echo # + +set @save_tmp_memory_table_size=@@tmp_memory_table_size; +set @save_max_heap_table_size=@@max_heap_table_size; + +CREATE TABLE t1 (id INT, v VARCHAR(1024), w VARCHAR(1024)) ENGINE=MyISAM; +INSERT INTO t1 +SELECT (seq+1) DIV 2, + CONCAT('v', LPAD((seq+1) DIV 2, 6, '0')), + CONCAT('w', LPAD((seq+1) DIV 2, 6, '0')) +FROM seq_1_to_2000; + +SELECT COUNT(*), COUNT(DISTINCT v, w) FROM t1; + +--echo # +--echo # ================================================================ +--echo # Run 1: the temporary tables overflow and are converted +--echo # ================================================================ +--echo # + +set @@tmp_memory_table_size=1024*1024*2; +set @@max_heap_table_size=1024*1024*2; + +--echo # --- SELECT DISTINCT (end_write path) --- +FLUSH STATUS; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT DISTINCT v, w FROM t1) dt; +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; + +--echo # --- SELECT DISTINCT with ORDER BY --- +FLUSH STATUS; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT DISTINCT v, w FROM t1 ORDER BY v, w) dt; +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; + +--echo # --- GROUP BY (end_update path) --- +FLUSH STATUS; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT v, w FROM t1 GROUP BY v, w) dt; +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; + +--echo # --- UNION DISTINCT (select_unit::write_record path) --- +FLUSH STATUS; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM ((SELECT v, w FROM t1) UNION (SELECT v, w FROM t1)) dt; +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; + +--echo # --- INSERT SELECT DISTINCT: the duplicate reaches a user table --- +FLUSH STATUS; +CREATE TABLE t2 (v VARCHAR(1024), w VARCHAR(1024)) ENGINE=MyISAM; +INSERT INTO t2 SELECT DISTINCT v, w FROM t1; +SELECT COUNT(*) AS rows_stored, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM t2; +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +DROP TABLE t2; + +--echo # +--echo # ================================================================ +--echo # Run 2: the same statements with enough memory to stay in HEAP +--echo # ================================================================ +--echo # + +set @@tmp_memory_table_size=1024*1024*512; +set @@max_heap_table_size=1024*1024*512; + +FLUSH STATUS; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT DISTINCT v, w FROM t1) dt; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT DISTINCT v, w FROM t1 ORDER BY v, w) dt; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM (SELECT v, w FROM t1 GROUP BY v, w) dt; +SELECT COUNT(*) AS rows_returned, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM ((SELECT v, w FROM t1) UNION (SELECT v, w FROM t1)) dt; +CREATE TABLE t2 (v VARCHAR(1024), w VARCHAR(1024)) ENGINE=MyISAM; +INSERT INTO t2 SELECT DISTINCT v, w FROM t1; +SELECT COUNT(*) AS rows_stored, COUNT(DISTINCT CONCAT(v,'#',w)) AS distinct_rows +FROM t2; +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; + +--echo # Clean up + +set @@tmp_memory_table_size=@save_tmp_memory_table_size; +set @@max_heap_table_size=@save_max_heap_table_size; +DROP TABLE t1, t2; + +--enable_ps2_protocol +--enable_ps_protocol +--enable_cursor_protocol diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 31aada29e92..3a5fd9be9ea 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -24077,6 +24077,13 @@ bool create_internal_tmp_table(TABLE *table, KEY *org_keyinfo, Default row copier of create_internal_tmp_table_from_heap(): copy the rows in table scan order, then append the pending record[0], whose write filled the in-memory table, as a new row. + + The pending row is appended from write_pending_row(), after the bulk + insert of the copy has ended, because the on-disk tmp engine serves a + bulk insert into an internal table with the indexes of that table + disabled (@see ha_maria::start_bulk_insert()). The pending row is the + only row of the conversion that can be a duplicate, so it is also the + only one whose write must be checked. */ class Tmp_table_default_copier: public Tmp_table_row_copier @@ -24086,6 +24093,7 @@ public: : ignore_last_dupp_key_error(ignore_last_dupp_key_error_arg) {} int copy_rows(TABLE *from, TABLE *to) override; + int write_pending_row(TABLE *from, TABLE *to) override; private: bool ignore_last_dupp_key_error; @@ -24114,6 +24122,15 @@ int Tmp_table_default_copier::copy_rows(TABLE *from, TABLE *to) DBUG_RETURN(-1); // The error is already reported } + DBUG_RETURN(0); +} + + +int Tmp_table_default_copier::write_pending_row(TABLE *from, TABLE *to) +{ + int write_err; + DBUG_ENTER("Tmp_table_default_copier::write_pending_row"); + /* copy row that filled HEAP table */ if (unlikely((write_err= to->file->ha_write_tmp_row(from->record[0])))) { @@ -24366,6 +24383,16 @@ err: already in the table and must not be appended as an extra row. + DESCRIPTION OF THE COPY + The rows already stored in the HEAP table are copied by + Tmp_table_row_copier::copy_rows() within a bulk insert into the new + table, which the on-disk tmp engine may serve with the indexes of + that table disabled. Those rows already satisfy every unique + constraint of the table, so they need no checking. The pending row + does not, so it is written afterwards, by + Tmp_table_row_copier::write_pending_row(), once the bulk insert has + ended and the indexes are back in place. + DESCRIPTION Creates a new internal temporary table using the on-disk tmp engine (MyISAM or Aria) and copies all rows of the full HEAP table into it @@ -24461,6 +24488,13 @@ create_internal_tmp_table_from_heap(THD *thd, TABLE *table, if (end_err && !write_err) write_err= end_err; } + /* + Only now, with the indexes of the new table in place, can a write be + checked against them, so this is where the pending row goes. + */ + if (!write_err) + write_err= copier->write_pending_row(table, &new_table); + if (write_err) { if (write_err < 0) diff --git a/sql/sql_select.h b/sql/sql_select.h index ba85c3f91e3..404692404a6 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -2967,15 +2967,29 @@ class Tmp_table_row_copier public: virtual ~Tmp_table_row_copier() = default; /* - Copy all rows of 'from' into 'to', including the pending record[0] that - did not fit into 'from'. + Copy the rows stored in 'from' into 'to'. Runs inside the bulk insert + of 'to', which the on-disk tmp engine may serve with its indexes + disabled (@see ha_maria::start_bulk_insert()): the rows of 'from' + already satisfy every unique constraint of the table, so they need no + checking against each other. @retval 0 all rows copied @retval -1 failed, the error has been reported @retval >0 failed with this handler error, which the caller reports */ virtual int copy_rows(TABLE *from, TABLE *to)= 0; - /* Set by copy_rows() when the pending record[0] was an ignored duplicate */ + /* + Write the pending record[0], which overflowed 'from' and is therefore + stored nowhere yet, into 'to'. Called after the bulk insert has ended, + so that the write is checked against the indexes of 'to' and a + duplicate of an already copied row is detected. Does nothing for a + copier that has to write the pending row within copy_rows(). + + @retval 0 written, or an ignored duplicate + @retval >0 failed with this handler error, which the caller reports + */ + virtual int write_pending_row(TABLE *from, TABLE *to) { return 0; } + /* Set when the pending record[0] was an ignored duplicate */ bool duplicate_row_error= false; }; @@ -3014,6 +3028,15 @@ public: point into the handler's shared blob reassembly buffer (@see hp_read_blobs()), which reading the other rows overwrites, so they are first given memory of their own. + + That row is written in its own place in the sequence, and not from + write_pending_row() after the bulk insert as the default copier writes + its pending row, because its new position is only known once the rows + before it have been written. Nothing is lost by writing it with the + indexes of the new table still disabled: it replaces a row that is + already in the table rather than adding one, and an update of a window + function value can not collide with another row, as the columns a + deduplicating key of the table is built on are not the ones it changes. */ class Window_rowid_remapper : public Tmp_table_row_copier