database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2026-10-23 18:49:29 +03:00
Code Activity

MDEV-40957 various fixes for mixed DML path that suffered from uninit bitmaps and wrong SQL statements that failed in DuckDB.

Browse Source
This commit is contained in:
drrtuy
2026-09-12 21:57:31 +01:00
parent b6398296ae
commit 15174ecd8e
10 changed files with 356 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
storage/duckdb/convertor/dml_convertor.cc
View File

@@ -216,9 +216,9 @@ std::string DMLConvertor::translate()
void DMLConvertor::fill_index_fields_for_where(std::vector<Field *> &fields)
{
KEY *key_info= m_table->key_info;
if (key_info)
if (m_table->s->primary_key != MAX_KEY)
{
KEY *key_info= m_table->key_info + m_table->s->primary_key;
KEY_PART_INFO *key_part= key_info->key_part;
for (uint j= 0; j < key_info->user_defined_key_parts; j++, key_part++)
{

10
storage/duckdb/docs/architecture.md
View File

@@ -293,12 +293,14 @@ XA transactions are rejected (`reject_xa_if_active`).
## Batch DML (DeltaAppender)
For bulk INSERT (and mixed INSERT/UPDATE/DELETE within a single transaction):
For bulk INSERT, the insert-only path appends directly to the target table. For mixed
INSERT/UPDATE/DELETE within a transaction:
1. Creates a **temporary table** cloning the target schema + 3 auxiliary columns (`#mdb_delete_flag`, `#mdb_row_no`, `#mdb_trx_no`).
2. Writes rows via the `duckdb::Appender` API (much faster than per-row SQL INSERT).
3. On `flush()` (at commit/prepare): executes `INSERT INTO target SELECT … FROM tmp` and/or `DELETE FROM target WHERE pk IN (SELECT pk FROM tmp WHERE delete_flag)`.
4. The temp table lives in the connection-local temporary catalog — invisible to other sessions.
2. Writes inserted row versions and delete markers via the `duckdb::Appender` API.
3. On `flush()` (at commit/prepare), deletes target rows for every primary key present in the temporary table.
4. For each primary key, selects the temporary row with the greatest `#mdb_row_no` and inserts it unless that latest row is a delete marker.
5. The temp table lives in the connection-local temporary catalog — invisible to other sessions.
---

63
storage/duckdb/ha_duckdb.cc
View File

@@ -423,20 +423,36 @@ static bool calc_pk_difference(const uchar *old_row, const uchar *new_row,
return false;
}
static myduck::BatchState get_batch_state(THD *thd)
static int get_batch_state(THD *thd, bool insert_only,
myduck::BatchState *batch_state)
{
auto *ctx= get_duckdb_context(thd);
myduck::BatchState batch_state= ctx->get_batch_state();
*batch_state= ctx->get_batch_state();
if (batch_state == myduck::BatchState::UNDEFINED)
if (*batch_state == myduck::BatchState::IN_INSERT_ONLY_BATCH &&
!insert_only)
{
if (dml_in_batch)
batch_state= myduck::BatchState::IN_INSERT_ONLY_BATCH;
else
batch_state= myduck::BatchState::NOT_IN_BATCH;
ctx->set_batch_state(batch_state);
std::string error_msg;
if (ctx->flush_appenders(error_msg))
{
my_error(ER_GET_ERRMSG, MYF(0), HA_DUCKDB_APPEND_ERROR,
error_msg.c_str(), "DuckDB");
return HA_DUCKDB_APPEND_ERROR;
}
*batch_state= myduck::BatchState::UNDEFINED;
}
return batch_state;
if (*batch_state == myduck::BatchState::UNDEFINED)
{
if (!dml_in_batch)
*batch_state= myduck::BatchState::NOT_IN_BATCH;
else if (insert_only)
*batch_state= myduck::BatchState::IN_INSERT_ONLY_BATCH;
else
*batch_state= myduck::BatchState::IN_MIX_BATCH;
ctx->set_batch_state(*batch_state);
}
return 0;
}
/* Build duckdb type map of blob type */
@@ -490,7 +506,13 @@ int ha_duckdb::write_row(const uchar *buf)
DBUG_RETURN(ret);
}
myduck::BatchState batch_state= get_batch_state(thd);
myduck::BatchState batch_state;
ret= get_batch_state(thd, true, &batch_state);
if (ret)
{
dbug_tmp_restore_column_map(&table->read_set, org_bitmap);
DBUG_RETURN(ret);
}
if (batch_state == myduck::BatchState::NOT_IN_BATCH)
{
@@ -666,7 +688,10 @@ int ha_duckdb::update_row(const uchar *old_row, const uchar *new_row)
if (ret)
DBUG_RETURN(ret);
myduck::BatchState batch_state= get_batch_state(thd);
myduck::BatchState batch_state;
ret= get_batch_state(thd, false, &batch_state);
if (ret)
DBUG_RETURN(ret);
if (batch_state == myduck::BatchState::NOT_IN_BATCH)
{
@@ -711,7 +736,10 @@ int ha_duckdb::delete_row(const uchar *)
if (ret)
DBUG_RETURN(ret);
myduck::BatchState batch_state= get_batch_state(thd);
myduck::BatchState batch_state;
ret= get_batch_state(thd, false, &batch_state);
if (ret)
DBUG_RETURN(ret);
if (batch_state == myduck::BatchState::NOT_IN_BATCH)
{
@@ -789,10 +817,17 @@ int ha_duckdb::rnd_init(bool)
else
DBUG_RETURN(HA_ERR_INTERNAL_ERROR);
auto *ctx= get_duckdb_context(thd);
std::string error_msg;
if (ctx->flush_appenders(error_msg))
{
my_error(ER_GET_ERRMSG, MYF(0), HA_DUCKDB_APPEND_ERROR,
error_msg.c_str(), "DuckDB");
DBUG_RETURN(HA_DUCKDB_APPEND_ERROR);
}
std::string query= "SELECT * FROM " + quote_duckdb_identifier(schema_name) +
"." + quote_duckdb_identifier(table_name);
auto *ctx= get_duckdb_context(thd);
query_result= myduck::duckdb_query(ctx->get_connection(), query);
if (query_result->HasError())
{

1
storage/duckdb/ha_duckdb.h
View File

@@ -93,6 +93,7 @@ public:
{
return (HA_BINLOG_STMT_CAPABLE | HA_BINLOG_ROW_CAPABLE |
HA_NULL_IN_KEY | HA_CAN_INDEX_BLOBS |
HA_PRIMARY_KEY_REQUIRED_FOR_DELETE |
HA_CAN_DIRECT_UPDATE_AND_DELETE);
}

15
storage/duckdb/mysql-test/duckdb/r/mdev_40957.result Normal file
View File

@@ -0,0 +1,15 @@
CREATE TABLE invoice (
id UUID DEFAULT UUID() PRIMARY KEY,
product VARCHAR(64)
) ENGINE=DuckDB;
CREATE TABLE invoice_audit (n INT) ENGINE=InnoDB;
CREATE TRIGGER invoice_ad AFTER DELETE ON invoice
FOR EACH ROW INSERT INTO invoice_audit VALUES (1);
INSERT INTO invoice (product) VALUES ('one'), ('two'), ('three'), ('four');
DELETE FROM invoice;
SELECT COUNT(*) FROM invoice;
COUNT(*)
0
SELECT COUNT(*) FROM invoice_audit;
COUNT(*)
4

113
storage/duckdb/mysql-test/duckdb/r/mixed_dml.result Normal file
View File

@@ -0,0 +1,113 @@
# Commit and rollback after an insert-only to mixed batch transition
CREATE TABLE t1 (
id INT PRIMARY KEY,
value VARCHAR(16)
) ENGINE=DuckDB;
CREATE TABLE t1_audit (event CHAR(1)) ENGINE=InnoDB;
CREATE TRIGGER t1_ad AFTER DELETE ON t1
FOR EACH ROW INSERT INTO t1_audit VALUES ('D');
CREATE TRIGGER t1_au AFTER UPDATE ON t1
FOR EACH ROW INSERT INTO t1_audit VALUES ('U');
BEGIN;
INSERT INTO t1 VALUES (1, 'one'), (2, 'two'), (3, 'three');
DELETE FROM t1 WHERE value = 'two';
UPDATE t1 SET value = 'THREE' WHERE value = 'three';
COMMIT;
SELECT * FROM t1 ORDER BY id;
id value
1 one
3 THREE
SELECT * FROM t1_audit ORDER BY event;
event
D
U
BEGIN;
INSERT INTO t1 VALUES (4, 'four');
DELETE FROM t1 WHERE value = 'one';
UPDATE t1 SET value = 'changed' WHERE value = 'THREE';
ROLLBACK;
SELECT * FROM t1 ORDER BY id;
id value
1 one
3 THREE
SELECT * FROM t1_audit ORDER BY event;
event
D
U
# Repeated changes, delete followed by insert, and primary key update
CREATE TABLE t2 (
id INT PRIMARY KEY,
value VARCHAR(16)
) ENGINE=DuckDB;
CREATE TABLE t2_audit (event CHAR(1)) ENGINE=InnoDB;
CREATE TRIGGER t2_ad AFTER DELETE ON t2
FOR EACH ROW INSERT INTO t2_audit VALUES ('D');
CREATE TRIGGER t2_au AFTER UPDATE ON t2
FOR EACH ROW INSERT INTO t2_audit VALUES ('U');
INSERT INTO t2 VALUES (1, 'one'), (2, 'two'), (3, 'three');
BEGIN;
UPDATE t2 SET value = 'first' WHERE value = 'one';
UPDATE t2 SET value = 'second' WHERE value = 'first';
DELETE FROM t2 WHERE value = 'second';
DELETE FROM t2 WHERE value = 'two';
INSERT INTO t2 VALUES (2, 'reinserted');
UPDATE t2 SET value = 'first' WHERE value = 'three';
UPDATE t2 SET value = 'second' WHERE value = 'first';
UPDATE t2 SET id = 30, value = 'moved' WHERE value = 'second';
COMMIT;
SELECT * FROM t2 ORDER BY id;
id value
2 reinserted
30 moved
SELECT event, COUNT(*) FROM t2_audit GROUP BY event ORDER BY event;
event COUNT(*)
D 2
U 5
# Composite primary key
CREATE TABLE t3 (
a INT,
b INT,
value VARCHAR(16),
PRIMARY KEY (a, b)
) ENGINE=DuckDB;
CREATE TABLE t3_audit (event CHAR(1)) ENGINE=InnoDB;
CREATE TRIGGER t3_ad AFTER DELETE ON t3
FOR EACH ROW INSERT INTO t3_audit VALUES ('D');
CREATE TRIGGER t3_au AFTER UPDATE ON t3
FOR EACH ROW INSERT INTO t3_audit VALUES ('U');
INSERT INTO t3 VALUES (1, 1, 'alpha'), (1, 2, 'beta'), (2, 1, 'gamma');
BEGIN;
DELETE FROM t3 WHERE value = 'alpha';
UPDATE t3 SET value = 'BETA' WHERE value = 'beta';
UPDATE t3 SET a = 3, value = 'moved' WHERE value = 'gamma';
COMMIT;
SELECT * FROM t3 ORDER BY a, b;
a b value
1 2 BETA
3 1 moved
SELECT event, COUNT(*) FROM t3_audit GROUP BY event ORDER BY event;
event COUNT(*)
D 1
U 2
# A secondary index is not a row identity without a primary key
CREATE TABLE t4 (
k INT,
value VARCHAR(16),
KEY secondary_key (k)
) ENGINE=DuckDB;
CREATE TABLE t4_audit (value VARCHAR(16)) ENGINE=InnoDB;
CREATE TRIGGER t4_ad AFTER DELETE ON t4
FOR EACH ROW INSERT INTO t4_audit VALUES (OLD.value);
INSERT INTO t4 VALUES (1, 'delete-me'), (1, 'keep-me'), (2, 'other');
DELETE FROM t4 WHERE value = 'delete-me';
SELECT * FROM t4 ORDER BY k, value;
k value
1 keep-me
2 other
SELECT * FROM t4_audit ORDER BY value;
value
delete-me
delete_in_batch
5
update_in_batch
9

30
storage/duckdb/mysql-test/duckdb/t/mdev_40957.test Normal file
View File

@@ -0,0 +1,30 @@
--source ../include/have_duckdb.inc
--source include/not_msan.inc
--disable_query_log
--disable_warnings
SET @saved_duckdb_dml_in_batch = @@GLOBAL.duckdb_dml_in_batch;
SET GLOBAL duckdb_dml_in_batch = ON;
DROP DATABASE IF EXISTS mdev_40957;
CREATE DATABASE mdev_40957 CHARACTER SET utf8mb4;
USE mdev_40957;
--enable_warnings
--enable_query_log
CREATE TABLE invoice (
id UUID DEFAULT UUID() PRIMARY KEY,
product VARCHAR(64)
) ENGINE=DuckDB;
CREATE TABLE invoice_audit (n INT) ENGINE=InnoDB;
CREATE TRIGGER invoice_ad AFTER DELETE ON invoice
FOR EACH ROW INSERT INTO invoice_audit VALUES (1);
INSERT INTO invoice (product) VALUES ('one'), ('two'), ('three'), ('four');
DELETE FROM invoice;
SELECT COUNT(*) FROM invoice;
SELECT COUNT(*) FROM invoice_audit;
--disable_query_log
DROP DATABASE mdev_40957;
USE test;
SET GLOBAL duckdb_dml_in_batch = @saved_duckdb_dml_in_batch;
--enable_query_log

115
storage/duckdb/mysql-test/duckdb/t/mixed_dml.test Normal file
View File

@@ -0,0 +1,115 @@
--source ../include/have_duckdb.inc
--source include/not_msan.inc
--disable_query_log
--disable_warnings
SET @saved_duckdb_dml_in_batch = @@GLOBAL.duckdb_dml_in_batch;
SET @saved_duckdb_require_primary_key = @@GLOBAL.duckdb_require_primary_key;
SET GLOBAL duckdb_dml_in_batch = ON;
SET GLOBAL duckdb_require_primary_key = OFF;
DROP DATABASE IF EXISTS mixed_dml;
CREATE DATABASE mixed_dml CHARACTER SET utf8mb4;
USE mixed_dml;
--let $delete_before = query_get_value(SHOW STATUS LIKE 'Duckdb_rows_delete_in_batch', Value, 1)
--let $update_before = query_get_value(SHOW STATUS LIKE 'Duckdb_rows_update_in_batch', Value, 1)
--enable_warnings
--enable_query_log
--echo # Commit and rollback after an insert-only to mixed batch transition
CREATE TABLE t1 (
id INT PRIMARY KEY,
value VARCHAR(16)
) ENGINE=DuckDB;
CREATE TABLE t1_audit (event CHAR(1)) ENGINE=InnoDB;
CREATE TRIGGER t1_ad AFTER DELETE ON t1
FOR EACH ROW INSERT INTO t1_audit VALUES ('D');
CREATE TRIGGER t1_au AFTER UPDATE ON t1
FOR EACH ROW INSERT INTO t1_audit VALUES ('U');
BEGIN;
INSERT INTO t1 VALUES (1, 'one'), (2, 'two'), (3, 'three');
DELETE FROM t1 WHERE value = 'two';
UPDATE t1 SET value = 'THREE' WHERE value = 'three';
COMMIT;
SELECT * FROM t1 ORDER BY id;
SELECT * FROM t1_audit ORDER BY event;
BEGIN;
INSERT INTO t1 VALUES (4, 'four');
DELETE FROM t1 WHERE value = 'one';
UPDATE t1 SET value = 'changed' WHERE value = 'THREE';
ROLLBACK;
SELECT * FROM t1 ORDER BY id;
SELECT * FROM t1_audit ORDER BY event;
--echo # Repeated changes, delete followed by insert, and primary key update
CREATE TABLE t2 (
id INT PRIMARY KEY,
value VARCHAR(16)
) ENGINE=DuckDB;
CREATE TABLE t2_audit (event CHAR(1)) ENGINE=InnoDB;
CREATE TRIGGER t2_ad AFTER DELETE ON t2
FOR EACH ROW INSERT INTO t2_audit VALUES ('D');
CREATE TRIGGER t2_au AFTER UPDATE ON t2
FOR EACH ROW INSERT INTO t2_audit VALUES ('U');
INSERT INTO t2 VALUES (1, 'one'), (2, 'two'), (3, 'three');
BEGIN;
UPDATE t2 SET value = 'first' WHERE value = 'one';
UPDATE t2 SET value = 'second' WHERE value = 'first';
DELETE FROM t2 WHERE value = 'second';
DELETE FROM t2 WHERE value = 'two';
INSERT INTO t2 VALUES (2, 'reinserted');
UPDATE t2 SET value = 'first' WHERE value = 'three';
UPDATE t2 SET value = 'second' WHERE value = 'first';
UPDATE t2 SET id = 30, value = 'moved' WHERE value = 'second';
COMMIT;
SELECT * FROM t2 ORDER BY id;
SELECT event, COUNT(*) FROM t2_audit GROUP BY event ORDER BY event;
--echo # Composite primary key
CREATE TABLE t3 (
a INT,
b INT,
value VARCHAR(16),
PRIMARY KEY (a, b)
) ENGINE=DuckDB;
CREATE TABLE t3_audit (event CHAR(1)) ENGINE=InnoDB;
CREATE TRIGGER t3_ad AFTER DELETE ON t3
FOR EACH ROW INSERT INTO t3_audit VALUES ('D');
CREATE TRIGGER t3_au AFTER UPDATE ON t3
FOR EACH ROW INSERT INTO t3_audit VALUES ('U');
INSERT INTO t3 VALUES (1, 1, 'alpha'), (1, 2, 'beta'), (2, 1, 'gamma');
BEGIN;
DELETE FROM t3 WHERE value = 'alpha';
UPDATE t3 SET value = 'BETA' WHERE value = 'beta';
UPDATE t3 SET a = 3, value = 'moved' WHERE value = 'gamma';
COMMIT;
SELECT * FROM t3 ORDER BY a, b;
SELECT event, COUNT(*) FROM t3_audit GROUP BY event ORDER BY event;
--echo # A secondary index is not a row identity without a primary key
CREATE TABLE t4 (
k INT,
value VARCHAR(16),
KEY secondary_key (k)
) ENGINE=DuckDB;
CREATE TABLE t4_audit (value VARCHAR(16)) ENGINE=InnoDB;
CREATE TRIGGER t4_ad AFTER DELETE ON t4
FOR EACH ROW INSERT INTO t4_audit VALUES (OLD.value);
INSERT INTO t4 VALUES (1, 'delete-me'), (1, 'keep-me'), (2, 'other');
DELETE FROM t4 WHERE value = 'delete-me';
SELECT * FROM t4 ORDER BY k, value;
SELECT * FROM t4_audit ORDER BY value;
--let $delete_after = query_get_value(SHOW STATUS LIKE 'Duckdb_rows_delete_in_batch', Value, 1)
--let $update_after = query_get_value(SHOW STATUS LIKE 'Duckdb_rows_update_in_batch', Value, 1)
--disable_query_log
--eval SELECT $delete_after - $delete_before AS delete_in_batch
--eval SELECT $update_after - $update_before AS update_in_batch
DROP DATABASE mixed_dml;
USE test;
SET GLOBAL duckdb_dml_in_batch = @saved_duckdb_dml_in_batch;
SET GLOBAL duckdb_require_primary_key = @saved_duckdb_require_primary_key;
--enable_query_log

43
storage/duckdb/runtime/delta_appender.cc
View File

@@ -153,6 +153,13 @@ int DeltaAppender::append_row_update(TABLE *table, ulonglong trx_no,
int DeltaAppender::append_row_delete(TABLE *table, ulonglong trx_no,
const uchar *old_row)
{
if (!m_use_tmp_table)
{
my_error(ER_GET_ERRMSG, MYF(0), HA_DUCKDB_APPEND_ERROR,
"Delete requires a mixed batch", "DuckDB Appender");
return HA_DUCKDB_APPEND_ERROR;
}
++m_row_count;
m_has_delete= true;
@@ -241,10 +248,11 @@ bool DeltaAppender::Initialize(TABLE *table)
return true;
}
KEY *key_info= table->key_info;
if (!key_info)
if (table->s->primary_key == MAX_KEY)
return true;
KEY *key_info= table->key_info + table->s->primary_key;
if (my_bitmap_init(&m_pk_bitmap, nullptr, table->s->fields))
return true;
my_bitmap_init(&m_pk_bitmap, nullptr, table->s->fields);
KEY_PART_INFO *key_part= key_info->key_part;
for (uint i= 0; i < key_info->user_defined_key_parts; i++, key_part++)
{
@@ -470,18 +478,16 @@ int DeltaAppender::append_mysql_field(const Field *field_arg,
return 0;
}
static void appendSelectQuery(std::stringstream &ss,
const std::string &select_list,
const std::string &pk_list,
const std::string &table_name, int delete_flag)
static void appendLatestRowsQuery(std::stringstream &ss,
const std::string &select_list,
const std::string &pk_list,
const std::string &table_name)
{
ss << "SELECT UNNEST(r) FROM (SELECT LAST(ROW(" << select_list
<< ") ORDER BY \"#mdb_row_no\") AS r, "
"LAST(\"#mdb_delete_flag\" ORDER BY \"#mdb_row_no\") AS "
"\"#mdb_delete_flag\" FROM main."
<< quote_duckdb_identifier(table_name) << " GROUP BY " << pk_list << ")";
if (!delete_flag)
ss << " WHERE \"#mdb_delete_flag\" = " << delete_flag;
ss << "SELECT " << select_list << " FROM main."
<< quote_duckdb_identifier(table_name)
<< " QUALIFY ROW_NUMBER() OVER (PARTITION BY " << pk_list
<< " ORDER BY \"#mdb_row_no\" DESC) = 1 AND NOT "
"\"#mdb_delete_flag\"";
}
void DeltaAppender::generateQuery(std::stringstream &ss, bool delete_flag)
@@ -493,17 +499,16 @@ void DeltaAppender::generateQuery(std::stringstream &ss, bool delete_flag)
{
ss << "INSERT INTO " << quote_duckdb_identifier(m_schema_name) << "."
<< quote_duckdb_identifier(m_table_name) << " ";
appendSelectQuery(ss, m_col_list, m_pk_list, m_tmp_table_name,
delete_flag);
appendLatestRowsQuery(ss, m_col_list, m_pk_list, m_tmp_table_name);
ss << ";";
}
else
{
ss << "DELETE FROM " << quote_duckdb_identifier(m_schema_name) << "."
<< quote_duckdb_identifier(m_table_name) << " WHERE (" << m_pk_list
<< ") IN (";
appendSelectQuery(ss, m_pk_list, m_pk_list, m_tmp_table_name, delete_flag);
ss << ");";
<< ") IN (SELECT " << m_pk_list << " FROM main."
<< quote_duckdb_identifier(m_tmp_table_name) << " GROUP BY "
<< m_pk_list << ");";
}
}

2
storage/duckdb/runtime/delta_appender.h
View File

@@ -71,7 +71,7 @@ private:
std::string m_table_name;
std::string m_tmp_table_name;
MY_BITMAP m_pk_bitmap;
MY_BITMAP m_pk_bitmap{};
std::string m_pk_list{""};
std::string m_col_list{""};