mirror of
https://github.com/MariaDB/server.git
synced 2026-10-23 18:49:29 +03:00
MDEV-39143 Replicate MySQL binary JSON to a MariaDB slave
Let a MariaDB slave apply ROW-format binlog events from a MySQL master that contain JSON columns, both as full documents and as MySQL's partial (diff) JSON updates (PARTIAL_UPDATE_ROWS_EVENT).
This commit is contained in:
BIN
mysql-test/std_data/mdev39143_partial_json.000001
Normal file
BIN
mysql-test/std_data/mdev39143_partial_json.000001
Normal file
Binary file not shown.
BIN
mysql-test/std_data/mdev39143_partial_json_insert.000001
Normal file
BIN
mysql-test/std_data/mdev39143_partial_json_insert.000001
Normal file
Binary file not shown.
@@ -1,6 +1,7 @@
|
||||
include/master-slave.inc
|
||||
[connection master]
|
||||
connection slave;
|
||||
INSTALL SONAME 'type_mysql_json';
|
||||
include/stop_slave.inc
|
||||
connection master;
|
||||
include/rpl_stop_server.inc [server_number=1]
|
||||
@@ -25,7 +26,6 @@ CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=2297;
|
||||
START SLAVE IO_THREAD;
|
||||
START SLAVE SQL_THREAD;
|
||||
include/wait_for_slave_io_to_start.inc
|
||||
include/wait_for_slave_sql_error.inc [errno=1594]
|
||||
SELECT * FROM t1 ORDER BY a;
|
||||
a b c
|
||||
1 0
|
||||
@@ -35,16 +35,16 @@ a b c
|
||||
5 0 after compressed
|
||||
SELECT * FROM t2 ORDER BY a;
|
||||
a b
|
||||
1 {"a": "hulu", "b": "[zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy]", "c": "bulu"}
|
||||
STOP SLAVE IO_THREAD;
|
||||
include/wait_for_slave_io_to_stop.inc
|
||||
1 {"a": "hulu", "b": "oOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOo", "c": "bulu"}
|
||||
STOP SLAVE;
|
||||
include/wait_for_slave_to_stop.inc
|
||||
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4;
|
||||
START SLAVE IO_THREAD;
|
||||
START SLAVE SQL_THREAD;
|
||||
include/wait_for_slave_io_to_start.inc
|
||||
include/wait_for_slave_sql_to_start.inc
|
||||
DROP TABLE t1, t2;
|
||||
UNINSTALL SONAME 'type_mysql_json';
|
||||
CALL mtr.add_suppression('TRANSACTION_PAYLOAD_EVENT event. You can avoid this event by specifying');
|
||||
CALL mtr.add_suppression('PARTIAL_UPDATE_ROWS_EVENT event. You can avoid this event by specifying');
|
||||
connection master;
|
||||
include/rpl_end.inc
|
||||
|
||||
36
mysql-test/suite/rpl/r/rpl_json_partial_insert.result
Normal file
36
mysql-test/suite/rpl/r/rpl_json_partial_insert.result
Normal file
@@ -0,0 +1,36 @@
|
||||
include/master-slave.inc
|
||||
[connection master]
|
||||
connection slave;
|
||||
INSTALL SONAME 'type_mysql_json';
|
||||
include/stop_slave.inc
|
||||
connection master;
|
||||
include/rpl_stop_server.inc [server_number=1]
|
||||
include/rpl_start_server.inc [server_number=1]
|
||||
connection slave;
|
||||
CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
|
||||
call mtr.add_suppression("MDEV-39143: cannot apply a partial JSON update");
|
||||
call mtr.add_suppression("Slave SQL.*Error executing row event");
|
||||
START SLAVE;
|
||||
include/wait_for_slave_sql_error.inc [errno=1105]
|
||||
# Slave stopped, Last_SQL_Error matches the expected refusal:
|
||||
# Could not execute MySQL Update_rows_partial event on table jsontest.t1; MDEV-39143: cannot apply a partial JSON update that inserts at a from-the-end array index: path '$.arr[last-9]' on jsontest.t1.j. MariaDB's JSON_ARRAY_INSERT resolves '[last-N]' differently from MySQL, so the result would silently differ from the source., Error_code: 1105; handler error HA_ERR_GENERIC; the event's master log master-bin.000001, end_log_pos 2453
|
||||
#
|
||||
# Rows applied before the halt: id=1 got its object-member INSERT and
|
||||
# id=2 its array append. id=3 and id=4 keep the inserted values.
|
||||
SELECT id, JSON_QUERY(j,'$.arr') AS arr, JSON_VALUE(j,'$.newkey') AS newkey
|
||||
FROM jsontest.t1 ORDER BY id;
|
||||
id arr newkey
|
||||
1 [1, 2, 3] added
|
||||
2 [1, 2, 3, 99] NULL
|
||||
3 [1, 2, 3] NULL
|
||||
4 [1, 2, 3] NULL
|
||||
connection slave;
|
||||
STOP SLAVE IO_THREAD;
|
||||
include/wait_for_slave_io_to_stop.inc
|
||||
DROP DATABASE jsontest;
|
||||
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
|
||||
START SLAVE;
|
||||
connection master;
|
||||
connection slave;
|
||||
UNINSTALL SONAME 'type_mysql_json';
|
||||
include/rpl_end.inc
|
||||
27
mysql-test/suite/rpl/r/rpl_json_partial_update.result
Normal file
27
mysql-test/suite/rpl/r/rpl_json_partial_update.result
Normal file
@@ -0,0 +1,27 @@
|
||||
include/master-slave.inc
|
||||
[connection master]
|
||||
connection slave;
|
||||
INSTALL SONAME 'type_mysql_json';
|
||||
include/stop_slave.inc
|
||||
connection master;
|
||||
include/rpl_stop_server.inc [server_number=1]
|
||||
include/rpl_start_server.inc [server_number=1]
|
||||
connection slave;
|
||||
CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
|
||||
START SLAVE;
|
||||
# id=1: inserted as a full document, then modified by four partial-JSON
|
||||
# events (REPLACE of members a and n twice, REMOVE of arr element 2).
|
||||
SELECT id, JSON_VALUE(j,'$.a') AS a, JSON_VALUE(j,'$.n') AS n,
|
||||
JSON_QUERY(j,'$.arr') AS arr FROM testdb.tj ORDER BY id;
|
||||
id a n arr
|
||||
1 zzzz 44 [1, 2]
|
||||
connection slave;
|
||||
STOP SLAVE;
|
||||
include/wait_for_slave_to_stop.inc
|
||||
DROP DATABASE testdb;
|
||||
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
|
||||
START SLAVE;
|
||||
connection master;
|
||||
connection slave;
|
||||
UNINSTALL SONAME 'type_mysql_json';
|
||||
include/rpl_end.inc
|
||||
@@ -59,6 +59,7 @@
|
||||
|
||||
|
||||
--connection slave
|
||||
INSTALL SONAME 'type_mysql_json';
|
||||
--source include/stop_slave.inc
|
||||
|
||||
--connection master
|
||||
@@ -99,14 +100,26 @@ CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=2297;
|
||||
START SLAVE IO_THREAD;
|
||||
START SLAVE SQL_THREAD;
|
||||
--source include/wait_for_slave_io_to_start.inc
|
||||
--let $slave_sql_errno= 1594
|
||||
--source include/wait_for_slave_sql_error.inc
|
||||
|
||||
# MDEV-39143: PARTIAL_UPDATE_ROWS_EVENT is now applied instead of rejected.
|
||||
# Wait for the JSON_REPLACE partial update (the last event in the injected
|
||||
# file) to reach t2, rather than for a fixed position, since the exact byte
|
||||
# offset is fragile against any future re-recording of the fixture.
|
||||
# LIKE avoids '$' which mysqltest would interpolate inside --let; the
|
||||
# repeating unit is 3 chars ('oOo'), so 'oOooOo' is the substring it produces.
|
||||
--let $wait_condition= SELECT COUNT(*)=1 FROM test.t2 WHERE a=1 AND b LIKE '%oOooOo%'
|
||||
--source include/wait_condition.inc
|
||||
|
||||
SELECT * FROM t1 ORDER BY a;
|
||||
SELECT * FROM t2 ORDER BY a;
|
||||
|
||||
STOP SLAVE IO_THREAD;
|
||||
--source include/wait_for_slave_io_to_stop.inc
|
||||
# The SQL thread is genuinely caught up and running now that the partial
|
||||
# JSON update applies instead of erroring, so both threads must be stopped
|
||||
# before CHANGE MASTER (STOP SLAVE IO_THREAD alone left it running here
|
||||
# before MDEV-39143, when the SQL thread had already halted on the event 39
|
||||
# rejection).
|
||||
STOP SLAVE;
|
||||
--source include/wait_for_slave_to_stop.inc
|
||||
# Restart replication after the MySQL 8.0 file.
|
||||
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4;
|
||||
START SLAVE IO_THREAD;
|
||||
@@ -116,8 +129,8 @@ START SLAVE SQL_THREAD;
|
||||
--sync_with_master
|
||||
|
||||
DROP TABLE t1, t2;
|
||||
UNINSTALL SONAME 'type_mysql_json';
|
||||
CALL mtr.add_suppression('TRANSACTION_PAYLOAD_EVENT event. You can avoid this event by specifying');
|
||||
CALL mtr.add_suppression('PARTIAL_UPDATE_ROWS_EVENT event. You can avoid this event by specifying');
|
||||
|
||||
--connection master
|
||||
--source include/rpl_end.inc
|
||||
|
||||
95
mysql-test/suite/rpl/t/rpl_json_partial_insert.test
Normal file
95
mysql-test/suite/rpl/t/rpl_json_partial_insert.test
Normal file
@@ -0,0 +1,95 @@
|
||||
# MDEV-39143: INSERT diffs in a MySQL partial JSON update.
|
||||
#
|
||||
# Companion to rpl_json_partial_update.test and driven the same way - see the
|
||||
# header there for how the recorded MySQL binlog is injected. The fixture,
|
||||
# std_data/mdev39143_partial_json_insert.000001, was recorded on MySQL 9.2
|
||||
# with binlog_row_value_options=PARTIAL_JSON from:
|
||||
#
|
||||
# UPDATE t1 SET j=JSON_SET(j,'$.newkey','added') WHERE id=1;
|
||||
# UPDATE t1 SET j=JSON_SET(j,'$.arr[7]',99) WHERE id=2;
|
||||
# UPDATE t1 SET j=JSON_SET(j,'$.arr[last-9]',77) WHERE id=3;
|
||||
# UPDATE t1 SET j=JSON_SET(j,'$.arr[1]',55) WHERE id=4;
|
||||
# UPDATE t1 SET j=JSON_REMOVE(j,'$.arr[2]') WHERE id=4;
|
||||
#
|
||||
# MySQL logs the first as an INSERT diff on an object-member path and the
|
||||
# next two as INSERT diffs on array-cell paths; its own mysqlbinlog renders
|
||||
# them as JSON_INSERT, JSON_ARRAY_INSERT and JSON_ARRAY_INSERT respectively.
|
||||
# The slave must make the same distinction: id=3 is the discriminating case,
|
||||
# where the path resolves before the start of the array so the value is
|
||||
# prepended and the existing elements shift right. JSON_INSERT would leave
|
||||
# the document untouched there, silently.
|
||||
#
|
||||
# MySQL's own contents after those updates, read from the source table:
|
||||
# id=1 arr [1, 2, 3] newkey "added"
|
||||
# id=2 arr [1, 2, 3, 99]
|
||||
# id=3 arr [77, 1, 2, 3]
|
||||
# id=4 arr [1, 55]
|
||||
#
|
||||
# id=3 cannot be applied correctly yet and the slave deliberately stops on
|
||||
# it. MySQL emits an array-cell INSERT only for a cell that does not exist,
|
||||
# which for '[last-9]' means it clamped the position to 0 and prepended.
|
||||
# MariaDB's JSON_ARRAY_INSERT resolves a from-the-end index differently - it
|
||||
# appends once the index goes negative, and is off by one even in range -
|
||||
# while JSON_EXTRACT, JSON_REPLACE and JSON_REMOVE all agree with MySQL on
|
||||
# the same paths. That looks like a pre-existing JSON_ARRAY_INSERT bug
|
||||
# reachable from plain SQL, so rather than store a document that silently
|
||||
# differs from the source, the apply path refuses the diff and halts.
|
||||
#
|
||||
# This test therefore asserts the halt, and that the diffs before it applied
|
||||
# correctly: id=1 (object member) and id=2 (array append, where both
|
||||
# functions agree). id=4's REPLACE and REMOVE come after the failing event
|
||||
# and are not reached. When JSON_ARRAY_INSERT is fixed, drop the guard in
|
||||
# apply_partial_json_column() and this test should expect MySQL's values
|
||||
# above for all four rows.
|
||||
|
||||
--source include/have_innodb.inc
|
||||
--source include/have_binlog_format_row.inc
|
||||
--source include/master-slave.inc
|
||||
|
||||
--connection slave
|
||||
INSTALL SONAME 'type_mysql_json';
|
||||
--source include/stop_slave.inc
|
||||
|
||||
--connection master
|
||||
--let $datadir= `SELECT @@datadir`
|
||||
--let $rpl_server_number= 1
|
||||
--source include/rpl_stop_server.inc
|
||||
--remove_file $datadir/master-bin.000001
|
||||
--copy_file $MYSQL_TEST_DIR/std_data/mdev39143_partial_json_insert.000001 $datadir/master-bin.000001
|
||||
--source include/rpl_start_server.inc
|
||||
|
||||
--connection slave
|
||||
CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
|
||||
call mtr.add_suppression("MDEV-39143: cannot apply a partial JSON update");
|
||||
call mtr.add_suppression("Slave SQL.*Error executing row event");
|
||||
START SLAVE;
|
||||
|
||||
# The slave must stop on the id=3 diff rather than apply it.
|
||||
--let $slave_sql_errno= 1105
|
||||
--source include/wait_for_slave_sql_error.inc
|
||||
--let $err= query_get_value(SHOW SLAVE STATUS, Last_SQL_Error, 1)
|
||||
--echo # Slave stopped, Last_SQL_Error matches the expected refusal:
|
||||
--echo # $err
|
||||
|
||||
--echo #
|
||||
--echo # Rows applied before the halt: id=1 got its object-member INSERT and
|
||||
--echo # id=2 its array append. id=3 and id=4 keep the inserted values.
|
||||
SELECT id, JSON_QUERY(j,'$.arr') AS arr, JSON_VALUE(j,'$.newkey') AS newkey
|
||||
FROM jsontest.t1 ORDER BY id;
|
||||
|
||||
# Cleanup.
|
||||
# The SQL thread is already stopped by the error above, so only
|
||||
# the IO thread needs stopping before CHANGE MASTER; wait_for_slave_to_stop
|
||||
# would itself fail on the pending Last_SQL_Error.
|
||||
--connection slave
|
||||
STOP SLAVE IO_THREAD;
|
||||
--source include/wait_for_slave_io_to_stop.inc
|
||||
DROP DATABASE jsontest;
|
||||
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
|
||||
START SLAVE;
|
||||
--connection master
|
||||
--save_master_pos
|
||||
--connection slave
|
||||
--sync_with_master
|
||||
UNINSTALL SONAME 'type_mysql_json';
|
||||
--source include/rpl_end.inc
|
||||
75
mysql-test/suite/rpl/t/rpl_json_partial_update.test
Normal file
75
mysql-test/suite/rpl/t/rpl_json_partial_update.test
Normal file
@@ -0,0 +1,75 @@
|
||||
# MDEV-39143: replicate MySQL binary JSON (full documents and partial diffs)
|
||||
# from a MySQL master to a MariaDB slave.
|
||||
#
|
||||
# No MySQL server runs here. Both servers started by master-slave.inc below
|
||||
# are MariaDB; the MySQL side is supplied by a binlog recorded once on a real
|
||||
# MySQL 9.2 server and checked in as
|
||||
# std_data/mdev39143_partial_json.000001. The test stops the MariaDB master,
|
||||
# replaces the master's own master-bin.000001 with that file, and restarts it.
|
||||
# The master is then only a file server for those bytes, and the slave pulls
|
||||
# them over the ordinary replication protocol. What is under test is the
|
||||
# slave apply path: PARTIAL_UPDATE_ROWS_EVENT (39) parsing, Json_diff
|
||||
# decoding, and diff application - see unpack_row() in sql/rpl_record.cc.
|
||||
#
|
||||
# Important points:
|
||||
# - The recorded events are fixed. The SQL the MySQL master ran is not
|
||||
# visible in this file, only its effect (described at the SELECT below);
|
||||
# extending coverage needs a MySQL server, a re-recording, and a new file.
|
||||
# - The Json_diff encoding exercised is MySQL 9.2's specifically.
|
||||
# - have_binlog_format_row.inc constrains the MariaDB master's own binlog
|
||||
# format; the injected events are row events regardless of it.
|
||||
--source include/have_innodb.inc
|
||||
--source include/have_binlog_format_row.inc
|
||||
--source include/master-slave.inc
|
||||
|
||||
# The slave decodes MySQL binary JSON with the read-only mysql_json plugin,
|
||||
# which is not installed by default. Uninstalled again in the teardown.
|
||||
--connection slave
|
||||
INSTALL SONAME 'type_mysql_json';
|
||||
--source include/stop_slave.inc
|
||||
|
||||
# Substitute the recorded MySQL binlog for the master's own. The server must
|
||||
# be down while the file is swapped, and is restarted so it reopens the new
|
||||
# master-bin.000001 and starts a fresh master-bin.000002 of its own.
|
||||
--connection master
|
||||
--let $datadir= `SELECT @@datadir`
|
||||
--let $rpl_server_number= 1
|
||||
--source include/rpl_stop_server.inc
|
||||
--remove_file $datadir/master-bin.000001
|
||||
--copy_file $MYSQL_TEST_DIR/std_data/mdev39143_partial_json.000001 $datadir/master-bin.000001
|
||||
--source include/rpl_start_server.inc
|
||||
|
||||
--connection slave
|
||||
CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
|
||||
START SLAVE;
|
||||
|
||||
# Wait until the last partial JSON update has been applied to id=1.
|
||||
# (LIKE avoids '$' which mysqltest would interpolate inside --let.)
|
||||
--let $wait_condition= SELECT COUNT(*)=1 FROM testdb.tj WHERE id=1 AND j LIKE '%"a": "zzzz"%' AND j LIKE '%"n": 44%'
|
||||
--source include/wait_condition.inc
|
||||
|
||||
--echo # id=1: inserted as a full document, then modified by four partial-JSON
|
||||
--echo # events (REPLACE of members a and n twice, REMOVE of arr element 2).
|
||||
SELECT id, JSON_VALUE(j,'$.a') AS a, JSON_VALUE(j,'$.n') AS n,
|
||||
JSON_QUERY(j,'$.arr') AS arr FROM testdb.tj ORDER BY id;
|
||||
|
||||
# Cleanup.
|
||||
# The slave is sitting at the end of the injected binlog, which the
|
||||
# master did not write and knows nothing about, so save_master_pos/
|
||||
# sync_with_master cannot line the two up. Repoint the slave at
|
||||
# master-bin.000002 - the empty binlog the master opened on restart - so that
|
||||
# the two agree on a position and rpl_end.inc can shut down cleanly.
|
||||
--connection slave
|
||||
STOP SLAVE;
|
||||
--source include/wait_for_slave_to_stop.inc
|
||||
DROP DATABASE testdb;
|
||||
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
|
||||
START SLAVE;
|
||||
--connection master
|
||||
--save_master_pos
|
||||
--connection slave
|
||||
--sync_with_master
|
||||
# Remove the plugin again: it is installed into mysql.plugin, which MTR's
|
||||
# check-testcase compares against the pre-test server state.
|
||||
UNINSTALL SONAME 'type_mysql_json';
|
||||
--source include/rpl_end.inc
|
||||
@@ -1234,6 +1234,15 @@ Log_event *Log_event::read_log_event_no_checksum(
|
||||
case UPDATE_ROWS_EVENT:
|
||||
ev= new Update_rows_log_event(buf, event_len, fdle);
|
||||
break;
|
||||
// MySQL 8.0: partial JSON update
|
||||
case PARTIAL_UPDATE_ROWS_EVENT:
|
||||
/*
|
||||
MDEV-39143: same event structure as UPDATE_ROWS_EVENT; the after-image
|
||||
may carry JSON diffs, handled in unpack_row() via the
|
||||
is_partial_json_after_image flag.
|
||||
*/
|
||||
ev= new Update_rows_log_event(buf, event_len, fdle);
|
||||
break;
|
||||
case DELETE_ROWS_EVENT_V1:
|
||||
case DELETE_ROWS_EVENT:
|
||||
ev= new Delete_rows_log_event(buf, event_len, fdle);
|
||||
@@ -1302,14 +1311,6 @@ Log_event *Log_event::read_log_event_no_checksum(
|
||||
"'binlog_transaction_compression=0' in the MySQL server";
|
||||
ev= NULL;
|
||||
break;
|
||||
case PARTIAL_UPDATE_ROWS_EVENT: // MySQL 8.0
|
||||
*error=
|
||||
"Found incompatible MySQL 8.0 PARTIAL_UPDATE_ROWS_EVENT event. "
|
||||
"You can avoid this event by specifying "
|
||||
"'binlog-row-value-options=\"\"' in the MySQL server";
|
||||
ev= NULL;
|
||||
break;
|
||||
|
||||
case PRE_GA_WRITE_ROWS_EVENT:
|
||||
case PRE_GA_UPDATE_ROWS_EVENT:
|
||||
case PRE_GA_DELETE_ROWS_EVENT:
|
||||
|
||||
@@ -809,7 +809,9 @@ static inline bool LOG_EVENT_IS_UPDATE_ROW(enum Log_event_type type)
|
||||
{
|
||||
return type == UPDATE_ROWS_EVENT || type == UPDATE_ROWS_EVENT_V1 ||
|
||||
type == UPDATE_ROWS_COMPRESSED_EVENT ||
|
||||
type == UPDATE_ROWS_COMPRESSED_EVENT_V1;
|
||||
type == UPDATE_ROWS_COMPRESSED_EVENT_V1 ||
|
||||
// MySQL 8.0+ partial JSON update (MDEV-39143)
|
||||
type == PARTIAL_UPDATE_ROWS_EVENT;
|
||||
}
|
||||
|
||||
|
||||
@@ -5089,6 +5091,14 @@ protected:
|
||||
int find_row(rpl_group_info *);
|
||||
int update_sequence();
|
||||
|
||||
/*
|
||||
True for a MySQL PARTIAL_UPDATE_ROWS_EVENT (binlog_row_value_options=
|
||||
PARTIAL_JSON on the master). Its update after-image may carry JSON diffs
|
||||
instead of full JSON values for some columns.
|
||||
*/
|
||||
bool is_partial_json_update() const
|
||||
{ return m_type == PARTIAL_UPDATE_ROWS_EVENT; }
|
||||
|
||||
// Unpack the current row into m_table->record[0], but with
|
||||
// a different columns bitmap.
|
||||
int unpack_current_row(rpl_group_info *rgi, MY_BITMAP const *cols)
|
||||
@@ -5097,7 +5107,8 @@ protected:
|
||||
|
||||
ASSERT_OR_RETURN_ERROR(m_curr_row <= m_rows_end, HA_ERR_CORRUPT_EVENT);
|
||||
return ::unpack_row(rgi, m_table, m_width, m_curr_row, cols,
|
||||
&m_curr_row_end, m_rows_end);
|
||||
&m_curr_row_end, m_rows_end,
|
||||
is_partial_json_update() && cols == &m_cols_ai);
|
||||
}
|
||||
|
||||
// Unpack the current row into m_table->record[0]
|
||||
|
||||
@@ -22,6 +22,9 @@
|
||||
#include "slave.h" // Need to pull in slave_print_msg
|
||||
#include "rpl_utility.h"
|
||||
#include "rpl_rli.h"
|
||||
/* MDEV-39143: Item_func_json_insert / _remove / _format */
|
||||
#include "item_jsonfunc.h"
|
||||
#include "sql_array.h" // Dynamic_array
|
||||
|
||||
/*
|
||||
|
||||
@@ -274,6 +277,410 @@ static void convert_field(Field *result_field, Field *conv_field)
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
One decoded MySQL JSON diff from a PARTIAL_UPDATE_ROWS_EVENT. 'path' points
|
||||
at the JSON path text and 'value' at the diff's new value in MySQL binary
|
||||
JSON (JSONB); both point into the row buffer. 'value' is NULL for REMOVE.
|
||||
*/
|
||||
enum enum_mysql_json_diff_op
|
||||
{
|
||||
MYSQL_JSON_DIFF_REPLACE= 0,
|
||||
MYSQL_JSON_DIFF_INSERT= 1,
|
||||
MYSQL_JSON_DIFF_REMOVE= 2
|
||||
};
|
||||
|
||||
struct Mysql_json_diff
|
||||
{
|
||||
enum_mysql_json_diff_op op;
|
||||
const char *path;
|
||||
size_t path_length;
|
||||
const uchar *value; // NULL for REMOVE
|
||||
size_t value_length;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
Decode a MySQL Json_diff_vector, the payload of one partial JSON column in
|
||||
the after-image of a PARTIAL_UPDATE_ROWS_EVENT. The format, specified by
|
||||
MySQL WL#2955 section 4, F3, is
|
||||
|
||||
[ length : 4 bytes ] [ diff_1 ] ... [ diff_N ]
|
||||
|
||||
where 'length' counts the bytes of the diffs that follow (not itself), and
|
||||
each diff is
|
||||
|
||||
[ op : 1 ] [ path_length : net_field_length ] [ path : path_length ]
|
||||
( [ value_length : net_field_length ] [ value : value_length ] )?
|
||||
|
||||
the value part being present if and only if op is not REMOVE. The number
|
||||
of diffs is not stored; they are read until 'length' bytes are consumed.
|
||||
|
||||
@param[in,out] row_image_pos
|
||||
The caller's read cursor into the row image. On entry it
|
||||
points at the 4-byte length prefix; on success it is
|
||||
advanced to one past the last diff, i.e. to the next
|
||||
column's data. On failure it is left untouched, so the
|
||||
caller must not rely on it to locate the error.
|
||||
@param[in] end
|
||||
One past the last readable byte of the row image. Only an
|
||||
upper bound for validation: the extent of the vector itself
|
||||
comes from its own length prefix, which is checked to fit
|
||||
within [*row_image_pos, end). Every path and value length
|
||||
is likewise checked against the vector's end, so a corrupt
|
||||
length cannot make this function read outside the buffer.
|
||||
@param[out] out
|
||||
Decoded diffs are *appended*; existing elements are kept, so
|
||||
the caller passes a fresh array unless it deliberately wants
|
||||
to accumulate. On failure 'out' may already hold the diffs
|
||||
decoded before the bad one - a caller that continues after an
|
||||
error must clear it.
|
||||
|
||||
The 'path' and 'value' members point *into* the row image
|
||||
rather than at copies, so the decoded diffs are only valid
|
||||
while that buffer lives. 'value' is NULL and 'value_length'
|
||||
is 0 for a REMOVE diff. Values are still MySQL binary JSON
|
||||
here; decoding them to text is the caller's job.
|
||||
|
||||
@retval false Success; the whole vector was consumed.
|
||||
@retval true The buffer is truncated or corrupt (a length runs past the
|
||||
end of the vector, or an unknown operation code), or the
|
||||
array could not grow. Note the MariaDB convention: true
|
||||
means failure.
|
||||
*/
|
||||
static bool read_mysql_json_diff_vector(const uchar **row_image_pos,
|
||||
const uchar *end,
|
||||
Dynamic_array<Mysql_json_diff> &out)
|
||||
{
|
||||
const uchar *ptr= *row_image_pos;
|
||||
if (end - ptr < 4)
|
||||
return true;
|
||||
size_t length= uint4korr(ptr);
|
||||
ptr+= 4;
|
||||
if ((size_t) (end - ptr) < length)
|
||||
return true;
|
||||
const uchar *const diffs_end= ptr + length;
|
||||
|
||||
while (ptr < diffs_end)
|
||||
{
|
||||
Mysql_json_diff diff;
|
||||
diff.op= (enum_mysql_json_diff_op) *ptr++;
|
||||
if (diff.op > MYSQL_JSON_DIFF_REMOVE)
|
||||
return true;
|
||||
|
||||
size_t avail= (size_t) (diffs_end - ptr);
|
||||
diff.path_length= (size_t) safe_net_field_length_ll((uchar**) &ptr, avail);
|
||||
if ((size_t) (diffs_end - ptr) < diff.path_length)
|
||||
return true;
|
||||
diff.path= (const char*) ptr;
|
||||
ptr+= diff.path_length;
|
||||
|
||||
if (diff.op == MYSQL_JSON_DIFF_REMOVE)
|
||||
{
|
||||
diff.value= NULL;
|
||||
diff.value_length= 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
avail= (size_t) (diffs_end - ptr);
|
||||
diff.value_length= (size_t) safe_net_field_length_ll((uchar**) &ptr,
|
||||
avail);
|
||||
if ((size_t) (diffs_end - ptr) < diff.value_length)
|
||||
return true;
|
||||
diff.value= ptr;
|
||||
ptr+= diff.value_length;
|
||||
}
|
||||
|
||||
if (out.append(diff))
|
||||
return true; // out of memory
|
||||
}
|
||||
|
||||
*row_image_pos= diffs_end;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
Decide which MariaDB function an INSERT diff corresponds to.
|
||||
|
||||
MySQL's Json_diff has a single INSERT operation, but its meaning depends
|
||||
on what the path points at: sql-common/json_diff.h and WL#2955 section 4,
|
||||
F3 define it as JSON_ARRAY_INSERT() when the path names an array cell and
|
||||
JSON_INSERT() when it names an object member. The two differ for a cell
|
||||
that already exists - JSON_ARRAY_INSERT shifts the remaining elements
|
||||
right, while JSON_INSERT does nothing at all because the path is already
|
||||
present - so picking the wrong one corrupts the document silently.
|
||||
|
||||
@param path_obj Scratch path parser, its steps array already
|
||||
initialised; reused across the diffs of one column.
|
||||
@param path The diff's path text.
|
||||
@param path_length Its length in bytes.
|
||||
@param[out] is_array_cell
|
||||
True if the path's last step selects an array cell.
|
||||
@param[out] is_from_end
|
||||
True if that step counts from the end of the array,
|
||||
i.e. it was written as '[last]' or '[last-N]'. Such a
|
||||
path is stored with a negative n_item. The caller
|
||||
refuses these for now; see the note in
|
||||
apply_partial_json_column().
|
||||
|
||||
@retval false Path parsed; both out-parameters are set.
|
||||
@retval true The path could not be parsed.
|
||||
*/
|
||||
static bool json_diff_path_is_array_cell(json_path_t *path_obj,
|
||||
const char *path,
|
||||
size_t path_length,
|
||||
bool *is_array_cell,
|
||||
bool *is_from_end)
|
||||
{
|
||||
if (json_path_setup(path_obj, &my_charset_utf8mb4_bin,
|
||||
(const uchar *) path,
|
||||
(const uchar *) path + path_length))
|
||||
return true;
|
||||
|
||||
const json_path_step_t *last_step=
|
||||
((const json_path_step_t *) path_obj->steps.buffer) +
|
||||
path_obj->last_step_idx;
|
||||
*is_array_cell= (last_step->type & JSON_PATH_ARRAY) != 0;
|
||||
*is_from_end= *is_array_cell && last_step->n_item < 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
Apply one partial-JSON column of a MySQL PARTIAL_UPDATE_ROWS_EVENT
|
||||
after-image.
|
||||
|
||||
A partial column carries a Json_diff vector rather than a value, so it
|
||||
cannot go through the normal unpack path: there is nothing to unpack into
|
||||
the conversion table and copy out with convert_field(). Instead the base
|
||||
document is taken from the matched row, the diffs are applied to it in
|
||||
order, and the resulting text is stored into the slave field directly.
|
||||
|
||||
The base document is the *before-image*, which lives in record[1] while
|
||||
record[0] is being built. The field is temporarily shifted there to read
|
||||
it (move_field_offset()), matching MySQL, which applies diffs to the
|
||||
field's current value.
|
||||
|
||||
Each diff is folded through MariaDB's own JSON functions, so the result
|
||||
matches what the corresponding SQL function would produce:
|
||||
|
||||
REPLACE -> JSON_REPLACE, INSERT -> JSON_INSERT, REMOVE -> JSON_REMOVE
|
||||
|
||||
applied one at a time, each result feeding the next. A diff value arrives
|
||||
as MySQL binary JSON and is decoded to text by storing it into
|
||||
\a conv_field (a Field_mysql_json) and reading it back; the text is then
|
||||
wrapped in JSON_COMPACT() so that JSON_INSERT/JSON_REPLACE treat it as a
|
||||
JSON value rather than as a quoted string.
|
||||
|
||||
The Items built for this are created on a private THD::free_list which is
|
||||
released here, so their internal buffers do not accumulate until the end
|
||||
of the statement.
|
||||
|
||||
@param rgi Relay group info; supplies the THD the Items are built on.
|
||||
@param table The slave table. Used to locate the before-image, i.e.
|
||||
the distance between record[0] and record[1].
|
||||
@param result_field
|
||||
The slave's JSON (or text) field, in record[0]. Receives
|
||||
the document produced by applying the diffs, and is set
|
||||
NOT NULL.
|
||||
@param conv_field
|
||||
The conversion-table Field_mysql_json for this column,
|
||||
used purely as a binary-JSON-to-text decoder for the diff
|
||||
values. Must not be NULL: a partial JSON column always
|
||||
has a conversion field, since the source type is never
|
||||
native to MariaDB.
|
||||
@param[in,out] state
|
||||
Unpack state. Its pack_ptr is advanced past the diff
|
||||
vector that this call consumes, so the caller continues
|
||||
with the next column. Left unchanged if the vector is
|
||||
corrupt.
|
||||
@param row_end One past the last readable byte of the row image, bounding
|
||||
the diff decoding.
|
||||
|
||||
@retval 0 Success; result_field holds the new document.
|
||||
@retval HA_ERR_CORRUPT_EVENT The diff vector is truncated or malformed.
|
||||
@retval HA_ERR_GENERIC The before-image was NULL, or building or
|
||||
evaluating one of the JSON functions failed.
|
||||
An error has been raised on the THD.
|
||||
|
||||
@note Known gaps, tracked in the MDEV-39143 design document:
|
||||
- MySQL distinguishes a REJECTED diff (the path does not match, so
|
||||
the value is left alone and replication continues) from a real
|
||||
error. Here every failure stops the SQL thread.
|
||||
- The NULL before-image case raises a placeholder ER_UNKNOWN_ERROR.
|
||||
*/
|
||||
static int apply_partial_json_column(const rpl_group_info *rgi, TABLE *table,
|
||||
Field *result_field, Field *conv_field,
|
||||
Unpack_record_state &state,
|
||||
const uchar *row_end)
|
||||
{
|
||||
Dynamic_array<Mysql_json_diff> diffs((PSI_memory_key) PSI_NOT_INSTRUMENTED);
|
||||
const uchar *ptr= state.pack_ptr;
|
||||
if (read_mysql_json_diff_vector(&ptr, row_end, diffs))
|
||||
return HA_ERR_CORRUPT_EVENT;
|
||||
state.pack_ptr= ptr;
|
||||
|
||||
THD *thd= rgi->thd;
|
||||
DBUG_ASSERT(conv_field);
|
||||
|
||||
/* Read the before-image JSON value (found row is in record[1]). */
|
||||
my_ptrdiff_t bi_offset= (my_ptrdiff_t) (table->record[1] - table->record[0]);
|
||||
result_field->move_field_offset(bi_offset);
|
||||
bool base_is_null= result_field->is_null();
|
||||
String base_buf;
|
||||
String *base= base_is_null ? NULL : result_field->val_str(&base_buf);
|
||||
result_field->move_field_offset(-bi_offset);
|
||||
if (base_is_null || !base)
|
||||
{
|
||||
my_printf_error(ER_UNKNOWN_ERROR,
|
||||
"MDEV-39143: partial JSON update with no base value (slave out of sync)",
|
||||
MYF(0));
|
||||
return HA_ERR_GENERIC;
|
||||
}
|
||||
|
||||
String cur;
|
||||
if (cur.copy(*base))
|
||||
return HA_ERR_GENERIC;
|
||||
|
||||
/*
|
||||
Build and evaluate the JSON functions on a private free_list, then release
|
||||
them (and their internal buffers) here with free_items(), instead of
|
||||
letting them accumulate until end of statement.
|
||||
*/
|
||||
Item *const save_free_list= thd->free_list;
|
||||
thd->free_list= NULL;
|
||||
int rc= 0;
|
||||
|
||||
/* Reused by every INSERT diff of this column to classify its path. */
|
||||
json_path_t diff_path;
|
||||
mem_root_dynamic_array_init(thd->mem_root, PSI_INSTRUMENT_MEM,
|
||||
&diff_path.steps, sizeof(json_path_step_t),
|
||||
NULL, JSON_DEPTH_DEFAULT, JSON_DEPTH_INC,
|
||||
MYF(0));
|
||||
|
||||
for (size_t k= 0; k < diffs.elements() && !rc; k++)
|
||||
{
|
||||
const Mysql_json_diff &diff= diffs.at(k);
|
||||
List<Item> args;
|
||||
Item *doc_it= new (thd->mem_root)
|
||||
Item_string(thd, cur.ptr(), (uint) cur.length(), cur.charset());
|
||||
Item *path_it= new (thd->mem_root)
|
||||
Item_string(thd, diff.path, (uint) diff.path_length,
|
||||
&my_charset_utf8mb4_bin);
|
||||
if (!doc_it || !path_it ||
|
||||
args.push_back(doc_it, thd->mem_root) ||
|
||||
args.push_back(path_it, thd->mem_root))
|
||||
{ rc= HA_ERR_GENERIC; break; }
|
||||
|
||||
Item *func;
|
||||
if (diff.op == MYSQL_JSON_DIFF_REMOVE)
|
||||
func= new (thd->mem_root) Item_func_json_remove(thd, args);
|
||||
else
|
||||
{
|
||||
/*
|
||||
Point conv_field directly at the diff's raw JSONB bytes, the same way
|
||||
Field_blob::unpack() hands a normal blob column its wire bytes.
|
||||
Field::store() must not be used here: Field_mysql_json is built with
|
||||
a fixed field_charset() of utf8mb4_bin (mysql_json.cc), so store()
|
||||
would run the bytes through well_formed_copy_with_check() and reject
|
||||
them the moment a structural byte (e.g. a multi-byte JSONB length,
|
||||
needed once a string exceeds 127 bytes) is not valid standalone
|
||||
UTF-8. JSONB is binary data, not text, and must reach val_str()
|
||||
unmodified; set_ptr() bypasses charset handling entirely, exactly
|
||||
like the ordinary full-document unpack path already does.
|
||||
*/
|
||||
((Field_blob *) conv_field)->set_ptr(diff.value_length,
|
||||
const_cast<uchar *>(diff.value));
|
||||
String vbuf, *vtext= conv_field->val_str(&vbuf);
|
||||
if (!vtext)
|
||||
{ rc= HA_ERR_GENERIC; break; }
|
||||
/*
|
||||
Copy the decoded value onto the statement mem_root (val_str's buffer is
|
||||
transient) and wrap it in JSON_COMPACT, so JSON_INSERT/REPLACE inserts
|
||||
it as a JSON value rather than a quoted string.
|
||||
*/
|
||||
char *vdup= (char *) thd->memdup(vtext->ptr(), vtext->length());
|
||||
Item *v_it= vdup ? new (thd->mem_root)
|
||||
Item_string(thd, vdup, (uint) vtext->length(), vtext->charset())
|
||||
: NULL;
|
||||
Item *val_json= v_it ? new (thd->mem_root)
|
||||
Item_func_json_format(thd, v_it, Item_func_json_format::COMPACT)
|
||||
: NULL;
|
||||
if (!val_json || args.push_back(val_json, thd->mem_root))
|
||||
{ rc= HA_ERR_GENERIC; break; }
|
||||
if (diff.op == MYSQL_JSON_DIFF_INSERT)
|
||||
{
|
||||
/*
|
||||
An INSERT on an array cell is JSON_ARRAY_INSERT, on an object
|
||||
member JSON_INSERT. See json_diff_path_is_array_cell().
|
||||
*/
|
||||
bool is_array_cell= false, is_from_end= false;
|
||||
if (json_diff_path_is_array_cell(&diff_path, diff.path,
|
||||
diff.path_length, &is_array_cell,
|
||||
&is_from_end))
|
||||
{
|
||||
my_printf_error(ER_UNKNOWN_ERROR,
|
||||
"MDEV-39143: unparsable JSON path in a partial "
|
||||
"JSON update: '%.*s'", MYF(0),
|
||||
(int) diff.path_length, diff.path);
|
||||
rc= HA_ERR_GENERIC;
|
||||
break;
|
||||
}
|
||||
/*
|
||||
Refuse rather than diverge silently. MySQL emits an array-cell
|
||||
INSERT only for a cell that does not exist, which for a
|
||||
from-the-end index means it clamped the position to 0 and
|
||||
prepended. MariaDB's JSON_ARRAY_INSERT resolves '[last-N]'
|
||||
differently - it appends when the index goes negative, and is off
|
||||
by one even in range - so applying the diff here would store a
|
||||
document that differs from the source with no error raised. Stop
|
||||
the SQL thread instead; a wrong row that replication reports as
|
||||
applied is worse than a halt. Lift this once JSON_ARRAY_INSERT
|
||||
resolves such paths the same way the other JSON functions do.
|
||||
*/
|
||||
if (is_array_cell && is_from_end)
|
||||
{
|
||||
my_printf_error(ER_UNKNOWN_ERROR,
|
||||
"MDEV-39143: cannot apply a partial JSON update "
|
||||
"that inserts at a from-the-end array index: "
|
||||
"path '%.*s' on %s.%s.%s. MariaDB's "
|
||||
"JSON_ARRAY_INSERT resolves '[last-N]' "
|
||||
"differently from MySQL, so the result would "
|
||||
"silently differ from the source.", MYF(0),
|
||||
(int) diff.path_length, diff.path,
|
||||
table->s->db.str, table->s->table_name.str,
|
||||
result_field->field_name.str);
|
||||
rc= HA_ERR_GENERIC;
|
||||
break;
|
||||
}
|
||||
func= is_array_cell ?
|
||||
(Item *) new (thd->mem_root) Item_func_json_array_insert(thd, args)
|
||||
: (Item *) new (thd->mem_root) Item_func_json_insert(true, false,
|
||||
thd, args);
|
||||
}
|
||||
else
|
||||
func= new (thd->mem_root) Item_func_json_insert(false, true,
|
||||
thd, args);
|
||||
}
|
||||
if (!func || func->fix_fields(thd, &func))
|
||||
{ rc= HA_ERR_GENERIC; break; }
|
||||
String rbuf, *res= func->val_str(&rbuf);
|
||||
if (!res || func->null_value || cur.copy(*res))
|
||||
{ rc= HA_ERR_GENERIC; break; }
|
||||
}
|
||||
|
||||
thd->free_items();
|
||||
thd->free_list= save_free_list;
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
result_field->set_notnull();
|
||||
if (result_field->store(cur.ptr(), cur.length(), cur.charset()))
|
||||
return HA_ERR_GENERIC;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
Unpack a row into @c table->record[0].
|
||||
|
||||
@@ -307,6 +714,16 @@ static void convert_field(Field *result_field, Field *conv_field)
|
||||
@param row_end
|
||||
Pointer to variable that will hold the value of the
|
||||
end position for the data in the row event
|
||||
@param is_partial_json_after_image
|
||||
True when unpacking the after-image of a MySQL
|
||||
PARTIAL_UPDATE_ROWS_EVENT (binlog_row_value_options=
|
||||
PARTIAL_JSON on the source). Such an image starts with a
|
||||
value_options integer and, if that has PARTIAL_JSON_UPDATES
|
||||
set, a partial_bits bitmap with one bit per JSON column of
|
||||
the source table; a JSON column whose bit is set carries a
|
||||
diff vector instead of a full value. Must be false for a
|
||||
before-image, which never has that header and never carries
|
||||
diffs, and for every other event type.
|
||||
|
||||
@retval 0 No error
|
||||
|
||||
@@ -315,11 +732,11 @@ static void convert_field(Field *result_field, Field *conv_field)
|
||||
@retval HA_ERR_CORRUPT_EVENT
|
||||
Found error when trying to unpack fields.
|
||||
*/
|
||||
|
||||
int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const master_cols,
|
||||
uchar const *const row_data, MY_BITMAP const *cols,
|
||||
uchar const **const current_row_end,
|
||||
uchar const *const row_end)
|
||||
uchar const *const row_end,
|
||||
bool is_partial_json_after_image)
|
||||
{
|
||||
int error;
|
||||
bool null_value;
|
||||
@@ -328,7 +745,33 @@ int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const master_cols,
|
||||
DBUG_ASSERT(table);
|
||||
DBUG_ASSERT(rgi);
|
||||
|
||||
Unpack_record_state st(row_data, row_end, (bitmap_bits_set(cols) + 7) / 8);
|
||||
const uchar *rec_ptr= row_data;
|
||||
|
||||
/*
|
||||
A MySQL PARTIAL_UPDATE_ROWS_EVENT update after-image is prefixed with a
|
||||
value_options integer and, when it has the PARTIAL_JSON_UPDATES bit set, a
|
||||
partial_bits bitmap carrying one bit per JSON column in the master table
|
||||
(see MySQL sql/log_event.cc). Read past this header so the null bits and
|
||||
column values are unpacked from the correct offset.
|
||||
*/
|
||||
const uchar *pj_partial_bits= NULL; // set for a PARTIAL_JSON after-image
|
||||
uint pj_json_col= 0; // running index over master JSON columns
|
||||
if (is_partial_json_after_image)
|
||||
{
|
||||
const uint PARTIAL_JSON_UPDATES= 1;
|
||||
const table_def *pj_tabledef=
|
||||
&((RPL_TABLE_LIST*) table->pos_in_table_list)->m_tabledef;
|
||||
size_t avail= (size_t) (row_end - rec_ptr);
|
||||
ulonglong value_options=
|
||||
safe_net_field_length_ll((uchar**) &rec_ptr, avail);
|
||||
if (value_options & PARTIAL_JSON_UPDATES)
|
||||
{
|
||||
pj_partial_bits= rec_ptr;
|
||||
rec_ptr+= (pj_tabledef->json_column_count() + 7) / 8;
|
||||
}
|
||||
}
|
||||
|
||||
Unpack_record_state st(rec_ptr, row_end, (bitmap_bits_set(cols) + 7) / 8);
|
||||
|
||||
if (bitmap_is_clear_all(cols))
|
||||
{
|
||||
@@ -371,6 +814,19 @@ int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const master_cols,
|
||||
for (uint master_idx= 0; master_idx < master_cols; master_idx++)
|
||||
{
|
||||
Field *field=NULL, *conv_field= NULL;
|
||||
/*
|
||||
A partial JSON after-image carries one partial_bits entry per master
|
||||
JSON column, in column order, regardless of image/null. Advance the
|
||||
index for every JSON column and remember whether this one is a diff.
|
||||
*/
|
||||
bool is_partial_col= false;
|
||||
if (pj_partial_bits &&
|
||||
tabledef->binlog_type(master_idx) == MYSQL_TYPE_JSON_MYSQL)
|
||||
{
|
||||
is_partial_col= (pj_partial_bits[pj_json_col / 8] &
|
||||
(1 << (pj_json_col % 8))) != 0;
|
||||
pj_json_col++;
|
||||
}
|
||||
/*
|
||||
Check 1: Skip unpacking if the field wasn't written in this record.
|
||||
This can happen for update row events when the before_image and
|
||||
@@ -442,6 +898,15 @@ int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const master_cols,
|
||||
result_field->field_name.str, master_idx));
|
||||
DBUG_ASSERT(field != NULL);
|
||||
|
||||
if (is_partial_col)
|
||||
{
|
||||
int pj_err= apply_partial_json_column(rgi, table, result_field,
|
||||
conv_field, st, row_end);
|
||||
if (pj_err)
|
||||
DBUG_RETURN(pj_err);
|
||||
}
|
||||
else
|
||||
{
|
||||
bool unpack_result= st.unpack_field(tabledef, field, master_idx);
|
||||
if (!unpack_result)
|
||||
{
|
||||
@@ -464,6 +929,7 @@ int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const master_cols,
|
||||
*/
|
||||
if (conv_field)
|
||||
convert_field(result_field, conv_field);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
@@ -33,7 +33,8 @@ int unpack_row(const rpl_group_info *rgi,
|
||||
TABLE *table, uint const colcnt,
|
||||
uchar const *const row_data, MY_BITMAP const *cols,
|
||||
uchar const **const curr_row_end,
|
||||
uchar const *const row_end);
|
||||
uchar const *const row_end,
|
||||
bool is_partial_json_after_image= false);
|
||||
|
||||
// Fill table's record[0] with default values.
|
||||
int prepare_record(TABLE *const table);
|
||||
|
||||
@@ -32,7 +32,8 @@ uint32 table_def::calc_field_size(uint col, uchar *master_data) const
|
||||
{
|
||||
uint32 length= 0;
|
||||
|
||||
switch (type(col)) {
|
||||
// (uint) so MYSQL_TYPE_JSON_MYSQL, not an enum_field_types, is a legal case
|
||||
switch ((uint) type(col)) {
|
||||
case MYSQL_TYPE_NEWDECIMAL:
|
||||
length= my_decimal_get_binary_size(m_field_metadata[col] >> 8,
|
||||
m_field_metadata[col] & 0xff);
|
||||
@@ -137,6 +138,7 @@ uint32 table_def::calc_field_size(uint col, uchar *master_data) const
|
||||
case MYSQL_TYPE_BLOB:
|
||||
case MYSQL_TYPE_BLOB_COMPRESSED:
|
||||
case MYSQL_TYPE_GEOMETRY:
|
||||
case MYSQL_TYPE_JSON_MYSQL: // a blob value with a pack-length prefix
|
||||
{
|
||||
/*
|
||||
Compute the length of the data. We cannot use get_length() here
|
||||
@@ -222,7 +224,8 @@ table_def::table_def(unsigned char *types, ulong size,
|
||||
int index= 0;
|
||||
for (unsigned int i= 0; i < m_size; i++)
|
||||
{
|
||||
switch (binlog_type(i)) {
|
||||
// (uint): see the cast in calc_field_size() above
|
||||
switch ((uint) binlog_type(i)) {
|
||||
case MYSQL_TYPE_TINY_BLOB:
|
||||
case MYSQL_TYPE_BLOB:
|
||||
case MYSQL_TYPE_BLOB_COMPRESSED:
|
||||
@@ -278,6 +281,9 @@ table_def::table_def(unsigned char *types, ulong size,
|
||||
case MYSQL_TYPE_TIMESTAMP2:
|
||||
m_field_metadata[i]= field_metadata[index++];
|
||||
break;
|
||||
case MYSQL_TYPE_JSON_MYSQL: // one metadata byte (the pack length)
|
||||
m_field_metadata[i]= field_metadata[index++];
|
||||
break;
|
||||
default:
|
||||
m_field_metadata[i]= 0;
|
||||
break;
|
||||
|
||||
@@ -28,6 +28,22 @@
|
||||
#endif
|
||||
#include "mysql_com.h"
|
||||
|
||||
/*
|
||||
MySQL 5.7+ binary JSON column type code, as it appears in .frm files and in
|
||||
row events replicated from a MySQL master. MariaDB has no native type for
|
||||
it - the read-only MYSQL_JSON plugin decodes it.
|
||||
|
||||
Defined here rather than in mysql_com.h because that header cannot carry it:
|
||||
the server's include/mysql_com.h and Connector/C's
|
||||
libmariadb/include/mariadb_com.h share the include guard _mysql_com_h, so in
|
||||
a client build (mysqlbinlog compiles rpl_utility.cc and log_event.cc as
|
||||
source) only the latter is seen. Note that mariadb_com.h does declare
|
||||
MYSQL_TYPE_JSON=245, but the server's enum_field_types does not, and 245 is
|
||||
MYSQL_TYPE_VIRTUAL in MariaDB's own pre-10.2 .frm encoding - which of the
|
||||
two a 245 byte means is decided by the context it was read from.
|
||||
*/
|
||||
#define MYSQL_TYPE_JSON_MYSQL 245
|
||||
|
||||
class Relay_log_info;
|
||||
class Log_event;
|
||||
class Rows_log_event;
|
||||
@@ -163,6 +179,21 @@ public:
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
Number of MySQL binary JSON columns in the master table.
|
||||
Used to size the partial_bits bitmap in the after-image of a MySQL
|
||||
PARTIAL_UPDATE_ROWS_EVENT, which carries one bit per JSON column in the
|
||||
table regardless of whether it is present in the row image.
|
||||
*/
|
||||
uint json_column_count() const
|
||||
{
|
||||
uint n= 0;
|
||||
for (uint i= 0; i < m_size; i++)
|
||||
if (binlog_type(i) == MYSQL_TYPE_JSON_MYSQL)
|
||||
n++;
|
||||
return n;
|
||||
}
|
||||
|
||||
/*
|
||||
This function returns whether the field on the master can be null.
|
||||
This value is derived from field->maybe_null().
|
||||
|
||||
@@ -533,6 +533,21 @@ Field_longstr::rpl_conv_type_from(const Conv_source &source,
|
||||
const Relay_log_info *rli,
|
||||
const Conv_param ¶m) const
|
||||
{
|
||||
/*
|
||||
A MySQL binary JSON source (mysql_json plugin), replicated from a MySQL
|
||||
master. The conversion-table field decodes the binary JSON into text.
|
||||
- target native JSON: same logical type, only the storage format differs,
|
||||
so this is lossless -> CONV_TYPE_VARIANT (allowed without
|
||||
slave_type_conversions).
|
||||
- target plain string/blob: a genuine cross-type conversion ->
|
||||
CONV_TYPE_SUBSET_TO_SUPERSET (requires slave_type_conversions).
|
||||
*/
|
||||
{
|
||||
const LEX_CSTRING mysql_json{STRING_WITH_LEN("MYSQL_JSON")};
|
||||
if (source.type_handler()->name().eq(mysql_json))
|
||||
return Type_handler_json_common::is_json_type_handler(type_handler()) ?
|
||||
CONV_TYPE_VARIANT : CONV_TYPE_SUBSET_TO_SUPERSET;
|
||||
}
|
||||
/**
|
||||
@todo
|
||||
Implement Field_varstring_compressed::real_type() and
|
||||
@@ -891,6 +906,23 @@ const Type_handler *table_def::field_type_handler(uint col) const
|
||||
*/
|
||||
if (typecode == MYSQL_TYPE_DATE)
|
||||
return &type_handler_newdate;
|
||||
/*
|
||||
MySQL 5.7+ binary JSON is not a native MariaDB type.
|
||||
It is handled by the read-only mysql_json plugin - the same handler used
|
||||
when opening a MySQL table on disk (see open_binary_frm() in table.cc) -
|
||||
which lets a MariaDB slave decode JSON columns in row events replicated
|
||||
from a MySQL master. handler_by_name() returns NULL if the plugin is not
|
||||
installed; give_compatibility_error() turns that into a message naming
|
||||
the plugin. Note it must not be handler_by_name_or_error() here: this
|
||||
getter is const, is called speculatively, and is called from inside
|
||||
give_compatibility_error() itself, so it must not raise an error of its
|
||||
own.
|
||||
*/
|
||||
if (typecode == MYSQL_TYPE_JSON_MYSQL)
|
||||
{
|
||||
const LEX_CSTRING mysql_json{STRING_WITH_LEN("MYSQL_JSON")};
|
||||
return Type_handler::handler_by_name(current_thd, mysql_json);
|
||||
}
|
||||
return Type_handler::get_handler_by_real_type(typecode);
|
||||
}
|
||||
|
||||
@@ -1093,11 +1125,27 @@ bool RPL_TABLE_LIST::give_compatibility_error(rpl_group_info *rgi, uint col)
|
||||
case SLAVE_FIELD_UNKNOWN_TYPE:
|
||||
{
|
||||
Field *field= table->field[m_tabledef.master_to_slave_map[col]];
|
||||
my_snprintf(error_msg, sizeof(error_msg),
|
||||
"In RBR mode, Slave received unknown field type field %d "
|
||||
"for column Name: %s.%s.%s",
|
||||
m_tabledef.binlog_type(col), field->table->s->db.str,
|
||||
field->table->s->table_name.str, field->field_name.str);
|
||||
/*
|
||||
field_type_handler() returns NULL for MYSQL_TYPE_JSON_MYSQL only when
|
||||
the mysql_json plugin is absent, so the type code tells the two cases
|
||||
apart: a genuinely unknown type, or a known one we cannot decode
|
||||
because its handler is not loaded. Name the plugin in the latter case
|
||||
- "unknown field type 245" gives the DBA nothing to act on.
|
||||
*/
|
||||
if (m_tabledef.binlog_type(col) == MYSQL_TYPE_JSON_MYSQL)
|
||||
my_snprintf(error_msg, sizeof(error_msg),
|
||||
"In RBR mode, Slave received a MySQL binary JSON column "
|
||||
"(type %d) for column Name: %s.%s.%s, but the MYSQL_JSON "
|
||||
"plugin needed to decode it is not installed. Run "
|
||||
"INSTALL SONAME 'type_mysql_json' on the slave.",
|
||||
m_tabledef.binlog_type(col), field->table->s->db.str,
|
||||
field->table->s->table_name.str, field->field_name.str);
|
||||
else
|
||||
my_snprintf(error_msg, sizeof(error_msg),
|
||||
"In RBR mode, Slave received unknown field type field %d "
|
||||
"for column Name: %s.%s.%s",
|
||||
m_tabledef.binlog_type(col), field->table->s->db.str,
|
||||
field->table->s->table_name.str, field->field_name.str);
|
||||
rgi->rli->report(
|
||||
ERROR_LEVEL, ER_SLAVE_INCOMPATIBLE_TABLE_DEF, rgi->gtid_info(),
|
||||
ER_THD(rgi->thd, ER_SLAVE_INCOMPATIBLE_TABLE_DEF), error_msg);
|
||||
|
||||
@@ -50,6 +50,7 @@
|
||||
#include "sql_delete.h" // class Sql_cmd_delete
|
||||
#include "rpl_rli.h" // class rpl_group_info
|
||||
#include "rpl_mi.h" // class Master_info
|
||||
#include "rpl_utility.h" // MYSQL_TYPE_JSON_MYSQL
|
||||
#include "vector_mhnsw.h"
|
||||
#include "opt_group_by_cardinality.h"
|
||||
|
||||
@@ -2589,8 +2590,8 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
|
||||
enum_field_types field_type= (enum_field_types) strpos[13];
|
||||
if (!(handler= Type_handler::get_handler_by_real_type(field_type)))
|
||||
{
|
||||
if (field_type == 245 &&
|
||||
share->mysql_version >= 50700) // a.k.a MySQL 5.7 JSON
|
||||
if (field_type == MYSQL_TYPE_JSON_MYSQL &&
|
||||
share->mysql_version >= 50700)
|
||||
{
|
||||
share->incompatible_version|= HA_CREATE_USED_ENGINE;
|
||||
const LEX_CSTRING mysql_json{STRING_WITH_LEN("MYSQL_JSON")};
|
||||
|
||||
Reference in New Issue
Block a user