1
0
mirror of https://github.com/MariaDB/server.git synced 2026-10-23 18:49:29 +03:00

MDEV-39143 Replicate MySQL binary JSON to a MariaDB slave

Let a MariaDB slave apply ROW-format binlog events from a MySQL
master that contain JSON columns, both as full documents and as
MySQL's partial (diff) JSON updates (PARTIAL_UPDATE_ROWS_EVENT).
This commit is contained in:
Pekka Lampio
2026-09-18 16:25:05 +03:00
parent 172609526b
commit 16ce07df15
16 changed files with 844 additions and 33 deletions

Binary file not shown.

View File

@@ -1,6 +1,7 @@
include/master-slave.inc
[connection master]
connection slave;
INSTALL SONAME 'type_mysql_json';
include/stop_slave.inc
connection master;
include/rpl_stop_server.inc [server_number=1]
@@ -25,7 +26,6 @@ CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=2297;
START SLAVE IO_THREAD;
START SLAVE SQL_THREAD;
include/wait_for_slave_io_to_start.inc
include/wait_for_slave_sql_error.inc [errno=1594]
SELECT * FROM t1 ORDER BY a;
a b c
1 0
@@ -35,16 +35,16 @@ a b c
5 0 after compressed
SELECT * FROM t2 ORDER BY a;
a b
1 {"a": "hulu", "b": "[zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy][zyzzy]", "c": "bulu"}
STOP SLAVE IO_THREAD;
include/wait_for_slave_io_to_stop.inc
1 {"a": "hulu", "b": "oOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOooOo", "c": "bulu"}
STOP SLAVE;
include/wait_for_slave_to_stop.inc
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4;
START SLAVE IO_THREAD;
START SLAVE SQL_THREAD;
include/wait_for_slave_io_to_start.inc
include/wait_for_slave_sql_to_start.inc
DROP TABLE t1, t2;
UNINSTALL SONAME 'type_mysql_json';
CALL mtr.add_suppression('TRANSACTION_PAYLOAD_EVENT event. You can avoid this event by specifying');
CALL mtr.add_suppression('PARTIAL_UPDATE_ROWS_EVENT event. You can avoid this event by specifying');
connection master;
include/rpl_end.inc

View File

@@ -0,0 +1,36 @@
include/master-slave.inc
[connection master]
connection slave;
INSTALL SONAME 'type_mysql_json';
include/stop_slave.inc
connection master;
include/rpl_stop_server.inc [server_number=1]
include/rpl_start_server.inc [server_number=1]
connection slave;
CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
call mtr.add_suppression("MDEV-39143: cannot apply a partial JSON update");
call mtr.add_suppression("Slave SQL.*Error executing row event");
START SLAVE;
include/wait_for_slave_sql_error.inc [errno=1105]
# Slave stopped, Last_SQL_Error matches the expected refusal:
# Could not execute MySQL Update_rows_partial event on table jsontest.t1; MDEV-39143: cannot apply a partial JSON update that inserts at a from-the-end array index: path '$.arr[last-9]' on jsontest.t1.j. MariaDB's JSON_ARRAY_INSERT resolves '[last-N]' differently from MySQL, so the result would silently differ from the source., Error_code: 1105; handler error HA_ERR_GENERIC; the event's master log master-bin.000001, end_log_pos 2453
#
# Rows applied before the halt: id=1 got its object-member INSERT and
# id=2 its array append. id=3 and id=4 keep the inserted values.
SELECT id, JSON_QUERY(j,'$.arr') AS arr, JSON_VALUE(j,'$.newkey') AS newkey
FROM jsontest.t1 ORDER BY id;
id arr newkey
1 [1, 2, 3] added
2 [1, 2, 3, 99] NULL
3 [1, 2, 3] NULL
4 [1, 2, 3] NULL
connection slave;
STOP SLAVE IO_THREAD;
include/wait_for_slave_io_to_stop.inc
DROP DATABASE jsontest;
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
START SLAVE;
connection master;
connection slave;
UNINSTALL SONAME 'type_mysql_json';
include/rpl_end.inc

View File

@@ -0,0 +1,27 @@
include/master-slave.inc
[connection master]
connection slave;
INSTALL SONAME 'type_mysql_json';
include/stop_slave.inc
connection master;
include/rpl_stop_server.inc [server_number=1]
include/rpl_start_server.inc [server_number=1]
connection slave;
CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
START SLAVE;
# id=1: inserted as a full document, then modified by four partial-JSON
# events (REPLACE of members a and n twice, REMOVE of arr element 2).
SELECT id, JSON_VALUE(j,'$.a') AS a, JSON_VALUE(j,'$.n') AS n,
JSON_QUERY(j,'$.arr') AS arr FROM testdb.tj ORDER BY id;
id a n arr
1 zzzz 44 [1, 2]
connection slave;
STOP SLAVE;
include/wait_for_slave_to_stop.inc
DROP DATABASE testdb;
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
START SLAVE;
connection master;
connection slave;
UNINSTALL SONAME 'type_mysql_json';
include/rpl_end.inc

View File

@@ -59,6 +59,7 @@
--connection slave
INSTALL SONAME 'type_mysql_json';
--source include/stop_slave.inc
--connection master
@@ -99,14 +100,26 @@ CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=2297;
START SLAVE IO_THREAD;
START SLAVE SQL_THREAD;
--source include/wait_for_slave_io_to_start.inc
--let $slave_sql_errno= 1594
--source include/wait_for_slave_sql_error.inc
# MDEV-39143: PARTIAL_UPDATE_ROWS_EVENT is now applied instead of rejected.
# Wait for the JSON_REPLACE partial update (the last event in the injected
# file) to reach t2, rather than for a fixed position, since the exact byte
# offset is fragile against any future re-recording of the fixture.
# LIKE avoids '$' which mysqltest would interpolate inside --let; the
# repeating unit is 3 chars ('oOo'), so 'oOooOo' is the substring it produces.
--let $wait_condition= SELECT COUNT(*)=1 FROM test.t2 WHERE a=1 AND b LIKE '%oOooOo%'
--source include/wait_condition.inc
SELECT * FROM t1 ORDER BY a;
SELECT * FROM t2 ORDER BY a;
STOP SLAVE IO_THREAD;
--source include/wait_for_slave_io_to_stop.inc
# The SQL thread is genuinely caught up and running now that the partial
# JSON update applies instead of erroring, so both threads must be stopped
# before CHANGE MASTER (STOP SLAVE IO_THREAD alone left it running here
# before MDEV-39143, when the SQL thread had already halted on the event 39
# rejection).
STOP SLAVE;
--source include/wait_for_slave_to_stop.inc
# Restart replication after the MySQL 8.0 file.
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4;
START SLAVE IO_THREAD;
@@ -116,8 +129,8 @@ START SLAVE SQL_THREAD;
--sync_with_master
DROP TABLE t1, t2;
UNINSTALL SONAME 'type_mysql_json';
CALL mtr.add_suppression('TRANSACTION_PAYLOAD_EVENT event. You can avoid this event by specifying');
CALL mtr.add_suppression('PARTIAL_UPDATE_ROWS_EVENT event. You can avoid this event by specifying');
--connection master
--source include/rpl_end.inc

View File

@@ -0,0 +1,95 @@
# MDEV-39143: INSERT diffs in a MySQL partial JSON update.
#
# Companion to rpl_json_partial_update.test and driven the same way - see the
# header there for how the recorded MySQL binlog is injected. The fixture,
# std_data/mdev39143_partial_json_insert.000001, was recorded on MySQL 9.2
# with binlog_row_value_options=PARTIAL_JSON from:
#
# UPDATE t1 SET j=JSON_SET(j,'$.newkey','added') WHERE id=1;
# UPDATE t1 SET j=JSON_SET(j,'$.arr[7]',99) WHERE id=2;
# UPDATE t1 SET j=JSON_SET(j,'$.arr[last-9]',77) WHERE id=3;
# UPDATE t1 SET j=JSON_SET(j,'$.arr[1]',55) WHERE id=4;
# UPDATE t1 SET j=JSON_REMOVE(j,'$.arr[2]') WHERE id=4;
#
# MySQL logs the first as an INSERT diff on an object-member path and the
# next two as INSERT diffs on array-cell paths; its own mysqlbinlog renders
# them as JSON_INSERT, JSON_ARRAY_INSERT and JSON_ARRAY_INSERT respectively.
# The slave must make the same distinction: id=3 is the discriminating case,
# where the path resolves before the start of the array so the value is
# prepended and the existing elements shift right. JSON_INSERT would leave
# the document untouched there, silently.
#
# MySQL's own contents after those updates, read from the source table:
# id=1 arr [1, 2, 3] newkey "added"
# id=2 arr [1, 2, 3, 99]
# id=3 arr [77, 1, 2, 3]
# id=4 arr [1, 55]
#
# id=3 cannot be applied correctly yet and the slave deliberately stops on
# it. MySQL emits an array-cell INSERT only for a cell that does not exist,
# which for '[last-9]' means it clamped the position to 0 and prepended.
# MariaDB's JSON_ARRAY_INSERT resolves a from-the-end index differently - it
# appends once the index goes negative, and is off by one even in range -
# while JSON_EXTRACT, JSON_REPLACE and JSON_REMOVE all agree with MySQL on
# the same paths. That looks like a pre-existing JSON_ARRAY_INSERT bug
# reachable from plain SQL, so rather than store a document that silently
# differs from the source, the apply path refuses the diff and halts.
#
# This test therefore asserts the halt, and that the diffs before it applied
# correctly: id=1 (object member) and id=2 (array append, where both
# functions agree). id=4's REPLACE and REMOVE come after the failing event
# and are not reached. When JSON_ARRAY_INSERT is fixed, drop the guard in
# apply_partial_json_column() and this test should expect MySQL's values
# above for all four rows.
--source include/have_innodb.inc
--source include/have_binlog_format_row.inc
--source include/master-slave.inc
--connection slave
INSTALL SONAME 'type_mysql_json';
--source include/stop_slave.inc
--connection master
--let $datadir= `SELECT @@datadir`
--let $rpl_server_number= 1
--source include/rpl_stop_server.inc
--remove_file $datadir/master-bin.000001
--copy_file $MYSQL_TEST_DIR/std_data/mdev39143_partial_json_insert.000001 $datadir/master-bin.000001
--source include/rpl_start_server.inc
--connection slave
CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
call mtr.add_suppression("MDEV-39143: cannot apply a partial JSON update");
call mtr.add_suppression("Slave SQL.*Error executing row event");
START SLAVE;
# The slave must stop on the id=3 diff rather than apply it.
--let $slave_sql_errno= 1105
--source include/wait_for_slave_sql_error.inc
--let $err= query_get_value(SHOW SLAVE STATUS, Last_SQL_Error, 1)
--echo # Slave stopped, Last_SQL_Error matches the expected refusal:
--echo # $err
--echo #
--echo # Rows applied before the halt: id=1 got its object-member INSERT and
--echo # id=2 its array append. id=3 and id=4 keep the inserted values.
SELECT id, JSON_QUERY(j,'$.arr') AS arr, JSON_VALUE(j,'$.newkey') AS newkey
FROM jsontest.t1 ORDER BY id;
# Cleanup.
# The SQL thread is already stopped by the error above, so only
# the IO thread needs stopping before CHANGE MASTER; wait_for_slave_to_stop
# would itself fail on the pending Last_SQL_Error.
--connection slave
STOP SLAVE IO_THREAD;
--source include/wait_for_slave_io_to_stop.inc
DROP DATABASE jsontest;
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
START SLAVE;
--connection master
--save_master_pos
--connection slave
--sync_with_master
UNINSTALL SONAME 'type_mysql_json';
--source include/rpl_end.inc

View File

@@ -0,0 +1,75 @@
# MDEV-39143: replicate MySQL binary JSON (full documents and partial diffs)
# from a MySQL master to a MariaDB slave.
#
# No MySQL server runs here. Both servers started by master-slave.inc below
# are MariaDB; the MySQL side is supplied by a binlog recorded once on a real
# MySQL 9.2 server and checked in as
# std_data/mdev39143_partial_json.000001. The test stops the MariaDB master,
# replaces the master's own master-bin.000001 with that file, and restarts it.
# The master is then only a file server for those bytes, and the slave pulls
# them over the ordinary replication protocol. What is under test is the
# slave apply path: PARTIAL_UPDATE_ROWS_EVENT (39) parsing, Json_diff
# decoding, and diff application - see unpack_row() in sql/rpl_record.cc.
#
# Important points:
# - The recorded events are fixed. The SQL the MySQL master ran is not
# visible in this file, only its effect (described at the SELECT below);
# extending coverage needs a MySQL server, a re-recording, and a new file.
# - The Json_diff encoding exercised is MySQL 9.2's specifically.
# - have_binlog_format_row.inc constrains the MariaDB master's own binlog
# format; the injected events are row events regardless of it.
--source include/have_innodb.inc
--source include/have_binlog_format_row.inc
--source include/master-slave.inc
# The slave decodes MySQL binary JSON with the read-only mysql_json plugin,
# which is not installed by default. Uninstalled again in the teardown.
--connection slave
INSTALL SONAME 'type_mysql_json';
--source include/stop_slave.inc
# Substitute the recorded MySQL binlog for the master's own. The server must
# be down while the file is swapped, and is restarted so it reopens the new
# master-bin.000001 and starts a fresh master-bin.000002 of its own.
--connection master
--let $datadir= `SELECT @@datadir`
--let $rpl_server_number= 1
--source include/rpl_stop_server.inc
--remove_file $datadir/master-bin.000001
--copy_file $MYSQL_TEST_DIR/std_data/mdev39143_partial_json.000001 $datadir/master-bin.000001
--source include/rpl_start_server.inc
--connection slave
CHANGE MASTER TO Master_log_file='master-bin.000001', Master_log_pos=4, Master_use_gtid=No;
START SLAVE;
# Wait until the last partial JSON update has been applied to id=1.
# (LIKE avoids '$' which mysqltest would interpolate inside --let.)
--let $wait_condition= SELECT COUNT(*)=1 FROM testdb.tj WHERE id=1 AND j LIKE '%"a": "zzzz"%' AND j LIKE '%"n": 44%'
--source include/wait_condition.inc
--echo # id=1: inserted as a full document, then modified by four partial-JSON
--echo # events (REPLACE of members a and n twice, REMOVE of arr element 2).
SELECT id, JSON_VALUE(j,'$.a') AS a, JSON_VALUE(j,'$.n') AS n,
JSON_QUERY(j,'$.arr') AS arr FROM testdb.tj ORDER BY id;
# Cleanup.
# The slave is sitting at the end of the injected binlog, which the
# master did not write and knows nothing about, so save_master_pos/
# sync_with_master cannot line the two up. Repoint the slave at
# master-bin.000002 - the empty binlog the master opened on restart - so that
# the two agree on a position and rpl_end.inc can shut down cleanly.
--connection slave
STOP SLAVE;
--source include/wait_for_slave_to_stop.inc
DROP DATABASE testdb;
CHANGE MASTER TO Master_log_file='master-bin.000002', Master_log_pos=4, Master_use_gtid=No;
START SLAVE;
--connection master
--save_master_pos
--connection slave
--sync_with_master
# Remove the plugin again: it is installed into mysql.plugin, which MTR's
# check-testcase compares against the pre-test server state.
UNINSTALL SONAME 'type_mysql_json';
--source include/rpl_end.inc

View File

@@ -1234,6 +1234,15 @@ Log_event *Log_event::read_log_event_no_checksum(
case UPDATE_ROWS_EVENT:
ev= new Update_rows_log_event(buf, event_len, fdle);
break;
// MySQL 8.0: partial JSON update
case PARTIAL_UPDATE_ROWS_EVENT:
/*
MDEV-39143: same event structure as UPDATE_ROWS_EVENT; the after-image
may carry JSON diffs, handled in unpack_row() via the
is_partial_json_after_image flag.
*/
ev= new Update_rows_log_event(buf, event_len, fdle);
break;
case DELETE_ROWS_EVENT_V1:
case DELETE_ROWS_EVENT:
ev= new Delete_rows_log_event(buf, event_len, fdle);
@@ -1302,14 +1311,6 @@ Log_event *Log_event::read_log_event_no_checksum(
"'binlog_transaction_compression=0' in the MySQL server";
ev= NULL;
break;
case PARTIAL_UPDATE_ROWS_EVENT: // MySQL 8.0
*error=
"Found incompatible MySQL 8.0 PARTIAL_UPDATE_ROWS_EVENT event. "
"You can avoid this event by specifying "
"'binlog-row-value-options=\"\"' in the MySQL server";
ev= NULL;
break;
case PRE_GA_WRITE_ROWS_EVENT:
case PRE_GA_UPDATE_ROWS_EVENT:
case PRE_GA_DELETE_ROWS_EVENT:

View File

@@ -809,7 +809,9 @@ static inline bool LOG_EVENT_IS_UPDATE_ROW(enum Log_event_type type)
{
return type == UPDATE_ROWS_EVENT || type == UPDATE_ROWS_EVENT_V1 ||
type == UPDATE_ROWS_COMPRESSED_EVENT ||
type == UPDATE_ROWS_COMPRESSED_EVENT_V1;
type == UPDATE_ROWS_COMPRESSED_EVENT_V1 ||
// MySQL 8.0+ partial JSON update (MDEV-39143)
type == PARTIAL_UPDATE_ROWS_EVENT;
}
@@ -5089,6 +5091,14 @@ protected:
int find_row(rpl_group_info *);
int update_sequence();
/*
True for a MySQL PARTIAL_UPDATE_ROWS_EVENT (binlog_row_value_options=
PARTIAL_JSON on the master). Its update after-image may carry JSON diffs
instead of full JSON values for some columns.
*/
bool is_partial_json_update() const
{ return m_type == PARTIAL_UPDATE_ROWS_EVENT; }
// Unpack the current row into m_table->record[0], but with
// a different columns bitmap.
int unpack_current_row(rpl_group_info *rgi, MY_BITMAP const *cols)
@@ -5097,7 +5107,8 @@ protected:
ASSERT_OR_RETURN_ERROR(m_curr_row <= m_rows_end, HA_ERR_CORRUPT_EVENT);
return ::unpack_row(rgi, m_table, m_width, m_curr_row, cols,
&m_curr_row_end, m_rows_end);
&m_curr_row_end, m_rows_end,
is_partial_json_update() && cols == &m_cols_ai);
}
// Unpack the current row into m_table->record[0]

View File

@@ -22,6 +22,9 @@
#include "slave.h" // Need to pull in slave_print_msg
#include "rpl_utility.h"
#include "rpl_rli.h"
/* MDEV-39143: Item_func_json_insert / _remove / _format */
#include "item_jsonfunc.h"
#include "sql_array.h" // Dynamic_array
/*
@@ -274,6 +277,410 @@ static void convert_field(Field *result_field, Field *conv_field)
}
/*
One decoded MySQL JSON diff from a PARTIAL_UPDATE_ROWS_EVENT. 'path' points
at the JSON path text and 'value' at the diff's new value in MySQL binary
JSON (JSONB); both point into the row buffer. 'value' is NULL for REMOVE.
*/
enum enum_mysql_json_diff_op
{
MYSQL_JSON_DIFF_REPLACE= 0,
MYSQL_JSON_DIFF_INSERT= 1,
MYSQL_JSON_DIFF_REMOVE= 2
};
struct Mysql_json_diff
{
enum_mysql_json_diff_op op;
const char *path;
size_t path_length;
const uchar *value; // NULL for REMOVE
size_t value_length;
};
/**
Decode a MySQL Json_diff_vector, the payload of one partial JSON column in
the after-image of a PARTIAL_UPDATE_ROWS_EVENT. The format, specified by
MySQL WL#2955 section 4, F3, is
[ length : 4 bytes ] [ diff_1 ] ... [ diff_N ]
where 'length' counts the bytes of the diffs that follow (not itself), and
each diff is
[ op : 1 ] [ path_length : net_field_length ] [ path : path_length ]
( [ value_length : net_field_length ] [ value : value_length ] )?
the value part being present if and only if op is not REMOVE. The number
of diffs is not stored; they are read until 'length' bytes are consumed.
@param[in,out] row_image_pos
The caller's read cursor into the row image. On entry it
points at the 4-byte length prefix; on success it is
advanced to one past the last diff, i.e. to the next
column's data. On failure it is left untouched, so the
caller must not rely on it to locate the error.
@param[in] end
One past the last readable byte of the row image. Only an
upper bound for validation: the extent of the vector itself
comes from its own length prefix, which is checked to fit
within [*row_image_pos, end). Every path and value length
is likewise checked against the vector's end, so a corrupt
length cannot make this function read outside the buffer.
@param[out] out
Decoded diffs are *appended*; existing elements are kept, so
the caller passes a fresh array unless it deliberately wants
to accumulate. On failure 'out' may already hold the diffs
decoded before the bad one - a caller that continues after an
error must clear it.
The 'path' and 'value' members point *into* the row image
rather than at copies, so the decoded diffs are only valid
while that buffer lives. 'value' is NULL and 'value_length'
is 0 for a REMOVE diff. Values are still MySQL binary JSON
here; decoding them to text is the caller's job.
@retval false Success; the whole vector was consumed.
@retval true The buffer is truncated or corrupt (a length runs past the
end of the vector, or an unknown operation code), or the
array could not grow. Note the MariaDB convention: true
means failure.
*/
static bool read_mysql_json_diff_vector(const uchar **row_image_pos,
const uchar *end,
Dynamic_array<Mysql_json_diff> &out)
{
const uchar *ptr= *row_image_pos;
if (end - ptr < 4)
return true;
size_t length= uint4korr(ptr);
ptr+= 4;
if ((size_t) (end - ptr) < length)
return true;
const uchar *const diffs_end= ptr + length;
while (ptr < diffs_end)
{
Mysql_json_diff diff;
diff.op= (enum_mysql_json_diff_op) *ptr++;
if (diff.op > MYSQL_JSON_DIFF_REMOVE)
return true;
size_t avail= (size_t) (diffs_end - ptr);
diff.path_length= (size_t) safe_net_field_length_ll((uchar**) &ptr, avail);
if ((size_t) (diffs_end - ptr) < diff.path_length)
return true;
diff.path= (const char*) ptr;
ptr+= diff.path_length;
if (diff.op == MYSQL_JSON_DIFF_REMOVE)
{
diff.value= NULL;
diff.value_length= 0;
}
else
{
avail= (size_t) (diffs_end - ptr);
diff.value_length= (size_t) safe_net_field_length_ll((uchar**) &ptr,
avail);
if ((size_t) (diffs_end - ptr) < diff.value_length)
return true;
diff.value= ptr;
ptr+= diff.value_length;
}
if (out.append(diff))
return true; // out of memory
}
*row_image_pos= diffs_end;
return false;
}
/**
Decide which MariaDB function an INSERT diff corresponds to.
MySQL's Json_diff has a single INSERT operation, but its meaning depends
on what the path points at: sql-common/json_diff.h and WL#2955 section 4,
F3 define it as JSON_ARRAY_INSERT() when the path names an array cell and
JSON_INSERT() when it names an object member. The two differ for a cell
that already exists - JSON_ARRAY_INSERT shifts the remaining elements
right, while JSON_INSERT does nothing at all because the path is already
present - so picking the wrong one corrupts the document silently.
@param path_obj Scratch path parser, its steps array already
initialised; reused across the diffs of one column.
@param path The diff's path text.
@param path_length Its length in bytes.
@param[out] is_array_cell
True if the path's last step selects an array cell.
@param[out] is_from_end
True if that step counts from the end of the array,
i.e. it was written as '[last]' or '[last-N]'. Such a
path is stored with a negative n_item. The caller
refuses these for now; see the note in
apply_partial_json_column().
@retval false Path parsed; both out-parameters are set.
@retval true The path could not be parsed.
*/
static bool json_diff_path_is_array_cell(json_path_t *path_obj,
const char *path,
size_t path_length,
bool *is_array_cell,
bool *is_from_end)
{
if (json_path_setup(path_obj, &my_charset_utf8mb4_bin,
(const uchar *) path,
(const uchar *) path + path_length))
return true;
const json_path_step_t *last_step=
((const json_path_step_t *) path_obj->steps.buffer) +
path_obj->last_step_idx;
*is_array_cell= (last_step->type & JSON_PATH_ARRAY) != 0;
*is_from_end= *is_array_cell && last_step->n_item < 0;
return false;
}
/**
Apply one partial-JSON column of a MySQL PARTIAL_UPDATE_ROWS_EVENT
after-image.
A partial column carries a Json_diff vector rather than a value, so it
cannot go through the normal unpack path: there is nothing to unpack into
the conversion table and copy out with convert_field(). Instead the base
document is taken from the matched row, the diffs are applied to it in
order, and the resulting text is stored into the slave field directly.
The base document is the *before-image*, which lives in record[1] while
record[0] is being built. The field is temporarily shifted there to read
it (move_field_offset()), matching MySQL, which applies diffs to the
field's current value.
Each diff is folded through MariaDB's own JSON functions, so the result
matches what the corresponding SQL function would produce:
REPLACE -> JSON_REPLACE, INSERT -> JSON_INSERT, REMOVE -> JSON_REMOVE
applied one at a time, each result feeding the next. A diff value arrives
as MySQL binary JSON and is decoded to text by storing it into
\a conv_field (a Field_mysql_json) and reading it back; the text is then
wrapped in JSON_COMPACT() so that JSON_INSERT/JSON_REPLACE treat it as a
JSON value rather than as a quoted string.
The Items built for this are created on a private THD::free_list which is
released here, so their internal buffers do not accumulate until the end
of the statement.
@param rgi Relay group info; supplies the THD the Items are built on.
@param table The slave table. Used to locate the before-image, i.e.
the distance between record[0] and record[1].
@param result_field
The slave's JSON (or text) field, in record[0]. Receives
the document produced by applying the diffs, and is set
NOT NULL.
@param conv_field
The conversion-table Field_mysql_json for this column,
used purely as a binary-JSON-to-text decoder for the diff
values. Must not be NULL: a partial JSON column always
has a conversion field, since the source type is never
native to MariaDB.
@param[in,out] state
Unpack state. Its pack_ptr is advanced past the diff
vector that this call consumes, so the caller continues
with the next column. Left unchanged if the vector is
corrupt.
@param row_end One past the last readable byte of the row image, bounding
the diff decoding.
@retval 0 Success; result_field holds the new document.
@retval HA_ERR_CORRUPT_EVENT The diff vector is truncated or malformed.
@retval HA_ERR_GENERIC The before-image was NULL, or building or
evaluating one of the JSON functions failed.
An error has been raised on the THD.
@note Known gaps, tracked in the MDEV-39143 design document:
- MySQL distinguishes a REJECTED diff (the path does not match, so
the value is left alone and replication continues) from a real
error. Here every failure stops the SQL thread.
- The NULL before-image case raises a placeholder ER_UNKNOWN_ERROR.
*/
static int apply_partial_json_column(const rpl_group_info *rgi, TABLE *table,
Field *result_field, Field *conv_field,
Unpack_record_state &state,
const uchar *row_end)
{
Dynamic_array<Mysql_json_diff> diffs((PSI_memory_key) PSI_NOT_INSTRUMENTED);
const uchar *ptr= state.pack_ptr;
if (read_mysql_json_diff_vector(&ptr, row_end, diffs))
return HA_ERR_CORRUPT_EVENT;
state.pack_ptr= ptr;
THD *thd= rgi->thd;
DBUG_ASSERT(conv_field);
/* Read the before-image JSON value (found row is in record[1]). */
my_ptrdiff_t bi_offset= (my_ptrdiff_t) (table->record[1] - table->record[0]);
result_field->move_field_offset(bi_offset);
bool base_is_null= result_field->is_null();
String base_buf;
String *base= base_is_null ? NULL : result_field->val_str(&base_buf);
result_field->move_field_offset(-bi_offset);
if (base_is_null || !base)
{
my_printf_error(ER_UNKNOWN_ERROR,
"MDEV-39143: partial JSON update with no base value (slave out of sync)",
MYF(0));
return HA_ERR_GENERIC;
}
String cur;
if (cur.copy(*base))
return HA_ERR_GENERIC;
/*
Build and evaluate the JSON functions on a private free_list, then release
them (and their internal buffers) here with free_items(), instead of
letting them accumulate until end of statement.
*/
Item *const save_free_list= thd->free_list;
thd->free_list= NULL;
int rc= 0;
/* Reused by every INSERT diff of this column to classify its path. */
json_path_t diff_path;
mem_root_dynamic_array_init(thd->mem_root, PSI_INSTRUMENT_MEM,
&diff_path.steps, sizeof(json_path_step_t),
NULL, JSON_DEPTH_DEFAULT, JSON_DEPTH_INC,
MYF(0));
for (size_t k= 0; k < diffs.elements() && !rc; k++)
{
const Mysql_json_diff &diff= diffs.at(k);
List<Item> args;
Item *doc_it= new (thd->mem_root)
Item_string(thd, cur.ptr(), (uint) cur.length(), cur.charset());
Item *path_it= new (thd->mem_root)
Item_string(thd, diff.path, (uint) diff.path_length,
&my_charset_utf8mb4_bin);
if (!doc_it || !path_it ||
args.push_back(doc_it, thd->mem_root) ||
args.push_back(path_it, thd->mem_root))
{ rc= HA_ERR_GENERIC; break; }
Item *func;
if (diff.op == MYSQL_JSON_DIFF_REMOVE)
func= new (thd->mem_root) Item_func_json_remove(thd, args);
else
{
/*
Point conv_field directly at the diff's raw JSONB bytes, the same way
Field_blob::unpack() hands a normal blob column its wire bytes.
Field::store() must not be used here: Field_mysql_json is built with
a fixed field_charset() of utf8mb4_bin (mysql_json.cc), so store()
would run the bytes through well_formed_copy_with_check() and reject
them the moment a structural byte (e.g. a multi-byte JSONB length,
needed once a string exceeds 127 bytes) is not valid standalone
UTF-8. JSONB is binary data, not text, and must reach val_str()
unmodified; set_ptr() bypasses charset handling entirely, exactly
like the ordinary full-document unpack path already does.
*/
((Field_blob *) conv_field)->set_ptr(diff.value_length,
const_cast<uchar *>(diff.value));
String vbuf, *vtext= conv_field->val_str(&vbuf);
if (!vtext)
{ rc= HA_ERR_GENERIC; break; }
/*
Copy the decoded value onto the statement mem_root (val_str's buffer is
transient) and wrap it in JSON_COMPACT, so JSON_INSERT/REPLACE inserts
it as a JSON value rather than a quoted string.
*/
char *vdup= (char *) thd->memdup(vtext->ptr(), vtext->length());
Item *v_it= vdup ? new (thd->mem_root)
Item_string(thd, vdup, (uint) vtext->length(), vtext->charset())
: NULL;
Item *val_json= v_it ? new (thd->mem_root)
Item_func_json_format(thd, v_it, Item_func_json_format::COMPACT)
: NULL;
if (!val_json || args.push_back(val_json, thd->mem_root))
{ rc= HA_ERR_GENERIC; break; }
if (diff.op == MYSQL_JSON_DIFF_INSERT)
{
/*
An INSERT on an array cell is JSON_ARRAY_INSERT, on an object
member JSON_INSERT. See json_diff_path_is_array_cell().
*/
bool is_array_cell= false, is_from_end= false;
if (json_diff_path_is_array_cell(&diff_path, diff.path,
diff.path_length, &is_array_cell,
&is_from_end))
{
my_printf_error(ER_UNKNOWN_ERROR,
"MDEV-39143: unparsable JSON path in a partial "
"JSON update: '%.*s'", MYF(0),
(int) diff.path_length, diff.path);
rc= HA_ERR_GENERIC;
break;
}
/*
Refuse rather than diverge silently. MySQL emits an array-cell
INSERT only for a cell that does not exist, which for a
from-the-end index means it clamped the position to 0 and
prepended. MariaDB's JSON_ARRAY_INSERT resolves '[last-N]'
differently - it appends when the index goes negative, and is off
by one even in range - so applying the diff here would store a
document that differs from the source with no error raised. Stop
the SQL thread instead; a wrong row that replication reports as
applied is worse than a halt. Lift this once JSON_ARRAY_INSERT
resolves such paths the same way the other JSON functions do.
*/
if (is_array_cell && is_from_end)
{
my_printf_error(ER_UNKNOWN_ERROR,
"MDEV-39143: cannot apply a partial JSON update "
"that inserts at a from-the-end array index: "
"path '%.*s' on %s.%s.%s. MariaDB's "
"JSON_ARRAY_INSERT resolves '[last-N]' "
"differently from MySQL, so the result would "
"silently differ from the source.", MYF(0),
(int) diff.path_length, diff.path,
table->s->db.str, table->s->table_name.str,
result_field->field_name.str);
rc= HA_ERR_GENERIC;
break;
}
func= is_array_cell ?
(Item *) new (thd->mem_root) Item_func_json_array_insert(thd, args)
: (Item *) new (thd->mem_root) Item_func_json_insert(true, false,
thd, args);
}
else
func= new (thd->mem_root) Item_func_json_insert(false, true,
thd, args);
}
if (!func || func->fix_fields(thd, &func))
{ rc= HA_ERR_GENERIC; break; }
String rbuf, *res= func->val_str(&rbuf);
if (!res || func->null_value || cur.copy(*res))
{ rc= HA_ERR_GENERIC; break; }
}
thd->free_items();
thd->free_list= save_free_list;
if (rc)
return rc;
result_field->set_notnull();
if (result_field->store(cur.ptr(), cur.length(), cur.charset()))
return HA_ERR_GENERIC;
return 0;
}
/**
Unpack a row into @c table->record[0].
@@ -307,6 +714,16 @@ static void convert_field(Field *result_field, Field *conv_field)
@param row_end
Pointer to variable that will hold the value of the
end position for the data in the row event
@param is_partial_json_after_image
True when unpacking the after-image of a MySQL
PARTIAL_UPDATE_ROWS_EVENT (binlog_row_value_options=
PARTIAL_JSON on the source). Such an image starts with a
value_options integer and, if that has PARTIAL_JSON_UPDATES
set, a partial_bits bitmap with one bit per JSON column of
the source table; a JSON column whose bit is set carries a
diff vector instead of a full value. Must be false for a
before-image, which never has that header and never carries
diffs, and for every other event type.
@retval 0 No error
@@ -315,11 +732,11 @@ static void convert_field(Field *result_field, Field *conv_field)
@retval HA_ERR_CORRUPT_EVENT
Found error when trying to unpack fields.
*/
int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const master_cols,
uchar const *const row_data, MY_BITMAP const *cols,
uchar const **const current_row_end,
uchar const *const row_end)
uchar const *const row_end,
bool is_partial_json_after_image)
{
int error;
bool null_value;
@@ -328,7 +745,33 @@ int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const master_cols,
DBUG_ASSERT(table);
DBUG_ASSERT(rgi);
Unpack_record_state st(row_data, row_end, (bitmap_bits_set(cols) + 7) / 8);
const uchar *rec_ptr= row_data;
/*
A MySQL PARTIAL_UPDATE_ROWS_EVENT update after-image is prefixed with a
value_options integer and, when it has the PARTIAL_JSON_UPDATES bit set, a
partial_bits bitmap carrying one bit per JSON column in the master table
(see MySQL sql/log_event.cc). Read past this header so the null bits and
column values are unpacked from the correct offset.
*/
const uchar *pj_partial_bits= NULL; // set for a PARTIAL_JSON after-image
uint pj_json_col= 0; // running index over master JSON columns
if (is_partial_json_after_image)
{
const uint PARTIAL_JSON_UPDATES= 1;
const table_def *pj_tabledef=
&((RPL_TABLE_LIST*) table->pos_in_table_list)->m_tabledef;
size_t avail= (size_t) (row_end - rec_ptr);
ulonglong value_options=
safe_net_field_length_ll((uchar**) &rec_ptr, avail);
if (value_options & PARTIAL_JSON_UPDATES)
{
pj_partial_bits= rec_ptr;
rec_ptr+= (pj_tabledef->json_column_count() + 7) / 8;
}
}
Unpack_record_state st(rec_ptr, row_end, (bitmap_bits_set(cols) + 7) / 8);
if (bitmap_is_clear_all(cols))
{
@@ -371,6 +814,19 @@ int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const master_cols,
for (uint master_idx= 0; master_idx < master_cols; master_idx++)
{
Field *field=NULL, *conv_field= NULL;
/*
A partial JSON after-image carries one partial_bits entry per master
JSON column, in column order, regardless of image/null. Advance the
index for every JSON column and remember whether this one is a diff.
*/
bool is_partial_col= false;
if (pj_partial_bits &&
tabledef->binlog_type(master_idx) == MYSQL_TYPE_JSON_MYSQL)
{
is_partial_col= (pj_partial_bits[pj_json_col / 8] &
(1 << (pj_json_col % 8))) != 0;
pj_json_col++;
}
/*
Check 1: Skip unpacking if the field wasn't written in this record.
This can happen for update row events when the before_image and
@@ -442,6 +898,15 @@ int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const master_cols,
result_field->field_name.str, master_idx));
DBUG_ASSERT(field != NULL);
if (is_partial_col)
{
int pj_err= apply_partial_json_column(rgi, table, result_field,
conv_field, st, row_end);
if (pj_err)
DBUG_RETURN(pj_err);
}
else
{
bool unpack_result= st.unpack_field(tabledef, field, master_idx);
if (!unpack_result)
{
@@ -464,6 +929,7 @@ int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const master_cols,
*/
if (conv_field)
convert_field(result_field, conv_field);
}
}
}
else

View File

@@ -33,7 +33,8 @@ int unpack_row(const rpl_group_info *rgi,
TABLE *table, uint const colcnt,
uchar const *const row_data, MY_BITMAP const *cols,
uchar const **const curr_row_end,
uchar const *const row_end);
uchar const *const row_end,
bool is_partial_json_after_image= false);
// Fill table's record[0] with default values.
int prepare_record(TABLE *const table);

View File

@@ -32,7 +32,8 @@ uint32 table_def::calc_field_size(uint col, uchar *master_data) const
{
uint32 length= 0;
switch (type(col)) {
// (uint) so MYSQL_TYPE_JSON_MYSQL, not an enum_field_types, is a legal case
switch ((uint) type(col)) {
case MYSQL_TYPE_NEWDECIMAL:
length= my_decimal_get_binary_size(m_field_metadata[col] >> 8,
m_field_metadata[col] & 0xff);
@@ -137,6 +138,7 @@ uint32 table_def::calc_field_size(uint col, uchar *master_data) const
case MYSQL_TYPE_BLOB:
case MYSQL_TYPE_BLOB_COMPRESSED:
case MYSQL_TYPE_GEOMETRY:
case MYSQL_TYPE_JSON_MYSQL: // a blob value with a pack-length prefix
{
/*
Compute the length of the data. We cannot use get_length() here
@@ -222,7 +224,8 @@ table_def::table_def(unsigned char *types, ulong size,
int index= 0;
for (unsigned int i= 0; i < m_size; i++)
{
switch (binlog_type(i)) {
// (uint): see the cast in calc_field_size() above
switch ((uint) binlog_type(i)) {
case MYSQL_TYPE_TINY_BLOB:
case MYSQL_TYPE_BLOB:
case MYSQL_TYPE_BLOB_COMPRESSED:
@@ -278,6 +281,9 @@ table_def::table_def(unsigned char *types, ulong size,
case MYSQL_TYPE_TIMESTAMP2:
m_field_metadata[i]= field_metadata[index++];
break;
case MYSQL_TYPE_JSON_MYSQL: // one metadata byte (the pack length)
m_field_metadata[i]= field_metadata[index++];
break;
default:
m_field_metadata[i]= 0;
break;

View File

@@ -28,6 +28,22 @@
#endif
#include "mysql_com.h"
/*
MySQL 5.7+ binary JSON column type code, as it appears in .frm files and in
row events replicated from a MySQL master. MariaDB has no native type for
it - the read-only MYSQL_JSON plugin decodes it.
Defined here rather than in mysql_com.h because that header cannot carry it:
the server's include/mysql_com.h and Connector/C's
libmariadb/include/mariadb_com.h share the include guard _mysql_com_h, so in
a client build (mysqlbinlog compiles rpl_utility.cc and log_event.cc as
source) only the latter is seen. Note that mariadb_com.h does declare
MYSQL_TYPE_JSON=245, but the server's enum_field_types does not, and 245 is
MYSQL_TYPE_VIRTUAL in MariaDB's own pre-10.2 .frm encoding - which of the
two a 245 byte means is decided by the context it was read from.
*/
#define MYSQL_TYPE_JSON_MYSQL 245
class Relay_log_info;
class Log_event;
class Rows_log_event;
@@ -163,6 +179,21 @@ public:
return 0;
}
/*
Number of MySQL binary JSON columns in the master table.
Used to size the partial_bits bitmap in the after-image of a MySQL
PARTIAL_UPDATE_ROWS_EVENT, which carries one bit per JSON column in the
table regardless of whether it is present in the row image.
*/
uint json_column_count() const
{
uint n= 0;
for (uint i= 0; i < m_size; i++)
if (binlog_type(i) == MYSQL_TYPE_JSON_MYSQL)
n++;
return n;
}
/*
This function returns whether the field on the master can be null.
This value is derived from field->maybe_null().

View File

@@ -533,6 +533,21 @@ Field_longstr::rpl_conv_type_from(const Conv_source &source,
const Relay_log_info *rli,
const Conv_param &param) const
{
/*
A MySQL binary JSON source (mysql_json plugin), replicated from a MySQL
master. The conversion-table field decodes the binary JSON into text.
- target native JSON: same logical type, only the storage format differs,
so this is lossless -> CONV_TYPE_VARIANT (allowed without
slave_type_conversions).
- target plain string/blob: a genuine cross-type conversion ->
CONV_TYPE_SUBSET_TO_SUPERSET (requires slave_type_conversions).
*/
{
const LEX_CSTRING mysql_json{STRING_WITH_LEN("MYSQL_JSON")};
if (source.type_handler()->name().eq(mysql_json))
return Type_handler_json_common::is_json_type_handler(type_handler()) ?
CONV_TYPE_VARIANT : CONV_TYPE_SUBSET_TO_SUPERSET;
}
/**
@todo
Implement Field_varstring_compressed::real_type() and
@@ -891,6 +906,23 @@ const Type_handler *table_def::field_type_handler(uint col) const
*/
if (typecode == MYSQL_TYPE_DATE)
return &type_handler_newdate;
/*
MySQL 5.7+ binary JSON is not a native MariaDB type.
It is handled by the read-only mysql_json plugin - the same handler used
when opening a MySQL table on disk (see open_binary_frm() in table.cc) -
which lets a MariaDB slave decode JSON columns in row events replicated
from a MySQL master. handler_by_name() returns NULL if the plugin is not
installed; give_compatibility_error() turns that into a message naming
the plugin. Note it must not be handler_by_name_or_error() here: this
getter is const, is called speculatively, and is called from inside
give_compatibility_error() itself, so it must not raise an error of its
own.
*/
if (typecode == MYSQL_TYPE_JSON_MYSQL)
{
const LEX_CSTRING mysql_json{STRING_WITH_LEN("MYSQL_JSON")};
return Type_handler::handler_by_name(current_thd, mysql_json);
}
return Type_handler::get_handler_by_real_type(typecode);
}
@@ -1093,11 +1125,27 @@ bool RPL_TABLE_LIST::give_compatibility_error(rpl_group_info *rgi, uint col)
case SLAVE_FIELD_UNKNOWN_TYPE:
{
Field *field= table->field[m_tabledef.master_to_slave_map[col]];
my_snprintf(error_msg, sizeof(error_msg),
"In RBR mode, Slave received unknown field type field %d "
"for column Name: %s.%s.%s",
m_tabledef.binlog_type(col), field->table->s->db.str,
field->table->s->table_name.str, field->field_name.str);
/*
field_type_handler() returns NULL for MYSQL_TYPE_JSON_MYSQL only when
the mysql_json plugin is absent, so the type code tells the two cases
apart: a genuinely unknown type, or a known one we cannot decode
because its handler is not loaded. Name the plugin in the latter case
- "unknown field type 245" gives the DBA nothing to act on.
*/
if (m_tabledef.binlog_type(col) == MYSQL_TYPE_JSON_MYSQL)
my_snprintf(error_msg, sizeof(error_msg),
"In RBR mode, Slave received a MySQL binary JSON column "
"(type %d) for column Name: %s.%s.%s, but the MYSQL_JSON "
"plugin needed to decode it is not installed. Run "
"INSTALL SONAME 'type_mysql_json' on the slave.",
m_tabledef.binlog_type(col), field->table->s->db.str,
field->table->s->table_name.str, field->field_name.str);
else
my_snprintf(error_msg, sizeof(error_msg),
"In RBR mode, Slave received unknown field type field %d "
"for column Name: %s.%s.%s",
m_tabledef.binlog_type(col), field->table->s->db.str,
field->table->s->table_name.str, field->field_name.str);
rgi->rli->report(
ERROR_LEVEL, ER_SLAVE_INCOMPATIBLE_TABLE_DEF, rgi->gtid_info(),
ER_THD(rgi->thd, ER_SLAVE_INCOMPATIBLE_TABLE_DEF), error_msg);

View File

@@ -50,6 +50,7 @@
#include "sql_delete.h" // class Sql_cmd_delete
#include "rpl_rli.h" // class rpl_group_info
#include "rpl_mi.h" // class Master_info
#include "rpl_utility.h" // MYSQL_TYPE_JSON_MYSQL
#include "vector_mhnsw.h"
#include "opt_group_by_cardinality.h"
@@ -2589,8 +2590,8 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
enum_field_types field_type= (enum_field_types) strpos[13];
if (!(handler= Type_handler::get_handler_by_real_type(field_type)))
{
if (field_type == 245 &&
share->mysql_version >= 50700) // a.k.a MySQL 5.7 JSON
if (field_type == MYSQL_TYPE_JSON_MYSQL &&
share->mysql_version >= 50700)
{
share->incompatible_version|= HA_CREATE_USED_ENGINE;
const LEX_CSTRING mysql_json{STRING_WITH_LEN("MYSQL_JSON")};