1
0
mirror of https://github.com/MariaDB/server.git synced 2026-10-23 18:49:29 +03:00

MDEV-40383:innodb_gis.point_basic fails on replay

There are 2 problems: -
1. The REPLACE statement that is recorded doesn't store the
   value of geometry type field correctly.
2. The table definition that got recorded has fields with non-null constraint,
   and no default value is specified.
   Also, the "REPLACE INTO" statement that gets stored in the context,
   doesn't have any value specified for these non-null fields.

Solution is to: -
1. When using REPLACE INTO statement, store all the non-numeric values in HEX,
   whenever conversion from field's charset to output's charset is lossy.
2. Instead of storing only the column values that were projected in the
   query, store all the non-virtual column values into the recorded
   REPLACE INTO statement.

Implementation details: -
1. Introduce a new method is_charset_conversion_lossless() in filesort.cc,
   to check if the output charset to which field's data is being written to,
   results in a lossless conversion. If so, non-numeric values being witten
   using REPLACE INTO statement are stored in string representation,
   else they are converted to HEX.
2. Modify join_read_const(), and join_read_system() methods in sql_select.cc,
   and opt_sum_query() method in opt_sum.cc the following way: -
     a. Extend the read_set to make sure, we read all the non-virtual column
        using Optimizer_context_recorder::prepare_captured_row_read().
        This method also saves the original read_set.
     b. Read the row.
     c. Dump the row into the context when no error is noticed while
        reading. Irrespective of the error, restore back the read_set state to
        the original using Optimizer_context_recorder::finish_captured_row_read()
This commit is contained in:
bsrikanth-mariadb
2026-07-23 14:30:42 +05:30
committed by Sergei Petrunia
parent 8f44d4ceb7
commit a873569a47
9 changed files with 407 additions and 30 deletions

View File

@@ -568,5 +568,106 @@ nv
11
set optimizer_replay_context='';
drop table s1;
#
# MDEV-40383: innodb_gis.point_basic fails on replay
#
CREATE TABLE t1 (
a INT NOT NULL,
p POINT NOT NULL,
l LINESTRING NOT NULL,
g GEOMETRY NOT NULL,
PRIMARY KEY(p),
SPATIAL KEY `idx2` (p),
SPATIAL KEY `idx3` (l),
SPATIAL KEY `idx4` (g)
);
INSERT INTO t1 VALUES(
1, ST_GeomFromText('POINT(10 10)'),
ST_GeomFromText('LINESTRING(1 1, 5 5, 10 10)'),
ST_GeomFromText('POLYGON((30 30, 40 40, 50 50, 30 50, 30 40, 30 30))'));
INSERT INTO t1 VALUES(
2, ST_GeomFromText('POINT(20 20)'),
ST_GeomFromText('LINESTRING(2 3, 7 8, 9 10, 15 16)'),
ST_GeomFromText('POLYGON((10 30, 30 40, 40 50, 40 30, 30 20, 10 30))'));
set optimizer_record_context=1;
EXPLAIN SELECT a, ST_AsText(p) FROM t1 WHERE a = 2 AND p = ST_GeomFromText('POINT(20 20)');
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE t1 const PRIMARY,idx2 PRIMARY 27 const 1
select context into dumpfile "../../tmp/dump1.sql"
from information_schema.optimizer_context;
set optimizer_record_context=0;
drop table t1;
set optimizer_replay_context='opt_context';
# Same query as above, must have same explain:
EXPLAIN SELECT a, ST_AsText(p) FROM t1 WHERE a = 2 AND p = ST_GeomFromText('POINT(20 20)');
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE t1 const PRIMARY,idx2 PRIMARY 27 const 1
set optimizer_replay_context='';
SELECT a, ST_AsText(p), ST_AsText(l), ST_AsText(g) FROM t1;
a ST_AsText(p) ST_AsText(l) ST_AsText(g)
2 POINT(20 20) LINESTRING(2 3,7 8,9 10,15 16) POLYGON((10 30,30 40,40 50,40 30,30 20,10 30))
#
# MIN/MAX recording with geometry fields in the table
#
INSERT INTO t1 VALUES(
1, ST_GeomFromText('POINT(10 10)'),
ST_GeomFromText('LINESTRING(1 1, 5 5, 10 10)'),
ST_GeomFromText('POLYGON((30 30, 40 40, 50 50, 30 50, 30 40, 30 30))'));
alter table t1 add index(a);
select a from t1;
a
1
2
SELECT MIN(a) FROM t1;
MIN(a)
1
set optimizer_record_context=1;
EXPLAIN SELECT MIN(a) FROM t1;
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away
select context into dumpfile "../../tmp/dump1.sql"
from information_schema.optimizer_context;
set optimizer_record_context=0;
drop table t1;
set optimizer_replay_context='opt_context';
# Same query as above, must have same explain:
EXPLAIN SELECT MIN(a) FROM t1;
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away
set optimizer_replay_context='';
SELECT a, ST_AsText(p), ST_AsText(l), ST_AsText(g) FROM t1;
a ST_AsText(p) ST_AsText(l) ST_AsText(g)
1 POINT(10 10) LINESTRING(1 1,5 5,10 10) POLYGON((30 30,40 40,50 50,30 50,30 40,30 30))
drop table t1;
#
# MIN/MAX recording with a virtual column present.
#
CREATE TABLE t1 (
a INT NOT NULL,
b INT NOT NULL,
v INT AS (a + 100) VIRTUAL,
KEY(a)
) ENGINE=MyISAM;
INSERT INTO t1 (a,b) VALUES (1,10),(2,20),(3,30),(1,40);
set optimizer_record_context=1;
EXPLAIN SELECT MIN(a) FROM t1;
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away
select context into dumpfile "../../tmp/dump1.sql"
from information_schema.optimizer_context;
set optimizer_record_context=0;
drop table t1;
set optimizer_replay_context='opt_context';
# Same query as above, must have same explain:
EXPLAIN SELECT MIN(a) FROM t1;
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away
set optimizer_replay_context='';
# MIN(a) row is (1,10); the non-indexed NOT NULL column b must be
# captured (not defaulted to 0), and v must be recomputed as a+100=101:
SELECT a, b, v FROM t1;
a b v
1 10 101
drop table t1;
# End of 13.1 tests
drop database db1;

View File

@@ -376,6 +376,119 @@ select lastval(s1) as nv;
set optimizer_replay_context='';
--remove_file "$MYSQLTEST_VARDIR/tmp/dump1.sql"
drop table s1;
--echo #
--echo # MDEV-40383: innodb_gis.point_basic fails on replay
--echo #
CREATE TABLE t1 (
a INT NOT NULL,
p POINT NOT NULL,
l LINESTRING NOT NULL,
g GEOMETRY NOT NULL,
PRIMARY KEY(p),
SPATIAL KEY `idx2` (p),
SPATIAL KEY `idx3` (l),
SPATIAL KEY `idx4` (g)
);
INSERT INTO t1 VALUES(
1, ST_GeomFromText('POINT(10 10)'),
ST_GeomFromText('LINESTRING(1 1, 5 5, 10 10)'),
ST_GeomFromText('POLYGON((30 30, 40 40, 50 50, 30 50, 30 40, 30 30))'));
INSERT INTO t1 VALUES(
2, ST_GeomFromText('POINT(20 20)'),
ST_GeomFromText('LINESTRING(2 3, 7 8, 9 10, 15 16)'),
ST_GeomFromText('POLYGON((10 30, 30 40, 40 50, 40 30, 30 20, 10 30))'));
set optimizer_record_context=1;
EXPLAIN SELECT a, ST_AsText(p) FROM t1 WHERE a = 2 AND p = ST_GeomFromText('POINT(20 20)');
select context into dumpfile "../../tmp/dump1.sql"
from information_schema.optimizer_context;
set optimizer_record_context=0;
drop table t1;
--disable_query_log
--disable_result_log
--source "$MYSQLTEST_VARDIR/tmp/dump1.sql"
--enable_query_log
--enable_result_log
set optimizer_replay_context='opt_context';
--echo # Same query as above, must have same explain:
EXPLAIN SELECT a, ST_AsText(p) FROM t1 WHERE a = 2 AND p = ST_GeomFromText('POINT(20 20)');
set optimizer_replay_context='';
--remove_file "$MYSQLTEST_VARDIR/tmp/dump1.sql"
SELECT a, ST_AsText(p), ST_AsText(l), ST_AsText(g) FROM t1;
--echo #
--echo # MIN/MAX recording with geometry fields in the table
--echo #
INSERT INTO t1 VALUES(
1, ST_GeomFromText('POINT(10 10)'),
ST_GeomFromText('LINESTRING(1 1, 5 5, 10 10)'),
ST_GeomFromText('POLYGON((30 30, 40 40, 50 50, 30 50, 30 40, 30 30))'));
alter table t1 add index(a);
select a from t1;
SELECT MIN(a) FROM t1;
set optimizer_record_context=1;
EXPLAIN SELECT MIN(a) FROM t1;
select context into dumpfile "../../tmp/dump1.sql"
from information_schema.optimizer_context;
set optimizer_record_context=0;
drop table t1;
--disable_query_log
--disable_result_log
--source "$MYSQLTEST_VARDIR/tmp/dump1.sql"
--enable_query_log
--enable_result_log
set optimizer_replay_context='opt_context';
--echo # Same query as above, must have same explain:
EXPLAIN SELECT MIN(a) FROM t1;
set optimizer_replay_context='';
--remove_file "$MYSQLTEST_VARDIR/tmp/dump1.sql"
SELECT a, ST_AsText(p), ST_AsText(l), ST_AsText(g) FROM t1;
drop table t1;
--echo #
--echo # MIN/MAX recording with a virtual column present.
--echo #
CREATE TABLE t1 (
a INT NOT NULL,
b INT NOT NULL,
v INT AS (a + 100) VIRTUAL,
KEY(a)
) ENGINE=MyISAM;
INSERT INTO t1 (a,b) VALUES (1,10),(2,20),(3,30),(1,40);
set optimizer_record_context=1;
EXPLAIN SELECT MIN(a) FROM t1;
select context into dumpfile "../../tmp/dump1.sql"
from information_schema.optimizer_context;
set optimizer_record_context=0;
drop table t1;
--disable_query_log
--disable_result_log
--source "$MYSQLTEST_VARDIR/tmp/dump1.sql"
--enable_query_log
--enable_result_log
set optimizer_replay_context='opt_context';
--echo # Same query as above, must have same explain:
EXPLAIN SELECT MIN(a) FROM t1;
set optimizer_replay_context='';
--remove_file "$MYSQLTEST_VARDIR/tmp/dump1.sql"
--echo # MIN(a) row is (1,10); the non-indexed NOT NULL column b must be
--echo # captured (not defaulted to 0), and v must be recomputed as a+100=101:
SELECT a, b, v FROM t1;
drop table t1;
--echo # End of 13.1 tests
drop database db1;

View File

@@ -3024,6 +3024,7 @@ static uint make_packed_sortkey(Sort_param *param, uchar *to)
return length;
}
/*
@brief
Format the row record and store it in the output
@@ -3148,12 +3149,26 @@ void format_and_store_row(TABLE *table, const uchar *rec, bool print_names,
}
field->val_str(&tmp);
}
if (require_quote)
output.append('\'');
output.append_for_single_quote_opt_convert(tmp.ptr(), tmp.length(),
field->charset());
if (require_quote)
output.append('\'');
/*
Emit non-empty values as a hex literal whenever converting field's
charset to the output charset conversion is lossy; otherwise emit the
charset-converted value, quoted only when the type requires it.
*/
if (require_quote && tmp.length() &&
!String::is_charset_conversion_lossless(tmp.charset(),
output.charset()))
{
output.append(STRING_WITH_LEN("0x"));
output.append_hex(tmp.ptr(), tmp.length());
}
else
{
if (require_quote)
output.append('\'');
output.append_for_single_quote_opt_convert(tmp);
if (require_quote)
output.append('\'');
}
}
}
output.append(')');

View File

@@ -1121,7 +1121,7 @@ void Optimizer_context_recorder::record_records_in_range(
String min_key;
String max_key;
print_key_value(&min_key, key_part, min_range->key, min_range->length);
print_key_value(&max_key, key_part, min_range->key, min_range->length);
print_key_value(&max_key, key_part, max_range->key, max_range->length);
if (!(rec_in_range_ctx->min_key= strdup_root(mem_root, &min_key)))
return; // OOM
@@ -1144,10 +1144,13 @@ void Optimizer_context_recorder::record_table_row(TABLE *tbl, int row_index)
StringBuffer<512> output(&my_charset_utf8mb4_bin);
/*
The table could have fields that do not have a default value
but are not in the table->read_set.
The record doesn't have values for those.
Use a relaxed sql_mode setting so that REPLACE INTO doesn't fail.
Due to use of prepare_captured_row_read(), we have values for all
table columns.
However, the row that we're trying to dump might have been inserted into
the table with relaxed settings (no strict mode).
So, use relaxed @@sql_mode setting here also to make the REPLACE statement
is processed (i.e. not fails with an error).
*/
output.append(
STRING_WITH_LEN("SET STATEMENT sql_mode="
@@ -1953,7 +1956,7 @@ bool Optimizer_context_replay::infuse_records_in_range(
String max_key;
String tbl_name;
print_key_value(&min_key, key_part, min_range->key, min_range->length);
print_key_value(&max_key, key_part, min_range->key, min_range->length);
print_key_value(&max_key, key_part, max_range->key, max_range->length);
append_base_table_name(tbl, &tbl_name);
if (table_context_for_replay *tbl_ctx=
@@ -2371,3 +2374,80 @@ void clean_captured_ctx(THD *thd)
delete thd->captured_opt_ctx;
thd->captured_opt_ctx= nullptr;
}
/*
Point table->read_set at a private bitmap (table->tmp_set) covering every
stored column, so that a subsequent read/record captures the full row.
Virtual columns are excluded: they cannot be assigned in REPLACE INTO and are
recomputed on read. We copy s->all_set into the per-table tmp_set rather than
aliasing s->all_set directly -- s->all_set is shared across the whole
TABLE_SHARE and must never be mutated.
Precondition: table->tmp_set must stay free for the caller's use until
read_set is restored. This holds on the const-row and MIN/MAX read paths
precisely because we clear the virtual-column bits: with those bits unset,
TABLE::update_virtual_fields(VCOL_UPDATE_FOR_READ) skips them during the read
and so never reuses tmp_set as its own scratch. A caller on a path that
evaluates virtual columns into tmp_set would corrupt the widened read_set.
@return the previous read_set, which the caller must restore (via
column_bitmaps_set) once the row has been read and recorded.
*/
MY_BITMAP *widen_read_set_no_vcols(TABLE *table)
{
MY_BITMAP *saved_read_set= table->read_set;
/*
We are about to repurpose table->tmp_set as the widened read_set, so it
must not already be in use (e.g. as the current read_set). If this fires,
the caller is on a path that violates the tmp_set precondition documented
above.
*/
DBUG_ASSERT(saved_read_set != &table->tmp_set);
bitmap_copy(&table->tmp_set, &table->s->all_set);
for (Field **pfield= table->field; *pfield; pfield++)
{
/* virtual columns need not be stored. */
if ((*pfield)->vcol_info)
bitmap_clear_bit(&table->tmp_set, (*pfield)->field_index);
}
table->column_bitmaps_set(&table->tmp_set, table->write_set);
return saved_read_set;
}
/*
@brief
Prepare a TABLE to read a row which will be captured into the optimizer
context.
@detail
The query has set up table->read_set to only include columns of interest.
However, we need to read all non-virtual columns to produce a valid INSERT
statement. This may require disabling 'index-only' read.
@param table IN Table we're processing
@param state OUT Save the state here.
*/
void Optimizer_context_recorder::prepare_captured_row_read(TABLE *table,
Opt_ctx_recorder_state *state)
{
state->table= table;
state->keyread_state= table->file->ha_end_active_keyread();
state->saved_read_set= widen_read_set_no_vcols(table);
}
/*
@brief
Restore the table state previously saved by prepare_captured_row_read.
*/
void Optimizer_context_recorder::finish_captured_row_read(Opt_ctx_recorder_state *state)
{
if (state->saved_read_set)
state->table->column_bitmaps_set(state->saved_read_set, state->table->write_set);
state->table->file->ha_restart_keyread(state->keyread_state);
}

View File

@@ -39,6 +39,15 @@ class Multi_range_read_const_call_record;
void init_optimizer_context_recorder_if_needed(THD *thd,
const TABLE_LIST *query_tables);
class Opt_ctx_recorder_state
{
public:
TABLE *table= nullptr;
int keyread_state= 0;
MY_BITMAP *saved_read_set= nullptr;
};
/*
Recorder is used to capture the environment during query optimization run.
When the optimization is finished, one can save the captured context
@@ -51,6 +60,9 @@ public:
~Optimizer_context_recorder();
void prepare_captured_row_read(TABLE *table, Opt_ctx_recorder_state *state);
void finish_captured_row_read(Opt_ctx_recorder_state *state);
void record_multi_range_read_info_const(const TABLE *table, uint keynr,
Range_print_enumerator *ranges,
ha_rows rows,
@@ -66,11 +78,6 @@ public:
const KEY_PART_INFO *key_part, uint keynr,
const key_range *min_range,
const key_range *max_range, ha_rows records);
void record_const_table_row(TABLE *tbl)
{
/* use table->record[1] */
record_table_row(tbl, 1);
}
void record_current_table_row(TABLE *tbl)
{
/* use table->record[0] */
@@ -140,6 +147,13 @@ int fill_optimizer_context_capture_info(THD *thd, TABLE_LIST *tables, Item *);
void clean_captured_ctx(THD *thd);
/*
Widen read_set to all stored (non-virtual) columns via table->tmp_set, so the
next read/record captures the full row; returns the previous read_set for the
caller to restore. See the definition for the tmp_set precondition.
*/
MY_BITMAP *widen_read_set_no_vcols(TABLE *table);
/***************************************************************************
* Part 3: APIs for loading previously saved Optimizer Context and replaying
* it: making the optimizer work as if the environment was like it has been

View File

@@ -428,6 +428,12 @@ int opt_sum_query(THD *thd,
error= 0;
table->file->info_push(INFO_KIND_FORCE_LIMIT_BEGIN, &info_limit);
/* Prepare to capture the MIN/MAX row for the optimizer context */
Opt_ctx_recorder_state state;
if (thd->opt_ctx_recorder)
thd->opt_ctx_recorder->prepare_captured_row_read(table, &state);
if (!table->const_table)
{
if (likely(!(error= table->file->ha_index_init((uint) ref.key,
@@ -443,10 +449,12 @@ int opt_sum_query(THD *thd,
conds, range_fl, prefix_len))
error= HA_ERR_KEY_NOT_FOUND;
if (!error)
/* Capture the MIN/MAX row for the optimizer context */
if (Optimizer_context_recorder *rec= thd->opt_ctx_recorder)
{
if (Optimizer_context_recorder *rec= thd->opt_ctx_recorder)
if (!error)
rec->record_current_table_row(table);
rec->finish_captured_row_read(&state);
}
if (!table->const_table)
{

View File

@@ -46,6 +46,7 @@
#include "filesort_utils.h" // get_qsort_sort_cost
#include "sql_union.h" // mysql_union
#include "opt_subselect.h"
#include "opt_context_store_replay.h"
#include "sql_derived.h"
#include "sql_statistics.h"
#include "sql_cte.h"
@@ -25269,7 +25270,6 @@ join_read_const_table(THD *thd, JOIN_TAB *tab, POSITION *pos)
DBUG_RETURN(0);
}
/**
Read a constant table when there is at most one matching row, using a table
scan.
@@ -25287,9 +25287,23 @@ join_read_system(JOIN_TAB *tab)
int error;
if (table->status & STATUS_GARBAGE) // If first read
{
if (unlikely((error=
table->file->ha_read_first_row(table->record[0],
table->s->primary_key))))
/* Prepare to capture the constant row for the Optimizer Context */
Opt_ctx_recorder_state state;
if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder)
rec->prepare_captured_row_read(table, &state);
error= table->file->ha_read_first_row(table->record[0],
table->s->primary_key);
/* Capture the constant row for the Optimizer Context. */
if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder)
{
if (!error)
rec->record_current_table_row(table);
rec->finish_captured_row_read(&state);
}
if (unlikely(error))
{
if (error != HA_ERR_END_OF_FILE)
return report_error(table, error);
@@ -25298,9 +25312,8 @@ join_read_system(JOIN_TAB *tab)
empty_record(table); // Make empty record
return -1;
}
store_record(table,record[1]);
if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder)
rec->record_const_table_row(table);
store_record(table, record[1]); // cache the row the optimizer used
}
else if (!table->status) // Only happens with left join
restore_record(table,record[1]); // restore old record
@@ -25339,11 +25352,25 @@ join_read_const(JOIN_TAB *tab)
/* This is probably needed for analyze table */
tab->index= tab->ref.key;
}
/* Prepare to capture the constant row for the Optimizer Context */
Opt_ctx_recorder_state state;
if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder)
rec->prepare_captured_row_read(table, &state);
error= file->
ha_index_read_idx_map(table->record[0],tab->ref.key,
(uchar*) tab->ref.key_buff,
make_prev_keypart_map(tab->ref.key_parts),
HA_READ_KEY_EXACT);
/* Capture the constant row for the Optimizer Context */
if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder)
{
if (!error)
rec->record_current_table_row(table);
rec->finish_captured_row_read(&state);
}
file->ha_end_keyread();
}
if (unlikely(error))
@@ -25355,10 +25382,7 @@ join_read_const(JOIN_TAB *tab)
return report_error(table, error);
return -1;
}
store_record(table,record[1]);
if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder)
rec->record_const_table_row(table);
store_record(table, record[1]); // cache the row the optimizer used
}
else if (!(table->status & ~STATUS_NULL_ROW)) // Only happens with left join
{

View File

@@ -372,6 +372,26 @@ bool String::needs_conversion_on_storage(size_t arg_length,
}
bool String::is_charset_conversion_lossless(const CHARSET_INFO *from_cs,
const CHARSET_INFO *to_cs)
{
if (to_cs == &my_charset_bin)
return true; // binary swallows any bytes
if (from_cs == &my_charset_bin)
return false; // arbitrary bytes ⊄ text charset
if (my_charset_same(from_cs, to_cs))
return true; // same repertoire family
if (from_cs->state & MY_CS_PUREASCII) // ASCII-only source...
return my_charset_is_ascii_based(to_cs); // ...into any ASCII-based target
if ((to_cs->state &
MY_CS_UNICODE) && // Unicode target covers every repertoire,
(to_cs->state &
MY_CS_UNICODE_SUPPLEMENT)) // incl. non-BMP (so utf8mb4/utf16/utf32,
return true; // but NOT plain utf8mb3/ucs2)
return false; // unknown → treat as possibly lossy
}
/*
Copy a multi-byte character sets with adding leading zeros.

View File

@@ -945,6 +945,8 @@ public:
static bool needs_conversion_on_storage(size_t arg_length,
CHARSET_INFO *cs_from,
CHARSET_INFO *cs_to);
static bool is_charset_conversion_lossless(const CHARSET_INFO *from_cs,
const CHARSET_INFO *to_cs);
bool copy_aligned(const char *s, size_t arg_length, size_t offset,
CHARSET_INFO *cs);
bool set_or_copy_aligned(const char *s, size_t arg_length, CHARSET_INFO *cs);