database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2026-10-23 18:49:29 +03:00
Code Activity

MDEV-40920 Say whether a value cut in a group reached the answer

Browse Source 
A TEXT value longer than `group_concat_max_len` is cut on its way into
`blob_storage`, in `Field_blob::handle_group_concat()`. That happens
while the group is being built, not when the answer is put together, so
whether the answer is any shorter for it depends on whether the row
carrying the value reaches the answer at all. Every such cut was
reported as `ER_CUT_VALUE_GROUP_CONCAT`, which says that the answer lost
something the user asked for, and that is true of only some of them.

`Blob_mem_storage` now writes one byte in front of every value it
stores and returns the pointer past it, so `was_cut()` answers for any
value a reader holds a pointer to. `Field_blob::store()` sends every
blob of a table that has a `Blob_mem_storage` through
`handle_group_concat()`, so no value in that storage is without the
byte, and `Field_blob::get_ptr()` on the record of a row hands back
exactly the pointer that was stored.

`dump_leaf_key()` reads the mark off each row it appends and sets
`value_cut_in_result`. `val_str()` then reports:

1. **A warning**, `ER_CUT_VALUE_GROUP_CONCAT`, when a row that reached
   the answer carried a cut value. The answer is short by what was cut.
   The result being cut at `gconcat_max_len()` already gives that same
   warning, and a group that hits both is told once, not twice.
2. **A note**, `ER_CUT_VALUES_WHILE_PROCESSING`, when a value was cut
   but no row carrying one reached the answer. The answer may well be
   what a larger limit would have given. One note per aggregate is
   enough for a statement however many groups had a value cut, and
   `cleanup()` clears the mark so a statement run again gets its own.

Reporting the loss as a warning keeps a strict `sql_mode` aborting on
it, which it does because `THD::raise_condition()` promotes a warning
and never promotes a note.

`ST_COLLECT` is not affected. It reports `ER_CUT_VALUE_GROUP_CONCAT`
itself, against `group_collect_max_len`.

`main.gconcat_cut_note` covers the split with one group holding a short
value and a long one, where a `LIMIT` alone decides which of them the
answer is built from, over both the sort tree and the duplicate filter.
`main.func_gconcat` shows the granularity: of five groups at
`group_concat_max_len=499999`, the one holding exactly 499999 bytes is
the one that does not warn.

Note that `blob_storage` only exists when the aggregate has an
`ORDER BY` or a `DISTINCT` and a blob field, so this is the only shape
in which a value is cut this way.
This commit is contained in:
Arcadiy Ivanov
2026-09-02 15:46:47 -04:00
committed by Monty
parent 4346702fdb
commit ffc817f040
7 changed files with 308 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
mysql-test/main/gconcat_cut_note.result Normal file
View File

@@ -0,0 +1,105 @@
CREATE TABLE t1 (grp INT, t TEXT);
INSERT INTO t1 SELECT seq, REPEAT(CHAR(64 + seq), 200) FROM seq_1_to_3;
SET SESSION group_concat_max_len= 100;
#
# One row per group, so the result of each group is exactly the cut
# value. Every answer is short by what was cut off its only row,
# so every group warns.
#
SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
grp len
1 100
2 100
3 100
Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 3 was cut by group_concat()
#
# Two aggregates in one statement, each warning for each of its
# groups. The brackets JSON_ARRAYAGG() puts around the value push
# its result past the limit, so it is cut a second time.
#
SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS gc_len,
LENGTH(JSON_ARRAYAGG(t ORDER BY t)) AS ja_len FROM t1 GROUP BY grp;
grp gc_len ja_len
1 100 102
2 100 102
3 100 102
Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 1 was cut by json_arrayagg()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 2 was cut by json_arrayagg()
Warning 1260 Row 3 was cut by group_concat()
Warning 1260 Row 3 was cut by json_arrayagg()
#
# The same statement run again reports for itself.
#
SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
grp len
1 100
2 100
3 100
Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 3 was cut by group_concat()
#
# All three rows in one group, so the result is cut on top of the
# values. Both are the same loss to the user and both are already
# a warning, so it is given once.
#
SELECT LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1;
len
100
Warnings:
Warning 1260 Row 2 was cut by group_concat()
#
# Nothing is cut, so nothing is said.
#
SET SESSION group_concat_max_len= DEFAULT;
SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
grp len
1 200
2 200
3 200
DROP TABLE t1;
#
# Which of the two is given depends on the row, not on the group.
# One group holds a short value and a long one, and a LIMIT decides
# which of them the answer is built from.
#
CREATE TABLE t2 (t TEXT);
INSERT INTO t2 VALUES ('aa'), (REPEAT('z', 200)), ('bb');
SET SESSION group_concat_max_len= 100;
#
# The answer is built from the two short rows. The long value was
# cut on its way in, but it is not in the answer, which is the same
# answer a larger limit would have given: a note.
#
SELECT GROUP_CONCAT(t ORDER BY t LIMIT 2) AS gc FROM t2;
gc
aa,bb
Warnings:
Note 4273 Some values were cut while processing group_concat(). Increase group_concat_max_len if you want to avoid the cut
#
# The same data, and the offset moves the answer onto the row that
# was cut. The answer is now short by what was cut: a warning.
#
SELECT LENGTH(GROUP_CONCAT(t ORDER BY t LIMIT 1 OFFSET 2)) AS gc_len FROM t2;
gc_len
100
Warnings:
Warning 1260 Row 3 was cut by group_concat()
#
# DISTINCT without ORDER BY builds the answer from the duplicate
# filter instead of the sort tree, and is judged the same way.
#
SELECT LENGTH(GROUP_CONCAT(DISTINCT t LIMIT 1 OFFSET 2)) AS gc_len FROM t2;
gc_len
100
Warnings:
Warning 1260 Row 3 was cut by group_concat()
SET SESSION group_concat_max_len= DEFAULT;
DROP TABLE t2;

81
mysql-test/main/gconcat_cut_note.test Normal file
View File

@@ -0,0 +1,81 @@
#
# A TEXT value longer than group_concat_max_len is cut on its way into
# blob_storage, while the group is being built. Whether that changed the
# answer depends on whether the row carrying it reaches the answer at
# all. If it does, the answer is short by what was cut and that is a
# warning. If it does not, the answer may be exactly what it would have
# been without the limit, and all that can honestly be said is a note.
# One note is enough for the statement.
#
--source include/have_sequence.inc
CREATE TABLE t1 (grp INT, t TEXT);
INSERT INTO t1 SELECT seq, REPEAT(CHAR(64 + seq), 200) FROM seq_1_to_3;
SET SESSION group_concat_max_len= 100;
--echo #
--echo # One row per group, so the result of each group is exactly the cut
--echo # value. Every answer is short by what was cut off its only row,
--echo # so every group warns.
--echo #
SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
--echo #
--echo # Two aggregates in one statement, each warning for each of its
--echo # groups. The brackets JSON_ARRAYAGG() puts around the value push
--echo # its result past the limit, so it is cut a second time.
--echo #
SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS gc_len,
LENGTH(JSON_ARRAYAGG(t ORDER BY t)) AS ja_len FROM t1 GROUP BY grp;
--echo #
--echo # The same statement run again reports for itself.
--echo #
SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
--echo #
--echo # All three rows in one group, so the result is cut on top of the
--echo # values. Both are the same loss to the user and both are already
--echo # a warning, so it is given once.
--echo #
SELECT LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1;
--echo #
--echo # Nothing is cut, so nothing is said.
--echo #
SET SESSION group_concat_max_len= DEFAULT;
SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
DROP TABLE t1;
--echo #
--echo # Which of the two is given depends on the row, not on the group.
--echo # One group holds a short value and a long one, and a LIMIT decides
--echo # which of them the answer is built from.
--echo #
CREATE TABLE t2 (t TEXT);
INSERT INTO t2 VALUES ('aa'), (REPEAT('z', 200)), ('bb');
SET SESSION group_concat_max_len= 100;
--echo #
--echo # The answer is built from the two short rows. The long value was
--echo # cut on its way in, but it is not in the answer, which is the same
--echo # answer a larger limit would have given: a note.
--echo #
SELECT GROUP_CONCAT(t ORDER BY t LIMIT 2) AS gc FROM t2;
--echo #
--echo # The same data, and the offset moves the answer onto the row that
--echo # was cut. The answer is now short by what was cut: a warning.
--echo #
SELECT LENGTH(GROUP_CONCAT(t ORDER BY t LIMIT 1 OFFSET 2)) AS gc_len FROM t2;
--echo #
--echo # DISTINCT without ORDER BY builds the answer from the duplicate
--echo # filter instead of the sort tree, and is judged the same way.
--echo #
SELECT LENGTH(GROUP_CONCAT(DISTINCT t LIMIT 1 OFFSET 2)) AS gc_len FROM t2;
SET SESSION group_concat_max_len= DEFAULT;
DROP TABLE t2;

6
sql/field.cc
View File

@@ -8983,20 +8983,22 @@ int Field_blob::handle_group_concat(const char *from, size_t length,
size_t new_length= length;
size_t copy_length= table->in_use->variables.group_concat_max_len;
bool cut= false;
if (new_length > copy_length)
{
new_length= Well_formed_prefix(cs, from, copy_length, new_length).length();
cut= true;
table->blob_storage->set_truncated_value(true);
}
char *tmp;
if (with_zero_prefix)
{
tmp= table->blob_storage->store_with_zero_prefix(from, new_length);
tmp= table->blob_storage->store_with_zero_prefix(from, new_length, cut);
new_length++; // Count the extra 0
}
else
tmp= table->blob_storage->store(from, new_length);
tmp= table->blob_storage->store(from, new_length, cut);
if (!tmp)
goto oom_error;

77
sql/item_sum.cc
View File

@@ -3986,6 +3986,25 @@ static void report_cut_result_error(THD *thd, const char *fname)
}
/*
Tell the user that a value was cut to group_concat_max_len on its way
into blob_storage.
This is only a note. Whether the answer would have been different with
a bigger limit is not known here: the result may well have been cut at
the same place anyway, in which case nothing was lost that the user
could have seen.
*/
static void report_cut_value_note(THD *thd, const char *fname)
{
push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE,
ER_CUT_VALUES_WHILE_PROCESSING,
ER_THD(thd, ER_CUT_VALUES_WHILE_PROCESSING),
fname, "group_concat_max_len");
}
void Item_func_group_concat::cut_max_length(String *result,
uint old_length, uint max_length) const
{
@@ -4066,8 +4085,22 @@ int dump_leaf_key(void* key_arg, element_count count __attribute__((unused)),
uint offset= (field->offset(field->table->record[0]) -
table->s->null_bytes);
DBUG_ASSERT(offset < table->s->reclength);
const uchar *rec= key + offset + item->get_null_bytes();
res= item->get_str_from_field(*arg, field, &tmp, key,
offset + item->get_null_bytes());
/*
This row is going into the answer. If the value it carries was
cut on its way into blob_storage then the answer is short by
what was cut, which val_str() reports as a warning. A value cut
for a row that never gets here may have changed nothing.
*/
if (table->blob_storage && (field->flags & BLOB_FLAG))
{
/* A NULL blob was never stored, so there is no mark to read. */
const uchar *val= ((Field_blob*) field)->get_ptr(rec);
if (val && Blob_mem_storage::was_cut((const char*) val))
item->value_cut_in_result= true;
}
}
else
res= item->get_str_from_item(*arg, &tmp);
@@ -4121,8 +4154,9 @@ Item_func_group_concat(THD *thd, Name_resolution_context *context_arg,
arg_count_field(select_list->elements),
row_count(0),
distinct(distinct_arg),
warning_for_row(FALSE), result_cut(FALSE), walk_stopped(FALSE),
walk_failed(FALSE), always_null(FALSE),
warning_for_row(FALSE), result_cut(FALSE), cut_note_given(FALSE),
value_cut_in_result(FALSE), walk_stopped(FALSE), walk_failed(FALSE),
always_null(FALSE),
force_copy_fields(0), row_limit(NULL),
offset_limit(NULL), limit_clause(limit_clause),
copy_offset_limit(0), copy_row_limit(0), original(0)
@@ -4190,6 +4224,8 @@ Item_func_group_concat::Item_func_group_concat(THD *thd,
distinct(item->distinct),
warning_for_row(item->warning_for_row),
result_cut(item->result_cut),
cut_note_given(item->cut_note_given),
value_cut_in_result(item->value_cut_in_result),
walk_stopped(item->walk_stopped),
walk_failed(item->walk_failed),
always_null(item->always_null),
@@ -4262,6 +4298,7 @@ void Item_func_group_concat::cleanup()
row_count= 0;
DBUG_ASSERT(tree == 0);
}
cut_note_given= false;
/*
As the ORDER structures pointed to by the elements of the
'order' array may be modified in find_order_in_list() called
@@ -4292,6 +4329,7 @@ void Item_func_group_concat::clear()
null_value= TRUE;
warning_for_row= FALSE;
result_cut= FALSE;
value_cut_in_result= FALSE;
walk_failed= FALSE;
result_finalized= false;
if (offset_limit)
@@ -4892,19 +4930,44 @@ String* Item_func_group_concat::val_str(String* str)
DBUG_ASSERT(false); // Can't happen
}
if (result_cut ||
(table && table->blob_storage &&
table->blob_storage->is_truncated_value()))
/*
The answer came out short: it was cut at gconcat_max_len(), the sort
tree could not keep every row it was given, or a row that reached the
answer carried a value that was cut on its way into blob_storage.
Values the user asked for are missing, so this is a warning.
*/
if (result_cut || value_cut_in_result)
{
warning_for_row= true;
report_cut_value_error(current_thd, row_count, func_name());
/*
A cut value that reached the answer has now been reported as the
warning it is, so it must not be reported again as a note.
*/
if (value_cut_in_result && table && table->blob_storage)
table->blob_storage->set_truncated_value(false);
/*
Clear the marks so that we give only one warning per group, even
if val_str() is called more than once for this group.
*/
result_cut= false;
if (table && table->blob_storage)
table->blob_storage->set_truncated_value(false);
value_cut_in_result= false;
}
/*
A value was cut on its way into blob_storage, but no row carrying one
reached the answer, so the answer may well be what it would have been
without the limit: one note per statement.
*/
if (table && table->blob_storage &&
table->blob_storage->is_truncated_value())
{
if (!cut_note_given)
{
cut_note_given= true;
report_cut_value_note(current_thd, func_name());
}
table->blob_storage->set_truncated_value(false);
}
/*

14
sql/item_sum.h
View File

@@ -2136,6 +2136,20 @@ class Item_func_group_concat : public Item_sum_str
cut value warning for it.
*/
bool result_cut;
/*
Set once the note about values cut on their way into blob_storage has
been given for this aggregate. One note per statement is enough,
however many groups had a value cut. cleanup() clears it, so a
statement that is run again gets its own note.
*/
bool cut_note_given;
/*
Set by dump_leaf_key() when a row it appends carries a value that was
cut on its way into blob_storage. The answer is then short by what
was cut, which is a warning rather than the note a cut value that
never reached the answer gets.
*/
bool value_cut_in_result;
/*
Set by dump_leaf_key() when it is the one that stops a walk, so that
val_str() can tell that apart from the walk itself having failed.

2
sql/share/errmsg-utf8.txt
View File

@@ -12421,3 +12421,5 @@ ER_DDL_ILLEGAL_FK 42000
spa "No puedo %s una tabla referenciada en una restricción de clave foránea (%.192s)"
ER_RESULT_CUT_BY_LIMIT
eng "Result was cut by %s) because of %s limit"
ER_CUT_VALUES_WHILE_PROCESSING
eng "Some values were cut while processing %s). Increase %s if you want to avoid the cut"

42
sql/table.h
View File

@@ -1327,34 +1327,56 @@ public:
free_root(&storage, MYF(MY_MARK_BLOCKS_FREE));
truncated_value= false;
}
/*
Every value is stored behind one byte that records whether the value
was cut on its way in. The byte is not part of the value and the
returned pointer is what the record holds, so a reader that has the
record can ask was_cut() about a value without knowing which row it
came from.
*/
char *store_with_cut_mark(const char *from, size_t length, bool cut,
size_t prefix)
{
char *res= (char*) alloc_root(&storage, length + prefix + 1);
if (res)
{
*res++= (char) cut;
if (prefix)
res[0]= 0;
memcpy(res + prefix, from, length);
}
return res;
}
/**
Function creates duplicate of 'from'
string in 'storage' MEM_ROOT.
@param from string to copy
@param length string length
@param cut 'from' was cut to fit the limit
@retval Pointer to the copied string.
@retval 0 if an error occurred.
*/
char *store(const char *from, size_t length)
char *store(const char *from, size_t length, bool cut)
{
return (char*) memdup_root(&storage, from, length);
return store_with_cut_mark(from, length, cut, 0);
}
/*
Store string with a 0 prefix. This is used for storing
Field_blob_compressed fields in a not compressed format.
*/
char *store_with_zero_prefix(const char *from, size_t length)
char *store_with_zero_prefix(const char *from, size_t length, bool cut)
{
char *res= (char*) alloc_root(&storage, length+1);
if (res)
{
res[0]= 0;
memcpy(res+1, from, length);
}
return res;
return store_with_cut_mark(from, length, cut, 1);
}
/*
Whether the value at 'ptr' was cut on its way in. 'ptr' must be a
pointer this storage handed out, which is the case for every blob in
a table that has a Blob_mem_storage: Field_blob::store() sends all of
them through Field_blob::handle_group_concat().
*/
static bool was_cut(const char *ptr) { return ptr[-1] != 0; }
void set_truncated_value(bool is_truncated_value)
{
truncated_value= is_truncated_value;