mirror of
https://git.savannah.gnu.org/git/coreutils.git
synced 2026-11-17 18:02:09 +03:00
cut,expand,fold,unexpand: fix handling of multi-byte with short reads
This change is performance neutral as tested with:
$ yes $(yes eeeaae | head -n10K | paste -s -d,) | head -n10K > ll.in
$ hyperfine src/cut{,.orig}' -f10 -w ll.in >/dev/null'
* gl/lib/mbbuf.h (mbbuf_fill): Repeat the mbbuf_topup(), i.e. the read()
if the previous run didn't read enough. Note we add the explicit
common case short-circuit to avoid a 20% regression
for the test case above, with GCC 16.2 with -O3 -flto.
* tests/cut/mb-short-read.sh: A new test file.
* tests/local.mk: Reference the new test.
* NEWS: Mention the bug fix.
Discovered by Anthropic's OSS Scanner, as bug ANT-2026-JSCMJCN2.
This commit is contained in:
1 parent
4f20df5a92
commit
f66f9b1146
4 files changed
+76
-6
No files matched your search
@@ -95,6 +95,11 @@ GNU coreutils NEWS -*- outline -*-
|
||||
146077762747232966952776282887314548659.
|
||||
[bugs introduced in coreutils-8.20]
|
||||
|
||||
cut, expand, fold, and unexpand no longer treat a multi-byte character
|
||||
as separate bytes when a read() from a pipe or terminal returns only
|
||||
part of the character.
|
||||
[bug introduced in coreutils-9.11]
|
||||
|
||||
'fold' no longer prematurely wraps lines with a wide character towards
|
||||
the start of a line, followed by multiple backspaces. Also backspaces
|
||||
after a tab character are accounted for correctly.
|
||||
|
||||
+17
-6
@@ -76,7 +76,8 @@ mbbuf_init (mbbuf_t *mbbuf, char *buffer, idx_t size, FILE *fp)
|
||||
mbbuf->eof = false;
|
||||
}
|
||||
|
||||
/* Fill the input buffer with at least MCEL_LEN_MAX bytes if possible.
|
||||
/* Fill the input buffer with at least MCEL_LEN_MAX bytes if possible,
|
||||
though don't wait/retry if less than that are available.
|
||||
Return the number of bytes available from the current offset.
|
||||
At end of file, MBBUF.EOF is set, and zero will eventually be returned.
|
||||
Note feof() will _NOT_ be set on the MBBUF.FP. */
|
||||
@@ -123,15 +124,25 @@ mbbuf_fill (mbbuf_t *mbbuf)
|
||||
{
|
||||
idx_t available = mbbuf_avail (mbbuf);
|
||||
|
||||
if (available == 0)
|
||||
return mbbuf_topup (mbbuf);
|
||||
/* Short-circuit the common case. */
|
||||
if (MCEL_LEN_MAX <= available)
|
||||
return available;
|
||||
|
||||
if (available < MCEL_LEN_MAX && ! mbbuf->eof)
|
||||
if (available == 0)
|
||||
available = mbbuf_topup (mbbuf);
|
||||
|
||||
/* A single read() may return only part of a character, e.g., from a pipe
|
||||
or terminal. Keep reading until the character at the current offset
|
||||
is complete, or enough bytes are available to decide, or at EOF.
|
||||
Note mcel_scan does not distinguish invalid from incomplete sequences,
|
||||
so invalid bytes can also cause further reads and delay output. */
|
||||
while (0 < available && available < MCEL_LEN_MAX && ! mbbuf->eof)
|
||||
{
|
||||
mcel_t g = mcel_scan (mbbuf->buffer + mbbuf->offset,
|
||||
mbbuf->buffer + mbbuf->length);
|
||||
if (g.err)
|
||||
return mbbuf_topup (mbbuf);
|
||||
if (! g.err)
|
||||
break;
|
||||
available = mbbuf_topup (mbbuf);
|
||||
}
|
||||
|
||||
return available;
|
||||
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/bin/sh
|
||||
# Ensure a multi-byte character split across read() calls
|
||||
# is not processed as separate bytes.
|
||||
|
||||
# Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
|
||||
print_ver_ cut fold expand unexpand
|
||||
|
||||
{ test -z "$LOCALE_FR_UTF8" || test "$LOCALE_FR_UTF8" = none; } &&
|
||||
skip_ 'French UTF-8 locale not available'
|
||||
export LC_ALL="$LOCALE_FR_UTF8"
|
||||
|
||||
# Write each byte of U+20AC separately, so that each read() returns one.
|
||||
euro_in_pieces ()
|
||||
{
|
||||
printf '\342'; sleep .1; printf '\202'; sleep .1; printf '\254'; printf "$1"
|
||||
}
|
||||
|
||||
printf '\342\202\254ab\n' > exp || framework_failure_
|
||||
euro_in_pieces 'ab\n' | cut -c1-3 > out || fail=1
|
||||
compare exp out || fail=1
|
||||
|
||||
printf '\n' > exp || framework_failure_
|
||||
euro_in_pieces 'ab\n' | cut -b1-2 -n > out || fail=1
|
||||
compare exp out || fail=1
|
||||
|
||||
printf '\342\202\254a\nb\n' > exp || framework_failure_
|
||||
euro_in_pieces 'ab\n' | fold -w2 > out || fail=1
|
||||
compare exp out || fail=1
|
||||
|
||||
printf '\342\202\254 b\n' > exp || framework_failure_
|
||||
euro_in_pieces '\tb\n' | expand > out || fail=1
|
||||
compare exp out || fail=1
|
||||
|
||||
printf '\342\202\254\tb\n' > exp || framework_failure_
|
||||
euro_in_pieces ' b\n' | unexpand -a > out || fail=1
|
||||
compare exp out || fail=1
|
||||
|
||||
Exit $fail
|
||||
@@ -319,6 +319,7 @@ all_tests = \
|
||||
tests/misc/coreutils.sh \
|
||||
tests/cut/cut.pl \
|
||||
tests/cut/mb-non-utf8.sh \
|
||||
tests/cut/mb-short-read.sh \
|
||||
tests/cut/bounded-memory.sh \
|
||||
tests/cut/cut-huge-range.sh \
|
||||
tests/wc/mb-non-utf8.sh \
|
||||
|
||||
Reference in new issue
Block a user