cut,expand,fold,unexpand: fix handling of multi-byte with short reads

This change is performance neutral as tested with:
 $ yes $(yes eeeaae | head -n10K | paste -s -d,) | head -n10K > ll.in
 $ hyperfine src/cut{,.orig}' -f10 -w ll.in >/dev/null'

* gl/lib/mbbuf.h (mbbuf_fill): Repeat the mbbuf_topup(), i.e. the read()
if the previous run didn't read enough.  Note we add the explicit
common case short-circuit to avoid a 20% regression
for the test case above, with GCC 16.2 with -O3 -flto.
* tests/cut/mb-short-read.sh: A new test file.
* tests/local.mk: Reference the new test.
* NEWS: Mention the bug fix.

Discovered by Anthropic's OSS Scanner, as bug ANT-2026-JSCMJCN2.
This commit is contained in:
Pádraig Brady committed 2026-11-10 12:52:08 +00:00
1 parent 4f20df5a92
commit f66f9b1146
4 files changed
+76 -6

No files matched your search

+5
View File
@@ -95,6 +95,11 @@ GNU coreutils NEWS -*- outline -*-
146077762747232966952776282887314548659.
[bugs introduced in coreutils-8.20]
cut, expand, fold, and unexpand no longer treat a multi-byte character
as separate bytes when a read() from a pipe or terminal returns only
part of the character.
[bug introduced in coreutils-9.11]
'fold' no longer prematurely wraps lines with a wide character towards
the start of a line, followed by multiple backspaces. Also backspaces
after a tab character are accounted for correctly.
+17 -6
View File
@@ -76,7 +76,8 @@ mbbuf_init (mbbuf_t *mbbuf, char *buffer, idx_t size, FILE *fp)
mbbuf->eof = false;
}
/* Fill the input buffer with at least MCEL_LEN_MAX bytes if possible.
/* Fill the input buffer with at least MCEL_LEN_MAX bytes if possible,
though don't wait/retry if less than that are available.
Return the number of bytes available from the current offset.
At end of file, MBBUF.EOF is set, and zero will eventually be returned.
Note feof() will _NOT_ be set on the MBBUF.FP. */
@@ -123,15 +124,25 @@ mbbuf_fill (mbbuf_t *mbbuf)
{
idx_t available = mbbuf_avail (mbbuf);
if (available == 0)
return mbbuf_topup (mbbuf);
/* Short-circuit the common case. */
if (MCEL_LEN_MAX <= available)
return available;
if (available < MCEL_LEN_MAX && ! mbbuf->eof)
if (available == 0)
available = mbbuf_topup (mbbuf);
/* A single read() may return only part of a character, e.g., from a pipe
or terminal. Keep reading until the character at the current offset
is complete, or enough bytes are available to decide, or at EOF.
Note mcel_scan does not distinguish invalid from incomplete sequences,
so invalid bytes can also cause further reads and delay output. */
while (0 < available && available < MCEL_LEN_MAX && ! mbbuf->eof)
{
mcel_t g = mcel_scan (mbbuf->buffer + mbbuf->offset,
mbbuf->buffer + mbbuf->length);
if (g.err)
return mbbuf_topup (mbbuf);
if (! g.err)
break;
available = mbbuf_topup (mbbuf);
}
return available;
+53
View File
@@ -0,0 +1,53 @@
#!/bin/sh
# Ensure a multi-byte character split across read() calls
# is not processed as separate bytes.
# Copyright (C) 2026 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
print_ver_ cut fold expand unexpand
{ test -z "$LOCALE_FR_UTF8" || test "$LOCALE_FR_UTF8" = none; } &&
skip_ 'French UTF-8 locale not available'
export LC_ALL="$LOCALE_FR_UTF8"
# Write each byte of U+20AC separately, so that each read() returns one.
euro_in_pieces ()
{
printf '\342'; sleep .1; printf '\202'; sleep .1; printf '\254'; printf "$1"
}
printf '\342\202\254ab\n' > exp || framework_failure_
euro_in_pieces 'ab\n' | cut -c1-3 > out || fail=1
compare exp out || fail=1
printf '\n' > exp || framework_failure_
euro_in_pieces 'ab\n' | cut -b1-2 -n > out || fail=1
compare exp out || fail=1
printf '\342\202\254a\nb\n' > exp || framework_failure_
euro_in_pieces 'ab\n' | fold -w2 > out || fail=1
compare exp out || fail=1
printf '\342\202\254 b\n' > exp || framework_failure_
euro_in_pieces '\tb\n' | expand > out || fail=1
compare exp out || fail=1
printf '\342\202\254\tb\n' > exp || framework_failure_
euro_in_pieces ' b\n' | unexpand -a > out || fail=1
compare exp out || fail=1
Exit $fail
+1
View File
@@ -319,6 +319,7 @@ all_tests = \
tests/misc/coreutils.sh \
tests/cut/cut.pl \
tests/cut/mb-non-utf8.sh \
tests/cut/mb-short-read.sh \
tests/cut/bounded-memory.sh \
tests/cut/cut-huge-range.sh \
tests/wc/mb-non-utf8.sh \