diff --git a/NEWS b/NEWS index b6704f596..eb252320d 100644 --- a/NEWS +++ b/NEWS @@ -95,6 +95,11 @@ GNU coreutils NEWS -*- outline -*- 146077762747232966952776282887314548659. [bugs introduced in coreutils-8.20] + cut, expand, fold, and unexpand no longer treat a multi-byte character + as separate bytes when a read() from a pipe or terminal returns only + part of the character. + [bug introduced in coreutils-9.11] + 'fold' no longer prematurely wraps lines with a wide character towards the start of a line, followed by multiple backspaces. Also backspaces after a tab character are accounted for correctly. diff --git a/gl/lib/mbbuf.h b/gl/lib/mbbuf.h index 626dc737e..ccaad3d21 100644 --- a/gl/lib/mbbuf.h +++ b/gl/lib/mbbuf.h @@ -76,7 +76,8 @@ mbbuf_init (mbbuf_t *mbbuf, char *buffer, idx_t size, FILE *fp) mbbuf->eof = false; } -/* Fill the input buffer with at least MCEL_LEN_MAX bytes if possible. +/* Fill the input buffer with at least MCEL_LEN_MAX bytes if possible, + though don't wait/retry if less than that are available. Return the number of bytes available from the current offset. At end of file, MBBUF.EOF is set, and zero will eventually be returned. Note feof() will _NOT_ be set on the MBBUF.FP. */ @@ -123,15 +124,25 @@ mbbuf_fill (mbbuf_t *mbbuf) { idx_t available = mbbuf_avail (mbbuf); - if (available == 0) - return mbbuf_topup (mbbuf); + /* Short-circuit the common case. */ + if (MCEL_LEN_MAX <= available) + return available; - if (available < MCEL_LEN_MAX && ! mbbuf->eof) + if (available == 0) + available = mbbuf_topup (mbbuf); + + /* A single read() may return only part of a character, e.g., from a pipe + or terminal. Keep reading until the character at the current offset + is complete, or enough bytes are available to decide, or at EOF. + Note mcel_scan does not distinguish invalid from incomplete sequences, + so invalid bytes can also cause further reads and delay output. */ + while (0 < available && available < MCEL_LEN_MAX && ! mbbuf->eof) { mcel_t g = mcel_scan (mbbuf->buffer + mbbuf->offset, mbbuf->buffer + mbbuf->length); - if (g.err) - return mbbuf_topup (mbbuf); + if (! g.err) + break; + available = mbbuf_topup (mbbuf); } return available; diff --git a/tests/cut/mb-short-read.sh b/tests/cut/mb-short-read.sh new file mode 100755 index 000000000..9edc5278c --- /dev/null +++ b/tests/cut/mb-short-read.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Ensure a multi-byte character split across read() calls +# is not processed as separate bytes. + +# Copyright (C) 2026 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src +print_ver_ cut fold expand unexpand + +{ test -z "$LOCALE_FR_UTF8" || test "$LOCALE_FR_UTF8" = none; } && + skip_ 'French UTF-8 locale not available' +export LC_ALL="$LOCALE_FR_UTF8" + +# Write each byte of U+20AC separately, so that each read() returns one. +euro_in_pieces () +{ + printf '\342'; sleep .1; printf '\202'; sleep .1; printf '\254'; printf "$1" +} + +printf '\342\202\254ab\n' > exp || framework_failure_ +euro_in_pieces 'ab\n' | cut -c1-3 > out || fail=1 +compare exp out || fail=1 + +printf '\n' > exp || framework_failure_ +euro_in_pieces 'ab\n' | cut -b1-2 -n > out || fail=1 +compare exp out || fail=1 + +printf '\342\202\254a\nb\n' > exp || framework_failure_ +euro_in_pieces 'ab\n' | fold -w2 > out || fail=1 +compare exp out || fail=1 + +printf '\342\202\254 b\n' > exp || framework_failure_ +euro_in_pieces '\tb\n' | expand > out || fail=1 +compare exp out || fail=1 + +printf '\342\202\254\tb\n' > exp || framework_failure_ +euro_in_pieces ' b\n' | unexpand -a > out || fail=1 +compare exp out || fail=1 + +Exit $fail diff --git a/tests/local.mk b/tests/local.mk index 2199b39c2..0a49022a9 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -319,6 +319,7 @@ all_tests = \ tests/misc/coreutils.sh \ tests/cut/cut.pl \ tests/cut/mb-non-utf8.sh \ + tests/cut/mb-short-read.sh \ tests/cut/bounded-memory.sh \ tests/cut/cut-huge-range.sh \ tests/wc/mb-non-utf8.sh \