From f66f9b1146ccbc3346b59a2f64f7b99165148a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draig=20Brady?=
Date: Mon, 9 Nov 2026 16:32:42 +0000
Subject: [PATCH] cut,expand,fold,unexpand: fix handling of multi-byte with
short reads
This change is performance neutral as tested with:
$ yes $(yes eeeaae | head -n10K | paste -s -d,) | head -n10K > ll.in
$ hyperfine src/cut{,.orig}' -f10 -w ll.in >/dev/null'
* gl/lib/mbbuf.h (mbbuf_fill): Repeat the mbbuf_topup(), i.e. the read()
if the previous run didn't read enough. Note we add the explicit
common case short-circuit to avoid a 20% regression
for the test case above, with GCC 16.2 with -O3 -flto.
* tests/cut/mb-short-read.sh: A new test file.
* tests/local.mk: Reference the new test.
* NEWS: Mention the bug fix.
Discovered by Anthropic's OSS Scanner, as bug ANT-2026-JSCMJCN2.
---
NEWS | 5 ++++
gl/lib/mbbuf.h | 23 ++++++++++++-----
tests/cut/mb-short-read.sh | 53 ++++++++++++++++++++++++++++++++++++++
tests/local.mk | 1 +
4 files changed, 76 insertions(+), 6 deletions(-)
create mode 100755 tests/cut/mb-short-read.sh
diff --git a/NEWS b/NEWS
index b6704f596..eb252320d 100644
--- a/NEWS
+++ b/NEWS
@@ -95,6 +95,11 @@ GNU coreutils NEWS -*- outline -*-
146077762747232966952776282887314548659.
[bugs introduced in coreutils-8.20]
+ cut, expand, fold, and unexpand no longer treat a multi-byte character
+ as separate bytes when a read() from a pipe or terminal returns only
+ part of the character.
+ [bug introduced in coreutils-9.11]
+
'fold' no longer prematurely wraps lines with a wide character towards
the start of a line, followed by multiple backspaces. Also backspaces
after a tab character are accounted for correctly.
diff --git a/gl/lib/mbbuf.h b/gl/lib/mbbuf.h
index 626dc737e..ccaad3d21 100644
--- a/gl/lib/mbbuf.h
+++ b/gl/lib/mbbuf.h
@@ -76,7 +76,8 @@ mbbuf_init (mbbuf_t *mbbuf, char *buffer, idx_t size, FILE *fp)
mbbuf->eof = false;
}
-/* Fill the input buffer with at least MCEL_LEN_MAX bytes if possible.
+/* Fill the input buffer with at least MCEL_LEN_MAX bytes if possible,
+ though don't wait/retry if less than that are available.
Return the number of bytes available from the current offset.
At end of file, MBBUF.EOF is set, and zero will eventually be returned.
Note feof() will _NOT_ be set on the MBBUF.FP. */
@@ -123,15 +124,25 @@ mbbuf_fill (mbbuf_t *mbbuf)
{
idx_t available = mbbuf_avail (mbbuf);
- if (available == 0)
- return mbbuf_topup (mbbuf);
+ /* Short-circuit the common case. */
+ if (MCEL_LEN_MAX <= available)
+ return available;
- if (available < MCEL_LEN_MAX && ! mbbuf->eof)
+ if (available == 0)
+ available = mbbuf_topup (mbbuf);
+
+ /* A single read() may return only part of a character, e.g., from a pipe
+ or terminal. Keep reading until the character at the current offset
+ is complete, or enough bytes are available to decide, or at EOF.
+ Note mcel_scan does not distinguish invalid from incomplete sequences,
+ so invalid bytes can also cause further reads and delay output. */
+ while (0 < available && available < MCEL_LEN_MAX && ! mbbuf->eof)
{
mcel_t g = mcel_scan (mbbuf->buffer + mbbuf->offset,
mbbuf->buffer + mbbuf->length);
- if (g.err)
- return mbbuf_topup (mbbuf);
+ if (! g.err)
+ break;
+ available = mbbuf_topup (mbbuf);
}
return available;
diff --git a/tests/cut/mb-short-read.sh b/tests/cut/mb-short-read.sh
new file mode 100755
index 000000000..9edc5278c
--- /dev/null
+++ b/tests/cut/mb-short-read.sh
@@ -0,0 +1,53 @@
+#!/bin/sh
+# Ensure a multi-byte character split across read() calls
+# is not processed as separate bytes.
+
+# Copyright (C) 2026 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see