From da5cecf5532ed98d7677275af590e9199d2181e2 Mon Sep 17 00:00:00 2001 From: Paul Eggert Date: Thu, 13 Aug 2026 13:18:55 -0700 Subject: [PATCH] sed: 'l' and printable multibyte characters Fix a longstanding bug where the 'l' command mistakenly escapes bytes in a multibyte character. * basicdefs.h (ISPRINT): Remove. All uses removed. * bootstrap.conf (gnulib_modules): Add wctype-h, which is now needed directly instead of indirectly, as we now use iswprint. * sed/debug.c (debug_print_char): * sed/execute.c (do_list): Use iswprint instead of ISPRINT, so that things work correctly with multibyte locales. * sed/debug.c (debug_print_char): New args P and CLEN. All uses changed. * testsuite/cmd-l.sh: Test for the bug. --- NEWS | 4 ++++ basicdefs.h | 5 +--- bootstrap.conf | 1 + sed/debug.c | 21 +++++++++------- sed/execute.c | 60 ++++++++++++++++++++++++++++------------------ sed/sed.h | 2 +- testsuite/cmd-l.sh | 11 +++++++++ 7 files changed, 68 insertions(+), 36 deletions(-) diff --git a/NEWS b/NEWS index 7ebe0b2..6063410 100644 --- a/NEWS +++ b/NEWS @@ -8,6 +8,10 @@ GNU sed NEWS -*- outline -*- 's' command. [bug introduced in sed 4.10] + The 'l' command no longer mishandles a printable multibyte character + by escaping its individual bytes. + [bug present since the beginning] + File names with odd characters are now quoted in diagnostics. The quoting style is the same as 'ls --quoting-style=shell-escape'. [bug present since the beginning] diff --git a/basicdefs.h b/basicdefs.h index 1903a83..210093a 100644 --- a/basicdefs.h +++ b/basicdefs.h @@ -44,8 +44,7 @@ "... Some ctype macros are valid only for character codes that isascii says are ASCII (SGI's IRIX-4.0.5 is one such system --when - using /bin/cc or gcc but without giving an ansi option). So, all - ctype uses should be through macros like ISPRINT... If + using /bin/cc or gcc but without giving an ansi option).... If STDC_HEADERS is defined, then autoconf has verified that the ctype macros don't need to be guarded with references to isascii. ... Defining isascii to 1 should let any compiler worth its salt @@ -65,8 +64,6 @@ # define ISBLANK(c) ((c) == ' ' || (c) == '\t') #endif -#undef ISPRINT -#define ISPRINT(c) (ISASCII (c) && isprint (c)) #define ISSPACE(c) (ISASCII (c) && isspace (c)) #ifndef initialize_main diff --git a/bootstrap.conf b/bootstrap.conf index 40b0879..158ed46 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -70,6 +70,7 @@ update-copyright version-etc-fsf wcrtomb wctob +wctype-h xalloc ' gnulib_name=libsed diff --git a/sed/debug.c b/sed/debug.c index fe0b29c..1738a04 100644 --- a/sed/debug.c +++ b/sed/debug.c @@ -31,11 +31,11 @@ static off_t block_level = 0; void -debug_print_char (char c) +debug_print_char (int c, char const *p, idx_t clen) { - if (ISPRINT (c) && c != '\\') + if (0 <= c && iswprint (c) && c != '\\') { - putchar (c); + fwrite (p, 1, clen, stdout); return; } @@ -65,7 +65,8 @@ debug_print_char (char c) break; default: - printf ("o%03o", (unsigned int) c); + for (idx_t i = 0; i < clen; i++) + printf (&"\\o%03o"[!i], (unsigned int) {(unsigned char) {p[i]}}); } } @@ -73,13 +74,17 @@ static void debug_print_regex_pattern (const char *pat, idx_t len) { const char *p = pat; - while (len--) + mbstate_t mbs; mbszero (&mbs); + while (len) { - if (*p == '/') + int ch; + idx_t chlen = mbrtowc1 (&ch, p, len, &mbs); + if (ch == '/') fputs ("\\/", stdout); else - debug_print_char (*p); - ++p; + debug_print_char (ch, p, chlen); + p += chlen; + len -= chlen; } } diff --git a/sed/execute.c b/sed/execute.c index a32dad0..e46ebdb 100644 --- a/sed/execute.c +++ b/sed/execute.c @@ -894,30 +894,29 @@ match_address_p (struct sed_cmd *cmd, struct input *input) static void do_list (intmax_t line_len) { - unsigned char *p = (unsigned char *)line.active; + char *p = line.active; idx_t len = line.length; idx_t width = 0; FILE *fp = output_file.fp; + mbstate_t mbs; mbszero (&mbs); output_missing_newline (&output_file); - for (; len--; ++p) { - char obuf[sizeof "\\377" - 1]; + while (len) + { + char obuf[(sizeof "\\377" - 1) * MB_LEN_MAX]; char *o = obuf; - - /* Some locales define 8-bit characters as printable. This makes the - testsuite fail at 8to7.sed because the 'l' command in fact will not - convert the 8-bit characters. */ -#if defined isascii || defined HAVE_ISASCII - if (isascii (*p) && ISPRINT (*p)) { -#else - if (ISPRINT (*p)) { -#endif - *o++ = *p; - if (*p == '\\') + int ch; + idx_t chlen = mbrtowc1 (&ch, p, len, &mbs); + if (0 <= ch && iswprint (ch)) + { + o = mempcpy (o, p, chlen); + if (ch == '\\') *o++ = '\\'; - } else { + } + else + { *o++ = '\\'; - switch (*p) { + switch (ch) { case '\a': *o++ = 'a'; break; case '\b': *o++ = 'b'; break; case '\f': *o++ = 'f'; break; @@ -926,12 +925,20 @@ do_list (intmax_t line_len) case '\t': *o++ = 't'; break; case '\v': *o++ = 'v'; break; default: - *o++ = '0' + ((*p & 0300) >> 6); - *o++ = '0' + ((*p & 0070) >> 3); - *o++ = '0' + ((*p & 0007) >> 0); + for (idx_t i = 0; i < chlen; i++) + { + *o = '\\'; + o += !!i; + *o++ = '0' + ((p[i] & 0300) >> 6); + *o++ = '0' + ((p[i] & 0070) >> 3); + *o++ = '0' + ((p[i] & 0007) >> 0); + } break; } - } + } + p += chlen; + len -= chlen; + idx_t olen = o - obuf; if (0 < line_len && line_len - olen <= width) { ck_fwrite ("\\", 1, 1, fp); @@ -940,7 +947,7 @@ do_list (intmax_t line_len) } ck_fwrite (obuf, 1, olen, fp); width += olen; - } + } ck_fwrite ("$", 1, 1, fp); ck_fwrite (&buffer_delimiter, 1, 1, fp); flush_output (fp); @@ -1254,8 +1261,15 @@ debug_print_line (struct line *ln) const char *p = src; fputs ( (ln == &hold) ? "HOLD: ":"PATTERN: ", stdout); - while (l--) - debug_print_char (*p++); + mbstate_t mbs; mbszero (&mbs); + while (l) + { + int ch; + idx_t chlen = mbrtowc1 (&ch, p, l, &mbs); + debug_print_char (ch, p, chlen); + p += chlen; + l -= chlen; + } putchar ('\n'); } diff --git a/sed/sed.h b/sed/sed.h index cfbc19f..9904e36 100644 --- a/sed/sed.h +++ b/sed/sed.h @@ -229,7 +229,7 @@ debug_print_command (const struct vector *program, const struct sed_cmd *sc); void debug_print_program (const struct vector *program); void -debug_print_char (char c); +debug_print_char (int c, char const *p, idx_t clen); int process_files (struct vector *, char **argv); diff --git a/testsuite/cmd-l.sh b/testsuite/cmd-l.sh index b8b1e00..db133fe 100755 --- a/testsuite/cmd-l.sh +++ b/testsuite/cmd-l.sh @@ -18,11 +18,17 @@ . "${srcdir=.}/testsuite/init.sh"; path_prepend_ ./sed print_ver_ sed +require_en_utf8_locale_ + # 10 20 30 40 50 60 70 75 cat <<\EOF >in1 || framework_failure_ 0123456789012345678901234567890123456789012345678901234567890123456789012345 EOF +cat <<\EOF >in-non-ascii || framework_failure_ +“”‘’ …–—•°§¶™©®€£¥éüñà +EOF + # default: 70 characters (including the \n) cat <<\EOF >exp-def || framework_failure_ 012345678901234567890123456789012345678901234567890123456789012345678\ @@ -72,4 +78,9 @@ compare_ exp-11 out-ln-11 || fail=1 returns_ 1 sed --posix -n l11 in1 2>err-posix-ln || fail=1 compare_ exp-err-posix-ln err-posix-ln || fail=1 +# l with non-ASCII printable characters +LC_ALL=en_US.UTF-8 sed 's/$/\$/' exp-non-ascii || fail=1 +LC_ALL=en_US.UTF-8 sed -n l out-non-ascii || fail=1 +LC_ALL=en_US.UTF-8 compare_ exp-non-ascii out-non-ascii || fail=1 + Exit $fail