sed: simplify multibyte substitution

When handling the y command, or the case-changing replacement,
stay a bit more in the wchar_t world, to simplify allocation
and help make future changes simpler.
* basicdefs.h (STREQ_LEN): Remove; no longer used.
* cfg.mk (sc_prohibit_strncmp): Don’t suggest STREQ_LEN.
* sed/compile.c (compile_program):
* sed/execute.c (str_append_modified, translate_mb):
Instead of preserving the bytes of a multibyte character, just
save its wchar_t counterpart, or negative of its single-byte
encoding error.
* sed/debug.c (debug_print_mb): New static function.
(debug_print_translation): Use it to adjust to new data structure.
* sed/execute.c (translate_mb): Args are now NPAIRS and PAIR, not
TRANS.  Caller changed.
* sed/sed.h (struct sed_cmd.translate.mb): Now a struct with
an npairs and pair members, instead of a single char **.
All uses changed.
This commit is contained in:
Paul Eggert committed 2026-08-17 01:17:23 -07:00
1 parent bd3b429b80
commit 4e2995cd01
6 files changed
+111 -72

No files matched your search

-2
View File
@@ -31,8 +31,6 @@
#define obstack_chunk_alloc xzalloc
#define obstack_chunk_free free
#define STREQ_LEN(a, b, n) (strncmp (a, b, n) == 0)
/* MAX_PATH is not defined in some platforms, most notably GNU/Hurd.
In that case we define it here to some constant. Note however that
this relies in the fact that sed does reallocation if a buffer
+2 -3
View File
@@ -228,11 +228,10 @@ sc_gitignore_missing:
'entries to lib/.gitignore' >&2; exit 1; } || :
# Similar to the gnulib maint.mk rule for sc_prohibit_strcmp
# Use STREQ_LEN rather than comparing strncmp == 0, or != 0.
# Similar to the gnulib maint.mk rule for sc_prohibit_strcmp.
sc_prohibit_strncmp:
@prohibit='^[^#].*str''ncmp *\(' \
halt='use STREQ_LEN instead of str''ncmp' \
halt='use memeq instead of str''ncmp' \
$(_sc_search_regexp)
# Ensure that tests don't include a redundant fail=0.
+22 -30
View File
@@ -1207,55 +1207,47 @@ compile_program (struct vector *vector)
if (mb_cur_max > 1)
{
idx_t i, j, idx, src_char_num;
idx_t *src_lens = XNMALLOC (len, idx_t);
char **trans_pairs;
mbstate_t cur_stat; mbszero (&cur_stat);
/* Enumerate how many character the source buffer has. */
for (i = 0, j = 0; i < len;)
{
size_t mbclen = mbrlen1 (src_buf + i, len - i, &cur_stat);
src_lens[j++] = mbclen;
i += mbclen;
}
src_char_num = j;
idx_t src_char_num = 0;
for (idx_t i = 0; i < len;
i += mbrlen1 (src_buf + i, len - i, &cur_stat))
src_char_num++;
mbszero (&cur_stat);
mbstate_t tr_stat; mbszero (&tr_stat);
idx = 0;
idx_t idx = 0;
/* trans_pairs = {src(0), dest(0), src(1), dest(1), ..., NULL}
src(i) : pointer to i-th source character.
dest(i) : pointer to i-th destination character.
NULL : terminator */
trans_pairs = XNMALLOC (2 * src_char_num + 1, char *);
cur_cmd->x.translate.mb = trans_pairs;
for (i = 0; i < src_char_num; i++)
/* trans_pairs = {src(0), dest(0), src(1), dest(1), ... }
src(i) : i-th source character.
dest(i) : i-th destination character. */
int *trans_pairs = xnmalloc (src_char_num,
2 * sizeof *trans_pairs);
cur_cmd->x.translate.mb.npairs = src_char_num;
cur_cmd->x.translate.mb.pair = trans_pairs;
for (idx_t i = 0; i < src_char_num; i++)
{
if (idx >= dest_len)
bad_prog ("'y' command strings have different lengths");
/* Set the i-th source character. */
trans_pairs[2 * i] = XNMALLOC (src_lens[i] + 1, char);
memcpy (trans_pairs[2 * i], src_buf, src_lens[i]);
trans_pairs[2 * i][src_lens[i]] = '\0';
src_buf += src_lens[i]; /* Forward to next character. */
int wc;
idx_t mbclen = mbrtowc1 (&wc, src_buf, len, &cur_stat);
trans_pairs[2 * i] = wc;
src_buf += mbclen; /* Forward to next character. */
len -= mbclen;
/* Fetch the i-th destination character. */
size_t mbclen = mbrlen1 (dest_buf + idx, dest_len - idx,
&tr_stat);
mbclen = mbrtowc1 (&wc, dest_buf + idx, dest_len - idx,
&tr_stat);
/* Set the i-th destination character. */
trans_pairs[2 * i + 1] = XNMALLOC (mbclen + 1, char);
memcpy (trans_pairs[2 * i + 1], dest_buf + idx, mbclen);
trans_pairs[2 * i + 1][mbclen] = '\0';
trans_pairs[2 * i + 1] = wc;
idx += mbclen; /* Forward to next character. */
}
trans_pairs[2 * i] = NULL;
if (idx != dest_len)
bad_prog ("'y' command strings have different lengths");
IF_LINT (free (src_lens));
}
else
{
+22 -4
View File
@@ -231,6 +231,22 @@ debug_print_subst (const struct subst *s)
}
}
static void
debug_print_mb (int ch, mbstate_t *ps)
{
if (ch < 0)
{
fputc (-ch, stdout);
mbszero (ps);
}
else
{
char buf[MB_LEN_MAX];
size_t s = wcrtomb (buf, ch, ps);
fwrite (buf, 1, s, stdout);
}
}
static void
debug_print_translation (const struct sed_cmd *sc)
{
@@ -238,11 +254,13 @@ debug_print_translation (const struct sed_cmd *sc)
{
/* multibyte translation */
putchar ('/');
for (idx_t i = 0; sc->x.translate.mb[2 * i] != NULL; i++)
fputs (sc->x.translate.mb[2 * i], stdout);
mbstate_t mbs; mbszero (&mbs);
for (idx_t i = 0; i < sc->x.translate.mb.npairs; i++)
debug_print_mb (sc->x.translate.mb.pair[2 * i], &mbs);
putchar ('/');
for (idx_t i = 0; sc->x.translate.mb[2 * i] != NULL; i++)
fputs (sc->x.translate.mb[2 * i + 1], stdout);
mbszero (&mbs);
for (idx_t i = 0; i < sc->x.translate.mb.npairs; i++)
debug_print_mb (sc->x.translate.mb.pair[2 * i + 1], &mbs);
putchar ('/');
}
else
+31 -20
View File
@@ -190,11 +190,11 @@ str_append_modified (struct line *to, const char *string, idx_t length,
from_stat = to->mbstate;
while (length)
{
wchar_t wc;
size_t n = MBRTOWC (&wc, string, length, &from_stat);
int wc;
idx_t n = mbrtowc1 (&wc, string, length, &from_stat);
/* Treat an invalid or null sequence like a single-byte character. */
if (n == (size_t) {-1} || n == (size_t) {-2} || n == 0)
/* Treat an invalid sequence like a single-byte character. */
if (wc < 0)
{
type &= ~(REPL_LOWERCASE_FIRST | REPL_UPPERCASE_FIRST);
if (type == REPL_ASIS)
@@ -224,14 +224,14 @@ str_append_modified (struct line *to, const char *string, idx_t length,
if (type == REPL_ASIS)
{
/* Copy the new wide character to the end of the string. */
n = WCRTOMB (to->active + to->length, wc, &to->mbstate);
if (n == (size_t) -1 || n == (size_t) -2)
size_t w = wcrtomb (to->active + to->length, wc, &to->mbstate);
if (w == (size_t) {-1})
{
fprintf (stderr,
_("case conversion produced an invalid character"));
abort ();
}
to->length += n;
to->length += w;
str_append (to, string, length);
return;
}
@@ -242,13 +242,13 @@ str_append_modified (struct line *to, const char *string, idx_t length,
wc = towlower (wc);
/* Copy the new wide character to the end of the string. */
n = WCRTOMB (to->active + to->length, wc, &to->mbstate);
if (n == (size_t) {-1} || n == (size_t) {-2})
size_t w = wcrtomb (to->active + to->length, wc, &to->mbstate);
if (w == (size_t) {-1})
{
fprintf (stderr, _("case conversion produced an invalid character"));
abort ();
}
to->length += n;
to->length += w;
}
}
@@ -1169,23 +1169,34 @@ do_subst (struct subst *sub)
/* Translate the global input LINE via TRANS.
This function handles the multi-byte case. */
static void
translate_mb (char *const *trans)
translate_mb (idx_t npairs, int *pair)
{
idx_t idx; /* index in the input line. */
mbstate_t mbstate; mbszero (&mbstate);
mbstate_t trstate; mbszero (&trstate);
for (idx = 0; idx < line.length;)
{
idx_t i;
size_t mbclen = mbrlen1 (line.active + idx, line.length - idx, &mbstate);
int iwc;
idx_t mbclen = mbrtowc1 (&iwc, line.active + idx, line.length - idx,
&mbstate);
/* 'i' indicate i-th translate pair. */
for (i = 0; trans[2*i] != NULL; i++)
for (idx_t i = 0; i < npairs; i++)
{
if (STREQ_LEN (line.active + idx, trans[2*i], mbclen))
if (pair[2 * i] == iwc)
{
bool move_remain_buffer = false;
const char *tr = trans[2*i+1];
idx_t trans_len = *tr == '\0' ? 1 : strlen (tr);
int tr = pair[2 * i + 1];
char mbuf[MB_LEN_MAX];
idx_t trans_len;
if (tr < 0)
{
mbuf[0] = -tr;
mbszero (&trstate);
trans_len = 1;
}
else
trans_len = wcrtomb (mbuf, tr, &trstate);
if (mbclen < trans_len)
{
@@ -1211,8 +1222,7 @@ translate_mb (char *const *trans)
line.length += move_offset;
idx += move_offset;
}
memcpy (line.active + prev_idx, trans[2*i+1],
trans_len);
memcpy (line.active + prev_idx, mbuf, trans_len);
break;
}
}
@@ -1578,7 +1588,8 @@ execute_program (struct vector *vec, struct input *input)
case 'y':
if (mb_cur_max > 1)
translate_mb (cur_cmd->x.translate.mb);
translate_mb (cur_cmd->x.translate.mb.npairs,
cur_cmd->x.translate.mb.pair);
else
{
unsigned char *p, *e;
+34 -13
View File
@@ -185,8 +185,18 @@ struct sed_cmd {
/* This is used for the y command. */
union
{
/* An array of UCHAR_MAX + 1 bytes. sb[I] is the translation of I. */
unsigned char *sb;
char **mb;
/* The number of translation pairs, and an array of pairs.
For 0 <= I < 2 * npairs, pair[I + 1] is the translation of pair[I]
when I is even, and a negative pair[I] stands for the encoding
error byte -pair[I] instead of the usual wide character pair[I]. */
struct
{
idx_t npairs;
int *pair;
} mb;
} translate;
/* This is used for the ':' command (debug only). */
@@ -271,18 +281,29 @@ extern bool sandbox;
/* If set, print debugging information. */
extern bool debug;
#define MBRTOWC(pwc, s, n, ps) \
(mb_cur_max == 1 ? \
(*(pwc) = btowc (*(unsigned char *) (s)), 1) : \
mbrtowc ((pwc), (s), (n), (ps)))
#define WCRTOMB(s, wc, ps) \
(mb_cur_max == 1 ? \
(*(s) = wctob ((wint_t) (wc)), 1) : \
wcrtomb ((s), (wc), (ps)))
#define MBSINIT(s) \
(mb_cur_max == 1 ? 1 : mbsinit ((s)))
/* Store into *PWC the representation of the first part of buffer S of size N.
If the buffer starts with a multibyte character, set *PWC = the
corresponding wide character and return the number of bytes in the
character; otherwise, set *PWC = -(unsigned char) {*S} and return 1.
*PWC is of type int, not wchar_t, so it can hold that negative value.
N must be positive. PS is the conversion status.
The returned value is in the range 1..MB_CUR_MAX. */
SED_INLINE idx_t
mbrtowc1 (int *restrict pwc, char const *restrict s, idx_t n,
mbstate_t *restrict ps)
{
wchar_t wc;
size_t r = mbrtowc (&wc, s, n, ps);
if (0 < r && r < (size_t) {-2})
{
*pwc = wc;
return r;
}
/* Treat a NUL byte or an encoding error as a single byte. */
mbszero (ps);
*pwc = -(unsigned char) {*s};
return 1;
}
/* If buffer S of size N starts with a multibyte character return its length,
otherwise return 1. N must be positive. PS is the conversion status.