sed: simplify multibyte substitution
When handling the y command, or the case-changing replacement, stay a bit more in the wchar_t world, to simplify allocation and help make future changes simpler. * basicdefs.h (STREQ_LEN): Remove; no longer used. * cfg.mk (sc_prohibit_strncmp): Don’t suggest STREQ_LEN. * sed/compile.c (compile_program): * sed/execute.c (str_append_modified, translate_mb): Instead of preserving the bytes of a multibyte character, just save its wchar_t counterpart, or negative of its single-byte encoding error. * sed/debug.c (debug_print_mb): New static function. (debug_print_translation): Use it to adjust to new data structure. * sed/execute.c (translate_mb): Args are now NPAIRS and PAIR, not TRANS. Caller changed. * sed/sed.h (struct sed_cmd.translate.mb): Now a struct with an npairs and pair members, instead of a single char **. All uses changed.
This commit is contained in:
1 parent
bd3b429b80
commit
4e2995cd01
6 files changed
+111
-72
No files matched your search
@@ -31,8 +31,6 @@
|
||||
#define obstack_chunk_alloc xzalloc
|
||||
#define obstack_chunk_free free
|
||||
|
||||
#define STREQ_LEN(a, b, n) (strncmp (a, b, n) == 0)
|
||||
|
||||
/* MAX_PATH is not defined in some platforms, most notably GNU/Hurd.
|
||||
In that case we define it here to some constant. Note however that
|
||||
this relies in the fact that sed does reallocation if a buffer
|
||||
|
||||
@@ -228,11 +228,10 @@ sc_gitignore_missing:
|
||||
'entries to lib/.gitignore' >&2; exit 1; } || :
|
||||
|
||||
|
||||
# Similar to the gnulib maint.mk rule for sc_prohibit_strcmp
|
||||
# Use STREQ_LEN rather than comparing strncmp == 0, or != 0.
|
||||
# Similar to the gnulib maint.mk rule for sc_prohibit_strcmp.
|
||||
sc_prohibit_strncmp:
|
||||
@prohibit='^[^#].*str''ncmp *\(' \
|
||||
halt='use STREQ_LEN instead of str''ncmp' \
|
||||
halt='use memeq instead of str''ncmp' \
|
||||
$(_sc_search_regexp)
|
||||
|
||||
# Ensure that tests don't include a redundant fail=0.
|
||||
|
||||
+22
-30
@@ -1207,55 +1207,47 @@ compile_program (struct vector *vector)
|
||||
|
||||
if (mb_cur_max > 1)
|
||||
{
|
||||
idx_t i, j, idx, src_char_num;
|
||||
idx_t *src_lens = XNMALLOC (len, idx_t);
|
||||
char **trans_pairs;
|
||||
mbstate_t cur_stat; mbszero (&cur_stat);
|
||||
|
||||
/* Enumerate how many character the source buffer has. */
|
||||
for (i = 0, j = 0; i < len;)
|
||||
{
|
||||
size_t mbclen = mbrlen1 (src_buf + i, len - i, &cur_stat);
|
||||
src_lens[j++] = mbclen;
|
||||
i += mbclen;
|
||||
}
|
||||
src_char_num = j;
|
||||
idx_t src_char_num = 0;
|
||||
for (idx_t i = 0; i < len;
|
||||
i += mbrlen1 (src_buf + i, len - i, &cur_stat))
|
||||
src_char_num++;
|
||||
|
||||
mbszero (&cur_stat);
|
||||
mbstate_t tr_stat; mbszero (&tr_stat);
|
||||
idx = 0;
|
||||
idx_t idx = 0;
|
||||
|
||||
/* trans_pairs = {src(0), dest(0), src(1), dest(1), ..., NULL}
|
||||
src(i) : pointer to i-th source character.
|
||||
dest(i) : pointer to i-th destination character.
|
||||
NULL : terminator */
|
||||
trans_pairs = XNMALLOC (2 * src_char_num + 1, char *);
|
||||
cur_cmd->x.translate.mb = trans_pairs;
|
||||
for (i = 0; i < src_char_num; i++)
|
||||
/* trans_pairs = {src(0), dest(0), src(1), dest(1), ... }
|
||||
src(i) : i-th source character.
|
||||
dest(i) : i-th destination character. */
|
||||
int *trans_pairs = xnmalloc (src_char_num,
|
||||
2 * sizeof *trans_pairs);
|
||||
cur_cmd->x.translate.mb.npairs = src_char_num;
|
||||
cur_cmd->x.translate.mb.pair = trans_pairs;
|
||||
for (idx_t i = 0; i < src_char_num; i++)
|
||||
{
|
||||
if (idx >= dest_len)
|
||||
bad_prog ("'y' command strings have different lengths");
|
||||
|
||||
/* Set the i-th source character. */
|
||||
trans_pairs[2 * i] = XNMALLOC (src_lens[i] + 1, char);
|
||||
memcpy (trans_pairs[2 * i], src_buf, src_lens[i]);
|
||||
trans_pairs[2 * i][src_lens[i]] = '\0';
|
||||
src_buf += src_lens[i]; /* Forward to next character. */
|
||||
int wc;
|
||||
idx_t mbclen = mbrtowc1 (&wc, src_buf, len, &cur_stat);
|
||||
trans_pairs[2 * i] = wc;
|
||||
src_buf += mbclen; /* Forward to next character. */
|
||||
len -= mbclen;
|
||||
|
||||
/* Fetch the i-th destination character. */
|
||||
size_t mbclen = mbrlen1 (dest_buf + idx, dest_len - idx,
|
||||
&tr_stat);
|
||||
mbclen = mbrtowc1 (&wc, dest_buf + idx, dest_len - idx,
|
||||
&tr_stat);
|
||||
|
||||
/* Set the i-th destination character. */
|
||||
trans_pairs[2 * i + 1] = XNMALLOC (mbclen + 1, char);
|
||||
memcpy (trans_pairs[2 * i + 1], dest_buf + idx, mbclen);
|
||||
trans_pairs[2 * i + 1][mbclen] = '\0';
|
||||
trans_pairs[2 * i + 1] = wc;
|
||||
idx += mbclen; /* Forward to next character. */
|
||||
}
|
||||
trans_pairs[2 * i] = NULL;
|
||||
if (idx != dest_len)
|
||||
bad_prog ("'y' command strings have different lengths");
|
||||
|
||||
IF_LINT (free (src_lens));
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
+22
-4
@@ -231,6 +231,22 @@ debug_print_subst (const struct subst *s)
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
debug_print_mb (int ch, mbstate_t *ps)
|
||||
{
|
||||
if (ch < 0)
|
||||
{
|
||||
fputc (-ch, stdout);
|
||||
mbszero (ps);
|
||||
}
|
||||
else
|
||||
{
|
||||
char buf[MB_LEN_MAX];
|
||||
size_t s = wcrtomb (buf, ch, ps);
|
||||
fwrite (buf, 1, s, stdout);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
debug_print_translation (const struct sed_cmd *sc)
|
||||
{
|
||||
@@ -238,11 +254,13 @@ debug_print_translation (const struct sed_cmd *sc)
|
||||
{
|
||||
/* multibyte translation */
|
||||
putchar ('/');
|
||||
for (idx_t i = 0; sc->x.translate.mb[2 * i] != NULL; i++)
|
||||
fputs (sc->x.translate.mb[2 * i], stdout);
|
||||
mbstate_t mbs; mbszero (&mbs);
|
||||
for (idx_t i = 0; i < sc->x.translate.mb.npairs; i++)
|
||||
debug_print_mb (sc->x.translate.mb.pair[2 * i], &mbs);
|
||||
putchar ('/');
|
||||
for (idx_t i = 0; sc->x.translate.mb[2 * i] != NULL; i++)
|
||||
fputs (sc->x.translate.mb[2 * i + 1], stdout);
|
||||
mbszero (&mbs);
|
||||
for (idx_t i = 0; i < sc->x.translate.mb.npairs; i++)
|
||||
debug_print_mb (sc->x.translate.mb.pair[2 * i + 1], &mbs);
|
||||
putchar ('/');
|
||||
}
|
||||
else
|
||||
|
||||
+31
-20
@@ -190,11 +190,11 @@ str_append_modified (struct line *to, const char *string, idx_t length,
|
||||
from_stat = to->mbstate;
|
||||
while (length)
|
||||
{
|
||||
wchar_t wc;
|
||||
size_t n = MBRTOWC (&wc, string, length, &from_stat);
|
||||
int wc;
|
||||
idx_t n = mbrtowc1 (&wc, string, length, &from_stat);
|
||||
|
||||
/* Treat an invalid or null sequence like a single-byte character. */
|
||||
if (n == (size_t) {-1} || n == (size_t) {-2} || n == 0)
|
||||
/* Treat an invalid sequence like a single-byte character. */
|
||||
if (wc < 0)
|
||||
{
|
||||
type &= ~(REPL_LOWERCASE_FIRST | REPL_UPPERCASE_FIRST);
|
||||
if (type == REPL_ASIS)
|
||||
@@ -224,14 +224,14 @@ str_append_modified (struct line *to, const char *string, idx_t length,
|
||||
if (type == REPL_ASIS)
|
||||
{
|
||||
/* Copy the new wide character to the end of the string. */
|
||||
n = WCRTOMB (to->active + to->length, wc, &to->mbstate);
|
||||
if (n == (size_t) -1 || n == (size_t) -2)
|
||||
size_t w = wcrtomb (to->active + to->length, wc, &to->mbstate);
|
||||
if (w == (size_t) {-1})
|
||||
{
|
||||
fprintf (stderr,
|
||||
_("case conversion produced an invalid character"));
|
||||
abort ();
|
||||
}
|
||||
to->length += n;
|
||||
to->length += w;
|
||||
str_append (to, string, length);
|
||||
return;
|
||||
}
|
||||
@@ -242,13 +242,13 @@ str_append_modified (struct line *to, const char *string, idx_t length,
|
||||
wc = towlower (wc);
|
||||
|
||||
/* Copy the new wide character to the end of the string. */
|
||||
n = WCRTOMB (to->active + to->length, wc, &to->mbstate);
|
||||
if (n == (size_t) {-1} || n == (size_t) {-2})
|
||||
size_t w = wcrtomb (to->active + to->length, wc, &to->mbstate);
|
||||
if (w == (size_t) {-1})
|
||||
{
|
||||
fprintf (stderr, _("case conversion produced an invalid character"));
|
||||
abort ();
|
||||
}
|
||||
to->length += n;
|
||||
to->length += w;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1169,23 +1169,34 @@ do_subst (struct subst *sub)
|
||||
/* Translate the global input LINE via TRANS.
|
||||
This function handles the multi-byte case. */
|
||||
static void
|
||||
translate_mb (char *const *trans)
|
||||
translate_mb (idx_t npairs, int *pair)
|
||||
{
|
||||
idx_t idx; /* index in the input line. */
|
||||
mbstate_t mbstate; mbszero (&mbstate);
|
||||
mbstate_t trstate; mbszero (&trstate);
|
||||
for (idx = 0; idx < line.length;)
|
||||
{
|
||||
idx_t i;
|
||||
size_t mbclen = mbrlen1 (line.active + idx, line.length - idx, &mbstate);
|
||||
int iwc;
|
||||
idx_t mbclen = mbrtowc1 (&iwc, line.active + idx, line.length - idx,
|
||||
&mbstate);
|
||||
|
||||
/* 'i' indicate i-th translate pair. */
|
||||
for (i = 0; trans[2*i] != NULL; i++)
|
||||
for (idx_t i = 0; i < npairs; i++)
|
||||
{
|
||||
if (STREQ_LEN (line.active + idx, trans[2*i], mbclen))
|
||||
if (pair[2 * i] == iwc)
|
||||
{
|
||||
bool move_remain_buffer = false;
|
||||
const char *tr = trans[2*i+1];
|
||||
idx_t trans_len = *tr == '\0' ? 1 : strlen (tr);
|
||||
int tr = pair[2 * i + 1];
|
||||
char mbuf[MB_LEN_MAX];
|
||||
idx_t trans_len;
|
||||
if (tr < 0)
|
||||
{
|
||||
mbuf[0] = -tr;
|
||||
mbszero (&trstate);
|
||||
trans_len = 1;
|
||||
}
|
||||
else
|
||||
trans_len = wcrtomb (mbuf, tr, &trstate);
|
||||
|
||||
if (mbclen < trans_len)
|
||||
{
|
||||
@@ -1211,8 +1222,7 @@ translate_mb (char *const *trans)
|
||||
line.length += move_offset;
|
||||
idx += move_offset;
|
||||
}
|
||||
memcpy (line.active + prev_idx, trans[2*i+1],
|
||||
trans_len);
|
||||
memcpy (line.active + prev_idx, mbuf, trans_len);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -1578,7 +1588,8 @@ execute_program (struct vector *vec, struct input *input)
|
||||
|
||||
case 'y':
|
||||
if (mb_cur_max > 1)
|
||||
translate_mb (cur_cmd->x.translate.mb);
|
||||
translate_mb (cur_cmd->x.translate.mb.npairs,
|
||||
cur_cmd->x.translate.mb.pair);
|
||||
else
|
||||
{
|
||||
unsigned char *p, *e;
|
||||
|
||||
@@ -185,8 +185,18 @@ struct sed_cmd {
|
||||
/* This is used for the y command. */
|
||||
union
|
||||
{
|
||||
/* An array of UCHAR_MAX + 1 bytes. sb[I] is the translation of I. */
|
||||
unsigned char *sb;
|
||||
char **mb;
|
||||
|
||||
/* The number of translation pairs, and an array of pairs.
|
||||
For 0 <= I < 2 * npairs, pair[I + 1] is the translation of pair[I]
|
||||
when I is even, and a negative pair[I] stands for the encoding
|
||||
error byte -pair[I] instead of the usual wide character pair[I]. */
|
||||
struct
|
||||
{
|
||||
idx_t npairs;
|
||||
int *pair;
|
||||
} mb;
|
||||
} translate;
|
||||
|
||||
/* This is used for the ':' command (debug only). */
|
||||
@@ -271,18 +281,29 @@ extern bool sandbox;
|
||||
/* If set, print debugging information. */
|
||||
extern bool debug;
|
||||
|
||||
#define MBRTOWC(pwc, s, n, ps) \
|
||||
(mb_cur_max == 1 ? \
|
||||
(*(pwc) = btowc (*(unsigned char *) (s)), 1) : \
|
||||
mbrtowc ((pwc), (s), (n), (ps)))
|
||||
|
||||
#define WCRTOMB(s, wc, ps) \
|
||||
(mb_cur_max == 1 ? \
|
||||
(*(s) = wctob ((wint_t) (wc)), 1) : \
|
||||
wcrtomb ((s), (wc), (ps)))
|
||||
|
||||
#define MBSINIT(s) \
|
||||
(mb_cur_max == 1 ? 1 : mbsinit ((s)))
|
||||
/* Store into *PWC the representation of the first part of buffer S of size N.
|
||||
If the buffer starts with a multibyte character, set *PWC = the
|
||||
corresponding wide character and return the number of bytes in the
|
||||
character; otherwise, set *PWC = -(unsigned char) {*S} and return 1.
|
||||
*PWC is of type int, not wchar_t, so it can hold that negative value.
|
||||
N must be positive. PS is the conversion status.
|
||||
The returned value is in the range 1..MB_CUR_MAX. */
|
||||
SED_INLINE idx_t
|
||||
mbrtowc1 (int *restrict pwc, char const *restrict s, idx_t n,
|
||||
mbstate_t *restrict ps)
|
||||
{
|
||||
wchar_t wc;
|
||||
size_t r = mbrtowc (&wc, s, n, ps);
|
||||
if (0 < r && r < (size_t) {-2})
|
||||
{
|
||||
*pwc = wc;
|
||||
return r;
|
||||
}
|
||||
/* Treat a NUL byte or an encoding error as a single byte. */
|
||||
mbszero (ps);
|
||||
*pwc = -(unsigned char) {*s};
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* If buffer S of size N starts with a multibyte character return its length,
|
||||
otherwise return 1. N must be positive. PS is the conversion status.
|
||||
|
||||
Reference in new issue
Block a user