sed: fix minor multibyte parsing bug

Previously sed would parse multibyte characters incorrectly in two scenarios:

1. Slash following an incomplete-yet-valid multibyte sequence (match_slash):
    $ LC_ALL=en_US.UTF-8 sed $'s/\316/X/'
    sed: -e expression #1, char 6: unterminated `s' command

2. Open/close brackets as part of a valid mutilbyte string inside a character
class (snarf_char_class). In the example below, '\203]' is a valid
multibyte character in SHIFT-JIS locale:

    $ LC_ALL=ja_JP.shiftjis sed $'/[\203]/]/p'
    sed: -e expression #1, char #5: Unmatched [ or [^

Both cases stem from mbcs.c:brlen() being non-intuitive:
It returned 1 for valid single-byte character, invalid multibyte-character,
and a for the last byte of a valid multibyte sequence - making it
non-trivial to use correctly.

This commit replaces brlen() with a simpler is_mb_char() function:
returns non-zero for multibyte sequences, zero for single/invalid sequences.

* sed/sed.h: (BRLEN, brlen): Remove delaration.
(IS_MB_CHAR,is_mb_char): Add macro and function declaration.
* sed/mbcs.c: (brlen): Remove function. (is_mb_char): New function.
* sed/compile.c: (snarf_char_class, match_slash): Use IS_MB_CHAR instead of
BRLEN; Adjust local variables accordingly.
* testsuite/mb-match-slash.sh: New test for scenario 1.
* testsuite/mb-charclass-non-utf8.sh: New test for scenario 2,
requires SHIFT-JIS locale.
* testsuite/Makefile.am: Add new tests
* testsuite/init.cfg: (require_ja_shiftjis_locale_): New function.
* NEWS: Mention bug fix.
This commit is contained in:
Assaf Gordon committed 2016-07-04 14:26:09 -04:00
1 parent 6ad4acfbd5
commit 0052daf1e5
8 files changed
+248 -24

No files matched your search

+7
View File
@@ -54,6 +54,13 @@ GNU sed NEWS -*- outline -*-
Other characters after the second backslash are rejected (e.g. '\c\d').
[bug introduced in the sed-4.0.* releases]
sed no longer mishandles incomplete multibyte sequences in s,y commands
and valid multibyte SHIFT-JIS characters in character classes.
Previously, the following commands would fail:
LC_ALL=en_US.UTF-8 sed $'s/\316/X/'
LC_ALL=ja_JP.shiftjis sed $'/[\203]/]/p'
[bug introduced some time after sed-4.1.5 and before sed-4.2.1]
** Feature removal
The "L" command (format a paragraph like the fmt(1) command would)
+8 -8
View File
@@ -432,7 +432,6 @@ snarf_char_class (struct buffer *b, mbstate_t *cur_stat)
int ch;
int state = 0;
int delim IF_LINT ( = 0) ;
bool pending_mb = 0;
ch = inchar();
if (ch == '^')
@@ -448,7 +447,7 @@ snarf_char_class (struct buffer *b, mbstate_t *cur_stat)
for (;; ch = add_then_next (b, ch))
{
pending_mb = BRLEN (ch, cur_stat) != 1;
const int mb_char = IS_MB_CHAR (ch, cur_stat);
switch (ch)
{
@@ -459,7 +458,7 @@ snarf_char_class (struct buffer *b, mbstate_t *cur_stat)
case '.':
case ':':
case '=':
if (pending_mb)
if (mb_char)
continue;
if (state == 1)
@@ -475,7 +474,7 @@ snarf_char_class (struct buffer *b, mbstate_t *cur_stat)
continue;
case OPEN_BRACKET:
if (pending_mb)
if (mb_char)
continue;
if (state == 0)
@@ -483,7 +482,7 @@ snarf_char_class (struct buffer *b, mbstate_t *cur_stat)
continue;
case CLOSE_BRACKET:
if (pending_mb)
if (mb_char)
continue;
if (state == 0 || state == 1)
@@ -512,7 +511,7 @@ match_slash (int slash, int regex)
mbstate_t cur_stat = { 0, };
/* We allow only 1 byte characters for a slash. */
if (BRLEN (slash, &cur_stat) == -2)
if (IS_MB_CHAR (slash, &cur_stat))
bad_prog (BAD_DELIM);
memset (&cur_stat, 0, sizeof cur_stat);
@@ -520,8 +519,9 @@ match_slash (int slash, int regex)
b = init_buffer();
while ((ch = inchar()) != EOF && ch != '\n')
{
bool pending_mb = !MBSINIT (&cur_stat);
if (BRLEN (ch, &cur_stat) == 1 && !pending_mb)
const int mb_char = IS_MB_CHAR (ch, &cur_stat);
if (!mb_char)
{
if (ch == slash)
return b;
+32 -13
View File
@@ -24,24 +24,43 @@
int mb_cur_max;
bool is_utf8;
/* Add a byte to the multibyte character represented by the state
CUR_STAT, and answer its length if a character is completed,
or -2 if it is yet to be completed. */
int brlen (int ch, mbstate_t *cur_stat)
/* Return non-zero if CH is part of a valid multibyte sequence:
Either incomplete yet valid sequence (in case of a leading byte),
or the last byte of a valid multibyte sequence.
Return zero in all other cases:
CH is a valid single-byte character (e.g. 0x01-0x7F in UTF-8 locales);
CH is an invalid byte in a multibyte sequence for the currentl locale,
CH is the NUL byte.
Reset CUR_STAT in the case of an invalid byte.
*/
int
is_mb_char (int ch, mbstate_t *cur_stat)
{
char c = ch;
const char c = ch ;
const int mb_pending = !mbsinit (cur_stat);
const int result = mbrtowc (NULL, &c, 1, cur_stat);
/* If we use the generic brlen, then MBRLEN == mbrlen. */
int result = mbrtowc(NULL, &c, 1, cur_stat);
/* An invalid sequence is treated like a single-byte character. */
if (result == -1)
switch (result)
{
memset (cur_stat, 0, sizeof (mbstate_t));
case -2: /* Beginning or middle of valid multibyte sequence */
return 1;
}
return result;
case -1: /* Invalid sequence, byte treated like a single-byte character */
memset (cur_stat, 0, sizeof (mbstate_t));
return 0;
case 1: /* A valid byte, check if part of on-going multibyte sequence */
return mb_pending;
case 0: /* Special case of mbrtowc(3): the NUL character */
/* TODO: test this */
return 1;
default: /* Should never happen, as per mbrtowc(3) documentation */
panic ("is_mb_char: mbrtowc (0x%x) returned %d",ch,result);
}
}
void
+3 -3
View File
@@ -254,10 +254,10 @@ extern bool is_utf8;
#define MBRLEN(s, n, ps) \
(mb_cur_max == 1 ? 1 : mbrtowc (NULL, s, n, ps))
#define BRLEN(ch, ps) \
(mb_cur_max == 1 ? 1 : brlen (ch, ps))
#define IS_MB_CHAR(ch, ps) \
(mb_cur_max == 1 ? 0 : is_mb_char (ch, ps))
extern int brlen (int ch, mbstate_t *ps);
extern int is_mb_char (int ch, mbstate_t *ps);
extern void initialize_mbcs (void);
extern void register_cleanup_file (char const *file);
extern void cancel_cleanup (void);
+2
View File
@@ -19,6 +19,8 @@ T = \
in-place-hyphen.sh \
in-place-suffix-backup.sh \
invalid-mb-seq-UMR.sh \
mb-charclass-non-utf8.sh \
mb-match-slash.sh \
normalize-text.sh \
nulldata.sh \
panic-tests.sh \
+19
View File
@@ -51,3 +51,22 @@ require_en_utf8_locale_()
*) skip_ 'en_US.UTF-8 locale not found' ;;
esac
}
# Some tests would fail without this particular locale.
# If the locale is not available, just skip the test.
# The exact spelling differs between operating systems
# (ja_JP.shiftjis on Ubuntu, ja_JP.sjis on Debian, ja_JP.SJIS on Mac OS X).
# If a sjift-jis locale is found the function sets shell variable
# 'LOCALE_JA_SJIS' to the locale name.
require_ja_shiftjis_locale_()
{
path_prepend_ .
LOCALE_JA_SJIS=
for l in shiftjis sjis SJIS ; do
n=$(get-mb-cur-max ja_JP.$l) || continue
test 2 -eq "$n" || continue
LOCALE_JA_SJIS="ja_JP.$l"
break
done
test -z "$LOCALE_JA_SJIS" && skip_ 'ja_JP shift-jis locale not found'
}
+129
View File
@@ -0,0 +1,129 @@
#!/bin/sh
# Test multibyte locale which is not UTF-8 (ja_JP.shift_jis)
# This is a stateful locale. Same byte value can be either
# a single-byte character, or the second byte of a multibyte
# character.
# Copyright (C) 2016 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
. "${srcdir=.}/init.sh"; path_prepend_ ../sed
print_ver_ sed
# If found, LOCALE_JA_SJIS will contain the locale name.
require_ja_shiftjis_locale_
fail=0
# This test uses two characters:
# Unicode Character 'KATAKANA LETTER ZE' (U+30BC)
# Unicode Character 'KATAKANA LETTER ZO' (U+30BE)
#
# In SHIFT-JIS locale, these multibyte characters contain
# open/close brackets (ASCII 0x5B/0x5D) as the trailing byte.
#
# See also:
# https://en.wikipedia.org/wiki/Shift_JIS
# http://www.rikai.com/library/kanjitables/kanji_codes.sjis.shtml
# Unicode Character 'KATAKANA LETTER ZE' (U+30BC)
#
# UTF-8: hex: 0xE3 0x82 0xBC
# bin: 11100011 10000010 10111100
#
# Shift-jis hex: 0x83 0x5B
# oct: 203 133
# bin: 10000011 01011011
#
# Conversion example:
# $ printf '\x83\x5B' | iconv -f SHIFT-JIS -t UTF-8 | od -tx1o1c
# 0000000 e3 82 bc
# 343 202 274
# 343 202 274
# Unicode Character 'KATAKANA LETTER ZO' (U+30BE)
#
# UTF-8: hex: 0xE3 0x82 0xBE
# bin: 11100011 10000010 10111110
#
# Shift-jis hex: 0x83 0x5D
# oct: 203 135
# bin: 10000011 01011101
#
# Conversion example:
# $ printf '\x83\x5D' | iconv -f SHIFT-JIS -t UTF-8 | od -tx1o1c
# 0000000 e3 82 be
# 343 202 276
# 343 202 276
#
#
# Tests 1,2: Test y/// command with multibyte, non-utf8 seqeunce.
# Implmenetation notes: str_append() has special code path for non-utf8 cases.
#
# Test 1: valid multibyte seqeunce
printf 'y/a/\203\133/' > p1 || framework_failure_
echo Xa > in1 || framework_failure_
printf 'X\203\133\n' > exp1 || framework_failure_
LC_ALL="$LOCALE_JA_SJIS" sed -f p1 <in1 >out1 || fail=1
compare_ exp1 out1 || fail=1
# Test 2: invalid multibyte seqeunce, treated as two single-byte characters.
printf 'y/aa/\203\060/' > p2 || framework_failure_
LC_ALL="$LOCALE_JA_SJIS" sed -f p2 </dev/null 2>out2 || fail=1
compare_ /dev/null out2 || fail=1
#
# Test 3: multibyte character class with these characters.
#
# Before sed-4.3, snarf_char_class would parse it incorrectly,
# Treating the first closing-bracket as closing the character-class,
# instead of being part of a multibyte sequence.
printf '/[\203]/]/p' > p3 || framework_failure_
LC_ALL="$LOCALE_JA_SJIS" sed -f p3 </dev/null >out3 || fail=1
compare_ /dev/null out3 || fail=1
# Test 4:
# Same as test 3, but with the other multibyte character.
# (this did not cause a failure before sed-4.3, but the code was incorrect).
# Keep this test for code-coverage purposes.
printf '/[\203[/]/p' > p4 || framework_failure_
LC_ALL="$LOCALE_JA_SJIS" sed -f p4 </dev/null >out4 || fail=1
compare_ /dev/null out4 || fail=1
# TODO: Find a locale in which ':.=' can be part of a valid multibyte octet.
#
# snarf_char_class specifically tests for five bytes: ':.=[]' .
# '[' and ']' are tested above, yet '.:=' are not valid as part of a
# multibyte shift-jis sequence.
#
# valid:
# $ printf '\203]' | iconv -f SHIFT-JIS -t utf-8
# $ printf '\203[' | iconv -f SHIFT-JIS -t utf-8
#
# invalid:
# $ printf '\203:' | iconv -f SHIFT-JIS -t utf-8
# iconv: (stdin):1:0: cannot convert
#
# $ printf '\203=' | iconv -f SHIFT-JIS -t utf-8
# iconv: (stdin):1:0: cannot convert
#
# $ printf '\203.' | iconv -f SHIFT-JIS -t utf-8
# iconv: (stdin):0:0: cannot convert
Exit $fail
+48
View File
@@ -0,0 +1,48 @@
#!/bin/sh
# Test slash following an incomplete multibyte character
# Copyright (C) 2016 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
. "${srcdir=.}/init.sh"; path_prepend_ ../sed
print_ver_ sed
require_en_utf8_locale_
fail=0
# before sed-4.3, a slash following an incomplete multibyte character
# would be ignored during program compilation, leading to an error.
# Test 1: match_slash in 's' command.
# Before sed-4.3, this would fail with "unterminated `s' command".
printf 's/\316/X/' > p1 || framework_failure_
LC_ALL=en_US.UTF-8 sed -f p1 </dev/null >out1 || fail=1
compare_ /dev/null out1 || fail=1
# Test 2: match_slash in address regex.
# Before sed-4.3, this would fail with "unterminated address regex".
printf '/\316/p' >p2 || framework_failure_
LC_ALL=en_US.UTF-8 sed -f p2 </dev/null >out2 || fail=1
compare_ /dev/null out2 || fail=1
# Test 3: match_slash in 'y' command..
# Before sed-4.3, this would fail with "unterminated `y' command".
printf 'y/\316/X/' >p3 || framework_failure_
LC_ALL=en_US.UTF-8 sed -f p3 </dev/null >out3 || fail=1
compare_ /dev/null out3 || fail=1
Exit $fail