From 41e0cf2dac2364bc5c46c516143eccb59408e4ff Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 29 Sep 2008 09:31:27 +0200 Subject: [PATCH] add `z' extension 2008-09-29 Paolo Bonzini * BUGS: Document s/.*.// behavior with invalid multibyte sequences. * NEWS: Document `z' extension. * doc/sed-in.texi: Document both things. * sed/compile.c (compile_program): Recognize `z'. * sed/execute.c (execute_program): Execute `z'. * testsuite/Makefile.am: Add badenc test. * testsuite/Makefile.tests: Add badenc test. * testsuite/badenc.good: New. * testsuite/badenc.inp: New. * testsuite/badenc.sed: New. --- BUGS | 17 ++++++++++++++--- ChangeLog | 13 +++++++++++++ NEWS | 16 ++++++++++++---- doc/sed-in.texi | 34 ++++++++++++++++++++++++++++++++++ doc/sed.texi | 34 ++++++++++++++++++++++++++++++++++ sed/compile.c | 1 + sed/execute.c | 4 ++++ testsuite/Makefile.am | 3 ++- testsuite/Makefile.tests | 15 +++++++++++++++ testsuite/badenc.good | 9 +++++++++ testsuite/badenc.inp | 9 +++++++++ testsuite/badenc.sed | 1 + 12 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 testsuite/badenc.good create mode 100644 testsuite/badenc.inp create mode 100644 testsuite/badenc.sed diff --git a/BUGS b/BUGS index 499662b..138f3cd 100644 --- a/BUGS +++ b/BUGS @@ -103,6 +103,7 @@ regex syntax clashes (problems with backslashes) `[a-z]' is case insensitive +`s/.*//' does not clear pattern space You are encountering problems with locales. POSIX mandates that `[a-z]' uses the current locale's collation order -- in C parlance, that means @@ -117,6 +118,16 @@ regex syntax clashes (problems with backslashes) comes after `a' and before `b'; `ll' behaves similarly in Spanish locales, or `ij' in Dutch locales. - To work around these problems, which may cause bugs in shell scripts, - set the LC_ALL environment variable to `C', or set the locale on a - more fine-grained basis with the other LC_* environment variables. + Another common localization-related problem happens if your input stream + includes invalid multibyte sequences. POSIX mandates that such + sequences are _not_ matched by `.', so that `s/.*//' will not clear + pattern space as you would expect. In fact, there is no way to clear + sed's buffers in the middle of the script in most multibyte locales + (including UTF-8 locales). For this reason, GNU sed provides a `z' + command (for `zap') as an extension. + + However, to work around both of these problems, which may cause bugs + in shell scripts, you can set the LC_ALL environment variable to `C', + or set the locale on a more fine-grained basis with the other LC_* + environment variables. + diff --git a/ChangeLog b/ChangeLog index d1bc4b5..f394bc4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,16 @@ +2008-09-29 Paolo Bonzini + + * BUGS: Document s/.*.// behavior with invalid multibyte sequences. + * NEWS: Document `z' extension. + * doc/sed-in.texi: Document both things. + * sed/compile.c (compile_program): Recognize `z'. + * sed/execute.c (execute_program): Execute `z'. + * testsuite/Makefile.am: Add badenc test. + * testsuite/Makefile.tests: Add badenc test. + * testsuite/badenc.good: New. + * testsuite/badenc.inp: New. + * testsuite/badenc.sed: New. + 2008-09-28 Paolo Bonzini * basicdefs.h: Replace alloca cruft with alloca.h. diff --git a/NEWS b/NEWS index 952920f..96b2b0d 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,18 @@ +Sed 4.1f + +* added a new extension `z` to clear pattern space even in the presence +of invalid multibyte sequences + +* multibyte processing fixed + +---------------------------------------------------------------------------- Sed 4.1e * fixed --follow-symlinks to avoid attempting the creation of cross-device links. ---------------------------------------------------------------------------- -Sed 4.1d (final release candidate for sed 4.2) +Sed 4.1 * fixed bug in 'i\' giving a segmentation violation if given alone. @@ -13,12 +21,12 @@ Sed 4.1d (final release candidate for sed 4.2) * GPLv3 adopted ---------------------------------------------------------------------------- -Sed 4.1c (release candidate for sed 4.2) +Sed 4.1c * portability fixes ---------------------------------------------------------------------------- -Sed 4.1b (release candidate for sed 4.2) +Sed 4.1b * --posix disables all extensions to regular expressions. @@ -30,7 +38,7 @@ Sed 4.1b (release candidate for sed 4.2) indistinguishable from a correct implementation. ---------------------------------------------------------------------------- -Sed 4.1a (release candidate for sed 4.2) +Sed 4.1a * much improved portability diff --git a/doc/sed-in.texi b/doc/sed-in.texi index cee2505..c8bb21d 100644 --- a/doc/sed-in.texi +++ b/doc/sed-in.texi @@ -1489,6 +1489,18 @@ This command enables all @value{SSEDEXT} even if Write to the given filename the portion of the pattern space up to the first newline. Everything said under the @code{w} command about file handling holds here too. + +@item z +@findex z (Zap) command +@cindex @value{SSEDEXT}, emptying pattern space +@cindex Emptying pattern space +This command empties the content of pattern space. It is +usually the same as @samp{s/.*//}, but is more efficient +and works in the presence of invalid multibyte sequences +in the input stream. @sc{posix} mandates that such sequences +are @emph{not} matched by @samp{.}, so that there is no portable +way to clear @command{sed}'s buffers in the middle of the +script in most multibyte locales (including UTF-8 locales). @end table @node Escapes @@ -2738,6 +2750,10 @@ in a read-only directory, and will break hard or symbolic links when @option{-i} is used on such a file. @item @code{0a} does not work (gives an error) +@cindex @code{0} address +@cindex @acronym{GNU} extensions, @code{0} address +@cindex Non-bugs, @code{0} address + There is no line 0. 0 is a special address that is only used to treat addresses like @code{0,/@var{RE}/} as active when the script starts: if you write @code{1,/abc/d} and the first line includes the word @samp{abc}, @@ -2748,6 +2764,8 @@ is obtained with @code{0,/abc/d}. @ifclear PERL @item @code{[a-z]} is case insensitive +@cindex Non-bugs, localization-related + You are encountering problems with locales. POSIX mandates that @code{[a-z]} uses the current locale's collation order -- in C parlance, that means using @code{strcoll(3)} instead of @code{strcmp(3)}. Some locales have a @@ -2762,6 +2780,22 @@ because this is a single collating symbol that comes after @samp{a} and before @samp{b}; @samp{ll} behaves similarly in Spanish locales, or @samp{ij} in Dutch locales. +To work around these problems, which may cause bugs in shell scripts, set +the @env{LC_COLLATE} and @env{LC_CTYPE} environment variables to @samp{C}. + +@item @code{s/.*//} does not clear pattern space +@cindex Non-bugs, localization-related +@cindex @value{SSEDEXT}, emptying pattern space +@cindex Emptying pattern space + +This happens if your input stream includes invalid multibyte +sequences. @sc{posix} mandates that such sequences +are @emph{not} matched by @samp{.}, so that @samp{s/.*//} will not clear +pattern space as you would expect. In fact, there is no way to clear +sed's buffers in the middle of the script in most multibyte locales +(including UTF-8 locales). For this reason, @value{SSED} provides a `z' +command (for `zap') as an extension. + To work around these problems, which may cause bugs in shell scripts, set the @env{LC_COLLATE} and @env{LC_CTYPE} environment variables to @samp{C}. @end ifclear diff --git a/doc/sed.texi b/doc/sed.texi index 5b74a51..72408e5 100644 --- a/doc/sed.texi +++ b/doc/sed.texi @@ -1490,6 +1490,18 @@ This command enables all @value{SSEDEXT} even if Write to the given filename the portion of the pattern space up to the first newline. Everything said under the @code{w} command about file handling holds here too. + +@item z +@findex z (Zap) command +@cindex @value{SSEDEXT}, emptying pattern space +@cindex Emptying pattern space +This command empties the content of pattern space. It is +usually the same as @samp{s/.*//}, but is more efficient +and works in the presence of invalid multibyte sequences +in the input stream. @sc{posix} mandates that such sequences +are @emph{not} matched by @samp{.}, so that there is no portable +way to clear @command{sed}'s buffers in the middle of the +script in most multibyte locales (including UTF-8 locales). @end table @node Escapes @@ -2905,6 +2917,10 @@ in a read-only directory, and will break hard or symbolic links when @option{-i} is used on such a file. @item @code{0a} does not work (gives an error) +@cindex @code{0} address +@cindex @acronym{GNU} extensions, @code{0} address +@cindex Non-bugs, @code{0} address + There is no line 0. 0 is a special address that is only used to treat addresses like @code{0,/@var{RE}/} as active when the script starts: if you write @code{1,/abc/d} and the first line includes the word @samp{abc}, @@ -2915,6 +2931,8 @@ is obtained with @code{0,/abc/d}. @ifclear PERL @item @code{[a-z]} is case insensitive +@cindex Non-bugs, localization-related + You are encountering problems with locales. POSIX mandates that @code{[a-z]} uses the current locale's collation order -- in C parlance, that means using @code{strcoll(3)} instead of @code{strcmp(3)}. Some locales have a @@ -2929,6 +2947,22 @@ because this is a single collating symbol that comes after @samp{a} and before @samp{b}; @samp{ll} behaves similarly in Spanish locales, or @samp{ij} in Dutch locales. +To work around these problems, which may cause bugs in shell scripts, set +the @env{LC_COLLATE} and @env{LC_CTYPE} environment variables to @samp{C}. + +@item @code{s/.*//} does not clear pattern space +@cindex Non-bugs, localization-related +@cindex @value{SSEDEXT}, emptying pattern space +@cindex Emptying pattern space + +This happens if your input stream includes invalid multibyte +sequences. @sc{posix} mandates that such sequences +are @emph{not} matched by @samp{.}, so that @samp{s/.*//} will not clear +pattern space as you would expect. In fact, there is no way to clear +sed's buffers in the middle of the script in most multibyte locales +(including UTF-8 locales). For this reason, @value{SSED} provides a `z' +command (for `zap') as an extension. + To work around these problems, which may cause bugs in shell scripts, set the @env{LC_COLLATE} and @env{LC_CTYPE} environment variables to @samp{C}. @end ifclear diff --git a/sed/compile.c b/sed/compile.c index a5a6cca..2b471ee 100644 --- a/sed/compile.c +++ b/sed/compile.c @@ -1227,6 +1227,7 @@ compile_program(vector) case 'N': case 'p': case 'P': + case 'z': case 'x': ch = in_nonblank(); if (ch == CLOSE_BRACE || ch == '#') diff --git a/sed/execute.c b/sed/execute.c index c4ff1fd..8048cfc 100644 --- a/sed/execute.c +++ b/sed/execute.c @@ -1695,6 +1695,10 @@ execute_program(vec, input) } break; + case 'z': + line.length = 0; + break; + case '=': output_missing_newline(&output_file); fprintf(output_file.fp, "%lu\n", diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am index 95d016a..660c53b 100644 --- a/testsuite/Makefile.am +++ b/testsuite/Makefile.am @@ -23,7 +23,7 @@ SEDTESTS += \ head madding mac-mf empty xbxcx xbxcx3 recall recall2 xemacs \ fasts uniq manis khadafy linecnt eval distrib 8to7 y-bracket \ y-newline allsub cv-vars classes middle bsd stdin flipcase \ - insens subwrite writeout readin insert \ + insens subwrite writeout readin insert badenc \ help version file quiet \ factor binary3 binary2 binary dc @@ -79,6 +79,7 @@ EXTRA_DIST = \ sep.good sep.inp sep.sed \ subwrite.inp subwrite.sed subwrt1.good subwrt2.good \ uniq.good uniq.inp uniq.sed \ + badenc.good badenc.inp badenc.sed \ version.gin \ writeout.inp writeout.sed wrtout1.good wrtout2.good \ xabcx.good xabcx.inp xabcx.sed \ diff --git a/testsuite/Makefile.tests b/testsuite/Makefile.tests index fb64f2e..aca7ae3 100644 --- a/testsuite/Makefile.tests +++ b/testsuite/Makefile.tests @@ -25,6 +25,21 @@ y-bracket y-newline insert:: $(CMP) $(srcdir)/$@.good $@.out @$(RM) $@.out +badenc:: + LANG=ru_RU.UTF-8 $(TIME) $(SED) -nf $(srcdir)/$@.sed \ + < $(srcdir)/$@.inp > $@.out + $(CMP) $(srcdir)/$@.good $@.out + LANG=it_IT.UTF-8 $(TIME) $(SED) -nf $(srcdir)/$@.sed \ + < $(srcdir)/$@.inp > $@.out + $(CMP) $(srcdir)/$@.good $@.out + LANG=en_US.UTF-8 $(TIME) $(SED) -nf $(srcdir)/$@.sed \ + < $(srcdir)/$@.inp > $@.out + $(CMP) $(srcdir)/$@.good $@.out + LANG=en_GB.UTF-8 $(TIME) $(SED) -nf $(srcdir)/$@.sed \ + < $(srcdir)/$@.inp > $@.out + $(CMP) $(srcdir)/$@.good $@.out + @$(RM) $@.out + # This checks for a bug in 3.02 and 3.02.80 stdin:: ($(SEDENV) $(SED) d; $(SEDENV) $(SED) G) < $(srcdir)/numsub.inp > $@.1out diff --git a/testsuite/badenc.good b/testsuite/badenc.good new file mode 100644 index 0000000..bf6b206 --- /dev/null +++ b/testsuite/badenc.good @@ -0,0 +1,9 @@ +abc +deÐf +ghi +jkl +mno +pqr +stu +vwx +yz diff --git a/testsuite/badenc.inp b/testsuite/badenc.inp new file mode 100644 index 0000000..bf6b206 --- /dev/null +++ b/testsuite/badenc.inp @@ -0,0 +1,9 @@ +abc +deÐf +ghi +jkl +mno +pqr +stu +vwx +yz diff --git a/testsuite/badenc.sed b/testsuite/badenc.sed new file mode 100644 index 0000000..dc80982 --- /dev/null +++ b/testsuite/badenc.sed @@ -0,0 +1 @@ +/.*/ { H ; g ; s/\n// ; p ; z ; x }