diff --git a/ChangeLog b/ChangeLog index 0ad87b8..a3e9288 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,16 @@ +2008-09-29 Paolo Bonzini + + * NEWS: Document `z' extension. + * doc/sed-in.texi: Document `z' extension, and GNU sed's s/.*.// + behavior with invalid multibyte sequences. + * sed/compile.c (compile_program): Recognize `z'. + * sed/execute.c (execute_program): Execute `z'. + * testsuite/Makefile.am: Add badenc test. + * testsuite/Makefile.tests: Add badenc test. + * testsuite/badenc.good: New. + * testsuite/badenc.inp: New. + * testsuite/badenc.sed: New. + 2008-08-27 Paolo Bonzini * sed/execute.c (closedown): Close the input file! diff --git a/doc/sed-in.texi b/doc/sed-in.texi index bd15ddd..2109670 100644 --- a/doc/sed-in.texi +++ b/doc/sed-in.texi @@ -1489,6 +1489,18 @@ This command enables all @value{SSEDEXT} even if Write to the given filename the portion of the pattern space up to the first newline. Everything said under the @code{w} command about file handling holds here too. + +@item z +@findex z (Zap) command +@cindex @value{SSEDEXT}, emptying pattern space +@cindex Emptying pattern space +This command empties the content of pattern space. It is +usually the same as @samp{s/.*//}, but is more efficient +and works in the presence of invalid multibyte sequences +in the input stream. @sc{posix} mandates that such sequences +are @emph{not} matched by @samp{.}, so that there is no portable +way to clear @command{sed}'s buffers in the middle of the +script in most multibyte locales (including UTF-8 locales). @end table @node Escapes @@ -2738,6 +2750,10 @@ in a read-only directory, and will break hard or symbolic links when @option{-i} is used on such a file. @item @code{0a} does not work (gives an error) +@cindex @code{0} address +@cindex @acronym{GNU} extensions, @code{0} address +@cindex Non-bugs, @code{0} address + There is no line 0. 0 is a special address that is only used to treat addresses like @code{0,/@var{RE}/} as active when the script starts: if you write @code{1,/abc/d} and the first line includes the word @samp{abc}, @@ -2748,6 +2764,8 @@ is obtained with @code{0,/abc/d}. @ifclear PERL @item @code{[a-z]} is case insensitive +@cindex Non-bugs, localization-related + You are encountering problems with locales. POSIX mandates that @code{[a-z]} uses the current locale's collation order -- in C parlance, that means using @code{strcoll(3)} instead of @code{strcmp(3)}. Some locales have a @@ -2763,6 +2781,22 @@ because this is a single collating symbol that comes after @samp{a} and before @samp{b}; @samp{ll} behaves similarly in Spanish locales, or @samp{ij} in Dutch locales. +To work around these problems, which may cause bugs in shell scripts, set +the @env{LC_COLLATE} and @env{LC_CTYPE} environment variables to @samp{C}. + +@item @code{s/.*//} does not clear pattern space +@cindex Non-bugs, localization-related +@cindex @value{SSEDEXT}, emptying pattern space +@cindex Emptying pattern space + +This happens if your input stream includes invalid multibyte +sequences. @sc{posix} mandates that such sequences +are @emph{not} matched by @samp{.}, so that @samp{s/.*//} will not clear +pattern space as you would expect. In fact, there is no way to clear +sed's buffers in the middle of the script in most multibyte locales +(including UTF-8 locales). For this reason, @value{SSED} provides a `z' +command (for `zap') as an extension. + To work around these problems, which may cause bugs in shell scripts, set the @env{LC_COLLATE} and @env{LC_CTYPE} environment variables to @samp{C}. @end ifclear diff --git a/doc/sed.texi b/doc/sed.texi index f135bd1..998a6e2 100644 --- a/doc/sed.texi +++ b/doc/sed.texi @@ -1490,6 +1490,18 @@ This command enables all @value{SSEDEXT} even if Write to the given filename the portion of the pattern space up to the first newline. Everything said under the @code{w} command about file handling holds here too. + +@item z +@findex z (Zap) command +@cindex @value{SSEDEXT}, emptying pattern space +@cindex Emptying pattern space +This command empties the content of pattern space. It is +usually the same as @samp{s/.*//}, but is more efficient +and works in the presence of invalid multibyte sequences +in the input stream. @sc{posix} mandates that such sequences +are @emph{not} matched by @samp{.}, so that there is no portable +way to clear @command{sed}'s buffers in the middle of the +script in most multibyte locales (including UTF-8 locales). @end table @node Escapes @@ -2905,6 +2917,10 @@ in a read-only directory, and will break hard or symbolic links when @option{-i} is used on such a file. @item @code{0a} does not work (gives an error) +@cindex @code{0} address +@cindex @acronym{GNU} extensions, @code{0} address +@cindex Non-bugs, @code{0} address + There is no line 0. 0 is a special address that is only used to treat addresses like @code{0,/@var{RE}/} as active when the script starts: if you write @code{1,/abc/d} and the first line includes the word @samp{abc}, @@ -2915,6 +2931,8 @@ is obtained with @code{0,/abc/d}. @ifclear PERL @item @code{[a-z]} is case insensitive +@cindex Non-bugs, localization-related + You are encountering problems with locales. POSIX mandates that @code{[a-z]} uses the current locale's collation order -- in C parlance, that means using @code{strcoll(3)} instead of @code{strcmp(3)}. Some locales have a @@ -2930,6 +2948,22 @@ because this is a single collating symbol that comes after @samp{a} and before @samp{b}; @samp{ll} behaves similarly in Spanish locales, or @samp{ij} in Dutch locales. +To work around these problems, which may cause bugs in shell scripts, set +the @env{LC_COLLATE} and @env{LC_CTYPE} environment variables to @samp{C}. + +@item @code{s/.*//} does not clear pattern space +@cindex Non-bugs, localization-related +@cindex @value{SSEDEXT}, emptying pattern space +@cindex Emptying pattern space + +This happens if your input stream includes invalid multibyte +sequences. @sc{posix} mandates that such sequences +are @emph{not} matched by @samp{.}, so that @samp{s/.*//} will not clear +pattern space as you would expect. In fact, there is no way to clear +sed's buffers in the middle of the script in most multibyte locales +(including UTF-8 locales). For this reason, @value{SSED} provides a `z' +command (for `zap') as an extension. + To work around these problems, which may cause bugs in shell scripts, set the @env{LC_COLLATE} and @env{LC_CTYPE} environment variables to @samp{C}. @end ifclear diff --git a/sed/compile.c b/sed/compile.c index a5a6cca..2b471ee 100644 --- a/sed/compile.c +++ b/sed/compile.c @@ -1227,6 +1227,7 @@ compile_program(vector) case 'N': case 'p': case 'P': + case 'z': case 'x': ch = in_nonblank(); if (ch == CLOSE_BRACE || ch == '#') diff --git a/sed/execute.c b/sed/execute.c index 8d948e1..0dc25bb 100644 --- a/sed/execute.c +++ b/sed/execute.c @@ -1693,6 +1693,10 @@ execute_program(vec, input) } break; + case 'z': + line.length = 0; + break; + case '=': output_missing_newline(&output_file); fprintf(output_file.fp, "%lu\n", diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am index 86570fc..e841895 100644 --- a/testsuite/Makefile.am +++ b/testsuite/Makefile.am @@ -14,7 +14,7 @@ SEDTESTS = \ head madding mac-mf empty xbxcx xbxcx3 recall recall2 xemacs \ fasts uniq manis khadafy linecnt eval distrib 8to7 y-bracket \ y-newline allsub cv-vars classes middle bsd stdin flipcase \ - insens subwrite writeout readin insert \ + insens subwrite writeout readin insert badenc \ help version file quiet \ factor binary3 binary2 binary dc @@ -69,6 +69,7 @@ EXTRA_DIST = runtest Makefile.tests \ sep.good sep.inp sep.sed \ subwrite.inp subwrite.sed subwrt1.good subwrt2.good \ uniq.good uniq.inp uniq.sed \ + badenc.good badenc.inp badenc.sed \ version.gin \ writeout.inp writeout.sed wrtout1.good wrtout2.good \ xabcx.good xabcx.inp xabcx.sed \ diff --git a/testsuite/Makefile.tests b/testsuite/Makefile.tests index f322fb8..3a44ad5 100644 --- a/testsuite/Makefile.tests +++ b/testsuite/Makefile.tests @@ -25,6 +25,21 @@ y-bracket y-newline insert:: $(CMP) $(srcdir)/$@.good $@.out @$(RM) $@.out +badenc:: + LANG=ru_RU.UTF-8 $(TIME) $(SED) -nf $(srcdir)/$@.sed \ + < $(srcdir)/$@.inp > $@.out + $(CMP) $(srcdir)/$@.good $@.out + LANG=it_IT.UTF-8 $(TIME) $(SED) -nf $(srcdir)/$@.sed \ + < $(srcdir)/$@.inp > $@.out + $(CMP) $(srcdir)/$@.good $@.out + LANG=en_US.UTF-8 $(TIME) $(SED) -nf $(srcdir)/$@.sed \ + < $(srcdir)/$@.inp > $@.out + $(CMP) $(srcdir)/$@.good $@.out + LANG=en_GB.UTF-8 $(TIME) $(SED) -nf $(srcdir)/$@.sed \ + < $(srcdir)/$@.inp > $@.out + $(CMP) $(srcdir)/$@.good $@.out + @$(RM) $@.out + # This checks for a bug in 3.02 and 3.02.80 stdin:: ($(SEDENV) $(SED) d; $(SEDENV) $(SED) G) < $(srcdir)/numsub.inp > $@.1out diff --git a/testsuite/badenc.good b/testsuite/badenc.good new file mode 100644 index 0000000..bf6b206 --- /dev/null +++ b/testsuite/badenc.good @@ -0,0 +1,9 @@ +abc +deÐf +ghi +jkl +mno +pqr +stu +vwx +yz diff --git a/testsuite/badenc.inp b/testsuite/badenc.inp new file mode 100644 index 0000000..bf6b206 --- /dev/null +++ b/testsuite/badenc.inp @@ -0,0 +1,9 @@ +abc +deÐf +ghi +jkl +mno +pqr +stu +vwx +yz diff --git a/testsuite/badenc.sed b/testsuite/badenc.sed new file mode 100644 index 0000000..dc80982 --- /dev/null +++ b/testsuite/badenc.sed @@ -0,0 +1 @@ +/.*/ { H ; g ; s/\n// ; p ; z ; x }