diff --git a/ChangeLog b/ChangeLog index 6ade021..40ac47e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,32 @@ +2004-12-26 Paolo Bonzini + + Do not use leftmost-longest matching for addresses. + + * NEWS: Add a note about this. + * testsuite/recall2.good, testsuite/recall2.inp, + testsuite/recall2.sed: New test. + * testsuite/Makefile.am, testsuite/Makefile.tests: Add the + recall2 test. + * sed/sed.h (struct regex): New. + (struct addr, struct subst, compile_regex, match_regex, + release_regex): Use it instead of regex_t. + * sed/compile.c (compile_program): Update for new meaning of + the third parameter of compile_regex. + * sed/execute.c (do_subst): Pass less conservative value to + the regsize parameter of match_regex. + * sed/regexp.c (compile_regex_1): New, extracted out of + compile_regex. The third parameter, needed_sub, now includes + \0 (so 10 means that \0 .. \9 are needed). Pass RE_NO_SUB + if needed_sub is zero. + (compile_regex): Accept a struct regex instead of a regex_t. + Save the regular expression's text. + (match_regex): Accept a struct regex instead of a regex_t. + Recompile the pattern if it was compiled with RE_NO_SUB. + (release_regex): Accept a struct regex instead of a regex_t. + + * doc/Makefile.am: Generate sed.texi correctly when + building outside srcdir. + 2004-12-26 Paolo Bonzini * doc/sed-in.texi [!PERL]: Likewise. diff --git a/NEWS b/NEWS index 7d1db27..e06b5e3 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,13 @@ +Sed 3.62 (aligned with GNU sed 4.1.3) + +* added a note to BUGS and the manual about changed interpretation + of `s|abc\|def||', and about localization issues. + +* fixed --disable-nls build problems on Solaris. + +* fixed `make check' in non-English locales. + +----------------------------------------------------------------------- Sed 3.61a * sed 3.61 had GNU sed 4.1's toplevel Makefile by mistake. Fixed. diff --git a/doc/Makefile.am b/doc/Makefile.am index 51df281..2bfb211 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -13,8 +13,9 @@ SED = $(top_builddir)/sed/sed # scripts we group comments with lines following them; # since mantaining the "@group...@end group" manually # is a burden, we do this automatically -sed.texi: sed-in.texi groupify.sed - sed -nf groupify.sed < $(srcdir)/sed-in.texi > $(srcdir)/sed.texi +$(srcdir)/sed.texi: sed-in.texi $(srcdir)/groupify.sed + sed -nf $(srcdir)/groupify.sed \ + < $(srcdir)/sed-in.texi > $(srcdir)/sed.texi sed.1: $(top_srcdir)/sed/sed.c $(top_srcdir)/configure.ac $(srcdir)/sed.x $(HELP2MAN) -p sed --include $(srcdir)/sed.x $(SED) > $(srcdir)/sed.1 diff --git a/sed/compile.c b/sed/compile.c index 869817f..e6ab4f4 100644 --- a/sed/compile.c +++ b/sed/compile.c @@ -1261,7 +1261,7 @@ compile_program(vector) flags = mark_subst_opts(cur_cmd->x.cmd_subst); cur_cmd->x.cmd_subst->regx = - compile_regex(b, flags, cur_cmd->x.cmd_subst->max_id); + compile_regex(b, flags, cur_cmd->x.cmd_subst->max_id + 1); free_buffer(b); } break; diff --git a/sed/execute.c b/sed/execute.c index 2249a24..fb3f509 100644 --- a/sed/execute.c +++ b/sed/execute.c @@ -1072,7 +1072,6 @@ do_subst(sub) countT count = 0; /* number of matches found */ bool again = true; -#define MAX_BACKREFERENCES 10 static struct re_registers regs; if (s_accum.alloc == 0) @@ -1082,7 +1081,7 @@ do_subst(sub) /* The first part of the loop optimizes s/xxx// when xxx is at the start, and s/xxx$// */ if (!match_regex(sub->regx, line.active, line.length, start, - ®s, MAX_BACKREFERENCES)) + ®s, sub->max_id + 1)) return; if (!sub->replacement && sub->numb <= 1) @@ -1160,7 +1159,7 @@ do_subst(sub) while (again && start <= line.length && match_regex(sub->regx, line.active, line.length, start, - ®s, MAX_BACKREFERENCES)); + ®s, sub->max_id + 1)); /* Copy stuff to the right of the last match into the output string. */ if (start < line.length) diff --git a/sed/regexp.c b/sed/regexp.c index 228bf07..070ad95 100644 --- a/sed/regexp.c +++ b/sed/regexp.c @@ -41,16 +41,90 @@ static const char errors[] = -regex_t * +static void +compile_regex_1 (new_regex, needed_sub) + struct regex *new_regex; + int needed_sub; +{ +#ifdef REG_PERL + int errcode; + errcode = regncomp(&new_regex->pattern, new_regex->re, new_regex->sz, + (needed_sub ? 0 : REG_NOSUB) + | new_regex->flags + | extended_regexp_flags); + + if (errcode) + { + char errorbuf[200]; + regerror(errcode, NULL, errorbuf, 200); + bad_prog(gettext(errorbuf)); + } +#else + const char *error; + new_regex->pattern.fastmap = malloc (1 << (sizeof (char) * 8)); + int syntax = ((extended_regexp_flags & REG_EXTENDED) + ? RE_SYNTAX_POSIX_EXTENDED + : RE_SYNTAX_POSIX_BASIC) + & ~RE_UNMATCHED_RIGHT_PAREN_ORD; + + syntax |= RE_NO_POSIX_BACKTRACKING; +#ifdef RE_ICASE + syntax |= (new_regex->flags & REG_ICASE) ? RE_ICASE : 0; +#endif +#ifdef RE_NO_SUB + syntax |= needed_sub ? 0 : RE_NO_SUB; +#endif + + /* If REG_NEWLINE is set, newlines are treated differently. */ + if (new_regex->flags & REG_NEWLINE) + { + /* REG_NEWLINE implies neither . nor [^...] match newline. */ + syntax &= ~RE_DOT_NEWLINE; + syntax |= RE_HAT_LISTS_NOT_NEWLINE; + } + + re_set_syntax (syntax); + error = re_compile_pattern (new_regex->re, new_regex->sz, + &new_regex->pattern); + new_regex->pattern.newline_anchor = (new_regex->flags & REG_NEWLINE) != 0; + + new_regex->pattern.translate = NULL; +#ifndef RE_ICASE + if (new_regex->flags & REG_ICASE) + { + static char translate[1 << (sizeof(char) * 8)]; + int i; + for (i = 0; i < sizeof(translate) / sizeof(char); i++) + translate[i] = tolower (i); + + new_regex->pattern.translate = translate; + } +#endif + + if (error) + bad_prog(error); +#endif + + /* Just to be sure, I mark this as not POSIXLY_CORRECT behavior */ + if (needed_sub + && new_regex->pattern.re_nsub < needed_sub - 1 + && posixicity == POSIXLY_EXTENDED) + { + char buf[200]; + sprintf(buf, _("invalid reference \\%d on `s' command's RHS"), + needed_sub); + bad_prog(buf); + } +} + +struct regex * compile_regex(b, flags, needed_sub) struct buffer *b; int flags; int needed_sub; { - regex_t *new_regex; - - char *last_re = NULL; - size_t last_re_len; + struct regex *new_regex; + size_t re_len; /* // matches the last RE */ if (size_buffer(b) == 0) @@ -60,82 +134,19 @@ compile_regex(b, flags, needed_sub) return NULL; } - last_re_len = size_buffer(b); - last_re = ck_memdup(get_buffer(b), last_re_len); - - new_regex = MALLOC(1, regex_t); + re_len = size_buffer(b); + new_regex = ck_malloc(sizeof (struct regex) + re_len - 1); + new_regex->flags = flags; + memcpy (new_regex->re, get_buffer(b), re_len); #ifdef REG_PERL - { - int errcode; - errcode = regncomp(new_regex, last_re, last_re_len, - (needed_sub ? 0 : REG_NOSUB) - | flags - | extended_regexp_flags); - - if (errcode) - { - char errorbuf[200]; - regerror(errcode, NULL, errorbuf, 200); - bad_prog(gettext(errorbuf)); - } - } + new_regex->sz = re_len; #else - new_regex->fastmap = malloc (1 << (sizeof (char) * 8)); - { - const char *error; - int syntax = ((extended_regexp_flags & REG_EXTENDED) - ? RE_SYNTAX_POSIX_EXTENDED - : RE_SYNTAX_POSIX_BASIC) - & ~RE_UNMATCHED_RIGHT_PAREN_ORD; - - syntax |= RE_NO_POSIX_BACKTRACKING; -#ifdef RE_ICASE - syntax |= (flags & REG_ICASE) ? RE_ICASE : 0; + /* GNU regex does not process \t & co. */ + new_regex->sz = normalize_text(new_regex->re, re_len, TEXT_REGEX); #endif - /* If REG_NEWLINE is set, newlines are treated differently. */ - if (flags & REG_NEWLINE) - { /* REG_NEWLINE implies neither . nor [^...] match newline. */ - syntax &= ~RE_DOT_NEWLINE; - syntax |= RE_HAT_LISTS_NOT_NEWLINE; - } - - /* GNU regex does not process \t & co. */ - last_re_len = normalize_text(last_re, last_re_len, TEXT_REGEX); - re_set_syntax (syntax); - error = re_compile_pattern (last_re, last_re_len, new_regex); - new_regex->newline_anchor = (flags & REG_NEWLINE) != 0; - - new_regex->translate = NULL; -#ifndef RE_ICASE - if (flags & REG_ICASE) - { - static char translate[1 << (sizeof(char) * 8)]; - int i; - for (i = 0; i < sizeof(translate) / sizeof(char); i++) - translate[i] = tolower (i); - - new_regex->translate = translate; - } -#endif - - if (error) - bad_prog(error); - } -#endif - - FREE(last_re); - - /* Just to be sure, I mark this as not POSIXLY_CORRECT behavior */ - if (new_regex->re_nsub < needed_sub && posixicity == POSIXLY_EXTENDED) - { - char buf[200]; - sprintf(buf, _("invalid reference \\%d on `s' command's RHS"), - needed_sub); - bad_prog(buf); - } - + compile_regex_1 (new_regex, needed_sub); return new_regex; } @@ -179,7 +190,7 @@ copy_regs (regs, pmatch, nregs) int match_regex(regex, buf, buflen, buf_start_offset, regarray, regsize) - regex_t *regex; + struct regex *regex; char *buf; size_t buflen; size_t buf_start_offset; @@ -187,7 +198,7 @@ match_regex(regex, buf, buflen, buf_start_offset, regarray, regsize) int regsize; { int ret; - static regex_t *regex_last; + static struct regex *regex_last; #ifdef REG_PERL regmatch_t rm[10], *regmatch = rm; if (regsize > 10) @@ -209,16 +220,19 @@ match_regex(regex, buf, buflen, buf_start_offset, regarray, regsize) #ifdef REG_PERL regmatch[0].rm_so = CAST(int)buf_start_offset; regmatch[0].rm_eo = CAST(int)buflen; - ret = regexec (regex, buf, regsize, regmatch, REG_STARTEND); + ret = regexec (®ex->pattern, buf, regsize, regmatch, REG_STARTEND); if (regsize) copy_regs (regarray, regmatch, regsize); return (ret == 0); #else - regex->regs_allocated = REGS_REALLOCATE; + if (regex->pattern.no_sub && regsize) + compile_regex_1 (regex, regsize); - ret = re_search (regex, buf, buflen, buf_start_offset, + regex->pattern.regs_allocated = REGS_REALLOCATE; + + ret = re_search (®ex->pattern, buf, buflen, buf_start_offset, buflen - buf_start_offset, regsize ? regarray : NULL); @@ -230,9 +244,9 @@ match_regex(regex, buf, buflen, buf_start_offset, regarray, regsize) #ifdef DEBUG_LEAKS void release_regex(regex) - regex_t *regex; + struct regex *regex; { - regfree(regex); + regfree(®ex->pattern); FREE(regex); } #endif /*DEBUG_LEAKS*/ diff --git a/sed/sed.h b/sed/sed.h index 224d4a0..6aad93c 100644 --- a/sed/sed.h +++ b/sed/sed.h @@ -52,6 +52,13 @@ struct text_buf { size_t text_length; }; +struct regex { + regex_t pattern; + int flags; + size_t sz; + char re[1]; +}; + enum replacement_types { REPL_ASIS = 0, REPL_UPPERCASE = 1, @@ -99,7 +106,7 @@ struct addr { enum addr_types addr_type; countT addr_number; countT addr_step; - regex_t *addr_regex; + struct regex *addr_regex; }; @@ -112,7 +119,7 @@ struct replacement { }; struct subst { - regex_t *regx; + struct regex *regx; struct replacement *replacement; countT numb; /* if >0, only substitute for match number "numb" */ struct output *outf; /* 'w' option given */ @@ -187,12 +194,12 @@ void check_final_program P_((struct vector *)); void rewind_read_files P_((void)); void finish_program P_((struct vector *)); -regex_t *compile_regex P_((struct buffer *b, int flags, int needed_sub)); -int match_regex P_((regex_t *regex, +struct regex *compile_regex P_((struct buffer *b, int flags, int needed_sub)); +int match_regex P_((struct regex *regex, char *buf, size_t buflen, size_t buf_start_offset, struct re_registers *regarray, int regsize)); #ifdef DEBUG_LEAKS -void release_regex P_((regex_t *)); +void release_regex P_((struct regex *)); #endif int process_files P_((struct vector *, char **argv)); diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am index 6319a73..ab4cd45 100644 --- a/testsuite/Makefile.am +++ b/testsuite/Makefile.am @@ -11,7 +11,7 @@ AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/lib -I$(top_srcdir)/pcre \ SEDTESTS = \ space enable sep inclib 8bit newjis xabcx dollar noeol noeolw \ numsub numsub2 numsub3 numsub4 numsub5 0range bkslashes \ - head madding mac-mf empty xbxcx xbxcx3 recall xemacs \ + head madding mac-mf empty xbxcx xbxcx3 recall recall2 xemacs \ fasts uniq manis khadafy linecnt eval distrib 8to7 \ allsub cv-vars classes middle bsd stdin flipcase \ insens subwrite writeout readin \ @@ -62,6 +62,7 @@ EXTRA_DIST = runtest Makefile.tests \ numsub5.good numsub5.inp numsub5.sed \ readin.good readin.in2 readin.inp readin.sed \ recall.good recall.inp recall.sed \ + recall2.good recall2.inp recall2.sed \ sep.good sep.inp sep.sed \ subwrite.inp subwrite.sed subwrt1.good subwrt2.good \ uniq.good uniq.inp uniq.sed \ diff --git a/testsuite/Makefile.tests b/testsuite/Makefile.tests index 88a3a01..eaf393b 100644 --- a/testsuite/Makefile.tests +++ b/testsuite/Makefile.tests @@ -12,7 +12,7 @@ CMP=cmp RM=rm -f enable sep inclib 8bit 8to7 newjis xabcx dollar noeol bkslashes \ -numsub head madding mac-mf empty xbxcx xbxcx3 recall xemacs \ +numsub head madding mac-mf empty xbxcx xbxcx3 recall recall2 xemacs \ fasts uniq manis linecnt khadafy allsub flipcase space:: $(SEDENV) $(SED) -f $(srcdir)/$@.sed \ < $(srcdir)/$@.inp > $@.out diff --git a/testsuite/recall2.good b/testsuite/recall2.good new file mode 100644 index 0000000..74c01ea --- /dev/null +++ b/testsuite/recall2.good @@ -0,0 +1 @@ +>abb<||>abbbb< diff --git a/testsuite/recall2.inp b/testsuite/recall2.inp new file mode 100644 index 0000000..9046d59 --- /dev/null +++ b/testsuite/recall2.inp @@ -0,0 +1 @@ +ababb||abbbabbbb diff --git a/testsuite/recall2.sed b/testsuite/recall2.sed new file mode 100644 index 0000000..f668773 --- /dev/null +++ b/testsuite/recall2.sed @@ -0,0 +1,5 @@ +# Starting from sed 4.1.3, regexes are compiled with REG_NOSUB +# if they are used in an address, so that the matcher does not +# have to obey leftmost-longest. The tricky part is to recompile +# them if they are then used in a substitution. +/\(ab*\)\+/ s//>\1