Fix memory leak in regexp compiler (BZ #17069)

ChangeLog

@@ -1,3 +1,15 @@
+2014-06-19  Andreas Schwab  <schwab@linux-m68k.org>
+
+	[BZ #17069]
+	* posix/regcomp.c (parse_expression): Deallocate partially
+	constructed tree before returning error.
+	* posix/Makefile.c (tests): Add bug-regex36.
+	(generated): Add bug-regex36.mtrace.
+	(tests-special): Add $(objpfx)bug-regex36-mem.out
+	(bug-regex36-ENV): New variable.
+	($(objpfx)bug-regex36-mem.out): New rule.
+	* posix/bug-regex36.c: New file.
+
 2014-06-19  Will Newton  <will.newton@linaro.org>
 
 	* malloc/malloc.c (systrim): If extra is zero then return

posix/Makefile

@@ -87,7 +87,7 @@ tests		:= tstgetopt testfnm runtests runptests	     \
 		   bug-getopt1 bug-getopt2 bug-getopt3 bug-getopt4 \
 		   bug-getopt5 tst-getopt_long1 bug-regex34 bug-regex35 \
 		   tst-pathconf tst-getaddrinfo4 tst-rxspencer-no-utf8 \
-		   tst-fnmatch3
+		   tst-fnmatch3 bug-regex36
 xtests		:= bug-ga2
 ifeq (yes,$(build-shared))
 test-srcs	:= globtest
@@ -113,7 +113,7 @@ generated += $(addprefix wordexp-test-result, 1 2 3 4 5 6 7 8 9 10) \
 	     tst-boost.mtrace bug-ga2.mtrace bug-ga2-mem.out \
 	     bug-glob2.mtrace bug-glob2-mem.out tst-vfork3-mem.out \
 	     tst-vfork3.mtrace getconf.speclist tst-fnmatch-mem.out \
-	     tst-fnmatch.mtrace
+	     tst-fnmatch.mtrace bug-regex36.mtrace
 
 ifeq ($(run-built-tests),yes)
 ifeq (yes,$(build-shared))
@@ -130,7 +130,7 @@ tests-special += $(objpfx)bug-regex2-mem.out $(objpfx)bug-regex14-mem.out \
 		 $(objpfx)tst-rxspencer-no-utf8-mem.out $(objpfx)tst-pcre-mem.out \
 		 $(objpfx)tst-boost-mem.out $(objpfx)tst-getconf.out \
 		 $(objpfx)bug-glob2-mem.out $(objpfx)tst-vfork3-mem.out \
-		 $(objpfx)tst-fnmatch-mem.out
+		 $(objpfx)tst-fnmatch-mem.out $(objpfx)bug-regex36-mem.out
 xtests-special += $(objpfx)bug-ga2-mem.out
 endif
 
@@ -260,6 +260,12 @@ $(objpfx)bug-regex31-mem.out: $(objpfx)bug-regex31.out
 	$(common-objpfx)malloc/mtrace $(objpfx)bug-regex31.mtrace > $@; \
 	$(evaluate-test)
 
+bug-regex36-ENV = MALLOC_TRACE=$(objpfx)bug-regex36.mtrace
+
+$(objpfx)bug-regex36-mem.out: $(objpfx)bug-regex36.out
+	$(common-objpfx)malloc/mtrace $(objpfx)bug-regex36.mtrace > $@; \
+	$(evaluate-test)
+
 tst-vfork3-ENV = MALLOC_TRACE=$(objpfx)tst-vfork3.mtrace
 
 $(objpfx)tst-vfork3-mem.out: $(objpfx)tst-vfork3.out

posix/bug-regex36.c

@@ -0,0 +1,29 @@
+/* Test regcomp not leaking memory on invalid repetition operator
+   Copyright (C) 2014 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <mcheck.h>
+#include <regex.h>
+
+int
+main (int argc, char **argv)
+{
+  regex_t r;
+  mtrace ();
+  regcomp (&r, "[a]\\{-2,}", 0);
+  regfree (&r);
+}

posix/regcomp.c

@@ -2415,14 +2415,21 @@ parse_expression (re_string_t *regexp, regex_t *preg, re_token_t *token,
   while (token->type == OP_DUP_ASTERISK || token->type == OP_DUP_PLUS
 	 || token->type == OP_DUP_QUESTION || token->type == OP_OPEN_DUP_NUM)
     {
-      tree = parse_dup_op (tree, regexp, dfa, token, syntax, err);
+      bin_tree_t *dup_tree = parse_dup_op (tree, regexp, dfa, token, syntax, err);
-      if (BE (*err != REG_NOERROR && tree == NULL, 0))
+      if (BE (*err != REG_NOERROR && dup_tree == NULL, 0))
-	return NULL;
+	{
+	  if (tree != NULL)
+	    postorder (tree, free_tree, NULL);
+	  return NULL;
+	}
+      tree = dup_tree;
       /* In BRE consecutive duplications are not allowed.  */
       if ((syntax & RE_CONTEXT_INVALID_DUP)
 	  && (token->type == OP_DUP_ASTERISK
 	      || token->type == OP_OPEN_DUP_NUM))
 	{
+	  if (tree != NULL)
+	    postorder (tree, free_tree, NULL);
 	  *err = REG_BADRPT;
 	  return NULL;
 	}