Fix BZ #18820 -- fmemopen may leak memory on failure.

ChangeLog

@@ -1,3 +1,11 @@
+2015-08-12  Paul Pluzhnikov  <ppluzhnikov@google.com>
+
+	[BZ #18820]
+	* libio/Makefile (test-fmemopen-mem): New test.
+	* libio/test-fmemopen.c (do_bz18820): New test.
+	* libio/fmemopen.c (__fmemopen): Fix memory leak.
+	* libio/oldfmemopen.c (__old_fmemopen): Likewise.
+
 2015-08-12  Paul Pluzhnikov  <ppluzhnikov@google.com>
 
 	[BZ #16734]

NEWS

@@ -10,7 +10,7 @@ Version 2.23
 * The following bugs are resolved with this release:
 
   16517, 16519, 16734, 17905, 18086, 18265, 18480, 18525, 18618, 18647,
-  18661, 18674, 18778, 18781, 18787, 18789, 18790.
+  18661, 18674, 18778, 18781, 18787, 18789, 18790, 18820.
 
 
 Version 2.22

libio/Makefile

@@ -148,8 +148,10 @@ CFLAGS-tst_putwc.c = -DOBJPFX=\"$(objpfx)\"
 
 tst_wprintf2-ARGS = "Some Text"
 
+test-fmemopen-ENV = MALLOC_TRACE=$(objpfx)test-fmemopen.mtrace
 tst-fopenloc-ENV = MALLOC_TRACE=$(objpfx)tst-fopenloc.mtrace
 
+generated += test-fmemopen.mtrace test-fmemopen.check
 generated += tst-fopenloc.mtrace tst-fopenloc.check
 
 aux	:= fileops genops stdfiles stdio strops
@@ -164,7 +166,7 @@ shared-only-routines = oldiofopen oldiofdopen oldiofclose oldfileops	\
 		       oldiofsetpos64
 
 ifeq ($(run-built-tests),yes)
-tests-special += $(objpfx)test-freopen.out
+tests-special += $(objpfx)test-freopen.out $(objpfx)test-fmemopen-mem.out
 ifeq (yes,$(build-shared))
 # Run tst-fopenloc-cmp.out and tst-openloc-mem.out only if shared
 # library is enabled since they depend on tst-fopenloc.out.
@@ -184,6 +186,10 @@ $(objpfx)tst-fopenloc-cmp.out: ../iconvdata/testdata/ISO-8859-1..UTF8 \
 	cmp $^ > $@; \
 	$(evaluate-test)
 
+$(objpfx)test-fmemopen-mem.out: $(objpfx)test-fmemopen.out
+	$(common-objpfx)malloc/mtrace $(objpfx)test-fmemopen.mtrace > $@; \
+	$(evaluate-test)
+
 $(objpfx)tst-fopenloc-mem.out: $(objpfx)tst-fopenloc.out
 	$(common-objpfx)malloc/mtrace $(objpfx)tst-fopenloc.mtrace > $@; \
 	$(evaluate-test)

libio/fmemopen.c

@@ -149,6 +149,7 @@ __fmemopen (void *buf, size_t len, const char *mode)
 {
   cookie_io_functions_t iof;
   fmemopen_cookie_t *c;
+  FILE *result;
 
   c = (fmemopen_cookie_t *) calloc (sizeof (fmemopen_cookie_t), 1);
   if (c == NULL)
@@ -209,7 +210,16 @@ __fmemopen (void *buf, size_t len, const char *mode)
   iof.seek = fmemopen_seek;
   iof.close = fmemopen_close;
 
-  return _IO_fopencookie (c, mode, iof);
+  result = _IO_fopencookie (c, mode, iof);
+  if (__glibc_unlikely (result == NULL))
+    {
+      if (c->mybuffer)
+	free (c->buffer);
+
+      free (c);
+    }
+
+  return result;
 }
 libc_hidden_def (__fmemopen)
 versioned_symbol (libc, __fmemopen, fmemopen, GLIBC_2_22);

libio/oldfmemopen.c

@@ -204,6 +204,7 @@ __old_fmemopen (void *buf, size_t len, const char *mode)
 {
   cookie_io_functions_t iof;
   fmemopen_cookie_t *c;
+  FILE *result;
 
   if (__glibc_unlikely (len == 0))
     {
@@ -259,7 +260,16 @@ __old_fmemopen (void *buf, size_t len, const char *mode)
   iof.seek = fmemopen_seek;
   iof.close = fmemopen_close;
 
-  return _IO_fopencookie (c, mode, iof);
+  result = _IO_fopencookie (c, mode, iof);
+  if (__glibc_unlikely (result == NULL))
+    {
+      if (c->mybuffer)
+	free (c->buffer);
+
+      free (c);
+    }
+
+  return result;
 }
 compat_symbol (libc, __old_fmemopen, fmemopen, GLIBC_2_2);
 #endif

libio/test-fmemopen.c

@@ -22,6 +22,32 @@ static char buffer[] = "foobar";
 #include <stdio.h>
 #include <string.h>
 #include <errno.h>
+#include <mcheck.h>
+
+static int
+do_bz18820 (void)
+{
+  char ch;
+  FILE *stream;
+
+  stream = fmemopen (&ch, 1, "?");
+  if (stream)
+    {
+      printf ("fmemopen: expected NULL, got %p\n", stream);
+      fclose (stream);
+      return 1;
+    }
+
+  stream = fmemopen (NULL, 42, "?");
+  if (stream)
+    {
+      printf ("fmemopen: expected NULL, got %p\n", stream);
+      fclose (stream);
+      return 2;
+    }
+
+  return 0;
+}
 
 static int
 do_test (void)
@@ -30,6 +56,8 @@ do_test (void)
   FILE *stream;
   int ret = 0;
 
+  mtrace ();
+
   stream = fmemopen (buffer, strlen (buffer), "r+");
 
   while ((ch = fgetc (stream)) != EOF)
@@ -44,7 +72,7 @@ do_test (void)
 
   fclose (stream);
 
-  return ret;
+  return ret + do_bz18820 ();
 }
 
 #define TEST_FUNCTION do_test ()