Patch from Marko Kreen:

pgcrypto crypt()/md5 and hmac() leak memory when compiled against
OpenSSL as openssl.c digest ->reset will do two DigestInit calls
against a context.  This happened to work with OpenSSL 0.9.6
but not with 0.9.7+.

Reason for the messy code was that I tried to avoid creating
wrapper structure to transport algorithm info and tried to use
OpenSSL context for it.  The fix is to create wrapper structure.

It also uses newer digest API to avoid memory allocations
on reset with newer OpenSSLs.

Thanks to Daniel Blaisdell for reporting it.

contrib/pgcrypto/openssl.c

@@ -26,7 +26,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $Id: openssl.c,v 1.12.4.1 2005/03/12 06:55:14 neilc Exp $
+ * $Id: openssl.c,v 1.12.4.2 2006/02/18 20:49:00 neilc Exp $
  */
 
 #include <postgres.h>
@@ -35,61 +35,90 @@
 
 #include <openssl/evp.h>
 
+/*
+ * Backwards compatibility code for digest.
+ */
+#if OPENSSL_VERSION_NUMBER < 0x00907000L
+
+static void EVP_MD_CTX_init(EVP_MD_CTX *ctx)
+{
+	memset(ctx, 0, sizeof(*ctx));
+}
+
+static int EVP_MD_CTX_cleanup(EVP_MD_CTX *ctx)
+{
+	memset(ctx, 0, sizeof(*ctx));
+	return 1;
+}
+
+static int EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *md, void *engine)
+{
+	EVP_DigestInit(ctx, md);
+	return 1;
+}
+
+static int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *res, unsigned int *len)
+{
+	EVP_DigestFinal(ctx, res, len);
+	return 1;
+}
+
+#endif
+
 /*
  * Hashes
  */
+
+typedef struct OSSLDigest {
+	const EVP_MD *algo;
+	EVP_MD_CTX ctx;
+} OSSLDigest;
+
 static unsigned
 digest_result_size(PX_MD * h)
 {
-	return EVP_MD_CTX_size((EVP_MD_CTX *) h->p.ptr);
+	OSSLDigest *digest = (OSSLDigest *)h->p.ptr;
+	return EVP_MD_CTX_size(&digest->ctx);
 }
 
 static unsigned
 digest_block_size(PX_MD * h)
 {
-	return EVP_MD_CTX_block_size((EVP_MD_CTX *) h->p.ptr);
+	OSSLDigest *digest = (OSSLDigest *)h->p.ptr;
+	return EVP_MD_CTX_block_size(&digest->ctx);
 }
 
 static void
 digest_reset(PX_MD * h)
 {
-	EVP_MD_CTX *ctx = (EVP_MD_CTX *) h->p.ptr;
+	OSSLDigest *digest = (OSSLDigest *)h->p.ptr;
-	const EVP_MD *md;
 
-	md = EVP_MD_CTX_md(ctx);
+	EVP_DigestInit_ex(&digest->ctx, digest->algo, NULL);
-
-	EVP_DigestInit(ctx, md);
 }
 
 static void
 digest_update(PX_MD * h, const uint8 *data, unsigned dlen)
 {
-	EVP_MD_CTX *ctx = (EVP_MD_CTX *) h->p.ptr;
+	OSSLDigest *digest = (OSSLDigest *)h->p.ptr;
 
-	EVP_DigestUpdate(ctx, data, dlen);
+	EVP_DigestUpdate(&digest->ctx, data, dlen);
 }
 
 static void
 digest_finish(PX_MD * h, uint8 *dst)
 {
-	EVP_MD_CTX *ctx = (EVP_MD_CTX *) h->p.ptr;
+	OSSLDigest *digest = (OSSLDigest *)h->p.ptr;
-	const EVP_MD *md = EVP_MD_CTX_md(ctx);
 
-	EVP_DigestFinal(ctx, dst, NULL);
+	EVP_DigestFinal_ex(&digest->ctx, dst, NULL);
-
-	/*
-	 * Some builds of 0.9.7x clear all of ctx in EVP_DigestFinal.
-	 * Fix it by reinitializing ctx.
-	 */
-	EVP_DigestInit(ctx, md);
 }
 
 static void
 digest_free(PX_MD * h)
 {
-	EVP_MD_CTX *ctx = (EVP_MD_CTX *) h->p.ptr;
+	OSSLDigest *digest = (OSSLDigest *)h->p.ptr;
 
-	px_free(ctx);
+	EVP_MD_CTX_cleanup(&digest->ctx);
+	px_free(digest);
 	px_free(h);
 }
 
@@ -101,8 +130,8 @@ int
 px_find_digest(const char *name, PX_MD ** res)
 {
 	const EVP_MD *md;
-	EVP_MD_CTX *ctx;
 	PX_MD	   *h;
+	OSSLDigest *digest;
 
 	if (!px_openssl_initialized)
 	{
@@ -114,8 +143,12 @@ px_find_digest(const char *name, PX_MD ** res)
 	if (md == NULL)
 		return -1;
 
-	ctx = px_alloc(sizeof(*ctx));
+	digest = px_alloc(sizeof(*digest));
-	EVP_DigestInit(ctx, md);
+	digest->algo = md;
+	
+	EVP_MD_CTX_init(&digest->ctx);
+	if (EVP_DigestInit_ex(&digest->ctx, digest->algo, NULL) == 0)
+		return -1;
 
 	h = px_alloc(sizeof(*h));
 	h->result_size = digest_result_size;
@@ -124,7 +157,7 @@ px_find_digest(const char *name, PX_MD ** res)
 	h->update = digest_update;
 	h->finish = digest_finish;
 	h->free = digest_free;
-	h->p.ptr = (void *) ctx;
+	h->p.ptr = (void *) digest;
 
 	*res = h;
 	return 0;