add OpenSSL 1.1.0-pre1 compatibility

* close https://github.com/libssh2/libssh2/issues/69
* sync a declaration with the rest of similar ones
* handle EVP_MD_CTX_new() returning NULL with OpenSSL 1.1.0
* fix potential memory leak with OpenSSL 1.1.0 in
  _libssh2_*_init() functions, when EVP_MD_CTX_new() succeeds,
  but EVP_DigestInit() fails.

src/openssl.c

@@ -569,14 +569,43 @@ _libssh2_dsa_sha1_sign(libssh2_dsa_ctx * dsactx,
 int
 _libssh2_sha1_init(libssh2_sha1_ctx *ctx)
 {
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+    *ctx = EVP_MD_CTX_new();
+
+    if (*ctx == NULL)
+        return 0;
+
+    if (EVP_DigestInit(*ctx, EVP_get_digestbyname("sha1")))
+        return 1;
+
+    EVP_MD_CTX_free(*ctx);
+    *ctx = NULL;
+
+    return 0;
+#else
     EVP_MD_CTX_init(ctx);
     return EVP_DigestInit(ctx, EVP_get_digestbyname("sha1"));
+#endif
 }
 
 int
 _libssh2_sha1(const unsigned char *message, unsigned long len,
               unsigned char *out)
 {
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+    EVP_MD_CTX * ctx = EVP_MD_CTX_new();
+
+    if (ctx == NULL)
+        return 1; /* error */
+
+    if (EVP_DigestInit(ctx, EVP_get_digestbyname("sha1"))) {
+        EVP_DigestUpdate(ctx, message, len);
+        EVP_DigestFinal(ctx, out, NULL);
+        EVP_MD_CTX_free(ctx);
+        return 0; /* success */
+    }
+    EVP_MD_CTX_free(ctx);
+#else
     EVP_MD_CTX ctx;
 
     EVP_MD_CTX_init(&ctx);
@@ -585,20 +614,50 @@ _libssh2_sha1(const unsigned char *message, unsigned long len,
         EVP_DigestFinal(&ctx, out, NULL);
         return 0; /* success */
     }
+#endif
     return 1; /* error */
 }
 
 int
 _libssh2_sha256_init(libssh2_sha256_ctx *ctx)
 {
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+    *ctx = EVP_MD_CTX_new();
+
+    if (*ctx == NULL)
+        return 0;
+
+    if (EVP_DigestInit(*ctx, EVP_get_digestbyname("sha256")))
+        return 1;
+
+    EVP_MD_CTX_free(*ctx);
+    *ctx = NULL;
+
+    return 0;
+#else
     EVP_MD_CTX_init(ctx);
     return EVP_DigestInit(ctx, EVP_get_digestbyname("sha256"));
+#endif
 }
 
 int
 _libssh2_sha256(const unsigned char *message, unsigned long len,
                 unsigned char *out)
 {
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+    EVP_MD_CTX * ctx = EVP_MD_CTX_new();
+
+    if (ctx == NULL)
+        return 1; /* error */
+
+    if(EVP_DigestInit(ctx, EVP_get_digestbyname("sha256"))) {
+        EVP_DigestUpdate(ctx, message, len);
+        EVP_DigestFinal(ctx, out, NULL);
+        EVP_MD_CTX_free(ctx);
+        return 0; /* success */
+    }
+    EVP_MD_CTX_free(ctx);
+#else
     EVP_MD_CTX ctx;
 
     EVP_MD_CTX_init(&ctx);
@@ -607,14 +666,30 @@ _libssh2_sha256(const unsigned char *message, unsigned long len,
         EVP_DigestFinal(&ctx, out, NULL);
         return 0; /* success */
     }
+#endif
     return 1; /* error */
 }
 
 int
 _libssh2_md5_init(libssh2_md5_ctx *ctx)
 {
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+    *ctx = EVP_MD_CTX_new();
+
+    if (*ctx == NULL)
+        return 0;
+
+    if (EVP_DigestInit(*ctx, EVP_get_digestbyname("md5")))
+        return 1;
+
+    EVP_MD_CTX_free(*ctx);
+    *ctx = NULL;
+
+    return 0;
+#else
     EVP_MD_CTX_init(ctx);
     return EVP_DigestInit(ctx, EVP_get_digestbyname("md5"));
+#endif
 }
 
 static unsigned char *

src/openssl.h

@@ -114,36 +114,91 @@
 
 #define _libssh2_random(buf, len) RAND_bytes ((buf), (len))
 
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+#define libssh2_sha1_ctx EVP_MD_CTX *
+#else
 #define libssh2_sha1_ctx EVP_MD_CTX
+#endif
 
 /* returns 0 in case of failure */
 int _libssh2_sha1_init(libssh2_sha1_ctx *ctx);
 #define libssh2_sha1_init(x) _libssh2_sha1_init(x)
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+#define libssh2_sha1_update(ctx, data, len) EVP_DigestUpdate(ctx, data, len)
+#define libssh2_sha1_final(ctx, out) do { \
+                                         EVP_DigestFinal(ctx, out, NULL); \
+                                         EVP_MD_CTX_free(ctx); \
+                                     } while(0)
+#else
 #define libssh2_sha1_update(ctx, data, len) EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_sha1_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)
+#endif
 int _libssh2_sha1(const unsigned char *message, unsigned long len,
                   unsigned char *out);
 #define libssh2_sha1(x,y,z) _libssh2_sha1(x,y,z)
 
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+#define libssh2_sha256_ctx EVP_MD_CTX *
+#else
 #define libssh2_sha256_ctx EVP_MD_CTX
+#endif
 
 /* returns 0 in case of failure */
 int _libssh2_sha256_init(libssh2_sha256_ctx *ctx);
 #define libssh2_sha256_init(x) _libssh2_sha256_init(x)
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+#define libssh2_sha256_update(ctx, data, len) EVP_DigestUpdate(ctx, data, len)
+#define libssh2_sha256_final(ctx, out) do { \
+                                           EVP_DigestFinal(ctx, out, NULL); \
+                                           EVP_MD_CTX_free(ctx); \
+                                       } while(0)
+#else
 #define libssh2_sha256_update(ctx, data, len) EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_sha256_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)
+#endif
 int _libssh2_sha256(const unsigned char *message, unsigned long len,
                   unsigned char *out);
 #define libssh2_sha256(x,y,z) _libssh2_sha256(x,y,z)
 
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+#define libssh2_md5_ctx EVP_MD_CTX *
+#else
 #define libssh2_md5_ctx EVP_MD_CTX
+#endif
 
 /* returns 0 in case of failure */
-int _libssh2_md5_init(libssh2_md5_ctx *);
+int _libssh2_md5_init(libssh2_md5_ctx *ctx);
 #define libssh2_md5_init(x) _libssh2_md5_init(x)
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+#define libssh2_md5_update(ctx, data, len) EVP_DigestUpdate(ctx, data, len)
+#define libssh2_md5_final(ctx, out) do { \
+                                        EVP_DigestFinal(ctx, out, NULL); \
+                                        EVP_MD_CTX_free(ctx); \
+                                    } while(0)
+#else
 #define libssh2_md5_update(ctx, data, len) EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_md5_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)
+#endif
 
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+#define libssh2_hmac_ctx HMAC_CTX *
+#define libssh2_hmac_ctx_init(ctx) ctx = HMAC_CTX_new()
+#define libssh2_hmac_sha1_init(ctx, key, keylen) \
+  HMAC_Init_ex(*(ctx), key, keylen, EVP_sha1(), NULL)
+#define libssh2_hmac_md5_init(ctx, key, keylen) \
+  HMAC_Init_ex(*(ctx), key, keylen, EVP_md5(), NULL)
+#define libssh2_hmac_ripemd160_init(ctx, key, keylen) \
+  HMAC_Init_ex(*(ctx), key, keylen, EVP_ripemd160(), NULL)
+#define libssh2_hmac_sha256_init(ctx, key, keylen) \
+  HMAC_Init_ex(*(ctx), key, keylen, EVP_sha256(), NULL)
+#define libssh2_hmac_sha512_init(ctx, key, keylen) \
+  HMAC_Init_ex(*(ctx), key, keylen, EVP_sha512(), NULL)
+
+#define libssh2_hmac_update(ctx, data, datalen) \
+  HMAC_Update(ctx, data, datalen)
+#define libssh2_hmac_final(ctx, data) HMAC_Final(ctx, data, NULL)
+#define libssh2_hmac_cleanup(ctx) HMAC_CTX_free(*(ctx))
+#else
 #define libssh2_hmac_ctx HMAC_CTX
 #define libssh2_hmac_ctx_init(ctx) \
   HMAC_CTX_init(&ctx)
@@ -162,6 +217,7 @@ int _libssh2_md5_init(libssh2_md5_ctx *);
   HMAC_Update(&(ctx), data, datalen)
 #define libssh2_hmac_final(ctx, data) HMAC_Final(&(ctx), data, NULL)
 #define libssh2_hmac_cleanup(ctx) HMAC_cleanup(ctx)
+#endif
 
 #define libssh2_crypto_init() \
   OpenSSL_add_all_algorithms(); \