openssl: require EVP_aes_128_ctr() support

libssh2 built with OpenSSL and without its `EVP_aes_128_ctr()`, aka
`HAVE_EVP_AES_128_CTR`, option are working incorrectly. This option
wasn't always auto-detected by autotools up until recently (#811).
Non-cmake, non-autotools build methods never enabled it automatically.

OpenSSL supports this options since at least v1.0.2, which is already
EOLed and considered obsolete. OpenSSL forks (LibreSSL, BoringSSL)
supported it all along.

In this patch we enable this option unconditionally, now requiring
OpenSSL supporting this function, or one of its forks.

Also modernize OpenSSL lib references to what 1.0.2 and newer versions
have been using.

Fixes #739

acinclude.m4

@@ -419,17 +419,8 @@ m4_case([$1],
   LIBSSH2_LIB_HAVE_LINKFLAGS([ssl], [crypto], [#include <openssl/ssl.h>], [
     AC_DEFINE(LIBSSH2_OPENSSL, 1, [Use $1])
     LIBSREQUIRED="$LIBSREQUIRED${LIBSREQUIRED:+ }libssl libcrypto"
-
-    # Not all OpenSSL have AES-CTR functions.
-    libssh2_save_LIBS="$LIBS"
-    # Duplicate $LIBS to make binutils ld (known to be fatally
-    # sensitive to lib order) happy.
-    LIBS="$LIBS $LIBSSL $LIBS"
-    AC_CHECK_FUNCS(EVP_aes_128_ctr)
-    LIBS="$libssh2_save_LIBS"
-
     found_crypto="$1"
-    found_crypto_str="OpenSSL (AES-CTR: ${ac_cv_func_EVP_aes_128_ctr:-N/A})"
+    found_crypto_str="OpenSSL"
   ])
 ],
 

os400/libssh2_config.h

@@ -62,9 +62,6 @@
 /* disabled non-blocking sockets */
 #undef HAVE_DISABLED_NONBLOCKING
 
-/* Define to 1 if you have the `EVP_aes_128_ctr' function. */
-#undef HAVE_EVP_AES_128_CTR
-
 /* use FIONBIO for non-blocking sockets */
 #undef HAVE_FIONBIO
 

src/CMakeLists.txt

@@ -76,7 +76,7 @@ if(CRYPTO_BACKEND STREQUAL "OpenSSL" OR NOT CRYPTO_BACKEND)
       list(APPEND PC_LIBS -lcrypt32 -lbcrypt)
 
       find_file(DLL_LIBCRYPTO
-        NAMES libeay32.dll crypto.dll libcrypto.dll
+        NAMES crypto.dll libcrypto.dll
           libcrypto-1_1.dll libcrypto-1_1-x64.dll
           libcrypto-3.dll libcrypto-3-x64.dll
         HINTS ${_OPENSSL_ROOT_HINTS} PATHS ${_OPENSSL_ROOT_PATHS}
@@ -87,7 +87,7 @@ if(CRYPTO_BACKEND STREQUAL "OpenSSL" OR NOT CRYPTO_BACKEND)
       endif()
 
       find_file(DLL_LIBSSL
-        NAMES ssleay32.dll ssl.dll libssl.dll
+        NAMES ssl.dll libssl.dll
           libssl-1_1.dll libssl-1_1-x64.dll
           libssl-3.dll libssl-3-x64.dll
         HINTS ${_OPENSSL_ROOT_HINTS} PATHS ${_OPENSSL_ROOT_PATHS}
@@ -101,16 +101,6 @@ if(CRYPTO_BACKEND STREQUAL "OpenSSL" OR NOT CRYPTO_BACKEND)
         list(APPEND _RUNTIME_DEPENDENCIES ${DLL_LIBCRYPTO} ${DLL_LIBSSL})
       endif()
     endif()
-
-    # Not all OpenSSL have AES-CTR functions.
-    cmake_push_check_state()
-    set(CMAKE_REQUIRED_LIBRARIES ${OPENSSL_LIBRARIES})
-    if(WIN32)
-      # For OpenSSL and LibreSSL
-      set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}" "ws2_32" "bcrypt")
-    endif()
-    check_function_exists(EVP_aes_128_ctr HAVE_EVP_AES_128_CTR)
-    cmake_pop_check_state()
   endif()
 endif()
 

src/libssh2_config_cmake.h.in

@@ -62,9 +62,6 @@
 #cmakedefine HAVE_STRTOI64
 #cmakedefine HAVE_SNPRINTF
 
-/* OpenSSL functions */
-#cmakedefine HAVE_EVP_AES_128_CTR
-
 /* Socket non-blocking support */
 #cmakedefine HAVE_O_NONBLOCK
 #cmakedefine HAVE_FIONBIO

src/openssl.c

@@ -496,215 +496,6 @@ _libssh2_cipher_crypt(_libssh2_cipher_ctx * ctx,
     return rc;
 }
 
-#if LIBSSH2_AES_CTR && !defined(HAVE_EVP_AES_128_CTR)
-
-#include <openssl/aes.h>
-#include <openssl/evp.h>
-
-typedef struct
-{
-    AES_KEY       key;
-    EVP_CIPHER_CTX *aes_ctx;
-    unsigned char ctr[AES_BLOCK_SIZE];
-} aes_ctr_ctx;
-
-static EVP_CIPHER * aes_128_ctr_cipher = NULL;
-static EVP_CIPHER * aes_192_ctr_cipher = NULL;
-static EVP_CIPHER * aes_256_ctr_cipher = NULL;
-
-static int
-aes_ctr_init(EVP_CIPHER_CTX *ctx, const unsigned char *key,
-             const unsigned char *iv, int enc) /* init key */
-{
-    /*
-     * variable "c" is leaked from this scope, but is later freed
-     * in aes_ctr_cleanup
-     */
-    aes_ctr_ctx *c;
-    const EVP_CIPHER *aes_cipher;
-    (void) enc;
-
-    switch(EVP_CIPHER_CTX_key_length(ctx)) {
-    case 16:
-        aes_cipher = EVP_aes_128_ecb();
-        break;
-    case 24:
-        aes_cipher = EVP_aes_192_ecb();
-        break;
-    case 32:
-        aes_cipher = EVP_aes_256_ecb();
-        break;
-    default:
-        return 0;
-    }
-
-    c = malloc(sizeof(*c));
-    if(c == NULL)
-        return 0;
-
-#ifdef HAVE_OPAQUE_STRUCTS
-    c->aes_ctx = EVP_CIPHER_CTX_new();
-#else
-    c->aes_ctx = malloc(sizeof(EVP_CIPHER_CTX));
-#endif
-    if(c->aes_ctx == NULL) {
-        free(c);
-        return 0;
-    }
-
-    if(EVP_EncryptInit(c->aes_ctx, aes_cipher, key, NULL) != 1) {
-#ifdef HAVE_OPAQUE_STRUCTS
-        EVP_CIPHER_CTX_free(c->aes_ctx);
-#else
-        free(c->aes_ctx);
-#endif
-        free(c);
-        return 0;
-    }
-
-    EVP_CIPHER_CTX_set_padding(c->aes_ctx, 0);
-
-    memcpy(c->ctr, iv, AES_BLOCK_SIZE);
-
-    EVP_CIPHER_CTX_set_app_data(ctx, c);
-
-    return 1;
-}
-
-static int
-aes_ctr_do_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
-                  const unsigned char *in,
-                  size_t inl) /* encrypt/decrypt data */
-{
-    aes_ctr_ctx *c = EVP_CIPHER_CTX_get_app_data(ctx);
-    unsigned char b1[AES_BLOCK_SIZE];
-    int outlen = 0;
-
-    if(inl != 16) /* libssh2 only ever encrypt one block */
-        return 0;
-
-    if(c == NULL) {
-        return 0;
-    }
-
-/*
-  To encrypt a packet P=P1||P2||...||Pn (where P1, P2, ..., Pn are each
-  blocks of length L), the encryptor first encrypts <X> with <cipher>
-  to obtain a block B1.  The block B1 is then XORed with P1 to generate
-  the ciphertext block C1.  The counter X is then incremented
-*/
-
-    if(EVP_EncryptUpdate(c->aes_ctx, b1, &outlen,
-                         c->ctr, AES_BLOCK_SIZE) != 1) {
-        return 0;
-    }
-
-    _libssh2_xor_data(out, in, b1, AES_BLOCK_SIZE);
-    _libssh2_aes_ctr_increment(c->ctr, AES_BLOCK_SIZE);
-
-    return 1;
-}
-
-static int
-aes_ctr_cleanup(EVP_CIPHER_CTX *ctx) /* cleanup ctx */
-{
-    aes_ctr_ctx *c = EVP_CIPHER_CTX_get_app_data(ctx);
-
-    if(c == NULL) {
-        return 1;
-    }
-
-    if(c->aes_ctx != NULL) {
-#ifdef HAVE_OPAQUE_STRUCTS
-        EVP_CIPHER_CTX_free(c->aes_ctx);
-#else
-        _libssh2_cipher_dtor(c->aes_ctx);
-        free(c->aes_ctx);
-#endif
-    }
-
-    free(c);
-
-    return 1;
-}
-
-static const EVP_CIPHER *
-make_ctr_evp (size_t keylen, EVP_CIPHER **aes_ctr_cipher, int type)
-{
-#ifdef HAVE_OPAQUE_STRUCTS
-    *aes_ctr_cipher = EVP_CIPHER_meth_new(type, 16, keylen);
-    if(*aes_ctr_cipher) {
-        EVP_CIPHER_meth_set_iv_length(*aes_ctr_cipher, 16);
-        EVP_CIPHER_meth_set_init(*aes_ctr_cipher, aes_ctr_init);
-        EVP_CIPHER_meth_set_do_cipher(*aes_ctr_cipher, aes_ctr_do_cipher);
-        EVP_CIPHER_meth_set_cleanup(*aes_ctr_cipher, aes_ctr_cleanup);
-    }
-#else
-    (*aes_ctr_cipher)->nid = type;
-    (*aes_ctr_cipher)->block_size = 16;
-    (*aes_ctr_cipher)->key_len = keylen;
-    (*aes_ctr_cipher)->iv_len = 16;
-    (*aes_ctr_cipher)->init = aes_ctr_init;
-    (*aes_ctr_cipher)->do_cipher = aes_ctr_do_cipher;
-    (*aes_ctr_cipher)->cleanup = aes_ctr_cleanup;
-#endif
-
-    return *aes_ctr_cipher;
-}
-
-const EVP_CIPHER *
-_libssh2_EVP_aes_128_ctr(void)
-{
-#ifdef HAVE_OPAQUE_STRUCTS
-    return !aes_128_ctr_cipher ?
-        make_ctr_evp(16, &aes_128_ctr_cipher, NID_aes_128_ctr) :
-        aes_128_ctr_cipher;
-#else
-    static EVP_CIPHER aes_ctr_cipher;
-    if(!aes_128_ctr_cipher) {
-        aes_128_ctr_cipher = &aes_ctr_cipher;
-        make_ctr_evp(16, &aes_128_ctr_cipher, 0);
-    }
-    return aes_128_ctr_cipher;
-#endif
-}
-
-const EVP_CIPHER *
-_libssh2_EVP_aes_192_ctr(void)
-{
-#ifdef HAVE_OPAQUE_STRUCTS
-    return !aes_192_ctr_cipher ?
-        make_ctr_evp(24, &aes_192_ctr_cipher, NID_aes_192_ctr) :
-        aes_192_ctr_cipher;
-#else
-    static EVP_CIPHER aes_ctr_cipher;
-    if(!aes_192_ctr_cipher) {
-        aes_192_ctr_cipher = &aes_ctr_cipher;
-        make_ctr_evp(24, &aes_192_ctr_cipher, 0);
-    }
-    return aes_192_ctr_cipher;
-#endif
-}
-
-const EVP_CIPHER *
-_libssh2_EVP_aes_256_ctr(void)
-{
-#ifdef HAVE_OPAQUE_STRUCTS
-    return !aes_256_ctr_cipher ?
-        make_ctr_evp(32, &aes_256_ctr_cipher, NID_aes_256_ctr) :
-        aes_256_ctr_cipher;
-#else
-    static EVP_CIPHER aes_ctr_cipher;
-    if(!aes_256_ctr_cipher) {
-        aes_256_ctr_cipher = &aes_ctr_cipher;
-        make_ctr_evp(32, &aes_256_ctr_cipher, 0);
-    }
-    return aes_256_ctr_cipher;
-#endif
-}
-
-#endif /* LIBSSH2_AES_CTR && !defined(HAVE_EVP_AES_128_CTR) */
-
 void _libssh2_openssl_crypto_init(void)
 {
 #if OPENSSL_VERSION_NUMBER >= 0x10100000L && \
@@ -722,34 +513,10 @@ void _libssh2_openssl_crypto_init(void)
     ENGINE_register_all_complete();
 #endif
 #endif
-#if LIBSSH2_AES_CTR && !defined(HAVE_EVP_AES_128_CTR)
-    aes_128_ctr_cipher = (EVP_CIPHER *) _libssh2_EVP_aes_128_ctr();
-    aes_192_ctr_cipher = (EVP_CIPHER *) _libssh2_EVP_aes_192_ctr();
-    aes_256_ctr_cipher = (EVP_CIPHER *) _libssh2_EVP_aes_256_ctr();
-#endif
 }
 
 void _libssh2_openssl_crypto_exit(void)
 {
-#if LIBSSH2_AES_CTR && !defined(HAVE_EVP_AES_128_CTR)
-#ifdef HAVE_OPAQUE_STRUCTS
-    if(aes_128_ctr_cipher) {
-        EVP_CIPHER_meth_free(aes_128_ctr_cipher);
-    }
-
-    if(aes_192_ctr_cipher) {
-        EVP_CIPHER_meth_free(aes_192_ctr_cipher);
-    }
-
-    if(aes_256_ctr_cipher) {
-        EVP_CIPHER_meth_free(aes_256_ctr_cipher);
-    }
-#endif
-
-    aes_128_ctr_cipher = NULL;
-    aes_192_ctr_cipher = NULL;
-    aes_256_ctr_cipher = NULL;
-#endif
 }
 
 /* TODO: Optionally call a passphrase callback specified by the

src/openssl.h

@@ -67,10 +67,6 @@
 #define OPENSSL_NO_DES
 #endif
 
-#ifdef EVP_aes_128_ctr
-#define HAVE_EVP_AES_128_CTR
-#endif
-
 /* wolfSSL doesn't support Blowfish or CAST. */
 #define OPENSSL_NO_BF
 #define OPENSSL_NO_CAST
@@ -387,15 +383,9 @@ libssh2_curve_type;
 #define _libssh2_cipher_aes256 EVP_aes_256_cbc
 #define _libssh2_cipher_aes192 EVP_aes_192_cbc
 #define _libssh2_cipher_aes128 EVP_aes_128_cbc
-#ifdef HAVE_EVP_AES_128_CTR
 #define _libssh2_cipher_aes128ctr EVP_aes_128_ctr
 #define _libssh2_cipher_aes192ctr EVP_aes_192_ctr
 #define _libssh2_cipher_aes256ctr EVP_aes_256_ctr
-#else
-#define _libssh2_cipher_aes128ctr _libssh2_EVP_aes_128_ctr
-#define _libssh2_cipher_aes192ctr _libssh2_EVP_aes_192_ctr
-#define _libssh2_cipher_aes256ctr _libssh2_EVP_aes_256_ctr
-#endif
 #define _libssh2_cipher_blowfish EVP_bf_cbc
 #define _libssh2_cipher_arcfour EVP_rc4
 #define _libssh2_cipher_cast5 EVP_cast5_cbc

win32/Makefile.Watcom

@@ -171,13 +171,12 @@ $(LINK_ARG): $(__MAKEFILES__)
 !ifdef %use_zlib
 	@%append $^@ library '$(ZLIB_ROOT)\zlib.lib'
 !endif
-!ifdef %use_wincng
+!ifndef %use_wincng
-	@%append $^@ library bcrypt.lib
+	@%append $^@ library '$(OPENSSL_ROOT)\lib\crypt.lib'
-	@%append $^@ library crypt32.lib
+	@%append $^@ library '$(OPENSSL_ROOT)\lib\ssl.lib'
-!else
-	@%append $^@ library '$(OPENSSL_ROOT)\out32\libeay32.lib'
-	@%append $^@ library '$(OPENSSL_ROOT)\out32\ssleay32.lib'
 !endif
+@%append $^@ library bcrypt.lib
+@%append $^@ library crypt32.lib
 
 $(LIB_ARG): $(__MAKEFILES__)
 	%create $^@

win32/config.mk

@@ -35,7 +35,7 @@ CPPFLAGS=$(CPPFLAGS) /DLIBSSH2_WINCNG
 # LIBS=bcrypt.lib crypt32.lib
 !else
 CPPFLAGS=$(CPPFLAGS) /DLIBSSH2_OPENSSL /I$(OPENSSLINC)
-LIBS=$(LIBS) $(OPENSSLLIB)\libeay32.lib $(OPENSSLLIB)\ssleay32.lib
+LIBS=$(LIBS) $(OPENSSLLIB)\lib\crypto.lib $(OPENSSLLIB)\lib\ssl.lib
 !endif
 
 !if "$(WITH_ZLIB)" == "1"

win32/msvcproj.head

@@ -62,7 +62,7 @@ BSC32=bscmake.exe
 # ADD BSC32 /nologo
 LINK32=link.exe
 # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386
-# ADD LINK32 gdi32.lib advapi32.lib user32.lib kernel32.lib ws2_32.lib libeay32.lib zlib.lib /nologo /dll /map /debug /machine:I386
+# ADD LINK32 gdi32.lib advapi32.lib user32.lib kernel32.lib ws2_32.lib crypto.lib zlib.lib /nologo /dll /map /debug /machine:I386
 
 !ELSEIF  "$(CFG)" == "libssh2 - Win32 OpenSSL DLL Debug"
 
@@ -89,7 +89,7 @@ BSC32=bscmake.exe
 # ADD BSC32 /nologo
 LINK32=link.exe
 # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /pdbtype:sept
-# ADD LINK32 gdi32.lib advapi32.lib user32.lib kernel32.lib ws2_32.lib libeay32.lib zlib.lib /nologo /dll /incremental:no /map /debug /machine:I386 /pdbtype:sept
+# ADD LINK32 gdi32.lib advapi32.lib user32.lib kernel32.lib ws2_32.lib crypto.lib zlib.lib /nologo /dll /incremental:no /map /debug /machine:I386 /pdbtype:sept
 # SUBTRACT LINK32 /nodefaultlib
 
 !ELSEIF  "$(CFG)" == "libssh2 - Win32 OpenSSL LIB Release"

win32/tests.dsp

@@ -50,7 +50,7 @@ BSC32=bscmake.exe
 # ADD BSC32 /nologo
 LINK32=link.exe
 # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386
-# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib libeay32.lib ssleay32.lib ws2_32.lib zlib.lib libssh2.lib /nologo /subsystem:console /machine:I386 /libpath:"Release" /out:"simple.exe"
+# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib crypto.lib ssl.lib ws2_32.lib zlib.lib libssh2.lib /nologo /subsystem:console /machine:I386 /libpath:"Release" /out:"simple.exe"
 
 !ELSEIF  "$(CFG)" == "tests - Win32 Debug"
 
@@ -74,7 +74,7 @@ BSC32=bscmake.exe
 # ADD BSC32 /nologo
 LINK32=link.exe
 # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept
-# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib libeay32.lib ssleay32.lib ws2_32.lib zlib.lib libssh2d.lib /nologo /subsystem:console /debug /machine:I386 /nodefaultlib:"msvcrt.lib" /pdbtype:sept /libpath:"Debug" /out:"simple.exe"
+# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib crypto.lib ssl.lib ws2_32.lib zlib.lib libssh2d.lib /nologo /subsystem:console /debug /machine:I386 /nodefaultlib:"msvcrt.lib" /pdbtype:sept /libpath:"Debug" /out:"simple.exe"
 
 !ENDIF