build: add support for clang-cl, add CI job

- ci/appveyor: add clang-cl job.
- ci/appvayor: optimize setting an env.
- build: fix clang-cl builds.
- build: fix `-Wcast-function-type` compiler warnings for OpenSSL 3.
- build: use `stdint.h` with MSVC when supported.
- src: use `PRId64` for MSVC where supported.
- src: avoid recursive macro definition for `recv()` and `send()`.
- session: silence `-Wcast-function-type` for `libssh2_session_callback_set2()`.
  Sadly this function is still not fully warning-clean, and it
  seems we'd need separate setter-getters for each callback
  to avoid all warnings.

Closes #1484

appveyor.yml

@@ -35,6 +35,14 @@ environment:
   SKIP_CTEST: 'yes'  # Connection to test server has been failing consistently since 2024-08-29
 
   matrix:
+    - job_name: 'VS2022, OpenSSL 3, x64, Server 2019, clang-cl'
+      APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2022'
+      GENERATOR: 'Visual Studio 17 2022'
+      PLATFORM: 'x64'
+      CRYPTO_BACKEND: 'OpenSSL'
+      OPENSSL_ROOT_DIR: 'C:/OpenSSL-v33-Win64'
+      TOOLSET: 'ClangCl'
+
     - job_name: 'VS2022, OpenSSL 3, x64, Server 2019'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2022'
       GENERATOR: 'Visual Studio 17 2022'
@@ -80,7 +88,6 @@ environment:
       BUILD_SHARED_LIBS: 'OFF'
       CRYPTO_BACKEND: 'OpenSSL'
       OPENSSL_ROOT_DIR: 'C:/OpenSSL-Win64'
-      SKIP_CTEST: 'yes'
 
     - job_name: 'VS2008, WinCNG, x86, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2015'
@@ -88,7 +95,6 @@ environment:
       PLATFORM: 'x86'
       CRYPTO_BACKEND: 'WinCNG'
       ENABLE_ECDSA_WINCNG: 'ON'
-      SKIP_CTEST: 'yes'
 
     - job_name: 'VS2010, WinCNG, x64, Build-only, non-unity'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2015'
@@ -97,7 +103,6 @@ environment:
       CRYPTO_BACKEND: 'WinCNG'
       ENABLE_ECDSA_WINCNG: 'ON'
       UNITY: 'OFF'
-      SKIP_CTEST: 'yes'
 
     - job_name: 'VS2022, WinCNG, x64, Server 2019, Logging'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2022'
@@ -148,6 +153,10 @@ build_script:
         $options += "-A$env:PLATFORM"
       }
 
+      if($env:TOOLSET) {
+        $options += "-T $env:TOOLSET"
+      }
+
       $options += "-DCRYPTO_BACKEND=$env:CRYPTO_BACKEND"
       if($env:OPENSSL_ROOT_DIR -and $env:OPENSSL_ROOT_DIR -ne '') {
         if(Test-Path $env:OPENSSL_ROOT_DIR) {

example/x11.c

@@ -347,8 +347,15 @@ int main(int argc, char *argv[])
         libssh2_trace(session, LIBSSH2_TRACE_CONN);
 
     /* Set X11 Callback */
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcast-function-type"
+#endif
     libssh2_session_callback_set2(session, LIBSSH2_CALLBACK_X11,
                                   (libssh2_cb_generic *)x11_callback);
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
 
     /* Authenticate via password */
     rc = libssh2_userauth_password(session, username, password);

include/libssh2.h

@@ -123,7 +123,7 @@ extern "C" {
 # include <sys/uio.h>
 #endif
 
-#ifdef _MSC_VER
+#if defined(_MSC_VER) && (_MSC_VER < 1600)
 typedef unsigned char uint8_t;
 typedef unsigned short int uint16_t;
 typedef unsigned int uint32_t;
@@ -132,16 +132,17 @@ typedef __int64 int64_t;
 typedef unsigned __int64 uint64_t;
 typedef unsigned __int64 libssh2_uint64_t;
 typedef __int64 libssh2_int64_t;
-#if (!defined(HAVE_SSIZE_T) && !defined(ssize_t))
-typedef SSIZE_T ssize_t;
-#define HAVE_SSIZE_T
-#endif
 #else
 #include <stdint.h>
 typedef unsigned long long libssh2_uint64_t;
 typedef long long libssh2_int64_t;
 #endif
 
+#if defined(_MSC_VER) && !defined(HAVE_SSIZE_T) && !defined(ssize_t)
+typedef SSIZE_T ssize_t;
+#define HAVE_SSIZE_T
+#endif
+
 #ifdef _WIN32
 typedef SOCKET libssh2_socket_t;
 #define LIBSSH2_INVALID_SOCKET INVALID_SOCKET

src/libssh2_priv.h

@@ -1257,10 +1257,10 @@ size_t plain_method(char *method, size_t method_len);
 #define ARRAY_SIZE(a) (sizeof ((a)) / sizeof ((a)[0]))
 
 /* define to output the libssh2_int64_t type in a *printf() */
-#if defined(__BORLANDC__) || defined(_MSC_VER)
+#if defined(__MINGW32__) || (defined(_MSC_VER) && (_MSC_VER >= 1800))
-#define LIBSSH2_INT64_T_FORMAT "I64d"
-#elif defined(__MINGW32__)
 #define LIBSSH2_INT64_T_FORMAT PRId64
+#elif defined(_WIN32)
+#define LIBSSH2_INT64_T_FORMAT "I64d"
 #else
 #define LIBSSH2_INT64_T_FORMAT "lld"
 #endif

src/misc.c

@@ -50,8 +50,11 @@
 
 #ifdef _WIN32
 /* Force parameter type. */
-#define recv(s, b, l, f)  recv((s), (b), (int)(l), (f))
+#define libssh2_recv(s, b, l, f)  recv((s), (b), (int)(l), (f))
-#define send(s, b, l, f)  send((s), (b), (int)(l), (f))
+#define libssh2_send(s, b, l, f)  send((s), (b), (int)(l), (f))
+#else
+#define libssh2_recv  recv
+#define libssh2_send  send
 #endif
 
 /* snprintf not in Visual Studio CRT and _snprintf dangerously incompatible.
@@ -159,7 +162,7 @@ _libssh2_recv(libssh2_socket_t sock, void *buffer, size_t length,
 
     (void)abstract;
 
-    rc = recv(sock, buffer, length, flags);
+    rc = libssh2_recv(sock, buffer, length, flags);
     if(rc < 0) {
         int err;
 #ifdef _WIN32
@@ -198,7 +201,7 @@ _libssh2_send(libssh2_socket_t sock, const void *buffer, size_t length,
 
     (void)abstract;
 
-    rc = send(sock, buffer, length, flags);
+    rc = libssh2_send(sock, buffer, length, flags);
     if(rc < 0) {
         int err;
 #ifdef _WIN32

src/openssl.c

@@ -44,6 +44,10 @@
 #include <stdlib.h>
 #include <assert.h>
 
+#ifdef USE_OPENSSL_3
+#define USE_PEM_READ_BIO_PRIVATEKEY
+#endif
+
 int _libssh2_hmac_ctx_init(libssh2_hmac_ctx *ctx)
 {
 #ifdef USE_OPENSSL_3
@@ -1161,7 +1165,7 @@ void _libssh2_openssl_crypto_exit(void)
  * calling program
  */
 static int
-passphrase_cb(char *buf, int size, int rwflag, char *passphrase)
+passphrase_cb(char *buf, int size, int rwflag, void *passphrase)
 {
     int passphrase_len = (int) strlen(passphrase);
 
@@ -1176,8 +1180,13 @@ passphrase_cb(char *buf, int size, int rwflag, char *passphrase)
     return passphrase_len;
 }
 
+#ifdef USE_PEM_READ_BIO_PRIVATEKEY
+typedef EVP_PKEY * (*pem_read_bio_func)(BIO *, EVP_PKEY **, pem_password_cb *,
+                                        void *u);
+#else
 typedef void * (*pem_read_bio_func)(BIO *, void **, pem_password_cb *,
                                     void *u);
+#endif
 
 static int
 read_private_key_from_memory(void **key_ctx,
@@ -1241,7 +1250,7 @@ _libssh2_rsa_new_private_frommemory(libssh2_rsa_ctx ** rsa,
 {
     int rc;
 
-#if defined(USE_OPENSSL_3)
+#ifdef USE_PEM_READ_BIO_PRIVATEKEY
     pem_read_bio_func read_rsa =
         (pem_read_bio_func) &PEM_read_bio_PrivateKey;
 #else
@@ -1649,7 +1658,7 @@ _libssh2_rsa_new_private(libssh2_rsa_ctx ** rsa,
 {
     int rc;
 
-#if defined(USE_OPENSSL_3)
+#ifdef USE_PEM_READ_BIO_PRIVATEKEY
     pem_read_bio_func read_rsa =
         (pem_read_bio_func) &PEM_read_bio_PrivateKey;
 #else
@@ -1681,7 +1690,7 @@ _libssh2_dsa_new_private_frommemory(libssh2_dsa_ctx ** dsa,
 {
     int rc;
 
-#if defined(USE_OPENSSL_3)
+#ifdef USE_PEM_READ_BIO_PRIVATEKEY
     pem_read_bio_func read_dsa =
         (pem_read_bio_func) &PEM_read_bio_PrivateKey;
 #else
@@ -2006,7 +2015,7 @@ _libssh2_dsa_new_private(libssh2_dsa_ctx ** dsa,
 {
     int rc;
 
-#if defined(USE_OPENSSL_3)
+#ifdef USE_PEM_READ_BIO_PRIVATEKEY
     pem_read_bio_func read_dsa =
         (pem_read_bio_func) &PEM_read_bio_PrivateKey;
 #else
@@ -2038,7 +2047,7 @@ _libssh2_ecdsa_new_private_frommemory(libssh2_ecdsa_ctx ** ec_ctx,
 {
     int rc;
 
-#if defined(USE_OPENSSL_3)
+#ifdef USE_PEM_READ_BIO_PRIVATEKEY
     pem_read_bio_func read_ec =
         (pem_read_bio_func) &PEM_read_bio_PrivateKey;
 #else
@@ -4004,7 +4013,7 @@ _libssh2_ecdsa_new_private(libssh2_ecdsa_ctx ** ec_ctx,
 {
     int rc;
 
-#if defined(USE_OPENSSL_3)
+#ifdef USE_PEM_READ_BIO_PRIVATEKEY
     pem_read_bio_func read_ec =
         (pem_read_bio_func) &PEM_read_bio_PrivateKey;
 #else
@@ -4037,7 +4046,7 @@ _libssh2_ecdsa_new_private_sk(libssh2_ecdsa_ctx ** ec_ctx,
 {
     int rc;
 
-#if defined(USE_OPENSSL_3)
+#ifdef USE_PEM_READ_BIO_PRIVATEKEY
     pem_read_bio_func read_ec =
         (pem_read_bio_func) &PEM_read_bio_PrivateKey;
 #else
@@ -4981,7 +4990,7 @@ _libssh2_sk_pub_openssh_keyfilememory(LIBSSH2_SESSION *session,
     return rc;
 }
 
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#ifdef USE_OPENSSL_3
 #define HAVE_SSLERROR_BAD_DECRYPT
 #endif
 

src/session.c

@@ -489,6 +489,10 @@ libssh2_session_init_ex(LIBSSH2_ALLOC_FUNC((*my_alloc)),
  * Set (or reset) a callback function
  * Returns the prior address
  */
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcast-function-type"
+#endif
 LIBSSH2_API libssh2_cb_generic *
 libssh2_session_callback_set2(LIBSSH2_SESSION *session, int cbtype,
                               libssh2_cb_generic *callback)
@@ -553,6 +557,9 @@ libssh2_session_callback_set2(LIBSSH2_SESSION *session, int cbtype,
 
     return NULL;
 }
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
 
 /*
  * libssh2_session_callback_set (DEPRECATED, DO NOT USE!)