src: add Unicode, long-filename, and secure fopen() support (Windows)

Low-level logic copied from curl, and adapted to libssh2.

Also: use secure fopen CRT functions.

Credits-to: Jay Satiro
Credits-to: Viktor Szakats

Closes #2318

example/.checksrc

@@ -3,6 +3,7 @@
 
 allowfunc accept
 allowfunc atoi
+allowfunc fopen
 allowfunc fprintf
 allowfunc fstat
 allowfunc printf

scripts/checksrc-all.pl

@@ -16,7 +16,6 @@ my @options = (
     "-acalloc",
     "-aclose",
     "-afclose",
-    "-afopen",
     "-afree",
     "-amalloc",
     "-arealloc",

src/knownhost.c

@@ -930,7 +930,7 @@ int libssh2_knownhost_readfile(LIBSSH2_KNOWNHOSTS *hosts,
         return ssh2_err(hosts->session, LIBSSH2_ERROR_METHOD_NOT_SUPPORTED,
                         "Unsupported type of known-host information store");
 
-    fp = fopen(filename, FOPEN_READTEXT);
+    fp = libssh2_fopen(filename, FOPEN_READTEXT);
     if(fp) {
         while(fgets(buf, sizeof(buf), fp)) {
             if(libssh2_knownhost_readline(hosts, buf, strlen(buf), type)) {
@@ -1159,7 +1159,7 @@ int libssh2_knownhost_writefile(LIBSSH2_KNOWNHOSTS *hosts,
         return ssh2_err(hosts->session, LIBSSH2_ERROR_METHOD_NOT_SUPPORTED,
                         "Unsupported type of known-host information store");
 
-    fp = fopen(filename, FOPEN_WRITETEXT);
+    fp = libssh2_fopen(filename, FOPEN_WRITETEXT);
     if(!fp)
         return ssh2_err(hosts->session, LIBSSH2_ERROR_FILE,
                         "Failed to open file");

src/libgcrypt.c

@@ -265,7 +265,7 @@ int ssh2_rsa_new_private(ssh2_rsa_ctx **rsa,
     unsigned char *n, *e, *d, *p, *q, *e1, *e2, *coeff;
     unsigned int nlen, elen, dlen, plen, qlen, e1len, e2len, coefflen;
 
-    fp = fopen(filename, "rb");
+    fp = libssh2_fopen(filename, "rb");
     if(!fp)
         return -1;
 
@@ -379,7 +379,7 @@ int ssh2_dsa_new_private(ssh2_dsa_ctx **dsa,
     unsigned char *p, *q, *g, *y, *x;
     unsigned int plen, qlen, glen, ylen, xlen;
 
-    fp = fopen(filename, "rb");
+    fp = libssh2_fopen(filename, "rb");
     if(!fp)
         return -1;
 

src/libssh2_setup.h

@@ -75,10 +75,13 @@
 #  define _FILE_OFFSET_BITS 64
 #elif defined(_MSC_VER)
 #  ifndef _CRT_SECURE_NO_WARNINGS
-#  define _CRT_SECURE_NO_WARNINGS  /* for fopen() */
+#  define _CRT_SECURE_NO_WARNINGS  /* for vsnprintf() */
 #  endif
 #  if !defined(LIBSSH2_LIBRARY) || defined(LIBSSH2_TESTS)
-    /* apply to examples and tests only */
+     /* apply to examples and tests only */
+#    ifndef _CRT_SECURE_NO_WARNINGS
+#    define _CRT_SECURE_NO_WARNINGS  /* for fopen(), getenv() */
+#    endif
 #    ifndef _CRT_NONSTDC_NO_DEPRECATE
 #    define _CRT_NONSTDC_NO_DEPRECATE  /* for write() */
 #    endif

src/mbedtls.c

@@ -1155,7 +1155,7 @@ int ssh2_ecdsa_new_private(ssh2_ecdsa_ctx **ec_ctx,
 
     mbedtls_pk_init(&pkey);
 
-    fp = fopen(filename, "rb");
+    fp = libssh2_fopen(filename, "rb");
     if(!fp)
         goto cleanup;
     if(fseek(fp, 0, SEEK_END))

src/misc.c

@@ -996,3 +996,235 @@ int ssh2_sk_pub_keyfilememory(LIBSSH2_SESSION *session,
                     SSH2_CRYPTO_ENGINE_NAME " backend");
 }
 #endif
+
+#ifdef _WIN32
+#include <share.h>  /* for _SH_DENYNO */
+#include <stdlib.h>  /* for malloc(), free() */
+#include <tchar.h>  /* for _tcsncmp() */
+
+#ifdef _UNICODE
+static wchar_t *ssh2_win32_fn_convert_UTF8_to_wchar(const char *str_utf8)
+{
+    wchar_t *str_w = NULL;
+
+    if(str_utf8) {
+        int str_w_len = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS,
+                                            str_utf8, -1, NULL, 0);
+        if(str_w_len > 0) {
+            str_w = malloc(str_w_len * sizeof(wchar_t));
+            if(str_w) {
+                if(MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS,
+                                       str_utf8, -1, str_w, str_w_len) == 0) {
+                    free(str_w);
+                    return NULL;
+                }
+            }
+        }
+    }
+    return str_w;
+}
+#endif
+
+/* declare GetFullPathNameW for mingw-w64 UWP builds targeting old Windows */
+#if defined(LIBSSH2_WINDOWS_UWP) && defined(__MINGW32__) && \
+  (_WIN32_WINNT < _WIN32_WINNT_WIN10)
+WINBASEAPI DWORD WINAPI GetFullPathNameW(LPCWSTR, DWORD, LPWSTR, LPWSTR *);
+#endif
+
+/* Fix excessive paths (paths that exceed MAX_PATH length of 260).
+ *
+ * This is a helper function to fix paths that would exceed the MAX_PATH
+ * limitation check done by Windows APIs. It does so by normalizing the passed
+ * in filename or path 'in' to its full canonical path, and if that path is
+ * longer than MAX_PATH then setting 'out' to "\\?\" prefix + that full path.
+ *
+ * For example 'in' filename255chars in current directory C:\foo\bar is
+ * fixed as \\?\C:\foo\bar\filename255chars for 'out' which tells Windows
+ * it is ok to access that filename even though the actual full path is longer
+ * than 260 chars.
+ *
+ * For non-Unicode builds this function may fail sometimes because only the
+ * Unicode versions of some Windows API functions can access paths longer than
+ * MAX_PATH, for example GetFullPathNameW which is used in this function. When
+ * the full path is then converted from Unicode to multibyte that fails if any
+ * directories in the path contain characters not in the current codepage.
+ */
+static int ssh2_win32_fix_excessive_path(const TCHAR *in, TCHAR **out)
+{
+    size_t needed, count;
+    const wchar_t *in_w;
+    wchar_t *fbuf = NULL;
+
+    /* MS-documented "approximate" limit for the maximum path length */
+    const size_t max_path_len = 32767;
+
+#ifndef _UNICODE
+    wchar_t *ibuf = NULL;
+    char *obuf = NULL;
+#endif
+
+    *out = NULL;
+
+    /* skip paths already normalized */
+    if(!_tcsncmp(in, _TEXT("\\\\?\\"), 4))
+        goto cleanup;
+
+#ifndef _UNICODE
+    /* convert multibyte input to unicode */
+    if(mbstowcs_s(&needed, NULL, 0, in, 0))
+        goto cleanup;
+    if(!needed || needed >= max_path_len)
+        goto cleanup;
+    ibuf = malloc(needed * sizeof(wchar_t));
+    if(!ibuf)
+        goto cleanup;
+    if(mbstowcs_s(&count, ibuf, needed, in, needed - 1))
+        goto cleanup;
+    if(count != needed)
+        goto cleanup;
+    in_w = ibuf;
+#else
+    in_w = in;
+#endif
+
+    /* GetFullPathNameW returns the normalized full path in unicode. It
+       converts forward slashes to backslashes, processes .. to remove
+       directory segments, etc. Unlike GetFullPathNameA it can process
+       paths that exceed MAX_PATH. */
+    needed = (size_t)GetFullPathNameW(in_w, 0, NULL, NULL);
+    if(!needed || needed > max_path_len)
+        goto cleanup;
+    /* skip paths that are not excessive and do not need modification */
+    if(needed <= MAX_PATH)
+        goto cleanup;
+    fbuf = malloc(needed * sizeof(wchar_t));
+    if(!fbuf)
+        goto cleanup;
+    count = (size_t)GetFullPathNameW(in_w, (DWORD)needed, fbuf, NULL);
+    if(!count || count >= needed)
+        goto cleanup;
+
+    /* prepend \\?\ or \\?\UNC\ to the excessively long path.
+     *
+     * c:\longpath            --->    \\?\c:\longpath
+     * \\.\c:\longpath        --->    \\?\c:\longpath
+     * \\?\c:\longpath        --->    \\?\c:\longpath  (unchanged)
+     * \\server\c$\longpath   --->    \\?\UNC\server\c$\longpath
+     *
+     * https://learn.microsoft.com/dotnet/standard/io/file-path-formats
+     */
+    if(!wcsncmp(fbuf, L"\\\\?\\", 4))
+        ; /* do nothing */
+    else if(!wcsncmp(fbuf, L"\\\\.\\", 4))
+        fbuf[2] = '?';
+    else if(!wcsncmp(fbuf, L"\\\\.", 3) || !wcsncmp(fbuf, L"\\\\?", 3))
+        /* Unexpected, not UNC. The formatting doc does not allow this
+           AFAICT. */
+        goto cleanup;
+    else {
+        wchar_t *temp;
+
+        if(!wcsncmp(fbuf, L"\\\\", 2)) {
+            /* "\\?\UNC\" + full path without "\\" + null */
+            needed = 8 + (count - 2) + 1;
+            if(needed > max_path_len)
+                goto cleanup;
+
+            temp = malloc(needed * sizeof(wchar_t));
+            if(!temp)
+                goto cleanup;
+
+            if(wcsncpy_s(temp, needed, L"\\\\?\\UNC\\", 8)) {
+                free(temp);
+                goto cleanup;
+            }
+            if(wcscpy_s(temp + 8, needed, fbuf + 2)) {
+                free(temp);
+                goto cleanup;
+            }
+        }
+        else {
+            /* "\\?\" + full path + null */
+            needed = 4 + count + 1;
+            if(needed > max_path_len)
+                goto cleanup;
+
+            temp = malloc(needed * sizeof(wchar_t));
+            if(!temp)
+                goto cleanup;
+
+            if(wcsncpy_s(temp, needed, L"\\\\?\\", 4)) {
+                free(temp);
+                goto cleanup;
+            }
+            if(wcscpy_s(temp + 4, needed, fbuf)) {
+                free(temp);
+                goto cleanup;
+            }
+        }
+
+        free(fbuf);
+        fbuf = temp;
+    }
+
+#ifndef _UNICODE
+    /* convert unicode full path to multibyte output */
+    if(wcstombs_s(&needed, NULL, 0, fbuf, 0))
+        goto cleanup;
+    if(!needed || needed >= max_path_len)
+        goto cleanup;
+    obuf = malloc(needed);
+    if(!obuf)
+        goto cleanup;
+    if(wcstombs_s(&count, obuf, needed, fbuf, needed - 1))
+        goto cleanup;
+    if(count != needed)
+        goto cleanup;
+    *out = obuf;
+    obuf = NULL;
+#else
+    *out = fbuf;
+    fbuf = NULL;
+#endif
+
+cleanup:
+    free(fbuf);
+#ifndef _UNICODE
+    free(ibuf);
+    free(obuf);
+#endif
+    return !!*out;
+}
+
+FILE *libssh2_win32_fopen(const char *filename, const char *mode)
+{
+    FILE *fp = NULL;
+    TCHAR *fixed = NULL;
+    const TCHAR *target = NULL;
+
+#ifdef _UNICODE
+    wchar_t *filename_w = ssh2_win32_fn_convert_UTF8_to_wchar(filename);
+    wchar_t *mode_w = ssh2_win32_fn_convert_UTF8_to_wchar(mode);
+    if(filename_w && mode_w) {
+        if(ssh2_win32_fix_excessive_path(filename_w, &fixed))
+            target = fixed;
+        else
+            target = filename_w;
+        fp = _wfsopen(target, mode_w, _SH_DENYNO);
+    }
+    else
+        errno = EINVAL;
+    free(filename_w);
+    free(mode_w);
+#else
+    if(ssh2_win32_fix_excessive_path(filename, &fixed))
+        target = fixed;
+    else
+        target = filename;
+    fp = _fsopen(target, mode, _SH_DENYNO);
+#endif
+
+    free(fixed);
+    return fp;
+}
+#endif /* _WIN32 */

src/misc.h

@@ -34,6 +34,13 @@
 
 #include <errno.h>
 
+#ifdef _WIN32
+FILE *libssh2_win32_fopen(const char *filename, const char *mode);
+#define libssh2_fopen libssh2_win32_fopen
+#else
+#define libssh2_fopen fopen
+#endif
+
 #ifdef LIBSSH2_NO_CLEAR_MEMORY
 #define ssh2_explicit_zero(buf, size) \
     do {                              \

src/openssl.c

@@ -1280,7 +1280,7 @@ static int ossl_rsa_openssh_priv_new(ssh2_rsa_ctx **rsa,
 
     OSSL_INIT_IF_NEEDED();
 
-    fp = fopen(filename, "r");
+    fp = libssh2_fopen(filename, "r");
     if(!fp) {
         ssh2_err(session, LIBSSH2_ERROR_FILE,
                  "Unable to open OpenSSH RSA private key file");
@@ -1590,7 +1590,7 @@ static int ossl_dsa_openssh_priv_new(ssh2_dsa_ctx **dsa,
 
     OSSL_INIT_IF_NEEDED();
 
-    fp = fopen(filename, "r");
+    fp = libssh2_fopen(filename, "r");
     if(!fp) {
         ssh2_err(session, LIBSSH2_ERROR_FILE,
                  "Unable to open OpenSSH DSA private key file");
@@ -2102,7 +2102,7 @@ int ssh2_ed25519_new_private(ssh2_ed25519_ctx **ed_ctx,
 
     OSSL_INIT_IF_NEEDED();
 
-    fp = fopen(filename, "r");
+    fp = libssh2_fopen(filename, "r");
     if(!fp) {
         ssh2_err(session, LIBSSH2_ERROR_FILE,
                  "Unable to open ED25519 private key file");
@@ -3048,7 +3048,7 @@ static int ossl_ecdsa_openssh_priv_new(ssh2_ecdsa_ctx **ec_ctx,
 
     OSSL_INIT_IF_NEEDED();
 
-    fp = fopen(filename, "r");
+    fp = libssh2_fopen(filename, "r");
     if(!fp) {
         ssh2_err(session, LIBSSH2_ERROR_FILE,
                  "Unable to open OpenSSH ECDSA private key file");
@@ -3570,7 +3570,7 @@ static int ossl_key_from_openssh_file(LIBSSH2_SESSION *session,
 
     OSSL_INIT_IF_NEEDED();
 
-    fp = fopen(privatekey, "r");
+    fp = libssh2_fopen(privatekey, "r");
     if(!fp) {
         ssh2_err(session, LIBSSH2_ERROR_FILE,
                  "Unable to open private key file");

src/os400qc3.c

@@ -2016,7 +2016,7 @@ static int load_rsa_private_file(LIBSSH2_SESSION *session,
                                  loadkeyproc proc1, loadkeyproc proc8,
                                  void *loadkeydata)
 {
-    FILE *fp = fopen(filename, fopenrbmode);
+    FILE *fp = libssh2_fopen(filename, fopenrbmode);
     unsigned char *data = NULL;
     size_t datalen = 0;
     int ret;

src/userauth.c

@@ -642,7 +642,7 @@ static int userauth_read_file_pubkey(
     ssh2_deb((session, LIBSSH2_TRACE_AUTH, "Loading public key file: %s",
               pubkeyfile));
     /* Read Public Key */
-    fd = fopen(pubkeyfile, "rb");
+    fd = libssh2_fopen(pubkeyfile, "rb");
     if(!fd)
         return ssh2_err(session, LIBSSH2_ERROR_FILE,
                         "Unable to open public key file");

src/wincng.c

@@ -910,7 +910,7 @@ static int wcng_load_pem(LIBSSH2_SESSION *session,
     FILE *fp;
     int ret;
 
-    fp = fopen(filename, "rb");
+    fp = libssh2_fopen(filename, "rb");
     if(!fp)
         return -1;
 
@@ -2442,7 +2442,7 @@ int ssh2_ecdsa_new_private(OUT ssh2_ecdsa_ctx **ec_ctx,
                         "Passphrase-protected ECDSA private key "
                         "files are unsupported");
 
-    fp = fopen(filename, "rb");
+    fp = libssh2_fopen(filename, "rb");
     if(!fp) {
         result = ssh2_err(session, LIBSSH2_ERROR_INVAL,
                           "Opening the private key file failed");

tests/.checksrc

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 allowfunc atoi
+allowfunc fopen
 allowfunc fprintf
 allowfunc printf
 allowfunc vsnprintf

tests/ossfuzz/.checksrc

@@ -1,6 +1,7 @@
 # Copyright (C) The libssh2 project and its contributors.
 # SPDX-License-Identifier: BSD-3-Clause
 
+allowfunc fopen
 allowfunc fprintf
 allowfunc printf
 allowfunc send

vms/.checksrc

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 allowfunc atoi
+allowfunc fopen
 allowfunc fprintf
 allowfunc printf
 allowfunc strdup