Make OS/400 implementation work again (#953)

* os400: support QADRT development files in a non-standard directory

This enables the possibility to compile libssh2 even if the ascii
runtime development files are not installed system-wide.

* userauth_kbd_packet: fix a pointer target type mismatch.

A temporary variable matching the parameter type is used before copying
to the real target and checking for overflow (that should not occur!).

* os400qc3: move and fix big number procedures

A bug added by a previous code style cleaning is fixed.
_libssh2_random() now checks and return the success status.

* os400qc3: fix cipher definition block lengths

They were wrongly set to the key size.

* Diffie-Hellman min/max modulus sizes are dependent of crypto-backend

In particular, os400qc3 limits the maximum group size to 2048-bits.
Move definitions of these parameters to crypto backend header files.

* kex: return an error if Diffie-Hellman key pair generation fails

* os400: add an ascii assert.h header file

* os400qc3: implement RSA SHA2 256/512

docs/HACKING-CRYPTO

@@ -406,6 +406,21 @@ TripleDES-CBC algorithm identifier initializer.
 
 5) Diffie-Hellman support.
 
+LIBSSH2_DH_GEX_MINGROUP
+The minimum Diffie-Hellman group length in bits supported by the backend.
+Usually defined as 2048.
+
+LIBSSH2_DH_GEX_OPTGROUP
+The preferred Diffie-Hellman group length in bits. Usually defined as 4096.
+
+LIBSSH2_DH_GEX_MAXGROUP
+The maximum Diffie-Hellman group length in bits supported by the backend.
+Usually defined as 8192.
+ 
+LIBSSH2_DH_MAX_MODULUS_BITS
+The maximum Diffie-Hellman modulus bit count accepted from the server. This
+value must be supported by the backend.  Usually 16384.
+
 5.1) Diffie-Hellman context.
 _libssh2_dh_ctx
 Type of a Diffie-Hellman computation context.
@@ -658,7 +673,28 @@ the allocated signature at (signature, signature_len).
 Signature buffer must be allocated from the given session.
 Returns 0 if OK, else -1.
 This procedure is already prototyped in crypto.h.
-Note: this procedure is not used if macro _libssh2_rsa_sha1_signv() is defined.
+Note: this procedure is not used if both macros _libssh2_rsa_sha2_256_signv()
+and _libssh2_rsa_sha2_512_signv are defined.
+
+int _libssh2_rsa_sha2_256_signv(LIBSSH2_SESSION *session,
+                                unsigned char **sig, size_t *siglen,
+                                int count, const struct iovec vector[],
+                                libssh2_rsa_ctx *ctx);
+RSA signs the SHA-256 hash computed over the count data chunks in vector.
+Signature is stored at (sig, siglen).
+Signature buffer must be allocated from the given session.
+Returns 0 if OK, else -1.
+Note: this procedure is optional: if provided, it MUST be defined as a macro.
+
+int _libssh2_rsa_sha2_512_signv(LIBSSH2_SESSION *session,
+                                unsigned char **sig, size_t *siglen,
+                                int count, const struct iovec vector[],
+                                libssh2_rsa_ctx *ctx);
+RSA signs the SHA-512 hash computed over the count data chunks in vector.
+Signature is stored at (sig, siglen).
+Signature buffer must be allocated from the given session.
+Returns 0 if OK, else -1.
+Note: this procedure is optional: if provided, it MUST be defined as a macro.
 
 int _libssh2_rsa_sha2_verify(libssh2_rsa_ctx * rsa,
                              size_t hash_len,

include/libssh2.h

@@ -228,14 +228,6 @@ typedef off_t libssh2_struct_stat_size;
 #define LIBSSH2_SSH_DEFAULT_BANNER            LIBSSH2_SSH_BANNER
 #define LIBSSH2_SSH_DEFAULT_BANNER_WITH_CRLF  LIBSSH2_SSH_DEFAULT_BANNER "\r\n"
 
-/* Default generate and safe prime sizes for
-   diffie-hellman-group-exchange-sha1 */
-#define LIBSSH2_DH_GEX_MINGROUP     2048
-#define LIBSSH2_DH_GEX_OPTGROUP     4096
-#define LIBSSH2_DH_GEX_MAXGROUP     8192
-
-#define LIBSSH2_DH_MAX_MODULUS_BITS 16384
-
 /* Defaults for pty requests */
 #define LIBSSH2_TERM_WIDTH      80
 #define LIBSSH2_TERM_HEIGHT     24

os400/README400

@@ -38,7 +38,8 @@ familiar with.
 
 _ As a prerequisite, QADRT development environment must be installed.
 _ Install the libssh2 sources directory in IFS.
-_ Enter shell (QSH)
+_ Enter shell (QSH). You may need to change the LANG environment variable
+  to be in phase with the libssh2 source files CCSID.
 _ Change current directory to the libssh2 sources installation directory
 _ Change current directory to os400
 _ Edit file iniscript.sh. You may want to change tunable configuration

os400/include/assert.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2023 Patrick Monnerat <patrick@monnerat.net>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms,
+ * with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ *   Redistributions of source code must retain the above
+ *   copyright notice, this list of conditions and the
+ *   following disclaimer.
+ *
+ *   Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials
+ *   provided with the distribution.
+ *
+ *   Neither the name of the copyright holder nor the names
+ *   of any other contributors may be used to endorse or
+ *   promote products derived from this software without
+ *   specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+ * OF SUCH DAMAGE.
+ */
+
+#ifndef LIBSSH2_ASSERT_H
+#define LIBSSH2_ASSERT_H
+
+#include <stdio.h>
+#include <stdlib.h>
+
+/* Ascii assert() macro. */
+
+#ifndef NDEBUG
+#pragma convert(819)
+
+#define assert(expr)    ((expr)? ((void) 0): (fprintf(stderr,           \
+                "Assertion failed: %s in file %s line %u. Aborting\n",  \
+                #expr, __FILE__, __LINE__), abort()))
+#else
+#define assert(expr)    ((void) 0)
+#endif
+#endif
+
+/* vim: set expandtab ts=4 sw=4: */

os400/initscript.sh

@@ -49,8 +49,9 @@ setenv TGTCCSID         '500'                   # Target CCSID of objects.
 setenv DEBUG            '*ALL'                  # Debug level.
 setenv OPTIMIZE         '10'                    # Optimisation level
 setenv OUTPUT           '*NONE'                 # Compilation output option.
-setenv TGTRLS           'V6R1M0'                # Target OS release.
+setenv TGTRLS           'V7R3M0'                # Target OS release.
 setenv IFSDIR           '/libssh2'              # Installation IFS directory.
+setenv QADRTDIR         '/QIBM/ProdData/qadrt'  # QADRT IFS directory.
 
 #       Define ZLIB availability and locations.
 
@@ -182,7 +183,7 @@ make_module()
         CMD="${CMD} SYSIFCOPT(*IFS64IO) OPTION(*INCDIRFIRST)"
         CMD="${CMD} LOCALETYPE(*LOCALE) FLAG(10)"
         CMD="${CMD} INCDIR('${TOPDIR}/os400/include'"
-        CMD="${CMD} '/QIBM/ProdData/qadrt/include' '${TOPDIR}/include'"
+        CMD="${CMD} '${QADRTDIR}/include' '${TOPDIR}/include'"
         CMD="${CMD} '${TOPDIR}/os400' '${SRCDIR}'"
 
         if [ "${WITH_ZLIB}" != "0" ]

src/kex.c

@@ -263,8 +263,11 @@ static int diffie_hellman_sha_algo(LIBSSH2_SESSION *session,
 
         rc = libssh2_dh_key_pair(&exchange_state->x, exchange_state->e, g, p,
                                  group_order, exchange_state->ctx);
-        if(rc)
+        if(rc) {
+            ret = _libssh2_error(session, LIBSSH2_ERROR_KEX_FAILURE,
+                                 "dh key pair generation failed");
             goto clean_exit;
+        }
 
         /* Send KEX init */
         /* packet_type(1) + String Length(4) + leading 0(1) */

src/libgcrypt.h

@@ -222,6 +222,14 @@
 #define _libssh2_bn_bits(bn) gcry_mpi_get_nbits (bn)
 #define _libssh2_bn_free(bn) gcry_mpi_release(bn)
 
+/* Default generate and safe prime sizes for
+   diffie-hellman-group-exchange-sha1 */
+#define LIBSSH2_DH_GEX_MINGROUP     2048
+#define LIBSSH2_DH_GEX_OPTGROUP     4096
+#define LIBSSH2_DH_GEX_MAXGROUP     8192
+
+#define LIBSSH2_DH_MAX_MODULUS_BITS 16384
+
 #define _libssh2_dh_ctx struct gcry_mpi *
 #define libssh2_dh_init(dhctx) _libssh2_dh_init(dhctx)
 #define libssh2_dh_key_pair(dhctx, public, g, p, group_order, bnctx) \

src/mbedtls.h

@@ -429,6 +429,14 @@ typedef enum {
  * mbedTLS backend: Diffie-Hellman support.
  */
 
+/* Default generate and safe prime sizes for
+   diffie-hellman-group-exchange-sha1 */
+#define LIBSSH2_DH_GEX_MINGROUP     2048
+#define LIBSSH2_DH_GEX_OPTGROUP     4096
+#define LIBSSH2_DH_GEX_MAXGROUP     8192
+
+#define LIBSSH2_DH_MAX_MODULUS_BITS 16384
+
 #define _libssh2_dh_ctx mbedtls_mpi *
 #define libssh2_dh_init(dhctx) _libssh2_dh_init(dhctx)
 #define libssh2_dh_key_pair(dhctx, public, g, p, group_order, bnctx) \

src/openssl.h

@@ -419,6 +419,14 @@ libssh2_curve_type;
 #define _libssh2_bn_bits(bn) BN_num_bits(bn)
 #define _libssh2_bn_free(bn) BN_clear_free(bn)
 
+/* Default generate and safe prime sizes for
+   diffie-hellman-group-exchange-sha1 */
+#define LIBSSH2_DH_GEX_MINGROUP     2048
+#define LIBSSH2_DH_GEX_OPTGROUP     4096
+#define LIBSSH2_DH_GEX_MAXGROUP     8192
+
+#define LIBSSH2_DH_MAX_MODULUS_BITS 16384
+
 #define _libssh2_dh_ctx BIGNUM *
 #define libssh2_dh_init(dhctx) _libssh2_dh_init(dhctx)
 #define libssh2_dh_key_pair(dhctx, public, g, p, group_order, bnctx) \

src/os400qc3.c

@@ -1,6 +1,6 @@
 /*
  * Copyright (C) 2015-2016 Patrick Monnerat, D+H <patrick.monnerat@dh.com>
- * Copyright (C) 2020 Patrick Monnerat <patrick@monnerat.net>.
+ * Copyright (C) 2020-2023 Patrick Monnerat <patrick@monnerat.net>.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms,
@@ -348,6 +348,169 @@ static asn1Element  lastbytebitcount = {
 };
 
 
+/*******************************************************************
+ *
+ * OS/400 QC3 crypto-library backend: big numbers support.
+ *
+ *******************************************************************/
+
+int
+_libssh2_random(unsigned char *buf, size_t len)
+{
+    Qus_EC_t errcode;
+
+    set_EC_length(errcode, sizeof errcode);
+    Qc3GenPRNs(buf, len,
+        Qc3PRN_TYPE_NORMAL, Qc3PRN_NO_PARITY, (char *) &errcode);
+    return errcode.Bytes_Available? -1: 0;
+}
+
+_libssh2_bn *
+_libssh2_bn_init(void)
+{
+    _libssh2_bn *bignum;
+
+    bignum = (_libssh2_bn *) malloc(sizeof *bignum);
+    if(bignum) {
+        bignum->bignum = NULL;
+        bignum->length = 0;
+    }
+
+    return bignum;
+}
+
+void
+_libssh2_bn_free(_libssh2_bn *bn)
+{
+    if(bn) {
+        if(bn->bignum) {
+            if(bn->length)
+                _libssh2_explicit_zero(bn->bignum, bn->length);
+
+            free(bn->bignum);
+        }
+
+        free((char *) bn);
+    }
+}
+
+static int
+_libssh2_bn_resize(_libssh2_bn *bn, size_t newlen)
+{
+    unsigned char *bignum;
+
+    if(!bn)
+        return -1;
+    if(newlen == bn->length)
+        return 0;
+
+    if(!bn->bignum)
+        bignum = (unsigned char *) malloc(newlen);
+    else {
+        if(newlen < bn->length)
+            _libssh2_explicit_zero(bn->bignum + newlen, bn->length - newlen);
+
+        if(!newlen) {
+            free((char *) bn->bignum);
+            bn->bignum = NULL;
+            bn->length = 0;
+            return 0;
+        }
+        bignum = (unsigned char *) realloc((char *) bn->bignum, newlen);
+    }
+
+    if(!bignum)
+        return -1;
+
+    if(newlen > bn->length)
+        memset((char *) bignum + bn->length, 0, newlen - bn->length);
+
+    bn->bignum = bignum;
+    bn->length = newlen;
+    return 0;
+}
+
+unsigned long
+_libssh2_bn_bits(_libssh2_bn *bn)
+{
+    unsigned int i;
+    unsigned char b;
+
+    if(bn && bn->bignum) {
+        for(i = bn->length; i--;) {
+            b = bn->bignum[i];
+            if(b) {
+                i *= 8;
+                do {
+                    i++;
+                } while(b >>= 1);
+                return i;
+            }
+        }
+    }
+
+    return 0;
+}
+
+int
+_libssh2_bn_from_bin(_libssh2_bn *bn, int len, const unsigned char *val)
+{
+    int i;
+
+    if(!bn || (len && !val))
+        return -1;
+
+    for(; len && !*val; len--)
+        val++;
+
+    if(_libssh2_bn_resize(bn, len))
+        return -1;
+
+    for(i = len; i--;)
+        bn->bignum[i] = *val++;
+
+    return 0;
+}
+
+int
+_libssh2_bn_set_word(_libssh2_bn *bn, unsigned long val)
+{
+    val = htonl(val);
+    return _libssh2_bn_from_bin(bn, sizeof val, (unsigned char *) &val);
+}
+
+int
+_libssh2_bn_to_bin(_libssh2_bn *bn, unsigned char *val)
+{
+    int i;
+
+    if(!bn || !val)
+        return -1;
+
+    for(i = bn->length; i--;)
+        *val++ = bn->bignum[i];
+
+    return 0;
+}
+
+static int
+_libssh2_bn_from_bn(_libssh2_bn *to, _libssh2_bn *from)
+{
+    int i;
+
+    if(!to || !from)
+        return -1;
+
+    if(_libssh2_bn_resize(to, from->length))
+        return -1;
+
+    for(i = to->length; i--;)
+        to->bignum[i] = from->bignum[i];
+
+    return 0;
+}
+
+
 /*******************************************************************
  *
  * OS/400 QC3 crypto-library backend: ASN.1 support.
@@ -727,167 +890,6 @@ rsaprivatekeyinfo(asn1Element *privkey)
     return privkeyinfo;
 }
 
-/*******************************************************************
- *
- * OS/400 QC3 crypto-library backend: big numbers support.
- *
- *******************************************************************/
-
-
-_libssh2_bn *
-_libssh2_bn_init(void)
-{
-    _libssh2_bn *bignum;
-
-    bignum = (_libssh2_bn *) malloc(sizeof *bignum);
-    if(bignum) {
-        bignum->bignum = NULL;
-        bignum->length = 0;
-    }
-
-    return bignum;
-}
-
-void
-_libssh2_bn_free(_libssh2_bn *bn)
-{
-    if(bn) {
-        if(bn->bignum) {
-            if(bn->length)
-                _libssh2_explicit_zero(bn->bignum, bn->length);
-
-            free(bn->bignum);
-        }
-
-        free((char *) bn);
-    }
-}
-
-static int
-_libssh2_bn_resize(_libssh2_bn *bn, size_t newlen)
-{
-    unsigned char *bignum;
-
-    if(!bn)
-        return -1;
-    if(newlen == bn->length)
-        return 0;
-
-    if(!bn->bignum)
-        bignum = (unsigned char *) malloc(newlen);
-    else {
-        if(newlen < bn->length)
-            _libssh2_explicit_zero(bn->bignum + newlen, bn->length - newlen);
-
-        if(!newlen) {
-            free((char *) bn->bignum);
-            bn->bignum = NULL;
-            bn->length = 0;
-            return 0;
-        }
-        bignum = (unsigned char *) realloc((char *) bn->bignum, newlen);
-    }
-
-    if(!bignum)
-        return -1;
-
-    if(newlen > bn->length)
-        memset((char *) bignum + bn->length, 0, newlen - bn->length);
-
-    bn->bignum = bignum;
-    bn->length = newlen;
-    return 0;
-}
-
-unsigned long
-_libssh2_bn_bits(_libssh2_bn *bn)
-{
-    unsigned int i;
-    unsigned char b;
-
-    if(bn && bn->bignum) {
-        for(i = bn->length; i--;)
-            b = bn->bignum[i];
-            if(b) {
-                i *= 8;
-                do {
-                    i++;
-                } while(b >>= 1);
-                return i;
-            }
-    }
-
-    return 0;
-}
-
-int
-_libssh2_bn_from_bin(_libssh2_bn *bn, int len, const unsigned char *val)
-{
-    int i;
-
-    if(!bn || (len && !val))
-        return -1;
-
-    for(; len && !*val; len--)
-        val++;
-
-    if(_libssh2_bn_resize(bn, len))
-        return -1;
-
-    for(i = len; i--;)
-        bn->bignum[i] = *val++;
-
-    return 0;
-}
-
-int
-_libssh2_bn_set_word(_libssh2_bn *bn, unsigned long val)
-{
-    val = htonl(val);
-    return _libssh2_bn_from_bin(bn, sizeof val, (unsigned char *) &val);
-}
-
-int
-_libssh2_bn_to_bin(_libssh2_bn *bn, unsigned char *val)
-{
-    int i;
-
-    if(!bn || !val)
-        return -1;
-
-    for(i = bn->length; i--;)
-        *val++ = bn->bignum[i];
-
-    return 0;
-}
-
-static int
-_libssh2_bn_from_bn(_libssh2_bn *to, _libssh2_bn *from)
-{
-    int i;
-
-    if(!to || !from)
-        return -1;
-
-    if(_libssh2_bn_resize(to, from->length))
-        return -1;
-
-    for(i = to->length; i--;)
-        to->bignum[i] = from->bignum[i];
-
-    return 0;
-}
-
-int
-_libssh2_random(unsigned char *buf, size_t len)
-{
-    Qc3GenPRNs(buf, len,
-        Qc3PRN_TYPE_NORMAL, Qc3PRN_NO_PARITY, (char *) &ecnull);
-    /* FIXME: any error is silently discarded! But Qc3GenPRNs could fail,
-       including if "The system seed digest is not ready" dixit IBM doc. */
-    return 0;
-}
-
 
 /*******************************************************************
  *
@@ -1145,8 +1147,6 @@ _libssh2_rsa_new(libssh2_rsa_ctx **rsa,
     _libssh2_bn *coeff = NULL;
     asn1Element *key = NULL;
     asn1Element *structkey = NULL;
-    Qc3_Format_ALGD0400_T algd;
-    Qus_EC_t errcode;
     int keytype;
     int ret = 0;
     int i;
@@ -1192,23 +1192,11 @@ _libssh2_rsa_new(libssh2_rsa_ctx **rsa,
     if(!key || !structkey)
         ret = -1;
 
-    set_EC_length(errcode, sizeof errcode);
-
-    if(!ret) {
-        /* Create the algorithm context. */
-        algd.Public_Key_Alg = Qc3_RSA;
-        algd.PKA_Block_Format = Qc3_PKCS1_01;
-        memset(algd.Reserved, 0, sizeof algd.Reserved);
-        algd.Signing_Hash_Alg = Qc3_SHA1;
-        Qc3CreateAlgorithmContext((char *) &algd, Qc3_Alg_Public_Key,
-                                  ctx->hash.Alg_Context_Token, &errcode);
-        if(errcode.Bytes_Available)
-            ret = -1;
-        ctx->hash.Final_Op_Flag = Qc3_Continue;
-    }
-
     /* Create the key context. */
     if(!ret) {
+        Qus_EC_t errcode;
+
+        set_EC_length(errcode, sizeof errcode);
         i = structkey->end - structkey->header;
         Qc3CreateKeyContext(structkey->header, &i, berstring, &keytype,
                             qc3clear, NULL, NULL, ctx->key.Key_Context_Token,
@@ -2105,26 +2093,12 @@ _libssh2_rsa_new_private(libssh2_rsa_ctx **rsa, LIBSSH2_SESSION *session,
 {
     libssh2_rsa_ctx *ctx = libssh2_init_crypto_ctx(NULL);
     int ret;
-    Qc3_Format_ALGD0400_T algd;
-    Qus_EC_t errcode;
 
     if(!ctx)
         return -1;
     ret = load_rsa_private_file(session, filename, passphrase,
                                 rsapkcs1privkey, rsapkcs8privkey,
                                 (void *) ctx);
-    if(!ret) {
-        /* Create the algorithm context. */
-        algd.Public_Key_Alg = Qc3_RSA;
-        algd.PKA_Block_Format = Qc3_PKCS1_01;
-        memset(algd.Reserved, 0, sizeof algd.Reserved);
-        algd.Signing_Hash_Alg = Qc3_SHA1;
-        set_EC_length(errcode, sizeof errcode);
-        Qc3CreateAlgorithmContext((char *) &algd, Qc3_Alg_Public_Key,
-                                  ctx->hash.Alg_Context_Token, &errcode);
-        if(errcode.Bytes_Available)
-            ret = -1;
-    }
     if(ret) {
         _libssh2_os400qc3_crypto_dtor(ctx);
         ctx = NULL;
@@ -2185,8 +2159,6 @@ _libssh2_rsa_new_private_frommemory(libssh2_rsa_ctx **rsa,
     unsigned char *data = NULL;
     unsigned int datalen = 0;
     int ret;
-    Qc3_Format_ALGD0400_T algd;
-    Qus_EC_t errcode;
 
     if(!ctx)
         return -1;
@@ -2238,19 +2210,6 @@ _libssh2_rsa_new_private_frommemory(libssh2_rsa_ctx **rsa,
     if(data)
         LIBSSH2_FREE(session, data);
 
-    if(!ret) {
-        /* Create the algorithm context. */
-        algd.Public_Key_Alg = Qc3_RSA;
-        algd.PKA_Block_Format = Qc3_PKCS1_01;
-        memset(algd.Reserved, 0, sizeof algd.Reserved);
-        algd.Signing_Hash_Alg = Qc3_SHA1;
-        set_EC_length(errcode, sizeof errcode);
-        Qc3CreateAlgorithmContext((char *) &algd, Qc3_Alg_Public_Key,
-                                  ctx->hash.Alg_Context_Token, &errcode);
-        if(errcode.Bytes_Available)
-            ret = -1;
-    }
-
     if(ret) {
         _libssh2_os400qc3_crypto_dtor(ctx);
         ctx = NULL;
@@ -2373,24 +2332,52 @@ _libssh2_sk_pub_keyfilememory(LIBSSH2_SESSION *session,
 }
 
 int
-_libssh2_rsa_sha1_verify(libssh2_rsa_ctx *rsa,
+_libssh2_rsa_sha2_verify(libssh2_rsa_ctx *rsa, size_t hash_len,
                          const unsigned char *sig, size_t sig_len,
                          const unsigned char *m, size_t m_len)
 {
     Qus_EC_t errcode;
+    Qc3_Format_ALGD0400_T algd;
     int slen = (int)sig_len;
     int mlen = (int)m_len;
 
+    memset(&algd, 0, sizeof algd);
+    algd.Public_Key_Alg = Qc3_RSA;
+    algd.PKA_Block_Format = Qc3_PKCS1_01;
+    switch(hash_len) {
+    case SHA_DIGEST_LENGTH:
+        algd.Signing_Hash_Alg = Qc3_SHA1;
+        break;
+    case SHA256_DIGEST_LENGTH:
+        algd.Signing_Hash_Alg = Qc3_SHA256;
+        break;
+    case SHA512_DIGEST_LENGTH:
+        algd.Signing_Hash_Alg = Qc3_SHA512;
+        break;
+    default:
+        return -1;
+    }
+
     set_EC_length(errcode, sizeof errcode);
     Qc3VerifySignature((char *) sig, &slen, (char *) m, &mlen, Qc3_Data,
-                       rsa->hash.Alg_Context_Token, Qc3_Alg_Token,
+                       (char *) &algd, Qc3_Alg_Public_Key,
-                       rsa->key.Key_Context_Token, Qc3_Key_Token, anycsp,
+                       (char *) &rsa->key, Qc3_Key_Token, anycsp,
                        NULL, (char *) &errcode);
     return errcode.Bytes_Available? -1: 0;
 }
 
 int
-_libssh2_os400qc3_rsa_sha1_signv(LIBSSH2_SESSION *session,
+_libssh2_rsa_sha1_verify(libssh2_rsa_ctx *rsa,
+                         const unsigned char *sig, size_t sig_len,
+                         const unsigned char *m, size_t m_len)
+{
+    return _libssh2_rsa_sha2_verify(rsa, SHA_DIGEST_LENGTH,
+                                    sig, sig_len, m, m_len);
+}
+
+int
+_libssh2_os400qc3_rsa_signv(LIBSSH2_SESSION *session,
+                            int algo,
                             unsigned char **signature,
                             size_t *signature_len,
                             int veccount,
@@ -2398,19 +2385,22 @@ _libssh2_os400qc3_rsa_sha1_signv(LIBSSH2_SESSION *session,
                             libssh2_rsa_ctx *ctx)
 {
     Qus_EC_t errcode;
+    Qc3_Format_ALGD0400_T algd;
     int siglen;
     unsigned char *sig;
     char sigbuf[8192];
     int sigbufsize = sizeof sigbuf;
 
-    ctx->hash.Final_Op_Flag = Qc3_Final;
+    algd.Public_Key_Alg = Qc3_RSA;
+    algd.PKA_Block_Format = Qc3_PKCS1_01;
+    memset(algd.Reserved, 0, sizeof algd.Reserved);
+    algd.Signing_Hash_Alg = algo;
     set_EC_length(errcode, sizeof errcode);
     Qc3CalculateSignature((char *) vector, &veccount, Qc3_Array,
-                          (char *) &ctx->hash, Qc3_Alg_Token,
+                          (char *) &algd, Qc3_Alg_Public_Key,
                           (char *) &ctx->key, Qc3_Key_Token,
                           anycsp, NULL, sigbuf, &sigbufsize, &siglen,
                           (char *) &errcode);
-    ctx->hash.Final_Op_Flag = Qc3_Continue;
     if(errcode.Bytes_Available)
         return -1;
     sig = LIBSSH2_ALLOC(session, siglen);
@@ -2434,8 +2424,11 @@ _libssh2_supported_key_sign_algorithms(LIBSSH2_SESSION *session,
                                        size_t key_method_len)
 {
     (void)session;
-    (void)key_method;
+
-    (void)key_method_len;
+    if(key_method_len == 7 &&
+       memcmp(key_method, "ssh-rsa", key_method_len) == 0) {
+        return "rsa-sha2-512,rsa-sha2-256,ssh-rsa";
+    }
 
     return NULL;
 }

src/os400qc3.h

@@ -2,7 +2,7 @@
 #define __LIBSSH2_OS400QC3_H
 /*
  * Copyright (C) 2015-2016 Patrick Monnerat, D+H <patrick.monnerat@dh.com>
- * Copyright (C) 2020 Patrick Monnerat <patrick@monnerat.net>.
+ * Copyright (C) 2020-2023 Patrick Monnerat <patrick@monnerat.net>.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms,
@@ -177,7 +177,7 @@
 #define LIBSSH2_3DES            1
 
 #define LIBSSH2_RSA             1
-#define LIBSSH2_RSA_SHA2        0
+#define LIBSSH2_RSA_SHA2        1
 #define LIBSSH2_DSA             0
 #define LIBSSH2_ECDSA           0
 #define LIBSSH2_ED25519         0
@@ -316,19 +316,19 @@ typedef struct {        /* Diffie-Hellman context. */
 #define _libssh2_cipher_type(name)  _libssh2_os400qc3_cipher_t name
 #define _libssh2_cipher_aes128 {Qc3_Alg_Block_Cipher, Qc3_AES, 16,          \
                                 Qc3_CBC, 16}
-#define _libssh2_cipher_aes192 {Qc3_Alg_Block_Cipher, Qc3_AES, 24,          \
+#define _libssh2_cipher_aes192 {Qc3_Alg_Block_Cipher, Qc3_AES, 16,          \
                                 Qc3_CBC, 24}
-#define _libssh2_cipher_aes256 {Qc3_Alg_Block_Cipher, Qc3_AES, 32,          \
+#define _libssh2_cipher_aes256 {Qc3_Alg_Block_Cipher, Qc3_AES, 16,          \
                                 Qc3_CBC, 32}
 #define _libssh2_cipher_aes128ctr {Qc3_Alg_Block_Cipher, Qc3_AES, 16,       \
                                    Qc3_CTR, 16}
-#define _libssh2_cipher_aes192ctr {Qc3_Alg_Block_Cipher, Qc3_AES, 24,       \
+#define _libssh2_cipher_aes192ctr {Qc3_Alg_Block_Cipher, Qc3_AES, 16,       \
                                    Qc3_CTR, 24}
-#define _libssh2_cipher_aes256ctr {Qc3_Alg_Block_Cipher, Qc3_AES, 32,       \
+#define _libssh2_cipher_aes256ctr {Qc3_Alg_Block_Cipher, Qc3_AES, 16,       \
                                    Qc3_CTR, 32}
-#define _libssh2_cipher_3des {Qc3_Alg_Block_Cipher, Qc3_TDES, 0,            \
+#define _libssh2_cipher_3des {Qc3_Alg_Block_Cipher, Qc3_TDES, 8,            \
                               Qc3_CBC, 24}
-#define _libssh2_cipher_arcfour {Qc3_Alg_Stream_Cipher, Qc3_RC4, 0, 0, 16}
+#define _libssh2_cipher_arcfour {Qc3_Alg_Stream_Cipher, Qc3_RC4, 8, 0, 16}
 
 #define _libssh2_cipher_dtor(ctx) _libssh2_os400qc3_crypto_dtor(ctx)
 
@@ -338,8 +338,22 @@ typedef struct {        /* Diffie-Hellman context. */
 #define libssh2_prepare_iovec(vec, len) memset((char *) (vec), 0,           \
                                                (len) * sizeof(struct iovec))
 #define _libssh2_rsa_sha1_signv(session, sig, siglen, count, vector, ctx)   \
-            _libssh2_os400qc3_rsa_sha1_signv(session, sig, siglen,          \
+            _libssh2_os400qc3_rsa_signv(session, Qc3_SHA1, sig, siglen,     \
                                              count, vector, ctx)
+#define _libssh2_rsa_sha2_256_signv(session, sig, siglen, cnt, vector, ctx) \
+            _libssh2_os400qc3_rsa_signv(session, Qc3_SHA256, sig, siglen,   \
+                                             cnt, vector, ctx)
+#define _libssh2_rsa_sha2_512_signv(session, sig, siglen, cnt, vector, ctx) \
+            _libssh2_os400qc3_rsa_signv(session, Qc3_SHA512, sig, siglen,   \
+                                             cnt, vector, ctx)
+
+/* Default generate and safe prime sizes for diffie-hellman-group-exchange-sha1
+   Qc3 is limited to a maximum 2048-bit modulus/key size. */
+#define LIBSSH2_DH_GEX_MINGROUP     1024
+#define LIBSSH2_DH_GEX_OPTGROUP     1536
+#define LIBSSH2_DH_GEX_MAXGROUP     2048
+
+#define LIBSSH2_DH_MAX_MODULUS_BITS 2048
 
 #define _libssh2_dh_ctx         _libssh2_os400qc3_dh_ctx
 #define libssh2_dh_init(dhctx)  _libssh2_os400qc3_dh_init(dhctx)
@@ -383,7 +397,7 @@ extern void     libssh2_os400qc3_hmac_update(_libssh2_os400qc3_crypto_ctx *ctx,
                                              int len);
 extern void     libssh2_os400qc3_hmac_final(_libssh2_os400qc3_crypto_ctx *ctx,
                                             unsigned char *out);
-extern int      _libssh2_os400qc3_rsa_sha1_signv(LIBSSH2_SESSION *session,
+extern int      _libssh2_os400qc3_rsa_signv(LIBSSH2_SESSION *session, int algo,
                                             unsigned char **signature,
                                             size_t *signature_len,
                                             int veccount,

src/userauth_kbd_packet.c

@@ -44,6 +44,7 @@ int userauth_keyboard_interactive_decode_info_request(LIBSSH2_SESSION *session)
     size_t language_tag_len;
     unsigned int i;
     unsigned char packet_type;
+    uint32_t tmp_u32;
 
     struct string_buf decoded;
 
@@ -95,7 +96,8 @@ int userauth_keyboard_interactive_decode_info_request(LIBSSH2_SESSION *session)
     }
 
     /* int       num-prompts */
-    if(_libssh2_get_u32(&decoded, &session->userauth_kybd_num_prompts) == -1) {
+    if(_libssh2_get_u32(&decoded, &tmp_u32) == -1 ||
+       (session->userauth_kybd_num_prompts = tmp_u32) != tmp_u32) {
         _libssh2_error(session, LIBSSH2_ERROR_BUFFER_TOO_SMALL,
                        "Unable to decode "
                        "keyboard-interactive number of keyboard prompts");

src/wincng.h

@@ -423,6 +423,14 @@ _libssh2_bn *_libssh2_wincng_bignum_init(void);
  * Windows CNG backend: Diffie-Hellman support
  */
 
+/* Default generate and safe prime sizes for
+   diffie-hellman-group-exchange-sha1 */
+#define LIBSSH2_DH_GEX_MINGROUP     2048
+#define LIBSSH2_DH_GEX_OPTGROUP     4096
+#define LIBSSH2_DH_GEX_MAXGROUP     8192
+
+#define LIBSSH2_DH_MAX_MODULUS_BITS 16384
+
 typedef struct {
     /* holds our private and public key components */
     BCRYPT_KEY_HANDLE dh_handle;