pki: Fail to sign when using wrong hash algorithm

Do not allow using SSH_DIGEST_AUTO for any algorithm other than
ed25519.

Do not allow using incompatible hash algorithms when signing or
verifying signatures.

Added negative tests for all combinations of signature and hash
algorithms.

Signed-off-by: Anderson Toshiyuki Sasaki <ansasaki@redhat.com>
Reviewed-by: Jakub Jelen <jjelen@redhat.com>
Reviewed-by: Andreas Schneider <asn@cryptomilk.org>

include/libssh/pki_priv.h

@@ -61,6 +61,8 @@ int pki_key_compare(const ssh_key k1,
                     const ssh_key k2,
                     enum ssh_keycmp_e what);
 
+int pki_key_check_hash_compatible(ssh_key key,
+                                  enum ssh_digest_e hash_type);
 /* SSH Private Key Functions */
 enum ssh_keytypes_e pki_privatekey_type_from_string(const char *privkey);
 ssh_key pki_private_key_from_base64(const char *b64_key,

src/pki.c

@@ -344,8 +344,10 @@ enum ssh_digest_e ssh_key_type_to_hash(ssh_session session,
                                        enum ssh_keytypes_e type)
 {
     switch (type) {
+    case SSH_KEYTYPE_DSS_CERT01:
     case SSH_KEYTYPE_DSS:
         return SSH_DIGEST_SHA1;
+    case SSH_KEYTYPE_RSA_CERT01:
     case SSH_KEYTYPE_RSA:
         if (ssh_key_algorithm_allowed(session, "rsa-sha2-512") &&
             (session->extensions & SSH_EXT_SIG_RSA_SHA512)) {
@@ -2070,6 +2072,79 @@ int ssh_pki_import_signature_blob(const ssh_string sig_blob,
     return SSH_OK;
 }
 
+/**
+ * @internal
+ *
+ * @brief Check if the provided key can be used with the provided hash type for
+ * data signing or signature verification.
+ *
+ * @param[in]   key         The key to be checked.
+ * @param[in]   hash_type   The digest algorithm to be checked.
+ *
+ * @return  SSH_OK if compatible; SSH_ERROR otherwise
+ */
+int pki_key_check_hash_compatible(ssh_key key,
+                                  enum ssh_digest_e hash_type)
+{
+    if (key == NULL) {
+        SSH_LOG(SSH_LOG_TRACE, "Null pointer provided as key to "
+                               "pki_key_check_hash_compatible()");
+        return SSH_ERROR;
+    }
+
+    switch(key->type) {
+    case SSH_KEYTYPE_DSS_CERT01:
+    case SSH_KEYTYPE_DSS:
+        if (hash_type == SSH_DIGEST_SHA1) {
+            return SSH_OK;
+        }
+        break;
+    case SSH_KEYTYPE_RSA_CERT01:
+    case SSH_KEYTYPE_RSA:
+        if (hash_type == SSH_DIGEST_SHA1 ||
+            hash_type == SSH_DIGEST_SHA256 ||
+            hash_type == SSH_DIGEST_SHA512)
+        {
+            return SSH_OK;
+        }
+        break;
+    case SSH_KEYTYPE_ECDSA_P256_CERT01:
+    case SSH_KEYTYPE_ECDSA_P256:
+        if (hash_type == SSH_DIGEST_SHA256) {
+            return SSH_OK;
+        }
+        break;
+    case SSH_KEYTYPE_ECDSA_P384_CERT01:
+    case SSH_KEYTYPE_ECDSA_P384:
+        if (hash_type == SSH_DIGEST_SHA384) {
+            return SSH_OK;
+        }
+        break;
+    case SSH_KEYTYPE_ECDSA_P521_CERT01:
+    case SSH_KEYTYPE_ECDSA_P521:
+        if (hash_type == SSH_DIGEST_SHA512) {
+            return SSH_OK;
+        }
+        break;
+    case SSH_KEYTYPE_ED25519_CERT01:
+    case SSH_KEYTYPE_ED25519:
+        if (hash_type == SSH_DIGEST_AUTO) {
+            return SSH_OK;
+        }
+        break;
+    case SSH_KEYTYPE_RSA1:
+    case SSH_KEYTYPE_ECDSA:
+    case SSH_KEYTYPE_UNKNOWN:
+        SSH_LOG(SSH_LOG_WARN, "Unknown key type %d", key->type);
+        return SSH_ERROR;
+    }
+
+    SSH_LOG(SSH_LOG_WARN, "Key type %d incompatible with hash type  %d",
+            key->type, hash_type);
+
+    return SSH_ERROR;
+}
+
 int ssh_pki_signature_verify(ssh_session session,
                              ssh_signature sig,
                              const ssh_key key,
@@ -2096,6 +2171,12 @@ int ssh_pki_signature_verify(ssh_session session,
         return SSH_ERROR;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(key, sig->hash_type);
+    if (rc != SSH_OK) {
+        return SSH_ERROR;
+    }
+
     rc = pki_signature_verify(session, sig, key, input, input_len);
 
     return rc;
@@ -2106,12 +2187,20 @@ ssh_signature pki_do_sign(const ssh_key privkey,
                           size_t input_len,
                           enum ssh_digest_e hash_type)
 {
+    int rc;
+
     if (privkey == NULL || input == NULL) {
         SSH_LOG(SSH_LOG_TRACE, "Bad parameter provided to "
                                "pki_do_sign()");
         return NULL;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(privkey, hash_type);
+    if (rc != SSH_OK) {
+        return NULL;
+    }
+
     if (privkey->type == SSH_KEYTYPE_ED25519 ||
         privkey->type == SSH_KEYTYPE_ED25519_CERT01)
     {

src/pki_crypto.c

@@ -583,9 +583,13 @@ int pki_key_generate_ecdsa(ssh_key key, int parameter) {
             key->type = SSH_KEYTYPE_ECDSA_P521;
             break;
         case 256:
-        default:
             key->ecdsa_nid = NID_X9_62_prime256v1;
             key->type = SSH_KEYTYPE_ECDSA_P256;
+            break;
+        default:
+            SSH_LOG(SSH_LOG_WARN, "Invalid parameter %d for ECDSA key "
+                    "generation", parameter);
+            return SSH_ERROR;
     }
 
     key->ecdsa = EC_KEY_new_by_curve_name(key->ecdsa_nid);
@@ -1922,6 +1926,12 @@ int pki_signature_verify(ssh_session session,
         return SSH_ERROR;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(key, sig->hash_type);
+    if (rc != SSH_OK) {
+        return SSH_ERROR;
+    }
+
     /* For ed25519 keys, verify using the input directly */
     if (key->type == SSH_KEYTYPE_ED25519 ||
         key->type == SSH_KEYTYPE_ED25519_CERT01)
@@ -1957,9 +1967,9 @@ static const EVP_MD *pki_digest_to_md(enum ssh_digest_e hash_type)
         md = EVP_sha512();
         break;
     case SSH_DIGEST_SHA1:
-    case SSH_DIGEST_AUTO:
         md = EVP_sha1();
         break;
+    case SSH_DIGEST_AUTO:
     default:
         SSH_LOG(SSH_LOG_TRACE, "Unknown hash algorithm for type: %d",
                 hash_type);
@@ -2063,6 +2073,12 @@ ssh_signature pki_sign_data(const ssh_key privkey,
         return NULL;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(privkey, hash_type);
+    if (rc != SSH_OK) {
+        return NULL;
+    }
+
     /* Set hash algorithm to be used */
     md = pki_digest_to_md(hash_type);
     if (md == NULL) {
@@ -2187,6 +2203,12 @@ int pki_verify_data_signature(ssh_signature signature,
         return SSH_ERROR;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(pubkey, signature->hash_type);
+    if (rc != SSH_OK) {
+        return SSH_ERROR;
+    }
+
     /* Get the signature to be verified */
     raw_sig_data = ssh_string_data(signature->raw_sig);
     raw_sig_len = ssh_string_len(signature->raw_sig);

src/pki_gcrypt.c

@@ -2107,6 +2107,12 @@ int pki_signature_verify(ssh_session session,
         return SSH_ERROR;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(key, sig->hash_type);
+    if (rc != SSH_OK) {
+        return SSH_ERROR;
+    }
+
     /* For ed25519 keys, verify using the input directly */
     if (key->type == SSH_KEYTYPE_ED25519 ||
         key->type == SSH_KEYTYPE_ED25519_CERT01)
@@ -2171,7 +2177,6 @@ ssh_signature pki_do_sign_hash(const ssh_key privkey,
         case SSH_KEYTYPE_RSA:
             switch (hash_type) {
             case SSH_DIGEST_SHA1:
-            case SSH_DIGEST_AUTO:
                 hash_c = "sha1";
                 break;
             case SSH_DIGEST_SHA256:
@@ -2180,8 +2185,9 @@ ssh_signature pki_do_sign_hash(const ssh_key privkey,
             case SSH_DIGEST_SHA512:
                 hash_c = "sha512";
                 break;
+            case SSH_DIGEST_AUTO:
             default:
-                SSH_LOG(SSH_LOG_WARN, "Incomplatible key algorithm");
+                SSH_LOG(SSH_LOG_WARN, "Incompatible key algorithm");
                 return NULL;
             }
             err = gcry_sexp_build(&sexp,
@@ -2261,6 +2267,7 @@ ssh_signature pki_sign_data(const ssh_key privkey,
 {
     unsigned char hash[SHA512_DIGEST_LEN] = {0};
     uint32_t hlen = 0;
+    int rc;
 
     if (privkey == NULL || !ssh_key_is_private(privkey) || input == NULL) {
         SSH_LOG(SSH_LOG_TRACE, "Bad parameter provided to "
@@ -2268,6 +2275,12 @@ ssh_signature pki_sign_data(const ssh_key privkey,
         return NULL;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(privkey, hash_type);
+    if (rc != SSH_OK) {
+        return NULL;
+    }
+
     switch (hash_type) {
     case SSH_DIGEST_SHA256:
         sha256(input, input_len, hash);
@@ -2281,11 +2294,11 @@ ssh_signature pki_sign_data(const ssh_key privkey,
         sha512(input, input_len, hash);
         hlen = SHA512_DIGEST_LEN;
         break;
-    case SSH_DIGEST_AUTO:
     case SSH_DIGEST_SHA1:
         sha1(input, input_len, hash);
         hlen = SHA_DIGEST_LEN;
         break;
+    case SSH_DIGEST_AUTO:
     default:
         SSH_LOG(SSH_LOG_TRACE, "Unknown hash algorithm for type: %d",
                 hash_type);
@@ -2321,6 +2334,8 @@ int pki_verify_data_signature(ssh_signature signature,
     unsigned char *hash = ghash + 1;
     uint32_t hlen = 0;
 
+    int rc;
+
     if (pubkey == NULL || ssh_key_is_private(pubkey) || input == NULL ||
         signature == NULL)
     {
@@ -2329,6 +2344,12 @@ int pki_verify_data_signature(ssh_signature signature,
         return SSH_ERROR;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(pubkey, signature->hash_type);
+    if (rc != SSH_OK) {
+        return SSH_ERROR;
+    }
+
     switch (signature->hash_type) {
     case SSH_DIGEST_SHA256:
         sha256(input, input_len, hash);
@@ -2345,12 +2366,12 @@ int pki_verify_data_signature(ssh_signature signature,
         hlen = SHA512_DIGEST_LEN;
         hash_type = "sha512";
         break;
-    case SSH_DIGEST_AUTO:
     case SSH_DIGEST_SHA1:
         sha1(input, input_len, hash);
         hlen = SHA_DIGEST_LEN;
         hash_type = "sha1";
         break;
+    case SSH_DIGEST_AUTO:
     default:
         SSH_LOG(SSH_LOG_TRACE, "Unknown sig->hash_type: %d", signature->hash_type);
         return SSH_ERROR;

src/pki_mbedcrypto.c

@@ -1040,6 +1040,12 @@ int pki_signature_verify(ssh_session session, const ssh_signature sig, const
         return SSH_ERROR;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(key, sig->hash_type);
+    if (rc != SSH_OK) {
+        return SSH_ERROR;
+    }
+
     /* For ed25519 keys, verify using the input directly */
     if (key->type == SSH_KEYTYPE_ED25519 ||
         key->type == SSH_KEYTYPE_ED25519_CERT01)
@@ -1073,7 +1079,6 @@ static ssh_string rsa_do_sign_hash(const unsigned char *digest,
 
     switch (hash_type) {
     case SSH_DIGEST_SHA1:
-    case SSH_DIGEST_AUTO:
         md = MBEDTLS_MD_SHA1;
         break;
     case SSH_DIGEST_SHA256:
@@ -1082,8 +1087,9 @@ static ssh_string rsa_do_sign_hash(const unsigned char *digest,
     case SSH_DIGEST_SHA512:
         md = MBEDTLS_MD_SHA512;
         break;
+    case SSH_DIGEST_AUTO:
     default:
-        SSH_LOG(SSH_LOG_WARN, "Incomplatible key algorithm");
+        SSH_LOG(SSH_LOG_WARN, "Incompatible key algorithm");
         return NULL;
     }
 
@@ -1208,6 +1214,7 @@ ssh_signature pki_sign_data(const ssh_key privkey,
 {
     unsigned char hash[SHA512_DIGEST_LEN] = {0};
     uint32_t hlen = 0;
+    int rc;
 
     if (privkey == NULL || !ssh_key_is_private(privkey) || input == NULL) {
         SSH_LOG(SSH_LOG_TRACE, "Bad parameter provided to "
@@ -1215,6 +1222,12 @@ ssh_signature pki_sign_data(const ssh_key privkey,
         return NULL;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(privkey, hash_type);
+    if (rc != SSH_OK) {
+        return NULL;
+    }
+
     switch (hash_type) {
     case SSH_DIGEST_SHA256:
         sha256(input, input_len, hash);
@@ -1228,11 +1241,11 @@ ssh_signature pki_sign_data(const ssh_key privkey,
         sha512(input, input_len, hash);
         hlen = SHA512_DIGEST_LEN;
         break;
-    case SSH_DIGEST_AUTO:
     case SSH_DIGEST_SHA1:
         sha1(input, input_len, hash);
         hlen = SHA_DIGEST_LEN;
         break;
+    case SSH_DIGEST_AUTO:
     default:
         SSH_LOG(SSH_LOG_TRACE, "Unknown hash algorithm for type: %d",
                 hash_type);
@@ -1276,6 +1289,12 @@ int pki_verify_data_signature(ssh_signature signature,
         return SSH_ERROR;
     }
 
+    /* Check if public key and hash type are compatible */
+    rc = pki_key_check_hash_compatible(pubkey, signature->hash_type);
+    if (rc != SSH_OK) {
+        return SSH_ERROR;
+    }
+
     switch (signature->hash_type) {
     case SSH_DIGEST_SHA256:
         sha256(input, input_len, hash);
@@ -1292,12 +1311,12 @@ int pki_verify_data_signature(ssh_signature signature,
         hlen = SHA512_DIGEST_LEN;
         md = MBEDTLS_MD_SHA512;
         break;
-    case SSH_DIGEST_AUTO:
     case SSH_DIGEST_SHA1:
         sha1(input, input_len, hash);
         hlen = SHA_DIGEST_LEN;
         md = MBEDTLS_MD_SHA1;
         break;
+    case SSH_DIGEST_AUTO:
     default:
         SSH_LOG(SSH_LOG_TRACE, "Unknown sig->hash_type: %d",
                 signature->hash_type);

tests/unittests/torture_pki.c

@@ -10,8 +10,8 @@
 #include "torture_key.h"
 #include "pki.c"
 
-const unsigned char HASH[] = "1234567890123456789012345678901234567890"
+const unsigned char INPUT[] = "1234567890123456789012345678901234567890"
-                             "123456789012345678901234";
+                              "123456789012345678901234";
 
 const char template[] = "temp_dir_XXXXXX";
 
@@ -114,50 +114,137 @@ struct key_attrs {
     int size_arg;
     int sig_length;
     const char *sig_type_c;
+    int expect_success;
 };
 
-struct key_attrs key_attrs_list[] = {
+struct key_attrs key_attrs_list[][5] = {
-    {0, 0, "", 0, 0, ""},                                        /* UNKNOWN */
+    {
+        {0, 0, "", 0, 0, "", 0}, /* UNKNOWN, AUTO */
+        {0, 0, "", 0, 0, "", 0}, /* UNKNOWN, SHA1 */
+        {0, 0, "", 0, 0, "", 0}, /* UNKNOWN, SHA256 */
+        {0, 0, "", 0, 0, "", 0}, /* UNKNOWN, SHA384 */
+        {0, 0, "", 0, 0, "", 0}, /* UNKNOWN, SHA512 */
+    },
 #ifdef HAVE_DSA
-    {1, 1, "ssh-dss", 1024, 20, "ssh-dss" },                     /* DSS */
+    {
+        {1, 1, "ssh-dss", 1024, 0, "", 0},         /* DSS, AUTO */
+        {1, 1, "ssh-dss", 1024, 20, "ssh-dss", 1}, /* DSS, SHA1 */
+        {1, 1, "ssh-dss", 1024, 0, "", 0},         /* DSS, SHA256 */
+        {1, 1, "ssh-dss", 1024, 0, "", 0},         /* DSS, SHA384 */
+        {1, 1, "ssh-dss", 1024, 0, "", 0},         /* DSS, SHA512 */
+    },
 #else
-    {0, 0, "", 0, 0, ""},                                        /* DSS */
+    {
-#endif
+        {0, 0, "", 0, 0, "", 0}, /* DSS, AUTO */
-    {1, 1, "ssh-rsa", 2048, 20, "ssh-rsa"},                      /* RSA */
+        {0, 0, "", 0, 0, "", 0}, /* DSS, SHA1 */
-    {0, 0, "", 0, 0, ""},                                        /* RSA1 */
+        {0, 0, "", 0, 0, "", 0}, /* DSS, SHA256 */
-    {0, 0, "", 0, 0, ""},                                        /* ECDSA */
+        {0, 0, "", 0, 0, "", 0}, /* DSS, SHA384 */
-    {1, 1, "ssh-ed25519", 0, 33, "ssh-ed25519"},                 /* ED25519 */
+        {0, 0, "", 0, 0, "", 0}, /* DSS, SHA512 */
+    },
+#endif /* HAVE_DSA */
+    {
+        {1, 1, "ssh-rsa", 2048, 0, "", 0},              /* RSA, AUTO */
+        {1, 1, "ssh-rsa", 2048, 20, "ssh-rsa", 1},      /* RSA, SHA1 */
+        {1, 1, "ssh-rsa", 2048, 32, "rsa-sha2-256", 1}, /* RSA, SHA256 */
+        {1, 1, "ssh-rsa", 2048, 0, "", 0},              /* RSA, SHA384 */
+        {1, 1, "ssh-rsa", 2048, 64, "rsa-sha2-512", 1}, /* RSA, SHA512 */
+    },
+    {
+        {0, 0, "", 0, 0, "", 0}, /* RSA1, AUTO */
+        {0, 0, "", 0, 0, "", 0}, /* RSA1, SHA1 */
+        {0, 0, "", 0, 0, "", 0}, /* RSA1, SHA256 */
+        {0, 0, "", 0, 0, "", 0}, /* RSA1, SHA384 */
+        {0, 0, "", 0, 0, "", 0}, /* RSA1, SHA512 */
+    },
+    {
+        {0, 1, "", 256, 0, "", 0}, /* ECDSA, AUTO */
+        {0, 1, "", 256, 0, "", 0}, /* ECDSA, SHA1 */
+        {0, 1, "", 256, 0, "", 0}, /* ECDSA, SHA256 */
+        {0, 1, "", 384, 0, "", 0}, /* ECDSA, SHA384 */
+        {0, 1, "", 521, 0, "", 0}, /* ECDSA, SHA512 */
+    },
+    {
+        {1, 1, "ssh-ed25519", 0, 33, "ssh-ed25519", 1}, /* ED25519, AUTO */
+        {1, 1, "ssh-ed25519", 0, 0, "", 0},             /* ED25519, SHA1 */
+        {1, 1, "ssh-ed25519", 0, 0, "", 0},             /* ED25519, SHA256 */
+        {1, 1, "ssh-ed25519", 0, 0, "", 0},             /* ED25519, SHA384 */
+        {1, 1, "ssh-ed25519", 0, 0, "", 0},             /* ED25519, SHA512 */
+    },
 #ifdef HAVE_DSA
-    {0, 1, "", 0, 0, ""},                                        /* DSS CERT */
+    {
+        {0, 1, "", 0, 0, "", 0}, /* DSS CERT, AUTO */
+        {0, 1, "", 0, 0, "", 0}, /* DSS CERT, SHA1 */
+        {0, 1, "", 0, 0, "", 0}, /* DSS CERT, SHA256 */
+        {0, 1, "", 0, 0, "", 0}, /* DSS CERT, SHA384 */
+        {0, 1, "", 0, 0, "", 0}, /* DSS CERT, SHA512 */
+    },
 #else
-    {0, 0, "", 0, 0, ""},                                        /* DSS CERT */
+    {
-#endif
+        {0, 0, "", 0, 0, "", 0}, /* DSS CERT, AUTO */
-    {0, 1, "", 0, 0, ""},                                        /* RSA CERT */
+        {0, 0, "", 0, 0, "", 0}, /* DSS CERT, SHA1 */
-    {1, 1, "ecdsa-sha2-nistp256", 0, 64, "ecdsa-sha2-nistp256"}, /* ECDSA P256 */
+        {0, 0, "", 0, 0, "", 0}, /* DSS CERT, SHA256 */
-    {1, 1, "ecdsa-sha2-nistp384", 0, 64, "ecdsa-sha2-nistp384"}, /* ECDSA P384 */
+        {0, 0, "", 0, 0, "", 0}, /* DSS CERT, SHA384 */
-    {1, 1, "ecdsa-sha2-nistp521", 0, 64, "ecdsa-sha2-nistp521"}, /* ECDSA P521 */
+        {0, 0, "", 0, 0, "", 0}, /* DSS CERT, SHA512 */
-    {0, 1, "", 0, 0, ""},                                        /* ECDSA P256 CERT */
+    },
-    {0, 1, "", 0, 0, ""},                                        /* ECDSA P384 CERT */
+#endif /* HAVE_DSA */
-    {0, 1, "", 0, 0, ""},                                        /* ECDSA P521 CERT */
+    {
-    {0, 1, "", 0, 0, ""},                                        /* ED25519 CERT */
+        {0, 1, "", 0, 0, "", 0}, /* RSA CERT, AUTO */
-};
+        {0, 1, "", 0, 0, "", 0}, /* RSA CERT, SHA1 */
-
+        {0, 1, "", 0, 0, "", 0}, /* RSA CERT, SHA256 */
-/* Maps to enum ssh_digest_e */
+        {0, 1, "", 0, 0, "", 0}, /* RSA CERT, SHA384 */
-const char *hash_signatures[] = {
+        {0, 1, "", 0, 0, "", 0}, /* RSA CERT, SHA512 */
-    "", /* Not used here */
+    },
-    "ssh-rsa",
+#ifdef HAVE_ECC
-    "rsa-sha2-256",
+    {
-    "", /* Not used; there is no rsa-sha2-384 */
+        {1, 1, "ecdsa-sha2-nistp256", 256, 0, "", 0},                     /* ECDSA P256, AUTO */
-    "rsa-sha2-512",
+        {1, 1, "ecdsa-sha2-nistp256", 256, 0, "", 0},                     /* ECDSA P256, SHA1 */
-};
+        {1, 1, "ecdsa-sha2-nistp256", 256, 32, "ecdsa-sha2-nistp256", 1}, /* ECDSA P256, SHA256 */
-
+        {1, 1, "ecdsa-sha2-nistp256", 256, 0, "", 0},                     /* ECDSA P256, SHA384 */
-/* Maps to enum ssh_digest_e */
+        {1, 1, "ecdsa-sha2-nistp256", 256, 0, "", 0},                     /* ECDSA P256, SHA512 */
-int hash_lengths[] = {
+    },
-    0, /* Not used here */
+    {
-    20,
+        {1, 1, "ecdsa-sha2-nistp384", 384, 0, "", 0},                     /* ECDSA P384, AUTO */
-    32,
+        {1, 1, "ecdsa-sha2-nistp384", 384, 0, "", 0},                     /* ECDSA P384, SHA1 */
-    48, /* Not used; there is no rsa-sha2-384 */
+        {1, 1, "ecdsa-sha2-nistp384", 384, 0, "", 0},                     /* ECDSA P384, SHA256 */
-    64,
+        {1, 1, "ecdsa-sha2-nistp384", 384, 48, "ecdsa-sha2-nistp384", 1}, /* ECDSA P384, SHA384 */
+        {1, 1, "ecdsa-sha2-nistp384", 384, 0, "", 0},                     /* ECDSA P384, SHA512 */
+    },
+    {
+        {1, 1, "ecdsa-sha2-nistp521", 521, 0, "", 0},                     /* ECDSA P521, AUTO */
+        {1, 1, "ecdsa-sha2-nistp521", 521, 0, "", 0},                     /* ECDSA P521, SHA1 */
+        {1, 1, "ecdsa-sha2-nistp521", 521, 0, "", 0},                     /* ECDSA P521, SHA256 */
+        {1, 1, "ecdsa-sha2-nistp521", 521, 0, "", 0},                     /* ECDSA P521, SHA384 */
+        {1, 1, "ecdsa-sha2-nistp521", 521, 64, "ecdsa-sha2-nistp521", 1}, /* ECDSA P521, SHA512 */
+    },
+    {
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P256 CERT, AUTO */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P256 CERT, SHA1 */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P256 CERT, SHA256 */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P256 CERT, SHA384 */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P256 CERT, SHA512 */
+    },
+    {
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P384 CERT, AUTO */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P384 CERT, SHA1 */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P384 CERT, SHA256 */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P384 CERT, SHA384 */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P384 CERT, SHA512 */
+    },
+    {
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P521 CERT, AUTO */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P521 CERT, SHA1 */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P521 CERT, SHA256 */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P521 CERT, SHA384 */
+        {0, 1, "", 0, 0, "", 0}, /* ECDSA P521 CERT, SHA512 */
+    },
+#endif /* HAVE_ECC */
+    {
+        {0, 1, "", 0, 0, "", 0}, /* ED25519 CERT, AUTO */
+        {0, 1, "", 0, 0, "", 0}, /* ED25519 CERT, SHA1 */
+        {0, 1, "", 0, 0, "", 0}, /* ED25519 CERT, SHA256 */
+        {0, 1, "", 0, 0, "", 0}, /* ED25519 CERT, SHA384 */
+        {0, 1, "", 0, 0, "", 0}, /* ED25519 CERT, SHA512 */
+    },
 };
 
 /* This tests all the base types and their signatures against each other */
@@ -171,7 +258,7 @@ static void torture_pki_verify_mismatch(void **state)
     ssh_session session = ssh_new();
     enum ssh_keytypes_e key_type, sig_type;
     enum ssh_digest_e hash;
-    int hash_length;
+    size_t input_length = sizeof(INPUT);
     struct key_attrs skey_attrs, vkey_attrs;
 
     (void) state;
@@ -179,45 +266,46 @@ static void torture_pki_verify_mismatch(void **state)
     ssh_options_set(session, SSH_OPTIONS_LOG_VERBOSITY, &verbosity);
 
     for (sig_type = SSH_KEYTYPE_DSS;
-         sig_type <= SSH_KEYTYPE_ED25519;
+         sig_type <= SSH_KEYTYPE_ED25519_CERT01;
-         sig_type++) {
+         sig_type++)
-        skey_attrs = key_attrs_list[sig_type];
+    {
-        if (!skey_attrs.sign) {
-            continue;
-        }
-        rc = ssh_pki_generate(sig_type, skey_attrs.size_arg, &key);
-        assert_true(rc == SSH_OK);
-        assert_non_null(key);
-        assert_int_equal(key->type, sig_type);
-        assert_string_equal(key->type_c, skey_attrs.type_c);
-        rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
-        assert_int_equal(rc, SSH_OK);
-        assert_non_null(pubkey);
-
         for (hash = SSH_DIGEST_AUTO;
              hash <= SSH_DIGEST_SHA512;
-             hash++) {
+             hash++)
-            hash_length = ((hash == SSH_DIGEST_AUTO) ?
+        {
-                              skey_attrs.sig_length : hash_lengths[hash]);
+            skey_attrs = key_attrs_list[sig_type][hash];
 
-            /* SHA384 is used only internaly for ECDSA. Skip it. */
+            if (!skey_attrs.sign) {
-            if (hash == SSH_DIGEST_SHA384) {
                 continue;
             }
 
+            rc = ssh_pki_generate(sig_type, skey_attrs.size_arg, &key);
+            assert_true(rc == SSH_OK);
+            assert_non_null(key);
+            assert_int_equal(key->type, sig_type);
+            assert_string_equal(key->type_c, skey_attrs.type_c);
+
             SSH_LOG(SSH_LOG_TRACE, "Creating signature %d with hash %d",
                     sig_type, hash);
 
+            if (skey_attrs.expect_success == 0) {
+                /* Expect error */
+                sign = pki_do_sign(key, INPUT, input_length, hash);
+                assert_null(sign);
+
+                SSH_KEY_FREE(key);
+                continue;
+            }
+
+            rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
+            assert_int_equal(rc, SSH_OK);
+            assert_non_null(pubkey);
+
             /* Create a valid signature using this key */
-            sign = pki_do_sign(key, HASH, hash_length, hash);
+            sign = pki_do_sign(key, INPUT, input_length, hash);
             assert_non_null(sign);
             assert_int_equal(sign->type, key->type);
-            if (hash == SSH_DIGEST_AUTO) {
+            assert_string_equal(sign->type_c, skey_attrs.sig_type_c);
-                assert_string_equal(sign->type_c, key->type_c);
-                assert_string_equal(sign->type_c, skey_attrs.sig_type_c);
-            } else {
-                assert_string_equal(sign->type_c, hash_signatures[hash]);
-            }
 
             /* Create a signature blob that can be imported and verified */
             blob = pki_signature_to_blob(sign);
@@ -231,28 +319,24 @@ static void torture_pki_verify_mismatch(void **state)
                                                  hash);
             assert_non_null(import_sig);
             assert_int_equal(import_sig->type, key->type);
-            if (hash == SSH_DIGEST_AUTO) {
+            assert_string_equal(import_sig->type_c, skey_attrs.sig_type_c);
-                assert_string_equal(import_sig->type_c, key->type_c);
-                assert_string_equal(import_sig->type_c, skey_attrs.sig_type_c);
-            } else {
-                assert_string_equal(import_sig->type_c, hash_signatures[hash]);
-            }
 
-            /* Internal API: Should work */
             rc = pki_signature_verify(session,
                                       import_sig,
                                       pubkey,
-                                      HASH,
+                                      INPUT,
-                                      hash_length);
+                                      input_length);
             assert_true(rc == SSH_OK);
 
             for (key_type = SSH_KEYTYPE_DSS;
                  key_type <= SSH_KEYTYPE_ED25519_CERT01;
-                 key_type++) {
+                 key_type++)
-                vkey_attrs = key_attrs_list[key_type];
+            {
+                vkey_attrs = key_attrs_list[key_type][hash];
                 if (!vkey_attrs.verify) {
                     continue;
                 }
+
                 SSH_LOG(SSH_LOG_TRACE, "Trying key %d with signature %d",
                         key_type, sig_type);
 
@@ -274,16 +358,16 @@ static void torture_pki_verify_mismatch(void **state)
                 rc = pki_signature_verify(session,
                                           sign,
                                           verify_pubkey,
-                                          HASH,
+                                          INPUT,
-                                          hash_length);
+                                          input_length);
                 assert_true(rc != SSH_OK);
 
                 /* Try the same with the imported signature */
                 rc = pki_signature_verify(session,
                                           import_sig,
                                           verify_pubkey,
-                                          HASH,
+                                          INPUT,
-                                          hash_length);
+                                          input_length);
                 assert_true(rc != SSH_OK);
 
                 /* Try to import the signature blob with different key */
@@ -291,23 +375,18 @@ static void torture_pki_verify_mismatch(void **state)
                                                   blob,
                                                   sig_type,
                                                   import_sig->hash_type);
-                if (ssh_key_type_plain(key_type) == sig_type) {
+                if (ssh_key_type_plain(verify_pubkey->type) == sig_type) {
                     /* Importing with the same key type should work */
                     assert_non_null(new_sig);
                     assert_int_equal(new_sig->type, key->type);
-                    if (ssh_key_type_plain(key_type) == SSH_KEYTYPE_RSA &&
+                    assert_string_equal(new_sig->type_c, skey_attrs.sig_type_c);
-                    new_sig->hash_type != SSH_DIGEST_AUTO) {
+
-                        assert_string_equal(new_sig->type_c, hash_signatures[new_sig->hash_type]);
-                    } else {
-                        assert_string_equal(new_sig->type_c, key->type_c);
-                        assert_string_equal(new_sig->type_c, skey_attrs.sig_type_c);
-                    }
                     /* The verification should not work */
                     rc = pki_signature_verify(session,
                                               new_sig,
                                               verify_pubkey,
-                                              HASH,
+                                              INPUT,
-                                              hash_length);
+                                              input_length);
                     assert_true(rc != SSH_OK);
 
                     ssh_signature_free(new_sig);
@@ -322,15 +401,10 @@ static void torture_pki_verify_mismatch(void **state)
             ssh_signature_free(sign);
             ssh_signature_free(import_sig);
 
-            /* XXX Test all the hash versions only with RSA. */
+            SSH_KEY_FREE(key);
-            if (sig_type != SSH_KEYTYPE_RSA || hash == SSH_DIGEST_SHA512) {
+            SSH_KEY_FREE(pubkey);
-                break;
+            key = NULL;
-            }
         }
-
-        SSH_KEY_FREE(key);
-        SSH_KEY_FREE(pubkey);
-        key = NULL;
     }
 
     ssh_free(session);

tests/unittests/torture_pki_dsa.c

@@ -15,7 +15,7 @@
 #define LIBSSH_DSA_TESTKEY_PASSPHRASE "libssh_testkey_passphrase.id_dsa"
 
 const char template[] = "temp_dir_XXXXXX";
-const unsigned char DSA_HASH[] = "12345678901234567890";
+const unsigned char INPUT[] = "12345678901234567890";
 
 struct pki_st {
     char *cwd;
@@ -208,26 +208,83 @@ static void torture_pki_sign_data_dsa(void **state)
     assert_int_equal(rc, SSH_OK);
     assert_non_null(key);
 
-    /* Test using automatic digest */
-    rc = test_sign_verify_data(key, SSH_DIGEST_AUTO, DSA_HASH, 20);
-    assert_int_equal(rc, SSH_OK);
-
     /* Test using SHA1 */
-    rc = test_sign_verify_data(key, SSH_DIGEST_SHA1, DSA_HASH, 20);
+    rc = test_sign_verify_data(key, SSH_DIGEST_SHA1, INPUT, sizeof(INPUT));
-    assert_int_equal(rc, SSH_OK);
-
-    /* Test using SHA256 */
-    rc = test_sign_verify_data(key, SSH_DIGEST_SHA256, DSA_HASH, 20);
-    assert_int_equal(rc, SSH_OK);
-
-    /* Test using SHA512 */
-    rc = test_sign_verify_data(key, SSH_DIGEST_SHA512, DSA_HASH, 20);
     assert_int_equal(rc, SSH_OK);
 
     /* Cleanup */
     SSH_KEY_FREE(key);
 }
 
+static void torture_pki_fail_sign_with_incompatible_hash(void **state)
+{
+    int rc;
+    ssh_key key = NULL;
+    ssh_key pubkey = NULL;
+    ssh_signature sig, bad_sig;
+
+    (void) state;
+
+    /* Setup */
+    rc = ssh_pki_generate(SSH_KEYTYPE_DSS, 2048, &key);
+    assert_int_equal(rc, SSH_OK);
+    assert_non_null(key);
+
+    /* Get the public key to verify signature */
+    rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
+    assert_int_equal(rc, SSH_OK);
+    assert_non_null(pubkey);
+
+    /* Sign the buffer */
+    sig = pki_sign_data(key, SSH_DIGEST_SHA1, INPUT, sizeof(INPUT));
+    assert_non_null(sig);
+
+    /* Verify signature */
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_equal(rc, SSH_OK);
+
+    /* Test if signature fails with SSH_DIGEST_AUTO */
+    bad_sig = pki_sign_data(key, SSH_DIGEST_AUTO, INPUT, sizeof(INPUT));
+    assert_null(bad_sig);
+
+    /* Test if verification fails with SSH_DIGEST_AUTO */
+    sig->hash_type = SSH_DIGEST_AUTO;
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_not_equal(rc, SSH_OK);
+
+    /* Test if signature fails with SSH_DIGEST_SHA256 */
+    bad_sig = pki_sign_data(key, SSH_DIGEST_SHA256, INPUT, sizeof(INPUT));
+    assert_null(bad_sig);
+
+    /* Test if verification fails with SSH_DIGEST_SHA256 */
+    sig->hash_type = SSH_DIGEST_SHA256;
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_not_equal(rc, SSH_OK);
+
+    /* Test if signature fails with SSH_DIGEST_SHA384 */
+    bad_sig = pki_sign_data(key, SSH_DIGEST_SHA384, INPUT, sizeof(INPUT));
+    assert_null(bad_sig);
+
+    /* Test if verification fails with SSH_DIGEST_SHA384 */
+    sig->hash_type = SSH_DIGEST_SHA384;
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_not_equal(rc, SSH_OK);
+
+    /* Test if signature fails with SSH_DIGEST_SHA512 */
+    bad_sig = pki_sign_data(key, SSH_DIGEST_SHA512, INPUT, sizeof(INPUT));
+    assert_null(bad_sig);
+
+    /* Test if verification fails with SSH_DIGEST_SHA512 */
+    sig->hash_type = SSH_DIGEST_SHA512;
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_not_equal(rc, SSH_OK);
+
+    /* Cleanup */
+    ssh_signature_free(sig);
+    SSH_KEY_FREE(pubkey);
+    SSH_KEY_FREE(key);
+}
+
 #ifdef HAVE_LIBCRYPTO
 static void torture_pki_dsa_write_privkey(void **state)
 {
@@ -684,9 +741,9 @@ static void torture_pki_dsa_generate_key(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, DSA_HASH, 20, SSH_DIGEST_AUTO);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA1);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, DSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     ssh_signature_free(sign);
     SSH_KEY_FREE(key);
@@ -698,9 +755,9 @@ static void torture_pki_dsa_generate_key(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, DSA_HASH, 20, SSH_DIGEST_AUTO);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA1);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, DSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     ssh_signature_free(sign);
     SSH_KEY_FREE(key);
@@ -712,9 +769,9 @@ static void torture_pki_dsa_generate_key(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, DSA_HASH, 20, SSH_DIGEST_AUTO);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA1);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, DSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     ssh_signature_free(sign);
     SSH_KEY_FREE(key);
@@ -743,9 +800,9 @@ static void torture_pki_dsa_cert_verify(void **state)
     assert_true(rc == 0);
     assert_non_null(cert);
 
-    sign = pki_do_sign(privkey, DSA_HASH, 20, SSH_DIGEST_AUTO);
+    sign = pki_do_sign(privkey, INPUT, sizeof(INPUT), SSH_DIGEST_SHA1);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, cert, DSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, cert, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     ssh_signature_free(sign);
     SSH_KEY_FREE(privkey);
@@ -782,6 +839,7 @@ int torture_run_tests(void)
                                  teardown),
 #endif
         cmocka_unit_test(torture_pki_sign_data_dsa),
+        cmocka_unit_test(torture_pki_fail_sign_with_incompatible_hash),
         cmocka_unit_test(torture_pki_dsa_import_privkey_base64_passphrase),
         cmocka_unit_test(torture_pki_dsa_import_openssh_privkey_base64_passphrase),
 

tests/unittests/torture_pki_ecdsa.c

@@ -14,7 +14,7 @@
 #define LIBSSH_ECDSA_TESTKEY_PASSPHRASE "libssh_testkey_passphrase.id_ecdsa"
 
 const char template[] = "temp_dir_XXXXXX";
-const unsigned char ECDSA_HASH[] = "12345678901234567890";
+const unsigned char INPUT[] = "12345678901234567890";
 
 struct pki_st {
     char *cwd;
@@ -477,9 +477,9 @@ static void torture_pki_generate_key_ecdsa(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, ECDSA_HASH, 20, SSH_DIGEST_SHA256);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA256);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, ECDSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     type = ssh_key_type(key);
     assert_true(type == SSH_KEYTYPE_ECDSA_P256);
@@ -499,9 +499,9 @@ static void torture_pki_generate_key_ecdsa(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, ECDSA_HASH, 20, SSH_DIGEST_SHA256);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA256);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, ECDSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     type = ssh_key_type(key);
     assert_true(type == SSH_KEYTYPE_ECDSA_P256);
@@ -520,9 +520,9 @@ static void torture_pki_generate_key_ecdsa(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, ECDSA_HASH, 20, SSH_DIGEST_SHA384);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA384);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, ECDSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     type = ssh_key_type(key);
     assert_true(type == SSH_KEYTYPE_ECDSA_P384);
@@ -542,9 +542,9 @@ static void torture_pki_generate_key_ecdsa(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, ECDSA_HASH, 20, SSH_DIGEST_SHA384);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA384);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, ECDSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     type = ssh_key_type(key);
     assert_true(type == SSH_KEYTYPE_ECDSA_P384);
@@ -563,9 +563,9 @@ static void torture_pki_generate_key_ecdsa(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, ECDSA_HASH, 20, SSH_DIGEST_SHA512);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA512);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, ECDSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     type = ssh_key_type(key);
     assert_true(type == SSH_KEYTYPE_ECDSA_P521);
@@ -585,9 +585,9 @@ static void torture_pki_generate_key_ecdsa(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, ECDSA_HASH, 20, SSH_DIGEST_SHA512);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA512);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, ECDSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     type = ssh_key_type(key);
     assert_true(type == SSH_KEYTYPE_ECDSA_P521);
@@ -609,6 +609,7 @@ static void torture_pki_ecdsa_cert_verify(void **state)
     ssh_key privkey = NULL, cert = NULL;
     ssh_signature sign = NULL;
     ssh_session session=ssh_new();
+    enum ssh_digest_e hash_type;
     (void) state;
 
     rc = ssh_pki_import_privkey_file(LIBSSH_ECDSA_TESTKEY,
@@ -623,9 +624,12 @@ static void torture_pki_ecdsa_cert_verify(void **state)
     assert_true(rc == 0);
     assert_non_null(cert);
 
-    sign = pki_do_sign(privkey, ECDSA_HASH, 20, SSH_DIGEST_SHA256);
+    /* Get the hash type to be used in the signature based on the key type */
+    hash_type = ssh_key_type_to_hash(session, privkey->type);
+
+    sign = pki_do_sign(privkey, INPUT, sizeof(INPUT), hash_type);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, cert, ECDSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, cert, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     ssh_signature_free(sign);
     SSH_KEY_FREE(privkey);
@@ -674,26 +678,83 @@ static void torture_pki_sign_data_ecdsa(void **state)
     assert_int_equal(rc, SSH_OK);
     assert_non_null(key);
 
-    /* Test using automatic digest */
-    rc = test_sign_verify_data(key, SSH_DIGEST_AUTO, ECDSA_HASH, 20);
-    assert_int_equal(rc, SSH_OK);
-
-    /* Test using SHA1 */
-    rc = test_sign_verify_data(key, SSH_DIGEST_SHA1, ECDSA_HASH, 20);
-    assert_int_equal(rc, SSH_OK);
-
     /* Test using SHA256 */
-    rc = test_sign_verify_data(key, SSH_DIGEST_SHA256, ECDSA_HASH, 20);
+    rc = test_sign_verify_data(key, SSH_DIGEST_SHA256, INPUT, sizeof(INPUT));
-    assert_int_equal(rc, SSH_OK);
-
-    /* Test using SHA512 */
-    rc = test_sign_verify_data(key, SSH_DIGEST_SHA512, ECDSA_HASH, 20);
     assert_int_equal(rc, SSH_OK);
 
     /* Cleanup */
     SSH_KEY_FREE(key);
 }
 
+static void torture_pki_fail_sign_with_incompatible_hash(void **state)
+{
+    int rc;
+    ssh_key key = NULL;
+    ssh_key pubkey = NULL;
+    ssh_signature sig, bad_sig;
+
+    (void) state;
+
+    /* Setup */
+    rc = ssh_pki_generate(SSH_KEYTYPE_ECDSA_P256, 256, &key);
+    assert_int_equal(rc, SSH_OK);
+    assert_non_null(key);
+
+    /* Get the public key to verify signature */
+    rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
+    assert_int_equal(rc, SSH_OK);
+    assert_non_null(pubkey);
+
+    /* Sign the buffer */
+    sig = pki_sign_data(key, SSH_DIGEST_SHA256, INPUT, sizeof(INPUT));
+    assert_non_null(sig);
+
+    /* Verify signature */
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_equal(rc, SSH_OK);
+
+    /* Test if signature fails with SSH_DIGEST_AUTO */
+    bad_sig = pki_sign_data(key, SSH_DIGEST_AUTO, INPUT, sizeof(INPUT));
+    assert_null(bad_sig);
+
+    /* Test if verification fails with SSH_DIGEST_AUTO */
+    sig->hash_type = SSH_DIGEST_AUTO;
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_not_equal(rc, SSH_OK);
+
+    /* Test if signature fails with SSH_DIGEST_SHA1 */
+    bad_sig = pki_sign_data(key, SSH_DIGEST_SHA1, INPUT, sizeof(INPUT));
+    assert_null(bad_sig);
+
+    /* Test if verification fails with SSH_DIGEST_SHA1 */
+    sig->hash_type = SSH_DIGEST_SHA1;
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_not_equal(rc, SSH_OK);
+
+    /* Test if signature fails with SSH_DIGEST_SHA384 */
+    bad_sig = pki_sign_data(key, SSH_DIGEST_SHA384, INPUT, sizeof(INPUT));
+    assert_null(bad_sig);
+
+    /* Test if verification fails with SSH_DIGEST_SHA384 */
+    sig->hash_type = SSH_DIGEST_SHA384;
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_not_equal(rc, SSH_OK);
+
+    /* Test if signature fails with SSH_DIGEST_SHA512 */
+    bad_sig = pki_sign_data(key, SSH_DIGEST_SHA512, INPUT, sizeof(INPUT));
+    assert_null(bad_sig);
+
+    /* Test if verification fails with SSH_DIGEST_SHA512 */
+    sig->hash_type = SSH_DIGEST_SHA512;
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_not_equal(rc, SSH_OK);
+
+    /* Cleanup */
+    ssh_signature_free(sig);
+    SSH_KEY_FREE(pubkey);
+    SSH_KEY_FREE(key);
+}
+
 #ifdef HAVE_LIBCRYPTO
 static void torture_pki_ecdsa_write_privkey(void **state)
 {
@@ -924,6 +985,7 @@ int torture_run_tests(void) {
                                         teardown),
 #endif /* HAVE_LIBCRYPTO */
         cmocka_unit_test(torture_pki_sign_data_ecdsa),
+        cmocka_unit_test(torture_pki_fail_sign_with_incompatible_hash),
         cmocka_unit_test_setup_teardown(torture_pki_ecdsa_name256,
                                         setup_ecdsa_key_256,
                                         teardown),

tests/unittests/torture_pki_rsa.c

@@ -15,10 +15,8 @@
 #define LIBSSH_RSA_TESTKEY_PASSPHRASE "libssh_testkey_passphrase.id_rsa"
 
 const char template[] = "temp_dir_XXXXXX";
-const unsigned char RSA_HASH[] = "12345678901234567890";
+const unsigned char INPUT[] = "1234567890123456789012345678901234567890"
-const unsigned char SHA256_HASH[] = "12345678901234567890123456789012";
+                              "123456789012345678901234";
-const unsigned char SHA512_HASH[] = "1234567890123456789012345678901234567890"
-                                    "123456789012345678901234";
 
 struct pki_st {
     char *cwd;
@@ -472,9 +470,9 @@ static void torture_pki_rsa_generate_key(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, RSA_HASH, 20, SSH_DIGEST_SHA256);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA256);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, RSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     ssh_signature_free(sign);
     SSH_KEY_FREE(key);
@@ -488,9 +486,9 @@ static void torture_pki_rsa_generate_key(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, RSA_HASH, 20, SSH_DIGEST_SHA256);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA256);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, RSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     ssh_signature_free(sign);
     SSH_KEY_FREE(key);
@@ -504,9 +502,9 @@ static void torture_pki_rsa_generate_key(void **state)
     rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
-    sign = pki_do_sign(key, RSA_HASH, 20, SSH_DIGEST_SHA256);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA256);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, RSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_true(rc == SSH_OK);
     ssh_signature_free(sign);
     SSH_KEY_FREE(key);
@@ -541,42 +539,41 @@ static void torture_pki_rsa_sha2(void **state)
     assert_int_equal(rc, SSH_OK);
     assert_non_null(pubkey);
 
-    /* Sign using automatic digest */
-    sign = pki_do_sign(key, RSA_HASH, 20, SSH_DIGEST_AUTO);
-    assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, RSA_HASH, 20);
-    assert_ssh_return_code(session, rc);
-    rc = pki_signature_verify(session, sign, cert, RSA_HASH, 20);
-    assert_ssh_return_code(session, rc);
-    ssh_signature_free(sign);
-
     /* Sign using old SHA1 digest */
-    sign = pki_do_sign(key, RSA_HASH, 20, SSH_DIGEST_SHA1);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA1);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, RSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_ssh_return_code(session, rc);
-    rc = pki_signature_verify(session, sign, cert, RSA_HASH, 20);
+    rc = pki_signature_verify(session, sign, cert, INPUT, sizeof(INPUT));
     assert_ssh_return_code(session, rc);
     ssh_signature_free(sign);
 
     /* Sign using new SHA256 digest */
-    sign = pki_do_sign(key, SHA256_HASH, 32, SSH_DIGEST_SHA256);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA256);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, SHA256_HASH, 32);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_ssh_return_code(session, rc);
-    rc = pki_signature_verify(session, sign, cert, SHA256_HASH, 32);
+    rc = pki_signature_verify(session, sign, cert, INPUT, sizeof(INPUT));
     assert_ssh_return_code(session, rc);
     ssh_signature_free(sign);
 
     /* Sign using rsa-sha2-512 algorithm */
-    sign = pki_do_sign(key, SHA512_HASH, 64, SSH_DIGEST_SHA512);
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA512);
     assert_non_null(sign);
-    rc = pki_signature_verify(session, sign, pubkey, SHA512_HASH, 64);
+    rc = pki_signature_verify(session, sign, pubkey, INPUT, sizeof(INPUT));
     assert_ssh_return_code(session, rc);
-    rc = pki_signature_verify(session, sign, cert, SHA512_HASH, 64);
+    rc = pki_signature_verify(session, sign, cert, INPUT, sizeof(INPUT));
     assert_ssh_return_code(session, rc);
     ssh_signature_free(sign);
 
+    /* Test that it fails when using DIGEST_AUTO */
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_AUTO);
+    assert_null(sign);
+
+    /* Test that it fails when using SHA384 */
+    sign = pki_do_sign(key, INPUT, sizeof(INPUT), SSH_DIGEST_SHA384);
+    assert_null(sign);
+
     /* Cleanup */
     SSH_KEY_FREE(key);
     SSH_KEY_FREE(pubkey);
@@ -624,26 +621,64 @@ static void torture_pki_sign_data_rsa(void **state)
     assert_int_equal(rc, SSH_OK);
     assert_non_null(key);
 
-    /* Test using automatic digest */
-    rc = test_sign_verify_data(key, SSH_DIGEST_AUTO, RSA_HASH, 20);
-    assert_int_equal(rc, SSH_OK);
-
     /* Test using SHA1 */
-    rc = test_sign_verify_data(key, SSH_DIGEST_SHA1, RSA_HASH, 20);
+    rc = test_sign_verify_data(key, SSH_DIGEST_SHA1, INPUT, sizeof(INPUT));
     assert_int_equal(rc, SSH_OK);
 
     /* Test using SHA256 */
-    rc = test_sign_verify_data(key, SSH_DIGEST_SHA256, RSA_HASH, 20);
+    rc = test_sign_verify_data(key, SSH_DIGEST_SHA256, INPUT, sizeof(INPUT));
     assert_int_equal(rc, SSH_OK);
 
     /* Test using SHA512 */
-    rc = test_sign_verify_data(key, SSH_DIGEST_SHA512, RSA_HASH, 20);
+    rc = test_sign_verify_data(key, SSH_DIGEST_SHA512, INPUT, sizeof(INPUT));
     assert_int_equal(rc, SSH_OK);
 
     /* Cleanup */
     SSH_KEY_FREE(key);
 }
 
+static void torture_pki_fail_sign_with_incompatible_hash(void **state)
+{
+    int rc;
+    ssh_key key = NULL;
+    ssh_key pubkey = NULL;
+    ssh_signature sig, bad_sig;
+
+    (void) state;
+
+    /* Setup */
+    rc = ssh_pki_generate(SSH_KEYTYPE_RSA, 2048, &key);
+    assert_int_equal(rc, SSH_OK);
+    assert_non_null(key);
+
+    /* Get the public key to verify signature */
+    rc = ssh_pki_export_privkey_to_pubkey(key, &pubkey);
+    assert_int_equal(rc, SSH_OK);
+    assert_non_null(pubkey);
+
+    /* Sign the buffer */
+    sig = pki_sign_data(key, SSH_DIGEST_SHA1, INPUT, sizeof(INPUT));
+    assert_non_null(sig);
+
+    /* Verify signature */
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_equal(rc, SSH_OK);
+
+    /* Test if signature fails with SSH_DIGEST_AUTO */
+    bad_sig = pki_sign_data(key, SSH_DIGEST_AUTO, INPUT, sizeof(INPUT));
+    assert_null(bad_sig);
+
+    /* Test if verification fails with SSH_DIGEST_AUTO */
+    sig->hash_type = SSH_DIGEST_AUTO;
+    rc = pki_verify_data_signature(sig, pubkey, INPUT, sizeof(INPUT));
+    assert_int_not_equal(rc, SSH_OK);
+
+    /* Cleanup */
+    ssh_signature_free(sig);
+    SSH_KEY_FREE(pubkey);
+    SSH_KEY_FREE(key);
+}
+
 #ifdef HAVE_LIBCRYPTO
 static void torture_pki_rsa_write_privkey(void **state)
 {
@@ -864,6 +899,7 @@ int torture_run_tests(void) {
                                         teardown),
 #endif /* HAVE_LIBCRYPTO */
         cmocka_unit_test(torture_pki_sign_data_rsa),
+        cmocka_unit_test(torture_pki_fail_sign_with_incompatible_hash),
         cmocka_unit_test_setup_teardown(torture_pki_rsa_sha2,
                                         setup_rsa_key,
                                         teardown),