md5: allow disabling old-style encrypted private keys at build-time

Before this patch, this happened at runtime when using an old (pre-3.0),
FIPS-enabled OpenSSL backend.

This patch makes it possible to disable this via the build-time option
`LIBSSH2_NO_MD5_PEM`.

Also:
- make sure to exclude all MD5 internal APIs when both the above and
  `LIBSSH2_NO_MD5` are enabled.
- fix tests to support build with`LIBSSH2_NO_MD5`, `LIBSSH2_NO_MD5_PEM`
  and `LIBSSH2_NO_3DES`.
- add FIXME to apply this change to `os400qc3.*`.

Old-style encrypted private keys require MD5 and they look like this:
```
-----BEGIN RSA PRIVATE KEY-----
 Proc-Type: 4,ENCRYPTED
 DEK-Info: AES-128-CBC,<MD5-hex>

 <base64>
 -----END RSA PRIVATE KEY-----
```

E.g.: `tests/key_rsa_encrypted`

Ref: https://github.com/libssh2/www/issues/20
Closes #1181

tests/session_fixture.c

@@ -91,6 +91,10 @@ static char const *skip_crypt[] = {
     "rijndael-cbc@lysator.liu.se",
 #endif
 
+#if !LIBSSH2_3DES
+    "3des-cbc",
+#endif
+
 #if defined(LIBSSH2_LIBGCRYPT) || defined(LIBSSH2_OS400QC3) || \
     defined(LIBSSH2_WINCNG)
     /* Support for AES-GCM hasn't been added to these back-ends yet */
@@ -101,6 +105,15 @@ static char const *skip_crypt[] = {
     NULL
 };
 
+/* List of MAC protocols for which tests are skipped */
+static char const *skip_mac[] = {
+#if !LIBSSH2_MD5
+    "hmac-md5",
+    "hmac-md5-96",
+#endif
+    NULL
+};
+
 LIBSSH2_SESSION *start_session_fixture(int *skipped, int *err)
 {
     int rc;
@@ -112,11 +125,23 @@ LIBSSH2_SESSION *start_session_fixture(int *skipped, int *err)
     *err = LIBSSH2_ERROR_NONE;
 
     if(crypt) {
-        char const * const *cr;
-        for(cr = skip_crypt; *cr; ++cr) {
-            if(strcmp(*cr, crypt) == 0) {
-                fprintf(stderr, "crypt algorithm (%s) skipped "
-                                "for this crypto backend.\n", crypt);
+        char const * const *sk;
+        for(sk = skip_crypt; *sk; ++sk) {
+            if(strcmp(*sk, crypt) == 0) {
+                fprintf(stderr, "unsupported crypt algorithm (%s) skipped.\n",
+                                crypt);
+                *skipped = 1;
+                return NULL;
+            }
+        }
+    }
+
+    if(mac) {
+        char const * const *sk;
+        for(sk = skip_mac; *sk; ++sk) {
+            if(strcmp(*sk, mac) == 0) {
+                fprintf(stderr, "unsupported MAC algorithm (%s) skipped.\n",
+                                mac);
                 *skipped = 1;
                 return NULL;
             }

tests/test_auth_pubkey_ok_rsa_encrypted.c

@@ -7,7 +7,7 @@
 
 int test(LIBSSH2_SESSION *session)
 {
-#if LIBSSH2_RSA_SHA1
+#if LIBSSH2_RSA_SHA1 && LIBSSH2_MD5_PEM
     /* set in Dockerfile */
     return test_auth_pubkey(session, 0,
                             "libssh2",

tests/test_hostkey_hash.c

@@ -20,8 +20,10 @@ static const char *EXPECTED_ECDSA_HOSTKEY =
 static const char *EXPECTED_ED25519_HOSTKEY =
     "AAAAC3NzaC1lZDI1NTE5AAAAIIxtdyg2ZRXE70UwyPVUH3UyfDBV8GX5cPF636P6hjom";
 
+#if LIBSSH2_MD5
 static const char *EXPECTED_RSA_MD5_HASH_DIGEST =
     "0C0ED1A5BB10275F76924CE187CE5C5E";
+#endif
 
 static const char *EXPECTED_RSA_SHA1_HASH_DIGEST =
     "F3CD59E2913F4422B80F7B0A82B2B89EAE449387";
@@ -29,8 +31,10 @@ static const char *EXPECTED_RSA_SHA1_HASH_DIGEST =
 static const char *EXPECTED_RSA_SHA256_HASH_DIGEST =
     "92E3DA49DF3C7F99A828F505ED8239397A5D1F62914459760F878F7510F563A3";
 
+#if LIBSSH2_MD5
 static const char *EXPECTED_ECDSA_MD5_HASH_DIGEST =
     "0402E4D897580BBC911379CBD88BCD3D";
+#endif
 
 static const char *EXPECTED_ECDSA_SHA1_HASH_DIGEST =
     "12FDAD1E3B31B10BABB00F2A8D1B9A62C326BD2F";
@@ -41,7 +45,9 @@ static const char *EXPECTED_ECDSA_SHA256_HASH_DIGEST =
 static const char *EXPECTED_ED25519_SHA256_HASH_DIGEST =
     "2638B020F6121FA750A7F4754B718419F621814C6E779D68ADF26AA68814ADDF";
 
+#if LIBSSH2_MD5
 static const int MD5_HASH_SIZE = 16;
+#endif
 static const int SHA1_HASH_SIZE = 20;
 static const int SHA256_HASH_SIZE = 32;
 
@@ -62,7 +68,9 @@ int test(LIBSSH2_SESSION *session)
     char buf[BUFSIZ];
 
     const char *hostkey;
+#if LIBSSH2_MD5
     const char *md5_hash;
+#endif
     const char *sha1_hash;
     const char *sha256_hash;
     int type;
@@ -100,6 +108,7 @@ int test(LIBSSH2_SESSION *session)
     }
     else if(type == LIBSSH2_HOSTKEY_TYPE_ECDSA_256) {
 
+#if LIBSSH2_MD5
         md5_hash = libssh2_hostkey_hash(session, LIBSSH2_HOSTKEY_HASH_MD5);
         if(!md5_hash) {
             print_last_session_error(
@@ -115,6 +124,7 @@ int test(LIBSSH2_SESSION *session)
                     buf, EXPECTED_ECDSA_MD5_HASH_DIGEST);
             return 1;
         }
+#endif
 
         sha1_hash = libssh2_hostkey_hash(session, LIBSSH2_HOSTKEY_HASH_SHA1);
         if(!sha1_hash) {
@@ -151,6 +161,7 @@ int test(LIBSSH2_SESSION *session)
     }
     else if(type == LIBSSH2_HOSTKEY_TYPE_RSA) {
 
+#if LIBSSH2_MD5
         md5_hash = libssh2_hostkey_hash(session, LIBSSH2_HOSTKEY_HASH_MD5);
         if(!md5_hash) {
             print_last_session_error(
@@ -166,6 +177,7 @@ int test(LIBSSH2_SESSION *session)
                     buf, EXPECTED_RSA_MD5_HASH_DIGEST);
             return 1;
         }
+#endif
 
         sha1_hash = libssh2_hostkey_hash(session, LIBSSH2_HOSTKEY_HASH_SHA1);
         if(!sha1_hash) {