src: silence compiler warnings 3 (change types)

Apply type changes to avoid casts and warnings. In most cases this
means changing to a larger type, usually `size_t` or `ssize_t`.

Change signedness in a few places.

Also introduce new variables to avoid reusing them for multiple
purposes, to avoid casts and warnings.

- add FIXME for public `libssh2_sftp_readdir_ex()` return type.

- fix `_libssh2_mbedtls_rsa_sha2_verify()` to verify if `sig_len`
  is large enough.

- fix `_libssh2_dh_key_pair()` in `wincng.c` to return error if
  `group_order` input is negative.

  Maybe we should also reject zero?

- bump `_libssh2_random()` size type `int` -> `size_t`. Add checks
  for WinCNG and OpenSSL to return error if requested more than they
  support (`ULONG_MAX`, `INT_MAX` respectively).

- change `_libssh2_ntohu32()` return value `unsigned int` -> `uint32_t`.

- fix `_libssh2_mbedtls_bignum_random()` to check for a negative `top`
  input.

- size down `_libssh2_wincng_key_sha_verify()` `hashlen` to match
  Windows'.

- fix `session_disconnect()` to limit length of `lang_len`
  (to 256 bytes).

- fix bad syntax in an `assert()`.

- add a few `const` to casts.

- `while(1)` -> `for(;;)`.

- add casts that didn't fit into #876.

- update `docs/HACKING-CRYPTO` with new sizes.

May need review for OS400QC3: /cc @monnerat @jonrumsey

See warning details in the PR's individual commits.

Cherry-picked from #846
Closes #879

src/openssl.h

@@ -181,7 +181,8 @@
 
 #define EC_MAX_POINT_LEN ((528 * 2 / 8) + 1)
 
-#define _libssh2_random(buf, len) (RAND_bytes((buf), (len)) == 1 ? 0 : -1)
+#define _libssh2_random(buf, len) \
+  _libssh2_openssl_random((buf), (len))
 
 #define libssh2_prepare_iovec(vec, len)  /* Empty. */
 
@@ -204,7 +205,7 @@ int _libssh2_sha1_init(libssh2_sha1_ctx *ctx);
 #define libssh2_sha1_update(ctx, data, len) EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_sha1_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)
 #endif
-int _libssh2_sha1(const unsigned char *message, unsigned long len,
+int _libssh2_sha1(const unsigned char *message, size_t len,
                   unsigned char *out);
 #define libssh2_sha1(x,y,z) _libssh2_sha1(x,y,z)
 
@@ -228,8 +229,8 @@ int _libssh2_sha256_init(libssh2_sha256_ctx *ctx);
     EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_sha256_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)
 #endif
-int _libssh2_sha256(const unsigned char *message, unsigned long len,
-                  unsigned char *out);
+int _libssh2_sha256(const unsigned char *message, size_t len,
+                    unsigned char *out);
 #define libssh2_sha256(x,y,z) _libssh2_sha256(x,y,z)
 
 #ifdef HAVE_OPAQUE_STRUCTS
@@ -252,7 +253,7 @@ int _libssh2_sha384_init(libssh2_sha384_ctx *ctx);
     EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_sha384_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)
 #endif
-int _libssh2_sha384(const unsigned char *message, unsigned long len,
+int _libssh2_sha384(const unsigned char *message, size_t len,
                     unsigned char *out);
 #define libssh2_sha384(x,y,z) _libssh2_sha384(x,y,z)
 
@@ -276,7 +277,7 @@ int _libssh2_sha512_init(libssh2_sha512_ctx *ctx);
     EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_sha512_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)
 #endif
-int _libssh2_sha512(const unsigned char *message, unsigned long len,
+int _libssh2_sha512(const unsigned char *message, size_t len,
                     unsigned char *out);
 #define libssh2_sha512(x,y,z) _libssh2_sha512(x,y,z)
 
@@ -427,6 +428,8 @@ extern int _libssh2_dh_secret(_libssh2_dh_ctx *dhctx, _libssh2_bn *secret,
                               _libssh2_bn_ctx *bnctx);
 extern void _libssh2_dh_dtor(_libssh2_dh_ctx *dhctx);
 
+extern int _libssh2_openssl_random(void *buf, size_t len);
+
 const EVP_CIPHER *_libssh2_EVP_aes_128_ctr(void);
 const EVP_CIPHER *_libssh2_EVP_aes_192_ctr(void);
 const EVP_CIPHER *_libssh2_EVP_aes_256_ctr(void);