* RSA_decrypt now checks the integrity of the first 11 bytes.

* The size of the output buffer in RSA_decrypt is now checked and cleared.
* get_random now returns an error code
* Various system calls now check the return code to remove gcc warnings.

git-svn-id: svn://svn.code.sf.net/p/axtls/code/trunk@237 9a5d90b5-6617-0410-8a86-bb477d3ed2e3

crypto/crypto.h

@@ -203,7 +203,7 @@ void RSA_pub_key_new(RSA_CTX **rsa_ctx,
         const uint8_t *pub_exp, int pub_len);
 void RSA_free(RSA_CTX *ctx);
 int RSA_decrypt(const RSA_CTX *ctx, const uint8_t *in_data, uint8_t *out_data,
-        int is_decryption);
+        int out_len, int is_decryption);
 bigint *RSA_private(const RSA_CTX *c, bigint *bi_msg);
 #if defined(CONFIG_SSL_CERT_VERIFICATION) || defined(CONFIG_SSL_GENERATE_X509_CERT)
 bigint *RSA_sign_verify(BI_CTX *ctx, const uint8_t *sig, int sig_len,
@@ -220,8 +220,8 @@ void RSA_print(const RSA_CTX *ctx);
 EXP_FUNC void STDCALL RNG_initialize(void);
 EXP_FUNC void STDCALL RNG_custom_init(const uint8_t *seed_buf, int size);
 EXP_FUNC void STDCALL RNG_terminate(void);
-EXP_FUNC void STDCALL get_random(int num_rand_bytes, uint8_t *rand_data);
-void get_random_NZ(int num_rand_bytes, uint8_t *rand_data);
+EXP_FUNC int STDCALL get_random(int num_rand_bytes, uint8_t *rand_data);
+int get_random_NZ(int num_rand_bytes, uint8_t *rand_data);
 
 #ifdef __cplusplus
 }

crypto/crypto_misc.c

@@ -156,11 +156,12 @@ EXP_FUNC void STDCALL RNG_terminate(void)
 /**
  * Set a series of bytes with a random number. Individual bytes can be 0
  */
-EXP_FUNC void STDCALL get_random(int num_rand_bytes, uint8_t *rand_data)
+EXP_FUNC int STDCALL get_random(int num_rand_bytes, uint8_t *rand_data)
 {   
 #if !defined(WIN32) && defined(CONFIG_USE_DEV_URANDOM)
-    /* use the Linux default */
-    read(rng_fd, rand_data, num_rand_bytes);    /* read from /dev/urandom */
+    /* use the Linux default - read from /dev/urandom */
+    if (read(rng_fd, rand_data, num_rand_bytes) < 0) 
+        return -1;
 #elif defined(WIN32) && defined(CONFIG_WIN32_USE_CRYPTO_LIB)
     /* use Microsoft Crypto Libraries */
     CryptGenRandom(gCryptProv, num_rand_bytes, rand_data);
@@ -198,21 +199,25 @@ EXP_FUNC void STDCALL get_random(int num_rand_bytes, uint8_t *rand_data)
     /* insert the digest at the start of the entropy pool */
     memcpy(entropy_pool, digest, MD5_SIZE);
 #endif
+    return 0;
 }
 
 /**
  * Set a series of bytes with a random number. Individual bytes are not zero.
  */
-void get_random_NZ(int num_rand_bytes, uint8_t *rand_data)
+int get_random_NZ(int num_rand_bytes, uint8_t *rand_data)
 {
     int i;
-    get_random(num_rand_bytes, rand_data);
+    if (get_random(num_rand_bytes, rand_data))
+        return -1;
 
     for (i = 0; i < num_rand_bytes; i++)
     {
         while (rand_data[i] == 0)  /* can't be 0 */
             rand_data[i] = (uint8_t)(rand());
     }
+
+    return 0;
 }
 
 /**

crypto/rsa.c

@@ -134,21 +134,26 @@ void RSA_free(RSA_CTX *rsa_ctx)
 /**
  * @brief Use PKCS1.5 for decryption/verification.
  * @param ctx [in] The context
- * @param in_data [in] The data to encrypt (must be < modulus size-11)
- * @param out_data [out] The encrypted data.
+ * @param in_data [in] The data to decrypt (must be < modulus size-11)
+ * @param out_data [out] The decrypted data.
+ * @param out_len [int] The size of the decrypted buffer in bytes
  * @param is_decryption [in] Decryption or verify operation.
  * @return  The number of bytes that were originally encrypted. -1 on error.
  * @see http://www.rsasecurity.com/rsalabs/node.asp?id=2125
  */
 int RSA_decrypt(const RSA_CTX *ctx, const uint8_t *in_data, 
-                            uint8_t *out_data, int is_decryption)
+                            uint8_t *out_data, int out_len, int is_decryption)
 {
     const int byte_size = ctx->num_octets;
-    int i, size;
+    int i = 0, size;
     bigint *decrypted_bi, *dat_bi;
     uint8_t *block = (uint8_t *)alloca(byte_size);
+    int pad_count = 0;
 
-    memset(out_data, 0, byte_size); /* initialise */
+    if (out_len < byte_size)        /* check output has enough size */
+        return -1;
+
+    memset(out_data, 0, out_len);   /* initialise */
 
     /* decrypt */
     dat_bi = bi_import(ctx->bi_ctx, in_data, byte_size);
@@ -162,28 +167,37 @@ int RSA_decrypt(const RSA_CTX *ctx, const uint8_t *in_data,
     /* convert to a normal block */
     bi_export(ctx->bi_ctx, decrypted_bi, block, byte_size);
 
-    i = 10; /* start at the first possible non-padded byte */
+    if (block[i++] != 0)             /* leading 0? */
+        return -1;
 
 #ifdef CONFIG_SSL_CERT_VERIFICATION
     if (is_decryption == 0) /* PKCS1.5 signing pads with "0xff"s */
     {
-        while (block[i++] == 0xff && i < byte_size);
+        if (block[i++] != 0x01)     /* BT correct? */
+            return -1;
 
-        if (block[i-2] != 0xff)
-            i = byte_size;     /*ensure size is 0 */   
+        while (block[i++] == 0xff && i < byte_size)
+            pad_count++;
     }
     else                    /* PKCS1.5 encryption padding is random */
 #endif
     {
-        while (block[i++] && i < byte_size);
+        if (block[i++] != 0x02)     /* BT correct? */
+            return -1;
+
+        while (block[i++] && i < byte_size)
+            pad_count++;
     }
+
+    /* check separator byte - and padding must be 8 or more bytes */
+    if (i == byte_size || pad_count < 8) 
+        return -1;
+
     size = byte_size - i;
 
     /* get only the bit we want */
-    if (size > 0)
-        memcpy(out_data, &block[i], size);
-    
-    return size ? size : -1;
+    memcpy(out_data, &block[i], size);
+    return size;
 }
 
 /**
@@ -249,7 +263,8 @@ int RSA_encrypt(const RSA_CTX *ctx, const uint8_t *in_data, uint16_t in_len,
     else /* randomize the encryption padding with non-zero bytes */   
     {
         out_data[1] = 2;
-        get_random_NZ(num_pads_needed, &out_data[2]);
+        if (get_random_NZ(num_pads_needed, &out_data[2]) < 0)
+            return -1;
     }
 
     out_data[2+num_pads_needed] = 0;