MDEV-30389 Ensure correct dlen during encryption

This patch ensures that all direct and indirect calls to
encryption_crypt provide a `dlen` value correctly initialized to the
destination buffer length, allowing encryption plugins to verify
available space. It also adds assertions to verify related invariants.

Signed-off-by: Trevor Gross <tmgross@umich.edu>

include/mysql/plugin_audit.h.pp

@@ -57,10 +57,13 @@ static inline int encryption_crypt(const unsigned char* src, unsigned int slen,
 {
   void *ctx= alloca(encryption_handler.encryption_ctx_size_func((key_id),(key_version)));
   int res1, res2;
-  unsigned int d1, d2;
+  unsigned int d1, d2= *dlen;
+  assert(*dlen >= slen);
+  assert((dst[*dlen - 1]= 1));
   if ((res1= encryption_handler.encryption_ctx_init_func((ctx),(key),(klen),(iv),(ivlen),(flags),(key_id),(key_version))))
     return res1;
   res1= encryption_handler.encryption_ctx_update_func((ctx),(src),(slen),(dst),(&d1));
+  d2-= d1;
   res2= encryption_handler.encryption_ctx_finish_func((ctx),(dst + d1),(&d2));
   *dlen= d1 + d2;
   return res1 ? res1 : res2;

include/mysql/plugin_auth.h.pp

@@ -57,10 +57,13 @@ static inline int encryption_crypt(const unsigned char* src, unsigned int slen,
 {
   void *ctx= alloca(encryption_handler.encryption_ctx_size_func((key_id),(key_version)));
   int res1, res2;
-  unsigned int d1, d2;
+  unsigned int d1, d2= *dlen;
+  assert(*dlen >= slen);
+  assert((dst[*dlen - 1]= 1));
   if ((res1= encryption_handler.encryption_ctx_init_func((ctx),(key),(klen),(iv),(ivlen),(flags),(key_id),(key_version))))
     return res1;
   res1= encryption_handler.encryption_ctx_update_func((ctx),(src),(slen),(dst),(&d1));
+  d2-= d1;
   res2= encryption_handler.encryption_ctx_finish_func((ctx),(dst + d1),(&d2));
   *dlen= d1 + d2;
   return res1 ? res1 : res2;

include/mysql/plugin_data_type.h.pp

@@ -57,10 +57,13 @@ static inline int encryption_crypt(const unsigned char* src, unsigned int slen,
 {
   void *ctx= alloca(encryption_handler.encryption_ctx_size_func((key_id),(key_version)));
   int res1, res2;
-  unsigned int d1, d2;
+  unsigned int d1, d2= *dlen;
+  assert(*dlen >= slen);
+  assert((dst[*dlen - 1]= 1));
   if ((res1= encryption_handler.encryption_ctx_init_func((ctx),(key),(klen),(iv),(ivlen),(flags),(key_id),(key_version))))
     return res1;
   res1= encryption_handler.encryption_ctx_update_func((ctx),(src),(slen),(dst),(&d1));
+  d2-= d1;
   res2= encryption_handler.encryption_ctx_finish_func((ctx),(dst + d1),(&d2));
   *dlen= d1 + d2;
   return res1 ? res1 : res2;

include/mysql/plugin_encryption.h

@@ -96,8 +96,11 @@ struct st_mariadb_encryption
   /**
     processes (encrypts or decrypts) a chunk of data
 
-    writes the output to th dst buffer. note that it might write
+    writes the output to the dst buffer. note that it might write
     more bytes that were in the input. or less. or none at all.
+
+    dlen points to the starting lenght of the output buffer. Upon return, it
+    should be set to the number of bytes written.
   */
   int (*crypt_ctx_update)(void *ctx, const unsigned char* src, unsigned int slen,
                           unsigned char* dst, unsigned int* dlen);
@@ -123,4 +126,3 @@ struct st_mariadb_encryption
 }
 #endif
 #endif
-

include/mysql/plugin_encryption.h.pp

@@ -57,10 +57,13 @@ static inline int encryption_crypt(const unsigned char* src, unsigned int slen,
 {
   void *ctx= alloca(encryption_handler.encryption_ctx_size_func((key_id),(key_version)));
   int res1, res2;
-  unsigned int d1, d2;
+  unsigned int d1, d2= *dlen;
+  assert(*dlen >= slen);
+  assert((dst[*dlen - 1]= 1));
   if ((res1= encryption_handler.encryption_ctx_init_func((ctx),(key),(klen),(iv),(ivlen),(flags),(key_id),(key_version))))
     return res1;
   res1= encryption_handler.encryption_ctx_update_func((ctx),(src),(slen),(dst),(&d1));
+  d2-= d1;
   res2= encryption_handler.encryption_ctx_finish_func((ctx),(dst + d1),(&d2));
   *dlen= d1 + d2;
   return res1 ? res1 : res2;

include/mysql/plugin_ftparser.h.pp

@@ -57,10 +57,13 @@ static inline int encryption_crypt(const unsigned char* src, unsigned int slen,
 {
   void *ctx= alloca(encryption_handler.encryption_ctx_size_func((key_id),(key_version)));
   int res1, res2;
-  unsigned int d1, d2;
+  unsigned int d1, d2= *dlen;
+  assert(*dlen >= slen);
+  assert((dst[*dlen - 1]= 1));
   if ((res1= encryption_handler.encryption_ctx_init_func((ctx),(key),(klen),(iv),(ivlen),(flags),(key_id),(key_version))))
     return res1;
   res1= encryption_handler.encryption_ctx_update_func((ctx),(src),(slen),(dst),(&d1));
+  d2-= d1;
   res2= encryption_handler.encryption_ctx_finish_func((ctx),(dst + d1),(&d2));
   *dlen= d1 + d2;
   return res1 ? res1 : res2;

include/mysql/plugin_function.h.pp

@@ -57,10 +57,13 @@ static inline int encryption_crypt(const unsigned char* src, unsigned int slen,
 {
   void *ctx= alloca(encryption_handler.encryption_ctx_size_func((key_id),(key_version)));
   int res1, res2;
-  unsigned int d1, d2;
+  unsigned int d1, d2= *dlen;
+  assert(*dlen >= slen);
+  assert((dst[*dlen - 1]= 1));
   if ((res1= encryption_handler.encryption_ctx_init_func((ctx),(key),(klen),(iv),(ivlen),(flags),(key_id),(key_version))))
     return res1;
   res1= encryption_handler.encryption_ctx_update_func((ctx),(src),(slen),(dst),(&d1));
+  d2-= d1;
   res2= encryption_handler.encryption_ctx_finish_func((ctx),(dst + d1),(&d2));
   *dlen= d1 + d2;
   return res1 ? res1 : res2;

include/mysql/plugin_password_validation.h.pp

@@ -57,10 +57,13 @@ static inline int encryption_crypt(const unsigned char* src, unsigned int slen,
 {
   void *ctx= alloca(encryption_handler.encryption_ctx_size_func((key_id),(key_version)));
   int res1, res2;
-  unsigned int d1, d2;
+  unsigned int d1, d2= *dlen;
+  assert(*dlen >= slen);
+  assert((dst[*dlen - 1]= 1));
   if ((res1= encryption_handler.encryption_ctx_init_func((ctx),(key),(klen),(iv),(ivlen),(flags),(key_id),(key_version))))
     return res1;
   res1= encryption_handler.encryption_ctx_update_func((ctx),(src),(slen),(dst),(&d1));
+  d2-= d1;
   res2= encryption_handler.encryption_ctx_finish_func((ctx),(dst + d1),(&d2));
   *dlen= d1 + d2;
   return res1 ? res1 : res2;

include/mysql/service_encryption.h

@@ -36,6 +36,9 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#ifndef MYSQL_ABI_CHECK
+#include <assert.h>
+#endif
 
 /* returned from encryption_key_get_latest_version() */
 #define ENCRYPTION_KEY_VERSION_INVALID        (~(unsigned int)0)
@@ -101,6 +104,7 @@ static inline unsigned int encryption_key_version_exists(unsigned int id, unsign
   return encryption_key_get(id, version, NULL, &unused) != ENCRYPTION_KEY_VERSION_INVALID;
 }
 
+/* main entrypoint to perform encryption or decryption */
 static inline int encryption_crypt(const unsigned char* src, unsigned int slen,
                                    unsigned char* dst, unsigned int* dlen,
                                    const unsigned char* key, unsigned int klen,
@@ -109,11 +113,18 @@ static inline int encryption_crypt(const unsigned char* src, unsigned int slen,
 {
   void *ctx= alloca(encryption_ctx_size(key_id, key_version));
   int res1, res2;
-  unsigned int d1, d2;
+  unsigned int d1, d2= *dlen;
+
+  // Verify dlen is initialized properly. See MDEV-30389
+  assert(*dlen >= slen);
+  assert((dst[*dlen - 1]= 1));
+
   if ((res1= encryption_ctx_init(ctx, key, klen, iv, ivlen, flags, key_id, key_version)))
     return res1;
   res1= encryption_ctx_update(ctx, src, slen, dst, &d1);
+  d2-= d1;
   res2= encryption_ctx_finish(ctx, dst + d1, &d2);
+
   *dlen= d1 + d2;
   return res1 ? res1 : res2;
 }
@@ -124,4 +135,3 @@ static inline int encryption_crypt(const unsigned char* src, unsigned int slen,
 
 #define MYSQL_SERVICE_ENCRYPTION_INCLUDED
 #endif
-