Fix crash caused by calling DES_ENCRYPT() without the --des-key-file

option having been passed to the server. (Bug #11643)

BitKeeper/etc/config

@@ -24,7 +24,7 @@ description: MySQL - fast and reliable SQL database
 # repository is commercial it can be an internal email address or "none"
 # to disable logging.
 # 
-logging:  logging@openlogging.org
+logging: none
 # 
 # If this field is set, all checkins will appear to be made by this user,
 # in effect making this a single user package.  Single user packages are

mysql-test/r/func_des_encrypt.result

@@ -0,0 +1,3 @@
+select des_encrypt('hello');
+des_encrypt('hello')
+<EFBFBD><EFBFBD>2nV<6E><56>}

mysql-test/t/func_des_encrypt.test

@@ -0,0 +1,9 @@
+-- source include/have_openssl.inc
+
+# This test can't be in func_encrypt.test, because it requires
+# --des-key-file to not be set.
+
+#
+# Bug #11643: des_encrypt() causes server to die
+#
+select des_encrypt('hello');

sql/des_key_file.cc

@@ -22,7 +22,17 @@
 struct st_des_keyschedule des_keyschedule[10];
 uint   des_default_key;
 pthread_mutex_t LOCK_des_key_file;
-static int initialized;
+static int initialized= 0;
+
+void
+init_des_key_file()
+{
+  if (!initialized)
+  {
+    initialized=1;
+    pthread_mutex_init(&LOCK_des_key_file,MY_MUTEX_INIT_FAST);
+  }
+}
 
 /*
  Function which loads DES keys from plaintext file into memory on MySQL
@@ -45,11 +55,7 @@ load_des_key_file(const char *file_name)
   DBUG_ENTER("load_des_key_file");
   DBUG_PRINT("enter",("name: %s",file_name));
 
-  if (!initialized)
+  init_des_key_file();
-  {
-    initialized=1;
-    pthread_mutex_init(&LOCK_des_key_file,MY_MUTEX_INIT_FAST);
-  }
 
   VOID(pthread_mutex_lock(&LOCK_des_key_file));
   if ((file=my_open(file_name,O_RDONLY | O_BINARY ,MYF(MY_WME))) < 0 ||

sql/item_strfunc.cc

@@ -388,6 +388,9 @@ String *Item_func_des_encrypt::val_str(String *str)
 
   if (arg_count == 1)
   {
+    /* Make sure LOCK_des_key_file was initialized. */
+    init_des_key_file();
+
     /* Protect against someone doing FLUSH DES_KEY_FILE */
     VOID(pthread_mutex_lock(&LOCK_des_key_file));
     keyschedule= des_keyschedule[key_number=des_default_key];
@@ -398,6 +401,10 @@ String *Item_func_des_encrypt::val_str(String *str)
     key_number= (uint) args[1]->val_int();
     if (key_number > 9)
       goto error;
+
+    /* Make sure LOCK_des_key_file was initialized. */
+    init_des_key_file();
+
     VOID(pthread_mutex_lock(&LOCK_des_key_file));
     keyschedule= des_keyschedule[key_number];
     VOID(pthread_mutex_unlock(&LOCK_des_key_file));
@@ -485,6 +492,10 @@ String *Item_func_des_decrypt::val_str(String *str)
     // Check if automatic key and that we have privilege to uncompress using it
     if (!(current_thd->master_access & SUPER_ACL) || key_number > 9)
       goto error;
+
+    /* Make sure LOCK_des_key_file was initialized. */
+    init_des_key_file();
+
     VOID(pthread_mutex_lock(&LOCK_des_key_file));
     keyschedule= des_keyschedule[key_number];
     VOID(pthread_mutex_unlock(&LOCK_des_key_file));

sql/mysql_priv.h

@@ -624,6 +624,7 @@ extern char *des_key_file;
 extern struct st_des_keyschedule des_keyschedule[10];
 extern uint des_default_key;
 extern pthread_mutex_t LOCK_des_key_file;
+void init_des_key_file();
 bool load_des_key_file(const char *file_name);
 void free_des_key_file();
 #endif /* HAVE_OPENSSL */