Bug#9112 - Merge table with composite index producing invalid results with some queries

The problem was an ab-use of last_rkey_length.
Formerly we saved the packed key length (of the search key)
in this element. But in certain cases it got replaced by 
the (packed) result key length.
Now we use a new element of MI_INFO to save the packed key 
length of the search key.


myisam/mi_dbug.c:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  Fixed the recognition of NULL values in _mi_print_key().
myisam/mi_rkey.c:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  Saved the packed key length in a new element of MI_INFO.
myisam/mi_search.c:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  Added a comment and trace prints.
myisam/myisamdef.h:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  Added a new element to store the packed key length
  for use by the MyISAMMRG engine.
myisammrg/myrg_rkey.c:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  Changed to use the new element of MI_INFO to get at the
  packed key length.
mysql-test/r/merge.result:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  The test result.
mysql-test/t/merge.test:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  The test case.

myisam/mi_dbug.c

@@ -40,12 +40,12 @@ void _mi_print_key(FILE *stream, register MI_KEYSEG *keyseg,
     end= key+ keyseg->length;
     if (keyseg->flag & HA_NULL_PART)
     {
-      if (!*key)
+      /* A NULL value is encoded by a 1-byte flag. Zero means NULL. */
+      if (! *(key++))
       {
 	fprintf(stream,"NULL");
 	continue;
       }
-      key++;
     }
 
     switch (keyseg->type) {

myisam/mi_rkey.c

@@ -31,8 +31,8 @@ int mi_rkey(MI_INFO *info, byte *buf, int inx, const byte *key, uint key_len,
   MI_KEYSEG *last_used_keyseg;
   uint pack_key_length, use_key_length, nextflag;
   DBUG_ENTER("mi_rkey");
-  DBUG_PRINT("enter",("base: %lx  inx: %d  search_flag: %d",
+  DBUG_PRINT("enter", ("base: %p  buf: %p  inx: %d  search_flag: %d",
-		      info,inx,search_flag));
+                       info, buf, inx, search_flag));
 
   if ((inx = _mi_check_index(info,inx)) < 0)
     DBUG_RETURN(my_errno);
@@ -44,9 +44,12 @@ int mi_rkey(MI_INFO *info, byte *buf, int inx, const byte *key, uint key_len,
   {
     if (key_len == 0)
       key_len=USE_WHOLE_KEY;
+    /* Save the packed key for later use in the second buffer of lastkey. */
     key_buff=info->lastkey+info->s->base.max_key_length;
     pack_key_length=_mi_pack_key(info, (uint) inx, key_buff, (uchar*) key,
 				 key_len, &last_used_keyseg);
+    /* Save packed_key_length for use by the MERGE engine. */
+    info->pack_key_length= pack_key_length;
     DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE, keyinfo->seg,
 				     key_buff, pack_key_length););
   }

myisam/mi_search.c

@@ -1237,11 +1237,21 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
   reg1 MI_KEYSEG *keyseg;
   uchar *start_key,*page,*page_end,*from,*from_end;
   uint length,tmp;
+  DBUG_ENTER("_mi_get_binary_pack_key");
 
   page= *page_pos;
   page_end=page+MI_MAX_KEY_BUFF+1;
   start_key=key;
 
+  /*
+    Keys are compressed the following way:
+
+    prefix length    Packed length of prefix for the prev key. (1 or 3 bytes)
+    for each key segment:
+      [is null]        Null indicator if can be null (1 byte, zero means null)
+      [length]         Packed length if varlength (1 or 3 bytes)
+    pointer          Reference to the data file (last_keyseg->length).
+  */
   get_key_length(length,page);
   if (length)
   {
@@ -1251,7 +1261,7 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
                           length, keyinfo->maxlength, *page_pos));
       DBUG_DUMP("key",(char*) *page_pos,16);
       my_errno=HA_ERR_CRASHED;
-      return 0;                                 /* Wrong key */
+      DBUG_RETURN(0);                                 /* Wrong key */
     }
     from=key;  from_end=key+length;
   }
@@ -1312,12 +1322,12 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
     {
       DBUG_PRINT("error",("Error when unpacking key"));
       my_errno=HA_ERR_CRASHED;
-      return 0;                                 /* Error */
+      DBUG_RETURN(0);                                 /* Error */
     }
     memcpy((byte*) key,(byte*) from,(size_t) length);
     *page_pos= from+length;
   }
-  return((uint) (key-start_key)+keyseg->length);
+  DBUG_RETURN((uint) (key-start_key)+keyseg->length);
 }
 
 

myisam/myisamdef.h

@@ -255,6 +255,7 @@ struct st_myisam_info {
   uint	lastkey_length;			/* Length of key in lastkey */
   uint	last_rkey_length;		/* Last length in mi_rkey() */
   uint  save_lastkey_length;
+  uint  pack_key_length;                /* For MYISAMMRG */
   int	errkey;				/* Got last error on this key */
   int   lock_type;			/* How database was locked */
   int   tmp_lock_type;			/* When locked by readinfo */

myisammrg/myrg_rkey.c

@@ -44,11 +44,12 @@ int myrg_rkey(MYRG_INFO *info,byte *buf,int inx, const byte *key,
   MYRG_TABLE *table;
   MI_INFO *mi;
   int err;
+  DBUG_ENTER("myrg_rkey");
   LINT_INIT(key_buff);
   LINT_INIT(pack_key_length);
 
   if (_myrg_init_queue(info,inx,search_flag))
-    return my_errno;
+    DBUG_RETURN(my_errno);
 
   for (table=info->open_tables ; table != info->end_table ; table++)
   {
@@ -57,8 +58,9 @@ int myrg_rkey(MYRG_INFO *info,byte *buf,int inx, const byte *key,
     if (table == info->open_tables)
     {
       err=mi_rkey(mi,0,inx,key,key_len,search_flag);
+      /* Get the saved packed key and packed key length. */
       key_buff=(byte*) mi->lastkey+mi->s->base.max_key_length;
-      pack_key_length=mi->last_rkey_length;
+      pack_key_length=mi->pack_key_length;
     }
     else
     {
@@ -72,16 +74,21 @@ int myrg_rkey(MYRG_INFO *info,byte *buf,int inx, const byte *key,
     {
       if (err == HA_ERR_KEY_NOT_FOUND)
 	continue;
-      return err;
+      DBUG_PRINT("exit", ("err: %d", err));
+      DBUG_RETURN(err);
     }
     /* adding to queue */
     queue_insert(&(info->by_key),(byte *)table);
 
   }
 
+  DBUG_PRINT("info", ("tables with matches: %u", info->by_key.elements));
   if (!info->by_key.elements)
-    return HA_ERR_KEY_NOT_FOUND;
+    DBUG_RETURN(HA_ERR_KEY_NOT_FOUND);
 
   mi=(info->current_table=(MYRG_TABLE *)queue_top(&(info->by_key)))->table;
-  return _myrg_mi_read_record(mi,buf);
+  DBUG_PRINT("info", ("using table no: %d",
+                      info->current_table - info->open_tables + 1));
+  DBUG_DUMP("result key", (byte*) mi->lastkey, mi->lastkey_length);
+  DBUG_RETURN(_myrg_mi_read_record(mi,buf));
 }

mysql-test/r/merge.result

@@ -619,3 +619,29 @@ INSERT TABLE 't1' isn't allowed in FROM table list
 create table t3 engine=merge union=(t1, t2) select * from t2;
 INSERT TABLE 't2' isn't allowed in FROM table list
 drop table t1, t2;
+create table t1 (
+a double(16,6),
+b varchar(10),
+index (a,b)
+) engine=merge union=(t2,t3);
+create table t2 (
+a double(16,6),
+b varchar(10),
+index (a,b)
+) engine=myisam;
+create table t3 (
+a double(16,6),
+b varchar(10),
+index (a,b)
+) engine=myisam;
+insert into t2 values ( null, '');
+insert into t2 values ( 9999999999.999999, '');
+insert into t3 select * from t2;
+select min(a), max(a) from t1;
+min(a)	max(a)
+9999999999.999998	9999999999.999998
+flush tables;
+select min(a), max(a) from t1;
+min(a)	max(a)
+9999999999.999998	9999999999.999998
+drop table t1, t2, t3;

mysql-test/t/merge.test

@@ -264,3 +264,36 @@ create table t3 engine=merge union=(t1, t2) select * from t1;
 --error 1093
 create table t3 engine=merge union=(t1, t2) select * from t2;
 drop table t1, t2;
+
+#
+# Bug#9112 - Merge table with composite index producing invalid results with some queries
+# This test case will fail only without the bugfix and some
+# non-deterministic circumstances. It depends on properly initialized
+# "un-initialized" memory. At the time it happens with a standard
+# non-debug build. But there is no guarantee that this will be always so.
+#
+create table t1 (
+ a double(16,6),
+ b varchar(10),
+ index (a,b)
+) engine=merge union=(t2,t3);
+
+create table t2 (
+ a double(16,6),
+ b varchar(10),
+ index (a,b)
+) engine=myisam;
+
+create table t3 (
+ a double(16,6),
+ b varchar(10),
+ index (a,b)
+) engine=myisam;
+
+insert into t2 values ( null, '');
+insert into t2 values ( 9999999999.999999, '');
+insert into t3 select * from t2;
+select min(a), max(a) from t1;
+flush tables;
+select min(a), max(a) from t1;
+drop table t1, t2, t3;