From cb4fa7f401267bf887066100726c53f10b712e6d Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Wed, 5 Jan 2011 15:03:30 -0800
Subject: [PATCH] Fixed LP bug #697557. When stored in a key buffer any varchar
 field has a length prefix that always takes 2 bytes.

---
 mysql-test/r/join_cache.result | 20 ++++++++++++++++++++
 mysql-test/t/join_cache.test   | 22 ++++++++++++++++++++++
 sql/key.cc                     | 22 ++--------------------
 3 files changed, 44 insertions(+), 20 deletions(-)

diff --git a/mysql-test/r/join_cache.result b/mysql-test/r/join_cache.result
index 930be0bae6d..e660876ec1a 100644
--- a/mysql-test/r/join_cache.result
+++ b/mysql-test/r/join_cache.result
@@ -6200,4 +6200,24 @@ a	a	b
 2	NULL	42
 SET SESSION join_cache_level = DEFAULT;
 DROP TABLE t1,t2;
+#
+# Bug #697557: hash join on a varchar field 
+#
+CREATE TABLE t1 ( f1 varchar(10) , f2 int(11) , KEY (f1));
+INSERT INTO t1 VALUES ('r',1), ('m',2);
+CREATE TABLE t2 ( f1 varchar(10) , f2 int(11) , KEY (f1));
+INSERT INTO t2 VALUES
+('hgtofubn',1), ('GDOXZ',91), ('n',2), ('fggxgalh',88),
+('hgtofu',1), ('GDO',101), ('n',3), ('fggxga',55),
+('hgtofu',3), ('GDO',33), ('nn',3), ('fggxgarrr',77);
+SET SESSION join_cache_level=3;
+EXPLAIN 
+SELECT * FROM t1,t2 WHERE t2.f1 = t1.f1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	f1	NULL	NULL	NULL	2	Using where
+1	SIMPLE	t2	ref	f1	f1	13	test.t1.f1	2	Using join buffer (flat, BNLH join)
+SELECT * FROM t1,t2 WHERE t2.f1 = t1.f1;
+f1	f2	f1	f2
+SET SESSION join_cache_level = DEFAULT;
+DROP TABLE t1,t2;
 set @@optimizer_switch=@save_optimizer_switch;
diff --git a/mysql-test/t/join_cache.test b/mysql-test/t/join_cache.test
index 2f0ee8224d2..c9c93c74f46 100644
--- a/mysql-test/t/join_cache.test
+++ b/mysql-test/t/join_cache.test
@@ -2823,6 +2823,28 @@ SET SESSION join_cache_level = DEFAULT;
 
 DROP TABLE t1,t2;
   
+--echo #
+--echo # Bug #697557: hash join on a varchar field 
+--echo #
 
+CREATE TABLE t1 ( f1 varchar(10) , f2 int(11) , KEY (f1));
+INSERT INTO t1 VALUES ('r',1), ('m',2);
+
+CREATE TABLE t2 ( f1 varchar(10) , f2 int(11) , KEY (f1));
+INSERT INTO t2 VALUES
+  ('hgtofubn',1), ('GDOXZ',91), ('n',2), ('fggxgalh',88),
+  ('hgtofu',1), ('GDO',101), ('n',3), ('fggxga',55),
+  ('hgtofu',3), ('GDO',33), ('nn',3), ('fggxgarrr',77);
+
+SET SESSION join_cache_level=3;
+
+EXPLAIN 
+SELECT * FROM t1,t2 WHERE t2.f1 = t1.f1;
+SELECT * FROM t1,t2 WHERE t2.f1 = t1.f1;
+
+SET SESSION join_cache_level = DEFAULT;
+
+DROP TABLE t1,t2;
+  
 # this must be the last command in the file
 set @@optimizer_switch=@save_optimizer_switch;
diff --git a/sql/key.cc b/sql/key.cc
index 780cc6733c1..fd5c129eee8 100644
--- a/sql/key.cc
+++ b/sql/key.cc
@@ -689,20 +689,12 @@ ulong key_hashnr(KEY *key_info, uint used_key_parts, const uchar *key)
       pack_length= 0;
       break;
     case HA_KEYTYPE_VARTEXT1:
-      cs= key_part->field->charset();
-      length= (uint)(pos[0]);
-      pack_length= 1;
-      break;
-    case HA_KEYTYPE_VARBINARY1:
-      cs= &my_charset_bin;
-      length= (uint)(pos[0]);
-      pack_length= 1;
-      break;
     case HA_KEYTYPE_VARTEXT2:
       cs= key_part->field->charset();
       length= uint2korr(pos);
       pack_length= 2;
       break;
+    case HA_KEYTYPE_VARBINARY1:
     case HA_KEYTYPE_VARBINARY2:
       cs= &my_charset_bin;
       length= uint2korr(pos);
@@ -806,23 +798,13 @@ bool key_buf_cmp(KEY *key_info, uint used_key_parts,
       pack_length= 0;
       break;
     case HA_KEYTYPE_VARTEXT1:
-      cs= key_part->field->charset();
-      length1= (uint)(pos1[0]);
-      length2= (uint)(pos2[0]);
-      pack_length= 1;
-      break;
-    case HA_KEYTYPE_VARBINARY1:
-      cs= &my_charset_bin;
-      length1= (uint)(pos1[0]);
-      length2= (uint)(pos2[0]);
-      pack_length= 1;
-      break;
     case HA_KEYTYPE_VARTEXT2:
       cs= key_part->field->charset();
       length1= uint2korr(pos1);
       length2= uint2korr(pos2);
       pack_length= 2;
       break;
+    case HA_KEYTYPE_VARBINARY1:
     case HA_KEYTYPE_VARBINARY2:
       cs= &my_charset_bin;
       length1= uint2korr(pos1);