From 85d1853e875a5cbde7da69b3899891077d44726a Mon Sep 17 00:00:00 2001
From: unknown <mattiasj@mattiasj-laptop.(none)>
Date: Mon, 12 Nov 2007 14:51:14 +0100
Subject: [PATCH] Bug#31705 Partitions: crash if varchar length > 65530

Buffer overflow due to wrong key length in partitioning

Changed to the correct key_length function.


sql/opt_range.cc:
  Bug#31705 Partitions: crash if varchar length > 65530
  Problem: buffer overflow due to wrong key-length

  Fix: Using correct key_length function
mysql-test/r/partition_datatype.result:
  Bug#31705 Partitions: crash if varchar length > 65530
  New test-result case for testing all column types
  used in key-partitioning.

  (For verifying correct key-length use)
mysql-test/t/partition_datatype.test:
  Bug#31705 Partitions: crash if varchar length > 65530
  New test case for testing all column types
  used in key-partitioning.

  (For verifying correct key-length used)
---
 mysql-test/r/partition_datatype.result | 282 +++++++++++++++++++++++++
 mysql-test/t/partition_datatype.test   | 213 +++++++++++++++++++
 sql/opt_range.cc                       |  10 +-
 3 files changed, 499 insertions(+), 6 deletions(-)
 create mode 100644 mysql-test/r/partition_datatype.result
 create mode 100644 mysql-test/t/partition_datatype.test

diff --git a/mysql-test/r/partition_datatype.result b/mysql-test/r/partition_datatype.result
new file mode 100644
index 00000000000..c6506178b03
--- /dev/null
+++ b/mysql-test/r/partition_datatype.result
@@ -0,0 +1,282 @@
+drop table if exists t1;
+create table t1 (a tinyint not null) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+a
+2
+drop table t1;
+create table t1 (a smallint not null) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+a
+2
+drop table t1;
+create table t1 (a mediumint not null) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+a
+2
+drop table t1;
+create table t1 (a int not null) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+a
+2
+drop table t1;
+create table t1 (a bigint not null) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+a
+2
+drop table t1;
+create table t1 (a float not null) partition by key (a);
+insert into t1 values (2.1);
+select * from t1 where a = 2.1;
+a
+drop table t1;
+create table t1 (a double not null) partition by key (a);
+insert into t1 values (2.1);
+select * from t1 where a = 2.1;
+a
+2.1
+drop table t1;
+create table t1 (a decimal not null) partition by key (a);
+insert into t1 values (2.1);
+Warnings:
+Note	1265	Data truncated for column 'a' at row 1
+select * from t1 where a = 2.1;
+a
+drop table t1;
+create table t1 (a date not null) partition by key (a);
+insert into t1 values ('2001-01-01');
+select * from t1 where a = '2001-01-01';
+a
+2001-01-01
+drop table t1;
+create table t1 (a datetime not null) partition by key (a);
+insert into t1 values ('2001-01-01 01:02:03');
+select * from t1 where a = '2001-01-01 01:02:03';
+a
+2001-01-01 01:02:03
+drop table t1;
+create table t1 (a timestamp not null) partition by key (a);
+insert into t1 values ('2001-01-01 01:02:03');
+select * from t1 where a = '2001-01-01 01:02:03';
+a
+2001-01-01 01:02:03
+drop table t1;
+create table t1 (a time not null) partition by key (a);
+insert into t1 values ('01:02:03');
+select * from t1 where a = '01:02:03';
+a
+01:02:03
+drop table t1;
+create table t1 (a year not null) partition by key (a);
+insert into t1 values ('2001');
+select * from t1 where a = '2001';
+a
+2001
+drop table t1;
+create table t1 (a varchar(10) character set utf8 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a varchar(300) character set utf8 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a varchar(10) character set latin1 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a varchar(300) character set latin1 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a char(10) character set utf8 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a char(10) character set latin1 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a enum('y','n') not null) partition by key (a);
+insert into t1 values ('y');
+select * from t1 where a = 'y';
+a
+y
+drop table t1;
+create table t1 (a set('y','n') not null) partition by key (a);
+insert into t1 values ('y');
+select * from t1 where a = 'y';
+a
+y
+drop table t1;
+create table t1 (a tinyint) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+a
+2
+drop table t1;
+create table t1 (a smallint) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+a
+2
+drop table t1;
+create table t1 (a mediumint) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+a
+2
+drop table t1;
+create table t1 (a int) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+a
+2
+drop table t1;
+create table t1 (a bigint) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+a
+2
+drop table t1;
+create table t1 (a float) partition by key (a);
+insert into t1 values (2.1);
+select * from t1 where a = 2.1;
+a
+drop table t1;
+create table t1 (a double) partition by key (a);
+insert into t1 values (2.1);
+select * from t1 where a = 2.1;
+a
+2.1
+drop table t1;
+create table t1 (a decimal) partition by key (a);
+insert into t1 values (2.1);
+Warnings:
+Note	1265	Data truncated for column 'a' at row 1
+select * from t1 where a = 2.1;
+a
+drop table t1;
+create table t1 (a date) partition by key (a);
+insert into t1 values ('2001-01-01');
+select * from t1 where a = '2001-01-01';
+a
+2001-01-01
+drop table t1;
+create table t1 (a datetime) partition by key (a);
+insert into t1 values ('2001-01-01 01:02:03');
+select * from t1 where a = '2001-01-01 01:02:03';
+a
+2001-01-01 01:02:03
+drop table t1;
+create table t1 (a timestamp null) partition by key (a);
+insert into t1 values ('2001-01-01 01:02:03');
+select * from t1 where a = '2001-01-01 01:02:03';
+a
+2001-01-01 01:02:03
+drop table t1;
+create table t1 (a time) partition by key (a);
+insert into t1 values ('01:02:03');
+select * from t1 where a = '01:02:03';
+a
+01:02:03
+drop table t1;
+create table t1 (a year) partition by key (a);
+insert into t1 values ('2001');
+select * from t1 where a = '2001';
+a
+2001
+drop table t1;
+create table t1 (a varchar(10) character set utf8) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a varchar(300) character set utf8) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a varchar(10) character set latin1) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a varchar(300) character set latin1) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a char(10) character set utf8) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a char(10) character set latin1) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+a
+abc
+drop table t1;
+create table t1 (a enum('y','n')) partition by key (a);
+insert into t1 values ('y');
+select * from t1 where a = 'y';
+a
+y
+drop table t1;
+create table t1 (a set('y','n')) partition by key (a);
+insert into t1 values ('y');
+select * from t1 where a = 'y';
+a
+y
+drop table t1;
+create table t1 (a varchar(65531)) partition by key (a);
+insert into t1 values ('bbbb');
+insert into t1 values ('aaaa');
+select * from t1 where a = 'aaa%';
+a
+select * from t1 where a like 'aaa%';
+a
+aaaa
+drop table t1;
+create table t1 (a varchar(65532)) partition by key (a);
+insert into t1 values ('bbbb');
+insert into t1 values ('aaaa');
+select * from t1 where a = 'aaa%';
+a
+select * from t1 where a like 'aaa%';
+a
+aaaa
+drop table t1;
+create table t1 (a varchar(65533) not null) partition by key (a);
+insert into t1 values ('aaaa');
+select * from t1 where a = 'aaa%';
+a
+drop table t1;
+create table t1 (a varchar(65533)) partition by key (a);
+ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 65535. You have to change some columns to TEXT or BLOBs
+create table t1 (a varchar(65534) not null) partition by key (a);
+ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 65535. You have to change some columns to TEXT or BLOBs
+create table t1 (a varchar(65535)) partition by key (a);
+ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 65535. You have to change some columns to TEXT or BLOBs
diff --git a/mysql-test/t/partition_datatype.test b/mysql-test/t/partition_datatype.test
new file mode 100644
index 00000000000..61d3cb42c7b
--- /dev/null
+++ b/mysql-test/t/partition_datatype.test
@@ -0,0 +1,213 @@
+#
+# Simple test for the partition storage engine
+# with most datatypes and null / not null
+# as partition by key
+# Created to verify the fix for Bug#31705
+# Partitions: crash if varchar length > 65530
+#
+-- source include/have_partition.inc
+
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+
+# FIXME: disabled this test because of valgrind error
+#create table t1 (a bit not null) partition by key (a);
+#insert into t1 values (b'1');
+#select * from t1 where a = b'1';
+#drop table t1;
+create table t1 (a tinyint not null) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+drop table t1;
+create table t1 (a smallint not null) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+drop table t1;
+create table t1 (a mediumint not null) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+drop table t1;
+create table t1 (a int not null) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+drop table t1;
+create table t1 (a bigint not null) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+drop table t1;
+create table t1 (a float not null) partition by key (a);
+insert into t1 values (2.1);
+select * from t1 where a = 2.1;
+drop table t1;
+create table t1 (a double not null) partition by key (a);
+insert into t1 values (2.1);
+select * from t1 where a = 2.1;
+drop table t1;
+create table t1 (a decimal not null) partition by key (a);
+insert into t1 values (2.1);
+select * from t1 where a = 2.1;
+drop table t1;
+create table t1 (a date not null) partition by key (a);
+insert into t1 values ('2001-01-01');
+select * from t1 where a = '2001-01-01';
+drop table t1;
+create table t1 (a datetime not null) partition by key (a);
+insert into t1 values ('2001-01-01 01:02:03');
+select * from t1 where a = '2001-01-01 01:02:03';
+drop table t1;
+create table t1 (a timestamp not null) partition by key (a);
+insert into t1 values ('2001-01-01 01:02:03');
+select * from t1 where a = '2001-01-01 01:02:03';
+drop table t1;
+create table t1 (a time not null) partition by key (a);
+insert into t1 values ('01:02:03');
+select * from t1 where a = '01:02:03';
+drop table t1;
+create table t1 (a year not null) partition by key (a);
+insert into t1 values ('2001');
+select * from t1 where a = '2001';
+drop table t1;
+create table t1 (a varchar(10) character set utf8 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a varchar(300) character set utf8 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a varchar(10) character set latin1 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a varchar(300) character set latin1 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a char(10) character set utf8 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a char(10) character set latin1 not null) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a enum('y','n') not null) partition by key (a);
+insert into t1 values ('y');
+select * from t1 where a = 'y';
+drop table t1;
+create table t1 (a set('y','n') not null) partition by key (a);
+insert into t1 values ('y');
+select * from t1 where a = 'y';
+drop table t1;
+# FIXME: disabled this test because of valgrind error
+#create table t1 (a bit) partition by key (a);
+#insert into t1 values (b'1');
+#select * from t1 where a = b'1';
+#drop table t1;
+create table t1 (a tinyint) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+drop table t1;
+create table t1 (a smallint) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+drop table t1;
+create table t1 (a mediumint) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+drop table t1;
+create table t1 (a int) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+drop table t1;
+create table t1 (a bigint) partition by key (a);
+insert into t1 values (2);
+select * from t1 where a = 2;
+drop table t1;
+create table t1 (a float) partition by key (a);
+insert into t1 values (2.1);
+select * from t1 where a = 2.1;
+drop table t1;
+create table t1 (a double) partition by key (a);
+insert into t1 values (2.1);
+select * from t1 where a = 2.1;
+drop table t1;
+create table t1 (a decimal) partition by key (a);
+insert into t1 values (2.1);
+select * from t1 where a = 2.1;
+drop table t1;
+create table t1 (a date) partition by key (a);
+insert into t1 values ('2001-01-01');
+select * from t1 where a = '2001-01-01';
+drop table t1;
+create table t1 (a datetime) partition by key (a);
+insert into t1 values ('2001-01-01 01:02:03');
+select * from t1 where a = '2001-01-01 01:02:03';
+drop table t1;
+create table t1 (a timestamp null) partition by key (a);
+insert into t1 values ('2001-01-01 01:02:03');
+select * from t1 where a = '2001-01-01 01:02:03';
+drop table t1;
+create table t1 (a time) partition by key (a);
+insert into t1 values ('01:02:03');
+select * from t1 where a = '01:02:03';
+drop table t1;
+create table t1 (a year) partition by key (a);
+insert into t1 values ('2001');
+select * from t1 where a = '2001';
+drop table t1;
+create table t1 (a varchar(10) character set utf8) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a varchar(300) character set utf8) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a varchar(10) character set latin1) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a varchar(300) character set latin1) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a char(10) character set utf8) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a char(10) character set latin1) partition by key (a);
+insert into t1 values ('abc');
+select * from t1 where a = 'abc';
+drop table t1;
+create table t1 (a enum('y','n')) partition by key (a);
+insert into t1 values ('y');
+select * from t1 where a = 'y';
+drop table t1;
+create table t1 (a set('y','n')) partition by key (a);
+insert into t1 values ('y');
+select * from t1 where a = 'y';
+drop table t1;
+create table t1 (a varchar(65531)) partition by key (a);
+insert into t1 values ('bbbb');
+insert into t1 values ('aaaa');
+select * from t1 where a = 'aaa%';
+select * from t1 where a like 'aaa%';
+drop table t1;
+create table t1 (a varchar(65532)) partition by key (a);
+insert into t1 values ('bbbb');
+insert into t1 values ('aaaa');
+select * from t1 where a = 'aaa%';
+select * from t1 where a like 'aaa%';
+drop table t1;
+create table t1 (a varchar(65533) not null) partition by key (a);
+insert into t1 values ('aaaa');
+select * from t1 where a = 'aaa%';
+drop table t1;
+-- error ER_TOO_BIG_ROWSIZE
+create table t1 (a varchar(65533)) partition by key (a);
+-- error ER_TOO_BIG_ROWSIZE
+create table t1 (a varchar(65534) not null) partition by key (a);
+-- error ER_TOO_BIG_ROWSIZE
+create table t1 (a varchar(65535)) partition by key (a);
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index 99c28be36b0..b89e8b3dcbb 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -3344,18 +3344,16 @@ static bool create_partition_index_description(PART_PRUNE_PARAM *ppar)
   {
     key_part->key=          0;
     key_part->part=	    part;
-    key_part->length=       (uint16) (*field)->pack_length_in_rec();
-    /* 
-      psergey-todo: check yet again if this is correct for tricky field types,
-      e.g. see "Fix a fatal error in decimal key handling" in open_binary_frm()
-    */
-    key_part->store_length= (uint16) (*field)->pack_length();
+    key_part->store_length= key_part->length= (uint16) (*field)->key_length();
     if ((*field)->real_maybe_null())
       key_part->store_length+= HA_KEY_NULL_LENGTH;
     if ((*field)->type() == MYSQL_TYPE_BLOB || 
         (*field)->real_type() == MYSQL_TYPE_VARCHAR)
       key_part->store_length+= HA_KEY_BLOB_LENGTH;
 
+    DBUG_PRINT("info", ("part %u length %u store_length %u", part,
+                         key_part->length, key_part->store_length));
+
     key_part->field=        (*field);
     key_part->image_type =  Field::itRAW;
     /*