From 660d66263ba1e9c5ec197cc29ddc425b200b58ca Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 24 Oct 2007 16:09:30 +0500 Subject: [PATCH 01/12] BUG#31159 - fulltext search on ucs2 column crashes server ucs2 doesn't provide required by fulltext ctype array. Crash happens because fulltext attempts to use unitialized ctype array. Fixed by converting ucs2 fields to compatible utf8 analogue. include/my_sys.h: Added a function to find compatible character set with ctype array available. Currently used by fulltext search to find compatible substitute for ucs2 collations. mysql-test/r/ctype_ucs.result: A test case for BUG#31159. mysql-test/t/ctype_ucs.test: A test case for BUG#31159. mysys/charset.c: Added a function to find compatible character set with ctype array available. Currently used by fulltext search to find compatible substitute for ucs2 collations. sql/item_func.cc: Convert ucs2 fields to utf8. Fulltext requires ctype array, but ucs2 doesn't provide it. --- include/my_sys.h | 2 ++ mysql-test/r/ctype_ucs.result | 6 ++++++ mysql-test/t/ctype_ucs.test | 8 +++++++ mysys/charset.c | 40 +++++++++++++++++++++++++++++++++++ sql/item_func.cc | 33 ++++++++++++++++++++++++++++- 5 files changed, 88 insertions(+), 1 deletion(-) diff --git a/include/my_sys.h b/include/my_sys.h index 759531fa649..4a0586b9f2d 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -784,6 +784,8 @@ extern CHARSET_INFO *get_charset(uint cs_number, myf flags); extern CHARSET_INFO *get_charset_by_name(const char *cs_name, myf flags); extern CHARSET_INFO *get_charset_by_csname(const char *cs_name, uint cs_flags, myf my_flags); +extern CHARSET_INFO *get_compatible_charset_with_ctype(CHARSET_INFO + *original_cs); extern void free_charsets(void); extern char *get_charsets_dir(char *buf); extern my_bool my_charset_same(CHARSET_INFO *cs1, CHARSET_INFO *cs2); diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result index bf827209795..fef13b19ae8 100644 --- a/mysql-test/r/ctype_ucs.result +++ b/mysql-test/r/ctype_ucs.result @@ -803,4 +803,10 @@ quote(name) ???????? ???????????????? drop table bug20536; +CREATE TABLE t1(a TEXT CHARSET ucs2 COLLATE ucs2_unicode_ci); +INSERT INTO t1 VALUES('abcd'); +SELECT * FROM t1 WHERE MATCH(a) AGAINST ('+abcd' IN BOOLEAN MODE); +a +abcd +DROP TABLE t1; End of 4.1 tests diff --git a/mysql-test/t/ctype_ucs.test b/mysql-test/t/ctype_ucs.test index 10559d33eb3..57f741597e0 100644 --- a/mysql-test/t/ctype_ucs.test +++ b/mysql-test/t/ctype_ucs.test @@ -535,4 +535,12 @@ select quote(name) from bug20536; drop table bug20536; +# +# BUG#31159 - fulltext search on ucs2 column crashes server +# +CREATE TABLE t1(a TEXT CHARSET ucs2 COLLATE ucs2_unicode_ci); +INSERT INTO t1 VALUES('abcd'); +SELECT * FROM t1 WHERE MATCH(a) AGAINST ('+abcd' IN BOOLEAN MODE); +DROP TABLE t1; + --echo End of 4.1 tests diff --git a/mysys/charset.c b/mysys/charset.c index 6f2d4d3c347..f0ac61ceed5 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -673,3 +673,43 @@ CHARSET_INFO *fs_character_set() return fs_cset_cache; } #endif + + +/** + @brief Find compatible character set with ctype. + + @param[in] original_cs Original character set + + @note + 128 my_charset_ucs2_general_uca ->192 my_charset_utf8_general_uca_ci + 129 my_charset_ucs2_icelandic_uca_ci ->193 my_charset_utf8_icelandic_uca_ci + 130 my_charset_ucs2_latvian_uca_ci ->194 my_charset_utf8_latvian_uca_ci + 131 my_charset_ucs2_romanian_uca_ci ->195 my_charset_utf8_romanian_uca_ci + 132 my_charset_ucs2_slovenian_uca_ci ->196 my_charset_utf8_slovenian_uca_ci + 133 my_charset_ucs2_polish_uca_ci ->197 my_charset_utf8_polish_uca_ci + 134 my_charset_ucs2_estonian_uca_ci ->198 my_charset_utf8_estonian_uca_ci + 135 my_charset_ucs2_spanish_uca_ci ->199 my_charset_utf8_spanish_uca_ci + 136 my_charset_ucs2_swedish_uca_ci ->200 my_charset_utf8_swedish_uca_ci + 137 my_charset_ucs2_turkish_uca_ci ->201 my_charset_utf8_turkish_uca_ci + 138 my_charset_ucs2_czech_uca_ci ->202 my_charset_utf8_czech_uca_ci + 139 my_charset_ucs2_danish_uca_ci ->203 my_charset_utf8_danish_uca_ci + 140 my_charset_ucs2_lithuanian_uca_ci->204 my_charset_utf8_lithuanian_uca_ci + 141 my_charset_ucs2_slovak_uca_ci ->205 my_charset_utf8_slovak_uca_ci + 142 my_charset_ucs2_spanish2_uca_ci ->206 my_charset_utf8_spanish2_uca_ci + 143 my_charset_ucs2_roman_uca_ci ->207 my_charset_utf8_roman_uca_ci + 144 my_charset_ucs2_persian_uca_ci ->208 my_charset_utf8_persian_uca_ci + + @return Compatible character set or NULL. +*/ + +CHARSET_INFO *get_compatible_charset_with_ctype(CHARSET_INFO *original_cs) +{ + CHARSET_INFO *compatible_cs= 0; + DBUG_ENTER("get_compatible_charset_with_ctype"); + if (!strcmp(original_cs->csname, "ucs2") && + (compatible_cs= get_charset(original_cs->number + 64, MYF(0))) && + (!compatible_cs->ctype || + strcmp(original_cs->name + 4, compatible_cs->name + 4))) + compatible_cs= 0; + DBUG_RETURN(compatible_cs); +} diff --git a/sql/item_func.cc b/sql/item_func.cc index f71297515d6..6bfb920d7c8 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -3135,13 +3135,44 @@ bool Item_func_match::fix_fields(THD *thd, TABLE_LIST *tlist, Item **ref) my_error(ER_WRONG_ARGUMENTS,MYF(0),"MATCH"); return 1; } - table=((Item_field *)item)->field->table; + /* + With prepared statements Item_func_match::fix_fields is called twice. + When it is called first time we have original item tree here and add + conversion layer for character sets that do not have ctype array a few + lines below. When it is called second time, we already have conversion + layer in item tree. + */ + table= (item->type() == Item::FIELD_ITEM) ? + ((Item_field *)item)->field->table : + ((Item_field *)((Item_func_conv *)item)->key_item())->field->table; if (!(table->file->table_flags() & HA_CAN_FULLTEXT)) { my_error(ER_TABLE_CANT_HANDLE_FT, MYF(0)); return 1; } table->fulltext_searched=1; + /* A workaround for ucs2 character set */ + if (!args[1]->collation.collation->ctype) + { + CHARSET_INFO *compatible_cs= + get_compatible_charset_with_ctype(args[1]->collation.collation); + bool rc= 1; + if (compatible_cs) + { + Item_string *conv_item= new Item_string("", 0, compatible_cs, + DERIVATION_EXPLICIT); + item= args[0]; + args[0]= conv_item; + rc= agg_item_charsets(cmp_collation, func_name(), args, arg_count, + MY_COLL_ALLOW_SUPERSET_CONV | + MY_COLL_ALLOW_COERCIBLE_CONV | + MY_COLL_DISALLOW_NONE); + args[0]= item; + } + else + my_error(ER_WRONG_ARGUMENTS, MYF(0), "MATCH"); + return rc; + } return agg_arg_collations_for_comparison(cmp_collation, args+1, arg_count-1); } From 0724eb88e2540cdbecb3c58a02dc79dba2358359 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 25 Oct 2007 17:23:12 -0700 Subject: [PATCH 02/12] protect bug31473 from regressing into 5.0 --- mysql-test/r/csv.result | 42 +++++++++++++++++++++++++++++++++++++++++ mysql-test/t/csv.test | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/mysql-test/r/csv.result b/mysql-test/r/csv.result index 3900597d2a6..dca4e349c8a 100644 --- a/mysql-test/r/csv.result +++ b/mysql-test/r/csv.result @@ -5029,4 +5029,46 @@ F7 FE þ LATIN SMALL LETTER THORN FF ÿ LATIN SMALL LETTER Y WITH DIAERESIS drop table t1; +create table t1(a datetime) engine=csv; +insert into t1 values(); +select * from t1; +a +0000-00-00 00:00:00 +drop table t1; +create table t1(a set('foo','bar')) engine=csv; +insert into t1 values(); +select * from t1; +a + +drop table t1; +create table t1(a varchar(32)) engine=csv; +insert into t1 values(); +select * from t1; +a + +drop table t1; +create table t1(a int) engine=csv; +insert into t1 values(); +select * from t1; +a +0 +drop table t1; +create table t1(a blob) engine=csv; +insert into t1 values(); +select * from t1; +a + +drop table t1; +create table t1(a bit(1)) engine=csv; +insert into t1 values(); +select BIN(a) from t1; +BIN(a) +0 +drop table t1; +create table t1(a enum('foo','bar') default 'foo') engine=csv; +insert into t1 values(); +select * from t1; +a +foo +drop table t1; End of 5.0 tests diff --git a/mysql-test/t/csv.test b/mysql-test/t/csv.test index b724b4ce47c..db5cb92c3e6 100644 --- a/mysql-test/t/csv.test +++ b/mysql-test/t/csv.test @@ -1427,4 +1427,37 @@ insert into t1 values (0xFF,'LATIN SMALL LETTER Y WITH DIAERESIS'); select hex(c), c, name from t1 order by 1; drop table t1; +# +# Bug #31473: does not work with NULL value in datetime field +# This bug is a 5.1 but is here to prevent 5.0 regression. +# +create table t1(a datetime) engine=csv; +insert into t1 values(); +select * from t1; +drop table t1; +create table t1(a set('foo','bar')) engine=csv; +insert into t1 values(); +select * from t1; +drop table t1; +create table t1(a varchar(32)) engine=csv; +insert into t1 values(); +select * from t1; +drop table t1; +create table t1(a int) engine=csv; +insert into t1 values(); +select * from t1; +drop table t1; +create table t1(a blob) engine=csv; +insert into t1 values(); +select * from t1; +drop table t1; +create table t1(a bit(1)) engine=csv; +insert into t1 values(); +select BIN(a) from t1; +drop table t1; +create table t1(a enum('foo','bar') default 'foo') engine=csv; +insert into t1 values(); +select * from t1; +drop table t1; + --echo End of 5.0 tests From b698b6fd945e5cd0e469e7494518b0a482f61e6e Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 30 Oct 2007 14:46:43 +0400 Subject: [PATCH 03/12] BUG#11392 - fulltext search bug Fulltext boolean mode phrase search may crash server on platforms where size of pointer is not equal to size of unsigned integer (in other words some 64-bit platforms). The problem was integer overflow. Affects 4.1 only. myisam/ft_boolean_search.c: my_match_t::beg is unsigned int, that means type of expression (m[0].beg - 1) has unsigned type too. It may happen that instr() finds substring in the beggining of passed string, returning m[0].beg equal to 0. In this case value of expression (m[0].beg - 1) is equal to MAX_UINT. This is not a problem on platforms where sizeof(pointer) equals to sizeof(uint). That means ptr[(uint)-1] = ptr[(uint)MAX_UINT] = ptr - 1. On some 64-bit platforms where sizeof(pointer) is 8 and sizeof(uint) is 4, wrong address gets accessed. In other words ptr[(uint)-1] is equal to ptr + MAX_UINT. mysql-test/r/fulltext.result: A test case for BUG#11392. mysql-test/t/fulltext.test: A test case for BUG#11392. --- myisam/ft_boolean_search.c | 3 ++- mysql-test/r/fulltext.result | 6 ++++++ mysql-test/t/fulltext.test | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/myisam/ft_boolean_search.c b/myisam/ft_boolean_search.c index f1ff8f6d886..fad25abcc6c 100644 --- a/myisam/ft_boolean_search.c +++ b/myisam/ft_boolean_search.c @@ -446,7 +446,8 @@ static int _ftb_strstr(const byte *s0, const byte *e0, { if (cs->coll->instr(cs, p0, e0 - p0, s1, e1 - s1, m, 2) != 2) return(0); - if ((!s_after || p0 + m[1].beg == s0 || !true_word_char(cs, p0[m[1].beg-1])) && + if ((!s_after || p0 + m[1].beg == s0 || + !true_word_char(cs, p0[(int) m[1].beg - 1])) && (!e_before || p0 + m[1].end == e0 || !true_word_char(cs, p0[m[1].end]))) return(1); p0+= m[1].beg; diff --git a/mysql-test/r/fulltext.result b/mysql-test/r/fulltext.result index 3700ace4b19..af41adf3a24 100644 --- a/mysql-test/r/fulltext.result +++ b/mysql-test/r/fulltext.result @@ -454,3 +454,9 @@ ALTER TABLE t1 DISABLE KEYS; SELECT * FROM t1 WHERE MATCH(a) AGAINST('test'); ERROR HY000: Can't find FULLTEXT index matching the column list DROP TABLE t1; +CREATE TABLE t1(a TEXT); +INSERT INTO t1 VALUES(' aaaaa aaaa'); +SELECT * FROM t1 WHERE MATCH(a) AGAINST ('"aaaa"' IN BOOLEAN MODE); +a + aaaaa aaaa +DROP TABLE t1; diff --git a/mysql-test/t/fulltext.test b/mysql-test/t/fulltext.test index 1a9a6b578dc..661e93d8d87 100644 --- a/mysql-test/t/fulltext.test +++ b/mysql-test/t/fulltext.test @@ -379,4 +379,12 @@ ALTER TABLE t1 DISABLE KEYS; SELECT * FROM t1 WHERE MATCH(a) AGAINST('test'); DROP TABLE t1; +# +# BUG#11392 - fulltext search bug +# +CREATE TABLE t1(a TEXT); +INSERT INTO t1 VALUES(' aaaaa aaaa'); +SELECT * FROM t1 WHERE MATCH(a) AGAINST ('"aaaa"' IN BOOLEAN MODE); +DROP TABLE t1; + # End of 4.1 tests From db4ef35a4f833718da76f4ca239636086bde1903 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 1 Nov 2007 16:27:01 +0400 Subject: [PATCH 04/12] BUG#31950 - repair table hangs while processing multicolumn utf8 fulltext index Having a table with broken multibyte characters may cause fulltext parser dead-loop. Since normally it is not possible to insert broken multibyte sequence into a table, this problem may arise only if table is damaged. Affected statements are: - CHECK/REPAIR against damaged table with fulltext index; - boolean mode phrase search against damaged table with or without fulltext inex; - boolean mode searches without index; - nlq searches. No test case for this fix. Affects 5.0 only. myisam/ft_parser.c: When skipping leading spaces, skip broken characters as well (broken characters a identified by mbl == 0). --- myisam/ft_parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myisam/ft_parser.c b/myisam/ft_parser.c index 6d68542e4e2..1d3a19dd8c6 100644 --- a/myisam/ft_parser.c +++ b/myisam/ft_parser.c @@ -188,7 +188,7 @@ byte ft_simple_get_word(CHARSET_INFO *cs, byte **start, const byte *end, do { - for (;; doc+= mbl) + for (;; doc+= (mbl ? mbl : 1)) { if (doc >= end) DBUG_RETURN(0); if (true_word_char(cs, *doc)) break; From d787eb9541cec4be2310f43628a3ad677d3e243e Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 1 Nov 2007 15:03:09 +0100 Subject: [PATCH 05/12] Bug#31909 - New gis.test creates warnings files Comment sign of -- at line begin in test files lead to warnings from mysqltest. Changed -- to #. mysql-test/include/gis_keys.inc: Bug#31909 - New gis.test creates warnings files Changed -- to # at comment begin to avoid warnings files. --- mysql-test/include/gis_keys.inc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mysql-test/include/gis_keys.inc b/mysql-test/include/gis_keys.inc index 295e0c48234..c75311f062a 100644 --- a/mysql-test/include/gis_keys.inc +++ b/mysql-test/include/gis_keys.inc @@ -13,20 +13,20 @@ CREATE TABLE t2 (p POINT, INDEX(p)); INSERT INTO t1 VALUES (POINTFROMTEXT('POINT(1 2)')); INSERT INTO t2 VALUES (POINTFROMTEXT('POINT(1 2)')); --- no index, returns 1 as expected +# no index, returns 1 as expected SELECT COUNT(*) FROM t1 WHERE p=POINTFROMTEXT('POINT(1 2)'); --- with index, returns 1 as expected --- EXPLAIN shows that the index is not used though --- due to the "most rows covered anyway, so a scan is more effective" rule +# with index, returns 1 as expected +# EXPLAIN shows that the index is not used though +# due to the "most rows covered anyway, so a scan is more effective" rule EXPLAIN SELECT COUNT(*) FROM t2 WHERE p=POINTFROMTEXT('POINT(1 2)'); SELECT COUNT(*) FROM t2 WHERE p=POINTFROMTEXT('POINT(1 2)'); --- adding another row to the table so that --- the "most rows covered" rule doesn't kick in anymore --- now EXPLAIN shows the index used on the table --- and we're getting the wrong result again +# adding another row to the table so that +# the "most rows covered" rule doesn't kick in anymore +# now EXPLAIN shows the index used on the table +# and we're getting the wrong result again INSERT INTO t1 VALUES (POINTFROMTEXT('POINT(1 2)')); INSERT INTO t2 VALUES (POINTFROMTEXT('POINT(1 2)')); EXPLAIN From 7b6b909ac04aca9bfd2ede8b0ec4aa70c2899a6f Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 1 Nov 2007 15:04:23 +0100 Subject: [PATCH 06/12] Post-merge fix --- mysql-test/t/variables.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/t/variables.test b/mysql-test/t/variables.test index 63bc3c5e273..b661d0e8ae7 100644 --- a/mysql-test/t/variables.test +++ b/mysql-test/t/variables.test @@ -139,7 +139,7 @@ show global variables like 'net_%'; show session variables like 'net_%'; set net_buffer_length=1; show variables like 'net_buffer_length'; ---warning 1292 +#warning 1292 set net_buffer_length=2000000000; show variables like 'net_buffer_length'; From 6f97c22925f074db15d39b9e8a08d8fd9623af82 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 2 Nov 2007 10:11:26 +0100 Subject: [PATCH 07/12] Bug#31030 - rpl000015.test fails if $MYSQL_TCP_PORT != 3306 Preliminarily disabled test case --- mysql-test/t/disabled.def | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/t/disabled.def b/mysql-test/t/disabled.def index 9bfe9567d83..80b28e11da6 100644 --- a/mysql-test/t/disabled.def +++ b/mysql-test/t/disabled.def @@ -10,3 +10,4 @@ # ############################################################################## +rpl000015 : Bug#31030 - rpl000015.test fails if $MYSQL_TCP_PORT != 3306 From abd6ff862251b1228f011acaf725458494a4294e Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 5 Nov 2007 14:37:00 +0100 Subject: [PATCH 08/12] Bug#32107 - ctype_uca.test produces warnings files Comment sign of -- at line begin in test files lead to warnings from mysqltest. Changed -- to #. mysql-test/t/ctype_uca.test: Bug#32107 - ctype_uca.test produces warnings files Changed -- to # at comment begin to avoid warnings files. --- mysql-test/t/ctype_uca.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/t/ctype_uca.test b/mysql-test/t/ctype_uca.test index 17632e7c8fc..695a21adbf5 100644 --- a/mysql-test/t/ctype_uca.test +++ b/mysql-test/t/ctype_uca.test @@ -530,7 +530,7 @@ create table t1 ( a varchar(255), key a(a) ) character set utf8 collate utf8_czech_ci; --- In Czech 'ch' is a single letter between 'h' and 'i' +# In Czech 'ch' is a single letter between 'h' and 'i' insert into t1 values ('b'),('c'),('d'),('e'),('f'),('g'),('h'),('ch'),('i'),('j'); select * from t1 where a like 'c%'; From 6f12c924392e0d9ec480f60a7bcc1f6ea87e871b Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 5 Nov 2007 14:44:38 +0100 Subject: [PATCH 09/12] Bug#32108 - subselect.test produces warnings files Comment sign of -- at line begin in test files lead to warnings from mysqltest. Changed -- to #. mysql-test/t/subselect.test: Bug#32108 - subselect.test produces warnings files Changed -- to # at comment begin to avoid warnings files. --- mysql-test/t/subselect.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index d076ca6bd33..86d2aec870c 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -2970,7 +2970,7 @@ DROP TABLE t1,t2; CREATE TABLE t1 (a INT, b INT); INSERT INTO t1 VALUES (1, 2), (1,3), (1,4), (2,1), (2,2); --- returns no rows, when it should +# returns no rows, when it should SELECT a1.a, COUNT(*) FROM t1 a1 WHERE a1.a = 1 AND EXISTS( SELECT a2.a FROM t1 a2 WHERE a2.a = a1.a) GROUP BY a1.a; From 42829c61b6a912b51e4cebf8697307cbddc7bffd Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 6 Nov 2007 13:41:32 +0100 Subject: [PATCH 10/12] Bug#4692 - DISABLE/ENABLE KEYS waste a space Disabling and enabling indexes on a non-empty table grows the index file. Disabling indexes just sets a flag per non-unique index and does not free the index blocks of the affected indexes. Re-enabling indexes creates new indexes with new blocks. The old blocks remain unused in the index file. Fixed by dropping and re-creating all indexes if non-empty disabled indexes exist when enabling indexes. Dropping all indexes resets the internal end-of-file marker to the end of the index file header. It also clears the root block pointers of every index and clears the deleted blocks chains. This way all blocks are declared as free. myisam/mi_check.c: Bug#4692 - DISABLE/ENABLE KEYS waste a space Added function mi_drop_all_indexes() to support drop of all indexes in case we want to re-enable non-empty disabled indexes. Changed mi_repair(), mi_repair_by_sort(), and mi_repair_parallel() to use the new function instead of duplicate drop index code. mysql-test/r/myisam.result: Bug#4692 - DISABLE/ENABLE KEYS waste a space Added test result. mysql-test/t/myisam.test: Bug#4692 - DISABLE/ENABLE KEYS waste a space Added test. --- myisam/mi_check.c | 224 +++++++++++++++++++++++++++---------- mysql-test/r/myisam.result | 25 +++++ mysql-test/t/myisam.test | 26 +++++ 3 files changed, 218 insertions(+), 57 deletions(-) diff --git a/myisam/mi_check.c b/myisam/mi_check.c index ed0a84e737d..ba308f75d24 100644 --- a/myisam/mi_check.c +++ b/myisam/mi_check.c @@ -1375,6 +1375,139 @@ int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend) } /* chk_data_link */ +/** + @brief Drop all indexes + + @param[in] param check parameters + @param[in] info MI_INFO handle + @param[in] force if to force drop all indexes + + @return status + @retval 0 OK + @retval != 0 Error + + @note + Once allocated, index blocks remain part of the key file forever. + When indexes are disabled, no block is freed. When enabling indexes, + no block is freed either. The new indexes are create from new + blocks. (Bug #4692) + + Before recreating formerly disabled indexes, the unused blocks + must be freed. There are two options to do this: + - Follow the tree of disabled indexes, add all blocks to the + deleted blocks chain. Would require a lot of random I/O. + - Drop all blocks by clearing all index root pointers and all + delete chain pointers and resetting key_file_length to the end + of the index file header. This requires to recreate all indexes, + even those that may still be intact. + The second method is probably faster in most cases. + + When disabling indexes, MySQL disables either all indexes or all + non-unique indexes. When MySQL [re-]enables disabled indexes + (T_CREATE_MISSING_KEYS), then we either have "lost" blocks in the + index file, or there are no non-unique indexes. In the latter case, + mi_repair*() would not be called as there would be no disabled + indexes. + + If there would be more unique indexes than disabled (non-unique) + indexes, we could do the first method. But this is not implemented + yet. By now we drop and recreate all indexes when repair is called. + + However, there is an exception. Sometimes MySQL disables non-unique + indexes when the table is empty (e.g. when copying a table in + mysql_alter_table()). When enabling the non-unique indexes, they + are still empty. So there is no index block that can be lost. This + optimization is implemented in this function. + + Note that in normal repair (T_CREATE_MISSING_KEYS not set) we + recreate all enabled indexes unconditonally. We do not change the + key_map. Otherwise we invert the key map temporarily (outside of + this function) and recreate the then "seemingly" enabled indexes. + When we cannot use the optimization, and drop all indexes, we + pretend that all indexes were disabled. By the inversion, we will + then recrate all indexes. +*/ + +static int mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, my_bool force) +{ + MYISAM_SHARE *share= info->s; + MI_STATE_INFO *state= &share->state; + uint i; + int error; + DBUG_ENTER("mi_drop_all_indexes"); + + /* + If any of the disabled indexes has a key block assigned, we must + drop and recreate all indexes to avoid losing index blocks. + + If we want to recreate disabled indexes only _and_ all of these + indexes are empty, we don't need to recreate the existing indexes. + */ + if (!force && (param->testflag & T_CREATE_MISSING_KEYS)) + { + DBUG_PRINT("repair", ("creating missing indexes")); + for (i= 0; i < share->base.keys; i++) + { + DBUG_PRINT("repair", ("index #: %u key_root: 0x%lx active: %d", + i, (long) state->key_root[i], + mi_is_key_active(state->key_map, i))); + if ((state->key_root[i] != HA_OFFSET_ERROR) && + !mi_is_key_active(state->key_map, i)) + { + /* + This index has at least one key block and it is disabled. + We would lose its block(s) if would just recreate it. + So we need to drop and recreate all indexes. + */ + DBUG_PRINT("repair", ("nonempty and disabled: recreate all")); + break; + } + } + if (i >= share->base.keys) + { + /* + All of the disabled indexes are empty. We can just recreate them. + Flush dirty blocks of this index file from key cache and remove + all blocks of this index file from key cache. + */ + DBUG_PRINT("repair", ("all disabled are empty: create missing")); + error= flush_key_blocks(share->key_cache, share->kfile, + FLUSH_FORCE_WRITE); + goto end; + } + /* + We do now drop all indexes and declare them disabled. With the + T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all + disabled indexes and enable them. + */ + mi_clear_all_keys_active(state->key_map); + DBUG_PRINT("repair", ("declared all indexes disabled")); + } + + /* Remove all key blocks of this index file from key cache. */ + if ((error= flush_key_blocks(share->key_cache, share->kfile, + FLUSH_IGNORE_CHANGED))) + goto end; + + /* Clear index root block pointers. */ + for (i= 0; i < share->base.keys; i++) + state->key_root[i]= HA_OFFSET_ERROR; + + /* Clear the delete chains. */ + for (i= 0; i < state->header.max_block_size; i++) + state->key_del[i]= HA_OFFSET_ERROR; + + /* Reset index file length to end of index file header. */ + info->state->key_file_length= share->base.keystart; + + DBUG_PRINT("repair", ("dropped all indexes")); + /* error= 0; set by last (error= flush_key_bocks()). */ + + end: + DBUG_RETURN(error); +} + + /* Recover old table by reading each record and writing all keys */ /* Save new datafile-name in temp_filename */ @@ -1382,7 +1515,6 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, my_string name, int rep_quick) { int error,got_error; - uint i; ha_rows start_records,new_header_length; my_off_t del; File new_file; @@ -1486,25 +1618,10 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); - /* - Clear all keys. Note that all key blocks allocated until now remain - "dead" parts of the key file. (Bug #4692) - */ - for (i=0 ; i < info->s->base.keys ; i++) - share->state.key_root[i]= HA_OFFSET_ERROR; - - /* Drop the delete chain. */ - for (i=0 ; i < share->state.header.max_block_size ; i++) - share->state.key_del[i]= HA_OFFSET_ERROR; - - /* - If requested, activate (enable) all keys in key_map. In this case, - all indexes will be (re-)built. - */ + /* This function always recreates all enabled indexes. */ if (param->testflag & T_CREATE_MISSING_KEYS) mi_set_all_keys_active(share->state.key_map, share->base.keys); - - info->state->key_file_length=share->base.keystart; + mi_drop_all_indexes(param, info, TRUE); lock_memory(param); /* Everything is alloced */ @@ -2105,8 +2222,9 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, ulong *rec_per_key_part; char llbuff[22]; SORT_INFO sort_info; - ulonglong key_map=share->state.key_map; + ulonglong key_map; DBUG_ENTER("mi_repair_by_sort"); + LINT_INIT(key_map); start_records=info->state->records; got_error=1; @@ -2179,25 +2297,14 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, } info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); - if (!(param->testflag & T_CREATE_MISSING_KEYS)) + + /* Optionally drop indexes and optionally modify the key_map. */ + mi_drop_all_indexes(param, info, FALSE); + key_map= share->state.key_map; + if (param->testflag & T_CREATE_MISSING_KEYS) { - /* - Flush key cache for this file if we are calling this outside - myisamchk - */ - flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED); - /* Clear the pointers to the given rows */ - for (i=0 ; i < share->base.keys ; i++) - share->state.key_root[i]= HA_OFFSET_ERROR; - for (i=0 ; i < share->state.header.max_block_size ; i++) - share->state.key_del[i]= HA_OFFSET_ERROR; - info->state->key_file_length=share->base.keystart; - } - else - { - if (flush_key_blocks(share->key_cache,share->kfile, FLUSH_FORCE_WRITE)) - goto err; - key_map= ~key_map; /* Create the missing keys */ + /* Invert the copied key_map to recreate all disabled indexes. */ + key_map= ~key_map; } sort_info.info=info; @@ -2240,6 +2347,10 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, sort_param.read_cache=param->read_cache; sort_param.keyinfo=share->keyinfo+sort_param.key; sort_param.seg=sort_param.keyinfo->seg; + /* + Skip this index if it is marked disabled in the copied + (and possibly inverted) key_map. + */ if (! mi_is_key_active(key_map, sort_param.key)) { /* Remember old statistics for key */ @@ -2247,6 +2358,8 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, (char*) (share->state.rec_per_key_part + (uint) (rec_per_key_part - param->rec_per_key_part)), sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part)); + DBUG_PRINT("repair", ("skipping seemingly disabled index #: %u", + sort_param.key)); continue; } @@ -2302,8 +2415,11 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, if (param->testflag & T_STATISTICS) update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique, param->stats_method == MI_STATS_METHOD_IGNORE_NULLS? - sort_param.notnull: NULL,(ulonglong) info->state->records); + sort_param.notnull: NULL, + (ulonglong) info->state->records); + /* Enable this index in the permanent (not the copied) key_map. */ mi_set_key_active(share->state.key_map, sort_param.key); + DBUG_PRINT("repair", ("set enabled index #: %u", sort_param.key)); if (sort_param.fix_datafile) { @@ -2504,9 +2620,10 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info, IO_CACHE new_data_cache; /* For non-quick repair. */ IO_CACHE_SHARE io_share; SORT_INFO sort_info; - ulonglong key_map=share->state.key_map; + ulonglong key_map; pthread_attr_t thr_attr; DBUG_ENTER("mi_repair_parallel"); + LINT_INIT(key_map); start_records=info->state->records; got_error=1; @@ -2608,25 +2725,14 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info, } info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); - if (!(param->testflag & T_CREATE_MISSING_KEYS)) + + /* Optionally drop indexes and optionally modify the key_map. */ + mi_drop_all_indexes(param, info, FALSE); + key_map= share->state.key_map; + if (param->testflag & T_CREATE_MISSING_KEYS) { - /* - Flush key cache for this file if we are calling this outside - myisamchk - */ - flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED); - /* Clear the pointers to the given rows */ - for (i=0 ; i < share->base.keys ; i++) - share->state.key_root[i]= HA_OFFSET_ERROR; - for (i=0 ; i < share->state.header.max_block_size ; i++) - share->state.key_del[i]= HA_OFFSET_ERROR; - info->state->key_file_length=share->base.keystart; - } - else - { - if (flush_key_blocks(share->key_cache,share->kfile, FLUSH_FORCE_WRITE)) - goto err; - key_map= ~key_map; /* Create the missing keys */ + /* Invert the copied key_map to recreate all disabled indexes. */ + key_map= ~key_map; } sort_info.info=info; @@ -2682,6 +2788,10 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info, sort_param[i].key=key; sort_param[i].keyinfo=share->keyinfo+key; sort_param[i].seg=sort_param[i].keyinfo->seg; + /* + Skip this index if it is marked disabled in the copied + (and possibly inverted) key_map. + */ if (! mi_is_key_active(key_map, key)) { /* Remember old statistics for key */ diff --git a/mysql-test/r/myisam.result b/mysql-test/r/myisam.result index 7fc29cd13ca..176d0e97012 100644 --- a/mysql-test/r/myisam.result +++ b/mysql-test/r/myisam.result @@ -1806,4 +1806,29 @@ SELECT a FROM t1 FORCE INDEX (inx) WHERE a=1; a 1 DROP TABLE t1; +CREATE TABLE t1 (c1 INT, c2 INT, UNIQUE INDEX (c1), INDEX (c2)) ENGINE=MYISAM; +SHOW TABLE STATUS LIKE 't1'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment +t1 MyISAM 10 Fixed 0 # # # 1024 # # # # # # # +INSERT INTO t1 VALUES (1,1); +SHOW TABLE STATUS LIKE 't1'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment +t1 MyISAM 10 Fixed 1 # # # 3072 # # # # # # # +ALTER TABLE t1 DISABLE KEYS; +SHOW TABLE STATUS LIKE 't1'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment +t1 MyISAM 10 Fixed 1 # # # 3072 # # # # # # # +ALTER TABLE t1 ENABLE KEYS; +SHOW TABLE STATUS LIKE 't1'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment +t1 MyISAM 10 Fixed 1 # # # 3072 # # # # # # # +ALTER TABLE t1 DISABLE KEYS; +SHOW TABLE STATUS LIKE 't1'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment +t1 MyISAM 10 Fixed 1 # # # 3072 # # # # # # # +ALTER TABLE t1 ENABLE KEYS; +SHOW TABLE STATUS LIKE 't1'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment +t1 MyISAM 10 Fixed 1 # # # 3072 # # # # # # # +DROP TABLE t1; End of 5.0 tests diff --git a/mysql-test/t/myisam.test b/mysql-test/t/myisam.test index d5f403616c8..ad223dc2664 100644 --- a/mysql-test/t/myisam.test +++ b/mysql-test/t/myisam.test @@ -1161,4 +1161,30 @@ ALTER TABLE t1 ENABLE KEYS; SELECT a FROM t1 FORCE INDEX (inx) WHERE a=1; DROP TABLE t1; +# +# Bug#4692 - DISABLE/ENABLE KEYS waste a space +# +CREATE TABLE t1 (c1 INT, c2 INT, UNIQUE INDEX (c1), INDEX (c2)) ENGINE=MYISAM; +--replace_column 6 # 7 # 8 # 10 # 11 # 12 # 13 # 14 # 15 # 16 # +SHOW TABLE STATUS LIKE 't1'; +INSERT INTO t1 VALUES (1,1); +--replace_column 6 # 7 # 8 # 10 # 11 # 12 # 13 # 14 # 15 # 16 # +SHOW TABLE STATUS LIKE 't1'; +ALTER TABLE t1 DISABLE KEYS; +--replace_column 6 # 7 # 8 # 10 # 11 # 12 # 13 # 14 # 15 # 16 # +SHOW TABLE STATUS LIKE 't1'; +ALTER TABLE t1 ENABLE KEYS; +--replace_column 6 # 7 # 8 # 10 # 11 # 12 # 13 # 14 # 15 # 16 # +SHOW TABLE STATUS LIKE 't1'; +ALTER TABLE t1 DISABLE KEYS; +--replace_column 6 # 7 # 8 # 10 # 11 # 12 # 13 # 14 # 15 # 16 # +SHOW TABLE STATUS LIKE 't1'; +ALTER TABLE t1 ENABLE KEYS; +--replace_column 6 # 7 # 8 # 10 # 11 # 12 # 13 # 14 # 15 # 16 # +SHOW TABLE STATUS LIKE 't1'; +#--exec ls -log var/master-data/test/t1.MYI +#--exec myisamchk -dvv var/master-data/test/t1.MYI +#--exec myisamchk -iev var/master-data/test/t1.MYI +DROP TABLE t1; + --echo End of 5.0 tests From ff4b438be005a3cffb5100db93e10da0f7980922 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 6 Nov 2007 18:09:33 +0400 Subject: [PATCH 11/12] BUG#32111 - Security Breach via DATA/INDEX DIRECORY and RENAME TABLE RENAME TABLE against a table with DATA/INDEX DIRECTORY overwrites the file to which the symlink points. This is security issue, because it is possible to create a table with some name in some non-system database and set DATA/INDEX DIRECTORY to mysql system database. Renaming this table to one of mysql system tables (e.g. user, host) would overwrite the system table. Return an error when the file to which the symlink points exist. mysql-test/r/symlink.result: A test case for BUG#32111. mysql-test/t/symlink.test: A test case for BUG#32111. mysys/my_symlink2.c: Return an error when the file to which the symlink points exist. --- mysql-test/r/symlink.result | 6 ++++++ mysql-test/t/symlink.test | 12 ++++++++++++ mysys/my_symlink2.c | 11 ++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/symlink.result b/mysql-test/r/symlink.result index d07a8613883..e2b26cb508c 100644 --- a/mysql-test/r/symlink.result +++ b/mysql-test/r/symlink.result @@ -84,3 +84,9 @@ t1 CREATE TABLE `t1` ( `b` int(11) default NULL ) TYPE=MyISAM drop table t1; +CREATE TABLE t1(a INT) +DATA DIRECTORY='TEST_DIR/var/master-data/mysql' +INDEX DIRECTORY='TEST_DIR/var/master-data/mysql'; +RENAME TABLE t1 TO user; +Can't create/write to file 'TEST_DIR/var/master-data/mysql/user.MYI' (Errcode: 17) +DROP TABLE t1; diff --git a/mysql-test/t/symlink.test b/mysql-test/t/symlink.test index 7a42a60054e..9750e6fdfdd 100644 --- a/mysql-test/t/symlink.test +++ b/mysql-test/t/symlink.test @@ -112,3 +112,15 @@ eval alter table t1 index directory="$MYSQL_TEST_DIR/var/log"; enable_query_log; show create table t1; drop table t1; + +# +# BUG#32111 - Security Breach via DATA/INDEX DIRECORY and RENAME TABLE +# +--replace_result $MYSQL_TEST_DIR TEST_DIR +eval CREATE TABLE t1(a INT) +DATA DIRECTORY='$MYSQL_TEST_DIR/var/master-data/mysql' +INDEX DIRECTORY='$MYSQL_TEST_DIR/var/master-data/mysql'; +--replace_result $MYSQL_TEST_DIR TEST_DIR +--error 1 +RENAME TABLE t1 TO user; +DROP TABLE t1; diff --git a/mysys/my_symlink2.c b/mysys/my_symlink2.c index 913f632fbb4..4d58699412a 100644 --- a/mysys/my_symlink2.c +++ b/mysys/my_symlink2.c @@ -120,6 +120,7 @@ int my_rename_with_symlink(const char *from, const char *to, myf MyFlags) int was_symlink= (!my_disable_symlinks && !my_readlink(link_name, from, MYF(0))); int result=0; + int name_is_different; DBUG_ENTER("my_rename_with_symlink"); if (!was_symlink) @@ -128,6 +129,14 @@ int my_rename_with_symlink(const char *from, const char *to, myf MyFlags) /* Change filename that symlink pointed to */ strmov(tmp_name, to); fn_same(tmp_name,link_name,1); /* Copy dir */ + name_is_different= strcmp(link_name, tmp_name); + if (name_is_different && !access(tmp_name, F_OK)) + { + my_errno= EEXIST; + if (MyFlags & MY_WME) + my_error(EE_CANTCREATEFILE, MYF(0), tmp_name, EEXIST); + DBUG_RETURN(1); + } /* Create new symlink */ if (my_symlink(tmp_name, to, MyFlags)) @@ -139,7 +148,7 @@ int my_rename_with_symlink(const char *from, const char *to, myf MyFlags) the same basename and different directories. */ - if (strcmp(link_name, tmp_name) && my_rename(link_name, tmp_name, MyFlags)) + if (name_is_different && my_rename(link_name, tmp_name, MyFlags)) { int save_errno=my_errno; my_delete(to, MyFlags); /* Remove created symlink */ From 88701b45a935016c31af3bd6817dd4ee4c7ccd37 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 12 Nov 2007 15:15:16 +0400 Subject: [PATCH 12/12] After merge fix. --- mysql-test/r/symlink.result | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/r/symlink.result b/mysql-test/r/symlink.result index cc54233107a..81798b2dc20 100644 --- a/mysql-test/r/symlink.result +++ b/mysql-test/r/symlink.result @@ -94,7 +94,7 @@ CREATE TABLE t1(a INT) DATA DIRECTORY='TEST_DIR/var/master-data/mysql' INDEX DIRECTORY='TEST_DIR/var/master-data/mysql'; RENAME TABLE t1 TO user; -Can't create/write to file 'TEST_DIR/var/master-data/mysql/user.MYI' (Errcode: 17) +ERROR HY000: Can't create/write to file 'TEST_DIR/var/master-data/mysql/user.MYI' (Errcode: 17) DROP TABLE t1; show create table t1; Table Create Table