From ef920b28793302a532b87b4babfc224bb732941e Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 20 Jun 2005 11:43:38 +0200 Subject: [PATCH 01/32] Fix for BUG#10151: In Item_func_case::find_item don't assume that parameter str != &(this->str_value) mysql-test/r/case.result: Testcase for BUG#10151 mysql-test/t/case.test: Testcase for BUG#10151 --- mysql-test/r/case.result | 15 +++++++++++++++ mysql-test/t/case.test | 11 +++++++++++ sql/item_cmpfunc.cc | 4 +++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/case.result b/mysql-test/r/case.result index 541560afd36..fb0bc19a67e 100644 --- a/mysql-test/r/case.result +++ b/mysql-test/r/case.result @@ -154,3 +154,18 @@ t1 CREATE TABLE `t1` ( `COALESCE('a' COLLATE latin1_bin,'b')` char(1) character set latin1 collate latin1_bin NOT NULL default '' ) ENGINE=MyISAM DEFAULT CHARSET=latin1 DROP TABLE t1; +SELECT 'case+union+test' +UNION +SELECT CASE LOWER('1') WHEN LOWER('2') THEN 'BUG' ELSE 'nobug' END; +case+union+test +case+union+test +nobug +SELECT CASE LOWER('1') WHEN LOWER('2') THEN 'BUG' ELSE 'nobug' END; +CASE LOWER('1') WHEN LOWER('2') THEN 'BUG' ELSE 'nobug' END +nobug +SELECT 'case+union+test' +UNION +SELECT CASE '1' WHEN '2' THEN 'BUG' ELSE 'nobug' END; +case+union+test +case+union+test +nobug diff --git a/mysql-test/t/case.test b/mysql-test/t/case.test index 87e456baba7..ac60d7298ce 100644 --- a/mysql-test/t/case.test +++ b/mysql-test/t/case.test @@ -107,3 +107,14 @@ explain extended SELECT COALESCE('a' COLLATE latin1_bin,'b'); SHOW CREATE TABLE t1; DROP TABLE t1; + +# Test for BUG#10151 +SELECT 'case+union+test' +UNION +SELECT CASE LOWER('1') WHEN LOWER('2') THEN 'BUG' ELSE 'nobug' END; + +SELECT CASE LOWER('1') WHEN LOWER('2') THEN 'BUG' ELSE 'nobug' END; + +SELECT 'case+union+test' +UNION +SELECT CASE '1' WHEN '2' THEN 'BUG' ELSE 'nobug' END; diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 3098e5dc77e..f24638d1a93 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -1174,6 +1174,8 @@ Item *Item_func_case::find_item(String *str) String *first_expr_str,*tmp; longlong first_expr_int; double first_expr_real; + char buff[MAX_FIELD_WIDTH]; + String buff_str(buff,sizeof(buff),default_charset()); /* These will be initialized later */ LINT_INIT(first_expr_str); @@ -1186,7 +1188,7 @@ Item *Item_func_case::find_item(String *str) { case STRING_RESULT: // We can't use 'str' here as this may be overwritten - if (!(first_expr_str= args[first_expr_num]->val_str(&str_value))) + if (!(first_expr_str= args[first_expr_num]->val_str(&buff_str))) return else_expr_num != -1 ? args[else_expr_num] : 0; // Impossible break; case INT_RESULT: From 900fe718d131434d33daf5fd01bf8250bc936bd3 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 22 Jun 2005 17:12:02 +0200 Subject: [PATCH 02/32] BUG#9361: Added test cases mysql-test/r/rpl_multi_update3.result: Added test cases mysql-test/t/rpl_multi_update3.test: Added test cases --- mysql-test/r/rpl_multi_update3.result | 81 ++++++++++++++++ mysql-test/t/rpl_multi_update3.test | 133 +++++++++++++++++++++++++- 2 files changed, 210 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/rpl_multi_update3.result b/mysql-test/r/rpl_multi_update3.result index 708b230b19f..1b757b1400c 100644 --- a/mysql-test/r/rpl_multi_update3.result +++ b/mysql-test/r/rpl_multi_update3.result @@ -4,6 +4,8 @@ reset master; reset slave; drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; start slave; + +-------- Test for BUG#9361 -------- CREATE TABLE t1 ( a int unsigned not null auto_increment primary key, b int unsigned @@ -41,3 +43,82 @@ a b 1 6 2 6 drop table t1,t2; + +-------- Test 1 for BUG#9361 -------- +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +CREATE TABLE t1 ( +a1 char(30), +a2 int, +a3 int, +a4 char(30), +a5 char(30) +); +CREATE TABLE t2 ( +b1 int, +b2 char(30) +); +INSERT INTO t1 VALUES ('Yes', 1, NULL, 'foo', 'bar'); +INSERT INTO t2 VALUES (1, 'baz'); +UPDATE t1 a, t2 +SET a.a1 = 'No' +WHERE a.a2 = +(SELECT b1 +FROM t2 +WHERE b2 = 'baz') +AND a.a3 IS NULL +AND a.a4 = 'foo' +AND a.a5 = 'bar'; +SELECT * FROM t1; +a1 a2 a3 a4 a5 +No 1 NULL foo bar +SELECT * FROM t2; +b1 b2 +1 baz +DROP TABLE t1, t2; + +-------- Test 2 for BUG#9361 -------- +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +CREATE TABLE t1 ( +i INT, +j INT, +x INT, +y INT, +z INT +); +CREATE TABLE t2 ( +i INT, +k INT, +x INT, +y INT, +z INT +); +CREATE TABLE t3 ( +j INT, +k INT, +x INT, +y INT, +z INT +); +INSERT INTO t1 VALUES ( 1, 2,13,14,15); +INSERT INTO t2 VALUES ( 1, 3,23,24,25); +INSERT INTO t3 VALUES ( 2, 3, 1,34,35), ( 2, 3, 1,34,36); +UPDATE t1 AS a +INNER JOIN t2 AS b +ON a.i = b.i +INNER JOIN t3 AS c +ON a.j = c.j AND b.k = c.k +SET a.x = b.x, +a.y = b.y, +a.z = ( +SELECT sum(z) +FROM t3 +WHERE y = 34 +) +WHERE b.x = 23; +SELECT * FROM t1; +i j x y z +1 2 23 24 71 +DROP TABLE t1, t2, t3; diff --git a/mysql-test/t/rpl_multi_update3.test b/mysql-test/t/rpl_multi_update3.test index b8c8ed79532..80b0603eb60 100644 --- a/mysql-test/t/rpl_multi_update3.test +++ b/mysql-test/t/rpl_multi_update3.test @@ -1,7 +1,13 @@ +source include/master-slave.inc; + +############################################################################## +# # Let's verify that multi-update with a subselect does not cause the slave to crash # (BUG#10442) - -source include/master-slave.inc; +# +--disable_query_log +SELECT '-------- Test for BUG#9361 --------' as ""; +--enable_query_log CREATE TABLE t1 ( a int unsigned not null auto_increment primary key, @@ -25,10 +31,129 @@ UPDATE t2, (SELECT a FROM t1) AS t SET t2.b = t.a+5 ; SELECT * FROM t1 ORDER BY a; SELECT * FROM t2 ORDER BY a; -save_master_pos; +sync_slave_with_master; connection slave; -sync_with_master; SELECT * FROM t1 ORDER BY a; SELECT * FROM t2 ORDER BY a; +connection master; drop table t1,t2; + +############################################################################## +# +# Test for BUG#9361: +# Subselects should work inside multi-updates +# +--disable_query_log +SELECT '-------- Test 1 for BUG#9361 --------' as ""; +--enable_query_log + +connection master; + +--disable_warnings +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +--enable_warnings + +CREATE TABLE t1 ( + a1 char(30), + a2 int, + a3 int, + a4 char(30), + a5 char(30) +); + +CREATE TABLE t2 ( + b1 int, + b2 char(30) +); + +# Insert one row per table +INSERT INTO t1 VALUES ('Yes', 1, NULL, 'foo', 'bar'); +INSERT INTO t2 VALUES (1, 'baz'); + +# This should update the row in t1 +UPDATE t1 a, t2 + SET a.a1 = 'No' + WHERE a.a2 = + (SELECT b1 + FROM t2 + WHERE b2 = 'baz') + AND a.a3 IS NULL + AND a.a4 = 'foo' + AND a.a5 = 'bar'; + +sync_slave_with_master; +connection slave; +SELECT * FROM t1; +SELECT * FROM t2; + +connection master; +DROP TABLE t1, t2; + +############################################################################## +# +# Second test for BUG#9361 +# + +--disable_query_log +SELECT '-------- Test 2 for BUG#9361 --------' as ""; +--enable_query_log + +connection master; + +--disable_warnings +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +--enable_warnings + +CREATE TABLE t1 ( + i INT, + j INT, + x INT, + y INT, + z INT +); + +CREATE TABLE t2 ( + i INT, + k INT, + x INT, + y INT, + z INT +); + +CREATE TABLE t3 ( + j INT, + k INT, + x INT, + y INT, + z INT +); + +INSERT INTO t1 VALUES ( 1, 2,13,14,15); +INSERT INTO t2 VALUES ( 1, 3,23,24,25); +INSERT INTO t3 VALUES ( 2, 3, 1,34,35), ( 2, 3, 1,34,36); + +UPDATE t1 AS a +INNER JOIN t2 AS b + ON a.i = b.i +INNER JOIN t3 AS c + ON a.j = c.j AND b.k = c.k +SET a.x = b.x, + a.y = b.y, + a.z = ( + SELECT sum(z) + FROM t3 + WHERE y = 34 + ) +WHERE b.x = 23; + +sync_slave_with_master; +connection slave; + +SELECT * FROM t1; + +connection master; +DROP TABLE t1, t2, t3; From e4296f586851746ad265e52c18e8e33080eb1a86 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 10:56:44 +0300 Subject: [PATCH 03/32] Fix for BUG#11185. The source of the problem is in Field_longlong::cmp. If 'this' is an unsigned number, the method casts both the current value, and the constant that we compare with to an unsigned number. As a result if the constant we compare with is a negative number, it wraps to some unsigned number, and the comparison is incorrect. When the optimizer chooses the "range" access method, this problem causes handler::read_range_next to reject the current key when the upper bound key is a negative number because handler::compare_key incorrectly considers the positive and negative keys to be equal. The current patch does not correct the source of the problem in Field_longlong::cmp because it is not easy to propagate sign information about the constant at query execution time. Instead the patch changes the range optimizer so that it never compares unsiged fields with negative constants. As an added benefit, queries that do such comparisons will execute faster because the range optimizer replaces conditions like: (a) (unsigned_int [< | <=] negative_constant) == FALSE (b) (unsigned_int [> | >=] negative_constant) == TRUE with the corresponding constants. In some cases this may even result in constant time execution. mysql-test/r/range.result: - Changed incorrect result of an old test - Added new results for BUG#11185 mysql-test/t/range.test: - Added new tests for BUG#11185 - Deleted an old comment because now the problem is fixed sql/opt_range.cc: Added a new optimization to the range optimizer where we detect that an UNSIGNED field is compared with a negative constant. Depending on the comparison operator, we know directly that the result of the comparison is either TRUE or FALSE for all input values, and we need not check each value. This optimization is also necessary so that the index range access method produces correct results when comparing unsigned fields with negative constants. --- mysql-test/r/range.result | 33 ++++++++++++++++++++++++++++++++- mysql-test/t/range.test | 22 +++++++++++++++++++--- sql/opt_range.cc | 38 +++++++++++++++++++++++++++++++++++--- 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/mysql-test/r/range.result b/mysql-test/r/range.result index afd91ce5fe9..07e96aee22b 100644 --- a/mysql-test/r/range.result +++ b/mysql-test/r/range.result @@ -556,11 +556,42 @@ count(*) 0 select count(*) from t1 where x > -16; count(*) -1 +2 select count(*) from t1 where x = 18446744073709551601; count(*) 1 drop table t1; +create table t1 (a bigint unsigned); +create index t1i on t1(a); +insert into t1 select 18446744073709551615; +insert into t1 select 18446744073709551614; +explain select * from t1 where a <> -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index t1i t1i 9 NULL 2 Using where; Using index +select * from t1 where a <> -1; +a +18446744073709551614 +18446744073709551615 +explain select * from t1 where a > -1 or a < -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index t1i t1i 9 NULL 2 Using where; Using index +select * from t1 where a > -1 or a < -1; +a +18446744073709551614 +18446744073709551615 +explain select * from t1 where a > -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index t1i t1i 9 NULL 2 Using where; Using index +select * from t1 where a > -1; +a +18446744073709551614 +18446744073709551615 +explain select * from t1 where a < -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +select * from t1 where a < -1; +a +drop table t1; set names latin1; create table t1 (a char(10), b text, key (a)) character set latin1; INSERT INTO t1 (a) VALUES diff --git a/mysql-test/t/range.test b/mysql-test/t/range.test index b51a79fba77..92b7b848a8a 100644 --- a/mysql-test/t/range.test +++ b/mysql-test/t/range.test @@ -423,14 +423,30 @@ select count(*) from t1 where x=0; select count(*) from t1 where x<0; select count(*) from t1 where x < -16; select count(*) from t1 where x = -16; -# The following query returns wrong value because the range optimizer can't -# handle search on a signed value for an unsigned parameter. This will be fixed in -# 5.0 select count(*) from t1 where x > -16; select count(*) from t1 where x = 18446744073709551601; drop table t1; +# +# Bug #11185 incorrect comparison of unsigned int to signed constant +# +create table t1 (a bigint unsigned); +create index t1i on t1(a); +insert into t1 select 18446744073709551615; +insert into t1 select 18446744073709551614; + +explain select * from t1 where a <> -1; +select * from t1 where a <> -1; +explain select * from t1 where a > -1 or a < -1; +select * from t1 where a > -1 or a < -1; +explain select * from t1 where a > -1; +select * from t1 where a > -1; +explain select * from t1 where a < -1; +select * from t1 where a < -1; + +drop table t1; + # # Bug #6045: Binary Comparison regression in MySQL 4.1 # Binary searches didn't use a case insensitive index. diff --git a/sql/opt_range.cc b/sql/opt_range.cc index bd1befb436f..75cf9e6b3f0 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -960,7 +960,9 @@ get_mm_parts(PARAM *param, COND *cond_func, Field *field, if (sel_arg->type == SEL_ARG::IMPOSSIBLE) { tree->type=SEL_TREE::IMPOSSIBLE; - DBUG_RETURN(tree); + /* If this is an NE_FUNC, we still need to check GT_FUNC. */ + if (!ne_func) + DBUG_RETURN(tree); } } else @@ -979,8 +981,9 @@ get_mm_parts(PARAM *param, COND *cond_func, Field *field, SEL_TREE *tree2= get_mm_parts(param, cond_func, field, Item_func::GT_FUNC, value, cmp_type); - if (tree2) - tree= tree_or(param,tree,tree2); + if (!tree2) + DBUG_RETURN(0) + tree= tree_or(param,tree,tree2); } DBUG_RETURN(tree); } @@ -1159,6 +1162,35 @@ get_mm_leaf(PARAM *param, COND *conf_func, Field *field, KEY_PART *key_part, if (!(tree=new SEL_ARG(field,str,str2))) DBUG_RETURN(0); // out of memory + /* + Check if we are comparing an UNSIGNED integer with a negative constant. + In this case we know that: + (a) (unsigned_int [< | <=] negative_constant) == FALSE + (b) (unsigned_int [> | >=] negative_constant) == TRUE + In case (a) the condition is false for all values, and in case (b) it + is true for all values, so we can avoid unnecessary retrieval and condition + testing, and we also get correct comparison of unsinged integers with + negative integers (which otherwise fails because at query execution time + negative integers are cast to unsigned if compared with unsigned). + */ + Item_result field_result_type= field->result_type(); + Item_result value_result_type= value->result_type(); + if (field_result_type == INT_RESULT && value_result_type == INT_RESULT && + ((Field_num*)field)->unsigned_flag && !((Item_int*)value)->unsigned_flag) + { + longlong item_val= value->val_int(); + if (item_val < 0) + { + if (type == Item_func::LT_FUNC || type == Item_func::LE_FUNC) + { + tree->type= SEL_ARG::IMPOSSIBLE; + DBUG_RETURN(tree); + } + if (type == Item_func::GT_FUNC || type == Item_func::GE_FUNC) + DBUG_RETURN(0); + } + } + switch (type) { case Item_func::LT_FUNC: if (field_is_equal_to_item(field,value)) From c4a8325da2583f783cd81d231a309b2080e34954 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 02:08:30 -0700 Subject: [PATCH 04/32] opt_range.cc: Fixed buf #11487. Added a call of QUICK_RANGE_SELECT::init to the QUICK_RANGE_SELECT::reset method. Without it the second evaluation of a subquery employing the range access failed. subselect.result, subselect.test: Added a test case for bug #11487. mysql-test/t/subselect.test: Added a test case for bug #11487. mysql-test/r/subselect.result: Added a test case for bug #11487. sql/opt_range.cc: Fixed buf #11487. Added a call of QUICK_RANGE_SELECT::init to the QUICK_RANGE_SELECT::reset method. Without it the second evaluation of a subquery employing the range access failed. --- mysql-test/r/subselect.result | 21 +++++++++++++++++++++ mysql-test/t/subselect.test | 22 ++++++++++++++++++++++ sql/opt_range.cc | 5 ++++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index 6703147c635..736559f8569 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -2816,3 +2816,24 @@ select * from t1; EMPNUM E1 DROP TABLE t1,t2; +CREATE TABLE t1(select_id BIGINT, values_id BIGINT); +INSERT INTO t1 VALUES (1, 1); +CREATE TABLE t2 (select_id BIGINT, values_id BIGINT, +PRIMARY KEY(select_id,values_id)); +INSERT INTO t2 VALUES (0, 1), (0, 2), (0, 3), (1, 5); +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 +WHERE select_id IN (1, 0)); +values_id +1 +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 +WHERE select_id BETWEEN 0 AND 1); +values_id +1 +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 +WHERE select_id = 0 OR select_id = 1); +values_id +1 +DROP TABLE t1, t2; diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index 2e6cea8468b..1e4930d385d 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -1837,3 +1837,25 @@ WHERE t1.EMPNUM NOT IN WHERE t1.EMPNUM = t2.EMPNUM); select * from t1; DROP TABLE t1,t2; + +# +# Test for bug #11487: range access in a subquery +# + +CREATE TABLE t1(select_id BIGINT, values_id BIGINT); +INSERT INTO t1 VALUES (1, 1); +CREATE TABLE t2 (select_id BIGINT, values_id BIGINT, + PRIMARY KEY(select_id,values_id)); +INSERT INTO t2 VALUES (0, 1), (0, 2), (0, 3), (1, 5); + +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 + WHERE select_id IN (1, 0)); +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 + WHERE select_id BETWEEN 0 AND 1); +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 + WHERE select_id = 0 OR select_id = 1); + +DROP TABLE t1, t2; diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 0a8627b1fa0..200e6dfbabb 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -5992,7 +5992,10 @@ int QUICK_RANGE_SELECT::reset() next=0; range= NULL; cur_range= (QUICK_RANGE**) ranges.buffer; - + + if (file->inited == handler::NONE && (error= file->ha_index_init(index))) + DBUG_RETURN(error); + /* Do not allocate the buffers twice. */ if (multi_range_length) { From 0beb0abf5a7572a461aff86eeaedbf8307bfc911 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 11:30:40 +0200 Subject: [PATCH 05/32] - fixed text file generation - the node names in the info page had changed BitKeeper/deleted/.del-generate-flag-images~f77476753fff8186: Delete: Docs/Support/generate-flag-images Docs/Makefile.am: - fixed node names for the new info file Docs/Support/generate-text-files.pl: - stop printing if the index was reached --- Docs/Makefile.am | 8 ++++---- Docs/Support/generate-flag-images | 31 ----------------------------- Docs/Support/generate-text-files.pl | 2 +- 3 files changed, 5 insertions(+), 36 deletions(-) delete mode 100755 Docs/Support/generate-flag-images diff --git a/Docs/Makefile.am b/Docs/Makefile.am index 8577fee52cb..dba3a4f819c 100644 --- a/Docs/Makefile.am +++ b/Docs/Makefile.am @@ -36,19 +36,19 @@ CLEAN_FILES: $(txt_files) GT = $(srcdir)/Support/generate-text-files.pl ../INSTALL-SOURCE: mysql.info $(GT) - perl -w $(GT) mysql.info "Installing" "Tutorial" > $@ + perl -w $(GT) mysql.info "installing-source" "windows-source-build" > $@ # We put the description for the binary installation here so that # people who download source wont have to see it. It is moved up to # the toplevel by the script that makes the binary tar files. INSTALL-BINARY: mysql.info $(GT) - perl -w $(GT) mysql.info "Installing binary" "Installing source" > $@ + perl -w $(GT) mysql.info "installing-binary" "installing-source" > $@ ../EXCEPTIONS-CLIENT: mysql.info $(GT) - perl -w $(GT) mysql.info "MySQL FLOSS License Exception" "Function Index" > $@ + perl -w $(GT) mysql.info "mysql-floss-license-exception" "function-index" > $@ ../support-files/MacOSX/ReadMe.txt: mysql.info $(GT) - perl -w $(GT) mysql.info "Mac OS X installation" "NetWare installation" > $@ + perl -w $(GT) mysql.info "mac-os-x-installation" "netware-installation" > $@ # Don't update the files from bitkeeper %::SCCS/s.% diff --git a/Docs/Support/generate-flag-images b/Docs/Support/generate-flag-images deleted file mode 100755 index 21140388012..00000000000 --- a/Docs/Support/generate-flag-images +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -flags=`grep @image mirrors.texi | cut -d" " -f1 | cut -d/ -f2 | tr -d "}" | sort | uniq` - -set -x -cd Flags - -for c in $flags -do - # For PNM, to be used later - giftopnm ../Raw-Flags/$c.gif | pnmscale -xsize 30 > $c-tmp.pnm - pnmpaste $c-tmp.pnm 1 1 ../Images/flag-background.pnm > $c.pnm - rm -f $c-tmp.pnm - - # For GIF version - ppmtogif $c.pnm > $c.gif - # or cjpeg -optimize -quality 70 -outfile $c.jpg - - # For EPS version - pnmtops -noturn $c.pnm > $c.eps - - # For PDF version - ps2pdf $c.eps $c.pdf - - # For text version - echo -n "" > $c.txt - - # PNM isn't really needed - rm -f $c.pnm - -done diff --git a/Docs/Support/generate-text-files.pl b/Docs/Support/generate-text-files.pl index 6470baaa6e9..0829525f679 100755 --- a/Docs/Support/generate-text-files.pl +++ b/Docs/Support/generate-text-files.pl @@ -13,7 +13,7 @@ while () { if ($in) { - if (/Node: $tnode,/) + if (/Node: $tnode,/ || /\[index/) { $in = 0; } From 089d20959e2f5551473f843f320495a811b6191b Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 16:04:10 +0500 Subject: [PATCH 06/32] WL#2286 - Compile MySQL w/YASSL support Fix for "multiple definition of __cxa_pure_virtual" link failure when compiling with icc. extra/yassl/taocrypt/include/runtime.hpp: Do not define __cxa_pure_virtual for ICC. Fixes "multiple definition of __cxa_pure_virtual" link failure on production. --- extra/yassl/taocrypt/include/runtime.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/yassl/taocrypt/include/runtime.hpp b/extra/yassl/taocrypt/include/runtime.hpp index 70768bb01d1..f506040f0d8 100644 --- a/extra/yassl/taocrypt/include/runtime.hpp +++ b/extra/yassl/taocrypt/include/runtime.hpp @@ -25,7 +25,7 @@ -#if !defined(yaSSL_NEW_HPP) && defined(__GNUC__) +#if !defined(yaSSL_NEW_HPP) && defined(__GNUC__) && !defined(__ICC) #define yaSSL_NEW_HPP From dfaf7a01845e137769b4146c8d98a1b749271bc0 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 04:10:43 -0700 Subject: [PATCH 07/32] opt_range.cc: Identation correction. sql/opt_range.cc: Identation correction. --- sql/opt_range.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 1d6acb3aee1..59ec1140d15 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -5994,7 +5994,7 @@ int QUICK_RANGE_SELECT::reset() cur_range= (QUICK_RANGE**) ranges.buffer; if (file->inited == handler::NONE && (error= file->ha_index_init(index))) - DBUG_RETURN(error); + DBUG_RETURN(error); /* Do not allocate the buffers twice. */ if (multi_range_length) From 98253bd64d51d42f914c59ebabe89e3975ba9cc8 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 06:15:50 -0700 Subject: [PATCH 08/32] func_str.result, func_str.test: Added a test case for bug #10124. sql_select.h, item_subselect.cc, sql_select.cc: Fixed bug #10124. The copy method of the store_key classes can return STORE_KEY_OK=0, STORE_KEY_FATAL=1, STORE_KEY_CONV=2 now. field.cc: Fixed bug #10124. When ussuing a warning the store methods return 2 instead of 1 now. sql/field.cc: Fixed bug #10124. When ussuing a warning the store methods return 2 instead of 1 now. sql/sql_select.cc: Fixed bug #10124. The copy method of the store_key classes can return STORE_KEY_OK=0, STORE_KEY_FATAL=1, STORE_KEY_CONV=2 now. sql/item_subselect.cc: Fixed bug #10124. The copy method of the store_key classes can return STORE_KEY_OK=0, STORE_KEY_FATAL=1, STORE_KEY_CONV=2 now. sql/sql_select.h: Fixed bug #10124. The copy method of the store_key classes can return STORE_KEY_OK=0, STORE_KEY_FATAL=1, STORE_KEY_CONV=2 now. mysql-test/t/func_str.test: Added a test case for bug #10124. mysql-test/r/func_str.result: Added a test case for bug #10124. --- mysql-test/r/func_str.result | 11 ++++++++++ mysql-test/t/func_str.test | 14 ++++++++++++ sql/field.cc | 41 +++++++++++++++++++++--------------- sql/item_subselect.cc | 4 ++-- sql/sql_select.cc | 8 +++++-- sql/sql_select.h | 25 ++++++++++++++-------- 6 files changed, 73 insertions(+), 30 deletions(-) diff --git a/mysql-test/r/func_str.result b/mysql-test/r/func_str.result index ea1efbc7c0a..60c77d91ca5 100644 --- a/mysql-test/r/func_str.result +++ b/mysql-test/r/func_str.result @@ -789,3 +789,14 @@ field(0,NULL,1,0) field("",NULL,"bar","") field(0.0,NULL,1.0,0.0) select field(NULL,1,2,NULL), field(NULL,1,2,0); field(NULL,1,2,NULL) field(NULL,1,2,0) 0 0 +CREATE TABLE t1 (str varchar(20) PRIMARY KEY); +CREATE TABLE t2 (num int primary key); +INSERT INTO t1 VALUES ('notnumber'); +INSERT INTO t2 VALUES (0), (1); +SELECT * FROM t1, t2 WHERE num=str; +str num +notnumber 0 +SELECT * FROM t1, t2 WHERE num=substring(str from 1 for 6); +str num +notnumber 0 +DROP TABLE t1,t2; diff --git a/mysql-test/t/func_str.test b/mysql-test/t/func_str.test index a5536f7a0be..36cfac16ff3 100644 --- a/mysql-test/t/func_str.test +++ b/mysql-test/t/func_str.test @@ -527,3 +527,17 @@ DROP TABLE t1, t2; # select field(0,NULL,1,0), field("",NULL,"bar",""), field(0.0,NULL,1.0,0.0); select field(NULL,1,2,NULL), field(NULL,1,2,0); + +# +# Bug #10124: access by integer index with a string key that is not a number +# + +CREATE TABLE t1 (str varchar(20) PRIMARY KEY); +CREATE TABLE t2 (num int primary key); +INSERT INTO t1 VALUES ('notnumber'); +INSERT INTO t2 VALUES (0), (1); + +SELECT * FROM t1, t2 WHERE num=str; +SELECT * FROM t1, t2 WHERE num=substring(str from 1 for 6); + +DROP TABLE t1,t2; diff --git a/sql/field.cc b/sql/field.cc index 692f123097a..a330ffb7262 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -2473,7 +2473,10 @@ int Field_long::store(const char *from,uint len,CHARSET_INFO *cs) if (error || (from+len != end && table->in_use->count_cuted_fields && !test_if_int(from,len,end,cs))) - error= 1; + { + if (error != 1) + error= 2; + } #if SIZEOF_LONG > 4 if (unsigned_flag) { @@ -2501,10 +2504,7 @@ int Field_long::store(const char *from,uint len,CHARSET_INFO *cs) } #endif if (error) - { set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); - error= 1; - } #ifdef WORDS_BIGENDIAN if (table->db_low_byte_first) { @@ -2770,8 +2770,11 @@ int Field_longlong::store(const char *from,uint len,CHARSET_INFO *cs) (from+len != end && table->in_use->count_cuted_fields && !test_if_int(from,len,end,cs))) { - set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); - error= 1; + if (error != 1) + { + set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); + error= 2; + } } #ifdef WORDS_BIGENDIAN if (table->db_low_byte_first) @@ -2991,7 +2994,7 @@ int Field_float::store(const char *from,uint len,CHARSET_INFO *cs) double nr= my_strntod(cs,(char*) from,len,&end,&error); if (error || ((uint) (end-from) != len && table->in_use->count_cuted_fields)) { - error= 1; + error= 2; set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); } Field_float::store(nr); @@ -3277,7 +3280,7 @@ int Field_double::store(const char *from,uint len,CHARSET_INFO *cs) double nr= my_strntod(cs,(char*) from, len, &end, &error); if (error || ((uint) (end-from) != len && table->in_use->count_cuted_fields)) { - error= 1; + error= 2; set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); } Field_double::store(nr); @@ -3659,6 +3662,8 @@ int Field_timestamp::store(const char *from,uint len,CHARSET_INFO *cs) error= 1; } } + if (error > 1) + error= 2; #ifdef WORDS_BIGENDIAN if (table->db_low_byte_first) @@ -3947,7 +3952,7 @@ int Field_time::store(const char *from,uint len,CHARSET_INFO *cs) if (str_to_time(from, len, <ime, &error)) { tmp=0L; - error= 1; + error= 2; set_datetime_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, from, len, MYSQL_TIMESTAMP_TIME, 1); } @@ -3969,6 +3974,8 @@ int Field_time::store(const char *from,uint len,CHARSET_INFO *cs) from, len, MYSQL_TIMESTAMP_TIME, !error); error= 1; } + if (error > 1) + error= 2; } if (ltime.neg) @@ -4298,7 +4305,7 @@ int Field_date::store(const char *from, uint len,CHARSET_INFO *cs) if (str_to_datetime(from, len, &l_time, 1, &error) <= MYSQL_TIMESTAMP_ERROR) { tmp=0; - error= 1; + error= 2; } else tmp=(uint32) l_time.year*10000L + (uint32) (l_time.month*100+l_time.day); @@ -4489,7 +4496,7 @@ int Field_newdate::store(const char *from,uint len,CHARSET_INFO *cs) if (str_to_datetime(from, len, &l_time, 1, &error) <= MYSQL_TIMESTAMP_ERROR) { tmp=0L; - error= 1; + error= 2; } else tmp= l_time.day + l_time.month*32 + l_time.year*16*32; @@ -4931,7 +4938,7 @@ int Field_string::store(const char *from,uint length,CHARSET_INFO *cs) from= tmpstr.ptr(); length= tmpstr.length(); if (conv_errors) - error= 1; + error= 2; } /* @@ -4955,7 +4962,7 @@ int Field_string::store(const char *from,uint length,CHARSET_INFO *cs) from+= field_charset->cset->scan(field_charset, from, end, MY_SEQ_SPACES); if (from != end) - error= 1; + error= 2; } if (error) set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); @@ -5210,12 +5217,12 @@ int Field_varstring::store(const char *from,uint length,CHARSET_INFO *cs) from= tmpstr.ptr(); length= tmpstr.length(); if (conv_errors) - error= 1; + error= 2; } if (length > field_length) { length=field_length; - error= 1; + error= 2; } if (error) set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); @@ -5568,7 +5575,7 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs) from= tmpstr.ptr(); length= tmpstr.length(); if (conv_errors) - error= 1; + error= 2; } copy_length= max_data_length(); @@ -5583,7 +5590,7 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs) copy_length, &well_formed_error); if (copy_length < length) - error= 1; + error= 2; Field_blob::store_length(copy_length); if (was_conversion || table->copy_blobs || copy_length <= MAX_FIELD_WIDTH) { // Must make a copy diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index ebc08545566..82954a664c0 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -1361,7 +1361,7 @@ int subselect_uniquesubquery_engine::exec() TABLE *table= tab->table; for (store_key **copy=tab->ref.key_copy ; *copy ; copy++) { - if (tab->ref.key_err= (*copy)->copy()) + if ((tab->ref.key_err= (*copy)->copy()) & 1) { table->status= STATUS_NOT_FOUND; DBUG_RETURN(1); @@ -1414,7 +1414,7 @@ int subselect_indexsubquery_engine::exec() for (store_key **copy=tab->ref.key_copy ; *copy ; copy++) { - if (tab->ref.key_err= (*copy)->copy()) + if ((tab->ref.key_err= (*copy)->copy()) & 1) { table->status= STATUS_NOT_FOUND; DBUG_RETURN(1); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 5bafe1a7df4..2f165565ce1 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -8223,11 +8223,15 @@ cp_buffer_from_ref(THD *thd, TABLE_REF *ref) enum enum_check_fields save_count_cuted_fields= thd->count_cuted_fields; thd->count_cuted_fields= CHECK_FIELD_IGNORE; for (store_key **copy=ref->key_copy ; *copy ; copy++) - if ((*copy)->copy()) + { + int res; + if ((res= (*copy)->copy())) { thd->count_cuted_fields= save_count_cuted_fields; - return 1; // Something went wrong + if ((res= res & 1)) + return res; // Something went wrong } + } thd->count_cuted_fields= save_count_cuted_fields; return 0; } diff --git a/sql/sql_select.h b/sql/sql_select.h index 7e69eca4683..c7440fe4c3a 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -356,6 +356,7 @@ class store_key :public Sql_alloc char *null_ptr; char err; public: + enum store_key_result { STORE_KEY_OK, STORE_KEY_FATAL, STORE_KEY_CONV }; store_key(THD *thd, Field *field_arg, char *ptr, char *null, uint length) :null_ptr(null),err(0) { @@ -371,7 +372,7 @@ class store_key :public Sql_alloc } } virtual ~store_key() {} /* Not actually needed */ - virtual bool copy()=0; + virtual enum store_key_result copy()=0; virtual const char *name() const=0; }; @@ -392,10 +393,10 @@ class store_key_field: public store_key copy_field.set(to_field,from_field,0); } } - bool copy() + enum store_key_result copy() { copy_field.do_copy(©_field); - return err != 0; + return err != 0 ? STORE_KEY_FATAL : STORE_KEY_OK; } const char *name() const { return field_name; } }; @@ -412,9 +413,11 @@ public: null_ptr_arg ? null_ptr_arg : item_arg->maybe_null ? &err : NullS, length), item(item_arg) {} - bool copy() + enum store_key_result copy() { - return item->save_in_field(to_field, 1) || err != 0; + int res= item->save_in_field(to_field, 1); + return (err != 0 || res > 2 ? STORE_KEY_FATAL : (store_key_result) res); + } const char *name() const { return "func"; } }; @@ -432,15 +435,19 @@ public: &err : NullS, length, item_arg), inited(0) { } - bool copy() + enum store_key_result copy() { + int res; if (!inited) { inited=1; - if (item->save_in_field(to_field, 1)) - err= 1; + if ((res= item->save_in_field(to_field, 1))) + { + if (!err) + err= res; + } } - return err != 0; + return (err > 2 ? STORE_KEY_FATAL : (store_key_result) err); } const char *name() const { return "const"; } }; From a5e742fedd4324d29867a15a6cabb54959108fbb Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 18:29:10 +0300 Subject: [PATCH 09/32] fixed environment restoring in case of error during SP function execution (BUG#9503) #define macro improvement mysql-test/r/sp-security.result: BUG#9503: reseting correct parameters of thread after error in SP function mysql-test/t/sp-security.test: BUG#9503: reseting correct parameters of thread after error in SP function sql/item_func.cc: fixed environment restoring in case of error during SP function execution sql/protocol.cc: added debug print sql/sql_class.h: fixed #defines to force them to be alvaise in piar, and variable name made more complex for accident repeating in other code --- mysql-test/r/sp-security.result | 17 ++++++++++++++++ mysql-test/t/sp-security.test | 36 +++++++++++++++++++++++++++++++++ sql/item_func.cc | 29 +++++++++++++------------- sql/protocol.cc | 5 +++++ sql/sql_class.h | 6 +++--- 5 files changed, 76 insertions(+), 17 deletions(-) diff --git a/mysql-test/r/sp-security.result b/mysql-test/r/sp-security.result index ee72fde7324..4ace6f59411 100644 --- a/mysql-test/r/sp-security.result +++ b/mysql-test/r/sp-security.result @@ -194,3 +194,20 @@ use test; drop database sptest; delete from mysql.user where user='usera' or user='userb' or user='userc'; delete from mysql.procs_priv where user='usera' or user='userb' or user='userc'; +drop function if exists bug_9503; +create database mysqltest// +use mysqltest// +create table t1 (s1 int)// +grant select on t1 to user1@localhost// +create function bug_9503 () returns int sql security invoker begin declare v int; +select min(s1) into v from t1; return v; end// +use mysqltest; +select bug_9503(); +ERROR 42000: execute command denied to user 'user1'@'localhost' for routine 'mysqltest.bug_9503' +grant execute on function bug_9503 to user1@localhost; +do 1; +use test; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM user1@localhost; +drop function bug_9503; +use test; +drop database mysqltest; diff --git a/mysql-test/t/sp-security.test b/mysql-test/t/sp-security.test index e1d8043ccda..72fe6c332bf 100644 --- a/mysql-test/t/sp-security.test +++ b/mysql-test/t/sp-security.test @@ -304,3 +304,39 @@ drop database sptest; delete from mysql.user where user='usera' or user='userb' or user='userc'; delete from mysql.procs_priv where user='usera' or user='userb' or user='userc'; +# +# BUG#9503: reseting correct parameters of thread after error in SP function +# +connect (root,localhost,root,,test); +connection root; + +--disable_warnings +drop function if exists bug_9503; +--enable_warnings +delimiter //; +create database mysqltest// +use mysqltest// +create table t1 (s1 int)// +grant select on t1 to user1@localhost// +create function bug_9503 () returns int sql security invoker begin declare v int; +select min(s1) into v from t1; return v; end// +delimiter ;// + +connect (user1,localhost,user1,,test); +connection user1; +use mysqltest; +-- error 1370 +select bug_9503(); + +connection root; +grant execute on function bug_9503 to user1@localhost; + +connection user1; +do 1; +use test; + +connection root; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM user1@localhost; +drop function bug_9503; +use test; +drop database mysqltest; diff --git a/sql/item_func.cc b/sql/item_func.cc index 57f68bbc2a0..de497014240 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -4807,42 +4807,37 @@ Item_func_sp::execute(Item **itp) DBUG_ENTER("Item_func_sp::execute"); THD *thd= current_thd; ulong old_client_capabilites; - int res; + int res= -1; bool save_in_sub_stmt= thd->transaction.in_sub_stmt; + my_bool nsok; #ifndef NO_EMBEDDED_ACCESS_CHECKS st_sp_security_context save_ctx; #endif - if (! m_sp) + if (! m_sp && ! (m_sp= sp_find_function(thd, m_name, TRUE))) { - if (!(m_sp= sp_find_function(thd, m_name, TRUE))) - { - my_error(ER_SP_DOES_NOT_EXIST, MYF(0), "FUNCTION", m_name->m_qname.str); - DBUG_RETURN(-1); - } + my_error(ER_SP_DOES_NOT_EXIST, MYF(0), "FUNCTION", m_name->m_qname.str); + goto error; } old_client_capabilites= thd->client_capabilities; thd->client_capabilities &= ~CLIENT_MULTI_RESULTS; #ifndef EMBEDDED_LIBRARY - my_bool nsok= thd->net.no_send_ok; + nsok= thd->net.no_send_ok; thd->net.no_send_ok= TRUE; #endif + res= -1; #ifndef NO_EMBEDDED_ACCESS_CHECKS if (check_routine_access(thd, EXECUTE_ACL, m_sp->m_db.str, m_sp->m_name.str, 0, 0)) - DBUG_RETURN(-1); + goto error_check; sp_change_security_context(thd, m_sp, &save_ctx); if (save_ctx.changed && check_routine_access(thd, EXECUTE_ACL, m_sp->m_db.str, m_sp->m_name.str, 0, 0)) - { - sp_restore_security_context(thd, m_sp, &save_ctx); - thd->client_capabilities|= old_client_capabilites & CLIENT_MULTI_RESULTS; - DBUG_RETURN(-1); - } + goto error_check; #endif /* Like for SPs, we don't binlog the substatements. If the statement which @@ -4850,6 +4845,7 @@ Item_func_sp::execute(Item **itp) it's not (e.g. SELECT myfunc()) it won't be binlogged (documented known problem). */ + tmp_disable_binlog(thd); /* don't binlog the substatements */ thd->transaction.in_sub_stmt= TRUE; @@ -4864,16 +4860,21 @@ Item_func_sp::execute(Item **itp) ER_FAILED_ROUTINE_BREAK_BINLOG, ER(ER_FAILED_ROUTINE_BREAK_BINLOG)); +error_check_ctx: #ifndef NO_EMBEDDED_ACCESS_CHECKS sp_restore_security_context(thd, m_sp, &save_ctx); #endif + thd->client_capabilities|= old_client_capabilites & CLIENT_MULTI_RESULTS; + +error_check: #ifndef EMBEDDED_LIBRARY thd->net.no_send_ok= nsok; #endif thd->client_capabilities|= old_client_capabilites & CLIENT_MULTI_RESULTS; +error: DBUG_RETURN(res); } diff --git a/sql/protocol.cc b/sql/protocol.cc index 57922cdc677..1c399a89a99 100644 --- a/sql/protocol.cc +++ b/sql/protocol.cc @@ -294,7 +294,12 @@ send_ok(THD *thd, ha_rows affected_rows, ulonglong id, const char *message) DBUG_ENTER("send_ok"); if (net->no_send_ok || !net->vio) // hack for re-parsing queries + { + DBUG_PRINT("info", ("no send ok: %s, vio present: %s", + (net->no_send_ok ? "YES" : "NO"), + (net->vio ? "YES" : "NO"))); DBUG_VOID_RETURN; + } buff[0]=0; // No fields pos=net_store_length(buff+1,(ulonglong) affected_rows); diff --git a/sql/sql_class.h b/sql/sql_class.h index dd4b8310e51..9f263dbebf3 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -1433,10 +1433,10 @@ public: }; #define tmp_disable_binlog(A) \ - ulong save_options= (A)->options; \ - (A)->options&= ~OPTION_BIN_LOG; + {ulong tmp_disable_binlog__save_options= (A)->options; \ + (A)->options&= ~OPTION_BIN_LOG -#define reenable_binlog(A) (A)->options= save_options; +#define reenable_binlog(A) (A)->options= tmp_disable_binlog__save_options;} /* Flags for the THD::system_thread (bitmap) variable */ #define SYSTEM_THREAD_DELAYED_INSERT 1 From 744f6a1a9966b83e21ae39270d07c1da6ba87fbf Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 17:38:40 +0200 Subject: [PATCH 10/32] mysql-test-run.pl: Might need a restart after test with special TZ Removed unused argument to run_mysqltest() mysql-test/mysql-test-run.pl: Might need a restart after test with special TZ Removed unused argument to run_mysqltest() --- mysql-test/mysql-test-run.pl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 4f673fe567d..1a31bbee1d5 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -303,7 +303,7 @@ sub mysqld_arguments ($$$$$); sub stop_masters_slaves (); sub stop_masters (); sub stop_slaves (); -sub run_mysqltest ($$); +sub run_mysqltest ($); sub usage ($); ###################################################################### @@ -1345,10 +1345,11 @@ sub run_testcase ($) { if ( ! $glob_use_running_server and ! $glob_use_embedded_server ) { - if ( $tinfo->{'master_restart'} or $master->[0]->{'uses_special_flags'} ) + if ( $tinfo->{'master_restart'} or + $master->[0]->{'running_master_is_special'} ) { stop_masters(); - $master->[0]->{'uses_special_flags'}= 0; # Forget about why we stopped + $master->[0]->{'running_master_is_special'}= 0; # Forget why we stopped } # ---------------------------------------------------------------------- @@ -1426,9 +1427,9 @@ sub run_testcase ($) { } } - if ( @{$tinfo->{'master_opt'}} ) + if ( $tinfo->{'master_restart'} ) { - $master->[0]->{'uses_special_flags'}= 1; + $master->[0]->{'running_master_is_special'}= 1; } } @@ -1475,7 +1476,7 @@ sub run_testcase ($) { } unlink($path_timefile); - my $res= run_mysqltest($tinfo, $tinfo->{'master_opt'}); + my $res= run_mysqltest($tinfo); if ( $res == 0 ) { @@ -1975,9 +1976,8 @@ sub stop_slaves () { } -sub run_mysqltest ($$) { +sub run_mysqltest ($) { my $tinfo= shift; - my $master_opts= shift; my $cmdline_mysqldump= "$exe_mysqldump --no-defaults -uroot " . "--socket=$master->[0]->{'path_mysock'} --password="; From 91180cb8ab4924fc9907ee16ba3479f88c309bc2 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 20:22:08 +0400 Subject: [PATCH 11/32] - implement inheritance of sp_instr: public Query_arena. We need every instruction to have its own arena, because we want to track instruction's state (INITIALIZED_FOR_SP -> EXECUTED). Because of `if' statements and other conditional instructions used in stored procedures, not every instruction of a stored procedure gets executed during the first (or even subsequent) execution of the procedure. So it's better if we track the execution state of every instruction independently. All instructions of a given procedure now also share sp_head's mem_root, but keep their own free_list. This simplifies juggling with free Item lists in sp_head::execute. - free_items() moved to be a member of Query_arena. - logic of 'backup_arena' debug member of Query_arena has been changed to support multi-backups. Until now, TRUE 'backup_arena' meant that there is exactly one active backup of the THD arena. Now it means simply that the arena is used for backup, so that we can't accidentally overwrite an existing backup. This allows doing multiple backups, e.g. in sp_head::execute and Cursor::fetch, when THD arena is already backed up but we want to set yet another arena (usually the 'permanent' arena, to save permanent transformations/optimizations of a parsed tree). sql/sp_head.cc: - use Query_arena support in sp_head::execute() as now sp_instr inherites from it. sql/sp_head.h: - inherite sp_instr from Query_arena sql/sql_class.cc: - changed the principle of Query_arena::backup_arena; free_items is now a member of Query_arena. sql/sql_class.h: - changed the principle of Query_arena::backup_arena; free_items is now a member of Query_arena. sql/sql_prepare.cc: free_items() is now a member of Query_arena. sql/sql_select.cc: free_items() now automatically sets free_list to zero. --- sql/sp_head.cc | 71 ++++++++++++++++++++-------------------------- sql/sp_head.h | 7 ++--- sql/sql_class.cc | 27 +++++++++++++----- sql/sql_class.h | 13 +++++++-- sql/sql_prepare.cc | 2 +- sql/sql_select.cc | 3 +- 6 files changed, 65 insertions(+), 58 deletions(-) diff --git a/sql/sp_head.cc b/sql/sp_head.cc index 29cee6da4d3..825ec34e410 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -509,7 +509,7 @@ sp_head::destroy() delete i; delete_dynamic(&m_instr); m_pcont->destroy(); - free_items(free_list); + free_items(); /* If we have non-empty LEX stack then we just came out of parser with @@ -596,7 +596,6 @@ sp_head::execute(THD *thd) ctx->clear_handler(); thd->query_error= 0; old_arena= thd->current_arena; - thd->current_arena= this; /* We have to save/restore this info when we are changing call level to @@ -636,23 +635,18 @@ sp_head::execute(THD *thd) break; DBUG_PRINT("execute", ("Instruction %u", ip)); thd->set_time(); // Make current_time() et al work - { - /* - We have to substitute free_list of executing statement to - current_arena to store there all new items created during execution - (for example '*' expanding, or items made during permanent subquery - transformation) - Note: Every statement have to have all its items listed in free_list - for correct cleaning them up - */ - Item *save_free_list= thd->current_arena->free_list; - thd->current_arena->free_list= i->free_list; - ret= i->execute(thd, &ip); - i->free_list= thd->current_arena->free_list; - thd->current_arena->free_list= save_free_list; - } + /* + We have to set thd->current_arena before executing the instruction + to store in the instruction free_list all new items, created + during the first execution (for example expanding of '*' or the + items made during other permanent subquery transformations). + */ + thd->current_arena= i; + ret= i->execute(thd, &ip); if (i->free_list) cleanup_items(i->free_list); + i->state= Query_arena::EXECUTED; + // Check if an exception has occurred and a handler has been found // Note: We havo to check even if ret==0, since warnings (and some // errors don't return a non-zero value. @@ -694,7 +688,6 @@ sp_head::execute(THD *thd) DBUG_ASSERT(!thd->derived_tables); thd->derived_tables= old_derived_tables; - cleanup_items(thd->current_arena->free_list); thd->current_arena= old_arena; state= EXECUTED; @@ -728,8 +721,8 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, Item **resp) sp_rcontext *nctx = NULL; uint i; int ret; - MEM_ROOT *old_mem_root, call_mem_root; - Item *old_free_list, *call_free_list; + MEM_ROOT call_mem_root; + Query_arena call_arena(&call_mem_root, INITIALIZED_FOR_SP), backup_arena; if (argcount != params) { @@ -741,14 +734,12 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, Item **resp) } init_alloc_root(&call_mem_root, MEM_ROOT_BLOCK_SIZE, 0); - old_mem_root= thd->mem_root; - thd->mem_root= &call_mem_root; - old_free_list= thd->free_list; // Keep the old list - thd->free_list= NULL; // Start a new one + + thd->set_n_backup_item_arena(&call_arena, &backup_arena); // QQ Should have some error checking here? (types, etc...) nctx= new sp_rcontext(csize, hmax, cmax); - nctx->callers_mem_root= old_mem_root; + nctx->callers_mem_root= backup_arena.mem_root; for (i= 0 ; i < argcount ; i++) { sp_pvar_t *pvar = m_pcont->find_pvar(i); @@ -780,9 +771,7 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, Item **resp) // Partially restore context now. // We still need the call mem root and free list for processing // of the result. - call_free_list= thd->free_list; - thd->free_list= old_free_list; - thd->mem_root= old_mem_root; + thd->restore_backup_item_arena(&call_arena, &backup_arena); if (m_type == TYPE_ENUM_FUNCTION && ret == 0) { @@ -802,8 +791,7 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, Item **resp) thd->spcont= octx; // Now get rid of the rest of the callee context - cleanup_items(call_free_list); - free_items(call_free_list); + call_arena.free_items(); free_root(&call_mem_root, MYF(0)); DBUG_RETURN(ret); @@ -835,8 +823,8 @@ sp_head::execute_procedure(THD *thd, List *args) sp_rcontext *octx = thd->spcont; sp_rcontext *nctx = NULL; my_bool tmp_octx = FALSE; // True if we have allocated a temporary octx - MEM_ROOT *old_mem_root, call_mem_root; - Item *old_free_list, *call_free_list; + MEM_ROOT call_mem_root; + Query_arena call_arena(&call_mem_root, INITIALIZED_FOR_SP), backup_arena; if (args->elements != params) { @@ -846,10 +834,7 @@ sp_head::execute_procedure(THD *thd, List *args) } init_alloc_root(&call_mem_root, MEM_ROOT_BLOCK_SIZE, 0); - old_mem_root= thd->mem_root; - thd->mem_root= &call_mem_root; - old_free_list= thd->free_list; // Keep the old list - thd->free_list= NULL; // Start a new one + thd->set_n_backup_item_arena(&call_arena, &backup_arena); if (csize > 0 || hmax > 0 || cmax > 0) { @@ -919,9 +904,7 @@ sp_head::execute_procedure(THD *thd, List *args) // Partially restore context now. // We still need the call mem root and free list for processing // of out parameters. - call_free_list= thd->free_list; - thd->free_list= old_free_list; - thd->mem_root= old_mem_root; + thd->restore_backup_item_arena(&call_arena, &backup_arena); if (!ret && csize > 0) { @@ -996,8 +979,7 @@ sp_head::execute_procedure(THD *thd, List *args) thd->spcont= octx; // Now get rid of the rest of the callee context - cleanup_items(call_free_list); - free_items(call_free_list); + call_arena.free_items(); thd->lex->unit.cleanup(); free_root(&call_mem_root, MYF(0)); @@ -1291,6 +1273,13 @@ void sp_head::add_instr(sp_instr *instr) { instr->free_list= m_thd->free_list; m_thd->free_list= 0; + /* + Memory root of every instruction is designated for permanent + transformations (optimizations) made on the parsed tree during + the first execution. It points to the memory root of the + entire stored procedure, as their life span is equal. + */ + instr->mem_root= &main_mem_root; insert_dynamic(&m_instr, (gptr)&instr); } diff --git a/sql/sp_head.h b/sql/sp_head.h index 2c75a320f30..aaef5a3d50e 100644 --- a/sql/sp_head.h +++ b/sql/sp_head.h @@ -274,7 +274,7 @@ private: // "Instructions"... // -class sp_instr : public Sql_alloc +class sp_instr :public Query_arena, public Sql_alloc { sp_instr(const sp_instr &); /* Prevent use of these */ void operator=(sp_instr &); @@ -282,17 +282,16 @@ class sp_instr : public Sql_alloc public: uint marked; - Item *free_list; // My Items uint m_ip; // My index sp_pcontext *m_ctx; // My parse context // Should give each a name or type code for debugging purposes? sp_instr(uint ip, sp_pcontext *ctx) - :Sql_alloc(), marked(0), free_list(0), m_ip(ip), m_ctx(ctx) + :Query_arena(0, INITIALIZED_FOR_SP), marked(0), m_ip(ip), m_ctx(ctx) {} virtual ~sp_instr() - { free_items(free_list); } + { free_items(); } // Execute this instrution. '*nextp' will be set to the index of the next // instruction to execute. (For most instruction this will be the diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 8abd7cbbe7d..20f48da9283 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -171,9 +171,6 @@ THD::THD() spcont(NULL) { current_arena= this; -#ifndef DBUG_OFF - backup_arena= 0; -#endif host= user= priv_user= db= ip= 0; catalog= (char*)"std"; // the only catalog we have for now host_or_ip= "connecting host"; @@ -528,7 +525,7 @@ void THD::cleanup_after_query() next_insert_id= 0; } /* Free Items that were created during this execution */ - free_items(free_list); + free_items(); /* In the rest of code we assume that free_list never points to garbage: Keep this predicate true. @@ -1485,6 +1482,21 @@ Query_arena::Type Query_arena::type() const } +void Query_arena::free_items() +{ + Item *next; + DBUG_ENTER("Query_arena::free_items"); + /* This works because items are allocated with sql_alloc() */ + for (; free_list; free_list= next) + { + next= free_list->next; + free_list->delete_self(); + } + /* Postcondition: free_list is 0 */ + DBUG_VOID_RETURN; +} + + /* Statement functions */ @@ -1556,11 +1568,11 @@ void THD::end_statement() void Query_arena::set_n_backup_item_arena(Query_arena *set, Query_arena *backup) { DBUG_ENTER("Query_arena::set_n_backup_item_arena"); - DBUG_ASSERT(backup_arena == 0); + DBUG_ASSERT(backup->is_backup_arena == FALSE); backup->set_item_arena(this); set_item_arena(set); #ifndef DBUG_OFF - backup_arena= 1; + backup->is_backup_arena= TRUE; #endif DBUG_VOID_RETURN; } @@ -1569,10 +1581,11 @@ void Query_arena::set_n_backup_item_arena(Query_arena *set, Query_arena *backup) void Query_arena::restore_backup_item_arena(Query_arena *set, Query_arena *backup) { DBUG_ENTER("Query_arena::restore_backup_item_arena"); + DBUG_ASSERT(backup->is_backup_arena); set->set_item_arena(this); set_item_arena(backup); #ifndef DBUG_OFF - backup_arena= 0; + backup->is_backup_arena= FALSE; #endif DBUG_VOID_RETURN; } diff --git a/sql/sql_class.h b/sql/sql_class.h index a635a126f84..adc164085f9 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -663,7 +663,10 @@ public: Item *free_list; MEM_ROOT *mem_root; // Pointer to current memroot #ifndef DBUG_OFF - bool backup_arena; + bool is_backup_arena; /* True if this arena is used for backup. */ +#define INIT_ARENA_DBUG_INFO is_backup_arena= 0 +#else +#define INIT_ARENA_DBUG_INFO #endif enum enum_state { @@ -681,12 +684,14 @@ public: Query_arena(MEM_ROOT *mem_root_arg, enum enum_state state_arg) : free_list(0), mem_root(mem_root_arg), state(state_arg) - {} + { INIT_ARENA_DBUG_INFO; } /* This constructor is used only when Query_arena is created as backup storage for another instance of Query_arena. */ - Query_arena() {}; + Query_arena() { INIT_ARENA_DBUG_INFO; } + +#undef INIT_ARENA_DBUG_INFO virtual Type type() const; virtual ~Query_arena() {}; @@ -726,6 +731,8 @@ public: void set_n_backup_item_arena(Query_arena *set, Query_arena *backup); void restore_backup_item_arena(Query_arena *set, Query_arena *backup); void set_item_arena(Query_arena *set); + + void free_items(); }; diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index eeea493d868..c97cb037f15 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -2429,7 +2429,7 @@ Prepared_statement::~Prepared_statement() { if (cursor) cursor->Cursor::~Cursor(); - free_items(free_list); + free_items(); delete lex->result; } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 1487dfbb436..96a25c7919b 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1914,8 +1914,7 @@ Cursor::close() } join= 0; unit= 0; - free_items(free_list); - free_list= 0; + free_items(); /* Must be last, as some memory might be allocated for free purposes, like in free_tmp_table() (TODO: fix this issue) From 74307f39ebee0a1e4a14d9a989fbc92fa97a8f16 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 21:29:44 +0500 Subject: [PATCH 12/32] WL#2286 - Compile MySQL w/YASSL support Fix for compilation failure with Forte Developer C++. configure.in: Export ARFLAGS, so innobase could pick it up. innobase/configure.in: Use ARFLAGS exported by parent configure script. --- configure.in | 2 +- innobase/configure.in | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/configure.in b/configure.in index ae26a0c44df..58682063593 100644 --- a/configure.in +++ b/configure.in @@ -342,7 +342,7 @@ AC_SUBST(CXXFLAGS) AC_SUBST(LD) AC_SUBST(INSTALL_SCRIPT) -export CC CXX CFLAGS LD LDFLAGS AR +export CC CXX CFLAGS LD LDFLAGS AR ARFLAGS if test "$GCC" = "yes" then diff --git a/innobase/configure.in b/innobase/configure.in index baf11272ab9..c56bd8274c4 100644 --- a/innobase/configure.in +++ b/innobase/configure.in @@ -117,6 +117,13 @@ case "$target" in CFLAGS="$CFLAGS -DUNIV_MUST_NOT_INLINE";; esac +# must go in pair with AR as set by MYSQL_CHECK_AR +if test -z "$ARFLAGS" +then + ARFLAGS="cru" +fi +AC_SUBST(ARFLAGS) + AC_OUTPUT(Makefile os/Makefile ut/Makefile btr/Makefile dnl buf/Makefile data/Makefile dnl dict/Makefile dyn/Makefile dnl From 2460a01b684edc9e064991d2cd4e64985765c449 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 19:33:19 +0300 Subject: [PATCH 13/32] removed unneed line --- sql/item_func.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/item_func.cc b/sql/item_func.cc index de497014240..1dbf28b67cb 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -4828,7 +4828,6 @@ Item_func_sp::execute(Item **itp) thd->net.no_send_ok= TRUE; #endif - res= -1; #ifndef NO_EMBEDDED_ACCESS_CHECKS if (check_routine_access(thd, EXECUTE_ACL, m_sp->m_db.str, m_sp->m_name.str, 0, 0)) From 624cc5223c09bf5863667e8f2da69ac23ed7a819 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 19:03:55 +0200 Subject: [PATCH 14/32] - reverted to using the shell version of mysql-test-run for "make test" until the remaining bugs in the Perl version have been resolved Makefile.am: - reverted to using the shell version of mysql-test-run until the remaining bugs in the Perl version have been resolved --- Makefile.am | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 9a6fcf3c95a..d7059c6adaf 100644 --- a/Makefile.am +++ b/Makefile.am @@ -101,12 +101,12 @@ tags: test: cd mysql-test; \ - perl mysql-test-run.pl && perl mysql-test-run.pl --ps-protocol + ./mysql-test-run && ./mysql-test-run --ps-protocol test-force: cd mysql-test; \ - perl mysql-test-run.pl --force ;\ - perl mysql-test-run.pl --ps-protocol --force + ./mysql-test-run --force ;\ + ./mysql-test-run --ps-protocol --force # Don't update the files from bitkeeper %::SCCS/s.% From d06446af84a875faa5b642cd21e759ef9fa9fa20 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 21:06:33 +0400 Subject: [PATCH 15/32] sql_parse.cc: Fix for fix for bug #9728 Error caused server hang on prepared insert ... select sql/sql_parse.cc: Fix for fix for bug #9728 Error caused server hang on prepared insert ... select --- sql/sql_parse.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 233104c9a90..2eeae8f7332 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2882,7 +2882,6 @@ unsent_create_error: } else res= -1; - first_local_table->next= tables; lex->select_lex.table_list.first= (byte*) first_local_table; break; } From 9a03ad151da975badcccaf7bf3517057aef8e5b8 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Jun 2005 22:41:26 +0200 Subject: [PATCH 16/32] - bumped up version number to 5.0.9 now that the 5.0.8 builds have been branched off --- configure.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.in b/configure.in index 58682063593..9e294a35223 100644 --- a/configure.in +++ b/configure.in @@ -6,7 +6,7 @@ AC_PREREQ(2.50)dnl Minimum Autoconf version required. AC_INIT(sql/mysqld.cc) AC_CANONICAL_SYSTEM # Don't forget to also update the NDB lines below. -AM_INIT_AUTOMAKE(mysql, 5.0.8-beta) +AM_INIT_AUTOMAKE(mysql, 5.0.9-beta) AM_CONFIG_HEADER(config.h) PROTOCOL_VERSION=10 @@ -17,7 +17,7 @@ SHARED_LIB_VERSION=14:0:0 # ndb version NDB_VERSION_MAJOR=5 NDB_VERSION_MINOR=0 -NDB_VERSION_BUILD=8 +NDB_VERSION_BUILD=9 NDB_VERSION_STATUS="beta" # Set all version vars based on $VERSION. How do we do this more elegant ? From 3d1172a10f3365b29bce370f37b8740d1798d11e Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 00:24:11 +0300 Subject: [PATCH 17/32] fixed encrypt() print (BUG#7024) mysql-test/r/view.result: using encrypt & substring_index in view mysql-test/t/view.test: using encrypt & substring_index in view sql/item_strfunc.h: fixed encrypt() print --- mysql-test/r/view.result | 8 ++++++++ mysql-test/t/view.test | 12 ++++++++++++ sql/item_strfunc.h | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/view.result b/mysql-test/r/view.result index 68cc0c4cb57..d4d6eb08cad 100644 --- a/mysql-test/r/view.result +++ b/mysql-test/r/view.result @@ -1831,3 +1831,11 @@ select * from v1; t 01:00 drop view v1; +CREATE VIEW v1 AS SELECT ENCRYPT("dhgdhgd"); +SELECT * FROM v1; +drop view v1; +CREATE VIEW v1 AS SELECT SUBSTRING_INDEX("dkjhgd:kjhdjh", ":", 1); +SELECT * FROM v1; +SUBSTRING_INDEX("dkjhgd:kjhdjh", ":", 1) +dkjhgd +drop view v1; diff --git a/mysql-test/t/view.test b/mysql-test/t/view.test index 13a5f8cef1f..f131f9d2604 100644 --- a/mysql-test/t/view.test +++ b/mysql-test/t/view.test @@ -1673,3 +1673,15 @@ create view v1(k, K) as select 1,2; create view v1 as SELECT TIME_FORMAT(SEC_TO_TIME(3600),'%H:%i') as t; select * from v1; drop view v1; + +# +# using encrypt & substring_index in view (BUG#7024) +# +CREATE VIEW v1 AS SELECT ENCRYPT("dhgdhgd"); +disable_result_log; +SELECT * FROM v1; +enable_result_log; +drop view v1; +CREATE VIEW v1 AS SELECT SUBSTRING_INDEX("dkjhgd:kjhdjh", ":", 1); +SELECT * FROM v1; +drop view v1; diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 8d2eb269915..c4beb3b08cb 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -322,7 +322,7 @@ public: Item_func_encrypt(Item *a, Item *b): Item_str_func(a,b) {} String *val_str(String *); void fix_length_and_dec() { maybe_null=1; max_length = 13; } - const char *func_name() const { return "ecrypt"; } + const char *func_name() const { return "encrypt"; } }; #include "sql_crypt.h" From fe80b7b5bf7a3d2191d268db7bff4704c9fdb022 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 01:30:52 +0300 Subject: [PATCH 18/32] fixed encrypt() name --- mysql-test/r/subselect.result | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index 736559f8569..400a2be01f1 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -1025,7 +1025,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 system NULL NULL NULL NULL 0 const row not found 2 UNCACHEABLE SUBQUERY t1 system NULL NULL NULL NULL 0 const row not found Warnings: -Note 1003 select sql_no_cache (select sql_no_cache ecrypt(_latin1'test') AS `ENCRYPT('test')` from `test`.`t1`) AS `(SELECT ENCRYPT('test') FROM t1)` from `test`.`t1` +Note 1003 select sql_no_cache (select sql_no_cache encrypt(_latin1'test') AS `ENCRYPT('test')` from `test`.`t1`) AS `(SELECT ENCRYPT('test') FROM t1)` from `test`.`t1` EXPLAIN EXTENDED SELECT (SELECT BENCHMARK(1,1) FROM t1) FROM t1; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 system NULL NULL NULL NULL 0 const row not found From 3c14168865716d329058876c284bf0e29538ab32 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 09:56:58 +0300 Subject: [PATCH 19/32] Fixed test result for BUG#11185. --- mysql-test/r/innodb.result | 5 +++-- mysql-test/r/range.result | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/innodb.result b/mysql-test/r/innodb.result index b07a6b03c8a..838e02c3a89 100644 --- a/mysql-test/r/innodb.result +++ b/mysql-test/r/innodb.result @@ -1721,12 +1721,13 @@ count(*) 0 explain select count(*) from t1 where x > -16; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range PRIMARY PRIMARY 8 NULL 1 Using where; Using index +1 SIMPLE t1 index PRIMARY PRIMARY 8 NULL 2 Using where; Using index select count(*) from t1 where x > -16; count(*) -1 +2 select * from t1 where x > -16; x +18446744073709551600 18446744073709551601 select count(*) from t1 where x = 18446744073709551601; count(*) diff --git a/mysql-test/r/range.result b/mysql-test/r/range.result index 16bf9a8247a..ae7d0474e24 100644 --- a/mysql-test/r/range.result +++ b/mysql-test/r/range.result @@ -544,7 +544,7 @@ count(*) 1 select count(*) from t2 where x > -16; count(*) -2 +1 select count(*) from t2 where x = 18446744073709551601; count(*) 0 From 77dc5c423fb9f22dfc931d1fd1a00715c1fca0e2 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 12:51:11 +0500 Subject: [PATCH 20/32] an improvement (bug #7851: C++ 'new' conflicts with kernel header asm/system.h). include/my_global.h: an improvement (bug #7851: C++ 'new' conflicts with kernel header asm/system.h). redefine 'new' before #include in any case. --- include/my_global.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/my_global.h b/include/my_global.h index f8ba555b150..9b53be66db0 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -274,10 +274,8 @@ C_MODE_START int __cxa_pure_virtual() {\ #include #endif #ifdef HAVE_ATOMIC_ADD -#if defined(__ia64__) #define new my_arg_new #define need_to_restore_new 1 -#endif C_MODE_START #include C_MODE_END From a60233121f524fc326d90bff7dcbf08dbd1cc959 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 01:21:18 -0700 Subject: [PATCH 21/32] field.cc: Correction after manula merge. sql/field.cc: Correction after manula merge. --- sql/field.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/field.cc b/sql/field.cc index 81f695454ff..fb244de4275 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -3317,7 +3317,7 @@ int Field_long::store(const char *from,uint len,CHARSET_INFO *cs) } if (error) { - error= 1; + error= error != MY_ERRNO_EDOM ? 1 : 2; set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, 1); } else if (from+len != end && table->in_use->count_cuted_fields && From 5aa793f72bfdf5e5903ad4999691fdb72f507de3 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 14:04:48 +0500 Subject: [PATCH 22/32] backport for #10568: Function 'LAST_DAY(date)' does not return NULL for invalid argument. --- mysql-test/r/func_time.result | 15 +++++++++++++++ mysql-test/t/func_time.test | 7 +++++++ sql/item_timefunc.cc | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index 4dd00ab74a1..fc872285acb 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -611,3 +611,18 @@ SELECT count(*) FROM t1 WHERE d>FROM_DAYS(TO_DAYS(@TMP)) AND d<=FROM_DAYS(TO_DAY count(*) 3 DROP TABLE t1; +select last_day('2005-00-00'); +last_day('2005-00-00') +NULL +Warnings: +Warning 1292 Truncated incorrect datetime value: '2005-00-00' +select last_day('2005-00-01'); +last_day('2005-00-01') +NULL +Warnings: +Warning 1292 Truncated incorrect datetime value: '2005-00-01' +select last_day('2005-01-00'); +last_day('2005-01-00') +NULL +Warnings: +Warning 1292 Truncated incorrect datetime value: '2005-01-00' diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index 0f495ef891d..9e2703da110 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -307,3 +307,10 @@ INSERT INTO t1 VALUES (NOW()); SELECT count(*) FROM t1 WHERE d>FROM_DAYS(TO_DAYS(@TMP)) AND d<=FROM_DAYS(TO_DAYS(@TMP)+1); DROP TABLE t1; +# +# Bug #10568 +# + +select last_day('2005-00-00'); +select last_day('2005-00-01'); +select last_day('2005-01-00'); diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index a3cf69035f3..0e1a8766e8f 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -2817,7 +2817,7 @@ String *Item_func_str_to_date::val_str(String *str) bool Item_func_last_day::get_date(TIME *ltime, uint fuzzy_date) { - if (get_arg0_date(ltime,fuzzy_date)) + if (get_arg0_date(ltime, fuzzy_date & ~TIME_FUZZY_DATE)) return 1; uint month_idx= ltime->month-1; ltime->day= days_in_month[month_idx]; From b743e2b4faedd59f655fa8cb31ee1bfcae430c70 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 11:56:20 +0200 Subject: [PATCH 23/32] mysql-test-run.sh: Corrected path to CA certificate mysql-test/mysql-test-run.sh: Corrected path to CA certificate --- mysql-test/mysql-test-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/mysql-test-run.sh b/mysql-test/mysql-test-run.sh index e3fcfdf2b1e..376946eb8aa 100644 --- a/mysql-test/mysql-test-run.sh +++ b/mysql-test/mysql-test-run.sh @@ -307,7 +307,7 @@ while test $# -gt 0; do --ssl-ca=$MYSQL_TEST_DIR/std_data/cacert.pem \ --ssl-cert=$MYSQL_TEST_DIR/std_data/server-cert.pem \ --ssl-key=$MYSQL_TEST_DIR/std_data/server-key.pem" - MYSQL_TEST_SSL_OPTS="--ssl-ca=$BASEDIR/SSL/cacert.pem \ + MYSQL_TEST_SSL_OPTS="--ssl-ca=$MYSQL_TEST_DIR/std_data/cacert.pem \ --ssl-cert=$MYSQL_TEST_DIR/std_data/client-cert.pem \ --ssl-key=$MYSQL_TEST_DIR/std_data/client-key.pem" ;; --no-manager | --skip-manager) USE_MANAGER=0 ;; From a7e66efc2cc37b8ca32868ac8c8e3e3c05675e48 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 17:47:09 +0200 Subject: [PATCH 24/32] Bug#10178 - failure to find a row in heap table by concurrent UPDATEs Moved the key statistics update to info(). The table is not locked in open(). This made wrong stats possible. No test case for the test suite. This happens only with heavy concurrency. A test script is added to the bug report. mysql-test/r/heap_hash.result: Bug#10178 - failure to find a row in heap table by concurrent UPDATEs Updated test results to reflect the new statistics behaviour. mysql-test/t/heap_hash.test: Bug#10178 - failure to find a row in heap table by concurrent UPDATEs Added a FLUSH TABLES to avoid statistics differences between normal and ps-protocol tests. sql/ha_heap.cc: Bug#10178 - failure to find a row in heap table by concurrent UPDATEs Moved the key statistics update to info(). The table is not locked in open(). This made wrong stats possible. sql/ha_heap.h: Bug#10178 - failure to find a row in heap table by concurrent UPDATEs Added an element to track the validity of the key statistics. --- mysql-test/r/heap_hash.result | 23 ++++++++++++----------- mysql-test/t/heap_hash.test | 2 ++ sql/ha_heap.cc | 31 ++++++++++++++++++++++++++----- sql/ha_heap.h | 4 +++- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/mysql-test/r/heap_hash.result b/mysql-test/r/heap_hash.result index d3673cd2a63..e5098ab8c87 100644 --- a/mysql-test/r/heap_hash.result +++ b/mysql-test/r/heap_hash.result @@ -231,18 +231,19 @@ explain select * from t1 where a='aaad'; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ref a a 8 const 1 Using where insert into t1 select * from t1; +flush tables; explain select * from t1 where a='aaaa'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaab'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaac'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaad'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where flush tables; explain select * from t1 where a='aaaa'; id select_type table type possible_keys key key_len ref rows Extra @@ -261,16 +262,16 @@ delete from t1; insert into t1 select * from t2; explain select * from t1 where a='aaaa'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaab'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaac'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaad'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where drop table t1, t2; create table t1 ( id int unsigned not null primary key auto_increment, @@ -345,15 +346,15 @@ insert into t3 select name, name from t1; show index from t3; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t3 1 a 1 a NULL NULL NULL NULL HASH -t3 1 a 2 b NULL 15 NULL NULL HASH +t3 1 a 2 b NULL 13 NULL NULL HASH show index from t3; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t3 1 a 1 a NULL NULL NULL NULL HASH -t3 1 a 2 b NULL 15 NULL NULL HASH +t3 1 a 2 b NULL 13 NULL NULL HASH explain select * from t1 ignore key(btree_idx), t3 where t1.name='matt' and t3.a = concat('',t1.name) and t3.b=t1.name; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ref heap_idx heap_idx 20 const 7 Using where -1 SIMPLE t3 ref a a 40 func,const 6 Using where +1 SIMPLE t3 ref a a 40 func,const 7 Using where drop table t1, t2, t3; create temporary table t1 ( a int, index (a) ) engine=memory; insert into t1 values (1),(2),(3),(4),(5); diff --git a/mysql-test/t/heap_hash.test b/mysql-test/t/heap_hash.test index 6d27f19dfad..59af50da932 100644 --- a/mysql-test/t/heap_hash.test +++ b/mysql-test/t/heap_hash.test @@ -169,6 +169,8 @@ explain select * from t1 where a='aaac'; explain select * from t1 where a='aaad'; insert into t1 select * from t1; +# avoid statistics differences between normal and ps-protocol tests +flush tables; explain select * from t1 where a='aaaa'; explain select * from t1 where a='aaab'; explain select * from t1 where a='aaac'; diff --git a/sql/ha_heap.cc b/sql/ha_heap.cc index 4fc0116a26a..f8c2e6cc338 100644 --- a/sql/ha_heap.cc +++ b/sql/ha_heap.cc @@ -60,7 +60,15 @@ int ha_heap::open(const char *name, int mode, uint test_if_locked) { /* Initialize variables for the opened table */ set_keys_for_scanning(); - update_key_stats(); + /* + We cannot run update_key_stats() here because we do not have a + lock on the table. The 'records' count might just be changed + temporarily at this moment and we might get wrong statistics (Bug + #10178). Instead we request for update. This will be done in + ha_heap::info(), which is always called before key statistics are + used. + */ + key_stats_ok= FALSE; } return (file ? 0 : 1); } @@ -112,6 +120,8 @@ void ha_heap::update_key_stats() } } records_changed= 0; + /* At the end of update_key_stats() we can proudly claim they are OK. */ + key_stats_ok= TRUE; } int ha_heap::write_row(byte * buf) @@ -125,7 +135,7 @@ int ha_heap::write_row(byte * buf) res= heap_write(file,buf); if (!res && ++records_changed*HEAP_STATS_UPDATE_THRESHOLD > file->s->records) - update_key_stats(); + key_stats_ok= FALSE; return res; } @@ -138,7 +148,7 @@ int ha_heap::update_row(const byte * old_data, byte * new_data) res= heap_update(file,old_data,new_data); if (!res && ++records_changed*HEAP_STATS_UPDATE_THRESHOLD > file->s->records) - update_key_stats(); + key_stats_ok= FALSE; return res; } @@ -149,7 +159,7 @@ int ha_heap::delete_row(const byte * buf) res= heap_delete(file,buf); if (!res && table->tmp_table == NO_TMP_TABLE && ++records_changed*HEAP_STATS_UPDATE_THRESHOLD > file->s->records) - update_key_stats(); + key_stats_ok= FALSE; return res; } @@ -262,6 +272,13 @@ void ha_heap::info(uint flag) delete_length= info.deleted * info.reclength; if (flag & HA_STATUS_AUTO) auto_increment_value= info.auto_increment; + /* + If info() is called for the first time after open(), we will still + have to update the key statistics. Hoping that a table lock is now + in place. + */ + if (! key_stats_ok) + update_key_stats(); } int ha_heap::extra(enum ha_extra_function operation) @@ -273,7 +290,7 @@ int ha_heap::delete_all_rows() { heap_clear(file); if (table->tmp_table == NO_TMP_TABLE) - update_key_stats(); + key_stats_ok= FALSE; return 0; } @@ -433,7 +450,11 @@ ha_rows ha_heap::records_in_range(uint inx, key_range *min_key, max_key->flag != HA_READ_AFTER_KEY) return HA_POS_ERROR; // Can only use exact keys else + { + /* Assert that info() did run. We need current statistics here. */ + DBUG_ASSERT(key_stats_ok); return key->rec_per_key[key->key_parts-1]; + } } diff --git a/sql/ha_heap.h b/sql/ha_heap.h index 33de0156074..cbe2474492d 100644 --- a/sql/ha_heap.h +++ b/sql/ha_heap.h @@ -29,8 +29,10 @@ class ha_heap: public handler key_map btree_keys; /* number of records changed since last statistics update */ uint records_changed; + bool key_stats_ok; public: - ha_heap(TABLE *table): handler(table), file(0), records_changed(0) {} + ha_heap(TABLE *table): handler(table), file(0), records_changed(0), + key_stats_ok(0) {} ~ha_heap() {} const char *table_type() const { return "HEAP"; } const char *index_type(uint inx) From fd3ac3829bec048fa7cfeae5c8d1fb88d16e6380 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 20:16:52 +0400 Subject: [PATCH 25/32] Fix bug#11325 Wrong date comparison in views Wrong comparing method were choosen which results in false comparison. Make Item_bool_func2::fix_length_and_dec() to get type and field from real_item() to make REF_ITEM pass the check. sql/item_cmpfunc.cc: Fix bug#11325 Wrong date comparison in views mysql-test/t/view.test: Test case for bug#11325 Wrong date comparison in views. mysql-test/r/view.result: Test case for bug#11325 Wrong date comparison in views. --- mysql-test/r/view.result | 11 +++++++++++ mysql-test/t/view.test | 11 +++++++++++ sql/item_cmpfunc.cc | 10 ++++++---- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/view.result b/mysql-test/r/view.result index 68cc0c4cb57..7c25608aa49 100644 --- a/mysql-test/r/view.result +++ b/mysql-test/r/view.result @@ -1831,3 +1831,14 @@ select * from v1; t 01:00 drop view v1; +create table t1 (f1 date); +insert into t1 values ('2005-01-01'),('2005-02-02'); +create view v1 as select * from t1; +select * from v1 where f1='2005.02.02'; +f1 +2005-02-02 +select * from v1 where '2005.02.02'=f1; +f1 +2005-02-02 +drop view v1; +drop table t1; diff --git a/mysql-test/t/view.test b/mysql-test/t/view.test index 13a5f8cef1f..e32f2c09575 100644 --- a/mysql-test/t/view.test +++ b/mysql-test/t/view.test @@ -1673,3 +1673,14 @@ create view v1(k, K) as select 1,2; create view v1 as SELECT TIME_FORMAT(SEC_TO_TIME(3600),'%H:%i') as t; select * from v1; drop view v1; + +# +# bug #11325 Wrong date comparison in views +# +create table t1 (f1 date); +insert into t1 values ('2005-01-01'),('2005-02-02'); +create view v1 as select * from t1; +select * from v1 where f1='2005.02.02'; +select * from v1 where '2005.02.02'=f1; +drop view v1; +drop table t1; diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 0442865b7f9..58a7f3316d7 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -238,9 +238,10 @@ void Item_bool_func2::fix_length_and_dec() return; } - if (args[0]->type() == FIELD_ITEM) + Item *real_item= args[0]->real_item(); + if (real_item->type() == FIELD_ITEM) { - Field *field=((Item_field*) args[0])->field; + Field *field= ((Item_field*) real_item)->field; if (field->can_be_compared_as_longlong()) { if (convert_constant_item(thd, field,&args[1])) @@ -251,9 +252,10 @@ void Item_bool_func2::fix_length_and_dec() } } } - if (args[1]->type() == FIELD_ITEM /* && !args[1]->const_item() */) + real_item= args[1]->real_item(); + if (real_item->type() == FIELD_ITEM) { - Field *field=((Item_field*) args[1])->field; + Field *field= ((Item_field*) real_item)->field; if (field->can_be_compared_as_longlong()) { if (convert_constant_item(thd, field,&args[0])) From a01d110680c6cf0cbf86d3de9b3ca777b870635b Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 18:23:59 +0200 Subject: [PATCH 26/32] mysql-test-run.pl: Corrected master server id's mysql-test/mysql-test-run.pl: Corrected master server id's --- mysql-test/mysql-test-run.pl | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 2051afc7745..74d1a8dc6f2 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -1554,11 +1554,12 @@ sub do_before_start_master ($$) { } } + # FIXME only remove the ones that are tied to this master # Remove old master.info and relay-log.info files - unlink("$opt_vardir/master-data/master.info"); - unlink("$opt_vardir/master-data/relay-log.info"); - unlink("$opt_vardir/master1-data/master.info"); - unlink("$opt_vardir/master1-data/relay-log.info"); + unlink("$master->[0]->{'path_myddir'}/master.info"); + unlink("$master->[0]->{'path_myddir'}/relay-log.info"); + unlink("$master->[1]->{'path_myddir'}/master.info"); + unlink("$master->[1]->{'path_myddir'}/relay-log.info"); # Run master initialization shell script if one exists if ( $init_script ) @@ -1652,12 +1653,15 @@ sub mysqld_arguments ($$$$$) { if ( $type eq 'master' ) { - mtr_add_arg($args, "%s--log-bin=%s/log/master-bin", $prefix, $opt_vardir); + my $id= $idx > 0 ? $idx + 101 : 1; + + mtr_add_arg($args, "%s--log-bin=%s/log/master-bin%s", $prefix, + $opt_vardir, $sidx); mtr_add_arg($args, "%s--pid-file=%s", $prefix, $master->[$idx]->{'path_mypid'}); mtr_add_arg($args, "%s--port=%d", $prefix, $master->[$idx]->{'path_myport'}); - mtr_add_arg($args, "%s--server-id=1", $prefix); + mtr_add_arg($args, "%s--server-id=%d", $prefix, $id); mtr_add_arg($args, "%s--socket=%s", $prefix, $master->[$idx]->{'path_mysock'}); mtr_add_arg($args, "%s--innodb_data_file_path=ibdata1:128M:autoextend", $prefix); @@ -1665,6 +1669,11 @@ sub mysqld_arguments ($$$$$) { mtr_add_arg($args, "%s--datadir=%s", $prefix, $master->[$idx]->{'path_myddir'}); + if ( $idx > 0 ) + { + mtr_add_arg($args, "%s--skip-innodb", $prefix); + } + if ( $opt_skip_ndbcluster ) { mtr_add_arg($args, "%s--skip-ndbcluster", $prefix); @@ -1674,7 +1683,7 @@ sub mysqld_arguments ($$$$$) { if ( $type eq 'slave' ) { my $slave_server_id= 2 + $idx; - my $slave_rpl_rank= $idx > 0 ? 2 : $slave_server_id; + my $slave_rpl_rank= $slave_server_id; mtr_add_arg($args, "%s--datadir=%s", $prefix, $slave->[$idx]->{'path_myddir'}); From 12a640e28023d9d1333fcecb387e7e3915762b2b Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 19:34:55 +0200 Subject: [PATCH 27/32] Bug#8321 - myisampack bug in compression algorithm Added 64-bit extensions, comments, extended statistics and trace prints. include/my_base.h: Bug#8321 - myisampack bug in compression algorithm Added a comment. myisam/mi_packrec.c: Bug#8321 - myisampack bug in compression algorithm Fixed a function comment. myisam/myisampack.c: Bug#8321 - myisampack bug in compression algorithm Enlarged the variables which hold Huffman codes to ulonglong and adjusted the functions accordingly. Added test code for long Huffman codes. Enlarged the distinct column values buffer (tree_buff) and added checks to stay in its range. Added statistics and trace prints. Added a lot of comments. mysys/tree.c: Bug#8321 - myisampack bug in compression algorithm Added a check against overflow of the tree element count. The tree element count is only 31 bits, but sometimes used for big numbers. There is however no application yet, which relies on exact tree element counts. --- include/my_base.h | 1 + myisam/mi_packrec.c | 15 +- myisam/myisampack.c | 1280 ++++++++++++++++++++++++++++++++++++++----- mysys/tree.c | 3 + 4 files changed, 1155 insertions(+), 144 deletions(-) diff --git a/include/my_base.h b/include/my_base.h index 25fa683744e..c76cf8c604e 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -367,6 +367,7 @@ enum ha_base_keytype { #define HA_STATE_EXTEND_BLOCK 2048 #define HA_STATE_RNEXT_SAME 4096 /* rnext_same was called */ +/* myisampack expects no more than 32 field types. */ enum en_fieldtype { FIELD_LAST=-1,FIELD_NORMAL,FIELD_SKIP_ENDSPACE,FIELD_SKIP_PRESPACE, FIELD_SKIP_ZERO,FIELD_BLOB,FIELD_CONSTANT,FIELD_INTERVALL,FIELD_ZERO, diff --git a/myisam/mi_packrec.c b/myisam/mi_packrec.c index 4b512dd89dd..c251e4dda4a 100644 --- a/myisam/mi_packrec.c +++ b/myisam/mi_packrec.c @@ -416,8 +416,19 @@ static uint find_longest_bitstream(uint16 *table, uint16 *end) } - /* Read record from datafile */ - /* Returns length of packed record, -1 if error */ +/* + Read record from datafile. + + SYNOPSIS + _mi_read_pack_record() + info A pointer to MI_INFO. + filepos File offset of the record. + buf RETURN The buffer to receive the record. + + RETURN + 0 on success + HA_ERR_WRONG_IN_RECORD or -1 on error +*/ int _mi_read_pack_record(MI_INFO *info, my_off_t filepos, byte *buf) { diff --git a/myisam/myisampack.c b/myisam/myisampack.c index 74bb541b220..70a32902da6 100644 --- a/myisam/myisampack.c +++ b/myisam/myisampack.c @@ -33,10 +33,10 @@ #include #include -#if INT_MAX > 32767 -#define BITS_SAVED 32 +#if SIZEOF_LONG_LONG > 4 +#define BITS_SAVED 64 #else -#define BITS_SAVED 16 +#define BITS_SAVED 32 #endif #define IS_OFFSET ((uint) 32768) /* Bit if offset or char in tree */ @@ -49,10 +49,10 @@ struct st_file_buffer { File file; - char *buffer,*pos,*end; + uchar *buffer,*pos,*end; my_off_t pos_in_file; int bits; - uint current_byte; + ulonglong bitbucket; }; struct st_huff_tree; @@ -69,13 +69,17 @@ typedef struct st_huff_counts { my_off_t end_space[8]; my_off_t pre_space[8]; my_off_t tot_end_space,tot_pre_space,zero_fields,empty_fields,bytes_packed; - TREE int_tree; - byte *tree_buff; - byte *tree_pos; + TREE int_tree; /* Tree for detecting distinct column values. */ + byte *tree_buff; /* Column values, 'field_length' each. */ + byte *tree_pos; /* Points to end of column values in 'tree_buff'. */ } HUFF_COUNTS; typedef struct st_huff_element HUFF_ELEMENT; +/* + WARNING: It is crucial for the optimizations in calc_packed_length() + that 'count' is the first element of 'HUFF_ELEMENT'. +*/ struct st_huff_element { my_off_t count; union un_element { @@ -98,7 +102,7 @@ typedef struct st_huff_tree { my_off_t bytes_packed; uint tree_pack_length; uint min_chr,max_chr,char_bits,offset_bits,max_offset,height; - ulong *code; + ulonglong *code; uchar *code_len; } HUFF_TREE; @@ -146,7 +150,7 @@ static uint join_same_trees(HUFF_COUNTS *huff_counts,uint trees); static int make_huff_decode_table(HUFF_TREE *huff_tree,uint trees); static void make_traverse_code_tree(HUFF_TREE *huff_tree, HUFF_ELEMENT *element,uint size, - ulong code); + ulonglong code); static int write_header(PACK_MRG_INFO *isam_file, uint header_length,uint trees, my_off_t tot_elements,my_off_t filelength); static void write_field_info(HUFF_COUNTS *counts, uint fields,uint trees); @@ -161,7 +165,7 @@ static char *make_old_name(char *new_name,char *old_name); static void init_file_buffer(File file,pbool read_buffer); static int flush_buffer(ulong neaded_length); static void end_file_buffer(void); -static void write_bits(ulong value,uint bits); +static void write_bits(ulonglong value, uint bits); static void flush_bits(void); static int save_state(MI_INFO *isam_file,PACK_MRG_INFO *mrg,my_off_t new_length, ha_checksum crc); @@ -170,13 +174,23 @@ static int save_state_mrg(File file,PACK_MRG_INFO *isam_file,my_off_t new_length static int mrg_close(PACK_MRG_INFO *mrg); static int mrg_rrnd(PACK_MRG_INFO *info,byte *buf); static void mrg_reset(PACK_MRG_INFO *mrg); +#if !defined(DBUG_OFF) +static void fakebigcodes(HUFF_COUNTS *huff_counts, HUFF_COUNTS *end_count); +static int fakecmp(my_off_t **count1, my_off_t **count2); +#endif static int error_on_write=0,test_only=0,verbose=0,silent=0, write_loop=0,force_pack=0, isamchk_neaded=0; static int tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL; static my_bool backup, opt_wait; -static uint tree_buff_length=8196-MALLOC_OVERHEAD; +/* + tree_buff_length is somewhat arbitrary. The bigger it is the better + the chance to win in terms of compression factor. On the other hand, + this table becomes part of the compressed file header. And its length + is coded with 16 bits in the header. Hence the limit is 2**16 - 1. +*/ +static uint tree_buff_length= 65536 - MALLOC_OVERHEAD; static char tmp_dir[FN_REFLEN]={0},*join_table; static my_off_t intervall_length; static ha_checksum glob_crc; @@ -225,7 +239,8 @@ int main(int argc, char **argv) } if (ok && isamchk_neaded && !silent) puts("Remember to run myisamchk -rq on compressed tables"); - VOID(fflush(stdout)); VOID(fflush(stderr)); + VOID(fflush(stdout)); + VOID(fflush(stderr)); free_defaults(default_argv); my_end(verbose ? MY_CHECK_ERROR | MY_GIVE_INFO : MY_CHECK_ERROR); exit(error ? 2 : 0); @@ -260,7 +275,7 @@ static struct my_option my_long_options[] = 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"test", 't', "Don't pack table, only test packing it.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"verbose", 'v', "Write info about progress and packing result.", + {"verbose", 'v', "Write info about progress and packing result. Use many -v for more verbosity!", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"version", 'V', "Output version information and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -273,7 +288,8 @@ static struct my_option my_long_options[] = static void print_version(void) { - printf("%s Ver 1.22 for %s on %s\n", my_progname, SYSTEM_TYPE, MACHINE_TYPE); + VOID(printf("%s Ver 1.22 for %s on %s\n", + my_progname, SYSTEM_TYPE, MACHINE_TYPE)); NETWARE_SET_SCREEN_MODE(1); } @@ -290,7 +306,7 @@ static void usage(void) puts("afterwards to update the keys."); puts("You should give the .MYI file as the filename argument."); - printf("\nUsage: %s [OPTIONS] filename...\n", my_progname); + VOID(printf("\nUsage: %s [OPTIONS] filename...\n", my_progname)); my_print_help(my_long_options); print_defaults("my", load_default_groups); my_print_variables(my_long_options); @@ -314,7 +330,10 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)), silent= 1; break; case 't': - test_only= verbose= 1; + test_only= 1; + /* Avoid to reset 'verbose' if it was already set > 1. */ + if (! verbose) + verbose= 1; break; case 'T': length= (uint) (strmov(tmp_dir, argument) - tmp_dir); @@ -325,7 +344,7 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)), } break; case 'v': - verbose= 1; + verbose++; /* Allow for selecting the level of verbosity. */ silent= 0; break; case '#': @@ -380,7 +399,7 @@ static MI_INFO *open_isam_file(char *name,int mode) (opt_wait ? HA_OPEN_WAIT_IF_LOCKED : HA_OPEN_ABORT_IF_LOCKED)))) { - VOID(fprintf(stderr,"%s gave error %d on open\n",name,my_errno)); + VOID(fprintf(stderr, "%s gave error %d on open\n", name, my_errno)); DBUG_RETURN(0); } share=isam_file->s; @@ -388,7 +407,7 @@ static MI_INFO *open_isam_file(char *name,int mode) { if (!force_pack) { - VOID(fprintf(stderr,"%s is already compressed\n",name)); + VOID(fprintf(stderr, "%s is already compressed\n", name)); VOID(mi_close(isam_file)); DBUG_RETURN(0); } @@ -400,7 +419,7 @@ static MI_INFO *open_isam_file(char *name,int mode) (share->state.state.records <= 1 || share->state.state.data_file_length < 1024)) { - VOID(fprintf(stderr,"%s is too small to compress\n",name)); + VOID(fprintf(stderr, "%s is too small to compress\n", name)); VOID(mi_close(isam_file)); DBUG_RETURN(0); } @@ -446,8 +465,8 @@ static bool open_isam_files(PACK_MRG_INFO *mrg,char **names,uint count) return 0; diff_file: - fprintf(stderr,"%s: Tables '%s' and '%s' are not identical\n", - my_progname,names[j],names[j+1]); + VOID(fprintf(stderr, "%s: Tables '%s' and '%s' are not identical\n", + my_progname, names[j], names[j+1])); error: while (i--) mi_close(mrg->file[i]); @@ -518,16 +537,25 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) mrg->records=0; for (i=0 ; i < mrg->count ; i++) mrg->records+=mrg->file[i]->s->state.state.records; + + DBUG_PRINT("info", ("Compressing %s: (%lu records)", + result_table ? new_name : org_name, + (ulong) mrg->records)); if (write_loop || verbose) { - printf("Compressing %s: (%lu records)\n", - result_table ? new_name : org_name,(ulong) mrg->records); + VOID(printf("Compressing %s: (%lu records)\n", + result_table ? new_name : org_name, (ulong) mrg->records)); } trees=fields=share->base.fields; huff_counts=init_huff_count(isam_file,mrg->records); QUICK_SAFEMALLOC; + + /* + Read the whole data file(s) for statistics. + */ + DBUG_PRINT("info", ("- Calculating statistics")); if (write_loop || verbose) - printf("- Calculating statistics\n"); + VOID(printf("- Calculating statistics\n")); if (get_statistic(mrg,huff_counts)) goto err; NORMAL_SAFEMALLOC; @@ -536,29 +564,74 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) old_length+= (mrg->file[i]->s->state.state.data_file_length - mrg->file[i]->s->state.state.empty); + /* + Create a global priority queue in preparation for making + temporary Huffman trees. + */ if (init_queue(&queue,256,0,0,compare_huff_elements,0)) goto err; + + /* + Check each column if we should use pre-space-compress, end-space- + compress, empty-field-compress or zero-field-compress. + */ check_counts(huff_counts,fields,mrg->records); + + /* + Build a Huffman tree for each column. + */ huff_trees=make_huff_trees(huff_counts,trees); + + /* + If the packed lengths of combined columns is less then the sum of + the non-combined columns, then create common Huffman trees for them. + We do this only for byte compressed columns, not for distinct values + compressed columns. + */ if ((int) (used_trees=join_same_trees(huff_counts,trees)) < 0) goto err; + + /* + Assign codes to all byte or column values. + */ if (make_huff_decode_table(huff_trees,fields)) goto err; + /* Prepare a file buffer. */ init_file_buffer(new_file,0); + + /* + Reserve space in the target file for the fixed compressed file header. + */ file_buffer.pos_in_file=HEAD_LENGTH; if (! test_only) VOID(my_seek(new_file,file_buffer.pos_in_file,MY_SEEK_SET,MYF(0))); + /* + Write field infos: field type, pack type, length bits, tree number. + */ write_field_info(huff_counts,fields,used_trees); + + /* + Write decode trees. + */ if (!(tot_elements=write_huff_tree(huff_trees,trees))) goto err; + + /* + Calculate the total length of the compression info header. + This includes the fixed compressed file header, the column compression + type descriptions, and the decode trees. + */ header_length=(uint) file_buffer.pos_in_file+ (uint) (file_buffer.pos-file_buffer.buffer); - /* Compress file */ + /* + Compress the source file into the target file. + */ + DBUG_PRINT("info", ("- Compressing file")); if (write_loop || verbose) - printf("- Compressing file\n"); + VOID(printf("- Compressing file\n")); error=compress_isam_file(mrg,huff_counts); new_length=file_buffer.pos_in_file; if (!error && !test_only) @@ -568,16 +641,28 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) error=my_write(file_buffer.file,buff,sizeof(buff), MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL)) != 0; } + + /* + Write the fixed compressed file header. + */ if (!error) error=write_header(mrg,header_length,used_trees,tot_elements, new_length); + + /* Flush the file buffer. */ end_file_buffer(); + /* Display statistics. */ + DBUG_PRINT("info", ("Min record length: %6d Max length: %6d " + "Mean total length: %6ld\n", + mrg->min_pack_length, mrg->max_pack_length, + (ulong) (mrg->records ? (new_length/mrg->records) : 0))); if (verbose && mrg->records) - printf("Min record length: %6d Max length: %6d Mean total length: %6ld\n", - mrg->min_pack_length,mrg->max_pack_length, - (ulong) (new_length/mrg->records)); + VOID(printf("Min record length: %6d Max length: %6d " + "Mean total length: %6ld\n", mrg->min_pack_length, + mrg->max_pack_length, (ulong) (new_length/mrg->records))); + /* Close source and target file. */ if (!test_only) { error|=my_close(new_file,MYF(MY_WME)); @@ -588,6 +673,7 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) } } + /* Cleanup. */ free_counts_and_tree_and_queue(huff_trees,trees,huff_counts,fields); if (! test_only && ! error) { @@ -629,15 +715,16 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) error|=my_close(join_isam_file,MYF(MY_WME)); if (error) { - VOID(fprintf(stderr,"Aborting: %s is not compressed\n",org_name)); + VOID(fprintf(stderr, "Aborting: %s is not compressed\n", org_name)); VOID(my_delete(new_name,MYF(MY_WME))); DBUG_RETURN(-1); } if (write_loop || verbose) { if (old_length) - printf("%.4g%% \n", (((longlong) (old_length -new_length))*100.0/ - (longlong) old_length)); + VOID(printf("%.4g%% \n", + (((longlong) (old_length - new_length)) * 100.0 / + (longlong) old_length))); else puts("Empty file saved in compressed format"); } @@ -650,7 +737,7 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) if (join_isam_file >= 0) VOID(my_close(join_isam_file,MYF(0))); mrg_close(mrg); - VOID(fprintf(stderr,"Aborted: %s is not compressed\n",org_name)); + VOID(fprintf(stderr, "Aborted: %s is not compressed\n", org_name)); DBUG_RETURN(-1); } @@ -677,6 +764,12 @@ static HUFF_COUNTS *init_huff_count(MI_INFO *info,my_off_t records) (type == FIELD_NORMAL || type == FIELD_SKIP_ZERO)) count[i].max_zero_fill= count[i].field_length; + /* + For every column initialize a tree, which is used to detect distinct + column values. 'int_tree' works together with 'tree_buff' and + 'tree_pos'. It's keys are implemented by pointers into 'tree_buff'. + This is accomplished by '-1' as the element size. + */ init_tree(&count[i].int_tree,0,0,-1,(qsort_cmp2) compare_tree,0, NULL, NULL); if (records && type != FIELD_BLOB && type != FIELD_VARCHAR) @@ -762,10 +855,13 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) ulong tot_blob_length=0; if (! error) { + /* glob_crc is a checksum over all bytes of all records. */ if (static_row_size) glob_crc+=mi_static_checksum(mrg->file[0],record); else glob_crc+=mi_checksum(mrg->file[0],record); + + /* Count the incidence of values separately for every column. */ for (pos=record,count=huff_counts ; count < end_count ; count++, @@ -773,15 +869,48 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) { next_pos=end_pos=(start_pos=pos)+count->field_length; - /* Put value in tree if there is room for it */ + /* + Put the whole column value in a tree if there is room for it. + 'int_tree' is used to quickly check for duplicate values. + 'tree_buff' collects as many distinct column values as + possible. If the field length is > 1, it is tree_buff_length, + else 2 bytes. Each value is 'field_length' bytes big. If there + are more distinct column values than fit into the buffer, we + give up with this tree. BLOBs and VARCHARs do not have a + tree_buff as it can only be used with fixed length columns. + For the special case of field length == 1, we handle only the + case that there is only one distinct value in the table(s). + Otherwise, we can have a maximum of 256 distinct values. This + is then handled by the normal Huffman tree build. + + Another limit for collecting distinct column values is the + number of values itself. Since we would need to build a + Huffman tree for the values, we are limited by the 'IS_OFFSET' + constant. This constant expresses a bit which is used to + determine if a tree element holds a final value or an offset + to a child element. Hence, all values and offsets need to be + smaller than 'IS_OFFSET'. A tree element is implemented with + two integer values, one for the left branch and one for the + right branch. For the extreme case that the first element + points to the last element, the number of integers in the tree + must be less or equal to IS_OFFSET. So the number of elements + must be less or equal to IS_OFFSET / 2. + + WARNING: At first, we insert a pointer into the record buffer + as the key for the tree. If we got a new distinct value, which + is really inserted into the tree, instead of being counted + only, we will copy the column value from the record buffer to + 'tree_buff' and adjust the key pointer of the tree accordingly. + */ if (count->tree_buff) { global_count=count; if (!(element=tree_insert(&count->int_tree,pos, 0, count->int_tree.custom_arg)) || (element->count == 1 && - count->tree_buff + tree_buff_length < - count->tree_pos + count->field_length) || + (count->tree_buff + tree_buff_length < + count->tree_pos + count->field_length)) || + (count->int_tree.elements_in_tree > IS_OFFSET / 2) || (count->field_length == 1 && count->int_tree.elements_in_tree > 1)) { @@ -791,10 +920,17 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) } else { + /* + If tree_insert() succeeds, it either creates a new element + or increments the counter of an existing element. + */ if (element->count == 1) - { /* New element */ + { + /* Copy the new column value into 'tree_buff'. */ memcpy(count->tree_pos,pos,(size_t) count->field_length); + /* Adjust the key pointer in the tree. */ tree_set_pointer(element,count->tree_pos); + /* Point behind the last column value so far. */ count->tree_pos+=count->field_length; } } @@ -804,15 +940,21 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) if (count->field_type == FIELD_NORMAL || count->field_type == FIELD_SKIP_ENDSPACE) { + /* Ignore trailing space. */ for ( ; end_pos > pos ; end_pos--) if (end_pos[-1] != ' ') break; + /* Empty fields are just counted. Go to the next record. */ if (end_pos == pos) { count->empty_fields++; count->max_zero_fill=0; continue; } + /* + Count the total of all trailing spaces and the number of + short trailing spaces. Remember the longest trailing space. + */ length= (uint) (next_pos-end_pos); count->tot_end_space+=length; if (length < 8) @@ -820,18 +962,25 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) if (count->max_end_space < length) count->max_end_space = length; } + if (count->field_type == FIELD_NORMAL || count->field_type == FIELD_SKIP_PRESPACE) { + /* Ignore leading space. */ for (pos=start_pos; pos < end_pos ; pos++) if (pos[0] != ' ') break; + /* Empty fields are just counted. Go to the next record. */ if (end_pos == pos) { count->empty_fields++; count->max_zero_fill=0; continue; } + /* + Count the total of all leading spaces and the number of + short leading spaces. Remember the longest leading space. + */ length= (uint) (pos-start_pos); count->tot_pre_space+=length; if (length < 8) @@ -839,6 +988,8 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) if (count->max_pre_space < length) count->max_pre_space = length; } + + /* Calculate pos, end_pos, and max_length for variable length fields. */ if (count->field_type == FIELD_BLOB) { uint field_length=count->field_length -mi_portable_sizeof_char_ptr; @@ -857,45 +1008,121 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) end_pos= pos+length; set_if_bigger(count->max_length,length); } + + /* Evaluate 'max_zero_fill' for short fields. */ if (count->field_length <= 8 && (count->field_type == FIELD_NORMAL || count->field_type == FIELD_SKIP_ZERO)) { uint i; + /* Zero fields are just counted. Go to the next record. */ if (!memcmp((byte*) start_pos,zero_string,count->field_length)) { count->zero_fields++; continue; } + /* + max_zero_fill starts with field_length. It is decreased every + time a shorter "zero trailer" is found. It is set to zero when + an empty field is found (see above). This suggests that the + variable should be called 'min_zero_fill'. + */ for (i =0 ; i < count->max_zero_fill && ! end_pos[-1 - (int) i] ; i++) ; if (i < count->max_zero_fill) count->max_zero_fill=i; } + + /* Ignore zero fields and check fields. */ if (count->field_type == FIELD_ZERO || count->field_type == FIELD_CHECK) continue; + + /* + Count the incidence of every byte value in the + significant field value. + */ for ( ; pos < end_pos ; pos++) count->counts[(uchar) *pos]++; + + /* Step to next field. */ } + if (tot_blob_length > max_blob_length) max_blob_length=tot_blob_length; record_count++; if (write_loop && record_count % WRITE_COUNT == 0) { - printf("%lu\r",(ulong) record_count); VOID(fflush(stdout)); + VOID(printf("%lu\r", (ulong) record_count)); + VOID(fflush(stdout)); } } else if (error != HA_ERR_RECORD_DELETED) { - fprintf(stderr,"Got error %d while reading rows",error); + VOID(fprintf(stderr, "Got error %d while reading rows", error)); break; } + + /* Step to next record. */ } if (write_loop) { - printf(" \r"); VOID(fflush(stdout)); + VOID(printf(" \r")); + VOID(fflush(stdout)); } + + /* + If --debug=d,fakebigcodes is set, fake the counts to get big Huffman + codes. + */ + DBUG_EXECUTE_IF("fakebigcodes", fakebigcodes(huff_counts, end_count);); + + DBUG_PRINT("info", ("Found the following number of incidents " + "of the byte codes:")); + if (verbose >= 2) + VOID(printf("Found the following number of incidents " + "of the byte codes:\n")); + for (count= huff_counts ; count < end_count; count++) + { + uint idx; + my_off_t total_count; + char llbuf[32]; + + DBUG_PRINT("info", ("column: %3u", count - huff_counts + 1)); + if (verbose >= 2) + VOID(printf("column: %3u\n", count - huff_counts + 1)); + if (count->tree_buff) + { + DBUG_PRINT("info", ("number of distinct values: %u", + (count->tree_pos - count->tree_buff) / + count->field_length)); + if (verbose >= 2) + VOID(printf("number of distinct values: %u\n", + (count->tree_pos - count->tree_buff) / + count->field_length)); + } + total_count= 0; + for (idx= 0; idx < 256; idx++) + { + if (count->counts[idx]) + { + total_count+= count->counts[idx]; + DBUG_PRINT("info", ("counts[0x%02x]: %12s", idx, + llstr((longlong) count->counts[idx], llbuf))); + if (verbose >= 2) + VOID(printf("counts[0x%02x]: %12s\n", idx, + llstr((longlong) count->counts[idx], llbuf))); + } + } + DBUG_PRINT("info", ("total: %12s", llstr((longlong) total_count, + llbuf))); + if ((verbose >= 2) && total_count) + { + VOID(printf("total: %12s\n", + llstr((longlong) total_count, llbuf))); + } + } + mrg->records=record_count; mrg->max_blob_length=max_blob_length; my_afree((gptr) record); @@ -944,9 +1171,14 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, huff_counts->field_type=FIELD_NORMAL; huff_counts->pack_type=0; + /* Check for zero-filled records (in this column), or zero records. */ if (huff_counts->zero_fields || ! records) { my_off_t old_space_count; + /* + If there are only zero filled records (in this column), + or no records at all, we are done. + */ if (huff_counts->zero_fields == records) { huff_counts->field_type= FIELD_ZERO; @@ -954,14 +1186,22 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, huff_counts->counts[0]=0; goto found_pack; } + /* Remeber the number of significant spaces. */ old_space_count=huff_counts->counts[' ']; - huff_counts->counts[' ']+=huff_counts->tot_end_space+ - huff_counts->tot_pre_space + - huff_counts->empty_fields * huff_counts->field_length; + /* Add all leading and trailing spaces. */ + huff_counts->counts[' ']+= (huff_counts->tot_end_space + + huff_counts->tot_pre_space + + huff_counts->empty_fields * + huff_counts->field_length); + /* Check, what the compressed length of this would be. */ old_length=calc_packed_length(huff_counts,0)+records/8; + /* Get the number of zero bytes. */ length=huff_counts->zero_fields*huff_counts->field_length; + /* Add it to the counts. */ huff_counts->counts[0]+=length; + /* Check, what the compressed length of this would be. */ new_length=calc_packed_length(huff_counts,0); + /* If the compression without the zeroes would be shorter, we are done. */ if (old_length < new_length && huff_counts->field_length > 1) { huff_counts->field_type=FIELD_SKIP_ZERO; @@ -969,9 +1209,16 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, huff_counts->bytes_packed=old_length- records/8; goto found_pack; } + /* Remove the insignificant spaces, but keep the zeroes. */ huff_counts->counts[' ']=old_space_count; } + /* Check, what the compressed length of this column would be. */ huff_counts->bytes_packed=calc_packed_length(huff_counts,0); + + /* + If there are enough empty records (in this column), + treating them specially may pay off. + */ if (huff_counts->empty_fields) { if (huff_counts->field_length > 2 && @@ -1003,6 +1250,11 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, } } } + + /* + If there are enough trailing spaces (in this column), + treating them specially may pay off. + */ if (huff_counts->tot_end_space) { huff_counts->counts[' ']+=huff_counts->tot_pre_space; @@ -1012,6 +1264,11 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, goto found_pack; huff_counts->counts[' ']-=huff_counts->tot_pre_space; } + + /* + If there are enough leading spaces (in this column), + treating them specially may pay off. + */ if (huff_counts->tot_pre_space) { if (test_space_compress(huff_counts,records,huff_counts->max_pre_space, @@ -1041,6 +1298,8 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, { HUFF_TREE tree; + DBUG_EXECUTE_IF("forceintervall", + huff_counts->bytes_packed= ~ (my_off_t) 0;); tree.element_buffer=0; if (!make_huff_tree(&tree,huff_counts) && tree.bytes_packed+tree.tree_pack_length < huff_counts->bytes_packed) @@ -1066,14 +1325,27 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, fill_zero_fields++; field_count[huff_counts->field_type]++; } + DBUG_PRINT("info", ("normal: %3d empty-space: %3d " + "empty-zero: %3d empty-fill: %3d", + field_count[FIELD_NORMAL],space_fields, + field_count[FIELD_SKIP_ZERO],fill_zero_fields)); + DBUG_PRINT("info", ("pre-space: %3d end-space: %3d " + "intervall-fields: %3d zero: %3d", + field_count[FIELD_SKIP_PRESPACE], + field_count[FIELD_SKIP_ENDSPACE], + field_count[FIELD_INTERVALL], + field_count[FIELD_ZERO])); if (verbose) - printf("\nnormal: %3d empty-space: %3d empty-zero: %3d empty-fill: %3d\npre-space: %3d end-space: %3d intervall-fields: %3d zero: %3d\n", - field_count[FIELD_NORMAL],space_fields, - field_count[FIELD_SKIP_ZERO],fill_zero_fields, - field_count[FIELD_SKIP_PRESPACE], - field_count[FIELD_SKIP_ENDSPACE], - field_count[FIELD_INTERVALL], - field_count[FIELD_ZERO]); + VOID(printf("\nnormal: %3d empty-space: %3d " + "empty-zero: %3d empty-fill: %3d\n" + "pre-space: %3d end-space: %3d " + "intervall-fields: %3d zero: %3d\n", + field_count[FIELD_NORMAL],space_fields, + field_count[FIELD_SKIP_ZERO],fill_zero_fields, + field_count[FIELD_SKIP_PRESPACE], + field_count[FIELD_SKIP_ENDSPACE], + field_count[FIELD_INTERVALL], + field_count[FIELD_ZERO])); DBUG_VOID_RETURN; } @@ -1170,8 +1442,24 @@ static HUFF_TREE* make_huff_trees(HUFF_COUNTS *huff_counts, uint trees) DBUG_RETURN(huff_tree); } - /* Update huff_tree according to huff_counts->counts or - huff_counts->tree_buff */ +/* + Build a Huffman tree. + + SYNOPSIS + make_huff_tree() + huff_tree The Huffman tree. + huff_counts The counts. + + DESCRIPTION + Build a Huffman tree according to huff_counts->counts or + huff_counts->tree_buff. tree_buff, if non-NULL contains up to + tree_buff_length of distinct column values. In that case, whole + values can be Huffman encoded instead of single bytes. + + RETURN + 0 OK + != 0 Error +*/ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) { @@ -1182,12 +1470,14 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) first=last=0; if (huff_counts->tree_buff) { + /* Calculate the number of distinct values in tree_buff. */ found= (uint) (huff_counts->tree_pos - huff_counts->tree_buff) / huff_counts->field_length; first=0; last=found-1; } else { + /* Count the number of byte codes found in the column. */ for (i=found=0 ; i < 256 ; i++) { if (huff_counts->counts[i]) @@ -1201,6 +1491,7 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) found=2; } + /* When using 'tree_buff' we can have more that 256 values. */ if (queue.max_elements < found) { delete_queue(&queue); @@ -1208,6 +1499,7 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) return -1; } + /* Allocate or reallocate an element buffer for the Huffman tree. */ if (!huff_tree->element_buffer) { if (!(huff_tree->element_buffer= @@ -1235,15 +1527,25 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) if (huff_counts->tree_buff) { huff_tree->elements=0; - tree_walk(&huff_counts->int_tree, - (int (*)(void*, element_count,void*)) save_counts_in_queue, - (gptr) huff_tree, left_root_right); huff_tree->tree_pack_length=(1+15+16+5+5+ (huff_tree->char_bits+1)*found+ (huff_tree->offset_bits+1)* (found-2)+7)/8 + (uint) (huff_tree->counts->tree_pos- huff_tree->counts->tree_buff); + /* + Put a HUFF_ELEMENT into the queue for every distinct column value. + + tree_walk() calls save_counts_in_queue() for every element in + 'int_tree'. This takes elements from the target trees element + buffer and places references to them into the buffer of the + priority queue. We insert in column value order, but the order is + in fact irrelevant here. We will establish the correct order + later. + */ + tree_walk(&huff_counts->int_tree, + (int (*)(void*, element_count,void*)) save_counts_in_queue, + (gptr) huff_tree, left_root_right); } else { @@ -1252,7 +1554,15 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) (huff_tree->char_bits+1)*found+ (huff_tree->offset_bits+1)* (found-2)+7)/8; + /* + Put a HUFF_ELEMENT into the queue for every byte code found in the column. + The elements are taken from the target trees element buffer. + Instead of using queue_insert(), we just place references to the + elements into the buffer of the priority queue. We insert in byte + value order, but the order is in fact irrelevant here. We will + establish the correct order later. + */ for (i=first, found=0 ; i <= last ; i++) { if (huff_counts->counts[i]) @@ -1264,8 +1574,13 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) queue.root[found]=(byte*) new_huff_el; } } + /* + If there is only a single byte value in this field in all records, + add a second element with zero incidence. This is required to enter + the loop, which builds the Huffman tree. + */ while (found < 2) - { /* Our huff_trees request at least 2 elements */ + { new_huff_el=huff_tree->element_buffer+(found++); new_huff_el->count=0; new_huff_el->a.leaf.null=0; @@ -1276,21 +1591,53 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) queue.root[found]=(byte*) new_huff_el; } } + + /* Make a queue from the queue buffer. */ queue.elements=found; + /* + Make a priority queue from the queue. Construct its index so that we + have a partially ordered tree. + */ for (i=found/2 ; i > 0 ; i--) _downheap(&queue,i); + + /* The Huffman algorithm. */ bytes_packed=0; bits_packed=0; for (i=1 ; i < found ; i++) { + /* + Pop the top element from the queue (the one with the least incidence). + Popping from a priority queue includes a re-ordering of the queue, + to get the next least incidence element to the top. + */ a=(HUFF_ELEMENT*) queue_remove(&queue,0); + /* + Copy the next least incidence element. The queue implementation + reserves root[0] for temporary purposes. root[1] is the top. + */ b=(HUFF_ELEMENT*) queue.root[1]; + /* Get a new element from the element buffer. */ new_huff_el=huff_tree->element_buffer+found+i; + /* The new element gets the sum of the two least incidence elements. */ new_huff_el->count=a->count+b->count; + /* + The Huffman algorithm assigns another bit to the code for a byte + every time that bytes incidence is combined (directly or indirectly) + to a new element as one of the two least incidence elements. + This means that one more bit per incidence of that byte is required + in the resulting file. So we add the new combined incidence as the + number of bits by which the result grows. + */ bits_packed+=(uint) (new_huff_el->count & 7); bytes_packed+=new_huff_el->count/8; - new_huff_el->a.nod.left=a; /* lesser in left */ + /* The new element points to its children, lesser in left. */ + new_huff_el->a.nod.left=a; new_huff_el->a.nod.right=b; + /* + Replace the copied top element by the new element and re-order the + queue. + */ queue.root[1]=(byte*) new_huff_el; queue_replaced(&queue); } @@ -1309,7 +1656,26 @@ static int compare_tree(void* cmp_arg __attribute__((unused)), return 0; } - /* Used by make_huff_tree to save intervall-counts in queue */ +/* + Organize distinct column values and their incidences into a priority queue. + + SYNOPSIS + save_counts_in_queue() + key The column value. + count The incidence of this value. + tree The Huffman tree to be built later. + + DESCRIPTION + We use the element buffer of the targeted tree. The distinct column + values are organized in a priority queue first. The Huffman + algorithm will later organize the elements into a Huffman tree. For + the time being, we just place references to the elements into the + queue buffer. The buffer will later be organized into a priority + queue. + + RETURN + 0 + */ static int save_counts_in_queue(byte *key, element_count count, HUFF_TREE *tree) @@ -1326,8 +1692,23 @@ static int save_counts_in_queue(byte *key, element_count count, } - /* Calculate length of file if given counts should be used */ - /* Its actually a faster version of make_huff_tree */ +/* + Calculate length of file if given counts should be used. + + SYNOPSIS + calc_packed_length() + huff_counts The counts for a column of the table(s). + add_tree_lenght If the decode tree length should be added. + + DESCRIPTION + We need to follow the Huffman algorithm until we know, how many bits + are required for each byte code. But we do not need the resulting + Huffman tree. Hence, we can leave out some steps which are essential + in make_huff_tree(). + + RETURN + Number of bytes required to compress this table column. +*/ static my_off_t calc_packed_length(HUFF_COUNTS *huff_counts, uint add_tree_lenght) @@ -1337,6 +1718,23 @@ static my_off_t calc_packed_length(HUFF_COUNTS *huff_counts, HUFF_ELEMENT element_buffer[256]; DBUG_ENTER("calc_packed_length"); + /* + WARNING: We use a small hack for efficiency: Instead of placing + references to HUFF_ELEMENTs into the queue, we just insert + references to the counts of the byte codes which appeared in this + table column. During the Huffman algorithm they are successively + replaced by references to HUFF_ELEMENTs. This works, because + HUFF_ELEMENTs have the incidence count at their beginning. + Regardless, wether the queue array contains references to counts of + type my_off_t or references to HUFF_ELEMENTs which have the count of + type my_off_t at their beginning, it always points to a count of the + same type. + + Instead of using queue_insert(), we just copy the references into + the buffer of the priority queue. We insert in byte value order, but + the order is in fact irrelevant here. We will establish the correct + order later. + */ first=last=0; for (i=found=0 ; i < 256 ; i++) { @@ -1345,31 +1743,73 @@ static my_off_t calc_packed_length(HUFF_COUNTS *huff_counts, if (! found++) first=i; last=i; + /* We start with root[1], which is the queues top element. */ queue.root[found]=(byte*) &huff_counts->counts[i]; } } if (!found) DBUG_RETURN(0); /* Empty tree */ + /* + If there is only a single byte value in this field in all records, + add a second element with zero incidence. This is required to enter + the loop, which follows the Huffman algorithm. + */ if (found < 2) queue.root[++found]=(byte*) &huff_counts->counts[last ? 0 : 1]; + /* Make a queue from the queue buffer. */ queue.elements=found; bytes_packed=0; bits_packed=0; + /* Add the length of the coding table, which would become part of the file. */ if (add_tree_lenght) bytes_packed=(8+9+5+5+(max_bit(last-first)+1)*found+ (max_bit(found-1)+1+1)*(found-2) +7)/8; + + /* + Make a priority queue from the queue. Construct its index so that we + have a partially ordered tree. + */ for (i=(found+1)/2 ; i > 0 ; i--) _downheap(&queue,i); + + /* The Huffman algorithm. */ for (i=0 ; i < found-1 ; i++) { - HUFF_ELEMENT *a,*b,*new_huff_el; - a=(HUFF_ELEMENT*) queue_remove(&queue,0); - b=(HUFF_ELEMENT*) queue.root[1]; - new_huff_el=element_buffer+i; - new_huff_el->count=a->count+b->count; + my_off_t *a; + my_off_t *b; + HUFF_ELEMENT *new_huff_el; + + /* + Pop the top element from the queue (the one with the least + incidence). Popping from a priority queue includes a re-ordering + of the queue, to get the next least incidence element to the top. + */ + a= (my_off_t*) queue_remove(&queue, 0); + /* + Copy the next least incidence element. The queue implementation + reserves root[0] for temporary purposes. root[1] is the top. + */ + b= (my_off_t*) queue.root[1]; + /* Create a new element in a local (automatic) buffer. */ + new_huff_el= element_buffer + i; + /* The new element gets the sum of the two least incidence elements. */ + new_huff_el->count= *a + *b; + /* + The Huffman algorithm assigns another bit to the code for a byte + every time that bytes incidence is combined (directly or indirectly) + to a new element as one of the two least incidence elements. + This means that one more bit per incidence of that byte is required + in the resulting file. So we add the new combined incidence as the + number of bits by which the result grows. + */ bits_packed+=(uint) (new_huff_el->count & 7); bytes_packed+=new_huff_el->count/8; + /* + Replace the copied top element by the new element and re-order the + queue. This successively replaces the references to counts by + references to HUFF_ELEMENTs. + */ queue.root[1]=(byte*) new_huff_el; queue_replaced(&queue); } @@ -1417,13 +1857,26 @@ static uint join_same_trees(HUFF_COUNTS *huff_counts, uint trees) } } } + DBUG_PRINT("info", ("Original trees: %d After join: %d", + trees, tree_number)); if (verbose) - printf("Original trees: %d After join: %d\n",trees,tree_number); + VOID(printf("Original trees: %d After join: %d\n", trees, tree_number)); return tree_number; /* Return trees left */ } - /* Fill in huff_tree decode tables */ +/* + Fill in huff_tree encode tables. + + SYNOPSIS + make_huff_decode_table() + huff_tree An array of HUFF_TREE which are to be encoded. + trees The number of HUFF_TREE in the array. + + RETURN + 0 success + != 0 error +*/ static int make_huff_decode_table(HUFF_TREE *huff_tree, uint trees) { @@ -1434,12 +1887,13 @@ static int make_huff_decode_table(HUFF_TREE *huff_tree, uint trees) { elements=huff_tree->counts->tree_buff ? huff_tree->elements : 256; if (!(huff_tree->code = - (ulong*) my_malloc(elements* - (sizeof(ulong)+sizeof(uchar)), - MYF(MY_WME | MY_ZEROFILL)))) + (ulonglong*) my_malloc(elements* + (sizeof(ulonglong) + sizeof(uchar)), + MYF(MY_WME | MY_ZEROFILL)))) return 1; huff_tree->code_len=(uchar*) (huff_tree->code+elements); - make_traverse_code_tree(huff_tree,huff_tree->root,32,0); + make_traverse_code_tree(huff_tree, huff_tree->root, + 8 * sizeof(ulonglong), LL(0)); } } return 0; @@ -1448,28 +1902,90 @@ static int make_huff_decode_table(HUFF_TREE *huff_tree, uint trees) static void make_traverse_code_tree(HUFF_TREE *huff_tree, HUFF_ELEMENT *element, - uint size, ulong code) + uint size, ulonglong code) { uint chr; if (!element->a.leaf.null) { chr=element->a.leaf.element_nr; - huff_tree->code_len[chr]=(uchar) (32-size); - huff_tree->code[chr]= (code >> size); - if (huff_tree->height < 32-size) - huff_tree->height= 32-size; + huff_tree->code_len[chr]= (uchar) (8 * sizeof(ulonglong) - size); + huff_tree->code[chr]= (code >> size); + if (huff_tree->height < 8 * sizeof(ulonglong) - size) + huff_tree->height= 8 * sizeof(ulonglong) - size; } else { size--; make_traverse_code_tree(huff_tree,element->a.nod.left,size,code); - make_traverse_code_tree(huff_tree,element->a.nod.right,size, - code+((ulong) 1L << size)); + make_traverse_code_tree(huff_tree, element->a.nod.right, size, + code + (((ulonglong) 1) << size)); } return; } +/* + Convert a value into binary digits. + + SYNOPSIS + bindigits() + value The value. + length The number of low order bits to convert. + + NOTE + The result string is in static storage. It is reused on every call. + So you cannot use it twice in one expression. + + RETURN + A pointer to a static NUL-terminated string. + */ + +static char *bindigits(ulonglong value, uint bits) +{ + static char digits[72]; + char *ptr= digits; + uint idx= bits; + + DBUG_ASSERT(idx < sizeof(digits)); + while (idx) + *(ptr++)= '0' + ((value >> (--idx)) & 1); + *ptr= '\0'; + return digits; +} + + +/* + Convert a value into hexadecimal digits. + + SYNOPSIS + hexdigits() + value The value. + + NOTE + The result string is in static storage. It is reused on every call. + So you cannot use it twice in one expression. + + RETURN + A pointer to a static NUL-terminated string. + */ + +static char *hexdigits(ulonglong value) +{ + static char digits[20]; + char *ptr= digits; + uint idx= 2 * sizeof(value); /* Two hex digits per byte. */ + + DBUG_ASSERT(idx < sizeof(digits)); + while (idx) + { + if ((*(ptr++)= '0' + ((value >> (4 * (--idx))) & 0xf)) > '9') + *(ptr - 1)+= 'a' - '9' - 1; + } + *ptr= '\0'; + return digits; +} + + /* Write header to new packed data file */ static int write_header(PACK_MRG_INFO *mrg,uint head_length,uint trees, @@ -1503,15 +2019,64 @@ static void write_field_info(HUFF_COUNTS *counts, uint fields, uint trees) uint huff_tree_bits; huff_tree_bits=max_bit(trees ? trees-1 : 0); + DBUG_PRINT("info", ("")); + DBUG_PRINT("info", ("column types:")); + DBUG_PRINT("info", ("FIELD_NORMAL 0")); + DBUG_PRINT("info", ("FIELD_SKIP_ENDSPACE 1")); + DBUG_PRINT("info", ("FIELD_SKIP_PRESPACE 2")); + DBUG_PRINT("info", ("FIELD_SKIP_ZERO 3")); + DBUG_PRINT("info", ("FIELD_BLOB 4")); + DBUG_PRINT("info", ("FIELD_CONSTANT 5")); + DBUG_PRINT("info", ("FIELD_INTERVALL 6")); + DBUG_PRINT("info", ("FIELD_ZERO 7")); + DBUG_PRINT("info", ("FIELD_VARCHAR 8")); + DBUG_PRINT("info", ("FIELD_CHECK 9")); + DBUG_PRINT("info", ("")); + DBUG_PRINT("info", ("pack type as a set of flags:")); + DBUG_PRINT("info", ("PACK_TYPE_SELECTED 1")); + DBUG_PRINT("info", ("PACK_TYPE_SPACE_FIELDS 2")); + DBUG_PRINT("info", ("PACK_TYPE_ZERO_FILL 4")); + DBUG_PRINT("info", ("")); + if (verbose >= 2) + { + VOID(printf("\n")); + VOID(printf("column types:\n")); + VOID(printf("FIELD_NORMAL 0\n")); + VOID(printf("FIELD_SKIP_ENDSPACE 1\n")); + VOID(printf("FIELD_SKIP_PRESPACE 2\n")); + VOID(printf("FIELD_SKIP_ZERO 3\n")); + VOID(printf("FIELD_BLOB 4\n")); + VOID(printf("FIELD_CONSTANT 5\n")); + VOID(printf("FIELD_INTERVALL 6\n")); + VOID(printf("FIELD_ZERO 7\n")); + VOID(printf("FIELD_VARCHAR 8\n")); + VOID(printf("FIELD_CHECK 9\n")); + VOID(printf("\n")); + VOID(printf("pack type as a set of flags:\n")); + VOID(printf("PACK_TYPE_SELECTED 1\n")); + VOID(printf("PACK_TYPE_SPACE_FIELDS 2\n")); + VOID(printf("PACK_TYPE_ZERO_FILL 4\n")); + VOID(printf("\n")); + } for (i=0 ; i++ < fields ; counts++) { - write_bits((ulong) (int) counts->field_type,5); + write_bits((ulonglong) (int) counts->field_type, 5); write_bits(counts->pack_type,6); if (counts->pack_type & PACK_TYPE_ZERO_FILL) write_bits(counts->max_zero_fill,5); else write_bits(counts->length_bits,5); - write_bits((ulong) counts->tree->tree_number-1,huff_tree_bits); + write_bits((ulonglong) counts->tree->tree_number - 1, huff_tree_bits); + DBUG_PRINT("info", ("column: %3u type: %2u pack: %2u zero: %4u " + "lbits: %2u tree: %2u length: %4u", + i , counts->field_type, counts->pack_type, + counts->max_zero_fill, counts->length_bits, + counts->tree->tree_number, counts->field_length)); + if (verbose >= 2) + VOID(printf("column: %3u type: %2u pack: %2u zero: %4u lbits: %2u " + "tree: %2u length: %4u\n", i , counts->field_type, + counts->pack_type, counts->max_zero_fill, counts->length_bits, + counts->tree->tree_number, counts->field_length)); } flush_bits(); return; @@ -1524,45 +2089,72 @@ static void write_field_info(HUFF_COUNTS *counts, uint fields, uint trees) static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees) { uint i,int_length; + uint tree_no; + uint codes; + uint errors= 0; uint *packed_tree,*offset,length; my_off_t elements; + /* Find the highest number of elements in the trees. */ for (i=length=0 ; i < trees ; i++) if (huff_tree[i].tree_number > 0 && huff_tree[i].elements > length) length=huff_tree[i].elements; + /* + Allocate a buffer for packing a decode tree. Two numbers per element + (left child and right child). + */ if (!(packed_tree=(uint*) my_alloca(sizeof(uint)*length*2))) { my_error(EE_OUTOFMEMORY,MYF(ME_BELL),sizeof(uint)*length*2); return 0; } + DBUG_PRINT("info", ("")); + if (verbose >= 2) + VOID(printf("\n")); + tree_no= 0; intervall_length=0; for (elements=0; trees-- ; huff_tree++) { + /* Skip columns that have been joined with other columns. */ if (huff_tree->tree_number == 0) continue; /* Deleted tree */ + tree_no++; + DBUG_PRINT("info", ("")); + if (verbose >= 3) + VOID(printf("\n")); + /* Count the total number of elements (byte codes or column values). */ elements+=huff_tree->elements; huff_tree->max_offset=2; + /* Build a tree of offsets and codes for decoding in 'packed_tree'. */ if (huff_tree->elements <= 1) offset=packed_tree; else offset=make_offset_code_tree(huff_tree,huff_tree->root,packed_tree); + + /* This should be the same as 'length' above. */ huff_tree->offset_bits=max_bit(huff_tree->max_offset); + + /* + Since we check this during collecting the distinct column values, + this should never happen. + */ if (huff_tree->max_offset >= IS_OFFSET) { /* This should be impossible */ - VOID(fprintf(stderr,"Tree offset got too big: %d, aborted\n", - huff_tree->max_offset)); + VOID(fprintf(stderr, "Tree offset got too big: %d, aborted\n", + huff_tree->max_offset)); my_afree((gptr) packed_tree); return 0; } -#ifdef EXTRA_DBUG - printf("pos: %d elements: %d tree-elements: %d char_bits: %d\n", - (uint) (file_buffer.pos-file_buffer.buffer), - huff_tree->elements, (offset-packed_tree),huff_tree->char_bits); -#endif + DBUG_PRINT("info", ("pos: %lu elements: %u tree-elements: %lu " + "char_bits: %u\n", + (ulong) (file_buffer.pos - file_buffer.buffer), + huff_tree->elements, (ulong) (offset - packed_tree), + huff_tree->char_bits)); if (!huff_tree->counts->tree_buff) { + /* We do a byte compression on this column. Mark with bit 0. */ write_bits(0,1); write_bits(huff_tree->min_chr,8); write_bits(huff_tree->elements,9); @@ -1574,6 +2166,7 @@ static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees) { int_length=(uint) (huff_tree->counts->tree_pos - huff_tree->counts->tree_buff); + /* We have distinct column values for this column. Mark with bit 1. */ write_bits(1,1); write_bits(huff_tree->elements,15); write_bits(int_length,16); @@ -1581,10 +2174,29 @@ static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees) write_bits(huff_tree->offset_bits,5); intervall_length+=int_length; } + DBUG_PRINT("info", ("tree: %2u elements: %4u char_bits: %2u " + "offset_bits: %2u %s: %5u codelen: %2u", + tree_no, huff_tree->elements, huff_tree->char_bits, + huff_tree->offset_bits, huff_tree->counts->tree_buff ? + "bufflen" : "min_chr", huff_tree->counts->tree_buff ? + int_length : huff_tree->min_chr, huff_tree->height)); + if (verbose >= 2) + VOID(printf("tree: %2u elements: %4u char_bits: %2u offset_bits: %2u " + "%s: %5u codelen: %2u\n", tree_no, huff_tree->elements, + huff_tree->char_bits, huff_tree->offset_bits, + huff_tree->counts->tree_buff ? "bufflen" : "min_chr", + huff_tree->counts->tree_buff ? int_length : + huff_tree->min_chr, huff_tree->height)); + + /* Check that the code tree length matches the element count. */ length=(uint) (offset-packed_tree); if (length != huff_tree->elements*2-2) - printf("error: Huff-tree-length: %d != calc_length: %d\n", - length,huff_tree->elements*2-2); + { + VOID(fprintf(stderr, "error: Huff-tree-length: %d != calc_length: %d\n", + length, huff_tree->elements * 2 - 2)); + errors++; + break; + } for (i=0 ; i < length ; i++) { @@ -1593,16 +2205,122 @@ static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees) huff_tree->offset_bits+1); else write_bits(packed_tree[i]-huff_tree->min_chr,huff_tree->char_bits+1); + DBUG_PRINT("info", ("tree[0x%04x]: %s0x%04x", + i, (packed_tree[i] & IS_OFFSET) ? + " -> " : "", (packed_tree[i] & IS_OFFSET) ? + packed_tree[i] - IS_OFFSET + i : packed_tree[i])); + if (verbose >= 3) + VOID(printf("tree[0x%04x]: %s0x%04x\n", + i, (packed_tree[i] & IS_OFFSET) ? " -> " : "", + (packed_tree[i] & IS_OFFSET) ? + packed_tree[i] - IS_OFFSET + i : packed_tree[i])); } flush_bits(); + + /* + Display coding tables and check their correctness. + */ + codes= huff_tree->counts->tree_buff ? huff_tree->elements : 256; + for (i= 0; i < codes; i++) + { + ulonglong code; + uint bits; + uint len; + uint idx; + + if (! (len= huff_tree->code_len[i])) + continue; + DBUG_PRINT("info", ("code[0x%04x]: 0x%s bits: %2u bin: %s", i, + hexdigits(huff_tree->code[i]), huff_tree->code_len[i], + bindigits(huff_tree->code[i], + huff_tree->code_len[i]))); + if (verbose >= 3) + VOID(printf("code[0x%04x]: 0x%s bits: %2u bin: %s\n", i, + hexdigits(huff_tree->code[i]), huff_tree->code_len[i], + bindigits(huff_tree->code[i], huff_tree->code_len[i]))); + + /* Check that the encode table decodes correctly. */ + code= 0; + bits= 0; + idx= 0; + DBUG_EXECUTE_IF("forcechkerr1", len--;); + DBUG_EXECUTE_IF("forcechkerr2", bits= 8 * sizeof(code);); + DBUG_EXECUTE_IF("forcechkerr3", idx= length;); + for (;;) + { + if (! len) + { + VOID(fflush(stdout)); + VOID(fprintf(stderr, "error: code 0x%s with %u bits not found\n", + hexdigits(huff_tree->code[i]), huff_tree->code_len[i])); + errors++; + break; + } + code<<= 1; + code|= (huff_tree->code[i] >> (--len)) & 1; + bits++; + if (bits > 8 * sizeof(code)) + { + VOID(fflush(stdout)); + VOID(fprintf(stderr, "error: Huffman code too long: %u/%u\n", + bits, 8 * sizeof(code))); + errors++; + break; + } + idx+= code & 1; + if (idx >= length) + { + VOID(fflush(stdout)); + VOID(fprintf(stderr, "error: illegal tree offset: %u/%u\n", + idx, length)); + errors++; + break; + } + if (packed_tree[idx] & IS_OFFSET) + idx+= packed_tree[idx] & ~IS_OFFSET; + else + break; /* Hit a leaf. This contains the result value. */ + } + if (errors) + break; + + DBUG_EXECUTE_IF("forcechkerr4", packed_tree[idx]++;); + if (packed_tree[idx] != i) + { + VOID(fflush(stdout)); + VOID(fprintf(stderr, "error: decoded value 0x%04x should be: 0x%04x\n", + packed_tree[idx], i)); + errors++; + break; + } + } /*end for (codes)*/ + if (errors) + break; + + /* Write column values in case of distinct column value compression. */ if (huff_tree->counts->tree_buff) { for (i=0 ; i < int_length ; i++) - write_bits((uint) (uchar) huff_tree->counts->tree_buff[i],8); + { + write_bits((ulonglong) (uchar) huff_tree->counts->tree_buff[i], 8); + DBUG_PRINT("info", ("column_values[0x%04x]: 0x%02x", + i, (uchar) huff_tree->counts->tree_buff[i])); + if (verbose >= 3) + VOID(printf("column_values[0x%04x]: 0x%02x\n", + i, (uchar) huff_tree->counts->tree_buff[i])); + } } flush_bits(); } + DBUG_PRINT("info", ("")); + if (verbose >= 2) + VOID(printf("\n")); my_afree((gptr) packed_tree); + if (errors) + { + VOID(fprintf(stderr, "Error: Generated decode trees are corrupt. Stop.\n")); + return 0; + } return elements; } @@ -1613,23 +2331,43 @@ static uint *make_offset_code_tree(HUFF_TREE *huff_tree, HUFF_ELEMENT *element, uint *prev_offset; prev_offset= offset; + /* + 'a.leaf.null' takes the same place as 'a.nod.left'. If this is null, + then there is no left child and, hence no right child either. This + is a property of a binary tree. An element is either a node with two + childs, or a leaf without childs. + + The current element is always a node with two childs. Go left first. + */ if (!element->a.nod.left->a.leaf.null) { - offset[0] =(uint) element->a.nod.left->a.leaf.element_nr; + /* Store the byte code or the index of the column value. */ + prev_offset[0] =(uint) element->a.nod.left->a.leaf.element_nr; offset+=2; } else { + /* + Recursively traverse the tree to the left. Mark it as an offset to + another tree node (in contrast to a byte code or column value index). + */ prev_offset[0]= IS_OFFSET+2; offset=make_offset_code_tree(huff_tree,element->a.nod.left,offset+2); } + + /* Now, check the right child. */ if (!element->a.nod.right->a.leaf.null) { + /* Store the byte code or the index of the column value. */ prev_offset[1]=element->a.nod.right->a.leaf.element_nr; return offset; } else { + /* + Recursively traverse the tree to the right. Mark it as an offset to + another tree node (in contrast to a byte code or column value index). + */ uint temp=(uint) (offset-prev_offset-1); prev_offset[1]= IS_OFFSET+ temp; if (huff_tree->max_offset < temp) @@ -1656,6 +2394,7 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) uint i,max_calc_length,pack_ref_length,min_record_length,max_record_length, intervall,field_length,max_pack_length,pack_blob_length; my_off_t record_count; + char llbuf[32]; ulong length,pack_length; byte *record,*pos,*end_pos,*record_pos,*start_pos; HUFF_COUNTS *count,*end_count; @@ -1663,12 +2402,23 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) MI_INFO *isam_file=mrg->file[0]; DBUG_ENTER("compress_isam_file"); + /* Allocate a buffer for the records (excluding blobs). */ if (!(record=(byte*) my_alloca(isam_file->s->base.reclength))) return -1; + end_count=huff_counts+isam_file->s->base.fields; min_record_length= (uint) ~0; max_record_length=0; + /* + Calculate the maximum number of bits required to pack the records. + Remember to understand 'max_zero_fill' as 'min_zero_fill'. + The tree height determines the maximum number of bits per value. + Some fields skip leading or trailing spaces or zeroes. The skipped + number of bytes is encoded by 'length_bits' bits. + Empty blobs and varchar are encoded with a single 1 bit. Other blobs + and varchar get a leading 0 bit. + */ for (i=max_calc_length=0 ; i < isam_file->s->base.fields ; i++) { if (!(huff_counts[i].pack_type & PACK_TYPE_ZERO_FILL)) @@ -1687,14 +2437,16 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) (huff_counts[i].field_length - huff_counts[i].max_zero_fill)* huff_counts[i].tree->height+huff_counts[i].length_bits; } - max_calc_length/=8; + max_calc_length= (max_calc_length + 7) / 8; if (max_calc_length < 254) pack_ref_length=1; else if (max_calc_length <= 65535) pack_ref_length=3; else pack_ref_length=4; + record_count=0; + /* 'max_blob_length' is the max length of all blobs of a record. */ pack_blob_length=0; if (isam_file->s->base.blobs) { @@ -1707,6 +2459,7 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) } max_pack_length=pack_ref_length+pack_blob_length; + DBUG_PRINT("fields", ("===")); mrg_reset(mrg); while ((error=mrg_rrnd(mrg,record)) != HA_ERR_END_OF_FILE) { @@ -1722,15 +2475,29 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) end_pos=start_pos+(field_length=count->field_length); tree=count->tree; + DBUG_PRINT("fields", ("column: %3lu type: %2u pack: %2u zero: %4u " + "lbits: %2u tree: %2u length: %4u", + (ulong) (count - huff_counts + 1), + count->field_type, + count->pack_type, count->max_zero_fill, + count->length_bits, count->tree->tree_number, + count->field_length)); + + /* Check if the column contains spaces only. */ if (count->pack_type & PACK_TYPE_SPACE_FIELDS) { for (pos=start_pos ; *pos == ' ' && pos < end_pos; pos++) ; if (pos == end_pos) { + DBUG_PRINT("fields", + ("PACK_TYPE_SPACE_FIELDS spaces only, bits: 1")); + DBUG_PRINT("fields", ("---")); write_bits(1,1); start_pos=end_pos; continue; } + DBUG_PRINT("fields", + ("PACK_TYPE_SPACE_FIELDS not only spaces, bits: 1")); write_bits(0,1); } end_pos-=count->max_zero_fill; @@ -1740,65 +2507,129 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) case FIELD_SKIP_ZERO: if (!memcmp((byte*) start_pos,zero_string,field_length)) { + DBUG_PRINT("fields", ("FIELD_SKIP_ZERO zeroes only, bits: 1")); write_bits(1,1); start_pos=end_pos; break; } + DBUG_PRINT("fields", ("FIELD_SKIP_ZERO not only zeroes, bits: 1")); write_bits(0,1); /* Fall through */ case FIELD_NORMAL: + DBUG_PRINT("fields", ("FIELD_NORMAL %lu bytes", + (ulong) (end_pos - start_pos))); for ( ; start_pos < end_pos ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]))); write_bits(tree->code[(uchar) *start_pos], (uint) tree->code_len[(uchar) *start_pos]); + } break; case FIELD_SKIP_ENDSPACE: for (pos=end_pos ; pos > start_pos && pos[-1] == ' ' ; pos--) ; - length=(uint) (end_pos-pos); + length= (ulong) (end_pos - pos); if (count->pack_type & PACK_TYPE_SELECTED) { if (length > count->min_space) { + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE more than min_space, bits: 1")); + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); write_bits(1,1); write_bits(length,count->length_bits); } else { + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE not more than min_space, " + "bits: 1")); write_bits(0,1); pos=end_pos; } } else + { + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); write_bits(length,count->length_bits); + } + /* Encode all significant bytes. */ + DBUG_PRINT("fields", ("FIELD_SKIP_ENDSPACE %lu bytes", + (ulong) (pos - start_pos))); for ( ; start_pos < pos ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]))); write_bits(tree->code[(uchar) *start_pos], (uint) tree->code_len[(uchar) *start_pos]); + } start_pos=end_pos; break; case FIELD_SKIP_PRESPACE: for (pos=start_pos ; pos < end_pos && pos[0] == ' ' ; pos++) ; - length=(uint) (pos-start_pos); + length= (ulong) (pos - start_pos); if (count->pack_type & PACK_TYPE_SELECTED) { if (length > count->min_space) { + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE more than min_space, bits: 1")); + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); write_bits(1,1); write_bits(length,count->length_bits); } else { + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE not more than min_space, " + "bits: 1")); pos=start_pos; write_bits(0,1); } } else + { + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); write_bits(length,count->length_bits); + } + /* Encode all significant bytes. */ + DBUG_PRINT("fields", ("FIELD_SKIP_PRESPACE %lu bytes", + (ulong) (end_pos - start_pos))); for (start_pos=pos ; start_pos < end_pos ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]))); write_bits(tree->code[(uchar) *start_pos], (uint) tree->code_len[(uchar) *start_pos]); + } break; case FIELD_CONSTANT: case FIELD_ZERO: case FIELD_CHECK: + DBUG_PRINT("fields", ("FIELD_CONSTANT/ZERO/CHECK")); start_pos=end_pos; break; case FIELD_INTERVALL: @@ -1806,6 +2637,10 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) pos=(byte*) tree_search(&count->int_tree, start_pos, count->int_tree.custom_arg); intervall=(uint) (pos - count->tree_buff)/field_length; + DBUG_PRINT("fields", ("FIELD_INTERVALL")); + DBUG_PRINT("fields", ("index: %4u code: 0x%s bits: %2u", + intervall, hexdigits(tree->code[intervall]), + (uint) tree->code_len[intervall])); write_bits(tree->code[intervall],(uint) tree->code_len[intervall]); start_pos=end_pos; break; @@ -1814,21 +2649,36 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) ulong blob_length=_mi_calc_blob_length(field_length- mi_portable_sizeof_char_ptr, start_pos); + /* Empty blobs are encoded with a single 1 bit. */ if (!blob_length) { - write_bits(1,1); /* Empty blob */ + DBUG_PRINT("fields", ("FIELD_BLOB empty, bits: 1")); + write_bits(1,1); } else { byte *blob,*blob_end; + DBUG_PRINT("fields", ("FIELD_BLOB not empty, bits: 1")); write_bits(0,1); + /* Write the blob length. */ + DBUG_PRINT("fields", ("FIELD_BLOB %lu bytes, bits: %2u", + blob_length, count->length_bits)); write_bits(blob_length,count->length_bits); memcpy_fixed(&blob,end_pos-mi_portable_sizeof_char_ptr, sizeof(char*)); blob_end=blob+blob_length; + /* Encode the blob bytes. */ for ( ; blob < blob_end ; blob++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *blob, hexdigits(tree->code[(uchar) *blob]), + (uint) tree->code_len[(uchar) *blob], + bindigits(tree->code[(uchar) *start_pos], + (uint)tree->code_len[(uchar) *start_pos]))); write_bits(tree->code[(uchar) *blob], (uint) tree->code_len[(uchar) *blob]); + } tot_blob_length+=blob_length; } start_pos= end_pos; @@ -1839,18 +2689,34 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) uint pack_length= HA_VARCHAR_PACKLENGTH(count->field_length-1); ulong col_length= (pack_length == 1 ? (uint) *(uchar*) start_pos : uint2korr(start_pos)); + /* Empty varchar are encoded with a single 1 bit. */ if (!col_length) { + DBUG_PRINT("fields", ("FIELD_VARCHAR empty, bits: 1")); write_bits(1,1); /* Empty varchar */ } else { byte *end=start_pos+pack_length+col_length; + DBUG_PRINT("fields", ("FIELD_VARCHAR not empty, bits: 1")); write_bits(0,1); + /* Write the varchar length. */ + DBUG_PRINT("fields", ("FIELD_VARCHAR %lu bytes, bits: %2u", + col_length, count->length_bits)); write_bits(col_length,count->length_bits); + /* Encode the varchar bytes. */ for (start_pos+=pack_length ; start_pos < end ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint)tree->code_len[(uchar) *start_pos]))); write_bits(tree->code[(uchar) *start_pos], (uint) tree->code_len[(uchar) *start_pos]); + } } start_pos= end_pos; break; @@ -1859,12 +2725,17 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) abort(); /* Impossible */ } start_pos+=count->max_zero_fill; + DBUG_PRINT("fields", ("---")); } flush_bits(); - length=(ulong) (file_buffer.pos-record_pos)-max_pack_length; + length=(ulong) ((byte*) file_buffer.pos - record_pos) - max_pack_length; pack_length=save_pack_length(record_pos,length); if (pack_blob_length) pack_length+=save_pack_length(record_pos+pack_length,tot_blob_length); + DBUG_PRINT("fields", ("record: %lu length: %lu blob-length: %lu " + "length-bytes: %lu", (ulong) record_count, length, + tot_blob_length, pack_length)); + DBUG_PRINT("fields", ("===")); /* Correct file buffer if the header was smaller */ if (pack_length != max_pack_length) @@ -1876,9 +2747,11 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) min_record_length=(uint) length; if (length > (ulong) max_record_length) max_record_length=(uint) length; - if (write_loop && ++record_count % WRITE_COUNT == 0) + record_count++; + if (write_loop && record_count % WRITE_COUNT == 0) { - printf("%lu\r",(ulong) record_count); VOID(fflush(stdout)); + VOID(printf("%lu\r", (ulong) record_count)); + VOID(fflush(stdout)); } } else if (error != HA_ERR_RECORD_DELETED) @@ -1888,8 +2761,11 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) error=0; else { - fprintf(stderr,"%s: Got error %d reading records\n",my_progname,error); + VOID(fprintf(stderr, "%s: Got error %d reading records\n", + my_progname, error)); } + if (verbose >= 2) + VOID(printf("wrote %s records.\n", llstr((longlong) record_count, llbuf))); my_afree((gptr) record); mrg->ref_length=max_pack_length; @@ -1929,7 +2805,7 @@ static void init_file_buffer(File file, pbool read_buffer) file_buffer.pos=file_buffer.buffer; file_buffer.bits=BITS_SAVED; } - file_buffer.current_byte=0; + file_buffer.bitbucket= 0; } @@ -1972,7 +2848,8 @@ static int flush_buffer(ulong neaded_length) tmp=my_realloc(file_buffer.buffer, neaded_length,MYF(MY_WME)); if (!tmp) return 1; - file_buffer.pos= tmp + (ulong) (file_buffer.pos - file_buffer.buffer); + file_buffer.pos= ((uchar*) tmp + + (ulong) (file_buffer.pos - file_buffer.buffer)); file_buffer.buffer=tmp; file_buffer.end=tmp+neaded_length-8; } @@ -1987,68 +2864,59 @@ static void end_file_buffer(void) /* output `bits` low bits of `value' */ -static void write_bits (register ulong value, register uint bits) +static void write_bits(register ulonglong value, register uint bits) { - if ((file_buffer.bits-=(int) bits) >= 0) + DBUG_ASSERT(((bits < 8 * sizeof(value)) && ! (value >> bits)) || + (bits == 8 * sizeof(value))); + + if ((file_buffer.bits-= (int) bits) >= 0) { - file_buffer.current_byte|=value << file_buffer.bits; + file_buffer.bitbucket|= value << file_buffer.bits; } else { - reg3 uint byte_buff; + reg3 ulonglong bit_buffer; bits= (uint) -file_buffer.bits; - DBUG_ASSERT(bits <= 8 * sizeof(value)); - byte_buff= (file_buffer.current_byte | - ((bits != 8 * sizeof(value)) ? (uint) (value >> bits) : 0)); -#if BITS_SAVED == 32 - *file_buffer.pos++= (byte) (byte_buff >> 24) ; - *file_buffer.pos++= (byte) (byte_buff >> 16) ; + bit_buffer= (file_buffer.bitbucket | + ((bits != 8 * sizeof(value)) ? (value >> bits) : 0)); +#if BITS_SAVED == 64 + *file_buffer.pos++= (uchar) (bit_buffer >> 56); + *file_buffer.pos++= (uchar) (bit_buffer >> 48); + *file_buffer.pos++= (uchar) (bit_buffer >> 40); + *file_buffer.pos++= (uchar) (bit_buffer >> 32); #endif - *file_buffer.pos++= (byte) (byte_buff >> 8) ; - *file_buffer.pos++= (byte) byte_buff; + *file_buffer.pos++= (uchar) (bit_buffer >> 24); + *file_buffer.pos++= (uchar) (bit_buffer >> 16); + *file_buffer.pos++= (uchar) (bit_buffer >> 8); + *file_buffer.pos++= (uchar) (bit_buffer); - DBUG_ASSERT(bits <= 8 * sizeof(ulong)); if (bits != 8 * sizeof(value)) - value&= (((ulong) 1) << bits) - 1; -#if BITS_SAVED == 16 - if (bits >= sizeof(uint)) - { - bits-=8; - *file_buffer.pos++= (uchar) (value >> bits); - value&= (1 << bits)-1; - if (bits >= sizeof(uint)) - { - bits-=8; - *file_buffer.pos++= (uchar) (value >> bits); - value&= (1 << bits)-1; - } - } -#endif + value&= (((ulonglong) 1) << bits) - 1; if (file_buffer.pos >= file_buffer.end) VOID(flush_buffer(~ (ulong) 0)); file_buffer.bits=(int) (BITS_SAVED - bits); - file_buffer.current_byte=(uint) (value << (BITS_SAVED - bits)); + file_buffer.bitbucket= value << (BITS_SAVED - bits); } return; } /* Flush bits in bit_buffer to buffer */ -static void flush_bits (void) +static void flush_bits(void) { - uint bits,byte_buff; + int bits; + ulonglong bit_buffer; - bits=(file_buffer.bits) & ~7; - byte_buff = file_buffer.current_byte >> bits; - bits=BITS_SAVED - bits; + bits= file_buffer.bits & ~7; + bit_buffer= file_buffer.bitbucket >> bits; + bits= BITS_SAVED - bits; while (bits > 0) { - bits-=8; - *file_buffer.pos++= (byte) (uchar) (byte_buff >> bits) ; + bits-= 8; + *file_buffer.pos++= (uchar) (bit_buffer >> bits); } - file_buffer.bits=BITS_SAVED; - file_buffer.current_byte=0; - return; + file_buffer.bits= BITS_SAVED; + file_buffer.bitbucket= 0; } @@ -2196,3 +3064,131 @@ static int mrg_close(PACK_MRG_INFO *mrg) my_free((gptr) mrg->file,MYF(0)); return error; } + + +#if !defined(DBUG_OFF) +/* + Fake the counts to get big Huffman codes. + + SYNOPSIS + fakebigcodes() + huff_counts A pointer to the counts array. + end_count A pointer past the counts array. + + DESCRIPTION + + Huffman coding works by removing the two least frequent values from + the list of values and add a new value with the sum of their + incidences in a loop until only one value is left. Every time a + value is reused for a new value, it gets one more bit for its + encoding. Hence, the least frequent values get the longest codes. + + To get a maximum code length for a value, two of the values must + have an incidence of 1. As their sum is 2, the next infrequent value + must have at least an incidence of 2, then 4, 8, 16 and so on. This + means that one needs 2**n bytes (values) for a code length of n + bits. However, using more distinct values forces the use of longer + codes, or reaching the code length with less total bytes (values). + + To get 64(32)-bit codes, I sort the counts by decreasing incidence. + I assign counts of 1 to the two most frequent values, a count of 2 + for the next one, then 4, 8, and so on until 2**64-1(2**30-1). All + the remaining values get 1. That way every possible byte has an + assigned code, though not all codes are used if not all byte values + are present in the column. + + This strategy would work with distinct column values too, but + requires that at least 64(32) values are present. To make things + easier here, I cancel all distinct column values and force byte + compression for all columns. + + RETURN + void +*/ + +static void fakebigcodes(HUFF_COUNTS *huff_counts, HUFF_COUNTS *end_count) +{ + HUFF_COUNTS *count; + my_off_t *cur_count_p; + my_off_t *end_count_p; + my_off_t **cur_sort_p; + my_off_t **end_sort_p; + my_off_t *sort_counts[256]; + my_off_t total; + DBUG_ENTER("fakebigcodes"); + + for (count= huff_counts; count < end_count; count++) + { + /* + Remove distinct column values. + */ + if (huff_counts->tree_buff) + { + my_free((gptr) huff_counts->tree_buff, MYF(0)); + delete_tree(&huff_counts->int_tree); + huff_counts->tree_buff= NULL; + DBUG_PRINT("fakebigcodes", ("freed distinct column values")); + } + + /* + Sort counts by decreasing incidence. + */ + cur_count_p= count->counts; + end_count_p= cur_count_p + 256; + cur_sort_p= sort_counts; + while (cur_count_p < end_count_p) + *(cur_sort_p++)= cur_count_p++; + (void) qsort(sort_counts, 256, sizeof(my_off_t*), (qsort_cmp) fakecmp); + + /* + Assign faked counts. + */ + cur_sort_p= sort_counts; +#if SIZEOF_LONG_LONG > 4 + end_sort_p= sort_counts + 8 * sizeof(ulonglong) - 1; +#else + end_sort_p= sort_counts + 8 * sizeof(ulonglong) - 2; +#endif + /* Most frequent value gets a faked count of 1. */ + **(cur_sort_p++)= 1; + total= 1; + while (cur_sort_p < end_sort_p) + { + **(cur_sort_p++)= total; + total<<= 1; + } + /* Set the last value. */ + **(cur_sort_p++)= --total; + /* + Set the remaining counts. + */ + end_sort_p= sort_counts + 256; + while (cur_sort_p < end_sort_p) + **(cur_sort_p++)= 1; + } + DBUG_VOID_RETURN; +} + + +/* + Compare two counts for reverse sorting. + + SYNOPSIS + fakecmp() + count1 One count. + count2 Another count. + + RETURN + 1 count1 < count2 + 0 count1 == count2 + -1 count1 > count2 +*/ + +static int fakecmp(my_off_t **count1, my_off_t **count2) +{ + return ((**count1 < **count2) ? 1 : + (**count1 > **count2) ? -1 : 0); +} +#endif + + diff --git a/mysys/tree.c b/mysys/tree.c index bec1ec680f1..1780913961e 100644 --- a/mysys/tree.c +++ b/mysys/tree.c @@ -263,6 +263,9 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size, if (tree->flag & TREE_NO_DUPS) return(NULL); element->count++; + /* Avoid a wrap over of the count. */ + if (! element->count) + element->count--; } DBUG_EXECUTE("check_tree", test_rb_tree(tree->root);); return element; From 1960fbcf3d9dae7eb08833bdcf5b898224c2b817 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 19:47:19 +0200 Subject: [PATCH 28/32] Bug#10178 - failure to find a row in heap table by concurrent UPDATEs After merge fixes of test result. --- mysql-test/r/heap.result | 8 ++++---- mysql-test/r/heap_hash.result | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mysql-test/r/heap.result b/mysql-test/r/heap.result index 22304c4a93d..b905dae3aba 100644 --- a/mysql-test/r/heap.result +++ b/mysql-test/r/heap.result @@ -367,13 +367,13 @@ count(*) 9 explain select count(*) from t1 where v='a '; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref v v 13 const 9 Using where +1 SIMPLE t1 ref v v 13 const 10 Using where explain select count(*) from t1 where c='a '; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref c c 11 const 9 Using where +1 SIMPLE t1 ref c c 11 const 10 Using where explain select count(*) from t1 where t='a '; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref t t 13 const 9 Using where +1 SIMPLE t1 ref t t 13 const 10 Using where explain select count(*) from t1 where v like 'a%'; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL v NULL NULL NULL 271 Using where @@ -399,7 +399,7 @@ qq *a *a*a * explain select * from t1 where v='a'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref v v 13 const 9 Using where +1 SIMPLE t1 ref v v 13 const 10 Using where select v,count(*) from t1 group by v limit 10; v count(*) a 1 diff --git a/mysql-test/r/heap_hash.result b/mysql-test/r/heap_hash.result index 346fdd640ca..d8d89b786b5 100644 --- a/mysql-test/r/heap_hash.result +++ b/mysql-test/r/heap_hash.result @@ -353,8 +353,8 @@ t3 1 a 1 a NULL NULL NULL NULL HASH t3 1 a 2 b NULL 13 NULL NULL HASH explain select * from t1 ignore key(btree_idx), t3 where t1.name='matt' and t3.a = concat('',t1.name) and t3.b=t1.name; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 ref a a 44 const,const 7 Using where 1 SIMPLE t1 ref heap_idx heap_idx 22 const 7 Using where +1 SIMPLE t3 ref a a 44 const,const 7 Using where drop table t1, t2, t3; create temporary table t1 ( a int, index (a) ) engine=memory; insert into t1 values (1),(2),(3),(4),(5); From 97e78d60177b7e06040827902a9a91bc3d104c9d Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 22:48:12 +0400 Subject: [PATCH 29/32] - don't call JOIN::join_free(1) twice for every join in JOIN::cleanup(). The reason it happened was that both, JOIN::cleanup() and JOIN::join_free(), went over all nested joins and called cleanup/join_free for them. For that: - split recursive and non-recursive parts of JOIN::cleanup() and JOIN::join_free() - rename JOIN::cleanup to JOIN::destroy, as it actually destroys its argument - move the recursive part of JOIN::cleanup to st_select_lex::cleanup - move the non-recursive part of JOIN::join_free to the introduced method JOIN::cleanup(). sql/sql_lex.h: Add st_select_lex::cleanup, a counterpart of st_select_lex_unit::cleanup() sql/sql_select.cc: - remove two unused arguments from return_zero_rows - split JOIN::join_free and JOIN::cleanup to recursive and non-recursive parts. - note, the assert in JOIN::join_free _does_ fail in having.test. We have two options: a) propagate `full' flag to the nested joins. We did it before, and this patch didn't change it. If so, we can end up cleaning up an uncacheable JOIN (that is, the join that we might need again). b) evaluate own 'full' flag on every level. In this case, we might end up with tables freed in mysql_unlock_read_tables, but not cleaned up properly, and this may be even worse. The test suite passes with both approaches, but not with the assert. sql/sql_select.h: - declarations for JOIN::cleanup() and JOIN::join_free() sql/sql_union.cc: Add st_select_lex::cleanup, a counterpart of st_select_lex_unit::cleanup(): move the recursive part of JOIN::cleanup to it. --- sql/sql_lex.h | 5 ++ sql/sql_select.cc | 135 ++++++++++++++++++++++++---------------------- sql/sql_select.h | 10 +++- sql/sql_union.cc | 50 +++++++++-------- 4 files changed, 115 insertions(+), 85 deletions(-) diff --git a/sql/sql_lex.h b/sql/sql_lex.h index a9bfb6da926..5cf0b66598f 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -642,6 +642,11 @@ public: static void print_order(String *str, ORDER *order); void print_limit(THD *thd, String *str); void fix_prepare_information(THD *thd, Item **conds); + /* + Destroy the used execution plan (JOIN) of this subtree (this + SELECT_LEX and all nested SELECT_LEXes and SELECT_LEX_UNITs). + */ + bool cleanup(); }; typedef class st_select_lex SELECT_LEX; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 96a25c7919b..61234125d18 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -87,10 +87,9 @@ static void update_depend_map(JOIN *join, ORDER *order); static ORDER *remove_const(JOIN *join,ORDER *first_order,COND *cond, bool change_list, bool *simple_order); static int return_zero_rows(JOIN *join, select_result *res,TABLE_LIST *tables, - List &fields, bool send_row, - uint select_options, const char *info, - Item *having, Procedure *proc, - SELECT_LEX_UNIT *unit); + List &fields, bool send_row, + uint select_options, const char *info, + Item *having); static COND *build_equal_items(THD *thd, COND *cond, COND_EQUAL *inherited, List *join_list, @@ -1227,8 +1226,7 @@ JOIN::exec() send_row_on_empty_set(), select_options, zero_result_cause, - having, procedure, - unit); + having); DBUG_VOID_RETURN; } @@ -1437,7 +1435,7 @@ JOIN::exec() DBUG_VOID_RETURN; } end_read_record(&curr_join->join_tab->read_record); - curr_join->const_tables= curr_join->tables; // Mark free for join_free() + curr_join->const_tables= curr_join->tables; // Mark free for cleanup() curr_join->join_tab[0].table= 0; // Table is freed // No sum funcs anymore @@ -1667,9 +1665,9 @@ JOIN::exec() */ int -JOIN::cleanup() +JOIN::destroy() { - DBUG_ENTER("JOIN::cleanup"); + DBUG_ENTER("JOIN::destroy"); select_lex->join= 0; if (tmp_join) @@ -1684,12 +1682,11 @@ JOIN::cleanup() } tmp_join->tmp_join= 0; tmp_table_param.copy_field=0; - DBUG_RETURN(tmp_join->cleanup()); + DBUG_RETURN(tmp_join->destroy()); } cond_equal= 0; - lock=0; // It's faster to unlock later - join_free(1); + cleanup(1); if (exec_tmp_table1) free_tmp_table(thd, exec_tmp_table1); if (exec_tmp_table2) @@ -1697,12 +1694,6 @@ JOIN::cleanup() delete select; delete_dynamic(&keyuse); delete procedure; - for (SELECT_LEX_UNIT *lex_unit= select_lex->first_inner_unit(); - lex_unit != 0; - lex_unit= lex_unit->next_unit()) - { - error|= lex_unit->cleanup(); - } DBUG_RETURN(error); } @@ -1885,17 +1876,14 @@ Cursor::close() THD *thd= join->thd; DBUG_ENTER("Cursor::close"); - join->join_free(0); + /* + In case of UNIONs JOIN is freed inside of unit->cleanup(), + otherwise in select_lex->cleanup(). + */ if (unit) - { - /* In case of UNIONs JOIN is freed inside unit->cleanup() */ - unit->cleanup(); - } + (void) unit->cleanup(); else - { - join->cleanup(); - delete join; - } + (void) join->select_lex->cleanup(); { /* XXX: Another hack: closing tables used in the cursor */ DBUG_ASSERT(lock || open_tables || derived_tables); @@ -2071,8 +2059,7 @@ err: if (free_join) { thd->proc_info="end"; - err= join->cleanup(); - delete join; + err= select_lex->cleanup(); DBUG_RETURN(err || thd->net.report_error); } DBUG_RETURN(join->error); @@ -5905,29 +5892,75 @@ void JOIN_TAB::cleanup() } +void JOIN::join_free(bool full) +{ + SELECT_LEX_UNIT *unit; + SELECT_LEX *sl; + DBUG_ENTER("JOIN::join_free"); + + /* + Optimization: if not EXPLAIN and we are done with the JOIN, + free all tables. + */ + full= full || (!select_lex->uncacheable && !thd->lex->subqueries && + !thd->lex->describe); + + cleanup(full); + + for (unit= select_lex->first_inner_unit(); unit; unit= unit->next_unit()) + for (sl= unit->first_select_in_union(); sl; sl= sl->next_select()) + { + JOIN *join= sl->join; + if (join) + { + /* Check that we don't occasionally clean up an uncacheable JOIN */ +#if 0 + DBUG_ASSERT(! (!select_lex->uncacheable && sl->uncacheable)); +#endif + join->join_free(full); + } + } + + /* + We are not using tables anymore + Unlock all tables. We may be in an INSERT .... SELECT statement. + */ + if (full && lock && thd->lock && !(select_options & SELECT_NO_UNLOCK) && + !select_lex->subquery_in_having && + (select_lex == (thd->lex->unit.fake_select_lex ? + thd->lex->unit.fake_select_lex : &thd->lex->select_lex))) + { + /* + TODO: unlock tables even if the join isn't top level select in the + tree. + */ + mysql_unlock_read_tables(thd, lock); // Don't free join->lock + lock= 0; + } + + DBUG_VOID_RETURN; +} + + /* Free resources of given join SYNOPSIS - JOIN::join_free() + JOIN::cleanup() fill - true if we should free all resources, call with full==1 should be last, before it this function can be called with full==0 NOTE: with subquery this function definitely will be called several times, but even for simple query it can be called several times. */ -void -JOIN::join_free(bool full) -{ - JOIN_TAB *tab,*end; - DBUG_ENTER("JOIN::join_free"); - full= full || (!select_lex->uncacheable && - !thd->lex->subqueries && - !thd->lex->describe); // do not cleanup too early on EXPLAIN +void JOIN::cleanup(bool full) +{ + DBUG_ENTER("JOIN::cleanup"); if (table) { + JOIN_TAB *tab,*end; /* Only a sorted table may be cached. This sorted table is always the first non const table in join->table @@ -5938,16 +5971,6 @@ JOIN::join_free(bool full) filesort_free_buffers(table[const_tables]); } - for (SELECT_LEX_UNIT *unit= select_lex->first_inner_unit(); unit; - unit= unit->next_unit()) - { - JOIN *join; - for (SELECT_LEX *sl= unit->first_select_in_union(); sl; - sl= sl->next_select()) - if ((join= sl->join)) - join->join_free(full); - } - if (full) { for (tab= join_tab, end= tab+tables; tab != end; tab++) @@ -5964,23 +5987,10 @@ JOIN::join_free(bool full) } } } - /* We are not using tables anymore Unlock all tables. We may be in an INSERT .... SELECT statement. */ - if (full && lock && thd->lock && !(select_options & SELECT_NO_UNLOCK) && - !select_lex->subquery_in_having) - { - // TODO: unlock tables even if the join isn't top level select in the tree - if (select_lex == (thd->lex->unit.fake_select_lex ? - thd->lex->unit.fake_select_lex : &thd->lex->select_lex)) - { - mysql_unlock_read_tables(thd, lock); // Don't free join->lock - lock=0; - } - } - if (full) { group_fields.delete_elements(); @@ -6217,8 +6227,7 @@ remove_const(JOIN *join,ORDER *first_order, COND *cond, static int return_zero_rows(JOIN *join, select_result *result,TABLE_LIST *tables, List &fields, bool send_row, uint select_options, - const char *info, Item *having, Procedure *procedure, - SELECT_LEX_UNIT *unit) + const char *info, Item *having) { DBUG_ENTER("return_zero_rows"); diff --git a/sql/sql_select.h b/sql/sql_select.h index e5266944251..d88fbbfc73f 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -325,7 +325,7 @@ class JOIN :public Sql_alloc int optimize(); int reinit(); void exec(); - int cleanup(); + int destroy(); void restore_tmp(); bool alloc_func_list(); bool make_sum_func_list(List &all_fields, List &send_fields, @@ -349,7 +349,15 @@ class JOIN :public Sql_alloc int rollup_send_data(uint idx); int rollup_write_data(uint idx, TABLE *table); bool test_in_subselect(Item **where); + /* + Release memory and, if possible, the open tables held by this execution + plan (and nested plans). It's used to release some tables before + the end of execution in order to increase concurrency and reduce + memory consumption. + */ void join_free(bool full); + /* Cleanup this JOIN, possibly for reuse */ + void cleanup(bool full); void clear(); bool save_join_tab(); bool send_row_on_empty_set() diff --git a/sql/sql_union.cc b/sql/sql_union.cc index f59d7fffe85..87b67a5127a 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -553,7 +553,6 @@ bool st_select_lex_unit::exec() bool st_select_lex_unit::cleanup() { int error= 0; - JOIN *join; DBUG_ENTER("st_select_lex_unit::cleanup"); if (cleaned) @@ -572,29 +571,17 @@ bool st_select_lex_unit::cleanup() } for (SELECT_LEX *sl= first_select_in_union(); sl; sl= sl->next_select()) + error|= sl->cleanup(); + + if (fake_select_lex) { - if ((join= sl->join)) + JOIN *join; + if ((join= fake_select_lex->join)) { - error|= sl->join->cleanup(); - delete join; + join->tables_list= 0; + join->tables= 0; } - else - { - // it can be DO/SET with subqueries - for (SELECT_LEX_UNIT *lex_unit= sl->first_inner_unit(); - lex_unit != 0; - lex_unit= lex_unit->next_unit()) - { - error|= lex_unit->cleanup(); - } - } - } - if (fake_select_lex && (join= fake_select_lex->join)) - { - join->tables_list= 0; - join->tables= 0; - error|= join->cleanup(); - delete join; + error|= fake_select_lex->cleanup(); } DBUG_RETURN(error); @@ -650,3 +637,24 @@ bool st_select_lex_unit::change_result(select_subselect *result, res= fake_select_lex->join->change_result(result); return (res); } + + +bool st_select_lex::cleanup() +{ + bool error= FALSE; + DBUG_ENTER("st_select_lex::cleanup()"); + + if (join) + { + error|= join->destroy(); + delete join; + join= 0; + } + for (SELECT_LEX_UNIT *lex_unit= first_inner_unit(); lex_unit ; + lex_unit= lex_unit->next_unit()) + { + error|= lex_unit->cleanup(); + } + DBUG_RETURN(error); +} + From 024d232af5baef385f3c2fbe23bbab0ffb55620a Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Jun 2005 23:29:00 +0400 Subject: [PATCH 30/32] Remove an unrelevant assert. sql/sql_select.cc: This assert is not relevant because: - the correct assert is DBUG_ASSERT(! (full && sl->uncacheable)) (prevents freeing of uncacheable JOINs), it breaks view.test - it seems we can free internal JOINs, even if they are uncacheable: if the top level join is evaluated, we're not going to need the internal joins any more --- sql/sql_select.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 6feb495c940..da89fdf1675 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -5912,13 +5912,7 @@ void JOIN::join_free(bool full) { JOIN *join= sl->join; if (join) - { - /* Check that we don't occasionally clean up an uncacheable JOIN */ -#if 0 - DBUG_ASSERT(! (!select_lex->uncacheable && sl->uncacheable)); -#endif join->join_free(full); - } } /* From f6edb3f5c2a3030d0bbf3f22d3e6528fb95596f3 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 25 Jun 2005 00:27:40 +0400 Subject: [PATCH 31/32] Free unused JOINs early even if using subqueries. sql/sql_select.cc: According to the conclusion made in the previous patch, we can widen the range of cases when JOINs are fully freed early, and include subqueries to it. --- sql/sql_select.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index da89fdf1675..b487637ba9c 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -5902,8 +5902,7 @@ void JOIN::join_free(bool full) Optimization: if not EXPLAIN and we are done with the JOIN, free all tables. */ - full= full || (!select_lex->uncacheable && !thd->lex->subqueries && - !thd->lex->describe); + full= full || (!select_lex->uncacheable && !thd->lex->describe); cleanup(full); From 3bdac0a06e0310c2961c7c6f446db2c54a824566 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 27 Jun 2005 09:36:43 +0200 Subject: [PATCH 32/32] Fix for Intel compiler --- extra/yassl/taocrypt/src/integer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extra/yassl/taocrypt/src/integer.cpp b/extra/yassl/taocrypt/src/integer.cpp index 0f06bb4e044..460b2d31426 100644 --- a/extra/yassl/taocrypt/src/integer.cpp +++ b/extra/yassl/taocrypt/src/integer.cpp @@ -35,7 +35,8 @@ #endif -#if defined(_MSC_VER) && defined(_WIN64) // 64 bit X overflow intrinsic +#if defined(_MSC_VER) && defined(_WIN64) && \ + !defined(__INTEL_COMPILER) // 64 bit X overflow intrinsic #ifdef __ia64__ #define myUMULH __UMULH #elif __x86_64__