From 68f60cd695333372ff602b3030f5d6b7399ec375 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Tue, 17 Jun 2025 00:18:38 +0300 Subject: [PATCH 01/68] Add functions including into MTR tests --- .../t/MCOL-5572-autoincrement-filtering.test | 1 + .../columnstore/basic/t/cal_named_udfs.test | 1 + .../basic/t/mcs98_moda_function.test | 1 + .../columnstore/bugfixes/mcol_4465.test | 1 + mysql-test/columnstore/include/functions.inc | 63 +++++++++++++++++++ 5 files changed, 67 insertions(+) create mode 100644 mysql-test/columnstore/include/functions.inc diff --git a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test index 513d41df7..de9d88014 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test +++ b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test @@ -1,4 +1,5 @@ --source ../include/have_columnstore.inc +--source ../include/functions.inc SET default_storage_engine=columnstore; diff --git a/mysql-test/columnstore/basic/t/cal_named_udfs.test b/mysql-test/columnstore/basic/t/cal_named_udfs.test index f127716d2..d097511b8 100644 --- a/mysql-test/columnstore/basic/t/cal_named_udfs.test +++ b/mysql-test/columnstore/basic/t/cal_named_udfs.test @@ -1,4 +1,5 @@ -- source ../include/have_columnstore.inc +--source ../include/functions.inc --disable_warnings DROP DATABASE IF EXISTS cal_test_db; diff --git a/mysql-test/columnstore/basic/t/mcs98_moda_function.test b/mysql-test/columnstore/basic/t/mcs98_moda_function.test index 5d70bf61a..148f355b4 100644 --- a/mysql-test/columnstore/basic/t/mcs98_moda_function.test +++ b/mysql-test/columnstore/basic/t/mcs98_moda_function.test @@ -4,6 +4,7 @@ # # Test MODA with various numeric types -- source ../include/have_columnstore.inc +--source ../include/functions.inc --disable_warnings DROP DATABASE IF EXISTS mcs98_db; --enable_warnings diff --git a/mysql-test/columnstore/bugfixes/mcol_4465.test b/mysql-test/columnstore/bugfixes/mcol_4465.test index 59f078ca1..25503f6d5 100644 --- a/mysql-test/columnstore/bugfixes/mcol_4465.test +++ b/mysql-test/columnstore/bugfixes/mcol_4465.test @@ -1,4 +1,5 @@ --source ../include/have_columnstore.inc +--source ../include/functions.inc --disable_warnings DROP DATABASE IF EXISTS mcol_4465; --enable_warnings diff --git a/mysql-test/columnstore/include/functions.inc b/mysql-test/columnstore/include/functions.inc new file mode 100644 index 000000000..9d9602aa3 --- /dev/null +++ b/mysql-test/columnstore/include/functions.inc @@ -0,0 +1,63 @@ +--disable_query_log +CREATE OR REPLACE FUNCTION mcsgetstats RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calgetstats RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcssettrace RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calsettrace RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcssetparms RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calsetparms RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcsflushcache RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calflushcache RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcsgettrace RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calgettrace RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcsgetversion RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calgetversion RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calonlinealter RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcsviewtablelock RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calviewtablelock RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcscleartablelock RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calcleartablelock RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcslastinsertid RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION callastinsertid RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcsgetsqlcount RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calgetsqlcount RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idbpm RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idbdbroot RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idbsegment RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idbsegmentdir RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idbextentrelativerid RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idbblockid RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idbextentid RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idbextentmin RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idbextentmax RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idbpartition RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION idblocalpm RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcssystemready RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcssystemreadonly RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcssystemprimary RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcs_emindex_size RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION mcs_emindex_free RETURNS INTEGER SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION columnstore_dataload RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE AGGREGATE FUNCTION regr_avgx RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION regr_avgy RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION regr_count RETURNS INTEGER SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION regr_slope RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION regr_intercept RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION regr_r2 RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION corr RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION regr_sxx RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION regr_syy RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION regr_sxy RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION covar_pop RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION covar_samp RETURNS REAL SONAME 'libregr_mysql.so'; +CREATE OR REPLACE AGGREGATE FUNCTION distinct_count RETURNS INTEGER SONAME 'libudf_mysql.so'; +CREATE OR REPLACE FUNCTION caldisablepartitions RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calenablepartitions RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION caldroppartitions RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calshowpartitions RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION caldroppartitionsbyvalue RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION caldisablepartitionsbyvalue RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calenablepartitionsbyvalue RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE FUNCTION calshowpartitionsbyvalue RETURNS STRING SONAME 'ha_columnstore.so'; +CREATE OR REPLACE AGGREGATE FUNCTION moda RETURNS STRING SONAME 'libregr_mysql.so'; +CREATE OR REPLACE FUNCTION mcs_set_ddldebug_level RETURNS STRING SONAME 'ha_columnstore.so'; +--enable_query_log From 38a5a7edec03f6a0fe25b9b317221b73fdbd27ab Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 18 Jun 2025 18:01:51 +0300 Subject: [PATCH 02/68] Fix charset in MTR --- .../basic/r/MCOL-5744-utf8-in-ddl.result | 1 + .../basic/r/mcol641-aggregate.result | 1 + .../basic/r/mcs19_grant_revoke.result | 2 +- .../t/MCOL-5572-autoincrement-filtering.test | 1 + .../basic/t/MCOL-5744-utf8-in-ddl.test | 2 + ...artitioned-tables-in-crossengine-steps.opt | 1 + .../columnstore/basic/t/func_json_quote.test | 2 + mysql-test/columnstore/basic/t/mcol-5005.test | 1 + .../basic/t/mcol641-aggregate.test | 1 + .../columnstore/basic/t/mcol641-insert.opt | 1 + mysql-test/columnstore/basic/t/mcol_2000.test | 1 + .../basic/t/mcs117_create_utf8.test | 1 + .../basic/t/mcs12_alter_table.test | 2 +- .../basic/t/mcs13_alter_table_negative.test | 2 +- .../basic/t/mcs169_bin_functions.test | 1 + .../basic/t/mcs171_null_functions.test | 1 + .../basic/t/mcs173_coalesce_function.test | 1 + .../basic/t/mcs174_case_function.test | 1 + .../basic/t/mcs175_convert_functions.test | 1 + .../basic/t/mcs176_if_function.test | 1 + .../basic/t/mcs178_adddate_function.test | 1 + .../basic/t/mcs179_addtime_function.test | 1 + .../basic/t/mcs180_ascii_function.test | 1 + .../basic/t/mcs181_between_and.test | 1 + .../basic/t/mcs182_char_length_function.test | 1 + .../basic/t/mcs183_date_function.test | 1 + .../basic/t/mcs184_day_function.test | 1 + .../basic/t/mcs185_dayname_function.test | 1 + .../basic/t/mcs186_dayofyear_function.test | 1 + .../basic/t/mcs187_dayofmonth_function.test | 1 + .../basic/t/mcs19_grant_revoke.test | 4 +- .../basic/t/mcs1_create_table_data_types.test | 1 + .../basic/t/mcs229_data_compression_type.test | 1 + .../basic/t/mcs28_load_data_local_infile.test | 1 + .../mcs5_create_table_from_other_tables.test | 1 + .../basic/t/mcs74_check_constraint.test | 1 + .../t/mcs7_create_table_with_metadata.test | 1 + .../t/mcs8_create_table_with_constraints.test | 1 + mysql-test/columnstore/include/charset.inc | 5 ++ .../columnstore/include/syscatalog_mysql.sql | 46 +++++++++++++++++++ 40 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt create mode 100644 mysql-test/columnstore/basic/t/mcol641-insert.opt create mode 100644 mysql-test/columnstore/include/charset.inc create mode 100644 mysql-test/columnstore/include/syscatalog_mysql.sql diff --git a/mysql-test/columnstore/basic/r/MCOL-5744-utf8-in-ddl.result b/mysql-test/columnstore/basic/r/MCOL-5744-utf8-in-ddl.result index dfd4742cd..97a184f2e 100644 --- a/mysql-test/columnstore/basic/r/MCOL-5744-utf8-in-ddl.result +++ b/mysql-test/columnstore/basic/r/MCOL-5744-utf8-in-ddl.result @@ -1,5 +1,6 @@ DROP DATABASE IF EXISTS MCOL5744; CREATE DATABASE MCOL5744; +USE MCOL5744; SET old_mode=''; CREATE TABLE t(x text CHARACTER SET utf8 COLLATE utf8_general_ci) ENGINE=COLUMNSTORE; SHOW CREATE TABLE t; diff --git a/mysql-test/columnstore/basic/r/mcol641-aggregate.result b/mysql-test/columnstore/basic/r/mcol641-aggregate.result index 97f74000b..b16f2123d 100644 --- a/mysql-test/columnstore/basic/r/mcol641-aggregate.result +++ b/mysql-test/columnstore/basic/r/mcol641-aggregate.result @@ -497,3 +497,4 @@ DROP TABLE cs2; DROP TABLE cs3; DROP TABLE cs4; DROP TABLE cs5; +DROP DATABASE IF EXISTS test_mcol641_aggregate; diff --git a/mysql-test/columnstore/basic/r/mcs19_grant_revoke.result b/mysql-test/columnstore/basic/r/mcs19_grant_revoke.result index 6ab1854c4..9a7880b83 100644 --- a/mysql-test/columnstore/basic/r/mcs19_grant_revoke.result +++ b/mysql-test/columnstore/basic/r/mcs19_grant_revoke.result @@ -41,7 +41,7 @@ SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( `col` int(11) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci +) ENGINE=InnoDB INSERT INTO t1 VALUES(1); SELECT * FROM t1; col diff --git a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test index de9d88014..8d331cf40 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test +++ b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test @@ -1,5 +1,6 @@ --source ../include/have_columnstore.inc --source ../include/functions.inc +--source ../include/syscatalog_mysql.sql SET default_storage_engine=columnstore; diff --git a/mysql-test/columnstore/basic/t/MCOL-5744-utf8-in-ddl.test b/mysql-test/columnstore/basic/t/MCOL-5744-utf8-in-ddl.test index 60713bdc3..d7f83fe25 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5744-utf8-in-ddl.test +++ b/mysql-test/columnstore/basic/t/MCOL-5744-utf8-in-ddl.test @@ -4,8 +4,10 @@ --disable_warnings DROP DATABASE IF EXISTS MCOL5744; --enable_warnings +--source ../include/charset.inc CREATE DATABASE MCOL5744; +USE MCOL5744; SET old_mode=''; diff --git a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt new file mode 100644 index 000000000..d9562aa81 --- /dev/null +++ b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt @@ -0,0 +1 @@ +--skip-partition=0 diff --git a/mysql-test/columnstore/basic/t/func_json_quote.test b/mysql-test/columnstore/basic/t/func_json_quote.test index cf08dae8e..8f87df9f4 100644 --- a/mysql-test/columnstore/basic/t/func_json_quote.test +++ b/mysql-test/columnstore/basic/t/func_json_quote.test @@ -1,4 +1,6 @@ --source ../include/have_columnstore.inc +--source ../include/charset.inc + --disable_warnings DROP DATABASE IF EXISTS json_quote_db; diff --git a/mysql-test/columnstore/basic/t/mcol-5005.test b/mysql-test/columnstore/basic/t/mcol-5005.test index 988a4ab16..fde196e9f 100644 --- a/mysql-test/columnstore/basic/t/mcol-5005.test +++ b/mysql-test/columnstore/basic/t/mcol-5005.test @@ -4,6 +4,7 @@ # --source ../include/have_columnstore.inc --source include/have_innodb.inc +--source ../include/charset.inc # # If the calpontsys database does not exist, let's create it. diff --git a/mysql-test/columnstore/basic/t/mcol641-aggregate.test b/mysql-test/columnstore/basic/t/mcol641-aggregate.test index fb0203088..642287152 100644 --- a/mysql-test/columnstore/basic/t/mcol641-aggregate.test +++ b/mysql-test/columnstore/basic/t/mcol641-aggregate.test @@ -221,3 +221,4 @@ DROP TABLE cs2; DROP TABLE cs3; DROP TABLE cs4; DROP TABLE cs5; +DROP DATABASE test_mcol641_aggregate; diff --git a/mysql-test/columnstore/basic/t/mcol641-insert.opt b/mysql-test/columnstore/basic/t/mcol641-insert.opt new file mode 100644 index 000000000..0b781d76f --- /dev/null +++ b/mysql-test/columnstore/basic/t/mcol641-insert.opt @@ -0,0 +1 @@ +--secure-file-priv='' diff --git a/mysql-test/columnstore/basic/t/mcol_2000.test b/mysql-test/columnstore/basic/t/mcol_2000.test index 42e9c5e0e..5dc47ad55 100644 --- a/mysql-test/columnstore/basic/t/mcol_2000.test +++ b/mysql-test/columnstore/basic/t/mcol_2000.test @@ -4,6 +4,7 @@ -- source ../include/have_columnstore.inc -- source include/have_innodb.inc +--source ../include/charset.inc -- disable_warnings drop database if exists test_mcol2000; diff --git a/mysql-test/columnstore/basic/t/mcs117_create_utf8.test b/mysql-test/columnstore/basic/t/mcs117_create_utf8.test index 0c11cab5b..3f7894408 100644 --- a/mysql-test/columnstore/basic/t/mcs117_create_utf8.test +++ b/mysql-test/columnstore/basic/t/mcs117_create_utf8.test @@ -4,6 +4,7 @@ # -- source ../include/have_columnstore.inc -- source ../include/detect_maxscale.inc +--source ../include/charset.inc set names utf8; diff --git a/mysql-test/columnstore/basic/t/mcs12_alter_table.test b/mysql-test/columnstore/basic/t/mcs12_alter_table.test index 69fa2c1ff..175086f43 100644 --- a/mysql-test/columnstore/basic/t/mcs12_alter_table.test +++ b/mysql-test/columnstore/basic/t/mcs12_alter_table.test @@ -4,7 +4,7 @@ -- source ../include/have_columnstore.inc -- source include/have_innodb.inc -- source ../include/check_maxscale.inc - +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs12_db1; diff --git a/mysql-test/columnstore/basic/t/mcs13_alter_table_negative.test b/mysql-test/columnstore/basic/t/mcs13_alter_table_negative.test index 0c4b86415..ade63563a 100644 --- a/mysql-test/columnstore/basic/t/mcs13_alter_table_negative.test +++ b/mysql-test/columnstore/basic/t/mcs13_alter_table_negative.test @@ -3,7 +3,7 @@ # -- source ../include/have_columnstore.inc -- source ../include/check_maxscale.inc - +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs13_db1; diff --git a/mysql-test/columnstore/basic/t/mcs169_bin_functions.test b/mysql-test/columnstore/basic/t/mcs169_bin_functions.test index e8722e85f..7c10c3ace 100644 --- a/mysql-test/columnstore/basic/t/mcs169_bin_functions.test +++ b/mysql-test/columnstore/basic/t/mcs169_bin_functions.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs169_db; diff --git a/mysql-test/columnstore/basic/t/mcs171_null_functions.test b/mysql-test/columnstore/basic/t/mcs171_null_functions.test index 6f814dd36..997678a13 100644 --- a/mysql-test/columnstore/basic/t/mcs171_null_functions.test +++ b/mysql-test/columnstore/basic/t/mcs171_null_functions.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs171_db; diff --git a/mysql-test/columnstore/basic/t/mcs173_coalesce_function.test b/mysql-test/columnstore/basic/t/mcs173_coalesce_function.test index 8a8df2c0e..424d447ea 100644 --- a/mysql-test/columnstore/basic/t/mcs173_coalesce_function.test +++ b/mysql-test/columnstore/basic/t/mcs173_coalesce_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs173_db; diff --git a/mysql-test/columnstore/basic/t/mcs174_case_function.test b/mysql-test/columnstore/basic/t/mcs174_case_function.test index 72afc4e7c..774461e89 100644 --- a/mysql-test/columnstore/basic/t/mcs174_case_function.test +++ b/mysql-test/columnstore/basic/t/mcs174_case_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs174_db; diff --git a/mysql-test/columnstore/basic/t/mcs175_convert_functions.test b/mysql-test/columnstore/basic/t/mcs175_convert_functions.test index f6e7c34c5..8d2830532 100644 --- a/mysql-test/columnstore/basic/t/mcs175_convert_functions.test +++ b/mysql-test/columnstore/basic/t/mcs175_convert_functions.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs175_db; diff --git a/mysql-test/columnstore/basic/t/mcs176_if_function.test b/mysql-test/columnstore/basic/t/mcs176_if_function.test index 6add1bffe..1889804dc 100644 --- a/mysql-test/columnstore/basic/t/mcs176_if_function.test +++ b/mysql-test/columnstore/basic/t/mcs176_if_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs176_db; diff --git a/mysql-test/columnstore/basic/t/mcs178_adddate_function.test b/mysql-test/columnstore/basic/t/mcs178_adddate_function.test index 71d278b50..d788f9bbb 100644 --- a/mysql-test/columnstore/basic/t/mcs178_adddate_function.test +++ b/mysql-test/columnstore/basic/t/mcs178_adddate_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs178_db; diff --git a/mysql-test/columnstore/basic/t/mcs179_addtime_function.test b/mysql-test/columnstore/basic/t/mcs179_addtime_function.test index 9fa1372b7..4081c3d98 100644 --- a/mysql-test/columnstore/basic/t/mcs179_addtime_function.test +++ b/mysql-test/columnstore/basic/t/mcs179_addtime_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs179_db; diff --git a/mysql-test/columnstore/basic/t/mcs180_ascii_function.test b/mysql-test/columnstore/basic/t/mcs180_ascii_function.test index b39ad622c..2904cd22e 100644 --- a/mysql-test/columnstore/basic/t/mcs180_ascii_function.test +++ b/mysql-test/columnstore/basic/t/mcs180_ascii_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs180_db; diff --git a/mysql-test/columnstore/basic/t/mcs181_between_and.test b/mysql-test/columnstore/basic/t/mcs181_between_and.test index a11282789..6a949a261 100644 --- a/mysql-test/columnstore/basic/t/mcs181_between_and.test +++ b/mysql-test/columnstore/basic/t/mcs181_between_and.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs181_db; diff --git a/mysql-test/columnstore/basic/t/mcs182_char_length_function.test b/mysql-test/columnstore/basic/t/mcs182_char_length_function.test index b46523fe8..a1de4382e 100644 --- a/mysql-test/columnstore/basic/t/mcs182_char_length_function.test +++ b/mysql-test/columnstore/basic/t/mcs182_char_length_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs182_db; diff --git a/mysql-test/columnstore/basic/t/mcs183_date_function.test b/mysql-test/columnstore/basic/t/mcs183_date_function.test index 322ffdc29..1a937f4c2 100644 --- a/mysql-test/columnstore/basic/t/mcs183_date_function.test +++ b/mysql-test/columnstore/basic/t/mcs183_date_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs183_db; diff --git a/mysql-test/columnstore/basic/t/mcs184_day_function.test b/mysql-test/columnstore/basic/t/mcs184_day_function.test index 74fcd3837..d23b3d4ea 100644 --- a/mysql-test/columnstore/basic/t/mcs184_day_function.test +++ b/mysql-test/columnstore/basic/t/mcs184_day_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs184_db; diff --git a/mysql-test/columnstore/basic/t/mcs185_dayname_function.test b/mysql-test/columnstore/basic/t/mcs185_dayname_function.test index 65f8bcbbb..4c0a61314 100644 --- a/mysql-test/columnstore/basic/t/mcs185_dayname_function.test +++ b/mysql-test/columnstore/basic/t/mcs185_dayname_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs185_db; diff --git a/mysql-test/columnstore/basic/t/mcs186_dayofyear_function.test b/mysql-test/columnstore/basic/t/mcs186_dayofyear_function.test index 9e0b2c274..afa475c38 100644 --- a/mysql-test/columnstore/basic/t/mcs186_dayofyear_function.test +++ b/mysql-test/columnstore/basic/t/mcs186_dayofyear_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs186_db; diff --git a/mysql-test/columnstore/basic/t/mcs187_dayofmonth_function.test b/mysql-test/columnstore/basic/t/mcs187_dayofmonth_function.test index bcd51b27a..4686b7480 100644 --- a/mysql-test/columnstore/basic/t/mcs187_dayofmonth_function.test +++ b/mysql-test/columnstore/basic/t/mcs187_dayofmonth_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs187_db; diff --git a/mysql-test/columnstore/basic/t/mcs19_grant_revoke.test b/mysql-test/columnstore/basic/t/mcs19_grant_revoke.test index f514756ef..4ae71180d 100644 --- a/mysql-test/columnstore/basic/t/mcs19_grant_revoke.test +++ b/mysql-test/columnstore/basic/t/mcs19_grant_revoke.test @@ -5,7 +5,7 @@ -- source ../include/have_columnstore.inc -- source include/have_innodb.inc -- source ../include/check_maxscale.inc - +--source ../include/charset.inc SET default_storage_engine=InnoDB; @@ -50,7 +50,7 @@ USE mcs19_db; --error ER_DBACCESS_DENIED_ERROR CREATE DATABASE mcs19_db1; CREATE TABLE t1(col INT) ENGINE=InnoDB; ---replace_regex /( COLLATE=latin1_swedish_ci)// +--replace_regex /(\ DEFAULT CHARSET(.*))// SHOW CREATE TABLE t1; INSERT INTO t1 VALUES(1); SELECT * FROM t1; diff --git a/mysql-test/columnstore/basic/t/mcs1_create_table_data_types.test b/mysql-test/columnstore/basic/t/mcs1_create_table_data_types.test index e0e630333..bacda2ef9 100644 --- a/mysql-test/columnstore/basic/t/mcs1_create_table_data_types.test +++ b/mysql-test/columnstore/basic/t/mcs1_create_table_data_types.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs1_db; diff --git a/mysql-test/columnstore/basic/t/mcs229_data_compression_type.test b/mysql-test/columnstore/basic/t/mcs229_data_compression_type.test index 3737eaa68..8e4e60ff3 100644 --- a/mysql-test/columnstore/basic/t/mcs229_data_compression_type.test +++ b/mysql-test/columnstore/basic/t/mcs229_data_compression_type.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc # # If the calpontsys database does not exist, let's create it. diff --git a/mysql-test/columnstore/basic/t/mcs28_load_data_local_infile.test b/mysql-test/columnstore/basic/t/mcs28_load_data_local_infile.test index af2e11e78..6ab41785a 100755 --- a/mysql-test/columnstore/basic/t/mcs28_load_data_local_infile.test +++ b/mysql-test/columnstore/basic/t/mcs28_load_data_local_infile.test @@ -4,6 +4,7 @@ # -- source ../include/have_columnstore.inc --source ../include/detect_maxscale.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs28_db1; diff --git a/mysql-test/columnstore/basic/t/mcs5_create_table_from_other_tables.test b/mysql-test/columnstore/basic/t/mcs5_create_table_from_other_tables.test index c08c3aed3..acf966a28 100644 --- a/mysql-test/columnstore/basic/t/mcs5_create_table_from_other_tables.test +++ b/mysql-test/columnstore/basic/t/mcs5_create_table_from_other_tables.test @@ -3,6 +3,7 @@ # -- source ../include/have_columnstore.inc -- source include/have_innodb.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs5_db; diff --git a/mysql-test/columnstore/basic/t/mcs74_check_constraint.test b/mysql-test/columnstore/basic/t/mcs74_check_constraint.test index f1620cdc2..e25e9cf5c 100644 --- a/mysql-test/columnstore/basic/t/mcs74_check_constraint.test +++ b/mysql-test/columnstore/basic/t/mcs74_check_constraint.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs74_db; diff --git a/mysql-test/columnstore/basic/t/mcs7_create_table_with_metadata.test b/mysql-test/columnstore/basic/t/mcs7_create_table_with_metadata.test index dabd48eb0..3e511991a 100644 --- a/mysql-test/columnstore/basic/t/mcs7_create_table_with_metadata.test +++ b/mysql-test/columnstore/basic/t/mcs7_create_table_with_metadata.test @@ -2,6 +2,7 @@ #Test CREATE TABLE with metadata # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs7_db; diff --git a/mysql-test/columnstore/basic/t/mcs8_create_table_with_constraints.test b/mysql-test/columnstore/basic/t/mcs8_create_table_with_constraints.test index d5ef1fcf1..4144423d0 100644 --- a/mysql-test/columnstore/basic/t/mcs8_create_table_with_constraints.test +++ b/mysql-test/columnstore/basic/t/mcs8_create_table_with_constraints.test @@ -2,6 +2,7 @@ #Test CREATE TABLE with all supported constraints in Columnstore # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs8_db; diff --git a/mysql-test/columnstore/include/charset.inc b/mysql-test/columnstore/include/charset.inc new file mode 100644 index 000000000..15c273e8e --- /dev/null +++ b/mysql-test/columnstore/include/charset.inc @@ -0,0 +1,5 @@ +--disable_query_log +SET character_set_server = 'utf8mb3'; +SET collation_server = 'utf8mb3_general_ci'; +--enable_query_log + diff --git a/mysql-test/columnstore/include/syscatalog_mysql.sql b/mysql-test/columnstore/include/syscatalog_mysql.sql new file mode 100644 index 000000000..551efa69f --- /dev/null +++ b/mysql-test/columnstore/include/syscatalog_mysql.sql @@ -0,0 +1,46 @@ +--disable_query_log +create database if not exists calpontsys; + +use calpontsys; + +drop table if exists systable restrict; +drop table if exists syscolumn restrict; + +create table if not exists systable (tablename varchar(128), + `schema` varchar(128), + objectid int, + createdate date, + lastupdate date, + init int, + next int, + numofrows int, + avgrowlen int, + numofblocks int, + autoincrement int, + auxcolumnoid int not null default 0) engine=columnstore comment='SCHEMA SYNC ONLY'; + +create table if not exists syscolumn (`schema` varchar(128), + tablename varchar(128), + columnname varchar(128), + objectid integer, + dictobjectid integer, + listobjectid integer, + treeobjectid integer, + datatype integer, + columnlength integer, + columnposition integer, + lastupdate date, + defaultvalue varchar(64), + nullable integer, + scale integer, + prec integer, + autoincrement char(1), + distcount integer, + nullcount integer, + minvalue varchar(64), + `maxvalue` varchar(64), + compressiontype integer, + nextvalue bigint, + charsetnum int not null default 0) engine=columnstore comment='SCHEMA SYNC ONLY'; + +--enable_query_log From 2d400d919d5b36a2e38382b16abb7facf8d8df64 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Thu, 19 Jun 2025 15:02:46 +0300 Subject: [PATCH 03/68] add have_inndb to MTR --- mysql-test/columnstore/basic/r/mcol641-aggregate.result | 2 +- ...COL-5886-use-of-partitioned-tables-in-crossengine-steps.test | 2 ++ mysql-test/columnstore/basic/t/mcs22_insert_ignore.test | 1 + mysql-test/columnstore/basic/t/mcs_named_udfs.test | 2 ++ mysql-test/columnstore/bugfixes/mcol_4622.test | 1 + 5 files changed, 7 insertions(+), 1 deletion(-) diff --git a/mysql-test/columnstore/basic/r/mcol641-aggregate.result b/mysql-test/columnstore/basic/r/mcol641-aggregate.result index b16f2123d..bf2ec4f80 100644 --- a/mysql-test/columnstore/basic/r/mcol641-aggregate.result +++ b/mysql-test/columnstore/basic/r/mcol641-aggregate.result @@ -497,4 +497,4 @@ DROP TABLE cs2; DROP TABLE cs3; DROP TABLE cs4; DROP TABLE cs5; -DROP DATABASE IF EXISTS test_mcol641_aggregate; +DROP DATABASE test_mcol641_aggregate; diff --git a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test index 64ff80efa..ea4726ca9 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test +++ b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test @@ -1,3 +1,5 @@ +-- source ../include/have_columnstore.inc +--source include/have_innodb.inc --disable_warnings DROP DATABASE IF EXISTS MCOL5886; --enable_warnings diff --git a/mysql-test/columnstore/basic/t/mcs22_insert_ignore.test b/mysql-test/columnstore/basic/t/mcs22_insert_ignore.test index b0120cf5e..8e15e1b6f 100644 --- a/mysql-test/columnstore/basic/t/mcs22_insert_ignore.test +++ b/mysql-test/columnstore/basic/t/mcs22_insert_ignore.test @@ -3,6 +3,7 @@ # Author: Susil, susil.behera@mariadb.com # -- source ../include/have_columnstore.inc +--source include/have_innodb.inc --disable_warnings DROP DATABASE IF EXISTS mcs22_db; diff --git a/mysql-test/columnstore/basic/t/mcs_named_udfs.test b/mysql-test/columnstore/basic/t/mcs_named_udfs.test index d8ab21388..84859b2c6 100644 --- a/mysql-test/columnstore/basic/t/mcs_named_udfs.test +++ b/mysql-test/columnstore/basic/t/mcs_named_udfs.test @@ -1,4 +1,6 @@ -- source ../include/have_columnstore.inc +--source ../include/functions.inc + --disable_warnings DROP DATABASE IF EXISTS cal_test_db; diff --git a/mysql-test/columnstore/bugfixes/mcol_4622.test b/mysql-test/columnstore/bugfixes/mcol_4622.test index 862f43239..b4b164fad 100644 --- a/mysql-test/columnstore/bugfixes/mcol_4622.test +++ b/mysql-test/columnstore/bugfixes/mcol_4622.test @@ -1,4 +1,5 @@ -- source ../include/have_columnstore.inc +--source include/have_innodb.inc --disable_warnings DROP DATABASE IF EXISTS mcol_4622; --enable_warnings From e9c6c6b5d34163a001b054a9d0dd7d01b996c826 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Thu, 19 Jun 2025 18:37:54 +0300 Subject: [PATCH 04/68] enable sequence engine in the mcol-4786 test --- mysql-test/columnstore/basic/r/analyze_table.result | 6 ++++++ ...886-use-of-partitioned-tables-in-crossengine-steps.opt | 2 ++ mysql-test/columnstore/basic/t/analyze_table.test | 8 ++++---- mysql-test/columnstore/bugfixes/mcol-4786.opt | 1 + 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 mysql-test/columnstore/bugfixes/mcol-4786.opt diff --git a/mysql-test/columnstore/basic/r/analyze_table.result b/mysql-test/columnstore/basic/r/analyze_table.result index 65fd380e0..0086a049d 100644 --- a/mysql-test/columnstore/basic/r/analyze_table.result +++ b/mysql-test/columnstore/basic/r/analyze_table.result @@ -6,20 +6,26 @@ insert into t1 values (1, 2, 3), (2, 2, 2), (2, 3, 4); analyze table t1; Table Op Msg_type Msg_text analyze_table_db.t1 analyze status OK +analyze_table_db.t1 analyze status Engine-independent statistics collected create table t2 (a int, b double) engine=columnstore; insert into t2 values (2, 3), (3, 4); analyze table t2; Table Op Msg_type Msg_text analyze_table_db.t2 analyze status OK +analyze_table_db.t2 analyze status Engine-independent statistics collected create table t3 (a varchar(25)) engine=columnstore; insert into t3 values ("a"), ("b"); analyze table t3; Table Op Msg_type Msg_text +analyze_table_db.t3 analyze status Engine-independent statistics collected analyze_table_db.t3 analyze status OK analyze table t1, t2, t3; Table Op Msg_type Msg_text +analyze_table_db.t1 analyze status Engine-independent statistics collected analyze_table_db.t1 analyze status OK +analyze_table_db.t2 analyze status Engine-independent statistics collected analyze_table_db.t2 analyze status OK +analyze_table_db.t3 analyze status Engine-independent statistics collected analyze_table_db.t3 analyze status OK DROP TABLE t1; DROP TABLE t2; diff --git a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt index d9562aa81..02b69515b 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt +++ b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt @@ -1 +1,3 @@ --skip-partition=0 +--skip-sequence=0 + diff --git a/mysql-test/columnstore/basic/t/analyze_table.test b/mysql-test/columnstore/basic/t/analyze_table.test index 217b24920..a65ec3711 100644 --- a/mysql-test/columnstore/basic/t/analyze_table.test +++ b/mysql-test/columnstore/basic/t/analyze_table.test @@ -9,18 +9,18 @@ USE analyze_table_db; create table t1 (a int, b int, c int) engine=columnstore; insert into t1 values (1, 2, 3), (2, 2, 2), (2, 3, 4); -analyze table t1; +analyze table t1 PERSISTENT FOR ALL; create table t2 (a int, b double) engine=columnstore; insert into t2 values (2, 3), (3, 4); -analyze table t2; +analyze table t2 PERSISTENT FOR ALL; create table t3 (a varchar(25)) engine=columnstore; insert into t3 values ("a"), ("b"); -analyze table t3; +analyze table t3 PERSISTENT FOR ALL; -analyze table t1, t2, t3; +analyze table t1, t2, t3 PERSISTENT FOR ALL; DROP TABLE t1; DROP TABLE t2; diff --git a/mysql-test/columnstore/bugfixes/mcol-4786.opt b/mysql-test/columnstore/bugfixes/mcol-4786.opt new file mode 100644 index 000000000..bedbaf7de --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-4786.opt @@ -0,0 +1 @@ +--skip-sequence=0 From 5831f578fe616a1efc8d438eb331fb6ce058b0f1 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 20 Jun 2025 01:08:57 +0300 Subject: [PATCH 05/68] set charset in the mcs271_substring_index_function MTR test --- .../columnstore/basic/t/mcs271_substring_index_function.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/columnstore/basic/t/mcs271_substring_index_function.test b/mysql-test/columnstore/basic/t/mcs271_substring_index_function.test index 9ae4e0409..478b73def 100644 --- a/mysql-test/columnstore/basic/t/mcs271_substring_index_function.test +++ b/mysql-test/columnstore/basic/t/mcs271_substring_index_function.test @@ -3,6 +3,7 @@ # Author: Bharath, bharath.bokka@mariadb.com # -- source ../include/have_columnstore.inc +--source ../include/charset.inc --disable_warnings DROP DATABASE IF EXISTS mcs271_db; From 9d4173e3a879e149a5924304c4a6d902658d3c4e Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 20 Jun 2025 01:31:10 +0300 Subject: [PATCH 06/68] enable engnes for the test MCOL-5776-GROUP-BY-HAVING-functions-use --- .../bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.opt | 3 +++ .../bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.opt diff --git a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.opt b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.opt new file mode 100644 index 000000000..02b69515b --- /dev/null +++ b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.opt @@ -0,0 +1,3 @@ +--skip-partition=0 +--skip-sequence=0 + diff --git a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test index 129bfcbc2..7efed5121 100644 --- a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test +++ b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test @@ -1,3 +1,6 @@ +--source ../include/have_columnstore.inc +--source include/have_innodb.inc + --disable_warnings DROP DATABASE IF EXISTS MCOL5776; --enable_warnings From c0307b66367870367f637911ac8432348031ad07 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 20 Jun 2025 14:14:28 +0300 Subject: [PATCH 07/68] Fix analize_table test --- mysql-test/columnstore/basic/r/analyze_table.result | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mysql-test/columnstore/basic/r/analyze_table.result b/mysql-test/columnstore/basic/r/analyze_table.result index 0086a049d..08d8031e0 100644 --- a/mysql-test/columnstore/basic/r/analyze_table.result +++ b/mysql-test/columnstore/basic/r/analyze_table.result @@ -3,23 +3,23 @@ CREATE DATABASE analyze_table_db; USE analyze_table_db; create table t1 (a int, b int, c int) engine=columnstore; insert into t1 values (1, 2, 3), (2, 2, 2), (2, 3, 4); -analyze table t1; +analyze table t1 PERSISTENT FOR ALL; Table Op Msg_type Msg_text -analyze_table_db.t1 analyze status OK analyze_table_db.t1 analyze status Engine-independent statistics collected +analyze_table_db.t1 analyze status OK create table t2 (a int, b double) engine=columnstore; insert into t2 values (2, 3), (3, 4); -analyze table t2; +analyze table t2 PERSISTENT FOR ALL; Table Op Msg_type Msg_text -analyze_table_db.t2 analyze status OK analyze_table_db.t2 analyze status Engine-independent statistics collected +analyze_table_db.t2 analyze status OK create table t3 (a varchar(25)) engine=columnstore; insert into t3 values ("a"), ("b"); -analyze table t3; +analyze table t3 PERSISTENT FOR ALL; Table Op Msg_type Msg_text analyze_table_db.t3 analyze status Engine-independent statistics collected analyze_table_db.t3 analyze status OK -analyze table t1, t2, t3; +analyze table t1, t2, t3 PERSISTENT FOR ALL; Table Op Msg_type Msg_text analyze_table_db.t1 analyze status Engine-independent statistics collected analyze_table_db.t1 analyze status OK From a61325f780654150dea0a38639c3f417140c5b02 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 20 Jun 2025 18:00:35 +0300 Subject: [PATCH 08/68] dusable warings in the includes for MTR --- mysql-test/columnstore/include/syscatalog_mysql.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mysql-test/columnstore/include/syscatalog_mysql.sql b/mysql-test/columnstore/include/syscatalog_mysql.sql index 551efa69f..ef2b22039 100644 --- a/mysql-test/columnstore/include/syscatalog_mysql.sql +++ b/mysql-test/columnstore/include/syscatalog_mysql.sql @@ -1,4 +1,5 @@ --disable_query_log +--disable_warnings create database if not exists calpontsys; use calpontsys; @@ -43,4 +44,6 @@ create table if not exists syscolumn (`schema` varchar(128), nextvalue bigint, charsetnum int not null default 0) engine=columnstore comment='SCHEMA SYNC ONLY'; +--enable_warnings --enable_query_log + From 7f7552e896432d2b72cbe9cc76f4198423b604e3 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 20 Jun 2025 18:18:46 +0300 Subject: [PATCH 09/68] add functions loading to the MCOL-5776-GROUP-BY-HAVING-functions test --- .../bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test index 7efed5121..99c91b68d 100644 --- a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test +++ b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test @@ -1,5 +1,6 @@ --source ../include/have_columnstore.inc --source include/have_innodb.inc +--source ../include/functions.inc --disable_warnings DROP DATABASE IF EXISTS MCOL5776; From 3d64026d0a5c8ed89d167e7cb1e051f06a8c6574 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Sun, 22 Jun 2025 00:47:44 +0300 Subject: [PATCH 10/68] add cleanups to MTR --- .../MCOL-5572-autoincrement-filtering.result | 1 + .../basic/r/mcs13_alter_table_negative.result | 1 + .../t/MCOL-5572-autoincrement-filtering.test | 3 +- .../columnstore/basic/t/cal_named_udfs.test | 1 + .../basic/t/mcs13_alter_table_negative.test | 1 + .../basic/t/mcs98_moda_function.test | 2 +- .../columnstore/basic/t/mcs_named_udfs.test | 1 + ...OL-5776-GROUP-BY-HAVING-functions-use.test | 1 + ...ol-4741-strings-ranges-are-unsigned.result | 1 + ...mcol-4741-strings-ranges-are-unsigned.test | 2 +- .../columnstore/bugfixes/mcol-5074.result | 1 + .../columnstore/bugfixes/mcol-5074.test | 1 + .../columnstore/bugfixes/mcol-5669.result | 1 + .../columnstore/bugfixes/mcol-5669.test | 1 + .../columnstore/bugfixes/mcol_4465.test | 1 + .../columnstore/include/drop_functions.inc | 63 +++++++++++++++++++ 16 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 mysql-test/columnstore/include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result b/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result index f638c9e48..63263c955 100644 --- a/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result +++ b/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result @@ -18,3 +18,4 @@ select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoi tablename autoincrement nextvalue foo n 1 DROP DATABASE mcol5572; +DROP DATABASE calpontsys; diff --git a/mysql-test/columnstore/basic/r/mcs13_alter_table_negative.result b/mysql-test/columnstore/basic/r/mcs13_alter_table_negative.result index 288952088..2ce6f1013 100644 --- a/mysql-test/columnstore/basic/r/mcs13_alter_table_negative.result +++ b/mysql-test/columnstore/basic/r/mcs13_alter_table_negative.result @@ -58,3 +58,4 @@ a c1 disconnect addconroot1; disconnect addconroot2; DROP DATABASE mcs13_db1; +DROP DATABASE mcs13_db2; diff --git a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test index 8d331cf40..bee2d5a99 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test +++ b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test @@ -21,4 +21,5 @@ select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoi select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoincrement = 'n' and `schema`='mcol5572' and tablename='foo'; DROP DATABASE mcol5572; - +--source ../include/drop_functions.inc +DROP DATABASE calpontsys; diff --git a/mysql-test/columnstore/basic/t/cal_named_udfs.test b/mysql-test/columnstore/basic/t/cal_named_udfs.test index d097511b8..bd03af5bc 100644 --- a/mysql-test/columnstore/basic/t/cal_named_udfs.test +++ b/mysql-test/columnstore/basic/t/cal_named_udfs.test @@ -24,3 +24,4 @@ select calgetsqlcount(); DROP TABLE t1; DROP DATABASE cal_test_db; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/t/mcs13_alter_table_negative.test b/mysql-test/columnstore/basic/t/mcs13_alter_table_negative.test index ade63563a..52d08f77c 100644 --- a/mysql-test/columnstore/basic/t/mcs13_alter_table_negative.test +++ b/mysql-test/columnstore/basic/t/mcs13_alter_table_negative.test @@ -83,6 +83,7 @@ disconnect addconroot2; --disable_warnings DROP DATABASE mcs13_db1; +DROP DATABASE mcs13_db2; --enable_warnings # diff --git a/mysql-test/columnstore/basic/t/mcs98_moda_function.test b/mysql-test/columnstore/basic/t/mcs98_moda_function.test index 148f355b4..9b02a79a2 100644 --- a/mysql-test/columnstore/basic/t/mcs98_moda_function.test +++ b/mysql-test/columnstore/basic/t/mcs98_moda_function.test @@ -101,4 +101,4 @@ SELECT moda(floor(rl)) FROM t2; SELECT t, moda(tu) 'q1' FROM t2 GROUP BY t HAVING moda(tu) > 5; # Clean UP DROP DATABASE mcs98_db; - +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/t/mcs_named_udfs.test b/mysql-test/columnstore/basic/t/mcs_named_udfs.test index 84859b2c6..00ca03aa5 100644 --- a/mysql-test/columnstore/basic/t/mcs_named_udfs.test +++ b/mysql-test/columnstore/basic/t/mcs_named_udfs.test @@ -25,3 +25,4 @@ select mcslastinsertid("t1"); DROP TABLE t1; DROP DATABASE cal_test_db; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test index 99c91b68d..0edfd7823 100644 --- a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test +++ b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test @@ -103,3 +103,4 @@ INSERT INTO t(k) VALUES (1), (2), (2), (3), (3), (4), (4),(4),(4),(4),(995), (NU SELECT k + k a FROM t GROUP BY a HAVING a >= 8; DROP DATABASE MCOL5776; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result index c9e2db639..b3dcfdb62 100644 --- a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result +++ b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result @@ -10,3 +10,4 @@ LOAD DATA LOCAL infile 'MTR_SUITE_DIR/../std_data/mcol-4741-part-2-empty-strings SELECT COUNT(*) FROM t WHERE x='val'; COUNT(*) 2 +DROP DATABASE db4741; diff --git a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test index 31311d992..a3d2222a9 100644 --- a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test +++ b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test @@ -16,4 +16,4 @@ SELECT COUNT(*) FROM t WHERE x='val'; --eval LOAD DATA LOCAL infile '$MTR_SUITE_DIR/../std_data/mcol-4741-part-2-empty-strings.txt' INTO TABLE t SELECT COUNT(*) FROM t WHERE x='val'; - +DROP DATABASE db4741; diff --git a/mysql-test/columnstore/bugfixes/mcol-5074.result b/mysql-test/columnstore/bugfixes/mcol-5074.result index 772bf7a21..87c51361d 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5074.result +++ b/mysql-test/columnstore/bugfixes/mcol-5074.result @@ -32,3 +32,4 @@ Partner 5 2021-12-11 2 NULL NULL NULL Person Focal Group 4 2021-12-11 NULL NULL NULL 4 PFG 4 2021-12-11 NULL NULL NULL NULL Retail 4 2021-12-12 NULL 1 NULL NULL +DROP DATABASE mcol_5074_db; diff --git a/mysql-test/columnstore/bugfixes/mcol-5074.test b/mysql-test/columnstore/bugfixes/mcol-5074.test index aed1ff750..949210734 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5074.test +++ b/mysql-test/columnstore/bugfixes/mcol-5074.test @@ -40,4 +40,5 @@ select case when da.mtn in ('PFG','Person Focal Group') then count( acct_id) end + case when da.mtn = 'Person Focal Group' then count( acct_id) end as PFG_active_accounts from accnt da group by mtn, accts, act_created_dt order by mtn; + DROP DATABASE mcol_5074_db; diff --git a/mysql-test/columnstore/bugfixes/mcol-5669.result b/mysql-test/columnstore/bugfixes/mcol-5669.result index 661bd51c2..bb99111c4 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5669.result +++ b/mysql-test/columnstore/bugfixes/mcol-5669.result @@ -111,3 +111,4 @@ di_source_id brand_sku_id adjusted_dtm is_sit sit_uom sales_uom_daily_average 5304 1004 2024-01-01 1 0.2434 0.00000000 5389 1004 2024-01-01 1 0.2074 0.00000000 DROP DATABASE mcol_5669; +REVOKE ALL PRIVILEGES ON *.* FROM 'cejuser'@'localhost'; diff --git a/mysql-test/columnstore/bugfixes/mcol-5669.test b/mysql-test/columnstore/bugfixes/mcol-5669.test index 70f1b21d5..1506e0a34 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5669.test +++ b/mysql-test/columnstore/bugfixes/mcol-5669.test @@ -110,3 +110,4 @@ INSERT INTO `zzz_999999` VALUES ('2023-12-18',5298,684963,1004,1,1,2166,1.000),( ) as z; DROP DATABASE mcol_5669; +REVOKE ALL PRIVILEGES ON *.* FROM 'cejuser'@'localhost'; diff --git a/mysql-test/columnstore/bugfixes/mcol_4465.test b/mysql-test/columnstore/bugfixes/mcol_4465.test index 25503f6d5..6339eced9 100644 --- a/mysql-test/columnstore/bugfixes/mcol_4465.test +++ b/mysql-test/columnstore/bugfixes/mcol_4465.test @@ -10,3 +10,4 @@ INSERT INTO cs1 VALUES('i',0),('i',0),('ii',0); SELECT c,regr_count(d,length(c)) f FROM cs1 GROUP BY 1 ORDER BY 1; SELECT * FROM (SELECT c,regr_count(d,length(c)) f FROM cs1 GROUP BY 1 ORDER BY 1)a; DROP DATABASE mcol_4465; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/include/drop_functions.inc b/mysql-test/columnstore/include/drop_functions.inc new file mode 100644 index 000000000..a55af261c --- /dev/null +++ b/mysql-test/columnstore/include/drop_functions.inc @@ -0,0 +1,63 @@ +--disable_query_log +DROP FUNCTION mcsgetstats; +DROP FUNCTION calgetstats; +DROP FUNCTION mcssettrace; +DROP FUNCTION calsettrace; +DROP FUNCTION mcssetparms; +DROP FUNCTION calsetparms; +DROP FUNCTION mcsflushcache; +DROP FUNCTION calflushcache; +DROP FUNCTION mcsgettrace; +DROP FUNCTION calgettrace; +DROP FUNCTION mcsgetversion; +DROP FUNCTION calgetversion; +DROP FUNCTION calonlinealter; +DROP FUNCTION mcsviewtablelock; +DROP FUNCTION calviewtablelock; +DROP FUNCTION mcscleartablelock; +DROP FUNCTION calcleartablelock; +DROP FUNCTION mcslastinsertid; +DROP FUNCTION callastinsertid; +DROP FUNCTION mcsgetsqlcount; +DROP FUNCTION calgetsqlcount; +DROP FUNCTION idbpm; +DROP FUNCTION idbdbroot; +DROP FUNCTION idbsegment; +DROP FUNCTION idbsegmentdir; +DROP FUNCTION idbextentrelativerid; +DROP FUNCTION idbblockid; +DROP FUNCTION idbextentid; +DROP FUNCTION idbextentmin; +DROP FUNCTION idbextentmax; +DROP FUNCTION idbpartition; +DROP FUNCTION idblocalpm; +DROP FUNCTION mcssystemready; +DROP FUNCTION mcssystemreadonly; +DROP FUNCTION mcssystemprimary; +DROP FUNCTION mcs_emindex_size; +DROP FUNCTION mcs_emindex_free; +DROP FUNCTION columnstore_dataload; +DROP FUNCTION regr_avgx; +DROP FUNCTION regr_avgy; +DROP FUNCTION regr_count; +DROP FUNCTION regr_slope; +DROP FUNCTION regr_intercept; +DROP FUNCTION regr_r2; +DROP FUNCTION corr; +DROP FUNCTION regr_sxx; +DROP FUNCTION regr_syy; +DROP FUNCTION regr_sxy; +DROP FUNCTION covar_pop; +DROP FUNCTION covar_samp; +DROP FUNCTION distinct_count; +DROP FUNCTION caldisablepartitions; +DROP FUNCTION calenablepartitions; +DROP FUNCTION caldroppartitions; +DROP FUNCTION calshowpartitions; +DROP FUNCTION caldroppartitionsbyvalue; +DROP FUNCTION caldisablepartitionsbyvalue; +DROP FUNCTION calenablepartitionsbyvalue; +DROP FUNCTION calshowpartitionsbyvalue; +DROP FUNCTION moda; +DROP FUNCTION mcs_set_ddldebug_level; +--enable_query_log From bdf0a956ffdda9be8ef37e454d6d1b5d821d14fd Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 23 Jun 2025 15:06:25 +0300 Subject: [PATCH 11/68] Fix MCOL-5572-autoincrement-filtering test --- .../columnstore/basic/r/MCOL-5572-autoincrement-filtering.result | 1 - .../columnstore/basic/t/MCOL-5572-autoincrement-filtering.test | 1 - 2 files changed, 2 deletions(-) diff --git a/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result b/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result index 63263c955..f638c9e48 100644 --- a/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result +++ b/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result @@ -18,4 +18,3 @@ select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoi tablename autoincrement nextvalue foo n 1 DROP DATABASE mcol5572; -DROP DATABASE calpontsys; diff --git a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test index bee2d5a99..aef1b6f75 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test +++ b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test @@ -22,4 +22,3 @@ select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoi DROP DATABASE mcol5572; --source ../include/drop_functions.inc -DROP DATABASE calpontsys; From f277a1f3d0e4dd7039db7e191258b421e3ccecbd Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 23 Jun 2025 17:34:26 +0300 Subject: [PATCH 12/68] fix MCOL-5886-use-of-partitioned-tables-in-crossengine-steps test --- ...5886-use-of-partitioned-tables-in-crossengine-steps.result | 2 ++ ...OL-5886-use-of-partitioned-tables-in-crossengine-steps.opt | 1 - ...L-5886-use-of-partitioned-tables-in-crossengine-steps.test | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mysql-test/columnstore/basic/r/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.result b/mysql-test/columnstore/basic/r/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.result index 07db35568..7676dc807 100644 --- a/mysql-test/columnstore/basic/r/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.result +++ b/mysql-test/columnstore/basic/r/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.result @@ -13,3 +13,5 @@ COUNT(*) SELECT COUNT(*) FROM (SELECT * FROM t2 UNION ALL SELECT * FROM t1 PARTITION (p0)) tt; COUNT(*) 20 +REVOKE ALL PRIVILEGES ON *.* FROM 'cejuser'@'localhost'; +DROP DATABASE MCOL5886; diff --git a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt index 02b69515b..ba0d1d133 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt +++ b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.opt @@ -1,3 +1,2 @@ --skip-partition=0 --skip-sequence=0 - diff --git a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test index ea4726ca9..af3fbff86 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test +++ b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test @@ -8,6 +8,7 @@ USE MCOL5886; --exec $MCS_MCSSETCONFIG CrossEngineSupport User 'cejuser' --exec $MCS_MCSSETCONFIG CrossEngineSupport Password 'Vagrant1|0000001' +--exec $MCS_MCSSETCONFIG CrossEngineSupport Port $MASTER_MYPORT --disable_warnings CREATE USER IF NOT EXISTS'cejuser'@'localhost' IDENTIFIED BY 'Vagrant1|0000001'; @@ -23,4 +24,5 @@ CREATE TABLE IF NOT EXISTS t2 ( a DECIMAL(12, 2), b int ) ENGINE=COLUMNSTORE; SELECT COUNT(*) FROM (SELECT * FROM t1 PARTITION (p0)) tt; SELECT COUNT(*) FROM (SELECT * FROM t2 UNION ALL SELECT * FROM t1 PARTITION (p0)) tt; - +REVOKE ALL PRIVILEGES ON *.* FROM 'cejuser'@'localhost'; +DROP DATABASE MCOL5886; From 069bdf47f57401a1ba22ef39a50c24f30e991fdd Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 23 Jun 2025 23:17:33 +0300 Subject: [PATCH 13/68] fix MCOL-5572-autoincrement-filtering and mcol-5669 tests --- ...-5886-use-of-partitioned-tables-in-crossengine-steps.result | 1 + ...OL-5886-use-of-partitioned-tables-in-crossengine-steps.test | 1 + mysql-test/columnstore/bugfixes/mcol-5669.result | 2 ++ mysql-test/columnstore/bugfixes/mcol-5669.test | 3 +++ 4 files changed, 7 insertions(+) diff --git a/mysql-test/columnstore/basic/r/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.result b/mysql-test/columnstore/basic/r/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.result index 7676dc807..a7e29c42a 100644 --- a/mysql-test/columnstore/basic/r/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.result +++ b/mysql-test/columnstore/basic/r/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.result @@ -14,4 +14,5 @@ SELECT COUNT(*) FROM (SELECT * FROM t2 UNION ALL SELECT * FROM t1 PARTITION (p0) COUNT(*) 20 REVOKE ALL PRIVILEGES ON *.* FROM 'cejuser'@'localhost'; +DROP USER 'cejuser'@'localhost'; DROP DATABASE MCOL5886; diff --git a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test index af3fbff86..fbd3b3041 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test +++ b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test @@ -25,4 +25,5 @@ SELECT COUNT(*) FROM (SELECT * FROM t1 PARTITION (p0)) tt; SELECT COUNT(*) FROM (SELECT * FROM t2 UNION ALL SELECT * FROM t1 PARTITION (p0)) tt; REVOKE ALL PRIVILEGES ON *.* FROM 'cejuser'@'localhost'; +DROP USER 'cejuser'@'localhost'; DROP DATABASE MCOL5886; diff --git a/mysql-test/columnstore/bugfixes/mcol-5669.result b/mysql-test/columnstore/bugfixes/mcol-5669.result index bb99111c4..33eabaf91 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5669.result +++ b/mysql-test/columnstore/bugfixes/mcol-5669.result @@ -112,3 +112,5 @@ di_source_id brand_sku_id adjusted_dtm is_sit sit_uom sales_uom_daily_average 5389 1004 2024-01-01 1 0.2074 0.00000000 DROP DATABASE mcol_5669; REVOKE ALL PRIVILEGES ON *.* FROM 'cejuser'@'localhost'; +DROP USER 'cejuser'@'localhost'; + diff --git a/mysql-test/columnstore/bugfixes/mcol-5669.test b/mysql-test/columnstore/bugfixes/mcol-5669.test index 1506e0a34..890d750d7 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5669.test +++ b/mysql-test/columnstore/bugfixes/mcol-5669.test @@ -111,3 +111,6 @@ INSERT INTO `zzz_999999` VALUES ('2023-12-18',5298,684963,1004,1,1,2166,1.000),( DROP DATABASE mcol_5669; REVOKE ALL PRIVILEGES ON *.* FROM 'cejuser'@'localhost'; +DROP USER 'cejuser'@'localhost'; + + From ce8368defd7c43003073ccacf6ef813bf789924b Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Tue, 24 Jun 2025 10:57:08 +0300 Subject: [PATCH 14/68] Revert "add cleanups to MTR" This reverts commit 9dafef641a785507d447d3dbce81e8ba5a60a1f0. --- .../t/MCOL-5572-autoincrement-filtering.test | 2 +- .../columnstore/basic/t/cal_named_udfs.test | 1 - .../basic/t/mcs98_moda_function.test | 2 +- .../columnstore/basic/t/mcs_named_udfs.test | 1 - ...OL-5776-GROUP-BY-HAVING-functions-use.test | 1 - ...ol-4741-strings-ranges-are-unsigned.result | 1 - ...mcol-4741-strings-ranges-are-unsigned.test | 2 +- .../columnstore/bugfixes/mcol-5074.test | 1 - .../columnstore/bugfixes/mcol-5669.test | 2 - .../columnstore/bugfixes/mcol_4465.test | 1 - .../columnstore/include/drop_functions.inc | 63 ------------------- 11 files changed, 3 insertions(+), 74 deletions(-) delete mode 100644 mysql-test/columnstore/include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test index aef1b6f75..8d331cf40 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test +++ b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test @@ -21,4 +21,4 @@ select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoi select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoincrement = 'n' and `schema`='mcol5572' and tablename='foo'; DROP DATABASE mcol5572; ---source ../include/drop_functions.inc + diff --git a/mysql-test/columnstore/basic/t/cal_named_udfs.test b/mysql-test/columnstore/basic/t/cal_named_udfs.test index bd03af5bc..d097511b8 100644 --- a/mysql-test/columnstore/basic/t/cal_named_udfs.test +++ b/mysql-test/columnstore/basic/t/cal_named_udfs.test @@ -24,4 +24,3 @@ select calgetsqlcount(); DROP TABLE t1; DROP DATABASE cal_test_db; ---source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/t/mcs98_moda_function.test b/mysql-test/columnstore/basic/t/mcs98_moda_function.test index 9b02a79a2..148f355b4 100644 --- a/mysql-test/columnstore/basic/t/mcs98_moda_function.test +++ b/mysql-test/columnstore/basic/t/mcs98_moda_function.test @@ -101,4 +101,4 @@ SELECT moda(floor(rl)) FROM t2; SELECT t, moda(tu) 'q1' FROM t2 GROUP BY t HAVING moda(tu) > 5; # Clean UP DROP DATABASE mcs98_db; ---source ../include/drop_functions.inc + diff --git a/mysql-test/columnstore/basic/t/mcs_named_udfs.test b/mysql-test/columnstore/basic/t/mcs_named_udfs.test index 00ca03aa5..84859b2c6 100644 --- a/mysql-test/columnstore/basic/t/mcs_named_udfs.test +++ b/mysql-test/columnstore/basic/t/mcs_named_udfs.test @@ -25,4 +25,3 @@ select mcslastinsertid("t1"); DROP TABLE t1; DROP DATABASE cal_test_db; ---source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test index 0edfd7823..99c91b68d 100644 --- a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test +++ b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test @@ -103,4 +103,3 @@ INSERT INTO t(k) VALUES (1), (2), (2), (3), (3), (4), (4),(4),(4),(4),(995), (NU SELECT k + k a FROM t GROUP BY a HAVING a >= 8; DROP DATABASE MCOL5776; ---source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result index b3dcfdb62..c9e2db639 100644 --- a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result +++ b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result @@ -10,4 +10,3 @@ LOAD DATA LOCAL infile 'MTR_SUITE_DIR/../std_data/mcol-4741-part-2-empty-strings SELECT COUNT(*) FROM t WHERE x='val'; COUNT(*) 2 -DROP DATABASE db4741; diff --git a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test index a3d2222a9..31311d992 100644 --- a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test +++ b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test @@ -16,4 +16,4 @@ SELECT COUNT(*) FROM t WHERE x='val'; --eval LOAD DATA LOCAL infile '$MTR_SUITE_DIR/../std_data/mcol-4741-part-2-empty-strings.txt' INTO TABLE t SELECT COUNT(*) FROM t WHERE x='val'; -DROP DATABASE db4741; + diff --git a/mysql-test/columnstore/bugfixes/mcol-5074.test b/mysql-test/columnstore/bugfixes/mcol-5074.test index 949210734..aed1ff750 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5074.test +++ b/mysql-test/columnstore/bugfixes/mcol-5074.test @@ -40,5 +40,4 @@ select case when da.mtn in ('PFG','Person Focal Group') then count( acct_id) end + case when da.mtn = 'Person Focal Group' then count( acct_id) end as PFG_active_accounts from accnt da group by mtn, accts, act_created_dt order by mtn; - DROP DATABASE mcol_5074_db; diff --git a/mysql-test/columnstore/bugfixes/mcol-5669.test b/mysql-test/columnstore/bugfixes/mcol-5669.test index 890d750d7..5420034d8 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5669.test +++ b/mysql-test/columnstore/bugfixes/mcol-5669.test @@ -112,5 +112,3 @@ INSERT INTO `zzz_999999` VALUES ('2023-12-18',5298,684963,1004,1,1,2166,1.000),( DROP DATABASE mcol_5669; REVOKE ALL PRIVILEGES ON *.* FROM 'cejuser'@'localhost'; DROP USER 'cejuser'@'localhost'; - - diff --git a/mysql-test/columnstore/bugfixes/mcol_4465.test b/mysql-test/columnstore/bugfixes/mcol_4465.test index 6339eced9..25503f6d5 100644 --- a/mysql-test/columnstore/bugfixes/mcol_4465.test +++ b/mysql-test/columnstore/bugfixes/mcol_4465.test @@ -10,4 +10,3 @@ INSERT INTO cs1 VALUES('i',0),('i',0),('ii',0); SELECT c,regr_count(d,length(c)) f FROM cs1 GROUP BY 1 ORDER BY 1; SELECT * FROM (SELECT c,regr_count(d,length(c)) f FROM cs1 GROUP BY 1 ORDER BY 1)a; DROP DATABASE mcol_4465; ---source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/include/drop_functions.inc b/mysql-test/columnstore/include/drop_functions.inc deleted file mode 100644 index a55af261c..000000000 --- a/mysql-test/columnstore/include/drop_functions.inc +++ /dev/null @@ -1,63 +0,0 @@ ---disable_query_log -DROP FUNCTION mcsgetstats; -DROP FUNCTION calgetstats; -DROP FUNCTION mcssettrace; -DROP FUNCTION calsettrace; -DROP FUNCTION mcssetparms; -DROP FUNCTION calsetparms; -DROP FUNCTION mcsflushcache; -DROP FUNCTION calflushcache; -DROP FUNCTION mcsgettrace; -DROP FUNCTION calgettrace; -DROP FUNCTION mcsgetversion; -DROP FUNCTION calgetversion; -DROP FUNCTION calonlinealter; -DROP FUNCTION mcsviewtablelock; -DROP FUNCTION calviewtablelock; -DROP FUNCTION mcscleartablelock; -DROP FUNCTION calcleartablelock; -DROP FUNCTION mcslastinsertid; -DROP FUNCTION callastinsertid; -DROP FUNCTION mcsgetsqlcount; -DROP FUNCTION calgetsqlcount; -DROP FUNCTION idbpm; -DROP FUNCTION idbdbroot; -DROP FUNCTION idbsegment; -DROP FUNCTION idbsegmentdir; -DROP FUNCTION idbextentrelativerid; -DROP FUNCTION idbblockid; -DROP FUNCTION idbextentid; -DROP FUNCTION idbextentmin; -DROP FUNCTION idbextentmax; -DROP FUNCTION idbpartition; -DROP FUNCTION idblocalpm; -DROP FUNCTION mcssystemready; -DROP FUNCTION mcssystemreadonly; -DROP FUNCTION mcssystemprimary; -DROP FUNCTION mcs_emindex_size; -DROP FUNCTION mcs_emindex_free; -DROP FUNCTION columnstore_dataload; -DROP FUNCTION regr_avgx; -DROP FUNCTION regr_avgy; -DROP FUNCTION regr_count; -DROP FUNCTION regr_slope; -DROP FUNCTION regr_intercept; -DROP FUNCTION regr_r2; -DROP FUNCTION corr; -DROP FUNCTION regr_sxx; -DROP FUNCTION regr_syy; -DROP FUNCTION regr_sxy; -DROP FUNCTION covar_pop; -DROP FUNCTION covar_samp; -DROP FUNCTION distinct_count; -DROP FUNCTION caldisablepartitions; -DROP FUNCTION calenablepartitions; -DROP FUNCTION caldroppartitions; -DROP FUNCTION calshowpartitions; -DROP FUNCTION caldroppartitionsbyvalue; -DROP FUNCTION caldisablepartitionsbyvalue; -DROP FUNCTION calenablepartitionsbyvalue; -DROP FUNCTION calshowpartitionsbyvalue; -DROP FUNCTION moda; -DROP FUNCTION mcs_set_ddldebug_level; ---enable_query_log From f8b32bbf9a9dea1076a759a103fc159c60043b70 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Tue, 24 Jun 2025 15:59:58 +0300 Subject: [PATCH 15/68] Fix typos in bugfixes.mcol-5074 and bugfixes.mcol-5669 tests --- mysql-test/columnstore/bugfixes/mcol-5074.test | 2 +- mysql-test/columnstore/bugfixes/mcol-5669.result | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/mysql-test/columnstore/bugfixes/mcol-5074.test b/mysql-test/columnstore/bugfixes/mcol-5074.test index aed1ff750..6c4978590 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5074.test +++ b/mysql-test/columnstore/bugfixes/mcol-5074.test @@ -40,4 +40,4 @@ select case when da.mtn in ('PFG','Person Focal Group') then count( acct_id) end + case when da.mtn = 'Person Focal Group' then count( acct_id) end as PFG_active_accounts from accnt da group by mtn, accts, act_created_dt order by mtn; - +DROP DATABASE mcol_5074_db; diff --git a/mysql-test/columnstore/bugfixes/mcol-5669.result b/mysql-test/columnstore/bugfixes/mcol-5669.result index 33eabaf91..10cb76216 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5669.result +++ b/mysql-test/columnstore/bugfixes/mcol-5669.result @@ -113,4 +113,3 @@ di_source_id brand_sku_id adjusted_dtm is_sit sit_uom sales_uom_daily_average DROP DATABASE mcol_5669; REVOKE ALL PRIVILEGES ON *.* FROM 'cejuser'@'localhost'; DROP USER 'cejuser'@'localhost'; - From 4b8b5e3d75807a68dd7730ad1974a76972e2cadc Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Tue, 24 Jun 2025 16:10:03 +0300 Subject: [PATCH 16/68] add clean up to bugfixes.mcol-4741-strings-ranges-are-unsigned test --- .../bugfixes/mcol-4741-strings-ranges-are-unsigned.result | 2 +- .../bugfixes/mcol-4741-strings-ranges-are-unsigned.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result index c9e2db639..b531a6e1c 100644 --- a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result +++ b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result @@ -9,4 +9,4 @@ COUNT(*) LOAD DATA LOCAL infile 'MTR_SUITE_DIR/../std_data/mcol-4741-part-2-empty-strings.txt' INTO TABLE t; SELECT COUNT(*) FROM t WHERE x='val'; COUNT(*) -2 +DROP DATABASE IF EXISTS db4741; diff --git a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test index 31311d992..c57fbbb98 100644 --- a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test +++ b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.test @@ -16,4 +16,4 @@ SELECT COUNT(*) FROM t WHERE x='val'; --eval LOAD DATA LOCAL infile '$MTR_SUITE_DIR/../std_data/mcol-4741-part-2-empty-strings.txt' INTO TABLE t SELECT COUNT(*) FROM t WHERE x='val'; - +DROP DATABASE IF EXISTS db4741; From 22ad96e04a9848d506e9cd7d085deec06d8bb759 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 25 Jun 2025 00:26:56 +0300 Subject: [PATCH 17/68] Fix typo in the mcol-4741-strings-ranges-are-unsigned test --- .../bugfixes/mcol-4741-strings-ranges-are-unsigned.result | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result index b531a6e1c..778ef7959 100644 --- a/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result +++ b/mysql-test/columnstore/bugfixes/mcol-4741-strings-ranges-are-unsigned.result @@ -9,4 +9,5 @@ COUNT(*) LOAD DATA LOCAL infile 'MTR_SUITE_DIR/../std_data/mcol-4741-part-2-empty-strings.txt' INTO TABLE t; SELECT COUNT(*) FROM t WHERE x='val'; COUNT(*) +2 DROP DATABASE IF EXISTS db4741; From 3d5e10631082bbe361a36cbb7b0b6c8e1e7f0262 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 18 Jun 2025 18:01:51 +0300 Subject: [PATCH 18/68] Fix charset in MTR --- mysql-test/columnstore/include/syscatalog_mysql.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/columnstore/include/syscatalog_mysql.sql b/mysql-test/columnstore/include/syscatalog_mysql.sql index ef2b22039..43beaee2b 100644 --- a/mysql-test/columnstore/include/syscatalog_mysql.sql +++ b/mysql-test/columnstore/include/syscatalog_mysql.sql @@ -1,5 +1,6 @@ --disable_query_log --disable_warnings +-- SYSTABLE create database if not exists calpontsys; use calpontsys; @@ -46,4 +47,3 @@ create table if not exists syscolumn (`schema` varchar(128), --enable_warnings --enable_query_log - From e789dfd1bf75abbe3effbf805434a6860974967c Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Thu, 19 Jun 2025 18:37:54 +0300 Subject: [PATCH 19/68] enable sequence engine in the mcol-4786 test --- mysql-test/columnstore/basic/r/analyze_table.result | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/columnstore/basic/r/analyze_table.result b/mysql-test/columnstore/basic/r/analyze_table.result index 08d8031e0..5630c2c2a 100644 --- a/mysql-test/columnstore/basic/r/analyze_table.result +++ b/mysql-test/columnstore/basic/r/analyze_table.result @@ -7,12 +7,14 @@ analyze table t1 PERSISTENT FOR ALL; Table Op Msg_type Msg_text analyze_table_db.t1 analyze status Engine-independent statistics collected analyze_table_db.t1 analyze status OK +analyze_table_db.t1 analyze status Engine-independent statistics collected create table t2 (a int, b double) engine=columnstore; insert into t2 values (2, 3), (3, 4); analyze table t2 PERSISTENT FOR ALL; Table Op Msg_type Msg_text analyze_table_db.t2 analyze status Engine-independent statistics collected analyze_table_db.t2 analyze status OK +analyze_table_db.t2 analyze status Engine-independent statistics collected create table t3 (a varchar(25)) engine=columnstore; insert into t3 values ("a"), ("b"); analyze table t3 PERSISTENT FOR ALL; From 549b33f9452fe48bb8694422113746b5097d4d57 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 20 Jun 2025 14:14:28 +0300 Subject: [PATCH 20/68] Fix analize_table test --- mysql-test/columnstore/basic/r/analyze_table.result | 2 -- 1 file changed, 2 deletions(-) diff --git a/mysql-test/columnstore/basic/r/analyze_table.result b/mysql-test/columnstore/basic/r/analyze_table.result index 5630c2c2a..08d8031e0 100644 --- a/mysql-test/columnstore/basic/r/analyze_table.result +++ b/mysql-test/columnstore/basic/r/analyze_table.result @@ -7,14 +7,12 @@ analyze table t1 PERSISTENT FOR ALL; Table Op Msg_type Msg_text analyze_table_db.t1 analyze status Engine-independent statistics collected analyze_table_db.t1 analyze status OK -analyze_table_db.t1 analyze status Engine-independent statistics collected create table t2 (a int, b double) engine=columnstore; insert into t2 values (2, 3), (3, 4); analyze table t2 PERSISTENT FOR ALL; Table Op Msg_type Msg_text analyze_table_db.t2 analyze status Engine-independent statistics collected analyze_table_db.t2 analyze status OK -analyze_table_db.t2 analyze status Engine-independent statistics collected create table t3 (a varchar(25)) engine=columnstore; insert into t3 values ("a"), ("b"); analyze table t3 PERSISTENT FOR ALL; From 1465a5c6b3fd6d93c60b1e3c1c0515279f4b352f Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 20 Jun 2025 18:00:35 +0300 Subject: [PATCH 21/68] dusable warings in the includes for MTR --- mysql-test/columnstore/include/syscatalog_mysql.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/columnstore/include/syscatalog_mysql.sql b/mysql-test/columnstore/include/syscatalog_mysql.sql index 43beaee2b..23470f666 100644 --- a/mysql-test/columnstore/include/syscatalog_mysql.sql +++ b/mysql-test/columnstore/include/syscatalog_mysql.sql @@ -1,6 +1,7 @@ --disable_query_log --disable_warnings -- SYSTABLE + create database if not exists calpontsys; use calpontsys; From 1c6985cbf5992ed42ed14507d6871be4d722ff67 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Sun, 22 Jun 2025 00:47:44 +0300 Subject: [PATCH 22/68] add cleanups to MTR --- .../MCOL-5572-autoincrement-filtering.result | 1 + .../t/MCOL-5572-autoincrement-filtering.test | 3 +- .../columnstore/basic/t/cal_named_udfs.test | 1 + .../basic/t/mcs98_moda_function.test | 2 +- .../columnstore/basic/t/mcs_named_udfs.test | 1 + ...OL-5776-GROUP-BY-HAVING-functions-use.test | 1 + .../columnstore/bugfixes/mcol_4465.test | 1 + .../columnstore/include/drop_functions.inc | 63 +++++++++++++++++++ 8 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 mysql-test/columnstore/include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result b/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result index f638c9e48..63263c955 100644 --- a/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result +++ b/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result @@ -18,3 +18,4 @@ select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoi tablename autoincrement nextvalue foo n 1 DROP DATABASE mcol5572; +DROP DATABASE calpontsys; diff --git a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test index 8d331cf40..bee2d5a99 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test +++ b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test @@ -21,4 +21,5 @@ select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoi select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoincrement = 'n' and `schema`='mcol5572' and tablename='foo'; DROP DATABASE mcol5572; - +--source ../include/drop_functions.inc +DROP DATABASE calpontsys; diff --git a/mysql-test/columnstore/basic/t/cal_named_udfs.test b/mysql-test/columnstore/basic/t/cal_named_udfs.test index d097511b8..bd03af5bc 100644 --- a/mysql-test/columnstore/basic/t/cal_named_udfs.test +++ b/mysql-test/columnstore/basic/t/cal_named_udfs.test @@ -24,3 +24,4 @@ select calgetsqlcount(); DROP TABLE t1; DROP DATABASE cal_test_db; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/t/mcs98_moda_function.test b/mysql-test/columnstore/basic/t/mcs98_moda_function.test index 148f355b4..9b02a79a2 100644 --- a/mysql-test/columnstore/basic/t/mcs98_moda_function.test +++ b/mysql-test/columnstore/basic/t/mcs98_moda_function.test @@ -101,4 +101,4 @@ SELECT moda(floor(rl)) FROM t2; SELECT t, moda(tu) 'q1' FROM t2 GROUP BY t HAVING moda(tu) > 5; # Clean UP DROP DATABASE mcs98_db; - +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/t/mcs_named_udfs.test b/mysql-test/columnstore/basic/t/mcs_named_udfs.test index 84859b2c6..00ca03aa5 100644 --- a/mysql-test/columnstore/basic/t/mcs_named_udfs.test +++ b/mysql-test/columnstore/basic/t/mcs_named_udfs.test @@ -25,3 +25,4 @@ select mcslastinsertid("t1"); DROP TABLE t1; DROP DATABASE cal_test_db; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test index 99c91b68d..0edfd7823 100644 --- a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test +++ b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test @@ -103,3 +103,4 @@ INSERT INTO t(k) VALUES (1), (2), (2), (3), (3), (4), (4),(4),(4),(4),(995), (NU SELECT k + k a FROM t GROUP BY a HAVING a >= 8; DROP DATABASE MCOL5776; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/bugfixes/mcol_4465.test b/mysql-test/columnstore/bugfixes/mcol_4465.test index 25503f6d5..6339eced9 100644 --- a/mysql-test/columnstore/bugfixes/mcol_4465.test +++ b/mysql-test/columnstore/bugfixes/mcol_4465.test @@ -10,3 +10,4 @@ INSERT INTO cs1 VALUES('i',0),('i',0),('ii',0); SELECT c,regr_count(d,length(c)) f FROM cs1 GROUP BY 1 ORDER BY 1; SELECT * FROM (SELECT c,regr_count(d,length(c)) f FROM cs1 GROUP BY 1 ORDER BY 1)a; DROP DATABASE mcol_4465; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/include/drop_functions.inc b/mysql-test/columnstore/include/drop_functions.inc new file mode 100644 index 000000000..a55af261c --- /dev/null +++ b/mysql-test/columnstore/include/drop_functions.inc @@ -0,0 +1,63 @@ +--disable_query_log +DROP FUNCTION mcsgetstats; +DROP FUNCTION calgetstats; +DROP FUNCTION mcssettrace; +DROP FUNCTION calsettrace; +DROP FUNCTION mcssetparms; +DROP FUNCTION calsetparms; +DROP FUNCTION mcsflushcache; +DROP FUNCTION calflushcache; +DROP FUNCTION mcsgettrace; +DROP FUNCTION calgettrace; +DROP FUNCTION mcsgetversion; +DROP FUNCTION calgetversion; +DROP FUNCTION calonlinealter; +DROP FUNCTION mcsviewtablelock; +DROP FUNCTION calviewtablelock; +DROP FUNCTION mcscleartablelock; +DROP FUNCTION calcleartablelock; +DROP FUNCTION mcslastinsertid; +DROP FUNCTION callastinsertid; +DROP FUNCTION mcsgetsqlcount; +DROP FUNCTION calgetsqlcount; +DROP FUNCTION idbpm; +DROP FUNCTION idbdbroot; +DROP FUNCTION idbsegment; +DROP FUNCTION idbsegmentdir; +DROP FUNCTION idbextentrelativerid; +DROP FUNCTION idbblockid; +DROP FUNCTION idbextentid; +DROP FUNCTION idbextentmin; +DROP FUNCTION idbextentmax; +DROP FUNCTION idbpartition; +DROP FUNCTION idblocalpm; +DROP FUNCTION mcssystemready; +DROP FUNCTION mcssystemreadonly; +DROP FUNCTION mcssystemprimary; +DROP FUNCTION mcs_emindex_size; +DROP FUNCTION mcs_emindex_free; +DROP FUNCTION columnstore_dataload; +DROP FUNCTION regr_avgx; +DROP FUNCTION regr_avgy; +DROP FUNCTION regr_count; +DROP FUNCTION regr_slope; +DROP FUNCTION regr_intercept; +DROP FUNCTION regr_r2; +DROP FUNCTION corr; +DROP FUNCTION regr_sxx; +DROP FUNCTION regr_syy; +DROP FUNCTION regr_sxy; +DROP FUNCTION covar_pop; +DROP FUNCTION covar_samp; +DROP FUNCTION distinct_count; +DROP FUNCTION caldisablepartitions; +DROP FUNCTION calenablepartitions; +DROP FUNCTION caldroppartitions; +DROP FUNCTION calshowpartitions; +DROP FUNCTION caldroppartitionsbyvalue; +DROP FUNCTION caldisablepartitionsbyvalue; +DROP FUNCTION calenablepartitionsbyvalue; +DROP FUNCTION calshowpartitionsbyvalue; +DROP FUNCTION moda; +DROP FUNCTION mcs_set_ddldebug_level; +--enable_query_log From e95fef4f265bdc813e7185e032d17705fa3c3ce0 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 23 Jun 2025 15:06:25 +0300 Subject: [PATCH 23/68] Fix MCOL-5572-autoincrement-filtering test --- .../columnstore/basic/r/MCOL-5572-autoincrement-filtering.result | 1 - .../columnstore/basic/t/MCOL-5572-autoincrement-filtering.test | 1 - 2 files changed, 2 deletions(-) diff --git a/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result b/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result index 63263c955..f638c9e48 100644 --- a/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result +++ b/mysql-test/columnstore/basic/r/MCOL-5572-autoincrement-filtering.result @@ -18,4 +18,3 @@ select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoi tablename autoincrement nextvalue foo n 1 DROP DATABASE mcol5572; -DROP DATABASE calpontsys; diff --git a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test index bee2d5a99..aef1b6f75 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test +++ b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test @@ -22,4 +22,3 @@ select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoi DROP DATABASE mcol5572; --source ../include/drop_functions.inc -DROP DATABASE calpontsys; From 72753308cd6452064517da3609ee86cf0b051a7c Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Tue, 24 Jun 2025 10:57:08 +0300 Subject: [PATCH 24/68] Revert "add cleanups to MTR" This reverts commit 9dafef641a785507d447d3dbce81e8ba5a60a1f0. --- .../t/MCOL-5572-autoincrement-filtering.test | 2 +- .../columnstore/basic/t/cal_named_udfs.test | 1 - .../basic/t/mcs98_moda_function.test | 2 +- .../columnstore/basic/t/mcs_named_udfs.test | 1 - ...OL-5776-GROUP-BY-HAVING-functions-use.test | 1 - .../columnstore/bugfixes/mcol-5074.test | 1 + .../columnstore/bugfixes/mcol_4465.test | 1 - .../columnstore/include/drop_functions.inc | 63 ------------------- 8 files changed, 3 insertions(+), 69 deletions(-) delete mode 100644 mysql-test/columnstore/include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test index aef1b6f75..8d331cf40 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test +++ b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test @@ -21,4 +21,4 @@ select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoi select tablename, autoincrement, nextvalue from calpontsys.syscolumn where autoincrement = 'n' and `schema`='mcol5572' and tablename='foo'; DROP DATABASE mcol5572; ---source ../include/drop_functions.inc + diff --git a/mysql-test/columnstore/basic/t/cal_named_udfs.test b/mysql-test/columnstore/basic/t/cal_named_udfs.test index bd03af5bc..d097511b8 100644 --- a/mysql-test/columnstore/basic/t/cal_named_udfs.test +++ b/mysql-test/columnstore/basic/t/cal_named_udfs.test @@ -24,4 +24,3 @@ select calgetsqlcount(); DROP TABLE t1; DROP DATABASE cal_test_db; ---source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/t/mcs98_moda_function.test b/mysql-test/columnstore/basic/t/mcs98_moda_function.test index 9b02a79a2..148f355b4 100644 --- a/mysql-test/columnstore/basic/t/mcs98_moda_function.test +++ b/mysql-test/columnstore/basic/t/mcs98_moda_function.test @@ -101,4 +101,4 @@ SELECT moda(floor(rl)) FROM t2; SELECT t, moda(tu) 'q1' FROM t2 GROUP BY t HAVING moda(tu) > 5; # Clean UP DROP DATABASE mcs98_db; ---source ../include/drop_functions.inc + diff --git a/mysql-test/columnstore/basic/t/mcs_named_udfs.test b/mysql-test/columnstore/basic/t/mcs_named_udfs.test index 00ca03aa5..84859b2c6 100644 --- a/mysql-test/columnstore/basic/t/mcs_named_udfs.test +++ b/mysql-test/columnstore/basic/t/mcs_named_udfs.test @@ -25,4 +25,3 @@ select mcslastinsertid("t1"); DROP TABLE t1; DROP DATABASE cal_test_db; ---source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test index 0edfd7823..99c91b68d 100644 --- a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test +++ b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test @@ -103,4 +103,3 @@ INSERT INTO t(k) VALUES (1), (2), (2), (3), (3), (4), (4),(4),(4),(4),(995), (NU SELECT k + k a FROM t GROUP BY a HAVING a >= 8; DROP DATABASE MCOL5776; ---source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/bugfixes/mcol-5074.test b/mysql-test/columnstore/bugfixes/mcol-5074.test index 6c4978590..9ec6e6535 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5074.test +++ b/mysql-test/columnstore/bugfixes/mcol-5074.test @@ -40,4 +40,5 @@ select case when da.mtn in ('PFG','Person Focal Group') then count( acct_id) end + case when da.mtn = 'Person Focal Group' then count( acct_id) end as PFG_active_accounts from accnt da group by mtn, accts, act_created_dt order by mtn; + DROP DATABASE mcol_5074_db; diff --git a/mysql-test/columnstore/bugfixes/mcol_4465.test b/mysql-test/columnstore/bugfixes/mcol_4465.test index 6339eced9..25503f6d5 100644 --- a/mysql-test/columnstore/bugfixes/mcol_4465.test +++ b/mysql-test/columnstore/bugfixes/mcol_4465.test @@ -10,4 +10,3 @@ INSERT INTO cs1 VALUES('i',0),('i',0),('ii',0); SELECT c,regr_count(d,length(c)) f FROM cs1 GROUP BY 1 ORDER BY 1; SELECT * FROM (SELECT c,regr_count(d,length(c)) f FROM cs1 GROUP BY 1 ORDER BY 1)a; DROP DATABASE mcol_4465; ---source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/include/drop_functions.inc b/mysql-test/columnstore/include/drop_functions.inc deleted file mode 100644 index a55af261c..000000000 --- a/mysql-test/columnstore/include/drop_functions.inc +++ /dev/null @@ -1,63 +0,0 @@ ---disable_query_log -DROP FUNCTION mcsgetstats; -DROP FUNCTION calgetstats; -DROP FUNCTION mcssettrace; -DROP FUNCTION calsettrace; -DROP FUNCTION mcssetparms; -DROP FUNCTION calsetparms; -DROP FUNCTION mcsflushcache; -DROP FUNCTION calflushcache; -DROP FUNCTION mcsgettrace; -DROP FUNCTION calgettrace; -DROP FUNCTION mcsgetversion; -DROP FUNCTION calgetversion; -DROP FUNCTION calonlinealter; -DROP FUNCTION mcsviewtablelock; -DROP FUNCTION calviewtablelock; -DROP FUNCTION mcscleartablelock; -DROP FUNCTION calcleartablelock; -DROP FUNCTION mcslastinsertid; -DROP FUNCTION callastinsertid; -DROP FUNCTION mcsgetsqlcount; -DROP FUNCTION calgetsqlcount; -DROP FUNCTION idbpm; -DROP FUNCTION idbdbroot; -DROP FUNCTION idbsegment; -DROP FUNCTION idbsegmentdir; -DROP FUNCTION idbextentrelativerid; -DROP FUNCTION idbblockid; -DROP FUNCTION idbextentid; -DROP FUNCTION idbextentmin; -DROP FUNCTION idbextentmax; -DROP FUNCTION idbpartition; -DROP FUNCTION idblocalpm; -DROP FUNCTION mcssystemready; -DROP FUNCTION mcssystemreadonly; -DROP FUNCTION mcssystemprimary; -DROP FUNCTION mcs_emindex_size; -DROP FUNCTION mcs_emindex_free; -DROP FUNCTION columnstore_dataload; -DROP FUNCTION regr_avgx; -DROP FUNCTION regr_avgy; -DROP FUNCTION regr_count; -DROP FUNCTION regr_slope; -DROP FUNCTION regr_intercept; -DROP FUNCTION regr_r2; -DROP FUNCTION corr; -DROP FUNCTION regr_sxx; -DROP FUNCTION regr_syy; -DROP FUNCTION regr_sxy; -DROP FUNCTION covar_pop; -DROP FUNCTION covar_samp; -DROP FUNCTION distinct_count; -DROP FUNCTION caldisablepartitions; -DROP FUNCTION calenablepartitions; -DROP FUNCTION caldroppartitions; -DROP FUNCTION calshowpartitions; -DROP FUNCTION caldroppartitionsbyvalue; -DROP FUNCTION caldisablepartitionsbyvalue; -DROP FUNCTION calenablepartitionsbyvalue; -DROP FUNCTION calshowpartitionsbyvalue; -DROP FUNCTION moda; -DROP FUNCTION mcs_set_ddldebug_level; ---enable_query_log From d503221e6fa7b5a36ada4de410e142d5648973d5 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 25 Jun 2025 14:32:57 +0300 Subject: [PATCH 25/68] Drop functions in the bugfixes suite tests --- ...OL-5776-GROUP-BY-HAVING-functions-use.test | 1 + .../columnstore/bugfixes/mcol_4465.test | 1 + .../columnstore/include/drop_functions.inc | 63 +++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 mysql-test/columnstore/include/drop_functions.inc diff --git a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test index 99c91b68d..0edfd7823 100644 --- a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test +++ b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test @@ -103,3 +103,4 @@ INSERT INTO t(k) VALUES (1), (2), (2), (3), (3), (4), (4),(4),(4),(4),(995), (NU SELECT k + k a FROM t GROUP BY a HAVING a >= 8; DROP DATABASE MCOL5776; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/bugfixes/mcol_4465.test b/mysql-test/columnstore/bugfixes/mcol_4465.test index 25503f6d5..6339eced9 100644 --- a/mysql-test/columnstore/bugfixes/mcol_4465.test +++ b/mysql-test/columnstore/bugfixes/mcol_4465.test @@ -10,3 +10,4 @@ INSERT INTO cs1 VALUES('i',0),('i',0),('ii',0); SELECT c,regr_count(d,length(c)) f FROM cs1 GROUP BY 1 ORDER BY 1; SELECT * FROM (SELECT c,regr_count(d,length(c)) f FROM cs1 GROUP BY 1 ORDER BY 1)a; DROP DATABASE mcol_4465; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/include/drop_functions.inc b/mysql-test/columnstore/include/drop_functions.inc new file mode 100644 index 000000000..a55af261c --- /dev/null +++ b/mysql-test/columnstore/include/drop_functions.inc @@ -0,0 +1,63 @@ +--disable_query_log +DROP FUNCTION mcsgetstats; +DROP FUNCTION calgetstats; +DROP FUNCTION mcssettrace; +DROP FUNCTION calsettrace; +DROP FUNCTION mcssetparms; +DROP FUNCTION calsetparms; +DROP FUNCTION mcsflushcache; +DROP FUNCTION calflushcache; +DROP FUNCTION mcsgettrace; +DROP FUNCTION calgettrace; +DROP FUNCTION mcsgetversion; +DROP FUNCTION calgetversion; +DROP FUNCTION calonlinealter; +DROP FUNCTION mcsviewtablelock; +DROP FUNCTION calviewtablelock; +DROP FUNCTION mcscleartablelock; +DROP FUNCTION calcleartablelock; +DROP FUNCTION mcslastinsertid; +DROP FUNCTION callastinsertid; +DROP FUNCTION mcsgetsqlcount; +DROP FUNCTION calgetsqlcount; +DROP FUNCTION idbpm; +DROP FUNCTION idbdbroot; +DROP FUNCTION idbsegment; +DROP FUNCTION idbsegmentdir; +DROP FUNCTION idbextentrelativerid; +DROP FUNCTION idbblockid; +DROP FUNCTION idbextentid; +DROP FUNCTION idbextentmin; +DROP FUNCTION idbextentmax; +DROP FUNCTION idbpartition; +DROP FUNCTION idblocalpm; +DROP FUNCTION mcssystemready; +DROP FUNCTION mcssystemreadonly; +DROP FUNCTION mcssystemprimary; +DROP FUNCTION mcs_emindex_size; +DROP FUNCTION mcs_emindex_free; +DROP FUNCTION columnstore_dataload; +DROP FUNCTION regr_avgx; +DROP FUNCTION regr_avgy; +DROP FUNCTION regr_count; +DROP FUNCTION regr_slope; +DROP FUNCTION regr_intercept; +DROP FUNCTION regr_r2; +DROP FUNCTION corr; +DROP FUNCTION regr_sxx; +DROP FUNCTION regr_syy; +DROP FUNCTION regr_sxy; +DROP FUNCTION covar_pop; +DROP FUNCTION covar_samp; +DROP FUNCTION distinct_count; +DROP FUNCTION caldisablepartitions; +DROP FUNCTION calenablepartitions; +DROP FUNCTION caldroppartitions; +DROP FUNCTION calshowpartitions; +DROP FUNCTION caldroppartitionsbyvalue; +DROP FUNCTION caldisablepartitionsbyvalue; +DROP FUNCTION calenablepartitionsbyvalue; +DROP FUNCTION calshowpartitionsbyvalue; +DROP FUNCTION moda; +DROP FUNCTION mcs_set_ddldebug_level; +--enable_query_log From 06aff0bd0a273a5c1cbc9246bc45a3fc6ea753ce Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 25 Jun 2025 16:21:37 +0300 Subject: [PATCH 26/68] Drop functions in the basic suite tests --- mysql-test/columnstore/basic/t/cal_named_udfs.test | 1 + mysql-test/columnstore/basic/t/mcs98_moda_function.test | 2 +- mysql-test/columnstore/basic/t/mcs_named_udfs.test | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mysql-test/columnstore/basic/t/cal_named_udfs.test b/mysql-test/columnstore/basic/t/cal_named_udfs.test index d097511b8..bd03af5bc 100644 --- a/mysql-test/columnstore/basic/t/cal_named_udfs.test +++ b/mysql-test/columnstore/basic/t/cal_named_udfs.test @@ -24,3 +24,4 @@ select calgetsqlcount(); DROP TABLE t1; DROP DATABASE cal_test_db; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/t/mcs98_moda_function.test b/mysql-test/columnstore/basic/t/mcs98_moda_function.test index 148f355b4..9b02a79a2 100644 --- a/mysql-test/columnstore/basic/t/mcs98_moda_function.test +++ b/mysql-test/columnstore/basic/t/mcs98_moda_function.test @@ -101,4 +101,4 @@ SELECT moda(floor(rl)) FROM t2; SELECT t, moda(tu) 'q1' FROM t2 GROUP BY t HAVING moda(tu) > 5; # Clean UP DROP DATABASE mcs98_db; - +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/basic/t/mcs_named_udfs.test b/mysql-test/columnstore/basic/t/mcs_named_udfs.test index 84859b2c6..00ca03aa5 100644 --- a/mysql-test/columnstore/basic/t/mcs_named_udfs.test +++ b/mysql-test/columnstore/basic/t/mcs_named_udfs.test @@ -25,3 +25,4 @@ select mcslastinsertid("t1"); DROP TABLE t1; DROP DATABASE cal_test_db; +--source ../include/drop_functions.inc From d08766fd7d8f361d69627e4e25012d7e1e823f84 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 25 Jun 2025 19:42:39 +0300 Subject: [PATCH 27/68] disable basic.MCOL-5572-autoincrement-filtering when running without --extern --- .../basic/t/MCOL-5572-autoincrement-filtering.test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test index 8d331cf40..0a7f1af32 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test +++ b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test @@ -2,6 +2,12 @@ --source ../include/functions.inc --source ../include/syscatalog_mysql.sql +if ($MASTER_MYPORT) +{ + # Running without --extern + --skip Only works with --extern (will work without extern when cleanup issues are fixed) +} + SET default_storage_engine=columnstore; --disable_warnings From 4df58ffa963dfb6cbcf169dc68b888f6a90f552b Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Thu, 26 Jun 2025 02:09:06 +0300 Subject: [PATCH 28/68] Check for --extern before importing functions in the MCOL-5572-autoincrement-filtering.test --- .../basic/t/MCOL-5572-autoincrement-filtering.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test index 0a7f1af32..462157f6e 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test +++ b/mysql-test/columnstore/basic/t/MCOL-5572-autoincrement-filtering.test @@ -1,13 +1,13 @@ ---source ../include/have_columnstore.inc ---source ../include/functions.inc ---source ../include/syscatalog_mysql.sql - if ($MASTER_MYPORT) { # Running without --extern --skip Only works with --extern (will work without extern when cleanup issues are fixed) } +--source ../include/have_columnstore.inc +--source ../include/functions.inc +--source ../include/syscatalog_mysql.sql + SET default_storage_engine=columnstore; --disable_warnings From f1b68b9a19ab750b6ce74bf59436995ee2d9d390 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 30 Jun 2025 11:26:03 +0300 Subject: [PATCH 29/68] Fix merge error im the mysql-test/columnstore/include/syscatalog_mysql.sql --- mysql-test/columnstore/include/syscatalog_mysql.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/mysql-test/columnstore/include/syscatalog_mysql.sql b/mysql-test/columnstore/include/syscatalog_mysql.sql index 23470f666..aea3fdaad 100644 --- a/mysql-test/columnstore/include/syscatalog_mysql.sql +++ b/mysql-test/columnstore/include/syscatalog_mysql.sql @@ -1,6 +1,5 @@ --disable_query_log --disable_warnings --- SYSTABLE create database if not exists calpontsys; From 1adb39db41b28f6f0b438a78f371622cd52bc5bc Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 30 Jun 2025 11:30:05 +0300 Subject: [PATCH 30/68] Fix merge error im the mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps --- ...5886-use-of-partitioned-tables-in-crossengine-steps.test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test index fbd3b3041..2320e5848 100644 --- a/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test +++ b/mysql-test/columnstore/basic/t/MCOL-5886-use-of-partitioned-tables-in-crossengine-steps.test @@ -6,6 +6,12 @@ DROP DATABASE IF EXISTS MCOL5886; CREATE DATABASE MCOL5886; USE MCOL5886; +if (!$MASTER_MYPORT) +{ + # Running with --extern + let $MASTER_MYPORT=`SELECT @@port`; +} + --exec $MCS_MCSSETCONFIG CrossEngineSupport User 'cejuser' --exec $MCS_MCSSETCONFIG CrossEngineSupport Password 'Vagrant1|0000001' --exec $MCS_MCSSETCONFIG CrossEngineSupport Port $MASTER_MYPORT From 9c0accb44703f812485113ac5f189c3e601962ca Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 30 Jun 2025 15:15:01 +0300 Subject: [PATCH 31/68] disable tests inclompatible with 11.4 --- mysql-test/columnstore/basic/t/mcs114_comments.test | 1 + mysql-test/columnstore/basic/t/mcs11_show_db_table.test | 1 + mysql-test/columnstore/basic/t/mcs12_alter_table.test | 1 + .../columnstore/basic/t/mcs1_create_table_data_types.test | 1 + .../columnstore/basic/t/mcs45_write_crossengine_join.test | 1 + mysql-test/columnstore/bugfixes/mcol-4740.test | 4 ++-- mysql-test/columnstore/bugfixes/mcol-5480.test | 2 +- mysql-test/columnstore/bugfixes/mcol-5669.test | 1 + mysql-test/columnstore/include/disable_11.4.inc | 4 ++++ 9 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 mysql-test/columnstore/include/disable_11.4.inc diff --git a/mysql-test/columnstore/basic/t/mcs114_comments.test b/mysql-test/columnstore/basic/t/mcs114_comments.test index ad7406d16..66bc469c2 100644 --- a/mysql-test/columnstore/basic/t/mcs114_comments.test +++ b/mysql-test/columnstore/basic/t/mcs114_comments.test @@ -2,6 +2,7 @@ # Test comments # Author: Bharath, bharath.bokka@mariadb.com # +--source ../include/disable_11.4.inc -- source ../include/have_columnstore.inc --disable_warnings diff --git a/mysql-test/columnstore/basic/t/mcs11_show_db_table.test b/mysql-test/columnstore/basic/t/mcs11_show_db_table.test index f5469d63d..c0948a830 100644 --- a/mysql-test/columnstore/basic/t/mcs11_show_db_table.test +++ b/mysql-test/columnstore/basic/t/mcs11_show_db_table.test @@ -1,6 +1,7 @@ # #This test verifies show databases and tables with engine=columnstore. # +--source ../include/disable_11.4.inc -- source ../include/have_columnstore.inc --source ../include/detect_maxscale.inc -- source include/have_innodb.inc diff --git a/mysql-test/columnstore/basic/t/mcs12_alter_table.test b/mysql-test/columnstore/basic/t/mcs12_alter_table.test index 175086f43..dd8dda7fa 100644 --- a/mysql-test/columnstore/basic/t/mcs12_alter_table.test +++ b/mysql-test/columnstore/basic/t/mcs12_alter_table.test @@ -1,6 +1,7 @@ # # Test ALTER TABLE schemas in various possible ways. # +--source ../include/disable_11.4.inc -- source ../include/have_columnstore.inc -- source include/have_innodb.inc -- source ../include/check_maxscale.inc diff --git a/mysql-test/columnstore/basic/t/mcs1_create_table_data_types.test b/mysql-test/columnstore/basic/t/mcs1_create_table_data_types.test index bacda2ef9..167c65d8f 100644 --- a/mysql-test/columnstore/basic/t/mcs1_create_table_data_types.test +++ b/mysql-test/columnstore/basic/t/mcs1_create_table_data_types.test @@ -2,6 +2,7 @@ # Test CREATE TABLE with all data types supported in Columnstore # Author: Bharath, bharath.bokka@mariadb.com # +--source ../include/disable_11.4.inc -- source ../include/have_columnstore.inc --source ../include/charset.inc diff --git a/mysql-test/columnstore/basic/t/mcs45_write_crossengine_join.test b/mysql-test/columnstore/basic/t/mcs45_write_crossengine_join.test index 2ebaadbc9..f3ab5ec94 100644 --- a/mysql-test/columnstore/basic/t/mcs45_write_crossengine_join.test +++ b/mysql-test/columnstore/basic/t/mcs45_write_crossengine_join.test @@ -2,6 +2,7 @@ # Update and Delete using Cross engine join # Author: Bharath, bharath.bokka@mariadb.com # +--source ../include/disable_11.4.inc -- source include/have_innodb.inc -- source ../include/have_columnstore.inc diff --git a/mysql-test/columnstore/bugfixes/mcol-4740.test b/mysql-test/columnstore/bugfixes/mcol-4740.test index 85075fd99..b950908cb 100644 --- a/mysql-test/columnstore/bugfixes/mcol-4740.test +++ b/mysql-test/columnstore/bugfixes/mcol-4740.test @@ -1,7 +1,7 @@ # # MCOL-4740 UPDATE involving multi-tables returns wrong "Rows matched" # - +--source ../include/disable_11.4.inc --source ../include/have_columnstore.inc --disable_warnings @@ -79,4 +79,4 @@ SET main.status = 1; SELECT status FROM test1; -DROP DATABASE mcol_4740; \ No newline at end of file +DROP DATABASE mcol_4740; diff --git a/mysql-test/columnstore/bugfixes/mcol-5480.test b/mysql-test/columnstore/bugfixes/mcol-5480.test index ab32e1503..37de6bb1c 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5480.test +++ b/mysql-test/columnstore/bugfixes/mcol-5480.test @@ -2,7 +2,7 @@ # MCOL-5480 LDI loads values incorrectly for MEDIUMINT, TIME and TIMESTAMP # when cpimport is used for batch insert # - +--source ../include/disable_11.4.inc --source ../include/have_columnstore.inc --source ../include/detect_maxscale.inc diff --git a/mysql-test/columnstore/bugfixes/mcol-5669.test b/mysql-test/columnstore/bugfixes/mcol-5669.test index 5420034d8..c40e987f7 100644 --- a/mysql-test/columnstore/bugfixes/mcol-5669.test +++ b/mysql-test/columnstore/bugfixes/mcol-5669.test @@ -1,3 +1,4 @@ +--source ../include/disable_11.4.inc --source include/have_innodb.inc --source ../include/have_columnstore.inc --disable_warnings diff --git a/mysql-test/columnstore/include/disable_11.4.inc b/mysql-test/columnstore/include/disable_11.4.inc new file mode 100644 index 000000000..308d65d5a --- /dev/null +++ b/mysql-test/columnstore/include/disable_11.4.inc @@ -0,0 +1,4 @@ +if (`SELECT (sys.version_major(), sys.version_minor()) != (10, 6)`) +{ + skip Temporary disabled due to incompatibility with 11.4; +} From e9400a1959e9b785b77707749b06718fda025772 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 30 Jun 2025 23:43:48 +0300 Subject: [PATCH 32/68] add link to JIRA for disable tests --- mysql-test/columnstore/include/disable_11.4.inc | 2 +- mysql-test/columnstore/include/only_10.6.inc | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 mysql-test/columnstore/include/only_10.6.inc diff --git a/mysql-test/columnstore/include/disable_11.4.inc b/mysql-test/columnstore/include/disable_11.4.inc index 308d65d5a..3e225b452 100644 --- a/mysql-test/columnstore/include/disable_11.4.inc +++ b/mysql-test/columnstore/include/disable_11.4.inc @@ -1,4 +1,4 @@ if (`SELECT (sys.version_major(), sys.version_minor()) != (10, 6)`) { - skip Temporary disabled due to incompatibility with 11.4; + skip Temporary disabled due to incompatibility with 11.4, MCOL-5948; } diff --git a/mysql-test/columnstore/include/only_10.6.inc b/mysql-test/columnstore/include/only_10.6.inc new file mode 100644 index 000000000..931e97f8c --- /dev/null +++ b/mysql-test/columnstore/include/only_10.6.inc @@ -0,0 +1,4 @@ +if (`SELECT (sys.version_major(), sys.version_minor()) != (10, 6)`) +{ + skip This test works only with 10.6; +} From dd4cfebaf482e055c7a01ec994606b81ac779bf7 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Thu, 3 Jul 2025 01:26:12 +0300 Subject: [PATCH 33/68] change skip message for mcs114_comments test --- mysql-test/columnstore/basic/t/mcs114_comments.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/columnstore/basic/t/mcs114_comments.test b/mysql-test/columnstore/basic/t/mcs114_comments.test index 66bc469c2..8e4e1fe9c 100644 --- a/mysql-test/columnstore/basic/t/mcs114_comments.test +++ b/mysql-test/columnstore/basic/t/mcs114_comments.test @@ -2,7 +2,7 @@ # Test comments # Author: Bharath, bharath.bokka@mariadb.com # ---source ../include/disable_11.4.inc +--source ../include/only_10.6.inc -- source ../include/have_columnstore.inc --disable_warnings From 6eeb948ad283965ff39eddd770a036ab7596d228 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Tue, 24 Jun 2025 18:21:18 +0000 Subject: [PATCH 34/68] MCOL-6069: update thrift for gcc-15 --- cmake/thrift.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/thrift.cmake b/cmake/thrift.cmake index 11e1533d6..8a8088676 100644 --- a/cmake/thrift.cmake +++ b/cmake/thrift.cmake @@ -25,8 +25,8 @@ set(THRIFT_LIBRARY ${THRIFT_LIBRARY_DIRS}/${CMAKE_STATIC_LIBRARY_PREFIX}thrift${ ExternalProject_Add( external_thrift - URL https://github.com/apache/thrift/archive/refs/tags/v0.17.0.tar.gz - URL_HASH SHA256=f5888bcd3b8de40c2c2ab86896867ad9b18510deb412cba3e5da76fb4c604c29 + URL https://github.com/apache/thrift/archive/refs/tags/v0.22.0.tar.gz + URL_HASH SHA256=c4649c5879dd56c88f1e7a1c03e0fbfcc3b2a2872fb81616bffba5aa8a225a37 PREFIX ${INSTALL_LOCATION} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_LOCATION} -DBUILD_COMPILER=YES From 4a79f8890f7fd6173315b306ad12449aea8ae309 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 4 Jul 2025 13:46:07 +0300 Subject: [PATCH 35/68] add funtions loading to all MTRs --- .../1pmonly/t/mcs4009_autopilot_partition_management.test | 2 ++ .../autopilot/t/mcs6197_windowFunctions_REGR_AVGX.test | 2 ++ .../autopilot/t/mcs6198_windowFunctions_REGR_AVGY.test | 2 ++ .../autopilot/t/mcs6199_windowFunctions_REGR_COUNT.test | 2 ++ .../devregression/t/mcs7064_regression_bug3283.test | 3 ++- .../devregression/t/mcs7139_regression_bug4074.test | 2 ++ .../devregression/t/mcs7224_regression_MCOL-3503.test | 3 ++- .../devregression/t/mcs7257_regression_mcol1642.test | 3 ++- 8 files changed, 16 insertions(+), 3 deletions(-) diff --git a/mysql-test/columnstore/1pmonly/t/mcs4009_autopilot_partition_management.test b/mysql-test/columnstore/1pmonly/t/mcs4009_autopilot_partition_management.test index 074cd1e1b..2863ef856 100755 --- a/mysql-test/columnstore/1pmonly/t/mcs4009_autopilot_partition_management.test +++ b/mysql-test/columnstore/1pmonly/t/mcs4009_autopilot_partition_management.test @@ -5,6 +5,7 @@ # -------------------------------------------------------------- # --source ../include/have_columnstore.inc --source ../include/detect_maxscale.inc +--source ../include/functions.inc # --disable_warnings DROP DATABASE IF EXISTS mcs4009_db; @@ -60,3 +61,4 @@ select calshowpartitions('lineitem','l_orderkey'); # -------------------------------------------------------------- # # DROP DATABASE mcs4009_db; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/autopilot/t/mcs6197_windowFunctions_REGR_AVGX.test b/mysql-test/columnstore/autopilot/t/mcs6197_windowFunctions_REGR_AVGX.test index f7f52f9ec..3706fe973 100644 --- a/mysql-test/columnstore/autopilot/t/mcs6197_windowFunctions_REGR_AVGX.test +++ b/mysql-test/columnstore/autopilot/t/mcs6197_windowFunctions_REGR_AVGX.test @@ -5,6 +5,7 @@ # -------------------------------------------------------------- # # --source ../include/have_columnstore.inc +--source ../include/functions.inc # use tpch1m; # @@ -256,3 +257,4 @@ select o_custkey, REGR_AVGX(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custk select o_custkey, REGR_AVGX(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custkey)+2 ORDER BY o_custkey ,o_orderkey ,o_orderdate ROWS BETWEEN 15 PRECEDING AND 15 PRECEDING) from (select * from orders where o_custkey <= 20000) s order by 1, 2; select o_custkey, REGR_AVGX(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custkey)+2 ORDER BY o_custkey ,o_orderkey ,o_orderdate ROWS BETWEEN 15 PRECEDING AND 15 FOLLOWING) from (select * from orders where o_custkey <= 20000) s order by 1, 2; select o_custkey, REGR_AVGX(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custkey)+2 ORDER BY o_custkey ,o_orderkey ,o_orderdate ROWS BETWEEN 15 FOLLOWING AND 15 FOLLOWING) from (select * from orders where o_custkey <= 20000) s order by 1, 2; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/autopilot/t/mcs6198_windowFunctions_REGR_AVGY.test b/mysql-test/columnstore/autopilot/t/mcs6198_windowFunctions_REGR_AVGY.test index 74173983f..aaa216a3c 100644 --- a/mysql-test/columnstore/autopilot/t/mcs6198_windowFunctions_REGR_AVGY.test +++ b/mysql-test/columnstore/autopilot/t/mcs6198_windowFunctions_REGR_AVGY.test @@ -5,6 +5,7 @@ # -------------------------------------------------------------- # # --source ../include/have_columnstore.inc +--source ../include/functions.inc # use tpch1m; # @@ -256,3 +257,4 @@ select o_custkey, REGR_AVGY(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custk select o_custkey, REGR_AVGY(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custkey)+2 ORDER BY o_custkey ,o_orderkey ,o_orderdate ROWS BETWEEN 15 PRECEDING AND 15 PRECEDING) from (select * from orders where o_custkey <= 20000) s order by 1, 2; select o_custkey, REGR_AVGY(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custkey)+2 ORDER BY o_custkey ,o_orderkey ,o_orderdate ROWS BETWEEN 15 PRECEDING AND 15 FOLLOWING) from (select * from orders where o_custkey <= 20000) s order by 1, 2; select o_custkey, REGR_AVGY(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custkey)+2 ORDER BY o_custkey ,o_orderkey ,o_orderdate ROWS BETWEEN 15 FOLLOWING AND 15 FOLLOWING) from (select * from orders where o_custkey <= 20000) s order by 1, 2; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/autopilot/t/mcs6199_windowFunctions_REGR_COUNT.test b/mysql-test/columnstore/autopilot/t/mcs6199_windowFunctions_REGR_COUNT.test index 7c36d2496..e2da024ed 100644 --- a/mysql-test/columnstore/autopilot/t/mcs6199_windowFunctions_REGR_COUNT.test +++ b/mysql-test/columnstore/autopilot/t/mcs6199_windowFunctions_REGR_COUNT.test @@ -5,6 +5,7 @@ # -------------------------------------------------------------- # # --source ../include/have_columnstore.inc +--source ../include/functions.inc # use tpch1m; # @@ -256,3 +257,4 @@ select o_custkey, REGR_COUNT(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_cust select o_custkey, REGR_COUNT(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custkey)+2 ORDER BY o_custkey ,o_orderkey ,o_orderdate ROWS BETWEEN 15 PRECEDING AND 15 PRECEDING) from (select * from orders where o_custkey <= 20000) s order by 1, 2; select o_custkey, REGR_COUNT(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custkey)+2 ORDER BY o_custkey ,o_orderkey ,o_orderdate ROWS BETWEEN 15 PRECEDING AND 15 FOLLOWING) from (select * from orders where o_custkey <= 20000) s order by 1, 2; select o_custkey, REGR_COUNT(o_custkey,o_orderkey) OVER (PARTITION BY abs(o_custkey)+2 ORDER BY o_custkey ,o_orderkey ,o_orderdate ROWS BETWEEN 15 FOLLOWING AND 15 FOLLOWING) from (select * from orders where o_custkey <= 20000) s order by 1, 2; +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/devregression/t/mcs7064_regression_bug3283.test b/mysql-test/columnstore/devregression/t/mcs7064_regression_bug3283.test index de8dcb488..f5197a3d4 100644 --- a/mysql-test/columnstore/devregression/t/mcs7064_regression_bug3283.test +++ b/mysql-test/columnstore/devregression/t/mcs7064_regression_bug3283.test @@ -5,12 +5,13 @@ # -------------------------------------------------------------- # # --source ../include/have_columnstore.inc +--source ../include/functions.inc # USE tpch1; # # Negative test case for attempting to drop a non-existent partition. select caldroppartitions('lineitem', '4.1.1'); select caldisablepartitions('lineitem', '4.1.1'); - +--source ../include/drop_functions.inc # diff --git a/mysql-test/columnstore/devregression/t/mcs7139_regression_bug4074.test b/mysql-test/columnstore/devregression/t/mcs7139_regression_bug4074.test index 817d6637a..89fb8b5a1 100644 --- a/mysql-test/columnstore/devregression/t/mcs7139_regression_bug4074.test +++ b/mysql-test/columnstore/devregression/t/mcs7139_regression_bug4074.test @@ -6,6 +6,7 @@ # --source ../include/have_columnstore.inc --source ../include/detect_maxscale.inc +--source ../include/functions.inc # USE tpch1; # @@ -36,4 +37,5 @@ select 'q6', birthdate, age, id, col8 from bug3657 where col8=2; --disable_warnings drop table if exists bug3657; --enable_warnings +--source ../include/drop_functions.inc # diff --git a/mysql-test/columnstore/devregression/t/mcs7224_regression_MCOL-3503.test b/mysql-test/columnstore/devregression/t/mcs7224_regression_MCOL-3503.test index 3d31bbfd6..6f5cba866 100644 --- a/mysql-test/columnstore/devregression/t/mcs7224_regression_MCOL-3503.test +++ b/mysql-test/columnstore/devregression/t/mcs7224_regression_MCOL-3503.test @@ -5,10 +5,11 @@ # -------------------------------------------------------------- # # --source ../include/have_columnstore.inc +--source ../include/functions.inc # USE tpch1; # select depname, empno, moda(salary) over(partition by depname order by enroll_date) from empsalary order by depname, empno, enroll_date; select avg(salary),depname, moda(salary) over(partition by depname order by enroll_date) from empsalary group by depname order by depname, avg(salary); # - +--source ../include/drop_functions.inc diff --git a/mysql-test/columnstore/devregression/t/mcs7257_regression_mcol1642.test b/mysql-test/columnstore/devregression/t/mcs7257_regression_mcol1642.test index 43ac8bb06..88c60f04f 100644 --- a/mysql-test/columnstore/devregression/t/mcs7257_regression_mcol1642.test +++ b/mysql-test/columnstore/devregression/t/mcs7257_regression_mcol1642.test @@ -5,9 +5,10 @@ # -------------------------------------------------------------- # # --source ../include/have_columnstore.inc +--source ../include/functions.inc # USE tpch1; # select mcsSystemPrimary() as a; # - +--source ../include/drop_functions.inc From c9ba6732e1ac324f02fac5a9d85ef693363901af Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Fri, 4 Jul 2025 14:06:42 +0000 Subject: [PATCH 36/68] bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c6111e479..91ed5975d 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=23 COLUMNSTORE_VERSION_MINOR=10 -COLUMNSTORE_VERSION_PATCH=4 +COLUMNSTORE_VERSION_PATCH=5 COLUMNSTORE_VERSION_RELEASE=1 From 3083eb6d9579a87cc4656caf3967ef19310333db Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Thu, 19 Jun 2025 13:24:37 +0300 Subject: [PATCH 37/68] add args to max file system disk usage error --- writeengine/shared/we_define.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/writeengine/shared/we_define.cpp b/writeengine/shared/we_define.cpp index 382b49456..b6f452f96 100644 --- a/writeengine/shared/we_define.cpp +++ b/writeengine/shared/we_define.cpp @@ -284,8 +284,7 @@ std::string WErrorCodes::errorString(int code) case ERR_FILE_DISK_SPACE: { logging::Message::Args args; - std::string msgArg; // empty str arg; no extra info in this context - args.add(msgArg); + args.add("configured by 'WriteEngine' . 'MaxFileSystemDiskUsagePct' in columnstore.xml"); return logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_EXTENT_DISK_SPACE, args); break; } From 52351f2f29d8759c4551125da1287de34d7ada02 Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Thu, 19 Jun 2025 13:26:24 +0300 Subject: [PATCH 38/68] add args to try larger read buffer error --- writeengine/shared/we_define.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/shared/we_define.cpp b/writeengine/shared/we_define.cpp index b6f452f96..24077cf18 100644 --- a/writeengine/shared/we_define.cpp +++ b/writeengine/shared/we_define.cpp @@ -154,7 +154,7 @@ WErrorCodes::WErrorCodes() : fErrorCodes() fErrorCodes[ERR_BULK_SEND_MSG_ERR] = " in a bulk load send msg"; fErrorCodes[ERR_BULK_MISSING_EXTENT_ENTRY] = " missing Extent Entry when trying to save LBID info for CP"; fErrorCodes[ERR_BULK_MISSING_EXTENT_ROW] = " missing Extent Row when trying to save LBID info for CP"; - fErrorCodes[ERR_BULK_ROW_FILL_BUFFER] = " Single row fills read buffer; try larger read buffer."; + fErrorCodes[ERR_BULK_ROW_FILL_BUFFER] = " Single row fills read buffer; try larger read buffer via -c flag in cpimport"; fErrorCodes[ERR_BULK_DBROOT_CHANGE] = " Local PM DBRoot settings changed during bulk load."; fErrorCodes[ERR_BULK_ROLLBACK_MISS_ROOT] = " Mode3 automatic rollback not performed. DBRoot missing."; fErrorCodes[ERR_BULK_ROLLBACK_SEG_LIST] = " Error building segment file list in a directory."; From 5cb5ea1e0fbb94704639cb233f27ab9d1bbda58a Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Fri, 20 Jun 2025 14:15:23 +0300 Subject: [PATCH 39/68] add more description to Disk join error msg --- utils/joiner/joinpartition.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/joiner/joinpartition.cpp b/utils/joiner/joinpartition.cpp index dbf4521eb..be55ec706 100644 --- a/utils/joiner/joinpartition.cpp +++ b/utils/joiner/joinpartition.cpp @@ -821,7 +821,7 @@ uint64_t JoinPartition::writeByteStream(int which, ByteStream& bs) { fs.close(); ostringstream os; - os << "Disk join could not write file " << filename << ": " << strerror(saveErrno) << endl; + os << "Disk join could not write to configured path check 'HashJoin' in columnstore.xml " << filename << ": " << strerror(saveErrno) << endl; throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } @@ -845,7 +845,7 @@ uint64_t JoinPartition::writeByteStream(int which, ByteStream& bs) { fs.close(); ostringstream os; - os << "Disk join could not write file " << filename << ": " << strerror(saveErrno) << endl; + os << "Disk join could not write to configured path check 'HashJoin' in columnstore.xml " << filename << ": " << strerror(saveErrno) << endl; throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } From dc5bb881cf3fff59063627a1a45ac48f5a479882 Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Wed, 25 Jun 2025 16:32:13 +0300 Subject: [PATCH 40/68] fix DiskJoinStep error msg --- utils/joiner/joinpartition.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/joiner/joinpartition.cpp b/utils/joiner/joinpartition.cpp index be55ec706..d07a380cf 100644 --- a/utils/joiner/joinpartition.cpp +++ b/utils/joiner/joinpartition.cpp @@ -821,7 +821,7 @@ uint64_t JoinPartition::writeByteStream(int which, ByteStream& bs) { fs.close(); ostringstream os; - os << "Disk join could not write to configured path check 'HashJoin' in columnstore.xml " << filename << ": " << strerror(saveErrno) << endl; + os << "Disk join could not write to configured path SystemConfig.SystemTempFileDir in Columnstore.xml " << filename << ": " << strerror(saveErrno) << endl; throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } @@ -845,7 +845,7 @@ uint64_t JoinPartition::writeByteStream(int which, ByteStream& bs) { fs.close(); ostringstream os; - os << "Disk join could not write to configured path check 'HashJoin' in columnstore.xml " << filename << ": " << strerror(saveErrno) << endl; + os << "Disk join could not write to configured path SystemConfig.SystemTempFileDir in Columnstore.xml " << filename << ": " << strerror(saveErrno) << endl; throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } From dd3ce850b78468b31b4a3fd4d9a1444523a9f692 Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Thu, 26 Jun 2025 11:20:57 +0300 Subject: [PATCH 41/68] add filename to ExtentMap::save():open err log --- versioning/BRM/extentmap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index ac584b776..dab00c1cc 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -1826,7 +1826,7 @@ void ExtentMap::save(const string& filename) if (!out) { - log_errno("ExtentMap::save(): open"); + log_errno("ExtentMap::save(): can't open file " + filename); releaseFreeList(READ); releaseEMIndex(READ); releaseEMEntryTable(READ); From a29ac85c7c0d3a6004c7fc7148f2ebde0d069859 Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Fri, 27 Jun 2025 13:09:45 +0300 Subject: [PATCH 42/68] add partial support for unsupported query syntax --- dbcon/mysql/ha_mcs_ddl.cpp | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 32888b8a3..d270b75e5 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -761,7 +761,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta #ifdef MCS_DEBUG cout << "ProcessDDLStatement: " << schema << "." << table << ":" << ddlStatement << endl; #endif - + parser.setDefaultSchema(schema); parser.setDefaultCharset(default_table_charset); int rc = 0; @@ -2199,26 +2199,38 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta //@Bug 3602. Error message for MySql syntax for autoincrement algorithm::to_upper(ddlStatement); + std::string errMsg = "The syntax " if (ddlStatement.find("AUTO_INCREMENT") != string::npos) { - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, - "Use of the MySQL auto_increment syntax is not supported in Columnstore. If " - "you wish to create an auto increment column in Columnstore, please consult " - "the Columnstore SQL Syntax Guide for the correct usage."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; + errMsg += "auto_increment is not supported in Columnstore."; + } + else if(ddlStatement.find("RENAME COLUMN") != string::npos) + { + errMsg += "rename column is not supported by Columnstore."; + } + else if(ddlStatement.find("MAX_ROWS") != string::npos || ddlStatement.find("MIN_ROWS") != string::npos) + { + errMsg += "min_rows/max_rows is not supported by Columnstore."; + } + else if(ddlStatement.find("REPLACE TABLE") != string::npos) + { + errMsg += "replace table is not supported by Columnstore."; + } + else if(ddlStatement.find("DROP COLUMN IF EXISTS") != string::npos) + { + errMsg += "drop column if exists is not supported by Columnstore."; } else { //@Bug 1888,1885. update error message - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, - "The syntax or the data type(s) is not supported by Columnstore. Please check " - "the Columnstore syntax guide for supported syntax or data types."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; + errMsg += " or the data type(s) is not supported by Columnstore."; } + errMsg += " Please check the Columnstore syntax guide for supported syntax or data types."; + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, errMsg); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; } - + return rc; } From 40f4ee70085c8f04b41cc06f3daf9adf67c9ffbc Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Fri, 27 Jun 2025 14:35:41 +0300 Subject: [PATCH 43/68] add args to system catalog err --- dbcon/execplan/calpontsystemcatalog.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 685c66b37..4955e4e20 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -843,7 +843,8 @@ void CalpontSystemCatalog::getSysData(CalpontSelectExecutionPlan& csep, NJLSysDa if (tryCnt >= 5) // throw runtime_error("Error occurred when calling system catalog. ExeMgr is not functioning."); - throw IDBExcept(ERR_SYSTEM_CATALOG); + Message::Args& args = "Cannot connect to ExeMgr re-connections tries exceeded"; + throw IDBExcept(ERR_SYSTEM_CATALOG, args); } csep.sessionID(fSessionID); @@ -910,7 +911,9 @@ void CalpontSystemCatalog::getSysData_EC(CalpontSelectExecutionPlan& csep, NJLSy if (status >= 1000) // new error system throw IDBExcept(status); else - throw IDBExcept(ERR_SYSTEM_CATALOG); + Message::Args args; + args.add("rowGroup status: ", status); + throw IDBExcept(ERR_SYSTEM_CATALOG, args); } if (rowCount > 0) From 6831b82515e7bd368a1c8f4ac64afff9982cb57d Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Fri, 27 Jun 2025 14:36:37 +0300 Subject: [PATCH 44/68] fix err syntax - disk join --- writeengine/shared/we_define.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/shared/we_define.cpp b/writeengine/shared/we_define.cpp index 24077cf18..bc0aa2265 100644 --- a/writeengine/shared/we_define.cpp +++ b/writeengine/shared/we_define.cpp @@ -284,7 +284,7 @@ std::string WErrorCodes::errorString(int code) case ERR_FILE_DISK_SPACE: { logging::Message::Args args; - args.add("configured by 'WriteEngine' . 'MaxFileSystemDiskUsagePct' in columnstore.xml"); + args.add("configured by WriteEngine.MaxFileSystemDiskUsagePct in columnstore.xml"); return logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_EXTENT_DISK_SPACE, args); break; } From d0312398446164560a33b0a6aa8907f44e3536b7 Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Mon, 30 Jun 2025 11:39:40 +0300 Subject: [PATCH 45/68] add args to lost connection error when getting system data --- dbcon/execplan/calpontsystemcatalog.cpp | 22 +++++++++++++++------- dbcon/mysql/ha_mcs_ddl.cpp | 2 +- writeengine/shared/we_define.cpp | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 4955e4e20..316ee10ef 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -841,10 +841,12 @@ void CalpontSystemCatalog::getSysData(CalpontSelectExecutionPlan& csep, NJLSysDa } } - if (tryCnt >= 5) + if (tryCnt >= 5){ // throw runtime_error("Error occurred when calling system catalog. ExeMgr is not functioning."); - Message::Args& args = "Cannot connect to ExeMgr re-connections tries exceeded"; + Message::Args args; + args.add("Cannot connect to ExeMgr re-connections tries exceeded"); throw IDBExcept(ERR_SYSTEM_CATALOG, args); + } } csep.sessionID(fSessionID); @@ -910,10 +912,11 @@ void CalpontSystemCatalog::getSysData_EC(CalpontSelectExecutionPlan& csep, NJLSy { if (status >= 1000) // new error system throw IDBExcept(status); - else + else{ Message::Args args; - args.add("rowGroup status: ", status); + args.add("rowGroup status: " + status); throw IDBExcept(ERR_SYSTEM_CATALOG, args); + } } if (rowCount > 0) @@ -962,7 +965,9 @@ void CalpontSystemCatalog::getSysData_FE(const CalpontSelectExecutionPlan& csep, if (bs.length() == 0) { - throw IDBExcept(ERR_LOST_CONN_EXEMGR); + Message::Args args; + args.add("look in error.log on node, having dbroot1"); + throw IDBExcept(ERR_LOST_CONN_EXEMGR, args); } string emsgStr; @@ -993,8 +998,11 @@ void CalpontSystemCatalog::getSysData_FE(const CalpontSelectExecutionPlan& csep, // @bug 1782. check ExeMgr connection lost if (bs.length() == 0) - throw IDBExcept(ERR_LOST_CONN_EXEMGR); - + { + Message::Args args; + args.add("look in error.log on node, having dbroot1"); + throw IDBExcept(ERR_LOST_CONN_EXEMGR, args); + } if (!rowGroup) { rowGroup.reset(new RowGroup()); diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index d270b75e5..0c0bd468c 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -2199,7 +2199,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta //@Bug 3602. Error message for MySql syntax for autoincrement algorithm::to_upper(ddlStatement); - std::string errMsg = "The syntax " + std::string errMsg = "The syntax "; if (ddlStatement.find("AUTO_INCREMENT") != string::npos) { errMsg += "auto_increment is not supported in Columnstore."; diff --git a/writeengine/shared/we_define.cpp b/writeengine/shared/we_define.cpp index bc0aa2265..49bfeac18 100644 --- a/writeengine/shared/we_define.cpp +++ b/writeengine/shared/we_define.cpp @@ -284,7 +284,7 @@ std::string WErrorCodes::errorString(int code) case ERR_FILE_DISK_SPACE: { logging::Message::Args args; - args.add("configured by WriteEngine.MaxFileSystemDiskUsagePct in columnstore.xml"); + args.add("configured by WriteEngine.MaxFileSystemDiskUsagePct in Columnstore.xml"); return logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_EXTENT_DISK_SPACE, args); break; } From a1a017a475b2526d5575d20f3c53102a38dd9eca Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Mon, 30 Jun 2025 17:00:05 +0300 Subject: [PATCH 46/68] fix err printing --- dbcon/mysql/ha_mcs_ddl.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 0c0bd468c..cbd8b7430 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -2199,34 +2199,32 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta //@Bug 3602. Error message for MySql syntax for autoincrement algorithm::to_upper(ddlStatement); - std::string errMsg = "The syntax "; if (ddlStatement.find("AUTO_INCREMENT") != string::npos) { - errMsg += "auto_increment is not supported in Columnstore."; + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The syntax auto_increment is not supported in Columnstore. Please check the Columnstore syntax guide for supported syntax or data types."); } else if(ddlStatement.find("RENAME COLUMN") != string::npos) { - errMsg += "rename column is not supported by Columnstore."; + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The syntax rename column is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types."); } else if(ddlStatement.find("MAX_ROWS") != string::npos || ddlStatement.find("MIN_ROWS") != string::npos) { - errMsg += "min_rows/max_rows is not supported by Columnstore."; + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The syntax min_rows/max_rows is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types."); } else if(ddlStatement.find("REPLACE TABLE") != string::npos) { - errMsg += "replace table is not supported by Columnstore."; + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The syntax replace table is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types."); } else if(ddlStatement.find("DROP COLUMN IF EXISTS") != string::npos) { - errMsg += "drop column if exists is not supported by Columnstore."; + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The syntax drop column if exists is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types."); } else { //@Bug 1888,1885. update error message - errMsg += " or the data type(s) is not supported by Columnstore."; + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types."); } - errMsg += " Please check the Columnstore syntax guide for supported syntax or data types."; - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, errMsg); + ci->alterTableState = cal_connection_info::NOT_ALTER; ci->isAlter = false; } From bf3db183c7aa7d69f9224e46d0ebca2f594dc3e0 Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Tue, 1 Jul 2025 13:42:31 +0300 Subject: [PATCH 47/68] add transaction ID in error msg --- dbcon/ddlpackageproc/droptableprocessor.cpp | 3 ++- dmlproc/dmlprocessor.cpp | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dbcon/ddlpackageproc/droptableprocessor.cpp b/dbcon/ddlpackageproc/droptableprocessor.cpp index faa28b721..cd259f586 100644 --- a/dbcon/ddlpackageproc/droptableprocessor.cpp +++ b/dbcon/ddlpackageproc/droptableprocessor.cpp @@ -425,7 +425,8 @@ DropTableProcessor::DDLResult DropTableProcessor::processPackageInternal(ddlpack << endl; Message::Args args; Message message(9); - args.add("Error in dropping table from systables."); + args.add(fTxnid.id); + args.add(" Error in dropping table from systables."); args.add(errorMsg); message.format(args); result.result = (ResultCode)rc; diff --git a/dmlproc/dmlprocessor.cpp b/dmlproc/dmlprocessor.cpp index 765f5f19f..e905095e5 100644 --- a/dmlproc/dmlprocessor.cpp +++ b/dmlproc/dmlprocessor.cpp @@ -1155,6 +1155,8 @@ void PackageHandler::run() logging::Message message(1); args.add("dmlprocessor.cpp PackageHandler::run() package type"); args.add((uint64_t)fPackageType); + args.add(" ,transaction ID: "); + args.add(fTxnid); args.add(e.what()); message.format(args); ml.logErrorMessage(message); From 1dc794d4db9e5ba327a2f155557138ea04906126 Mon Sep 17 00:00:00 2001 From: Aleksei Bukhalov Date: Wed, 2 Jul 2025 15:33:06 +0200 Subject: [PATCH 48/68] MCOL-5906 expected error updated --- mysql-test/columnstore/basic/r/mcol641-create.result | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/columnstore/basic/r/mcol641-create.result b/mysql-test/columnstore/basic/r/mcol641-create.result index 52e1ef005..7f3635325 100644 --- a/mysql-test/columnstore/basic/r/mcol641-create.result +++ b/mysql-test/columnstore/basic/r/mcol641-create.result @@ -45,7 +45,7 @@ ALTER TABLE cs3 ADD COLUMN d5 DECIMAL(20,18) UNSIGNED; Warnings: Warning 1618 ZEROFILL is ignored in ColumnStore ALTER TABLE cs1 RENAME COLUMN d7 TO d17; -ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. +ERROR 42000: The storage engine for the table doesn't support The syntax rename column is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. ALTER TABLE cs2 ADD COLUMN (d5 DECIMAL(38,5), d6 DECIMAL(35,15)); ERROR 42000: The storage engine for the table doesn't support Multiple actions in alter table statement is currently not supported by Columnstore. ALTER TABLE cs3 MODIFY d1 DECIMAL(38) SIGNED; From e8b7442afaf009c3f0a3d0c3b25ebc77cef4fa28 Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Fri, 4 Jul 2025 11:01:21 +0300 Subject: [PATCH 49/68] record new test results --- mysql-test/columnstore/basic/r/mcs87_alter_column.result | 2 +- .../basic/r/mcs8_create_table_with_constraints.result | 4 ++-- .../columnstore/basic/r/mcs9_create_table_negative.result | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mysql-test/columnstore/basic/r/mcs87_alter_column.result b/mysql-test/columnstore/basic/r/mcs87_alter_column.result index f13f73f82..96de5b655 100644 --- a/mysql-test/columnstore/basic/r/mcs87_alter_column.result +++ b/mysql-test/columnstore/basic/r/mcs87_alter_column.result @@ -20,7 +20,7 @@ ERROR HY000: Internal error: CAL0001: Alter table Failed: Changing the datatype ALTER TABLE t1 CHANGE COLUMN c2 c2new CHAR(1); ALTER TABLE t1 CHANGE COLUMN c2new c2 CHAR(1); ALTER TABLE t1 DROP COLUMN IF EXISTS c11; -ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. +ERROR 42000: The storage engine for the table doesn't support The syntax drop column if exists is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. ALTER TABLE t1 DROP COLUMN c11; ALTER TABLE t1 DROP COLUMN IF EXISTS c11; Warnings: diff --git a/mysql-test/columnstore/basic/r/mcs8_create_table_with_constraints.result b/mysql-test/columnstore/basic/r/mcs8_create_table_with_constraints.result index 73051249f..a685e672d 100644 --- a/mysql-test/columnstore/basic/r/mcs8_create_table_with_constraints.result +++ b/mysql-test/columnstore/basic/r/mcs8_create_table_with_constraints.result @@ -25,7 +25,7 @@ t3 CREATE TABLE `t3` ( CREATE TABLE t4(col1 INT PRIMARY KEY AUTO_INCREMENT)ENGINE=Columnstore; ERROR 42000: Too many keys specified; max 0 keys allowed CREATE TABLE t4(col1 INT )MAX_ROWS=10, ENGINE=Columnstore; -ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. +ERROR 42000: The storage engine for the table doesn't support The syntax min_rows/max_rows is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. CREATE TABLE t5(col1 INT )MIN_ROWS=10, ENGINE=Columnstore; -ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. +ERROR 42000: The storage engine for the table doesn't support The syntax min_rows/max_rows is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. DROP DATABASE mcs8_db; diff --git a/mysql-test/columnstore/basic/r/mcs9_create_table_negative.result b/mysql-test/columnstore/basic/r/mcs9_create_table_negative.result index b332d2d3a..ee1fe7a3e 100644 --- a/mysql-test/columnstore/basic/r/mcs9_create_table_negative.result +++ b/mysql-test/columnstore/basic/r/mcs9_create_table_negative.result @@ -20,7 +20,7 @@ ERROR 42000: Too many keys specified; max 0 keys allowed CREATE TABLE t7(col1 INT UNIQUE)ENGINE=Columnstore; ERROR 42000: Too many keys specified; max 0 keys allowed CREATE TABLE t8(col1 INT)MIN_ROWS=10, ENGINE=Columnstore; -ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. +ERROR 42000: The storage engine for the table doesn't support The syntax min_rows/max_rows is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. CREATE TABLE t9(col1 INT)MAX_ROWS=1000, ENGINE=Columnstore; -ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. +ERROR 42000: The storage engine for the table doesn't support The syntax min_rows/max_rows is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. DROP DATABASE mcs9_db; From 7aac943ffeb54afc05f13823fdee1841a3956837 Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Fri, 4 Jul 2025 13:37:23 +0300 Subject: [PATCH 50/68] record test result --- mysql-test/columnstore/basic/r/mcs116_create_or_replace.result | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/columnstore/basic/r/mcs116_create_or_replace.result b/mysql-test/columnstore/basic/r/mcs116_create_or_replace.result index 56d4eac9d..34d3bd1e1 100644 --- a/mysql-test/columnstore/basic/r/mcs116_create_or_replace.result +++ b/mysql-test/columnstore/basic/r/mcs116_create_or_replace.result @@ -4,5 +4,5 @@ USE mcs116_db; CREATE OR REPLACE TABLE IF NOT EXISTS t1 (a INT); ERROR HY000: Incorrect usage of OR REPLACE and IF NOT EXISTS CREATE OR REPLACE TABLE t1 (a INT)ENGINE=Columnstore; -ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. +ERROR 42000: The storage engine for the table doesn't support The syntax replace table is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. DROP DATABASE mcs116_db; From a501ef87218654c30242e63ba857810c5292463b Mon Sep 17 00:00:00 2001 From: mariadb-KristinaPavlova Date: Fri, 4 Jul 2025 15:03:00 +0300 Subject: [PATCH 51/68] fix string args --- dbcon/execplan/calpontsystemcatalog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 316ee10ef..118fe1f6e 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -914,7 +914,7 @@ void CalpontSystemCatalog::getSysData_EC(CalpontSelectExecutionPlan& csep, NJLSy throw IDBExcept(status); else{ Message::Args args; - args.add("rowGroup status: " + status); + args.add("rowGroup status: " + std::to_string(status)); throw IDBExcept(ERR_SYSTEM_CATALOG, args); } } From 073f7abe6b3c28861d1896237aa4a84ef5f4f615 Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Thu, 26 Jun 2025 17:15:39 +0200 Subject: [PATCH 52/68] chore(ci) MCOL-6071 smoke and mtr reporting in bash --- .drone.jsonnet | 78 +++++++++----------------------------- build/report_test_stage.sh | 28 ++++++++------ build/run_mtr.sh | 16 ++++++-- build/run_regression.sh | 6 +++ build/run_smoke.sh | 21 +++++++--- 5 files changed, 69 insertions(+), 80 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 8cf14957c..bdaf4990c 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -56,10 +56,6 @@ local customBootstrapParamsForAdditionalPipelinesMap = { local any_branch = "**"; -local platforms_custom = platforms[current_branch]; -local platforms_arm_custom = platforms_arm[current_branch]; - -local platforms_mtr = platforms[current_branch]; local builddir = "verylongdirnameforverystrangecpackbehavior"; @@ -95,18 +91,6 @@ local upgrade_test_lists = { }, }; -local testRun(platform) = - local platform_map = { - "rockylinux:8": "ctest3 -R columnstore: -j $(nproc) --output-on-failure", - "rockylinux:9": "ctest3 -R columnstore: -j $(nproc) --output-on-failure", - "debian:12": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", - "ubuntu:20.04": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", - "ubuntu:22.04": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", - "ubuntu:24.04": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", - - }; - platform_map[platform]; - local gcc_version = "11"; local rockylinux8_deps = "dnf install -y 'dnf-command(config-manager)' " + @@ -139,15 +123,12 @@ local echo_running_on = ["echo running on ${DRONE_STAGE_MACHINE}", local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", customBootstrapParams="", customBuildEnvCommandsMapKey="") = { local pkg_format = if (std.split(platform, ":")[0] == "rockylinux") then "rpm" else "deb", - local init = if (pkg_format == "rpm") then "/usr/lib/systemd/systemd" else "systemd", local mtr_path = if (pkg_format == "rpm") then "/usr/share/mysql-test" else "/usr/share/mysql/mysql-test", local cmapi_path = "/usr/share/columnstore/cmapi", local etc_path = "/etc/columnstore", local socket_path = if (pkg_format == "rpm") then "/var/lib/mysql/mysql.sock" else "/run/mysqld/mysqld.sock", - local config_path_prefix = if (pkg_format == "rpm") then "/etc/my.cnf.d/" else "/etc/mysql/mariadb.conf.d/50-", local img = if (platform == "rockylinux:8") then platform else "detravi/" + std.strReplace(platform, "/", "-"), local branch_ref = if (branch == any_branch) then current_branch else branch, - // local regression_tests = if (std.startsWith(platform, 'debian') || std.startsWith(platform, 'ubuntu:20')) then 'test000.sh' else 'test000.sh,test001.sh', local branchp = if (branch == "**") then "" else branch + "/", local brancht = if (branch == "**") then "" else branch + "-", @@ -256,7 +237,12 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", " --packages-url " + packages_url + " --do-setup " + std.toString(do_setup) + '"', - local reportTestStage(containerName, result, stage) = 'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/report_test_stage.sh " + containerName + " " + result + " " + stage + '"', + local reportTestStage(containerName, result, stage) = + 'sh -c "apk add bash && bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/report_test_stage.sh' + + ' --container-name ' + containerName + + ' --result-path ' + result + + ' --stage ' + stage + '"', + _volumes:: { mdb: { @@ -275,21 +261,11 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", volumes: [pipeline._volumes.mdb, pipeline._volumes.docker], commands: [ prepareTestContainer(getContainerName("smoke"), result, true), - "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_smoke.sh " + getContainerName("smoke"), + "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_smoke.sh" + + ' --container-name ' + getContainerName("smoke") + + ' --result-path ' + result, ], }, - smokelog:: { - name: "smokelog", - depends_on: ["smoke"], - image: "docker:28.2.2", - volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], - commands: [ - reportTestStage(getContainerName("smoke"), result, "smoke"), - ], - when: { - status: ["success", "failure"], - }, - }, upgrade(version):: { name: "upgrade-test from " + version, depends_on: ["regressionlog"], @@ -301,7 +277,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", }, }, commands: [ - // why do we mount cgroups here, but miss it on other steps? prepareTestContainer(getContainerName("upgrade") + version, result, false), if (pkg_format == "deb") then execInnerDocker('bash -c "./upgrade_setup_deb.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"', @@ -348,21 +323,10 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", ' --container-name ' + getContainerName("mtr") + ' --distro ' + platform + ' --suite-list $${MTR_SUITE_LIST}' + + ' --result-path ' + result + ' --triggering-event ' + event, ], }, - mtrlog:: { - name: "mtrlog", - depends_on: ["mtr"], - image: "docker:28.2.2", - volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], - commands: [ - reportTestStage(getContainerName("mtr"), result, "mtr"), - ], - when: { - status: ["success", "failure"], - }, - }, regression(name, depends_on):: { name: name, depends_on: depends_on, @@ -411,7 +375,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", dockerfile:: { name: "dockerfile", depends_on: ["publish pkg", "publish cmapi build"], - //failure: 'ignore', image: "alpine/git:2.49.0", environment: { DOCKER_BRANCH_REF: "${DRONE_SOURCE_BRANCH}", @@ -498,7 +461,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", multi_node_mtr:: { name: "mtr", depends_on: ["dockerhub"], - //failure: 'ignore', image: "docker:28.2.2", volumes: [pipeline._volumes.docker], environment: { @@ -531,7 +493,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", type: "docker", name: std.join(" ", [branch, platform, event, arch, server, customBootstrapParams, customBuildEnvCommandsMapKey]), platform: { arch: arch }, - // [if arch == 'arm64' then 'node']: { arch: 'arm64' }, clone: { depth: 10 }, steps: [ { @@ -585,8 +546,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", SCCACHE_REGION: "us-east-1", SCCACHE_S3_USE_SSL: "true", SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + "${DRONE_PULL_REQUEST}", - //SCCACHE_ERROR_LOG: '/tmp/sccache_log.txt', - //SCCACHE_LOG: 'debug', }, commands: [ "mkdir /mdb/" + builddir + "/" + result, @@ -640,7 +599,10 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", commands: [ "cd /mdb/" + builddir, testPreparation(platform), - testRun(platform), + + if (platform == "rockylinux:8" || platform == "rockylinux:9") + then "ctest3 -R columnstore: -j $(nproc) --output-on-failure" + else "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", ], }, { @@ -676,12 +638,11 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", [pipeline.publish()] + (if (event == "cron") then [pipeline.publish("pkg latest", "latest")] else []) + [pipeline.smoke] + - [pipeline.smokelog] + - [pipeline.publish("smokelog")] + + [pipeline.publish("smoke")] + [pipeline.cmapitest] + [pipeline.cmapilog] + [pipeline.publish("cmapilog")] + - (if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish("mtrlog")]) + + (if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.publish("mtr")]) + [pipeline.regression(regression_tests[i], if (i == 0) then ["mtr", "publish pkg", "publish cmapi build"] else [regression_tests[i - 1]]) for i in indexes(regression_tests)] + [pipeline.regressionlog] + [pipeline.publish("regressionlog")] + @@ -694,9 +655,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", event: [event], branch: [branch], }, - // + (if event == 'cron' then { - // cron: ['nightly-' + std.strReplace(branch, '.', '-')], - // } else {}), }; local FinalPipeline(branch, event) = { @@ -749,11 +707,11 @@ local FinalPipeline(branch, event) = { [ Pipeline(any_branch, p, "custom", "amd64", "10.6-enterprise") - for p in platforms_custom + for p in platforms[current_branch] ] + // [ // Pipeline(any_branch, p, "custom", "arm64", "10.6-enterprise") -// for p in platforms_arm_custom +// for p in platforms_arm[current_branch]; // ] // + [ diff --git a/build/report_test_stage.sh b/build/report_test_stage.sh index 23546a382..1be7cf140 100755 --- a/build/report_test_stage.sh +++ b/build/report_test_stage.sh @@ -2,15 +2,27 @@ set -eo pipefail -CONTAINER_NAME=$1 -RESULT=$2 -STAGE=$3 - SCRIPT_LOCATION=$(dirname "$0") source "$SCRIPT_LOCATION"/utils.sh +optparse.define short=c long=container-name desc="Name of the Docker container where mtr tests will run" variable=CONTAINER_NAME +optparse.define short=r long=result-path desc="Path for logs and results" variable=RESULT +optparse.define short=s long=stage desc="Test stage name" variable=STAGE +source $(optparse.build) + echo "Arguments received: $@" + +cleanup() { + if [[ -n $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then + echo "Cleaning up container ${CONTAINER_NAME}..." + docker rm -f "${CONTAINER_NAME}" || echo "Can't remove container ${CONTAINER_NAME}!" + fi +} +#Remove the container on exit +trap cleanup EXIT + + if [[ "$EUID" -ne 0 ]]; then error "Please run script as root" exit 1 @@ -86,11 +98,3 @@ echo "Saved artifacts:" ls -R "/drone/src/${RESULT}/" echo "Done reporting ${STAGE}" -cleanup() { - if [[ -n $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then - echo "Cleaning up container ${CONTAINER_NAME}..." - docker rm -f "${CONTAINER_NAME}" || echo "Can't remove container ${CONTAINER_NAME}!" - fi -} -#Remove the container on exit -trap cleanup EXIT diff --git a/build/run_mtr.sh b/build/run_mtr.sh index 273fa1720..dd7271325 100755 --- a/build/run_mtr.sh +++ b/build/run_mtr.sh @@ -9,6 +9,7 @@ optparse.define short=c long=container-name desc="Name of the Docker container w optparse.define short=d long=distro desc="Linux distro for which mtr is runned" variable=DISTRO optparse.define short=s long=suite-list desc="Comma-separated list of test suites to run" variable=MTR_SUITE_LIST optparse.define short=e long=triggering-event desc="Event that triggers testrun" variable=EVENT +optparse.define short=r long=result-path desc="Path for logs and results" variable=RESULT source $(optparse.build) MTR_FULL_SET="basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly" @@ -20,16 +21,25 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -if [[ -z "${CONTAINER_NAME}" ]]; then - echo "Please provide mtr container name as a parameter, e.g. ./run_mtr.sh -c mtr183" +for flag in CONTAINER_NAME DISTRO EVENT MTR_SUITE_LIST RESULT; do + if [[ -z "${!flag}" ]]; then + error "Missing required flag: -${flag:0:1} / --${flag,,}" exit 1 -fi + fi +done if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then error "Container '${CONTAINER_NAME}' is not running." exit 1 fi +#Collect the logs on exit event +collect_logs() { + "${SCRIPT_LOCATION}/report_test_stage.sh" --container-name "${CONTAINER_NAME}" --result-path "${RESULT}" --stage "mtr" +} +trap collect_logs EXIT + + if [[ "$DISTRO" == *rocky* ]]; then SOCKET_PATH="/var/lib/mysql/mysql.sock" MTR_PATH="/usr/share/mysql-test" diff --git a/build/run_regression.sh b/build/run_regression.sh index 83b41fb02..e2bd3324a 100755 --- a/build/run_regression.sh +++ b/build/run_regression.sh @@ -24,6 +24,12 @@ if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then exit 1 fi +##Collect the logs on exit event +#collect_logs(){ +# "${SCRIPT_LOCATION}/report_test_stage.sh" --container-name "${CONTAINER_NAME}" --result-path "${RESULT}" --stage "regression" +#} +#trap collect_logs EXIT + BUILD_DIR="verylongdirnameforverystrangecpackbehavior" prepare_regression() { diff --git a/build/run_smoke.sh b/build/run_smoke.sh index c94164662..de6096987 100755 --- a/build/run_smoke.sh +++ b/build/run_smoke.sh @@ -5,20 +5,30 @@ set -eo pipefail SCRIPT_LOCATION=$(dirname "$0") source "$SCRIPT_LOCATION"/utils.sh -CONTAINER_NAME="$1" +optparse.define short=c long=container-name desc="Name of the Docker container where mtr tests will run" variable=CONTAINER_NAME +optparse.define short=r long=result-path desc="Path for logs and results" variable=RESULT +source $(optparse.build) echo "Arguments received: $@" -if [[ "$EUID" -ne 0 ]]; then - error "Please run script as root" - exit 1 -fi +for flag in CONTAINER_NAME RESULT; do + if [[ -z "${!flag}" ]]; then + error "Missing required flag: -${flag:0:1} / --${flag,,}" + exit 1 + fi +done if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then error "Container '${CONTAINER_NAME}' is not running." exit 1 fi +#Collect the logs on exit event +collect_logs(){ + "${SCRIPT_LOCATION}/report_test_stage.sh" --container-name "${CONTAINER_NAME}" --result-path "${RESULT}" --stage "smoke" +} +trap collect_logs EXIT + message "Running smoke checks..." # start mariadb and mariadb-columnstore services and run simple query @@ -38,3 +48,4 @@ execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb-columnstore' sleep 10 execInnerDocker "$CONTAINER_NAME" 'mariadb -e "INSERT INTO test.t1 VALUES (2); SELECT * FROM test.t1;"' + From a27f30c73e7ce72b36d9b2024bb2d3d939ad31f7 Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Fri, 27 Jun 2025 19:10:30 +0200 Subject: [PATCH 53/68] chore(ci) MCOL-6071 sccache installing to bootstrap_mcs.sh --- .drone.jsonnet | 5 ++--- build/bootstrap_mcs.sh | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index bdaf4990c..f733a3aa8 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -556,12 +556,11 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", 'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " + "--build-type RelWithDebInfo " + "--distro " + platform + " " + - "--build-packages --install-deps --sccache " + + "--build-packages --install-deps --sccache --sccache-arch " + arch + " " + customBootstrapParams + " " + customBootstrapParamsForExisitingPipelines(platform) + " | " + "/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " + - "/mdb/" + builddir + "/" + result + '/build.log"', - "sccache --show-stats", + "/mdb/" + builddir + "/" + result + '/build.log "' ], }, { diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index a82992c16..683828821 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -53,7 +53,6 @@ optparse.define short=O long=static desc="Build all with static libraries" varia optparse.define short=p long=build-packages desc="Build packages" variable=BUILD_PACKAGES default=false value=true optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core" optparse.define short=r long=restart-services variable=RESTART_SERVICES default=true value=false -optparse.define short=s long=sccache desc="Build with sccache" variable=SCCACHE default=false value=true optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true @@ -62,6 +61,7 @@ optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN defaul optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true optparse.define short=V long=add-branch-name-to-outdir desc="Add branch name to build output directory" variable=BRANCH_NAME_TO_OUTDIR default=false value=true optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true +optparse.define short=s long=sccache desc="Build with sccache" variable=SCCACHE default=false value=true source $(optparse.build) @@ -82,6 +82,36 @@ if [[ "$OS" == *"rocky"* ]]; then pkg_format="rpm" fi +install_sccache() { + if [[ "$SCCACHE" == false ]]; then + return + fi + + if [[ "$(arch)" == "x86_64" ]]; then + sccache_arch="x86_64" + else + sccache_arch="aarch64" + fi + + message "getting sccache..." + + rewrite_ubuntu_mirror #TODO + + if command -v apt-get &>/dev/null; then + apt-get clean + apt-get update -y + apt-get install -y curl + elif command -v yum &>/dev/null; then + yum install -y curl + fi || true + + curl -L -o sccache.tar.gz \ + "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${sccache_arch}-unknown-linux-musl.tar.gz" + + tar xzf sccache.tar.gz + install sccache*/sccache /usr/local/bin/ && echo "sccache installed" +} + install_deps() { if [[ $INSTALL_DEPS = false ]]; then return @@ -744,6 +774,7 @@ construct_cmake_flags init_submodules if [[ $BUILD_PACKAGES = true ]]; then + modify_packaging build_package message_splitted "PACKAGES BUILD FINISHED" From 26fd0ba4bf34f491745c379680a23ceb4a203eb3 Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Mon, 30 Jun 2025 17:37:20 +0200 Subject: [PATCH 54/68] chore(ci) MCOL-6071 refactorings --- .drone.jsonnet | 31 +++++++++---------------------- build/bootstrap_mcs.sh | 6 +++--- build/prepare_test_container.sh | 3 ++- build/report_test_stage.sh | 6 ++++++ build/run_regression.sh | 2 +- build/utils.sh | 18 +++++++++--------- 6 files changed, 30 insertions(+), 36 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index f733a3aa8..334237e3a 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -15,14 +15,9 @@ local platforms_arm = { "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"], }; -local rewrite_ubuntu_mirror = @"sed -i 's|//\\(us\\.\\)\\?archive\\.ubuntu\\.com|//us.archive.ubuntu.com|g' /etc/apt/sources.list || true; " + - @"sed -i 's|//\\(us\\.\\)\\?archive\\.ubuntu\\.com|//us.archive.ubuntu.com|g' /etc/apt/sources.list.d/ubuntu.sources || true; " + - "cat /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list | grep archive || true; "; - local customEnvCommandsMap = { // 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'], "clang-20": [ - rewrite_ubuntu_mirror, "apt-get clean && apt-get update", "apt-get install -y wget curl lsb-release software-properties-common gnupg", "wget https://apt.llvm.org/llvm.sh", @@ -144,14 +139,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", local server_remote = if (std.endsWith(server, "enterprise")) then "https://github.com/mariadb-corporation/MariaDBEnterprise" else "https://github.com/MariaDB/server", - local sccache_arch = if (arch == "amd64") then "x86_64" else "aarch64", - local get_sccache = ["echo getting sccache...", - rewrite_ubuntu_mirror, - "(apt-get clean && apt-get update -y && apt-get install -y curl || yum install -y curl || true)", - "curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-" + sccache_arch + "-unknown-linux-musl.tar.gz &&", - "tar xzf sccache.tar.gz", - "install sccache*/sccache /usr/local/bin/ && echo sccache installed"], - local pipeline = self, publish(step_prefix="pkg", eventp=event + "/${DRONE_BUILD_NUMBER}"):: { @@ -220,9 +207,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", local execInnerDocker(command, containerName, flags="") = "docker exec " + flags + " -t " + containerName + " " + command, - local execInnerDockerNoTTY(command, containerName, flags="") = - "docker exec " + flags + " " + containerName + " " + command, - local getContainerName(stepname) = stepname + "$${DRONE_BUILD_NUMBER}", local installCmapi(containerName, pkg_format) = @@ -278,12 +262,15 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", }, commands: [ prepareTestContainer(getContainerName("upgrade") + version, result, false), - if (pkg_format == "deb") - then execInnerDocker('bash -c "./upgrade_setup_deb.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"', - getContainerName("upgrade") + version), - if (pkg_format == "rpm") - then execInnerDocker('bash -c "./upgrade_setup_rpm.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"', - getContainerName("upgrade") + version), + + execInnerDocker('bash -c "./upgrade_setup_' + pkg_format + '.sh ' + + version + ' ' + + result + ' ' + + arch + ' ' + + repo_pkg_url_no_res + + ' $${UPGRADE_TOKEN}"', + getContainerName("upgrade") + version + ) ], }, upgradelog:: { diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 683828821..368b39501 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -95,8 +95,6 @@ install_sccache() { message "getting sccache..." - rewrite_ubuntu_mirror #TODO - if command -v apt-get &>/dev/null; then apt-get clean apt-get update -y @@ -109,9 +107,11 @@ install_sccache() { "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${sccache_arch}-unknown-linux-musl.tar.gz" tar xzf sccache.tar.gz - install sccache*/sccache /usr/local/bin/ && echo "sccache installed" + install sccache*/sccache /usr/local/bin/ && message "sccache installed" } +install_sccache + install_deps() { if [[ $INSTALL_DEPS = false ]]; then return diff --git a/build/prepare_test_container.sh b/build/prepare_test_container.sh index 924382bcf..e5bacee39 100755 --- a/build/prepare_test_container.sh +++ b/build/prepare_test_container.sh @@ -103,6 +103,7 @@ fi # install deps if [[ "$RESULT" == *rocky* ]]; then + execInnerDockerWithRetry "$CONTAINER_NAME" 'dnf install -y dnf-plugins-core && dnf config-manager --set-enabled crb' execInnerDockerWithRetry "$CONTAINER_NAME" 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release expect findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' else change_ubuntu_mirror_in_docker "$CONTAINER_NAME" "us" @@ -128,5 +129,5 @@ echo "PrepareTestStage completed in $CONTAINER_NAME" if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then start_container prepare_container -else warn "Container ${CONTAINER_NAME} is already running!" +else message "Container ${CONTAINER_NAME} is already running, skipping prepare step" fi diff --git a/build/report_test_stage.sh b/build/report_test_stage.sh index 1be7cf140..89a74d4b9 100755 --- a/build/report_test_stage.sh +++ b/build/report_test_stage.sh @@ -22,6 +22,12 @@ cleanup() { #Remove the container on exit trap cleanup EXIT +for flag in CONTAINER_NAME RESULT STAGE; do + if [[ -z "${!flag}" ]]; then + error "Missing required flag: -${flag:0:1} / --${flag,,}" + exit 1 + fi +done if [[ "$EUID" -ne 0 ]]; then error "Please run script as root" diff --git a/build/run_regression.sh b/build/run_regression.sh index e2bd3324a..029c417a3 100755 --- a/build/run_regression.sh +++ b/build/run_regression.sh @@ -12,7 +12,7 @@ optparse.define short=t long=regression-timeout desc="Timeout for the regression optparse.define short=n long=test-name desc="Name of regression test to execute" variable=TEST_NAME source "$(optparse.build)" -for flag in CONTAINER_NAME REGRESSION_BRANCH DISTRO; do +for flag in CONTAINER_NAME REGRESSION_BRANCH DISTRO REGRESSION_TIMEOUT TEST_NAME; do if [[ -z "${!flag}" ]]; then error "Missing required flag: -${flag:0:1} / --${flag,,}" exit 1 diff --git a/build/utils.sh b/build/utils.sh index d2551655a..42ee52077 100644 --- a/build/utils.sh +++ b/build/utils.sh @@ -548,15 +548,6 @@ function execInnerDockerNoTTY() { fi } -function change_ubuntu_mirror() { - local region="$1" - message "Changing Ubuntu mirror to $region" - sed -i "s|//\(${region}\.\)\?archive\.ubuntu\.com|//${region}.archive.ubuntu.com|g" /etc/apt/sources.list 2>/dev/null || true - sed -i "s|//\(${region}\.\)\?archive\.ubuntu\.com|//${region}.archive.ubuntu.com|g" /etc/apt/sources.list.d/ubuntu.sources 2>/dev/null || true - cat /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null | grep archive || true - message_split -} - function execInnerDockerWithRetry() { local max_retries=5 local container_name=$1 @@ -583,6 +574,15 @@ function execInnerDockerWithRetry() { return 0 } +function change_ubuntu_mirror() { + local region="$1" + message "Changing Ubuntu mirror to $region" + sed -i "s|//\(${region}\.\)\?archive\.ubuntu\.com|//${region}.archive.ubuntu.com|g" /etc/apt/sources.list 2>/dev/null || true + sed -i "s|//\(${region}\.\)\?archive\.ubuntu\.com|//${region}.archive.ubuntu.com|g" /etc/apt/sources.list.d/ubuntu.sources 2>/dev/null || true + cat /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null | grep archive || true + message_split +} + change_ubuntu_mirror_in_docker() { local container_name=$1 local region=$2 From daab30a9874565de7c703fb684376c1b21cc2074 Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Mon, 30 Jun 2025 18:06:19 +0200 Subject: [PATCH 55/68] chore(ci) MCOL-6071 execute unittests during build step in bootstrap_mcs.sh --- .drone.jsonnet | 48 ++---------------------------------------- build/bootstrap_mcs.sh | 9 +++++--- 2 files changed, 8 insertions(+), 49 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 334237e3a..a8d8ffb6e 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -86,32 +86,6 @@ local upgrade_test_lists = { }, }; -local gcc_version = "11"; - -local rockylinux8_deps = "dnf install -y 'dnf-command(config-manager)' " + - "&& dnf config-manager --set-enabled powertools " + - "&& dnf install -y gcc-toolset-" + gcc_version + " libarchive cmake " + - "&& . /opt/rh/gcc-toolset-" + gcc_version + "/enable "; - -local rockylinux9_deps = "dnf install -y 'dnf-command(config-manager)' " + - "&& dnf config-manager --set-enabled crb " + - "&& dnf install -y gcc gcc-c++"; - -local rockylinux_common_deps = " && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel"; - -local deb_deps = rewrite_ubuntu_mirror + "apt-get clean && apt-get update && apt-get install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev"; - -local testPreparation(platform) = - local platform_map = { - "rockylinux:8": rockylinux8_deps + rockylinux_common_deps, - "rockylinux:9": rockylinux9_deps + rockylinux_common_deps, - "debian:12": deb_deps, - "ubuntu:20.04": deb_deps, - "ubuntu:22.04": deb_deps, - "ubuntu:24.04": deb_deps, - }; - platform_map[platform]; - local make_clickable_link(link) = "echo -e '\\e]8;;" + link + "\\e\\\\" + link + "\\e]8;;\\e\\\\'"; local echo_running_on = ["echo running on ${DRONE_STAGE_MACHINE}", make_clickable_link("https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=:${DRONE_STAGE_MACHINE};v=3;$case=tags:true%5C,client:false;$regex=tags:false%5C,client:false;sort=desc:launchTime")]; @@ -502,7 +476,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", }, commands: echo_running_on + [ - "echo $$SERVER_REF", + "echo $$SERVER_REF", "echo $$SERVER_REMOTE", "mkdir -p /mdb/" + builddir + " && cd /mdb/" + builddir, 'git config --global url."https://github.com/".insteadOf git@github.com:', @@ -537,7 +511,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", commands: [ "mkdir /mdb/" + builddir + "/" + result, ] - + get_sccache + customEnvCommands(customBuildEnvCommandsMapKey, builddir) + [ 'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " + @@ -574,26 +547,9 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/createrepo.sh --result " + result, ], }, - { - name: "unittests", - depends_on: ["createrepo"], - image: img, - volumes: [pipeline._volumes.mdb], - environment: { - DEBIAN_FRONTEND: "noninteractive", - }, - commands: [ - "cd /mdb/" + builddir, - testPreparation(platform), - - if (platform == "rockylinux:8" || platform == "rockylinux:9") - then "ctest3 -R columnstore: -j $(nproc) --output-on-failure" - else "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", - ], - }, { name: "pkg", - depends_on: ["unittests"], + depends_on: ["createrepo"], image: "alpine/git:2.49.0", when: { status: ["success", "failure"], diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 368b39501..ebb503127 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -11,7 +11,6 @@ export CLICOLOR_FORCE=1 #cmake output INSTALL_PREFIX="/usr/" DATA_DIR="/var/lib/mysql/data" CMAKE_BIN_NAME=cmake -CTEST_BIN_NAME=ctest RPM_CONFIG_DIR="/etc/my.cnf.d" DEB_CONFIG_DIR="/etc/mysql/mariadb.conf.d" @@ -77,9 +76,12 @@ if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " ]]; then detect_distro fi -pkg_format="deb" if [[ "$OS" == *"rocky"* ]]; then pkg_format="rpm" + CTEST_BIN_NAME=:"ctest3" +else + pkg_format="deb" + CTEST_BIN_NAME="ctest" fi install_sccache() { @@ -585,7 +587,7 @@ run_unit_tests() { message "Running unittests" cd $MARIA_BUILD_PATH - ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure + ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --output-on-failure cd - >/dev/null } @@ -778,6 +780,7 @@ if [[ $BUILD_PACKAGES = true ]]; then modify_packaging build_package message_splitted "PACKAGES BUILD FINISHED" + run_unit_tests exit 0 fi From 49864b4c64ef9ff4a4e47051a370cae3342d2f42 Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Mon, 30 Jun 2025 20:16:33 +0200 Subject: [PATCH 56/68] chore(ci) MCOL-6071 run_cmapi_test.sh --- .drone.jsonnet | 23 ++++----------- build/run_cmapi_test.sh | 64 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 18 deletions(-) create mode 100755 build/run_cmapi_test.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index a8d8ffb6e..cb1643571 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -93,8 +93,6 @@ local echo_running_on = ["echo running on ${DRONE_STAGE_MACHINE}", local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", customBootstrapParams="", customBuildEnvCommandsMapKey="") = { local pkg_format = if (std.split(platform, ":")[0] == "rockylinux") then "rpm" else "deb", local mtr_path = if (pkg_format == "rpm") then "/usr/share/mysql-test" else "/usr/share/mysql/mysql-test", - local cmapi_path = "/usr/share/columnstore/cmapi", - local etc_path = "/etc/columnstore", local socket_path = if (pkg_format == "rpm") then "/var/lib/mysql/mysql.sock" else "/run/mysqld/mysqld.sock", local img = if (platform == "rockylinux:8") then platform else "detravi/" + std.strReplace(platform, "/", "-"), local branch_ref = if (branch == any_branch) then current_branch else branch, @@ -183,10 +181,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", local getContainerName(stepname) = stepname + "$${DRONE_BUILD_NUMBER}", - local installCmapi(containerName, pkg_format) = - if (pkg_format == "deb") then execInnerDocker('bash -c "apt-get clean && apt-get update -y && apt-get install -y mariadb-columnstore-cmapi"', containerName) - else execInnerDocker('bash -c "yum update -y && yum install -y MariaDB-columnstore-cmapi"', containerName), - local prepareTestContainer(containerName, result, do_setup) = 'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/prepare_test_container.sh" + " --container-name " + containerName + @@ -394,17 +388,10 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", }, commands: [ prepareTestContainer(getContainerName("cmapi"), result, true), - installCmapi(getContainerName("cmapi"), pkg_format), - "cd cmapi", - "for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/$${i}/; done", - "docker cp run_tests.py cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/", - execInnerDocker("systemctl start mariadb-columnstore-cmapi", getContainerName("cmapi")), - // set API key to /etc/columnstore/cmapi_server.conf - execInnerDocker('bash -c "mcs cluster set api-key --key somekey123"', getContainerName("cmapi")), - // copy cmapi conf file for test purposes (there are api key already set inside) - execInnerDocker('bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], getContainerName("cmapi")), - execInnerDocker("systemctl stop mariadb-columnstore-cmapi", getContainerName("cmapi")), - execInnerDocker('bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', getContainerName("cmapi")), + + 'apk add bash && bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/run_cmapi_test.sh' + + ' --container-name ' + getContainerName("cmapi") + + ' --pkg-format ' + pkg_format, ], }, cmapilog:: { @@ -520,7 +507,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", " " + customBootstrapParams + " " + customBootstrapParamsForExisitingPipelines(platform) + " | " + "/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " + - "/mdb/" + builddir + "/" + result + '/build.log "' + "/mdb/" + builddir + "/" + result + '/build.log "', ], }, { diff --git a/build/run_cmapi_test.sh b/build/run_cmapi_test.sh new file mode 100755 index 000000000..0e9457741 --- /dev/null +++ b/build/run_cmapi_test.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -eo pipefail + +SCRIPT_LOCATION=$(dirname "$0") +COLUMNSTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../) +MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..) +CMAPI_PATH="/usr/share/columnstore/cmapi" +ETC_PATH="/etc/columnstore" + +source "$SCRIPT_LOCATION"/utils.sh + +optparse.define short=c long=container-name desc="Name of the Docker container where cmapi tests will run" variable=CONTAINER_NAME +optparse.define short=f long=pkg-format desc="Package format" variable=PKG_FORMAT +source $(optparse.build) +echo "Arguments received: $@" + +for flag in CONTAINER_NAME PKG_FORMAT; do + if [[ -z "${!flag}" ]]; then + error "Missing required flag: -${flag:0:1} / --${flag,,}" + exit 1 + fi +done + +on_exit() { + if [[ $? -eq 0 ]]; then + echo "Cmapi tests passed" + else + echo "Cmapi tests failed!" + fi +} +trap on_exit EXIT + +prepare_environment() { + echo "Preparing for cmapi test run..." + + if [[ "$PKG_FORMAT" == "deb" ]]; then + execInnerDocker $CONTAINER_NAME "apt-get clean && apt-get update -y && apt-get install -y mariadb-columnstore-cmapi" + else + execInnerDocker $CONTAINER_NAME "yum update -y && yum install -y MariaDB-columnstore-cmapi" + fi + + cd cmapi + + for i in mcs_node_control cmapi_server failover; do + docker cp "${i}/test" "$CONTAINER_NAME:${CMAPI_PATH}/${i}/" + done + + docker cp run_tests.py "$CONTAINER_NAME:${CMAPI_PATH}/" + execInnerDocker $CONTAINER_NAME "systemctl start mariadb-columnstore-cmapi" + + # set API key to /etc/columnstore/cmapi_server.conf + execInnerDocker $CONTAINER_NAME "mcs cluster set api-key --key somekey123" + # copy cmapi conf file for test purposes (there are api key already set inside) + execInnerDocker $CONTAINER_NAME "cp ${ETC_PATH}/cmapi_server.conf ${CMAPI_PATH}/cmapi_server/" + execInnerDocker $CONTAINER_NAME "systemctl stop mariadb-columnstore-cmapi" +} + +run_cmapi_test() { + execInnerDocker $CONTAINER_NAME "cd ${CMAPI_PATH} && python/bin/python3 run_tests.py" +} + +prepare_environment +run_cmapi_test From 76b86ae8394665d6a348b7066bd322f7921f2bc5 Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Mon, 30 Jun 2025 20:50:21 +0200 Subject: [PATCH 57/68] chore(ci) MCOL-6071 sccache show stats to bootstrap --- build/bootstrap_mcs.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index ebb503127..856f5e57f 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -781,6 +781,9 @@ if [[ $BUILD_PACKAGES = true ]]; then build_package message_splitted "PACKAGES BUILD FINISHED" run_unit_tests + if [[ $SCCACHE = true ]]; then + sccache --show-stats + fi exit 0 fi From 019b9054e6413816fcdd0f1d8a439a75a51ff43c Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Mon, 30 Jun 2025 21:13:51 +0200 Subject: [PATCH 58/68] chore(ci) fix publish step problem --- .drone.jsonnet | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index cb1643571..efa8ff111 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -131,10 +131,11 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", AWS_DEFAULT_REGION: "us-east-1", }, commands: [ + "sleep 10", "ls " + result, '[ -z "$(ls -A "' + result + '")" ] && echo Nothing to publish! && exit 1', - "aws s3 sync " + result + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors", + "aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors --debug", 'echo "Data uploaded to: ' + publish_pkg_url + '"', make_clickable_link(publish_pkg_url), "rm -rf " + result + "/*", @@ -574,10 +575,9 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", (if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.publish("mtr")]) + [pipeline.regression(regression_tests[i], if (i == 0) then ["mtr", "publish pkg", "publish cmapi build"] else [regression_tests[i - 1]]) for i in indexes(regression_tests)] + [pipeline.regressionlog] + - [pipeline.publish("regressionlog")] + // [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] + // (if (std.length(mdb_server_versions) == 0) then [] else [pipeline.upgradelog] + [pipeline.publish("upgradelog")]) + - (if (event == "cron") then [pipeline.publish("regressionlog latest", "latest")] else []), + (if (event == "cron") then [pipeline.publish("regressionlog latest", "latest")] else [pipeline.publish("regressionlog")]), volumes: [pipeline._volumes.mdb { temp: {} }, pipeline._volumes.docker { host: { path: "/var/run/docker.sock" } }], trigger: { From 2fc6e976b296d5af8f2a1fd69e090a7405263a5e Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Tue, 1 Jul 2025 10:47:45 +0200 Subject: [PATCH 59/68] chore(ci) MCOL-6071 cmapilog inside cmapi test --- .drone.jsonnet | 22 +++++----------------- build/run_cmapi_test.sh | 14 ++++++-------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index efa8ff111..7eb328dc2 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -390,23 +390,12 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", commands: [ prepareTestContainer(getContainerName("cmapi"), result, true), - 'apk add bash && bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/run_cmapi_test.sh' + - ' --container-name ' + getContainerName("cmapi") + - ' --pkg-format ' + pkg_format, + "apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_cmapi_test.sh" + + " --container-name " + getContainerName("cmapi") + + " --result-path " + result + + " --pkg-format " + pkg_format, ], }, - cmapilog:: { - name: "cmapilog", - depends_on: ["cmapi test"], - image: "docker:28.2.2", - volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], - commands: [ - reportTestStage(getContainerName("cmapi"), result, "cmapi"), - ], - when: { - status: ["success", "failure"], - }, - }, multi_node_mtr:: { name: "mtr", depends_on: ["dockerhub"], @@ -570,8 +559,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", [pipeline.smoke] + [pipeline.publish("smoke")] + [pipeline.cmapitest] + - [pipeline.cmapilog] + - [pipeline.publish("cmapilog")] + + [pipeline.publish("cmapi test")] + (if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.publish("mtr")]) + [pipeline.regression(regression_tests[i], if (i == 0) then ["mtr", "publish pkg", "publish cmapi build"] else [regression_tests[i - 1]]) for i in indexes(regression_tests)] + [pipeline.regressionlog] + diff --git a/build/run_cmapi_test.sh b/build/run_cmapi_test.sh index 0e9457741..a6d9da800 100755 --- a/build/run_cmapi_test.sh +++ b/build/run_cmapi_test.sh @@ -12,24 +12,22 @@ source "$SCRIPT_LOCATION"/utils.sh optparse.define short=c long=container-name desc="Name of the Docker container where cmapi tests will run" variable=CONTAINER_NAME optparse.define short=f long=pkg-format desc="Package format" variable=PKG_FORMAT +optparse.define short=r long=result-path desc="Path for logs and results" variable=RESULT source $(optparse.build) echo "Arguments received: $@" -for flag in CONTAINER_NAME PKG_FORMAT; do +for flag in CONTAINER_NAME PKG_FORMAT RESULT; do if [[ -z "${!flag}" ]]; then error "Missing required flag: -${flag:0:1} / --${flag,,}" exit 1 fi done -on_exit() { - if [[ $? -eq 0 ]]; then - echo "Cmapi tests passed" - else - echo "Cmapi tests failed!" - fi +#Collect the logs on exit event +collect_logs() { + "${SCRIPT_LOCATION}/report_test_stage.sh" --container-name "${CONTAINER_NAME}" --result-path "${RESULT}" --stage "cmapi" } -trap on_exit EXIT +trap collect_logs EXIT prepare_environment() { echo "Preparing for cmapi test run..." From 460cb17e37fb58c653dd0f7be4c491a220fb0b0a Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Thu, 3 Jul 2025 16:27:21 +0200 Subject: [PATCH 60/68] chore(ci) MCOL-6071 multinode mtr in bash script --- .drone.jsonnet | 20 ++++--------- build/run_multi_node_mtr.sh | 57 +++++++++++++++++++++++++++++++++++++ build/run_regression.sh | 6 ---- 3 files changed, 62 insertions(+), 21 deletions(-) create mode 100755 build/run_multi_node_mtr.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index 7eb328dc2..979ac308e 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -92,8 +92,6 @@ local echo_running_on = ["echo running on ${DRONE_STAGE_MACHINE}", local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", customBootstrapParams="", customBuildEnvCommandsMapKey="") = { local pkg_format = if (std.split(platform, ":")[0] == "rockylinux") then "rpm" else "deb", - local mtr_path = if (pkg_format == "rpm") then "/usr/share/mysql-test" else "/usr/share/mysql/mysql-test", - local socket_path = if (pkg_format == "rpm") then "/var/lib/mysql/mysql.sock" else "/run/mysqld/mysqld.sock", local img = if (platform == "rockylinux:8") then platform else "detravi/" + std.strReplace(platform, "/", "-"), local branch_ref = if (branch == any_branch) then current_branch else branch, @@ -400,7 +398,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", name: "mtr", depends_on: ["dockerhub"], image: "docker:28.2.2", - volumes: [pipeline._volumes.docker], + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], environment: { DOCKER_LOGIN: { from_secret: "dockerhub_user", @@ -412,18 +410,10 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", }, commands: [ "echo $$DOCKER_PASSWORD | docker login --username $$DOCKER_LOGIN --password-stdin", - "cd docker", - "cp .env_example .env", - 'sed -i "/^MCS_IMAGE_NAME=/s/=.*/=${MCS_IMAGE_NAME}/" .env', - 'sed -i "/^MAXSCALE=/s/=.*/=false/" .env', - "docker-compose up -d", - "docker exec mcs1 provision mcs1 mcs2 mcs3", - "docker cp ../mysql-test/columnstore mcs1:" + mtr_path + "/suite/", - "docker exec -t mcs1 chown mysql:mysql -R " + mtr_path, - 'docker exec -t mcs1 mariadb -e "create database if not exists test;"', - // delay for manual debugging on live instance - "sleep $${COMPOSE_DELAY_SECONDS:-1s}", - 'docker exec -t mcs1 bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/basic,columnstore/bugfixes"', + + "apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_multi_node_mtr.sh " + + "--columnstore-image-name $${MCS_IMAGE_NAME}" + + "--distro " + platform, ], }, diff --git a/build/run_multi_node_mtr.sh b/build/run_multi_node_mtr.sh new file mode 100755 index 000000000..a8404f1d8 --- /dev/null +++ b/build/run_multi_node_mtr.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -eo pipefail + +SCRIPT_LOCATION=$(dirname "$0") +source "$SCRIPT_LOCATION"/utils.sh + +optparse.define short=i long=columnstore-image-name desc="Name of columnstore docker image" variable=MCS_IMAGE_NAME +optparse.define short=d long=distro desc="Linux distro for which multinode mtr is executed" variable=DISTRO +source $(optparse.build) + +echo "Arguments received: $@" + +if [[ "$EUID" -ne 0 ]]; then + error "Please run script as root" + exit 1 +fi + +for flag in MCS_IMAGE_NAME DISTRO; do + if [[ -z "${!flag}" ]]; then + error "Missing required flag: -${flag:0:1} / --${flag,,}" + exit 1 + fi +done + + +if [[ "$DISTRO" == *rocky* ]]; then + SOCKET_PATH="/var/lib/mysql/mysql.sock" + MTR_PATH="/usr/share/mysql-test" +else + SOCKET_PATH="/run/mysqld/mysqld.sock" + MTR_PATH="/usr/share/mysql/mysql-test" +fi + +message "Running multinode mtr tests..." + +cd docker +cp .env_example .env +sed -i "/^MCS_IMAGE_NAME=/s|=.*|=${MCS_IMAGE_NAME}|" .env +sed -i "/^MAXSCALE=/s|=.*|=false|" .env + +docker-compose up -d +docker exec mcs1 provision mcs1 mcs2 mcs3 +docker cp ../mysql-test/columnstore mcs1:"${MTR_PATH}/suite/" +docker exec -t mcs1 chown -R mysql:mysql "${MTR_PATH}" +docker exec -t mcs1 mariadb -e "CREATE DATABASE IF NOT EXISTS test;" + +docker exec -t mcs1 bash -c "\ + cd '${MTR_PATH}' && \ + ./mtr \ + --extern socket='${SOCKET_PATH}' \ + --force \ + --print-core=detailed \ + --print-method=gdb \ + --max-test-fail=0 \ + --suite=columnstore/basic,columnstore/bugfixes \ +" diff --git a/build/run_regression.sh b/build/run_regression.sh index 029c417a3..16ed1db59 100755 --- a/build/run_regression.sh +++ b/build/run_regression.sh @@ -24,12 +24,6 @@ if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then exit 1 fi -##Collect the logs on exit event -#collect_logs(){ -# "${SCRIPT_LOCATION}/report_test_stage.sh" --container-name "${CONTAINER_NAME}" --result-path "${RESULT}" --stage "regression" -#} -#trap collect_logs EXIT - BUILD_DIR="verylongdirnameforverystrangecpackbehavior" prepare_regression() { From 6a57129416815009b0e9ca2c13de59464b7e994e Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Thu, 3 Jul 2025 20:07:45 +0200 Subject: [PATCH 61/68] remove result dir cleaning --- .drone.jsonnet | 4 +--- build/prepare_test_container.sh | 2 +- build/report_test_stage.sh | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 979ac308e..5a3edb040 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -130,13 +130,11 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", }, commands: [ "sleep 10", - "ls " + result, - '[ -z "$(ls -A "' + result + '")" ] && echo Nothing to publish! && exit 1', + "ls -lR " + result, "aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors --debug", 'echo "Data uploaded to: ' + publish_pkg_url + '"', make_clickable_link(publish_pkg_url), - "rm -rf " + result + "/*", ], }, diff --git a/build/prepare_test_container.sh b/build/prepare_test_container.sh index e5bacee39..8253215cc 100755 --- a/build/prepare_test_container.sh +++ b/build/prepare_test_container.sh @@ -104,7 +104,7 @@ fi # install deps if [[ "$RESULT" == *rocky* ]]; then execInnerDockerWithRetry "$CONTAINER_NAME" 'dnf install -y dnf-plugins-core && dnf config-manager --set-enabled crb' - execInnerDockerWithRetry "$CONTAINER_NAME" 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release expect findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' + execInnerDockerWithRetry "$CONTAINER_NAME" 'yum --nobest update -y && yum --nobest install -y cracklib-dicts diffutils elfutils epel-release expect findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' else change_ubuntu_mirror_in_docker "$CONTAINER_NAME" "us" execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y elfutils expect findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' diff --git a/build/report_test_stage.sh b/build/report_test_stage.sh index 89a74d4b9..15d2e6732 100755 --- a/build/report_test_stage.sh +++ b/build/report_test_stage.sh @@ -77,14 +77,14 @@ elif [[ "${CONTAINER_NAME}" == *upgrade* ]]; then elif [[ "${CONTAINER_NAME}" == *regression* ]]; then echo "---------- start columnstore regression short report ----------" - execInnerDocker "$CONTAINER_NAME" 'cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest; cat go.log || echo "missing go.log"' + execInnerDocker "$CONTAINER_NAME" 'cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest; cat go.log' || echo "missing go.log" echo "---------- end columnstore regression short report ----------" echo docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/" "/drone/src/${RESULT}/" || echo "missing regression logs" docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs.tgz" execInnerDocker "$CONTAINER_NAME" 'tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/' - execInnerDocker "$CONTAINER_NAME" 'cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest; tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore || echo "failed to grab regression results"' + execInnerDocker "$CONTAINER_NAME" 'cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest && tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore' || echo "failed to grab regression results" docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs2.tgz" docker cp "${CONTAINER_NAME}:regressionQueries.tgz" "/drone/src/${RESULT}/" || echo "missing regressionQueries.tgz" From 09f5df182cb394806aac0744bdcb7ee626abe1fe Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Fri, 4 Jul 2025 14:36:10 +0200 Subject: [PATCH 62/68] review fixes --- .drone.jsonnet | 4 ++-- build/bootstrap_mcs.sh | 12 ++++++------ build/build_cmapi.sh | 24 ++++++++++-------------- build/prepare_test_container.sh | 4 +++- build/report_test_stage.sh | 4 +++- build/run_mtr.sh | 3 ++- build/run_multi_node_mtr.sh | 3 ++- build/utils.sh | 10 ++++++++++ 8 files changed, 38 insertions(+), 26 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 5a3edb040..56a269b5b 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -481,7 +481,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", 'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " + "--build-type RelWithDebInfo " + "--distro " + platform + " " + - "--build-packages --install-deps --sccache --sccache-arch " + arch + + "--build-packages --install-deps --sccache" + " " + customBootstrapParams + " " + customBootstrapParamsForExisitingPipelines(platform) + " | " + "/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " + @@ -497,7 +497,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", DEBIAN_FRONTEND: "noninteractive", }, commands: [ - "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/build_cmapi.sh --distro " + platform + " --arch " + arch, + "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/build_cmapi.sh --distro " + platform, ], }, { diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 856f5e57f..eed9873c8 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -76,11 +76,11 @@ if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " ]]; then detect_distro fi -if [[ "$OS" == *"rocky"* ]]; then - pkg_format="rpm" +select_pkg_format ${OS} + +if [[ "$PKG_FORMAT" == "rpm" ]]; then CTEST_BIN_NAME=:"ctest3" else - pkg_format="deb" CTEST_BIN_NAME="ctest" fi @@ -277,7 +277,7 @@ modify_packaging() { echo "Modifying_packaging..." cd $MDB_SOURCE_PATH - if [[ $pkg_format == "deb" ]]; then + if [[ $PKG_FORMAT == "deb" ]]; then sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh fi @@ -289,7 +289,7 @@ modify_packaging() { grep mariadb /usr/share/lto-disabled-list/lto-disabled-list fi - if [[ $pkg_format == "deb" ]]; then + if [[ $PKG_FORMAT == "deb" ]]; then apt-cache madison liburing-dev | grep liburing-dev || { sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && @@ -498,7 +498,7 @@ generate_svgs() { build_package() { cd $MDB_SOURCE_PATH - if [[ $pkg_format == "rpm" ]]; then + if [[ $PKG_FORMAT == "rpm" ]]; then command="cmake ${MDB_CMAKE_FLAGS[@]} && make -j\$(nproc) package" else export DEBIAN_FRONTEND="noninteractive" diff --git a/build/build_cmapi.sh b/build/build_cmapi.sh index 241b3fc35..84c6ca5b0 100755 --- a/build/build_cmapi.sh +++ b/build/build_cmapi.sh @@ -13,7 +13,6 @@ MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..) source "$SCRIPT_LOCATION"/utils.sh optparse.define short=d long=distro desc="distro" variable=OS -optparse.define short=a long=arch desc="architecture" variable=ARCH source $(optparse.build) echo "Arguments received: $@" @@ -22,18 +21,15 @@ if [ "$EUID" -ne 0 ]; then exit 1 fi -if [[ -z "${OS:-}" || -z "${ARCH:-}" ]]; then - echo "Please provide provide --distro and --arch parameters, e.g. ./build_cmapi.sh --distro ubuntu:22.04 --arch amd64" +if [[ -z "${OS:-}" ]]; then + echo "Please provide provide --distro parameter, e.g. ./build_cmapi.sh --distro ubuntu:22.04" exit 1 fi -pkg_format="deb" -if [[ "$OS" == *"rocky"* ]]; then - pkg_format="rpm" -fi +select_pkg_format ${OS} -if [[ "$ARCH" == "arm64" ]]; then - export CC=gcc #TODO: what it is for? +if [[ "$(arch)" == "arm64" ]]; then + export CC=gcc fi on_exit() { @@ -55,18 +51,18 @@ install_deps() { retry_eval 5 "dnf config-manager --set-enabled devel && dnf update -q -y" #to make redhat-lsb-core available for rocky 9 fi - if [[ "$pkg_format" == "rpm" ]]; then + if [[ "$PKG_FORMAT" == "rpm" ]]; then retry_eval 5 "dnf update -q -y && dnf install -q -y epel-release wget zstd findutils gcc cmake make rpm-build redhat-lsb-core libarchive" else retry_eval 5 "apt-get update -qq -o Dpkg::Use-Pty=0 && apt-get install -qq -o Dpkg::Use-Pty=0 wget zstd findutils gcc cmake make dpkg-dev lsb-release" fi - if [ "$ARCH" == "amd64" ]; then + if [ "$(arch)" == "x86_64" ]; then PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v2-unknown-linux-gnu-pgo+lto-full.tar.zst" - elif [ "$ARCH" == "arm64" ]; then + elif [ "$(arch)" == "arm64" ]; then PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-aarch64-unknown-linux-gnu-noopt-full.tar.zst" else - echo "Unsupported architecture: $ARCH" + echo "Unsupported architecture: $(arch)" exit 1 fi @@ -84,7 +80,7 @@ install_deps() { build_cmapi() { cd "$COLUMNSTORE_SOURCE_PATH"/cmapi ./cleanup.sh - cmake -D"${pkg_format^^}"=1 -DSERVER_DIR="$MDB_SOURCE_PATH" . && make package + cmake -D"${PKG_FORMAT^^}"=1 -DSERVER_DIR="$MDB_SOURCE_PATH" . && make package } install_deps build_cmapi diff --git a/build/prepare_test_container.sh b/build/prepare_test_container.sh index 8253215cc..e43978973 100755 --- a/build/prepare_test_container.sh +++ b/build/prepare_test_container.sh @@ -27,8 +27,10 @@ if [[ -z "${CONTAINER_NAME:-}" || -z "${DOCKER_IMAGE:-}" || -z "${RESULT:-}" || exit 1 fi +select_pkg_format ${RESULT} + start_container() { - if [[ "$RESULT" == *rocky* ]]; then + if [[ $PKG_FORMAT == "rpm" ]]; then SYSTEMD_PATH="/usr/lib/systemd/systemd" MTR_PATH="/usr/share/mysql-test" else diff --git a/build/report_test_stage.sh b/build/report_test_stage.sh index 15d2e6732..cea115616 100755 --- a/build/report_test_stage.sh +++ b/build/report_test_stage.sh @@ -39,7 +39,9 @@ if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then exit 1 fi -if [[ "$RESULT" == *rocky* ]]; then +select_pkg_format ${RESULT} + +if [[ "$PKG_FORMAT" == "rpm" ]]; then SYSTEMD_PATH="/usr/lib/systemd/systemd" MTR_PATH="/usr/share/mysql-test" else diff --git a/build/run_mtr.sh b/build/run_mtr.sh index dd7271325..d2f664b34 100755 --- a/build/run_mtr.sh +++ b/build/run_mtr.sh @@ -39,8 +39,9 @@ collect_logs() { } trap collect_logs EXIT +select_pkg_format ${RESULT} -if [[ "$DISTRO" == *rocky* ]]; then +if [[ "$PKG_FORMAT" == "rpm" ]]; then SOCKET_PATH="/var/lib/mysql/mysql.sock" MTR_PATH="/usr/share/mysql-test" else diff --git a/build/run_multi_node_mtr.sh b/build/run_multi_node_mtr.sh index a8404f1d8..19829bd92 100755 --- a/build/run_multi_node_mtr.sh +++ b/build/run_multi_node_mtr.sh @@ -23,8 +23,9 @@ for flag in MCS_IMAGE_NAME DISTRO; do fi done +select_pkg_format ${DISTRO} -if [[ "$DISTRO" == *rocky* ]]; then +if [[ "$PKG_FORMAT" == "rpm" ]]; then SOCKET_PATH="/var/lib/mysql/mysql.sock" MTR_PATH="/usr/share/mysql-test" else diff --git a/build/utils.sh b/build/utils.sh index 42ee52077..f41dcd895 100644 --- a/build/utils.sh +++ b/build/utils.sh @@ -171,6 +171,16 @@ detect_distro() { message "Detected $color_yellow$OS $OS_VERSION$color_normal" } +select_pkg_format() { + local distro="$1" + + if [[ "$distro" == *rocky* ]]; then + export PKG_FORMAT="rpm" + else + export PKG_FORMAT="deb" + fi +} + menuStr="" function hideCursor() { From 7171742dab21e0cf46f8ad7893f71b323f5f1b45 Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Fri, 4 Jul 2025 17:07:41 +0200 Subject: [PATCH 63/68] put separate *log steps back --- .drone.jsonnet | 51 ++++++++++++++++++++++++++++++++++------- build/run_cmapi_test.sh | 9 +------- build/run_mtr.sh | 11 ++------- build/run_smoke.sh | 23 ++++++++----------- 4 files changed, 55 insertions(+), 39 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 56a269b5b..dcaed9695 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -211,10 +211,21 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", commands: [ prepareTestContainer(getContainerName("smoke"), result, true), "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_smoke.sh" + - ' --container-name ' + getContainerName("smoke") + - ' --result-path ' + result, + ' --container-name ' + getContainerName("smoke"), ], }, + smokelog:: { + name: "smokelog", + depends_on: ["smoke"], + image: "docker:28.2.2", + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], + commands: [ + reportTestStage(getContainerName("smoke"), result, "smoke"), + ], + when: { + status: ["success", "failure"], + }, + }, upgrade(version):: { name: "upgrade-test from " + version, depends_on: ["regressionlog"], @@ -275,10 +286,21 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", ' --container-name ' + getContainerName("mtr") + ' --distro ' + platform + ' --suite-list $${MTR_SUITE_LIST}' + - ' --result-path ' + result + ' --triggering-event ' + event, ], }, + mtrlog:: { + name: "mtrlog", + depends_on: ["mtr"], + image: "docker:28.2.2", + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], + commands: [ + reportTestStage(getContainerName("mtr"), result, "mtr"), + ], + when: { + status: ["success", "failure"], + }, + }, regression(name, depends_on):: { name: name, depends_on: depends_on, @@ -388,10 +410,21 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", "apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_cmapi_test.sh" + " --container-name " + getContainerName("cmapi") + - " --result-path " + result + " --pkg-format " + pkg_format, ], }, + cmapilog:: { + name: "cmapilog", + depends_on: ["cmapi test"], + image: "docker:28.2.2", + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], + commands: [ + reportTestStage(getContainerName("cmapi"), result, "cmapi"), + ], + when: { + status: ["success", "failure"], + }, + }, multi_node_mtr:: { name: "mtr", depends_on: ["dockerhub"], @@ -441,7 +474,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", }, commands: echo_running_on + [ - "echo $$SERVER_REF", + "echo $$SERVER_REF", "echo $$SERVER_REMOTE", "mkdir -p /mdb/" + builddir + " && cd /mdb/" + builddir, 'git config --global url."https://github.com/".insteadOf git@github.com:', @@ -545,10 +578,12 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", [pipeline.publish()] + (if (event == "cron") then [pipeline.publish("pkg latest", "latest")] else []) + [pipeline.smoke] + - [pipeline.publish("smoke")] + + [pipeline.smokelog] + + [pipeline.publish("smokelog")] + [pipeline.cmapitest] + - [pipeline.publish("cmapi test")] + - (if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.publish("mtr")]) + + [pipeline.cmapilog] + + [pipeline.publish("cmapilog")] + + (if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish("mtrlog")]) + [pipeline.regression(regression_tests[i], if (i == 0) then ["mtr", "publish pkg", "publish cmapi build"] else [regression_tests[i - 1]]) for i in indexes(regression_tests)] + [pipeline.regressionlog] + // [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] + diff --git a/build/run_cmapi_test.sh b/build/run_cmapi_test.sh index a6d9da800..6c5391965 100755 --- a/build/run_cmapi_test.sh +++ b/build/run_cmapi_test.sh @@ -12,23 +12,16 @@ source "$SCRIPT_LOCATION"/utils.sh optparse.define short=c long=container-name desc="Name of the Docker container where cmapi tests will run" variable=CONTAINER_NAME optparse.define short=f long=pkg-format desc="Package format" variable=PKG_FORMAT -optparse.define short=r long=result-path desc="Path for logs and results" variable=RESULT source $(optparse.build) echo "Arguments received: $@" -for flag in CONTAINER_NAME PKG_FORMAT RESULT; do +for flag in CONTAINER_NAME PKG_FORMAT; do if [[ -z "${!flag}" ]]; then error "Missing required flag: -${flag:0:1} / --${flag,,}" exit 1 fi done -#Collect the logs on exit event -collect_logs() { - "${SCRIPT_LOCATION}/report_test_stage.sh" --container-name "${CONTAINER_NAME}" --result-path "${RESULT}" --stage "cmapi" -} -trap collect_logs EXIT - prepare_environment() { echo "Preparing for cmapi test run..." diff --git a/build/run_mtr.sh b/build/run_mtr.sh index d2f664b34..1a83ce7d4 100755 --- a/build/run_mtr.sh +++ b/build/run_mtr.sh @@ -9,7 +9,6 @@ optparse.define short=c long=container-name desc="Name of the Docker container w optparse.define short=d long=distro desc="Linux distro for which mtr is runned" variable=DISTRO optparse.define short=s long=suite-list desc="Comma-separated list of test suites to run" variable=MTR_SUITE_LIST optparse.define short=e long=triggering-event desc="Event that triggers testrun" variable=EVENT -optparse.define short=r long=result-path desc="Path for logs and results" variable=RESULT source $(optparse.build) MTR_FULL_SET="basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly" @@ -21,7 +20,7 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -for flag in CONTAINER_NAME DISTRO EVENT MTR_SUITE_LIST RESULT; do +for flag in CONTAINER_NAME DISTRO EVENT MTR_SUITE_LIST; do if [[ -z "${!flag}" ]]; then error "Missing required flag: -${flag:0:1} / --${flag,,}" exit 1 @@ -33,13 +32,7 @@ if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then exit 1 fi -#Collect the logs on exit event -collect_logs() { - "${SCRIPT_LOCATION}/report_test_stage.sh" --container-name "${CONTAINER_NAME}" --result-path "${RESULT}" --stage "mtr" -} -trap collect_logs EXIT - -select_pkg_format ${RESULT} +select_pkg_format ${DISTRO} if [[ "$PKG_FORMAT" == "rpm" ]]; then SOCKET_PATH="/var/lib/mysql/mysql.sock" diff --git a/build/run_smoke.sh b/build/run_smoke.sh index de6096987..9ef5eb78a 100755 --- a/build/run_smoke.sh +++ b/build/run_smoke.sh @@ -6,29 +6,25 @@ SCRIPT_LOCATION=$(dirname "$0") source "$SCRIPT_LOCATION"/utils.sh optparse.define short=c long=container-name desc="Name of the Docker container where mtr tests will run" variable=CONTAINER_NAME -optparse.define short=r long=result-path desc="Path for logs and results" variable=RESULT source $(optparse.build) echo "Arguments received: $@" -for flag in CONTAINER_NAME RESULT; do - if [[ -z "${!flag}" ]]; then - error "Missing required flag: -${flag:0:1} / --${flag,,}" - exit 1 - fi -done +if [[ "$EUID" -ne 0 ]]; then + error "Please run script as root" + exit 1 +fi + +if [[ -z "${CONTAINER_NAME:-}" ]]; then + echo "Please provide provide --container-name parameter, e.g. ./run_smoke.sh --container-name smoke185" + exit 1 +fi if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then error "Container '${CONTAINER_NAME}' is not running." exit 1 fi -#Collect the logs on exit event -collect_logs(){ - "${SCRIPT_LOCATION}/report_test_stage.sh" --container-name "${CONTAINER_NAME}" --result-path "${RESULT}" --stage "smoke" -} -trap collect_logs EXIT - message "Running smoke checks..." # start mariadb and mariadb-columnstore services and run simple query @@ -48,4 +44,3 @@ execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb-columnstore' sleep 10 execInnerDocker "$CONTAINER_NAME" 'mariadb -e "INSERT INTO test.t1 VALUES (2); SELECT * FROM test.t1;"' - From 6e3cf16ce4603aef72a63b27983148636db98d0a Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Mon, 23 Jun 2025 15:33:28 +0300 Subject: [PATCH 64/68] chore(MCOL-6018) Fix leaks in Columnstore information tables Fix introduces unique_ptr around DBRM instance in fill result functions. Now all paths lead to DBRM instance freeing. --- dbcon/mysql/is_columnstore_extents.cpp | 12 ++++++------ dbcon/mysql/is_columnstore_files.cpp | 13 ++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index b8f0cacf0..814e3c95e 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -202,7 +202,7 @@ static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) TABLE* table = tables->table; BRM::DBRM::refreshShmWithLock(); - BRM::DBRM* emp = new BRM::DBRM(); + std::unique_ptr emp(new BRM::DBRM()); if (!emp || !emp->isDBRMReady()) { @@ -224,7 +224,7 @@ static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) if (strcasecmp(item_field->field_name.str, "object_id") == 0) { cond_oid = fitem->arguments()[1]->val_int(); - return generate_result(cond_oid, emp, table, thd); + return generate_result(cond_oid, emp.get(), table, thd); } } else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM && @@ -236,7 +236,7 @@ static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) if (strcasecmp(item_field->field_name.str, "object_id") == 0) { cond_oid = fitem->arguments()[0]->val_int(); - return generate_result(cond_oid, emp, table, thd); + return generate_result(cond_oid, emp.get(), table, thd); } } } @@ -250,7 +250,7 @@ static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) for (unsigned int i = 1; i < fitem->argument_count(); i++) { cond_oid = fitem->arguments()[i]->val_int(); - int result = generate_result(cond_oid, emp, table, thd); + int result = generate_result(cond_oid, emp.get(), table, thd); if (result) return 1; @@ -266,7 +266,7 @@ static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) while (ss >> cond_oid) { - int ret = generate_result(cond_oid, emp, table, thd); + int ret = generate_result(cond_oid, emp.get(), table, thd); if (ret) return 1; @@ -282,7 +282,7 @@ static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) for (BRM::OID_t oid = 3000; oid <= MaxOID; oid++) { - int result = generate_result(oid, emp, table, thd); + int result = generate_result(oid, emp.get(), table, thd); if (result) return 1; diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index 11f64e1d2..19aea021b 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -216,7 +216,7 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) { BRM::DBRM::refreshShmWithLock(); - BRM::DBRM* emp = new BRM::DBRM(); + std::unique_ptr emp(new BRM::DBRM()); BRM::OID_t cond_oid = 0; TABLE* table = tables->table; @@ -240,7 +240,7 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) if (strcasecmp(item_field->field_name.str, "object_id") == 0) { cond_oid = fitem->arguments()[1]->val_int(); - return generate_result(cond_oid, emp, table, thd); + return generate_result(cond_oid, emp.get(), table, thd); } } else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM && @@ -252,7 +252,7 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) if (strcasecmp(item_field->field_name.str, "object_id") == 0) { cond_oid = fitem->arguments()[0]->val_int(); - return generate_result(cond_oid, emp, table, thd); + return generate_result(cond_oid, emp.get(), table, thd); } } } @@ -266,7 +266,7 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) for (unsigned int i = 1; i < fitem->argument_count(); i++) { cond_oid = fitem->arguments()[i]->val_int(); - int result = generate_result(cond_oid, emp, table, thd); + int result = generate_result(cond_oid, emp.get(), table, thd); if (result) return 1; @@ -282,7 +282,7 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) while (ss >> cond_oid) { - int ret = generate_result(cond_oid, emp, table, thd); + int ret = generate_result(cond_oid, emp.get(), table, thd); if (ret) return 1; @@ -300,14 +300,13 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) { for (BRM::OID_t oid = 3000; oid <= MaxOID; oid++) { - int result = generate_result(oid, emp, table, thd); + int result = generate_result(oid, emp.get(), table, thd); if (result) return 1; } } - delete emp; return 0; } From 5b92723b5cdd551e4297cda0da5d5fb93a89caa6 Mon Sep 17 00:00:00 2001 From: "aleksei.bukhalov" Date: Sun, 6 Jul 2025 13:26:48 +0200 Subject: [PATCH 65/68] chore(ci): MCOL-6071 remove crb repo enabling for rocky in prepare_test_container.sh --- .drone.jsonnet | 2 +- build/prepare_test_container.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index dcaed9695..ac352a2aa 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -443,7 +443,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", "echo $$DOCKER_PASSWORD | docker login --username $$DOCKER_LOGIN --password-stdin", "apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_multi_node_mtr.sh " + - "--columnstore-image-name $${MCS_IMAGE_NAME}" + + "--columnstore-image-name $${MCS_IMAGE_NAME} " + "--distro " + platform, ], }, diff --git a/build/prepare_test_container.sh b/build/prepare_test_container.sh index e43978973..83abaeb5e 100755 --- a/build/prepare_test_container.sh +++ b/build/prepare_test_container.sh @@ -105,7 +105,6 @@ fi # install deps if [[ "$RESULT" == *rocky* ]]; then - execInnerDockerWithRetry "$CONTAINER_NAME" 'dnf install -y dnf-plugins-core && dnf config-manager --set-enabled crb' execInnerDockerWithRetry "$CONTAINER_NAME" 'yum --nobest update -y && yum --nobest install -y cracklib-dicts diffutils elfutils epel-release expect findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' else change_ubuntu_mirror_in_docker "$CONTAINER_NAME" "us" From 37b5f212a91e4654b2893fc5c28c1d69b1539a40 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Mon, 7 Jul 2025 19:56:54 +0000 Subject: [PATCH 66/68] chore(build): MCOL-6090 do not build columnstore if not defined target --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec22f91d0..ee826f6c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,8 +49,12 @@ include(compiler_flags) include(misc) include(cpack_manage) +add_subdirectory(dbcon/mysql) +if(NOT TARGET columnstore) + return() +endif() + set(COMPONENTS - dbcon/mysql utils oam/oamcpp dbcon/execplan From b6a5c1d71f799845501d877a20947ab10b521eda Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Tue, 24 Jun 2025 15:46:18 +0300 Subject: [PATCH 67/68] fix(cmapi): MCOL-5913 cmapi write local address 127.0.1.1 instead real ip address * feat(cmapi): NetworkManager class for some ip hostname opoerations. * fix(cmapi): Use NetworkManager class to resolve ip and hostname in node_manipulation.add_node function * fix(cmapi): Minor docstring and formatting fixes --- cmapi/cmapi_server/constants.py | 9 +- cmapi/cmapi_server/managers/network.py | 258 ++++++++++++++++++++++++ cmapi/cmapi_server/node_manipulation.py | 66 +++--- 3 files changed, 302 insertions(+), 31 deletions(-) create mode 100644 cmapi/cmapi_server/managers/network.py diff --git a/cmapi/cmapi_server/constants.py b/cmapi/cmapi_server/constants.py index 464b61d99..c4bbb8cd6 100644 --- a/cmapi/cmapi_server/constants.py +++ b/cmapi/cmapi_server/constants.py @@ -43,7 +43,14 @@ SECRET_KEY = 'MCSIsTheBestEver' # not just a random string! (base32) # network constants -LOCALHOSTS = ('localhost', '127.0.0.1', '::1') +# according to https://www.ibm.com/docs/en/storage-sentinel/1.1.2?topic=installation-map-your-local-host-loopback-address +LOCALHOSTS = ( + '127.0.0.1', + 'localhost', 'localhost.localdomain', + 'localhost4', 'localhost4.localdomain4', + '::1', + 'localhost6', 'localhost6.localdomain6', +) CMAPI_INSTALL_PATH = '/usr/share/columnstore/cmapi/' CMAPI_PYTHON_BIN = os.path.join(CMAPI_INSTALL_PATH, "python/bin/python3") diff --git a/cmapi/cmapi_server/managers/network.py b/cmapi/cmapi_server/managers/network.py new file mode 100644 index 000000000..bded9e392 --- /dev/null +++ b/cmapi/cmapi_server/managers/network.py @@ -0,0 +1,258 @@ +import errno +import fcntl +import logging +import socket +import struct +from ipaddress import ip_address +from typing import Optional + +try: + import psutil + _PSUTIL_AVAILABLE = True +except ImportError: + psutil = None + _PSUTIL_AVAILABLE = False + +from cmapi_server.exceptions import CMAPIBasicError + + +SIOCGIFADDR = 0x8915 # SIOCGIFADDR "socket ioctl get interface address" + + +class NetworkManager: + @classmethod + def get_ip_version(cls, ip_addr: str) -> int: + """Get version of a given IP address. + + :param ip_addr: IP to get version + :type ip_addr: str + :return: version of a given IP + :rtype: int + """ + return ip_address(ip_addr).version + + @classmethod + def is_ip(cls, input_str: str) -> bool: + """Check is input a valid IP or not. + + :param input_str: input string + :type input_str: str + :return: True if input is a valid IP + :rtype: bool + """ + try: + ip_address(input_str) + return True + except ValueError: + return False + + @classmethod + def resolve_hostname_to_ip( + cls, + hostname: str, + only_ipv4: bool = True, + exclude_loopback: bool = False + ) -> list[str]: + """Resolve a hostname to one or more IP addresses. + + :param hostname: Hostname to resolve. + :type hostname: str + :param only_ipv4: Return only IPv4 addresses (default: True). + :type only_ipv4: bool + :param exclude_loopback: Exclude loopback addresses like 127.x.x.x (default: True). + :type exclude_loopback: bool + :return: List of resolved IP addresses. + :rtype: list[str] + """ + sorted_ips: list[str] = [] + try: + addr_info = socket.getaddrinfo( + hostname, + None, + socket.AF_INET if only_ipv4 else socket.AF_UNSPEC, + socket.SOCK_STREAM + ) + ip_set = { + info[4][0] for info in addr_info + if not (exclude_loopback and ip_address(info[4][0]).is_loopback) + } + sorted_ips = sorted( + list(ip_set), + key=lambda ip: ( + not ip_address(ip).is_loopback, # loopback first (False < True) + ip_address(ip).version != 4, # IPv4 before IPv6 (False < True) + ip_address(ip) # lexical order + ) + ) + except socket.gaierror: + logging.error( + f'Standard name resolution failed for hostname: {hostname!r}', + exc_info=True + ) + + return sorted_ips + + @classmethod + def get_ip_address_by_nic(cls, ifname: str) -> str: + """Get IP address of a network interface. + + :param ifname: network interface name + :type ifname: str + :return: ip address + :rtype: str + """ + # doesn't work on Windows, + # OpenBSD and probably doesn't on FreeBSD/pfSense either + ip_addr: str = '' + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + ip_addr = socket.inet_ntoa( + fcntl.ioctl( + s.fileno(), + SIOCGIFADDR, + struct.pack('256s', bytes(ifname[:15], 'utf-8')) + )[20:24] + ) + except OSError as exc: + if exc.errno == errno.ENODEV: + logging.error( + f'Interface {ifname!r} doesn\'t exist.' + ) + else: + logging.error( + f'Unknown OSError code while getting IP for an {ifname!r}', + exc_info=True + ) + except Exception: + logging.error( + ( + 'Unknown exception while getting IP address of an ' + f'{ifname!r} interface', + ), + exc_info=True + ) + return ip_addr + + @classmethod + def get_current_node_ips( + cls, ignore_loopback: bool = False, only_ipv4: bool = True, + ) -> list[str]: + """Get all IP addresses for all existing network interfaces. + + + :param ignore_loopback: Ignore loopback addresses, defaults to False + :type ignore_loopback: bool, optional + :param only_ipv4: Return only IPv4 addresses, defaults to True + :type only_ipv4: bool, optional + :return: IP addresses for all node interfaces + :rtype: list[str] + :raises CMAPIBasicError: If no IPs are found + """ + ext_ips: list[str] = [] + loopback_ips: list[str] = [] + + if _PSUTIL_AVAILABLE: + try: + for _, addrs in psutil.net_if_addrs().items(): + for addr in addrs: + if only_ipv4 and addr.family != socket.AF_INET: + continue + try: + ip = ip_address(addr.address) + if ip.is_loopback: + loopback_ips.append(str(ip)) + else: + ext_ips.append(str(ip)) + except ValueError: + continue # Not a valid IP (e.g., MAC addresses) + except Exception: + logging.warning( + 'Failed to get IPs via psutil, falling back to ioctl', + exc_info=True + ) + + result = ext_ips if ignore_loopback else [*ext_ips, *loopback_ips] + if result: + return result + logging.warning( + 'psutil returned no valid IPs, trying fallback method' + ) + + ext_ips: list[str] = [] + loopback_ips: list[str] = [] + # Fallback to stdlib method using fcntl/ioctl + for _, nic_name in socket.if_nameindex(): + ip_addr = cls.get_ip_address_by_nic(nic_name) + if not ip_addr: + continue + if only_ipv4 and cls.get_ip_version(ip_addr) != 4: + continue + if ip_address(ip_addr).is_loopback: + loopback_ips.append(ip_addr) + else: + ext_ips.append(ip_addr) + + result = ext_ips if ignore_loopback else [*ext_ips, *loopback_ips] + if not result: + raise CMAPIBasicError('No IP addresses found on this node.') + return result + + @classmethod + def get_hostname(cls, ip_addr: str) -> Optional[str]: + """Get hostname for a given IP address. + + :param ip_addr: IP address to get hostname + :type ip_addr: str + :return: Hostname if it exists, otherwise None + :rtype: Optional[str] + """ + try: + hostnames = socket.gethostbyaddr(ip_addr) + return hostnames[0] + except socket.herror: + logging.error(f'No hostname found for address: {ip_addr!r}') + return None + + @classmethod + def is_only_loopback_hostname(cls, hostname: str) -> bool: + """Check if all IPs resolved from the hostname are loopback. + + :param hostname: Hostname to check + :type hostname: str + :return: True if all resolved IPs are loopback, False otherwise + :rtype: bool + """ + ips = cls.resolve_hostname_to_ip(hostname) + if not ips: + return False + for ip in ips: + if not ip_address(ip).is_loopback: + return False + return True + + @classmethod + def resolve_ip_and_hostname(cls, input_str: str) -> tuple[str, str]: + """Resolve input string to an (IP, hostname) pair. + + :param input_str: Input which may be an IP address or a hostname + :type input_str: str + :return: A tuple containing (ip, hostname) + :rtype: tuple[str, str] + :raises CMAPIBasicError: if hostname resolution yields no IPs + """ + ip: str = '' + hostname: str = None + + if cls.is_ip(input_str): + ip = input_str + hostname = cls.get_hostname(input_str) + else: + hostname = input_str + ip_list = cls.resolve_hostname_to_ip( + input_str, + exclude_loopback=not cls.is_only_loopback_hostname(input_str) + ) + if not ip_list: + raise CMAPIBasicError(f'No IPs found for {hostname!r}') + ip = ip_list[0] + return ip, hostname diff --git a/cmapi/cmapi_server/node_manipulation.py b/cmapi/cmapi_server/node_manipulation.py index 7eee0ad18..bccdb142a 100644 --- a/cmapi/cmapi_server/node_manipulation.py +++ b/cmapi/cmapi_server/node_manipulation.py @@ -20,6 +20,7 @@ from cmapi_server.constants import ( CMAPI_CONF_PATH, CMAPI_SINGLE_NODE_XML, DEFAULT_MCS_CONF_PATH, LOCALHOSTS, MCS_DATA_PATH, ) +from cmapi_server.managers.network import NetworkManager from mcs_node_control.models.node_config import NodeConfig @@ -928,7 +929,7 @@ def _remove_node_from_PMS(root, node): return pm_num -def _add_Module_entries(root, node): +def _add_Module_entries(root, node: str) -> None: ''' get new node id add ModuleIPAddr, ModuleHostName, ModuleDBRootCount (don't set ModuleDBRootID* here) @@ -937,47 +938,52 @@ def _add_Module_entries(root, node): ''' # XXXPAT: No guarantee these are the values used in the rest of the system. - # This will work best with a simple network configuration where there is 1 IP addr - # and 1 host name for a node. - ip4 = socket.gethostbyname(node) - if ip4 == node: # node is an IP addr - node_name = socket.gethostbyaddr(node)[0] - else: - node_name = node # node is a hostname + # TODO: what should we do with complicated network configs where node has + # several ips and\or several hostnames + ip4, hostname = NetworkManager.resolve_ip_and_hostname(node) + logging.info(f'Using ip address {ip4} and hostname {hostname}') - logging.info(f"_add_Module_entries(): using ip address {ip4} and hostname {node_name}") - - smc_node = root.find("./SystemModuleConfig") - mod_count_node = smc_node.find("./ModuleCount3") - nnid_node = root.find("./NextNodeId") + smc_node = root.find('./SystemModuleConfig') + mod_count_node = smc_node.find('./ModuleCount3') + nnid_node = root.find('./NextNodeId') nnid = int(nnid_node.text) current_module_count = int(mod_count_node.text) # look for existing entries and fix if they exist for i in range(1, nnid): - ip_node = smc_node.find(f"./ModuleIPAddr{i}-1-3") - name_node = smc_node.find(f"./ModuleHostName{i}-1-3") - # if we find a matching IP address, but it has a different hostname, update the addr - if ip_node is not None and ip_node.text == ip4: - logging.info(f"_add_Module_entries(): found ip address already at ModuleIPAddr{i}-1-3") - hostname = smc_node.find(f"./ModuleHostName{i}-1-3").text - if hostname != node_name: - new_ip_addr = socket.gethostbyname(hostname) - logging.info(f"_add_Module_entries(): hostname doesn't match, updating address to {new_ip_addr}") - smc_node.find(f"ModuleHostName{i}-1-3").text = new_ip_addr + curr_ip_node = smc_node.find(f'./ModuleIPAddr{i}-1-3') + curr_name_node = smc_node.find(f'./ModuleHostName{i}-1-3') + # TODO: NETWORK: seems it's useless even in very rare cases. + # Even simplier to rewrite resolved IP an Hostname + # if we find a matching IP address, but it has a different hostname, + # update the addr + if curr_ip_node is not None and curr_ip_node.text == ip4: + logging.info(f'Found ip address already at ModuleIPAddr{i}-1-3') + if curr_name_node != hostname: + new_ip_addr = NetworkManager.resolve_hostname_to_ip( + curr_name_node + ) + logging.info( + 'Hostname doesn\'t match, updating address to ' + f'{new_ip_addr!r}' + ) + smc_node.find(f'ModuleHostName{i}-1-3').text = new_ip_addr else: - logging.info(f"_add_Module_entries(): no update is necessary") + logging.info('No update for ModuleIPAddr{i}-1-3 is necessary') return # if we find a matching hostname, update the ip addr - if name_node is not None and name_node.text == node_name: - logging.info(f"_add_Module_entries(): found existing entry for {node_name}, updating its address to {ip4}") - ip_node.text = ip4 + if curr_name_node is not None and curr_name_node.text == hostname: + logging.info( + f'Found existing entry for {hostname!r}, updating its ' + f'address to {ip4!r}' + ) + curr_ip_node.text = ip4 return - etree.SubElement(smc_node, f"ModuleIPAddr{nnid}-1-3").text = ip4 - etree.SubElement(smc_node, f"ModuleHostName{nnid}-1-3").text = node_name - etree.SubElement(smc_node, f"ModuleDBRootCount{nnid}-3").text = "0" + etree.SubElement(smc_node, f'ModuleIPAddr{nnid}-1-3').text = ip4 + etree.SubElement(smc_node, f'ModuleHostName{nnid}-1-3').text = hostname + etree.SubElement(smc_node, f'ModuleDBRootCount{nnid}-3').text = '0' mod_count_node.text = str(current_module_count + 1) nnid_node.text = str(nnid + 1) From aa3db28074c813ec9e85568f8ae49cf499d74b00 Mon Sep 17 00:00:00 2001 From: Aleksei Bukhalov Date: Wed, 9 Jul 2025 12:20:21 +0200 Subject: [PATCH 68/68] MCOL-6071 drone refactor (#3641) * chore(ci): MCOL-6071 less sccache granularity * chore(ci): MCOL-6071 clean old versions of .deb/.rpm files in s3 when publishing * chore(ci): MCOL-6071 fix unittests execution * install sccache after deps, to have curl * chore(ci) MCOL-6071 review fixes --------- Co-authored-by: Leonid Fedorov --- .drone.jsonnet | 39 +++++++++++++++++++++++--------------- build/bootstrap_mcs.sh | 35 +++++++++++++--------------------- build/install_clang_deb.sh | 28 +++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 37 deletions(-) create mode 100755 build/install_clang_deb.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index ac352a2aa..6e687b951 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -4,27 +4,21 @@ local events = ["pull_request", "cron"]; local current_branch = "stable-23.10"; local servers = { - "stable-23.10": ["10.6-enterprise"], + [current_branch]: ["10.6-enterprise"], }; local platforms = { - "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"], + [current_branch]: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"], }; local platforms_arm = { - "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"], + [current_branch]: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"], }; +local builddir = "verylongdirnameforverystrangecpackbehavior"; local customEnvCommandsMap = { - // 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'], - "clang-20": [ - "apt-get clean && apt-get update", - "apt-get install -y wget curl lsb-release software-properties-common gnupg", - "wget https://apt.llvm.org/llvm.sh", - "bash llvm.sh 20", - "export CC=/usr/bin/clang", - "export CXX=/usr/bin/clang++", - ], + // 'clang-18': ["bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/install_clang_deb.sh 18"], + "clang-20": ["bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/install_clang_deb.sh 20"], }; local customEnvCommands(envkey, builddir) = @@ -52,7 +46,6 @@ local customBootstrapParamsForAdditionalPipelinesMap = { local any_branch = "**"; -local builddir = "verylongdirnameforverystrangecpackbehavior"; local mtr_suite_list = "basic,bugfixes"; local mtr_full_set = "basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly"; @@ -115,6 +108,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", name: "publish " + step_prefix, depends_on: [std.strReplace(step_prefix, " latest", ""), "createrepo"], image: "amazon/aws-cli:2.22.30", + volumes: [pipeline._volumes.mdb], when: { status: ["success", "failure"], }, @@ -132,7 +126,21 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", "sleep 10", "ls -lR " + result, - "aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors --debug", + //clean old versions of .deb/.rpm files + "source /mdb/" + builddir + "/storage/columnstore/columnstore/VERSION && " + + "CURRENT_VERSION=${COLUMNSTORE_VERSION_MAJOR}.${COLUMNSTORE_VERSION_MINOR}.${COLUMNSTORE_VERSION_PATCH} && " + + "aws s3 rm s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + "/ " + + "--recursive " + + "--exclude \"*\" " + + // include only debs/rpms with columnstore in names + "--include \"*columnstore*.deb\" " + + "--include \"*columnstore*.rpm\" " + + // but do not delete the ones matching CURRENT_VERSION + "--exclude \"*${CURRENT_VERSION}*.deb\" " + + "--exclude \"*${CURRENT_VERSION}*.rpm\" " + + "--only-show-errors", + + "aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors", 'echo "Data uploaded to: ' + publish_pkg_url + '"', make_clickable_link(publish_pkg_url), ], @@ -504,7 +512,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", SCCACHE_BUCKET: "cs-sccache", SCCACHE_REGION: "us-east-1", SCCACHE_S3_USE_SSL: "true", - SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + "${DRONE_PULL_REQUEST}", + SCCACHE_S3_KEY_PREFIX: result + branch + server + arch, }, commands: [ "mkdir /mdb/" + builddir + "/" + result, @@ -514,6 +522,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", 'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " + "--build-type RelWithDebInfo " + "--distro " + platform + " " + + "--build-path " + "/mdb/" + builddir + "/builddir " + "--build-packages --install-deps --sccache" + " " + customBootstrapParams + " " + customBootstrapParamsForExisitingPipelines(platform) + " | " + diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index eed9873c8..250cb245f 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -79,41 +79,30 @@ fi select_pkg_format ${OS} if [[ "$PKG_FORMAT" == "rpm" ]]; then - CTEST_BIN_NAME=:"ctest3" + CTEST_BIN_NAME="ctest3" else CTEST_BIN_NAME="ctest" fi install_sccache() { if [[ "$SCCACHE" == false ]]; then - return + return fi if [[ "$(arch)" == "x86_64" ]]; then - sccache_arch="x86_64" + sccache_arch="x86_64" else - sccache_arch="aarch64" + sccache_arch="aarch64" fi message "getting sccache..." - - if command -v apt-get &>/dev/null; then - apt-get clean - apt-get update -y - apt-get install -y curl - elif command -v yum &>/dev/null; then - yum install -y curl - fi || true - curl -L -o sccache.tar.gz \ - "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${sccache_arch}-unknown-linux-musl.tar.gz" + "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${sccache_arch}-unknown-linux-musl.tar.gz" tar xzf sccache.tar.gz install sccache*/sccache /usr/local/bin/ && message "sccache installed" } -install_sccache - install_deps() { if [[ $INSTALL_DEPS = false ]]; then return @@ -161,6 +150,7 @@ install_deps() { } install_deps +install_sccache cd $COLUMSNTORE_SOURCE_PATH COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD) @@ -776,15 +766,16 @@ construct_cmake_flags init_submodules if [[ $BUILD_PACKAGES = true ]]; then - modify_packaging - build_package - message_splitted "PACKAGES BUILD FINISHED" - run_unit_tests + + ( build_package && run_unit_tests ) + exit_code=$? + if [[ $SCCACHE = true ]]; then - sccache --show-stats + sccache --show-stats fi - exit 0 + + exit $exit_code fi stop_service diff --git a/build/install_clang_deb.sh b/build/install_clang_deb.sh new file mode 100755 index 000000000..b1c9fa9b3 --- /dev/null +++ b/build/install_clang_deb.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eo pipefail + +SCRIPT_LOCATION=$(dirname "$0") +source "$SCRIPT_LOCATION"/utils.sh + +CLANG_VERSION="$1" + +if [[ $# -ne 1 ]]; then + echo "Please pass clang-version as a first parameter" + exit 1 +fi + +change_ubuntu_mirror us + +message "Installing clang-${CLANG_VERSION}" + +retry_eval 5 apt-get clean && apt-get update && apt-get install -y wget curl lsb-release software-properties-common gnupg +wget https://apt.llvm.org/llvm.sh +bash llvm.sh $CLANG_VERSION +export CC=/usr/bin/clang +export CXX=/usr/bin/clang++ + + + + +