database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-10-28 19:54:55 +03:00
Code Activity

Merge branch 'stable-23.10' into tests-12.1

Browse Source
This commit is contained in:
Timofey Turenko
2025-10-24 20:28:32 +03:00
committed by GitHub
parent ccd3324e80 dd82f2d91d
commit ae8d8c68b7
34 changed files with 1745 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.drone.jsonnet
View File

@@ -17,7 +17,7 @@ local platforms = {
};
local extra_servers_platforms = {
[current_branch]: ["rockylinux:9", "debian:13", "ubuntu:24.04"],
[current_branch]: ["rockylinux:9", "debian:13", "ubuntu:24.04", "ubuntu:22.04"],
};
//local archs = ["amd64", "arm64"];

1
CMakeLists.txt
View File

@@ -87,6 +87,7 @@ foreach(component ${COMPONENTS})
add_subdirectory(${component})
endforeach()
add_dependencies(rbo GenError)
add_dependencies(udf_mysql GenError)
add_dependencies(funcexp GenError)
add_dependencies(oamcpp GenError)

4
VERSION
View File

@@ -1,4 +1,4 @@
COLUMNSTORE_VERSION_MAJOR=23
COLUMNSTORE_VERSION_MAJOR=25
COLUMNSTORE_VERSION_MINOR=10
COLUMNSTORE_VERSION_PATCH=6
COLUMNSTORE_VERSION_PATCH=1
COLUMNSTORE_VERSION_RELEASE=1

2
build/prepare_test_container.sh
View File

@@ -58,7 +58,7 @@ start_container() {
if [[ "$CONTAINER_NAME" == *smoke* ]]; then
docker_run_args+=(--memory 3g)
elif [[ "$CONTAINER_NAME" == *mtr* ]]; then
docker_run_args+=(--shm-size=500m --memory 12g --env MYSQL_TEST_DIR="$MTR_PATH")
docker_run_args+=(--shm-size=500m --memory 13g --env MYSQL_TEST_DIR="$MTR_PATH")
elif [[ "$CONTAINER_NAME" == *cmapi* ]]; then
docker_run_args+=(--env PYTHONPATH="${PYTHONPATH}")
elif [[ "$CONTAINER_NAME" == *upgrade* ]]; then

8
cmake/compiler_flags.cmake
View File

@@ -95,6 +95,14 @@ set(GNU_FLAGS # suppressed warnings
set(ASAN_FLAGS -U_FORTIFY_SOURCE -fsanitize=address -fsanitize-address-use-after-scope -fPIC)
# } end Sanitizers
# Check if built with enterprise configuration
if(MYSQL_SERVER_SUFFIX STREQUAL "-enterprise")
message(STATUS "ColumnStore: Compiling with ENTERPRISE features enabled")
my_check_and_set_compiler_flag("-DCOLUMNSTORE_COMPILED_WITH_ENTERPRISE")
else()
message(STATUS "ColumnStore: Compiling with COMMUNITY features")
endif()
# configured by cmake/configureEngine.cmake {
if(MASK_LONGDOUBLE)
my_check_and_set_compiler_flag("-DMASK_LONGDOUBLE")

5
cmapi/cmapi_server/process_dispatchers/systemd.py
View File

@@ -165,11 +165,6 @@ class SystemdDispatcher(BaseDispatcher):
"""
service_name = service
if service_name == 'mcs-workernode':
# Run pre-stop lock reset before saving BRM
# These stale locks can occur if the controllernode couldn't stop correctly
# and they cause mcs-savebrm.py to hang
release_shmem_locks(logging.getLogger(__name__))
service_name = f'{service_name}@1.service {service_name}@2.service'
cls._workernode_enable(False, use_sudo)

28
datatypes/mcs_datatype.cpp
View File

@@ -23,6 +23,7 @@
#include <sstream>
#include <cerrno>
#include <cstring>
#include <limits>
#include <utility>
#include <cassert>
@@ -427,6 +428,7 @@ const TypeHandler* TypeHandler::find_by_ddltype(const ddlpackage::ColumnType& ct
case ddlpackage::DDL_CLOB: return &mcs_type_handler_clob;
case ddlpackage::DDL_BLOB: return &mcs_type_handler_blob;
case ddlpackage::DDL_TEXT: return &mcs_type_handler_text;
case ddlpackage::DDL_JSON: return &mcs_type_handler_text;
case ddlpackage::DDL_UNSIGNED_TINYINT: return &mcs_type_handler_uint8;
case ddlpackage::DDL_UNSIGNED_SMALLINT: return &mcs_type_handler_uint16;
@@ -782,6 +784,32 @@ string TypeHandlerVarchar::formatPartitionInfo(const SystemCatalog::TypeAttribut
return formatPartitionInfoSInt64(attr, pi);
}
string TypeHandlerTemporal::formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr,
const MinMaxInfo& pi) const
{
ostringstreamL output;
// Check for empty/null partition
// For 4-byte temporal types (DATE), check int32 sentinels
// For 8-byte temporal types (DATETIME/TIMESTAMP), check int64 sentinels
bool isEmpty = false;
if (attr.colWidth == 4)
{
isEmpty = pi.isEmptyOrNullSInt32();
}
else
{
isEmpty = pi.isEmptyOrNullSInt64();
}
if (isEmpty)
output << setw(30) << "N/A" << setw(30) << "N/A";
else
output << setw(30) << format(SimpleValueSInt64(pi.min), attr) << setw(30)
<< format(SimpleValueSInt64(pi.max), attr);
return output.str();
}
/****************************************************************************/
execplan::SimpleColumn* TypeHandlerSInt8::newSimpleColumn(const DatabaseQualifiedColumnName& name,

10
datatypes/mcs_datatype.h
View File

@@ -745,6 +745,11 @@ class MinMaxInfo
: greaterThan<int64_t, int64_t>(mm.min, mm.max);
}
}
bool isEmptyOrNullSInt32() const
{
return min == std::numeric_limits<int32_t>::max() && max == std::numeric_limits<int32_t>::min();
}
bool isEmptyOrNullSInt64() const
{
return min == std::numeric_limits<int64_t>::max() && max == std::numeric_limits<int64_t>::min();
@@ -2414,10 +2419,7 @@ class TypeHandlerTemporal : public TypeHandler
{
public:
std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr,
const MinMaxInfo& i) const override
{
return formatPartitionInfoSInt64(attr, i);
}
const MinMaxInfo& i) const override;
execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name,
SystemCatalog::TypeHolderStd& ct,
const SimpleColumnParam& prm) const override;

1
dbcon/ddlpackage/ddl.l
View File

@@ -134,6 +134,7 @@ INDEX {return INDEX;}
INITIALLY {return INITIALLY;}
INT {return IDB_INT;}
INTEGER {return INTEGER;}
JSON { return JSON; }
KEY {return KEY;}
MATCH {return MATCH;}
MAX_ROWS {return MAX_ROWS;}

30
dbcon/ddlpackage/ddl.y
View File

@@ -56,7 +56,7 @@ int ddllex(YYSTYPE* ddllval, void* yyscanner);
void ddlerror(struct pass_to_bison* x, char const *s);
char* copy_string(const char *str);
void fix_column_length_and_charset(SchemaObject* elem, const CHARSET_INFO* def_cs, myf utf8_flag)
void postprocess_column_information(SchemaObject* elem, const CHARSET_INFO* def_cs, myf utf8_flag)
{
auto* column = dynamic_cast<ColumnDef*>(elem);
@@ -104,6 +104,17 @@ void fix_column_length_and_charset(SchemaObject* elem, const CHARSET_INFO* def_c
else
column->fType->fLength = 16777215;
}
if (column->fType->fType == DDL_JSON)
{
CHARSET_INFO* cs = &my_charset_utf8mb4_bin;
column->fType->fCharset = cs->cs_name.str;
column->fType->fCollate = cs->coll_name.str;
column->fType->fCharsetNum = cs->number;
column->fType->fLength = 16777215;
column->fConstraints.push_back(new ColumnConstraintDef(DDL_VALIDATE_JSON));
}
}
%}
@@ -155,7 +166,7 @@ CHARACTER CHECK CLOB COLUMN
BOOL BOOLEAN
COLUMNS COMMENT CONSTRAINT CONSTRAINTS CREATE CURRENT_USER DATETIME DEC
DECIMAL DEFAULT DEFERRABLE DEFERRED IDB_DELETE DROP ENGINE
FOREIGN FULL IMMEDIATE INDEX INITIALLY IDB_INT INTEGER KEY LONGBLOB LONGTEXT
FOREIGN FULL IMMEDIATE INDEX INITIALLY IDB_INT INTEGER JSON KEY LONGBLOB LONGTEXT
MATCH MAX_ROWS MEDIUMBLOB MEDIUMTEXT MEDIUMINT
MIN_ROWS MODIFY NO NOT NULL_TOK NUMBER NUMERIC ON PARTIAL PRECISION PRIMARY
REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TINYBLOB TINYTEXT
@@ -355,7 +366,7 @@ create_table_statement:
{
for (auto* elem : *$6)
{
fix_column_length_and_charset(elem, x->default_table_charset, x->utf8_flag);
postprocess_column_information(elem, x->default_table_charset, x->utf8_flag);
}
$$ = new CreateTableStatement(new TableDef($4, $6, $8));
}
@@ -719,17 +730,17 @@ ata_add_column:
/* See the documentation for SchemaObject for an explanation of why we are using
* dynamic_cast here.
*/
ADD column_def { fix_column_length_and_charset($2, x->default_table_charset, x->utf8_flag); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($2));}
| ADD COLUMN column_def { fix_column_length_and_charset($3, x->default_table_charset, x->utf8_flag); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($3));}
ADD column_def { postprocess_column_information($2, x->default_table_charset, x->utf8_flag); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($2));}
| ADD COLUMN column_def { postprocess_column_information($3, x->default_table_charset, x->utf8_flag); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($3));}
| ADD '(' table_element_list ')' {
for (auto* elem : *$3) {
fix_column_length_and_charset(elem, x->default_table_charset, x->utf8_flag);
postprocess_column_information(elem, x->default_table_charset, x->utf8_flag);
}
$$ = new AtaAddColumns($3);
}
| ADD COLUMN '(' table_element_list ')' {
for (auto* elem : *$4) {
fix_column_length_and_charset(elem, x->default_table_charset, x->utf8_flag);
postprocess_column_information(elem, x->default_table_charset, x->utf8_flag);
}
$$ = new AtaAddColumns($4);
}
@@ -1067,6 +1078,11 @@ text_type:
$$ = new ColumnType(DDL_TEXT);
$$->fLength = 16777215;
}
| JSON
{
$$ = new ColumnType(DDL_JSON);
$$->fLength = 16777215;
}
;
numeric_type:

6
dbcon/ddlpackage/ddlpkg.h
View File

@@ -160,6 +160,7 @@ enum DDL_CONSTRAINTS
DDL_REFERENCES,
DDL_NOT_NULL,
DDL_AUTO_INCREMENT,
DDL_VALIDATE_JSON,
DDL_INVALID_CONSTRAINT
};
/** @brief
@@ -170,7 +171,8 @@ const std::string ConstraintString[] = {"primary",
"unique",
"references",
"not_null",
"auto_increment"
"auto_increment",
"validate_json",
""};
/** @brief Datatype List
@@ -210,6 +212,7 @@ enum DDL_DATATYPES
DDL_TEXT,
DDL_TIME,
DDL_TIMESTAMP,
DDL_JSON,
DDL_INVALID_DATATYPE
};
@@ -956,6 +959,7 @@ struct ColumnType
/** @brief Is the TEXT column has explicit defined length, ie TEXT(1717) */
bool fExplicitLength;
};
/** @brief A column constraint definition.

1
dbcon/ddlpackage/serialize.cpp
View File

@@ -1167,6 +1167,7 @@ int ColumnType::serialize(ByteStream& bytestream)
messageqcpp::ByteStream::octbyte nextVal = fNextvalue;
messageqcpp::ByteStream::quadbyte charsetNum = fCharsetNum;
// write column types
bytestream << ftype;
bytestream << length;

11
dbcon/ddlpackageproc/ddlpackageprocessor.cpp
View File

@@ -195,9 +195,10 @@ execplan::CalpontSystemCatalog::ColDataType DDLPackageProcessor::convertDataType
case ddlpackage::DDL_BLOB: colDataType = CalpontSystemCatalog::BLOB; break;
case ddlpackage::DDL_TEXT: colDataType = CalpontSystemCatalog::TEXT; break;
case ddlpackage::DDL_TEXT:
case ddlpackage::DDL_JSON: colDataType = CalpontSystemCatalog::TEXT; break;
default: throw runtime_error("Unsupported datatype!");
default: throw runtime_error("Unsupported DDL datatype!");
}
return colDataType;
@@ -228,6 +229,8 @@ std::string DDLPackageProcessor::buildTableConstraintName(const int oid, ddlpack
case ddlpackage::DDL_NOT_NULL: prefix = "nk_"; break;
case ddlpackage::DDL_VALIDATE_JSON: prefix = "jk_"; break;
default: throw runtime_error("Unsupported constraint type!"); break;
}
@@ -261,6 +264,8 @@ std::string DDLPackageProcessor::buildColumnConstraintName(const std::string& sc
case ddlpackage::DDL_NOT_NULL: prefix = "nk_"; break;
case ddlpackage::DDL_VALIDATE_JSON: prefix = "jk_"; break;
default: throw runtime_error("Unsupported constraint type!"); break;
}
@@ -288,6 +293,8 @@ char DDLPackageProcessor::getConstraintCode(ddlpackage::DDL_CONSTRAINTS type)
case ddlpackage::DDL_NOT_NULL: constraint_type = 'n'; break;
case ddlpackage::DDL_VALIDATE_JSON: constraint_type = 'j'; break;
default: constraint_type = '0'; break;
}

11
dbcon/mysql/ha_mcs.cpp
View File

@@ -77,6 +77,15 @@ pthread_mutex_t mcs_mutex;
#endif
#define DEBUG_RETURN return
bool isEnterprise()
{
#ifdef COLUMNSTORE_COMPILED_WITH_ENTERPRISE
return true;
#else
return false;
#endif
}
/**
@brief
Function we use in the creation of our hash to get key.
@@ -1866,7 +1875,7 @@ static int columnstore_init_func(void* p)
mcs_hton->create_unit = create_columnstore_unit_handler;
mcs_hton->db_type = DB_TYPE_AUTOASSIGN;
if (get_innodb_queries_uses_mcs())
if (isEnterprise() && get_innodb_queries_uses_mcs())
{
std::cerr << "Columnstore: innodb_queries_uses_mcs is set, redirecting all InnoDB queries to Columnstore."
<< std::endl;

25
dbcon/mysql/ha_mcs_ddl.cpp
View File

@@ -125,7 +125,7 @@ CalpontSystemCatalog::ColDataType convertDataType(const ddlpackage::ColumnType&
const datatypes::TypeHandler* h = datatypes::TypeHandler::find_by_ddltype(ct);
if (!h)
{
throw runtime_error("Unsupported datatype!");
throw runtime_error("Unsupported datatype to convert from!");
return CalpontSystemCatalog::UNDEFINED;
}
return h->code();
@@ -822,10 +822,11 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta
if (createTable->fTableDef->fColumns[i]->fConstraints.size() > 0)
{
// support default value and NOT NULL constraint
// support default value, JSON validation and NOT NULL constraint
for (uint32_t j = 0; j < createTable->fTableDef->fColumns[i]->fConstraints.size(); j++)
{
if (createTable->fTableDef->fColumns[i]->fConstraints[j]->fConstraintType != DDL_NOT_NULL)
auto ctype = createTable->fTableDef->fColumns[i]->fConstraints[j]->fConstraintType;
if (ctype != DDL_NOT_NULL && ctype != DDL_VALIDATE_JSON)
{
rc = 1;
thd->get_stmt_da()->set_overwrite_status(true);
@@ -1226,7 +1227,8 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta
// support default value and NOT NULL constraint
for (uint32_t j = 0; j < addColumnPtr->fColumnDef->fConstraints.size(); j++)
{
if (addColumnPtr->fColumnDef->fConstraints[j]->fConstraintType != DDL_NOT_NULL)
auto ctype = addColumnPtr->fColumnDef->fConstraints[j]->fConstraintType;
if (ctype != DDL_NOT_NULL && ctype != DDL_VALIDATE_JSON)
{
rc = 1;
thd->get_stmt_da()->set_overwrite_status(true);
@@ -1359,6 +1361,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta
}
// For TIMESTAMP, if no constraint is given, default to NOT NULL
// XXX: see same code conditionally enabled for specific MariaDB version.
if (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_TIMESTAMP &&
addColumnPtr->fColumnDef->fConstraints.empty())
{
@@ -1611,7 +1614,8 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta
//@Bug 5274. support default value and NOT NULL constraint
for (uint32_t j = 0; j < addColumnsPtr->fColumns[0]->fConstraints.size(); j++)
{
if (addColumnsPtr->fColumns[0]->fConstraints[j]->fConstraintType != DDL_NOT_NULL)
auto ctype = addColumnsPtr->fColumns[0]->fConstraints[j]->fConstraintType;
if (ctype != DDL_NOT_NULL && ctype != DDL_VALIDATE_JSON)
{
rc = 1;
thd->get_stmt_da()->set_overwrite_status(true);
@@ -1744,6 +1748,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta
}
// For TIMESTAMP, if no constraint is given, default to NOT NULL
// XXX: please see conditional to MariaDB version enablement of similar code.
if (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_TIMESTAMP &&
addColumnsPtr->fColumns[0]->fConstraints.empty())
{
@@ -2073,6 +2078,16 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta
return rc;
}
}
else if (renameColumnsPtr->fConstraints[j]->fConstraintType == DDL_VALIDATE_JSON)
{
rc = 1;
thd->get_stmt_da()->set_overwrite_status(true);
thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED,
(IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str());
ci->alterTableState = cal_connection_info::NOT_ALTER;
ci->isAlter = false;
return rc;
}
else
{
rc = 1;

35
dbcon/mysql/ha_mcs_execplan_walks.cpp
View File

@@ -70,6 +70,31 @@ class RecursionCounter
cal_impl_if::gp_walk_info* fgwip;
};
// RAII guard to automatically clean up work stacks on error.
// If a fatal parse error occurs, this ensures allocated objects
// are deleted when the guard goes out of scope, preventing memory leaks.
class CleanupGuard
{
private:
CleanupGuard() = delete;
CleanupGuard(const CleanupGuard&) = delete;
CleanupGuard& operator=(const CleanupGuard&) = delete;
public:
explicit CleanupGuard(cal_impl_if::gp_walk_info* gwip) : fgwip(gwip)
{
}
~CleanupGuard()
{
if (fgwip->fatalParseError)
{
clearDeleteStacks(*fgwip);
}
}
cal_impl_if::gp_walk_info* fgwip;
};
bool isSecondArgumentConstItem(Item_func* ifp)
{
return (ifp->argument_count() == 2 && ifp->arguments()[1]->type() == Item::CONST_ITEM);
@@ -89,6 +114,9 @@ void gp_walk(const Item* item, void* arg)
cal_impl_if::gp_walk_info* gwip = static_cast<cal_impl_if::gp_walk_info*>(arg);
idbassert(gwip);
// RAII guard: automatically cleans up work stacks on error when guard goes out of scope
CleanupGuard cleanup(gwip);
// Bailout...
if (gwip->fatalParseError)
return;
@@ -815,12 +843,7 @@ void gp_walk(const Item* item, void* arg)
}
}
// Clean up allocated objects if a fatal parse error occurred
if (gwip->fatalParseError)
{
clearDeleteStacks(*gwip);
}
// CleanupGuard will automatically clean up if fatalParseError is set
return;
}

32
dbcon/mysql/ha_mcs_sysvars.cpp
View File

@@ -21,6 +21,23 @@
#include "ha_mcs_sysvars.h"
#include "mcsconfig.h"
#if MYSQL_VERSION_ID >= 110800
#define DEFINE_TYPELIB(A) { \
array_elements(A) - 1, \
#A, \
A, \
nullptr, \
nullptr \
}
#else
#define DEFINE_TYPELIB(A) { \
array_elements(A) - 1, \
#A, \
A, \
nullptr \
}
#endif
const char* mcs_compression_type_names[] = {"SNAPPY", // 0
"SNAPPY", // 1
"SNAPPY", // 2
@@ -29,9 +46,7 @@ const char* mcs_compression_type_names[] = {"SNAPPY", // 0
#endif
NullS};
static TYPELIB mcs_compression_type_names_lib = {array_elements(mcs_compression_type_names) - 1,
"mcs_compression_type_names", mcs_compression_type_names,
NULL};
static TYPELIB mcs_compression_type_names_lib = DEFINE_TYPELIB(mcs_compression_type_names);
// compression type
static MYSQL_THDVAR_ENUM(compression_type, PLUGIN_VAR_RQCMDARG,
@@ -63,9 +78,7 @@ static MYSQL_THDVAR_ULONGLONG(original_option_bits, PLUGIN_VAR_NOSYSVAR | PLUGIN
const char* mcs_select_handler_mode_values[] = {"OFF", "ON", "AUTO", NullS};
static TYPELIB mcs_select_handler_mode_values_lib = {array_elements(mcs_select_handler_mode_values) - 1,
"mcs_select_handler_mode_values",
mcs_select_handler_mode_values, NULL};
static TYPELIB mcs_select_handler_mode_values_lib = DEFINE_TYPELIB(mcs_select_handler_mode_values);
static MYSQL_THDVAR_ENUM(select_handler, PLUGIN_VAR_RQCMDARG,
"Set the MCS select_handler to Disabled, Enabled, or Automatic",
@@ -178,9 +191,8 @@ static MYSQL_THDVAR_ULONG(import_for_batchinsert_enclosed_by, PLUGIN_VAR_RQCMDAR
const char* mcs_use_import_for_batchinsert_mode_values[] = {"OFF", "ON", "ALWAYS", NullS};
static TYPELIB mcs_use_import_for_batchinsert_mode_values_lib = {
array_elements(mcs_use_import_for_batchinsert_mode_values) - 1,
"mcs_use_import_for_batchinsert_mode_values", mcs_use_import_for_batchinsert_mode_values, NULL};
static TYPELIB mcs_use_import_for_batchinsert_mode_values_lib =
DEFINE_TYPELIB(mcs_use_import_for_batchinsert_mode_values);
static MYSQL_THDVAR_ENUM(use_import_for_batchinsert, PLUGIN_VAR_RQCMDARG,
"LOAD DATA INFILE and INSERT..SELECT will use cpimport internally",
@@ -694,3 +706,5 @@ bool get_innodb_queries_uses_mcs()
{
return SYSVAR(innodb_queries_use_mcs);
}
#undef DEFINE_TYPELIB

12
docs/QueryAccelerator.md
View File

@@ -1,11 +1,11 @@
# What is Query Accelarator
# What is Query Accelerator
Query Accelarator is a feature that allows MariaDB to use ColumnStore to execute queries that are otherwise executed by InnoDB.
Query Accelerator is a feature that allows MariaDB to use ColumnStore to execute queries that are otherwise executed by InnoDB.
Under the hood Columnstore:
- receives a query
- searches for applicable Engine Independent statistics for InnoDB table index column
- applies RBO rule to transform its InnoDB tables into a number of UNIONs over non-overlapping ranges of a suitable InnoDB table index
- retrives the data in parallel from MariaDB and runs it using Columnstore runtime
- retrieves the data in parallel from MariaDB and runs it using Columnstore runtime
# How to enable Query Accelerator
@@ -26,11 +26,11 @@ analyze table <table_name> persistent for columns (<column_name>) indexes();
- `columnstore_unstable_optimizer`: enables unstable optimizer that is required for Query Accelerator RBO rule
- `columnstore_select_handler`: enables/disables ColumnStore processing for InnoDB tables
- `columnstore_query_accel_parallel_factor` : controls the number of parallel ranges to be used for Query Accelerator
- `columnstore_query_accel_parallel_factor`: controls the number of parallel ranges to be used for Query Accelerator
Watch out `max_connections`. If you set `columnstore_query_accel_parallel_factor` to a high value, you may need to increase `max_connections` to avoid connection pool exhaustion.
# How to verify QA is being used
There are two ways to verify QA is being used:
# How to verify Query Accelerator is being used
There are two ways to verify Query Accelerator is being used:
1. Use `select mcs_get_plan('rules')` to get a list of the rules that were applied to the query.
2. Look for patterns like `derived table - $added_sub_#db_name_#table_name_X` in the optimized plan using `select mcs_get_plan('optimized')`.

1
mysql-test/columnstore/basic/disabled.def
View File

@@ -1,4 +1,5 @@
mcs80_set_operations : BUG#MCOL-4273 2020-08-27 susil.behera@mariadb.com
mcs82_update_join: unstable MCOL-6201 2025-10-06 roman.nozdrin@mariadb.com
mcs211_idbExtentId_function : 2021-07-12 david.halla@mariadb.com
mcs212_idbExtentMax_function : 2020-11-30 bharath.bokka@mariadb.com
mcs213_idbExtentMin_function : 2020-11-30 bharath.bokka@mariadb.com

38
mysql-test/columnstore/basic/r/JSON-type.result Normal file
View File

@@ -0,0 +1,38 @@
DROP DATABASE IF EXISTS json_type;
CREATE DATABASE json_type;
USE json_type;
CREATE TABLE tj(j JSON) ENGINE=COLUMNSTORE;
INSERT INTO tj(j) VALUES ('()');
ERROR 23000: CONSTRAINT `tj.j` failed for `json_type`.`tj`
INSERT INTO tj(j) VALUES ('{');
ERROR 23000: CONSTRAINT `tj.j` failed for `json_type`.`tj`
INSERT INTO tj(j) VALUES ('[1, 2,');
ERROR 23000: CONSTRAINT `tj.j` failed for `json_type`.`tj`
INSERT INTO tj(j) VALUES ('"unclosed string');
ERROR 23000: CONSTRAINT `tj.j` failed for `json_type`.`tj`
INSERT INTO tj(j) VALUES ('{"key": value}');
ERROR 23000: CONSTRAINT `tj.j` failed for `json_type`.`tj`
INSERT INTO tj(j) VALUES ('');
ERROR 23000: CONSTRAINT `tj.j` failed for `json_type`.`tj`
INSERT INTO tj(j) VALUES ('{"good":1}'), ('bad'), ('[]');
ERROR 23000: CONSTRAINT `tj.j` failed for `json_type`.`tj`
INSERT INTO tj(j) VALUES ('{"nested": [1, true, null]}');
INSERT INTO tj(j) VALUES ('[ "hello", 42, false ]');
INSERT INTO tj(j) VALUES (NULL);
SELECT * FROM tj WHERE j IS NULL;
j
NULL
INSERT INTO tj(j) VALUES ('null');
SELECT * FROM tj WHERE j = 'null';
j
null
INSERT INTO tj(j) VALUES ('[]'), ('{}'), ('"a"');
SELECT * FROM tj WHERE j = '"A"';
j
SELECT * FROM tj WHERE j = '"a"';
j
"a"
INSERT INTO tj(j) VALUES ('{"a":"b", "b":"a"}');
SELECT * FROM tj WHERE j = '{"b":"a","a":"b"}';
j
DROP DATABASE json_type;

2
mysql-test/columnstore/basic/r/mcs59_nonscalar_datatypes.result
View File

@@ -5,6 +5,4 @@ CREATE TABLE t_enum(col ENUM('min','max','avg'))ENGINE=Columnstore;
ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types.
CREATE TABLE t_set(col SET('x','y','z'))ENGINE=Columnstore;
ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types.
CREATE TABLE t_json(col JSON)ENGINE=Columnstore;
ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types.
DROP DATABASE mcs59_db;

2
mysql-test/columnstore/basic/r/mcs9_create_table_negative.result
View File

@@ -7,8 +7,6 @@ CREATE TABLE t2(t2_binary BINARY(3))ENGINE=Columnstore;
ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types.
CREATE TABLE t3(t3_set SET('a','b'))ENGINE=Columnstore;
ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types.
CREATE TABLE t4(t4_json JSON)ENGINE=Columnstore;
ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types.
CREATE TABLE $table(col1 INT)ENGINE=columnstore;
ERROR 42000: The storage engine for the table doesn't support The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types.
CREATE TABLE abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm(col1 VARCHAR(90))ENGINE=Columnstore;

39
mysql-test/columnstore/basic/t/JSON-type.test Normal file
View File

@@ -0,0 +1,39 @@
--disable_warnings
DROP DATABASE IF EXISTS json_type;
--enable_warnings
CREATE DATABASE json_type;
USE json_type;
CREATE TABLE tj(j JSON) ENGINE=COLUMNSTORE;
--error 4025
INSERT INTO tj(j) VALUES ('()');
--error 4025
INSERT INTO tj(j) VALUES ('{');
--error 4025
INSERT INTO tj(j) VALUES ('[1, 2,');
--error 4025
INSERT INTO tj(j) VALUES ('"unclosed string');
--error 4025
INSERT INTO tj(j) VALUES ('{"key": value}');
--error 4025
INSERT INTO tj(j) VALUES ('');
# batch insert with mix of valid and invalid, should fail entirely
--error 4025
INSERT INTO tj(j) VALUES ('{"good":1}'), ('bad'), ('[]');
INSERT INTO tj(j) VALUES ('{"nested": [1, true, null]}');
INSERT INTO tj(j) VALUES ('[ "hello", 42, false ]');
INSERT INTO tj(j) VALUES (NULL);
SELECT * FROM tj WHERE j IS NULL;
INSERT INTO tj(j) VALUES ('null');
SELECT * FROM tj WHERE j = 'null';
INSERT INTO tj(j) VALUES ('[]'), ('{}'), ('"a"'); # valid
SELECT * FROM tj WHERE j = '"A"'; # empty set.
SELECT * FROM tj WHERE j = '"a"'; # single row.
INSERT INTO tj(j) VALUES ('{"a":"b", "b":"a"}');
SELECT * FROM tj WHERE j = '{"b":"a","a":"b"}'; # empty set, comparison is not structural.
# UPDATE is not tested because it does not work.
#UPDATE tj SET j = CONCAT(j,'()');
DROP DATABASE json_type;

2
mysql-test/columnstore/basic/t/mcs59_nonscalar_datatypes.test
View File

@@ -15,8 +15,6 @@ USE mcs59_db;
CREATE TABLE t_enum(col ENUM('min','max','avg'))ENGINE=Columnstore;
--error ER_CHECK_NOT_IMPLEMENTED
CREATE TABLE t_set(col SET('x','y','z'))ENGINE=Columnstore;
--error ER_CHECK_NOT_IMPLEMENTED
CREATE TABLE t_json(col JSON)ENGINE=Columnstore;
# Clean UP
DROP DATABASE mcs59_db;

2
mysql-test/columnstore/basic/t/mcs9_create_table_negative.test
View File

@@ -17,8 +17,6 @@ CREATE TABLE t1(t1_enum ENUM('one','two','three'))ENGINE=Columnstore;
CREATE TABLE t2(t2_binary BINARY(3))ENGINE=Columnstore;
--error 1178
CREATE TABLE t3(t3_set SET('a','b'))ENGINE=Columnstore;
--error 1178
CREATE TABLE t4(t4_json JSON)ENGINE=Columnstore;
# with unsupported table name
--error 1178

17
mysql-test/columnstore/bugfixes/MCOL-6198-segfault-crossengine-join.result Normal file
View File

@@ -0,0 +1,17 @@
DROP DATABASE IF EXISTS mcol_6198;
CREATE DATABASE mcol_6198;
SELECT
v.c36 AS hst,
m.p6 AS g,
COALESCE(g0.eg_id,'MARIA') AS g01,
COALESCE(g1.eg_id,'MARIA') AS g02,
SUM(v.c758 * m.p42 / 100 + v.c759 * m.p42 / 100 + v.c760 * m.p42 / 100) AS sval
FROM
c AS v
JOIN p m on (v.c4 = m.a4)
LEFT OUTER JOIN group_g01 AS g0 ON g0.key_id=m.p6
LEFT OUTER JOIN group_g02 AS g1 ON g1.key_id=m.p6
WHERE
1=1
GROUP BY c36,p6,g01,g02;
DROP DATABASE mcol_6198;

1234
mysql-test/columnstore/bugfixes/MCOL-6198-segfault-crossengine-join.test Normal file
View File

File diff suppressed because it is too large Load Diff

34
mysql-test/columnstore/bugfixes/mcol_4623.result Normal file
View File

@@ -0,0 +1,34 @@
DROP DATABASE IF EXISTS mcol_4623;
CREATE DATABASE mcol_4623;
USE mcol_4623;
CREATE TABLE t1 (a DOUBLE UNSIGNED) Engine=ColumnStore;
INSERT INTO t1 VALUES (1000);
SELECT SEC_TO_TIME(a) FROM t1 ORDER BY 1;
SEC_TO_TIME(a)
00:16:40.000000
DROP TABLE t1;
CREATE TABLE t1 (a TIME) ENGINE=ColumnStore;
INSERT INTO t1 VALUES ('17:31:27');
SELECT SEC_TO_TIME(a) FROM t1 ORDER BY 1;
SEC_TO_TIME(a)
17:31:27
DROP TABLE t1;
CREATE TABLE t1 (a DATE) ENGINE=ColumnStore;
INSERT INTO t1 VALUES ('2023-07-23');
SELECT SEC_TO_TIME(a) FROM t1 ORDER BY 1;
SEC_TO_TIME(a)
838:59:59
DROP TABLE t1;
CREATE TABLE t1 (a DATETIME) ENGINE=ColumnStore;
INSERT INTO t1 VALUES ('2009-03-17 14:30:45');
SELECT SEC_TO_TIME(a) FROM t1 ORDER BY 1;
SEC_TO_TIME(a)
838:59:59
DROP TABLE t1;
CREATE TABLE t1 (a TIMESTAMP) ENGINE=ColumnStore;
INSERT INTO t1 VALUES ('2017-01-01 04:30:45');
SELECT SEC_TO_TIME(a) FROM t1 ORDER BY 1;
SEC_TO_TIME(a)
838:59:59
DROP TABLE t1;
DROP DATABASE mcol_4623;

45
mysql-test/columnstore/bugfixes/mcol_4623.test Normal file
View File

@@ -0,0 +1,45 @@
-- source ../include/have_columnstore.inc
--disable_warnings
DROP DATABASE IF EXISTS mcol_4623;
--enable_warnings
CREATE DATABASE mcol_4623;
USE mcol_4623;
CREATE TABLE t1 (a DOUBLE UNSIGNED) Engine=ColumnStore;
INSERT INTO t1 VALUES (1000);
SELECT SEC_TO_TIME(a) FROM t1 ORDER BY 1;
--disable_warnings
DROP TABLE t1;
--enable_warnings
CREATE TABLE t1 (a TIME) ENGINE=ColumnStore;
INSERT INTO t1 VALUES ('17:31:27');
SELECT SEC_TO_TIME(a) FROM t1 ORDER BY 1;
--disable_warnings
DROP TABLE t1;
--enable_warnings
CREATE TABLE t1 (a DATE) ENGINE=ColumnStore;
INSERT INTO t1 VALUES ('2023-07-23');
SELECT SEC_TO_TIME(a) FROM t1 ORDER BY 1;
--disable_warnings
DROP TABLE t1;
--enable_warnings
CREATE TABLE t1 (a DATETIME) ENGINE=ColumnStore;
INSERT INTO t1 VALUES ('2009-03-17 14:30:45');
SELECT SEC_TO_TIME(a) FROM t1 ORDER BY 1;
--disable_warnings
DROP TABLE t1;
--enable_warnings
CREATE TABLE t1 (a TIMESTAMP) ENGINE=ColumnStore;
INSERT INTO t1 VALUES ('2017-01-01 04:30:45');
SELECT SEC_TO_TIME(a) FROM t1 ORDER BY 1;
--disable_warnings
DROP TABLE t1;
--enable_warnings
--disable_warnings
DROP DATABASE mcol_4623;
--enable_warnings

42
mysql-test/columnstore/bugfixes/mcs_calshowpartitions_empty_date.result Normal file
View File

@@ -0,0 +1,42 @@
DROP DATABASE IF EXISTS test_empty_date;
CREATE DATABASE test_empty_date;
USE test_empty_date;
CREATE TABLE td (d DATE) ENGINE=ColumnStore;
SELECT calShowPartitions('td','d');
calShowPartitions('td','d')
Part# Min Max Status
0.0.1 N/A N/A Enabled
SELECT MIN(d) FROM td;
MIN(d)
NULL
SELECT calShowPartitions('td','d');
calShowPartitions('td','d')
Part# Min Max Status
0.0.1 N/A N/A Enabled
# Test with DATETIME as well
CREATE TABLE tdt (dt DATETIME) ENGINE=ColumnStore;
SELECT calShowPartitions('tdt','dt');
calShowPartitions('tdt','dt')
Part# Min Max Status
0.0.1 N/A N/A Enabled
SELECT MIN(dt) FROM tdt;
MIN(dt)
NULL
SELECT calShowPartitions('tdt','dt');
calShowPartitions('tdt','dt')
Part# Min Max Status
0.0.1 N/A N/A Enabled
# Test with TIMESTAMP
CREATE TABLE tts (ts TIMESTAMP) ENGINE=ColumnStore;
SELECT calShowPartitions('tts','ts');
calShowPartitions('tts','ts')
Part# Min Max Status
0.0.1 N/A N/A Enabled
SELECT MIN(ts) FROM tts;
MIN(ts)
NULL
SELECT calShowPartitions('tts','ts');
calShowPartitions('tts','ts')
Part# Min Max Status
0.0.1 N/A N/A Enabled
DROP DATABASE test_empty_date;

38
mysql-test/columnstore/bugfixes/mcs_calshowpartitions_empty_date.test Normal file
View File

@@ -0,0 +1,38 @@
--source ../include/have_columnstore.inc
--source ../include/functions.inc
--disable_warnings
DROP DATABASE IF EXISTS test_empty_date;
--enable_warnings
CREATE DATABASE test_empty_date;
USE test_empty_date;
CREATE TABLE td (d DATE) ENGINE=ColumnStore;
--replace_regex / 0\.0\.. / 0.0.1 /
SELECT calShowPartitions('td','d');
SELECT MIN(d) FROM td;
--replace_regex / 0\.0\.. / 0.0.1 /
SELECT calShowPartitions('td','d');
--echo # Test with DATETIME as well
CREATE TABLE tdt (dt DATETIME) ENGINE=ColumnStore;
--replace_regex / 0\.0\.. / 0.0.1 /
SELECT calShowPartitions('tdt','dt');
SELECT MIN(dt) FROM tdt;
--replace_regex / 0\.0\.. / 0.0.1 /
SELECT calShowPartitions('tdt','dt');
--echo # Test with TIMESTAMP
CREATE TABLE tts (ts TIMESTAMP) ENGINE=ColumnStore;
--replace_regex / 0\.0\.. / 0.0.1 /
SELECT calShowPartitions('tts','ts');
SELECT MIN(ts) FROM tts;
--replace_regex / 0\.0\.. / 0.0.1 /
SELECT calShowPartitions('tts','ts');
DROP DATABASE test_empty_date;
--source ../include/drop_functions.inc

50
primitives/primproc/batchprimitiveprocessor.cpp
View File

@@ -72,6 +72,24 @@ using namespace logging;
using namespace utils;
using namespace joblist;
#define idblog(x) \
do \
{ \
{ \
std::ostringstream os; \
\
os << __FILE__ << "@" << __LINE__ << ": \'" << x << "\'"; \
std::cerr << os.str() << std::endl; \
logging::MessageLog logger((logging::LoggingID())); \
logging::Message message; \
logging::Message::Args args; \
\
args.add(os.str()); \
message.format(args); \
logger.logErrorMessage(message); \
} \
} while (0)
namespace primitiveprocessor
{
#ifdef PRIMPROC_STOPWATCH
@@ -386,15 +404,6 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs)
smallSideRGRecvd = true;
}
for (uint j = 0; j < processorThreads; ++j)
{
auto tlHasher = TupleJoiner::TypelessDataHasher(&outputRG, &tlLargeSideKeyColumns[i],
mSmallSideKeyColumnsPtr, mSmallSideRGPtr);
auto tlComparator = TupleJoiner::TypelessDataComparator(&outputRG, &tlLargeSideKeyColumns[i],
mSmallSideKeyColumnsPtr, mSmallSideRGPtr);
tlJoiners[i][j].reset(new TLJoiner(10, tlHasher, tlComparator,
utils::STLPoolAllocator<TLJoiner::value_type>(resourceManager)));
}
}
}
@@ -408,6 +417,7 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs)
{
deserializeVector(bs, smallSideRGs);
idbassert(smallSideRGs.size() == joinerCount);
mSmallSideRGPtr = mSmallSideRGPtr ? &smallSideRGs[0] : nullptr;
smallSideRowLengths.reset(new uint32_t[joinerCount]);
smallSideRowData.reset(new RGData[joinerCount]);
smallNullRowData.reset(new RGData[joinerCount]);
@@ -438,6 +448,24 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs)
bs >> largeSideRG;
bs >> joinedRG;
}
for (i = 0; i < joinerCount; i++)
{
if (!typelessJoin[i])
{
continue;
}
for (uint j = 0; j < processorThreads; ++j)
{
auto tlHasher = TupleJoiner::TypelessDataHasher(&outputRG, &tlLargeSideKeyColumns[i],
mSmallSideKeyColumnsPtr, mSmallSideRGPtr);
auto tlComparator = TupleJoiner::TypelessDataComparator(&outputRG, &tlLargeSideKeyColumns[i],
mSmallSideKeyColumnsPtr, mSmallSideRGPtr);
tlJoiners[i][j].reset(new TLJoiner(10, tlHasher, tlComparator,
utils::STLPoolAllocator<TLJoiner::value_type>(resourceManager)));
}
}
}
pthread_mutex_unlock(&objLock);
@@ -2412,8 +2440,7 @@ SBPP BatchPrimitiveProcessor::duplicate()
bpp->hasJoinFEFilters = hasJoinFEFilters;
bpp->hasSmallOuterJoin = hasSmallOuterJoin;
bpp->mJOINHasSkewedKeyColumn = mJOINHasSkewedKeyColumn;
bpp->mSmallSideRGPtr = mSmallSideRGPtr;
bpp->mSmallSideKeyColumnsPtr = mSmallSideKeyColumnsPtr;
bpp->mSmallSideKeyColumnsPtr = &(*bpp->tlSmallSideKeyColumns);
if (!getTupleJoinRowGroupData && mJOINHasSkewedKeyColumn)
{
idbassert(!smallSideRGs.empty());
@@ -2440,6 +2467,7 @@ SBPP BatchPrimitiveProcessor::duplicate()
bpp->smallNullPointers = smallNullPointers;
bpp->joinedRG = joinedRG;
}
bpp->mSmallSideRGPtr = &bpp->smallSideRGs[0];
#ifdef __FreeBSD__
pthread_mutex_unlock(&bpp->objLock);

44
utils/funcexp/func_sec_to_time.cpp
View File

@@ -67,33 +67,73 @@ string Func_sec_to_time::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool&
case execplan::CalpontSystemCatalog::USMALLINT:
{
val = parm[0]->data()->getIntVal(row, isNull);
break;
}
break;
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::UDOUBLE:
{
datatypes::TDouble d(parm[0]->data()->getDoubleVal(row, isNull));
val = d.toMCSSInt64Round();
break;
}
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UFLOAT:
{
datatypes::TDouble d(parm[0]->data()->getFloatVal(row, isNull));
val = d.toMCSSInt64Round();
break;
}
case execplan::CalpontSystemCatalog::LONGDOUBLE:
{
datatypes::TLongDouble d(parm[0]->data()->getLongDoubleVal(row, isNull));
val = d.toMCSSInt64Round();
break;
}
break;
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
{
val = parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round();
break;
}
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::VARCHAR:
case execplan::CalpontSystemCatalog::TEXT:
{
val = parm[0]->data()->getIntVal(row, isNull);
break;
}
case execplan::CalpontSystemCatalog::TIME:
{
int64_t timeVal = parm[0]->data()->getTimeIntVal(row, isNull);
uint32_t hour = (uint32_t)((timeVal >> 40) & 0xfff);
uint32_t minute = (uint32_t)((timeVal >> 32) & 0xff);
uint32_t second = (uint32_t)((timeVal >> 24) & 0xff);
val = (int64_t)(hour * 3600 + minute * 60 + second);
break;
}
case execplan::CalpontSystemCatalog::DATE:
case execplan::CalpontSystemCatalog::DATETIME:
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
return "838:59:59";
break;
}
case execplan::CalpontSystemCatalog::BLOB:
case execplan::CalpontSystemCatalog::CLOB:
case execplan::CalpontSystemCatalog::VARBINARY:
case execplan::CalpontSystemCatalog::STRINT:
case execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE:
case execplan::CalpontSystemCatalog::UNDEFINED:
{
val = parm[0]->data()->getIntVal(row, isNull);
break;
}

3
writeengine/server/we_ddlcommon.h
View File

@@ -178,7 +178,8 @@ inline int convertDataType(int dataType)
case ddlpackage::DDL_BLOB: calpontDataType = execplan::CalpontSystemCatalog::BLOB; break;
case ddlpackage::DDL_TEXT: calpontDataType = execplan::CalpontSystemCatalog::TEXT; break;
case ddlpackage::DDL_TEXT:
case ddlpackage::DDL_JSON: calpontDataType = execplan::CalpontSystemCatalog::TEXT; break;
case ddlpackage::DDL_UNSIGNED_TINYINT: calpontDataType = execplan::CalpontSystemCatalog::UTINYINT; break;