database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-07 03:22:57 +03:00
Code Activity

Merge branch 'develop' into MCOL-1052

Browse Source
This commit is contained in:
Andrew Hutchings
2018-05-05 06:29:04 +01:00
committed by GitHub
parent 00990ca5bd 47ccb3f259
commit c2b9016430
138 changed files with 3426 additions and 553 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dbcon/mysql/columnstore_info.sql
View File

@@ -52,21 +52,21 @@ CREATE PROCEDURE table_usage (IN t_schema char(64), IN t_name char(64))
CREATE TABLE columnstore_info.columnstore_files engine=myisam as (select * from information_schema.columnstore_files);
ALTER TABLE columnstore_info.columnstore_files ADD INDEX `object_id` (`object_id`);
IF t_name IS NOT NULL THEN
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
FROM
columnstore_info.columnstore_columns ics where table_name = t_name and (table_schema = t_schema or t_schema IS NULL)
group by table_schema, table_name
) q;
ELSEIF t_schema IS NOT NULL THEN
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
FROM
columnstore_info.columnstore_columns ics where table_schema = t_schema
group by table_schema, table_name
) q;
ELSE
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
FROM
columnstore_info.columnstore_columns ics

20
dbcon/mysql/ha_calpont_ddl.cpp
View File

@@ -174,6 +174,10 @@ uint32_t convertDataType(int dataType)
calpontDataType = CalpontSystemCatalog::DATETIME;
break;
case ddlpackage::DDL_TIME:
calpontDataType = CalpontSystemCatalog::TIME;
break;
case ddlpackage::DDL_CLOB:
calpontDataType = CalpontSystemCatalog::CLOB;
break;
@@ -2183,6 +2187,7 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti
THD* thd = current_thd;
string emsg;
ostringstream stmt1;
pair<string, string> fromPair;
pair<string, string> toPair;
string stmt;
@@ -2210,16 +2215,15 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti
return -1;
}
stmt = thd->query();
stmt += ';';
stmt1 << "alter table " << fromPair.second << " rename to " << toPair.second << ";";
stmt = stmt1.str();
string db;
if ( thd->db )
db = thd->db;
else if ( fromPair.first.length() != 0 )
if ( fromPair.first.length() != 0 )
db = fromPair.first;
else
db = toPair.first;
else if ( thd->db )
db = thd->db;
int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg);
@@ -2266,7 +2270,7 @@ extern "C"
int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg, compressiontype);
if (rc != 0)
push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str());
push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str());
return rc;
}

59
dbcon/mysql/ha_calpont_dml.cpp
View File

@@ -136,7 +136,8 @@ int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table)
(*field)->type() == MYSQL_TYPE_STRING ||
(*field)->type() == MYSQL_TYPE_DATE ||
(*field)->type() == MYSQL_TYPE_DATETIME ||
(*field)->type() == MYSQL_TYPE_DATETIME2 )
(*field)->type() == MYSQL_TYPE_DATETIME2 ||
(*field)->type() == MYSQL_TYPE_TIME )
vals.append("'");
while (ptr < end_ptr)
@@ -166,7 +167,8 @@ int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table)
(*field)->type() == MYSQL_TYPE_STRING ||
(*field)->type() == MYSQL_TYPE_DATE ||
(*field)->type() == MYSQL_TYPE_DATETIME ||
(*field)->type() == MYSQL_TYPE_DATETIME2 )
(*field)->type() == MYSQL_TYPE_DATETIME2 ||
(*field)->type() == MYSQL_TYPE_TIME )
vals.append("'");
}
}
@@ -838,11 +840,23 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_
// mariadb 10.1 compatibility -- MYSQL_TYPE_DATETIME2 introduced in mysql 5.6
MYSQL_TIME ltime;
const uchar* pos = buf;
longlong tmp = my_datetime_packed_from_binary(pos, 0);
longlong tmp = my_datetime_packed_from_binary(pos, table->field[colpos]->decimals());
TIME_from_longlong_datetime_packed(&ltime, tmp);
fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c",
ltime.year, ltime.month, ltime.day,
ltime.hour, ltime.minute, ltime.second, ci.delimiter);
if (!ltime.second_part)
{
fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c",
ltime.year, ltime.month, ltime.day,
ltime.hour, ltime.minute, ltime.second, ci.delimiter);
}
else
{
fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d.%ld%c",
ltime.year, ltime.month, ltime.day,
ltime.hour, ltime.minute, ltime.second,
ltime.second_part, ci.delimiter);
}
buf += table->field[colpos]->pack_length();
}
else
@@ -866,6 +880,39 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_
break;
}
case CalpontSystemCatalog::TIME:
{
if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT))
{
fprintf(ci.filePtr, "%c", ci.delimiter);
buf += table->field[colpos]->pack_length();
}
else
{
MYSQL_TIME ltime;
const uchar* pos = buf;
longlong tmp = my_time_packed_from_binary(pos, table->field[colpos]->decimals());
TIME_from_longlong_time_packed(&ltime, tmp);
if (!ltime.second_part)
{
fprintf(ci.filePtr, "%02d:%02d:%02d%c",
ltime.hour, ltime.minute, ltime.second, ci.delimiter);
}
else
{
fprintf(ci.filePtr, "%02d:%02d:%02d.%ld%c",
ltime.hour, ltime.minute, ltime.second,
ltime.second_part, ci.delimiter);
}
buf += table->field[colpos]->pack_length();
}
break;
}
case CalpontSystemCatalog::CHAR:
{
if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT))

199
dbcon/mysql/ha_calpont_execplan.cpp
View File

@@ -2681,6 +2681,11 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item)
ct.colDataType = CalpontSystemCatalog::DATETIME;
ct.colWidth = 8;
}
else if (item->field_type() == MYSQL_TYPE_TIME)
{
ct.colDataType = CalpontSystemCatalog::TIME;
ct.colWidth = 8;
}
if (item->field_type() == MYSQL_TYPE_BLOB)
{
@@ -2969,6 +2974,13 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp
break;
}
case Item::COND_ITEM:
{
// MCOL-1196: Allow COND_ITEM thru. They will be picked up
// by further logic. It may become desirable to add code here.
break;
}
default:
{
gwi.fatalParseError = true;
@@ -3530,8 +3542,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non
if (ifp->field_type() == MYSQL_TYPE_DATETIME ||
ifp->field_type() == MYSQL_TYPE_DATETIME2 ||
ifp->field_type() == MYSQL_TYPE_TIMESTAMP ||
ifp->field_type() == MYSQL_TYPE_TIMESTAMP2 ||
funcName == "add_time")
ifp->field_type() == MYSQL_TYPE_TIMESTAMP2)
{
CalpontSystemCatalog::ColType ct;
ct.colDataType = CalpontSystemCatalog::DATETIME;
@@ -3545,6 +3556,13 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non
ct.colWidth = 4;
fc->resultType(ct);
}
else if (ifp->field_type() == MYSQL_TYPE_TIME)
{
CalpontSystemCatalog::ColType ct;
ct.colDataType = CalpontSystemCatalog::TIME;
ct.colWidth = 8;
fc->resultType(ct);
}
#if 0
@@ -3653,124 +3671,109 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS
if (((Item_func_case*)item)->get_first_expr_num() == -1)
funcName = "case_searched";
if (gwi.clauseType == SELECT || gwi.clauseType == HAVING || gwi.clauseType == GROUP_BY) // select clause
funcParms.reserve(item->argument_count());
// so buildXXXcolumn function will not pop stack.
ClauseType realClauseType = gwi.clauseType;
gwi.clauseType = SELECT;
// We ought to be able to just build from the stack, and would
// be able to if there were any way to know which stack had the
// next case item. Unfortunately, parameters may have been pushed
// onto the ptWorkStack or rcWorkStack or neither, depending on type
// and position. We can't tell which at this point, so we
// rebuild the item from the arguments directly and then try to
// figure what to pop, if anything, in order to sync the stacks.
//
// MCOL-1341 - With MariaDB 10.2.14 onwards CASE is now in the order:
// [case,]when1,when2,...,then1,then2,...[,else]
// See server commit bf1ca14ff3f3faa9f7a018097b25aa0f66d068cd for more
// information.
int32_t arg_offset = 0;
if ((item->argument_count() - 1) % 2)
{
// the first argument
if (funcName == "case_searched")
arg_offset = (item->argument_count() - 1) / 2;
}
else
{
arg_offset = item->argument_count() / 2;
}
for (int32_t i = item->argument_count() - 1; i >= 0; i--)
{
// For case_searched, we know the items for the WHEN clause will
// not be ReturnedColumns. We do this separately just to save
// some cpu cycles trying to build a ReturnedColumn as below.
// Every even numbered arg is a WHEN. In between are the THEN.
// An odd number of args indicates an ELSE residing in the last spot.
if (funcName == "case_searched" &&
(i < arg_offset))
{
for (uint32_t i = 0; i < item->argument_count(); i++)
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top()->data() == sptp->data())
{
if (i % 2 == 0 && i != 1 && i != item->argument_count() - 1)
{
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
funcParms.push_back(sptp);
}
else
{
ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport);
if (parm)
{
sptp.reset(new ParseTree(parm));
}
else
{
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
}
funcParms.push_back(sptp);
}
gwi.ptWorkStack.pop();
}
}
else
{
for (uint32_t i = 0; i < item->argument_count(); i++)
// First try building a ReturnedColumn. It may or may not succeed
// depending on the types involved. There's also little correlation
// between buildReturnedColumn and the existance of the item on
// rwWorkStack or ptWorkStack.
// For example, simple predicates, such as 1=1 or 1=0, land in the
// ptWorkStack but other stuff might land in the rwWorkStack
ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport);
if (parm)
{
ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport);
sptp.reset(new ParseTree(parm));
if (parm)
// We need to pop whichever stack is holding it, if any.
if ((!gwi.rcWorkStack.empty()) &&
*gwi.rcWorkStack.top() == parm)
{
sptp.reset(new ParseTree(parm));
gwi.rcWorkStack.pop();
}
else
else if (!gwi.ptWorkStack.empty())
{
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
}
ReturnedColumn* ptrc = dynamic_cast<ReturnedColumn*>(gwi.ptWorkStack.top()->data());
funcParms.push_back(sptp);
}
}
}
else // where clause
{
// so buildXXXcolumn function will not pop stack.
gwi.clauseType = SELECT;
if (funcName == "case_searched")
{
for (uint32_t i = 0; i < item->argument_count(); i++)
{
if (i % 2 == 0 && i != item->argument_count() - 1)
{
// build item from arguments to avoid parm sequence complexity
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
funcParms.push_back(sptp);
if (!gwi.ptWorkStack.empty())
if (ptrc && *ptrc == *parm)
gwi.ptWorkStack.pop();
}
else
{
ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport);
if (parm)
{
sptp.reset(new ParseTree(parm));
if (!gwi.rcWorkStack.empty())
gwi.rcWorkStack.pop();
}
else
{
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
if (!gwi.ptWorkStack.empty())
gwi.ptWorkStack.pop();
}
funcParms.push_back(sptp);
}
}
}
else // simple_case
{
for (uint32_t i = 0; i < item->argument_count(); i++)
else
{
ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport);
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
if (parm)
// We need to pop whichever stack is holding it, if any.
if ((!gwi.ptWorkStack.empty()) &&
*gwi.ptWorkStack.top()->data() == sptp->data())
{
sptp.reset(new ParseTree(parm));
gwi.ptWorkStack.pop();
}
else if (!gwi.rcWorkStack.empty())
{
// Probably won't happen, but it might have been on the
// rcWorkStack all along.
ReturnedColumn* ptrc = dynamic_cast<ReturnedColumn*>(sptp->data());
if (!gwi.rcWorkStack.empty())
if (ptrc && *ptrc == *gwi.rcWorkStack.top())
{
gwi.rcWorkStack.pop();
}
}
else
{
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
if (!gwi.ptWorkStack.empty())
gwi.ptWorkStack.pop();
}
funcParms.push_back(sptp);
}
}
// recover clause type
gwi.clauseType = WHERE;
funcParms.insert(funcParms.begin(), sptp);
}
// recover clause type
gwi.clauseType = realClauseType;
if (gwi.fatalParseError)
{
setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi);
@@ -4842,9 +4845,9 @@ void gp_walk(const Item* item, void* arg)
// bug 3137. If filter constant like 1=0, put it to ptWorkStack
// MariaDB bug 750. Breaks if compare is an argument to a function.
if ((int32_t)gwip->rcWorkStack.size() <= (gwip->rcBookMarkStack.empty() ? 0 : gwip->rcBookMarkStack.top())
&& isPredicateFunction(ifp, gwip))
// if (isPredicateFunction(ifp, gwip))
// if ((int32_t)gwip->rcWorkStack.size() <= (gwip->rcBookMarkStack.empty() ? 0 : gwip->rcBookMarkStack.top())
// && isPredicateFunction(ifp, gwip))
if (isPredicateFunction(ifp, gwip))
gwip->ptWorkStack.push(new ParseTree(cc));
else
gwip->rcWorkStack.push(cc);
@@ -6271,7 +6274,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i
// @bug 1706
String funcStr;
ifp->print(&funcStr, QT_INFINIDB);
gwi.selectCols.push_back(string(funcStr.c_ptr()) + " `" + escapeBackTick(ifp->name) + "`");
string valStr;
valStr.assign(funcStr.ptr(), funcStr.length());
gwi.selectCols.push_back(valStr + " `" + escapeBackTick(ifp->name) + "`");
// clear the error set by buildFunctionColumn
gwi.fatalParseError = false;
gwi.parseErrorText = "";

17
dbcon/mysql/ha_calpont_impl.cpp
View File

@@ -590,8 +590,21 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h
* based on the result set.
*/
/* MCOL-683: UTF-8 datetime no msecs is 57, this sometimes happens! */
if (((*f)->field_length > 19) && ((*f)->field_length != 57))
(*f)->field_length = strlen(tmp);
// if (((*f)->field_length > 19) && ((*f)->field_length != 57))
// (*f)->field_length = strlen(tmp);
Field_varstring* f2 = (Field_varstring*)*f;
f2->store(tmp, strlen(tmp), f2->charset());
break;
}
case CalpontSystemCatalog::TIME:
{
if ((*f)->null_ptr)
*(*f)->null_ptr &= ~(*f)->null_bit;
intColVal = row.getUintField<8>(s);
DataConvert::timeToString(intColVal, tmp, 255);
Field_varstring* f2 = (Field_varstring*)*f;
f2->store(tmp, strlen(tmp), f2->charset());

11
dbcon/mysql/ha_calpont_partition.cpp
View File

@@ -125,6 +125,9 @@ string name(CalpontSystemCatalog::ColType& ct)
case CalpontSystemCatalog::DATETIME:
return "DATETIME";
case CalpontSystemCatalog::TIME:
return "TIME";
case CalpontSystemCatalog::DECIMAL:
return "DECIMAL";
@@ -201,6 +204,7 @@ bool CP_type(CalpontSystemCatalog::ColType& ct)
ct.colDataType == CalpontSystemCatalog::BIGINT ||
ct.colDataType == CalpontSystemCatalog::DATE ||
ct.colDataType == CalpontSystemCatalog::DATETIME ||
ct.colDataType == CalpontSystemCatalog::TIME ||
ct.colDataType == CalpontSystemCatalog::DECIMAL ||
ct.colDataType == CalpontSystemCatalog::UTINYINT ||
ct.colDataType == CalpontSystemCatalog::USMALLINT ||
@@ -261,6 +265,9 @@ const string format(int64_t v, CalpontSystemCatalog::ColType& ct)
oss << DataConvert::datetimeToString(v);
break;
case CalpontSystemCatalog::TIME:
oss << DataConvert::timeToString(v);
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
{
@@ -387,6 +394,10 @@ const int64_t IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t&
v = boost::any_cast<uint64_t>(anyVal);
break;
case CalpontSystemCatalog::TIME:
v = boost::any_cast<int64_t>(anyVal);
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE)

2
dbcon/mysql/ha_window_function.cpp
View File

@@ -602,6 +602,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n
case CalpontSystemCatalog::DATE:
case CalpontSystemCatalog::DATETIME:
case CalpontSystemCatalog::TIME:
if (!frm.fIsRange)
boundTypeErr = true;
else if (dynamic_cast<IntervalColumn*>(frm.fStart.fVal.get()) == NULL)
@@ -653,6 +654,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n
case CalpontSystemCatalog::DATE:
case CalpontSystemCatalog::DATETIME:
case CalpontSystemCatalog::TIME:
if (!frm.fIsRange)
boundTypeErr = true;
else if (dynamic_cast<IntervalColumn*>(frm.fEnd.fVal.get()) == NULL)