database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL-1484 I_S condition pushdowns

Browse Source 
Add condition pushdowns to the information_schema tables to give a
performance improvement when a relevant WHERE condition is provided. In
addition there is a new table_usage() stored procedure designed to use
the pushdowns.
This commit is contained in:
Andrew Hutchings
2018-06-19 15:12:44 +01:00
parent 4a20da68ac
commit d3d322ed7b
5 changed files with 505 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
dbcon/mysql/columnstore_info.sql
View File

@ -37,43 +37,56 @@ DROP PROCEDURE IF EXISTS `table_usage` //
CREATE PROCEDURE table_usage (IN t_schema char(64), IN t_name char(64))
`table_usage`: BEGIN
DECLARE done INTEGER DEFAULT 0;
DECLARE dbname VARCHAR(64);
DECLARE tbname VARCHAR(64);
DECLARE object_ids TEXT;
DECLARE dictionary_object_ids TEXT;
DECLARE `locker` TINYINT UNSIGNED DEFAULT IS_USED_LOCK('table_usage');
DECLARE columns_list CURSOR FOR SELECT TABLE_SCHEMA, TABLE_NAME, GROUP_CONCAT(object_id) OBJECT_IDS, GROUP_CONCAT(dictionary_object_id) DICT_OBJECT_IDS FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS WHERE table_name = t_name and table_schema = t_schema GROUP BY table_schema, table_name;
DECLARE columns_list_sc CURSOR FOR SELECT TABLE_SCHEMA, TABLE_NAME, GROUP_CONCAT(object_id) OBJECT_IDS, GROUP_CONCAT(dictionary_object_id) DICT_OBJECT_IDS FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS WHERE table_schema = t_schema GROUP BY table_schema, table_name;
DECLARE columns_list_all CURSOR FOR SELECT TABLE_SCHEMA, TABLE_NAME, GROUP_CONCAT(object_id) OBJECT_IDS, GROUP_CONCAT(dictionary_object_id) DICT_OBJECT_IDS FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS GROUP BY table_schema, table_name;
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1;
IF `locker` IS NOT NULL THEN
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Error acquiring table_usage lock';
LEAVE `table_usage`;
END IF;
DO GET_LOCK('table_usage', 0);
DROP TABLE IF EXISTS columnstore_info.columnstore_columns;
DROP TABLE IF EXISTS columnstore_info.columnstore_files;
CREATE TABLE columnstore_info.columnstore_columns engine=myisam as (select * from information_schema.columnstore_columns);
ALTER TABLE columnstore_info.columnstore_columns ADD INDEX `object_id` (`object_id`);
ALTER TABLE columnstore_info.columnstore_columns ADD INDEX `dictionary_object_id` (`dictionary_object_id`);
CREATE TABLE columnstore_info.columnstore_files engine=myisam as (select * from information_schema.columnstore_files);
ALTER TABLE columnstore_info.columnstore_files ADD INDEX `object_id` (`object_id`);
CREATE TEMPORARY TABLE columnstore_info.columnstore_files (TABLE_SCHEMA VARCHAR(64), TABLE_NAME VARCHAR(64), DATA BIGINT, DICT BIGINT);
IF t_name IS NOT NULL THEN
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
FROM
columnstore_info.columnstore_columns ics where table_name = t_name and (table_schema = t_schema or t_schema IS NULL)
group by table_schema, table_name
) q;
OPEN columns_list;
ELSEIF t_schema IS NOT NULL THEN
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
FROM
columnstore_info.columnstore_columns ics where table_schema = t_schema
group by table_schema, table_name
) q;
OPEN columns_list_sc;
ELSE
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
FROM
columnstore_info.columnstore_columns ics
group by table_schema, table_name
) q;
OPEN columns_list_all;
END IF;
DROP TABLE IF EXISTS columnstore_info.columnstore_columns;
files_table: LOOP
IF t_name IS NOT NULL THEN
FETCH columns_list INTO dbname, tbname, object_ids, dictionary_object_ids;
ELSEIF t_schema IS NOT NULL THEN
FETCH columns_list_sc INTO dbname, tbname, object_ids, dictionary_object_ids;
ELSE
FETCH columns_list_all INTO dbname, tbname, object_ids, dictionary_object_ids;
END IF;
IF done = 1 THEN LEAVE files_table;
END IF;
INSERT INTO columnstore_info.columnstore_files (SELECT dbname, tbname, sum(file_size), 0 FROM information_schema.columnstore_files WHERE find_in_set(object_id, object_ids));
IF dictionary_object_ids IS NOT NULL THEN
UPDATE columnstore_info.columnstore_files SET DICT = (SELECT sum(file_size) FROM information_schema.columnstore_files WHERE find_in_set(object_id, dictionary_object_ids)) WHERE TABLE_SCHEMA = dbname AND TABLE_NAME = tbname;
END IF;
END LOOP;
IF t_name IS NOT NULL THEN
CLOSE columns_list;
ELSEIF t_schema IS NOT NULL THEN
CLOSE columns_list_sc;
ELSE
CLOSE columns_list_all;
END IF;
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(DATA) as DATA_DISK_USAGE, columnstore_info.format_filesize(DICT) as DICT_DATA_USAGE, columnstore_info.format_filesize(DATA + COALESCE(DICT, 0)) as TOTAL_USAGE FROM columnstore_info.columnstore_files;
DROP TABLE IF EXISTS columnstore_info.columnstore_files;
DO RELEASE_LOCK('table_usage');
END //

71
dbcon/mysql/is_columnstore_columns.cpp
View File

@ -56,10 +56,59 @@ ST_FIELD_INFO is_columnstore_columns_fields[] =
};
static void get_cond_item(Item_func *item, String **table, String **db)
{
char tmp_char[MAX_FIELD_WIDTH];
Item_field *item_field = (Item_field*) item->arguments()[0]->real_item();
if (strcasecmp(item_field->field_name, "table_name") == 0)
{
String str_buf(tmp_char, sizeof(tmp_char), system_charset_info);
*table = item->arguments()[1]->val_str(&str_buf);
return;
}
else if (strcasecmp(item_field->field_name, "table_schema") == 0)
{
String str_buf(tmp_char, sizeof(tmp_char), system_charset_info);
*db = item->arguments()[1]->val_str(&str_buf);
return;
}
}
static void get_cond_items(COND *cond, String **table, String **db)
{
if (cond->type() == Item::FUNC_ITEM)
{
Item_func* fitem = (Item_func*) cond;
if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM &&
fitem->arguments()[1]->const_item())
{
get_cond_item(fitem, table, db);
}
}
else if ((cond->type() == Item::COND_ITEM) && (((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC))
{
List_iterator<Item> li(*((Item_cond*) cond)->argument_list());
Item *item;
while ((item= li++))
{
if (item->type() == Item::FUNC_ITEM)
{
get_cond_item((Item_func*)item, table, db);
}
else
{
get_cond_items(item, table, db);
}
}
}
}
static int is_columnstore_columns_fill(THD *thd, TABLE_LIST *tables, COND *cond)
{
CHARSET_INFO *cs = system_charset_info;
TABLE *table = tables->table;
String *table_name = NULL;
String *db_name = NULL;
boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id));
@ -69,9 +118,29 @@ static int is_columnstore_columns_fill(THD *thd, TABLE_LIST *tables, COND *cond)
systemCatalogPtr->identity(execplan::CalpontSystemCatalog::FE);
if (cond)
{
get_cond_items(cond, &table_name, &db_name);
}
for (std::vector<std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >::const_iterator it = catalog_tables.begin();
it != catalog_tables.end(); ++it)
{
if (db_name)
{
if ((*it).second.schema.compare(db_name->ptr()) != 0)
{
continue;
}
}
if (table_name)
{
if ((*it).second.table.compare(table_name->ptr()) != 0)
{
continue;
}
}
execplan::CalpontSystemCatalog::RIDList column_rid_list;
// Note a table may get dropped as you iterate over the list of tables.
// So simply ignore the dropped table.
@ -168,8 +237,6 @@ static int is_columnstore_columns_fill(THD *thd, TABLE_LIST *tables, COND *cond)
}
}
return 0;
}

97
dbcon/mysql/is_columnstore_extents.cpp
View File

@ -52,28 +52,16 @@ ST_FIELD_INFO is_columnstore_extents_fields[] =
{0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
};
static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *thd)
{
CHARSET_INFO *cs = system_charset_info;
TABLE *table = tables->table;
std::vector<struct BRM::EMEntry> entries;
std::vector<struct BRM::EMEntry>::iterator iter;
std::vector<struct BRM::EMEntry>::iterator end;
BRM::DBRM *emp = new BRM::DBRM();
if (!emp || !emp->isDBRMReady())
{
return 1;
}
execplan::ObjectIDManager oidm;
BRM::OID_t MaxOID = oidm.size();
for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
{
emp->getExtents(oid, entries, false, false, true);
if (entries.size() == 0)
continue;
return 0;
iter = entries.begin();
end = entries.end();
@ -175,8 +163,89 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
iter++;
}
return 0;
}
static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
{
BRM::OID_t cond_oid = 0;
TABLE *table = tables->table;
BRM::DBRM *emp = new BRM::DBRM();
if (!emp || !emp->isDBRMReady())
{
return 1;
}
if (cond && cond->type() == Item::FUNC_ITEM)
{
Item_func* fitem = (Item_func*) cond;
if ((fitem->functype() == Item_func::EQ_FUNC) && (fitem->argument_count() == 2))
{
if(fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM &&
fitem->arguments()[1]->const_item())
{
// WHERE object_id = value
Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item();
if (strcasecmp(item_field->field_name, "object_id") == 0)
{
cond_oid = fitem->arguments()[1]->val_int();
return generate_result(cond_oid, emp, table, thd);
}
}
else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM &&
fitem->arguments()[0]->const_item())
{
// WHERE value = object_id
Item_field *item_field = (Item_field*) fitem->arguments()[1]->real_item();
if (strcasecmp(item_field->field_name, "object_id") == 0)
{
cond_oid = fitem->arguments()[0]->val_int();
return generate_result(cond_oid, emp, table, thd);
}
}
}
else if (fitem->functype() == Item_func::IN_FUNC)
{
// WHERE object_id in (value1, value2)
Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item();
if (strcasecmp(item_field->field_name, "object_id") == 0)
{
for (unsigned int i=1; i < fitem->argument_count(); i++)
{
cond_oid = fitem->arguments()[i]->val_int();
int result = generate_result(cond_oid, emp, table, thd);
if (result)
return 1;
}
}
}
else if (fitem->functype() == Item_func::UNKNOWN_FUNC &&
strcasecmp(fitem->func_name(), "find_in_set") == 0)
{
// WHERE FIND_IN_SET(object_id, values)
String *tmp_var = fitem->arguments()[1]->val_str();
std::stringstream ss(tmp_var->ptr());
while (ss >> cond_oid)
{
int ret = generate_result(cond_oid, emp, table, thd);
if (ret)
return 1;
if (ss.peek() == ',')
ss.ignore();
}
}
}
execplan::ObjectIDManager oidm;
BRM::OID_t MaxOID = oidm.size();
for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
{
int result = generate_result(oid, emp, table, thd);
if (result)
return 1;
}
delete emp;
return 0;
}

105
dbcon/mysql/is_columnstore_files.cpp
View File

@ -82,12 +82,10 @@ static bool get_file_sizes(messageqcpp::MessageQueueClient *msgQueueClient, cons
}
}
static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *thd)
{
BRM::DBRM *emp = new BRM::DBRM();
std::vector<struct BRM::EMEntry> entries;
CHARSET_INFO *cs = system_charset_info;
TABLE *table = tables->table;
char oidDirName[WriteEngine::FILE_NAME_SIZE];
char fullFileName[WriteEngine::FILE_NAME_SIZE];
@ -101,19 +99,9 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
oam::Oam oam_instance;
int pmId = 0;
if (!emp || !emp->isDBRMReady())
{
return 1;
}
execplan::ObjectIDManager oidm;
BRM::OID_t MaxOID = oidm.size();
for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
{
emp->getExtents(oid, entries, false, false, true);
if (entries.size() == 0)
continue;
return 0;
std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
while ( iter != entries.end() ) //organize extents into files
@ -122,7 +110,7 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
if (iter->blockOffset > 0)
{
iter++;
continue;
return 0;
}
try
@ -133,7 +121,7 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
{
// MCOL-1116: If we are here a DBRoot is offline/missing
iter++;
continue;
return 0;
}
table->field[0]->store(oid);
table->field[1]->store(iter->segmentNum);
@ -189,6 +177,91 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient);
msgQueueClient = NULL;
}
return 0;
}
static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
{
BRM::DBRM *emp = new BRM::DBRM();
BRM::OID_t cond_oid = 0;
TABLE *table = tables->table;
if (!emp || !emp->isDBRMReady())
{
return 1;
}
if (cond && cond->type() == Item::FUNC_ITEM)
{
Item_func* fitem = (Item_func*) cond;
if ((fitem->functype() == Item_func::EQ_FUNC) && (fitem->argument_count() == 2))
{
if(fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM &&
fitem->arguments()[1]->const_item())
{
// WHERE object_id = value
Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item();
if (strcasecmp(item_field->field_name, "object_id") == 0)
{
cond_oid = fitem->arguments()[1]->val_int();
return generate_result(cond_oid, emp, table, thd);
}
}
else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM &&
fitem->arguments()[0]->const_item())
{
// WHERE value = object_id
Item_field *item_field = (Item_field*) fitem->arguments()[1]->real_item();
if (strcasecmp(item_field->field_name, "object_id") == 0)
{
cond_oid = fitem->arguments()[0]->val_int();
return generate_result(cond_oid, emp, table, thd);
}
}
}
else if (fitem->functype() == Item_func::IN_FUNC)
{
// WHERE object_id in (value1, value2)
Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item();
if (strcasecmp(item_field->field_name, "object_id") == 0)
{
for (unsigned int i=1; i < fitem->argument_count(); i++)
{
cond_oid = fitem->arguments()[i]->val_int();
int result = generate_result(cond_oid, emp, table, thd);
if (result)
return 1;
}
}
}
else if (fitem->functype() == Item_func::UNKNOWN_FUNC &&
strcasecmp(fitem->func_name(), "find_in_set") == 0)
{
// WHERE FIND_IN_SET(object_id, values)
String *tmp_var = fitem->arguments()[1]->val_str();
std::stringstream ss(tmp_var->ptr());
while (ss >> cond_oid)
{
int ret = generate_result(cond_oid, emp, table, thd);
if (ret)
return 1;
if (ss.peek() == ',')
ss.ignore();
}
}
}
execplan::ObjectIDManager oidm;
BRM::OID_t MaxOID = oidm.size();
if (!cond_oid)
{
for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
{
int result = generate_result(oid, emp, table, thd);
if (result)
return 1;
}
}
delete emp;
return 0;

69
dbcon/mysql/is_columnstore_tables.cpp
View File

@ -42,22 +42,91 @@ ST_FIELD_INFO is_columnstore_tables_fields[] =
{0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
};
static void get_cond_item(Item_func *item, String **table, String **db)
{
char tmp_char[MAX_FIELD_WIDTH];
Item_field *item_field = (Item_field*) item->arguments()[0]->real_item();
if (strcasecmp(item_field->field_name, "table_name") == 0)
{
String str_buf(tmp_char, sizeof(tmp_char), system_charset_info);
*table = item->arguments()[1]->val_str(&str_buf);
return;
}
else if (strcasecmp(item_field->field_name, "table_schema") == 0)
{
String str_buf(tmp_char, sizeof(tmp_char), system_charset_info);
*db = item->arguments()[1]->val_str(&str_buf);
return;
}
}
static void get_cond_items(COND *cond, String **table, String **db)
{
if (cond->type() == Item::FUNC_ITEM)
{
Item_func* fitem = (Item_func*) cond;
if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM &&
fitem->arguments()[1]->const_item())
{
get_cond_item(fitem, table, db);
}
}
else if ((cond->type() == Item::COND_ITEM) && (((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC))
{
List_iterator<Item> li(*((Item_cond*) cond)->argument_list());
Item *item;
while ((item= li++))
{
if (item->type() == Item::FUNC_ITEM)
{
get_cond_item((Item_func*)item, table, db);
}
else
{
get_cond_items(item, table, db);
}
}
}
}
static int is_columnstore_tables_fill(THD *thd, TABLE_LIST *tables, COND *cond)
{
CHARSET_INFO *cs = system_charset_info;
TABLE *table = tables->table;
String *table_name = NULL;
String *db_name = NULL;
boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id));
systemCatalogPtr->identity(execplan::CalpontSystemCatalog::FE);
if (cond)
{
get_cond_items(cond, &table_name, &db_name);
}
const std::vector< std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> > catalog_tables
= systemCatalogPtr->getTables();
for (std::vector<std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >::const_iterator it = catalog_tables.begin();
it != catalog_tables.end(); ++it)
{
if (db_name)
{
if ((*it).second.schema.compare(db_name->ptr()) != 0)
{
continue;
}
}
if (table_name)
{
if ((*it).second.table.compare(table_name->ptr()) != 0)
{
continue;
}
}
execplan::CalpontSystemCatalog::TableInfo tb_info = systemCatalogPtr->tableInfo((*it).second);
std::string create_date = dataconvert::DataConvert::dateToString((*it).second.create_date);
table->field[0]->store((*it).second.schema.c_str(), (*it).second.schema.length(), cs);