database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-406 Improved Information Schema

Browse Source 
* Add INFORMATION_SCHEMA.COLUMNSTORE_FILES which contains information
  about files
* Remove file information from COLUMNSTORE_EXTENTS (due to above)
* Hide columns with Object ID < 3000 (internal columns)
* Fix bad calculation in data_size columns
* Fix minor memory leak
* Add compressedSize() function to IDBFileSystem to get the used file
  size for a compressed file
* Add columnstore_info schema with utility stored procedures to access
  the information_schema tables
This commit is contained in:
Andrew Hutchings
2016-11-23 22:11:26 +00:00
parent 606284ad63
commit 3b1de94cd8
13 changed files with 422 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
dbcon/mysql/CMakeLists.txt
View File

@ -62,12 +62,25 @@ target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LI
set_target_properties(is_columnstore_extents PROPERTIES PREFIX "")
set_target_properties(is_columnstore_extents PROPERTIES VERSION 1.0.0 SOVERSION 1)
install(TARGETS calmysql is_columnstore_tables is_columnstore_columns is_columnstore_extents DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine)
SET ( is_columnstore_files_SRCS
is_columnstore_files.cpp
)
add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS})
target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool)
# Don't prepend .so file with 'lib'
set_target_properties(is_columnstore_files PROPERTIES PREFIX "")
set_target_properties(is_columnstore_files PROPERTIES VERSION 1.0.0 SOVERSION 1)
install(TARGETS calmysql is_columnstore_tables is_columnstore_columns is_columnstore_extents is_columnstore_files DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine)
install(FILES syscatalog_mysql.sql
dumpcat_mysql.sql
calsetuserpriority.sql
calremoveuserpriority.sql
calshowprocesslist.sql
columnstore_info.sql
my.cnf
DESTINATION ${ENGINE_MYSQLDIR} COMPONENT storage-engine)
install(PROGRAMS install_calpont_mysql.sh mysql-Columnstore dumpcat.pl

50
dbcon/mysql/columnstore_info.sql Normal file
View File

@ -0,0 +1,50 @@
CREATE DATABASE columnstore_info;
USE columnstore_info;
DELIMITER //
CREATE FUNCTION `format_filesize`(filesize FLOAT) RETURNS varchar(20) CHARSET utf8 DETERMINISTIC
BEGIN
DECLARE n INT DEFAULT 1;
LOOP
IF filesize < 1024 THEN
RETURN concat(round(filesize, 2), ' ', elt(n, 'Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB', 'BB'));
END IF;
SET filesize = filesize / 1024;
SET n = n + 1;
END LOOP;
END //
CREATE PROCEDURE total_usage ()
BEGIN
SELECT
(SELECT format_filesize(sum(data_size)) TOTAL_DATA_SIZE FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS) TOTAL_DATA_SIZE,
(SELECT format_filesize(sum(file_size)) TOTAL_DISK_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_FILES) TOTAL_DISK_USAGE;
END //
CREATE PROCEDURE table_usage (IN t_name char(64))
BEGIN
IF t_name IS NOT NULL THEN
SELECT TABLE_NAME, format_filesize(sum(cf.file_size)) DATA_DISK_USAGE, format_filesize(sum(IFNULL(ccf.file_size, 0))) DICT_DISK_USAGE, format_filesize(sum(cf.file_size) + sum(IFNULL(ccf.file_size, 0))) TOTAL_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS cc
JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON cc.object_id = cf.object_id
LEFT JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES ccf ON cc.dictionary_object_id = ccf.object_id
WHERE table_name = t_name GROUP BY table_name;
ELSE
SELECT TABLE_NAME, format_filesize(sum(cf.file_size)) DATA_DISK_USAGE, format_filesize(sum(IFNULL(ccf.file_size, 0))) DICT_DISK_USAGE, format_filesize(sum(cf.file_size) + sum(IFNULL(ccf.file_size, 0))) TOTAL_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS cc
JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON cc.object_id = cf.object_id
LEFT JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES ccf ON cc.dictionary_object_id = ccf.object_id
GROUP BY table_name;
END IF;
END //
CREATE PROCEDURE compression_ratio()
BEGIN
SELECT CONCAT(((1 - (sum(data_size) / sum(compressed_data_size))) * 100), '%') COMPRESSION_RATIO FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS ce
JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON ce.object_id = cf.object_id
WHERE compressed_data_size IS NOT NULL;
END //
DELIMITER ;

2
dbcon/mysql/install_calpont_mysql.sh
View File

@ -33,6 +33,7 @@ INSTALL PLUGIN infinidb SONAME 'libcalmysql.so';
INSTALL PLUGIN columnstore_tables SONAME 'is_columnstore_tables.so';
INSTALL PLUGIN columnstore_columns SONAME 'is_columnstore_columns.so';
INSTALL PLUGIN columnstore_extents SONAME 'is_columnstore_extents.so';
INSTALL PLUGIN columnstore_files SONAME 'is_columnstore_files.so';
-- these are deprecated names
DELETE FROM mysql.func WHERE name='caldisablepartition';
DELETE FROM mysql.func WHERE name='caldroppartition';
@ -133,6 +134,7 @@ $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/d
$installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calsetuserpriority.sql
$installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calremoveuserpriority.sql
$installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calshowprocesslist.sql
$installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/columnstore_info.sql
sed -i 's/infinidb_compression_type=1/infinidb_compression_type=2/' $installdir/mysql/my.cnf >/dev/null 2>&1

34
dbcon/mysql/is_columnstore_extents.cpp
View File

@ -24,13 +24,9 @@
#include <vector>
#include <limits>
#include <boost/shared_ptr.hpp>
#include "dbrm.h"
#include "objectidmanager.h"
#include "we_convertor.h"
#include "we_define.h"
#include "IDBPolicy.h"
#include "we_config.h"
// Required declaration as it isn't in a MairaDB include
bool schema_table_store_record(THD *thd, TABLE *table);
@ -53,8 +49,6 @@ ST_FIELD_INFO is_columnstore_extents_fields[] =
{"STATE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, // 13
{"STATUS", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0}, // 14
{"DATA_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0}, // 15
{"FILENAME", 1024, MYSQL_TYPE_STRING, 0, 0, 0, 0}, // 16
{"FILE_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 17
{0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
};
@ -65,9 +59,6 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
std::vector<struct BRM::EMEntry> entries;
std::vector<struct BRM::EMEntry>::iterator iter;
std::vector<struct BRM::EMEntry>::iterator end;
char oidDirName[WriteEngine::FILE_NAME_SIZE];
char fullFileName[WriteEngine::FILE_NAME_SIZE];
char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE];
BRM::DBRM *emp = new BRM::DBRM();
if (!emp || !emp->isDBRMReady())
@ -78,7 +69,7 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
execplan::ObjectIDManager oidm;
BRM::OID_t MaxOID = oidm.size();
for(BRM::OID_t oid = 0; oid <= MaxOID; oid++)
for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
{
emp->getExtents(oid, entries, false, false, true);
if (entries.size() == 0)
@ -161,31 +152,20 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
default:
table->field[14]->store("Unknown", strlen("Unknown"), cs);
}
table->field[15]->store((iter->HWM + 1) * 8192);
WriteEngine::Convertor::oid2FileName(oid, oidDirName, dbDir, iter->partitionNum, iter->segmentNum);
snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", WriteEngine::Config::getDBRootByNum(iter->dbRoot).c_str(), oidDirName);
table->field[16]->store(fullFileName, strlen(fullFileName), cs);
if (idbdatafile::IDBPolicy::exists(fullFileName))
{
table->field[17]->set_notnull();
table->field[17]->store(idbdatafile::IDBPolicy::size(fullFileName));
}
else
{
table->field[17]->set_null();
}
table->field[15]->store((iter->HWM - iter->blockOffset + 1) * 8192);
if (schema_table_store_record(thd, table))
{
delete emp;
return 1;
}
iter++;
}
}
delete emp;
return 0;
}

163
dbcon/mysql/is_columnstore_files.cpp Normal file
View File

@ -0,0 +1,163 @@
/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil
* vi: set shiftwidth=4 tabstop=4 expandtab:
* :indentSize=4:tabSize=4:noTabs=true:
*
* Copyright (C) 2016 MariaDB Corporaton
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; version 2 of
* the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "idb_mysql.h"
#include "dbrm.h"
#include "objectidmanager.h"
#include "we_convertor.h"
#include "we_define.h"
#include "IDBPolicy.h"
#include "we_config.h"
#include "we_brm.h"
// Required declaration as it isn't in a MairaDB include
bool schema_table_store_record(THD *thd, TABLE *table);
ST_FIELD_INFO is_columnstore_files_fields[] =
{
{"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
{"SEGMENT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
{"PARTITION_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
{"FILENAME", 1024, MYSQL_TYPE_STRING, 0, 0, 0, 0},
{"FILE_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
{"COMPRESSED_DATA_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
{0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
};
static bool is_columnstore_files_get_entries(THD *thd, TABLE_LIST *tables, BRM::OID_t oid, std::vector<struct BRM::EMEntry> &entries)
{
CHARSET_INFO *cs = system_charset_info;
TABLE *table = tables->table;
char oidDirName[WriteEngine::FILE_NAME_SIZE];
char fullFileName[WriteEngine::FILE_NAME_SIZE];
char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE];
WriteEngine::Config config;
config.initConfigCache();
std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
while ( iter != entries.end() ) //organize extents into files
{
// Don't include files more than once at different block offsets
if (iter->blockOffset > 0)
{
iter++;
continue;
}
table->field[0]->store(oid);
table->field[1]->store(iter->segmentNum);
table->field[2]->store(iter->partitionNum);
WriteEngine::Convertor::oid2FileName(oid, oidDirName, dbDir, iter->partitionNum, iter->segmentNum);
snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", WriteEngine::Config::getDBRootByNum(iter->dbRoot).c_str(), oidDirName);
table->field[3]->store(fullFileName, strlen(fullFileName), cs);
if (idbdatafile::IDBPolicy::exists(fullFileName))
{
table->field[4]->set_notnull();
table->field[4]->store(idbdatafile::IDBPolicy::size(fullFileName));
off64_t comp_size;
if ((comp_size = idbdatafile::IDBPolicy::compressedSize(fullFileName)) != -1)
{
table->field[5]->set_notnull();
table->field[5]->store(comp_size);
}
else
{
table->field[5]->set_null();
}
}
else
{
table->field[4]->set_null();
table->field[5]->set_null();
}
if (schema_table_store_record(thd, table))
return 1;
iter++;
}
return 0;
}
static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
{
BRM::DBRM *emp = new BRM::DBRM();
std::vector<struct BRM::EMEntry> entries;
if (!emp || !emp->isDBRMReady())
{
return 1;
}
execplan::ObjectIDManager oidm;
BRM::OID_t MaxOID = oidm.size();
for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
{
emp->getExtents(oid, entries, false, false, true);
if (entries.size() == 0)
continue;
if (is_columnstore_files_get_entries(thd, tables, oid, entries))
{
delete emp;
return 1;
}
}
delete emp;
return 0;
}
static int is_columnstore_files_plugin_init(void *p)
{
ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p;
schema->fields_info = is_columnstore_files_fields;
schema->fill_table = is_columnstore_files_fill;
return 0;
}
static struct st_mysql_information_schema is_columnstore_files_plugin_version =
{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION };
maria_declare_plugin(is_columnstore_files_plugin)
{
MYSQL_INFORMATION_SCHEMA_PLUGIN,
&is_columnstore_files_plugin_version,
"COLUMNSTORE_FILES",
"MariaDB Corporaton",
"An information schema plugin to list ColumnStore filess",
PLUGIN_LICENSE_GPL,
is_columnstore_files_plugin_init,
//is_columnstore_files_plugin_deinit,
NULL,
0x0100,
NULL,
NULL,
"1.0",
MariaDB_PLUGIN_MATURITY_STABLE
}
maria_declare_plugin_end;