database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-987 Add LZ4 compression.

Browse Source 
* Adds CompressInterfaceLZ4 which uses LZ4 API for compress/uncompress.
* Adds CMake machinery to search LZ4 on running host.
* All methods which use static data and do not modify any internal data - become `static`,
  so we can use them without creation of the specific object. This is possible, because
  the header specification has not been modified. We still use 2 sections in header, first
  one with file meta data, the second one with pointers for compressed chunks.
* Methods `compress`, `uncompress`, `maxCompressedSize`, `getUncompressedSize` - become
  pure virtual, so we can override them for the other compression algos.
* Adds method `getChunkMagicNumber`, so we can verify chunk magic number
  for each compression algo.
* Renames "s/IDBCompressInterface/CompressInterface/g" according to requirement.
This commit is contained in:
Denis Khalikov
2021-04-01 17:26:38 +03:00
parent dd12bd3cd0
commit cc1c3629c5
45 changed files with 1311 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dbcon/mysql/columnstore_info.sql
View File

@ -95,7 +95,11 @@ DROP PROCEDURE IF EXISTS `compression_ratio` //
CREATE PROCEDURE compression_ratio() SQL SECURITY INVOKER
BEGIN
SELECT CONCAT((SELECT SUM(data_size) FROM information_schema.columnstore_extents ce left join information_schema.columnstore_columns cc on ce.object_id = cc.object_id where compression_type='Snappy') / (SELECT SUM(compressed_data_size) FROM information_schema.columnstore_files WHERE compressed_data_size IS NOT NULL), ':1') COMPRESSION_RATIO;
SELECT 'Snappy' as compression_method, CONCAT((SELECT SUM(data_size) FROM information_schema.columnstore_extents ce left join information_schema.columnstore_columns cc on ce.object_id = cc.object_id where compression_type='Snappy') / (SELECT SUM(compressed_data_size) FROM information_schema.columnstore_files co left join information_schema.columnstore_columns cc on (co.object_id = cc.object_id) left join information_schema.columnstore_extents ce on (ce.object_id = co.object_id) where compression_type='Snappy' and compressed_data_size IS NOT NULL /* could be a situation when compressed_data_size != NULL but data_size == 0, in this case we will get wrong ratio */ and data_size > 0), ':1') compression_ratio
UNION ALL
SELECT 'LZ4' as compression_method, CONCAT((SELECT SUM(data_size) FROM information_schema.columnstore_extents ce left join information_schema.columnstore_columns cc on ce.object_id = cc.object_id where compression_type='LZ4') / (SELECT SUM(compressed_data_size) FROM information_schema.columnstore_files co left join information_schema.columnstore_columns cc on (co.object_id = cc.object_id) left join information_schema.columnstore_extents ce on (ce.object_id = co.object_id) where compression_type='LZ4' and compressed_data_size IS NOT NULL /* could be a situation when compressed_data_size != NULL but data_size == 0, in this case we will get wrong ratio */ and data_size > 0), ':1') as compression_ratio;
END //
create or replace procedure columnstore_upgrade() SQL SECURITY INVOKER

22
dbcon/mysql/ha_mcs_ddl.cpp
View File

@ -777,7 +777,6 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
parser.setDefaultSchema(schema);
parser.setDefaultCharset(default_table_charset);
int rc = 0;
IDBCompressInterface idbCompress;
parser.Parse(ddlStatement.c_str());
if (get_fe_conn_info_ptr() == NULL)
@ -981,7 +980,9 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
if (compressionType == 1) compressionType = 2;
if (( compressionType > 0 ) && !(idbCompress.isCompressionAvail( compressionType )))
if ((compressionType > 0) &&
!(compress::CompressInterface::isCompressionAvail(
compressionType)))
{
rc = 1;
ci->alterTableState = cal_connection_info::NOT_ALTER;
@ -1368,7 +1369,9 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
return rc;
}
if (( compressionType > 0 ) && !(idbCompress.isCompressionAvail( compressionType )))
if ((compressionType > 0) &&
!(compress::CompressInterface::isCompressionAvail(
compressionType)))
{
rc = 1;
thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str());
@ -1713,7 +1716,9 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
return rc;
}
if (( compressionType > 0 ) && !(idbCompress.isCompressionAvail( compressionType )))
if ((compressionType > 0) &&
!(compress::CompressInterface::isCompressionAvail(
compressionType)))
{
rc = 1;
thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str());
@ -1842,7 +1847,9 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
return rc;
}
if (( compressionType > 0 ) && !(idbCompress.isCompressionAvail( compressionType )))
if ((compressionType > 0) &&
!(compress::CompressInterface::isCompressionAvail(
compressionType)))
{
rc = 1;
thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str());
@ -2364,9 +2371,8 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea
if (compressiontype == 1) compressiontype = 2;
IDBCompressInterface idbCompress;
if ( ( compressiontype > 0 ) && !(idbCompress.isCompressionAvail( compressiontype )) )
if ((compressiontype > 0) &&
!(compress::CompressInterface::isCompressionAvail(compressiontype)))
{
string emsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE);
setError(thd, ER_INTERNAL_ERROR, emsg);

9
dbcon/mysql/ha_mcs_sysvars.cpp
View File

@ -21,8 +21,10 @@
#include "ha_mcs_sysvars.h"
const char* mcs_compression_type_names[] = {
"SNAPPY",
"SNAPPY",
"SNAPPY", // 0
"SNAPPY", // 1
"SNAPPY", // 2
"LZ4", // 3
NullS
};
@ -39,7 +41,8 @@ static MYSQL_THDVAR_ENUM(
PLUGIN_VAR_RQCMDARG,
"Controls compression algorithm for create tables. Possible values are: "
"NO_COMPRESSION segment files aren't compressed; "
"SNAPPY segment files are Snappy compressed (default);",
"SNAPPY segment files are Snappy compressed (default);"
"LZ4 segment files are LZ4 compressed;",
NULL, // check
NULL, // update
1, //default

3
dbcon/mysql/ha_mcs_sysvars.h
View File

@ -30,7 +30,8 @@ extern char cs_commit_hash[];
// compression_type
enum mcs_compression_type_t {
NO_COMPRESSION = 0,
SNAPPY = 2
SNAPPY = 2,
LZ4 = 3
};
// use_import_for_batchinsert mode

4
dbcon/mysql/is_columnstore_columns.cpp
View File

@ -183,6 +183,10 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond)
compression_type = "Snappy";
break;
case 3:
compression_type = "LZ4";
break;
default:
compression_type = "Unknown";
break;