database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-2000 Process charset definitions in the DDL

Browse Source 
MCOL-2000 Process charset definitions in the ALTER TABLE .. ADD COLUMN

MCOL-2000 Yet another fixes for column charsets

* make respect for column (including table/db/server default) charsets
  for the TEXT(n) fields
* round TEXT(n) column length up to the next default length of TEXT-like
  subtypes, 255 (TINYTEXT), 65535 (TEXT) and so on up to 2100000000
  (LONGTEXT)
This commit is contained in:
Alexey Antipovsky
2020-11-24 15:56:14 +03:00
parent 6b13fbc210
commit d2e7c9d98d
6 changed files with 281 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
dbcon/ddlpackage/ddl.y
View File

@ -49,6 +49,9 @@
#include "ddl-gram.h"
#endif
#include "my_global.h"
#include "my_sys.h"
#define scanner x->scanner
using namespace std;
@ -57,6 +60,42 @@ using namespace ddlpackage;
int ddllex(YYSTYPE* ddllval, void* yyscanner);
void ddlerror(struct pass_to_bison* x, char const *s);
char* copy_string(const char *str);
void fix_column_length(SchemaObject* elem, const CHARSET_INFO* def_cs) {
auto* column = dynamic_cast<ColumnDef*>(elem);
if (column == NULL || column->fType == NULL)
{
return;
}
if (column->fType->fType == DDL_VARCHAR ||
column->fType->fType == DDL_CHAR ||
(column->fType->fType == DDL_TEXT && column->fType->fExplicitLength))
{
unsigned mul = def_cs ? def_cs->mbmaxlen : 1;
if (column->fType->fCharset) {
const CHARSET_INFO* cs = get_charset_by_csname(column->fType->fCharset, MY_CS_PRIMARY, MYF(0));
if (cs)
mul = cs->mbmaxlen;
}
column->fType->fLength *= mul;
}
if (column->fType->fType == DDL_TEXT && column->fType->fExplicitLength)
{
// Rounding the resulting length of TEXT(N) field to the next default length
if (column->fType->fLength <= 255)
column->fType->fLength = 255;
else if (column->fType->fLength <= 65535)
column->fType->fLength = 65535;
else if (column->fType->fLength <= 16777215)
column->fType->fLength = 16777215;
else if (column->fType->fLength <= 2100000000)
column->fType->fLength = 2100000000;
// otherwise leave the decision to a caller code
}
}
%}
%expect 17
@ -204,6 +243,7 @@ ZEROFILL
%type <sqlStmt> rename_table_statement
%type <str> ident
%type <str> opt_quoted_literal
%type <str> opt_column_charset
%%
stmtblock: stmtmulti { x->fParseTree = $1; }
;
@ -283,15 +323,14 @@ opt_table_options:
;
create_table_statement:
CREATE TABLE opt_if_not_exists table_name '(' table_element_list ')' table_options
CREATE TABLE opt_if_not_exists table_name '(' table_element_list ')' opt_table_options
{
for (auto* elem : *$6)
{
fix_column_length(elem, x->default_table_charset);
}
$$ = new CreateTableStatement(new TableDef($4, $6, $8));
}
|
CREATE TABLE opt_if_not_exists table_name '(' table_element_list ')'
{
$$ = new CreateTableStatement(new TableDef($4, $6, NULL));
}
;
opt_if_not_exists:
@ -647,10 +686,20 @@ ata_add_column:
/* See the documentation for SchemaObject for an explanation of why we are using
* dynamic_cast here.
*/
ADD column_def {$$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($2));}
| ADD COLUMN column_def {$$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($3));}
| ADD '(' table_element_list ')' {$$ = new AtaAddColumns($3);}
| ADD COLUMN '(' table_element_list ')' {$$ = new AtaAddColumns($4);}
ADD column_def { fix_column_length($2, x->default_table_charset); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($2));}
| ADD COLUMN column_def { fix_column_length($3, x->default_table_charset); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($3));}
| ADD '(' table_element_list ')' {
for (auto* elem : *$3) {
fix_column_length(elem, x->default_table_charset);
}
$$ = new AtaAddColumns($3);
}
| ADD COLUMN '(' table_element_list ')' {
for (auto* elem : *$4) {
fix_column_length(elem, x->default_table_charset);
}
$$ = new AtaAddColumns($4);
}
;
column_name:
@ -737,9 +786,9 @@ optional_braces:
;
opt_column_charset:
/* empty */ {}
/* empty */ { $$ = NULL; }
|
IDB_CHAR SET opt_quoted_literal {}
IDB_CHAR SET opt_quoted_literal { $$ = $3; }
;
opt_column_collate:
@ -749,12 +798,18 @@ opt_column_collate:
;
data_type:
character_string_type opt_column_charset opt_column_collate
character_string_type opt_column_charset opt_column_collate {
$1->fCharset = $2;
$$ = $1;
}
| binary_string_type
| numeric_type
| datetime_type
| blob_type
| text_type opt_column_charset opt_column_collate
| text_type opt_column_charset opt_column_collate {
$1->fCharset = $2;
$$ = $1;
}
| IDB_BLOB
{
$$ = new ColumnType(DDL_BLOB);
@ -955,6 +1010,7 @@ text_type:
{
$$ = new ColumnType(DDL_TEXT);
$$->fLength = atol($3);
$$->fExplicitLength = true;
}
| TEXT
{

8
dbcon/ddlpackage/ddlpkg.cpp
View File

@ -67,7 +67,9 @@ ColumnType::ColumnType(int prec, int scale) :
fLength(0),
fPrecision(prec),
fScale(scale),
fWithTimezone(false)
fWithTimezone(false),
fCharset(NULL),
fExplicitLength(false)
{
fLength = utils::widthByPrecision(fPrecision);
}
@ -76,7 +78,9 @@ ColumnType::ColumnType(int type) :
fType(type),
fLength(0),
fScale(0),
fWithTimezone(false)
fWithTimezone(false),
fCharset(NULL),
fExplicitLength(false)
{
switch ( type )
{

8
dbcon/ddlpackage/ddlpkg.h
View File

@ -1004,7 +1004,7 @@ struct ColumnType
EXPORT virtual int serialize(messageqcpp::ByteStream& bs);
/** @brief For deserialization. */
ColumnType()
ColumnType() : fCharset(NULL), fExplicitLength(false)
{}
friend std::ostream& operator<<(std::ostream& os, const ColumnType& ac);
@ -1044,6 +1044,12 @@ struct ColumnType
uint64_t fNextvalue;
/** @brief Column charset (CHAR, VARCHAR and TEXT only) */
const char* fCharset;
/** @brief Is the TEXT column has explicit defined length, ie TEXT(1717) */
bool fExplicitLength;
};

5
dbcon/ddlpackage/sqlparser.cpp
View File

@ -63,6 +63,11 @@ void SqlParser::setDefaultSchema(std::string schema)
x.fDBSchema = schema;
}
void SqlParser::setDefaultCharset(const CHARSET_INFO* default_charset)
{
x.default_table_charset = default_charset;
}
int SqlParser::Parse(const char* sqltext)
{
ddllex_init_extra(&scanData, &x.scanner);

12
dbcon/ddlpackage/sqlparser.h
View File

@ -27,6 +27,8 @@
*/
#include <stdexcept>
#include <my_global.h>
#include <my_sys.h>
#include "ddlpkg.h"
#if defined(_MSC_VER) && defined(xxxDDLPKGSQLPARSER_DLLEXPORT)
@ -86,8 +88,9 @@ struct pass_to_bison
ParseTree* fParseTree;
std::string fDBSchema;
void* scanner;
const CHARSET_INFO* default_table_charset;
pass_to_bison(ParseTree* pt) : fParseTree(pt), scanner(NULL) {};
pass_to_bison(ParseTree* pt) : fParseTree(pt), scanner(NULL), default_table_charset(NULL) {};
};
class SqlParser
@ -120,6 +123,13 @@ public:
*/
EXPORT void setDefaultSchema(std::string schema);
/** @brief Set the default table charset. Can be overriden by column
* or table options
*
* @param default_charset the default CHARSET_INFO pointer
*/
EXPORT void setDefaultCharset(const CHARSET_INFO* default_charset);
protected:
ParseTree fParseTree;
std::string fDBSchema;

348
dbcon/mysql/ha_mcs_ddl.cpp
View File

@ -113,21 +113,21 @@ static void decode_objectname(char *buf, const char *path, size_t buf_size)
static void decode_file_path(const char *path, char *decoded_dbname,
char *decoded_tbname)
{
// The format cont ains './' in the beginning of a path.
char *dbname_start = (char*) path + 2;
char *dbname_end = dbname_start;
while (*dbname_end != '/')
// The format cont ains './' in the beginning of a path.
char *dbname_start = (char*) path + 2;
char *dbname_end = dbname_start;
while (*dbname_end != '/')
dbname_end++;
int cnt = dbname_end - dbname_start;
char *dbname = (char *)my_alloca(cnt + 1);
memcpy(dbname, dbname_start, cnt);
dbname[cnt] = '\0';
decode_objectname(decoded_dbname, dbname, FN_REFLEN);
my_afree(dbname);
int cnt = dbname_end - dbname_start;
char *dbname = (char *)my_alloca(cnt + 1);
memcpy(dbname, dbname_start, cnt);
dbname[cnt] = '\0';
decode_objectname(decoded_dbname, dbname, FN_REFLEN);
my_afree(dbname);
char *tbname_start = dbname_end + 1;
decode_objectname(decoded_tbname, tbname_start, FN_REFLEN);
char *tbname_start = dbname_end + 1;
decode_objectname(decoded_tbname, tbname_start, FN_REFLEN);
}
@ -158,7 +158,7 @@ int parseCompressionComment ( std::string comment )
{
//Find the pattern, now get the compression type
string compType (&(*(what[0].second)));
//; is the seperator between compression and autoincrement comments.
//; is the separator between compression and autoincrement comments.
unsigned i = compType.find_first_of(";");
if ( i <= compType.length() )
@ -759,7 +759,8 @@ bool anyNullInTheColumn (THD* thd, string& schema, string& table, string& column
}
int ProcessDDLStatement(string& ddlStatement, string& schema, const string& table, int sessionID,
string& emsg, int compressionTypeIn = 2, bool isAnyAutoincreCol = false, int64_t nextvalue = 1, std::string autoiColName = "")
string& emsg, int compressionTypeIn = 2, bool isAnyAutoincreCol = false, int64_t nextvalue = 1, std::string autoiColName = "",
const CHARSET_INFO* default_table_charset = NULL)
{
SqlParser parser;
THD* thd = current_thd;
@ -768,6 +769,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
#endif
parser.setDefaultSchema(schema);
parser.setDefaultCharset(default_table_charset);
int rc = 0;
IDBCompressInterface idbCompress;
parser.Parse(ddlStatement.c_str());
@ -1477,23 +1479,23 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
return rc;
}
/* else if ( dynamic_cast<AtaSetColumnDefault*> (actionList[i]) )
{
rc = 1;
thd->get_stmt_da()->set_overwrite_status(true);
thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str());
ci->alterTableState = cal_connection_info::NOT_ALTER;
ci->isAlter = false;
return rc;
}
else if ( dynamic_cast<AtaDropColumnDefault*> (actionList[i]) )
{
rc = 1;
thd->get_stmt_da()->set_overwrite_status(true);
thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str());
ci->alterTableState = cal_connection_info::NOT_ALTER;
ci->isAlter = false;
return rc;
}
{
rc = 1;
thd->get_stmt_da()->set_overwrite_status(true);
thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str());
ci->alterTableState = cal_connection_info::NOT_ALTER;
ci->isAlter = false;
return rc;
}
else if ( dynamic_cast<AtaDropColumnDefault*> (actionList[i]) )
{
rc = 1;
thd->get_stmt_da()->set_overwrite_status(true);
thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str());
ci->alterTableState = cal_connection_info::NOT_ALTER;
ci->isAlter = false;
return rc;
}
*/
else if ( ddlpackage::AtaAddColumns* addColumnsPtr = dynamic_cast<AtaAddColumns*>(actionList[i]))
{
@ -1879,14 +1881,14 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
//@Bug 5444. rename column doen't need to check this.
/* if (tblInfo.tablewithautoincr == 1)
{
rc = 1;
thd->get_stmt_da()->set_overwrite_status(true);
thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)).c_str());
ci->alterTableState = cal_connection_info::NOT_ALTER;
ci->isAlter = false;
return rc;
} */
{
rc = 1;
thd->get_stmt_da()->set_overwrite_status(true);
thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)).c_str());
ci->alterTableState = cal_connection_info::NOT_ALTER;
ci->isAlter = false;
return rc;
} */
if (!validateAutoincrementDatatype(renameColumnsPtr->fNewType->fType))
{
@ -2113,68 +2115,68 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
static bool get_field_default_value(THD *thd, Field *field, String *def_value,
bool quoted)
{
bool has_default;
enum enum_field_types field_type= field->type();
bool has_default;
enum enum_field_types field_type= field->type();
has_default= (field->default_value ||
(!(field->flags & NO_DEFAULT_VALUE_FLAG) &&
field->unireg_check != Field::NEXT_NUMBER));
has_default= (field->default_value ||
(!(field->flags & NO_DEFAULT_VALUE_FLAG) &&
field->unireg_check != Field::NEXT_NUMBER));
def_value->length(0);
if (has_default)
{
StringBuffer<MAX_FIELD_WIDTH> str(field->charset());
if (field->default_value)
def_value->length(0);
if (has_default)
{
field->default_value->print(&str);
if (field->default_value->expr->need_parentheses_in_default())
{
def_value->set_charset(&my_charset_utf8mb4_general_ci);
def_value->append('(');
def_value->append(str);
def_value->append(')');
}
else
def_value->append(str);
}
else if (!field->is_null())
{ // Not null by default
if (field_type == MYSQL_TYPE_BIT)
{
str.qs_append('b');
str.qs_append('\'');
str.qs_append(field->val_int(), 2);
str.qs_append('\'');
quoted= 0;
}
else
{
field->val_str(&str);
if (!field->str_needs_quotes())
quoted= 0;
}
if (str.length())
{
StringBuffer<MAX_FIELD_WIDTH> def_val;
uint dummy_errors;
/* convert to system_charset_info == utf8 */
def_val.copy(str.ptr(), str.length(), field->charset(),
system_charset_info, &dummy_errors);
if (quoted)
append_unescaped(def_value, def_val.ptr(), def_val.length());
StringBuffer<MAX_FIELD_WIDTH> str(field->charset());
if (field->default_value)
{
field->default_value->print(&str);
if (field->default_value->expr->need_parentheses_in_default())
{
def_value->set_charset(&my_charset_utf8mb4_general_ci);
def_value->append('(');
def_value->append(str);
def_value->append(')');
}
else
def_value->append(str);
}
else if (!field->is_null())
{ // Not null by default
if (field_type == MYSQL_TYPE_BIT)
{
str.qs_append('b');
str.qs_append('\'');
str.qs_append(field->val_int(), 2);
str.qs_append('\'');
quoted= 0;
}
else
{
field->val_str(&str);
if (!field->str_needs_quotes())
quoted= 0;
}
if (str.length())
{
StringBuffer<MAX_FIELD_WIDTH> def_val;
uint dummy_errors;
/* convert to system_charset_info == utf8 */
def_val.copy(str.ptr(), str.length(), field->charset(),
system_charset_info, &dummy_errors);
if (quoted)
append_unescaped(def_value, def_val.ptr(), def_val.length());
else
def_value->append(def_val);
}
else if (quoted)
def_value->set(STRING_WITH_LEN("''"), system_charset_info);
}
else if (field->maybe_null() && quoted)
def_value->set(STRING_WITH_LEN("NULL"), system_charset_info); // Null as default
else
def_value->append(def_val);
}
else if (quoted)
def_value->set(STRING_WITH_LEN("''"), system_charset_info);
}
else if (field->maybe_null() && quoted)
def_value->set(STRING_WITH_LEN("NULL"), system_charset_info); // Null as default
else
return 0;
return 0;
}
return has_default;
}
return has_default;
}
/*
@ -2358,67 +2360,75 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea
return 1;
}
// Check if this is one of
// Check if this is one of
// * "CREATE TABLE ... LIKE "
// * "ALTER TABLE ... ENGINE=Columnstore"
// * "CREATE TABLE ... AS ..."
// If so generate a full create table statement using the properties of
// the source table. Note that source table has to be a columnstore table and
// we only check for currently supported options.
//
// If so generate a full create table statement using the properties of
// the source table. Note that source table has to be a columnstore table and
// we only check for currently supported options.
//
if ((thd->lex->sql_command == SQLCOM_CREATE_TABLE && thd->lex->used_tables) ||
if ((thd->lex->sql_command == SQLCOM_CREATE_TABLE && thd->lex->used_tables) ||
(thd->lex->sql_command == SQLCOM_ALTER_TABLE && create_info->used_fields & HA_CREATE_USED_ENGINE) ||
(thd->lex->create_info.like()))
{
TABLE_SHARE *share = table_arg->s;
my_bitmap_map *old_map; // To save the read_set
char datatype_buf[MAX_FIELD_WIDTH], def_value_buf[MAX_FIELD_WIDTH];
String datatype, def_value;
ostringstream oss;
{
TABLE_SHARE *share = table_arg->s;
my_bitmap_map *old_map; // To save the read_set
char datatype_buf[MAX_FIELD_WIDTH], def_value_buf[MAX_FIELD_WIDTH];
String datatype, def_value;
ostringstream oss;
string tbl_name = string(share->db.str) + "." + string(share->table_name.str);
// Save the current read_set map and mark it for read
old_map= tmp_use_all_columns(table_arg, table_arg->read_set);
// Save the current read_set map and mark it for read
old_map= tmp_use_all_columns(table_arg, table_arg->read_set);
oss << "CREATE TABLE " << tbl_name << " (";
oss << "CREATE TABLE " << tbl_name << " (";
restore_record(table_arg, s->default_values);
for (Field **field= table_arg->field; *field; field++)
{
uint flags = (*field)->flags;
datatype.set(datatype_buf, sizeof(datatype_buf), system_charset_info);
(*field)->sql_type(datatype);
if (field != table_arg->field)
oss << ", ";
oss << (*field)->field_name.str << " " << datatype.ptr();
restore_record(table_arg, s->default_values);
for (Field **field= table_arg->field; *field; field++)
{
uint flags = (*field)->flags;
datatype.set(datatype_buf, sizeof(datatype_buf), system_charset_info);
(*field)->sql_type(datatype);
if (field != table_arg->field)
oss << ", ";
oss << (*field)->field_name.str << " " << datatype.ptr();
if (flags & NOT_NULL_FLAG)
oss << " NOT NULL";
if (flags & NOT_NULL_FLAG)
oss << " NOT NULL";
def_value.set(def_value_buf, sizeof(def_value_buf), system_charset_info);
if (get_field_default_value(thd, *field, &def_value, true)) {
oss << " DEFAULT " << def_value.c_ptr();
}
if ((*field)->comment.length)
{
String comment;
append_unescaped(&comment, (*field)->comment.str, (*field)->comment.length);
oss << " COMMENT ";
oss << comment.c_ptr();
}
def_value.set(def_value_buf, sizeof(def_value_buf), system_charset_info);
if (get_field_default_value(thd, *field, &def_value, true))
{
oss << " DEFAULT " << def_value.c_ptr();
}
}
// End the list of columns
oss<< ") ENGINE=columnstore ";
const CHARSET_INFO* field_cs = (*field)->charset();
if (field_cs && (!share->table_charset || field_cs->number != share->table_charset->number))
{
oss << " CHARACTER SET " << field_cs->csname;
}
// Process table level options
if ((*field)->comment.length)
{
String comment;
append_unescaped(&comment, (*field)->comment.str, (*field)->comment.length);
oss << " COMMENT ";
oss << comment.c_ptr();
}
}
// End the list of columns
oss << ") ENGINE=columnstore ";
// Process table level options
/* TODO: uncomment when we support AUTO_INCREMENT
if (create_info->auto_increment_value > 1)
{
oss << " AUTO_INCREMENT=" << create_info->auto_increment_value;
}
if (create_info->auto_increment_value > 1)
{
oss << " AUTO_INCREMENT=" << create_info->auto_increment_value;
}
*/
if (create_info->auto_increment_value > 1)
@ -2427,40 +2437,42 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea
}
if (share->table_charset)
{
oss << " DEFAULT CHARSET=" << share->table_charset->csname;
}
if (share->table_charset)
{
oss << " DEFAULT CHARSET=" << share->table_charset->csname;
}
// Process table level options such as MIN_ROWS, MAX_ROWS, COMMENT
// Process table level options such as MIN_ROWS, MAX_ROWS, COMMENT
if (share->min_rows)
{
char buff[80];
longlong10_to_str(share->min_rows, buff, 10);
oss << " MIN_ROWS=" << buff;
}
if (share->min_rows)
{
char buff[80];
longlong10_to_str(share->min_rows, buff, 10);
oss << " MIN_ROWS=" << buff;
}
if (share->max_rows) {
char buff[80];
longlong10_to_str(share->max_rows, buff, 10);
oss << " MAX_ROWS=" << buff;
}
if (share->max_rows)
{
char buff[80];
longlong10_to_str(share->max_rows, buff, 10);
oss << " MAX_ROWS=" << buff;
}
if (share->comment.length) {
String comment;
append_unescaped(&comment, share->comment.str, share->comment.length);
oss << " COMMENT ";
oss << comment.c_ptr();
}
if (share->comment.length)
{
String comment;
append_unescaped(&comment, share->comment.str, share->comment.length);
oss << " COMMENT ";
oss << comment.c_ptr();
}
oss << ";";
stmt = oss.str();
oss << ";";
stmt = oss.str();
tmp_restore_column_map(table_arg->read_set, old_map);
}
tmp_restore_column_map(table_arg->read_set, old_map);
}
rc = ProcessDDLStatement(stmt, db, tbl, tid2sid(thd->thread_id), emsg, compressiontype, isAnyAutoincreCol, startValue, columnName);
rc = ProcessDDLStatement(stmt, db, tbl, tid2sid(thd->thread_id), emsg, compressiontype, isAnyAutoincreCol, startValue, columnName, create_info->default_table_charset);
if (rc != 0)
{