database/mariadb
database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-04-18 21:44:20 +03:00
Code

MDEV-34740 mariadb-import: optimize index and constraint creation

Browse Source 
For InnoDB tables, parse the CREATE TABLE statement to defer index and
constraint creation until after data loading. For other storage engines,
the DISABLE/ENABLE KEYS commands achieve similar optimization.

This behavior is controlled by a new option, innodb-optimize-keys
(default: ON), compatible with mydumper.

Additionally, this commit separates the table creation phase from data
loading. Running DDL statements (such as DROP IF EXISTS) in a single
thread avoids the "table not locked" issue from MDEV-34741. As a bonus,
view creation no longer requires a separate step.
This commit is contained in:
Vladislav Vaintroub 2025-01-23 15:15:03 +01:00
parent 9255206b86
commit 3bd23b76c5
10 changed files with 971 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
client/CMakeLists.txt
View File

@ -56,9 +56,17 @@ TARGET_LINK_LIBRARIES(mariadb-check ${CLIENT_LIB})
MYSQL_ADD_EXECUTABLE(mariadb-dump mysqldump.cc ../sql-common/my_user.c connection_pool.cc)
TARGET_LINK_LIBRARIES(mariadb-dump ${CLIENT_LIB})
ADD_LIBRARY(import_util STATIC import_util.cc import_util.h)
IF (PCRE2_DEBIAN_HACK)
SET_SOURCE_FILES_PROPERTIES(import_util.cc PROPERTIES COMPILE_FLAGS "${PCRE2_DEBIAN_HACK}")
ENDIF()
TARGET_LINK_LIBRARIES(import_util PRIVATE pcre2-posix pcre2-8)
TARGET_INCLUDE_DIRECTORIES(import_util PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
MYSQL_ADD_EXECUTABLE(mariadb-import mysqlimport.cc)
target_include_directories(mariadb-import PRIVATE ${CMAKE_SOURCE_DIR}/tpool)
target_link_libraries(mariadb-import PRIVATE tpool ${CLIENT_LIB})
TARGET_INCLUDE_DIRECTORIES(mariadb-import PRIVATE ${CMAKE_SOURCE_DIR}/tpool)
TARGET_LINK_LIBRARIES(mariadb-import PRIVATE tpool ${CLIENT_LIB} import_util)
MYSQL_ADD_EXECUTABLE(mariadb-upgrade mysql_upgrade.c COMPONENT Server)
TARGET_LINK_LIBRARIES(mariadb-upgrade ${CLIENT_LIB})
@ -101,3 +109,6 @@ FOREACH(t mariadb mariadb-test mariadb-check mariadb-dump mariadb-import mariadb
ENDFOREACH()
ADD_DEFINITIONS(-DHAVE_DLOPEN)
IF(WITH_UNIT_TESTS)
ADD_SUBDIRECTORY(${PROJECT_SOURCE_DIR}/unittest/client ${PROJECT_BINARY_DIR}/unittest/client)
ENDIF()

217
client/import_util.cc Normal file
View File

@ -0,0 +1,217 @@
/*
Copyright (c) 2024, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA
*/
/*
This file contains some routines to do client-side parsing of CREATE TABLE
statements. The goal is to extract the primary key, constraints, and
secondary key. his is useful for optimizing the import process, to delay
secondary index creation until after the data has been loaded.
*/
#include <string>
#include <vector>
#include <pcre2posix.h>
#include "import_util.h"
#include <assert.h>
/**
* Extract the first CREATE TABLE statement from a script.
*
* @param script The input script containing SQL statements.
* @return std::string The first CREATE TABLE statement found, or an empty
* string if not found.
*/
std::string extract_first_create_table(const std::string &script)
{
regex_t create_table_regex;
regmatch_t match[2];
const char *pattern= "(CREATE\\s+TABLE\\s+[^;]+;)\\s*\\n";
regcomp(&create_table_regex, pattern, REG_EXTENDED);
if (regexec(&create_table_regex, script.c_str(), 2, match, 0) == 0)
{
std::string result=
script.substr(match[1].rm_so, match[1].rm_eo - match[1].rm_so);
regfree(&create_table_regex);
return result;
}
regfree(&create_table_regex);
return "";
}
TableDDLInfo::TableDDLInfo(const std::string &create_table_stmt)
{
regex_t primary_key_regex, constraint_regex, index_regex, engine_regex,
table_name_regex;
constexpr size_t MAX_MATCHES= 10;
regmatch_t match[10];
regcomp(&primary_key_regex, "\\n\\s*(PRIMARY\\s+KEY\\s+(.*?)),?\\n",
REG_EXTENDED);
regcomp(&constraint_regex,
"\\n\\s*(CONSTRAINT\\s+(`?(?:[^`]|``)+`?)\\s+.*?),?\\n",
REG_EXTENDED);
regcomp(&index_regex,
"\\n\\s*(((?:UNIQUE|FULLTEXT|VECTOR|SPATIAL)\\s+)?(INDEX|KEY)\\s+(`(?:[^`]|``)+`)\\s+.*?),?\\n",
REG_EXTENDED);
regcomp(&engine_regex, "\\bENGINE\\s*=\\s*(\\w+)", REG_EXTENDED);
regcomp(&table_name_regex, "CREATE\\s+TABLE\\s+(`?(?:[^`]|``)+`?)\\s*\\(",
REG_EXTENDED);
const char *stmt= create_table_stmt.c_str();
const char *search_start= stmt;
// Extract primary key
if (regexec(&primary_key_regex, search_start, MAX_MATCHES, match, 0) == 0)
{
primary_key= {std::string(stmt + match[1].rm_so, match[1].rm_eo - match[1].rm_so),
"PRIMARY"};
}
// Extract constraints and foreign keys
search_start= stmt;
while (regexec(&constraint_regex, search_start, MAX_MATCHES, match, 0) == 0)
{
assert(match[2].rm_so != -1);
assert(match[1].rm_so != -1);
std::string name(search_start + match[2].rm_so, match[2].rm_eo - match[2].rm_so);
std::string definition(search_start + match[1].rm_so, match[1].rm_eo - match[1].rm_so);
constraints.push_back({definition, name});
search_start+= match[0].rm_eo - 1;
}
// Extract secondary indexes
search_start= stmt;
while (regexec(&index_regex, search_start, MAX_MATCHES, match, 0) == 0)
{
assert(match[4].rm_so != -1);
std::string name(search_start + match[4].rm_so, match[4].rm_eo - match[4].rm_so);
std::string definition(search_start + match[1].rm_so, match[1].rm_eo - match[1].rm_so);
secondary_indexes.push_back({definition, name});
search_start+= match[0].rm_eo -1;
}
// Extract storage engine
if (regexec(&engine_regex, stmt, MAX_MATCHES, match, 0) == 0)
{
storage_engine= std::string(stmt + match[1].rm_so, match[1].rm_eo - match[1].rm_so);
}
// Extract table name
if (regexec(&table_name_regex, stmt, MAX_MATCHES, match, 0) == 0)
{
table_name= std::string(stmt + match[1].rm_so, match[1].rm_eo - match[1].rm_so);
}
if (primary_key.definition.empty() && storage_engine == "InnoDB")
{
for (const auto &index : secondary_indexes)
{
if (index.definition.find("UNIQUE") != std::string::npos)
{
non_pk_clustering_key_name= index.name;
break;
}
}
}
regfree(&primary_key_regex);
regfree(&constraint_regex);
regfree(&index_regex);
regfree(&engine_regex);
regfree(&table_name_regex);
}
/**
Convert a KeyOrConstraintDefinitionType enum value to its
corresponding string representation.
@param type The KeyOrConstraintDefinitionType enum value.
@return std::string The string representation of the
KeyOrConstraintDefinitionType.
*/
static std::string to_string(KeyOrConstraintType type)
{
switch (type)
{
case KeyOrConstraintType::CONSTRAINT:
return "CONSTRAINT";
case KeyOrConstraintType::INDEX:
return "INDEX";
default:
return "UNKNOWN";
}
}
std::string TableDDLInfo::generate_alter_add(
const std::vector<KeyDefinition> &definitions,
KeyOrConstraintType type) const
{
if (definitions.empty() ||
(type == KeyOrConstraintType::INDEX && definitions.size() == 1
&& !non_pk_clustering_key_name.empty()))
{
return "";
}
std::string sql= "ALTER TABLE " + table_name + " ";
bool need_comma= false;
for (const auto &definition : definitions)
{
/*
Do not add or drop clustering secondary index
*/
if (type == KeyOrConstraintType::INDEX &&
definition.name == non_pk_clustering_key_name)
continue;
if (need_comma)
sql+= ", ";
else
need_comma= true;
sql+= "ADD " + definition.definition;
}
return sql;
}
std::string TableDDLInfo::generate_alter_drop(
const std::vector<KeyDefinition> &definitions, KeyOrConstraintType type) const
{
if (definitions.empty() ||
(type == KeyOrConstraintType::INDEX && definitions.size() == 1 &&
!non_pk_clustering_key_name.empty()))
{
return "";
}
std::string sql= "ALTER TABLE " + table_name + " ";
bool need_comma= false;
for (const auto &definition : definitions)
{
if (type == KeyOrConstraintType::INDEX &&
definition.name == non_pk_clustering_key_name)
continue;
if (need_comma)
sql+= ", ";
else
need_comma= true;
sql+= "DROP " + to_string(type) + " " +
definition.name;
}
return sql;
}

89
client/import_util.h Normal file
View File

@ -0,0 +1,89 @@
/*
Copyright (c) 2024, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA
*/
#pragma once
#include <string>
#include <vector>
/* TABLE DDL INFO - representation of parsed CREATE TABLE Statement */
enum class KeyOrConstraintType
{
CONSTRAINT,
INDEX,
UNKNOWN
};
/**
* Struct representing a table keyor constraint definition
*/
struct KeyDefinition
{
/** Full key or constraint definition string,
e.g UNIQUE KEY `uniq_idx` (`col`) */
std::string definition;
/** The name of key or constraint, including escape chars */
std::string name;
};
/**
Information about keys and constraints, extracted from
CREATE TABLE statement
*/
struct TableDDLInfo
{
TableDDLInfo(const std::string &create_table_stmt);
KeyDefinition primary_key;
std::vector<KeyDefinition> constraints;
std::vector<KeyDefinition> secondary_indexes;
std::string storage_engine;
std::string table_name;
/* Innodb is using first UNIQUE key for clustering, if no PK is set*/
std::string non_pk_clustering_key_name;
/**
Generate ALTER TABLE ADD/DROP statements for keys or constraints.
The goal is to remove indexes/constraints before the data is imported
and recreate them after import.
PRIMARY key is not affected by these operations
*/
std::string generate_alter_add(const std::vector<KeyDefinition> &defs,
KeyOrConstraintType type) const;
std::string generate_alter_drop(const std::vector<KeyDefinition> &defs,
KeyOrConstraintType type) const;
std::string drop_constraints_sql() const
{
return generate_alter_drop(constraints, KeyOrConstraintType::CONSTRAINT);
}
std::string add_constraints_sql() const
{
return generate_alter_add(constraints, KeyOrConstraintType::CONSTRAINT);
}
std::string drop_secondary_indexes_sql() const
{
return generate_alter_drop(secondary_indexes,
KeyOrConstraintType::INDEX);
}
std::string add_secondary_indexes_sql() const
{
return generate_alter_add(secondary_indexes,
KeyOrConstraintType::INDEX);
}
};
std::string extract_first_create_table(const std::string &script);

332
client/mysqlimport.cc
View File

@ -43,6 +43,7 @@
#include <vector>
#include <unordered_set>
#include <my_dir.h>
#include "import_util.h"
tpool::thread_pool *thread_pool;
static std::vector<MYSQL *> all_tp_connections;
@ -55,7 +56,10 @@ static char *field_escape(char *to,const char *from,uint length);
static char *add_load_option(char *ptr,const char *object,
const char *statement);
static my_bool verbose=0,lock_tables=0,ignore_errors=0,opt_delete=0,
static std::string parse_sql_script(const char *filepath, bool *tz_utc,
std::vector<std::string> *trigger_defs);
static my_bool verbose=0,lock_tables=0,ignore_errors=0,opt_delete=0,
replace, silent, ignore, ignore_foreign_keys,
opt_compress, opt_low_priority, tty_password;
static my_bool debug_info_flag= 0, debug_check_flag= 0;
@ -70,6 +74,7 @@ static char * opt_mysql_unix_port=0;
static char *opt_plugin_dir= 0, *opt_default_auth= 0;
static longlong opt_ignore_lines= -1;
static char *opt_dir;
static char opt_innodb_optimize_keys;
#include <sslopt-vars.h>
@ -82,9 +87,26 @@ struct table_load_params
std::string data_file; /* name of the file to load with LOAD DATA INFILE */
std::string sql_file; /* name of the file that contains CREATE TABLE or
CREATE VIEW */
std::string tablename; /* name of the table */
std::string dbname; /* name of the database */
bool tz_utc= false; /* true if the script sets the timezone to UTC */
bool is_view= false; /* true if the script is for a VIEW */
std::vector<std::string> triggers; /* CREATE TRIGGER statements */
ulonglong size= 0; /* size of the data file */
std::string sql_text; /* content of the SQL file, without triggers */
TableDDLInfo ddl_info; /* parsed CREATE TABLE statement */
table_load_params(const char* dfile, const char* sqlfile,
const char* db, ulonglong data_size)
: data_file(dfile), sql_file(sqlfile),
dbname(db), triggers(),
size(data_size),
sql_text(parse_sql_script(sqlfile, &tz_utc, &triggers)),
ddl_info(sql_text)
{
is_view= ddl_info.table_name.empty();
}
int create_table_or_view(MYSQL *);
int load_data(MYSQL *);
};
std::unordered_set<std::string> ignore_databases;
@ -173,6 +195,9 @@ static struct my_option my_long_options[] =
"--ignore-table=database.table. Only takes effect when used together with "
"--dir option",
0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"innodb-optimize-keys", 0, "Create secondary indexes after data load (Innodb only).",
&opt_innodb_optimize_keys, &opt_innodb_optimize_keys, 0, GET_BOOL, NO_ARG,
1, 0, 0, 0, 0, 0},
{"lines-terminated-by", 0,
"Lines in the input file are terminated by the given string.",
&lines_terminated, &lines_terminated, 0, GET_STR,
@ -512,8 +537,7 @@ static int exec_sql(MYSQL *mysql, const std::string& s)
@return content of the file as a string, excluding CREATE TRIGGER statements
*/
static std::string parse_sql_script(const char *filepath, bool *tz_utc, std::vector<std::string> *create_trigger_Defs,
std::string *engine)
static std::string parse_sql_script(const char *filepath, bool *tz_utc, std::vector<std::string> *create_trigger_Defs)
{
/*Read full file to string*/
std::ifstream t(filepath);
@ -552,16 +576,6 @@ static std::string parse_sql_script(const char *filepath, bool *tz_utc, std::vec
with --tz-utc option
*/
*tz_utc= sql_text.find("SET TIME_ZONE='+00:00'") != std::string::npos;
*engine= "";
auto engine_pos= sql_text.find("ENGINE=");
if (engine_pos != std::string::npos)
{
engine_pos+= 7;
auto end_pos= sql_text.find_first_of(" ", engine_pos);
if (end_pos != std::string::npos)
*engine= sql_text.substr(engine_pos, end_pos - engine_pos);
}
return sql_text;
}
@ -584,64 +598,88 @@ static int create_db_if_not_exists(MYSQL *mysql, const char *dbname)
return 0;
}
static int handle_one_table(const table_load_params *params, MYSQL *mysql)
#include "import_util.h"
int table_load_params::create_table_or_view(MYSQL* mysql)
{
if (sql_file.empty())
return 0;
if (verbose && !sql_file.empty())
{
fprintf(stdout, "Executing SQL script %s\n", sql_file.c_str());
}
if (!dbname.empty())
{
if (mysql_select_db(mysql, dbname.c_str()))
{
if (create_db_if_not_exists(mysql, dbname.c_str()))
return 1;
if (mysql_select_db(mysql, dbname.c_str()))
{
db_error(mysql);
return 1;
}
}
}
if (sql_text.empty())
{
fprintf(stderr, "Error: CREATE TABLE statement not found in %s\n",
sql_file.c_str());
return 1;
}
if (execute_sql_batch(mysql, sql_text.c_str(), sql_file.c_str()))
return 1;
/*
Temporarily drop from table definitions, if --innodb-optimize-keys is given. We'll add them back
later, after the data is loaded.
*/
auto drop_constraints_sql= ddl_info.drop_constraints_sql();
if (!drop_constraints_sql.empty())
{
if (exec_sql(mysql, drop_constraints_sql))
return 1;
}
return 0;
}
int table_load_params::load_data(MYSQL *mysql)
{
char tablename[FN_REFLEN], hard_path[FN_REFLEN],
escaped_name[FN_REFLEN * 2 + 1],
sql_statement[FN_REFLEN*16+256], *end;
DBUG_ENTER("handle_one_table");
DBUG_PRINT("enter",("datafile: %s",params->data_file.c_str()));
DBUG_ENTER("table_load_params::load");
DBUG_PRINT("enter",("datafile: %s",data_file.c_str()));
DBUG_ASSERT(!dbname.empty());
if (data_file.empty())
DBUG_RETURN(0);
if (aborting)
return 1;
if (verbose && !params->sql_file.empty())
DBUG_RETURN(0);
if (!dbname.empty() && mysql_select_db(mysql, dbname.c_str()))
{
fprintf(stdout, "Executing SQL script %s\n", params->sql_file.c_str());
db_error(mysql);
DBUG_RETURN(1);
}
if (!params->dbname.empty())
{
if (mysql_select_db(mysql, params->dbname.c_str()))
{
if (create_db_if_not_exists(mysql, params->dbname.c_str()))
DBUG_RETURN(1);
if (mysql_select_db(mysql, params->dbname.c_str()))
db_error(mysql);
}
}
const char *filename= params->data_file.c_str();
if (!filename[0])
filename= params->sql_file.c_str();
const char *filename= data_file.c_str();
fn_format(tablename, filename, "", "", 1 | 2); /* removes path & ext. */
const char *db= current_db ? current_db : params->dbname.c_str();
const char *db= current_db ? current_db : dbname.c_str();
std::string full_tablename= quote_identifier(db);
full_tablename+= ".";
full_tablename+= quote_identifier(tablename);
bool tz_utc= false;
std::string engine;
std::vector<std::string> triggers;
if (!params->sql_file.empty())
{
std::string sql_text= parse_sql_script(params->sql_file.c_str(), &tz_utc, &triggers,&engine);
if (execute_sql_batch(mysql, sql_text.c_str(),params->sql_file.c_str()))
DBUG_RETURN(1);
if (params->data_file.empty())
{
/*
We only use .sql extension for VIEWs, so we're done
with this file, there is no data to load.
*/
DBUG_RETURN(0);
}
if (tz_utc && exec_sql(mysql, "SET TIME_ZONE='+00:00';"))
DBUG_RETURN(1);
if (exec_sql(mysql, std::string("ALTER TABLE ") + full_tablename + " DISABLE KEYS"))
DBUG_RETURN(1);
}
if (tz_utc && exec_sql(mysql, "SET TIME_ZONE='+00:00';"))
DBUG_RETURN(1);
if (exec_sql(mysql,
std::string("ALTER TABLE ") + full_tablename + " DISABLE KEYS"))
DBUG_RETURN(1);
if (!opt_local_file)
strmov(hard_path,filename);
else
@ -656,9 +694,21 @@ static int handle_one_table(const table_load_params *params, MYSQL *mysql)
if (exec_sql(mysql, sql_statement))
DBUG_RETURN(1);
}
bool recreate_secondary_keys= false;
if (opt_innodb_optimize_keys && ddl_info.storage_engine == "InnoDB")
{
auto drop_secondary_keys_sql= ddl_info.drop_secondary_indexes_sql();
if (!drop_secondary_keys_sql.empty())
{
recreate_secondary_keys= true;
if (exec_sql(mysql, drop_secondary_keys_sql))
DBUG_RETURN(1);
}
}
if (exec_sql(mysql, "SET collation_database=binary"))
DBUG_RETURN(1);
to_unix_path(hard_path);
if (verbose)
{
@ -707,34 +757,50 @@ static int handle_one_table(const table_load_params *params, MYSQL *mysql)
fprintf(stdout, "%s.%s: %s\n", db, tablename, info);
}
/* Create triggers after loading data */
for (const auto &trigger: triggers)
if (exec_sql(mysql, std::string("ALTER TABLE ") + full_tablename + " ENABLE KEYS;"))
DBUG_RETURN(1);
if (ddl_info.storage_engine == "MyISAM" || ddl_info.storage_engine == "Aria")
{
if (mysql_query(mysql,trigger.c_str()))
{
db_error_with_table(mysql, tablename);
/* Avoid "table was not properly closed" warnings */
if (exec_sql(mysql, std::string("FLUSH TABLE ").append(full_tablename).c_str()))
DBUG_RETURN(1);
}
}
if (!params->sql_file.empty())
if (recreate_secondary_keys)
{
if (exec_sql(mysql, std::string("ALTER TABLE ") + full_tablename + " ENABLE KEYS;"))
DBUG_RETURN(1);
if (engine == "MyISAM" || engine == "Aria")
auto create_secondary_keys_sql= ddl_info.add_secondary_indexes_sql();
if (!create_secondary_keys_sql.empty())
{
/* Avoid "table was not properly closed" warnings */
if (exec_sql(mysql, std::string("FLUSH TABLE ").append(full_tablename).c_str()))
if (verbose)
{
fprintf(stdout, "Adding secondary indexes to table %s\n", ddl_info.table_name.c_str());
}
if (exec_sql(mysql, create_secondary_keys_sql))
DBUG_RETURN(1);
}
}
if (tz_utc)
{
if (exec_sql(mysql, "SET TIME_ZONE=@save_tz;"))
DBUG_RETURN(1);
}
/* Restore constrains and triggers */
for (const auto &create_trigger_def : triggers)
{
if (exec_sql(mysql, create_trigger_def))
return 1;
}
if (opt_innodb_optimize_keys && ddl_info.storage_engine == "InnoDB")
{
std::string constraints= ddl_info.add_constraints_sql();
if (!constraints.empty())
{
if (exec_sql(mysql, constraints))
return 1;
}
}
DBUG_RETURN(0);
}
@ -921,8 +987,10 @@ static void db_error(MYSQL *mysql)
const char *info= mysql_info(mysql);
auto err= mysql_errno(mysql);
auto err_text = mysql_error(mysql);
my_printf_error(0,"Error: %d %s %s", MYF(0), err, err_text, info);
if (info)
my_printf_error(0,"Error: %d %s %s", MYF(0), err, err_text, info);
else
my_printf_error(0, "Error %d %s", MYF(0), err, err_text);
safe_exit(1, mysql);
}
@ -986,10 +1054,11 @@ void set_exitcode(int code)
static thread_local MYSQL *thread_local_mysql;
void load_single_table(void *arg)
static void load_single_table(void *arg)
{
int error;
if((error= handle_one_table((const table_load_params *) arg, thread_local_mysql)))
table_load_params *params= (table_load_params *) arg;
if ((error= params->load_data(thread_local_mysql)))
set_exitcode(error);
}
@ -1050,8 +1119,7 @@ static void tpool_thread_exit(void)
@note files are sorted by size, descending
*/
static void scan_backup_dir(const char *dir,
std::vector<table_load_params> &files,
std::vector<table_load_params> &views)
std::vector<table_load_params> &files)
{
MY_DIR *dir_info;
std::vector<std::string> subdirs;
@ -1095,8 +1163,6 @@ static void scan_backup_dir(const char *dir,
}
for (size_t j= 0; j < dir_info2->number_of_files; j++)
{
table_load_params par;
par.dbname= dbname;
fi= &dir_info2->dir_entry[j];
if (has_extension(fi->name, ".sql") || has_extension(fi->name, ".txt"))
{
@ -1123,13 +1189,14 @@ static void scan_backup_dir(const char *dir,
/* test file*/
if (has_extension(fi->name, ".txt"))
{
par.data_file= file;
par.size= fi->mystat->st_size;
par.sql_file= file.substr(0, file.size() - 4) + ".sql";
if (access(par.sql_file.c_str(), F_OK))
auto sql_file= file.substr(0, file.size() - 4) + ".sql";
if (access(sql_file.c_str(), F_OK))
{
fatal_error("Expected file '%s' is missing",par.sql_file.c_str());
fatal_error("Expected file '%s' is missing",sql_file.c_str());
}
table_load_params par(file.c_str(), sql_file.c_str(), dbname,
fi->mystat->st_size);
files.push_back(par);
}
else if (has_extension(fi->name, ".sql"))
@ -1141,9 +1208,9 @@ static void scan_backup_dir(const char *dir,
std::string txt_file= file.substr(0, file.size() - 4) + ".txt";
if (access(txt_file.c_str(), F_OK))
{
par.sql_file= file;
par.size= fi->mystat->st_size;
views.push_back(par);
table_load_params par("", file.c_str(), dbname,
fi->mystat->st_size);
files.push_back(par);
}
}
else
@ -1160,16 +1227,17 @@ static void scan_backup_dir(const char *dir,
std::sort(files.begin(), files.end(),
[](const table_load_params &a, const table_load_params &b) -> bool
{
/* Sort views after base tables */
if (a.is_view && !b.is_view)
return false;
if (!a.is_view && b.is_view)
return true;
/* Sort by size descending */
if (a.size > b.size)
return true;
if (a.size < b.size)
return false;
return a.sql_file < b.sql_file;
});
std::sort(views.begin(), views.end(),
[](const table_load_params &a, const table_load_params &b) -> bool
{
/* If sizes are equal, sort by name */
return a.sql_file < b.sql_file;
});
}
@ -1193,79 +1261,79 @@ int main(int argc, char **argv)
free_defaults(argv_to_free);
return(1);
}
if (opt_use_threads > MAX_THREADS)
{
fatal_error("Too many connections, max value for --parallel is %d\n",
MAX_THREADS);
}
sf_leaking_memory=0; /* from now on we cleanup properly */
std::vector<table_load_params> files_to_load, views_to_load;
std::vector<table_load_params> files_to_load;
if (opt_dir)
{
ignore_foreign_keys= 1;
if (argc)
fatal_error("Invalid arguments for --dir option");
scan_backup_dir(opt_dir, files_to_load, views_to_load);
scan_backup_dir(opt_dir, files_to_load);
}
else
{
for (; *argv != NULL; argv++)
{
table_load_params p{};
p.data_file= *argv;
table_load_params p(*argv, "", current_db, 0);
files_to_load.push_back(p);
}
}
if (files_to_load.empty())
{
fatal_error("No files to load");
return 1;
}
MYSQL *mysql=
db_connect(current_host, current_db, current_user, opt_password);
if (!mysql)
{
free_defaults(argv_to_free);
return 1;
}
for (auto &f : files_to_load)
{
if (f.create_table_or_view(mysql))
set_exitcode(1);
}
if (opt_use_threads && !lock_tables)
{
if (opt_use_threads > MAX_THREADS)
{
fatal_error("Too many connections, max value for --parallel is %d\n",
MAX_THREADS);
}
init_tp_connections(opt_use_threads);
thread_pool= tpool::create_thread_pool_generic(opt_use_threads,opt_use_threads);
thread_pool->set_thread_callbacks(tpool_thread_init,tpool_thread_exit);
std::vector<tpool::task> all_tasks;
for (const auto &f: files_to_load)
all_tasks.push_back(tpool::task(load_single_table, (void *)&f));
std::vector<tpool::task> load_tasks;
for (const auto &f : files_to_load)
{
load_tasks.push_back(tpool::task(load_single_table, (void *) &f));
}
for (auto &t: all_tasks)
for (auto &t: load_tasks)
thread_pool->submit_task(&t);
delete thread_pool;
close_tp_connections();
thread_pool= nullptr;
files_to_load.clear();
}
/*
The following block handles single-threaded load.
Also views that must be created after the base tables, are created here.
BUG: funny case would be views that select from other views, won't generally work.
It won't work in mysqldump either, but it's not a common case.
*/
if (!files_to_load.empty() || !views_to_load.empty())
else
{
MYSQL *mysql= db_connect(current_host,current_db,current_user,opt_password);
if (!mysql)
{
free_defaults(argv_to_free);
return(1); /* purecov: dead code */
}
if (lock_tables)
lock_table(mysql, argc, argv);
for (const auto &f : files_to_load)
if ((error= handle_one_table(&f, mysql)))
set_exitcode(error);
for (const auto &v : views_to_load)
if ((error= handle_one_table(&v, mysql)))
for (auto &f : files_to_load)
{
if ((error= f.load_data(mysql)))
set_exitcode(error);
db_disconnect(current_host, mysql);
}
}
mysql_close(mysql);
safe_exit(0, 0);
return(exitcode);
}

300
mysql-test/main/mariadb-import.result
View File

@ -77,89 +77,88 @@ test_suppressions.sql
test_suppressions.txt
Connecting to localhost
Executing SQL script vardir/tmp/dump/mysql/help_topic.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/help_topic.txt into help_topic
mysql.help_topic: Records: 839 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/time_zone_transition.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_transition.txt into time_zone_transition
mysql.time_zone_transition: Records: 394 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mtr/global_suppressions.sql
Loading data from LOCAL file: vardir/tmp/dump/mtr/global_suppressions.txt into global_suppressions
mtr.global_suppressions: Records: 99 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/help_keyword.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/help_keyword.txt into help_keyword
mysql.help_keyword: Records: 106 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/help_relation.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/help_relation.txt into help_relation
mysql.help_relation: Records: 202 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/help_category.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/help_category.txt into help_category
mysql.help_category: Records: 50 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/time_zone_transition_type.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_transition_type.txt into time_zone_transition_type
mysql.time_zone_transition_type: Records: 32 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/global_priv.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/global_priv.txt into global_priv
mysql.global_priv: Records: 5 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/time_zone_leap_second.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_leap_second.txt into time_zone_leap_second
mysql.time_zone_leap_second: Records: 23 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/proxies_priv.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/proxies_priv.txt into proxies_priv
mysql.proxies_priv: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/tables_priv.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/tables_priv.txt into tables_priv
mysql.tables_priv: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/time_zone_name.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_name.txt into time_zone_name
mysql.time_zone_name: Records: 7 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/time_zone.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone.txt into time_zone
mysql.time_zone: Records: 6 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/test/t1.sql
Loading data from LOCAL file: vardir/tmp/dump/test/t1.txt into t1
test.t1: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/column_stats.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/column_stats.txt into column_stats
mysql.column_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/columns_priv.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/columns_priv.txt into columns_priv
mysql.columns_priv: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/db.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/db.txt into db
mysql.db: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/func.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/func.txt into func
mysql.func: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/gtid_slave_pos.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/gtid_slave_pos.txt into gtid_slave_pos
mysql.gtid_slave_pos: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/index_stats.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/index_stats.txt into index_stats
mysql.index_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/plugin.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/plugin.txt into plugin
mysql.plugin: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/procs_priv.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/procs_priv.txt into procs_priv
mysql.procs_priv: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/roles_mapping.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/roles_mapping.txt into roles_mapping
mysql.roles_mapping: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/servers.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/servers.txt into servers
mysql.servers: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/table_stats.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/table_stats.txt into table_stats
mysql.table_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/mysql/event.sql
Executing SQL script vardir/tmp/dump/mysql/general_log.sql
Executing SQL script vardir/tmp/dump/mysql/innodb_index_stats.sql
Executing SQL script vardir/tmp/dump/mysql/innodb_table_stats.sql
Executing SQL script vardir/tmp/dump/mysql/slow_log.sql
Executing SQL script vardir/tmp/dump/mysql/transaction_registry.sql
Executing SQL script vardir/tmp/dump/mysql/help_keyword.sql
Executing SQL script vardir/tmp/dump/mysql/help_relation.sql
Executing SQL script vardir/tmp/dump/mysql/help_category.sql
Executing SQL script vardir/tmp/dump/mysql/time_zone_transition_type.sql
Executing SQL script vardir/tmp/dump/mysql/global_priv.sql
Executing SQL script vardir/tmp/dump/mysql/time_zone_leap_second.sql
Executing SQL script vardir/tmp/dump/mysql/proxies_priv.sql
Executing SQL script vardir/tmp/dump/mysql/tables_priv.sql
Executing SQL script vardir/tmp/dump/mysql/time_zone_name.sql
Executing SQL script vardir/tmp/dump/mysql/time_zone.sql
Executing SQL script vardir/tmp/dump/test/t1.sql
Executing SQL script vardir/tmp/dump/mysql/column_stats.sql
Executing SQL script vardir/tmp/dump/mysql/columns_priv.sql
Executing SQL script vardir/tmp/dump/mysql/db.sql
Executing SQL script vardir/tmp/dump/mysql/func.sql
Executing SQL script vardir/tmp/dump/mysql/gtid_slave_pos.sql
Executing SQL script vardir/tmp/dump/mysql/index_stats.sql
Executing SQL script vardir/tmp/dump/mysql/plugin.sql
Executing SQL script vardir/tmp/dump/mysql/procs_priv.sql
Executing SQL script vardir/tmp/dump/mysql/roles_mapping.sql
Executing SQL script vardir/tmp/dump/mysql/servers.sql
Executing SQL script vardir/tmp/dump/mysql/table_stats.sql
Executing SQL script vardir/tmp/dump/mysql/user.sql
Executing SQL script vardir/tmp/dump/mysql/transaction_registry.sql
Executing SQL script vardir/tmp/dump/mysql/slow_log.sql
Executing SQL script vardir/tmp/dump/test/v1.sql
Disconnecting from localhost
Executing SQL script vardir/tmp/dump/mysql/general_log.sql
Loading data from LOCAL file: vardir/tmp/dump/mysql/help_topic.txt into help_topic
mysql.help_topic: Records: 839 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_transition.txt into time_zone_transition
mysql.time_zone_transition: Records: 394 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mtr/global_suppressions.txt into global_suppressions
mtr.global_suppressions: Records: 99 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/help_keyword.txt into help_keyword
mysql.help_keyword: Records: 106 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/help_relation.txt into help_relation
mysql.help_relation: Records: 202 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/help_category.txt into help_category
mysql.help_category: Records: 50 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_transition_type.txt into time_zone_transition_type
mysql.time_zone_transition_type: Records: 32 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/global_priv.txt into global_priv
mysql.global_priv: Records: 5 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_leap_second.txt into time_zone_leap_second
mysql.time_zone_leap_second: Records: 23 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/proxies_priv.txt into proxies_priv
mysql.proxies_priv: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/tables_priv.txt into tables_priv
mysql.tables_priv: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_name.txt into time_zone_name
mysql.time_zone_name: Records: 7 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone.txt into time_zone
mysql.time_zone: Records: 6 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/test/t1.txt into t1
test.t1: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/column_stats.txt into column_stats
mysql.column_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/columns_priv.txt into columns_priv
mysql.columns_priv: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/db.txt into db
mysql.db: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/func.txt into func
mysql.func: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/gtid_slave_pos.txt into gtid_slave_pos
mysql.gtid_slave_pos: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/index_stats.txt into index_stats
mysql.index_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/plugin.txt into plugin
mysql.plugin: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/procs_priv.txt into procs_priv
mysql.procs_priv: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/roles_mapping.txt into roles_mapping
mysql.roles_mapping: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/servers.txt into servers
mysql.servers: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from LOCAL file: vardir/tmp/dump/mysql/table_stats.txt into table_stats
mysql.table_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0
drop table t1;
drop view v1;
create database db2;
@ -171,13 +170,35 @@ PRIMARY KEY (id)
CREATE TABLE child (
id INT,
parent_id INT,
c CHAR(4),
INDEX par_ind (parent_id),
UNIQUE INDEX(c),
FOREIGN KEY (parent_id)
REFERENCES parent(id)
ON DELETE CASCADE
ON DELETE CASCADE,
CHECK (c >= 'a')
) ENGINE=INNODB;
insert into parent values(1),(2);
insert into child values (1,1),(1,2),(2,1),(2,2);
insert into child values (1,1,'a'),(1,2,'b'),(2,1,'c'),(2,2,'d');
CREATE TABLE offices (
id int NOT NULL AUTO_INCREMENT,
PRIMARY KEY (id)
) ENGINE=InnoDB;
CREATE TABLE users (
id int NOT NULL AUTO_INCREMENT,
office_id int DEFAULT NULL,
slogan text GENERATED ALWAYS AS (concat('Hello world #',office_id)) STORED,
PRIMARY KEY (id),
KEY office_id (office_id),
CONSTRAINT users_ibfk_1 FOREIGN KEY (office_id) REFERENCES offices (id)
) ENGINE=InnoDB;
insert into offices values();
insert into offices values();
insert into offices values();
insert into offices values();
insert into users (office_id) values (1);
insert into users (office_id) values (2);
insert into users (office_id) values (3);
drop database db2;
use db2;
select * from parent;
@ -185,13 +206,116 @@ id
1
2
select * from child;
id parent_id
1 1
1 2
2 1
2 2
drop table child;
drop table parent;
id parent_id c
1 1 a
1 2 b
2 1 c
2 2 d
show create table parent;
Table Create Table
parent CREATE TABLE `parent` (
`id` int(11) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci
show create table child;
Table Create Table
child CREATE TABLE `child` (
`id` int(11) DEFAULT NULL,
`parent_id` int(11) DEFAULT NULL,
`c` char(4) DEFAULT NULL,
UNIQUE KEY `c` (`c`),
KEY `par_ind` (`parent_id`),
CONSTRAINT `child_ibfk_1` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE,
CONSTRAINT `CONSTRAINT_1` CHECK (`c` >= 'a')
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci
drop database db2;
# Repeat import with --verbose to see "Adding secondary keys" in the output
Connecting to localhost
Executing SQL script vardir/tmp/dump/db2/users.sql
Executing SQL script vardir/tmp/dump/db2/child.sql
Executing SQL script vardir/tmp/dump/db2/offices.sql
Executing SQL script vardir/tmp/dump/db2/parent.sql
Loading data from SERVER file: vardir/tmp/dump/db2/users.txt into users
db2.users: Records: 3 Deleted: 0 Skipped: 0 Warnings: 0
Adding secondary indexes to table `users`
Loading data from SERVER file: vardir/tmp/dump/db2/child.txt into child
db2.child: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0
Adding secondary indexes to table `child`
Loading data from SERVER file: vardir/tmp/dump/db2/offices.txt into offices
db2.offices: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from SERVER file: vardir/tmp/dump/db2/parent.txt into parent
db2.parent: Records: 2 Deleted: 0 Skipped: 0 Warnings: 0
# Repeat import with --verbose and --innodb-optimize-indexes=0, to "not" see "Adding secondary indexes"
Connecting to localhost
Executing SQL script vardir/tmp/dump/db2/users.sql
Executing SQL script vardir/tmp/dump/db2/child.sql
Executing SQL script vardir/tmp/dump/db2/offices.sql
Executing SQL script vardir/tmp/dump/db2/parent.sql
Loading data from SERVER file: vardir/tmp/dump/db2/users.txt into users
db2.users: Records: 3 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from SERVER file: vardir/tmp/dump/db2/child.txt into child
db2.child: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from SERVER file: vardir/tmp/dump/db2/offices.txt into offices
db2.offices: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0
Loading data from SERVER file: vardir/tmp/dump/db2/parent.txt into parent
db2.parent: Records: 2 Deleted: 0 Skipped: 0 Warnings: 0
drop database db2;
create database db2;
use db2;
create table vec (id int auto_increment primary key, v vector(5) not null,
vector index (v)) ENGINE=InnoDB;
insert vec(v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
(x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'),
(x'f09baa3ea172763f123def3e0c7fe53e288bf33e'),
(x'b97a523f2a193e3eb4f62e3f2d23583e9dd60d3f'),
(x'f7c5df3e984b2b3e65e59d3d7376db3eac63773e'),
(x'de01453ffa486d3f10aa4d3fdd66813c71cb163f'),
(x'76edfc3e4b57243f10f8423fb158713f020bda3e'),
(x'56926c3fdf098d3e2c8c5e3d1ad4953daa9d0b3e'),
(x'7b713f3e5258323f80d1113d673b2b3f66e3583f'),
(x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
create table ft(v text, fulltext(v)) ENGINE=InnoDB;
insert into ft(v) values ('Once upon a time'),
('There was a wicked witch'), ('Who ate everybody up');
create table locations (id int auto_increment primary key, geom geometry NOT NULL) ENGINE=InnoDB;
create spatial index idx_geom on locations (geom);
insert into locations (geom) values (ST_GeomFromText('POINT(40.785091 -73.968285)'));
# use --verbose to see "Adding secondary indexes" in the output
Connecting to localhost
Executing SQL script vardir/tmp/dump/db2/vec.sql
Executing SQL script vardir/tmp/dump/db2/ft.sql
Executing SQL script vardir/tmp/dump/db2/locations.sql
Loading data from SERVER file: vardir/tmp/dump/db2/vec.txt into vec
db2.vec: Records: 10 Deleted: 0 Skipped: 0 Warnings: 0
Adding secondary indexes to table `vec`
Loading data from SERVER file: vardir/tmp/dump/db2/ft.txt into ft
db2.ft: Records: 3 Deleted: 0 Skipped: 0 Warnings: 0
Adding secondary indexes to table `ft`
Loading data from SERVER file: vardir/tmp/dump/db2/locations.txt into locations
db2.locations: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0
Adding secondary indexes to table `locations`
show index from vec;
Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored
vec 0 PRIMARY 1 id A 10 NULL NULL BTREE NO
vec 1 v 1 v A NULL NULL NULL VECTOR NO
show index from locations;
Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored
locations 0 PRIMARY 1 id A 0 NULL NULL BTREE NO
locations 1 idx_geom 1 geom A NULL 32 NULL SPATIAL NO
show index from ft;
Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored
ft 1 v 1 v NULL NULL NULL NULL YES FULLTEXT NO
select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from vec order by d limit 3;
id d
9 0.47199
10 0.50690
3 0.58656
select * from ft where match(v) against('wicked');
v
There was a wicked witch
drop database db2;
create database db2;
use db2;
CREATE TABLE animals (id mediumint(9)
NOT NULL AUTO_INCREMENT,
name char(30) NOT NULL,
@ -216,12 +340,11 @@ use test;
drop database db2;
Connecting to localhost
Executing SQL script vardir/tmp/dump/db2/animals.sql
Executing SQL script vardir/tmp/dump/db2/animal_count.sql
Loading data from LOCAL file: vardir/tmp/dump/db2/animals.txt into animals
db2.animals: Records: 2 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/db2/animal_count.sql
Loading data from LOCAL file: vardir/tmp/dump/db2/animal_count.txt into animal_count
db2.animal_count: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0
Disconnecting from localhost
use db2;
# Content of tables after import
select * from animals;
@ -239,10 +362,9 @@ use test;
drop database db2;
Connecting to localhost
Executing SQL script vardir/tmp/dump/db2/t1.sql
Executing SQL script vardir/tmp/dump/db2/a1.sql
Loading data from LOCAL file: vardir/tmp/dump/db2/t1.txt into t1
db2.t1: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0
Executing SQL script vardir/tmp/dump/db2/a1.sql
Disconnecting from localhost
use db2;
select * from t1;
val

93
mysql-test/main/mariadb-import.test
View File

@ -28,7 +28,7 @@ select * from t1;
# Test --dir
--replace_result $MYSQLTEST_VARDIR vardir
# Ignore mtr.test_suppressions (may have suppressions or now), mysql.proc is smaller without perfschema/sys schema
# Ignore mtr.test_suppressions (may have suppressions or not), mysql.proc is smaller without perfschema/sys schema
--exec $MYSQL_IMPORT --local --verbose --dir $MYSQLTEST_VARDIR/tmp/dump --ignore-table=mtr.test_suppressions --ignore-table=mysql.proc
drop table t1;
@ -45,28 +45,108 @@ CREATE TABLE parent (
CREATE TABLE child (
id INT,
parent_id INT,
c CHAR(4),
INDEX par_ind (parent_id),
UNIQUE INDEX(c),
FOREIGN KEY (parent_id)
REFERENCES parent(id)
ON DELETE CASCADE
ON DELETE CASCADE,
CHECK (c >= 'a')
) ENGINE=INNODB;
insert into parent values(1),(2);
insert into child values (1,1),(1,2),(2,1),(2,2);
insert into child values (1,1,'a'),(1,2,'b'),(2,1,'c'),(2,2,'d');
# Example from https://github.com/mydumper/mydumper/issues/395 (can't repeat now)
CREATE TABLE offices (
id int NOT NULL AUTO_INCREMENT,
PRIMARY KEY (id)
) ENGINE=InnoDB;
CREATE TABLE users (
id int NOT NULL AUTO_INCREMENT,
office_id int DEFAULT NULL,
slogan text GENERATED ALWAYS AS (concat('Hello world #',office_id)) STORED,
PRIMARY KEY (id),
KEY office_id (office_id),
CONSTRAINT users_ibfk_1 FOREIGN KEY (office_id) REFERENCES offices (id)
) ENGINE=InnoDB;
insert into offices values();
insert into offices values();
insert into offices values();
insert into offices values();
insert into users (office_id) values (1);
insert into users (office_id) values (2);
insert into users (office_id) values (3);
--mkdir $MYSQLTEST_VARDIR/tmp/dump
--exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump --all-databases
drop database db2;
--replace_result $MYSQLTEST_VARDIR vardir
--exec $MYSQL_IMPORT --local --silent --dir $MYSQLTEST_VARDIR/tmp/dump --database=db2 --parallel=2
use db2;
select * from parent;
select * from child;
drop table child;
drop table parent;
show create table parent;
show create table child;
drop database db2;
--echo # Repeat import with --verbose to see "Adding secondary keys" in the output
--replace_result $MYSQLTEST_VARDIR vardir
--exec $MYSQL_IMPORT --verbose --dir $MYSQLTEST_VARDIR/tmp/dump --database=db2
--echo # Repeat import with --verbose and --innodb-optimize-indexes=0, to "not" see "Adding secondary indexes"
--replace_result $MYSQLTEST_VARDIR vardir
--exec $MYSQL_IMPORT --verbose --dir $MYSQLTEST_VARDIR/tmp/dump --database=db2 --innodb-optimize-keys=0
--rmdir $MYSQLTEST_VARDIR/tmp/dump
# Test with triggers (using https://mariadb.com/kb/en/trigger-overview/ example)
drop database db2;
create database db2;
use db2;
# Test with vector/fulltext/spatial indexes
create table vec (id int auto_increment primary key, v vector(5) not null,
vector index (v)) ENGINE=InnoDB;
insert vec(v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
(x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'),
(x'f09baa3ea172763f123def3e0c7fe53e288bf33e'),
(x'b97a523f2a193e3eb4f62e3f2d23583e9dd60d3f'),
(x'f7c5df3e984b2b3e65e59d3d7376db3eac63773e'),
(x'de01453ffa486d3f10aa4d3fdd66813c71cb163f'),
(x'76edfc3e4b57243f10f8423fb158713f020bda3e'),
(x'56926c3fdf098d3e2c8c5e3d1ad4953daa9d0b3e'),
(x'7b713f3e5258323f80d1113d673b2b3f66e3583f'),
(x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
create table ft(v text, fulltext(v)) ENGINE=InnoDB;
insert into ft(v) values ('Once upon a time'),
('There was a wicked witch'), ('Who ate everybody up');
create table locations (id int auto_increment primary key, geom geometry NOT NULL) ENGINE=InnoDB;
create spatial index idx_geom on locations (geom);
insert into locations (geom) values (ST_GeomFromText('POINT(40.785091 -73.968285)'));
--mkdir $MYSQLTEST_VARDIR/tmp/dump
--exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump db2
--echo # use --verbose to see "Adding secondary indexes" in the output
--replace_result $MYSQLTEST_VARDIR vardir
--exec $MYSQL_IMPORT --verbose --dir $MYSQLTEST_VARDIR/tmp/dump --database=db2
# smoke-test restored tables
show index from vec;
show index from locations;
show index from ft;
--replace_regex /(\.\d{5})\d+/\1/
select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from vec order by d limit 3;
select * from ft where match(v) against('wicked');
--rmdir $MYSQLTEST_VARDIR/tmp/dump
drop database db2;
create database db2;
use db2;
# Test with triggers (using https://mariadb.com/kb/en/trigger-overview/ example)
CREATE TABLE animals (id mediumint(9)
NOT NULL AUTO_INCREMENT,
name char(30) NOT NULL,
@ -145,4 +225,3 @@ use test;
--exec $MYSQL_IMPORT --dir $MYSQLTEST_VARDIR/tmp/dump --parallel=300 2>&1
--rmdir $MYSQLTEST_VARDIR/tmp/dump

3
mysql-test/main/mysqldump.result
View File

@ -4617,7 +4617,8 @@ Abernathy
aberrant
aberration
drop table words;
mariadb-import: Error: 1146, Table 'test.words' doesn't exist, when using table: words
Error: 1146, Table 'test.words' doesn't exist, when using statement: ALTER TABLE `test`.`words` DISABLE KEYS
mariadb-import: Error 1146 Table 'test.words' doesn't exist
drop table t1;
drop table t2;
drop table words2;

3
unittest/client/CMakeLists.txt Normal file
View File

@ -0,0 +1,3 @@
ADD_EXECUTABLE(import_util-t import_util-t.cc)
TARGET_LINK_LIBRARIES(import_util-t PRIVATE import_util mytap)
ADD_TEST(import_util import_util-t)

149
unittest/client/import_util-t.cc Normal file
View File

@ -0,0 +1,149 @@
/* Copyright (c) 2024, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA
*/
#include "my_config.h"
#include "import_util.h"
#include <tap.h>
#include <string>
inline bool operator==(const KeyDefinition &lhs, const KeyDefinition &rhs)
{
return lhs.definition == rhs.definition && lhs.name == rhs.name;
}
/*
Test parsing of CREATE TABLE in mariadb-import utility
*/
static void test_ddl_parser()
{
std::string script= R"(
-- Some SQL script
CREATE TABLE `book` (
`id` mediumint(8) unsigned NOT NULL AUTO_INCREMENT,
`title` varchar(200) NOT NULL,
`author_id` smallint(5) unsigned NOT NULL,
`publisher_id` smallint(5) unsigned NOT NULL,
`excerpt` text,
PRIMARY KEY (`id`),
KEY `fk_book_author` (`author_id`),
KEY `fk_book_publisher` (`publisher_id`),
UNIQUE KEY `title_author` (`title`,`author`),
FULLTEXT KEY `excerpt` (`excerpt`),
CONSTRAINT `fk_book_author` FOREIGN KEY (`author_id`) REFERENCES `author` (`id`) ON DELETE CASCADE
CONSTRAINT `fk_book_publisher` FOREIGN KEY (`publisher_id`) REFERENCES `publisher` (`id`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci;
)";
auto create_table_stmt= extract_first_create_table(script);
ok(!create_table_stmt.empty(), "CREATE TABLE statement found");
TableDDLInfo ddl_info(create_table_stmt);
const std::string& table_name= ddl_info.table_name;
const std::string& storage_engine= ddl_info.storage_engine;
ok(table_name == "`book`", "Table name is OK");
ok(storage_engine == "InnoDB", "Storage engine is OK");
ok(ddl_info.primary_key == KeyDefinition{"PRIMARY KEY (`id`)", "PRIMARY"},
"Primary key def is OK");
ok(ddl_info.secondary_indexes.size() == 4, "Secondary index size is OK");
const auto &sec_indexes= ddl_info.secondary_indexes;
ok(sec_indexes[0] == KeyDefinition{"KEY `fk_book_author` (`author_id`)","`fk_book_author`"},
"First key is OK");
ok(sec_indexes[1] ==
KeyDefinition{"KEY `fk_book_publisher` (`publisher_id`)",
"`fk_book_publisher`"},
"Second key is OK");
ok(ddl_info.constraints.size() == 2, "Constraints size correct");
ok(ddl_info.constraints[0] ==
KeyDefinition{"CONSTRAINT `fk_book_author` FOREIGN KEY (`author_id`) REFERENCES "
"`author` (`id`) ON DELETE CASCADE","`fk_book_author`"},
"First constraint OK");
std::string drop_constraints= ddl_info.drop_constraints_sql();
ok(drop_constraints ==
"ALTER TABLE `book` DROP CONSTRAINT `fk_book_author`, DROP CONSTRAINT `fk_book_publisher`",
"Drop constraints SQL is \"%s\"", drop_constraints.c_str());
std::string add_constraints= ddl_info.add_constraints_sql();
ok(add_constraints ==
"ALTER TABLE `book` ADD CONSTRAINT `fk_book_author` FOREIGN KEY (`author_id`) "
"REFERENCES `author` (`id`) ON DELETE CASCADE, "
"ADD CONSTRAINT `fk_book_publisher` FOREIGN KEY (`publisher_id`) "
"REFERENCES `publisher` (`id`) ON DELETE CASCADE",
"Add constraints SQL is \"%s\"",add_constraints.c_str());
std::string drop_secondary_indexes=
ddl_info.drop_secondary_indexes_sql();
ok(drop_secondary_indexes ==
"ALTER TABLE `book` "
"DROP INDEX `fk_book_author`, "
"DROP INDEX `fk_book_publisher`, "
"DROP INDEX `title_author`, "
"DROP INDEX `excerpt`",
"Drop secondary indexes SQL is \"%s\"", drop_secondary_indexes.c_str());
std::string add_secondary_indexes=
ddl_info.add_secondary_indexes_sql();
ok(add_secondary_indexes ==
"ALTER TABLE `book` ADD KEY `fk_book_author` (`author_id`), "
"ADD KEY `fk_book_publisher` (`publisher_id`), "
"ADD UNIQUE KEY `title_author` (`title`,`author`), "
"ADD FULLTEXT KEY `excerpt` (`excerpt`)",
"Add secondary indexes SQL is \"%s\"", add_secondary_indexes.c_str());
}
/*
For Innodb table without PK, and but with Unique key
(which is used for clustering, instead of PK)
this key will not be added and dropped by
the import utility
*/
static void innodb_non_pk_clustering_key()
{
auto create_table_stmt= R"(
CREATE TABLE `book` (
`id` mediumint(8),
`uniq` varchar(200),
UNIQUE KEY `id` (`id`),
UNIQUE KEY `uniq` (`uniq`),
KEY `id_uniq` (`id`,`uniq`)
) ENGINE=InnoDB;
)";
TableDDLInfo ddl_info(create_table_stmt);
ok(ddl_info.non_pk_clustering_key_name == "`id`",
"Non-PK clustering key is %s",
ddl_info.non_pk_clustering_key_name.c_str());
ok(ddl_info.primary_key.definition.empty(),
"Primary key is %s", ddl_info.primary_key.definition.c_str());
ok(ddl_info.secondary_indexes.size() == 3,
"Secondary indexes size is %zu",
ddl_info.secondary_indexes.size());
ok(!ddl_info.add_secondary_indexes_sql().empty(),
"Some secondary indexes to add");
ok(!ddl_info.drop_secondary_indexes_sql().empty(),
"Some secondary indexes to drop");
}
int main()
{
plan(18);
diag("Testing DDL parser");
test_ddl_parser();
innodb_non_pk_clustering_key();
return exit_status();
}

1
unittest/mytap/CMakeLists.txt
View File

@ -15,3 +15,4 @@
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include)
ADD_LIBRARY(mytap tap.c)
TARGET_INCLUDE_DIRECTORIES(mytap PUBLIC ${CMAKE_SOURCE_DIR}/unittest/mytap)