From 9e25d6f0cc3ce965ebb7b861813b4a2f7aedf04c Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 3 Jun 2024 12:23:53 +0200 Subject: [PATCH] MDEV-33627 : Implement option --dir in mariadb-import With that, it is possible to restore the full "instance" from a backup made with mariadb-dump --dir The patch implements executing DDL (tables, views, triggers) using statements that are stored in .sql file, created by mariadb-dump --dir . Care is taken of creating triggers correctly after the data is loaded, disabling foreign keys and unique key checks etc. The files are loaded in descending order by datafile size - to ensure better work distribution when running with --parallel option. In addition to --dir option, following options are implemented for partial restore include-only options: --database - import one or several databases --table - import one or several tables exclude options: --ignore-database -. ignore one or several databases when importing --ignore-table - to ignore one or several tables when importing All options above are only valid together with --dir option, and can be specified multiple times. --- client/client_priv.h | 1 + client/mysqlimport.cc | 616 +++++++++++++++++++++++--- mysql-test/main/mariadb-import.result | 266 +++++++++++ mysql-test/main/mariadb-import.test | 148 +++++++ 4 files changed, 979 insertions(+), 52 deletions(-) create mode 100644 mysql-test/main/mariadb-import.result create mode 100644 mysql-test/main/mariadb-import.test diff --git a/client/client_priv.h b/client/client_priv.h index f559b204b12..542060bf7ec 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -73,6 +73,7 @@ enum options_client OPT_DO_SERVER_IDS, OPT_SSL_FP, OPT_SSL_FPLIST, OPT_UPDATE_HISTORY, + OPT_DATABASE, OPT_MAX_CLIENT_OPTION /* should be always the last */ }; diff --git a/client/mysqlimport.cc b/client/mysqlimport.cc index df52808256f..e68ac69d862 100644 --- a/client/mysqlimport.cc +++ b/client/mysqlimport.cc @@ -36,7 +36,14 @@ #include /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ +#include +#include +#include #include +#include +#include +#include + tpool::thread_pool *thread_pool; static void db_error_with_table(MYSQL *mysql, char *table); @@ -59,19 +66,44 @@ static uint opt_mysql_port= 0, opt_protocol= 0; static char * opt_mysql_unix_port=0; static char *opt_plugin_dir= 0, *opt_default_auth= 0; static longlong opt_ignore_lines= -1; +static char *opt_dir; #include static char **argv_to_free; +static void safe_exit(int error, MYSQL *mysql); +static void set_exitcode(int code); + +struct table_load_params +{ + std::string data_file; /* name of the file to load with LOAD DATA INFILE */ + std::string sql_file; /* name of the file that contains CREATE TABLE or + CREATE VIEW */ + std::string tablename; /* name of the table */ + std::string dbname; /* name of the database */ + ulonglong size; /* size of the data file */ +}; + +std::unordered_set ignore_databases; +std::unordered_set ignore_tables; +std::unordered_set include_databases; +std::unordered_set include_tables; static struct my_option my_long_options[] = { {"character-sets-dir", 0, "Directory for character set files.", (char**) &charsets_dir, (char**) &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"database", OPT_DATABASE, + "Restore the specified database, ignoring others.To specify more than one " + "database to include, use the directive multiple times, once for each database. " + "Only takes effect when used together with --dir option", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"default-character-set", 0, "Set the default character set.", &default_charset, &default_charset, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"dir", 0, "Restore all tables from backup directory created using mariadb-dump --dir", + &opt_dir, &opt_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"columns", 'c', "Use only these columns to import the data to. Give the column names in a comma separated list. This is same as giving columns to LOAD DATA INFILE.", &opt_columns, &opt_columns, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, @@ -125,6 +157,18 @@ static struct my_option my_long_options[] = {"ignore-lines", 0, "Ignore first n lines of data infile.", &opt_ignore_lines, &opt_ignore_lines, 0, GET_LL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"ignore-database", OPT_IGNORE_DATABASE, + "Do not restore the specified database. To specify more than one database " + "to ignore, use the directive multiple times, once for each database. Only " + "takes effect when used together with --dir option", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"ignore-table", OPT_IGNORE_TABLE, + "Do not restore the specified table. To specify more than one table to " + "ignore, use the directive multiple times, once for each table. Each " + "table must be specified with both database and table names, e.g., " + "--ignore-table=database.table. Only takes effect when used together with " + "--dir option", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"lines-terminated-by", 0, "Lines in the input file are terminated by the given string.", &lines_terminated, &lines_terminated, 0, GET_STR, @@ -168,6 +212,12 @@ static struct my_option my_long_options[] = {"socket", 'S', "The socket file to use for connection.", &opt_mysql_unix_port, &opt_mysql_unix_port, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"table", OPT_TABLES, + "Restore the specified table ignoring others. Use --table=dbname.tablename with this option. " + "To specify more than one table to include, use the directive multiple times, once for each " + "table. Only takes effect when used together with --dir option", + 0, 0, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, #include {"use-threads", 0, "Synonym for --parallel option", &opt_use_threads, &opt_use_threads, 0, @@ -272,6 +322,30 @@ get_one_option(const struct my_option *opt, const char *argument, } } break; + case (int) OPT_IGNORE_TABLE: + if (!strchr(argument, '.')) + { + fprintf(stderr, + "Illegal use of option --ignore-table=.\n"); + exit(1); + } + ignore_tables.insert(argument); + break; + case (int) OPT_TABLES: + if (!strchr(argument, '.')) + { + fprintf(stderr, + "Illegal use of option --table=.
\n"); + exit(1); + } + include_tables.insert(argument); + break; + case (int) OPT_IGNORE_DATABASE: + ignore_databases.insert(argument); + break; + case (int) OPT_DATABASE: + include_databases.insert(argument); + break; case '#': DBUG_PUSH(argument ? argument : "d:t:o"); debug_check_flag= 1; @@ -308,29 +382,257 @@ static int get_options(int *argc, char ***argv) fprintf(stderr, "You can't use --ignore (-i) and --replace (-r) at the same time.\n"); return(1); } - if (*argc < 2) + if (*argc < 2 && !opt_dir) { usage(); return 1; } - current_db= *((*argv)++); - (*argc)--; + if (!opt_dir) + { + current_db= *((*argv)++); + (*argc)--; + } if (tty_password) opt_password=my_get_tty_password(NullS); return(0); } +/** + Check if file has given extension + @param filename - name of the file + @param ext - extension to check for, including the dot + we assume that ext is always 4 characters long +*/ +static bool has_extension(const char *filename, const char *ext) +{ + constexpr size_t ext_len= 4; + DBUG_ASSERT(strlen(ext) == ext_len); + size_t len= strlen(filename); + return len >= ext_len && !strcmp(filename + len - ext_len, ext); +} -static int write_to_table(char *filename, MYSQL *mysql) +/** + Quote an identifier, e.g table name, or dbname + + Adds ` around the string, and replaces with ` with `` inside the string +*/ +static std::string quote_identifier(const char *name) +{ + std::string res; + res.reserve(strlen(name)+2); + res+= '`'; + for (const char *p= name; *p; p++) + { + if (*p == '`') + res += '`'; + res += *p; + } + res += '`'; + return res; +} + +/** + Execute a batch of SQL statements + + @param mysql - connection to the server + @param sql - SQL statements to execute, comma separated. + @param filename - name of the file that contains the SQL statements + + @return 0 if successful, 1 if there was an error +*/ +static int execute_sql_batch(MYSQL *mysql, const char *sql, + const char *filename) +{ + /* Execute batch */ + if (mysql_query(mysql, sql)) + { + my_printf_error(0, "Error: %d, %s, when using script: %s", MYF(0), + mysql_errno(mysql), mysql_error(mysql), filename); + safe_exit(1, mysql); + return 1; + } + + /* After we executed multi-statement batch, we need to read/check all + * results. */ + for (int stmt_count= 1;; stmt_count++) + { + int res= mysql_next_result(mysql); + switch (res) + { + case -1: + return 0; + case 0: + break; + default: + my_printf_error( + 0, "Error: %d, %s, when using script: %s, statement count = %d", + MYF(0), mysql_errno(mysql), mysql_error(mysql), filename, + stmt_count + 1); + safe_exit(1, mysql); + return 1; + } + } + return 0; +} + +static int exec_sql(MYSQL *mysql, const std::string& s) +{ + if (mysql_query(mysql, s.c_str())) + { + fprintf(stdout,"Error: %d, %s, when using statement: %s\n", + mysql_errno(mysql), mysql_error(mysql), s.c_str()); + db_error(mysql); + return 1; + } + return 0; +} + +/** + Prefix and suffix for CREATE TRIGGER statement in .sql file +*/ +#define CREATE_TRIGGER_PREFIX "\nDELIMITER ;;\n" +#define CREATE_TRIGGER_SUFFIX ";;\nDELIMITER ;\n" + +/** + Parse the SQL script file, and return the content of the file as a string + + @param filepath - path to .sql file + @param tz_utc - true if the script sets the timezone to UTC + @param create_trigger_Defs - will be filled with CREATE TRIGGER statements + + @return content of the file as a string, excluding CREATE TRIGGER statements +*/ +static std::string parse_sql_script(const char *filepath, bool *tz_utc, std::vector *create_trigger_Defs, + std::string *engine) +{ + /*Read full file to string*/ + std::ifstream t(filepath); + std::stringstream sql; + sql << t.rdbuf(); + + std::string sql_text= sql.str(); + + /* + This is how triggers are defined in .sql file by mysqldump + + DELIMITER ;; + CREATE TRIGGER ;; + DELIMITER ; + Now, DELIMITER is not a statement, but a command for the mysql client. + Thus we can't sent it as part of the batch, so we transform the above + by removing DELIMITER lines, and extra semicolon at the end of the + CREATE TRIGGER statement. + */ + for (;;) + { + auto pos= sql_text.find(CREATE_TRIGGER_PREFIX); + if (pos == std::string::npos) + break; + auto end_pos= sql_text.find(CREATE_TRIGGER_SUFFIX, pos); + if (end_pos == std::string::npos) + break; + create_trigger_Defs->push_back(sql_text.substr(pos + sizeof(CREATE_TRIGGER_PREFIX)-1, + end_pos - pos - sizeof(CREATE_TRIGGER_PREFIX) +1)); + sql_text.erase(pos, end_pos - pos + sizeof(CREATE_TRIGGER_SUFFIX) - 1); + } + + /* + Find out if dump was made using UTC timezone, we'd need the same for the + loading. output in UTC timezone is default in mysqldump, but can be controlled + with --tz-utc option + */ + *tz_utc= sql_text.find("SET TIME_ZONE='+00:00'") != std::string::npos; + + *engine= ""; + auto engine_pos= sql_text.find("ENGINE="); + if (engine_pos != std::string::npos) + { + engine_pos+= 7; + auto end_pos= sql_text.find_first_of(" ", engine_pos); + if (end_pos != std::string::npos) + *engine= sql_text.substr(engine_pos, end_pos - engine_pos); + } + return sql_text; +} + +/* + Creates database if it does not yet exists. + + @param mysql - connection to the server + @param dbname - name of the database +*/ +static int create_db_if_not_exists(MYSQL *mysql, const char *dbname) +{ + /* Create database if it does not yet exist */ + std::string create_db_if_not_exists= "CREATE DATABASE IF NOT EXISTS "; + create_db_if_not_exists+= quote_identifier(dbname); + if (mysql_query(mysql, create_db_if_not_exists.c_str())) + { + db_error(mysql); + return 1; + } + return 0; +} + +static int handle_one_table(const table_load_params *params, MYSQL *mysql) { char tablename[FN_REFLEN], hard_path[FN_REFLEN], escaped_name[FN_REFLEN * 2 + 1], - sql_statement[FN_REFLEN*16+256], *end, *pos; - DBUG_ENTER("write_to_table"); - DBUG_PRINT("enter",("filename: %s",filename)); + sql_statement[FN_REFLEN*16+256], *end; + DBUG_ENTER("handle_one_table"); + DBUG_PRINT("enter",("datafile: %s",params->data_file.c_str())); + + if (verbose && !params->sql_file.empty()) + { + fprintf(stdout, "Executing SQL script %s\n", params->sql_file.c_str()); + } + + if (!params->dbname.empty()) + { + if (mysql_select_db(mysql, params->dbname.c_str())) + { + if (create_db_if_not_exists(mysql, params->dbname.c_str())) + DBUG_RETURN(1); + if (mysql_select_db(mysql, params->dbname.c_str())) + db_error(mysql); + } + } + + const char *filename= params->data_file.c_str(); + if (!filename[0]) + filename= params->sql_file.c_str(); fn_format(tablename, filename, "", "", 1 | 2); /* removes path & ext. */ + + const char *db= current_db ? current_db : params->dbname.c_str(); + std::string full_tablename= quote_identifier(db); + full_tablename+= "."; + full_tablename+= quote_identifier(tablename); + + bool tz_utc= false; + std::string engine; + std::vector triggers; + if (!params->sql_file.empty()) + { + std::string sql_text= parse_sql_script(params->sql_file.c_str(), &tz_utc, &triggers,&engine); + if (execute_sql_batch(mysql, sql_text.c_str(),params->sql_file.c_str())) + DBUG_RETURN(1); + if (params->data_file.empty()) + { + /* + We only use .sql extension for VIEWs, so we're done + with this file, there is no data to load. + */ + DBUG_RETURN(0); + } + if (tz_utc && exec_sql(mysql, "SET TIME_ZONE='+00:00';")) + DBUG_RETURN(1); + if (exec_sql(mysql, std::string("LOCK TABLE ") + full_tablename + "WRITE")) + DBUG_RETURN(1); + if (exec_sql(mysql, std::string("ALTER TABLE ") + full_tablename + " DISABLE KEYS")) + DBUG_RETURN(1); + } if (!opt_local_file) strmov(hard_path,filename); else @@ -340,42 +642,31 @@ static int write_to_table(char *filename, MYSQL *mysql) { if (verbose) fprintf(stdout, "Deleting the old data from table %s\n", tablename); - snprintf(sql_statement, FN_REFLEN*16+256, "DELETE FROM %s", tablename); - if (mysql_query(mysql, sql_statement)) - { - db_error_with_table(mysql, tablename); + snprintf(sql_statement, FN_REFLEN * 16 + 256, "DELETE FROM %s", + full_tablename.c_str()); + if (exec_sql(mysql, sql_statement)) DBUG_RETURN(1); - } } to_unix_path(hard_path); if (verbose) { - if (opt_local_file) - fprintf(stdout, "Loading data from LOCAL file: %s into %s\n", - hard_path, tablename); - else - fprintf(stdout, "Loading data from SERVER file: %s into %s\n", - hard_path, tablename); + fprintf(stdout, "Loading data from %s file: %s into %s\n", + (opt_local_file) ? "LOCAL" : "SERVER", hard_path, tablename); } mysql_real_escape_string(mysql, escaped_name, hard_path, (unsigned long) strlen(hard_path)); sprintf(sql_statement, "LOAD DATA %s %s INFILE '%s'", - opt_low_priority ? "LOW_PRIORITY" : "", - opt_local_file ? "LOCAL" : "", escaped_name); + opt_low_priority ? "LOW_PRIORITY" : "", + opt_local_file ? "LOCAL" : "", escaped_name); + end= strend(sql_statement); if (replace) end= strmov(end, " REPLACE"); if (ignore) end= strmov(end, " IGNORE"); - end= strmov(end, " INTO TABLE `"); - /* Turn any ` into `` in table name. */ - for (pos= tablename; *pos; pos++) - { - if (*pos == '`') - *end++= '`'; - *end++= *pos; - } - end= strmov(end, "`"); + end= strmov(end, " INTO TABLE "); + + end= strmov(end,full_tablename.c_str()); if (fields_terminated || enclosed || opt_enclosed || escaped) end= strmov(end, " FIELDS"); @@ -399,12 +690,41 @@ static int write_to_table(char *filename, MYSQL *mysql) } if (!silent) { - if (mysql_info(mysql)) /* If NULL-pointer, print nothing */ + const char *info= mysql_info(mysql); + if (info) /* If NULL-pointer, print nothing */ + fprintf(stdout, "%s.%s: %s\n", db, tablename, info); + } + + /* Create triggers after loading data */ + for (const auto &trigger: triggers) + { + if (mysql_query(mysql,trigger.c_str())) { - fprintf(stdout, "%s.%s: %s\n", current_db, tablename, - mysql_info(mysql)); + db_error_with_table(mysql, tablename); + DBUG_RETURN(1); } } + + if (!params->sql_file.empty()) + { + if (exec_sql(mysql, std::string("ALTER TABLE ") + full_tablename + " ENABLE KEYS;")) + DBUG_RETURN(1); + + if (engine == "MyISAM" || engine == "Aria") + { + /* Avoid "table was not properly closed" warnings */ + if (exec_sql(mysql, std::string("FLUSH TABLE ").append(full_tablename).c_str())) + DBUG_RETURN(1); + } + if (exec_sql(mysql, "UNLOCK TABLES")) + DBUG_RETURN(1); + } + + if (tz_utc) + { + if (exec_sql(mysql, "SET TIME_ZONE=@save_tz;")) + DBUG_RETURN(1); + } DBUG_RETURN(0); } @@ -472,25 +792,34 @@ static MYSQL *db_connect(char *host, char *database, "program_name", "mysqlimport"); if (!(mysql_real_connect(mysql,host,user,passwd, database,opt_mysql_port,opt_mysql_unix_port, - 0))) + opt_dir?CLIENT_MULTI_STATEMENTS:0))) { ignore_errors=0; /* NO RETURN FROM db_error */ db_error(mysql); } reconnect= 0; mysql_options(mysql, MYSQL_OPT_RECONNECT, &reconnect); - if (verbose) - fprintf(stdout, "Selecting database %s\n", database); - if (mysql_select_db(mysql, database)) + if (database) { - ignore_errors=0; - db_error(mysql); + if (verbose) + fprintf(stdout, "Selecting database %s\n", database); + if (mysql_select_db(mysql, database)) + { + ignore_errors= 0; + db_error(mysql); + } } if (ignore_foreign_keys) mysql_query(mysql, "set foreign_key_checks= 0;"); if (mysql_query(mysql, "/*!40101 set @@character_set_database=binary */;")) db_error(mysql); + if (mysql_query(mysql, "set @save_tz=@@session.time_zone")) + db_error(mysql); + if (mysql_query(mysql, "set unique_checks= 0")) + db_error(mysql); + if (mysql_query(mysql, "/*M!100200 set check_constraint_checks=0*/")) + db_error(mysql); return mysql; } @@ -536,11 +865,25 @@ static void db_error_with_table(MYSQL *mysql, char *table) safe_exit(1, mysql); } +static void fatal_error(const char *format, ...) +{ + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + fflush(stderr); + ignore_errors= 0; + safe_exit(1, 0); +} static void db_error(MYSQL *mysql) { - my_printf_error(0,"Error: %d %s", MYF(0), mysql_errno(mysql), mysql_error(mysql)); + const char *info= mysql_info(mysql); + auto err= mysql_errno(mysql); + auto err_text = mysql_error(mysql); + + my_printf_error(0,"Error: %d %s %s", MYF(0), err, err_text, info); safe_exit(1, mysql); } @@ -601,18 +944,13 @@ void set_exitcode(int code) exitcode.compare_exchange_strong(expected,code); } -thread_local MYSQL *thread_local_mysql; +static thread_local MYSQL *thread_local_mysql; void load_single_table(void *arg) { int error; - char *raw_table_name= (char *)arg; - MYSQL *mysql= thread_local_mysql; - /* - We are not currently catching the error here. - */ - if((error= write_to_table(raw_table_name, mysql))) + if((error= handle_one_table((const table_load_params *) arg, thread_local_mysql))) set_exitcode(error); } @@ -621,6 +959,7 @@ static void tpool_thread_init(void) mysql_thread_init(); thread_local_mysql= db_connect(current_host,current_db,current_user,opt_password); } + static void tpool_thread_exit(void) { if (thread_local_mysql) @@ -628,8 +967,144 @@ static void tpool_thread_exit(void) mysql_thread_end(); } +/** + Get files to load, for --dir case + Enumerates all files in the subdirectories, and returns only *.txt files + (table data files), or .sql files, there is no corresponding .txt file + (view definitions) + + @param dir - directory to scan + @param files - vector to store the files + + @note files are sorted by size, descending +*/ +static void scan_backup_dir(const char *dir, + std::vector &files, + std::vector &views) +{ + MY_DIR *dir_info; + std::vector subdirs; + int stat_err; + struct stat st; + if ((stat_err= stat(dir, &st)) != 0 || (st.st_mode & S_IFDIR) == 0) + { + fatal_error("%s: Path '%s' specified by option '--dir' %s\n", + my_progname_short, dir, + stat_err ? "does not exist" : "is not a directory"); + } + dir_info= my_dir(dir, MYF(MY_DONT_SORT | MY_WANT_STAT | MY_WME)); + if (!dir_info) + { + fatal_error("Can't read directory '%s', error %d", opt_dir, errno); + return; + } + for (size_t i= 0; i < dir_info->number_of_files; i++) + { + const fileinfo *fi= &dir_info->dir_entry[i]; + if (!(fi->mystat->st_mode & S_IFDIR)) + continue; + if (!strcmp(fi->name, ".") || !strcmp(fi->name, "..")) + continue; + if (ignore_databases.find(fi->name) != ignore_databases.end()) + continue; + if (include_databases.size() && + include_databases.find(fi->name) == include_databases.end()) + continue; + std::string subdir= dir; + subdir+= "/"; + subdir+= fi->name; + const char *dbname = fi->name; + // subdirs.push_back(subdir); + MY_DIR *dir_info2= + my_dir(subdir.c_str(), MYF(MY_DONT_SORT | MY_WANT_STAT | MY_WME)); + if (!dir_info2) + { + fatal_error("Can't read directory %s , error %d", subdir.c_str(), errno); + return; + } + for (size_t j= 0; j < dir_info2->number_of_files; j++) + { + table_load_params par{}; + par.dbname= dbname; + fi= &dir_info2->dir_entry[j]; + if (has_extension(fi->name, ".sql") || has_extension(fi->name, ".txt")) + { + std::string full_table_name= + std::string(dbname) + "." + std::string(fi->name); + full_table_name.resize(full_table_name.size() - 4); + if (ignore_tables.find(full_table_name) != ignore_tables.end()) + { + continue; + } + if (include_tables.size() && + include_tables.find(full_table_name) == include_tables.end()) + { + continue; + } + } + + std::string file= subdir; + file+= "/"; + file+= fi->name; + DBUG_ASSERT(access(file.c_str(), F_OK) == 0); + if (!MY_S_ISDIR(fi->mystat->st_mode)) + { + /* test file*/ + if (has_extension(fi->name, ".txt")) + { + par.data_file= file; + par.size= fi->mystat->st_size; + par.sql_file= file.substr(0, file.size() - 4) + ".sql"; + if (access(par.sql_file.c_str(), F_OK)) + { + fatal_error("Expected file '%s' is missing",par.sql_file.c_str()); + } + files.push_back(par); + } + else if (has_extension(fi->name, ".sql")) + { + /* + Check whether it is a view definition, without a + corresponding .txt file, add .sql file then + */ + std::string txt_file= file.substr(0, file.size() - 4) + ".txt"; + if (access(txt_file.c_str(), F_OK)) + { + par.sql_file= file; + par.size= fi->mystat->st_size; + views.push_back(par); + } + } + else + { + fatal_error("Unexpected file '%s' in directory '%s'", fi->name,subdir.c_str()); + } + } + } + my_dirend(dir_info2); + } + my_dirend(dir_info); + + /* sort files by size, descending. Put view definitions at the end of the list.*/ + std::sort(files.begin(), files.end(), + [](const table_load_params &a, const table_load_params &b) -> bool + { + if (a.size > b.size) + return true; + if (a.size < b.size) + return false; + return a.sql_file < b.sql_file; + }); + + std::sort(views.begin(), views.end(), + [](const table_load_params &a, const table_load_params &b) -> bool + { + return a.sql_file < b.sql_file; + }); +} + +#define MAX_THREADS 256 -#include int main(int argc, char **argv) { int error=0; @@ -650,22 +1125,54 @@ int main(int argc, char **argv) sf_leaking_memory=0; /* from now on we cleanup properly */ + std::vector files_to_load, views_to_load; + + if (opt_dir) + { + ignore_foreign_keys= 1; + if (argc) + fatal_error("Invalid arguments for --dir option"); + scan_backup_dir(opt_dir, files_to_load, views_to_load); + } + else + { + for (; *argv != NULL; argv++) + { + table_load_params p{}; + p.data_file= *argv; + files_to_load.push_back(p); + } + } + if (opt_use_threads && !lock_tables) { + if (opt_use_threads > MAX_THREADS) + { + fatal_error("Too many connections, max value for --parallel is %d\n", + MAX_THREADS); + } thread_pool= tpool::create_thread_pool_generic(opt_use_threads,opt_use_threads); thread_pool->set_thread_callbacks(tpool_thread_init,tpool_thread_exit); std::vector all_tasks; - for (int i=0; argv[i]; i++) - all_tasks.push_back(tpool::task(load_single_table, argv[i])); + for (const auto &f: files_to_load) + all_tasks.push_back(tpool::task(load_single_table, (void *)&f)); for (auto &t: all_tasks) thread_pool->submit_task(&t); delete thread_pool; thread_pool= nullptr; + files_to_load.clear(); } - else + /* + The following block handles single-threaded load. + Also views that must be created after the base tables, are created here. + + BUG: funny case would be views that select from other views, won't generally work. + It won't work in mysqldump either, but it's not a common case. + */ + if (!files_to_load.empty() || !views_to_load.empty()) { MYSQL *mysql= db_connect(current_host,current_db,current_user,opt_password); if (!mysql) @@ -676,9 +1183,14 @@ int main(int argc, char **argv) if (lock_tables) lock_table(mysql, argc, argv); - for (; *argv != NULL; argv++) - if ((error= write_to_table(*argv, mysql))) + for (const auto &f : files_to_load) + if ((error= handle_one_table(&f, mysql))) set_exitcode(error); + + for (const auto &v : views_to_load) + if ((error= handle_one_table(&v, mysql))) + set_exitcode(error); + db_disconnect(current_host, mysql); } safe_exit(0, 0); diff --git a/mysql-test/main/mariadb-import.result b/mysql-test/main/mariadb-import.result new file mode 100644 index 00000000000..aa683070eac --- /dev/null +++ b/mysql-test/main/mariadb-import.result @@ -0,0 +1,266 @@ +create table t1(i int); +insert t1 values(100); +create view v1 as select 1; +drop table t1; +test.t1: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 +select * from t1; +i +100 +# Content of dump directory +mtr +mysql +test +# Content of 'test' dump subdirectory +t1.sql +t1.txt +v1.sql +# Content of 'mysql' dump subdirectory +column_stats.sql +column_stats.txt +columns_priv.sql +columns_priv.txt +db.sql +db.txt +event.sql +func.sql +func.txt +general_log.sql +global_priv.sql +global_priv.txt +gtid_slave_pos.sql +gtid_slave_pos.txt +help_category.sql +help_category.txt +help_keyword.sql +help_keyword.txt +help_relation.sql +help_relation.txt +help_topic.sql +help_topic.txt +index_stats.sql +index_stats.txt +innodb_index_stats.sql +innodb_table_stats.sql +plugin.sql +plugin.txt +proc.sql +proc.txt +procs_priv.sql +procs_priv.txt +proxies_priv.sql +proxies_priv.txt +roles_mapping.sql +roles_mapping.txt +servers.sql +servers.txt +slow_log.sql +table_stats.sql +table_stats.txt +tables_priv.sql +tables_priv.txt +time_zone.sql +time_zone.txt +time_zone_leap_second.sql +time_zone_leap_second.txt +time_zone_name.sql +time_zone_name.txt +time_zone_transition.sql +time_zone_transition.txt +time_zone_transition_type.sql +time_zone_transition_type.txt +transaction_registry.sql +user.sql +# Content of 'mtr' dump subdirectory +global_suppressions.sql +global_suppressions.txt +test_suppressions.sql +test_suppressions.txt +Connecting to localhost +Executing SQL script vardir/tmp/dump/mysql/help_topic.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/help_topic.txt into help_topic +mysql.help_topic: Records: 839 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/time_zone_transition.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_transition.txt into time_zone_transition +mysql.time_zone_transition: Records: 394 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mtr/global_suppressions.sql +Loading data from LOCAL file: vardir/tmp/dump/mtr/global_suppressions.txt into global_suppressions +mtr.global_suppressions: Records: 99 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/help_keyword.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/help_keyword.txt into help_keyword +mysql.help_keyword: Records: 106 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/help_relation.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/help_relation.txt into help_relation +mysql.help_relation: Records: 202 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/help_category.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/help_category.txt into help_category +mysql.help_category: Records: 50 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/time_zone_transition_type.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_transition_type.txt into time_zone_transition_type +mysql.time_zone_transition_type: Records: 32 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/global_priv.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/global_priv.txt into global_priv +mysql.global_priv: Records: 5 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/time_zone_leap_second.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_leap_second.txt into time_zone_leap_second +mysql.time_zone_leap_second: Records: 23 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/proxies_priv.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/proxies_priv.txt into proxies_priv +mysql.proxies_priv: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/tables_priv.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/tables_priv.txt into tables_priv +mysql.tables_priv: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/time_zone_name.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_name.txt into time_zone_name +mysql.time_zone_name: Records: 7 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/time_zone.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone.txt into time_zone +mysql.time_zone: Records: 6 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/test/t1.sql +Loading data from LOCAL file: vardir/tmp/dump/test/t1.txt into t1 +test.t1: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/column_stats.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/column_stats.txt into column_stats +mysql.column_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/columns_priv.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/columns_priv.txt into columns_priv +mysql.columns_priv: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/db.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/db.txt into db +mysql.db: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/func.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/func.txt into func +mysql.func: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/gtid_slave_pos.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/gtid_slave_pos.txt into gtid_slave_pos +mysql.gtid_slave_pos: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/index_stats.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/index_stats.txt into index_stats +mysql.index_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/plugin.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/plugin.txt into plugin +mysql.plugin: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/procs_priv.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/procs_priv.txt into procs_priv +mysql.procs_priv: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/roles_mapping.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/roles_mapping.txt into roles_mapping +mysql.roles_mapping: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/servers.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/servers.txt into servers +mysql.servers: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/table_stats.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/table_stats.txt into table_stats +mysql.table_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/mysql/event.sql +Executing SQL script vardir/tmp/dump/mysql/general_log.sql +Executing SQL script vardir/tmp/dump/mysql/innodb_index_stats.sql +Executing SQL script vardir/tmp/dump/mysql/innodb_table_stats.sql +Executing SQL script vardir/tmp/dump/mysql/slow_log.sql +Executing SQL script vardir/tmp/dump/mysql/transaction_registry.sql +Executing SQL script vardir/tmp/dump/mysql/user.sql +Executing SQL script vardir/tmp/dump/test/v1.sql +Disconnecting from localhost +drop table t1; +drop view v1; +create database db2; +use db2; +CREATE TABLE parent ( +id INT NOT NULL, +PRIMARY KEY (id) +) ENGINE=INNODB; +CREATE TABLE child ( +id INT, +parent_id INT, +INDEX par_ind (parent_id), +FOREIGN KEY (parent_id) +REFERENCES parent(id) +ON DELETE CASCADE +) ENGINE=INNODB; +insert into parent values(1),(2); +insert into child values (1,1),(1,2),(2,1),(2,2); +drop database db2; +use db2; +select * from parent; +id +1 +2 +select * from child; +id parent_id +1 1 +1 2 +2 1 +2 2 +drop table child; +drop table parent; +CREATE TABLE animals (id mediumint(9) +NOT NULL AUTO_INCREMENT, +name char(30) NOT NULL, +PRIMARY KEY (`id`)); +CREATE TABLE animal_count (animals int); +INSERT INTO animal_count (animals) VALUES(0); +CREATE TRIGGER increment_animal +AFTER INSERT ON animals +FOR EACH ROW +UPDATE animal_count SET animal_count.animals = animal_count.animals+1; +INSERT INTO animals (name) VALUES('aardvark'); +INSERT INTO animals (name) VALUES('baboon'); +# Content of tables before backup +select * from animals; +id name +1 aardvark +2 baboon +select * from animal_count; +animals +2 +use test; +drop database db2; +Connecting to localhost +Executing SQL script vardir/tmp/dump/db2/animals.sql +Loading data from LOCAL file: vardir/tmp/dump/db2/animals.txt into animals +db2.animals: Records: 2 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/db2/animal_count.sql +Loading data from LOCAL file: vardir/tmp/dump/db2/animal_count.txt into animal_count +db2.animal_count: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 +Disconnecting from localhost +use db2; +# Content of tables after import +select * from animals; +id name +1 aardvark +2 baboon +select * from animal_count; +animals +2 +drop table animals; +drop table animal_count; +create table t1 as select 1 as val; +create view a1 as select * from t1; +use test; +drop database db2; +Connecting to localhost +Executing SQL script vardir/tmp/dump/db2/t1.sql +Loading data from LOCAL file: vardir/tmp/dump/db2/t1.txt into t1 +db2.t1: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 +Executing SQL script vardir/tmp/dump/db2/a1.sql +Disconnecting from localhost +use db2; +select * from t1; +val +1 +select * from a1; +val +1 +drop database db2; +use test; +create database db; +use db; +create table t1 as select 1 as val; +use test; +drop database db; +use db; +ERROR 42000: Unknown database 'db' +use test; +# Test non-existing --dir +mariadb-import: Path 'MYSQLTEST_VARDIR/tmp/non_existing' specified by option '--dir' does not exist +# Test too many threads, builtin limit 256 +Too many connections, max value for --parallel is 256 diff --git a/mysql-test/main/mariadb-import.test b/mysql-test/main/mariadb-import.test new file mode 100644 index 00000000000..5aee5304e13 --- /dev/null +++ b/mysql-test/main/mariadb-import.test @@ -0,0 +1,148 @@ +--source include/not_embedded.inc +--source include/have_innodb.inc + +# Basic test case for --table (restore single table) +create table t1(i int); +insert t1 values(100); +create view v1 as select 1; + +--mkdir $MYSQLTEST_VARDIR/tmp/dump +--exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump test +drop table t1; +--exec $MYSQL_IMPORT --table=test.t1 --dir=$MYSQLTEST_VARDIR/tmp/dump +select * from t1; +--rmdir $MYSQLTEST_VARDIR/tmp/dump + +# Test cases for --dir +# test --all-databases +--mkdir $MYSQLTEST_VARDIR/tmp/dump +--exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump --all-databases +--echo # Content of dump directory +--list_files $MYSQLTEST_VARDIR/tmp/dump +--echo # Content of 'test' dump subdirectory +--list_files $MYSQLTEST_VARDIR/tmp/dump/test +--echo # Content of 'mysql' dump subdirectory +--list_files $MYSQLTEST_VARDIR/tmp/dump/mysql +--echo # Content of 'mtr' dump subdirectory +--list_files $MYSQLTEST_VARDIR/tmp/dump/mtr + +# Test --dir +--replace_result $MYSQLTEST_VARDIR vardir +# Ignore mtr.test_suppressions (may have suppressions or now), mysql.proc is smaller without perfschema/sys schema +--exec $MYSQL_IMPORT --local --verbose --dir $MYSQLTEST_VARDIR/tmp/dump --ignore-table=mtr.test_suppressions --ignore-table=mysql.proc + +drop table t1; +drop view v1; +--rmdir $MYSQLTEST_VARDIR/tmp/dump + +# Test foreign keys +create database db2; +use db2; +CREATE TABLE parent ( + id INT NOT NULL, + PRIMARY KEY (id) +) ENGINE=INNODB; +CREATE TABLE child ( + id INT, + parent_id INT, + INDEX par_ind (parent_id), + FOREIGN KEY (parent_id) + REFERENCES parent(id) + ON DELETE CASCADE +) ENGINE=INNODB; +insert into parent values(1),(2); +insert into child values (1,1),(1,2),(2,1),(2,2); + +--mkdir $MYSQLTEST_VARDIR/tmp/dump +--exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump --all-databases +drop database db2; +--replace_result $MYSQLTEST_VARDIR vardir +--exec $MYSQL_IMPORT --local --silent --dir $MYSQLTEST_VARDIR/tmp/dump --database=db2 --parallel=2 +use db2; +select * from parent; +select * from child; +drop table child; +drop table parent; +--rmdir $MYSQLTEST_VARDIR/tmp/dump + +# Test with triggers (using https://mariadb.com/kb/en/trigger-overview/ example) + +CREATE TABLE animals (id mediumint(9) +NOT NULL AUTO_INCREMENT, +name char(30) NOT NULL, +PRIMARY KEY (`id`)); + +CREATE TABLE animal_count (animals int); +INSERT INTO animal_count (animals) VALUES(0); + +CREATE TRIGGER increment_animal +AFTER INSERT ON animals +FOR EACH ROW +UPDATE animal_count SET animal_count.animals = animal_count.animals+1; + +INSERT INTO animals (name) VALUES('aardvark'); +INSERT INTO animals (name) VALUES('baboon'); + + +--echo # Content of tables before backup +select * from animals; +select * from animal_count; + +--mkdir $MYSQLTEST_VARDIR/tmp/dump +--exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump db2 +use test; +drop database db2; +--replace_result $MYSQLTEST_VARDIR vardir +--exec $MYSQL_IMPORT --local --verbose --dir $MYSQLTEST_VARDIR/tmp/dump +use db2; +--echo # Content of tables after import +select * from animals; +select * from animal_count; +drop table animals; +drop table animal_count; + +# Test VIEW +create table t1 as select 1 as val; +create view a1 as select * from t1; +--rmdir $MYSQLTEST_VARDIR/tmp/dump +--mkdir $MYSQLTEST_VARDIR/tmp/dump +--exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump db2 +use test; +drop database db2; +--replace_result $MYSQLTEST_VARDIR vardir +--exec $MYSQL_IMPORT --local --verbose --dir $MYSQLTEST_VARDIR/tmp/dump +use db2; +select * from t1; +select * from a1; +drop database db2; +--rmdir $MYSQLTEST_VARDIR/tmp/dump +use test; + +# Test --ignore-database +# Do full backup, drop one db, restore with --ignore-database=db +# Check that database does not exist anymore +create database db; +use db; +create table t1 as select 1 as val; +--mkdir $MYSQLTEST_VARDIR/tmp/dump +--exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump --all-databases +use test; +drop database db; +--replace_result $MYSQLTEST_VARDIR vardir +--exec $MYSQL_IMPORT --local --silent --dir $MYSQLTEST_VARDIR/tmp/dump --ignore-database=db +--error ER_BAD_DB_ERROR +use db; +use test; + +--echo # Test non-existing --dir +--replace_result mariadb-import.exe mariadb-import $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 1 +--exec $MYSQL_IMPORT --dir $MYSQLTEST_VARDIR/tmp/non_existing 2>&1 + +--echo # Test too many threads, builtin limit 256 +--replace_result mariadb-import.exe mariadb-import $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 1 +--exec $MYSQL_IMPORT --dir $MYSQLTEST_VARDIR/tmp/dump --parallel=300 2>&1 + +--rmdir $MYSQLTEST_VARDIR/tmp/dump +