diff --git a/client/mysql.cc b/client/mysql.cc index d22a3d49b16..d66b7706f16 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -3577,6 +3577,7 @@ print_field_types(MYSQL_RES *result) BinaryStringBuffer<128> data_type_metadata_str; metadata.print_data_type_related_attributes(&data_type_metadata_str); tee_fprintf(PAGER, "Field %3u: `%s`\n" + "Org_field: `%s`\n" "Catalog: `%s`\n" "Database: `%s`\n" "Table: `%s`\n" @@ -3588,8 +3589,8 @@ print_field_types(MYSQL_RES *result) "Decimals: %u\n" "Flags: %s\n\n", ++i, - field->name, field->catalog, field->db, field->table, - field->org_table, fieldtype2str(field->type), + field->name, field->org_name, field->catalog, field->db, + field->table, field->org_table, fieldtype2str(field->type), data_type_metadata_str.length() ? " (" : "", data_type_metadata_str.length(), data_type_metadata_str.ptr(), data_type_metadata_str.length() ? ")" : "", diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 4268714e979..e695d27b4c2 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -22,12 +22,14 @@ if [[ $TRAVIS ]] || [[ $GITLAB_CI ]] then # On both Travis and Gitlab the output log must stay under 4MB so make the # build less verbose - # MCOL-4149: ColumnStore builds are so slow and big that they must be skipped on - # both Travis-CI and Gitlab-CI - sed -e 's|$(CMAKEFLAGS)|$(CMAKEFLAGS) -DPLUGIN_COLUMNSTORE=NO|' \ - -i debian/rules + sed '/Add support for verbose builds/,/^$/d' -i debian/rules elif [ -d storage/columnstore/columnstore/debian ] then + # ColumnStore is explicitly disabled in the native Debian build, so allow it + # now when build is triggered by autobake-deb.sh (MariaDB.org) and when the + # build is not running on Travis or Gitlab-CI + sed '/-DPLUGIN_COLUMNSTORE=NO/d' -i debian/rules + # Take the files and part of control from MCS directory cp -v storage/columnstore/columnstore/debian/mariadb-plugin-columnstore.* debian/ echo >> debian/control cat storage/columnstore/columnstore/debian/control >> debian/control diff --git a/debian/rules b/debian/rules index a1075ffb9ac..2f6938b04a4 100755 --- a/debian/rules +++ b/debian/rules @@ -38,6 +38,16 @@ else NUMJOBS = 1 endif +# RocksDB cannot build on 32-bit platforms +ifeq (32,$(DEB_HOST_ARCH_BITS)) + CMAKEFLAGS += -DPLUGIN_ROCKSDB=NO +endif + +# ColumnStore can build only on amd64 and arm64 +ifneq (,$(filter $(DEB_HOST_ARCH_CPU),amd64 arm64)) + CMAKEFLAGS += -DPLUGIN_COLUMNSTORE=NO +endif + # Cross building requires stack direction instruction ifneq ($(DEB_BUILD_ARCH),$(DEB_HOST_ARCH)) ifneq (,$(filter $(DEB_HOST_ARCH_CPU),alpha amd64 arm arm64 i386 ia64 m68k mips64el mipsel powerpc ppc64 ppc64el riscv64 s390x sh4 sparc64)) @@ -75,13 +85,15 @@ endif echo "server:Version=$(DEB_VERSION)" >> debian/substvars - # RocksDB and Column Store cannot build on 32-bit platforms + # Don't build ColumnStore as part of the native build as it does not meet the + # quality standards in Debian. Also building it requires an extra 4 GB of disk + # space which makes native Debian builds fail as the total disk space needed + # for MariaDB becomes over 10 GB. Only build CS via autobake-deb.sh. PATH=$${MYSQL_BUILD_PATH:-"/usr/lib/ccache:/usr/local/bin:/usr/bin:/bin"} \ NO_UPDATE_BUILD_VERSION=1 \ dh_auto_configure --builddirectory=$(BUILDDIR) -- \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ $(CMAKEFLAGS) \ - $(if $(findstring $(DEB_HOST_ARCH_BITS),32),-DPLUGIN_ROCKSDB=NO -DPLUGIN_COLUMNSTORE=NO) \ $(if $(filter $(DEB_BUILD_ARCH),$(DEB_HOST_ARCH)),,-DIMPORT_EXECUTABLES=$(CURDIR)/builddir-native/import_executables.cmake) \ -DCOMPILATION_COMMENT="mariadb.org binary distribution" \ -DMYSQL_SERVER_SUFFIX="-$(DEB_VERSION_REVISION)" \ @@ -90,6 +102,7 @@ endif -DBUILD_CONFIG=mysql_release \ -DCONC_DEFAULT_CHARSET=utf8mb4 \ -DPLUGIN_AWS_KEY_MANAGEMENT=NO \ + -DPLUGIN_COLUMNSTORE=NO \ -DIGNORE_AIO_CHECK=YES \ -DWITH_URING=yes \ -DDEB=$(DEB_VENDOR) diff --git a/debian/salsa-ci.yml b/debian/salsa-ci.yml index 101afcefcc5..0039e1be393 100644 --- a/debian/salsa-ci.yml +++ b/debian/salsa-ci.yml @@ -299,6 +299,8 @@ mariadb-10.1 Stretch to mariadb-10.7 upgrade: - apt-get install -y 'default-mysql*' 'mariadb-*' 'libmariadbd*' 'libmariadbclient*' # Verify installation of MariaDB from Stretch - *test-verify-initial + # Remove manpages 4.10 that conflicts with manpages-dev 5.10 on console_ioctl.4.gz + - apt-get remove -y manpages - *test-enable-sid-repos - *test-install - service mysql status @@ -507,28 +509,8 @@ default-libmysqlclient-dev Stretch upgrade: - *test-prepare-container - apt-get install -y pkg-config default-libmysqlclient-dev - pkg-config --list-all - - *test-enable-sid-repos - - *test-install-all-libs - - *test-verify-libs - except: - variables: - - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/ - -mariadb-connector-c Stretch upgrade: - stage: upgrade from Buster/Stretch - needs: - - job: build - artifacts: true - image: debian:stretch - artifacts: - when: always - name: "$CI_BUILD_NAME" - paths: - - ${WORKING_DIR}/debug - script: - - *test-prepare-container - - apt-get install -y pkg-config libmariadb2 libmariadb-dev libmariadb-dev-compat - - pkg-config --list-all + # Remove manpages 4.10 that conflicts with manpages-dev 5.10 on console_ioctl.4.gz + - apt-get remove -y manpages - *test-enable-sid-repos - *test-install-all-libs - *test-verify-libs @@ -733,7 +715,7 @@ mariadb.org-10.2 to mariadb-10.7 upgrade: # prepending with --defaults-file=/etc/mysql/debian.cnf is needed in upstream 5.5–10.3 - mysql --defaults-file=/etc/mysql/debian.cnf --skip-column-names -e "SELECT @@version, @@version_comment" - echo 'SHOW DATABASES;' | mysql --defaults-file=/etc/mysql/debian.cnf - - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost" + - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT * FROM mysql.user; SHOW CREATE USER root@localhost" - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT * FROM mysql.plugin; SHOW PLUGINS" - *test-install - service mysql status diff --git a/debian/source/lintian-overrides b/debian/source/lintian-overrides index c25c6fc1dd7..66e4ff5b962 100644 --- a/debian/source/lintian-overrides +++ b/debian/source/lintian-overrides @@ -22,3 +22,9 @@ version-substvar-for-external-package mariadb-client-10.7 -> mysql-client-core-8 version-substvar-for-external-package libmariadbd-dev -> libmariadbclient-dev # ColumnStore not used in Debian, safe to ignore. Reported upstream in https://jira.mariadb.org/browse/MDEV-24124 source-is-missing storage/columnstore/columnstore/utils/jemalloc/libjemalloc.so.2 +# Must be fixed upstream +source-is-missing storage/mroonga/vendor/groonga/examples/dictionary/html/js/jquery-ui-1.8.18.custom.js line 58 is 273 characters long (>256) +# Intentional control relationships +version-substvar-for-external-package Replaces (line 216) ${source:Version} libmariadbd-dev -> libmariadbclient-dev +version-substvar-for-external-package Replaces (line 66) ${source:Version} libmariadb-dev -> libmysqlclient-dev +version-substvar-for-external-package Replaces (line 66) ${source:Version} libmariadb-dev -> libmysqld-dev diff --git a/include/handler_ername.h b/include/handler_ername.h index d03790b8e64..026d7c8bb8c 100644 --- a/include/handler_ername.h +++ b/include/handler_ername.h @@ -78,3 +78,4 @@ { "HA_ERR_ABORTED_BY_USER", HA_ERR_ABORTED_BY_USER, "" }, { "HA_ERR_DISK_FULL", HA_ERR_DISK_FULL, "" }, { "HA_ERR_INCOMPATIBLE_DEFINITION", HA_ERR_INCOMPATIBLE_DEFINITION, "" }, +{ "HA_ERR_COMMIT_ERROR", HA_ERR_COMMIT_ERROR, "" }, diff --git a/include/my_base.h b/include/my_base.h index 053bf3fbb69..80ee527ccf0 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -527,7 +527,8 @@ enum ha_base_keytype { #define HA_ERR_TABLESPACE_MISSING 194 /* Missing Tablespace */ #define HA_ERR_SEQUENCE_INVALID_DATA 195 #define HA_ERR_SEQUENCE_RUN_OUT 196 -#define HA_ERR_LAST 196 /* Copy of last error nr * */ +#define HA_ERR_COMMIT_ERROR 197 +#define HA_ERR_LAST 197 /* Copy of last error nr * */ /* Number of different errors */ #define HA_ERR_ERRORS (HA_ERR_LAST - HA_ERR_FIRST + 1) diff --git a/include/my_handler_errors.h b/include/my_handler_errors.h index faf7b9f459e..69b1566557d 100644 --- a/include/my_handler_errors.h +++ b/include/my_handler_errors.h @@ -107,7 +107,8 @@ static const char *handler_error_messages[]= "Foreign key cascade delete/update exceeds max depth", "Tablespace is missing for a table", "Sequence has been run out", - "Sequence values are conflicting" + "Sequence values are conflicting", + "Error during commit" }; #endif /* MYSYS_MY_HANDLER_ERRORS_INCLUDED */ diff --git a/include/myisamchk.h b/include/myisamchk.h index f9a55ba467f..c494c672ec7 100644 --- a/include/myisamchk.h +++ b/include/myisamchk.h @@ -87,7 +87,7 @@ typedef struct st_handler_check_param /* Following is used to check if rows are visible */ ulonglong max_trid, max_found_trid; ulonglong not_visible_rows_found; - ulonglong sort_buffer_length; + ulonglong sort_buffer_length, orig_sort_buffer_length; ulonglong use_buffers; /* Used as param to getopt() */ size_t read_buffer_length, write_buffer_length, sort_key_blocks; time_t backup_time; /* To sign backup files */ diff --git a/include/wsrep.h b/include/wsrep.h index 835c8d77c72..20159048442 100644 --- a/include/wsrep.h +++ b/include/wsrep.h @@ -26,10 +26,6 @@ #define WSREP_MYSQL_DB (char *)"mysql" -#define WSREP_TO_ISOLATION_BEGIN_IF(db_, table_, table_list_) \ - if (WSREP_ON && WSREP(thd) && \ - wsrep_to_isolation_begin(thd, db_, table_, table_list_)) - #define WSREP_TO_ISOLATION_BEGIN(db_, table_, table_list_) \ if (WSREP_ON && WSREP(thd) && \ wsrep_to_isolation_begin(thd, db_, table_, table_list_)) \ @@ -57,10 +53,6 @@ wsrep_to_isolation_begin(thd, db_, table_, table_list_)) \ goto wsrep_error_label; -#define WSREP_TO_ISOLATION_BEGIN_FK_TABLES(db_, table_, table_list_, fk_tables) \ - if (WSREP(thd) && !thd->lex->no_write_to_binlog \ - && wsrep_to_isolation_begin(thd, db_, table_, table_list_, NULL, fk_tables)) - #define WSREP_SYNC_WAIT(thd_, before_) \ { if (WSREP_CLIENT(thd_) && \ wsrep_sync_wait(thd_, before_)) goto wsrep_error_label; } @@ -75,7 +67,6 @@ #define WSREP_ERROR(...) #define WSREP_TO_ISOLATION_BEGIN(db_, table_, table_list_) do { } while(0) #define WSREP_TO_ISOLATION_BEGIN_ALTER(db_, table_, table_list_, alter_info_, fk_tables_, create_info_) -#define WSREP_TO_ISOLATION_BEGIN_FK_TABLES(db_, table_, table_list_, fk_tables_) #define WSREP_TO_ISOLATION_BEGIN_CREATE(db_, table_, table_list_, create_info_) #define WSREP_TO_ISOLATION_BEGIN_WRTCHK(db_, table_, table_list_) #define WSREP_SYNC_WAIT(thd_, before_) diff --git a/mysql-test/main/alias.result b/mysql-test/main/alias.result index defd44f2548..0d14607f613 100644 --- a/mysql-test/main/alias.result +++ b/mysql-test/main/alias.result @@ -218,3 +218,22 @@ DELETE ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 't1 WHERE 1=1' at line 1 connection default; disconnect c1; +# +# MDEV-23519 +# +create or replace table t1 (a int); +create or replace table t2 (b int); +insert into t1 values(1<<30),(1<<29); +insert into t2 values(1),(2); +select t1.a as a1 from t1 as t1,t2 order by t2.b,t1.a; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t1 t1 a a1 3 11 10 Y 32768 0 63 +a1 +536870912 +1073741824 +536870912 +1073741824 +drop table t1,t2; +# +# End of 10.4 tests +# diff --git a/mysql-test/main/alias.test b/mysql-test/main/alias.test index c02ebe2f5ff..2408509ba10 100644 --- a/mysql-test/main/alias.test +++ b/mysql-test/main/alias.test @@ -226,3 +226,18 @@ connection c1; DELETE ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZROM t1 WHERE 1=1; connection default; disconnect c1; +--echo # +--echo # MDEV-23519 +--echo # +create or replace table t1 (a int); +create or replace table t2 (b int); +insert into t1 values(1<<30),(1<<29); +insert into t2 values(1),(2); +--enable_metadata +select t1.a as a1 from t1 as t1,t2 order by t2.b,t1.a; +--disable_metadata +drop table t1,t2; + +--echo # +--echo # End of 10.4 tests +--echo # diff --git a/mysql-test/main/mysql-metadata.result b/mysql-test/main/mysql-metadata.result index ce6c77ed7fa..51b3eb24f2b 100644 --- a/mysql-test/main/mysql-metadata.result +++ b/mysql-test/main/mysql-metadata.result @@ -13,6 +13,7 @@ SELECT * FROM t1 -------------- Field 1: `js0` +Org_field: `js0` Catalog: `def` Database: `test` Table: `t1` @@ -25,6 +26,7 @@ Decimals: 0 Flags: BLOB BINARY Field 2: `js1` +Org_field: `js1` Catalog: `def` Database: `test` Table: `t1` @@ -37,6 +39,7 @@ Decimals: 0 Flags: BLOB Field 3: `js2` +Org_field: `js2` Catalog: `def` Database: `test` Table: `t1` @@ -49,6 +52,7 @@ Decimals: 0 Flags: BLOB Field 4: `js3` +Org_field: `js3` Catalog: `def` Database: `test` Table: `t1` @@ -69,6 +73,7 @@ SELECT JSON_COMPACT(js0) FROM t1 -------------- Field 1: `JSON_COMPACT(js0)` +Org_field: `` Catalog: `def` Database: `` Table: `` diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index f9995ebbec2..70220a1e01e 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -33,7 +33,6 @@ galera_parallel_simple : MDEV-20318 galera.galera_parallel_simple fails galera_pc_ignore_sb : MDEV-20888 galera.galera_pc_ignore_sb galera_pc_recovery : MDEV-25199 cluster fails to start up galera_shutdown_nonprim : MDEV-21493 galera.galera_shutdown_nonprim -galera_ssl_upgrade : MDEV-19950 Galera test failure on galera_ssl_upgrade galera_sst_mysqldump : MDEV-26501 : galera.galera_sst_mysqldump MTR failed: galera SST with mysqldump failed galera_toi_ddl_nonconflicting : MDEV-21518 galera.galera_toi_ddl_nonconflicting galera_toi_truncate : MDEV-22996 Hang on galera_toi_truncate test case diff --git a/mysql-test/suite/galera/r/galera_fk_truncate.result b/mysql-test/suite/galera/r/galera_fk_truncate.result new file mode 100644 index 00000000000..a90f3e27ab7 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_fk_truncate.result @@ -0,0 +1,47 @@ +connection node_2; +connection node_1; +CREATE TABLE author ( +id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, +name VARCHAR(100) NOT NULL +) ENGINE = InnoDB; +CREATE TABLE book ( +id MEDIUMINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, +title VARCHAR(200) NOT NULL, +author_id SMALLINT UNSIGNED NOT NULL, +CONSTRAINT `fk_book_author` + FOREIGN KEY (author_id) REFERENCES author (id) +ON DELETE CASCADE +ON UPDATE RESTRICT +) ENGINE = InnoDB; +INSERT INTO author (name) VALUES ('Abdul Alhazred'); +INSERT INTO book (title, author_id) VALUES ('Necronomicon', LAST_INSERT_ID()); +TRUNCATE TABLE book; +SELECT * FROM author; +id name +1 Abdul Alhazred +SELECT * FROM book; +id title author_id +connection node_2; +SELECT * FROM author; +id name +1 Abdul Alhazred +SELECT * FROM book; +id title author_id +INSERT INTO author (name) VALUES ('Abdul Alhazred'); +INSERT INTO book (title, author_id) VALUES ('Necronomicon', LAST_INSERT_ID()); +TRUNCATE TABLE book; +SELECT * FROM author; +id name +1 Abdul Alhazred +2 Abdul Alhazred +SELECT * FROM book; +id title author_id +connection node_1; +TRUNCATE TABLE book; +SELECT * FROM author; +id name +1 Abdul Alhazred +2 Abdul Alhazred +SELECT * FROM book; +id title author_id +DROP TABLE book, author; diff --git a/mysql-test/suite/galera/r/galera_ssl_upgrade.result b/mysql-test/suite/galera/r/galera_ssl_upgrade.result index 1443e34d041..134d1d1b605 100644 --- a/mysql-test/suite/galera/r/galera_ssl_upgrade.result +++ b/mysql-test/suite/galera/r/galera_ssl_upgrade.result @@ -1,3 +1,5 @@ +connection node_2; +connection node_1; connection node_1; call mtr.add_suppression("WSREP: write_handler(): protocol is shutdown.*"); connection node_2; @@ -24,5 +26,6 @@ connection node_1; SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; VARIABLE_VALUE = 2 1 +connection node_2; disconnect node_2; disconnect node_1; diff --git a/mysql-test/suite/galera/t/galera_fk_truncate.test b/mysql-test/suite/galera/t/galera_fk_truncate.test new file mode 100644 index 00000000000..9d54a720432 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_fk_truncate.test @@ -0,0 +1,39 @@ +--source include/galera_cluster.inc + +CREATE TABLE author ( + id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100) NOT NULL +) ENGINE = InnoDB; + +CREATE TABLE book ( + id MEDIUMINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + title VARCHAR(200) NOT NULL, + author_id SMALLINT UNSIGNED NOT NULL, + CONSTRAINT `fk_book_author` + FOREIGN KEY (author_id) REFERENCES author (id) + ON DELETE CASCADE + ON UPDATE RESTRICT +) ENGINE = InnoDB; + +INSERT INTO author (name) VALUES ('Abdul Alhazred'); +INSERT INTO book (title, author_id) VALUES ('Necronomicon', LAST_INSERT_ID()); + +TRUNCATE TABLE book; +SELECT * FROM author; +SELECT * FROM book; + +--connection node_2 +SELECT * FROM author; +SELECT * FROM book; +INSERT INTO author (name) VALUES ('Abdul Alhazred'); +INSERT INTO book (title, author_id) VALUES ('Necronomicon', LAST_INSERT_ID()); +TRUNCATE TABLE book; +SELECT * FROM author; +SELECT * FROM book; + +--connection node_1 +TRUNCATE TABLE book; +SELECT * FROM author; +SELECT * FROM book; + +DROP TABLE book, author; diff --git a/mysql-test/suite/galera_sr/r/MDEV-21613.result b/mysql-test/suite/galera_sr/r/MDEV-21613.result new file mode 100644 index 00000000000..67af2d49331 --- /dev/null +++ b/mysql-test/suite/galera_sr/r/MDEV-21613.result @@ -0,0 +1,20 @@ +connection node_2; +connection node_1; +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY); +connection node_1; +SET SESSION wsrep_trx_fragment_size = 1; +SET DEBUG_SYNC = "wsrep_before_fragment_removal SIGNAL fragment_removal_reached WAIT_FOR fragment_removal_continue"; +START TRANSACTION; +INSERT INTO t1 VALUES(1), (2); +COMMIT; +connect node_ctrl, 127.0.0.1, root, , test, $NODE_MYPORT_1; +connection node_ctrl; +SET DEBUG_SYNC = "now WAIT_FOR fragment_removal_reached"; +connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; +connection node_1a; +TRUNCATE TABLE t1; +connection node_1; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +connection node_ctrl; +SET DEBUG_SYNC = 'RESET'; +DROP TABLE t1; diff --git a/mysql-test/suite/galera_sr/t/MDEV-21613.cnf b/mysql-test/suite/galera_sr/t/MDEV-21613.cnf new file mode 100644 index 00000000000..d8e3488f044 --- /dev/null +++ b/mysql-test/suite/galera_sr/t/MDEV-21613.cnf @@ -0,0 +1,7 @@ +# Set thread-handling as a workaround to avoid MDEV-26528. +# The file can be removed once fixed. + +!include ../galera_2nodes.cnf + +[mysqld.1] +thread-handling=pool-of-threads diff --git a/mysql-test/suite/galera_sr/t/MDEV-21613.test b/mysql-test/suite/galera_sr/t/MDEV-21613.test new file mode 100644 index 00000000000..8a1fea1b7f5 --- /dev/null +++ b/mysql-test/suite/galera_sr/t/MDEV-21613.test @@ -0,0 +1,36 @@ +# +# MDEV-21613 - galera_sr.GCF-1018B MTR failed: +# Failed to open table mysql.wsrep_streaming_log for writing +# +# A BF abort right before fragment removal caused this error to +# be logged to the error log. +# +--source include/galera_cluster.inc +--source include/have_debug_sync.inc + +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY); + +--connection node_1 +SET SESSION wsrep_trx_fragment_size = 1; +SET DEBUG_SYNC = "wsrep_before_fragment_removal SIGNAL fragment_removal_reached WAIT_FOR fragment_removal_continue"; +START TRANSACTION; +INSERT INTO t1 VALUES(1), (2); +--send COMMIT + +--connect node_ctrl, 127.0.0.1, root, , test, $NODE_MYPORT_1 +--connection node_ctrl +SET DEBUG_SYNC = "now WAIT_FOR fragment_removal_reached"; + + +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 +--connection node_1a +TRUNCATE TABLE t1; + + +--connection node_1 +--error ER_LOCK_DEADLOCK +--reap + +--connection node_ctrl +SET DEBUG_SYNC = 'RESET'; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/r/instant_alter_import.result b/mysql-test/suite/innodb/r/instant_alter_import.result index 2d9a39f7886..008a26ea6af 100644 --- a/mysql-test/suite/innodb/r/instant_alter_import.result +++ b/mysql-test/suite/innodb/r/instant_alter_import.result @@ -70,3 +70,11 @@ select * from t1; ERROR HY000: Tablespace has been discarded for table `t1` drop table t2; drop table t1; +CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT, i1 INT) ENGINE=INNODB; +CREATE TABLE t2 (id INT PRIMARY KEY AUTO_INCREMENT, i1 INT, i2 INT) ENGINE=INNODB; +ALTER TABLE t2 DROP COLUMN i2, ALGORITHM=INSTANT; +ALTER TABLE t2 DISCARD TABLESPACE; +FLUSH TABLE t1 FOR EXPORT; +UNLOCK TABLES; +ALTER TABLE t2 IMPORT TABLESPACE; +DROP TABLE t2, t1; diff --git a/mysql-test/suite/innodb/t/instant_alter_import.test b/mysql-test/suite/innodb/t/instant_alter_import.test index 208bc423a11..c6fd2f344cf 100644 --- a/mysql-test/suite/innodb/t/instant_alter_import.test +++ b/mysql-test/suite/innodb/t/instant_alter_import.test @@ -83,3 +83,22 @@ select * from t1; drop table t2; drop table t1; + + +--let $MYSQLD_DATADIR= `SELECT @@datadir` + +CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT, i1 INT) ENGINE=INNODB; + +CREATE TABLE t2 (id INT PRIMARY KEY AUTO_INCREMENT, i1 INT, i2 INT) ENGINE=INNODB; +ALTER TABLE t2 DROP COLUMN i2, ALGORITHM=INSTANT; +ALTER TABLE t2 DISCARD TABLESPACE; + +FLUSH TABLE t1 FOR EXPORT; + +--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_DATADIR/test/t2.cfg + +UNLOCK TABLES; +ALTER TABLE t2 IMPORT TABLESPACE; + +DROP TABLE t2, t1; diff --git a/mysql-test/suite/perfschema/t/transaction_nested_events-master.opt b/mysql-test/suite/perfschema/t/transaction_nested_events-master.opt index d68f0498edf..0583801ec98 100644 --- a/mysql-test/suite/perfschema/t/transaction_nested_events-master.opt +++ b/mysql-test/suite/perfschema/t/transaction_nested_events-master.opt @@ -3,7 +3,7 @@ --loose-performance_schema_events_statements_history_size=20 --loose-performance_schema_events_statements_history_long_size=50 --loose-performance_schema_events_transactions_history_long_size=50 ---loose-performance_schema_max_thread_instances=30 +--loose-performance_schema_max_thread_instances=60 --loose-performance-schema-consumer-events-stages-current=OFF --loose-performance-schema-consumer-events-stages-history=OFF diff --git a/mysys/mf_tempfile.c b/mysys/mf_tempfile.c index 51a3efa05ad..0f1c6d6b1bc 100644 --- a/mysys/mf_tempfile.c +++ b/mysys/mf_tempfile.c @@ -121,7 +121,7 @@ File create_temp_file(char *to, const char *dir, const char *prefix, /* explictly don't use O_EXCL here has it has a different meaning with O_TMPFILE */ - if ((file= open(dir, mode | O_TMPFILE | O_CLOEXEC, + if ((file= open(dir, (mode & ~O_CREAT) | O_TMPFILE | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)) >= 0) { my_snprintf(to, FN_REFLEN, "%s/#sql/fd=%d", dir, file); diff --git a/plugin/type_inet/mysql-test/type_inet/type_inet6_mysql.result b/plugin/type_inet/mysql-test/type_inet/type_inet6_mysql.result index 868b9902f5c..7ed7b5c4422 100644 --- a/plugin/type_inet/mysql-test/type_inet/type_inet6_mysql.result +++ b/plugin/type_inet/mysql-test/type_inet/type_inet6_mysql.result @@ -1,5 +1,6 @@ CREATE TABLE t1 (a INET6); Field 1: `a` +Org_field: `a` Catalog: `def` Database: `test` Table: `t1` @@ -12,6 +13,7 @@ Decimals: 0 Flags: UNSIGNED BINARY Field 2: `b` +Org_field: `` Catalog: `def` Database: `` Table: `` @@ -24,6 +26,7 @@ Decimals: 0 Flags: NOT_NULL UNSIGNED Field 3: `c` +Org_field: `` Catalog: `def` Database: `` Table: `` diff --git a/plugin/type_test/mysql-test/type_test/type_test_mysql.result b/plugin/type_test/mysql-test/type_test/type_test_mysql.result index 402e8265181..7dfbfce3340 100644 --- a/plugin/type_test/mysql-test/type_test/type_test_mysql.result +++ b/plugin/type_test/mysql-test/type_test/type_test_mysql.result @@ -1,5 +1,6 @@ CREATE TABLE t1 (a TEST_INT8, b TEST_DOUBLE); Field 1: `a` +Org_field: `a` Catalog: `def` Database: `test` Table: `t1` @@ -12,6 +13,7 @@ Decimals: 0 Flags: NUM Field 2: `b` +Org_field: `b` Catalog: `def` Database: `test` Table: `t1` diff --git a/sql/handler.cc b/sql/handler.cc index 7a220068a7c..cce50dc9289 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4532,6 +4532,9 @@ void handler::print_error(int error, myf errflag) case HA_ERR_UNDO_REC_TOO_BIG: textno= ER_UNDO_RECORD_TOO_BIG; break; + case HA_ERR_COMMIT_ERROR: + textno= ER_ERROR_DURING_COMMIT; + break; default: { /* The error was "unknown" to this function. diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index 839ea083e5c..6ba08ebd55a 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -477,16 +477,17 @@ static bool wsrep_toi_replication(THD *thd, TABLE_LIST *tables) /* now TOI replication, with no locks held */ if (keys.empty()) { - WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, tables); - } else { - WSREP_TO_ISOLATION_BEGIN_FK_TABLES(NULL, NULL, tables, &keys) { + if (!thd->lex->no_write_to_binlog && + wsrep_to_isolation_begin(thd, NULL, NULL, tables)) + return true; + } + else + { + if (!thd->lex->no_write_to_binlog && + wsrep_to_isolation_begin(thd, NULL, NULL, tables, NULL, &keys)) return true; - } } return false; - - wsrep_error_label: - return true; } #endif /* WITH_WSREP */ diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b80ebed2f30..ce99e68c59c 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -18281,7 +18281,14 @@ Field *Item_sum::create_tmp_field(MEM_ROOT *root, bool group, TABLE *table) /** Create a temporary field for Item_field (or its descendant), either direct or referenced by an Item_ref. + + param->modify_item is set when we create a field for an internal temporary + table. In this case we have to ensure the new field name is identical to + the original field name as the field will info will be sent to the client. + In other cases, the field name is set from orig_item or name if org_item is + not set. */ + Field * Item_field::create_tmp_field_from_item_field(MEM_ROOT *root, TABLE *new_table, Item_ref *orig_item, @@ -18289,6 +18296,10 @@ Item_field::create_tmp_field_from_item_field(MEM_ROOT *root, TABLE *new_table, { DBUG_ASSERT(!is_result_field()); Field *result; + LEX_CSTRING *new_name= (orig_item ? &orig_item->name : + !param->modify_item() ? &name : + &field->field_name); + /* If item have to be able to store NULLs but underlaid field can't do it, create_tmp_field_from_field() can't be used for tmp field creation. @@ -18307,8 +18318,7 @@ Item_field::create_tmp_field_from_item_field(MEM_ROOT *root, TABLE *new_table, Record_addr rec(orig_item ? orig_item->maybe_null() : maybe_null()); const Type_handler *handler= type_handler()-> type_handler_for_tmp_table(this); - result= handler->make_and_init_table_field(root, - orig_item ? &orig_item->name : &name, + result= handler->make_and_init_table_field(root, new_name, rec, *this, new_table); } else if (param->table_cant_handle_bit_fields() && @@ -18316,18 +18326,17 @@ Item_field::create_tmp_field_from_item_field(MEM_ROOT *root, TABLE *new_table, { const Type_handler *handler= Type_handler::type_handler_long_or_longlong(max_char_length(), true); - result= handler->make_and_init_table_field(root, &name, + result= handler->make_and_init_table_field(root, new_name, Record_addr(maybe_null()), *this, new_table); } else { - LEX_CSTRING *tmp= orig_item ? &orig_item->name : &name; bool tmp_maybe_null= param->modify_item() ? maybe_null() : field->maybe_null(); result= field->create_tmp_field(root, new_table, tmp_maybe_null); - if (result) - result->field_name= *tmp; + if (result && ! param->modify_item()) + result->field_name= *new_name; } if (result && param->modify_item()) result_field= result; diff --git a/sql/sql_truncate.cc b/sql/sql_truncate.cc index 1613c5a6586..c6af72c5979 100644 --- a/sql/sql_truncate.cc +++ b/sql/sql_truncate.cc @@ -467,15 +467,13 @@ bool Sql_cmd_truncate_table::truncate_table(THD *thd, TABLE_LIST *table_ref) { if (keys.empty()) { - WSREP_TO_ISOLATION_BEGIN_IF(table_ref->db.str, table_ref->table_name.str, NULL) - { + if (wsrep_to_isolation_begin(thd, table_ref->db.str, table_ref->table_name.str, NULL)) DBUG_RETURN(TRUE); - } - } else { - WSREP_TO_ISOLATION_BEGIN_FK_TABLES(NULL, NULL, table_ref, &keys) - { + } + else + { + if (wsrep_to_isolation_begin(thd, NULL, NULL, table_ref, NULL, &keys)) DBUG_RETURN(TRUE); - } } } } diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index ad0f7b6c137..87c95616107 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -80,7 +80,7 @@ int yylex(void *yylval, void *yythd); #define yyoverflow(A,B,C,D,E,F) \ { \ - size_t val= *(F); \ + size_t val= *(F); \ if (unlikely(my_yyoverflow((B), (D), &val))) \ { \ yyerror(thd, (char*) (A)); \ diff --git a/sql/wsrep_client_service.cc b/sql/wsrep_client_service.cc index a7480aaddb8..6ab2834a219 100644 --- a/sql/wsrep_client_service.cc +++ b/sql/wsrep_client_service.cc @@ -192,6 +192,7 @@ cleanup: int Wsrep_client_service::remove_fragments() { DBUG_ENTER("Wsrep_client_service::remove_fragments"); + DEBUG_SYNC(m_thd, "wsrep_before_fragment_removal"); if (wsrep_schema->remove_fragments(m_thd, Wsrep_server_state::instance().id(), m_thd->wsrep_trx().id(), diff --git a/sql/wsrep_schema.cc b/sql/wsrep_schema.cc index 1bfce7874ae..9d77577ed42 100644 --- a/sql/wsrep_schema.cc +++ b/sql/wsrep_schema.cc @@ -270,13 +270,7 @@ static int open_table(THD* thd, thd->lex->query_tables_own_last= 0; if (!open_n_lock_single_table(thd, &tables, tables.lock_type, flags)) { - if (thd->is_error()) { - WSREP_WARN("Can't lock table %s.%s : %d (%s)", - schema_name->str, table_name->str, - thd->get_stmt_da()->sql_errno(), thd->get_stmt_da()->message()); - } close_thread_tables(thd); - my_error(ER_NO_SUCH_TABLE, MYF(0), schema_name->str, table_name->str); DBUG_RETURN(1); } @@ -292,8 +286,15 @@ static int open_for_write(THD* thd, const char* table_name, TABLE** table) { LEX_CSTRING table_str= { table_name, strlen(table_name) }; if (Wsrep_schema_impl::open_table(thd, &schema_str, &table_str, TL_WRITE, table)) { - WSREP_ERROR("Failed to open table %s.%s for writing", - schema_str.str, table_name); + // No need to log an error if the query was bf aborted, + // thd client will get ER_LOCK_DEADLOCK in the end. + const bool interrupted= thd->killed || + (thd->is_error() && + (thd->get_stmt_da()->sql_errno() == ER_QUERY_INTERRUPTED)); + if (!interrupted) { + WSREP_ERROR("Failed to open table %s.%s for writing", + schema_str.str, table_name); + } return 1; } empty_record(*table); diff --git a/storage/connect/tabrest.h b/storage/connect/tabrest.h index 1f6a288c27d..901d9102e95 100644 --- a/storage/connect/tabrest.h +++ b/storage/connect/tabrest.h @@ -6,7 +6,7 @@ #pragma once #if defined(_WIN32) -#else // !__WIN__ +#else // !_WIN32 #define stricmp strcasecmp #endif // !_WIN32 diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 794efb3ec98..6dfd00ca971 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -219,7 +219,7 @@ btr_root_block_get( or RW_X_LATCH */ mtr_t* mtr) /*!< in: mtr */ { - if (!index->table || !index->table->space) { + if (!index->table || !index->table->space || index->page == FIL_NULL) { return NULL; } @@ -259,6 +259,7 @@ btr_root_block_get( /**************************************************************//** Gets the root node of a tree and sx-latches it for segment access. @return root page, sx-latched */ +static page_t* btr_root_get( /*=========*/ @@ -272,38 +273,6 @@ btr_root_get( return(root ? buf_block_get_frame(root) : NULL); } -/**************************************************************//** -Gets the height of the B-tree (the level of the root, when the leaf -level is assumed to be 0). The caller must hold an S or X latch on -the index. -@return tree height (level of the root) */ -ulint -btr_height_get( -/*===========*/ - const dict_index_t* index, /*!< in: index tree */ - mtr_t* mtr) /*!< in/out: mini-transaction */ -{ - ulint height=0; - buf_block_t* root_block; - - ut_ad(srv_read_only_mode - || mtr->memo_contains_flagged(&index->lock, MTR_MEMO_S_LOCK - | MTR_MEMO_X_LOCK - | MTR_MEMO_SX_LOCK)); - - /* S latches the page */ - root_block = btr_root_block_get(index, RW_S_LATCH, mtr); - - if (root_block) { - height = btr_page_get_level(buf_block_get_frame(root_block)); - - /* Release the S latch on the root page. */ - mtr->memo_release(root_block, MTR_MEMO_PAGE_S_FIX); - } - - return(height); -} - /**************************************************************//** Checks a file segment header within a B-tree root page and updates the segment header space id. @@ -561,50 +530,6 @@ btr_page_alloc( index, hint_page_no, file_direction, level, mtr, init_mtr); } -/**************************************************************//** -Gets the number of pages in a B-tree. -@return number of pages, or ULINT_UNDEFINED if the index is unavailable */ -ulint -btr_get_size( -/*=========*/ - const dict_index_t* index, /*!< in: index */ - ulint flag, /*!< in: BTR_N_LEAF_PAGES or BTR_TOTAL_SIZE */ - mtr_t* mtr) /*!< in/out: mini-transaction where index - is s-latched */ -{ - ulint n=0; - - ut_ad(srv_read_only_mode - || mtr->memo_contains(index->lock, MTR_MEMO_S_LOCK)); - ut_ad(flag == BTR_N_LEAF_PAGES || flag == BTR_TOTAL_SIZE); - - if (index->page == FIL_NULL - || dict_index_is_online_ddl(index) - || !index->is_committed() - || !index->table->space) { - return(ULINT_UNDEFINED); - } - - buf_block_t* root = btr_root_block_get(index, RW_SX_LATCH, mtr); - if (!root) { - return ULINT_UNDEFINED; - } - mtr->x_lock_space(index->table->space); - if (flag == BTR_N_LEAF_PAGES) { - fseg_n_reserved_pages(*root, PAGE_HEADER + PAGE_BTR_SEG_LEAF - + root->frame, &n, mtr); - } else { - ulint dummy; - n = fseg_n_reserved_pages(*root, PAGE_HEADER + PAGE_BTR_SEG_TOP - + root->frame, &dummy, mtr); - n += fseg_n_reserved_pages(*root, - PAGE_HEADER + PAGE_BTR_SEG_LEAF - + root->frame, &dummy, mtr); - } - - return(n); -} - /**************************************************************//** Frees a page used in an ibuf tree. Puts the page to the free list of the ibuf tree. */ diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 7b4a466eb96..8a705c5aa64 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -137,17 +137,6 @@ can be released by page reorganize, then it is reorganized */ #define BTR_BLOB_HDR_SIZE 8 /*!< Size of a BLOB part header, in bytes */ -/** Estimated table level stats from sampled value. -@param value sampled stats -@param index index being sampled -@param sample number of sampled rows -@param ext_size external stored data size -@param not_empty table not empty -@return estimated table wide stats from sampled value */ -#define BTR_TABLE_STATS_FROM_SAMPLE(value, index, sample, ext_size, not_empty) \ - (((value) * static_cast(index->stat_n_leaf_pages) \ - + (sample) - 1 + (ext_size) + (not_empty)) / ((sample) + (ext_size))) - /* @} */ /*******************************************************************//** @@ -411,15 +400,15 @@ unreadable: return DB_CORRUPTION; } - page_t* root = btr_root_get(index, mtr); + buf_block_t* root = btr_root_block_get(index, RW_SX_LATCH, mtr); - if (!root || btr_cur_instant_root_init(index, root)) { + if (!root || btr_cur_instant_root_init(index, root->frame)) { goto unreadable; } ut_ad(index->n_core_null_bytes != dict_index_t::NO_CORE_NULL_BYTES); - if (fil_page_get_type(root) == FIL_PAGE_INDEX) { + if (fil_page_get_type(root->frame) == FIL_PAGE_INDEX) { ut_ad(!index->is_instant()); return DB_SUCCESS; } @@ -6552,339 +6541,6 @@ btr_estimate_n_rows_in_range( index, tuple1, tuple2, 1); } -/*******************************************************************//** -Record the number of non_null key values in a given index for -each n-column prefix of the index where 1 <= n <= dict_index_get_n_unique(index). -The estimates are eventually stored in the array: -index->stat_n_non_null_key_vals[], which is indexed from 0 to n-1. */ -static -void -btr_record_not_null_field_in_rec( -/*=============================*/ - ulint n_unique, /*!< in: dict_index_get_n_unique(index), - number of columns uniquely determine - an index entry */ - const rec_offs* offsets, /*!< in: rec_get_offsets(rec, index), - its size could be for all fields or - that of "n_unique" */ - ib_uint64_t* n_not_null) /*!< in/out: array to record number of - not null rows for n-column prefix */ -{ - ulint i; - - ut_ad(rec_offs_n_fields(offsets) >= n_unique); - - if (n_not_null == NULL) { - return; - } - - for (i = 0; i < n_unique; i++) { - if (rec_offs_nth_sql_null(offsets, i)) { - break; - } - - n_not_null[i]++; - } -} - -/** Estimates the number of different key values in a given index, for -each n-column prefix of the index where 1 <= n <= dict_index_get_n_unique(index). -The estimates are stored in the array index->stat_n_diff_key_vals[] (indexed -0..n_uniq-1) and the number of pages that were sampled is saved in -result.n_sample_sizes[]. -If innodb_stats_method is nulls_ignored, we also record the number of -non-null values for each prefix and stored the estimates in -array result.n_non_null_key_vals. -@param[in] index index -@return vector with statistics information -empty vector if the index is unavailable. */ -std::vector -btr_estimate_number_of_different_key_vals(dict_index_t* index) -{ - btr_cur_t cursor; - page_t* page; - rec_t* rec; - ulint n_cols; - ib_uint64_t* n_diff; - ib_uint64_t* n_not_null; - ibool stats_null_not_equal; - uintmax_t n_sample_pages=1; /* number of pages to sample */ - ulint not_empty_flag = 0; - ulint total_external_size = 0; - ulint i; - ulint j; - uintmax_t add_on; - mtr_t mtr; - mem_heap_t* heap = NULL; - rec_offs* offsets_rec = NULL; - rec_offs* offsets_next_rec = NULL; - - std::vector result; - - /* For spatial index, there is no such stats can be - fetched. */ - ut_ad(!dict_index_is_spatial(index)); - - n_cols = dict_index_get_n_unique(index); - - heap = mem_heap_create((sizeof *n_diff + sizeof *n_not_null) - * n_cols - + dict_index_get_n_fields(index) - * (sizeof *offsets_rec - + sizeof *offsets_next_rec)); - - n_diff = (ib_uint64_t*) mem_heap_zalloc( - heap, n_cols * sizeof(n_diff[0])); - - n_not_null = NULL; - - /* Check srv_innodb_stats_method setting, and decide whether we - need to record non-null value and also decide if NULL is - considered equal (by setting stats_null_not_equal value) */ - switch (srv_innodb_stats_method) { - case SRV_STATS_NULLS_IGNORED: - n_not_null = (ib_uint64_t*) mem_heap_zalloc( - heap, n_cols * sizeof *n_not_null); - /* fall through */ - - case SRV_STATS_NULLS_UNEQUAL: - /* for both SRV_STATS_NULLS_IGNORED and SRV_STATS_NULLS_UNEQUAL - case, we will treat NULLs as unequal value */ - stats_null_not_equal = TRUE; - break; - - case SRV_STATS_NULLS_EQUAL: - stats_null_not_equal = FALSE; - break; - - default: - ut_error; - } - - if (srv_stats_sample_traditional) { - /* It makes no sense to test more pages than are contained - in the index, thus we lower the number if it is too high */ - if (srv_stats_transient_sample_pages > index->stat_index_size) { - if (index->stat_index_size > 0) { - n_sample_pages = index->stat_index_size; - } - } else { - n_sample_pages = srv_stats_transient_sample_pages; - } - } else { - /* New logaritmic number of pages that are estimated. - Number of pages estimated should be between 1 and - index->stat_index_size. - - If we have only 0 or 1 index pages then we can only take 1 - sample. We have already initialized n_sample_pages to 1. - - So taking index size as I and sample as S and log(I)*S as L - - requirement 1) we want the out limit of the expression to not exceed I; - requirement 2) we want the ideal pages to be at least S; - so the current expression is min(I, max( min(S,I), L) - - looking for simplifications: - - case 1: assume S < I - min(I, max( min(S,I), L) -> min(I , max( S, L)) - - but since L=LOG2(I)*S and log2(I) >=1 L>S always so max(S,L) = L. - - so we have: min(I , L) - - case 2: assume I < S - min(I, max( min(S,I), L) -> min(I, max( I, L)) - - case 2a: L > I - min(I, max( I, L)) -> min(I, L) -> I - - case 2b: when L < I - min(I, max( I, L)) -> min(I, I ) -> I - - so taking all case2 paths is I, our expression is: - n_pages = S < I? min(I,L) : I - */ - if (index->stat_index_size > 1) { - n_sample_pages = (srv_stats_transient_sample_pages < index->stat_index_size) - ? ut_min(index->stat_index_size, - static_cast( - log2(double(index->stat_index_size)) - * double(srv_stats_transient_sample_pages))) - : index->stat_index_size; - } - } - - /* Sanity check */ - ut_ad(n_sample_pages > 0 && n_sample_pages <= (index->stat_index_size <= 1 ? 1 : index->stat_index_size)); - - /* We sample some pages in the index to get an estimate */ - - for (i = 0; i < n_sample_pages; i++) { - mtr_start(&mtr); - - bool available; - - available = btr_cur_open_at_rnd_pos(index, BTR_SEARCH_LEAF, - &cursor, &mtr); - - if (!available) { - mtr_commit(&mtr); - mem_heap_free(heap); - - return result; - } - - /* Count the number of different key values for each prefix of - the key on this index page. If the prefix does not determine - the index record uniquely in the B-tree, then we subtract one - because otherwise our algorithm would give a wrong estimate - for an index where there is just one key value. */ - - if (!index->is_readable()) { - mtr_commit(&mtr); - goto exit_loop; - } - - page = btr_cur_get_page(&cursor); - - rec = page_rec_get_next(page_get_infimum_rec(page)); - const ulint n_core = page_is_leaf(page) - ? index->n_core_fields : 0; - - if (!page_rec_is_supremum(rec)) { - not_empty_flag = 1; - offsets_rec = rec_get_offsets(rec, index, offsets_rec, - n_core, - ULINT_UNDEFINED, &heap); - - if (n_not_null != NULL) { - btr_record_not_null_field_in_rec( - n_cols, offsets_rec, n_not_null); - } - } - - while (!page_rec_is_supremum(rec)) { - ulint matched_fields; - rec_t* next_rec = page_rec_get_next(rec); - if (page_rec_is_supremum(next_rec)) { - total_external_size += - btr_rec_get_externally_stored_len( - rec, offsets_rec); - break; - } - - offsets_next_rec = rec_get_offsets(next_rec, index, - offsets_next_rec, - n_core, - ULINT_UNDEFINED, - &heap); - - cmp_rec_rec(rec, next_rec, - offsets_rec, offsets_next_rec, - index, stats_null_not_equal, - &matched_fields); - - for (j = matched_fields; j < n_cols; j++) { - /* We add one if this index record has - a different prefix from the previous */ - - n_diff[j]++; - } - - if (n_not_null != NULL) { - btr_record_not_null_field_in_rec( - n_cols, offsets_next_rec, n_not_null); - } - - total_external_size - += btr_rec_get_externally_stored_len( - rec, offsets_rec); - - rec = next_rec; - /* Initialize offsets_rec for the next round - and assign the old offsets_rec buffer to - offsets_next_rec. */ - { - rec_offs* offsets_tmp = offsets_rec; - offsets_rec = offsets_next_rec; - offsets_next_rec = offsets_tmp; - } - } - - if (n_cols == dict_index_get_n_unique_in_tree(index) - && page_has_siblings(page)) { - - /* If there is more than one leaf page in the tree, - we add one because we know that the first record - on the page certainly had a different prefix than the - last record on the previous index page in the - alphabetical order. Before this fix, if there was - just one big record on each clustered index page, the - algorithm grossly underestimated the number of rows - in the table. */ - - n_diff[n_cols - 1]++; - } - - mtr_commit(&mtr); - } - -exit_loop: - /* If we saw k borders between different key values on - n_sample_pages leaf pages, we can estimate how many - there will be in index->stat_n_leaf_pages */ - - /* We must take into account that our sample actually represents - also the pages used for external storage of fields (those pages are - included in index->stat_n_leaf_pages) */ - - result.reserve(n_cols); - - for (j = 0; j < n_cols; j++) { - index_field_stats_t stat; - - stat.n_diff_key_vals - = BTR_TABLE_STATS_FROM_SAMPLE( - n_diff[j], index, n_sample_pages, - total_external_size, not_empty_flag); - - /* If the tree is small, smaller than - 10 * n_sample_pages + total_external_size, then - the above estimate is ok. For bigger trees it is common that we - do not see any borders between key values in the few pages - we pick. But still there may be n_sample_pages - different key values, or even more. Let us try to approximate - that: */ - - add_on = index->stat_n_leaf_pages - / (10 * (n_sample_pages - + total_external_size)); - - if (add_on > n_sample_pages) { - add_on = n_sample_pages; - } - - stat.n_diff_key_vals += add_on; - - stat.n_sample_sizes = n_sample_pages; - - if (n_not_null != NULL) { - stat.n_non_null_key_vals = - BTR_TABLE_STATS_FROM_SAMPLE( - n_not_null[j], index, n_sample_pages, - total_external_size, not_empty_flag); - } - - result.push_back(stat); - } - - mem_heap_free(heap); - - return result; -} - /*================== EXTERNAL STORAGE OF BIG FIELDS ===================*/ /***********************************************************//** diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 3187df9926a..fe156c14b2f 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -2229,6 +2229,15 @@ furious_flush: unemployed: buf_flush_async_lsn= 0; buf_pool.page_cleaner_set_idle(true); + + DBUG_EXECUTE_IF("ib_log_checkpoint_avoid", continue;); + + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + + if (!recv_recovery_is_on() && srv_operation == SRV_OPERATION_NORMAL) + log_checkpoint(); + + mysql_mutex_lock(&buf_pool.flush_list_mutex); continue; } diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index d7466ae5f8a..f92beac0f67 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1024,6 +1024,367 @@ dict_stats_snapshot_free( dict_stats_table_clone_free(t); } +/** Statistics for one field of an index. */ +struct index_field_stats_t +{ + ib_uint64_t n_diff_key_vals; + ib_uint64_t n_sample_sizes; + ib_uint64_t n_non_null_key_vals; + + index_field_stats_t(ib_uint64_t n_diff_key_vals= 0, + ib_uint64_t n_sample_sizes= 0, + ib_uint64_t n_non_null_key_vals= 0) + : n_diff_key_vals(n_diff_key_vals), n_sample_sizes(n_sample_sizes), + n_non_null_key_vals(n_non_null_key_vals) + { + } +}; + +/*******************************************************************//** +Record the number of non_null key values in a given index for +each n-column prefix of the index where 1 <= n <= dict_index_get_n_unique(index). +The estimates are eventually stored in the array: +index->stat_n_non_null_key_vals[], which is indexed from 0 to n-1. */ +static +void +btr_record_not_null_field_in_rec( +/*=============================*/ + ulint n_unique, /*!< in: dict_index_get_n_unique(index), + number of columns uniquely determine + an index entry */ + const rec_offs* offsets, /*!< in: rec_get_offsets(rec, index), + its size could be for all fields or + that of "n_unique" */ + ib_uint64_t* n_not_null) /*!< in/out: array to record number of + not null rows for n-column prefix */ +{ + ulint i; + + ut_ad(rec_offs_n_fields(offsets) >= n_unique); + + if (n_not_null == NULL) { + return; + } + + for (i = 0; i < n_unique; i++) { + if (rec_offs_nth_sql_null(offsets, i)) { + break; + } + + n_not_null[i]++; + } +} + +/** Estimated table level stats from sampled value. +@param value sampled stats +@param index index being sampled +@param sample number of sampled rows +@param ext_size external stored data size +@param not_empty table not empty +@return estimated table wide stats from sampled value */ +#define BTR_TABLE_STATS_FROM_SAMPLE(value, index, sample, ext_size, not_empty) \ + (((value) * static_cast(index->stat_n_leaf_pages) \ + + (sample) - 1 + (ext_size) + (not_empty)) / ((sample) + (ext_size))) + +/** Estimates the number of different key values in a given index, for +each n-column prefix of the index where 1 <= n <= dict_index_get_n_unique(index). +The estimates are stored in the array index->stat_n_diff_key_vals[] (indexed +0..n_uniq-1) and the number of pages that were sampled is saved in +result.n_sample_sizes[]. +If innodb_stats_method is nulls_ignored, we also record the number of +non-null values for each prefix and stored the estimates in +array result.n_non_null_key_vals. +@param index B-tree index +@param bulk_trx_id the value of index->table->bulk_trx_id at the start +@return vector with statistics information +empty vector if the index is unavailable. */ +static +std::vector +btr_estimate_number_of_different_key_vals(dict_index_t* index, + trx_id_t bulk_trx_id) +{ + btr_cur_t cursor; + page_t* page; + rec_t* rec; + ulint n_cols; + ib_uint64_t* n_diff; + ib_uint64_t* n_not_null; + ibool stats_null_not_equal; + uintmax_t n_sample_pages=1; /* number of pages to sample */ + ulint not_empty_flag = 0; + ulint total_external_size = 0; + ulint i; + ulint j; + uintmax_t add_on; + mtr_t mtr; + mem_heap_t* heap = NULL; + rec_offs* offsets_rec = NULL; + rec_offs* offsets_next_rec = NULL; + + std::vector result; + + ut_ad(!index->is_spatial()); + + n_cols = dict_index_get_n_unique(index); + + heap = mem_heap_create((sizeof *n_diff + sizeof *n_not_null) + * n_cols + + dict_index_get_n_fields(index) + * (sizeof *offsets_rec + + sizeof *offsets_next_rec)); + + n_diff = (ib_uint64_t*) mem_heap_zalloc( + heap, n_cols * sizeof(n_diff[0])); + + n_not_null = NULL; + + /* Check srv_innodb_stats_method setting, and decide whether we + need to record non-null value and also decide if NULL is + considered equal (by setting stats_null_not_equal value) */ + switch (srv_innodb_stats_method) { + case SRV_STATS_NULLS_IGNORED: + n_not_null = (ib_uint64_t*) mem_heap_zalloc( + heap, n_cols * sizeof *n_not_null); + /* fall through */ + + case SRV_STATS_NULLS_UNEQUAL: + /* for both SRV_STATS_NULLS_IGNORED and SRV_STATS_NULLS_UNEQUAL + case, we will treat NULLs as unequal value */ + stats_null_not_equal = TRUE; + break; + + case SRV_STATS_NULLS_EQUAL: + stats_null_not_equal = FALSE; + break; + + default: + ut_error; + } + + if (srv_stats_sample_traditional) { + /* It makes no sense to test more pages than are contained + in the index, thus we lower the number if it is too high */ + if (srv_stats_transient_sample_pages > index->stat_index_size) { + if (index->stat_index_size > 0) { + n_sample_pages = index->stat_index_size; + } + } else { + n_sample_pages = srv_stats_transient_sample_pages; + } + } else { + /* New logaritmic number of pages that are estimated. + Number of pages estimated should be between 1 and + index->stat_index_size. + + If we have only 0 or 1 index pages then we can only take 1 + sample. We have already initialized n_sample_pages to 1. + + So taking index size as I and sample as S and log(I)*S as L + + requirement 1) we want the out limit of the expression to not exceed I; + requirement 2) we want the ideal pages to be at least S; + so the current expression is min(I, max( min(S,I), L) + + looking for simplifications: + + case 1: assume S < I + min(I, max( min(S,I), L) -> min(I , max( S, L)) + + but since L=LOG2(I)*S and log2(I) >=1 L>S always so max(S,L) = L. + + so we have: min(I , L) + + case 2: assume I < S + min(I, max( min(S,I), L) -> min(I, max( I, L)) + + case 2a: L > I + min(I, max( I, L)) -> min(I, L) -> I + + case 2b: when L < I + min(I, max( I, L)) -> min(I, I ) -> I + + so taking all case2 paths is I, our expression is: + n_pages = S < I? min(I,L) : I + */ + if (index->stat_index_size > 1) { + n_sample_pages = (srv_stats_transient_sample_pages < index->stat_index_size) + ? ut_min(index->stat_index_size, + static_cast( + log2(double(index->stat_index_size)) + * double(srv_stats_transient_sample_pages))) + : index->stat_index_size; + } + } + + /* Sanity check */ + ut_ad(n_sample_pages > 0 && n_sample_pages <= (index->stat_index_size <= 1 ? 1 : index->stat_index_size)); + + /* We sample some pages in the index to get an estimate */ + + for (i = 0; i < n_sample_pages; i++) { + mtr.start(); + + bool available; + + available = btr_cur_open_at_rnd_pos(index, BTR_SEARCH_LEAF, + &cursor, &mtr); + + if (!available || index->table->bulk_trx_id != bulk_trx_id) { + mtr.commit(); + mem_heap_free(heap); + + return result; + } + + /* Count the number of different key values for each prefix of + the key on this index page. If the prefix does not determine + the index record uniquely in the B-tree, then we subtract one + because otherwise our algorithm would give a wrong estimate + for an index where there is just one key value. */ + + if (!index->is_readable()) { + mtr.commit(); + goto exit_loop; + } + + page = btr_cur_get_page(&cursor); + + rec = page_rec_get_next(page_get_infimum_rec(page)); + const ulint n_core = page_is_leaf(page) + ? index->n_core_fields : 0; + + if (!page_rec_is_supremum(rec)) { + not_empty_flag = 1; + offsets_rec = rec_get_offsets(rec, index, offsets_rec, + n_core, + ULINT_UNDEFINED, &heap); + + if (n_not_null != NULL) { + btr_record_not_null_field_in_rec( + n_cols, offsets_rec, n_not_null); + } + } + + while (!page_rec_is_supremum(rec)) { + ulint matched_fields; + rec_t* next_rec = page_rec_get_next(rec); + if (page_rec_is_supremum(next_rec)) { + total_external_size += + btr_rec_get_externally_stored_len( + rec, offsets_rec); + break; + } + + offsets_next_rec = rec_get_offsets(next_rec, index, + offsets_next_rec, + n_core, + ULINT_UNDEFINED, + &heap); + + cmp_rec_rec(rec, next_rec, + offsets_rec, offsets_next_rec, + index, stats_null_not_equal, + &matched_fields); + + for (j = matched_fields; j < n_cols; j++) { + /* We add one if this index record has + a different prefix from the previous */ + + n_diff[j]++; + } + + if (n_not_null != NULL) { + btr_record_not_null_field_in_rec( + n_cols, offsets_next_rec, n_not_null); + } + + total_external_size + += btr_rec_get_externally_stored_len( + rec, offsets_rec); + + rec = next_rec; + /* Initialize offsets_rec for the next round + and assign the old offsets_rec buffer to + offsets_next_rec. */ + { + rec_offs* offsets_tmp = offsets_rec; + offsets_rec = offsets_next_rec; + offsets_next_rec = offsets_tmp; + } + } + + if (n_cols == dict_index_get_n_unique_in_tree(index) + && page_has_siblings(page)) { + + /* If there is more than one leaf page in the tree, + we add one because we know that the first record + on the page certainly had a different prefix than the + last record on the previous index page in the + alphabetical order. Before this fix, if there was + just one big record on each clustered index page, the + algorithm grossly underestimated the number of rows + in the table. */ + + n_diff[n_cols - 1]++; + } + + mtr.commit(); + } + +exit_loop: + /* If we saw k borders between different key values on + n_sample_pages leaf pages, we can estimate how many + there will be in index->stat_n_leaf_pages */ + + /* We must take into account that our sample actually represents + also the pages used for external storage of fields (those pages are + included in index->stat_n_leaf_pages) */ + + result.reserve(n_cols); + + for (j = 0; j < n_cols; j++) { + index_field_stats_t stat; + + stat.n_diff_key_vals + = BTR_TABLE_STATS_FROM_SAMPLE( + n_diff[j], index, n_sample_pages, + total_external_size, not_empty_flag); + + /* If the tree is small, smaller than + 10 * n_sample_pages + total_external_size, then + the above estimate is ok. For bigger trees it is common that we + do not see any borders between key values in the few pages + we pick. But still there may be n_sample_pages + different key values, or even more. Let us try to approximate + that: */ + + add_on = index->stat_n_leaf_pages + / (10 * (n_sample_pages + + total_external_size)); + + if (add_on > n_sample_pages) { + add_on = n_sample_pages; + } + + stat.n_diff_key_vals += add_on; + + stat.n_sample_sizes = n_sample_pages; + + if (n_not_null != NULL) { + stat.n_non_null_key_vals = + BTR_TABLE_STATS_FROM_SAMPLE( + n_not_null[j], index, n_sample_pages, + total_external_size, not_empty_flag); + } + + result.push_back(stat); + } + + mem_heap_free(heap); + + return result; +} + /*********************************************************************//** Calculates new estimates for index statistics. This function is relatively quick and is used to calculate transient statistics that @@ -1054,39 +1415,48 @@ dummy_empty: } else if (ibuf_debug && !dict_index_is_clust(index)) { goto dummy_empty; #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ + } else if (dict_index_is_online_ddl(index) || !index->is_committed() + || !index->table->space) { + goto dummy_empty; } else { mtr_t mtr; - ulint size; mtr.start(); mtr_s_lock_index(index, &mtr); - size = btr_get_size(index, BTR_TOTAL_SIZE, &mtr); - - if (size != ULINT_UNDEFINED) { - index->stat_index_size = size; - - size = btr_get_size( - index, BTR_N_LEAF_PAGES, &mtr); + buf_block_t* root = btr_root_block_get(index, RW_SX_LATCH, + &mtr); + if (!root) { +invalid: + mtr.commit(); + goto dummy_empty; } + const auto bulk_trx_id = index->table->bulk_trx_id; + if (bulk_trx_id && trx_sys.find(nullptr, bulk_trx_id, false)) { + goto invalid; + } + + mtr.x_lock_space(index->table->space); + + ulint dummy, size; + index->stat_index_size + = fseg_n_reserved_pages(*root, PAGE_HEADER + + PAGE_BTR_SEG_LEAF + + root->frame, &size, &mtr) + + fseg_n_reserved_pages(*root, PAGE_HEADER + + PAGE_BTR_SEG_TOP + + root->frame, &dummy, &mtr); + mtr.commit(); - switch (size) { - case ULINT_UNDEFINED: - goto dummy_empty; - case 0: - /* The root node of the tree is a leaf */ - size = 1; - } - - index->stat_n_leaf_pages = size; + index->stat_n_leaf_pages = size ? size : 1; /* Do not continue if table decryption has failed or table is already marked as corrupted. */ if (index->is_readable()) { std::vector stats = btr_estimate_number_of_different_key_vals( - index); + index, bulk_trx_id); if (!stats.empty()) { index->table->stats_mutex_lock(); @@ -2037,7 +2407,7 @@ struct index_stats_t { stats.reserve(n_uniq); for (ulint i= 0; i < n_uniq; ++i) - stats.push_back(index_field_stats_t(0, 1, 0)); + stats.push_back(index_field_stats_t{0, 1, 0}); } }; @@ -2123,15 +2493,12 @@ stat_n_leaf_pages. This function can be slow. @return index stats */ static index_stats_t dict_stats_analyze_index(dict_index_t* index) { - ulint root_level; - ulint level; bool level_is_analyzed; ulint n_uniq; ulint n_prefix; ib_uint64_t total_recs; ib_uint64_t total_pages; mtr_t mtr; - ulint size; index_stats_t result(index->n_uniq); DBUG_ENTER("dict_stats_analyze_index"); @@ -2150,30 +2517,43 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) mtr.start(); mtr_s_lock_index(index, &mtr); - size = btr_get_size(index, BTR_TOTAL_SIZE, &mtr); + uint16_t root_level; - if (size != ULINT_UNDEFINED) { - result.index_size = size; - size = btr_get_size(index, BTR_N_LEAF_PAGES, &mtr); + { + buf_block_t* root; + root = btr_root_block_get(index, RW_SX_LATCH, &mtr); + if (!root) { +empty_index: + mtr.commit(); + dict_stats_assert_initialized_index(index); + DBUG_RETURN(result); + } + + root_level = btr_page_get_level(root->frame); + + mtr.x_lock_space(index->table->space); + ulint dummy, size; + result.index_size + = fseg_n_reserved_pages(*root, PAGE_HEADER + + PAGE_BTR_SEG_LEAF + + root->frame, &size, &mtr) + + fseg_n_reserved_pages(*root, PAGE_HEADER + + PAGE_BTR_SEG_TOP + + root->frame, &dummy, &mtr); + result.n_leaf_pages = size ? size : 1; + } + + const auto bulk_trx_id = index->table->bulk_trx_id; + if (bulk_trx_id && trx_sys.find(nullptr, bulk_trx_id, false)) { + result.index_size = 1; + result.n_leaf_pages = 1; + goto empty_index; } - /* Release the X locks on the root page taken by btr_get_size() */ mtr.commit(); - switch (size) { - case ULINT_UNDEFINED: - dict_stats_assert_initialized_index(index); - DBUG_RETURN(result); - case 0: - /* The root node of the tree is a leaf */ - size = 1; - } - - result.n_leaf_pages = size; - mtr.start(); mtr_sx_lock_index(index, &mtr); - root_level = btr_height_get(index, &mtr); n_uniq = dict_index_get_n_unique(index); @@ -2251,7 +2631,7 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) So if we find that the first level containing D distinct keys (on n_prefix columns) is L, we continue from L when searching for D distinct keys on n_prefix-1 columns. */ - level = root_level; + auto level = root_level; level_is_analyzed = false; for (n_prefix = n_uniq; n_prefix >= 1; n_prefix--) { @@ -2265,7 +2645,10 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) mtr.commit(); mtr.start(); mtr_sx_lock_index(index, &mtr); - if (root_level != btr_height_get(index, &mtr)) { + buf_block_t *root = btr_root_block_get(index, RW_S_LATCH, + &mtr); + if (!root || root_level != btr_page_get_level(root->frame) + || index->table->bulk_trx_id != bulk_trx_id) { /* Just quit if the tree has changed beyond recognition here. The old stats from previous runs will remain in the values that we have @@ -2277,6 +2660,8 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) break; } + mtr.memo_release(root, MTR_MEMO_PAGE_S_FIX); + /* check whether we should pick the current level; we pick level 1 even if it does not have enough distinct records because we do not want to scan the diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index bd879c7c275..188643b3a2f 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -228,6 +228,14 @@ static void innodb_max_purge_lag_wait_update(THD *thd, st_mysql_sys_var *, { if (thd_kill_level(thd)) break; + /* Adjust for purge_coordinator_state::refresh() */ + mysql_mutex_lock(&log_sys.mutex); + const lsn_t last= log_sys.last_checkpoint_lsn, + max_age= log_sys.max_checkpoint_age; + mysql_mutex_unlock(&log_sys.mutex); + const lsn_t lsn= log_sys.get_lsn(); + if ((lsn - last) / 4 >= max_age / 5) + buf_flush_ahead(last + max_age / 5, false); srv_wake_purge_thread_if_not_active(); std::this_thread::sleep_for(std::chrono::milliseconds(100)); } diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index 65608552986..40293df21e5 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -187,16 +187,6 @@ void btr_corruption_report(const buf_block_t* block,const dict_index_t* index); != index->table->not_redundant()) \ btr_corruption_report(block, index) -/**************************************************************//** -Gets the root node of a tree and sx-latches it for segment access. -@return root page, sx-latched */ -page_t* -btr_root_get( -/*=========*/ - const dict_index_t* index, /*!< in: index tree */ - mtr_t* mtr) /*!< in: mtr */ - MY_ATTRIBUTE((nonnull)); - /**************************************************************//** Checks and adjusts the root node of a tree during IMPORT TABLESPACE. @return error code, or DB_SUCCESS */ @@ -206,18 +196,6 @@ btr_root_adjust_on_import( const dict_index_t* index) /*!< in: index tree */ MY_ATTRIBUTE((warn_unused_result)); -/**************************************************************//** -Gets the height of the B-tree (the level of the root, when the leaf -level is assumed to be 0). The caller must hold an S or X latch on -the index. -@return tree height (level of the root) */ -ulint -btr_height_get( -/*===========*/ - const dict_index_t* index, /*!< in: index tree */ - mtr_t* mtr) /*!< in/out: mini-transaction */ - MY_ATTRIBUTE((warn_unused_result)); - /** Get an index page and declare its latching order level. @param[in] index index tree @param[in] page page number @@ -547,17 +525,6 @@ btr_discard_page( btr_cur_t* cursor, /*!< in: cursor on the page to discard: not on the root page */ mtr_t* mtr); /*!< in: mtr */ -/**************************************************************//** -Gets the number of pages in a B-tree. -@return number of pages, or ULINT_UNDEFINED if the index is unavailable */ -ulint -btr_get_size( -/*=========*/ - const dict_index_t* index, /*!< in: index */ - ulint flag, /*!< in: BTR_N_LEAF_PAGES or BTR_TOTAL_SIZE */ - mtr_t* mtr) /*!< in/out: mini-transaction where index - is s-latched */ - MY_ATTRIBUTE((warn_unused_result)); /**************************************************************//** Allocates a new file page to be used in an index tree. NOTE: we assume @@ -624,7 +591,6 @@ btr_root_block_get( rw_lock_type_t mode, /*!< in: either RW_S_LATCH or RW_X_LATCH */ mtr_t* mtr); /*!< in: mtr */ - /*************************************************************//** Reorganizes an index page. diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index c7f25aff4b7..911c79c29ba 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -575,37 +575,6 @@ btr_estimate_n_rows_in_range( btr_pos_t* range_start, btr_pos_t* range_end); - -/** Statistics for one field of an index. */ -struct index_field_stats_t -{ - ib_uint64_t n_diff_key_vals; - ib_uint64_t n_sample_sizes; - ib_uint64_t n_non_null_key_vals; - - index_field_stats_t(ib_uint64_t n_diff_key_vals= 0, - ib_uint64_t n_sample_sizes= 0, - ib_uint64_t n_non_null_key_vals= 0) - : n_diff_key_vals(n_diff_key_vals), n_sample_sizes(n_sample_sizes), - n_non_null_key_vals(n_non_null_key_vals) - { - } -}; - -/** Estimates the number of different key values in a given index, for -each n-column prefix of the index where 1 <= n <= dict_index_get_n_unique(index). -The estimates are stored in the array index->stat_n_diff_key_vals[] (indexed -0..n_uniq-1) and the number of pages that were sampled is saved in -index->stat_n_sample_sizes[]. -If innodb_stats_method is nulls_ignored, we also record the number of -non-null values for each prefix and stored the estimates in -array index->stat_n_non_null_key_vals. -@param[in] index index -@return stat vector if the index is available and we get the estimated numbers, -empty vector if the index is unavailable. */ -std::vector -btr_estimate_number_of_different_key_vals(dict_index_t* index); - /** Gets the externally stored size of a record, in units of a database page. @param[in] rec record @param[in] offsets array returned by rec_get_offsets() diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index bfc4520150e..d7e10405853 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1722,10 +1722,15 @@ public: /** Hash table with singly-linked overflow lists. @see hash_table_t */ struct page_hash_table { + static_assert(CPU_LEVEL1_DCACHE_LINESIZE >= 64, "less than 64 bytes"); + static_assert(!(CPU_LEVEL1_DCACHE_LINESIZE & 63), + "not a multiple of 64 bytes"); + /** Number of array[] elements per page_hash_latch. Must be one less than a power of 2. */ - static constexpr size_t ELEMENTS_PER_LATCH= CPU_LEVEL1_DCACHE_LINESIZE / - sizeof(void*) - 1; + static constexpr size_t ELEMENTS_PER_LATCH= 64 / sizeof(void*) - 1; + static constexpr size_t EMPTY_SLOTS_PER_LATCH= + ((CPU_LEVEL1_DCACHE_LINESIZE / 64) - 1) * (64 / sizeof(void*)); /** number of payload elements in array[] */ Atomic_relaxed n_cells; @@ -1742,7 +1747,12 @@ public: /** @return the index of an array element */ ulint calc_hash(ulint fold) const { return calc_hash(fold, n_cells); } /** @return raw array index converted to padded index */ - static ulint pad(ulint h) { return 1 + (h / ELEMENTS_PER_LATCH) + h; } + static ulint pad(ulint h) + { + ulint latches= h / ELEMENTS_PER_LATCH; + ulint empty_slots= latches * EMPTY_SLOTS_PER_LATCH; + return 1 + latches + empty_slots + h; + } private: /** @return the hash value before any ELEMENTS_PER_LATCH padding */ static ulint hash(ulint fold, ulint n) { return ut_hash_ulint(fold, n); } diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index f1be99e81f1..f0aa41d2e5b 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -2462,7 +2462,7 @@ inline void dict_index_t::clear_instant_alter() { return a.col->ind < b.col->ind; }); table->instant = NULL; if (ai_col) { - auto a = std::find_if(begin, end, + auto a = std::find_if(fields, end, [ai_col](const dict_field_t& f) { return f.col == ai_col; }); table->persistent_autoinc = (a == end) diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index 859441afcc0..c2fef3baaac 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -609,8 +609,9 @@ public: /** Number of array[] elements per hash_latch. Must be LATCH less than a power of 2. */ - static constexpr size_t ELEMENTS_PER_LATCH= CPU_LEVEL1_DCACHE_LINESIZE / - sizeof(void*) - LATCH; + static constexpr size_t ELEMENTS_PER_LATCH= (64 / sizeof(void*)) - LATCH; + static constexpr size_t EMPTY_SLOTS_PER_LATCH= + ((CPU_LEVEL1_DCACHE_LINESIZE / 64) - 1) * (64 / sizeof(void*)); /** number of payload elements in array[]. Protected by lock_sys.latch. */ ulint n_cells; @@ -632,9 +633,15 @@ public: /** @return the index of an array element */ inline ulint calc_hash(ulint fold) const; + /** @return raw array index converted to padded index */ static ulint pad(ulint h) - { return LATCH + LATCH * (h / ELEMENTS_PER_LATCH) + h; } + { + ulint latches= LATCH * (h / ELEMENTS_PER_LATCH); + ulint empty_slots= (h / ELEMENTS_PER_LATCH) * EMPTY_SLOTS_PER_LATCH; + return LATCH + latches + empty_slots + h; + } + /** Get a latch. */ static hash_latch *latch(hash_cell_t *cell) { diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 29ceb0dabc1..010ec224460 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -115,8 +115,7 @@ row_purge_remove_clust_if_poss_low( retry: purge_sys.check_stop_FTS(); dict_sys.lock(SRW_LOCK_CALL); - table = dict_table_open_on_id( - table_id, true, DICT_TABLE_OP_OPEN_ONLY_IF_CACHED); + table = dict_sys.find_table(table_id); if (!table) { dict_sys.unlock(); } else if (table->n_rec_locks) { @@ -141,7 +140,6 @@ removed: mtr.commit(); close_and_exit: if (table) { - dict_table_close(table, true); dict_sys.unlock(); } return success; @@ -166,7 +164,7 @@ close_and_exit: if (const uint32_t space_id = dict_drop_index_tree( &node->pcur, nullptr, &mtr)) { if (table) { - if (table->release()) { + if (table->get_ref_count() == 0) { dict_sys.remove(table); } else if (table->space_id == space_id) { table->space = nullptr; @@ -181,7 +179,6 @@ close_and_exit: mtr.commit(); if (table) { - dict_table_close(table, true); dict_sys.unlock(); table = nullptr; } @@ -917,7 +914,7 @@ skip_secondaries: index->set_modified(mtr); - /* NOTE: we must also acquire an X-latch to the + /* NOTE: we must also acquire a U latch to the root page of the tree. We will need it when we free pages from the tree. If the tree is of height 1, the tree X-latch does NOT protect the root page, @@ -926,7 +923,7 @@ skip_secondaries: latching order if we would only later latch the root page of such a tree! */ - btr_root_get(index, &mtr); + btr_root_block_get(index, RW_SX_LATCH, &mtr); block = buf_page_get( page_id_t(rseg.space->id, page_no), @@ -1038,12 +1035,6 @@ try_again: goto err_exit; } - /* FIXME: We are acquiring exclusive dict_sys.latch only to - avoid increased wait times in - trx_purge_get_next_rec() and trx_purge_truncate_history(). */ - dict_sys.lock(SRW_LOCK_CALL); - dict_sys.unlock(); - already_locked: ut_ad(!node->table->is_temporary()); diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 088b98eba38..7416bbd415d 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -528,7 +528,26 @@ struct purge_coordinator_state /** Snapshot of the last history length before the purge call.*/ uint32 m_history_length; Atomic_counter m_running; - purge_coordinator_state() : m_history_length(), m_running(0) {} +private: + ulint count; + ulint n_use_threads; + ulint n_threads; + + ulint lsn_lwm; + ulint lsn_hwm; + ulonglong start_time; + ulint lsn_age_factor; + + static constexpr ulint adaptive_purge_threshold= 20; + static constexpr ulint safety_net= 20; + ulint series[innodb_purge_threads_MAX + 1]; + + inline void compute_series(); + inline void lazy_init(); + void refresh(bool full); + +public: + inline void do_purge(); }; static purge_coordinator_state purge_state; @@ -1329,7 +1348,6 @@ bool srv_any_background_activity() static void purge_worker_callback(void*); static void purge_coordinator_callback(void*); -static void purge_coordinator_timer_callback(void*); static tpool::task_group purge_task_group; tpool::waitable_task purge_worker_task(purge_worker_callback, nullptr, @@ -1621,24 +1639,25 @@ void srv_shutdown(bool ibuf_merge) /** The periodic master task controlling the server. */ void srv_master_callback(void*) { - static ulint old_activity_count; + static ulint old_activity_count; - ut_a(srv_shutdown_state <= SRV_SHUTDOWN_INITIATED); + ut_a(srv_shutdown_state <= SRV_SHUTDOWN_INITIATED); - MONITOR_INC(MONITOR_MASTER_THREAD_SLEEP); - ut_d(srv_master_do_disabled_loop()); - purge_coordinator_timer_callback(nullptr); - ulonglong counter_time = microsecond_interval_timer(); - srv_sync_log_buffer_in_background(); - MONITOR_INC_TIME_IN_MICRO_SECS( - MONITOR_SRV_LOG_FLUSH_MICROSECOND, counter_time); + MONITOR_INC(MONITOR_MASTER_THREAD_SLEEP); + ut_d(srv_master_do_disabled_loop()); + if (!purge_state.m_running) + srv_wake_purge_thread_if_not_active(); + ulonglong counter_time= microsecond_interval_timer(); + srv_sync_log_buffer_in_background(); + MONITOR_INC_TIME_IN_MICRO_SECS(MONITOR_SRV_LOG_FLUSH_MICROSECOND, + counter_time); - if (srv_check_activity(&old_activity_count)) { - srv_master_do_active_tasks(counter_time); - } else { - srv_master_do_idle_tasks(counter_time); - } - srv_main_thread_op_info = "sleeping"; + if (srv_check_activity(&old_activity_count)) + srv_master_do_active_tasks(counter_time); + else + srv_master_do_idle_tasks(counter_time); + + srv_main_thread_op_info= "sleeping"; } /** @return whether purge should exit due to shutdown */ @@ -1711,86 +1730,173 @@ void srv_update_purge_thread_count(uint n) Atomic_counter srv_purge_thread_count_changed; -/** Do the actual purge operation. -@param[in,out] n_total_purged total number of purged pages -@return length of history list before the last purge batch. */ -static uint32_t srv_do_purge(ulint* n_total_purged) +inline void purge_coordinator_state::do_purge() { - ulint n_pages_purged; + ut_ad(!srv_read_only_mode); + lazy_init(); + ut_ad(n_threads); + bool wakeup= false; - static ulint count = 0; - static ulint n_use_threads = 0; - static uint32_t rseg_history_len = 0; - ulint old_activity_count = srv_get_activity_count(); - static ulint n_threads = srv_n_purge_threads; + purge_coordinator_timer->disarm(); - ut_a(n_threads > 0); - ut_ad(!srv_read_only_mode); + while (purge_sys.enabled() && !purge_sys.paused()) + { +loop: + wakeup= false; + const auto now= my_interval_timer(); + const auto sigcount= m_running; - /* Purge until there are no more records to purge and there is - no change in configuration or server state. If the user has - configured more than one purge thread then we treat that as a - pool of threads and only use the extra threads if purge can't - keep up with updates. */ + if (now - start_time >= 1000000) + { + refresh(false); + start_time= now; + } - if (n_use_threads == 0) { - n_use_threads = n_threads; - } + const auto old_activity_count= srv_sys.activity_count; + const auto history_size= trx_sys.history_size(); - do { - if (UNIV_UNLIKELY(srv_purge_thread_count_changed)) { - /* Read the fresh value of srv_n_purge_threads, reset - the changed flag. Both variables are protected by - purge_thread_count_mtx. + if (UNIV_UNLIKELY(srv_purge_thread_count_changed)) + { + /* Read the fresh value of srv_n_purge_threads, reset + the changed flag. Both are protected by purge_thread_count_mtx. - This code does not run concurrently, it is executed - by a single purge_coordinator thread, and no races - involving srv_purge_thread_count_changed are possible. - */ + This code does not run concurrently, it is executed + by a single purge_coordinator thread, and no races + involving srv_purge_thread_count_changed are possible. */ + { + std::lock_guard lk(purge_thread_count_mtx); + n_threads= n_use_threads= srv_n_purge_threads; + srv_purge_thread_count_changed= 0; + } + refresh(true); + start_time= now; + } + else if (history_size > m_history_length) + { + /* dynamically adjust the purge thread based on redo log fill factor */ + if (n_use_threads < n_threads && lsn_age_factor < lsn_lwm) + { +more_threads: + ++n_use_threads; + lsn_hwm= lsn_lwm; + lsn_lwm-= series[n_use_threads]; + } + else if (n_use_threads > 1 && lsn_age_factor >= lsn_hwm) + { +fewer_threads: + --n_use_threads; + lsn_lwm= lsn_hwm; + lsn_hwm+= series[n_use_threads]; + } + else if (n_use_threads == 1 && lsn_age_factor >= 100 - safety_net) + { + wakeup= true; + break; + } + } + else if (n_threads > n_use_threads && + srv_max_purge_lag && m_history_length > srv_max_purge_lag) + goto more_threads; + else if (n_use_threads > 1 && old_activity_count == srv_sys.activity_count) + goto fewer_threads; - std::lock_guard lk(purge_thread_count_mtx); - n_threads = n_use_threads = srv_n_purge_threads; - srv_purge_thread_count_changed = 0; - } else if (trx_sys.history_size_approx() > rseg_history_len - || (srv_max_purge_lag > 0 - && rseg_history_len > srv_max_purge_lag)) { - /* History length is now longer than what it was - when we took the last snapshot. Use more threads. */ + ut_ad(n_use_threads); + ut_ad(n_use_threads <= n_threads); - if (n_use_threads < n_threads) { - ++n_use_threads; - } + m_history_length= history_size; - } else if (srv_check_activity(&old_activity_count) - && n_use_threads > 1) { + if (history_size && + trx_purge(n_use_threads, + !(++count % srv_purge_rseg_truncate_frequency) || + purge_sys.truncate.current)) + continue; - /* History length same or smaller since last snapshot, - use fewer threads. */ + if (m_running == sigcount) + { + /* Purge was not woken up by srv_wake_purge_thread_if_not_active() */ - --n_use_threads; - } + /* The magic number 5000 is an approximation for the case where we have + cached undo log records which prevent truncate of rollback segments. */ + wakeup= history_size && + (history_size >= 5000 || + history_size != trx_sys.history_size_approx()); + break; + } + else if (!trx_sys.history_exists()) + break; - /* Ensure that the purge threads are less than what - was configured. */ + if (!srv_purge_should_exit()) + goto loop; + } - ut_a(n_use_threads > 0); - ut_a(n_use_threads <= n_threads); + if (wakeup) + purge_coordinator_timer->set_time(10, 0); - /* Take a snapshot of the history list before purge. */ - if (!(rseg_history_len = trx_sys.history_size())) { - break; - } + m_running= 0; +} - n_pages_purged = trx_purge( - n_use_threads, - !(++count % srv_purge_rseg_truncate_frequency) - || purge_sys.truncate.current); +inline void purge_coordinator_state::compute_series() +{ + ulint points= n_threads; + memset(series, 0, sizeof series); + constexpr ulint spread= 100 - adaptive_purge_threshold - safety_net; - *n_total_purged += n_pages_purged; - } while (n_pages_purged > 0 && !purge_sys.paused() - && !srv_purge_should_exit()); + /* We distribute spread across n_threads, + e.g.: spread of 60 is distributed across n_threads=4 as: 6+12+18+24 */ - return(rseg_history_len); + const ulint additional_points= (points * (points + 1)) / 2; + if (spread % additional_points == 0) + { + /* Arithmetic progression is possible. */ + const ulint delta= spread / additional_points; + ulint growth= delta; + do + { + series[points--]= growth; + growth += delta; + } + while (points); + return; + } + + /* Use average distribution to spread across the points */ + const ulint delta= spread / points; + ulint total= 0; + do + { + series[points--]= delta; + total+= delta; + } + while (points); + + for (points= 1; points <= n_threads && total++ < spread; ) + series[points++]++; +} + +inline void purge_coordinator_state::lazy_init() +{ + if (n_threads) + return; + n_threads= n_use_threads= srv_n_purge_threads; + refresh(true); + start_time= my_interval_timer(); +} + +void purge_coordinator_state::refresh(bool full) +{ + if (full) + { + compute_series(); + lsn_lwm= adaptive_purge_threshold; + lsn_hwm= adaptive_purge_threshold + series[n_threads]; + } + + mysql_mutex_lock(&log_sys.mutex); + const lsn_t last= log_sys.last_checkpoint_lsn, + max_age= log_sys.max_checkpoint_age; + mysql_mutex_unlock(&log_sys.mutex); + + lsn_age_factor= ulint(((log_sys.get_lsn() - last) * 100) / max_age); } @@ -1836,24 +1942,6 @@ static void release_thd(THD *thd, void *ctx) set_current_thd(0); } - -/** - Called by timer when purge coordinator decides - to delay processing of purge records. -*/ -static void purge_coordinator_timer_callback(void *) -{ - if (!purge_sys.enabled() || purge_sys.paused() || purge_state.m_running) - return; - - /* The magic number 5000 is an approximation for the case where we have - cached undo log records which prevent truncate of the rollback segments. */ - if (const auto history_size= trx_sys.history_size()) - if (purge_state.m_history_length >= 5000 || - purge_state.m_history_length != history_size) - srv_wake_purge_thread_if_not_active(); -} - static void purge_worker_callback(void*) { ut_ad(!current_thd); @@ -1866,57 +1954,19 @@ static void purge_worker_callback(void*) release_thd(thd,ctx); } -static void purge_coordinator_callback_low() -{ - ulint n_total_purged= ULINT_UNDEFINED; - purge_state.m_history_length= 0; - - if (!purge_sys.enabled() || purge_sys.paused()) - return; - do - { - n_total_purged = 0; - int sigcount= purge_state.m_running; - - purge_state.m_history_length= srv_do_purge(&n_total_purged); - - /* Check if purge was woken by srv_wake_purge_thread_if_not_active() */ - - bool woken_during_purge= purge_state.m_running > sigcount; - - /* If last purge batch processed less than 1 page and there is - still work to do, delay the next batch by 10ms. Unless - someone added work and woke us up. */ - if (n_total_purged == 0) - { - if (trx_sys.history_size() == 0) - return; - if (!woken_during_purge) - { - /* Delay next purge round*/ - purge_coordinator_timer->set_time(10, 0); - return; - } - } - } - while ((purge_sys.enabled() && !purge_sys.paused()) || - !srv_purge_should_exit()); -} - static void purge_coordinator_callback(void*) { void *ctx; THD *thd= acquire_thd(&ctx); - purge_coordinator_callback_low(); - release_thd(thd,ctx); - purge_state.m_running= 0; + purge_state.do_purge(); + release_thd(thd, ctx); } void srv_init_purge_tasks() { purge_create_background_thds(srv_n_purge_threads); purge_coordinator_timer= srv_thread_pool->create_timer - (purge_coordinator_timer_callback, nullptr); + (purge_coordinator_callback, nullptr); } static void srv_shutdown_purge_tasks() diff --git a/storage/maria/aria_chk.c b/storage/maria/aria_chk.c index 728b574c98c..266b11d99f5 100644 --- a/storage/maria/aria_chk.c +++ b/storage/maria/aria_chk.c @@ -434,8 +434,8 @@ static struct my_option my_long_options[] = ~0UL, (long) MALLOC_OVERHEAD, (long) 1L, 0}, { "sort_buffer_size", OPT_SORT_BUFFER_SIZE, "Size of sort buffer. Used by --recover", - &check_param.sort_buffer_length, - &check_param.sort_buffer_length, 0, GET_ULL, REQUIRED_ARG, + &check_param.orig_sort_buffer_length, + &check_param.orig_sort_buffer_length, 0, GET_ULL, REQUIRED_ARG, SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, MALLOC_OVERHEAD, 1L, 0}, { "sort_key_blocks", OPT_SORT_KEY_BLOCKS, "Internal buffer for sorting keys; Don't touch :)", diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 38db0042a83..0cec0f088dd 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -1489,7 +1489,7 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt) param->testflag= ((check_opt->flags & ~(T_EXTEND)) | T_SILENT | T_FORCE_CREATE | T_CALC_CHECKSUM | (check_opt->flags & T_EXTEND ? T_REP : T_REP_BY_SORT)); - param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + param->orig_sort_buffer_length= THDVAR(thd, sort_buffer_size); param->backup_time= check_opt->start_time; start_records= file->state->records; old_proc_info= thd_proc_info(thd, "Checking table"); @@ -1560,7 +1560,7 @@ int ha_maria::zerofill(THD * thd, HA_CHECK_OPT *check_opt) param->thd= thd; param->op_name= "zerofill"; param->testflag= check_opt->flags | T_SILENT | T_ZEROFILL; - param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + param->orig_sort_buffer_length= THDVAR(thd, sort_buffer_size); param->db_name= table->s->db.str; param->table_name= table->alias.c_ptr(); @@ -1596,7 +1596,7 @@ int ha_maria::optimize(THD * thd, HA_CHECK_OPT *check_opt) param->op_name= "optimize"; param->testflag= (check_opt->flags | T_SILENT | T_FORCE_CREATE | T_REP_BY_SORT | T_STATISTICS | T_SORT_INDEX); - param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + param->orig_sort_buffer_length= THDVAR(thd, sort_buffer_size); thd_progress_init(thd, 1); if ((error= repair(thd, param, 1)) && param->retry_repair) { @@ -2064,7 +2064,7 @@ int ha_maria::enable_indexes(uint mode) } param->myf_rw &= ~MY_WAIT_IF_FULL; - param->sort_buffer_length= THDVAR(thd,sort_buffer_size); + param->orig_sort_buffer_length= THDVAR(thd,sort_buffer_size); param->stats_method= (enum_handler_stats_method)THDVAR(thd,stats_method); param->tmpdir= &mysql_tmpdir_list; if ((error= (repair(thd, param, 0) != HA_ADMIN_OK)) && param->retry_repair) @@ -2913,7 +2913,7 @@ int ha_maria::external_lock(THD *thd, int lock_type) if (file->autocommit) { if (ma_commit(trn)) - result= HA_ERR_INTERNAL_ERROR; + result= HA_ERR_COMMIT_ERROR; thd_set_ha_data(thd, maria_hton, 0); } } @@ -3052,7 +3052,7 @@ int ha_maria::implicit_commit(THD *thd, bool new_trn) error= 0; if (unlikely(ma_commit(trn))) - error= 1; + error= HA_ERR_COMMIT_ERROR; if (!new_trn) { reset_thd_trn(thd, used_tables); @@ -3489,7 +3489,7 @@ static int maria_commit(handlerton *hton __attribute__ ((unused)), THD *thd, bool all) { TRN *trn= THD_TRN; - int res; + int res= 0; MARIA_HA *used_instances; DBUG_ENTER("maria_commit"); @@ -3508,7 +3508,8 @@ static int maria_commit(handlerton *hton __attribute__ ((unused)), trnman_reset_locked_tables(trn, 0); trnman_set_flags(trn, trnman_get_flags(trn) & ~TRN_STATE_INFO_LOGGED); trn->used_instances= 0; - res= ma_commit(trn); + if (ma_commit(trn)) + res= HA_ERR_COMMIT_ERROR; reset_thd_trn(thd, used_instances); thd_set_ha_data(thd, maria_hton, 0); DBUG_RETURN(res); diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index c097c6d1e2e..20baa429d76 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -114,7 +114,7 @@ void maria_chk_init(HA_CHECK *param) param->use_buffers= PAGE_BUFFER_INIT; param->read_buffer_length=READ_BUFFER_INIT; param->write_buffer_length=READ_BUFFER_INIT; - param->sort_buffer_length=SORT_BUFFER_INIT; + param->orig_sort_buffer_length=SORT_BUFFER_INIT; param->sort_key_blocks=BUFFERS_WHEN_SORTING; param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL; param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL); @@ -2485,6 +2485,8 @@ static int initialize_variables_for_repair(HA_CHECK *param, tmp= (size_t) MY_MIN(sort_info->filelength, (my_off_t) (SIZE_T_MAX/10/threads)); tmp= MY_MAX(tmp * 8 * threads, (size_t) 65536); /* Some margin */ + param->sort_buffer_length= MY_MIN(param->orig_sort_buffer_length, + tmp); set_if_smaller(param->sort_buffer_length, tmp); /* Protect against too big sort buffer length */ #if SIZEOF_SIZE_T >= 8 diff --git a/storage/maria/ma_checkpoint.c b/storage/maria/ma_checkpoint.c index 3af808478e4..2741f54d7d7 100644 --- a/storage/maria/ma_checkpoint.c +++ b/storage/maria/ma_checkpoint.c @@ -153,8 +153,10 @@ end: static int really_execute_checkpoint(void) { uint i, error= 0; + int error_errno= 0; /** @brief checkpoint_start_log_horizon will be stored there */ char *ptr; + const char *error_place= 0; LEX_STRING record_pieces[4]; /**< only malloc-ed pieces */ LSN min_page_rec_lsn, min_trn_rec_lsn, min_first_undo_lsn; TRANSLOG_ADDRESS checkpoint_start_log_horizon; @@ -191,13 +193,19 @@ static int really_execute_checkpoint(void) &record_pieces[1], &min_trn_rec_lsn, &min_first_undo_lsn))) + { + error_place= "trnman_collect_transaction"; goto err; + } /* STEP 3: fetch information about table files */ if (unlikely(collect_tables(&record_pieces[2], checkpoint_start_log_horizon))) + { + error_place= "collect_tables"; goto err; + } /* STEP 4: fetch information about dirty pages */ @@ -211,7 +219,10 @@ static int really_execute_checkpoint(void) if (unlikely(pagecache_collect_changed_blocks_with_lsn(maria_pagecache, &record_pieces[3], &min_page_rec_lsn))) + { + error_place= "collect_pages"; goto err; + } /* LAST STEP: now write the checkpoint log record */ @@ -240,7 +251,10 @@ static int really_execute_checkpoint(void) sizeof(log_array)/sizeof(log_array[0]), log_array, NULL, NULL) || translog_flush(lsn))) + { + error_place= "translog_write_record"; goto err; + } translog_lock(); /* This cannot be done as a inwrite_rec_hook of LOGREC_CHECKPOINT, because @@ -251,6 +265,8 @@ static int really_execute_checkpoint(void) max_trid_in_control_file, recovery_failures))) { + error_place= "ma_control_file_write"; + error_errno= my_errno; translog_unlock(); goto err; } @@ -287,7 +303,9 @@ static int really_execute_checkpoint(void) err: error= 1; - ma_message_no_user(0, "checkpoint failed"); + my_printf_error(HA_ERR_GENERIC, "Aria engine: checkpoint failed at %s with " + "error %d", MYF(ME_ERROR_LOG), + error_place, (error_errno ? error_errno : my_errno)); /* we were possibly not able to determine what pages to flush */ pages_to_flush_before_next_checkpoint= 0; diff --git a/storage/perfschema/pfs_engine_table.cc b/storage/perfschema/pfs_engine_table.cc index d2a42bd7f98..283ca28100b 100644 --- a/storage/perfschema/pfs_engine_table.cc +++ b/storage/perfschema/pfs_engine_table.cc @@ -226,7 +226,7 @@ bool PFS_table_context::is_item_set(ulong n) { ulong word= n / m_word_size; ulong bit= n % m_word_size; - return (m_map[word] & (1 << bit)); + return (m_map[word] & (1UL << bit)); } diff --git a/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result b/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result index 47f7bb923ba..01fa9dac7fd 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result @@ -20,7 +20,7 @@ set @tmp1=@@rocksdb_verify_row_debug_checksums; set rocksdb_verify_row_debug_checksums=1; set session debug_dbug= "+d,myrocks_simulate_bad_row_read1"; select * from t1 where pk=1; -ERROR HY000: Got error 202 'Found data corruption.' from ROCKSDB +ERROR HY000: Got error 203 'Found data corruption.' from ROCKSDB set session debug_dbug= "-d,myrocks_simulate_bad_row_read1"; set rocksdb_verify_row_debug_checksums=@tmp1; select * from t1 where pk=1; @@ -28,11 +28,11 @@ pk col1 1 1 set session debug_dbug= "+d,myrocks_simulate_bad_row_read2"; select * from t1 where pk=1; -ERROR HY000: Got error 202 'Found data corruption.' from ROCKSDB +ERROR HY000: Got error 203 'Found data corruption.' from ROCKSDB set session debug_dbug= "-d,myrocks_simulate_bad_row_read2"; set session debug_dbug= "+d,myrocks_simulate_bad_row_read3"; select * from t1 where pk=1; -ERROR HY000: Got error 202 'Found data corruption.' from ROCKSDB +ERROR HY000: Got error 203 'Found data corruption.' from ROCKSDB set session debug_dbug= "-d,myrocks_simulate_bad_row_read3"; insert into t1 values(4,'0123456789'); select * from t1; @@ -56,7 +56,7 @@ pk col1 ABCD 1 set session debug_dbug= "+d,myrocks_simulate_bad_pk_read1"; select * from t2; -ERROR HY000: Got error 202 'Found data corruption.' from ROCKSDB +ERROR HY000: Got error 203 'Found data corruption.' from ROCKSDB set session debug_dbug= "-d,myrocks_simulate_bad_pk_read1"; drop table t2; create table t2 ( @@ -69,6 +69,6 @@ pk col1 ABCD 1 set session debug_dbug= "+d,myrocks_simulate_bad_pk_read1"; select * from t2; -ERROR HY000: Got error 202 'Found data corruption.' from ROCKSDB +ERROR HY000: Got error 203 'Found data corruption.' from ROCKSDB set session debug_dbug= "-d,myrocks_simulate_bad_pk_read1"; drop table t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result b/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result index d96c40127a8..9de77014593 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result @@ -63,7 +63,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t0 ALL i1 NULL NULL NULL # Using where explain select * from t0 where (key1 < 3 or key2 <4) and key3 = 50; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t0 index_merge i1,i2,i3 i1,i2 4,4 NULL # Using sort_union(i1,i2); Using where +1 SIMPLE t0 ref i1,i2,i3 i3 4 const # Using where explain select * from t0 use index (i1,i2) where (key1 < 2 or key2 <3) and key3 = 50; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t0 index_merge i1,i2 i1,i2 4,4 NULL # Using sort_union(i1,i2); Using where @@ -121,11 +121,11 @@ id select_type table type possible_keys key key_len ref rows Extra explain select * from t0 where (key1 < 3 or key2 < 3) and (key3 < 70); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t0 index_merge i1,i2,i3 i1,i2 4,4 NULL # Using sort_union(i1,i2); Using where +1 SIMPLE t0 range i1,i2,i3 i3 4 NULL # Using index condition; Using where explain select * from t0 where (key1 < 3 or key2 < 3) and (key3 < 1000); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t0 index_merge i1,i2,i3 i1,i2 4,4 NULL # Using sort_union(i1,i2); Using where +1 SIMPLE t0 range i1,i2,i3 i3 4 NULL # Using index condition; Using where explain select * from t0 where ((key1 < 3 or key2 < 3) and (key2 <4 or key3 < 3)) or @@ -287,7 +287,7 @@ id select_type table type possible_keys key key_len ref rows Extra NULL UNION RESULT ALL NULL NULL NULL NULL NULL explain select * from (select * from t1 where key1 = 3 or key2 =3) as Z where key8 >5; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index_merge i1,i2,i8 i1,i2 4,4 NULL 4 Using union(i1,i2); Using where +1 SIMPLE t1 range i1,i2,i8 i8 4 NULL 2 Using index condition; Using where create table t3 like t0; insert into t3 select * from t0; alter table t3 add key9 int not null, add index i9(key9); diff --git a/storage/rocksdb/mysql-test/rocksdb/r/tbl_opt_data_index_dir.result b/storage/rocksdb/mysql-test/rocksdb/r/tbl_opt_data_index_dir.result index 95dae68b4e6..90f163b7d4b 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/tbl_opt_data_index_dir.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/tbl_opt_data_index_dir.result @@ -1,16 +1,16 @@ DROP TABLE IF EXISTS t1; CREATE TABLE t1 (a INT PRIMARY KEY, b CHAR(8)) ENGINE=rocksdb DATA DIRECTORY = '/foo/bar/data'; -ERROR HY000: Can't create table `test`.`t1` (errno: 198 "Unknown error 198") -show warnings; -Level Code Message -Error 1005 Can't create table `test`.`t1` (errno: 198 "Unknown error 198") -Warning 1296 Got error 198 'Specifying DATA DIRECTORY for an individual table is not supported.' from ROCKSDB -CREATE TABLE t1 (a INT PRIMARY KEY, b CHAR(8)) ENGINE=rocksdb INDEX DIRECTORY = '/foo/bar/index'; ERROR HY000: Can't create table `test`.`t1` (errno: 199 "Unknown error 199") show warnings; Level Code Message Error 1005 Can't create table `test`.`t1` (errno: 199 "Unknown error 199") -Warning 1296 Got error 199 'Specifying INDEX DIRECTORY for an individual table is not supported.' from ROCKSDB +Warning 1296 Got error 199 'Specifying DATA DIRECTORY for an individual table is not supported.' from ROCKSDB +CREATE TABLE t1 (a INT PRIMARY KEY, b CHAR(8)) ENGINE=rocksdb INDEX DIRECTORY = '/foo/bar/index'; +ERROR HY000: Can't create table `test`.`t1` (errno: 200 "Unknown error 200") +show warnings; +Level Code Message +Error 1005 Can't create table `test`.`t1` (errno: 200 "Unknown error 200") +Warning 1296 Got error 200 'Specifying INDEX DIRECTORY for an individual table is not supported.' from ROCKSDB CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY) ENGINE=rocksdb PARTITION BY RANGE (id) ( PARTITION P0 VALUES LESS THAN (1000) @@ -19,11 +19,11 @@ PARTITION P1 VALUES LESS THAN (2000) DATA DIRECTORY = '/foo/bar/data/', PARTITION P2 VALUES LESS THAN (MAXVALUE) ); -ERROR HY000: Can't create table `test`.`t1` (errno: 198 "Unknown error 198") +ERROR HY000: Can't create table `test`.`t1` (errno: 199 "Unknown error 199") show warnings; Level Code Message -Error 1005 Can't create table `test`.`t1` (errno: 198 "Unknown error 198") -Warning 1296 Got error 198 'Specifying DATA DIRECTORY for an individual table is not supported.' from ROCKSDB +Error 1005 Can't create table `test`.`t1` (errno: 199 "Unknown error 199") +Warning 1296 Got error 199 'Specifying DATA DIRECTORY for an individual table is not supported.' from ROCKSDB Error 6 Error on delete of './test/t1.par' (Errcode: 2 "No such file or directory") CREATE TABLE t1 (id int not null primary key) ENGINE=rocksdb PARTITION BY RANGE (id) ( @@ -33,9 +33,9 @@ PARTITION P1 VALUES LESS THAN (2000) INDEX DIRECTORY = '/foo/bar/data/', PARTITION P2 VALUES LESS THAN (MAXVALUE) ); -ERROR HY000: Can't create table `test`.`t1` (errno: 199 "Unknown error 199") +ERROR HY000: Can't create table `test`.`t1` (errno: 200 "Unknown error 200") show warnings; Level Code Message -Error 1005 Can't create table `test`.`t1` (errno: 199 "Unknown error 199") -Warning 1296 Got error 199 'Specifying INDEX DIRECTORY for an individual table is not supported.' from ROCKSDB +Error 1005 Can't create table `test`.`t1` (errno: 200 "Unknown error 200") +Warning 1296 Got error 200 'Specifying INDEX DIRECTORY for an individual table is not supported.' from ROCKSDB Error 6 Error on delete of './test/t1.par' (Errcode: 2 "No such file or directory") diff --git a/tpool/task.cc b/tpool/task.cc index 0b5253bc725..81ec88590ce 100644 --- a/tpool/task.cc +++ b/tpool/task.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2019, 2020, MariaDB Corporation. +/* Copyright (C) 2019, 2021, MariaDB Corporation. This program is free software; you can redistribute itand /or modify it under the terms of the GNU General Public License as published by @@ -21,21 +21,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111 - 1301 USA*/ namespace tpool { - -#ifndef DBUG_OFF -static callback_func_np after_task_callback; -void set_after_task_callback(callback_func_np cb) -{ - after_task_callback= cb; -} - -void execute_after_task_callback() -{ - if (after_task_callback) - after_task_callback(); -} -#endif - task::task(callback_func func, void* arg, task_group* group) : m_func(func), m_arg(arg), m_group(group) {} @@ -50,7 +35,6 @@ void execute_after_task_callback() { /* Execute directly. */ m_func(m_arg); - dbug_execute_after_task_callback(); release(); } } diff --git a/tpool/task_group.cc b/tpool/task_group.cc index 97fbb0911c8..b52fe7c0f67 100644 --- a/tpool/task_group.cc +++ b/tpool/task_group.cc @@ -53,7 +53,6 @@ namespace tpool if (t) { t->m_func(t->m_arg); - dbug_execute_after_task_callback(); t->release(); } lk.lock(); diff --git a/tpool/tpool.h b/tpool/tpool.h index d33c0608959..95e4205e601 100644 --- a/tpool/tpool.h +++ b/tpool/tpool.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2019, 2020, MariaDB Corporation. +/* Copyright (C) 2019, 2021, MariaDB Corporation. This program is free software; you can redistribute itand /or modify it under the terms of the GNU General Public License as published by @@ -184,18 +184,6 @@ class thread_pool; extern aio *create_simulated_aio(thread_pool *tp); -#ifndef DBUG_OFF -/* - This function is useful for debugging to make sure all mutexes are released - inside a task callback -*/ -void set_after_task_callback(callback_func_np cb); -void execute_after_task_callback(); -#define dbug_execute_after_task_callback() execute_after_task_callback() -#else -#define dbug_execute_after_task_callback() do{}while(0) -#endif - class thread_pool { protected: diff --git a/tpool/tpool_generic.cc b/tpool/tpool_generic.cc index 0c769d67c99..ecc489c3357 100644 --- a/tpool/tpool_generic.cc +++ b/tpool/tpool_generic.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2019, 2020, MariaDB Corporation. +/* Copyright (C) 2019, 2021, MariaDB Corporation. This program is free software; you can redistribute itand /or modify it under the terms of the GNU General Public License as published by @@ -311,7 +311,6 @@ public: return; m_callback(m_data); - dbug_execute_after_task_callback(); m_running = false; if (m_pool && m_period) diff --git a/tpool/tpool_win.cc b/tpool/tpool_win.cc index 09fd49d9411..88168b26eff 100644 --- a/tpool/tpool_win.cc +++ b/tpool/tpool_win.cc @@ -1,4 +1,4 @@ -/* Copyright(C) 2019 MariaDB +/* Copyright (C) 2019, 2021, MariaDB Corporation. This program is free software; you can redistribute itand /or modify it under the terms of the GNU General Public License as published by @@ -93,7 +93,6 @@ class thread_pool_win : public thread_pool return; } timer->m_func(timer->m_data); - dbug_execute_after_task_callback(); if (timer->m_period) timer->set_time(timer->m_period, timer->m_period); }