From d4760d8c016c6b65a473fbd6f66ca4db02749106 Mon Sep 17 00:00:00 2001 From: Tuukka Pasanen Date: Mon, 23 May 2022 10:00:13 +0300 Subject: [PATCH 1/3] MDEV-28642: Suspend obvious false-positive Lintian warnings MariaDB codebase is huge and Lintian has lots of test than can fire false-positive warnings which leads to situation where real problems can't be spotted. Suspend obvious false-positive Lintian warnings and let Lintian problems that needs some love shine out. Suspends in package mariadb-test-data Supporting BSD family needs to use '/usr/bin/env perl' and not '/usr/bin/perl' Perl script are for testing and not for production in mariadb-test-data package: * incorrect-path-for-interpreter There is several files with national-encoding which are test file so they can't be in unicode charset * national-encoding Serveral test paths are intentionally repeated: * repeated-path-segment Suspends in package mariadb-test Supporting BSD family needs to use '/usr/bin/env perl' and not '/usr/bin/perl' Perl script are for testing and not for production in mariadb-test-data package: * incorrect-path-for-interpreter Suspends in package source package Remade some 'version-substvar-for-external-package' to use regex. MGroonga is missing source file 'jquery-ui-1.8.18.custom.js' correct lintian suspend with regex: * source-is-missing There is several files with very long line lenghts. Add suspends for those that can't be corrected in several places. Most of them are test result files, SQL test files or intentional long lines that can't be splitted. * very-long-line-length-in-source-file There is several autogenerated C++ files which probably should not be there but they should not do any harm: * source-contains-autogenerated-visual-c++-file --- debian/mariadb-test-data.lintian-overrides | 30 +++++++ debian/mariadb-test.lintian-overrides | 5 ++ debian/source/lintian-overrides | 93 +++++++++++++++++----- 3 files changed, 108 insertions(+), 20 deletions(-) diff --git a/debian/mariadb-test-data.lintian-overrides b/debian/mariadb-test-data.lintian-overrides index 4ff05a3be2a..2fb5bc6e604 100644 --- a/debian/mariadb-test-data.lintian-overrides +++ b/debian/mariadb-test-data.lintian-overrides @@ -1,3 +1,33 @@ # These should be moved, see https://jira.mariadb.org/browse/MDEV-21654 arch-dependent-file-in-usr-share usr/share/mysql/mysql-test/suite/plugins/pam/pam_mariadb_mtr.so arch-independent-package-contains-binary-or-object usr/share/mysql/mysql-test/suite/plugins/pam/pam_mariadb_mtr.so +# Mainly for support for *BSD family. Not right way to do but this is test package and not for production +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/std_data/checkDBI_DBD-MariaDB.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/suite/engines/rr_trx/run_stress_tx_rr.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/suite/funcs_1/lib/DataGen_local.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/suite/funcs_1/lib/DataGen_modify.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/suite/funcs_2/lib/gen_charset_utf8.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/suite/rpl/extension/checksum.pl] +# Intentional for test files +national-encoding usr/share/mysql/mysql-test/* +# Extra test documentation files that really need to be kept in context in test directory +package-contains-documentation-outside-usr-share-doc usr/share/mysql/mysql-test/* +# Intentional directory structure +repeated-path-segment auth_gssapi usr/share/mysql/mysql-test/plugin/auth_gssapi/auth_gssapi/ +repeated-path-segment connect usr/share/mysql/mysql-test/plugin/connect/connect/ +repeated-path-segment disks usr/share/mysql/mysql-test/plugin/disks/disks/ +repeated-path-segment func_test usr/share/mysql/mysql-test/plugin/func_test/func_test/ +repeated-path-segment metadata_lock_info usr/share/mysql/mysql-test/plugin/metadata_lock_info/metadata_lock_info/ +repeated-path-segment mroonga usr/share/mysql/mysql-test/plugin/mroonga/mroonga/ +repeated-path-segment mroonga usr/share/mysql/mysql-test/plugin/mroonga/mroonga/include/mroonga/ +repeated-path-segment oqgraph usr/share/mysql/mysql-test/plugin/oqgraph/oqgraph/ +repeated-path-segment query_response_time usr/share/mysql/mysql-test/plugin/query_response_time/query_response_time/ +repeated-path-segment rocksdb usr/share/mysql/mysql-test/plugin/rocksdb/rocksdb/ +repeated-path-segment sequence usr/share/mysql/mysql-test/plugin/sequence/sequence/ +repeated-path-segment sphinx usr/share/mysql/mysql-test/plugin/sphinx/sphinx/ +repeated-path-segment spider usr/share/mysql/mysql-test/plugin/spider/spider/ +repeated-path-segment tokudb usr/share/mysql/mysql-test/plugin/tokudb/tokudb/ +repeated-path-segment type_inet usr/share/mysql/mysql-test/plugin/type_inet/type_inet/ +repeated-path-segment type_test usr/share/mysql/mysql-test/plugin/type_test/type_test/ +repeated-path-segment user_variables usr/share/mysql/mysql-test/plugin/user_variables/user_variables/ +repeated-path-segment wsrep_info usr/share/mysql/mysql-test/plugin/wsrep_info/wsrep_info/ diff --git a/debian/mariadb-test.lintian-overrides b/debian/mariadb-test.lintian-overrides index 9de1b3784ff..b9f45b862b5 100644 --- a/debian/mariadb-test.lintian-overrides +++ b/debian/mariadb-test.lintian-overrides @@ -1,3 +1,8 @@ # These should be moved, see https://jira.mariadb.org/browse/MDEV-21653 arch-dependent-file-in-usr-share usr/share/mysql/mysql-test/lib/My/SafeProcess/my_safe_process arch-dependent-file-in-usr-share usr/share/mysql/mysql-test/lib/My/SafeProcess/wsrep_check_version +# Mainly for support for *BSD family. Not right way to do but this is test package and not for production +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/lib/process-purecov-annotations.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/lib/v1/mysql-test-run.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/mysql-stress-test.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/mysql-test-run.pl] diff --git a/debian/source/lintian-overrides b/debian/source/lintian-overrides index a9500a44598..2fab3096024 100644 --- a/debian/source/lintian-overrides +++ b/debian/source/lintian-overrides @@ -1,32 +1,85 @@ # Necessary for drop-in-place-replacement upgrades on mysql-server/-client # since package breaks/replaces these but at the same time also provides them -version-substvar-for-external-package mariadb-client-core-10.5 -> mysql-client-5.5 -version-substvar-for-external-package mariadb-server-10.5 -> mysql-server +version-substvar-for-external-package mariadb-client-core-10.5 -> mysql-client-5.* version-substvar-for-external-package mariadb-server-10.5 -> mysql-server version-substvar-for-external-package libmariadb-dev -> libmysqlclient-dev version-substvar-for-external-package libmariadb-dev -> libmysqld-dev -version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-5.5 -version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-5.6 -version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-5.7 -version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-8.0 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.1 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.1 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.5 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.5 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.6 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.6 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.7 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.7 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-8.0 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-8.0 +version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-5.* +version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-8.* +version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.* +version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-8.* version-substvar-for-external-package libmariadbd-dev -> libmariadbclient-dev # ColumnStore not used in Debian, safe to ignore. Reported upstream in https://jira.mariadb.org/browse/MDEV-24124 source-is-missing storage/columnstore/columnstore/utils/jemalloc/libjemalloc.so.2 # Must be fixed upstream -source-is-missing storage/mroonga/vendor/groonga/examples/dictionary/html/js/jquery-ui-1.8.18.custom.js * +source-is-missing storage/mroonga/vendor/groonga/examples/dictionary/html/js/jquery-ui-*.custom.js # Intentional control relationships -version-substvar-for-external-package Replaces (line 216) ${source:Version} libmariadbd-dev -> libmariadbclient-dev -version-substvar-for-external-package Replaces (line 66) ${source:Version} libmariadb-dev -> libmysqlclient-dev -version-substvar-for-external-package Replaces (line 66) ${source:Version} libmariadb-dev -> libmysqld-dev +version-substvar-for-external-package Replaces * libmariadbd-dev -> libmariadbclient-dev +version-substvar-for-external-package Replaces * libmariadb-dev -> libmysqlclient-dev +version-substvar-for-external-package Replaces * libmariadb-dev -> libmysqld-dev # We can't change build dependencies on a stable branch (10.5..10.8) so just override this missing-build-dependency-for-dh-addon systemd * +# Data or test files where long lines are justified +very-long-line-length-in-source-file *.test * +very-long-line-length-in-source-file *.result * +very-long-line-length-in-source-file BUILD/compile-* +very-long-line-length-in-source-file *COPYING.rtf * +# These are mainly found under extra/wolfssl +very-long-line-length-in-source-file *.cproject * +very-long-line-length-in-source-file *.md * +very-long-line-length-in-source-file *.scfg * +very-long-line-length-in-source-file *.launch * +very-long-line-length-in-source-file extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/test/test_wolfssl.c * +very-long-line-length-in-source-file extra/wolfssl/wolfssl/configure.ac * +very-long-line-length-in-source-file extra/wolfssl/wolfssl/doc/formats/html/html_changes/tabs.css * +# Preprocessed C files which have long lines +very-long-line-length-in-source-file extra/wolfssl/wolfssl/wolfcrypt/src/*.i * +# These are all results for test cases and similar so they can be +# especially formatted to be too long +very-long-line-length-in-source-file mysql-test/*.dump * +very-long-line-length-in-source-file mysql-test/*.inc * +very-long-line-length-in-source-file mysql-test/*.rdiff * +very-long-line-length-in-source-file mysql-test/*.txt * +very-long-line-length-in-source-file mysql-test/*.weekly * +# Test file +very-long-line-length-in-source-file plugin/handler_socket/regtest/test_01_lib/test19.expected * +# SQL source file that has very long inserts/selects +very-long-line-length-in-source-file mysql-test/std_data/init_file_longline_3816.sql * +very-long-line-length-in-source-file scripts/fill_help_tables.sql * +very-long-line-length-in-source-file scripts/mysql_system_tables.sql * +very-long-line-length-in-source-file scripts/mysql_test_data_timezone.sql * +# Machine formated HTML +very-long-line-length-in-source-file sql/share/charsets/languages.html * +very-long-line-length-in-source-file sql/share/errmsg-utf8.txt * +# Very long test string +very-long-line-length-in-source-file storage/archive/archive_test.c line 30 is 1051 characters long (>512) +# autogenerated thrift file +very-long-line-length-in-source-file storage/cassandra/gen-cpp/cassandra_types.h * +# ColumnStore ignores +# In Directory mysql-test are some long test includes +very-long-line-length-in-source-file storage/columnstore/columnstore/.drone.jsonnet * +very-long-line-length-in-source-file storage/columnstore/columnstore/CMakeLists.txt * +very-long-line-length-in-source-file storage/columnstore/columnstore/mysql-test/columnstore/csinternal/include/autopilot_create_datatypetestm_tables.inc * +very-long-line-length-in-source-file storage/columnstore/columnstore/mysql-test/columnstore/csinternal/include/autopilot_create_datatypeupdate_table.inc * +very-long-line-length-in-source-file storage/columnstore/columnstore/*.xmi * +very-long-line-length-in-source-file storage/columnstore/columnstore/dbcon/doc/q19_plan.txt * +very-long-line-length-in-source-file storage/columnstore/columnstore/utils/udfsdk/docs/source/reference/mcsv1Context.rst * +very-long-line-length-in-source-file storage/columnstore/columnstore/utils/winport/win_setup_mysql_part1.sql * +# Minified CSS files. These appear in several places +very-long-line-length-in-source-file *badge_only.css * +very-long-line-length-in-source-file *theme.css line * +# General storage ignores +very-long-line-length-in-source-file storage/mroonga/vendor/groonga/examples/dictionary/html/css/smoothness/jquery-ui-1.8.12.custom.css * +very-long-line-length-in-source-file storage/rocksdb/mysql-test/rocksdb/t/bypass_select_basic_bloom-master.opt * +very-long-line-length-in-source-file storage/rocksdb/mysql-test/rocksdb/t/type_enum.inc * +very-long-line-length-in-source-file storage/rocksdb/mysql-test/rocksdb/t/type_set.inc * +very-long-line-length-in-source-file storage/rocksdb/rocksdb/docs/_includes/footer.html * +very-long-line-length-in-source-file storage/rocksdb/rocksdb/docs/_posts/*.markdown line * +very-long-line-length-in-source-file storage/spider/mysql-test/spider/bugfix/include/sql_mode_init.inc * +very-long-line-length-in-source-file storage/tokudb/PerconaFT/cmake_modules/TokuBuildTagDatabases.cmake * +very-long-line-length-in-source-file storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/m4/po.m4 * +# These are generated files which should not make any harm +source-contains-autogenerated-visual-c++-file storage/columnstore/columnstore/*.rc +source-contains-autogenerated-visual-c++-file storage/columnstore/columnstore/*.h +source-contains-autogenerated-visual-c++-file win/upgrade_wizard/resource.h +source-contains-autogenerated-visual-c++-file win/upgrade_wizard/upgrade.rc From 06e9ce798c582ffb5a2f3cfbb0f8e33fe221feae Mon Sep 17 00:00:00 2001 From: Nayuta Yanagisawa Date: Fri, 29 Oct 2021 19:04:53 +0900 Subject: [PATCH 2/3] MDEV-26127 Assertion `err != DB_DUPLICATE_KEY' failed or InnoDB: Failing assertion: id != 0 on ALTER ... REBUILD PARTITION During rebuild of partition, the partitioning engine calls alter_close_table(), which does not unlock and close some table instances of the target table. Then, the engine fails to rename partitions because there are table instances that are still locked. Closing all the table instance of the target table fixes the bug. --- .../suite/parts/inc/part_alter_values.inc | 8 +++++ .../parts/r/partition_alter_innodb.result | 7 ++++ .../parts/r/partition_alter_maria.result | 7 ++++ .../parts/r/partition_alter_myisam.result | 7 ++++ sql/sql_partition.cc | 35 ++++++++++++++----- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/mysql-test/suite/parts/inc/part_alter_values.inc b/mysql-test/suite/parts/inc/part_alter_values.inc index ca18faa5758..d3b63a4610f 100644 --- a/mysql-test/suite/parts/inc/part_alter_values.inc +++ b/mysql-test/suite/parts/inc/part_alter_values.inc @@ -78,3 +78,11 @@ if (`SELECT IF('$engine' != 'InnoDB', 1, 0)`) --remove_files_wildcard $MYSQLTEST_VARDIR/tmp/mdev_27065 * --rmdir $MYSQLTEST_VARDIR/tmp/mdev_27065 + +--echo # +--echo # MDEV-26127 Assertion `err != DB_DUPLICATE_KEY' failed or InnoDB: Failing assertion: id != 0 on ALTER ... REBUILD PARTITION +--echo # +--eval CREATE TABLE t1 (c INT) ENGINE=$engine PARTITION BY KEY(c) PARTITIONS 4; +LOCK TABLES t1 WRITE, t1 AS a READ; +ALTER TABLE t1 REBUILD PARTITION p0; +DROP TABLE t1; diff --git a/mysql-test/suite/parts/r/partition_alter_innodb.result b/mysql-test/suite/parts/r/partition_alter_innodb.result index ae3caaa4981..6afa133f989 100644 --- a/mysql-test/suite/parts/r/partition_alter_innodb.result +++ b/mysql-test/suite/parts/r/partition_alter_innodb.result @@ -61,3 +61,10 @@ PARTITION p1 VALUES LESS THAN MAXVALUE Warnings: Warning 1618 table option of old schema is ignored DROP TABLE t1; +# +# MDEV-26127 Assertion `err != DB_DUPLICATE_KEY' failed or InnoDB: Failing assertion: id != 0 on ALTER ... REBUILD PARTITION +# +CREATE TABLE t1 (c INT) ENGINE=InnoDB PARTITION BY KEY(c) PARTITIONS 4;; +LOCK TABLES t1 WRITE, t1 AS a READ; +ALTER TABLE t1 REBUILD PARTITION p0; +DROP TABLE t1; diff --git a/mysql-test/suite/parts/r/partition_alter_maria.result b/mysql-test/suite/parts/r/partition_alter_maria.result index 358ffbdfbe7..eca8378430f 100644 --- a/mysql-test/suite/parts/r/partition_alter_maria.result +++ b/mysql-test/suite/parts/r/partition_alter_maria.result @@ -95,3 +95,10 @@ PARTITION p1 VALUES LESS THAN MAXVALUE Warnings: Warning 1618 table option of old schema is ignored DROP TABLE t2; +# +# MDEV-26127 Assertion `err != DB_DUPLICATE_KEY' failed or InnoDB: Failing assertion: id != 0 on ALTER ... REBUILD PARTITION +# +CREATE TABLE t1 (c INT) ENGINE=Aria PARTITION BY KEY(c) PARTITIONS 4;; +LOCK TABLES t1 WRITE, t1 AS a READ; +ALTER TABLE t1 REBUILD PARTITION p0; +DROP TABLE t1; diff --git a/mysql-test/suite/parts/r/partition_alter_myisam.result b/mysql-test/suite/parts/r/partition_alter_myisam.result index 9d76881fdfa..ba1a0fe05c4 100644 --- a/mysql-test/suite/parts/r/partition_alter_myisam.result +++ b/mysql-test/suite/parts/r/partition_alter_myisam.result @@ -68,6 +68,13 @@ PARTITION p1 VALUES LESS THAN MAXVALUE Warnings: Warning 1618 table option of old schema is ignored DROP TABLE t2; +# +# MDEV-26127 Assertion `err != DB_DUPLICATE_KEY' failed or InnoDB: Failing assertion: id != 0 on ALTER ... REBUILD PARTITION +# +CREATE TABLE t1 (c INT) ENGINE=MyISAM PARTITION BY KEY(c) PARTITIONS 4;; +LOCK TABLES t1 WRITE, t1 AS a READ; +ALTER TABLE t1 REBUILD PARTITION p0; +DROP TABLE t1; create table t1 ( c1 int, c2 int, c3 varchar(100)) delay_key_write=1 partition by key(c1) ( partition p01 data directory = 'MYSQL_TMP_DIR' diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index 1cf2d009d37..78f56992f46 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2005, 2017, Oracle and/or its affiliates. - Copyright (c) 2009, 2020, MariaDB + Copyright (c) 2009, 2022, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -6817,16 +6817,33 @@ static bool alter_partition_lock_handling(ALTER_PARTITION_PARAM_TYPE *lpt) static int alter_close_table(ALTER_PARTITION_PARAM_TYPE *lpt) { - int error= 0; + THD *thd= lpt->thd; + TABLE_SHARE *share= lpt->table->s; DBUG_ENTER("alter_close_table"); - if (lpt->table->db_stat) - { - error= mysql_lock_remove(lpt->thd, lpt->thd->lock, lpt->table); - error= lpt->table->file->ha_close(); - lpt->table->db_stat= 0; // Mark file closed - } - DBUG_RETURN(error); + TABLE *table= thd->open_tables; + do { + table= find_locked_table(table, share->db.str, share->table_name.str); + if (!table) + { + DBUG_RETURN(0); + } + + if (table->db_stat) + { + if (int error= mysql_lock_remove(thd, thd->lock, table)) + { + DBUG_RETURN(error); + } + if (int error= table->file->ha_close()) + { + DBUG_RETURN(error); + } + table->db_stat= 0; // Mark file closed + } + } while ((table= table->next)); + + DBUG_RETURN(0); } From 4849d94fe6bd918fd3dc6dc21424681a0ab5fa02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 14 Jun 2022 09:14:24 +0300 Subject: [PATCH 3/3] MDEV-28828 SIGSEGV in buf_flush_LRU_list_batch In commit 73fee39ea62037780c59161507e89dd76c10b7a3 (MDEV-27985) a regression was introduced that would cause bpage=nullptr to be referenced. buf_flush_LRU_list_batch(): Always terminate the loop upon encountering a null pointer. --- storage/innobase/buf/buf0flu.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index def598934c9..3d2669e2cc5 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1294,7 +1294,6 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n) n->flushed + n->evicted < max) || recv_recovery_is_on()); ++scanned) { - retry: buf_page_t *prev= UT_LIST_GET_PREV(LRU, bpage); const lsn_t oldest_modification= bpage->oldest_modification(); buf_pool.lru_hp.set(prev); @@ -1326,7 +1325,6 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n) mysql_mutex_lock(&buf_pool.mutex); if (p.second) buf_pool.stat.n_pages_written+= p.second; - bpage= buf_pool.lru_hp.get(); goto retry; } else @@ -1357,6 +1355,7 @@ reacquire_mutex: else /* Can't evict or dispatch this block. Go to previous. */ ut_ad(buf_pool.lru_hp.is_hp(prev)); + retry: bpage= buf_pool.lru_hp.get(); }