From 78119d1ae5e376f487e7a3c079f7df5e0cb28823 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 13 Oct 2024 14:15:25 +0200 Subject: [PATCH] MDEV-33410 VECTOR data type --- libmysqld/CMakeLists.txt | 2 +- mysql-test/main/blackhole.result | 2 +- mysql-test/main/blackhole.test | 2 +- mysql-test/main/vector,myisam.rdiff | 48 +-- mysql-test/main/vector.result | 110 +++--- mysql-test/main/vector.test | 38 +-- mysql-test/main/vector2.result | 109 ++++-- mysql-test/main/vector2.test | 89 +++-- mysql-test/main/vector2_notembedded.result | 8 +- mysql-test/main/vector2_notembedded.test | 4 +- mysql-test/main/vector_funcs.result | Bin 4970 -> 4975 bytes mysql-test/main/vector_funcs.test | 2 +- mysql-test/main/vector_innodb.result | 45 +-- mysql-test/main/vector_innodb.test | 36 +- mysql-test/main/vector_partition.result | 6 +- mysql-test/main/vector_partition.test | 6 +- mysql-test/suite/mariabackup/vector.result | 4 +- mysql-test/suite/mariabackup/vector.test | 2 +- sql/CMakeLists.txt | 2 +- sql/sql_basic_types.h | 2 + sql/sql_table.cc | 32 +- sql/sql_type.cc | 3 + sql/sql_type.h | 6 - sql/sql_type_vector.cc | 317 ++++++++++++++++++ sql/sql_type_vector.h | 130 +++++++ sql/vector_mhnsw.cc | 19 +- .../mysql-test/rocksdb/r/vector.result | 2 +- .../rocksdb/mysql-test/rocksdb/t/vector.test | 2 +- 28 files changed, 737 insertions(+), 291 deletions(-) create mode 100644 sql/sql_type_vector.cc create mode 100644 sql/sql_type_vector.h diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 4495078aa80..dad7631d2ed 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -136,7 +136,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/sql_mode.cc ../sql/sql_type_string.cc ../sql/sql_type_json.cc - ../sql/sql_type_geom.cc + ../sql/sql_type_geom.cc ../sql/sql_type_vector.cc ../sql/table_cache.cc ../sql/mf_iocache_encr.cc ../sql/wsrep_dummy.cc ../sql/encryption.cc ../sql/item_windowfunc.cc ../sql/sql_window.cc diff --git a/mysql-test/main/blackhole.result b/mysql-test/main/blackhole.result index 493cb1af5a0..f6e38cbff1d 100644 --- a/mysql-test/main/blackhole.result +++ b/mysql-test/main/blackhole.result @@ -39,7 +39,7 @@ DROP TABLE `t`; # # MDEV-35035 Assertion failure in ha_blackhole::position upon INSERT into blackhole table with vector index # -create table t (a int, v blob not null, vector index (v)) engine=blackhole; +create table t (a int, v vector(1) not null, vector index (v)) engine=blackhole; insert into t values (1,x'00000000'); drop table t; # End of 11.7 tests diff --git a/mysql-test/main/blackhole.test b/mysql-test/main/blackhole.test index c823ef23003..9e0a41cfb20 100644 --- a/mysql-test/main/blackhole.test +++ b/mysql-test/main/blackhole.test @@ -58,7 +58,7 @@ DROP TABLE `t`; --echo # --echo # MDEV-35035 Assertion failure in ha_blackhole::position upon INSERT into blackhole table with vector index --echo # -create table t (a int, v blob not null, vector index (v)) engine=blackhole; +create table t (a int, v vector(1) not null, vector index (v)) engine=blackhole; insert into t values (1,x'00000000'); drop table t; diff --git a/mysql-test/main/vector,myisam.rdiff b/mysql-test/main/vector,myisam.rdiff index 06f474a9b6c..b33fa194569 100644 --- a/mysql-test/main/vector,myisam.rdiff +++ b/mysql-test/main/vector,myisam.rdiff @@ -1,7 +1,7 @@ --- a/mysql-test/main/vector.result +++ b/mysql-test/main/vector.result @@ -305,8 +305,30 @@ - create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); + create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); insert t1 (id, v) values (1, x'e360d63ebe554f3fcdbc523f4522193f5236083d'); truncate table t1; + @@ -33,7 +33,7 @@ id hex(v) @@ -324,26 +346,32 @@ # Test RENAME TABLE with vector index - create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); + create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); db.opt -t1#i#01.ibd +t1#i#01.MYD @@ -106,7 +106,7 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -415,13 +415,15 @@ - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -143,7 +143,7 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -445,13 +447,15 @@ - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -180,7 +180,7 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -479,13 +479,15 @@ - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -217,7 +217,7 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -509,24 +511,27 @@ - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -233,7 +233,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -250,7 +250,7 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -539,24 +539,27 @@ - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -266,7 +266,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -283,7 +283,7 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -567,7 +567,7 @@ - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -292,7 +292,7 @@ alter table t1 add index a(id), algorithm=inplace; ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY @@ -590,31 +590,15 @@ - alter table t1 modify column v mediumblob not null, algorithm=inplace; + alter table t1 modify column v vector(6) not null, algorithm=inplace; ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY # ADD/CHANGE/DROP/MODIFY COLUMN, ALGORITHM=INPLACE (non-vector) -alter table t1 add column a varchar(10), algorithm=inplace; @@ -307,7 +307,7 @@ -Table Create Table -t1 CREATE TABLE `t1` ( - `id` int(11) NOT NULL AUTO_INCREMENT, -- `v` blob NOT NULL, +- `v` vector(5) NOT NULL, - PRIMARY KEY (`id`), - VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -328,7 +328,7 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -606,13 +606,15 @@ - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -347,7 +347,7 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -622,14 +622,16 @@ - `w` blob NOT NULL, + `w` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`w`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -367,7 +367,7 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -639,7 +639,7 @@ - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `w` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -387,7 +387,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -409,7 +409,7 @@ -Table Create Table -t1 CREATE TABLE `t1` ( - `id` int(11) NOT NULL AUTO_INCREMENT, -- `v` blob NOT NULL, +- `v` vector(5) NOT NULL, - PRIMARY KEY (`id`), - VECTOR KEY `v` (`v`) -) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -422,7 +422,7 @@ -Table Create Table -t1 CREATE TABLE `t1` ( - `id` int(11) NOT NULL AUTO_INCREMENT, -- `v` blob NOT NULL, +- `v` vector(5) NOT NULL, - PRIMARY KEY (`id`), - VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -437,7 +437,7 @@ -Table Create Table -t2 CREATE TABLE `t2` ( - `id` int(11) NOT NULL AUTO_INCREMENT, -- `v` blob NOT NULL, +- `v` vector(5) NOT NULL, - PRIMARY KEY (`id`), - VECTOR KEY `v` (`v`) -) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -450,7 +450,7 @@ -Table Create Table -t1 CREATE TABLE `t1` ( - `id` int(11) NOT NULL AUTO_INCREMENT, -- `v` blob NOT NULL, +- `v` vector(5) NOT NULL, - PRIMARY KEY (`id`), - VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -459,7 +459,7 @@ ERROR HY000: Incorrect arguments to VECTOR INDEX @@ -671,9 +672,11 @@ ERROR 42000: All parts of a VECTOR index must be NOT NULL - alter table t1 modify column v tinyblob not null; + alter table t1 modify column v vector(7) not null; db.opt -t1#i#01.ibd +t1#i#01.MYD @@ -472,12 +472,12 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -681,12 +684,14 @@ - `v` tinyblob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci - alter table t1 change column v v blob not null; + alter table t1 change column v v vector(5) not null; db.opt -t1#i#01.ibd +t1#i#01.MYD @@ -490,7 +490,7 @@ Table Create Table t1 CREATE TABLE `t1` ( @@ -699,15 +699,16 @@ - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) -) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci diff --git a/mysql-test/main/vector.result b/mysql-test/main/vector.result index 1dd955a4cd9..aba69708500 100644 --- a/mysql-test/main/vector.result +++ b/mysql-test/main/vector.result @@ -1,52 +1,52 @@ -create temporary table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create temporary table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); ERROR HY000: Cannot create VECTOR index on temporary MyISAM table create table t1 (id int auto_increment primary key, -u blob not null, vector index (u), -v blob not null, vector index (v)); +u vector(5) not null, vector index (u), +v vector(5) not null, vector index (v)); ERROR 42000: This version of MariaDB doesn't yet support 'multiple VECTOR indexes' -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci show keys from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored t1 0 PRIMARY 1 id A 0 NULL NULL BTREE NO -t1 1 v 1 v A NULL 1 NULL VECTOR NO +t1 1 v 1 v A NULL NULL NULL VECTOR NO drop table t1; set mhnsw_max_edges_per_node=@@mhnsw_max_edges_per_node+1; -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) `max_edges_per_node`=7 ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci show keys from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored t1 0 PRIMARY 1 id A 0 NULL NULL BTREE NO -t1 1 v 1 v A NULL 1 NULL VECTOR NO +t1 1 v 1 v A NULL NULL NULL VECTOR NO drop table t1; -create table t1 (id int auto_increment primary key, v blob not null, +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v) max_edges_per_node=5); show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) `max_edges_per_node`=5 ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci show keys from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored t1 0 PRIMARY 1 id A 0 NULL NULL BTREE NO -t1 1 v 1 v A NULL 1 NULL VECTOR NO +t1 1 v 1 v A NULL NULL NULL VECTOR NO set mhnsw_max_edges_per_node=default; select * from information_schema.statistics where table_name='t1'; TABLE_CATALOG def @@ -76,7 +76,7 @@ SEQ_IN_INDEX 1 COLUMN_NAME v COLLATION A CARDINALITY NULL -SUB_PART 1 +SUB_PART NULL PACKED NULL NULLABLE INDEX_TYPE VECTOR @@ -326,28 +326,28 @@ id d 18 0.73444 16 0.76710 insert t1 (v) values (''); -ERROR 22007: Incorrect vector value: '...' for column `test`.`t1`.`v` at row 1 +ERROR 22007: Incorrect vector value: '' for column `test`.`t1`.`v` at row 1 insert t1 (v) values (x'1234'); -ERROR 22007: Incorrect vector value: '...' for column `test`.`t1`.`v` at row 1 +ERROR 22007: Incorrect vector value: '\x124' for column `test`.`t1`.`v` at row 1 insert t1 (v) values (x'12345678'); -ERROR 22007: Incorrect vector value: '...' for column `test`.`t1`.`v` at row 1 +ERROR 22007: Incorrect vector value: '\x124Vx' for column `test`.`t1`.`v` at row 1 drop table t1; db.opt # Check if CREATE TABLE ... LIKE inherits VECTOR index -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); create table t2 like t1; show create table t2; Table Create Table t2 CREATE TABLE `t2` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci drop table t1, t2; db.opt # Test insert ... select with vector index -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); create table t2 like t1; insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'), (x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'), @@ -385,7 +385,7 @@ id hex(v) 20 6CA1D43E9DF91B3FE580DA3E1C247D3F147CF33E drop table t1, t2; db.opt -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); insert t1 (id, v) values (1, x'e360d63ebe554f3fcdbc523f4522193f5236083d'); truncate table t1; insert t1 (id, v) values (1, x'e360d63ebe554f3fcdbc523f4522193f5236083d'); @@ -398,14 +398,14 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=MyISAM AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci drop table t1; db.opt # Test RENAME TABLE with vector index -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); db.opt t1#i#01.ibd t1.frm @@ -433,13 +433,13 @@ db.opt # # Cosine distance # -create table t1 (id int auto_increment primary key, v blob not null, +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v) distance_function=cosine); show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) `distance_function`=cosine ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -490,7 +490,7 @@ select vec_distance_cosine(@a, @a), vec_distance_euclidean(@a, @a); vec_distance_cosine(@a, @a) vec_distance_euclidean(@a, @a) 0 0 # Test ALTER TABLE, CREATE/DROP INDEX -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'), (x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'), (x'f09baa3ea172763f123def3e0c7fe53e288bf33e'), @@ -511,7 +511,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, `a` int(11) DEFAULT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) @@ -525,7 +525,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -539,7 +539,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), KEY `a` (`id`), VECTOR KEY `v` (`v`) @@ -553,7 +553,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -567,7 +567,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), KEY `a` (`id`), VECTOR KEY `v` (`v`) @@ -581,7 +581,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -595,7 +595,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, `a` int(11) DEFAULT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) @@ -609,7 +609,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -622,7 +622,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci alter table t1 add vector index v(v), algorithm=copy; @@ -634,7 +634,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -647,7 +647,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci create vector index v on t1(v) algorithm=copy; @@ -659,7 +659,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -678,11 +678,11 @@ alter table t1 add vector index v(v), algorithm=inplace; ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY alter table t1 add vector index v(v); # CHANGE/DROP/MODIFY COLUMN, ALGORITHM=INPLACE (vector) -alter table t1 change column v v mediumblob not null, algorithm=inplace; +alter table t1 change column v v vector(6) not null, algorithm=inplace; ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY alter table t1 drop column v, algorithm=inplace; ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY -alter table t1 modify column v mediumblob not null, algorithm=inplace; +alter table t1 modify column v vector(7) not null, algorithm=inplace; ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY # ADD/CHANGE/DROP/MODIFY COLUMN, ALGORITHM=INPLACE (non-vector) alter table t1 add column a varchar(10), algorithm=inplace; @@ -697,7 +697,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -714,7 +714,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -728,7 +728,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `w` blob NOT NULL, + `w` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`w`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -743,7 +743,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `w` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -763,7 +763,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci create vector index if not exists v on t1(v); @@ -783,7 +783,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -796,7 +796,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -811,7 +811,7 @@ show create table t2; Table Create Table t2 CREATE TABLE `t2` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -824,7 +824,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -833,11 +833,11 @@ alter table t1 modify column v int; ERROR HY000: Incorrect arguments to VECTOR INDEX alter table t1 change column v v int; ERROR HY000: Incorrect arguments to VECTOR INDEX -alter table t1 modify column v blob; +alter table t1 modify column v vector(5); ERROR 42000: All parts of a VECTOR index must be NOT NULL -alter table t1 change column v v blob; +alter table t1 change column v v vector(6); ERROR 42000: All parts of a VECTOR index must be NOT NULL -alter table t1 modify column v tinyblob not null; +alter table t1 modify column v vector(7) not null; db.opt t1#i#01.ibd t1.frm @@ -846,11 +846,11 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` tinyblob NOT NULL, + `v` vector(7) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci -alter table t1 change column v v blob not null; +alter table t1 change column v v vector(5) not null; db.opt t1#i#01.ibd t1.frm @@ -859,7 +859,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -874,6 +874,6 @@ t1 CREATE TABLE `t1` ( PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci drop table t1; -create table t1(v blob not null, vector index(v)); +create table t1(v vector(5) not null, vector index(v)); alter table t1 add column a int; drop table t1; diff --git a/mysql-test/main/vector.test b/mysql-test/main/vector.test index e1a6fc37205..b90866b375c 100644 --- a/mysql-test/main/vector.test +++ b/mysql-test/main/vector.test @@ -1,24 +1,24 @@ replace_result InnoDB MyISAM; error ER_NO_INDEX_ON_TEMPORARY; -create temporary table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create temporary table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); error ER_NOT_SUPPORTED_YET; create table t1 (id int auto_increment primary key, - u blob not null, vector index (u), - v blob not null, vector index (v)); + u vector(5) not null, vector index (u), + v vector(5) not null, vector index (v)); -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); replace_result InnoDB MyISAM; show create table t1; show keys from t1; drop table t1; set mhnsw_max_edges_per_node=@@mhnsw_max_edges_per_node+1; -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); replace_result InnoDB MyISAM; show create table t1; show keys from t1; drop table t1; -create table t1 (id int auto_increment primary key, v blob not null, +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v) max_edges_per_node=5); replace_result InnoDB MyISAM; show create table t1; @@ -132,7 +132,7 @@ let $datadir=`select @@datadir`; list_files $datadir/test; --echo # Check if CREATE TABLE ... LIKE inherits VECTOR index -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); create table t2 like t1; replace_result InnoDB MyISAM; show create table t2; @@ -140,7 +140,7 @@ drop table t1, t2; list_files $datadir/test; --echo # Test insert ... select with vector index -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); create table t2 like t1; insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'), (x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'), @@ -158,7 +158,7 @@ select id, hex(v) from t1; drop table t1, t2; list_files $datadir/test; -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); insert t1 (id, v) values (1, x'e360d63ebe554f3fcdbc523f4522193f5236083d'); truncate table t1; if ($MTR_COMBINATION_MYISAM) { @@ -179,7 +179,7 @@ drop table t1; list_files $datadir/test; --echo # Test RENAME TABLE with vector index -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); list_files $datadir/test; rename table t1 to t2; list_files $datadir/test; @@ -204,7 +204,7 @@ list_files $datadir/test; --echo # --echo # Cosine distance --echo # -create table t1 (id int auto_increment primary key, v blob not null, +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v) distance_function=cosine); replace_result InnoDB MyISAM; show create table t1; @@ -245,7 +245,7 @@ set @a=vec_fromtext('[94.542572,8.735560,60.050098,74.043800,90.068710,28.212160 select vec_distance_cosine(@a, @a), vec_distance_euclidean(@a, @a); --echo # Test ALTER TABLE, CREATE/DROP INDEX -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'), (x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'), (x'f09baa3ea172763f123def3e0c7fe53e288bf33e'), @@ -324,11 +324,11 @@ alter table t1 add vector index v(v); --echo # CHANGE/DROP/MODIFY COLUMN, ALGORITHM=INPLACE (vector) --error ER_ALTER_OPERATION_NOT_SUPPORTED -alter table t1 change column v v mediumblob not null, algorithm=inplace; +alter table t1 change column v v vector(6) not null, algorithm=inplace; --error ER_ALTER_OPERATION_NOT_SUPPORTED alter table t1 drop column v, algorithm=inplace; --error ER_ALTER_OPERATION_NOT_SUPPORTED -alter table t1 modify column v mediumblob not null, algorithm=inplace; +alter table t1 modify column v vector(7) not null, algorithm=inplace; --echo # ADD/CHANGE/DROP/MODIFY COLUMN, ALGORITHM=INPLACE (non-vector) if ($MTR_COMBINATION_INNODB) { @@ -392,13 +392,13 @@ alter table t1 modify column v int; --error ER_WRONG_ARGUMENTS alter table t1 change column v v int; --error ER_INDEX_CANNOT_HAVE_NULL -alter table t1 modify column v blob; +alter table t1 modify column v vector(5); --error ER_INDEX_CANNOT_HAVE_NULL -alter table t1 change column v v blob; -alter table t1 modify column v tinyblob not null; +alter table t1 change column v v vector(6); +alter table t1 modify column v vector(7) not null; list_files $datadir/test; show create table t1; -alter table t1 change column v v blob not null; +alter table t1 change column v v vector(5) not null; list_files $datadir/test; show create table t1; alter table t1 drop column v; @@ -410,6 +410,6 @@ drop table t1; # This is supposed to test crash while filling indexes_option_struct array, # which doesn't happen because alloc_root(0) returns something. Add a test # anyway and fix indexes_option_struct array allocation. -create table t1(v blob not null, vector index(v)); +create table t1(v vector(5) not null, vector index(v)); alter table t1 add column a int; drop table t1; diff --git a/mysql-test/main/vector2.result b/mysql-test/main/vector2.result index 1c59a52c395..18f91c2e698 100644 --- a/mysql-test/main/vector2.result +++ b/mysql-test/main/vector2.result @@ -1,38 +1,93 @@ +# +# MDEV-33410 VECTOR data type +# +create table t1 (a int, b vector); +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near ')' at line 1 +create table t1 (a int, b vector(0)); +ERROR 42000: Incorrect column specifier for column 'b' +create table t1 (a int, b vector(10) collate utf8mb3_general_ci); +ERROR 42000: Incorrect column specifier for column 'b' +create table t1 (a int, b vector(10)); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` vector(10) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +insert t1 values (1, 1); +ERROR HY000: Cannot cast 'int' as 'vector' in assignment of `test`.`t1`.`b` +insert t1 values (1, 1.1); +ERROR HY000: Cannot cast 'decimal' as 'vector' in assignment of `test`.`t1`.`b` +insert t1 values (1, 1e1); +ERROR HY000: Cannot cast 'double' as 'vector' in assignment of `test`.`t1`.`b` +insert t1 values (1, now()); +ERROR HY000: Cannot cast 'timestamp' as 'vector' in assignment of `test`.`t1`.`b` +insert t1 values (1, repeat(x'56', 10)); +ERROR 22007: Incorrect vector value: 'VVVVVVVVVV' for column `test`.`t1`.`b` at row 1 +insert t1 values (1, repeat(x'66', 40)); +ERROR 22007: Incorrect vector value: 'ffffffffffffffffffffffffffffffffffffffff' for column `test`.`t1`.`b` at row 1 +insert t1 values (1, repeat(x'56', 40)); +select * from t1; +a b +1 VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV +select cast(b as char) from t1; +ERROR HY000: Illegal parameter data type vector for operation 'cast_as_char' +create table t2 as select b, cast(b as binary) from t1; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `b` vector(10) DEFAULT NULL, + `cast(b as binary)` varbinary(40) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1, t2; +create table t1 (a int, b vector(1) not null); +insert into t1 values (1,x'00000000'); +alter table t1 modify b vector(2) not null; +insert into t1 values (1,x'0000000000000000'); +select a, vec_totext(b) from t1; +a vec_totext(b) +1 [0.000000,0.000000] +1 [0.000000,0.000000] +drop table t1; +create table t1(v blob not null, vector index(v)); +ERROR HY000: Incorrect arguments to VECTOR INDEX +create table t1(v varbinary(100) not null, vector index(v)); +ERROR HY000: Incorrect arguments to VECTOR INDEX +create table t1(v binary not null, vector index(v)); +ERROR HY000: Incorrect arguments to VECTOR INDEX +create table t1 (a int, b vector(1536) not null, vector index(b)); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` vector(1536) NOT NULL, + VECTOR KEY `b` (`b`) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +# +# MDEV-34919 Aria crashes with high-level (vector) indexes +# # Aria doesn't support VECTOR yet -create table t1(v blob not null, vector index(v)) engine=aria; +create table t1(v vector(10) not null, vector index(v)) engine=aria; ERROR HY000: Table storage engine 'Aria' does not support the create option 'VECTOR' # # MDEV-35038 Server crash in Index_statistics::get_avg_frequency upon EITS collection for vector index # -create table t (a int, v blob not null, vector index (v)); +create table t (a int, v vector(10) not null, vector index (v)); analyze table t persistent for columns() indexes (v); Table Op Msg_type Msg_text test.t analyze status Engine-independent statistics collected test.t analyze status Table is already up to date drop table t; # -# MDEV-35033 LeakSanitizer errors in my_malloc / safe_mutex_lazy_init_deadlock_detection / MHNSW_Context::alloc_node and alike -# -call mtr.add_suppression('mariadbd: Can''t find record in ''t'''); -create table t (a int, v blob not null, vector index (v)); -insert into t values (1,vec_fromtext('[0]')); -update t set v = vec_fromtext('[0,0]'); -ERROR 22007: Incorrect vector value: '...' for column `test`.`t`.`v` at row 1 -update t set a = 2; -ERROR HY000: Can't find record in 't' -set global mhnsw_cache_size = 1048576; -insert into t values (2,x'00000000'); -drop table t; -set global mhnsw_cache_size = default; -# # MDEV-35029 ASAN errors in Lex_ident::is_valid_ident upon DDL on table with vector index # -create table t (a int, v blob not null, vector key (v) distance_function=euclidean); +create table t (a int, v vector(10) not null, vector key (v) distance_function=euclidean); show create table t; Table Create Table t CREATE TABLE `t` ( `a` int(11) DEFAULT NULL, - `v` blob NOT NULL, + `v` vector(10) NOT NULL, VECTOR KEY `v` (`v`) `distance_function`=euclidean ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci set session mhnsw_max_edges_per_node = @@mhnsw_max_edges_per_node + 1; @@ -42,27 +97,27 @@ drop table t, t2; # # MDEV-35043 Unsuitable error upon an attempt to create MEMORY table with vector key # -create table t (v binary(255) not null, vector index(v)) engine=memory; +create table t (v vector(31) not null, vector index(v)) engine=memory; ERROR HY000: Table storage engine 'MEMORY' does not support the create option 'VECTOR' # # MDEV-35042 Vector indexes are allowed for MERGE tables, but do not # -create table t (a int, v blob not null, vector index(v)) engine=myisam; -create table tm (a int, v blob not null, vector index(v)) engine=merge union=(t); +create table t (a int, v vector(10) not null, vector index(v)) engine=myisam; +create table tm (a int, v vector(10) not null, vector index(v)) engine=merge union=(t); ERROR HY000: Table storage engine 'MERGE' does not support the create option 'VECTOR' drop table t; # # MDEV-35078 Server crash or ASAN errors in mhnsw_insert # set session mhnsw_max_edges_per_node = 4; -create table t (a int, v blob not null); +create table t (a int, v vector(1) not null); insert into t select seq, x'00000000' from seq_1_to_10; alter table t add vector(v); show create table t; Table Create Table t CREATE TABLE `t` ( `a` int(11) DEFAULT NULL, - `v` blob NOT NULL, + `v` vector(1) NOT NULL, VECTOR KEY `v` (`v`) `max_edges_per_node`=4 ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci create table x like t; @@ -70,7 +125,7 @@ show create table x; Table Create Table x CREATE TABLE `x` ( `a` int(11) DEFAULT NULL, - `v` blob NOT NULL, + `v` vector(1) NOT NULL, VECTOR KEY `v` (`v`) `max_edges_per_node`=4 ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci insert into t values (11,x'00000000'); @@ -80,7 +135,7 @@ set session mhnsw_max_edges_per_node = default; # MDEV-35092 Server crash, hang or ASAN errors in mysql_create_frm_image upon using non-default table options and system variables # set mhnsw_distance_function= cosine; -create table t (a int, v blob not null); +create table t (a int, v vector(10) not null); prepare stmt from 'alter table t drop index if exists v, add vector (v) max_edges_per_node=10'; execute stmt; Warnings: @@ -90,7 +145,7 @@ show create table t; Table Create Table t CREATE TABLE `t` ( `a` int(11) DEFAULT NULL, - `v` blob NOT NULL, + `v` vector(10) NOT NULL, VECTOR KEY `v` (`v`) `max_edges_per_node`=10 `distance_function`='cosine' ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci drop table t; @@ -98,7 +153,7 @@ set mhnsw_distance_function= default; # # MDEV-35105 Assertion `tab->join->order' fails upon vector search with DISTINCT # -create table t (a int, v blob not null, vector(v)); +create table t (a int, v vector(1) not null, vector(v)); insert into t values(1,x'00000000'),(2,x'00000000'); select distinct a from t order by vec_distance_euclidean(v,vec_fromtext('[1]')) limit 1; a diff --git a/mysql-test/main/vector2.test b/mysql-test/main/vector2.test index 79c7e6ab397..6e5c0bb342d 100644 --- a/mysql-test/main/vector2.test +++ b/mysql-test/main/vector2.test @@ -3,40 +3,73 @@ # --source include/have_sequence.inc +--echo # +--echo # MDEV-33410 VECTOR data type +--echo # +--error ER_PARSE_ERROR +create table t1 (a int, b vector); +--error ER_WRONG_FIELD_SPEC +create table t1 (a int, b vector(0)); +--error ER_WRONG_FIELD_SPEC +create table t1 (a int, b vector(10) collate utf8mb3_general_ci); +create table t1 (a int, b vector(10)); +show create table t1; +--error ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION +insert t1 values (1, 1); +--error ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION +insert t1 values (1, 1.1); +--error ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION +insert t1 values (1, 1e1); +--error ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION +insert t1 values (1, now()); +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +insert t1 values (1, repeat(x'56', 10)); +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +insert t1 values (1, repeat(x'66', 40)); +insert t1 values (1, repeat(x'56', 40)); +select * from t1; +--error ER_ILLEGAL_PARAMETER_DATA_TYPE_FOR_OPERATION +select cast(b as char) from t1; +create table t2 as select b, cast(b as binary) from t1; +show create table t2; +drop table t1, t2; + +create table t1 (a int, b vector(1) not null); +insert into t1 values (1,x'00000000'); +alter table t1 modify b vector(2) not null; +insert into t1 values (1,x'0000000000000000'); +select a, vec_totext(b) from t1; +drop table t1; + +--error ER_WRONG_ARGUMENTS +create table t1(v blob not null, vector index(v)); +--error ER_WRONG_ARGUMENTS +create table t1(v varbinary(100) not null, vector index(v)); +--error ER_WRONG_ARGUMENTS +create table t1(v binary not null, vector index(v)); + +create table t1 (a int, b vector(1536) not null, vector index(b)); +show create table t1; +drop table t1; + +--echo # +--echo # MDEV-34919 Aria crashes with high-level (vector) indexes +--echo # --echo # Aria doesn't support VECTOR yet --error ER_ILLEGAL_HA_CREATE_OPTION -create table t1(v blob not null, vector index(v)) engine=aria; +create table t1(v vector(10) not null, vector index(v)) engine=aria; --echo # --echo # MDEV-35038 Server crash in Index_statistics::get_avg_frequency upon EITS collection for vector index --echo # -create table t (a int, v blob not null, vector index (v)); +create table t (a int, v vector(10) not null, vector index (v)); analyze table t persistent for columns() indexes (v); drop table t; ---echo # ---echo # MDEV-35033 LeakSanitizer errors in my_malloc / safe_mutex_lazy_init_deadlock_detection / MHNSW_Context::alloc_node and alike ---echo # -call mtr.add_suppression('mariadbd: Can''t find record in ''t'''); -create table t (a int, v blob not null, vector index (v)); -insert into t values (1,vec_fromtext('[0]')); - ---error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD -update t set v = vec_fromtext('[0,0]'); - ---error ER_KEY_NOT_FOUND -update t set a = 2; - -set global mhnsw_cache_size = 1048576; -insert into t values (2,x'00000000'); - -drop table t; -set global mhnsw_cache_size = default; - --echo # --echo # MDEV-35029 ASAN errors in Lex_ident::is_valid_ident upon DDL on table with vector index --echo # -create table t (a int, v blob not null, vector key (v) distance_function=euclidean); +create table t (a int, v vector(10) not null, vector key (v) distance_function=euclidean); show create table t; set session mhnsw_max_edges_per_node = @@mhnsw_max_edges_per_node + 1; create table t2 like t; @@ -47,21 +80,21 @@ drop table t, t2; --echo # MDEV-35043 Unsuitable error upon an attempt to create MEMORY table with vector key --echo # --error ER_ILLEGAL_HA_CREATE_OPTION -create table t (v binary(255) not null, vector index(v)) engine=memory; +create table t (v vector(31) not null, vector index(v)) engine=memory; --echo # --echo # MDEV-35042 Vector indexes are allowed for MERGE tables, but do not --echo # -create table t (a int, v blob not null, vector index(v)) engine=myisam; +create table t (a int, v vector(10) not null, vector index(v)) engine=myisam; --error ER_ILLEGAL_HA_CREATE_OPTION -create table tm (a int, v blob not null, vector index(v)) engine=merge union=(t); +create table tm (a int, v vector(10) not null, vector index(v)) engine=merge union=(t); drop table t; --echo # --echo # MDEV-35078 Server crash or ASAN errors in mhnsw_insert --echo # set session mhnsw_max_edges_per_node = 4; -create table t (a int, v blob not null); +create table t (a int, v vector(1) not null); insert into t select seq, x'00000000' from seq_1_to_10; alter table t add vector(v); show create table t; @@ -75,7 +108,7 @@ set session mhnsw_max_edges_per_node = default; --echo # MDEV-35092 Server crash, hang or ASAN errors in mysql_create_frm_image upon using non-default table options and system variables --echo # set mhnsw_distance_function= cosine; -create table t (a int, v blob not null); +create table t (a int, v vector(10) not null); prepare stmt from 'alter table t drop index if exists v, add vector (v) max_edges_per_node=10'; execute stmt; execute stmt; @@ -86,7 +119,7 @@ set mhnsw_distance_function= default; --echo # --echo # MDEV-35105 Assertion `tab->join->order' fails upon vector search with DISTINCT --echo # -create table t (a int, v blob not null, vector(v)); +create table t (a int, v vector(1) not null, vector(v)); insert into t values(1,x'00000000'),(2,x'00000000'); --replace_column 1 # select distinct a from t order by vec_distance_euclidean(v,vec_fromtext('[1]')) limit 1; diff --git a/mysql-test/main/vector2_notembedded.result b/mysql-test/main/vector2_notembedded.result index 1a72ca7e0b3..ea21f5b133d 100644 --- a/mysql-test/main/vector2_notembedded.result +++ b/mysql-test/main/vector2_notembedded.result @@ -1,7 +1,7 @@ # # mysqldump # -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); insert t1 (v) values (Vec_Fromtext('[0.418,0.809,0.823,0.598,0.033]')), (Vec_Fromtext('[0.687,0.789,0.496,0.574,0.917]')), (Vec_Fromtext('[0.333,0.962,0.467,0.448,0.475]')), @@ -22,7 +22,7 @@ id /*!40101 SET character_set_client = utf8 */; CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci; @@ -42,7 +42,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `id` int(11) NOT NULL AUTO_INCREMENT, - `v` blob NOT NULL, + `v` vector(5) NOT NULL, PRIMARY KEY (`id`), VECTOR KEY `v` (`v`) ) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci @@ -67,7 +67,7 @@ drop table t1; # # MDEV-35044 ALTER on a table with vector index attempts to bypass unsupported locking limitation, server crashes in THD::free_tmp_table_share # -create table t (a int primary key, v blob not null, vector index(v)); +create table t (a int primary key, v vector(10) not null, vector index(v)); alter table t modify a int auto_increment, lock=none; ERROR 0A000: LOCK=NONE is not supported. Reason: CHANGE COLUMN ... AUTO_INCREMENT. Try LOCK=SHARED drop table t; diff --git a/mysql-test/main/vector2_notembedded.test b/mysql-test/main/vector2_notembedded.test index 7af77da4f1d..ef581cb9470 100644 --- a/mysql-test/main/vector2_notembedded.test +++ b/mysql-test/main/vector2_notembedded.test @@ -3,7 +3,7 @@ --echo # --echo # mysqldump --echo # -create table t1 (id int auto_increment primary key, v blob not null, vector index (v)); +create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)); insert t1 (v) values (Vec_Fromtext('[0.418,0.809,0.823,0.598,0.033]')), (Vec_Fromtext('[0.687,0.789,0.496,0.574,0.917]')), (Vec_Fromtext('[0.333,0.962,0.467,0.448,0.475]')), @@ -28,7 +28,7 @@ drop table t1; --echo # --echo # MDEV-35044 ALTER on a table with vector index attempts to bypass unsupported locking limitation, server crashes in THD::free_tmp_table_share --echo # -create table t (a int primary key, v blob not null, vector index(v)); +create table t (a int primary key, v vector(10) not null, vector index(v)); --error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON alter table t modify a int auto_increment, lock=none; drop table t; diff --git a/mysql-test/main/vector_funcs.result b/mysql-test/main/vector_funcs.result index d64d5d0ed87d41d7e8fd550d0f1d89007c4e1be1..5ca3584aa88259d84d7eb46dd1f76d9c919e079c 100644 GIT binary patch delta 20 bcmaE*_Fip*8E08)a!G!XhN + typedef ulonglong sql_mode_t; typedef int64 query_id_t; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 83a94b307d1..93c62b7582b 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2762,34 +2762,6 @@ Type_handler_blob_common::Key_part_spec_init_ft(Key_part_spec *part, } -bool Type_handler_string::Key_part_spec_init_vector(Key_part_spec *part, - const Column_definition &def) - const -{ - part->length= 0; - return def.charset != &my_charset_bin; -} - - -bool Type_handler_varchar::Key_part_spec_init_vector(Key_part_spec *part, - const Column_definition &def) - const -{ - part->length= 0; - return def.charset != &my_charset_bin; -} - - -bool -Type_handler_blob_common::Key_part_spec_init_vector(Key_part_spec *part, - const Column_definition &def) - const -{ - part->length= 1; - return def.charset != &my_charset_bin; -} - - static bool key_add_part_check_null(const handler *file, KEY *key_info, const Column_definition *sql_field, @@ -3578,7 +3550,7 @@ mysql_prepare_create_table_finalize(THD *thd, HA_CREATE_INFO *create_info, DBUG_RETURN(TRUE); } if (key_part_length > file->max_key_part_length() && - key->type != Key::FULLTEXT) + key->type != Key::FULLTEXT && key->type != Key::VECTOR) { if (key->type == Key::MULTIPLE) { @@ -3641,7 +3613,7 @@ mysql_prepare_create_table_finalize(THD *thd, HA_CREATE_INFO *create_info, if (key_info->key_length > max_key_length && key->type == Key::UNIQUE) is_hash_field_needed= true; // for case "a BLOB UNIQUE" if (key_length > max_key_length && key->type != Key::FULLTEXT && - !is_hash_field_needed) + key->type != Key::VECTOR && !is_hash_field_needed) { my_error(ER_TOO_LONG_KEY, MYF(0), max_key_length); DBUG_RETURN(TRUE); diff --git a/sql/sql_type.cc b/sql/sql_type.cc index d4d9b520bfe..27a716df355 100644 --- a/sql/sql_type.cc +++ b/sql/sql_type.cc @@ -17,6 +17,7 @@ #include "mariadb.h" #include "sql_type.h" #include "sql_type_geom.h" +#include "sql_type_vector.h" #include "sql_const.h" #include "sql_class.h" #include "sql_time.h" @@ -289,6 +290,8 @@ Type_handler::handler_by_name(THD *thd, const LEX_CSTRING &name) } const Type_handler *ha= Type_collection_geometry_handler_by_name(name); + if (!ha && type_handler_vector.name().eq(name)) + return &type_handler_vector; return ha; } diff --git a/sql/sql_type.h b/sql/sql_type.h index 95766222801..8d070b26610 100644 --- a/sql/sql_type.h +++ b/sql/sql_type.h @@ -7228,8 +7228,6 @@ public: ulonglong table_flags) const override; bool Key_part_spec_init_ft(Key_part_spec *part, const Column_definition &def) const override; - bool Key_part_spec_init_vector(Key_part_spec *part, - const Column_definition &def) const override; Field *make_table_field(MEM_ROOT *root, const LEX_CSTRING *name, const Record_addr &addr, @@ -7327,8 +7325,6 @@ public: ulonglong table_flags) const override; bool Key_part_spec_init_ft(Key_part_spec *part, const Column_definition &def) const override; - bool Key_part_spec_init_vector(Key_part_spec *part, - const Column_definition &def) const override; Field *make_table_field(MEM_ROOT *root, const LEX_CSTRING *name, const Record_addr &addr, @@ -7431,8 +7427,6 @@ public: uchar *buff) const override; bool Key_part_spec_init_ft(Key_part_spec *part, const Column_definition &def) const override; - bool Key_part_spec_init_vector(Key_part_spec *part, - const Column_definition &def) const override; bool Key_part_spec_init_primary(Key_part_spec *part, const Column_definition &def, const handler *file) const override; diff --git a/sql/sql_type_vector.cc b/sql/sql_type_vector.cc new file mode 100644 index 00000000000..073e19ffc76 --- /dev/null +++ b/sql/sql_type_vector.cc @@ -0,0 +1,317 @@ +/* + Copyright (c) 2024, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + +#include "sql_type_vector.h" +#include "sql_class.h" +#include "sql_lex.h" + +Named_type_handler type_handler_vector("vector"); +Type_collection_vector type_collection_vector; + +const Type_collection *Type_handler_vector::type_collection() const +{ + return &type_collection_vector; +} + +const Type_handler *Type_collection_vector::aggregate_for_comparison( + const Type_handler *a, const Type_handler *b) const +{ + if (a->type_collection() == this) + swap_variables(const Type_handler *, a, b); + if (a == &type_handler_vector || a == &type_handler_hex_hybrid || + a == &type_handler_tiny_blob || a == &type_handler_blob || + a == &type_handler_medium_blob || a == &type_handler_long_blob || + a == &type_handler_varchar || a == &type_handler_string) + return b; + return NULL; +} + +const Type_handler *Type_collection_vector::aggregate_for_result( + const Type_handler *a, const Type_handler *b) const +{ + return aggregate_for_comparison(a,b); +} + +const Type_handler *Type_collection_vector::aggregate_for_min_max( + const Type_handler *a, const Type_handler *b) const +{ + return aggregate_for_comparison(a,b); +} + +const Type_handler *Type_collection_vector::aggregate_for_num_op( + const Type_handler *a, const Type_handler *b) const +{ + return NULL; +} + +const Type_handler *Type_handler_vector::type_handler_for_comparison() const +{ + return &type_handler_vector; +} + +Field *Type_handler_vector::make_conversion_table_field( + MEM_ROOT *root, TABLE *table, uint metadata, const Field *target) const +{ + return new (root) Field_vector(NULL, (uchar *) "", 1, Field::NONE, + &empty_clex_str, table->s, metadata); +} + +bool Type_handler_vector::Column_definition_fix_attributes( + Column_definition *def) const +{ + if (def->length == 0 || def->charset != &my_charset_bin) + { + my_error(ER_WRONG_FIELD_SPEC, MYF(0), def->field_name.str); + return true; + } + def->length*= sizeof(float); + return false; +} + +bool Type_handler_vector::Key_part_spec_init_vector(Key_part_spec *part, + const Column_definition &def) const +{ + if (part->length) + { + my_error(ER_WRONG_SUB_KEY, MYF(0)); + return true; + } + return false; +} + +Item *Type_handler_vector::create_typecast_item(THD *thd, Item *item, + const Type_cast_attributes &attr) const +{ + //return new (thd->mem_root) Item_typecast_vector(thd, item); + return NULL; +} + +Field *Type_handler_vector::make_table_field(MEM_ROOT *root, + const LEX_CSTRING *name, const Record_addr &addr, + const Type_all_attributes &attr, TABLE_SHARE *share) const +{ + return new (root) Field_vector(addr.ptr(), addr.null_ptr(), addr.null_bit(), + Field::NONE, name, share, attr.max_length); +} + +bool Type_handler_vector::Item_hybrid_func_fix_attributes(THD *thd, + const LEX_CSTRING &func_name, Type_handler_hybrid_field_type *handler, + Type_all_attributes *func, Item **items, uint nitems) const +{ + if (func->aggregate_attributes_string(func_name, items, nitems)) + return true; + //func->set_type_maybe_null(true); + return false; +} + +bool Type_handler_vector::Item_sum_sum_fix_length_and_dec( + Item_sum_sum *item) const +{ + LEX_CSTRING name= {STRING_WITH_LEN("sum") }; + return Item_func_or_sum_illegal_param(name); +} + +bool Type_handler_vector::Item_sum_avg_fix_length_and_dec( + Item_sum_avg *item) const +{ + LEX_CSTRING name= {STRING_WITH_LEN("avg") }; + return Item_func_or_sum_illegal_param(name); +} + +bool Type_handler_vector::Item_sum_variance_fix_length_and_dec( + Item_sum_variance *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_func_round_fix_length_and_dec( + Item_func_round *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_func_int_val_fix_length_and_dec( + Item_func_int_val *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_func_abs_fix_length_and_dec(Item_func_abs *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_func_neg_fix_length_and_dec(Item_func_neg *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_func_signed_fix_length_and_dec( + Item_func_signed *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_func_unsigned_fix_length_and_dec( + Item_func_unsigned *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_double_typecast_fix_length_and_dec( + Item_double_typecast *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_float_typecast_fix_length_and_dec( + Item_float_typecast *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_decimal_typecast_fix_length_and_dec( + Item_decimal_typecast *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_time_typecast_fix_length_and_dec( + Item_time_typecast *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_date_typecast_fix_length_and_dec( + Item_date_typecast *item) const +{ + return Item_func_or_sum_illegal_param(item); +} + +bool Type_handler_vector::Item_datetime_typecast_fix_length_and_dec( + Item_datetime_typecast *item) const +{ + return Item_func_or_sum_illegal_param(item); + +} + +bool Type_handler_vector::Item_char_typecast_fix_length_and_dec( + Item_char_typecast *item) const +{ + if (item->cast_charset() != &my_charset_bin) // XXX todo + return Item_func_or_sum_illegal_param(item); // CAST(vector AS CHAR) + item->fix_length_and_dec_str(); + return false; // CAST(vector AS BINARY) +} + +Field *Type_handler_vector::make_table_field_from_def(TABLE_SHARE *share, + MEM_ROOT *root, const LEX_CSTRING *name, const Record_addr &rec, + const Bit_addr &bit, const Column_definition_attributes *attr, + uint32 flags) const +{ + return new (root) Field_vector(rec.ptr(), rec.null_ptr(), rec.null_bit(), + attr->unireg_check, name, share, static_cast(attr->length)); +} + +/*****************************************************************/ +void Field_vector::sql_type(String &res) const +{ + res.set_ascii(STRING_WITH_LEN("vector")); + res.append_parenthesized(field_length/sizeof(float)); +} + +int Field_vector::reset() +{ + int res= Field_varstring::reset(); + store_length(field_length); + return res; +} + +static void do_copy_vec(const Copy_field *copy) +{ + uint from_length_bytes= 1 + (copy->from_length > 258); + uint to_length_bytes= 1 + (copy->to_length > 258); + uint from_length= copy->from_length - from_length_bytes; + uint to_length= copy->to_length - to_length_bytes; + uchar *from= copy->from_ptr + from_length_bytes; + uchar *to= copy->to_ptr + to_length_bytes; + + if (to_length_bytes == 1) + *copy->to_ptr= to_length; + else + int2store(copy->to_ptr, to_length); + + if (from_length > to_length) + memcpy(to, from, to_length); + else + { + memcpy(to, from, from_length); + bzero(to + from_length, to_length - from_length); + } +} + +Field::Copy_func *Field_vector::get_copy_func(const Field *from) const +{ + if (field_length == from->field_length) + return do_field_eq; + return do_copy_vec; +} + +int Field_vector::report_wrong_value(const ErrConv &val) const +{ + my_error(ER_TRUNCATED_WRONG_VALUE_FOR_FIELD, MYF(0), "vector", + val.ptr(), table->s->db.str, table->s->table_name.str, + field_name.str, table->in_use->get_stmt_da()->current_row_for_warning()); + return -1; +} + +int Field_vector::store(double nr) +{ + DBUG_ASSERT(0); + return report_wrong_value(ErrConvDouble(nr)); +} + +int Field_vector::store(longlong nr, bool unsigned_val) +{ + DBUG_ASSERT(0); + return report_wrong_value(ErrConvInteger(Longlong_hybrid(nr, unsigned_val))); +} + +int Field_vector::store_decimal(const my_decimal *nr) +{ + DBUG_ASSERT(0); + return report_wrong_value(ErrConvDecimal(nr)); +} + +int Field_vector::store(const char *from, size_t length, CHARSET_INFO *cs) +{ + if (cs != &my_charset_bin) // XXX todo + return report_wrong_value(ErrConvString(from, length, cs)); + + if (length != field_length) + return report_wrong_value(ErrConvString(from, length, cs)); + + float abs2= 0.0f; + for (const char *v= from, *end= from+length; v < end; v+= sizeof(float)) + { + float val= get_float(v); + abs2+= val*val; + } + if (!std::isfinite(abs2)) + return report_wrong_value(ErrConvString(from, length, cs)); + + return Field_varstring::store(from, length, cs); +} diff --git a/sql/sql_type_vector.h b/sql/sql_type_vector.h new file mode 100644 index 00000000000..62b0ace3c13 --- /dev/null +++ b/sql/sql_type_vector.h @@ -0,0 +1,130 @@ +#ifndef SQL_TYPE_VECTOR_INCLUDED +#define SQL_TYPE_VECTOR_INCLUDED +/* + Copyright (c) 2024 MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + +#include "sql_type.h" + +class Type_handler_vector: public Type_handler_varchar +{ +public: + virtual ~Type_handler_vector() {} + const Type_collection *type_collection() const override; + const Type_handler *type_handler_for_comparison() const override; + virtual Item *create_typecast_item(THD *thd, Item *item, + const Type_cast_attributes &attr) const override; + bool subquery_type_allows_materialization(const Item *, const Item *, bool) + const override + { + return false; // XXX for simplicity + } + Field *make_conversion_table_field(MEM_ROOT *root, + TABLE *table, uint metadata, + const Field *target) const override; + Log_event_data_type user_var_log_event_data_type(uint charset_nr) + const override + { + return Log_event_data_type(name().lex_cstring(), result_type(), + charset_nr, false); + } + + bool Column_definition_fix_attributes(Column_definition *c) const override; + bool Key_part_spec_init_vector(Key_part_spec *part, + const Column_definition &def) const override; + Field *make_table_field(MEM_ROOT *root, const LEX_CSTRING *name, + const Record_addr &addr, const Type_all_attributes &attr, + TABLE_SHARE *share) const override; + + Field *make_table_field_from_def(TABLE_SHARE *share, MEM_ROOT *mem_root, + const LEX_CSTRING *name, const Record_addr &addr, + const Bit_addr &bit, const Column_definition_attributes *attr, + uint32 flags) const override; + + bool can_return_int() const override { return false; } + bool can_return_decimal() const override { return false; } + bool can_return_real() const override { return false; } + bool can_return_text() const override { return false; } // XXX + bool can_return_date() const override { return false; } + bool can_return_time() const override { return false; } + bool Item_func_round_fix_length_and_dec(Item_func_round *) const override; + bool Item_func_int_val_fix_length_and_dec(Item_func_int_val *) const override; + bool Item_func_abs_fix_length_and_dec(Item_func_abs *) const override; + bool Item_func_neg_fix_length_and_dec(Item_func_neg *) const override; + bool Item_hybrid_func_fix_attributes(THD *thd, const LEX_CSTRING &name, + Type_handler_hybrid_field_type *h, Type_all_attributes *attr, + Item **items, uint nitems) const override; + bool Item_sum_sum_fix_length_and_dec(Item_sum_sum *) const override; + bool Item_sum_avg_fix_length_and_dec(Item_sum_avg *) const override; + bool Item_sum_variance_fix_length_and_dec(Item_sum_variance *) const override; + bool Item_func_signed_fix_length_and_dec(Item_func_signed *) const override; + bool Item_func_unsigned_fix_length_and_dec(Item_func_unsigned *) const + override; + bool Item_double_typecast_fix_length_and_dec(Item_double_typecast *) const + override; + bool Item_float_typecast_fix_length_and_dec(Item_float_typecast *) const + override; + bool Item_decimal_typecast_fix_length_and_dec(Item_decimal_typecast *) const + override; + bool Item_char_typecast_fix_length_and_dec(Item_char_typecast *) const + override; + bool Item_time_typecast_fix_length_and_dec(Item_time_typecast *) const + override; + bool Item_date_typecast_fix_length_and_dec(Item_date_typecast *) const + override; + bool Item_datetime_typecast_fix_length_and_dec(Item_datetime_typecast *) const + override; +}; + +extern Named_type_handler type_handler_vector; + +class Type_collection_vector: public Type_collection +{ +public: + const Type_handler *aggregate_for_result( + const Type_handler *a, const Type_handler *b) const override; + const Type_handler *aggregate_for_comparison( + const Type_handler *a, const Type_handler *b) const override; + const Type_handler *aggregate_for_min_max( + const Type_handler *a, const Type_handler *b) const override; + const Type_handler *aggregate_for_num_op( + const Type_handler *a, const Type_handler *b) const override; +}; + +#include "field.h" + +class Field_vector:public Field_varstring +{ + int report_wrong_value(const ErrConv &val) const; +public: + Field_vector(uchar *ptr_arg, uchar *null_ptr_arg, uchar null_bit_arg, + enum utype unireg_check_arg, const LEX_CSTRING *field_name_arg, + TABLE_SHARE *share, uint len_arg) + : Field_varstring(ptr_arg, len_arg, len_arg < 256 ? 1 :2, null_ptr_arg, + null_bit_arg, unireg_check_arg, field_name_arg, share, + &my_charset_bin) { } + const Type_handler *type_handler() const override + { return &type_handler_vector; } + void sql_type(String &str) const override; + int reset() override; + Copy_func *get_copy_func(const Field *from) const override; + int store(const char *to, size_t length, CHARSET_INFO *charset) override; + int store(double nr) override; + int store(longlong nr, bool unsigned_val) override; + int store_decimal(const my_decimal *) override; + uint size_of() const override { return sizeof(*this); } +}; + +#endif // SQL_TYPE_VECTOR_INCLUDED diff --git a/sql/vector_mhnsw.cc b/sql/vector_mhnsw.cc index 98f064685a4..ff5e5e2cc40 100644 --- a/sql/vector_mhnsw.cc +++ b/sql/vector_mhnsw.cc @@ -1122,15 +1122,6 @@ static int search_layer(MHNSW_Share *ctx, TABLE *graph, const FVector *target, } -static int bad_value_on_insert(Field *f) -{ - my_error(ER_TRUNCATED_WRONG_VALUE_FOR_FIELD, MYF(0), "vector", "...", - f->table->s->db.str, f->table->s->table_name.str, f->field_name.str, - f->table->in_use->get_stmt_da()->current_row_for_warning()); - return my_errno= HA_ERR_GENERIC; -} - - int mhnsw_insert(TABLE *table, KEY *keyinfo) { THD *thd= table->in_use; @@ -1148,13 +1139,7 @@ int mhnsw_insert(TABLE *table, KEY *keyinfo) DBUG_ASSERT(vec_field->cmp_type() == STRING_RESULT); DBUG_ASSERT(res); // ER_INDEX_CANNOT_HAVE_NULL DBUG_ASSERT(table->file->ref_length <= graph->field[FIELD_TREF]->field_length); - - // XXX returning an error here will rollback the insert in InnoDB - // but in MyISAM the row will stay inserted, making the index out of sync: - // invalid vector values are present in the table but cannot be found - // via an index. The easiest way to fix it is with a VECTOR(N) type - if (res->length() == 0 || res->length() % 4) - return bad_value_on_insert(vec_field); + DBUG_ASSERT(res->length() > 0 && res->length() % 4 == 0); table->file->position(table->record[0]); @@ -1175,7 +1160,7 @@ int mhnsw_insert(TABLE *table, KEY *keyinfo) } if (ctx->byte_len != res->length()) - return bad_value_on_insert(vec_field); + return my_errno= HA_ERR_CRASHED; MEM_ROOT_SAVEPOINT memroot_sv; root_make_savepoint(thd->mem_root, &memroot_sv); diff --git a/storage/rocksdb/mysql-test/rocksdb/r/vector.result b/storage/rocksdb/mysql-test/rocksdb/r/vector.result index b75ad358f79..300a4cfdfa9 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/vector.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/vector.result @@ -1,3 +1,3 @@ # RocksDB doesn't support VECTOR yet -create table t1(v blob not null, vector index(v)) engine=rocksdb; +create table t1(v vector(1) not null, vector index(v)) engine=rocksdb; ERROR HY000: Table storage engine 'RocksDB' does not support the create option 'VECTOR' diff --git a/storage/rocksdb/mysql-test/rocksdb/t/vector.test b/storage/rocksdb/mysql-test/rocksdb/t/vector.test index 4b0bb0eeffa..0629be480ba 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/vector.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/vector.test @@ -2,4 +2,4 @@ --echo # RocksDB doesn't support VECTOR yet --error ER_ILLEGAL_HA_CREATE_OPTION -create table t1(v blob not null, vector index(v)) engine=rocksdb; +create table t1(v vector(1) not null, vector index(v)) engine=rocksdb;