Add tests for the trigram tokenizer. Fix minor issues.

FossilOrigin-Name: 897ced99b44085012aa44d3264940dcbd4c77b295a894a1b58fb2c03a0f7fee8

ext/fts5/fts5_expr.c

@@ -284,6 +284,14 @@ int sqlite3Fts5ExprNew(
   return sParse.rc;
 }
 
+/*
+** This function is only called when using the special 'trigram' tokenizer.
+** Argument zText contains the text of a LIKE or GLOB pattern matched
+** against column iCol. This function creates and compiles an FTS5 MATCH
+** expression that will match a superset of the rows matched by the LIKE or
+** GLOB. If successful, SQLITE_OK is returned. Otherwise, an SQLite error
+** code.
+*/
 int sqlite3Fts5ExprPattern(
   Fts5Config *pConfig, int iCol, const char *zText, Fts5Expr **pp
 ){

ext/fts5/fts5_tokenize.c

@@ -1261,10 +1261,9 @@ static int fts5PorterTokenize(
 /**************************************************************************
 ** Start of trigram implementation.
 */
-
 typedef struct TrigramTokenizer TrigramTokenizer;
 struct TrigramTokenizer {
-  int bFold;
+  int bFold;                      /* True to fold to lower-case */
 };
 
 /*
@@ -1359,6 +1358,17 @@ static int fts5TriTokenize(
   return rc;
 }
 
+/*
+** Argument xCreate is a pointer to a constructor function for a tokenizer.
+** pTok is a tokenizer previously created using the same method. This function
+** returns one of FTS5_PATTERN_NONE, FTS5_PATTERN_LIKE or FTS5_PATTERN_GLOB
+** indicating the style of pattern matching that the tokenizer can support.
+** In practice, this is:
+**
+**     "trigram" tokenizer, case_sensitive=1 - FTS5_PATTERN_GLOB
+**     "trigram" tokenizer, case_sensitive=0 (the default) - FTS5_PATTERN_LIKE
+**     all other tokenizers - FTS5_PATTERN_NONE
+*/
 int sqlite3Fts5TokenizerPattern(
     int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
     Fts5Tokenizer *pTok

ext/fts5/test/fts5faultD.test

@@ -14,7 +14,7 @@
 
 source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
-set testprefix fts5faultA
+set testprefix fts5faultD
 
 # If SQLITE_ENABLE_FTS3 is defined, omit this file.
 ifcapable !fts5 {

ext/fts5/test/fts5faultE.test

@@ -0,0 +1,53 @@
+# 2016 February 2
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultE
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+faultsim_save_and_close
+do_faultsim_test 1 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { CREATE VIRTUAL TABLE t1 USING fts5(x, y, tokenize=trigram) }
+} -test {
+  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} 
+}
+
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, tokenize=trigram);
+}
+
+faultsim_save_and_close
+do_faultsim_test 2 -faults ioerr-t* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    INSERT INTO t1 VALUES('abcdefghijklmnopqrstuvwxyz', NULL);
+    SELECT count(*) FROM t1 WHERE x LIKE '%mnop%' AND t1 MATCH 'jkl';
+  }
+} -test {
+  faultsim_test_result {0 1} {1 {vtable constructor failed: t1}}
+}
+
+
+finish_test
+

ext/fts5/test/fts5trigram.test

@@ -117,5 +117,18 @@ foreach {tn like res} {
   } $res
 }
 
+#-------------------------------------------------------------------------
+reset_db
+do_catchsql_test 3.1 {
+  CREATE VIRTUAL TABLE ttt USING fts5(c, tokenize="trigram case_sensitive 2");
+} {1 {error in tokenizer constructor}}
+do_catchsql_test 3.2 {
+  CREATE VIRTUAL TABLE ttt USING fts5(c, tokenize="trigram case_sensitive 11");
+} {1 {error in tokenizer constructor}}
+do_catchsql_test 3.3 {
+  CREATE VIRTUAL TABLE ttt USING fts5(c, "tokenize=trigram case_sensitive 1");
+} {0 {}}
+
+
 finish_test