Fix another case where malformed utf-8 was being mishandled in fts5. Fix for [df46a6f3].

FossilOrigin-Name: 1c0a05b09a97e6e2e9b11c31ed6ec7e6484686614b587ebfd0cfe27d973ba461

ext/fts5/fts5_index.c

@@ -5318,10 +5318,13 @@ int sqlite3Fts5IndexCharlenToBytelen(
   for(i=0; i<nChar; i++){
     if( n>=nByte ) return 0;      /* Input contains fewer than nChar chars */
     if( (unsigned char)p[n++]>=0xc0 ){
-      if( n>=nByte ) break;
+      if( n>=nByte ) return 0;
       while( (p[n] & 0xc0)==0x80 ){
         n++;
-        if( n>=nByte ) break;
+        if( n>=nByte ){
+          if( i+1==nChar ) break;
+          return 0;
+        }
       }
     }
   }

ext/fts5/test/fts5misc.test

@@ -266,5 +266,22 @@ do_execsql_test 10.2 {
   INSERT INTO vt1(vt1) VALUES('integrity-check');
 }
 
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 11.0 {
+  CREATE VIRTUAL TABLE vt0 USING fts5(
+      c0, prefix = 71, tokenize = "porter ascii", prefix = 9
+  );
+} {}
+
+do_execsql_test 11.1 {
+  BEGIN;
+  INSERT INTO vt0(c0) VALUES (x'e8');
+}
+do_execsql_test 11.2 {
+  INSERT INTO vt0(vt0) VALUES('integrity-check');
+}
+
 finish_test
 

manifest

@@ -1,5 +1,5 @@
-C Fix\san\sassert()\sadded\sas\spart\sof\scommit\s[a11b393dc]\sthat\scan\sfail\sif\sfts5\sdatabase\srecords\sare\scorrupt.
-D 2019-12-24T15:35:53.224
+C Fix\sanother\scase\swhere\smalformed\sutf-8\swas\sbeing\smishandled\sin\sfts5.\sFix\sfor\s[df46a6f3].
+D 2019-12-24T16:20:05.755
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -116,7 +116,7 @@ F ext/fts5/fts5_buffer.c 5a5fe0159752c0fb0a5a93c722e9db2662822709490769d482b76a6
 F ext/fts5/fts5_config.c b447948f35ad3354e8fe5e242e0a7e7b5b941555400b9404259944e3aa570037
 F ext/fts5/fts5_expr.c 2be456484786333d559dc2987a00f2750981fab91d52db8452a8046278c5f22e
 F ext/fts5/fts5_hash.c 1cc0095646f5f3b46721aa112fb4f9bf29ae175cb5338f89dcec66ed97acfe75
-F ext/fts5/fts5_index.c 784559cf785ea1a7ff99c65724f39e45a1e74b627e458928a1e96859594c7140
+F ext/fts5/fts5_index.c f704b61e134109de766ce98100a35204ec3e64efc37bf87b30e04ce17755452d
 F ext/fts5/fts5_main.c 9db1f173d299466aeff89bd949fb1eb0a181265726fb56f11e07ea292dcc9a73
 F ext/fts5/fts5_storage.c 3ecda8edadc1f62a355d6789776be0da609f8658c50d72e422674093ab7e1528
 F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95
@@ -189,7 +189,7 @@ F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc27826807405
 F ext/fts5/test/fts5matchinfo.test 50d86da66ec5b27603dcd90ba0227f5d9deb10351cbc52974a88e24f6fc9b076
 F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6
 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2
-F ext/fts5/test/fts5misc.test b294b1d7ad814da30e473905a8165de1bfe137822d243f2ab8cbf20ecc37bd1e
+F ext/fts5/test/fts5misc.test bd6256df641e76fc0ccd9558707e39fed80c033ce3abb8c9a162cd68e0c18dda
 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581
 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45
 F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd
@@ -1852,7 +1852,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0a70f5ddaf3567f335cc0fd0fcd5303e2afb5e38a6388faddf5cab1f2a74e3dd
-R c925e095c0b909e03163107ccc245144
+P 4630c1ec013a4c2e54a34c9a64b2c803ac3912450de660497eb34ee21c91f426
+R 1a8908752b44d0f45dd43aab7ad0fcac
 U dan
-Z 4752e99a5dbec9811e8b98a3225a221a
+Z 13978aa5759e244af756578de8231027

manifest.uuid

@@ -1 +1 @@
-4630c1ec013a4c2e54a34c9a64b2c803ac3912450de660497eb34ee21c91f426
+1c0a05b09a97e6e2e9b11c31ed6ec7e6484686614b587ebfd0cfe27d973ba461