From feb5dcf1a67073c8d48f05d0ee8658139ea403b0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 17 Nov 2019 02:41:06 +0000
Subject: [PATCH] Better detection of corruption in the %_stat and %_docsize
 shadow tables of FTS3.

FossilOrigin-Name: 1e449687881f4d388e54a0e51bcabba41ab10cf7e596ff65e31e88a23c70d497
---
 ext/fts3/fts3.c         | 26 ++++++++++++++++++++++++++
 ext/fts3/fts3Int.h      |  1 +
 ext/fts3/fts3_snippet.c | 39 ++++++++++++++++++++++++++++++++-------
 ext/fts3/fts3_write.c   |  2 +-
 manifest                | 20 ++++++++++----------
 manifest.uuid           |  2 +-
 test/fts4aa.test        | 38 ++++++++++++++++++++++++++++++++++++++
 7 files changed, 109 insertions(+), 19 deletions(-)

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index e8e909a97b..f177538805 100644
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -391,6 +391,32 @@ int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
   return (int)(p - pStart);
 }
 
+/* 
+** Read a 64-bit variable-length integer from memory starting at p[0] and
+** not extending past pEnd[-1].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+int sqlite3Fts3GetVarintBounded(
+  const char *pBuf,
+  const char *pEnd,
+  sqlite_int64 *v
+){
+  const unsigned char *p = (const unsigned char*)pBuf;
+  const unsigned char *pStart = p;
+  const unsigned char *pX = (const unsigned char*)pEnd;
+  u64 b = 0;
+  int shift;
+  for(shift=0; shift<=63; shift+=7){
+    u64 c = p<pX ? *p : 0;
+    p++;
+    b += (c&0x7F) << shift;
+    if( (c & 0x80)==0 ) break;
+  }
+  *v = b;
+  return (int)(p - pStart);
+}
+
 /*
 ** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to 
 ** a non-negative 32-bit integer before it is returned.
diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h
index 3c58e49583..3833e067e4 100644
--- a/ext/fts3/fts3Int.h
+++ b/ext/fts3/fts3Int.h
@@ -577,6 +577,7 @@ int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
 void sqlite3Fts3ErrMsg(char**,const char*,...);
 int sqlite3Fts3PutVarint(char *, sqlite3_int64);
 int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
+int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*);
 int sqlite3Fts3GetVarint32(const char *, int *);
 int sqlite3Fts3VarintLen(sqlite3_uint64);
 void sqlite3Fts3Dequote(char *);
diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c
index 6db3fd7d96..b8e2bbaac9 100644
--- a/ext/fts3/fts3_snippet.c
+++ b/ext/fts3/fts3_snippet.c
@@ -1038,11 +1038,15 @@ static int fts3MatchinfoSelectDoctotal(
   Fts3Table *pTab,
   sqlite3_stmt **ppStmt,
   sqlite3_int64 *pnDoc,
-  const char **paLen
+  const char **paLen,
+  const char **ppEnd
 ){
   sqlite3_stmt *pStmt;
   const char *a;
+  const char *pEnd;
   sqlite3_int64 nDoc;
+  int n;
+
 
   if( !*ppStmt ){
     int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
@@ -1051,12 +1055,23 @@ static int fts3MatchinfoSelectDoctotal(
   pStmt = *ppStmt;
   assert( sqlite3_data_count(pStmt)==1 );
 
+  n = sqlite3_column_bytes(pStmt, 0);
+  if( n==0 ){
+    return FTS_CORRUPT_VTAB;
+  }
   a = sqlite3_column_blob(pStmt, 0);
-  a += sqlite3Fts3GetVarint(a, &nDoc);
-  if( nDoc==0 ) return FTS_CORRUPT_VTAB;
+  if( a==0 ){
+    return SQLITE_NOMEM;
+  }
+  pEnd = a + n;
+  a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc);
+  if( nDoc==0 || a>pEnd ){
+    return FTS_CORRUPT_VTAB;
+  }
   *pnDoc = (u32)nDoc;
 
   if( paLen ) *paLen = a;
+  if( ppEnd ) *ppEnd = pEnd;
   return SQLITE_OK;
 }
 
@@ -1237,7 +1252,7 @@ static int fts3MatchinfoValues(
       case FTS3_MATCHINFO_NDOC:
         if( bGlobal ){
           sqlite3_int64 nDoc = 0;
-          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0);
+          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0, 0);
           pInfo->aMatchinfo[0] = (u32)nDoc;
         }
         break;
@@ -1246,14 +1261,19 @@ static int fts3MatchinfoValues(
         if( bGlobal ){
           sqlite3_int64 nDoc;     /* Number of rows in table */
           const char *a;          /* Aggregate column length array */
+          const char *pEnd;       /* First byte past end of length array */
 
-          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a);
+          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a, &pEnd);
           if( rc==SQLITE_OK ){
             int iCol;
             for(iCol=0; iCol<pInfo->nCol; iCol++){
               u32 iVal;
               sqlite3_int64 nToken;
               a += sqlite3Fts3GetVarint(a, &nToken);
+              if( a>pEnd ){
+                rc = SQLITE_CORRUPT_VTAB;
+                break;
+              }
               iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc);
               pInfo->aMatchinfo[iCol] = iVal;
             }
@@ -1267,9 +1287,14 @@ static int fts3MatchinfoValues(
         if( rc==SQLITE_OK ){
           int iCol;
           const char *a = sqlite3_column_blob(pSelectDocsize, 0);
+          const char *pEnd = a + sqlite3_column_bytes(pSelectDocsize, 0);
           for(iCol=0; iCol<pInfo->nCol; iCol++){
             sqlite3_int64 nToken;
-            a += sqlite3Fts3GetVarint(a, &nToken);
+            a += sqlite3Fts3GetVarintBounded(a, pEnd, &nToken);
+            if( a>pEnd ){
+              rc = SQLITE_CORRUPT_VTAB;
+              break;
+            }
             pInfo->aMatchinfo[iCol] = (u32)nToken;
           }
         }
@@ -1300,7 +1325,7 @@ static int fts3MatchinfoValues(
         if( rc!=SQLITE_OK ) break;
         if( bGlobal ){
           if( pCsr->pDeferred ){
-            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
+            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0);
             if( rc!=SQLITE_OK ) break;
           }
           rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 3b36389053..e004c0816d 100644
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -67,7 +67,7 @@ int test_fts3_node_chunk_threshold = (4*1024)*4;
 #endif
 
 /*
-** The two values that may be meaningfully bound to the :1 parameter in
+** The values that may be meaningfully bound to the :1 parameter in
 ** statements SQL_REPLACE_STAT and SQL_SELECT_STAT.
 */
 #define FTS_STAT_DOCTOTAL      0
diff --git a/manifest b/manifest
index 6c4876e2f3..daeedd7d2d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sa\sreachable\sNEVER()\sin\sFTS3.
-D 2019-11-17T00:08:24.504
+C Better\sdetection\sof\scorruption\sin\sthe\s%_stat\sand\s%_docsize\sshadow\stables\nof\sFTS3.
+D 2019-11-17T02:41:06.494
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -81,16 +81,16 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c ad760e49a68d13c11f611d7532d7f2277924df6d4a43cbaf8acf7693e1e83aab
+F ext/fts3/fts3.c 493e74700036b725287fe03c0c597c9d0b22708acedc2c86fb323a5d8f519ce7
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 31f2ceee083d93254d84ab7a69eddb253e079bfe7e35c15413c0132f8947d6b6
+F ext/fts3/fts3Int.h 7201d419716896951044f980350f0ccb376fdc15932ff1a81f0d43f5e7051a45
 F ext/fts3/fts3_aux.c 96708c8b3a7d9b8ca1b68ea2b7e503e283f20e95f145becadedfad096dbd0f34
 F ext/fts3/fts3_expr.c b132af223e90e35b9f9efa9fe63d6ae737d34153a3b6066736086df8abc78a1f
 F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7
 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
 F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116
 F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009
-F ext/fts3/fts3_snippet.c 7963dd25ec81013c31f3c61697d0a1f3d06be21af3565774645c08d3dedf1fa7
+F ext/fts3/fts3_snippet.c 70e8aa4a42a9e0338d0d73d0ec996db29ce5421404e29a86703cb8c39949b0cc
 F ext/fts3/fts3_term.c f45a1e7c6ef464abb1231245d123dae12266b69e05cc56e14045b76591ae92d1
 F ext/fts3/fts3_test.c 73b16e229e517c1b1f0fb8e1046182a4e5dbc8dbe6eea8a5d4353fcce7dbbf39
 F ext/fts3/fts3_tokenize_vtab.c 1de9a61acfa2a0445ed989310c31839c57f6b6086dd9d5c97177ae734a17fd8b
@@ -99,7 +99,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d
 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f
-F ext/fts3/fts3_write.c f31351fc0405852d6ed32aa272e7770e55dffa33fc1d4c028aadef50a84213ab
+F ext/fts3/fts3_write.c a96fd13026e5109c54f2912d755aa14988140dd0673c92a606363f3c85c5cb64
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
@@ -973,7 +973,7 @@ F test/fts3sort.test ed34c716a11cc2009a35210e84ad5f9c102362ca
 F test/fts3tok1.test a663f4cac22a9505400bc22aacb818d7055240409c28729669ea7d4cc2120d15
 F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d
 F test/fts3varint.test 0b84a3fd4eba8a39f3687523804d18f3b322e6d4539a55bf342079c3614f2ada
-F test/fts4aa.test 10aac8e9d62c7357590acfabe3fad01e9a9ce1cb
+F test/fts4aa.test 9a90721c2a36ef07783aa4b74f1425df4b8b5ab14749029026949c202c35fc4d
 F test/fts4check.test 6259f856604445d7b684c9b306b2efb6346834c3f50e8fc4a59a2ca6d5319ad0
 F test/fts4content.test 1518195a9f92b711d94419f76409a31cc78755854fb0abb1da2b74b9e0cf843e
 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01
@@ -1849,7 +1849,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 51525f9c3235967bc00a090e84c70a6400698c897aa4742e817121c725b8c99d
-R 53c3e5a5ab84e0cd96e357e129ba281f
+P 8bd75bf636f72f32d66c6c38e1918f27daf2f13290f00a001f41d50838bbda47
+R 8cd04f5b2ab31e4e6a20c98f6ee3e287
 U drh
-Z 4e9166f4f5d94dbdb240d614319c425c
+Z 4bb0986828ea646d1a8a470b5245a8a1
diff --git a/manifest.uuid b/manifest.uuid
index 45e95d56cc..c5db53568d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8bd75bf636f72f32d66c6c38e1918f27daf2f13290f00a001f41d50838bbda47
\ No newline at end of file
+1e449687881f4d388e54a0e51bcabba41ab10cf7e596ff65e31e88a23c70d497
\ No newline at end of file
diff --git a/test/fts4aa.test b/test/fts4aa.test
index e6c7f9336e..3e85a77ecc 100644
--- a/test/fts4aa.test
+++ b/test/fts4aa.test
@@ -191,4 +191,42 @@ foreach {q r} [array get fts4aa_res] {
   } $r
 }
 
+# 2019-11-16 https://bugs.chromium.org/p/chromium/issues/detail?id=1025472
+#
+db close
+sqlite3 db :memory:
+do_execsql_test fts4aa-5.10 {
+  CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, d, e,f,g,h,i,j,k,l,m,n,o,p,q,r);
+  INSERT INTO t1 VALUES('X Y', '2', '3', '4', '5', '6', '7', '8', '9', '0',
+                        'a','b','c','d','e','f','g','h');
+  UPDATE t1_docsize SET size=x'88' WHERE docid=1;
+} {}
+do_catchsql_test fts4aa-5.20 {
+  SELECT quote(matchinfo(t1, 'l')) FROM t1 WHERE t1 MATCH 'X Y';
+} {1 {database disk image is malformed}}
+do_execsql_test fts4aa-5.30 {
+  DROP TABLE t1;
+  CREATE VIRTUAL TABLE t1 USING fts4(a,b,c,d);
+  INSERT INTO t1 VALUES('one two','three four','five six','seven eight');
+} {}
+do_catchsql_test fts4aa-5.40 {
+  UPDATE t1_stat SET value=x'01010101' WHERE id=0;
+  SELECT quote(matchinfo(t1,'a')) FROM t1 WHERE t1 MATCH 'one two';
+} {1 {database disk image is malformed}}
+do_catchsql_test fts4aa-5.50 {
+  UPDATE t1_stat SET value=x'010101' WHERE id=0;
+  SELECT quote(matchinfo(t1,'a')) FROM t1 WHERE t1 MATCH 'one two';
+} {1 {database disk image is malformed}}
+do_catchsql_test fts4aa-5.60 {
+  UPDATE t1_stat SET value=x'01' WHERE id=0;
+  SELECT quote(matchinfo(t1,'a')) FROM t1 WHERE t1 MATCH 'one two';
+} {1 {database disk image is malformed}}
+do_catchsql_test fts4aa-5.70 {
+  UPDATE t1_stat SET value=x'' WHERE id=0;
+  SELECT quote(matchinfo(t1,'a')) FROM t1 WHERE t1 MATCH 'one two';
+} {1 {database disk image is malformed}}
+
+
+
+
 finish_test