Add extra defenses against strategically corrupt databases to fts3/4.

FossilOrigin-Name: d44318f59044162e229a444582692e9788f17b5c404b4eb702f4c2114b22fefe

ext/fts3/fts3.c

@@ -1821,7 +1821,7 @@ static int fts3ScanInteriorNode(
   const char *zCsr = zNode;       /* Cursor to iterate through node */
   const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
   char *zBuffer = 0;              /* Buffer to load terms into */
-  int nAlloc = 0;                 /* Size of allocated buffer */
+  i64 nAlloc = 0;                 /* Size of allocated buffer */
   int isFirstTerm = 1;            /* True when processing first term on page */
   sqlite3_int64 iChild;           /* Block id of child node to descend to */
 
@@ -1859,14 +1859,14 @@ static int fts3ScanInteriorNode(
     zCsr += fts3GetVarint32(zCsr, &nSuffix);
     
     assert( nPrefix>=0 && nSuffix>=0 );
-    if( &zCsr[nSuffix]>zEnd ){
+    if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
       rc = FTS_CORRUPT_VTAB;
       goto finish_scan;
     }
-    if( nPrefix+nSuffix>nAlloc ){
+    if( (i64)nPrefix+nSuffix>nAlloc ){
       char *zNew;
-      nAlloc = (nPrefix+nSuffix) * 2;
+      nAlloc = ((i64)nPrefix+nSuffix) * 2;
-      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+      zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
       if( !zNew ){
         rc = SQLITE_NOMEM;
         goto finish_scan;

ext/fts3/fts3_write.c

@@ -1374,15 +1374,19 @@ static int fts3SegReaderNext(
   ** safe (no risk of overread) even if the node data is corrupted. */
   pNext += fts3GetVarint32(pNext, &nPrefix);
   pNext += fts3GetVarint32(pNext, &nSuffix);
-  if( nPrefix<0 || nSuffix<=0 
+  if( nSuffix<=0 
-   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
+   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
+   || nPrefix>pReader->nTermAlloc
   ){
     return FTS_CORRUPT_VTAB;
   }
 
-  if( nPrefix+nSuffix>pReader->nTermAlloc ){
+  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
-    int nNew = (nPrefix+nSuffix)*2;
+  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
-    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+  ** overflow - hence the (i64) casts.  */
+  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
+    i64 nNew = ((i64)nPrefix+nSuffix)*2;
+    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
     if( !zNew ){
       return SQLITE_NOMEM;
     }
@@ -1404,7 +1408,7 @@ static int fts3SegReaderNext(
   ** b-tree node. And that the final byte of the doclist is 0x00. If either 
   ** of these statements is untrue, then the data structure is corrupt.
   */
-  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
+  if( (&pReader->aNode[pReader->nNode] - pReader->aDoclist)<pReader->nDoclist
    || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
   ){
     return FTS_CORRUPT_VTAB;
@@ -3730,6 +3734,9 @@ static int nodeReaderNext(NodeReader *p){
     }
     p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
 
+    if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
+      return SQLITE_CORRUPT_VTAB;
+    }
     blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
     if( rc==SQLITE_OK ){
       memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
@@ -3737,6 +3744,9 @@ static int nodeReaderNext(NodeReader *p){
       p->iOff += nSuffix;
       if( p->iChild==0 ){
         p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
+        if( (p->nNode-p->iOff)<p->nDoclist ){
+          return SQLITE_CORRUPT_VTAB;
+        }
         p->aDoclist = &p->aNode[p->iOff];
         p->iOff += p->nDoclist;
       }
@@ -3744,7 +3754,6 @@ static int nodeReaderNext(NodeReader *p){
   }
 
   assert( p->iOff<=p->nNode );
-
   return rc;
 }
 

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sassert()\sin\sthe\squery\splanner\sthat\scan\sarise\swhen\sdoing\srow-value\noperations\son\sa\sPRIMARY\sKEY\sthat\scontains\sduplicate\scolumns.\nTicket\s[1a84668dcfdebaf12415d].
+C Add\sextra\sdefenses\sagainst\sstrategically\scorrupt\sdatabases\sto\sfts3/4.
-D 2018-11-03T13:11:24.271
+D 2018-11-03T16:51:30.810
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in edbb6e20bb1decf65f6c64c9e61004a69bdf8afb39cdce5337c916b03dfcd1e3
@@ -80,7 +80,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 53020b61dac31c167668f098ec83c0387ecd65b9e89ba3659f2f8d3e76efdf1e
+F ext/fts3/fts3.c 012384f01df28345a8d0d0c7fe465ab211598a91d78a7331b3664533756ed9fa
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 3378157f383540857a466420b8279626204434c3eb0dc948ad9bcd3991fc41f5
 F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1
@@ -98,7 +98,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d
 F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c
-F ext/fts3/fts3_write.c b583dede85eb0c3c3026f8d7ccb781ea4e845ae583754fecb2ca425b5907d87d
+F ext/fts3/fts3_write.c a85bc4885fde7f1b44c9de013b62f7cd3332dc59e208053d878729b1d04745bc
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
@@ -905,6 +905,7 @@ F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f
 F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb
 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
 F test/fts3corrupt3.test 56e0ee83e90b57f5f3644cb7d1b36a067b7b8b19cdf0dedce45e5e13cf752f65
+F test/fts3corrupt4.test 98022cbacbd6ddc4708f210768f5684f041f50ce330c461f2631752492611d96
 F test/fts3cov.test 9c3681325b9a850bca8dd75cc29dde73e9a87972bb75204e97d826f13c7181f9
 F test/fts3d.test d3e9c8fb75135ada06bf3bab4f9666224965d708
 F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963
@@ -1175,7 +1176,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 8749e292c8f7e98072049543da8c0fe60e1625f0a9f49068623060984c176bbf
+F test/permutations.test 6316c8edc7ac2c5e1263102aa3022ad3b12ac599e4e3f5931a9c1be407c41ece
 F test/pg_common.tcl 301ac19c1a52fd55166d26db929b3b89165c634d52b5f8ad76ea8cb06960db30
 F test/pragma.test 7c8cfc328a1717a95663cf8edb06c52ddfeaf97bb0aee69ae7457132e8d39e7d
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
@@ -1775,7 +1776,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1fa74930ab56171e2e840d4a5b259abafb0ad1e0320fc3030066570a6dd10002
+P dcb8c73594ea6b12bad98dc883a585d3e6b925c2ead267dc40332b3d266db5e8
-R 707185102d60512231af7b837ac929dd
+R b9cceefc94c9fa3822116514098d3797
-U drh
+U dan
-Z 05c1146263aa5b30b6bada73e8bbc541
+Z 9bf320f36f33a9474fd95119b2eded7a

manifest.uuid

@@ -1 +1 @@
-dcb8c73594ea6b12bad98dc883a585d3e6b925c2ead267dc40332b3d266db5e8
+d44318f59044162e229a444582692e9788f17b5c404b4eb702f4c2114b22fefe

test/fts3corrupt4.test

@@ -0,0 +1,147 @@
+# 2006 September 9
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS3 module.
+#
+# $Id: fts3aa.test,v 1.1 2007/08/20 17:38:42 shess Exp $
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts3corrupt4
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  BEGIN;
+    CREATE VIRTUAL TABLE ft USING fts3;
+    INSERT INTO ft VALUES('aback');
+    INSERT INTO ft VALUES('abaft');
+    INSERT INTO ft VALUES('abandon');
+  COMMIT;
+}
+
+proc blob {a} { binary decode hex $a }
+db func blob blob
+
+do_execsql_test 1.1 {
+  SELECT quote(root) FROM ft_segdir;
+} {X'0005616261636B03010200030266740302020003046E646F6E03030200'}
+
+do_execsql_test 1.2 {
+  UPDATE ft_segdir SET root = blob(
+    '0005616261636B03010200 FFFFFFFF0702 66740302020003046E646F6E03030200'
+  );
+}
+
+do_catchsql_test 1.3 {
+  SELECT * FROM ft WHERE ft MATCH 'abandon';
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0.0 {
+  CREATE VIRTUAL TABLE ft USING fts3;
+  INSERT INTO ft(ft) VALUES('nodesize=32');
+}
+do_test 2.0.1 {
+  for {set i 0} {$i < 12} {incr i} {
+    execsql {
+      BEGIN;
+        INSERT INTO ft VALUES('abc' || $i);
+        INSERT INTO ft VALUES('abc' || $i || 'x' );
+        INSERT INTO ft VALUES('abc' || $i || 'xx' );
+      COMMIT
+    }
+  }
+  execsql {
+    SELECT count(*) FROM ft_segdir;
+    SELECT count(*) FROM ft_segments;
+  }
+} {12 0}
+
+do_execsql_test 2.1 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+  SELECT count(*) FROM ft_segdir;
+  SELECT count(*) FROM ft_segments;
+} {12 3}
+
+do_execsql_test 2.2 {
+  SELECT quote(block) FROM ft_segments WHERE blockid=2
+} {X'00056162633130031F0200'}
+
+db func blob blob
+do_execsql_test 2.3.1 {
+  UPDATE ft_segments SET block = 
+    blob('00056162633130031F0200 FFFFFFFF07FF55 66740302020003046E646F6E03030200')
+    WHERE blockid=2;
+} {}
+do_catchsql_test 2.3.2 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+} {1 {database disk image is malformed}}
+
+do_execsql_test 2.4.1 {
+  UPDATE ft_segments SET block = 
+    blob('00056162633130031F0200 02FFFFFFFF07 66740302020003046E646F6E03030200')
+    WHERE blockid=2;
+} {}
+do_catchsql_test 2.4.2 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+} {1 {database disk image is malformed}}
+
+do_execsql_test 2.5.1 {
+  UPDATE ft_segments SET block = 
+    blob('00056162633130031F0200 0202 6674 FFFFFF070302020003046E646F6E030200')
+    WHERE blockid=2;
+} {}
+do_catchsql_test 2.5.2 {
+  INSERT INTO ft(ft) VALUES('merge=1,4');
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0.0 {
+  CREATE VIRTUAL TABLE ft USING fts3;
+  INSERT INTO ft(ft) VALUES('nodesize=32');
+}
+do_test 3.0.1 {
+  execsql BEGIN
+  for {set i 0} {$i < 20} {incr i} {
+    execsql { INSERT INTO ft VALUES('abc' || $i) }
+  }
+  execsql {
+    COMMIT;
+    SELECT count(*) FROM ft_segdir;
+    SELECT count(*) FROM ft_segments;
+  }
+} {1 5}
+
+do_execsql_test 3.1 {
+  SELECT quote(root) FROM ft_segdir
+} {X'0101056162633132040136030132030136'}
+
+db func blob blob
+do_execsql_test 3.2 {
+  UPDATE ft_segdir 
+  SET root = blob('0101056162633132FFFFFFFF070236030132030136');
+}
+
+do_catchsql_test 3.1 {
+  SELECT * FROM ft WHERE ft MATCH 'abc20'
+} {1 {database disk image is malformed}}
+
+finish_test
+
+

test/permutations.test

@@ -262,6 +262,7 @@ test_suite "fts3" -prefix "" -description {
   fts3am.test fts3an.test fts3ao.test fts3atoken.test
   fts3auto.test fts3aux1.test fts3aux2.test fts3b.test
   fts3comp1.test fts3conf.test fts3corrupt2.test fts3corrupt.test
+  fts3corrupt4.test
   fts3cov.test fts3c.test fts3defer2.test fts3defer3.test
   fts3defer.test fts3drop.test fts3d.test fts3e.test
   fts3expr2.test fts3expr3.test fts3expr4.test fts3expr5.test