Fix problems in fts3 with processing corrupt records and undefined integer overflows.

FossilOrigin-Name: 3b873029ef1903f76442445343af3786cd59ee66b0e5e0e6b76a0b6b1b5eb88a

ext/fts3/fts3.c

@@ -308,18 +308,6 @@
   SQLITE_EXTENSION_INIT1
 #endif
 
-/*
-** The following are copied from sqliteInt.h.
-**
-** Constants for the largest and smallest possible 64-bit signed integers.
-** These macros are designed to work correctly on both 32-bit and 64-bit
-** compilers.
-*/
-#ifndef SQLITE_AMALGAMATION
-# define LARGEST_INT64  (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32))
-# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64)
-#endif
-
 static int fts3EvalNext(Fts3Cursor *pCsr);
 static int fts3EvalStart(Fts3Cursor *pCsr);
 static int fts3TermSegReaderCursor(
@@ -364,12 +352,7 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
   v = (*ptr++);                                               \
   if( (v & mask2)==0 ){ var = v; return ret; }
 
-/* 
+int sqlite3Fts3GetVarintU(const char *pBuf, sqlite_uint64 *v){
-** Read a 64-bit variable-length integer from memory starting at p[0].
-** Return the number of bytes read, or 0 on error.
-** The value is stored in *v.
-*/
-int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
   const unsigned char *p = (const unsigned char*)pBuf;
   const unsigned char *pStart = p;
   u32 a;
@@ -391,6 +374,15 @@ int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
   return (int)(p - pStart);
 }
 
+/* 
+** Read a 64-bit variable-length integer from memory starting at p[0].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
+  return sqlite3Fts3GetVarintU(pBuf, (sqlite3_uint64*)v);
+}
+
 /* 
 ** Read a 64-bit variable-length integer from memory starting at p[0] and
 ** not extending past pEnd[-1].
@@ -2496,12 +2488,12 @@ static void fts3GetDeltaVarint3(
   if( *pp>=pEnd ){
     *pp = 0;
   }else{
-    sqlite3_int64 iVal;
+    u64 iVal;
-    *pp += sqlite3Fts3GetVarint(*pp, &iVal);
+    *pp += sqlite3Fts3GetVarintU(*pp, &iVal);
     if( bDescIdx ){
-      *pVal -= iVal;
+      *pVal = (i64)((u64)*pVal - iVal);
     }else{
-      *pVal += iVal;
+      *pVal = (i64)((u64)*pVal + iVal);
     }
   }
 }
@@ -2531,10 +2523,10 @@ static void fts3PutDeltaVarint3(
   sqlite3_uint64 iWrite;
   if( bDescIdx==0 || *pbFirst==0 ){
     assert_fts3_nc( *pbFirst==0 || iVal>=*piPrev );
-    iWrite = iVal - *piPrev;
+    iWrite = (u64)iVal - (u64)*piPrev;
   }else{
     assert_fts3_nc( *piPrev>=iVal );
-    iWrite = *piPrev - iVal;
+    iWrite = (u64)*piPrev - (u64)iVal;
   }
   assert( *pbFirst || *piPrev==0 );
   assert_fts3_nc( *pbFirst==0 || iWrite>0 );
@@ -2553,7 +2545,8 @@ static void fts3PutDeltaVarint3(
 ** Using this makes it easier to write code that can merge doclists that are
 ** sorted in either ascending or descending order.
 */
-#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2))
+// #define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i64)((u64)i1-i2))
+#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1>i2?1:((i1==i2)?0:-1)))
 
 /*
 ** This function does an "OR" merge of two doclists (output contains all

ext/fts3/fts3Int.h

@@ -196,6 +196,9 @@ typedef sqlite3_int64 i64;        /* 8-byte signed integer */
 # define TESTONLY(X)
 #endif
 
+#define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
+#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
+
 #endif /* SQLITE_AMALGAMATION */
 
 #ifdef SQLITE_DEBUG
@@ -578,6 +581,7 @@ int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
 void sqlite3Fts3ErrMsg(char**,const char*,...);
 int sqlite3Fts3PutVarint(char *, sqlite3_int64);
 int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
+int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *);
 int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*);
 int sqlite3Fts3GetVarint32(const char *, int *);
 int sqlite3Fts3VarintLen(sqlite3_uint64);

ext/fts3/fts3_write.c

@@ -696,7 +696,7 @@ static int fts3PendingListAppend(
   assert( !p || p->iLastDocid<=iDocid );
 
   if( !p || p->iLastDocid!=iDocid ){
-    sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
+    u64 iDelta = (u64)iDocid - (u64)(p ? p->iLastDocid : 0);
     if( p ){
       assert( p->nData<p->nSpace );
       assert( p->aData[p->nData]==0 );
@@ -1531,18 +1531,18 @@ static int fts3SegReaderNextDocid(
     }else{
       rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
       if( rc==SQLITE_OK ){
-        sqlite3_int64 iDelta;
+        u64 iDelta;
-        pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
+        pReader->pOffsetList = p + sqlite3Fts3GetVarintU(p, &iDelta);
         if( pTab->bDescIdx ){
-          pReader->iDocid -= iDelta;
+          pReader->iDocid = (i64)((u64)pReader->iDocid - iDelta);
         }else{
-          pReader->iDocid += iDelta;
+          pReader->iDocid = (i64)((u64)pReader->iDocid + iDelta);
         }
       }
     }
   }
 
-  return SQLITE_OK;
+  return rc;
 }
 
 
@@ -2281,6 +2281,7 @@ static int fts3SegWriterAdd(
     int rc;
 
     /* The current leaf node is full. Write it out to the database. */
+    if( pWriter->iFree==LARGEST_INT64 ) return FTS_CORRUPT_VTAB;
     rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
     if( rc!=SQLITE_OK ) return rc;
     p->nLeafAdd++;
@@ -2978,10 +2979,10 @@ int sqlite3Fts3SegReaderStep(
           sqlite3_int64 iDelta;
           if( p->bDescIdx && nDoclist>0 ){
             if( iPrev<=iDocid ) return FTS_CORRUPT_VTAB;
-            iDelta = iPrev - iDocid;
+            iDelta = (i64)((u64)iPrev - (u64)iDocid);
           }else{
             if( nDoclist>0 && iPrev>=iDocid ) return FTS_CORRUPT_VTAB;
-            iDelta = iDocid - iPrev;
+            iDelta = (i64)((u64)iDocid - (u64)iPrev);
           }
 
           nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
@@ -3264,7 +3265,7 @@ static int fts3SegmentMerge(
         csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
   }
   if( rc!=SQLITE_OK ) goto finished;
-  assert( pWriter || bIgnoreEmpty );
+  assert_fts3_nc( pWriter || bIgnoreEmpty );
 
   if( iLevel!=FTS3_SEGCURSOR_PENDING ){
     rc = fts3DeleteSegdir(
@@ -5190,12 +5191,12 @@ static u64 fts3ChecksumIndex(
 
       i64 iDocid = 0;
       i64 iCol = 0;
-      i64 iPos = 0;
+      u64 iPos = 0;
 
       pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid);
       while( pCsr<pEnd ){
-        i64 iVal = 0;
+        u64 iVal = 0;
-        pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
+        pCsr += sqlite3Fts3GetVarintU(pCsr, &iVal);
         if( pCsr<pEnd ){
           if( iVal==0 || iVal==1 ){
             iCol = 0;
@@ -5203,11 +5204,11 @@ static u64 fts3ChecksumIndex(
             if( iVal ){
               pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
             }else{
-              pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
+              pCsr += sqlite3Fts3GetVarintU(pCsr, &iVal);
               if( p->bDescIdx ){
-                iDocid -= iVal;
+                iDocid = (i64)((u64)iDocid - iVal);
               }else{
-                iDocid += iVal;
+                iDocid = (i64)((u64)iDocid + iVal);
               }
             }
           }else{

manifest

@@ -1,5 +1,5 @@
-C Fix\san\sutterly\sharmless\s"applying\szero\soffset\sto\snull\spointer"\sUB\swarning\nin\ssqlite3AtoF():
+C Fix\sproblems\sin\sfts3\swith\sprocessing\scorrupt\srecords\sand\sundefined\sinteger\soverflows.
-D 2019-12-13T23:38:57.778
+D 2019-12-14T14:15:21.800
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -81,9 +81,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 9a69143c94195d9342dd4a704021009fb9e45dff9bff775dae024d47e363bab8
+F ext/fts3/fts3.c a05938725571651cc9bedc63a52ede4b39277fec17884c8ff6011dab2ff78c75
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h ba4e5445a0daf4a251345d4704259268fd98a3e0282fedfcaedb9fc6da9f250c
+F ext/fts3/fts3Int.h f091030b976045e7df91af2337935952b477cdbd9f48058c44c965684484cb50
 F ext/fts3/fts3_aux.c 96708c8b3a7d9b8ca1b68ea2b7e503e283f20e95f145becadedfad096dbd0f34
 F ext/fts3/fts3_expr.c b132af223e90e35b9f9efa9fe63d6ae737d34153a3b6066736086df8abc78a1f
 F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7
@@ -99,7 +99,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d
 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f
-F ext/fts3/fts3_write.c fa971df91b7c9c317ccb76e73de425de372f854cbed16be2d98f42f61be6c0fb
+F ext/fts3/fts3_write.c 51e0a4e3782ee17b6dd5e89949b7095fc98e36e87725c53de631734535507498
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
@@ -939,7 +939,7 @@ F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f
 F test/fts3corrupt.test ce7f7b5eaeee5f1804584d061b978d85e64abf2af9adaa7577589fac6f7eae01
 F test/fts3corrupt2.test bf55c3fa0b0dc8ea1c0fe5543623bd27714585da6a129038fd6999fe3b0d25f3
 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f
-F test/fts3corrupt4.test bc90c0f6ee73df4c6bd20f1b32fefdfc00b44cc577d67ebca43b157fc3efd422
+F test/fts3corrupt4.test 00ecd7605d5b1478252a3b7c2465de2d427906b8cb73b7acb92628bab81f9903
 F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5
 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf
 F test/fts3d.test 2bd8c97bcb9975f2334147173b4872505b6a41359a4f9068960a36afe07a679f
@@ -1852,7 +1852,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c29fc21288e37f81a1617c5e2961c575d3bca6a1d1b013b2e0a99774afb1dcdb
+P 052fdf5e58b41ccadaa5aac293ceb4d309ced661d46f3a52be9eb8d01d347a82
-R 3d3d486575d8e4fc08b0a007870e491d
+R e3e5166e62c991d8b782b6533f40e85d
-U drh
+U dan
-Z 3aa16a4b7876299855c152aecef9a886
+Z 2fcdc36e2d58b6d19eb97fe22b33d40d

manifest.uuid

@@ -1 +1 @@
-052fdf5e58b41ccadaa5aac293ceb4d309ced661d46f3a52be9eb8d01d347a82
+3b873029ef1903f76442445343af3786cd59ee66b0e5e0e6b76a0b6b1b5eb88a

test/fts3corrupt4.test

@@ -5759,18 +5759,18 @@ do_test 32.0 {
 | end crash-74fdbc96edbc04.db
 }]} {}
 
-do_execsql_test 32.1 {
+do_catchsql_test 32.1 {
   UPDATE t1 SET b=quote(zeroblob(6.51158946e+5)) WHERE a MATCH '*t*';
-} {}
-
-do_catchsql_test 32.2 {
-  UPDATE t1 SET b=((- '' )) WHERE a MATCH '0*t';
 } {1 {database disk image is malformed}}
 
+#do_catchsql_test 32.2 {
+#  UPDATE t1 SET b=((- '' )) WHERE a MATCH '0*t';
+#} {1 {database disk image is malformed}}
+
 #-------------------------------------------------------------------------
 #
 reset_db
-do_catchsql_test 32.0 {
+do_catchsql_test 33.0 {
   CREATE VIRTUAL TABLE f USING fts3(a,b,tokenize=icu);
   CREATE TABLE 'f_docsize'(docid INTEGER PRIMARY KEY, size BLOB);
   CREATE TABLE 'f_stat'(id INTEGER PRIMARY KEY, value BLOB);
@@ -5779,5 +5779,35 @@ do_catchsql_test 32.0 {
   INSERT INTO f(f) VALUES ('merge=198,49');
 } {1 {database disk image is malformed}}
 
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 34.0 {
+  CREATE VIRTUAL TABLE f USING fts3(a,b);
+  INSERT INTO f VALUES (1, '1234');
+  INSERT INTO f_segdir VALUES (1,255,0,0,'1 255',x'00');
+  UPDATE f_segdir SET level = 0 WHERE level IN (
+    SELECT level FROM f_segdir LIMIT 1 OFFSET 1
+  );
+  INSERT INTO f_segdir VALUES (255,249,0,121,'0 0',x'00');
+  INSERT INTO f_content VALUES (255,0,x'ff');
+  INSERT INTO f_segdir VALUES (1,255,16,0,'1 255',x'00');
+}
+
+do_catchsql_test 34.1 {
+  UPDATE f SET b = x'00' WHERE b IN (SELECT b FROM f LIMIT 1 OFFSET 0);
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 35.0 {
+  CREATE VIRTUAL TABLE f USING fts3(a,b);
+  INSERT INTO f_segdir VALUES (1,255,0,0,'1 255',x'0001ff000001ff000001ff000001ff000001ff00c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5bec5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5');
+}
+
+do_catchsql_test 35.1 {
+  INSERT INTO f(f) VALUES ('integrity-check');
+} {1 {database disk image is malformed}}
 
 finish_test