Logically store updates as (insert+delete) within the FTS tree. This allows keys to be annihilated more quickly under some circumstances.

FossilOrigin-Name: 50fae1f0006c0e946b5214e73eedf2687a0016f9

ext/fts5/fts5_index.c

@@ -1771,30 +1771,9 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
 ** points to a delete marker. A delete marker is an entry with a 0 byte
 ** position-list.
 */
-static int fts5SegIterIsDelete(
-  Fts5Index *p,                   /* FTS5 backend object */
-  Fts5SegIter *pIter              /* Iterator to advance */
-){
-  int bRet = 0;
-  Fts5Data *pLeaf = pIter->pLeaf;
-  if( p->rc==SQLITE_OK && pLeaf ){
-    bRet = pIter->nPos==0;
-    /* bRet = pIter->bDel; */
-#if 0
-    if( pIter->iLeafOffset<pLeaf->n ){
-      bRet = ((pLeaf->p[pIter->iLeafOffset] & 0xFE)==0x00);
-    }else{
-      Fts5Data *pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
-            pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno+1
-      ));
-      if( pNew ){
-        bRet = ((pNew->p[4] & 0xFE)==0x00);
-        fts5DataRelease(pNew);
-      }
-    }
-#endif
-  }
-  return bRet;
+static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5MultiSegIter *pIter){
+  Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
+  return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0);
 }
 
 /*
@@ -2318,7 +2297,10 @@ static int fts5MultiIterDoCompare(Fts5MultiSegIter *pIter, int iOut){
       assert( i2>i1 );
       assert( i2!=0 );
       pRes->bTermEq = 1;
-      if( p1->iRowid==p2->iRowid ) return i2;
+      if( p1->iRowid==p2->iRowid ){
+        p1->bDel = p2->bDel;
+        return i2;
+      }
       res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1;
     }
     assert( res!=0 );
@@ -2513,9 +2495,7 @@ static void fts5MultiIterNext(
       fts5AssertMultiIterSetup(p, pIter);
 
       bUseFrom = 0;
-    }while( pIter->bSkipEmpty 
-         && fts5SegIterIsDelete(p, &pIter->aSeg[pIter->aFirst[1].iFirst])
-    );
+    }while( pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter) );
   }
 }
 
@@ -2611,9 +2591,7 @@ static void fts5MultiIterNew(
     }
     fts5AssertMultiIterSetup(p, pNew);
 
-    if( pNew->bSkipEmpty 
-     && fts5SegIterIsDelete(p, &pNew->aSeg[pNew->aFirst[1].iFirst]) 
-    ){
+    if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
       fts5MultiIterNext(p, pNew, 0, 0);
     }
   }else{
@@ -3408,36 +3386,38 @@ fflush(stdout);
   ){
     Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
     Fts5ChunkIter sPos;           /* Used to iterate through position list */
+    int nPos;                     /* position-list size field value */
+    int nTerm;
+    const u8 *pTerm;
+
+    /* Check for key annihilation. */
+    if( pSeg->nPos==0 && (bOldest || pSeg->bDel==0) ) continue;
 
-    /* If the segment being written is the oldest in the entire index and
-    ** the position list is empty (i.e. the entry is a delete marker), no
-    ** entry need be written to the output.  */
     fts5ChunkIterInit(p, pSeg, &sPos);
-    if( bOldest==0 || sPos.nRem>0 ){
-      int nTerm;
-      const u8 *pTerm = fts5MultiIterTerm(pIter, &nTerm);
-      if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){
-        if( pnRem && writer.nLeafWritten>nRem ){
-          fts5ChunkIterRelease(&sPos);
-          break;
-        }
 
-        /* This is a new term. Append a term to the output segment. */
-        if( bRequireDoclistTerm ){
-          fts5WriteAppendZerobyte(p, &writer);
-        }
-        fts5WriteAppendTerm(p, &writer, nTerm, pTerm);
-        fts5BufferSet(&p->rc, &term, nTerm, pTerm);
-        bRequireDoclistTerm = 1;
+    pTerm = fts5MultiIterTerm(pIter, &nTerm);
+    if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){
+      if( pnRem && writer.nLeafWritten>nRem ){
+        fts5ChunkIterRelease(&sPos);
+        break;
       }
 
-      /* Append the rowid to the output */
-      /* WRITEPOSLISTSIZE */
-      fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter), sPos.nRem*2);
-
-      for(/* noop */; !fts5ChunkIterEof(p, &sPos); fts5ChunkIterNext(p, &sPos)){
-        fts5WriteAppendPoslistData(p, &writer, sPos.p, sPos.n);
+      /* This is a new term. Append a term to the output segment. */
+      if( bRequireDoclistTerm ){
+        fts5WriteAppendZerobyte(p, &writer);
       }
+      fts5WriteAppendTerm(p, &writer, nTerm, pTerm);
+      fts5BufferSet(&p->rc, &term, nTerm, pTerm);
+      bRequireDoclistTerm = 1;
+    }
+
+    /* Append the rowid to the output */
+    /* WRITEPOSLISTSIZE */
+    nPos = pSeg->nPos*2 + pSeg->bDel;
+    fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter), nPos);
+
+    for(/* noop */; !fts5ChunkIterEof(p, &sPos); fts5ChunkIterNext(p, &sPos)){
+      fts5WriteAppendPoslistData(p, &writer, sPos.p, sPos.n);
     }
 
     fts5ChunkIterRelease(&sPos);
@@ -3882,13 +3862,15 @@ static void fts5BtreeIterInit(
   int i;
   nByte = sizeof(pIter->aLvl[0]) * (pSeg->nHeight-1);
   memset(pIter, 0, sizeof(*pIter));
-  pIter->nLvl = pSeg->nHeight-1;
-  pIter->iIdx = iIdx;
-  pIter->p = p;
-  pIter->pSeg = pSeg;
-  if( nByte && p->rc==SQLITE_OK ){
+  if( nByte ){
     pIter->aLvl = (Fts5BtreeIterLevel*)fts5IdxMalloc(p, nByte);
   }
+  if( p->rc==SQLITE_OK ){
+    pIter->nLvl = pSeg->nHeight-1;
+    pIter->iIdx = iIdx;
+    pIter->p = p;
+    pIter->pSeg = pSeg;
+  }
   for(i=0; p->rc==SQLITE_OK && i<pIter->nLvl; i++){
     i64 iRowid = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, i+1, 1);
     Fts5Data *pData;

ext/fts5/test/fts5corrupt.test

@@ -63,8 +63,8 @@ db func rnddoc fts5_rnddoc
 do_test 2.1 {
   for {set i 0} {$i < 500} {incr i} {
     execsql { INSERT INTO t2 VALUES(rnddoc(50)) }
-    execsql { INSERT INTO t2(t2) VALUES('integrity-check') }
   }
+  execsql { INSERT INTO t2(t2) VALUES('integrity-check') }
 } {}
 
 #--------------------------------------------------------------------

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\spreventing\sdoclist\sindexes\sfrom\sbeing\sloaded.
-D 2015-04-15T16:01:42.421
+C Logically\sstore\supdates\sas\s(insert+delete)\swithin\sthe\sFTS\stree.\sThis\sallows\skeys\sto\sbe\sannihilated\smore\squickly\sunder\ssome\scircumstances.
+D 2015-04-15T18:49:20.008
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 88a3e6261286db378fdffa1124cad11b3c05f5bb
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 3ba56cc6824c9f7b1e0695159e0a9c636f6b4a23
 F ext/fts5/fts5_config.c 0847facc8914f57ea4452c43ce109200dc65e894
 F ext/fts5/fts5_expr.c 5215137efab527577d36bdf9e44bfc2ec3e1be98
 F ext/fts5/fts5_hash.c 3cb5a3d04dd2030eb0ac8d544711dfd37c0e6529
-F ext/fts5/fts5_index.c 9556d405a12a38b3e1a323333a2620813b9f323a
+F ext/fts5/fts5_index.c 28f1bfadf3eb4f860c8b978f4d8d6ea0cf7c724d
 F ext/fts5/fts5_storage.c ac0f0937059c8d4f38a1f13aa5f2c2cd7edf3e0d
 F ext/fts5/fts5_tcl.c 617b6bb96545be8d9045de6967c688cd9cd15541
 F ext/fts5/fts5_tokenize.c c07f2c2f749282c1dbbf46bde1f6d7095c740b8b
@@ -134,7 +134,7 @@ F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592
 F ext/fts5/test/fts5al.test 6a5717faaf7f1e0e866360022d284903f3a4eede
 F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5content.test 8dc302fccdff834d946497e9d862750ea87d4517
-F ext/fts5/test/fts5corrupt.test 78eb076867e750a013b46b3bc06065870bc93c22
+F ext/fts5/test/fts5corrupt.test dbdcfe75749ed2f2eb3915cf68fd55d3dc3b058d
 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
 F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
 F ext/fts5/test/fts5fault1.test fbd8612889234849ff041f5b36f8e390feeed46e
@@ -1292,7 +1292,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 9341c070bb6140dbf559680952909674aa83fa55
-R c6ba726746ea3bc47027dcf89fd1186f
+P b29109a083e5cd442cdd19f29d7be45b09c4f661
+R 3f3d372bbefa9c63dd6c3e57000a1fa8
 U dan
-Z a5a728f32ef73ddd62455c44220623a7
+Z ff9a464604d7627b634f349ee851852e

manifest.uuid

@@ -1 +1 @@
-b29109a083e5cd442cdd19f29d7be45b09c4f661
+50fae1f0006c0e946b5214e73eedf2687a0016f9