Add "segment promotion" to fts5. This prevents the FTS index from growing indefinitely as data is added and deleted.

FossilOrigin-Name: ba359d78e166d78e0dc89e3c63a9a41e9ffea989

ext/fts5/fts5_index.c

@@ -655,6 +655,30 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
   }
 }
 
+static void fts5DebugStructure(
+  int *pRc,                       /* IN/OUT: error code */
+  Fts5Buffer *pBuf,
+  Fts5Structure *p
+){
+  int iLvl, iSeg;                 /* Iterate through levels, segments */
+
+  for(iLvl=0; iLvl<p->nLevel; iLvl++){
+    Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
+        " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge
+    );
+    for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
+      Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
+      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
+          " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, 
+          pSeg->pgnoFirst, pSeg->pgnoLast
+      );
+    }
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
+  }
+}
+
+
 
 static void fts5PutU16(u8 *aOut, u16 iVal){
   aOut[0] = (iVal>>8);
@@ -1080,6 +1104,181 @@ static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){
   fts5BufferFree(&buf);
 }
 
+#if 0
+static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){
+  int rc = SQLITE_OK;
+  Fts5Buffer buf;
+  memset(&buf, 0, sizeof(buf));
+  fts5DebugStructure(&rc, &buf, pStruct);
+  fprintf(stdout, "%s: %s\n", zCaption, buf.p);
+  fflush(stdout);
+  fts5BufferFree(&buf);
+}
+#else
+# define fts5PrintStructure(x,y)
+#endif
+
+/*
+** Return a copy of index structure pStruct. Except, promote as many segments
+** as possible to level iPromote. If an OOM occurs, NULL is returned.
+*/
+static void fts5StructurePromoteTo(
+  Fts5Index *p,
+  int iPromote,
+  int szPromote,
+  Fts5Structure *pStruct
+){
+  Fts5Structure *pNew;
+  u8 *pSpace;
+  int nSeg = fts5StructureCountSegments(pStruct);
+  int nLvl = pStruct->nLevel;
+  int nByte = (
+      sizeof(Fts5Structure) + 
+      sizeof(Fts5StructureLevel) * (nLvl+1) +
+      sizeof(Fts5StructureSegment) * (nSeg+nLvl+1)
+  );
+  int iTst;
+
+  pNew = fts5IdxMalloc(p, nByte);
+  if( !pNew ) return;
+  pNew->nWriteCounter = pStruct->nWriteCounter;
+  pNew->nLevel = pStruct->nLevel;
+  pSpace = (u8*)&pNew->aLevel[nLvl+1];
+
+  for(iTst=0; iTst<nLvl; iTst++){
+    int nCopy;
+    Fts5StructureLevel *pLvlOut = &pNew->aLevel[iTst];
+    pLvlOut->aSeg = (Fts5StructureSegment*)pSpace;
+
+    if( iTst==iPromote ){
+      int il, is;
+      int nSegCopy = 0;
+
+      /* Figure out the number of segments that will be promoted. */
+      for(il=iTst+1; il<pStruct->nLevel; il++){
+        Fts5StructureLevel *pLvl = &pStruct->aLevel[il];
+        if( pLvl->nMerge ) break;
+        for(is=pLvl->nSeg-1; is>=0; is--){
+          Fts5StructureSegment *pSeg = &pLvl->aSeg[is];
+          int sz = pSeg->pgnoLast - pSeg->pgnoFirst + 1;
+          if( sz>szPromote ){
+            il = pStruct->nLevel;
+            break;
+          }
+          nSegCopy++;
+        }
+      }
+      assert( nSegCopy>0 );
+      pSpace += (nSegCopy * sizeof(Fts5StructureSegment));
+      pLvlOut->nSeg = nSegCopy;
+
+      for(il=iTst+1; il<pStruct->nLevel && nSegCopy>0; il++){
+        Fts5StructureLevel *pLvl = &pStruct->aLevel[il];
+        for(is=pLvl->nSeg-1; is>=0 && nSegCopy>0; is--){
+          Fts5StructureSegment *pSeg = &pLvl->aSeg[is];
+          nSegCopy--;
+          memcpy(&pLvlOut->aSeg[nSegCopy], pSeg, sizeof(Fts5StructureSegment));
+          pLvl->nSeg--;
+        }
+      }
+      assert( nSegCopy==0 );
+    }
+
+    nCopy = pStruct->aLevel[iTst].nSeg * sizeof(Fts5StructureSegment);
+    if( nCopy ) memcpy(pSpace, pStruct->aLevel[iTst].aSeg, nCopy);
+    pSpace += (nCopy + sizeof(Fts5StructureSegment));
+    pLvlOut->nSeg += pStruct->aLevel[iTst].nSeg;
+  }
+
+  fts5PrintStructure("NEW", pNew);
+  memcpy(pStruct, pNew, nByte);
+  for(iTst=0; iTst<pNew->nLevel; iTst++){
+    int iOff = pNew->aLevel[iTst].aSeg - (Fts5StructureSegment*)pNew;
+    pStruct->aLevel[iTst].aSeg = &((Fts5StructureSegment*)pStruct)[iOff];
+  }
+  sqlite3_free(pNew);
+}
+
+/*
+** A new segment has just been written to level iLvl of index structure
+** pStruct. This function determines if any segments should be promoted
+** as a result. Segments are promoted in two scenarios:
+**
+**   a) If the segment just written is smaller than one or more segments
+**      within the previous populated level, it is promoted to the previous
+**      populated level.
+**
+**   b) If the segment just written is larger than the newest segment on
+**      the next populated level, then that segment, and any other adjacent
+**      segments that are also smaller than the one just written, are 
+**      promoted. 
+**
+** If one or more segments are promoted, the structure object is updated
+** to reflect this.
+*/
+static void fts5StructurePromote(
+  Fts5Index *p,                   /* FTS5 backend object */
+  int iLvl,                       /* Index level just updated */
+  Fts5Structure *pStruct          /* Index structure */
+){
+  if( p->rc==SQLITE_OK ){
+    int iTst;
+    int iPromote = -1;
+    int szPromote;                /* Promote anything this size or smaller */
+    Fts5StructureSegment *pSeg;   /* Segment just written */
+    Fts5StructureLevel *pTst;
+    int szSeg;                    /* Size of segment just written */
+
+
+    pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1];
+    szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst);
+
+    /* Check for condition (a) */
+    for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--);
+    pTst = &pStruct->aLevel[iTst];
+    if( iTst>=0 && pTst->nMerge==0 ){
+      int i;
+      int szMax = 0;
+      for(i=0; i<pTst->nSeg; i++){
+        int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1;
+        if( sz>szMax ) szMax = sz;
+      }
+      if( szMax>=szSeg ){
+        /* Condition (a) is true. Promote the newest segment on level 
+        ** iLvl to level iTst.  */
+        iPromote = iTst;
+        szPromote = szMax;
+      }
+    }
+
+    /* Check for condition (b) */
+    if( iPromote<0 ){
+      Fts5StructureLevel *pTst;
+      for(iTst=iLvl+1; iTst<pStruct->nLevel; iTst++){
+        pTst = &pStruct->aLevel[iTst];
+        if( pTst->nSeg ) break;
+      }
+      if( iTst<pStruct->nLevel && pTst->nMerge==0 ){
+        Fts5StructureSegment *pSeg2 = &pTst->aSeg[pTst->nSeg-1];
+        int sz = pSeg2->pgnoLast - pSeg2->pgnoFirst + 1;
+        if( sz<=szSeg ){
+          iPromote = iLvl;
+          szPromote = szSeg;
+        }
+      }
+    }
+
+    /* If iPromote is greater than or equal to zero at this point, then it
+    ** is the level number of a level to which segments that consist of
+    ** szPromote or fewer pages should be promoted. */ 
+    if( iPromote>=0 ){
+      fts5PrintStructure("BEFORE", pStruct);
+      fts5StructurePromoteTo(p, iPromote, szPromote, pStruct);
+      fts5PrintStructure("AFTER", pStruct);
+    }
+  }
+}
+
 
 /*
 ** If the pIter->iOff offset currently points to an entry indicating one
@@ -3151,6 +3350,7 @@ static void fts5IndexWork(
 
     if( nBest<p->nMinMerge && pStruct->aLevel[iBestLvl].nMerge==0 ) break;
     fts5IndexMergeLevel(p, iIdx, pStruct, iBestLvl, &nRem);
+    fts5StructurePromote(p, iBestLvl+1, pStruct);
     assert( nRem==0 || p->rc==SQLITE_OK );
   }
 }
@@ -3689,7 +3889,6 @@ static void fts5DecodeStructure(
   const u8 *pBlob, int nBlob
 ){
   int rc;                         /* Return code */
-  int iLvl, iSeg;                 /* Iterate through levels, segments */
   Fts5Structure *p = 0;           /* Decoded structure object */
 
   rc = fts5StructureDecode(pBlob, nBlob, &p);
@@ -3698,21 +3897,7 @@ static void fts5DecodeStructure(
     return;
   }
 
-  for(iLvl=0; iLvl<p->nLevel; iLvl++){
+  fts5DebugStructure(pRc, pBuf, p);
-    Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
-        " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge
-    );
-    for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
-      Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
-      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
-          " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, 
-          pSeg->pgnoFirst, pSeg->pgnoLast
-      );
-    }
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
-  }
-
   fts5StructureRelease(p);
 }
 

manifest

@@ -1,5 +1,5 @@
-C Avoid\swriting\sdelete\smarkers\sto\sthe\soldest\ssegment\sin\san\sFTS\sindex.
+C Add\s"segment\spromotion"\sto\sfts5.\sThis\sprevents\sthe\sFTS\sindex\sfrom\sgrowing\sindefinitely\sas\sdata\sis\sadded\sand\sdeleted.
-D 2014-08-06T20:04:14.831
+D 2014-08-07T18:47:33.788
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -110,7 +110,7 @@ F ext/fts5/fts5_aux.c 31e581413ecab0962ce2b37468f9f658f36f4b0e
 F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00
 F ext/fts5/fts5_config.c f4ebf143e141b8c77355e3b15aba81b7be51d710
 F ext/fts5/fts5_expr.c 7b8e380233176053841904a86006696ee8f6cd24
-F ext/fts5/fts5_index.c dab399c67cb6bdd23009d2f1280ea60a9585b47c
+F ext/fts5/fts5_index.c 1e001ed7dd4650a0a853b986f34b71c8d3f71ec1
 F ext/fts5/fts5_storage.c 2866e7e1de9dc851756c3a9c76b6e1d75e0facb7
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
@@ -604,7 +604,7 @@ F test/fts5af.test 9ebe23aa3875896076952c7bc6e8308813a63c74
 F test/fts5ag.test 0747bf3bade16d5165810cf891f875933b28b420
 F test/fts5ah.test 009b993a9b7ebc43f84c10e53bd778b1dc8ffbe7
 F test/fts5ai.test 4dee71c23ddbcf2b0fc5d5586f241002b883c10e
-F test/fts5aj.test d16f44bd1f7da9714ef99bd8b1996c5867aee8f5
+F test/fts5aj.test 67014e9fc7c069425d67d549b133742b67755047
 F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 3b19eba042bb2eeb1be60f8d58ebaa0a045d6a5c
+P 1baeb1cee61d9c56c718b50af034a24f1018a322
-R 26f7f3d5f6581939b65c2ce6e063db0c
+R c515f8340a51bac4920372c5a517f13d
 U dan
-Z 78bed80e24eecafbae841451888b9b1c
+Z e9b5b5b15db061fad6b53bb80e61f761

manifest.uuid

@@ -1 +1 @@
-1baeb1cee61d9c56c718b50af034a24f1018a322
+ba359d78e166d78e0dc89e3c63a9a41e9ffea989

test/fts5aj.test

@@ -57,15 +57,14 @@ for {set iTest 0} {$iTest < 50000} {incr iTest} {
   if {0==($iTest % 1000)} {
     set sz [db one {SELECT count(*) FROM t1_data}]
     set s [structure]
-    do_test 1.$iTest.$sz.{$s} {} {}
+    do_execsql_test 1.$iTest.$sz.{$s} {
+      INSERT INTO t1(t1) VALUES('integrity-check') 
+    }
   }
 }
 
-#db eval { SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
-
 do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }
 
 
-
 finish_test