diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 9f8f25908f..75fa0a83b4 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -655,6 +655,30 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ } } +static void fts5DebugStructure( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + Fts5Structure *p +){ + int iLvl, iSeg; /* Iterate through levels, segments */ + + for(iLvl=0; iLvlnLevel; iLvl++){ + Fts5StructureLevel *pLvl = &p->aLevel[iLvl]; + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, + " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge + ); + for(iSeg=0; iSegnSeg; iSeg++){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, + " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, + pSeg->pgnoFirst, pSeg->pgnoLast + ); + } + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); + } +} + + static void fts5PutU16(u8 *aOut, u16 iVal){ aOut[0] = (iVal>>8); @@ -1080,6 +1104,181 @@ static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){ fts5BufferFree(&buf); } +#if 0 +static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){ + int rc = SQLITE_OK; + Fts5Buffer buf; + memset(&buf, 0, sizeof(buf)); + fts5DebugStructure(&rc, &buf, pStruct); + fprintf(stdout, "%s: %s\n", zCaption, buf.p); + fflush(stdout); + fts5BufferFree(&buf); +} +#else +# define fts5PrintStructure(x,y) +#endif + +/* +** Return a copy of index structure pStruct. Except, promote as many segments +** as possible to level iPromote. If an OOM occurs, NULL is returned. +*/ +static void fts5StructurePromoteTo( + Fts5Index *p, + int iPromote, + int szPromote, + Fts5Structure *pStruct +){ + Fts5Structure *pNew; + u8 *pSpace; + int nSeg = fts5StructureCountSegments(pStruct); + int nLvl = pStruct->nLevel; + int nByte = ( + sizeof(Fts5Structure) + + sizeof(Fts5StructureLevel) * (nLvl+1) + + sizeof(Fts5StructureSegment) * (nSeg+nLvl+1) + ); + int iTst; + + pNew = fts5IdxMalloc(p, nByte); + if( !pNew ) return; + pNew->nWriteCounter = pStruct->nWriteCounter; + pNew->nLevel = pStruct->nLevel; + pSpace = (u8*)&pNew->aLevel[nLvl+1]; + + for(iTst=0; iTstaLevel[iTst]; + pLvlOut->aSeg = (Fts5StructureSegment*)pSpace; + + if( iTst==iPromote ){ + int il, is; + int nSegCopy = 0; + + /* Figure out the number of segments that will be promoted. */ + for(il=iTst+1; ilnLevel; il++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[il]; + if( pLvl->nMerge ) break; + for(is=pLvl->nSeg-1; is>=0; is--){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[is]; + int sz = pSeg->pgnoLast - pSeg->pgnoFirst + 1; + if( sz>szPromote ){ + il = pStruct->nLevel; + break; + } + nSegCopy++; + } + } + assert( nSegCopy>0 ); + pSpace += (nSegCopy * sizeof(Fts5StructureSegment)); + pLvlOut->nSeg = nSegCopy; + + for(il=iTst+1; ilnLevel && nSegCopy>0; il++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[il]; + for(is=pLvl->nSeg-1; is>=0 && nSegCopy>0; is--){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[is]; + nSegCopy--; + memcpy(&pLvlOut->aSeg[nSegCopy], pSeg, sizeof(Fts5StructureSegment)); + pLvl->nSeg--; + } + } + assert( nSegCopy==0 ); + } + + nCopy = pStruct->aLevel[iTst].nSeg * sizeof(Fts5StructureSegment); + if( nCopy ) memcpy(pSpace, pStruct->aLevel[iTst].aSeg, nCopy); + pSpace += (nCopy + sizeof(Fts5StructureSegment)); + pLvlOut->nSeg += pStruct->aLevel[iTst].nSeg; + } + + fts5PrintStructure("NEW", pNew); + memcpy(pStruct, pNew, nByte); + for(iTst=0; iTstnLevel; iTst++){ + int iOff = pNew->aLevel[iTst].aSeg - (Fts5StructureSegment*)pNew; + pStruct->aLevel[iTst].aSeg = &((Fts5StructureSegment*)pStruct)[iOff]; + } + sqlite3_free(pNew); +} + +/* +** A new segment has just been written to level iLvl of index structure +** pStruct. This function determines if any segments should be promoted +** as a result. Segments are promoted in two scenarios: +** +** a) If the segment just written is smaller than one or more segments +** within the previous populated level, it is promoted to the previous +** populated level. +** +** b) If the segment just written is larger than the newest segment on +** the next populated level, then that segment, and any other adjacent +** segments that are also smaller than the one just written, are +** promoted. +** +** If one or more segments are promoted, the structure object is updated +** to reflect this. +*/ +static void fts5StructurePromote( + Fts5Index *p, /* FTS5 backend object */ + int iLvl, /* Index level just updated */ + Fts5Structure *pStruct /* Index structure */ +){ + if( p->rc==SQLITE_OK ){ + int iTst; + int iPromote = -1; + int szPromote; /* Promote anything this size or smaller */ + Fts5StructureSegment *pSeg; /* Segment just written */ + Fts5StructureLevel *pTst; + int szSeg; /* Size of segment just written */ + + + pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1]; + szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst); + + /* Check for condition (a) */ + for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--); + pTst = &pStruct->aLevel[iTst]; + if( iTst>=0 && pTst->nMerge==0 ){ + int i; + int szMax = 0; + for(i=0; inSeg; i++){ + int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1; + if( sz>szMax ) szMax = sz; + } + if( szMax>=szSeg ){ + /* Condition (a) is true. Promote the newest segment on level + ** iLvl to level iTst. */ + iPromote = iTst; + szPromote = szMax; + } + } + + /* Check for condition (b) */ + if( iPromote<0 ){ + Fts5StructureLevel *pTst; + for(iTst=iLvl+1; iTstnLevel; iTst++){ + pTst = &pStruct->aLevel[iTst]; + if( pTst->nSeg ) break; + } + if( iTstnLevel && pTst->nMerge==0 ){ + Fts5StructureSegment *pSeg2 = &pTst->aSeg[pTst->nSeg-1]; + int sz = pSeg2->pgnoLast - pSeg2->pgnoFirst + 1; + if( sz<=szSeg ){ + iPromote = iLvl; + szPromote = szSeg; + } + } + } + + /* If iPromote is greater than or equal to zero at this point, then it + ** is the level number of a level to which segments that consist of + ** szPromote or fewer pages should be promoted. */ + if( iPromote>=0 ){ + fts5PrintStructure("BEFORE", pStruct); + fts5StructurePromoteTo(p, iPromote, szPromote, pStruct); + fts5PrintStructure("AFTER", pStruct); + } + } +} + /* ** If the pIter->iOff offset currently points to an entry indicating one @@ -3151,6 +3350,7 @@ static void fts5IndexWork( if( nBestnMinMerge && pStruct->aLevel[iBestLvl].nMerge==0 ) break; fts5IndexMergeLevel(p, iIdx, pStruct, iBestLvl, &nRem); + fts5StructurePromote(p, iBestLvl+1, pStruct); assert( nRem==0 || p->rc==SQLITE_OK ); } } @@ -3689,7 +3889,6 @@ static void fts5DecodeStructure( const u8 *pBlob, int nBlob ){ int rc; /* Return code */ - int iLvl, iSeg; /* Iterate through levels, segments */ Fts5Structure *p = 0; /* Decoded structure object */ rc = fts5StructureDecode(pBlob, nBlob, &p); @@ -3698,21 +3897,7 @@ static void fts5DecodeStructure( return; } - for(iLvl=0; iLvlnLevel; iLvl++){ - Fts5StructureLevel *pLvl = &p->aLevel[iLvl]; - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, - " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge - ); - for(iSeg=0; iSegnSeg; iSeg++){ - Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, - " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, - pSeg->pgnoFirst, pSeg->pgnoLast - ); - } - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); - } - + fts5DebugStructure(pRc, pBuf, p); fts5StructureRelease(p); } diff --git a/manifest b/manifest index 4503e360f1..ad4cd2c337 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\swriting\sdelete\smarkers\sto\sthe\soldest\ssegment\sin\san\sFTS\sindex. -D 2014-08-06T20:04:14.831 +C Add\s"segment\spromotion"\sto\sfts5.\sThis\sprevents\sthe\sFTS\sindex\sfrom\sgrowing\sindefinitely\sas\sdata\sis\sadded\sand\sdeleted. +D 2014-08-07T18:47:33.788 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5_aux.c 31e581413ecab0962ce2b37468f9f658f36f4b0e F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00 F ext/fts5/fts5_config.c f4ebf143e141b8c77355e3b15aba81b7be51d710 F ext/fts5/fts5_expr.c 7b8e380233176053841904a86006696ee8f6cd24 -F ext/fts5/fts5_index.c dab399c67cb6bdd23009d2f1280ea60a9585b47c +F ext/fts5/fts5_index.c 1e001ed7dd4650a0a853b986f34b71c8d3f71ec1 F ext/fts5/fts5_storage.c 2866e7e1de9dc851756c3a9c76b6e1d75e0facb7 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 @@ -604,7 +604,7 @@ F test/fts5af.test 9ebe23aa3875896076952c7bc6e8308813a63c74 F test/fts5ag.test 0747bf3bade16d5165810cf891f875933b28b420 F test/fts5ah.test 009b993a9b7ebc43f84c10e53bd778b1dc8ffbe7 F test/fts5ai.test 4dee71c23ddbcf2b0fc5d5586f241002b883c10e -F test/fts5aj.test d16f44bd1f7da9714ef99bd8b1996c5867aee8f5 +F test/fts5aj.test 67014e9fc7c069425d67d549b133742b67755047 F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef @@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3b19eba042bb2eeb1be60f8d58ebaa0a045d6a5c -R 26f7f3d5f6581939b65c2ce6e063db0c +P 1baeb1cee61d9c56c718b50af034a24f1018a322 +R c515f8340a51bac4920372c5a517f13d U dan -Z 78bed80e24eecafbae841451888b9b1c +Z e9b5b5b15db061fad6b53bb80e61f761 diff --git a/manifest.uuid b/manifest.uuid index 3d1cd0f81b..c812fc95e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1baeb1cee61d9c56c718b50af034a24f1018a322 \ No newline at end of file +ba359d78e166d78e0dc89e3c63a9a41e9ffea989 \ No newline at end of file diff --git a/test/fts5aj.test b/test/fts5aj.test index 31c8b71282..cb8e2d2a2f 100644 --- a/test/fts5aj.test +++ b/test/fts5aj.test @@ -57,15 +57,14 @@ for {set iTest 0} {$iTest < 50000} {incr iTest} { if {0==($iTest % 1000)} { set sz [db one {SELECT count(*) FROM t1_data}] set s [structure] - do_test 1.$iTest.$sz.{$s} {} {} + do_execsql_test 1.$iTest.$sz.{$s} { + INSERT INTO t1(t1) VALUES('integrity-check') + } } } -#db eval { SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} - do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') } - finish_test