diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 75fa0a83b4..737c226051 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -66,7 +66,13 @@ ** recorded in a single record within the %_data table. The record is a list ** of SQLite varints. ** -** For each level from 0 to nMax: +** The record begins with three varints: +** +** + number of levels, +** + total number of segments on all levels, +** + value of write counter. +** +** Then, for each level from 0 to nMax: ** ** + number of input segments in ongoing merge. ** + total number of segments in level. @@ -707,6 +713,18 @@ static void *fts5IdxMalloc(Fts5Index *p, int nByte){ return pRet; } +static void *fts5MallocZero(int *pRc, int nByte){ + void *pRet = 0; + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc(nByte); + if( pRet==0 && nByte>0 ){ + *pRc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} /* ** Compare the contents of the pLeft buffer with the pRight/nRight blob. @@ -973,27 +991,24 @@ static int fts5StructureDecode( int iLvl; int nLevel = 0; int nSegment = 0; - int nByte; /* Bytes of space to allocate */ - Fts5Structure *pRet = 0; + int nByte; /* Bytes of space to allocate at pRet */ + Fts5Structure *pRet = 0; /* Structure object to return */ /* Read the total number of levels and segments from the start of the - ** structure record. Use these values to allocate space for the deserialized - ** version of the record. */ + ** structure record. */ i = getVarint32(&pData[i], nLevel); i += getVarint32(&pData[i], nSegment); nByte = ( - sizeof(Fts5Structure) + - sizeof(Fts5StructureLevel) * (nLevel+1) + - sizeof(Fts5StructureSegment) * (nSegment+nLevel+1) + sizeof(Fts5Structure) + /* Main structure */ + sizeof(Fts5StructureLevel) * (nLevel) /* aLevel[] array */ ); - pRet = (Fts5Structure*)sqlite3_malloc(nByte); + pRet = (Fts5Structure*)fts5MallocZero(&rc, nByte); if( pRet ){ - u8 *pSpace = (u8*)&pRet->aLevel[nLevel+1]; - memset(pRet, 0, nByte); pRet->nLevel = nLevel; i += sqlite3GetVarint(&pData[i], &pRet->nWriteCounter); - for(iLvl=0; iLvlaLevel[iLvl]; int nTotal; int iSeg; @@ -1001,26 +1016,82 @@ static int fts5StructureDecode( i += getVarint32(&pData[i], pLvl->nMerge); i += getVarint32(&pData[i], nTotal); assert( nTotal>=pLvl->nMerge ); - pLvl->nSeg = nTotal; - pLvl->aSeg = (Fts5StructureSegment*)pSpace; - pSpace += ((nTotal+1) * sizeof(Fts5StructureSegment)); + pLvl->aSeg = (Fts5StructureSegment*)fts5MallocZero(&rc, + nTotal * sizeof(Fts5StructureSegment) + ); - for(iSeg=0; iSegaSeg[iSeg].iSegid); - i += getVarint32(&pData[i], pLvl->aSeg[iSeg].nHeight); - i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst); - i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast); + if( rc==SQLITE_OK ){ + pLvl->nSeg = nTotal; + for(iSeg=0; iSegaSeg[iSeg].iSegid); + i += getVarint32(&pData[i], pLvl->aSeg[iSeg].nHeight); + i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst); + i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast); + } } } - pRet->aLevel[nLevel].aSeg = (Fts5StructureSegment*)pSpace; - }else{ - rc = SQLITE_NOMEM; } *ppOut = pRet; return rc; } +/* +** +*/ +static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ + if( *pRc==SQLITE_OK ){ + Fts5Structure *pStruct = *ppStruct; + int nLevel = pStruct->nLevel; + int nByte = ( + sizeof(Fts5Structure) + /* Main structure */ + sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */ + ); + + pStruct = sqlite3_realloc(pStruct, nByte); + if( pStruct ){ + memset(&pStruct->aLevel[nLevel], 0, sizeof(Fts5StructureLevel)); + pStruct->nLevel++; + *ppStruct = pStruct; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Extend level iLvl so that there is room for at least nExtra more +** segments. +*/ +static void fts5StructureExtendLevel( + int *pRc, + Fts5Structure *pStruct, + int iLvl, + int nExtra, + int bInsert +){ + if( *pRc==SQLITE_OK ){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + Fts5StructureSegment *aNew; + int nByte; + + nByte = (pLvl->nSeg + nExtra) * sizeof(Fts5StructureSegment); + aNew = sqlite3_realloc(pLvl->aSeg, nByte); + if( aNew ){ + if( bInsert==0 ){ + memset(&aNew[pLvl->nSeg], 0, sizeof(Fts5StructureSegment) * nExtra); + }else{ + int nMove = pLvl->nSeg * sizeof(Fts5StructureSegment); + memmove(&aNew[nExtra], aNew, nMove); + memset(aNew, 0, sizeof(Fts5StructureSegment) * nExtra); + } + pLvl->aSeg = aNew; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + /* ** Read, deserialize and return the structure record for index iIdx. ** @@ -1051,6 +1122,10 @@ static Fts5Structure *fts5StructureRead(Fts5Index *p, int iIdx){ ** call to fts5StructureRead() or fts5StructureDecode(). */ static void fts5StructureRelease(Fts5Structure *pStruct){ + int i; + for(i=0; inLevel; i++){ + sqlite3_free(pStruct->aLevel[i].aSeg); + } sqlite3_free(pStruct); } @@ -1118,6 +1193,10 @@ static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){ # define fts5PrintStructure(x,y) #endif +static int fts5SegmentSize(Fts5StructureSegment *pSeg){ + return 1 + pSeg->pgnoLast - pSeg->pgnoFirst; +} + /* ** Return a copy of index structure pStruct. Except, promote as many segments ** as possible to level iPromote. If an OOM occurs, NULL is returned. @@ -1128,75 +1207,21 @@ static void fts5StructurePromoteTo( int szPromote, Fts5Structure *pStruct ){ - Fts5Structure *pNew; - u8 *pSpace; - int nSeg = fts5StructureCountSegments(pStruct); - int nLvl = pStruct->nLevel; - int nByte = ( - sizeof(Fts5Structure) + - sizeof(Fts5StructureLevel) * (nLvl+1) + - sizeof(Fts5StructureSegment) * (nSeg+nLvl+1) - ); - int iTst; + int il, is; + Fts5StructureLevel *pOut = &pStruct->aLevel[iPromote]; - pNew = fts5IdxMalloc(p, nByte); - if( !pNew ) return; - pNew->nWriteCounter = pStruct->nWriteCounter; - pNew->nLevel = pStruct->nLevel; - pSpace = (u8*)&pNew->aLevel[nLvl+1]; - - for(iTst=0; iTstaLevel[iTst]; - pLvlOut->aSeg = (Fts5StructureSegment*)pSpace; - - if( iTst==iPromote ){ - int il, is; - int nSegCopy = 0; - - /* Figure out the number of segments that will be promoted. */ - for(il=iTst+1; ilnLevel; il++){ - Fts5StructureLevel *pLvl = &pStruct->aLevel[il]; - if( pLvl->nMerge ) break; - for(is=pLvl->nSeg-1; is>=0; is--){ - Fts5StructureSegment *pSeg = &pLvl->aSeg[is]; - int sz = pSeg->pgnoLast - pSeg->pgnoFirst + 1; - if( sz>szPromote ){ - il = pStruct->nLevel; - break; - } - nSegCopy++; - } - } - assert( nSegCopy>0 ); - pSpace += (nSegCopy * sizeof(Fts5StructureSegment)); - pLvlOut->nSeg = nSegCopy; - - for(il=iTst+1; ilnLevel && nSegCopy>0; il++){ - Fts5StructureLevel *pLvl = &pStruct->aLevel[il]; - for(is=pLvl->nSeg-1; is>=0 && nSegCopy>0; is--){ - Fts5StructureSegment *pSeg = &pLvl->aSeg[is]; - nSegCopy--; - memcpy(&pLvlOut->aSeg[nSegCopy], pSeg, sizeof(Fts5StructureSegment)); - pLvl->nSeg--; - } - } - assert( nSegCopy==0 ); + for(il=iPromote+1; ilnLevel; il++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[il]; + for(is=pLvl->nSeg-1; is>=0; is--){ + int sz = fts5SegmentSize(&pLvl->aSeg[is]); + if( sz>szPromote ) return; + fts5StructureExtendLevel(&p->rc, pStruct, iPromote, 1, 1); + if( p->rc ) return; + memcpy(pOut->aSeg, &pLvl->aSeg[is], sizeof(Fts5StructureSegment)); + pOut->nSeg++; + pLvl->nSeg--; } - - nCopy = pStruct->aLevel[iTst].nSeg * sizeof(Fts5StructureSegment); - if( nCopy ) memcpy(pSpace, pStruct->aLevel[iTst].aSeg, nCopy); - pSpace += (nCopy + sizeof(Fts5StructureSegment)); - pLvlOut->nSeg += pStruct->aLevel[iTst].nSeg; } - - fts5PrintStructure("NEW", pNew); - memcpy(pStruct, pNew, nByte); - for(iTst=0; iTstnLevel; iTst++){ - int iOff = pNew->aLevel[iTst].aSeg - (Fts5StructureSegment*)pNew; - pStruct->aLevel[iTst].aSeg = &((Fts5StructureSegment*)pStruct)[iOff]; - } - sqlite3_free(pNew); } /* @@ -3306,9 +3331,10 @@ fflush(stdout); static void fts5IndexWork( Fts5Index *p, /* FTS5 backend object */ int iIdx, /* Index to work on */ - Fts5Structure *pStruct, /* Current structure of index */ + Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ int nLeaf /* Number of output leaves just written */ ){ + Fts5Structure *pStruct = *ppStruct; i64 nWrite; /* Initial value of write-counter */ int nWork; /* Number of work-quanta to perform */ int nRem; /* Number of leaf pages left to write */ @@ -3349,9 +3375,14 @@ static void fts5IndexWork( #endif if( nBestnMinMerge && pStruct->aLevel[iBestLvl].nMerge==0 ) break; + if( iBestLvl==pStruct->nLevel-1 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, iBestLvl+1, 1, 0); fts5IndexMergeLevel(p, iIdx, pStruct, iBestLvl, &nRem); fts5StructurePromote(p, iBestLvl+1, pStruct); assert( nRem==0 || p->rc==SQLITE_OK ); + *ppStruct = pStruct; } } @@ -3393,15 +3424,20 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){ fts5WriteFinish(p, &writer, &nHeight, &pgnoLast); /* Edit the Fts5Structure and write it back to the database. */ - if( pStruct->nLevel==0 ) pStruct->nLevel = 1; - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; - pSeg->iSegid = iSegid; - pSeg->nHeight = nHeight; - pSeg->pgnoFirst = 1; - pSeg->pgnoLast = pgnoLast; + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->nHeight = nHeight; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + } } - fts5IndexWork(p, iHash, pStruct, pgnoLast); + fts5IndexWork(p, iHash, &pStruct, pgnoLast); fts5StructureWrite(p, iHash, pStruct); fts5StructureRelease(pStruct); } diff --git a/manifest b/manifest index ad4cd2c337..1f1332f603 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s"segment\spromotion"\sto\sfts5.\sThis\sprevents\sthe\sFTS\sindex\sfrom\sgrowing\sindefinitely\sas\sdata\sis\sadded\sand\sdeleted. -D 2014-08-07T18:47:33.788 +C Use\smultiple\smemory\sallocations\sfor\sa\ssingle\sFts5Structure\sobject.\sThis\sis\sprobably\sless\sefficient\sbut\smuch\seasier\sto\sget\sright. +D 2014-08-09T18:02:27.223 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5_aux.c 31e581413ecab0962ce2b37468f9f658f36f4b0e F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00 F ext/fts5/fts5_config.c f4ebf143e141b8c77355e3b15aba81b7be51d710 F ext/fts5/fts5_expr.c 7b8e380233176053841904a86006696ee8f6cd24 -F ext/fts5/fts5_index.c 1e001ed7dd4650a0a853b986f34b71c8d3f71ec1 +F ext/fts5/fts5_index.c 75b2ebfa97ad6054bba98cb923cd2d3c6cc5b112 F ext/fts5/fts5_storage.c 2866e7e1de9dc851756c3a9c76b6e1d75e0facb7 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 @@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1baeb1cee61d9c56c718b50af034a24f1018a322 -R c515f8340a51bac4920372c5a517f13d +P ba359d78e166d78e0dc89e3c63a9a41e9ffea989 +R 6a35c04d12672be9d9a3f97537e4f2d1 U dan -Z e9b5b5b15db061fad6b53bb80e61f761 +Z fa46d6698c1e1fd549eedd5dc298f36b diff --git a/manifest.uuid b/manifest.uuid index c812fc95e2..ce1c2f3501 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba359d78e166d78e0dc89e3c63a9a41e9ffea989 \ No newline at end of file +2821825f7a481755a333dcdcad780b3e24448f20 \ No newline at end of file