diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index e7958d9c4d..8673ab7c3e 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1675,7 +1675,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ sqlite3Fts3ExprFree(pCsr->pExpr); sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); - sqlite3_free(pCsr->aMatchinfo); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_free(pCsr); return SQLITE_OK; @@ -3176,7 +3176,7 @@ static int fts3FilterMethod( /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); - sqlite3_free(pCsr->aMatchinfo); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 0748d916d0..9e98c0410f 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -197,6 +197,8 @@ typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; typedef struct Fts3MultiSegReader Fts3MultiSegReader; +typedef struct MatchinfoBuffer MatchinfoBuffer; + /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance @@ -306,9 +308,7 @@ struct Fts3Cursor { i64 iMinDocid; /* Minimum docid to return */ i64 iMaxDocid; /* Maximum docid to return */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ - u32 *aMatchinfo; /* Information about most recent match */ - int nMatchinfo; /* Number of elements in aMatchinfo[] */ - char *zMatchinfo; /* Matchinfo specification */ + MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ }; #define FTS3_EVAL_FILTER 0 @@ -564,6 +564,7 @@ void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, const char *, const char *, int, int ); void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 6abb169ebf..1eceafa050 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -92,6 +92,18 @@ struct MatchInfo { u32 *aMatchinfo; /* Pre-allocated buffer */ }; +/* +** An instance of this structure is used to manage a pair of buffers, each +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below +** for details. +*/ +struct MatchinfoBuffer { + u8 aRef[3]; + int nElem; + int bGlobal; /* Set if global data is loaded */ + char *zMatchinfo; + u32 aMatchinfo[0]; +}; /* @@ -107,6 +119,97 @@ struct StrBuffer { }; +/************************************************************************* +** Start of MatchinfoBuffer code. +*/ + +/* +** Allocate a two-slot MatchinfoBuffer object. +*/ +static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ + MatchinfoBuffer *pRet; + int nByte = sizeof(u32) * (2*nElem + 2) + sizeof(MatchinfoBuffer); + int nStr = strlen(zMatchinfo); + + pRet = sqlite3_malloc(nByte + nStr+1); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); + pRet->nElem = nElem; + pRet->zMatchinfo = ((char*)pRet) + nByte; + memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); + pRet->aRef[0] = 1; + } + + return pRet; +} + +static void fts3MIBufferFree(void *p){ + MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); + + assert( (u32*)p==&pBuf->aMatchinfo[1] + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + ); + if( (u32*)p==&pBuf->aMatchinfo[1] ){ + pBuf->aRef[1] = 0; + }else{ + pBuf->aRef[2] = 0; + } + + if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ + sqlite3_free(pBuf); + } +} + +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ + void (*xRet)(void*) = 0; + u32 *aOut = 0; + + if( p->aRef[1]==0 ){ + p->aRef[1] = 1; + aOut = &p->aMatchinfo[1]; + xRet = fts3MIBufferFree; + } + else if( p->aRef[2]==0 ){ + p->aRef[2] = 1; + aOut = &p->aMatchinfo[p->nElem+2]; + xRet = fts3MIBufferFree; + }else{ + aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32)); + if( aOut ){ + xRet = sqlite3_free; + if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + } + } + + *paOut = aOut; + return xRet; +} + +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ + p->bGlobal = 1; + memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); +} + +/* +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() +*/ +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ + if( p ){ + assert( p->aRef[0]==1 ); + p->aRef[0] = 0; + if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ + sqlite3_free(p); + } + } +} + +/* +** End of MatchinfoBuffer code. +*************************************************************************/ + + /* ** This function is used to help iterate through a position-list. A position ** list is a list of unique integers, sorted from smallest to largest. Each @@ -133,6 +236,8 @@ static void fts3GetDeltaPosition(char **pp, int *piPos){ *piPos += (iVal-2); } +static int fts3ExprLHitsCb(Fts3Expr*,int,void*); + /* ** Helper function for fts3ExprIterate() (see below). */ @@ -143,17 +248,21 @@ static int fts3ExprIterate2( void *pCtx /* Second argument to pass to callback */ ){ int rc; /* Return code */ - int eType = pExpr->eType; /* Type of expression node pExpr */ - if( eType!=FTSQUERY_PHRASE ){ - assert( pExpr->pLeft && pExpr->pRight ); - rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); - if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ - rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); - } + if( x==fts3ExprLHitsCb && pExpr->bEof ){ + rc = SQLITE_OK; }else{ - rc = x(pExpr, *piPhrase, pCtx); - (*piPhrase)++; + int eType = pExpr->eType; /* Type of expression node pExpr */ + if( eType!=FTSQUERY_PHRASE ){ + assert( pExpr->pLeft && pExpr->pRight ); + rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); + if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ + rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); + } + }else{ + rc = x(pExpr, *piPhrase, pCtx); + (*piPhrase)++; + } } return rc; } @@ -819,23 +928,15 @@ static int fts3ExprLHitsCb( int iPhrase, /* Phrase number */ void *pCtx /* Pointer to MatchInfo structure */ ){ - MatchInfo *p = (MatchInfo *)pCtx; - Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; int rc = SQLITE_OK; - int iStart = iPhrase * p->nCol; - Fts3Expr *pEof; /* Ancestor node already at EOF */ + MatchInfo *p = (MatchInfo *)pCtx; /* This must be a phrase */ assert( pExpr->pPhrase ); - /* Initialize all output integers to zero. */ - memset(&p->aMatchinfo[iStart], 0, sizeof(u32) * p->nCol); - - /* Check if this or any parent node is at EOF. If so, then all output - ** values are zero. */ - for(pEof=pExpr; pEof && pEof->bEof==0; pEof=pEof->pParent); - - if( pEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ + if( pExpr->iDocid==p->pCursor->iPrevId ){ + Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; + int iStart = iPhrase * p->nCol; Fts3Phrase *pPhrase = pExpr->pPhrase; char *pIter = pPhrase->doclist.pList; int iCol = 0; @@ -1149,9 +1250,12 @@ static int fts3MatchinfoValues( } break; - case FTS3_MATCHINFO_LHITS: + case FTS3_MATCHINFO_LHITS: { + int nZero = fts3MatchinfoSize(pInfo, FTS3_MATCHINFO_LHITS)*sizeof(u32); + memset(pInfo->aMatchinfo, 0, nZero); (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLHitsCb, (void*)pInfo); break; + } default: { Fts3Expr *pExpr; @@ -1185,6 +1289,7 @@ static int fts3MatchinfoValues( ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). */ static int fts3GetMatchinfo( + sqlite3_context *pCtx, /* Return results here */ Fts3Cursor *pCsr, /* FTS3 Cursor object */ const char *zArg /* Second argument to matchinfo() function */ ){ @@ -1193,6 +1298,9 @@ static int fts3GetMatchinfo( int rc = SQLITE_OK; int bGlobal = 0; /* Collect 'global' stats as well as local */ + u32 *aOut = 0; + void (*xDestroyOut)(void*) = 0; + memset(&sInfo, 0, sizeof(MatchInfo)); sInfo.pCursor = pCsr; sInfo.nCol = pTab->nColumn; @@ -1200,19 +1308,17 @@ static int fts3GetMatchinfo( /* If there is cached matchinfo() data, but the format string for the ** cache does not match the format string for this request, discard ** the cached data. */ - if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){ - assert( pCsr->aMatchinfo ); - sqlite3_free(pCsr->aMatchinfo); - pCsr->zMatchinfo = 0; - pCsr->aMatchinfo = 0; + if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + pCsr->pMIBuffer = 0; } - /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the + /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the ** matchinfo function has been called for this query. In this case ** allocate the array used to accumulate the matchinfo data and ** initialize those elements that are constant for every row. */ - if( pCsr->aMatchinfo==0 ){ + if( pCsr->pMIBuffer==0 ){ int nMatchinfo = 0; /* Number of u32 elements in match-info */ int nArg; /* Bytes in zArg */ int i; /* Used to iterate through zArg */ @@ -1227,23 +1333,35 @@ static int fts3GetMatchinfo( } /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ - nArg = (int)strlen(zArg); - pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1); - if( !pCsr->aMatchinfo ) return SQLITE_NOMEM; + pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); + if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; - pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo]; - pCsr->nMatchinfo = nMatchinfo; - memcpy(pCsr->zMatchinfo, zArg, nArg+1); - memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo); pCsr->isMatchinfoNeeded = 1; bGlobal = 1; } - sInfo.aMatchinfo = pCsr->aMatchinfo; - sInfo.nPhrase = pCsr->nPhrase; - if( pCsr->isMatchinfoNeeded ){ + if( rc==SQLITE_OK ){ + xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); + if( xDestroyOut==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + sInfo.aMatchinfo = aOut; + sInfo.nPhrase = pCsr->nPhrase; rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); - pCsr->isMatchinfoNeeded = 0; + if( bGlobal ){ + fts3MIBufferSetGlobal(pCsr->pMIBuffer); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + if( xDestroyOut ) xDestroyOut(aOut); + }else{ + int n = pCsr->pMIBuffer->nElem * sizeof(u32); + sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); } return rc; @@ -1568,15 +1686,8 @@ void sqlite3Fts3Matchinfo( } /* Retrieve matchinfo() data. */ - rc = fts3GetMatchinfo(pCsr, zFormat); + rc = fts3GetMatchinfo(pContext, pCsr, zFormat); sqlite3Fts3SegmentsClose(pTab); - - if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pContext, rc); - }else{ - int n = pCsr->nMatchinfo * sizeof(u32); - sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); - } } #endif diff --git a/manifest b/manifest index 7b6957d36a..738c7e07a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s#ifdef\sstatements\sto\stest_blob.c\sso\sthat\sit\swill\sbuild\nwith\sSQLITE_OMIT_INCRBLOB. -D 2015-05-05T11:08:02.278 +C Optimizations\sfor\sthe\smatchinfo()\sfunction,\sparticularly\sthe\s'y'\sflag. +D 2015-05-05T19:37:07.819 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 08728ecbeddca339c77bfd564d3484b523dffdb1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,16 +78,16 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 2fb98467f4b670c8934cdd97d1ba3ffa7382764c +F ext/fts3/fts3.c 341e9d9a3c7615bac8e815a8937d576265b22f78 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 59ecaa2d7af0da44c70b6aeaebdcfc070d14abab +F ext/fts3/fts3Int.h bf61766eeb57a6922a8458b894d85e50d1cfb04e F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1 F ext/fts3/fts3_expr.c 71c063da9c2a4167fb54aec089dd5ef33a58c9cb F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009 -F ext/fts3/fts3_snippet.c 40a96ba78e90aba7d7d6d014a18049bb218060fd +F ext/fts3/fts3_snippet.c a1c62f1b7c55d14a13385689ce11aa0e1ada55b8 F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7 F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038 F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860 @@ -1256,7 +1256,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d2cb1becc07fad5cbd48c206c676493ba90cada1 -R e1db2a6afd30f4051a55266b8de5e117 -U drh -Z df7e75264a470014004f32463950df05 +P b8f090e65d010c62df335d0520a36a24904e8bc6 +R db4b4f916c43503b3d5880cc35083941 +T *branch * fts3-matchinfo-y +T *sym-fts3-matchinfo-y * +T -sym-trunk * +U dan +Z 2cd9b208c5989e901293be093105f297 diff --git a/manifest.uuid b/manifest.uuid index c9040b853e..c76ab92b50 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8f090e65d010c62df335d0520a36a24904e8bc6 \ No newline at end of file +dddd7e182943a1d3a9d32830e819a63f1a228d6d \ No newline at end of file