database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-29 08:01:23 +03:00
Code Activity

Optimizations for the matchinfo() function, particularly the 'y' flag.

Browse Source 
FossilOrigin-Name: dddd7e182943a1d3a9d32830e819a63f1a228d6d
This commit is contained in:
dan
2015-05-05 19:37:07 +00:00
parent bfa582a3f3
commit b2bd727e52
5 changed files with 178 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ext/fts3/fts3.c
View File

@ -1675,7 +1675,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
sqlite3Fts3ExprFree(pCsr->pExpr);
sqlite3Fts3FreeDeferredTokens(pCsr);
sqlite3_free(pCsr->aDoclist);
sqlite3_free(pCsr->aMatchinfo);
sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
sqlite3_free(pCsr);
return SQLITE_OK;
@ -3176,7 +3176,7 @@ static int fts3FilterMethod(
/* In case the cursor has been used before, clear it now. */
sqlite3_finalize(pCsr->pStmt);
sqlite3_free(pCsr->aDoclist);
sqlite3_free(pCsr->aMatchinfo);
sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
sqlite3Fts3ExprFree(pCsr->pExpr);
memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));

7
ext/fts3/fts3Int.h
View File

@ -197,6 +197,8 @@ typedef struct Fts3DeferredToken Fts3DeferredToken;
typedef struct Fts3SegReader Fts3SegReader;
typedef struct Fts3MultiSegReader Fts3MultiSegReader;
typedef struct MatchinfoBuffer MatchinfoBuffer;
/*
** A connection to a fulltext index is an instance of the following
** structure. The xCreate and xConnect methods create an instance
@ -306,9 +308,7 @@ struct Fts3Cursor {
i64 iMinDocid; /* Minimum docid to return */
i64 iMaxDocid; /* Maximum docid to return */
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
u32 *aMatchinfo; /* Information about most recent match */
int nMatchinfo; /* Number of elements in aMatchinfo[] */
char *zMatchinfo; /* Matchinfo specification */
MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */
};
#define FTS3_EVAL_FILTER 0
@ -564,6 +564,7 @@ void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p);
/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,

207
ext/fts3/fts3_snippet.c
View File

@ -92,6 +92,18 @@ struct MatchInfo {
u32 *aMatchinfo; /* Pre-allocated buffer */
};
/*
** An instance of this structure is used to manage a pair of buffers, each
** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below
** for details.
*/
struct MatchinfoBuffer {
u8 aRef[3];
int nElem;
int bGlobal; /* Set if global data is loaded */
char *zMatchinfo;
u32 aMatchinfo[0];
};
/*
@ -107,6 +119,97 @@ struct StrBuffer {
};
/*************************************************************************
** Start of MatchinfoBuffer code.
*/
/*
** Allocate a two-slot MatchinfoBuffer object.
*/
static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){
MatchinfoBuffer *pRet;
int nByte = sizeof(u32) * (2*nElem + 2) + sizeof(MatchinfoBuffer);
int nStr = strlen(zMatchinfo);
pRet = sqlite3_malloc(nByte + nStr+1);
if( pRet ){
memset(pRet, 0, nByte);
pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet;
pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1);
pRet->nElem = nElem;
pRet->zMatchinfo = ((char*)pRet) + nByte;
memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1);
pRet->aRef[0] = 1;
}
return pRet;
}
static void fts3MIBufferFree(void *p){
MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]);
assert( (u32*)p==&pBuf->aMatchinfo[1]
|| (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2]
);
if( (u32*)p==&pBuf->aMatchinfo[1] ){
pBuf->aRef[1] = 0;
}else{
pBuf->aRef[2] = 0;
}
if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){
sqlite3_free(pBuf);
}
}
static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){
void (*xRet)(void*) = 0;
u32 *aOut = 0;
if( p->aRef[1]==0 ){
p->aRef[1] = 1;
aOut = &p->aMatchinfo[1];
xRet = fts3MIBufferFree;
}
else if( p->aRef[2]==0 ){
p->aRef[2] = 1;
aOut = &p->aMatchinfo[p->nElem+2];
xRet = fts3MIBufferFree;
}else{
aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32));
if( aOut ){
xRet = sqlite3_free;
if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32));
}
}
*paOut = aOut;
return xRet;
}
static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){
p->bGlobal = 1;
memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32));
}
/*
** Free a MatchinfoBuffer object allocated using fts3MIBufferNew()
*/
void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){
if( p ){
assert( p->aRef[0]==1 );
p->aRef[0] = 0;
if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){
sqlite3_free(p);
}
}
}
/*
** End of MatchinfoBuffer code.
*************************************************************************/
/*
** This function is used to help iterate through a position-list. A position
** list is a list of unique integers, sorted from smallest to largest. Each
@ -133,6 +236,8 @@ static void fts3GetDeltaPosition(char **pp, int *piPos){
*piPos += (iVal-2);
}
static int fts3ExprLHitsCb(Fts3Expr*,int,void*);
/*
** Helper function for fts3ExprIterate() (see below).
*/
@ -143,17 +248,21 @@ static int fts3ExprIterate2(
void *pCtx /* Second argument to pass to callback */
){
int rc; /* Return code */
int eType = pExpr->eType; /* Type of expression node pExpr */
if( eType!=FTSQUERY_PHRASE ){
assert( pExpr->pLeft && pExpr->pRight );
rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx);
if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){
rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx);
}
if( x==fts3ExprLHitsCb && pExpr->bEof ){
rc = SQLITE_OK;
}else{
rc = x(pExpr, *piPhrase, pCtx);
(*piPhrase)++;
int eType = pExpr->eType; /* Type of expression node pExpr */
if( eType!=FTSQUERY_PHRASE ){
assert( pExpr->pLeft && pExpr->pRight );
rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx);
if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){
rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx);
}
}else{
rc = x(pExpr, *piPhrase, pCtx);
(*piPhrase)++;
}
}
return rc;
}
@ -819,23 +928,15 @@ static int fts3ExprLHitsCb(
int iPhrase, /* Phrase number */
void *pCtx /* Pointer to MatchInfo structure */
){
MatchInfo *p = (MatchInfo *)pCtx;
Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
int rc = SQLITE_OK;
int iStart = iPhrase * p->nCol;
Fts3Expr *pEof; /* Ancestor node already at EOF */
MatchInfo *p = (MatchInfo *)pCtx;
/* This must be a phrase */
assert( pExpr->pPhrase );
/* Initialize all output integers to zero. */
memset(&p->aMatchinfo[iStart], 0, sizeof(u32) * p->nCol);
/* Check if this or any parent node is at EOF. If so, then all output
** values are zero. */
for(pEof=pExpr; pEof && pEof->bEof==0; pEof=pEof->pParent);
if( pEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
if( pExpr->iDocid==p->pCursor->iPrevId ){
Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
int iStart = iPhrase * p->nCol;
Fts3Phrase *pPhrase = pExpr->pPhrase;
char *pIter = pPhrase->doclist.pList;
int iCol = 0;
@ -1149,9 +1250,12 @@ static int fts3MatchinfoValues(
}
break;
case FTS3_MATCHINFO_LHITS:
case FTS3_MATCHINFO_LHITS: {
int nZero = fts3MatchinfoSize(pInfo, FTS3_MATCHINFO_LHITS)*sizeof(u32);
memset(pInfo->aMatchinfo, 0, nZero);
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprLHitsCb, (void*)pInfo);
break;
}
default: {
Fts3Expr *pExpr;
@ -1185,6 +1289,7 @@ static int fts3MatchinfoValues(
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
static int fts3GetMatchinfo(
sqlite3_context *pCtx, /* Return results here */
Fts3Cursor *pCsr, /* FTS3 Cursor object */
const char *zArg /* Second argument to matchinfo() function */
){
@ -1193,6 +1298,9 @@ static int fts3GetMatchinfo(
int rc = SQLITE_OK;
int bGlobal = 0; /* Collect 'global' stats as well as local */
u32 *aOut = 0;
void (*xDestroyOut)(void*) = 0;
memset(&sInfo, 0, sizeof(MatchInfo));
sInfo.pCursor = pCsr;
sInfo.nCol = pTab->nColumn;
@ -1200,19 +1308,17 @@ static int fts3GetMatchinfo(
/* If there is cached matchinfo() data, but the format string for the
** cache does not match the format string for this request, discard
** the cached data. */
if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
assert( pCsr->aMatchinfo );
sqlite3_free(pCsr->aMatchinfo);
pCsr->zMatchinfo = 0;
pCsr->aMatchinfo = 0;
if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){
sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
pCsr->pMIBuffer = 0;
}
/* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
/* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the
** matchinfo function has been called for this query. In this case
** allocate the array used to accumulate the matchinfo data and
** initialize those elements that are constant for every row.
*/
if( pCsr->aMatchinfo==0 ){
if( pCsr->pMIBuffer==0 ){
int nMatchinfo = 0; /* Number of u32 elements in match-info */
int nArg; /* Bytes in zArg */
int i; /* Used to iterate through zArg */
@ -1227,23 +1333,35 @@ static int fts3GetMatchinfo(
}
/* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
nArg = (int)strlen(zArg);
pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);
if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;
pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg);
if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM;
pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];
pCsr->nMatchinfo = nMatchinfo;
memcpy(pCsr->zMatchinfo, zArg, nArg+1);
memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
pCsr->isMatchinfoNeeded = 1;
bGlobal = 1;
}
sInfo.aMatchinfo = pCsr->aMatchinfo;
sInfo.nPhrase = pCsr->nPhrase;
if( pCsr->isMatchinfoNeeded ){
if( rc==SQLITE_OK ){
xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut);
if( xDestroyOut==0 ){
rc = SQLITE_NOMEM;
}
}
if( rc==SQLITE_OK ){
sInfo.aMatchinfo = aOut;
sInfo.nPhrase = pCsr->nPhrase;
rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
pCsr->isMatchinfoNeeded = 0;
if( bGlobal ){
fts3MIBufferSetGlobal(pCsr->pMIBuffer);
}
}
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
if( xDestroyOut ) xDestroyOut(aOut);
}else{
int n = pCsr->pMIBuffer->nElem * sizeof(u32);
sqlite3_result_blob(pCtx, aOut, n, xDestroyOut);
}
return rc;
@ -1568,15 +1686,8 @@ void sqlite3Fts3Matchinfo(
}
/* Retrieve matchinfo() data. */
rc = fts3GetMatchinfo(pCsr, zFormat);
rc = fts3GetMatchinfo(pContext, pCsr, zFormat);
sqlite3Fts3SegmentsClose(pTab);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pContext, rc);
}else{
int n = pCsr->nMatchinfo * sizeof(u32);
sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
}
}
#endif

21
manifest
View File

@ -1,5 +1,5 @@
C Add\s#ifdef\sstatements\sto\stest_blob.c\sso\sthat\sit\swill\sbuild\nwith\sSQLITE_OMIT_INCRBLOB.
D 2015-05-05T11:08:02.278
C Optimizations\sfor\sthe\smatchinfo()\sfunction,\sparticularly\sthe\s'y'\sflag.
D 2015-05-05T19:37:07.819
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 08728ecbeddca339c77bfd564d3484b523dffdb1
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -78,16 +78,16 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 2fb98467f4b670c8934cdd97d1ba3ffa7382764c
F ext/fts3/fts3.c 341e9d9a3c7615bac8e815a8937d576265b22f78
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h 59ecaa2d7af0da44c70b6aeaebdcfc070d14abab
F ext/fts3/fts3Int.h bf61766eeb57a6922a8458b894d85e50d1cfb04e
F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1
F ext/fts3/fts3_expr.c 71c063da9c2a4167fb54aec089dd5ef33a58c9cb
F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60
F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5
F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009
F ext/fts3/fts3_snippet.c 40a96ba78e90aba7d7d6d014a18049bb218060fd
F ext/fts3/fts3_snippet.c a1c62f1b7c55d14a13385689ce11aa0e1ada55b8
F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7
F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038
F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860
@ -1256,7 +1256,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P d2cb1becc07fad5cbd48c206c676493ba90cada1
R e1db2a6afd30f4051a55266b8de5e117
U drh
Z df7e75264a470014004f32463950df05
P b8f090e65d010c62df335d0520a36a24904e8bc6
R db4b4f916c43503b3d5880cc35083941
T *branch * fts3-matchinfo-y
T *sym-fts3-matchinfo-y *
T -sym-trunk *
U dan
Z 2cd9b208c5989e901293be093105f297

2
manifest.uuid
View File

@ -1 +1 @@
b8f090e65d010c62df335d0520a36a24904e8bc6
dddd7e182943a1d3a9d32830e819a63f1a228d6d