mirror of
https://github.com/sqlite/sqlite.git
synced 2025-07-30 19:03:16 +03:00
Modify fts3 to support a more complex expression syntax that allows parenthesis. The new syntax is not entirely backwards compatible, so is disabled by default. Use -DSQLITE_ENABLE_FTS3_PARENTHESIS to enable it. (CVS 6034)
FossilOrigin-Name: 7389b9ecb80294569845c40a23e0c832d07f7a45
This commit is contained in:
danielk1977
795
ext/fts3/fts3.c
795
ext/fts3/fts3.c
File diff suppressed because it is too large
Load Diff
864
ext/fts3/fts3_expr.c
Normal file
864
ext/fts3/fts3_expr.c
Normal file
@ -0,0 +1,864 @@
|
||||
/*
|
||||
** 2008 Nov 28
|
||||
**
|
||||
** The author disclaims copyright to this source code. In place of
|
||||
** a legal notice, here is a blessing:
|
||||
**
|
||||
** May you do good and not evil.
|
||||
** May you find forgiveness for yourself and forgive others.
|
||||
** May you share freely, never taking more than you give.
|
||||
**
|
||||
******************************************************************************
|
||||
**
|
||||
** This module contains code that implements a parser for fts3 query strings
|
||||
** (the right-hand argument to the MATCH operator). Because the supported
|
||||
** syntax is relatively simple, the whole tokenizer/parser system is
|
||||
** hand-coded. The public interface to this module is declared in source
|
||||
** code file "fts3_expr.h".
|
||||
*/
|
||||
|
||||
/*
|
||||
** By default, this module parses the legacy syntax that has been
|
||||
** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS
|
||||
** is defined, then it uses the new syntax. The differences between
|
||||
** the new and the old syntaxes are:
|
||||
**
|
||||
** a) The new syntax supports parenthesis. The old does not.
|
||||
**
|
||||
** b) The new syntax supports the AND and NOT operators. The old does not.
|
||||
**
|
||||
** c) The old syntax supports the "-" token qualifier. This is not
|
||||
** supported by the new syntax (it is replaced by the NOT operator).
|
||||
**
|
||||
** d) When using the old syntax, the OR operator has a greater precedence
|
||||
** than an implicit AND. When using the new, both implicity and explicit
|
||||
** AND operators have a higher precedence than OR.
|
||||
**
|
||||
** If compiled with SQLITE_TEST defined, then this module exports the
|
||||
** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
|
||||
** to zero causes the module to use the old syntax. If it is set to
|
||||
** non-zero the new syntax is activated. This is so both syntaxes can
|
||||
** be tested using a single build of testfixture.
|
||||
*/
|
||||
#ifdef SQLITE_TEST
|
||||
int sqlite3_fts3_enable_parentheses = 0;
|
||||
#else
|
||||
# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
|
||||
# define sqlite3_fts3_enable_parentheses 1
|
||||
# else
|
||||
# define sqlite3_fts3_enable_parentheses 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
** Default span for NEAR operators.
|
||||
*/
|
||||
#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
|
||||
|
||||
#include "fts3_expr.h"
|
||||
#include "sqlite3.h"
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
|
||||
typedef struct ParseContext ParseContext;
|
||||
struct ParseContext {
|
||||
sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
|
||||
const char **azCol; /* Array of column names for fts3 table */
|
||||
int nCol; /* Number of entries in azCol[] */
|
||||
int iDefaultCol; /* Default column to query */
|
||||
sqlite3_context *pCtx; /* Write error message here */
|
||||
int nNest; /* Number of nested brackets */
|
||||
};
|
||||
|
||||
/*
|
||||
** This function is equivalent to the standard isspace() function.
|
||||
**
|
||||
** The standard isspace() can be awkward to use safely, because although it
|
||||
** is defined to accept an argument of type int, its behaviour when passed
|
||||
** an integer that falls outside of the range of the unsigned char type
|
||||
** is undefined (and sometimes, "undefined" means segfault). This wrapper
|
||||
** is defined to accept an argument of type char, and always returns 0 for
|
||||
** any values that fall outside of the range of the unsigned char type (i.e.
|
||||
** negative values).
|
||||
*/
|
||||
static int safe_isspace(char c){
|
||||
return (c&0x80)==0 ? isspace(c) : 0;
|
||||
}
|
||||
|
||||
/*
|
||||
** Extract the next token from buffer z (length n) using the tokenizer
|
||||
** and other information (column names etc.) in pParse. Create an Fts3Expr
|
||||
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
|
||||
** single token and set *ppExpr to point to it. If the end of the buffer is
|
||||
** reached before a token is found, set *ppExpr to zero. It is the
|
||||
** responsibility of the caller to eventually deallocate the allocated
|
||||
** Fts3Expr structure (if any) by passing it to sqlite3_free().
|
||||
**
|
||||
** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation
|
||||
** fails.
|
||||
*/
|
||||
static int getNextToken(
|
||||
ParseContext *pParse, /* fts3 query parse context */
|
||||
int iCol, /* Value for Fts3Phrase.iColumn */
|
||||
const char *z, int n, /* Input string */
|
||||
Fts3Expr **ppExpr, /* OUT: expression */
|
||||
int *pnConsumed /* OUT: Number of bytes consumed */
|
||||
){
|
||||
sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
|
||||
sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
|
||||
int rc;
|
||||
sqlite3_tokenizer_cursor *pCursor;
|
||||
Fts3Expr *pRet = 0;
|
||||
int nConsumed = 0;
|
||||
|
||||
rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
|
||||
if( rc==SQLITE_OK ){
|
||||
const char *zToken;
|
||||
int nToken, iStart, iEnd, iPosition;
|
||||
int nByte; /* total space to allocate */
|
||||
|
||||
pCursor->pTokenizer = pTokenizer;
|
||||
rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
|
||||
|
||||
if( rc==SQLITE_OK ){
|
||||
nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
|
||||
pRet = (Fts3Expr *)sqlite3_malloc(nByte);
|
||||
if( !pRet ){
|
||||
rc = SQLITE_NOMEM;
|
||||
}else{
|
||||
memset(pRet, 0, nByte);
|
||||
pRet->eType = FTSQUERY_PHRASE;
|
||||
pRet->pPhrase = (Fts3Phrase *)&pRet[1];
|
||||
pRet->pPhrase->nToken = 1;
|
||||
pRet->pPhrase->iColumn = iCol;
|
||||
pRet->pPhrase->aToken[0].n = nToken;
|
||||
pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
|
||||
memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
|
||||
|
||||
if( iEnd<n && z[iEnd]=='*' ){
|
||||
pRet->pPhrase->aToken[0].isPrefix = 1;
|
||||
iEnd++;
|
||||
}
|
||||
if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
|
||||
pRet->pPhrase->isNot = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
nConsumed = iEnd;
|
||||
|
||||
pModule->xClose(pCursor);
|
||||
}
|
||||
|
||||
*pnConsumed = nConsumed;
|
||||
*ppExpr = pRet;
|
||||
return rc;
|
||||
}
|
||||
|
||||
void realloc_or_free(void **ppOrig, int nNew){
|
||||
void *pRet = sqlite3_realloc(*ppOrig, nNew);
|
||||
if( !pRet ){
|
||||
sqlite3_free(*ppOrig);
|
||||
}
|
||||
*ppOrig = pRet;
|
||||
}
|
||||
|
||||
/*
|
||||
** Buffer zInput, length nInput, contains the contents of a quoted string
|
||||
** that appeared as part of an fts3 query expression. Neither quote character
|
||||
** is included in the buffer. This function attempts to tokenize the entire
|
||||
** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE
|
||||
** containing the results.
|
||||
**
|
||||
** If successful, SQLITE_OK is returned and *ppExpr set to point at the
|
||||
** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory
|
||||
** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set
|
||||
** to 0.
|
||||
*/
|
||||
static int getNextString(
|
||||
ParseContext *pParse, /* fts3 query parse context */
|
||||
const char *zInput, int nInput, /* Input string */
|
||||
Fts3Expr **ppExpr /* OUT: expression */
|
||||
){
|
||||
sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
|
||||
sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
|
||||
int rc;
|
||||
Fts3Expr *p = 0;
|
||||
sqlite3_tokenizer_cursor *pCursor = 0;
|
||||
char *zTemp = 0;
|
||||
int nTemp = 0;
|
||||
|
||||
rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
|
||||
if( rc==SQLITE_OK ){
|
||||
int ii;
|
||||
pCursor->pTokenizer = pTokenizer;
|
||||
for(ii=0; rc==SQLITE_OK; ii++){
|
||||
const char *zToken;
|
||||
int nToken, iBegin, iEnd, iPos;
|
||||
rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
|
||||
if( rc==SQLITE_OK ){
|
||||
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
|
||||
realloc_or_free((void **)&p, nByte+ii*sizeof(struct PhraseToken));
|
||||
realloc_or_free((void **)&zTemp, nTemp + nToken);
|
||||
if( !p || !zTemp ){
|
||||
goto no_mem;
|
||||
}
|
||||
if( ii==0 ){
|
||||
memset(p, 0, nByte);
|
||||
p->pPhrase = (Fts3Phrase *)&p[1];
|
||||
p->eType = FTSQUERY_PHRASE;
|
||||
p->pPhrase->iColumn = pParse->iDefaultCol;
|
||||
}
|
||||
p->pPhrase = (Fts3Phrase *)&p[1];
|
||||
p->pPhrase->nToken = ii+1;
|
||||
p->pPhrase->aToken[ii].n = nToken;
|
||||
memcpy(&zTemp[nTemp], zToken, nToken);
|
||||
nTemp += nToken;
|
||||
if( iEnd<nInput && zInput[iEnd]=='*' ){
|
||||
p->pPhrase->aToken[ii].isPrefix = 1;
|
||||
}else{
|
||||
p->pPhrase->aToken[ii].isPrefix = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pModule->xClose(pCursor);
|
||||
pCursor = 0;
|
||||
}
|
||||
|
||||
if( rc==SQLITE_DONE ){
|
||||
int jj;
|
||||
char *zNew;
|
||||
int nNew = 0;
|
||||
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
|
||||
nByte += (p->pPhrase->nToken-1) * sizeof(struct PhraseToken);
|
||||
realloc_or_free((void **)&p, nByte + nTemp);
|
||||
if( !p ){
|
||||
goto no_mem;
|
||||
}
|
||||
p->pPhrase = (Fts3Phrase *)&p[1];
|
||||
zNew = &(((char *)p)[nByte]);
|
||||
memcpy(zNew, zTemp, nTemp);
|
||||
for(jj=0; jj<p->pPhrase->nToken; jj++){
|
||||
p->pPhrase->aToken[jj].z = &zNew[nNew];
|
||||
nNew += p->pPhrase->aToken[jj].n;
|
||||
}
|
||||
sqlite3_free(zTemp);
|
||||
rc = SQLITE_OK;
|
||||
}
|
||||
|
||||
*ppExpr = p;
|
||||
return rc;
|
||||
no_mem:
|
||||
|
||||
if( pCursor ){
|
||||
pModule->xClose(pCursor);
|
||||
}
|
||||
sqlite3_free(zTemp);
|
||||
sqlite3_free(p);
|
||||
*ppExpr = 0;
|
||||
return SQLITE_NOMEM;
|
||||
}
|
||||
|
||||
/*
|
||||
** Function getNextNode(), which is called by fts3ExprParse(), may itself
|
||||
** call fts3ExprParse(). So this forward declaration is required.
|
||||
*/
|
||||
static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
|
||||
|
||||
/*
|
||||
** The output variable *ppExpr is populated with an allocated Fts3Expr
|
||||
** structure, or set to 0 if the end of the input buffer is reached.
|
||||
**
|
||||
** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
|
||||
** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
|
||||
** If SQLITE_ERROR is returned, pContext is populated with an error message.
|
||||
*/
|
||||
static int getNextNode(
|
||||
ParseContext *pParse, /* fts3 query parse context */
|
||||
const char *z, int n, /* Input string */
|
||||
Fts3Expr **ppExpr, /* OUT: expression */
|
||||
int *pnConsumed /* OUT: Number of bytes consumed */
|
||||
){
|
||||
struct Fts3Keyword {
|
||||
char *z;
|
||||
int n;
|
||||
int eType;
|
||||
} aKeyword[] = {
|
||||
{ "OR" , 2, FTSQUERY_OR },
|
||||
{ "AND", 3, FTSQUERY_AND },
|
||||
{ "NOT", 3, FTSQUERY_NOT },
|
||||
{ "NEAR", 4, FTSQUERY_NEAR }
|
||||
};
|
||||
int ii;
|
||||
int iCol;
|
||||
int iColLen;
|
||||
int rc;
|
||||
Fts3Expr *pRet = 0;
|
||||
|
||||
const char *zInput = z;
|
||||
int nInput = n;
|
||||
|
||||
/* Skip over any whitespace before checking for a keyword, an open or
|
||||
** close bracket, or a quoted string.
|
||||
*/
|
||||
while( nInput>0 && safe_isspace(*zInput) ){
|
||||
nInput--;
|
||||
zInput++;
|
||||
}
|
||||
|
||||
/* See if we are dealing with a keyword. */
|
||||
for(ii=0; ii<sizeof(aKeyword)/sizeof(struct Fts3Keyword); ii++){
|
||||
struct Fts3Keyword *pKey = &aKeyword[ii];
|
||||
|
||||
if( (0==sqlite3_fts3_enable_parentheses)
|
||||
&& (pKey->eType==FTSQUERY_AND || pKey->eType==FTSQUERY_NOT)
|
||||
){
|
||||
continue;
|
||||
}
|
||||
|
||||
if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){
|
||||
int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
|
||||
int nKey = pKey->n;
|
||||
char cNext;
|
||||
|
||||
/* If this is a "NEAR" keyword, check for an explicit nearness. */
|
||||
if( pKey->eType==FTSQUERY_NEAR ){
|
||||
assert( nKey==4 );
|
||||
if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){
|
||||
nNear = 0;
|
||||
for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){
|
||||
nNear = nNear * 10 + (zInput[nKey] - '0');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* At this point this is probably a keyword. But for that to be true,
|
||||
** the next byte must contain either whitespace, an open or close
|
||||
** bracket, a quote character, or EOF.
|
||||
*/
|
||||
cNext = zInput[nKey];
|
||||
if( safe_isspace(cNext)
|
||||
|| cNext=='"' || cNext=='(' || cNext==')' || cNext==0
|
||||
){
|
||||
pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
|
||||
memset(pRet, 0, sizeof(Fts3Expr));
|
||||
pRet->eType = pKey->eType;
|
||||
pRet->nNear = nNear;
|
||||
*ppExpr = pRet;
|
||||
*pnConsumed = (zInput - z) + nKey;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
/* Turns out that wasn't a keyword after all. This happens if the
|
||||
** user has supplied a token such as "ORacle". Continue.
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
/* Check for an open bracket. */
|
||||
if( sqlite3_fts3_enable_parentheses ){
|
||||
if( *zInput=='(' ){
|
||||
int nConsumed;
|
||||
int rc;
|
||||
pParse->nNest++;
|
||||
rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
|
||||
*pnConsumed = (zInput - z) + 1 + nConsumed;
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* Check for a close bracket. */
|
||||
if( *zInput==')' ){
|
||||
pParse->nNest--;
|
||||
*pnConsumed = (zInput - z) + 1;
|
||||
return SQLITE_DONE;
|
||||
}
|
||||
}
|
||||
|
||||
/* See if we are dealing with a quoted phrase. If this is the case, then
|
||||
** search for the closing quote and pass the whole string to getNextString()
|
||||
** for processing. This is easy to do, as fts3 has no syntax for escaping
|
||||
** a quote character embedded in a string.
|
||||
*/
|
||||
if( *zInput=='"' ){
|
||||
for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);
|
||||
*pnConsumed = (zInput - z) + ii + 1;
|
||||
if( ii==nInput ){
|
||||
return SQLITE_ERROR;
|
||||
}
|
||||
return getNextString(pParse, &zInput[1], ii-1, ppExpr);
|
||||
}
|
||||
|
||||
|
||||
/* If control flows to this point, this must be a regular token, or
|
||||
** the end of the input. Read a regular token using the sqlite3_tokenizer
|
||||
** interface. Before doing so, figure out if there is an explicit
|
||||
** column specifier for the token.
|
||||
**
|
||||
** TODO: Strangely, it is not possible to associate a column specifier
|
||||
** with a quoted phrase, only with a single token. Not sure if this was
|
||||
** an implementation artifact or an intentional decision when fts3 was
|
||||
** first implemented. Whichever it was, this module duplicates the
|
||||
** limitation.
|
||||
*/
|
||||
iCol = pParse->iDefaultCol;
|
||||
iColLen = 0;
|
||||
for(ii=0; ii<pParse->nCol; ii++){
|
||||
const char *zStr = pParse->azCol[ii];
|
||||
int nStr = strlen(zStr);
|
||||
if( nInput>nStr && zInput[nStr]==':' && memcmp(zStr, zInput, nStr)==0 ){
|
||||
iCol = ii;
|
||||
iColLen = ((zInput - z) + nStr + 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed);
|
||||
*pnConsumed += iColLen;
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** The argument is an Fts3Expr structure for a binary operator (any type
|
||||
** except an FTSQUERY_PHRASE). Return an integer value representing the
|
||||
** precedence of the operator. Lower values have a higher precedence (i.e.
|
||||
** group more tightly). For example, in the C language, the == operator
|
||||
** groups more tightly than ||, and would therefore have a higher precedence.
|
||||
**
|
||||
** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS
|
||||
** is defined), the order of the operators in precedence from highest to
|
||||
** lowest is:
|
||||
**
|
||||
** NEAR
|
||||
** NOT
|
||||
** AND (including implicit ANDs)
|
||||
** OR
|
||||
**
|
||||
** Note that when using the old query syntax, the OR operator has a higher
|
||||
** precedence than the AND operator.
|
||||
*/
|
||||
static int opPrecedence(Fts3Expr *p){
|
||||
assert( p->eType!=FTSQUERY_PHRASE );
|
||||
if( sqlite3_fts3_enable_parentheses ){
|
||||
return p->eType;
|
||||
}else if( p->eType==FTSQUERY_NEAR ){
|
||||
return 1;
|
||||
}else if( p->eType==FTSQUERY_OR ){
|
||||
return 2;
|
||||
}
|
||||
assert( p->eType==FTSQUERY_AND );
|
||||
return 3;
|
||||
}
|
||||
|
||||
/*
|
||||
** Argument ppHead contains a pointer to the current head of a query
|
||||
** expression tree being parsed. pPrev is the expression node most recently
|
||||
** inserted into the tree. This function adds pNew, which is always a binary
|
||||
** operator node, into the expression tree based on the relative precedence
|
||||
** of pNew and the existing nodes of the tree. This may result in the head
|
||||
** of the tree changing, in which case *ppHead is set to the new root node.
|
||||
*/
|
||||
static void insertBinaryOperator(
|
||||
Fts3Expr **ppHead, /* Pointer to the root node of a tree */
|
||||
Fts3Expr *pPrev, /* Node most recently inserted into the tree */
|
||||
Fts3Expr *pNew /* New binary node to insert into expression tree */
|
||||
){
|
||||
Fts3Expr *pSplit = pPrev;
|
||||
while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){
|
||||
pSplit = pSplit->pParent;
|
||||
}
|
||||
|
||||
if( pSplit->pParent ){
|
||||
assert( pSplit->pParent->pRight==pSplit );
|
||||
pSplit->pParent->pRight = pNew;
|
||||
pNew->pParent = pSplit->pParent;
|
||||
}else{
|
||||
*ppHead = pNew;
|
||||
}
|
||||
pNew->pLeft = pSplit;
|
||||
pSplit->pParent = pNew;
|
||||
}
|
||||
|
||||
/*
|
||||
** Parse the fts3 query expression found in buffer z, length n. This function
|
||||
** returns either when the end of the buffer is reached or an unmatched
|
||||
** closing bracket - ')' - is encountered.
|
||||
**
|
||||
** If successful, SQLITE_OK is returned, *ppExpr is set to point to the
|
||||
** parsed form of the expression and *pnConsumed is set to the number of
|
||||
** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM
|
||||
** (out of memory error) or SQLITE_ERROR (parse error) is returned.
|
||||
*/
|
||||
static int fts3ExprParse(
|
||||
ParseContext *pParse, /* fts3 query parse context */
|
||||
const char *z, int n, /* Text of MATCH query */
|
||||
Fts3Expr **ppExpr, /* OUT: Parsed query structure */
|
||||
int *pnConsumed /* OUT: Number of bytes consumed */
|
||||
){
|
||||
Fts3Expr *pRet = 0;
|
||||
Fts3Expr *pPrev = 0;
|
||||
Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */
|
||||
int nIn = n;
|
||||
const char *zIn = z;
|
||||
int rc = SQLITE_OK;
|
||||
int isRequirePhrase = 1;
|
||||
|
||||
while( rc==SQLITE_OK ){
|
||||
Fts3Expr *p = 0;
|
||||
int nByte;
|
||||
rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
|
||||
if( rc==SQLITE_OK ){
|
||||
int isPhrase;
|
||||
|
||||
if( !sqlite3_fts3_enable_parentheses
|
||||
&& p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
|
||||
){
|
||||
/* Create an implicit NOT operator. */
|
||||
Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr));
|
||||
if( !pNot ){
|
||||
sqlite3Fts3ExprFree(p);
|
||||
rc = SQLITE_NOMEM;
|
||||
goto exprparse_out;
|
||||
}
|
||||
memset(pNot, 0, sizeof(Fts3Expr));
|
||||
pNot->eType = FTSQUERY_NOT;
|
||||
pNot->pRight = p;
|
||||
if( pNotBranch ){
|
||||
pNotBranch->pLeft = p;
|
||||
pNot->pRight = pNotBranch;
|
||||
}
|
||||
pNotBranch = pNot;
|
||||
}else{
|
||||
assert( p->eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
|
||||
isPhrase = (p->eType==FTSQUERY_PHRASE || p->pLeft);
|
||||
if( !isPhrase && isRequirePhrase ){
|
||||
sqlite3Fts3ExprFree(p);
|
||||
rc = SQLITE_ERROR;
|
||||
goto exprparse_out;
|
||||
}
|
||||
|
||||
if( isPhrase && !isRequirePhrase ){
|
||||
/* Insert an implicit AND operator. */
|
||||
Fts3Expr *pAnd;
|
||||
assert( pRet && pPrev );
|
||||
pAnd = sqlite3_malloc(sizeof(Fts3Expr));
|
||||
if( !pAnd ){
|
||||
sqlite3Fts3ExprFree(p);
|
||||
rc = SQLITE_NOMEM;
|
||||
goto exprparse_out;
|
||||
}
|
||||
memset(pAnd, 0, sizeof(Fts3Expr));
|
||||
pAnd->eType = FTSQUERY_AND;
|
||||
insertBinaryOperator(&pRet, pPrev, pAnd);
|
||||
pPrev = pAnd;
|
||||
}
|
||||
|
||||
if( pPrev && (
|
||||
(pPrev->eType==FTSQUERY_NEAR && p->eType!=FTSQUERY_PHRASE)
|
||||
|| (p->eType==FTSQUERY_NEAR && pPrev->eType!=FTSQUERY_PHRASE)
|
||||
)){
|
||||
/* This is an attempt to do "phrase NEAR (bracketed expression)"
|
||||
** or "(bracketed expression) NEAR phrase", both of which are
|
||||
** illegal. Return an error.
|
||||
*/
|
||||
sqlite3Fts3ExprFree(p);
|
||||
rc = SQLITE_ERROR;
|
||||
goto exprparse_out;
|
||||
}
|
||||
|
||||
if( isPhrase ){
|
||||
if( pRet ){
|
||||
assert( pPrev && pPrev->pLeft && pPrev->pRight==0 );
|
||||
pPrev->pRight = p;
|
||||
p->pParent = pPrev;
|
||||
}else{
|
||||
pRet = p;
|
||||
}
|
||||
}else{
|
||||
insertBinaryOperator(&pRet, pPrev, p);
|
||||
}
|
||||
isRequirePhrase = !isPhrase;
|
||||
}
|
||||
assert( nByte>0 );
|
||||
}
|
||||
nIn -= nByte;
|
||||
zIn += nByte;
|
||||
pPrev = p;
|
||||
}
|
||||
|
||||
if( rc==SQLITE_DONE && pRet && isRequirePhrase ){
|
||||
rc = SQLITE_ERROR;
|
||||
}
|
||||
|
||||
if( rc==SQLITE_DONE ){
|
||||
rc = SQLITE_OK;
|
||||
if( !sqlite3_fts3_enable_parentheses && pNotBranch ){
|
||||
if( !pRet ){
|
||||
rc = SQLITE_ERROR;
|
||||
}else{
|
||||
pNotBranch->pLeft = pRet;
|
||||
pRet = pNotBranch;
|
||||
}
|
||||
}
|
||||
}
|
||||
*pnConsumed = n - nIn;
|
||||
|
||||
exprparse_out:
|
||||
if( rc!=SQLITE_OK ){
|
||||
sqlite3Fts3ExprFree(pRet);
|
||||
sqlite3Fts3ExprFree(pNotBranch);
|
||||
pRet = 0;
|
||||
}
|
||||
*ppExpr = pRet;
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** Parameters z and n contain a pointer to and length of a buffer containing
|
||||
** an fts3 query expression, respectively. This function attempts to parse the
|
||||
** query expression and create a tree of Fts3Expr structures representing the
|
||||
** parsed expression. If successful, *ppExpr is set to point to the head
|
||||
** of the parsed expression tree and SQLITE_OK is returned. If an error
|
||||
** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse
|
||||
** error) is returned and *ppExpr is set to 0.
|
||||
**
|
||||
** If parameter n is a negative number, then z is assumed to point to a
|
||||
** nul-terminated string and the length is determined using strlen().
|
||||
**
|
||||
** The first parameter, pTokenizer, is passed the fts3 tokenizer module to
|
||||
** use to normalize query tokens while parsing the expression. The azCol[]
|
||||
** array, which is assumed to contain nCol entries, should contain the names
|
||||
** of each column in the target fts3 table, in order from left to right.
|
||||
** Column names must be nul-terminated strings.
|
||||
**
|
||||
** The iDefaultCol parameter should be passed the index of the table column
|
||||
** that appears on the left-hand-side of the MATCH operator (the default
|
||||
** column to match against for tokens for which a column name is not explicitly
|
||||
** specified as part of the query string), or -1 if tokens may by default
|
||||
** match any table column.
|
||||
*/
|
||||
int sqlite3Fts3ExprParse(
|
||||
sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
|
||||
char **azCol, /* Array of column names for fts3 table */
|
||||
int nCol, /* Number of entries in azCol[] */
|
||||
int iDefaultCol, /* Default column to query */
|
||||
const char *z, int n, /* Text of MATCH query */
|
||||
Fts3Expr **ppExpr /* OUT: Parsed query structure */
|
||||
){
|
||||
int nParsed;
|
||||
int rc;
|
||||
ParseContext sParse;
|
||||
sParse.pTokenizer = pTokenizer;
|
||||
sParse.azCol = (const char **)azCol;
|
||||
sParse.nCol = nCol;
|
||||
sParse.iDefaultCol = iDefaultCol;
|
||||
sParse.nNest = 0;
|
||||
if( z==0 ){
|
||||
*ppExpr = 0;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
if( n<0 ){
|
||||
n = strlen(z);
|
||||
}
|
||||
rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed);
|
||||
|
||||
/* Check for mismatched parenthesis */
|
||||
if( rc==SQLITE_OK && sParse.nNest ){
|
||||
rc = SQLITE_ERROR;
|
||||
sqlite3Fts3ExprFree(*ppExpr);
|
||||
*ppExpr = 0;
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
|
||||
*/
|
||||
void sqlite3Fts3ExprFree(Fts3Expr *p){
|
||||
if( p ){
|
||||
sqlite3Fts3ExprFree(p->pLeft);
|
||||
sqlite3Fts3ExprFree(p->pRight);
|
||||
sqlite3_free(p);
|
||||
}
|
||||
}
|
||||
|
||||
/****************************************************************************
|
||||
*****************************************************************************
|
||||
** Everything after this point is just test code.
|
||||
*/
|
||||
|
||||
#ifdef SQLITE_TEST
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
/*
|
||||
** Function to query the hash-table of tokenizers (see README.tokenizers).
|
||||
*/
|
||||
static int queryTokenizer(
|
||||
sqlite3 *db,
|
||||
const char *zName,
|
||||
const sqlite3_tokenizer_module **pp
|
||||
){
|
||||
int rc;
|
||||
sqlite3_stmt *pStmt;
|
||||
const char zSql[] = "SELECT fts3_tokenizer(?)";
|
||||
|
||||
*pp = 0;
|
||||
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
|
||||
if( rc!=SQLITE_OK ){
|
||||
return rc;
|
||||
}
|
||||
|
||||
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
|
||||
if( SQLITE_ROW==sqlite3_step(pStmt) ){
|
||||
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
|
||||
memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
|
||||
}
|
||||
}
|
||||
|
||||
return sqlite3_finalize(pStmt);
|
||||
}
|
||||
|
||||
/*
|
||||
** This function is part of the test interface for the query parser. It
|
||||
** writes a text representation of the query expression pExpr into the
|
||||
** buffer pointed to by argument zBuf. It is assumed that zBuf is large
|
||||
** enough to store the required text representation.
|
||||
*/
|
||||
static void exprToString(Fts3Expr *pExpr, char *zBuf){
|
||||
switch( pExpr->eType ){
|
||||
case FTSQUERY_PHRASE: {
|
||||
Fts3Phrase *pPhrase = pExpr->pPhrase;
|
||||
int i;
|
||||
zBuf += sprintf(zBuf, "PHRASE %d %d", pPhrase->iColumn, pPhrase->isNot);
|
||||
for(i=0; i<pPhrase->nToken; i++){
|
||||
zBuf += sprintf(zBuf," %.*s",pPhrase->aToken[i].n,pPhrase->aToken[i].z);
|
||||
zBuf += sprintf(zBuf,"%s", (pPhrase->aToken[i].isPrefix?"+":""));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
case FTSQUERY_NEAR:
|
||||
zBuf += sprintf(zBuf, "NEAR/%d ", pExpr->nNear);
|
||||
break;
|
||||
case FTSQUERY_NOT:
|
||||
zBuf += sprintf(zBuf, "NOT ");
|
||||
break;
|
||||
case FTSQUERY_AND:
|
||||
zBuf += sprintf(zBuf, "AND ");
|
||||
break;
|
||||
case FTSQUERY_OR:
|
||||
zBuf += sprintf(zBuf, "OR ");
|
||||
break;
|
||||
}
|
||||
|
||||
zBuf += sprintf(zBuf, "{");
|
||||
exprToString(pExpr->pLeft, zBuf);
|
||||
zBuf += strlen(zBuf);
|
||||
zBuf += sprintf(zBuf, "} ");
|
||||
|
||||
zBuf += sprintf(zBuf, "{");
|
||||
exprToString(pExpr->pRight, zBuf);
|
||||
zBuf += strlen(zBuf);
|
||||
zBuf += sprintf(zBuf, "}");
|
||||
}
|
||||
|
||||
/*
|
||||
** This is the implementation of a scalar SQL function used to test the
|
||||
** expression parser. It should be called as follows:
|
||||
**
|
||||
** fts3_exprtest(<tokenizer>, <expr>, <column 1>, ...);
|
||||
**
|
||||
** The first argument, <tokenizer>, is the name of the fts3 tokenizer used
|
||||
** to parse the query expression (see README.tokenizers). The second argument
|
||||
** is the query expression to parse. Each subsequent argument is the name
|
||||
** of a column of the fts3 table that the query expression may refer to.
|
||||
** For example:
|
||||
**
|
||||
** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
|
||||
*/
|
||||
static void fts3ExprTest(
|
||||
sqlite3_context *context,
|
||||
int argc,
|
||||
sqlite3_value **argv
|
||||
){
|
||||
sqlite3_tokenizer_module const *pModule = 0;
|
||||
sqlite3_tokenizer *pTokenizer;
|
||||
int rc;
|
||||
char **azCol = 0;
|
||||
const char *zExpr;
|
||||
int nExpr;
|
||||
int nCol;
|
||||
int ii;
|
||||
Fts3Expr *pExpr;
|
||||
sqlite3 *db = sqlite3_context_db_handle(context);
|
||||
|
||||
if( argc<3 ){
|
||||
sqlite3_result_error(context,
|
||||
"Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
rc = queryTokenizer(db, (const char *)sqlite3_value_text(argv[0]), &pModule);
|
||||
if( rc==SQLITE_NOMEM ){
|
||||
sqlite3_result_error_nomem(context);
|
||||
goto exprtest_out;
|
||||
}else if( !pModule ){
|
||||
sqlite3_result_error(context, "No such tokenizer module", -1);
|
||||
goto exprtest_out;
|
||||
}
|
||||
|
||||
rc = pModule->xCreate(0, 0, &pTokenizer);
|
||||
assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
|
||||
if( rc==SQLITE_NOMEM ){
|
||||
sqlite3_result_error_nomem(context);
|
||||
goto exprtest_out;
|
||||
}
|
||||
pTokenizer->pModule = pModule;
|
||||
|
||||
zExpr = (const char *)sqlite3_value_text(argv[1]);
|
||||
nExpr = sqlite3_value_bytes(argv[1]);
|
||||
nCol = argc-2;
|
||||
azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
|
||||
if( !azCol ){
|
||||
sqlite3_result_error_nomem(context);
|
||||
goto exprtest_out;
|
||||
}
|
||||
for(ii=0; ii<nCol; ii++){
|
||||
azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
|
||||
}
|
||||
|
||||
rc = sqlite3Fts3ExprParse(
|
||||
pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
|
||||
);
|
||||
if( rc==SQLITE_NOMEM ){
|
||||
sqlite3_result_error_nomem(context);
|
||||
goto exprtest_out;
|
||||
}else if( rc==SQLITE_OK ){
|
||||
char zBuf[4096];
|
||||
exprToString(pExpr, zBuf);
|
||||
sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
|
||||
sqlite3Fts3ExprFree(pExpr);
|
||||
}else{
|
||||
sqlite3_result_error(context, "Error parsing expression", -1);
|
||||
}
|
||||
|
||||
exprtest_out:
|
||||
if( pTokenizer ){
|
||||
rc = pModule->xDestroy(pTokenizer);
|
||||
}
|
||||
sqlite3_free(azCol);
|
||||
}
|
||||
|
||||
/*
|
||||
** Register the query expression parser test function fts3_exprtest()
|
||||
** with database connection db.
|
||||
*/
|
||||
void sqlite3Fts3ExprInitTestInterface(sqlite3* db){
|
||||
sqlite3_create_function(
|
||||
db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
|
||||
);
|
||||
}
|
||||
|
||||
#endif
|
||||
87
ext/fts3/fts3_expr.h
Normal file
87
ext/fts3/fts3_expr.h
Normal file
@ -0,0 +1,87 @@
|
||||
/*
|
||||
** 2008 Nov 28
|
||||
**
|
||||
** The author disclaims copyright to this source code. In place of
|
||||
** a legal notice, here is a blessing:
|
||||
**
|
||||
** May you do good and not evil.
|
||||
** May you find forgiveness for yourself and forgive others.
|
||||
** May you share freely, never taking more than you give.
|
||||
**
|
||||
******************************************************************************
|
||||
**
|
||||
*/
|
||||
|
||||
#include "fts3_tokenizer.h"
|
||||
#include "sqlite3.h"
|
||||
|
||||
/*
|
||||
** The following describes the syntax supported by the fts3 MATCH
|
||||
** operator in a similar format to that used by the lemon parser
|
||||
** generator. This module does not use actually lemon, it uses a
|
||||
** custom parser.
|
||||
**
|
||||
** phrase ::= TOKEN.
|
||||
** phrase ::= TOKEN:COLUMN.
|
||||
** phrase ::= "TOKEN TOKEN TOKEN...".
|
||||
** phrase ::= phrase near phrase.
|
||||
**
|
||||
** near ::= NEAR.
|
||||
** near ::= NEAR / INTEGER.
|
||||
**
|
||||
** query ::= -TOKEN.
|
||||
** query ::= phrase.
|
||||
** query ::= LP query RP.
|
||||
** query ::= query NOT query.
|
||||
** query ::= query OR query.
|
||||
** query ::= query AND query.
|
||||
*/
|
||||
|
||||
typedef struct Fts3Expr Fts3Expr;
|
||||
typedef struct Fts3Phrase Fts3Phrase;
|
||||
|
||||
struct Fts3Phrase {
|
||||
int nToken; /* Number of entries in aToken[] */
|
||||
int iColumn; /* Index of column this phrase must match */
|
||||
int isNot; /* Phrase prefixed by unary not (-) operator */
|
||||
struct PhraseToken {
|
||||
char *z;
|
||||
int n; /* Number of bytes in buffer pointed to by z */
|
||||
int isPrefix; /* True if token ends in with a "*" character */
|
||||
} aToken[1];
|
||||
};
|
||||
|
||||
struct Fts3Expr {
|
||||
int eType; /* One of the FTSQUERY_XXX values defined below */
|
||||
int nNear; /* Valid if eType==FTSQUERY_NEAR */
|
||||
Fts3Expr *pParent;
|
||||
Fts3Expr *pLeft;
|
||||
Fts3Expr *pRight;
|
||||
Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
|
||||
};
|
||||
|
||||
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int,
|
||||
const char *, int, Fts3Expr **);
|
||||
void sqlite3Fts3ExprFree(Fts3Expr *);
|
||||
|
||||
/*
|
||||
** Candidate values for Fts3Query.eType. Note that the order of the first
|
||||
** four values is in order of precedence when parsing expressions. For
|
||||
** example, the following:
|
||||
**
|
||||
** "a OR b AND c NOT d NEAR e"
|
||||
**
|
||||
** is equivalent to:
|
||||
**
|
||||
** "a OR (b AND (c NOT (d NEAR e)))"
|
||||
*/
|
||||
#define FTSQUERY_NEAR 1
|
||||
#define FTSQUERY_NOT 2
|
||||
#define FTSQUERY_AND 3
|
||||
#define FTSQUERY_OR 4
|
||||
#define FTSQUERY_PHRASE 5
|
||||
|
||||
#ifdef SQLITE_TEST
|
||||
void sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
|
||||
#endif
|
||||
|
||||
11
main.mk
11
main.mk
@ -52,7 +52,7 @@ TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3
|
||||
LIBOBJ+= alter.o analyze.o attach.o auth.o \
|
||||
bitvec.o btmutex.o btree.o build.o \
|
||||
callback.o complete.o date.o delete.o expr.o fault.o \
|
||||
fts3.o fts3_hash.o fts3_icu.o fts3_porter.o \
|
||||
fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
|
||||
fts3_tokenizer.o fts3_tokenizer1.o \
|
||||
func.o global.o hash.o \
|
||||
icu.o insert.o journal.o legacy.o loadext.o \
|
||||
@ -178,6 +178,8 @@ SRC += \
|
||||
SRC += \
|
||||
$(TOP)/ext/fts3/fts3.c \
|
||||
$(TOP)/ext/fts3/fts3.h \
|
||||
$(TOP)/ext/fts3/fts3_expr.c \
|
||||
$(TOP)/ext/fts3/fts3_expr.h \
|
||||
$(TOP)/ext/fts3/fts3_hash.c \
|
||||
$(TOP)/ext/fts3/fts3_hash.h \
|
||||
$(TOP)/ext/fts3/fts3_icu.c \
|
||||
@ -247,7 +249,8 @@ TESTSRC2 = \
|
||||
$(TOP)/src/pcache1.c $(TOP)/src/select.c $(TOP)/src/tokenize.c \
|
||||
$(TOP)/src/utf.c $(TOP)/src/util.c $(TOP)/src/vdbeapi.c $(TOP)/src/vdbeaux.c \
|
||||
$(TOP)/src/vdbe.c $(TOP)/src/vdbemem.c $(TOP)/src/where.c parse.c \
|
||||
$(TOP)/ext/fts3/fts3.c $(TOP)/ext/fts3/fts3_tokenizer.c
|
||||
$(TOP)/ext/fts3/fts3.c $(TOP)/ext/fts3/fts3_expr.c \
|
||||
$(TOP)/ext/fts3/fts3_tokenizer.c
|
||||
|
||||
# Header files used by all library source files.
|
||||
#
|
||||
@ -283,6 +286,7 @@ EXTHDR += \
|
||||
$(TOP)/ext/fts2/fts2_tokenizer.h
|
||||
EXTHDR += \
|
||||
$(TOP)/ext/fts3/fts3.h \
|
||||
$(TOP)/ext/fts3/fts3_expr.h \
|
||||
$(TOP)/ext/fts3/fts3_hash.h \
|
||||
$(TOP)/ext/fts3/fts3_tokenizer.h
|
||||
EXTHDR += \
|
||||
@ -412,6 +416,9 @@ fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR)
|
||||
fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR)
|
||||
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c
|
||||
|
||||
fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR)
|
||||
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c
|
||||
|
||||
fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR)
|
||||
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c
|
||||
|
||||
|
||||
21
manifest
21
manifest
@ -1,5 +1,5 @@
|
||||
C Version\s3.6.7\s(CVS\s6033)
|
||||
D 2008-12-16T18:15:59
|
||||
C Modify\sfts3\sto\ssupport\sa\smore\scomplex\sexpression\ssyntax\sthat\sallows\sparenthesis.\sThe\snew\ssyntax\sis\snot\sentirely\sbackwards\scompatible,\sso\sis\sdisabled\sby\sdefault.\sUse\s-DSQLITE_ENABLE_FTS3_PARENTHESIS\sto\senable\sit.\s(CVS\s6034)
|
||||
D 2008-12-17T15:18:18
|
||||
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
|
||||
F Makefile.in f7e4c81c347b04f7b0f1c1b081a168645d7b8af7
|
||||
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
|
||||
@ -52,8 +52,10 @@ F ext/fts2/fts2_tokenizer1.c 8545ce12b41922004da46e91a7b023b92b76f94e
|
||||
F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
|
||||
F ext/fts3/README.tokenizers 226644a0eab97724e8de83061912e8bb248461b6
|
||||
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
|
||||
F ext/fts3/fts3.c e67453b6ac421b79e600385491ed7f038b3bb271
|
||||
F ext/fts3/fts3.c 3aa6aef1eadc44606f6ed3c841062735a5210077
|
||||
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
|
||||
F ext/fts3/fts3_expr.c b141145197cc749accb03d2b970813443b723edd
|
||||
F ext/fts3/fts3_expr.h 4dad4d87cf5d41ea924a815fe89a6f87dc76f277
|
||||
F ext/fts3/fts3_hash.c e15e84d18f8df149ab290029872d4559c4c7c15a
|
||||
F ext/fts3/fts3_hash.h 004b759e1602ff16dfa02fea3ca1c77336ad6798
|
||||
F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
|
||||
@ -80,7 +82,7 @@ F ext/rtree/tkt3363.test 6662237ea75bb431cd5d262dfc9535e1023315fc
|
||||
F ext/rtree/viewrtree.tcl 09526398dae87a5a87c5aac2b3854dbaf8376869
|
||||
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
|
||||
F ltmain.sh 09fe5815427dc7d0abb188bbcdf0e34896577210
|
||||
F main.mk 5923e75b5ac4b265f322597c3953dda7175f4405
|
||||
F main.mk f6eb58a66f942bf672ab58e74e30e72cad39b93f
|
||||
F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a
|
||||
F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
|
||||
F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac
|
||||
@ -160,7 +162,7 @@ F src/sqliteLimit.h f435e728c6b620ef7312814d660a81f9356eb5c8
|
||||
F src/status.c 237b193efae0cf6ac3f0817a208de6c6c6ef6d76
|
||||
F src/table.c 23db1e5f27c03160987c122a078b4bb51ef0b2f8
|
||||
F src/tclsqlite.c 23afb60549af943e135ded441a631f4745be6040
|
||||
F src/test1.c 9c0502c3627162f969099e57932782057d9139b6
|
||||
F src/test1.c b193b8b80617bdb8297b25a87d00ee8d5a125d0d
|
||||
F src/test2.c 897528183edf2839c2a3c991d415905db56f1240
|
||||
F src/test3.c 88a246b56b824275300e6c899634fbac1dc94b14
|
||||
F src/test4.c f79ab52d27ff49b784b631a42e2ccd52cfd5c84c
|
||||
@ -364,6 +366,7 @@ F test/fts3b.test b3a25180a633873d37d86e1ccd00ed690d37237a
|
||||
F test/fts3c.test 4c7ef29b37aca3e8ebb6a39b57910caa6506034e
|
||||
F test/fts3d.test d92a47fe8ed59c9e53d2d8e6d2685bb380aadadc
|
||||
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
|
||||
F test/fts3expr.test 18143e61503845b940fd7caacce53bce4307426f
|
||||
F test/fts3near.test e8a9b4e16c63a795918b334b74d4aec14815bf8b
|
||||
F test/func.test a50f0a4b69ac251debe1dce3ba29da7476dc8c52
|
||||
F test/fuzz.test 62fc19dd36a427777fd671b569df07166548628a
|
||||
@ -675,7 +678,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
|
||||
F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
|
||||
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
|
||||
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
|
||||
P 8b8f6a6ab597e06e60557ab56c6ee7f8522ed570
|
||||
R a6ff965a91307b50874e830368f8fcd6
|
||||
U drh
|
||||
Z 76064440713dce47dc110c16fc71382b
|
||||
P f4f40370fb83d677df3fbf2c51c4bb4a3e5ccc7a
|
||||
R 0170cc023fb1827148ff15e7bc02335a
|
||||
U danielk1977
|
||||
Z 3401f60bb1566cbc5da4c344a91c4fb9
|
||||
|
||||
@ -1 +1 @@
|
||||
f4f40370fb83d677df3fbf2c51c4bb4a3e5ccc7a
|
||||
7389b9ecb80294569845c40a23e0c832d07f7a45
|
||||
@ -13,7 +13,7 @@
|
||||
** is not included in the SQLite library. It is used for automated
|
||||
** testing of the SQLite library.
|
||||
**
|
||||
** $Id: test1.c,v 1.337 2008/12/11 02:56:07 drh Exp $
|
||||
** $Id: test1.c,v 1.338 2008/12/17 15:18:18 danielk1977 Exp $
|
||||
*/
|
||||
#include "sqliteInt.h"
|
||||
#include "tcl.h"
|
||||
@ -4930,6 +4930,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
|
||||
extern int sqlite3_enable_in_opt;
|
||||
extern char sqlite3_query_plan[];
|
||||
static char *query_plan = sqlite3_query_plan;
|
||||
#ifdef SQLITE_ENABLE_FTS3
|
||||
extern int sqlite3_fts3_enable_parentheses;
|
||||
#endif
|
||||
#endif
|
||||
|
||||
for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){
|
||||
@ -5012,6 +5015,10 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
|
||||
#ifdef SQLITE_TEST
|
||||
Tcl_LinkVar(interp, "sqlite_enable_in_opt",
|
||||
(char*)&sqlite3_enable_in_opt, TCL_LINK_INT);
|
||||
#ifdef SQLITE_ENABLE_FTS3
|
||||
Tcl_LinkVar(interp, "sqlite_fts3_enable_parentheses",
|
||||
(char*)&sqlite3_fts3_enable_parentheses, TCL_LINK_INT);
|
||||
#endif
|
||||
#endif
|
||||
return TCL_OK;
|
||||
}
|
||||
|
||||
258
test/fts3expr.test
Normal file
258
test/fts3expr.test
Normal file
@ -0,0 +1,258 @@
|
||||
# 2006 September 9
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#*************************************************************************
|
||||
# This file implements regression tests for SQLite library. The
|
||||
# focus of this script is testing the FTS3 module.
|
||||
#
|
||||
# $Id: fts3expr.test,v 1.1 2008/12/17 15:18:18 danielk1977 Exp $
|
||||
#
|
||||
|
||||
set testdir [file dirname $argv0]
|
||||
source $testdir/tester.tcl
|
||||
|
||||
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
|
||||
ifcapable !fts3 {
|
||||
finish_test
|
||||
return
|
||||
}
|
||||
|
||||
set sqlite_fts3_enable_parentheses 1
|
||||
|
||||
proc test_fts3expr {expr} {
|
||||
db one {SELECT fts3_exprtest('simple', $expr, 'a', 'b', 'c')}
|
||||
}
|
||||
do_test fts3expr-1.0 {
|
||||
test_fts3expr "abcd"
|
||||
} {PHRASE 3 0 abcd}
|
||||
do_test fts3expr-1.1 {
|
||||
test_fts3expr " tag "
|
||||
} {PHRASE 3 0 tag}
|
||||
|
||||
do_test fts3expr-1.2 {
|
||||
test_fts3expr "ab AND cd"
|
||||
} {AND {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
|
||||
do_test fts3expr-1.3 {
|
||||
test_fts3expr "ab OR cd"
|
||||
} {OR {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
|
||||
do_test fts3expr-1.4 {
|
||||
test_fts3expr "ab NOT cd"
|
||||
} {NOT {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
|
||||
do_test fts3expr-1.5 {
|
||||
test_fts3expr "ab NEAR cd"
|
||||
} {NEAR/10 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
|
||||
do_test fts3expr-1.6 {
|
||||
test_fts3expr "ab NEAR/5 cd"
|
||||
} {NEAR/5 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
|
||||
|
||||
do_test fts3expr-1.7 {
|
||||
test_fts3expr {"one two three"}
|
||||
} {PHRASE 3 0 one two three}
|
||||
do_test fts3expr-1.8 {
|
||||
test_fts3expr {zero "one two three" four}
|
||||
} {AND {AND {PHRASE 3 0 zero} {PHRASE 3 0 one two three}} {PHRASE 3 0 four}}
|
||||
do_test fts3expr-1.9 {
|
||||
test_fts3expr {"one* two three*"}
|
||||
} {PHRASE 3 0 one+ two three+}
|
||||
|
||||
do_test fts3expr-1.10 {
|
||||
test_fts3expr {one* two}
|
||||
} {AND {PHRASE 3 0 one+} {PHRASE 3 0 two}}
|
||||
do_test fts3expr-1.11 {
|
||||
test_fts3expr {one two*}
|
||||
} {AND {PHRASE 3 0 one} {PHRASE 3 0 two+}}
|
||||
|
||||
do_test fts3expr-1.14 {
|
||||
test_fts3expr {a:one two}
|
||||
} {AND {PHRASE 0 0 one} {PHRASE 3 0 two}}
|
||||
do_test fts3expr-1.15 {
|
||||
test_fts3expr {one b:two}
|
||||
} {AND {PHRASE 3 0 one} {PHRASE 1 0 two}}
|
||||
|
||||
proc strip_phrase_data {L} {
|
||||
if {[lindex $L 0] eq "PHRASE"} {
|
||||
return [lrange $L 3 end]
|
||||
}
|
||||
return [list \
|
||||
[lindex $L 0] \
|
||||
[strip_phrase_data [lindex $L 1]] \
|
||||
[strip_phrase_data [lindex $L 2]] \
|
||||
]
|
||||
}
|
||||
proc test_fts3expr2 {expr} {
|
||||
strip_phrase_data [
|
||||
db one {SELECT fts3_exprtest('simple', $expr, 'a', 'b', 'c')}
|
||||
]
|
||||
}
|
||||
do_test fts3expr-2.1 {
|
||||
test_fts3expr2 "ab OR cd AND ef"
|
||||
} {OR ab {AND cd ef}}
|
||||
do_test fts3expr-2.2 {
|
||||
test_fts3expr2 "cd AND ef OR ab"
|
||||
} {OR {AND cd ef} ab}
|
||||
do_test fts3expr-2.3 {
|
||||
test_fts3expr2 "ab AND cd AND ef OR gh"
|
||||
} {OR {AND {AND ab cd} ef} gh}
|
||||
do_test fts3expr-2.4 {
|
||||
test_fts3expr2 "ab AND cd OR ef AND gh"
|
||||
} {OR {AND ab cd} {AND ef gh}}
|
||||
do_test fts3expr-2.5 {
|
||||
test_fts3expr2 "ab cd"
|
||||
} {AND ab cd}
|
||||
|
||||
do_test fts3expr-3.1 {
|
||||
test_fts3expr2 "(ab OR cd) AND ef"
|
||||
} {AND {OR ab cd} ef}
|
||||
do_test fts3expr-3.2 {
|
||||
test_fts3expr2 "ef AND (ab OR cd)"
|
||||
} {AND ef {OR ab cd}}
|
||||
do_test fts3expr-3.3 {
|
||||
test_fts3expr2 "(ab OR cd)"
|
||||
} {OR ab cd}
|
||||
do_test fts3expr-3.4 {
|
||||
test_fts3expr2 "(((ab OR cd)))"
|
||||
} {OR ab cd}
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
# The following tests, fts3expr-4.*, test the parsers response to syntax
|
||||
# errors in query expressions. This is done using a real fts3 table and
|
||||
# MATCH clauses, not the parser test interface.
|
||||
#
|
||||
do_test fts3expr-4.1 {
|
||||
execsql { CREATE VIRTUAL TABLE t1 USING fts3(a, b, c) }
|
||||
} {}
|
||||
|
||||
# Mismatched parenthesis:
|
||||
do_test fts3expr-4.2.1 {
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example AND (hello OR world))' }
|
||||
} {1 {SQL logic error or missing database}}
|
||||
do_test fts3expr-4.2.2 {
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example AND (hello OR world' }
|
||||
} {1 {SQL logic error or missing database}}
|
||||
|
||||
# Unterminated quotation marks:
|
||||
do_test fts3expr-4.3.1 {
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example OR "hello world' }
|
||||
} {1 {SQL logic error or missing database}}
|
||||
do_test fts3expr-4.3.2 {
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example OR hello world"' }
|
||||
} {1 {SQL logic error or missing database}}
|
||||
|
||||
# Binary operators without the required operands.
|
||||
do_test fts3expr-4.4.1 {
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'OR hello world' }
|
||||
} {1 {SQL logic error or missing database}}
|
||||
do_test fts3expr-4.4.2 {
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'hello world OR' }
|
||||
} {1 {SQL logic error or missing database}}
|
||||
do_test fts3expr-4.4.3 {
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'one (hello world OR) two' }
|
||||
} {1 {SQL logic error or missing database}}
|
||||
do_test fts3expr-4.4.4 {
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'one (OR hello world) two' }
|
||||
} {1 {SQL logic error or missing database}}
|
||||
|
||||
# NEAR operators with something other than phrases as arguments.
|
||||
do_test fts3expr-4.5.1 {
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH '(hello OR world) NEAR one' }
|
||||
} {1 {SQL logic error or missing database}}
|
||||
do_test fts3expr-4.5.2 {
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'one NEAR (hello OR world)' }
|
||||
} {1 {SQL logic error or missing database}}
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
# The following OOM tests are designed to cover cases in fts3_expr.c.
|
||||
#
|
||||
source $testdir/malloc_common.tcl
|
||||
do_malloc_test fts3expr-malloc-1 -sqlbody {
|
||||
SELECT fts3_exprtest('simple', 'a b c "d e f"', 'a', 'b', 'c')
|
||||
}
|
||||
do_malloc_test fts3expr-malloc-2 -tclprep {
|
||||
set sqlite_fts3_enable_parentheses 0
|
||||
} -sqlbody {
|
||||
SELECT fts3_exprtest('simple', 'a -b', 'a', 'b', 'c')
|
||||
} -cleanup {
|
||||
set sqlite_fts3_enable_parentheses 1
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
# The following tests are not very important. They cover error handling
|
||||
# cases in the test code, which makes test coverage easier to measure.
|
||||
#
|
||||
do_test fts3expr-5.1 {
|
||||
catchsql { SELECT fts3_exprtest('simple', 'a b') }
|
||||
} {1 {Usage: fts3_exprtest(tokenizer, expr, col1, ...}}
|
||||
do_test fts3expr-5.2 {
|
||||
catchsql { SELECT fts3_exprtest('doesnotexist', 'a b', 'c') }
|
||||
} {1 {No such tokenizer module}}
|
||||
do_test fts3expr-5.3 {
|
||||
catchsql { SELECT fts3_exprtest('simple', 'a b OR', 'c') }
|
||||
} {1 {Error parsing expression}}
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
# The next set of tests verifies that things actually work as they are
|
||||
# supposed to when using the new syntax.
|
||||
#
|
||||
do_test fts3expr-6.1 {
|
||||
execsql {
|
||||
CREATE VIRTUAL TABLE t1 USING fts3(a);
|
||||
}
|
||||
for {set ii 1} {$ii < 32} {incr ii} {
|
||||
set v [list]
|
||||
if {$ii & 1} { lappend v one }
|
||||
if {$ii & 2} { lappend v two }
|
||||
if {$ii & 4} { lappend v three }
|
||||
if {$ii & 8} { lappend v four }
|
||||
if {$ii & 16} { lappend v five }
|
||||
execsql { INSERT INTO t1 VALUES($v) }
|
||||
}
|
||||
|
||||
execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'five four one' ORDER BY rowid}
|
||||
} {25 27 29 31}
|
||||
|
||||
foreach {id expr res} {
|
||||
|
||||
2 "five four NOT one" {24 26 28 30}
|
||||
|
||||
3 "five AND four OR one"
|
||||
{1 3 5 7 9 11 13 15 17 19 21 23 24 25 26 27 28 29 30 31}
|
||||
|
||||
4 "five AND (four OR one)" {17 19 21 23 24 25 26 27 28 29 30 31}
|
||||
|
||||
5 "five NOT (four OR one)" {16 18 20 22}
|
||||
|
||||
6 "(five NOT (four OR one)) OR (five AND (four OR one))"
|
||||
{16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31}
|
||||
|
||||
7 "(five OR one) AND two AND three" {7 15 22 23 30 31}
|
||||
|
||||
8 "five OR one AND two AND three"
|
||||
{7 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31}
|
||||
|
||||
9 "five OR one two three"
|
||||
{7 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31}
|
||||
|
||||
10 "five OR \"one two three\""
|
||||
{7 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31}
|
||||
|
||||
11 "one two OR four five NOT three" {3 7 11 15 19 23 24 25 26 27 31}
|
||||
|
||||
12 "(one two OR four five) NOT three" {3 11 19 24 25 26 27}
|
||||
|
||||
13 "((((((one two OR four five)))))) NOT three" {3 11 19 24 25 26 27}
|
||||
|
||||
} {
|
||||
do_test fts3expr-6.$id {
|
||||
execsql { SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid }
|
||||
} $res
|
||||
}
|
||||
|
||||
set sqlite_fts3_enable_parentheses 0
|
||||
finish_test
|
||||
|
||||
Reference in New Issue
Block a user