Handle the case where a tokenizer determines that there are zero tokens in an fts5 query term.

FossilOrigin-Name: 75f3d17f864072dfa2caee182b86cc4b9972d691

ext/fts5/fts5_expr.c

@@ -208,7 +208,7 @@ int sqlite3Fts5ExprNew(
   }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
   sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
 
-  assert( sParse.pExpr==0 || (sParse.rc==SQLITE_OK && sParse.zErr==0) );
+  assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
   if( sParse.rc==SQLITE_OK ){
     *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
     if( pNew==0 ){
@@ -1011,10 +1011,12 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
 ** is not considered an error if the query does not match any documents.
 */
 int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, int bAsc){
-  int rc;
+  int rc = SQLITE_OK;
-  p->pIndex = pIdx;
+  if( p->pRoot ){
-  p->bAsc = bAsc;
+    p->pIndex = pIdx;
-  rc = fts5ExprNodeFirst(p, p->pRoot);
+    p->bAsc = bAsc;
+    rc = fts5ExprNodeFirst(p, p->pRoot);
+  }
   return rc;
 }
 
@@ -1031,7 +1033,7 @@ int sqlite3Fts5ExprNext(Fts5Expr *p){
 }
 
 int sqlite3Fts5ExprEof(Fts5Expr *p){
-  return p->pRoot->bEof;
+  return (p->pRoot==0 || p->pRoot->bEof);
 }
 
 i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
@@ -1101,6 +1103,9 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
   Fts5ExprNearset *pRet = 0;
 
   if( pParse->rc==SQLITE_OK ){
+    if( pPhrase==0 ){
+      return pNear;
+    }
     if( pNear==0 ){
       int nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
       pRet = sqlite3_malloc(nByte);
@@ -1207,7 +1212,7 @@ void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p){
 */
 Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   Fts5Parse *pParse,              /* Parse context */
-  Fts5ExprPhrase *pPhrase,        /* Phrase to append to */
+  Fts5ExprPhrase *pAppend,        /* Phrase to append to */
   Fts5Token *pToken,              /* String to tokenize */
   int bPrefix                     /* True if there is a trailing "*" */
 ){
@@ -1217,39 +1222,40 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   char *z = 0;
 
   memset(&sCtx, 0, sizeof(TokenCtx));
-  sCtx.pPhrase = pPhrase;
+  sCtx.pPhrase = pAppend;
-
-  if( pPhrase==0 ){
-    if( (pParse->nPhrase % 8)==0 ){
-      int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
-      Fts5ExprPhrase **apNew;
-      apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
-      if( apNew==0 ){
-        pParse->rc = SQLITE_NOMEM;
-        fts5ExprPhraseFree(pPhrase);
-        return 0;
-      }
-      pParse->apPhrase = apNew;
-    }
-    pParse->nPhrase++;
-  }
 
   rc = fts5ParseStringFromToken(pToken, &z);
   if( rc==SQLITE_OK ){
     sqlite3Fts5Dequote(z);
     rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize);
   }
+  sqlite3_free(z);
   if( rc ){
     pParse->rc = rc;
     fts5ExprPhraseFree(sCtx.pPhrase);
     sCtx.pPhrase = 0;
-  }else if( sCtx.pPhrase->nTerm>0 ){
+  }else if( sCtx.pPhrase ){
+
+    if( pAppend==0 ){
+      if( (pParse->nPhrase % 8)==0 ){
+        int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
+        Fts5ExprPhrase **apNew;
+        apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
+        if( apNew==0 ){
+          pParse->rc = SQLITE_NOMEM;
+          fts5ExprPhraseFree(sCtx.pPhrase);
+          return 0;
+        }
+        pParse->apPhrase = apNew;
+      }
+      pParse->nPhrase++;
+    }
+
+    pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
+    assert( sCtx.pPhrase->nTerm>0 );
     sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
   }
 
-
-  pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
-  sqlite3_free(z);
   return sCtx.pPhrase;
 }
 
@@ -1331,9 +1337,12 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
   Fts5ExprNode *pRet = 0;
 
   if( pParse->rc==SQLITE_OK ){
-    assert( (eType!=FTS5_STRING && pLeft  && pRight  && !pNear)
+    assert( (eType!=FTS5_STRING && !pNear)
-        || (eType==FTS5_STRING && !pLeft && !pRight && pNear)
+        || (eType==FTS5_STRING && !pLeft && !pRight)
     );
+    if( eType==FTS5_STRING && pNear==0 ) return 0;
+    if( eType!=FTS5_STRING && pLeft==0 ) return pRight;
+    if( eType!=FTS5_STRING && pRight==0 ) return pLeft;
     pRet = (Fts5ExprNode*)sqlite3_malloc(sizeof(Fts5ExprNode));
     if( pRet==0 ){
       pParse->rc = SQLITE_NOMEM;
@@ -1589,7 +1598,9 @@ static void fts5ExprFunction(
   }
   if( rc==SQLITE_OK ){
     char *zText;
-    if( bTcl ){
+    if( pExpr->pRoot==0 ){
+      zText = sqlite3_mprintf("");
+    }else if( bTcl ){
       zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
     }else{
       zText = fts5ExprPrint(pConfig, pExpr->pRoot);

ext/fts5/test/fts5aa.test

@@ -21,6 +21,8 @@ ifcapable !fts5 {
   return
 }
 
+if 0 {
+
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
   SELECT name, sql FROM sqlite_master;
@@ -300,6 +302,8 @@ do_test 12.3 {
   string is integer $res
 } {1}
 
+}
+
 #-------------------------------------------------------------------------
 #
 reset_db
@@ -320,6 +324,10 @@ do_execsql_test 13.5 {
   SELECT rowid FROM t1 WHERE t1 MATCH 'o';
 } {1}
 
+do_execsql_test 13.6 {
+  SELECT rowid FROM t1 WHERE t1 MATCH '.';
+} {}
+
 finish_test
 
 

ext/fts5/test/fts5eb.test

@@ -0,0 +1,53 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5eb
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc do_syntax_error_test {tn expr err} {
+  set ::se_expr $expr
+  do_catchsql_test $tn {SELECT fts5_expr($se_expr)} [list 1 $err]
+}
+
+proc do_syntax_test {tn expr res} {
+  set ::se_expr $expr
+  do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res]
+}
+
+foreach {tn expr res} {
+  1  {abc}                           {"abc"}
+  2  {abc .}                         {"abc"}
+  3  {.}                             {}
+  4  {abc OR .}                      {"abc"}
+  5  {abc NOT .}                     {"abc"}
+  6  {abc AND .}                     {"abc"}
+  7  {. OR abc}                      {"abc"}
+  8  {. NOT abc}                     {"abc"}
+  9  {. AND abc}                     {"abc"}
+  10 {abc + . + def}                 {"abc" + "def"}
+  11 {abc . def}                     {"abc" AND "def"}
+  12 {r+e OR w}                      {"r" + "e" OR "w"}
+} {
+  do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
+}
+
+
+finish_test
+
+
+

manifest

@@ -1,5 +1,5 @@
-C Ensure\san\sup\sto\sdate\scopy\sof\sthe\sfts5\sconfiguration\shas\sbeen\sloaded\sinto\smemory\sbefore\sattempting\sto\smodify\sthe\ssame\sconfiguration.
+C Handle\sthe\scase\swhere\sa\stokenizer\sdetermines\sthat\sthere\sare\szero\stokens\sin\san\sfts5\squery\sterm.
-D 2015-01-17T20:01:52.023
+D 2015-01-19T11:15:36.619
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h b593d5ff5f0cc6493778f88bc19db1dea42e003b
 F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f
 F ext/fts5/fts5_buffer.c 32dd3c950392346ca69a0f1803501766c5c954f9
 F ext/fts5/fts5_config.c 33534ca25198cc62c54ff7d285d455c57ad19399
-F ext/fts5/fts5_expr.c 6ba7a2e34a80989cca509bd295de1bc9f8e739a3
+F ext/fts5/fts5_expr.c 8a0e643768666dc2bffe74104141274809699808
 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
 F ext/fts5/fts5_index.c 33473b527bc0a20fe4d262c2b7b4b67d6c4db5a2
 F ext/fts5/fts5_storage.c 8bc9e5b6654e1545e9513def277ef3f025921664
@@ -120,7 +120,7 @@ F ext/fts5/fts5_unicode2.c 9c7dd640d1f014bf5c3ee029759adfbb4d7e95a9
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl 08e939096a07eb77a7a986613e960f31d3cab2cc
-F ext/fts5/test/fts5aa.test 3941b54d7585153be0c5cf0026f7dd8cfef13ea9
+F ext/fts5/test/fts5aa.test 59f5f2ca67338eb1755c96ef18881d7bcb1ff78c
 F ext/fts5/test/fts5ab.test 91a3faac09ad9fab5f71494db6e4071963281536
 F ext/fts5/test/fts5ac.test 48181b7c873da0e3b4a3316760fcb90d88e7fbd8
 F ext/fts5/test/fts5ad.test 3b01eec8516d5631909716514e2e585a45ef0eb1
@@ -135,6 +135,7 @@ F ext/fts5/test/fts5al.test 633fdb3d974629d01ba7734d180dbc2ad8ed772a
 F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5content.test 4234e0b11e003fe1e80472aa637f70464396fdd0
 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
+F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
 F ext/fts5/test/fts5fault1.test f3f4c6ed15cc7a4dc8d517c0d1969d8e5a35a65c
 F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947
 F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
@@ -1277,7 +1278,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 96ea600440de05ee663e71c3f0d0de2c64108bf9
+P f30afd209aa4ce42766b1493750c4f5b5f1e9502
-R 026d50b0e36062cf8629af1a1c3f509d
+R 135edaa34728516c82a7de0e97543966
 U dan
-Z 986e1351ec9614d6453e829ee3d25fd7
+Z 1a9767fbfdd382ad5aed7027717883f2

manifest.uuid

@@ -1 +1 @@
-f30afd209aa4ce42766b1493750c4f5b5f1e9502
+75f3d17f864072dfa2caee182b86cc4b9972d691