Add simple full-table-scan and rowid lookup support to fts5.

FossilOrigin-Name: 3515da85d09220c464979467b476c611da4a6a7a

ext/fts5/fts5.c

@@ -16,6 +16,7 @@
 #include "fts5Int.h"
 
 typedef struct Fts5Table Fts5Table;
+typedef struct Fts5Cursor Fts5Cursor;
 
 struct Fts5Table {
   sqlite3_vtab base;              /* Base class used by SQLite core */
@@ -24,6 +25,13 @@ struct Fts5Table {
   Fts5Storage *pStorage;          /* Document store */
 };
 
+struct Fts5Cursor {
+  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
+  int idxNum;                     /* idxNum passed to xFilter() */
+  sqlite3_stmt *pStmt;            /* Statement used to read %_content */
+  int bEof;                       /* True at EOF */
+};
+
 /*
 ** Close a virtual table handle opened by fts5InitVtab(). If the bDestroy
 ** argument is non-zero, attempt delete the shadow tables from teh database
@@ -146,14 +154,68 @@ static int fts5CreateMethod(
 }
 
 /*
-** Implementation of the xBestIndex method for FTS3 tables. There
+** The three query plans xBestIndex may choose between.
+*/
+#define FTS5_PLAN_SCAN    1       /* No usable constraint */
+#define FTS5_PLAN_MATCH   2       /* (<tbl> MATCH ?) */
+#define FTS5_PLAN_ROWID   3       /* (rowid = ?) */
+
+#define FTS5_PLAN(idxNum) ((idxNum) & 0x7)
+
+#define FTS5_ORDER_DESC   8       /* ORDER BY rowid DESC */
+#define FTS5_ORDER_ASC   16       /* ORDER BY rowid ASC */
+
+
+static int fts5FindConstraint(sqlite3_index_info *pInfo, int eOp, int iCol){
+  int i;
+
+  for(i=0; i<pInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
+    if( p->usable && p->iColumn==iCol && p->op==eOp ) return i;
+  }
+
+  return -1;
+}
+
+/* 
+** Implementation of the xBestIndex method for FTS5 tables. There
 ** are three possible strategies, in order of preference:
 **
-**   1. Direct lookup by rowid or docid. 
+**   1. Full-text search using a MATCH operator.
-**   2. Full-text search using a MATCH operator on a non-docid column.
+**   2. A by-rowid lookup.
-**   3. Linear scan of %_content table.
+**   3. A full-table scan.
 */
 static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
+  Fts5Table *pTab = (Fts5Table*)pVTab;
+  Fts5Config *pConfig = pTab->pConfig;
+  int iCons;
+  int ePlan = FTS5_PLAN_SCAN;
+
+  iCons = fts5FindConstraint(pInfo,SQLITE_INDEX_CONSTRAINT_MATCH,pConfig->nCol);
+  if( iCons>=0 ){
+    ePlan = FTS5_PLAN_MATCH;
+    pInfo->estimatedCost = 1.0;
+  }else{
+    iCons = fts5FindConstraint(pInfo, SQLITE_INDEX_CONSTRAINT_EQ, -1);
+    if( iCons>=0 ){
+      ePlan = FTS5_PLAN_ROWID;
+      pInfo->estimatedCost = 2.0;
+    }
+  }
+
+  if( iCons>=0 ){
+    pInfo->aConstraintUsage[iCons].argvIndex = 1;
+    pInfo->aConstraintUsage[iCons].omit = 1;
+  }else{
+    pInfo->estimatedCost = 10000000.0;
+  }
+
+  if( pInfo->nOrderBy==1 && pInfo->aOrderBy[0].iColumn<0 ){
+    pInfo->orderByConsumed = 1;
+    ePlan |= pInfo->aOrderBy[0].desc ? FTS5_ORDER_DESC : FTS5_ORDER_ASC;
+  }
+   
+  pInfo->idxNum = ePlan;
   return SQLITE_OK;
 }
 
@@ -161,7 +223,23 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
 ** Implementation of xOpen method.
 */
 static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
-  return SQLITE_OK;
+  Fts5Cursor *pCsr;
+  int rc = SQLITE_OK;
+  pCsr = (Fts5Cursor*)sqlite3_malloc(sizeof(Fts5Cursor));
+  if( pCsr ){
+    memset(pCsr, 0, sizeof(Fts5Cursor));
+  }else{
+    rc = SQLITE_NOMEM;
+  }
+  *ppCsr = (sqlite3_vtab_cursor*)pCsr;
+  return rc;
+}
+
+static int fts5StmtType(int idxNum){
+  if( FTS5_PLAN(idxNum)==FTS5_PLAN_SCAN ){
+    return (idxNum&FTS5_ORDER_ASC) ? FTS5_STMT_SCAN_ASC : FTS5_STMT_SCAN_DESC;
+  }
+  return FTS5_STMT_LOOKUP;
 }
 
 /*
@@ -169,6 +247,13 @@ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
 ** on the xClose method of the virtual table interface.
 */
 static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
+  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
+  if( pCsr->pStmt ){
+    int eStmt = fts5StmtType(pCsr->idxNum);
+    sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
+  }
+  sqlite3_free(pCsr);
   return SQLITE_OK;
 }
 
@@ -182,7 +267,22 @@ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
 ** subsequently to determine whether or not an EOF was hit.
 */
 static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
-  return SQLITE_OK;
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
+  int ePlan = FTS5_PLAN(pCsr->idxNum);
+  int rc = SQLITE_OK;
+
+  assert( ePlan!=FTS5_PLAN_MATCH );
+  if( ePlan!=FTS5_PLAN_MATCH ){
+    rc = sqlite3_step(pCsr->pStmt);
+    if( rc!=SQLITE_ROW ){
+      pCsr->bEof = 1;
+      rc = sqlite3_reset(pCsr->pStmt);
+    }else{
+      rc = SQLITE_OK;
+    }
+  }
+  
+  return rc;
 }
 
 /*
@@ -197,7 +297,25 @@ static int fts5FilterMethod(
   int nVal,                       /* Number of elements in apVal */
   sqlite3_value **apVal           /* Arguments for the indexing scheme */
 ){
-  return SQLITE_OK;
+  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
+  int rc = SQLITE_OK;
+  int ePlan = FTS5_PLAN(idxNum);
+  int eStmt = fts5StmtType(idxNum);
+
+  assert( ePlan!=FTS5_PLAN_MATCH );
+  memset(&pCursor[1], 0, sizeof(Fts5Cursor) - sizeof(sqlite3_vtab_cursor));
+  pCsr->idxNum = idxNum;
+
+  rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
+  if( ePlan==FTS5_PLAN_ROWID ){
+    sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = fts5NextMethod(pCursor);
+  }
+  return rc;
 }
 
 /* 
@@ -205,7 +323,8 @@ static int fts5FilterMethod(
 ** routine to find out if it has reached the end of a result set.
 */
 static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
-  return 1;
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
+  return pCsr->bEof;
 }
 
 /* 
@@ -215,6 +334,16 @@ static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
 ** rowid should be written to *pRowid.
 */
 static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
+  int ePlan = FTS5_PLAN(pCsr->idxNum);
+  
+  assert( pCsr->bEof==0 );
+  assert( ePlan!=FTS5_PLAN_MATCH );
+
+  if( ePlan!=FTS5_PLAN_MATCH ){
+    *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
+  }
+
   return SQLITE_OK;
 }
 
@@ -227,6 +356,14 @@ static int fts5ColumnMethod(
   sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
   int iCol                        /* Index of column to read value from */
 ){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
+  int ePlan = FTS5_PLAN(pCsr->idxNum);
+  
+  assert( pCsr->bEof==0 );
+  assert( ePlan!=FTS5_PLAN_MATCH );
+  if( ePlan!=FTS5_PLAN_MATCH ){
+    sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
+  }
   return SQLITE_OK;
 }
 
@@ -241,7 +378,7 @@ static int fts5ColumnMethod(
 ** error code if an error occurs.
 */
 static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){
-  const char *z = sqlite3_value_text(pVal);
+  const char *z = (const char*)sqlite3_value_text(pVal);
   int n = sqlite3_value_bytes(pVal);
   int rc = SQLITE_ERROR;
 

ext/fts5/fts5Int.h

@@ -226,6 +226,15 @@ int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*);
 
 int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
 
+#define FTS5_STMT_SCAN_ASC  0     /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
+#define FTS5_STMT_SCAN_DESC 1     /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
+#define FTS5_STMT_LOOKUP    2     /* SELECT rowid, * FROM ... WHERE rowid=? */
+
+int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
+void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
+
+
+
 /*
 ** End of interface to code in fts5_storage.c.
 **************************************************************************/

ext/fts5/fts5_storage.c

@@ -18,18 +18,26 @@ struct Fts5Storage {
   Fts5Config *pConfig;
   Fts5Index *pIndex;
 
-  sqlite3_stmt *aStmt[7];
+  sqlite3_stmt *aStmt[8];
 };
 
-#define FTS5_STMT_INSERT_CONTENT  0
-#define FTS5_STMT_REPLACE_CONTENT 1
 
-#define FTS5_STMT_DELETE_CONTENT  2
+#if FTS5_STMT_SCAN_ASC!=0 
-#define FTS5_STMT_INSERT_DOCSIZE  3
+# error "FTS5_STMT_SCAN_ASC mismatch" 
-#define FTS5_STMT_DELETE_DOCSIZE  4
+#endif
+#if FTS5_STMT_SCAN_DESC!=1 
+# error "FTS5_STMT_SCAN_DESC mismatch" 
+#endif
+#if FTS5_STMT_LOOKUP!=2
+# error "FTS5_STMT_LOOKUP mismatch" 
+#endif
 
-#define FTS5_STMT_SCAN_CONTENT    5
+#define FTS5_STMT_INSERT_CONTENT  3
-#define FTS5_STMT_SEEK_CONTENT    6
+#define FTS5_STMT_REPLACE_CONTENT 4
+
+#define FTS5_STMT_DELETE_CONTENT  5
+#define FTS5_STMT_INSERT_DOCSIZE  6
+#define FTS5_STMT_DELETE_DOCSIZE  7
 
 /*
 ** Prepare the two insert statements - Fts5Storage.pInsertContent and
@@ -47,13 +55,15 @@ static int fts5StorageGetStmt(
   assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
   if( p->aStmt[eStmt]==0 ){
     const char *azStmt[] = {
-      "INSERT INTO %Q.'%q_content' VALUES(%s)",       /* INSERT_CONTENT  */
+      "SELECT * FROM %Q.'%q_content' ORDER BY id ASC",  /* SCAN_ASC */
-      "REPLACE INTO %Q.'%q_content' VALUES(%s)",      /* REPLACE_CONTENT */
+      "SELECT * FROM %Q.'%q_content' ORDER BY id DESC", /* SCAN_DESC */
-      "DELETE FROM %Q.'%q_content' WHERE id=?",       /* DELETE_CONTENT  */
+      "SELECT * FROM %Q.'%q_content' WHERE rowid=?",    /* LOOKUP  */
-      "INSERT INTO %Q.'%q_docsize' VALUES(?,?)",      /* INSERT_DOCSIZE  */
+
-      "DELETE FROM %Q.'%q_docsize' WHERE id=?",       /* DELETE_DOCSIZE  */
+      "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
-      "SELECT * FROM %Q.'%q_content'",                /* SCAN_CONTENT  */
+      "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
-      "SELECT * FROM %Q.'%q_content' WHERE rowid=?",  /* SEEK_CONTENT  */
+      "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
+      "INSERT INTO %Q.'%q_docsize' VALUES(?,?)",        /* INSERT_DOCSIZE  */
+      "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
     };
     Fts5Config *pConfig = p->pConfig;
     char *zSql = 0;
@@ -253,7 +263,7 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
   sqlite3_stmt *pSeek;            /* SELECT to read row iDel from %_data */
   int rc;                         /* Return code */
 
-  rc = fts5StorageGetStmt(p, FTS5_STMT_SEEK_CONTENT, &pSeek);
+  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek);
   if( rc==SQLITE_OK ){
     int rc2;
     sqlite3_bind_int64(pSeek, 1, iDel);
@@ -377,7 +387,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
 
   /* Generate the expected index checksum based on the contents of the
   ** %_content table. This block stores the checksum in ctx.cksum. */
-  rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_CONTENT, &pScan);
+  rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_ASC, &pScan);
   if( rc==SQLITE_OK ){
     int rc2;
     while( SQLITE_ROW==sqlite3_step(pScan) ){
@@ -408,4 +418,44 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
   return rc;
 }
 
+/*
+** Obtain an SQLite statement handle that may be used to read data from the
+** %_content table.
+*/
+int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **pp){
+  int rc;
+  assert( eStmt==FTS5_STMT_SCAN_ASC 
+       || eStmt==FTS5_STMT_SCAN_DESC
+       || eStmt==FTS5_STMT_LOOKUP
+  );
+  rc = fts5StorageGetStmt(p, eStmt, pp);
+  if( rc==SQLITE_OK ){
+    assert( p->aStmt[eStmt]==*pp );
+    p->aStmt[eStmt] = 0;
+  }
+  return rc;
+}
+
+/*
+** Release an SQLite statement handle obtained via an earlier call to
+** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function
+** must match that passed to the sqlite3Fts5StorageStmt() call.
+*/
+void sqlite3Fts5StorageStmtRelease(
+  Fts5Storage *p, 
+  int eStmt, 
+  sqlite3_stmt *pStmt
+){
+  assert( eStmt==FTS5_STMT_SCAN_ASC
+       || eStmt==FTS5_STMT_SCAN_DESC
+       || eStmt==FTS5_STMT_LOOKUP
+  );
+  if( p->aStmt[eStmt]==0 ){
+    sqlite3_reset(pStmt);
+    p->aStmt[eStmt] = pStmt;
+  }else{
+    sqlite3_finalize(pStmt);
+  }
+}
+
 

manifest

@@ -1,5 +1,5 @@
-C Add\ssome\scode\sfor\san\sexperimental\sfts5\smodule.\sDoes\snot\swork\syet.
+C Add\ssimple\sfull-table-scan\sand\srowid\slookup\ssupport\sto\sfts5.
-D 2014-06-23T11:33:22.754
+D 2014-06-24T16:59:06.519
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -103,12 +103,12 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
-F ext/fts5/fts5.c 2cb2cc3c1acefa36d9e8ce8e68bceaac8515059a
+F ext/fts5/fts5.c 3efba544818662a02e8e5ebd73d57cff6182b2dd
-F ext/fts5/fts5Int.h cc41cf776a3e612aa3a461e96463647fd3957bed
+F ext/fts5/fts5Int.h 6f11697324ebaafe92872ee5b19f3661b2b621f1
 F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
 F ext/fts5/fts5_expr.c bdfb98dab7729cf967022d7a4a815828bbad8c23
 F ext/fts5/fts5_index.c 0548e8925a0664cfa00b2477ebe9afa18bc7848f
-F ext/fts5/fts5_storage.c aa1ff4b6b283303ffd8c5dc57a45ebe55e62a7b2
+F ext/fts5/fts5_storage.c 7848d8f8528d798bba159900ea310a6d4a279da8
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@@ -591,7 +591,8 @@ F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
 F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849
 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36
-F test/fts5aa.test bbea71fed733b1d433bf83dbc8d86077936d1efc
+F test/fts5aa.test c8d3b9694f6b2864161c7437408464a535d19343
+F test/fts5ab.test 0c44271259bfba089e9e2ab3c18c2760d8a5392c
 F test/fts5ea.test 814287a2cb25ac3e59abbe4ccbcabf6bda821868
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@@ -1187,10 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 07dda49c1bf8997a18c3368acb81b6d863ea38d6
+P 1e0648dcf283d4f1f6159db4d2433b6cc635992e
-R 66e5d0ccaa728e4d98b92edeb331ffb3
+R 21f33a2cea70ea3a9d3ce73abf49bcfc
-T *branch * fts5
-T *sym-fts5 *
-T -sym-trunk *
 U dan
-Z e3b7f827041011d2f1d78b39cdee11d7
+Z 81c134ce13df1e24ae1f1936c8a52cf8

manifest.uuid

@@ -1 +1 @@
-1e0648dcf283d4f1f6159db4d2433b6cc635992e
+3515da85d09220c464979467b476c611da4a6a7a

test/fts5aa.test

@@ -198,9 +198,6 @@ do_execsql_test 8.1 {
 }
 
 
-#finish_test
-
-
 #-------------------------------------------------------------------------
 #
 reset_db

test/fts5ab.test

@@ -0,0 +1,57 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5ab
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+  INSERT INTO t1 VALUES('hello', 'world');
+  INSERT INTO t1 VALUES('one two', 'three four');
+  INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five');
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM t1;
+} { forty five {one two} {three four} hello world }
+
+do_execsql_test 1.2 {
+  SELECT rowid FROM t1;
+} {45 2 1}
+
+do_execsql_test 1.3 {
+  SELECT rowid FROM t1 ORDER BY rowid ASC;
+} {1 2 45}
+
+do_execsql_test 1.4 {
+  SELECT * FROM t1 WHERE rowid=2;
+} {{one two} {three four}}
+
+do_execsql_test 1.5 {
+  SELECT * FROM t1 WHERE rowid=2.01;
+} {}
+
+do_execsql_test 1.6 {
+  SELECT * FROM t1 WHERE rowid=1.99;
+} {}
+
+finish_test