Further fixes and test cases related to external content tables.

FossilOrigin-Name: ce6a899baff7265a60c880098a9a57ea352b5415

ext/fts5/fts5_config.c

@@ -216,12 +216,13 @@ static char *fts5EscapeName(int *pRc, const char *z){
   char *pRet = 0;
   if( *pRc==SQLITE_OK ){
     int n = strlen(z);
-    pRet = (char*)sqlite3_malloc(2 * 2*n + 1);
+    pRet = (char*)sqlite3_malloc(2 + 2*n + 1);
     if( pRet==0 ){
       *pRc = SQLITE_NOMEM;
     }else{
       int i;
       char *p = pRet;
+      *p++ = '`';
       for(i=0; i<n; i++){
         if( z[i]=='`' ) *p++ = '`';
         *p++ = z[i];

ext/fts5/fts5_index.c

@@ -406,6 +406,7 @@ struct Fts5SegWriter {
 struct Fts5MultiSegIter {
   int nSeg;                       /* Size of aSeg[] array */
   int bRev;                       /* True to iterate in reverse order */
+  int bSkipEmpty;                 /* True to skip deleted entries */
   Fts5SegIter *aSeg;              /* Array of segment iterators */
   u16 *aFirst;                    /* Current merge state (see above) */
 };
@@ -1601,6 +1602,33 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
   }
 }
 
+/*
+** Return true if the iterator passed as the second argument currently
+** points to a delete marker. A delete marker is an entry with a 0 byte
+** position-list.
+*/
+static int fts5SegIterIsDelete(
+  Fts5Index *p,                   /* FTS5 backend object */
+  Fts5SegIter *pIter              /* Iterator to advance */
+){
+  int bRet = 0;
+  Fts5Data *pLeaf = pIter->pLeaf;
+  if( p->rc==SQLITE_OK && pLeaf ){
+    if( pIter->iLeafOffset<pLeaf->n ){
+      bRet = (pLeaf->p[pIter->iLeafOffset]==0x00);
+    }else{
+      Fts5Data *pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
+            pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno
+      ));
+      if( pNew ){
+        bRet = (pNew->p[4]==0x00);
+        fts5DataRelease(pNew);
+      }
+    }
+  }
+  return bRet;
+}
+
 /*
 ** Advance iterator pIter to the next entry. 
 **
@@ -2094,14 +2122,20 @@ static void fts5MultiIterNext(
   i64 iFrom                       /* Advance at least as far as this */
 ){
   if( p->rc==SQLITE_OK ){
-    int iFirst = pIter->aFirst[1];
-    Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
-    if( bFrom && pSeg->pDlidx ){
-      fts5SegIterNextFrom(p, pSeg, iFrom);
-    }else{
-      fts5SegIterNext(p, pSeg);
-    }
-    fts5MultiIterAdvanced(p, pIter, iFirst, 1);
+    int bUseFrom = bFrom;
+    do {
+      int iFirst = pIter->aFirst[1];
+      Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
+      if( bUseFrom && pSeg->pDlidx ){
+        fts5SegIterNextFrom(p, pSeg, iFrom);
+      }else{
+        fts5SegIterNext(p, pSeg);
+      }
+      fts5MultiIterAdvanced(p, pIter, iFirst, 1);
+      bUseFrom = 0;
+    }while( pIter->bSkipEmpty 
+         && fts5SegIterIsDelete(p, &pIter->aSeg[pIter->aFirst[1]])
+    );
   }
 }
 
@@ -2120,6 +2154,7 @@ static void fts5MultiIterNew(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
   Fts5Structure *pStruct,         /* Structure of specific index */
   int iIdx,                       /* Config.aHash[] index of FTS index */
+  int bSkipEmpty,
   int flags,                      /* True for >= */
   const u8 *pTerm, int nTerm,     /* Term to seek to (or NULL/0) */
   int iLevel,                     /* Level to iterate (-1 for all) */
@@ -2152,6 +2187,7 @@ static void fts5MultiIterNew(
   pNew->aSeg = (Fts5SegIter*)&pNew[1];
   pNew->aFirst = (u16*)&pNew->aSeg[nSlot];
   pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_ASC));
+  pNew->bSkipEmpty = bSkipEmpty;
 
   /* Initialize each of the component segment iterators. */
   if( iLevel<0 ){
@@ -2187,6 +2223,12 @@ static void fts5MultiIterNew(
         fts5MultiIterAdvanced(p, pNew, iEq, iIter);
       }
     }
+
+    if( pNew->bSkipEmpty 
+     && fts5SegIterIsDelete(p, &pNew->aSeg[pNew->aFirst[1]]) 
+    ){
+      fts5MultiIterNext(p, pNew, 0, 0);
+    }
   }else{
     fts5MultiIterFree(p, pNew);
     *ppOut = 0;
@@ -2958,7 +3000,7 @@ fprintf(stdout, "merging %d segments from level %d!", nInput, iLvl);
 fflush(stdout);
 #endif
 
-  for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, iLvl, nInput, &pIter);
+  for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, iLvl, nInput, &pIter);
       fts5MultiIterEof(p, pIter)==0;
       fts5MultiIterNext(p, pIter, 0, 0)
   ){
@@ -3689,7 +3731,7 @@ static void fts5SetupPrefixIter(
     Fts5Buffer doclist;
 
     memset(&doclist, 0, sizeof(doclist));
-    for(fts5MultiIterNew(p, pStruct, 0, 1, pToken, nToken, -1, 0, &p1);
+    for(fts5MultiIterNew(p, pStruct, 0, 1, 1, pToken, nToken, -1, 0, &p1);
         fts5MultiIterEof(p, p1)==0;
         fts5MultiIterNext(p, p1, 0, 0)
     ){
@@ -3770,7 +3812,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
   for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
     Fts5MultiSegIter *pIter;
     Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
-    for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, -1, 0, &pIter);
+    for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, -1, 0, &pIter);
         fts5MultiIterEof(p, pIter)==0;
         fts5MultiIterNext(p, pIter, 0, 0)
     ){
@@ -4031,7 +4073,7 @@ int sqlite3Fts5IndexQuery(
       pRet->pStruct = fts5StructureRead(p, iIdx);
       if( pRet->pStruct ){
         fts5MultiIterNew(p, pRet->pStruct, 
-            iIdx, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti
+            iIdx, 1, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti
         );
       }
     }else{

ext/fts5/fts5_storage.c

@@ -62,8 +62,8 @@ static int fts5StorageGetStmt(
   assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
   if( p->aStmt[eStmt]==0 ){
     const char *azStmt[] = {
-      "SELECT * FROM %s ORDER BY id ASC",               /* SCAN_ASC */
-      "SELECT * FROM %s ORDER BY id DESC",              /* SCAN_DESC */
+      "SELECT * FROM %s ORDER BY %s ASC",               /* SCAN_ASC */
+      "SELECT * FROM %s ORDER BY %s DESC",              /* SCAN_DESC */
       "SELECT * FROM %s WHERE %s=?",                    /* LOOKUP  */
 
       "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
@@ -82,9 +82,6 @@ static int fts5StorageGetStmt(
     switch( eStmt ){
       case FTS5_STMT_SCAN_ASC:
       case FTS5_STMT_SCAN_DESC:
-        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent);
-        break;
-
       case FTS5_STMT_LOOKUP:
         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent, pC->zContentRowid);
         break;
@@ -725,7 +722,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
 
   /* Check that the %_docsize and %_content tables contain the expected
   ** number of rows.  */
-  if( rc==SQLITE_OK ){
+  if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
     i64 nRow;
     rc = fts5StorageCount(p, "content", &nRow);
     if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = SQLITE_CORRUPT_VTAB;

ext/fts5/test/fts5aa.test

@@ -303,5 +303,26 @@ do_test 12.3 {
   string is integer $res
 } {1}
 
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 13.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
+} {}
+
+do_execsql_test 13.2 {
+  SELECT rowid FROM t1 WHERE t1 MATCH 'o';
+} {2 1}
+
+do_execsql_test 13.4 {
+  DELETE FROM t1 WHERE rowid=2;
+} {}
+
+do_execsql_test 13.5 {
+  SELECT rowid FROM t1 WHERE t1 MATCH 'o';
+} {1}
+
 finish_test
 
+

ext/fts5/test/fts5content.test

@@ -17,6 +17,9 @@ if {![info exists testdir]} {
 source $testdir/tester.tcl
 set testprefix fts5content
 
+#-------------------------------------------------------------------------
+# Contentless tables
+#
 do_execsql_test 1.1 {
   CREATE VIRTUAL TABLE f1 USING fts5(a, b, content='');
   INSERT INTO f1(rowid, a, b) VALUES(1, 'one',   'o n e');
@@ -83,17 +86,67 @@ do_execsql_test 1.15 {
   INSERT INTO f1(f1, rowid, a, b) VALUES('delete', 2, 'two', 't w o');
 } {}
 
-db eval { SELECT fts5_decode(id, block) AS d FROM f1_data } { puts $d }
-
-breakpoint
 do_execsql_test 1.16 {
   SELECT rowid FROM f1 WHERE f1 MATCH 'o';
 } {4 1}
+
 do_execsql_test 1.17 {
   SELECT rowid FROM f1;
 } {4 3 1}
 
+#-------------------------------------------------------------------------
+# External content tables
+#
+reset_db
+do_execsql_test 2.1 {
+  -- Create a table. And an external content fts5 table to index it.
+  CREATE TABLE tbl(a INTEGER PRIMARY KEY, b, c);
+  CREATE VIRTUAL TABLE fts_idx USING fts5(b, c, content='tbl', content_rowid='a');
 
+  -- Triggers to keep the FTS index up to date.
+  CREATE TRIGGER tbl_ai AFTER INSERT ON tbl BEGIN
+    INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c);
+  END;
+  CREATE TRIGGER tbl_ad AFTER DELETE ON tbl BEGIN
+    INSERT INTO fts_idx(fts_idx, rowid, b, c) 
+        VALUES('delete', old.a, old.b, old.c);
+  END;
+  CREATE TRIGGER tbl_au AFTER UPDATE ON tbl BEGIN
+    INSERT INTO fts_idx(fts_idx, rowid, b, c) 
+        VALUES('delete', old.a, old.b, old.c);
+    INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c);
+  END;
+}
+
+do_execsql_test 2.2 {
+  INSERT INTO tbl VALUES(1, 'one', 'o n e');
+  INSERT INTO tbl VALUES(NULL, 'two', 't w o');
+  INSERT INTO tbl VALUES(3, 'three', 't h r e e');
+}
+
+do_execsql_test 2.3 {
+  INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
+}
+
+do_execsql_test 2.4 {
+  DELETE FROM tbl WHERE rowid=2;
+  INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
+}
+
+do_execsql_test 2.5 {
+  UPDATE tbl SET c = c || ' x y z';
+  INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
+}
+
+do_execsql_test 2.6 {
+  SELECT * FROM fts_idx WHERE fts_idx MATCH 't AND x';
+} {three {t h r e e x y z}}
+
+do_execsql_test 2.7 {
+  SELECT highlight(fts_idx, 1, '[', ']') FROM fts_idx 
+  WHERE fts_idx MATCH 't AND x';
+} {{[t] h r e e [x] y z}}
 
 
 finish_test
+