Change the name of the offsets=0 option to "detail=column". Have the xInst, xPhraseFirst and other API functions work by parsing the original text for detail=column tables.

FossilOrigin-Name: 228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d
2025-12-24 14:17:58 +03:00 · 2015-12-28 19:55:00 +00:00
parent 8f9ced636f
commit 9f44deed93
13 changed files with 379 additions and 120 deletions
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -151,7 +151,7 @@ struct Fts5Config {
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
-  int bOffsets;                   /* "offsets=" option value (dflt==1) */
+  int eDetail;                    /* FTS5_DETAIL_XXX value */
  char *zContentExprlist;
  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;
@@ -180,6 +180,9 @@ struct Fts5Config {
 #define FTS5_CONTENT_NONE     1
 #define FTS5_CONTENT_EXTERNAL 2

+#define FTS5_DETAIL_FULL    0
+#define FTS5_DETAIL_NONE    1
+#define FTS5_DETAIL_COLUMNS 2



@@ -637,6 +640,11 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);

+Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*);
+int sqlite3Fts5ExprPopulatePoslists(
+    Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int
+);
+
 int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);

 /*******************************************
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -194,6 +194,33 @@ void sqlite3Fts5Dequote(char *z){
  }
 }

+
+struct Fts5Enum {
+  const char *zName;
+  int eVal;
+};
+typedef struct Fts5Enum Fts5Enum;
+
+static int fts5ConfigSetEnum(
+  const Fts5Enum *aEnum, 
+  const char *zEnum, 
+  int *peVal
+){
+  int nEnum = strlen(zEnum);
+  int i;
+  int iVal = -1;
+
+  for(i=0; aEnum[i].zName; i++){
+    if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
+      if( iVal>=0 ) return SQLITE_ERROR;
+      iVal = aEnum[i].eVal;
+    }
+  }
+
+  *peVal = iVal;
+  return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
+}
+
 /*
 ** Parse a "special" CREATE VIRTUAL TABLE directive and update
 ** configuration object pConfig as appropriate.
@@ -344,12 +371,16 @@ static int fts5ConfigParseSpecial(
    return rc;
  }

-  if( sqlite3_strnicmp("offsets", zCmd, nCmd)==0 ){
-    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
-      *pzErr = sqlite3_mprintf("malformed offsets=... directive");
-      rc = SQLITE_ERROR;
-    }else{
-      pConfig->bOffsets = (zArg[0]=='1');
+  if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
+    const Fts5Enum aDetail[] = {
+      { "none", FTS5_DETAIL_NONE },
+      { "full", FTS5_DETAIL_FULL },
+      { "columns", FTS5_DETAIL_COLUMNS },
+      { 0, 0 }
+    };
+
+    if( rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail) ){
+      *pzErr = sqlite3_mprintf("malformed detail=... directive");
    }
    return rc;
  }
@@ -509,7 +540,7 @@ int sqlite3Fts5ConfigParse(
  pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
  pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
  pRet->bColumnsize = 1;
-  pRet->bOffsets = 1;
+  pRet->eDetail = FTS5_DETAIL_FULL;
 #ifdef SQLITE_DEBUG
  pRet->bPrefixIndex = 1;
 #endif
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -1744,6 +1744,15 @@ void sqlite3Fts5ParseSetColset(
  Fts5ExprNearset *pNear, 
  Fts5Colset *pColset 
 ){
+  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
+    pParse->rc = SQLITE_ERROR;
+    pParse->zErr = sqlite3_mprintf(
+      "fts5: column queries are not supported (detail=none)"
+    );
+    sqlite3_free(pColset);
+    return;
+  }
+
  if( pNear ){
    pNear->pColset = pColset;
  }else{
@@ -1809,12 +1818,12 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
          if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){
            pRet->eType = FTS5_TERM;
          }
-        }else if( pParse->pConfig->bOffsets==0 ){
+        }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
          assert( pParse->rc==SQLITE_OK );
          pParse->rc = SQLITE_ERROR;
          assert( pParse->zErr==0 );
          pParse->zErr = sqlite3_mprintf(
-              "fts5: %s queries are not supported (offsets=0)", 
+              "fts5: %s queries are not supported (detail!=full)", 
              pNear->nPhrase==1 ? "phrase": "NEAR"
          );
          sqlite3_free(pRet);
@@ -1932,6 +1941,9 @@ static char *fts5ExprPrintTcl(
      for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
        char *zTerm = pPhrase->aTerm[iTerm].zTerm;
        zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
+        if( pPhrase->aTerm[iTerm].bPrefix ){
+          zRet = fts5PrintfAppend(zRet, "*");
+        }
      }

      if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
@@ -2244,3 +2256,74 @@ int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
  }
  return nRet;
 }
+
+Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){
+  int i;
+  Fts5PoslistWriter *pRet;
+  for(i=0; i<pExpr->nPhrase; i++){
+    Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
+    assert( pExpr->apExprPhrase[i]->nTerm==1 );
+    pBuf->n = 0;
+  }
+  pRet = sqlite3_malloc(sizeof(Fts5PoslistWriter)*pExpr->nPhrase);
+  if( pRet ){
+    memset(pRet, 0, sizeof(Fts5PoslistWriter)*pExpr->nPhrase);
+  }
+  return pRet;
+}
+
+struct Fts5ExprCtx {
+  Fts5Expr *pExpr;
+  Fts5PoslistWriter *aWriter;
+  i64 iOff;
+};
+typedef struct Fts5ExprCtx Fts5ExprCtx;
+
+static int fts5ExprPopulatePoslistsCb(
+  void *pCtx,                /* Copy of 2nd argument to xTokenize() */
+  int tflags,                /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken,        /* Pointer to buffer containing token */
+  int nToken,                /* Size of token in bytes */
+  int iStart,                /* Byte offset of token within input text */
+  int iEnd                   /* Byte offset of end of token within input text */
+){
+  Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
+  Fts5Expr *pExpr = p->pExpr;
+  int i;
+
+  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
+  for(i=0; i<pExpr->nPhrase; i++){
+    Fts5ExprTerm *pTerm;
+    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
+      int nTerm = strlen(pTerm->zTerm);
+      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
+       && memcmp(pTerm->zTerm, pToken, nTerm)==0
+      ){
+        int rc = sqlite3Fts5PoslistWriterAppend(
+            &pExpr->apExprPhrase[i]->poslist, &p->aWriter[i], p->iOff
+        );
+        if( rc ) return rc;
+        break;
+      }
+    }
+  }
+  return SQLITE_OK;
+}
+
+int sqlite3Fts5ExprPopulatePoslists(
+  Fts5Config *pConfig,
+  Fts5Expr *pExpr, 
+  Fts5PoslistWriter *aWriter,
+  int iCol, 
+  const char *z, int n
+){
+  Fts5ExprCtx sCtx;
+  sCtx.pExpr = pExpr;
+  sCtx.aWriter = aWriter;
+  sCtx.iOff = (((i64)iCol) << 32) - 1;
+
+  return sqlite3Fts5Tokenize(pConfig, 
+      FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
+  );
+}
+
--- a/ext/fts5/fts5_hash.c
+++ b/ext/fts5/fts5_hash.c
@@ -26,7 +26,7 @@ typedef struct Fts5HashEntry Fts5HashEntry;


 struct Fts5Hash {
-  int bOffsets;                   /* Copy of Fts5Config.bOffsets */
+  int eDetail;                    /* Copy of Fts5Config.eDetail */
  int *pnByte;                    /* Pointer to bytes counter */
  int nEntry;                     /* Number of entries currently in hash */
  int nSlot;                      /* Size of aSlot[] array */
@@ -91,7 +91,7 @@ int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
    int nByte;
    memset(pNew, 0, sizeof(Fts5Hash));
    pNew->pnByte = pnByte;
-    pNew->bOffsets = pConfig->bOffsets;
+    pNew->eDetail = pConfig->eDetail;

    pNew->nSlot = 1024;
    nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
@@ -216,7 +216,9 @@ int sqlite3Fts5HashWrite(
  Fts5HashEntry *p;
  u8 *pPtr;
  int nIncr = 0;                  /* Amount to increment (*pHash->pnByte) by */
-  int bNew = pHash->bOffsets;     /* If non-delete entry should be written */
+  int bNew;                       /* If non-delete entry should be written */
+  
+  bNew = (pHash->eDetail==FTS5_DETAIL_FULL);

  /* Attempt to locate an existing hash entry */
  iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
@@ -253,7 +255,7 @@ int sqlite3Fts5HashWrite(
    p->iSzPoslist = p->nData;
    p->nData += 1;
    p->iRowid = iRowid;
-    p->iCol = (pHash->bOffsets-1);
+    p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
    p->pHashNext = pHash->aSlot[iHash];
    pHash->aSlot[iHash] = p;
    pHash->nEntry++;
@@ -290,7 +292,7 @@ int sqlite3Fts5HashWrite(
    p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
    p->iSzPoslist = p->nData;
    p->nData += 1;
-    p->iCol = (pHash->bOffsets-1);
+    p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
    p->iPos = 0;
    p->iRowid = iRowid;
    bNew = 1;
@@ -300,14 +302,14 @@ int sqlite3Fts5HashWrite(
    /* Append a new column value, if necessary */
    assert( iCol>=p->iCol );
    if( iCol!=p->iCol ){
-      if( pHash->bOffsets==0 ){
-        bNew = 1;
-        p->iCol = iPos = iCol;
-      }else{
+      if( pHash->eDetail==FTS5_DETAIL_FULL ){
        pPtr[p->nData++] = 0x01;
        p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
        p->iCol = iCol;
        p->iPos = 0;
+      }else{
+        bNew = 1;
+        p->iCol = iPos = iCol;
      }
    }

--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4109,19 +4109,19 @@ static void fts5SegiterPoslist(
    if( pColset==0 ){
      fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
    }else{
-      if( p->pConfig->bOffsets==0 ){
-        PoslistOffsetsCtx sCtx;
-        memset(&sCtx, 0, sizeof(sCtx));
-        sCtx.pBuf = pBuf;
-        sCtx.pColset = pColset;
-        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
-      }else{
+      if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){
        PoslistCallbackCtx sCtx;
        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;
        sCtx.eState = fts5IndexColsetTest(pColset, 0);
        assert( sCtx.eState==0 || sCtx.eState==1 );
        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
+      }else{
+        PoslistOffsetsCtx sCtx;
+        memset(&sCtx, 0, sizeof(sCtx));
+        sCtx.pBuf = pBuf;
+        sCtx.pColset = pColset;
+        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
      }
    }
  }
@@ -4192,7 +4192,7 @@ static int fts5AppendPoslist(
    assert( fts5MultiIterEof(p, pMulti)==0 );
    assert( pSeg->nPos>0 );
    if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){
-      if( p->pConfig->bOffsets
+      if( p->pConfig->eDetail==FTS5_DETAIL_FULL
       && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
       && (pColset==0 || pColset->nCol==1)
      ){
@@ -4850,9 +4850,11 @@ int sqlite3Fts5IterPoslist(
  i64 *piRowid                    /* OUT: Current rowid */
 ){
  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
+  int eDetail = pIter->pIndex->pConfig->eDetail;
+
  assert( pIter->pIndex->rc==SQLITE_OK );
  *piRowid = pSeg->iRowid;
-  if( pIter->pIndex->pConfig->bOffsets 
+  if( eDetail==FTS5_DETAIL_FULL 
   && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
  ){
    u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
@@ -4871,7 +4873,9 @@ int sqlite3Fts5IterPoslist(
  }else{
    fts5BufferZero(&pIter->poslist);
    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
-    *pp = pIter->poslist.p;
+    if( eDetail==FTS5_DETAIL_FULL ){
+      *pp = pIter->poslist.p;
+    }
    *pn = pIter->poslist.n;
  }
  return fts5IndexReturn(pIter->pIndex);
@@ -5062,17 +5066,15 @@ static int fts5QueryCksum(
 ){
  u64 cksum = *pCksum;
  Fts5IndexIter *pIdxIter = 0;
+  Fts5Buffer buf = {0, 0, 0};
  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);

  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
-    i64 dummy;
-    const u8 *pPos;
-    int nPos;
    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
-    rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy);
+    rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
    if( rc==SQLITE_OK ){
      Fts5PoslistReader sReader;
-      for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader);
+      for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
          sReader.bEof==0;
          sqlite3Fts5PoslistReaderNext(&sReader)
      ){
@@ -5084,6 +5086,7 @@ static int fts5QueryCksum(
    }
  }
  sqlite3Fts5IterClose(pIdxIter);
+  fts5BufferFree(&buf);

  *pCksum = cksum;
  return rc;
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -226,6 +226,7 @@ struct Fts5Cursor {
 #define FTS5CSR_EOF               0x08
 #define FTS5CSR_FREE_ZRANK        0x10
 #define FTS5CSR_REQUIRE_RESEEK    0x20
+#define FTS5CSR_REQUIRE_POSLIST   0x40

 #define BitFlagAllTest(x,y) (((x) & (y))==(y))
 #define BitFlagTest(x,y)    (((x) & (y))!=0)
@@ -313,7 +314,7 @@ static int fts5IsContentless(Fts5Table *pTab){
 ** Return true if pTab is an offsetless table.
 */
 static int fts5IsOffsetless(Fts5Table *pTab){
-  return pTab->pConfig->bOffsets==0;
+  return pTab->pConfig->eDetail!=FTS5_DETAIL_FULL;
 }

 /*
@@ -646,6 +647,7 @@ static void fts5CsrNewrow(Fts5Cursor *pCsr){
      FTS5CSR_REQUIRE_CONTENT 
    | FTS5CSR_REQUIRE_DOCSIZE 
    | FTS5CSR_REQUIRE_INST 
+    | FTS5CSR_REQUIRE_POSLIST 
  );
 }

@@ -1603,6 +1605,8 @@ static int fts5RollbackMethod(sqlite3_vtab *pVtab){
  return rc;
 }

+static int fts5CsrPoslist(Fts5Cursor*, int, const u8**);
+
 static void *fts5ApiUserData(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return pCsr->pAux->pUserData;
@@ -1652,8 +1656,53 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
  return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
 }

+static int fts5ApiColumnText(
+  Fts5Context *pCtx, 
+  int iCol, 
+  const char **pz, 
+  int *pn
+){
+  int rc = SQLITE_OK;
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
+    *pz = 0;
+    *pn = 0;
+  }else{
+    rc = fts5SeekCursor(pCsr, 0);
+    if( rc==SQLITE_OK ){
+      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
+      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
+    }
+  }
+  return rc;
+}
+
 static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){
  int n;
+  int rc = SQLITE_OK;
+
+  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
+    Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
+    if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
+      Fts5PoslistWriter *aWriter;
+      int i;
+assert( pCsr->pSorter==0 );  /* fixme */
+      aWriter = sqlite3Fts5ExprClearPoslists(pCsr->pExpr);
+      if( aWriter==0 ) rc = SQLITE_NOMEM;
+      for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
+        int n; const char *z;
+        rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n);
+        if( rc==SQLITE_OK ){
+          rc = sqlite3Fts5ExprPopulatePoslists(
+              pConfig, pCsr->pExpr, aWriter, i, z, n
+          );
+        }
+      }
+      sqlite3_free(aWriter);
+    }
+    CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST);
+  }
+
  if( pCsr->pSorter ){
    Fts5Sorter *pSorter = pCsr->pSorter;
    int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
@@ -1756,10 +1805,12 @@ static int fts5ApiInst(
  ){
    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
      rc = SQLITE_RANGE;
+#if 0
    }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
      *piPhrase = pCsr->aInst[iIdx*3];
      *piCol = pCsr->aInst[iIdx*3 + 2];
      *piOff = -1;
+#endif
    }else{
      *piPhrase = pCsr->aInst[iIdx*3];
      *piCol = pCsr->aInst[iIdx*3 + 1];
@@ -1773,27 +1824,6 @@ static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
  return fts5CursorRowid((Fts5Cursor*)pCtx);
 }

-static int fts5ApiColumnText(
-  Fts5Context *pCtx, 
-  int iCol, 
-  const char **pz, 
-  int *pn
-){
-  int rc = SQLITE_OK;
-  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
-  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
-    *pz = 0;
-    *pn = 0;
-  }else{
-    rc = fts5SeekCursor(pCsr, 0);
-    if( rc==SQLITE_OK ){
-      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
-      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
-    }
-  }
-  return rc;
-}
-
 static int fts5ColumnSizeCb(
  void *pContext,                 /* Pointer to int */
  int tflags,
@@ -1925,11 +1955,13 @@ static void fts5ApiPhraseNext(
  if( pIter->a>=pIter->b ){
    *piCol = -1;
    *piOff = -1;
+#if 0
  }else if( fts5IsOffsetless((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab)) ){
    int iVal;
    pIter->a += fts5GetVarint32(pIter->a, iVal);
    *piCol += (iVal-2);
    *piOff = -1;
+#endif
  }else{
    int iVal;
    pIter->a += fts5GetVarint32(pIter->a, iVal);
@@ -1982,7 +2014,6 @@ static const Fts5ExtensionApi sFts5Api = {
  fts5ApiPhraseNext,
 };

-
 /*
 ** Implementation of API function xQueryPhrase().
 */
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -916,7 +916,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
        if( pConfig->abUnindexed[i] ) continue;
        ctx.iCol = i;
        ctx.szCol = 0;
-        if( pConfig->bOffsets==0 ){
+        if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
          rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
        }
        if( rc==SQLITE_OK ){
--- a/ext/fts5/test/fts5_common.tcl
+++ b/ext/fts5/test/fts5_common.tcl
@@ -239,7 +239,21 @@ proc nearset {aCol args} {
        set iFirst [expr $iFL - $O(-near) - [llength $p]]

        for {set i $iFirst} {$i <= $iFL} {incr i} {
-          if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i }
+          set lCand [lrange $col $i [expr $i+$nPm1]]
+
+          set bMatch 1
+          foreach tok $p term $lCand {
+            if {[string match $tok $term]==0} {
+              #puts "$tok $term failed"
+              set bMatch 0
+            }
+          }
+          if {$bMatch} { 
+            #puts "match at $i"
+            lappend B($iPhrase) $i 
+          }
+
+          #if {$lCand == $p} { lappend B($iPhrase) $i }
        }
        if {[llength $B($iPhrase)] == 0} break
      }
@@ -265,7 +279,7 @@ proc nearset {aCol args} {
    incr counter
  }

-  #puts $res
+  #puts "$aCol -> $res"
  sort_poslist $res
 }

--- a/ext/fts5/test/fts5ac.test
+++ b/ext/fts5/test/fts5ac.test
@@ -130,7 +130,7 @@ set data {
 # 
 #   CREATE VIRTUAL TABLE xy USING fts5(x, y);
 #
-# Assuming the table contains the same records as stored int the global 
+# Assuming the table contains the same records as stored in the global 
 # $::data array (see above), this function returns a list containing one
 # element for each match in the dataset. The elements are themselves lists
 # formatted as follows:
@@ -175,30 +175,22 @@ proc matchdata {bPos expr {bAsc 1}} {
 # End of test code
 #-------------------------------------------------------------------------

-proc fts5_test_poslist {cmd} {
-  set res [list]
-  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
-    lappend res [string map {{ } .} [$cmd xInst $i]]
-  }
-  set res
-}
-

 foreach {tn2 sql} {
  1  {}
  2  {BEGIN}
 } {
  reset_db
-  sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist
+  fts5_aux_test_functions db

-  do_execsql_test 1.0 {
+  do_execsql_test 1.$tn2.0 {
    CREATE VIRTUAL TABLE xx USING fts5(x,y);
    INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
  }

  execsql $sql

-  do_test $tn2.1.1 {
+  do_test 1.$tn2.1.1 {
    foreach {id x y} $data {
      execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
    }
@@ -224,7 +216,7 @@ foreach {tn2 sql} {
    set expr "\"$phrase\""
    set res [matchdata 1 $expr]

-    do_execsql_test $tn2.1.2.$tn.[llength $res] { 
+    do_execsql_test 1.$tn2.1.2.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res
  }
@@ -246,7 +238,7 @@ foreach {tn2 sql} {
    3.1 { a AND b AND c }
  } {
    set res [matchdata 1 $expr]
-    do_execsql_test $tn2.2.$tn.[llength $res] { 
+    do_execsql_test 1.$tn2.2.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res
  }
@@ -275,7 +267,7 @@ foreach {tn2 sql} {
    4.4 {{"y" y}:b}
  } {
    set res [matchdata 1 $expr]
-    do_execsql_test $tn2.3.$tn.[llength $res] { 
+    do_execsql_test 1.$tn2.3.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res
  }
@@ -296,7 +288,7 @@ foreach {tn2 sql} {
    9  { y : NEAR(r c) }
  } {
    set res [matchdata 1 $expr]
-    do_execsql_test $tn2.4.1.$tn.[llength $res] { 
+    do_execsql_test 1.$tn2.4.1.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res
  }
@@ -307,14 +299,14 @@ foreach {tn2 sql} {
  foreach {tn expr tclexpr} {
    1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
  } {
-    do_execsql_test $tn2.5.$tn {
+    do_execsql_test 1.$tn2.5.$tn {
      SELECT fts5_expr_tcl($expr, 'N $x')
    } [list $tclexpr]
  }

  #-------------------------------------------------------------------------
  #
-  do_execsql_test $tn2.6.integrity {
+  do_execsql_test 1.$tn2.6.integrity {
    INSERT INTO xx(xx) VALUES('integrity-check');
  }
  #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r}
@@ -346,14 +338,62 @@ foreach {tn2 sql} {
      19 { c NOT b OR a AND d }
    } {
      set res [matchdata 0 $expr $bAsc]
-      do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res
+      do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res
    }
  }
 }

-do_execsql_test 3.1 {
+do_execsql_test 2.1 {
  SELECT fts5_expr_tcl('a AND b');
 } {{AND [nearset -- {a}] [nearset -- {b}]}}

+# Some tests for detail=col tables and detail=none.
+#
+foreach {tn2 sql} {
+  1  {
+    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col);
+  }
+  2  {
+    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col);
+    BEGIN;
+  }
+  3  {
+    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none);
+    BEGIN;
+  }
+  4  {
+    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none);
+  }
+} {
+  reset_db
+  fts5_aux_test_functions db
+
+  execsql $sql
+
+  do_execsql_test 3.$tn2.0 {
+    INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
+  }
+
+
+  do_test 3.$tn2.1.1 {
+    foreach {id x y} $data {
+      execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
+    }
+    execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
+  } {}
+
+  foreach {tn q} {
+    1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L"
+    11 "o a" 12 "c AND d" 13 "o OR a" 12 "c OR d"
+  } {
+    set res [matchdata 1 $q]
+
+    do_execsql_test 3.$tn2.1.2.$tn.[llength $res] { 
+      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q
+    } $res
+  }
+
+}
+
 finish_test

--- a/ext/fts5/test/fts5ad.test
+++ b/ext/fts5/test/fts5ad.test
@@ -75,17 +75,17 @@ foreach {T create} {
  }

  6 {
-    CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0);
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }

  7 {
-    CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5");
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5");
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }

  8 {
-    CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5");
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5");
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    BEGIN;
  }
--- a/ext/fts5/test/fts5offsets.test
+++ b/ext/fts5/test/fts5offsets.test
@@ -13,7 +13,7 @@
 #

 source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5offsets
+set testprefix fts5detail

 # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
@@ -21,11 +21,13 @@ ifcapable !fts5 {
  return
 }

+fts5_aux_test_functions db
+
 #--------------------------------------------------------------------------
 # Simple tests.
 #
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col);
  INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
  INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
  INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
@@ -58,17 +60,18 @@ foreach {tn match res} {

 do_catchsql_test 1.3.1 {
  SELECT rowid FROM t1('h + d');
-} {1 {fts5: phrase queries are not supported (offsets=0)}}
+} {1 {fts5: phrase queries are not supported (detail!=full)}}

 do_catchsql_test 1.3.2 {
  SELECT rowid FROM t1('NEAR(h d)');
-} {1 {fts5: NEAR queries are not supported (offsets=0)}}
+} {1 {fts5: NEAR queries are not supported (detail!=full)}}
+

 #-------------------------------------------------------------------------
-# integrity-check with both offsets= and prefix= options.
+# integrity-check with both detail= and prefix= options.
 #
 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, offsets=0, prefix="1");
+  CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1");
  INSERT INTO t2(a) VALUES('aa ab');
 }

@@ -78,9 +81,17 @@ do_execsql_test 2.1 {
  INSERT INTO t2(t2) VALUES('integrity-check');
 }

+do_execsql_test 2.2 {
+  SELECT fts5_test_poslist(t2) FROM t2('aa');
+} {0.0.0}
+
+set ::pc 0
+#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
+#exit
+
 #-------------------------------------------------------------------------
 # Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs
-# work with offsets=0 tables.
+# work with detail=col tables.
 #
 set data {
  1  {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
@@ -124,9 +135,42 @@ set data {
  39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba}
  40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca}
 }
+
+set data {
+  1  {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
+}
+
+proc matchdata {expr {bAsc 1}} {
+
+  set tclexpr [db one {
+    SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z')
+  }]
+  set res [list]
+
+  #puts "$expr -> $tclexpr"
+  foreach {id x y z} $::data {
+    set cols [list $x $y $z]
+    set ::pc 0
+    #set hits [lsort -command instcompare [eval $tclexpr]]
+    set hits [eval $tclexpr]
+    if {[llength $hits]>0} {
+      lappend res [list $id $hits]
+    }
+  }
+
+  if {$bAsc} {
+    set res [lsort -integer -increasing -index 0 $res]
+  } else {
+    set res [lsort -integer -decreasing -index 0 $res]
+  }
+
+  return [concat {*}$res]
+}
+
 foreach {tn tbl} {
-  1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, offsets=0) }
+  1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) }
 } {
+#break
  reset_db
  fts5_aux_test_functions db
  execsql $tbl
@@ -139,17 +183,7 @@ foreach {tn tbl} {
    9 a*     10 b*   11 c*
  } {

-    set res [list]
-    foreach {id x y z} $data {
-      if {[lsearch [concat $x $y $z] $expr]>=0} {
-        lappend res $id
-        set inst [list]
-        if {[lsearch $x $expr]>=0} { lappend inst 0.0.-1 }
-        if {[lsearch $y $expr]>=0} { lappend inst 0.1.-1 }
-        if {[lsearch $z $expr]>=0} { lappend inst 0.2.-1 }
-        lappend res $inst
-      }
-    }
+    set res [matchdata $expr]

    do_execsql_test 3.$tn.$tn2.1 {
      SELECT rowid, fts5_test_poslist(t3) FROM t3($expr)
@@ -159,8 +193,21 @@ foreach {tn tbl} {
      SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr)
    } $res
  }
-
 }

+#-------------------------------------------------------------------------
+# Simple tests for detail=none tables.
+#
+breakpoint
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none);
+  INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g');
+  INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9');
+}
+
+do_catchsql_test 4.1 {
+  SELECT * FROM t4('a:a')
+} {1 {fts5: column queries are not supported (detail=none)}}
+
 finish_test

--- a/32
+++ b/32
@@ -1,5 +1,5 @@
-C Updates\sso\sthat\sfts5\sAPI\sfunctions\sxInst,\sxPhraseFirst\sand\sxPhraseNext\swork\swith\sthe\soffsets=0\soption.
-D 2015-12-22T18:54:16.763
+C Change\sthe\sname\sof\sthe\soffsets=0\soption\sto\s"detail=column".\sHave\sthe\sxInst,\sxPhraseFirst\sand\sother\sAPI\sfunctions\swork\sby\sparsing\sthe\soriginal\stext\sfor\sdetail=column\stables.
+D 2015-12-28T19:55:00.739
 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751
@@ -97,15 +97,15 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 7d6785c75afe23239dad9cbaffc6fc55803feb4b
-F ext/fts5/fts5Int.h e71739ed0b816758f1f77baaccca1b4a9064c4dc
+F ext/fts5/fts5Int.h 7328cfa7b0151e34bb3715d404d732c6d49d5361
 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35
 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
-F ext/fts5/fts5_config.c ba5248a05c28ec6a6fdf2599a86e9fd67e5c61e2
-F ext/fts5/fts5_expr.c 3b2c7ac54e6c03e732751a6a4bf9ced8f408e2a2
-F ext/fts5/fts5_hash.c d4a6b52faca0134cc7bcc880f03a257a0dec2636
-F ext/fts5/fts5_index.c 1d1939afbc434907993d7e9a0c631be630d5b0dc
-F ext/fts5/fts5_main.c b7d6540c63f347908d8ee937ca8a7987130c84b3
-F ext/fts5/fts5_storage.c 14e0bb6549a66da54adf4fa1564edbf24647cb22
+F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6
+F ext/fts5/fts5_expr.c c9522ae99f862ddb318ea69998a02147d2a00c8d
+F ext/fts5/fts5_hash.c 8b510868502ec31119409fc7022edc37c27b5c40
+F ext/fts5/fts5_index.c 24e81d8f2ce92f50eb19c15bffd5ada4ec91d8de
+F ext/fts5/fts5_main.c a71db65929e150031c5a195e3c782e773deb55a6
+F ext/fts5/fts5_storage.c 076a3356536a8831eb6e554195171d5c249cd179
 F ext/fts5/fts5_tcl.c c3cfff5f727b3d02f73c5717413d637b18b960f2
 F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3
 F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e
@@ -114,11 +114,11 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
 F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2
 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl 4a23a75f31dbb96c03c2840fae1bf85b4f64f062
+F ext/fts5/test/fts5_common.tcl 27f941596e241e25b4bd64c2446ae91dd4cb138c
 F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206
 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
-F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c
-F ext/fts5/test/fts5ad.test 21d87b12c7ec83b4ec48816d24503443dffb10a1
+F ext/fts5/test/fts5ac.test 5dea9e119ed39e0923b423dc2b3eaaef2ec3abd3
+F ext/fts5/test/fts5ad.test 0fd1a5bcb5dcc76a639bd26f27e2641034a508a3
 F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe
 F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a
 F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505
@@ -139,6 +139,7 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
 F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599
+F ext/fts5/test/fts5detail.test 71babe633c68c6c5153b2fe1be81a97e953ca485 w ext/fts5/test/fts5offsets.test
 F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d
 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
 F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
@@ -156,7 +157,6 @@ F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65
 F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2
 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367
 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc
-F ext/fts5/test/fts5offsets.test 7d32df518f6513131c14ce985006c3ac5873e79f
 F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c
 F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5
 F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 609a0bc7f34e6dae74ce756aff920f3df78fe828
-R 3d59533357e77f6a30434e9a4ea67f85
+P 69bffc1632c8a8f3bfe5bf92607e64fed982e48c
+R bce3f221a6a9e036b503a3a5522049ea
 U dan
-Z ae770209a83bcb452cb39fda88b8ef75
+Z 73bb0a8c75a5e4585189e64532bdf9a9
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-69bffc1632c8a8f3bfe5bf92607e64fed982e48c
+228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d