From e7036202bcc2bae5420332388dee8a74fce969a1 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 17 Aug 2016 19:05:12 +0000
Subject: [PATCH 1/6] Bias the fts5 snippet() function to return snippets that
 look like they start at the start of sentences.

FossilOrigin-Name: 60de159476edbd48dc363f7f77f09c32ea68422f
---
 ext/fts5/fts5_aux.c             | 209 +++++++++++++++++++++++++-------
 ext/fts5/test/fts5af.test       |   5 +-
 ext/fts5/test/fts5unicode2.test |   4 +-
 manifest                        |  16 +--
 manifest.uuid                   |   2 +-
 5 files changed, 181 insertions(+), 55 deletions(-)

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index d8147f8111..ca55fd6b01 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -246,6 +246,115 @@ static void fts5HighlightFunction(
 ** End of highlight() implementation.
 **************************************************************************/
 
+/*
+** Context object passed to the fts5SentenceFinderCb() function.
+*/
+typedef struct Fts5SFinder Fts5SFinder;
+struct Fts5SFinder {
+  int iPos;                       /* Current token position */
+  int nFirstAlloc;                /* Allocated size of aFirst[] */
+  int nFirst;                     /* Number of entries in aFirst[] */
+  int *aFirst;                    /* Array of first token in each sentence */
+  const char *zDoc;               /* Document being tokenized */
+};
+
+/*
+** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
+** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
+** error occurs.
+*/
+static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
+  if( p->nFirstAlloc==p->nFirst ){
+    int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
+    int *aNew;
+
+    aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int));
+    if( aNew==0 ) return SQLITE_NOMEM;
+    p->aFirst = aNew;
+    p->nFirstAlloc = nNew;
+  }
+  p->aFirst[p->nFirst++] = iAdd;
+  return SQLITE_OK;
+}
+
+/*
+** This function is an xTokenize() callback used by the auxiliary snippet()
+** function. Its job is to identify tokens that are the first in a sentence.
+** For each such token, an entry is added to the SFinder.aFirst[] array.
+*/
+static int fts5SentenceFinderCb(
+  void *pContext,                 /* Pointer to HighlightContext object */
+  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStartOff,                  /* Start offset of token */
+  int iEndOff                     /* End offset of token */
+){
+  int rc = SQLITE_OK;
+
+  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
+    Fts5SFinder *p = (Fts5SFinder*)pContext;
+    if( p->iPos>0 ){
+      int i;
+      char c = 0;
+      for(i=iStartOff-1; i>=0; i--){
+        c = p->zDoc[i];
+        if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
+      }
+      if( i!=iStartOff-1 && (c=='.' || c==':') ){
+        rc = fts5SentenceFinderAdd(p, p->iPos);
+      }
+    }else{
+      rc = fts5SentenceFinderAdd(p, 0);
+    }
+    p->iPos++;
+  }
+  return rc;
+}
+
+static int fts5SnippetScore(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  int nDocsize,                   /* Size of column in tokens */
+  unsigned char *aSeen,           /* Array with one element per query phrase */
+  int iCol,                       /* Column to score */
+  int iPos,                       /* Starting offset to score */
+  int nToken,                     /* Max tokens per snippet */
+  int *pnScore,                   /* OUT: Score */
+  int *piPos                      /* OUT: Adjusted offset */
+){
+  int rc;
+  int i;
+  int ip = 0;
+  int ic = 0;
+  int iOff = 0;
+  int iFirst = -1;
+  int nInst;
+  int nScore = 0;
+
+  rc = pApi->xInstCount(pFts, &nInst);
+  for(i=0; i<nInst && rc==SQLITE_OK; i++){
+    rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
+    if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){
+      nScore += (aSeen[ip] ? 1 : 1000);
+      aSeen[ip] = 1;
+      if( iFirst<0 ) iFirst = iOff;
+    }
+  }
+
+  *pnScore = nScore;
+  if( piPos ){
+    int iLast = iOff + pApi->xPhraseSize(pFts, ip);
+    int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
+
+    if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
+    if( iAdj<0 ) iAdj = 0;
+    *piPos = iAdj;
+  }
+
+  return rc;
+}
+
 /*
 ** Implementation of snippet() function.
 */
@@ -267,9 +376,10 @@ static void fts5SnippetFunction(
   unsigned char *aSeen;           /* Array of "seen instance" flags */
   int iBestCol;                   /* Column containing best snippet */
   int iBestStart = 0;             /* First token of best snippet */
-  int iBestLast;                  /* Last token of best snippet */
   int nBestScore = 0;             /* Score of best snippet */
   int nColSize = 0;               /* Total size of iBestCol in tokens */
+  Fts5SFinder sFinder;            /* Used to find the beginnings of sentences */
+  int nCol;
 
   if( nVal!=5 ){
     const char *zErr = "wrong number of arguments to function snippet()";
@@ -277,13 +387,13 @@ static void fts5SnippetFunction(
     return;
   }
 
+  nCol = pApi->xColumnCount(pFts);
   memset(&ctx, 0, sizeof(HighlightContext));
   iCol = sqlite3_value_int(apVal[0]);
   ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
   zEllips = (const char*)sqlite3_value_text(apVal[3]);
   nToken = sqlite3_value_int(apVal[4]);
-  iBestLast = nToken-1;
 
   iBestCol = (iCol>=0 ? iCol : 0);
   nPhrase = pApi->xPhraseCount(pFts);
@@ -291,43 +401,64 @@ static void fts5SnippetFunction(
   if( aSeen==0 ){
     rc = SQLITE_NOMEM;
   }
-
   if( rc==SQLITE_OK ){
     rc = pApi->xInstCount(pFts, &nInst);
   }
-  for(i=0; rc==SQLITE_OK && i<nInst; i++){
-    int ip, iSnippetCol, iStart;
-    memset(aSeen, 0, nPhrase);
-    rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart);
-    if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){
-      int nScore = 1000;
-      int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip);
-      int j;
-      aSeen[ip] = 1;
 
-      for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
-        int ic; int io; int iFinal;
-        rc = pApi->xInst(pFts, j, &ip, &ic, &io);
-        iFinal = io + pApi->xPhraseSize(pFts, ip) - 1;
-        if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){
-          nScore += aSeen[ip] ? 1000 : 1;
-          aSeen[ip] = 1;
-          if( iFinal>iLast ) iLast = iFinal;
+  memset(&sFinder, 0, sizeof(Fts5SFinder));
+  for(i=0; i<nCol; i++){
+    if( iCol<0 || iCol==i ){
+      int nDoc;
+      int nDocsize;
+      int ii;
+      sFinder.iPos = 0;
+      sFinder.nFirst = 0;
+      rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xTokenize(pFts, 
+          sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
+      );
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xColumnSize(pFts, i, &nDocsize);
+      if( rc!=SQLITE_OK ) break;
+
+      for(ii=0; rc==SQLITE_OK && ii<sFinder.nFirst; ii++){
+        int nScore;
+        memset(aSeen, 0, nPhrase);
+        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
+            sFinder.aFirst[ii], nToken, &nScore, 0
+        );
+
+        /* Bonus of 100 points for starting at the start of a sentence */
+        nScore += 100;            
+
+        if( rc==SQLITE_OK && nScore>nBestScore ){
+          nBestScore = nScore;
+          iBestCol = i;
+          iBestStart = sFinder.aFirst[ii];
+          nColSize = nDocsize;
         }
       }
 
-      if( rc==SQLITE_OK && nScore>nBestScore ){
-        iBestCol = iSnippetCol;
-        iBestStart = iStart;
-        iBestLast = iLast;
-        nBestScore = nScore;
+      for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
+        int ip, ic, io;
+        int nScore;
+        rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
+        if( ic!=i || rc!=SQLITE_OK ) continue;
+        memset(aSeen, 0, nPhrase);
+        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
+            io, nToken, &nScore, &io
+        );
+        if( rc==SQLITE_OK && nScore>nBestScore ){
+          nBestScore = nScore;
+          iBestCol = i;
+          iBestStart = io;
+          nColSize = nDocsize;
+        }
       }
     }
   }
 
-  if( rc==SQLITE_OK ){
-    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
-  }
   if( rc==SQLITE_OK ){
     rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
   }
@@ -336,14 +467,6 @@ static void fts5SnippetFunction(
       rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
     }
 
-    if( (iBestStart+nToken-1)>iBestLast ){
-      iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
-    }
-    if( iBestStart+nToken>nColSize ){
-      iBestStart = nColSize - nToken;
-    }
-    if( iBestStart<0 ) iBestStart = 0;
-
     ctx.iRangeStart = iBestStart;
     ctx.iRangeEnd = iBestStart + nToken - 1;
 
@@ -365,15 +488,15 @@ static void fts5SnippetFunction(
     }else{
       fts5HighlightAppend(&rc, &ctx, zEllips, -1);
     }
-
-    if( rc==SQLITE_OK ){
-      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
-    }else{
-      sqlite3_result_error_code(pCtx, rc);
-    }
-    sqlite3_free(ctx.zOut);
   }
+  if( rc==SQLITE_OK ){
+    sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_error_code(pCtx, rc);
+  }
+  sqlite3_free(ctx.zOut);
   sqlite3_free(aSeen);
+  sqlite3_free(sFinder.aFirst);
 }
 
 /************************************************************************/
diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test
index 21854662d3..903dd2910f 100644
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@@ -59,7 +59,6 @@ proc do_snippet_test {tn doc match res} {
 
 
 foreach {tn doc res} {
-
   1.1 {X o o o o o o} {[X] o o o o o o}
   1.2 {o X o o o o o} {o [X] o o o o o}
   1.3 {o o X o o o o} {o o [X] o o o o}
@@ -111,6 +110,10 @@ foreach {tn doc res} {
 
   8.1 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o}
       {...o o [X] [X] [X] o o...}
+  8.2 {o o o o. X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
+      {...[X] o o o o o o...}
+
+
 } {
   do_snippet_test 1.$tn $doc X $res
 }
diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test
index d3ff5128da..9fead57c07 100644
--- a/ext/fts5/test/fts5unicode2.test
+++ b/ext/fts5/test/fts5unicode2.test
@@ -160,11 +160,11 @@ foreach {tn query snippet} {
      the maximum x value.
   }
   4 "rollback" {
-     ...[ROLLBACK]. Instead, the pending statement
+     ...Instead, the pending statement
      will return SQLITE_ABORT upon next access after the [ROLLBACK].
   }
   5 "rOllback" {
-     ...[ROLLBACK]. Instead, the pending statement
+     ...Instead, the pending statement
      will return SQLITE_ABORT upon next access after the [ROLLBACK].
   }
   6 "lang*" {
diff --git a/manifest b/manifest
index 2dccb1db29..4fbf53dc5a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sin\sthe\sfts5\ssnippet()\sauxiliary\sfunction.
-D 2016-08-17T11:14:39.949
+C Bias\sthe\sfts5\ssnippet()\sfunction\sto\sreturn\ssnippets\sthat\slook\slike\sthey\sstart\sat\sthe\sstart\sof\ssentences.
+D 2016-08-17T19:05:12.783
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -99,7 +99,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f
-F ext/fts5/fts5_aux.c e4bec077c5190946dbaac72c6555defd823724ca
+F ext/fts5/fts5_aux.c 5921bbef4c78b86159111335135837a867f1ff8a
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 1ee97156421919e497595bfa962bb88ad1665401
@@ -122,7 +122,7 @@ F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
-F ext/fts5/test/fts5af.test b6afd7c28ad62d546c30f387fb971f3aaebaac0d
+F ext/fts5/test/fts5af.test 38bfc8c3f2fd4687a37ae3a0d4d64e029273df36
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5
 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df
@@ -190,7 +190,7 @@ F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
-F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
+F ext/fts5/test/fts5unicode2.test cb4918278514eeb6f398a28ce71ba52934f50ce5
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f9035b8e2ea331801402bcb62b203ab092949770
-R c6d38a9772924d9b143e2aa32916b131
+P a861713cc6a3868a1c89240e8340bc7b2b9559da
+R ddec9217c48a760254ac22da18301cf5
 U dan
-Z c5e29b567d02bec227e0b49d5b8a188c
+Z 28e0d85dd23359fe591994acb5271e6f
diff --git a/manifest.uuid b/manifest.uuid
index 77bfe841ee..cfefaabd19 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a861713cc6a3868a1c89240e8340bc7b2b9559da
\ No newline at end of file
+60de159476edbd48dc363f7f77f09c32ea68422f
\ No newline at end of file

From b39a5ac22946a1de8b70e48260d02d6bd4598d0c Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 18 Aug 2016 14:47:48 +0000
Subject: [PATCH 2/6] Adjust some tests to account for recent changes to the
 fts5 snippet function.

FossilOrigin-Name: 184ecbe9c0c3280a22cdeda5c7836a7b280c3e1f
---
 ext/fts5/test/fts5af.test       | 76 ++++++++++++++++++---------------
 ext/fts5/test/fts5unicode2.test |  8 ++--
 manifest                        | 14 +++---
 manifest.uuid                   |  2 +-
 4 files changed, 54 insertions(+), 46 deletions(-)

diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test
index 903dd2910f..3b71a9ff5f 100644
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@@ -59,6 +59,7 @@ proc do_snippet_test {tn doc match res} {
 
 
 foreach {tn doc res} {
+
   1.1 {X o o o o o o} {[X] o o o o o o}
   1.2 {o X o o o o o} {o [X] o o o o o}
   1.3 {o o X o o o o} {o o [X] o o o o}
@@ -71,49 +72,56 @@ foreach {tn doc res} {
   2.2 {o X o o o o o o} {o [X] o o o o o...}
   2.3 {o o X o o o o o} {o o [X] o o o o...}
   2.4 {o o o X o o o o} {o o o [X] o o o...}
-  2.5 {o o o o X o o o} {...o o o [X] o o o}
-  2.6 {o o o o o X o o} {...o o o o [X] o o}
-  2.7 {o o o o o o X o} {...o o o o o [X] o}
+  2.5 {o o o o X o o o} {o o o o [X] o o...}
+  2.6 {o o o o o X o o} {o o o o o [X] o...}
+  2.7 {o o o o o o X o} {o o o o o o [X]...}
   2.8 {o o o o o o o X} {...o o o o o o [X]}
 
+  2.9  {o o o o o o o X o}       {...o o o o o [X] o}
+  2.10 {o o o o o o o X o o}     {...o o o o [X] o o}
+  2.11 {o o o o o o o X o o o}   {...o o o [X] o o o}
+  2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...}
+
+
   3.1 {X o o o o o o o o} {[X] o o o o o o...}
   3.2 {o X o o o o o o o} {o [X] o o o o o...}
   3.3 {o o X o o o o o o} {o o [X] o o o o...}
   3.4 {o o o X o o o o o} {o o o [X] o o o...}
-  3.5 {o o o o X o o o o} {...o o o [X] o o o...}
-  3.6 {o o o o o X o o o} {...o o o [X] o o o}
-  3.7 {o o o o o o X o o} {...o o o o [X] o o}
-  3.8 {o o o o o o o X o} {...o o o o o [X] o}
-  3.9 {o o o o o o o o X} {...o o o o o o [X]}
+
+  3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...}
+  3.6 {o o o o o o o o X o o o} {...o o o [X] o o o}
+  3.7 {o o o o o o o o o X o o} {...o o o o [X] o o}
+  3.8 {o o o o o o o o o o X o} {...o o o o o [X] o}
+  3.9 {o o o o o o o o o o o X} {...o o o o o o [X]}
 
   4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
-  4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
-  4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
+  4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...}
+  4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]}
 
   5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
-  5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
-  5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
-  5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
+  5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...}
+  5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o}
+  5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]}
 
   6.1 {X o o o X o o o} {[X] o o o [X] o o...}
   6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
-  6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
-  6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
-  6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
+  6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...}
+  6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o}
+  6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]}
 
   7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
   7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
-  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
-  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
-  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
-  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
+  7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...}
+  7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o}
+  7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o}
+  7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]}
 
-  8.1 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o}
+  8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o}
       {...o o [X] [X] [X] o o...}
-  8.2 {o o o o. X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
-      {...[X] o o o o o o...}
-
-
+  8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} 
+      {...o o [X] o o o o...}
+  8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
+      {o o o o [X] o o...}
 } {
   do_snippet_test 1.$tn $doc X $res
 }
@@ -130,19 +138,19 @@ if {[detail_is_full]} {
     2.1 {X Y o o o o o o} {[X Y] o o o o o...}
     2.2 {o X Y o o o o o} {o [X Y] o o o o...}
     2.3 {o o X Y o o o o} {o o [X Y] o o o...}
-    2.4 {o o o X Y o o o} {...o o [X Y] o o o}
-    2.5 {o o o o X Y o o} {...o o o [X Y] o o}
-    2.6 {o o o o o X Y o} {...o o o o [X Y] o}
-    2.7 {o o o o o o X Y} {...o o o o o [X Y]}
+    2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o}
+    2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o}
+    2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o}
+    2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]}
 
     3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
     3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
     3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
-    3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
-    3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
-    3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
-    3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
-    3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
+    3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...}
+    3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o}
+    3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o}
+    3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o}
+    3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]}
   } {
     do_snippet_test 2.$tn $doc "X + Y" $res
   }
diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test
index 9fead57c07..8e5bb8e22b 100644
--- a/ext/fts5/test/fts5unicode2.test
+++ b/ext/fts5/test/fts5unicode2.test
@@ -160,12 +160,12 @@ foreach {tn query snippet} {
      the maximum x value.
   }
   4 "rollback" {
-     ...Instead, the pending statement
-     will return SQLITE_ABORT upon next access after the [ROLLBACK].
+     Pending statements no longer block [ROLLBACK]. Instead, the pending
+     statement will return SQLITE_ABORT upon...
   }
   5 "rOllback" {
-     ...Instead, the pending statement
-     will return SQLITE_ABORT upon next access after the [ROLLBACK].
+     Pending statements no longer block [ROLLBACK]. Instead, the pending
+     statement will return SQLITE_ABORT upon...
   }
   6 "lang*" {
      Added support for the FTS4 [languageid] option.
diff --git a/manifest b/manifest
index 4fbf53dc5a..9749b2bf53 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Bias\sthe\sfts5\ssnippet()\sfunction\sto\sreturn\ssnippets\sthat\slook\slike\sthey\sstart\sat\sthe\sstart\sof\ssentences.
-D 2016-08-17T19:05:12.783
+C Adjust\ssome\stests\sto\saccount\sfor\srecent\schanges\sto\sthe\sfts5\ssnippet\sfunction.
+D 2016-08-18T14:47:48.448
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -122,7 +122,7 @@ F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
-F ext/fts5/test/fts5af.test 38bfc8c3f2fd4687a37ae3a0d4d64e029273df36
+F ext/fts5/test/fts5af.test b601bbab5a35ca15dd3d1069ee7075a859567073
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5
 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df
@@ -190,7 +190,7 @@ F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
-F ext/fts5/test/fts5unicode2.test cb4918278514eeb6f398a28ce71ba52934f50ce5
+F ext/fts5/test/fts5unicode2.test 529ac7e8648c943bc87bfed1e427128a2f3f9e33
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a861713cc6a3868a1c89240e8340bc7b2b9559da
-R ddec9217c48a760254ac22da18301cf5
+P 60de159476edbd48dc363f7f77f09c32ea68422f
+R 8546174dc7a44b2b80755efe3cb03b8a
 U dan
-Z 28e0d85dd23359fe591994acb5271e6f
+Z b3f4b9f04ef4ae0622fcf2d28707aee5
diff --git a/manifest.uuid b/manifest.uuid
index cfefaabd19..eb9043217f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-60de159476edbd48dc363f7f77f09c32ea68422f
\ No newline at end of file
+184ecbe9c0c3280a22cdeda5c7836a7b280c3e1f
\ No newline at end of file

From 7ca1ea16ff99968db8994405c2d1c542e9a9d22f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 19 Aug 2016 14:25:38 +0000
Subject: [PATCH 3/6] Have the fts5 snippet() function avoid favouring snippets
 that begin with sentences that do not contain search terms. Add an extra bias
 in favour of the first sentence in the document.

FossilOrigin-Name: d8180af2adfc101dfce95a0498b5cd7b30643b30
---
 ext/fts5/fts5_aux.c | 47 ++++++++++++++++++++++++++-------------------
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 34 insertions(+), 27 deletions(-)

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index ca55fd6b01..8efe05968e 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -422,39 +422,46 @@ static void fts5SnippetFunction(
       rc = pApi->xColumnSize(pFts, i, &nDocsize);
       if( rc!=SQLITE_OK ) break;
 
-      for(ii=0; rc==SQLITE_OK && ii<sFinder.nFirst; ii++){
-        int nScore;
-        memset(aSeen, 0, nPhrase);
-        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
-            sFinder.aFirst[ii], nToken, &nScore, 0
-        );
-
-        /* Bonus of 100 points for starting at the start of a sentence */
-        nScore += 100;            
-
-        if( rc==SQLITE_OK && nScore>nBestScore ){
-          nBestScore = nScore;
-          iBestCol = i;
-          iBestStart = sFinder.aFirst[ii];
-          nColSize = nDocsize;
-        }
-      }
-
       for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
         int ip, ic, io;
+        int iAdj;
         int nScore;
+        int jj;
+
         rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
         if( ic!=i || rc!=SQLITE_OK ) continue;
         memset(aSeen, 0, nPhrase);
         rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
-            io, nToken, &nScore, &io
+            io, nToken, &nScore, &iAdj
         );
         if( rc==SQLITE_OK && nScore>nBestScore ){
           nBestScore = nScore;
           iBestCol = i;
-          iBestStart = io;
+          iBestStart = iAdj;
           nColSize = nDocsize;
         }
+
+        if( rc==SQLITE_OK && sFinder.nFirst ){
+          for(jj=0; jj<(sFinder.nFirst-1); jj++){
+            if( sFinder.aFirst[jj+1]>io ) break;
+          }
+
+          if( sFinder.aFirst[jj]<io ){
+            int nScore;
+            memset(aSeen, 0, nPhrase);
+            rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
+              sFinder.aFirst[jj], nToken, &nScore, 0
+            );
+
+            nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
+            if( rc==SQLITE_OK && nScore>nBestScore ){
+              nBestScore = nScore;
+              iBestCol = i;
+              iBestStart = sFinder.aFirst[jj];
+              nColSize = nDocsize;
+            }
+          }
+        }
       }
     }
   }
diff --git a/manifest b/manifest
index 9749b2bf53..f261342285 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Adjust\ssome\stests\sto\saccount\sfor\srecent\schanges\sto\sthe\sfts5\ssnippet\sfunction.
-D 2016-08-18T14:47:48.448
+C Have\sthe\sfts5\ssnippet()\sfunction\savoid\sfavouring\ssnippets\sthat\sbegin\swith\ssentences\sthat\sdo\snot\scontain\ssearch\sterms.\sAdd\san\sextra\sbias\sin\sfavour\sof\sthe\sfirst\ssentence\sin\sthe\sdocument.
+D 2016-08-19T14:25:38.928
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -99,7 +99,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f
-F ext/fts5/fts5_aux.c 5921bbef4c78b86159111335135837a867f1ff8a
+F ext/fts5/fts5_aux.c 26a2b7e5f02cdc6b7322b85f473e488de452a5be
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 1ee97156421919e497595bfa962bb88ad1665401
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 60de159476edbd48dc363f7f77f09c32ea68422f
-R 8546174dc7a44b2b80755efe3cb03b8a
+P 184ecbe9c0c3280a22cdeda5c7836a7b280c3e1f
+R 1ef5a5b8d26eab21fbb58bf0464e61ed
 U dan
-Z b3f4b9f04ef4ae0622fcf2d28707aee5
+Z ed743fa5b6cead3848c470a1b5a47844
diff --git a/manifest.uuid b/manifest.uuid
index eb9043217f..c3129c10b9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-184ecbe9c0c3280a22cdeda5c7836a7b280c3e1f
\ No newline at end of file
+d8180af2adfc101dfce95a0498b5cd7b30643b30
\ No newline at end of file

From c9e75fb23abf0a7ba620bbc886efa63d9fefd8b0 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 19 Aug 2016 18:37:35 +0000
Subject: [PATCH 4/6] Register any built-in fts5 module before loading
 automatic extensions. This allows automatic extensions to register fts5
 tokenizers and auxiliary functions.

FossilOrigin-Name: b10e31dce8e590e1ba30f512374a8c16656bcc72
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/main.c    | 17 ++++++++++-------
 3 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index f261342285..a68dc1c021 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\sthe\sfts5\ssnippet()\sfunction\savoid\sfavouring\ssnippets\sthat\sbegin\swith\ssentences\sthat\sdo\snot\scontain\ssearch\sterms.\sAdd\san\sextra\sbias\sin\sfavour\sof\sthe\sfirst\ssentence\sin\sthe\sdocument.
-D 2016-08-19T14:25:38.928
+C Register\sany\sbuilt-in\sfts5\smodule\sbefore\sloading\sautomatic\sextensions.\sThis\sallows\sautomatic\sextensions\sto\sregister\sfts5\stokenizers\sand\sauxiliary\sfunctions.
+D 2016-08-19T18:37:35.386
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -349,7 +349,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/insert.c 8f4e9fcbd8e95e85f15647ba8b413b18d556ec2b
 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e
 F src/loadext.c dd7a2b77902cc66c22555aef02e1a682554b7aec
-F src/main.c 16c1b2114eae8804caf3a8de8cb47bf2c6d83ad3
+F src/main.c 4f29b8d1b705fc3b73b26f892e85df2bfca31cff
 F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 184ecbe9c0c3280a22cdeda5c7836a7b280c3e1f
-R 1ef5a5b8d26eab21fbb58bf0464e61ed
+P d8180af2adfc101dfce95a0498b5cd7b30643b30
+R 60073adaf70faac8deb27fa21c3071ec
 U dan
-Z ed743fa5b6cead3848c470a1b5a47844
+Z db6d5df26cfba25421660b26226c4ab6
diff --git a/manifest.uuid b/manifest.uuid
index c3129c10b9..674031237c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d8180af2adfc101dfce95a0498b5cd7b30643b30
\ No newline at end of file
+b10e31dce8e590e1ba30f512374a8c16656bcc72
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index f31f865818..4e4836238b 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2948,11 +2948,20 @@ static int openDatabase(
   */
   sqlite3Error(db, SQLITE_OK);
   sqlite3RegisterPerConnectionBuiltinFunctions(db);
+  rc = sqlite3_errcode(db);
+
+#ifdef SQLITE_ENABLE_FTS5
+  /* Register any built-in FTS5 module before loading the automatic
+  ** extensions. This allows automatic extensions to register FTS5 
+  ** tokenizers and auxiliary functions.  */
+  if( !db->mallocFailed && rc==SQLITE_OK ){
+    rc = sqlite3Fts5Init(db);
+  }
+#endif
 
   /* Load automatic extensions - extensions that have been registered
   ** using the sqlite3_automatic_extension() API.
   */
-  rc = sqlite3_errcode(db);
   if( rc==SQLITE_OK ){
     sqlite3AutoLoadExtensions(db);
     rc = sqlite3_errcode(db);
@@ -2981,12 +2990,6 @@ static int openDatabase(
   }
 #endif
 
-#ifdef SQLITE_ENABLE_FTS5
-  if( !db->mallocFailed && rc==SQLITE_OK ){
-    rc = sqlite3Fts5Init(db);
-  }
-#endif
-
 #ifdef SQLITE_ENABLE_ICU
   if( !db->mallocFailed && rc==SQLITE_OK ){
     rc = sqlite3IcuInit(db);

From 783e778f3865bb86a5dfcd1bc830f746ed2705bb Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 20 Aug 2016 17:23:45 +0000
Subject: [PATCH 5/6] Fix other minor problems with the fts5 snippet()
 function.

FossilOrigin-Name: b56b2a85cf377686727f3929208ede43f797cf76
---
 ext/fts5/fts5_aux.c       |  5 ++++-
 ext/fts5/test/fts5af.test |  8 ++++++++
 manifest                  | 14 +++++++-------
 manifest.uuid             |  2 +-
 4 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index 8efe05968e..f2aacbff20 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -441,7 +441,7 @@ static void fts5SnippetFunction(
           nColSize = nDocsize;
         }
 
-        if( rc==SQLITE_OK && sFinder.nFirst ){
+        if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){
           for(jj=0; jj<(sFinder.nFirst-1); jj++){
             if( sFinder.aFirst[jj+1]>io ) break;
           }
@@ -469,6 +469,9 @@ static void fts5SnippetFunction(
   if( rc==SQLITE_OK ){
     rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
   }
+  if( rc==SQLITE_OK && nColSize==0 ){
+    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
+  }
   if( ctx.zIn ){
     if( rc==SQLITE_OK ){
       rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test
index 3b71a9ff5f..c8f815dfda 100644
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@@ -156,6 +156,14 @@ if {[detail_is_full]} {
   }
 }
 
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+  INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13');
+  SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz');
+} {
+  {1 2 3 4 5...}
+}
+
 } ;# foreach_detail_mode 
 
 finish_test
diff --git a/manifest b/manifest
index a68dc1c021..9d9fddff14 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Register\sany\sbuilt-in\sfts5\smodule\sbefore\sloading\sautomatic\sextensions.\sThis\sallows\sautomatic\sextensions\sto\sregister\sfts5\stokenizers\sand\sauxiliary\sfunctions.
-D 2016-08-19T18:37:35.386
+C Fix\sother\sminor\sproblems\swith\sthe\sfts5\ssnippet()\sfunction.
+D 2016-08-20T17:23:45.034
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -99,7 +99,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f
-F ext/fts5/fts5_aux.c 26a2b7e5f02cdc6b7322b85f473e488de452a5be
+F ext/fts5/fts5_aux.c 13246759ce03614a3b214894fa4b0f76891e97c0
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 1ee97156421919e497595bfa962bb88ad1665401
@@ -122,7 +122,7 @@ F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
-F ext/fts5/test/fts5af.test b601bbab5a35ca15dd3d1069ee7075a859567073
+F ext/fts5/test/fts5af.test b21abd9455c7351a9ef872b650d16f9dd8db2c49
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5
 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d8180af2adfc101dfce95a0498b5cd7b30643b30
-R 60073adaf70faac8deb27fa21c3071ec
+P b10e31dce8e590e1ba30f512374a8c16656bcc72
+R adacbf86d42eaec5a7639c5010230497
 U dan
-Z db6d5df26cfba25421660b26226c4ab6
+Z 520e1a56589636baad586098a109d286
diff --git a/manifest.uuid b/manifest.uuid
index 674031237c..eda24ca294 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b10e31dce8e590e1ba30f512374a8c16656bcc72
\ No newline at end of file
+b56b2a85cf377686727f3929208ede43f797cf76
\ No newline at end of file

From 37ff4d4c77662a01c7b1902486827426317142dd Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 24 Aug 2016 18:50:47 +0000
Subject: [PATCH 6/6] Fix a bug in the fts5 snippet function causing it to
 return text containing zero phrase instances.

FossilOrigin-Name: b174ed2bde43a1872374ce18b85ac9f067e86fbb
---
 ext/fts5/fts5_aux.c       |  4 ++--
 ext/fts5/test/fts5af.test | 11 +++++++++++
 manifest                  | 14 +++++++-------
 manifest.uuid             |  2 +-
 4 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index f2aacbff20..e171233050 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -331,6 +331,7 @@ static int fts5SnippetScore(
   int iFirst = -1;
   int nInst;
   int nScore = 0;
+  int iLast = 0;
 
   rc = pApi->xInstCount(pFts, &nInst);
   for(i=0; i<nInst && rc==SQLITE_OK; i++){
@@ -339,14 +340,13 @@ static int fts5SnippetScore(
       nScore += (aSeen[ip] ? 1 : 1000);
       aSeen[ip] = 1;
       if( iFirst<0 ) iFirst = iOff;
+      iLast = iOff + pApi->xPhraseSize(pFts, ip);
     }
   }
 
   *pnScore = nScore;
   if( piPos ){
-    int iLast = iOff + pApi->xPhraseSize(pFts, ip);
     int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
-
     if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
     if( iAdj<0 ) iAdj = 0;
     *piPos = iAdj;
diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test
index c8f815dfda..6aab55a557 100644
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@@ -164,6 +164,17 @@ do_execsql_test 4.0 {
   {1 2 3 4 5...}
 }
 
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE p1 USING fts5(a, b);
+  INSERT INTO p1 VALUES(
+    'x a a a a a a a a a a',
+    'a a a a a a a a a a a a a a a a a a a x'
+  );
+}
+do_execsql_test 5.1 {
+  SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
+} {{[x] a a a a a...}}
+
 } ;# foreach_detail_mode 
 
 finish_test
diff --git a/manifest b/manifest
index 9d9fddff14..6ce61f548e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sother\sminor\sproblems\swith\sthe\sfts5\ssnippet()\sfunction.
-D 2016-08-20T17:23:45.034
+C Fix\sa\sbug\sin\sthe\sfts5\ssnippet\sfunction\scausing\sit\sto\sreturn\stext\scontaining\szero\sphrase\sinstances.
+D 2016-08-24T18:50:47.193
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -99,7 +99,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f
-F ext/fts5/fts5_aux.c 13246759ce03614a3b214894fa4b0f76891e97c0
+F ext/fts5/fts5_aux.c 2f20784a344701d4c72986e2e692062dd47d568c
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 1ee97156421919e497595bfa962bb88ad1665401
@@ -122,7 +122,7 @@ F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
-F ext/fts5/test/fts5af.test b21abd9455c7351a9ef872b650d16f9dd8db2c49
+F ext/fts5/test/fts5af.test c92825778ed2adc80014832762c056bbe968ef88
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5
 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b10e31dce8e590e1ba30f512374a8c16656bcc72
-R adacbf86d42eaec5a7639c5010230497
+P b56b2a85cf377686727f3929208ede43f797cf76
+R d95861b85eb0fcd438ce646b06ec5daf
 U dan
-Z 520e1a56589636baad586098a109d286
+Z 1e5d85bd62c3d7d96772d546309a0d7d
diff --git a/manifest.uuid b/manifest.uuid
index eda24ca294..3fabbae740 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b56b2a85cf377686727f3929208ede43f797cf76
\ No newline at end of file
+b174ed2bde43a1872374ce18b85ac9f067e86fbb
\ No newline at end of file