database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-12-24 14:17:58 +03:00
Code Activity

Remove the "#include sqlite3Int.h" from fts5Int.h.

Browse Source 
FossilOrigin-Name: e008c3c8e29c843ec945ddad54b9688bbf2bdb44
This commit is contained in:
dan
2015-05-30 11:49:58 +00:00
parent a8c024905c
commit 3f3074e0c1
13 changed files with 457 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ext/fts5/fts5.c
View File

@@ -601,7 +601,7 @@ static int fts5SorterNext(Fts5Cursor *pCsr){
for(i=0; i<(pSorter->nIdx-1); i++){
int iVal;
a += getVarint32(a, iVal);
a += fts5GetVarint32(a, iVal);
iOff += iVal;
pSorter->aIdx[i] = iOff;
}

43
ext/fts5/fts5Int.h
View File

@@ -15,7 +15,27 @@
#define _FTS5INT_H
#include "fts5.h"
#include "sqliteInt.h"
#include <string.h>
#include <assert.h>
#ifndef SQLITE_AMALGAMATION
typedef unsigned char u8;
typedef unsigned int u32;
typedef unsigned short u16;
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;
#define ArraySize(x) (sizeof(x) / sizeof(x[0]))
#define testcase(x)
#define ALWAYS(x) 1
#define NEVER(x) 0
#define MIN(x,y) (((x) < (y)) ? (x) : (y))
#endif
/*
@@ -377,15 +397,26 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
int sqlite3Fts5GetVarintLen(u32 iVal);
/*
** End of interface to code in fts5_index.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_varint.c.
*/
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
int sqlite3Fts5GetVarintLen(u32 iVal);
u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
#define fts5GetVarint sqlite3Fts5GetVarint
/*
** End of interface to code in fts5_varint.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5.c.
*/

8
ext/fts5/fts5_buffer.c
View File

@@ -44,7 +44,7 @@ int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
*/
void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
if( sqlite3Fts5BufferGrow(pRc, pBuf, 9) ) return;
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iVal);
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal);
}
void sqlite3Fts5Put32(u8 *aBuf, int iVal){
@@ -169,11 +169,11 @@ int sqlite3Fts5PoslistNext64(
}else{
i64 iOff = *piOff;
int iVal;
i += getVarint32(&a[i], iVal);
i += fts5GetVarint32(&a[i], iVal);
if( iVal==1 ){
i += getVarint32(&a[i], iVal);
i += fts5GetVarint32(&a[i], iVal);
iOff = ((i64)iVal) << 32;
i += getVarint32(&a[i], iVal);
i += fts5GetVarint32(&a[i], iVal);
}
*piOff = iOff + (iVal-2);
*pi = i;

2
ext/fts5/fts5_expr.c
View File

@@ -708,7 +708,7 @@ static int fts5ExprExtractCol(
if( p==pEnd ) return 0;
}
*pa = p++;
p += getVarint32(p, iCurrent);
p += fts5GetVarint32(p, iCurrent);
}
/* Advance pointer p until it points to pEnd or an 0x01 byte that is

10
ext/fts5/fts5_hash.c
View File

@@ -188,7 +188,7 @@ static void fts5HashAddPoslistSize(Fts5HashEntry *p){
}else{
int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
sqlite3PutVarint(&pPtr[p->iSzPoslist], nPos);
sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
p->nData += (nByte-1);
}
p->bDel = 0;
@@ -239,7 +239,7 @@ int sqlite3Fts5HashWrite(
assert( iHash==fts5HashKey(pHash->nSlot, p->zKey, nToken+1) );
p->zKey[nToken+1] = '\0';
p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
p->nData += sqlite3PutVarint(&((u8*)p)[p->nData], iRowid);
p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
p->iSzPoslist = p->nData;
p->nData += 1;
p->iRowid = iRowid;
@@ -276,7 +276,7 @@ int sqlite3Fts5HashWrite(
** entry, and the new rowid for this entry. */
if( iRowid!=p->iRowid ){
fts5HashAddPoslistSize(p);
p->nData += sqlite3PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
p->iSzPoslist = p->nData;
p->nData += 1;
p->iCol = 0;
@@ -289,13 +289,13 @@ int sqlite3Fts5HashWrite(
assert( iCol>=p->iCol );
if( iCol!=p->iCol ){
pPtr[p->nData++] = 0x01;
p->nData += sqlite3PutVarint(&pPtr[p->nData], iCol);
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
p->iCol = iCol;
p->iPos = 0;
}
/* Append the new position offset */
p->nData += sqlite3PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
p->iPos = iPos;
}else{
/* This is a delete. Set the delete flag. */

138
ext/fts5/fts5_index.c
View File

@@ -612,81 +612,7 @@ static void fts5PutU16(u8 *aOut, u16 iVal){
static u16 fts5GetU16(const u8 *aIn){
return ((u16)aIn[0] << 8) + aIn[1];
}
/*
** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
** Except, this version does handle the single byte case that the core
** version depends on being handled before its function is called.
*/
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){
u32 a,b;
/* The 1-byte case. Overwhelmingly the most common. */
a = *p;
/* a: p0 (unmasked) */
if (!(a&0x80))
{
/* Values between 0 and 127 */
*v = a;
return 1;
}
/* The 2-byte case */
p++;
b = *p;
/* b: p1 (unmasked) */
if (!(b&0x80))
{
/* Values between 128 and 16383 */
a &= 0x7f;
a = a<<7;
*v = a | b;
return 2;
}
/* The 3-byte case */
p++;
a = a<<14;
a |= *p;
/* a: p0<<14 | p2 (unmasked) */
if (!(a&0x80))
{
/* Values between 16384 and 2097151 */
a &= (0x7f<<14)|(0x7f);
b &= 0x7f;
b = b<<7;
*v = a | b;
return 3;
}
/* A 32-bit varint is used to store size information in btrees.
** Objects are rarely larger than 2MiB limit of a 3-byte varint.
** A 3-byte varint is sufficient, for example, to record the size
** of a 1048569-byte BLOB or string.
**
** We only unroll the first 1-, 2-, and 3- byte cases. The very
** rare larger cases can be handled by the slower 64-bit varint
** routine.
*/
{
u64 v64;
u8 n;
p -= 2;
n = sqlite3GetVarint(p, &v64);
*v = (u32)v64;
assert( n>3 && n<=9 );
return n;
}
}
int sqlite3Fts5GetVarintLen(u32 iVal){
if( iVal<(1 << 7 ) ) return 1;
if( iVal<(1 << 14) ) return 2;
if( iVal<(1 << 21) ) return 3;
if( iVal<(1 << 28) ) return 4;
return 5;
}
}
/*
** Allocate and return a buffer at least nByte bytes in size.
@@ -986,7 +912,7 @@ static int fts5StructureDecode(
if( pRet ){
pRet->nLevel = nLevel;
pRet->nSegment = nSegment;
i += sqlite3GetVarint(&pData[i], &pRet->nWriteCounter);
i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter);
for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){
Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl];
@@ -1350,7 +1276,7 @@ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){
assert( pLvl->bEof==0 );
pLvl->iOff = 1;
pLvl->iOff += fts5GetVarint32(&pData->p[1], pLvl->iLeafPgno);
pLvl->iOff += getVarint(&pData->p[pLvl->iOff], (u64*)&pLvl->iRowid);
pLvl->iOff += fts5GetVarint(&pData->p[pLvl->iOff], (u64*)&pLvl->iRowid);
pLvl->iFirstOff = pLvl->iOff;
}else{
int iOff;
@@ -1361,7 +1287,7 @@ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){
if( iOff<pData->n ){
i64 iVal;
pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1;
iOff += getVarint(&pData->p[iOff], (u64*)&iVal);
iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal);
pLvl->iRowid += iVal;
pLvl->iOff = iOff;
}else{
@@ -1468,7 +1394,7 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){
if( (a[iOff-1] & 0x80)==0 ) break;
}
getVarint(&a[iOff], (u64*)&iVal);
fts5GetVarint(&a[iOff], (u64*)&iVal);
pLvl->iRowid -= iVal;
pLvl->iLeafPgno--;
@@ -1690,7 +1616,7 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
iOff = 4;
a = pIter->pLeaf->p;
}
iOff += sqlite3GetVarint(&a[iOff], (u64*)&pIter->iRowid);
iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
pIter->iLeafOffset = iOff;
}
@@ -1761,7 +1687,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){
i += fts5GetPoslistSize(&a[i], &nPos, &bDummy);
i += nPos;
if( i>=n ) break;
i += getVarint(&a[i], (u64*)&iDelta);
i += fts5GetVarint(&a[i], (u64*)&iDelta);
if( iDelta==0 ) break;
pIter->iRowid += iDelta;
@@ -1815,7 +1741,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
if( pIter->pLeaf ){
u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset];
pIter->iLeafOffset += getVarint(a, (u64*)&pIter->iRowid);
pIter->iLeafOffset += fts5GetVarint(a, (u64*)&pIter->iRowid);
break;
}else{
fts5DataRelease(pNew);
@@ -1865,7 +1791,7 @@ static void fts5SegIterNext(
pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
iOff += nPos;
getVarint(&a[iOff], (u64*)&iDelta);
fts5GetVarint(&a[iOff], (u64*)&iDelta);
pIter->iRowid -= iDelta;
fts5SegIterLoadNPos(p, pIter);
}else{
@@ -1886,7 +1812,7 @@ static void fts5SegIterNext(
if( iOff<n ){
/* The next entry is on the current page */
u64 iDelta;
iOff += sqlite3GetVarint(&a[iOff], &iDelta);
iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta);
pIter->iLeafOffset = iOff;
if( iDelta==0 ){
bNewTerm = 1;
@@ -1914,7 +1840,7 @@ static void fts5SegIterNext(
pIter->pLeaf->p = (u8*)pList;
pIter->pLeaf->n = nList;
sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm);
pIter->iLeafOffset = getVarint(pList, (u64*)&pIter->iRowid);
pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);
}
}else{
iOff = 0;
@@ -1924,7 +1850,7 @@ static void fts5SegIterNext(
pLeaf = pIter->pLeaf;
if( pLeaf==0 ) break;
if( (iOff = fts5GetU16(&pLeaf->p[0])) ){
iOff += sqlite3GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
pIter->iLeafOffset = iOff;
}
else if( (iOff = fts5GetU16(&pLeaf->p[2])) ){
@@ -1993,7 +1919,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
if( iOff>=pLeaf->n ) break;
/* Rowid delta. Or, if 0x00, the end of doclist marker. */
nPos = getVarint(&pLeaf->p[iOff], (u64*)&iDelta);
nPos = fts5GetVarint(&pLeaf->p[iOff], (u64*)&iDelta);
if( iDelta==0 ) break;
iOff += nPos;
}
@@ -2040,7 +1966,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
pIter->pLeaf = pLast;
pIter->iLeafPgno = pgnoLast;
fts5LeafHeader(pLast, &iOff, &dummy);
iOff += getVarint(&pLast->p[iOff], (u64*)&pIter->iRowid);
iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid);
pIter->iLeafOffset = iOff;
}
@@ -2072,7 +1998,7 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){
i64 iDelta;
/* iOff is currently the offset of the start of position list data */
iOff += getVarint(&pLeaf->p[iOff], (u64*)&iDelta);
iOff += fts5GetVarint(&pLeaf->p[iOff], (u64*)&iDelta);
if( iDelta==0 ) return;
assert_nc( iOff<pLeaf->n );
iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy);
@@ -2214,7 +2140,7 @@ static void fts5SegIterHashInit(
pLeaf->p = (u8*)pList;
pLeaf->n = nList;
pIter->pLeaf = pLeaf;
pIter->iLeafOffset = getVarint(pLeaf->p, (u64*)&pIter->iRowid);
pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid);
if( flags & FTS5INDEX_QUERY_DESC ){
pIter->flags |= FTS5_SEGITER_REVERSE;
@@ -2397,7 +2323,7 @@ static void fts5SegIterGotoPage(
if( iOff<4 || iOff>=n ){
p->rc = FTS5_CORRUPT;
}else{
iOff += getVarint(&a[iOff], (u64*)&pIter->iRowid);
iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
pIter->iLeafOffset = iOff;
fts5SegIterLoadNPos(p, pIter);
}
@@ -2692,7 +2618,7 @@ static void fts5MultiIterNew2(
pIter->flags = FTS5_SEGITER_ONETERM;
if( pData->n>0 ){
pIter->pLeaf = pData;
pIter->iLeafOffset = getVarint(pData->p, (u64*)&pIter->iRowid);
pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid);
pNew->aFirst[1].iFirst = 1;
if( bDesc ){
pNew->bRev = 1;
@@ -3024,8 +2950,8 @@ static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){
i64 iRowid;
int iOff;
iOff = 1 + getVarint(&pBuf->p[1], (u64*)&iRowid);
getVarint(&pBuf->p[iOff], (u64*)&iRowid);
iOff = 1 + fts5GetVarint(&pBuf->p[1], (u64*)&iRowid);
fts5GetVarint(&pBuf->p[iOff], (u64*)&iRowid);
return iRowid;
}
@@ -3253,7 +3179,7 @@ static void fts5WriteAppendPoslistData(
int nCopy = 0;
while( nCopy<nReq ){
i64 dummy;
nCopy += getVarint(&a[nCopy], (u64*)&dummy);
nCopy += fts5GetVarint(&a[nCopy], (u64*)&dummy);
}
fts5BufferAppendBlob(&p->rc, &pPage->buf, nCopy, a);
a += nCopy;
@@ -3759,7 +3685,7 @@ static void fts5FlushOneHash(Fts5Index *p){
** as well. */
if( writer.bFirstTermInPage==0 ){
int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm);
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nPre);
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nPre);
nSuffix = nTerm - nPre;
}else{
fts5PutU16(&pBuf->p[2], pBuf->n);
@@ -3772,7 +3698,7 @@ static void fts5FlushOneHash(Fts5Index *p){
}
nSuffix = nTerm;
}
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nSuffix);
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nSuffix);
fts5BufferSafeAppendBlob(pBuf, (const u8*)&zTerm[nTerm-nSuffix], nSuffix);
/* We just wrote a term into page writer.aWriter[0].pgno. If a
@@ -3798,18 +3724,18 @@ static void fts5FlushOneHash(Fts5Index *p){
int nPos;
int nCopy;
int bDummy;
iOff += getVarint(&pDoclist[iOff], (u64*)&iDelta);
iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta);
nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
nCopy += nPos;
iRowid += iDelta;
if( writer.bFirstRowidInPage ){
fts5PutU16(&pBuf->p[0], pBuf->n); /* first docid on page */
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iRowid);
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
writer.bFirstRowidInPage = 0;
fts5WriteDlidxAppend(p, &writer, iRowid);
}else{
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iDelta);
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta);
}
assert( pBuf->n<=pBuf->nSpace );
@@ -4012,10 +3938,10 @@ static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
int bDummy;
if( pIter->i ){
i64 iDelta;
pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&iDelta);
pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&iDelta);
pIter->iRowid += iDelta;
}else{
pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
}
pIter->i += fts5GetPoslistSize(
&pIter->a[pIter->i], &pIter->nPoslist, &bDummy
@@ -4866,7 +4792,7 @@ static void fts5TestTerm(
}else
if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){
u32 cksum3 = *pCksum;
const char *zTerm = &pPrev->p[1]; /* The term without the prefix-byte */
const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */
int nTerm = pPrev->n-1; /* Size of zTerm in bytes */
int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX);
int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX);
@@ -5006,7 +4932,7 @@ static void fts5IndexIntegrityCheckSegment(
if( pLeaf ){
i64 iRowid;
int iRowidOff = fts5GetU16(&pLeaf->p[0]);
getVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT;
fts5DataRelease(pLeaf);
}
@@ -5266,7 +5192,7 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
i64 iDocid;
int iOff = 0;
iOff = sqlite3GetVarint(&a[iOff], (u64*)&iDocid);
iOff = sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDocid);
sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
while( iOff<n ){
int nPos;
@@ -5275,7 +5201,7 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
if( iOff<n ){
i64 iDelta;
iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDelta);
iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta);
if( iDelta==0 ) return iOff;
iDocid += iDelta;
sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);

14
ext/fts5/fts5_storage.c
View File

@@ -233,14 +233,18 @@ int sqlite3Fts5StorageOpen(
if( bCreate ){
if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
int nDefn = 32 + pConfig->nCol*10;
char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
if( zDefn==0 ){
rc = SQLITE_NOMEM;
}else{
int i;
int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
int iOff;
sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY");
iOff = strlen(zDefn);
for(i=0; i<pConfig->nCol; i++){
iOff += sprintf(&zDefn[iOff], ", c%d", i);
sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
iOff += strlen(&zDefn[iOff]);
}
rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
}
@@ -395,9 +399,9 @@ static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){
if( rc==SQLITE_OK && buf.n ){
int i = 0;
int iCol;
i += getVarint(&buf.p[i], (u64*)&p->nTotalRow);
i += fts5GetVarint(&buf.p[i], (u64*)&p->nTotalRow);
for(iCol=0; i<buf.n && iCol<nCol; iCol++){
i += getVarint(&buf.p[i], (u64*)&p->aTotalSize[iCol]);
i += fts5GetVarint(&buf.p[i], (u64*)&p->aTotalSize[iCol]);
}
}
sqlite3_free(buf.p);
@@ -907,7 +911,7 @@ static int fts5StorageDecodeSizeArray(
int iOff = 0;
for(i=0; i<nCol; i++){
if( iOff>=nBlob ) return 1;
iOff += getVarint32(&aBlob[iOff], aCol[i]);
iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]);
}
return (iOff!=nBlob);
}

4
ext/fts5/fts5_tokenize.c
View File

@@ -13,9 +13,7 @@
#if defined(SQLITE_ENABLE_FTS5)
#include "fts5.h"
#include <string.h>
#include <assert.h>
#include "fts5Int.h"
/**************************************************************************
** Start of ascii tokenizer implementation.

344
ext/fts5/fts5_varint.c Normal file
View File

@@ -0,0 +1,344 @@
/*
** 2015 May 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** Routines for varint serialization and deserialization.
*/
#ifdef SQLITE_ENABLE_FTS5
#include "fts5Int.h"
/*
** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
** Except, this version does handle the single byte case that the core
** version depends on being handled before its function is called.
*/
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){
u32 a,b;
/* The 1-byte case. Overwhelmingly the most common. */
a = *p;
/* a: p0 (unmasked) */
if (!(a&0x80))
{
/* Values between 0 and 127 */
*v = a;
return 1;
}
/* The 2-byte case */
p++;
b = *p;
/* b: p1 (unmasked) */
if (!(b&0x80))
{
/* Values between 128 and 16383 */
a &= 0x7f;
a = a<<7;
*v = a | b;
return 2;
}
/* The 3-byte case */
p++;
a = a<<14;
a |= *p;
/* a: p0<<14 | p2 (unmasked) */
if (!(a&0x80))
{
/* Values between 16384 and 2097151 */
a &= (0x7f<<14)|(0x7f);
b &= 0x7f;
b = b<<7;
*v = a | b;
return 3;
}
/* A 32-bit varint is used to store size information in btrees.
** Objects are rarely larger than 2MiB limit of a 3-byte varint.
** A 3-byte varint is sufficient, for example, to record the size
** of a 1048569-byte BLOB or string.
**
** We only unroll the first 1-, 2-, and 3- byte cases. The very
** rare larger cases can be handled by the slower 64-bit varint
** routine.
*/
{
u64 v64;
u8 n;
p -= 2;
n = sqlite3Fts5GetVarint(p, &v64);
*v = (u32)v64;
assert( n>3 && n<=9 );
return n;
}
}
/*
** Bitmasks used by sqlite3GetVarint(). These precomputed constants
** are defined here rather than simply putting the constant expressions
** inline in order to work around bugs in the RVT compiler.
**
** SLOT_2_0 A mask for (0x7f<<14) | 0x7f
**
** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0
*/
#define SLOT_2_0 0x001fc07f
#define SLOT_4_2_0 0xf01fc07f
/*
** Read a 64-bit variable-length integer from memory starting at p[0].
** Return the number of bytes read. The value is stored in *v.
*/
u8 sqlite3Fts5GetVarint(const unsigned char *p, u64 *v){
u32 a,b,s;
a = *p;
/* a: p0 (unmasked) */
if (!(a&0x80))
{
*v = a;
return 1;
}
p++;
b = *p;
/* b: p1 (unmasked) */
if (!(b&0x80))
{
a &= 0x7f;
a = a<<7;
a |= b;
*v = a;
return 2;
}
/* Verify that constants are precomputed correctly */
assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) );
assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) );
p++;
a = a<<14;
a |= *p;
/* a: p0<<14 | p2 (unmasked) */
if (!(a&0x80))
{
a &= SLOT_2_0;
b &= 0x7f;
b = b<<7;
a |= b;
*v = a;
return 3;
}
/* CSE1 from below */
a &= SLOT_2_0;
p++;
b = b<<14;
b |= *p;
/* b: p1<<14 | p3 (unmasked) */
if (!(b&0x80))
{
b &= SLOT_2_0;
/* moved CSE1 up */
/* a &= (0x7f<<14)|(0x7f); */
a = a<<7;
a |= b;
*v = a;
return 4;
}
/* a: p0<<14 | p2 (masked) */
/* b: p1<<14 | p3 (unmasked) */
/* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
/* moved CSE1 up */
/* a &= (0x7f<<14)|(0x7f); */
b &= SLOT_2_0;
s = a;
/* s: p0<<14 | p2 (masked) */
p++;
a = a<<14;
a |= *p;
/* a: p0<<28 | p2<<14 | p4 (unmasked) */
if (!(a&0x80))
{
/* we can skip these cause they were (effectively) done above in calc'ing s */
/* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
/* b &= (0x7f<<14)|(0x7f); */
b = b<<7;
a |= b;
s = s>>18;
*v = ((u64)s)<<32 | a;
return 5;
}
/* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
s = s<<7;
s |= b;
/* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
p++;
b = b<<14;
b |= *p;
/* b: p1<<28 | p3<<14 | p5 (unmasked) */
if (!(b&0x80))
{
/* we can skip this cause it was (effectively) done above in calc'ing s */
/* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
a &= SLOT_2_0;
a = a<<7;
a |= b;
s = s>>18;
*v = ((u64)s)<<32 | a;
return 6;
}
p++;
a = a<<14;
a |= *p;
/* a: p2<<28 | p4<<14 | p6 (unmasked) */
if (!(a&0x80))
{
a &= SLOT_4_2_0;
b &= SLOT_2_0;
b = b<<7;
a |= b;
s = s>>11;
*v = ((u64)s)<<32 | a;
return 7;
}
/* CSE2 from below */
a &= SLOT_2_0;
p++;
b = b<<14;
b |= *p;
/* b: p3<<28 | p5<<14 | p7 (unmasked) */
if (!(b&0x80))
{
b &= SLOT_4_2_0;
/* moved CSE2 up */
/* a &= (0x7f<<14)|(0x7f); */
a = a<<7;
a |= b;
s = s>>4;
*v = ((u64)s)<<32 | a;
return 8;
}
p++;
a = a<<15;
a |= *p;
/* a: p4<<29 | p6<<15 | p8 (unmasked) */
/* moved CSE2 up */
/* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */
b &= SLOT_2_0;
b = b<<8;
a |= b;
s = s<<4;
b = p[-4];
b &= 0x7f;
b = b>>3;
s |= b;
*v = ((u64)s)<<32 | a;
return 9;
}
/*
** The variable-length integer encoding is as follows:
**
** KEY:
** A = 0xxxxxxx 7 bits of data and one flag bit
** B = 1xxxxxxx 7 bits of data and one flag bit
** C = xxxxxxxx 8 bits of data
**
** 7 bits - A
** 14 bits - BA
** 21 bits - BBA
** 28 bits - BBBA
** 35 bits - BBBBA
** 42 bits - BBBBBA
** 49 bits - BBBBBBA
** 56 bits - BBBBBBBA
** 64 bits - BBBBBBBBC
*/
#ifdef SQLITE_NOINLINE
# define FTS5_NOINLINE SQLITE_NOINLINE
#else
# define FTS5_NOINLINE
#endif
/*
** Write a 64-bit variable-length integer to memory starting at p[0].
** The length of data write will be between 1 and 9 bytes. The number
** of bytes written is returned.
**
** A variable-length integer consists of the lower 7 bits of each byte
** for all bytes that have the 8th bit set and one byte with the 8th
** bit clear. Except, if we get to the 9th byte, it stores the full
** 8 bits and is the last byte.
*/
static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){
int i, j, n;
u8 buf[10];
if( v & (((u64)0xff000000)<<32) ){
p[8] = (u8)v;
v >>= 8;
for(i=7; i>=0; i--){
p[i] = (u8)((v & 0x7f) | 0x80);
v >>= 7;
}
return 9;
}
n = 0;
do{
buf[n++] = (u8)((v & 0x7f) | 0x80);
v >>= 7;
}while( v!=0 );
buf[0] &= 0x7f;
assert( n<=9 );
for(i=0, j=n-1; j>=0; j--, i++){
p[i] = buf[j];
}
return n;
}
int sqlite3Fts5PutVarint(unsigned char *p, u64 v){
if( v<=0x7f ){
p[0] = v&0x7f;
return 1;
}
if( v<=0x3fff ){
p[0] = ((v>>7)&0x7f)|0x80;
p[1] = v&0x7f;
return 2;
}
return fts5PutVarint64(p,v);
}
int sqlite3Fts5GetVarintLen(u32 iVal){
if( iVal<(1 << 7 ) ) return 1;
if( iVal<(1 << 14) ) return 2;
if( iVal<(1 << 21) ) return 3;
if( iVal<(1 << 28) ) return 4;
return 5;
}
#endif /* SQLITE_ENABLE_FTS5 */

5
main.mk
View File

@@ -82,6 +82,7 @@ LIBOBJ += fts5_index.o
LIBOBJ += fts5_storage.o
LIBOBJ += fts5_tokenize.o
LIBOBJ += fts5_unicode2.o
LIBOBJ += fts5_varint.o
LIBOBJ += fts5_vocab.o
LIBOBJ += fts5parse.o
@@ -249,6 +250,7 @@ SRC += \
$(TOP)/ext/fts5/fts5_storage.c \
$(TOP)/ext/fts5/fts5_tokenize.c \
$(TOP)/ext/fts5/fts5_unicode2.c \
$(TOP)/ext/fts5/fts5_varint.c \
$(TOP)/ext/fts5/fts5_vocab.c
@@ -667,6 +669,9 @@ fts5_tokenize.o: $(TOP)/ext/fts5/fts5_tokenize.c $(HDR) $(EXTHDR)
fts5_unicode2.o: $(TOP)/ext/fts5/fts5_unicode2.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_unicode2.c
fts5_varint.o: $(TOP)/ext/fts5/fts5_varint.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_varint.c
fts5_vocab.o: $(TOP)/ext/fts5/fts5_vocab.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_vocab.c

31
manifest
View File

@@ -1,5 +1,5 @@
C Add\sextra\stests\sand\sfixes\sfor\smulti-column\smatches.
D 2015-05-29T19:00:22.571
C Remove\sthe\s"#include\ssqlite3Int.h"\sfrom\sfts5Int.h.
D 2015-05-30T11:49:58.614
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2c28e557780395095c307a6e5cb539419027eb5e
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -105,19 +105,20 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl ed0534dd51efce39878bce33944c6073d37a1e20
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
F ext/fts5/fts5.c 74d18b4dc7518c7cd85609f1541e83bc564619a2
F ext/fts5/fts5.c 34e5098e85ed14cc120004c5622536b77ddf4976
F ext/fts5/fts5.h 4266c6231094005b051dbfc8dd85d2bc57243d34
F ext/fts5/fts5Int.h 3bcecc469fe570ab188d123e1d33d6e5e11a5129
F ext/fts5/fts5Int.h 4c677f3b797acde90ba1b7730eca6a32e7def742
F ext/fts5/fts5_aux.c d53f00f31ad615ca4f139dd8751f9041afa00971
F ext/fts5/fts5_buffer.c 861599a0abe2383f0cd0352c57001140a26b0930
F ext/fts5/fts5_buffer.c 9ec57c75c81e81dca118568876b1caead0aadadf
F ext/fts5/fts5_config.c 11f969ed711a0a8b611d47431d74c372ad78c713
F ext/fts5/fts5_expr.c 1685b331ecb880cb8807e2dc7fc4184d2933bb96
F ext/fts5/fts5_hash.c 54dd25348a46ea62ea96322c572e08cd1fb37304
F ext/fts5/fts5_index.c 59b8a3dfde24ddb80c31088148a3dfc779db22ab
F ext/fts5/fts5_storage.c 5d2b51adb304643d8f825ba89283d628418b20c2
F ext/fts5/fts5_expr.c 6a683326d6ae4e58420792e84576af9c7a8a89e4
F ext/fts5/fts5_hash.c c1cfdb2cae0fad00b06fae38a40eaf9261563ccc
F ext/fts5/fts5_index.c 7cea402924cd3d8cd5943a7f9514c9153696571b
F ext/fts5/fts5_storage.c 04e6717656b78eb230a1c730cac3b935eb94889b
F ext/fts5/fts5_tcl.c 7ea165878e4ae3598e89acd470a0ee1b5a00e33c
F ext/fts5/fts5_tokenize.c 24649425adfea2c4877d8f69f2754b70374940ec
F ext/fts5/fts5_tokenize.c 97251d68d7a6a9415bde1203f9382864dfc1f989
F ext/fts5/fts5_unicode2.c da3cf712f05cd8347c8c5bc00964cc0361c88da9
F ext/fts5/fts5_varint.c 366452037bf9a000c351374b489badc1b3541796
F ext/fts5/fts5_vocab.c 1f8543b2c1ae4427f127a911bc8e60873fcd7bf9
F ext/fts5/fts5parse.y 4ee667932d561a150d96483cf563281b95a9e523
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
@@ -225,7 +226,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
F main.mk a17ff4be35788f0a007b80af5e2a4f3036f03882
F main.mk 5ff584ca0d7bd7d7006965d6f04c95b73a444824
F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
F mkopcodeh.awk d5e22023b5238985bb54a72d33e0ac71fe4f8a32
F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@@ -1297,7 +1298,7 @@ F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
F tool/mkpragmatab.tcl 94f196c9961e0ca3513e29f57125a3197808be2d
F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
F tool/mksqlite3c-noext.tcl 69bae8ce4aa52d2ff82d4a8a856bf283ec035b2e
F tool/mksqlite3c.tcl ddd1ab091b2b42cf5cef07db5003b4c88272754e
F tool/mksqlite3c.tcl 23c7cddd9f9ccd192e7a73758aaf46a8159441bb
F tool/mksqlite3h.tcl 44730d586c9031638cdd2eb443b801c0d2dbd9f8
F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b
F tool/mkvsix.tcl 3b58b9398f91c7dbf18d49eb87cefeee9efdbce1
@@ -1332,7 +1333,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 0fc0ea20920615f3e48ea2dbe2b7dcd979b0993e
R 358d1aaaac138ca9e6a38de66a871d33
P ae6794ffa23ef6191bd8834422abf322d978c11b
R bc0305687d74df992086e66a1770f40c
U dan
Z c843d103a5028eb0d546bbe1e7e6abdc
Z 008bbbc1e4c3598d73e809e2e8e489be

2
manifest.uuid
View File

@@ -1 +1 @@
ae6794ffa23ef6191bd8834422abf322d978c11b
e008c3c8e29c843ec945ddad54b9688bbf2bdb44

1
tool/mksqlite3c.tcl
View File

@@ -379,6 +379,7 @@ foreach file {
fts5_storage.c
fts5_tokenize.c
fts5_unicode2.c
fts5_varint.c
fts5_vocab.c
rtree.c