diff --git a/Makefile.in b/Makefile.in index 6da7bff345..883b79bc68 100644 --- a/Makefile.in +++ b/Makefile.in @@ -421,10 +421,12 @@ TESTSRC += \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/fts5/fts5_tcl.c \ + $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/Makefile.msc b/Makefile.msc index 64e2d26e07..f4200c44ba 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1095,10 +1095,12 @@ TESTEXT = \ $(TOP)\ext\misc\fuzzer.c \ fts5.c \ $(TOP)\ext\fts5\fts5_tcl.c \ + $(TOP)\ext\fts5\fts5_test_mi.c \ $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ + $(TOP)\ext\misc\series.c \ $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\totype.c \ $(TOP)\ext\misc\wholenumber.c diff --git a/VERSION b/VERSION index 73bb444b25..89a1ad7ad3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.11 +3.8.12 diff --git a/configure b/configure index b92c4e210e..2969759f3c 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.8.11. +# Generated by GNU Autoconf 2.69 for sqlite 3.8.12. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.11' -PACKAGE_STRING='sqlite 3.8.11' +PACKAGE_VERSION='3.8.12' +PACKAGE_STRING='sqlite 3.8.12' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1454,7 +1454,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.11 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.12 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1519,7 +1519,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.11:";; + short | recursive ) echo "Configuration of sqlite 3.8.12:";; esac cat <<\_ACEOF @@ -1634,7 +1634,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.11 +sqlite configure 3.8.12 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2053,7 +2053,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.11, which was +It was created by sqlite $as_me 3.8.12, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -11808,7 +11808,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.11, which was +This file was extended by sqlite $as_me 3.8.12, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -11874,7 +11874,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.8.11 +sqlite config.status 3.8.12 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/ext/async/sqlite3async.c b/ext/async/sqlite3async.c index 4ab39cac35..b6f4a4bd36 100644 --- a/ext/async/sqlite3async.c +++ b/ext/async/sqlite3async.c @@ -1636,6 +1636,7 @@ void sqlite3async_run(void){ ** Control/configure the asynchronous IO system. */ int sqlite3async_control(int op, ...){ + int rc = SQLITE_OK; va_list ap; va_start(ap, op); switch( op ){ @@ -1645,7 +1646,8 @@ int sqlite3async_control(int op, ...){ && eWhen!=SQLITEASYNC_HALT_NOW && eWhen!=SQLITEASYNC_HALT_IDLE ){ - return SQLITE_MISUSE; + rc = SQLITE_MISUSE; + break; } async.eHalt = eWhen; async_mutex_enter(ASYNC_MUTEX_QUEUE); @@ -1657,7 +1659,8 @@ int sqlite3async_control(int op, ...){ case SQLITEASYNC_DELAY: { int iDelay = va_arg(ap, int); if( iDelay<0 ){ - return SQLITE_MISUSE; + rc = SQLITE_MISUSE; + break; } async.ioDelay = iDelay; break; @@ -1668,7 +1671,8 @@ int sqlite3async_control(int op, ...){ async_mutex_enter(ASYNC_MUTEX_QUEUE); if( async.nFile || async.pQueueFirst ){ async_mutex_leave(ASYNC_MUTEX_QUEUE); - return SQLITE_MISUSE; + rc = SQLITE_MISUSE; + break; } async.bLockFiles = bLock; async_mutex_leave(ASYNC_MUTEX_QUEUE); @@ -1692,9 +1696,11 @@ int sqlite3async_control(int op, ...){ } default: - return SQLITE_ERROR; + rc = SQLITE_ERROR; + break; } - return SQLITE_OK; + va_end(ap); + return rc; } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ASYNCIO) */ diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 903d7d84fd..6a9b507fc0 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -4231,7 +4231,6 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ int bIncrOk = (bOptOk && pCsr->bDesc==pTab->bDescIdx && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 - && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 #ifdef SQLITE_TEST && pTab->bNoIncrDoclist==0 #endif diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index d9b30c620d..c123d6444c 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -32,6 +32,7 @@ typedef struct Fts5ExtensionApi Fts5ExtensionApi; typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; typedef void (*fts5_extension_function)( const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ @@ -41,6 +42,11 @@ typedef void (*fts5_extension_function)( sqlite3_value **apVal /* Array of trailing arguments */ ); +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + /* ** EXTENSION API FUNCTIONS ** @@ -60,11 +66,19 @@ typedef void (*fts5_extension_function)( ** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** -** xColumnCount: -** Returns the number of columns in the FTS5 table. +** xColumnCount(pFts): +** Return the number of columns in the table. ** -** xColumnSize: -** Reports the size in tokens of a column value from the current row. +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. ** ** xColumnText: ** This function attempts to retrieve the text of column iCol of the @@ -166,6 +180,30 @@ typedef void (*fts5_extension_function)( ** In other words, the same value that would be returned by: ** ** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iOff>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods. +** +** xPhraseNext() +** See xPhraseFirst above. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 1 */ @@ -197,6 +235,9 @@ struct Fts5ExtensionApi { ); int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); void *(*xGetAuxdata)(Fts5Context*, int bClear); + + void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); }; /* diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 149f6b6694..5298429437 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -148,7 +148,7 @@ struct Fts5Config { }; /* Current expected value of %_config table 'version' field */ -#define FTS5_CURRENT_VERSION 2 +#define FTS5_CURRENT_VERSION 3 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 @@ -301,7 +301,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p); */ /* -** Open a new iterator to iterate though all docids that match the +** Open a new iterator to iterate though all rowids that match the ** specified token or token prefix. */ int sqlite3Fts5IndexQuery( diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 3bf9e2376e..7e991fc21d 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -17,7 +17,7 @@ #include "fts5Int.h" -#define FTS5_DEFAULT_PAGE_SIZE 1000 +#define FTS5_DEFAULT_PAGE_SIZE 4050 #define FTS5_DEFAULT_AUTOMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 861b20863a..a713bb7c5a 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -34,7 +34,7 @@ void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*); struct Fts5Expr { Fts5Index *pIndex; Fts5ExprNode *pRoot; - int bDesc; /* Iterate in descending docid order */ + int bDesc; /* Iterate in descending rowid order */ int nPhrase; /* Number of phrases in expression */ Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ }; @@ -1865,6 +1865,15 @@ static void fts5ExprFunction( Fts5Config *pConfig = 0; int iArg = 1; + if( nArg<1 ){ + zErr = sqlite3_mprintf("wrong number of arguments to function %s", + bTcl ? "fts5_expr_tcl" : "fts5_expr" + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } + if( bTcl && nArg>1 ){ zNearsetCmd = (const char*)sqlite3_value_text(apVal[1]); iArg = 2; @@ -2003,7 +2012,7 @@ int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ ** Return the number of phrases in expression pExpr. */ int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){ - return pExpr->nPhrase; + return (pExpr ? pExpr->nPhrase : 0); } /* diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index ff440fa61c..e8052a2dad 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -133,7 +133,7 @@ void sqlite3Fts5HashClear(Fts5Hash *pHash){ pHash->nEntry = 0; } -static unsigned int fts5HashKey(int nSlot, const char *p, int n){ +static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){ int i; unsigned int h = 13; for(i=n-1; i>=0; i--){ @@ -142,7 +142,7 @@ static unsigned int fts5HashKey(int nSlot, const char *p, int n){ return (h % nSlot); } -static unsigned int fts5HashKey2(int nSlot, char b, const char *p, int n){ +static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){ int i; unsigned int h = 13; for(i=n-1; i>=0; i--){ @@ -170,7 +170,7 @@ static int fts5HashResize(Fts5Hash *pHash){ int iHash; Fts5HashEntry *p = apOld[i]; apOld[i] = p->pHashNext; - iHash = fts5HashKey(nNew, p->zKey, strlen(p->zKey)); + iHash = fts5HashKey(nNew, (u8*)p->zKey, strlen(p->zKey)); p->pHashNext = apNew[iHash]; apNew[iHash] = p; } @@ -210,12 +210,13 @@ int sqlite3Fts5HashWrite( char bByte, /* First byte of token */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ - unsigned int iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken); + unsigned int iHash; Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ /* Attempt to locate an existing hash entry */ + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( p->zKey[0]==bByte && memcmp(&p->zKey[1], pToken, nToken)==0 @@ -233,7 +234,7 @@ int sqlite3Fts5HashWrite( if( (pHash->nEntry*2)>=pHash->nSlot ){ int rc = fts5HashResize(pHash); if( rc!=SQLITE_OK ) return rc; - iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken); + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); } p = (Fts5HashEntry*)sqlite3_malloc(nByte); @@ -242,7 +243,7 @@ int sqlite3Fts5HashWrite( p->nAlloc = nByte; p->zKey[0] = bByte; memcpy(&p->zKey[1], pToken, nToken); - assert( iHash==fts5HashKey(pHash->nSlot, p->zKey, nToken+1) ); + assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); p->zKey[nToken+1] = '\0'; p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); @@ -414,7 +415,7 @@ int sqlite3Fts5HashQuery( const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ int *pnDoclist /* OUT: Size of doclist in bytes */ ){ - unsigned int iHash = fts5HashKey(pHash->nSlot, pTerm, nTerm); + unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm); Fts5HashEntry *p; for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 2d23607d3f..a7ab7c4c05 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -33,7 +33,7 @@ ** doclist, without loading it into memory. ** ** * large doclists that span many pages have associated "doclist index" -** records that contain a copy of the first docid on each page spanned by +** records that contain a copy of the first rowid on each page spanned by ** the doclist. This is used to speed up seek operations, and merges of ** large doclists with very small doclists. ** @@ -210,10 +210,10 @@ ** ** * Page number of fts index leaf page. As a varint. ** -** * First docid on page indicated by previous field. As a varint. +** * First rowid on page indicated by previous field. As a varint. ** ** * A list of varints, one for each subsequent termless page. A -** positive delta if the termless page contains at least one docid, +** positive delta if the termless page contains at least one rowid, ** or an 0x00 byte otherwise. ** ** Internal doclist index nodes are: @@ -223,9 +223,9 @@ ** ** * Page number of first child page. As a varint. ** -** * Copy of first docid on page indicated by previous field. As a varint. +** * Copy of first rowid on page indicated by previous field. As a varint. ** -** * A list of delta-encoded varints - the first docid on each subsequent +** * A list of delta-encoded varints - the first rowid on each subsequent ** child page. ** */ @@ -287,9 +287,8 @@ int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } ** without overreading if the records are corrupt. */ #define FTS5_DATA_ZERO_PADDING 8 +#define FTS5_DATA_PADDING 20 -typedef struct Fts5BtreeIter Fts5BtreeIter; -typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel; typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5DlidxLvl Fts5DlidxLvl; @@ -333,6 +332,9 @@ struct Fts5Index { sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ + sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ + sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ + sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ }; @@ -382,14 +384,13 @@ struct Fts5PageWriter { struct Fts5DlidxWriter { int pgno; /* Page number for this page */ int bPrevValid; /* True if iPrev is valid */ - i64 iPrev; /* Previous docid value written to page */ + i64 iPrev; /* Previous rowid value written to page */ Fts5Buffer buf; /* Buffer containing page data */ }; struct Fts5SegWriter { int iSegid; /* Segid to write to */ - int nWriter; /* Number of entries in aWriter */ - Fts5PageWriter *aWriter; /* Array of PageWriter objects */ - i64 iPrevRowid; /* Previous docid written to current leaf */ + Fts5PageWriter writer; /* PageWriter object */ + i64 iPrevRowid; /* Previous rowid written to current leaf */ u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */ u8 bFirstRowidInPage; /* True if next rowid is first in page */ u8 bFirstTermInPage; /* True if next term will be first in leaf */ @@ -398,11 +399,15 @@ struct Fts5SegWriter { int nDlidx; /* Allocated size of aDlidx[] array */ Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */ + + /* Values to insert into the %_idx table */ + Fts5Buffer btterm; /* Next term to insert into %_idx table */ + int iBtPage; /* Page number corresponding to btterm */ }; /* ** Object for iterating through the merged results of one or more segments, -** visiting each term/docid pair in the merged data. +** visiting each term/rowid pair in the merged data. ** ** nSeg is always a power of two greater than or equal to the number of ** segments that this object is merging data from. Both the aSeg[] and @@ -427,7 +432,7 @@ struct Fts5CResult { }; /* -** Object for iterating through a single segment, visiting each term/docid +** Object for iterating through a single segment, visiting each term/rowid ** pair in the segment. ** ** pSeg: @@ -459,7 +464,7 @@ struct Fts5CResult { ** ** FTS5_SEGITER_REVERSE: ** This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If -** it is set, iterate through docids in descending order instead of the +** it is set, iterate through rowid in descending order instead of the ** default ascending order. ** ** iRowidOffset/nRowidOffset/aRowidOffset: @@ -570,43 +575,6 @@ struct Fts5DlidxIter { -/* -** An Fts5BtreeIter object is used to iterate through all entries in the -** b-tree hierarchy belonging to a single fts5 segment. In this case the -** "b-tree hierarchy" is all b-tree nodes except leaves. Each entry in the -** b-tree hierarchy consists of the following: -** -** iLeaf: The page number of the leaf page the entry points to. -** -** term: A split-key that all terms on leaf page $iLeaf must be greater -** than or equal to. The "term" associated with the first b-tree -** hierarchy entry (the one that points to leaf page 1) is always -** an empty string. -** -** nEmpty: The number of empty (termless) leaf pages that immediately -** following iLeaf. -** -** The Fts5BtreeIter object is only used as part of the integrity-check code. -*/ -struct Fts5BtreeIterLevel { - Fts5NodeIter s; /* Iterator for the current node */ - Fts5Data *pData; /* Data for the current node */ -}; -struct Fts5BtreeIter { - Fts5Index *p; /* FTS5 backend object */ - Fts5StructureSegment *pSeg; /* Iterate through this segment's b-tree */ - int nLvl; /* Size of aLvl[] array */ - Fts5BtreeIterLevel *aLvl; /* Level for each tier of b-tree */ - - /* Output variables */ - Fts5Buffer term; /* Current term */ - int iLeaf; /* Leaf containing terms >= current term */ - int nEmpty; /* Number of "empty" leaves following iLeaf */ - int bEof; /* Set to true at EOF */ - int bDlidx; /* True if there exists a dlidx */ -}; - - /* ** The first argument passed to this macro is a pointer to an Fts5Buffer ** object. @@ -748,7 +716,7 @@ static Fts5Data *fts5DataReadOrBuffer( rc = SQLITE_NOMEM; } }else{ - int nSpace = nByte + FTS5_DATA_ZERO_PADDING; + int nSpace = nByte + FTS5_DATA_PADDING; pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data)); if( pRet ){ pRet->n = nByte; @@ -805,6 +773,23 @@ static void fts5DataRelease(Fts5Data *pData){ sqlite3_free(pData); } +static int fts5IndexPrepareStmt( + Fts5Index *p, + sqlite3_stmt **ppStmt, + char *zSql +){ + if( p->rc==SQLITE_OK ){ + if( zSql ){ + p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0); + }else{ + p->rc = SQLITE_NOMEM; + } + } + sqlite3_free(zSql); + return p->rc; +} + + /* ** INSERT OR REPLACE a record into the %_data table. */ @@ -814,17 +799,11 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ if( p->pWriter==0 ){ int rc = SQLITE_OK; Fts5Config *pConfig = p->pConfig; - char *zSql = sqlite3Fts5Mprintf(&rc, - "REPLACE INTO '%q'.%Q(id, block) VALUES(?,?)", pConfig->zDb, p->zDataTbl - ); - if( zSql ){ - rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pWriter, 0); - sqlite3_free(zSql); - } - if( rc!=SQLITE_OK ){ - p->rc = rc; - return; - } + fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf( + "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", + pConfig->zDb, pConfig->zName + )); + if( p->rc ) return; } sqlite3_bind_int64(p->pWriter, 1, iRowid); @@ -845,7 +824,8 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ int rc; Fts5Config *pConfig = p->pConfig; char *zSql = sqlite3_mprintf( - "DELETE FROM '%q'.%Q WHERE id>=? AND id<=?", pConfig->zDb, p->zDataTbl + "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", + pConfig->zDb, pConfig->zName ); if( zSql==0 ){ rc = SQLITE_NOMEM; @@ -872,6 +852,18 @@ static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0, 0); i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0, 0)-1; fts5DataDelete(p, iFirst, iLast); + if( p->pIdxDeleter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE segid=?", + pConfig->zDb, pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pIdxDeleter, 1, iSegid); + sqlite3_step(p->pIdxDeleter); + p->rc = sqlite3_reset(p->pIdxDeleter); + } } /* @@ -2247,7 +2239,7 @@ static void fts5LeafSeek( while( 1 ){ int nPos; - /* Skip past docid delta */ + /* Skip past rowid delta */ fts5IndexSkipVarint(a, iOff); /* Skip past position list */ @@ -2320,7 +2312,6 @@ static void fts5SegIterSeekInit( Fts5SegIter *pIter /* Object to populate */ ){ int iPg = 1; - int h; int bGe = (flags & FTS5INDEX_QUERY_SCAN); int bDlidx = 0; /* True if there is a doclist-index */ @@ -2334,12 +2325,23 @@ static void fts5SegIterSeekInit( /* This block sets stack variable iPg to the leaf page number that may ** contain term (pTerm/nTerm), if it is present in the segment. */ - for(h=pSeg->nHeight-1; h>0; h--){ - i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg); - fts5DataBuffer(p, pBuf, iRowid); - if( p->rc ) break; - iPg = fts5NodeSeek(pBuf, pTerm, nTerm, &bDlidx); + if( p->pIdxSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term<=? ORDER BY term DESC LIMIT 1", + pConfig->zDb, pConfig->zName + )); } + if( p->rc ) return; + sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid); + sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){ + i64 val = sqlite3_column_int(p->pIdxSelect, 0); + iPg = (int)(val>>1); + bDlidx = (val & 0x0001); + } + p->rc = sqlite3_reset(p->pIdxSelect); if( iPgpgnoFirst ){ iPg = pSeg->pgnoFirst; @@ -3169,52 +3171,51 @@ static int fts5WriteDlidxGrow( } /* -** If an "nEmpty" record must be written to the b-tree before the next -** term, write it now. +** If the current doclist-index accumulating in pWriter->aDlidx[] is large +** enough, flush it to disk and return 1. Otherwise discard it and return +** zero. */ -static void fts5WriteBtreeNEmpty(Fts5Index *p, Fts5SegWriter *pWriter){ - if( pWriter->nEmpty ){ - int bFlag = 0; - Fts5PageWriter *pPg; - pPg = &pWriter->aWriter[1]; +static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){ + int bFlag = 0; - /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written - ** to the database, also write the doclist-index to disk. */ - if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ - bFlag = 1; - } - fts5WriteDlidxClear(p, pWriter, bFlag); - fts5BufferAppendVarint(&p->rc, &pPg->buf, bFlag); - fts5BufferAppendVarint(&p->rc, &pPg->buf, pWriter->nEmpty); - pWriter->nEmpty = 0; - }else{ - fts5WriteDlidxClear(p, pWriter, 0); + /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written + ** to the database, also write the doclist-index to disk. */ + if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ + bFlag = 1; } - - assert( pWriter->nDlidx==0 || pWriter->aDlidx[0].buf.n==0 ); - assert( pWriter->nDlidx==0 || pWriter->aDlidx[0].bPrevValid==0 ); + fts5WriteDlidxClear(p, pWriter, bFlag); + pWriter->nEmpty = 0; + return bFlag; } -static void fts5WriteBtreeGrow(Fts5Index *p, Fts5SegWriter *pWriter){ +/* +** This function is called whenever processing of the doclist for the +** last term on leaf page (pWriter->iBtPage) is completed. +** +** The doclist-index for that term is currently stored in-memory within the +** Fts5SegWriter.aDlidx[] array. If it is large enough, this function +** writes it out to disk. Or, if it is too small to bother with, discards +** it. +** +** Fts5SegWriter.btterm currently contains the first term on page iBtPage. +*/ +static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){ + int bFlag; + + assert( pWriter->iBtPage || pWriter->nEmpty==0 ); + if( pWriter->iBtPage==0 ) return; + bFlag = fts5WriteFlushDlidx(p, pWriter); + if( p->rc==SQLITE_OK ){ - Fts5PageWriter *aNew; - Fts5PageWriter *pNew; - int nNew = sizeof(Fts5PageWriter) * (pWriter->nWriter+1); - - aNew = (Fts5PageWriter*)sqlite3_realloc(pWriter->aWriter, nNew); - if( aNew==0 ){ - p->rc = SQLITE_NOMEM; - return; - } - - pNew = &aNew[pWriter->nWriter]; - memset(pNew, 0, sizeof(Fts5PageWriter)); - pNew->pgno = 1; - fts5BufferAppendVarint(&p->rc, &pNew->buf, 1); - - pWriter->nWriter++; - pWriter->aWriter = aNew; + const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:""); + /* The following was already done in fts5WriteInit(): */ + /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */ + sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC); + sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1)); + sqlite3_step(p->pIdxWriter); + p->rc = sqlite3_reset(p->pIdxWriter); } + pWriter->iBtPage = 0; } /* @@ -3231,36 +3232,9 @@ static void fts5WriteBtreeTerm( Fts5SegWriter *pWriter, /* Writer object */ int nTerm, const u8 *pTerm /* First term on new page */ ){ - int iHeight; - for(iHeight=1; 1; iHeight++){ - Fts5PageWriter *pPage; - - if( iHeight>=pWriter->nWriter ){ - fts5WriteBtreeGrow(p, pWriter); - if( p->rc ) return; - } - pPage = &pWriter->aWriter[iHeight]; - - fts5WriteBtreeNEmpty(p, pWriter); - - if( pPage->buf.n>=p->pConfig->pgsz ){ - /* pPage will be written to disk. The term will be written into the - ** parent of pPage. */ - i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, iHeight, pPage->pgno); - fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); - fts5BufferZero(&pPage->buf); - fts5BufferZero(&pPage->term); - fts5BufferAppendVarint(&p->rc, &pPage->buf, pPage[-1].pgno); - pPage->pgno++; - }else{ - int nPre = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); - fts5BufferAppendVarint(&p->rc, &pPage->buf, nPre+2); - fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm-nPre); - fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm-nPre, pTerm+nPre); - fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm); - break; - } - } + fts5WriteFlushBtree(p, pWriter); + fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm); + pWriter->iBtPage = pWriter->writer.pgno; } /* @@ -3344,7 +3318,7 @@ static void fts5WriteDlidxAppend( if( pDlidx->bPrevValid ){ iVal = iRowid - pDlidx->iPrev; }else{ - i64 iPgno = (i==0 ? pWriter->aWriter[0].pgno : pDlidx[-1].pgno); + i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); assert( pDlidx->buf.n==0 ); sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone); sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno); @@ -3359,7 +3333,7 @@ static void fts5WriteDlidxAppend( static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; - Fts5PageWriter *pPage = &pWriter->aWriter[0]; + Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; if( pWriter->bFirstTermInPage ){ @@ -3398,11 +3372,11 @@ static void fts5WriteAppendTerm( int nTerm, const u8 *pTerm ){ int nPrefix; /* Bytes of prefix compression for term */ - Fts5PageWriter *pPage = &pWriter->aWriter[0]; + Fts5PageWriter *pPage = &pWriter->writer; assert( pPage->buf.n==0 || pPage->buf.n>4 ); if( pPage->buf.n==0 ){ - /* Zero the first term and first docid fields */ + /* Zero the first term and first rowid fields */ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); assert( pWriter->bFirstTermInPage ); @@ -3433,7 +3407,7 @@ static void fts5WriteAppendTerm( n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); } fts5WriteBtreeTerm(p, pWriter, n, pTerm); - pPage = &pWriter->aWriter[0]; + pPage = &pWriter->writer; } }else{ nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); @@ -3462,7 +3436,7 @@ static void fts5WriteAppendTerm( } /* -** Append a docid and position-list size field to the writers output. +** Append a rowid and position-list size field to the writers output. */ static void fts5WriteAppendRowid( Fts5Index *p, @@ -3471,17 +3445,17 @@ static void fts5WriteAppendRowid( int nPos ){ if( p->rc==SQLITE_OK ){ - Fts5PageWriter *pPage = &pWriter->aWriter[0]; + Fts5PageWriter *pPage = &pWriter->writer; - /* If this is to be the first docid written to the page, set the - ** docid-pointer in the page-header. Also append a value to the dlidx + /* If this is to be the first rowid written to the page, set the + ** rowid-pointer in the page-header. Also append a value to the dlidx ** buffer, in case a doclist-index is required. */ if( pWriter->bFirstRowidInPage ){ fts5PutU16(pPage->buf.p, pPage->buf.n); fts5WriteDlidxAppend(p, pWriter, iRowid); } - /* Write the docid. */ + /* Write the rowid. */ if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){ fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); }else{ @@ -3506,7 +3480,7 @@ static void fts5WriteAppendPoslistData( const u8 *aData, int nData ){ - Fts5PageWriter *pPage = &pWriter->aWriter[0]; + Fts5PageWriter *pPage = &pWriter->writer; const u8 *a = aData; int n = nData; @@ -3529,7 +3503,7 @@ static void fts5WriteAppendPoslistData( } static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){ - fts5BufferAppendVarint(&p->rc, &pWriter->aWriter[0].buf, 0); + fts5BufferAppendVarint(&p->rc, &pWriter->writer.buf, 0); } /* @@ -3543,8 +3517,8 @@ static void fts5WriteFinish( int *pnLeaf /* OUT: Number of leaf pages in b-tree */ ){ int i; + Fts5PageWriter *pLeaf = &pWriter->writer; if( p->rc==SQLITE_OK ){ - Fts5PageWriter *pLeaf = &pWriter->aWriter[0]; if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){ *pnLeaf = 0; *pnHeight = 0; @@ -3553,29 +3527,14 @@ static void fts5WriteFinish( fts5WriteFlushLeaf(p, pWriter); } *pnLeaf = pLeaf->pgno-1; - if( pWriter->nWriter==1 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ - fts5WriteBtreeGrow(p, pWriter); - } - if( pWriter->nWriter>1 ){ - fts5WriteBtreeNEmpty(p, pWriter); - } - *pnHeight = pWriter->nWriter; - for(i=1; inWriter; i++){ - Fts5PageWriter *pPg = &pWriter->aWriter[i]; - fts5DataWrite(p, - FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pPg->pgno), - pPg->buf.p, pPg->buf.n - ); - } + fts5WriteFlushBtree(p, pWriter); + *pnHeight = 0; } } - for(i=0; inWriter; i++){ - Fts5PageWriter *pPg = &pWriter->aWriter[i]; - fts5BufferFree(&pPg->term); - fts5BufferFree(&pPg->buf); - } - sqlite3_free(pWriter->aWriter); + fts5BufferFree(&pLeaf->term); + fts5BufferFree(&pLeaf->buf); + fts5BufferFree(&pWriter->btterm); for(i=0; inDlidx; i++){ sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf); @@ -3591,48 +3550,21 @@ static void fts5WriteInit( memset(pWriter, 0, sizeof(Fts5SegWriter)); pWriter->iSegid = iSegid; - pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, sizeof(Fts5PageWriter)); - if( fts5WriteDlidxGrow(p, pWriter, 1) ) return; - pWriter->nWriter = 1; - pWriter->nDlidx = 1; - pWriter->aWriter[0].pgno = 1; + fts5WriteDlidxGrow(p, pWriter, 1); + pWriter->writer.pgno = 1; pWriter->bFirstTermInPage = 1; -} + pWriter->iBtPage = 1; -static void fts5WriteInitForAppend( - Fts5Index *p, /* FTS5 backend object */ - Fts5SegWriter *pWriter, /* Writer to initialize */ - Fts5StructureSegment *pSeg /* Segment object to append to */ -){ - int nByte = pSeg->nHeight * sizeof(Fts5PageWriter); - memset(pWriter, 0, sizeof(Fts5SegWriter)); - pWriter->iSegid = pSeg->iSegid; - pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, nByte); - pWriter->aDlidx = (Fts5DlidxWriter*)fts5IdxMalloc(p, sizeof(Fts5DlidxWriter)); + if( p->pIdxWriter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( + "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)", + pConfig->zDb, pConfig->zName + )); + } if( p->rc==SQLITE_OK ){ - int pgno = 1; - int i; - pWriter->nDlidx = 1; - pWriter->nWriter = pSeg->nHeight; - pWriter->aWriter[0].pgno = pSeg->pgnoLast+1; - for(i=pSeg->nHeight-1; i>0; i--){ - i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pgno); - Fts5PageWriter *pPg = &pWriter->aWriter[i]; - pPg->pgno = pgno; - fts5DataBuffer(p, &pPg->buf, iRowid); - if( p->rc==SQLITE_OK ){ - Fts5NodeIter ss; - fts5NodeIterInit(pPg->buf.p, pPg->buf.n, &ss); - while( ss.aData ) fts5NodeIterNext(&p->rc, &ss); - fts5BufferSet(&p->rc, &pPg->term, ss.term.n, ss.term.p); - pgno = ss.iChild; - fts5NodeIterFree(&ss); - } - } - assert( p->rc!=SQLITE_OK || (pgno+pWriter->nEmpty)==pSeg->pgnoLast ); - pWriter->bFirstTermInPage = 1; - assert( pWriter->aWriter[0].term.n==0 ); + sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); } } @@ -3672,7 +3604,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){ fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]); fts5DataRelease(pData); pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; - fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1),iLeafRowid); + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1), iLeafRowid); fts5DataWrite(p, iLeafRowid, buf.p, buf.n); } } @@ -3719,8 +3651,11 @@ static void fts5IndexMergeLevel( pLvlOut = &pStruct->aLevel[iLvl+1]; assert( pLvlOut->nSeg>0 ); nInput = pLvl->nMerge; - fts5WriteInitForAppend(p, &writer, &pLvlOut->aSeg[pLvlOut->nSeg-1]); pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1]; + + fts5WriteInit(p, &writer, pSeg->iSegid); + writer.writer.pgno = pSeg->pgnoLast+1; + writer.iBtPage = 0; }else{ int iSegid = fts5AllocateSegid(p, pStruct); @@ -3811,7 +3746,7 @@ static void fts5IndexMergeLevel( pStruct->nSegment--; } }else{ - assert( pSeg->nHeight>0 && pSeg->pgnoLast>0 ); + assert( pSeg->pgnoLast>0 ); fts5TrimSegments(p, pIter); pLvl->nMerge = nInput; } @@ -3986,7 +3921,7 @@ static void fts5FlushOneHash(Fts5Index *p){ /* Pre-allocate the buffer used to assemble leaf pages to the target ** page size. */ assert( pgsz>0 ); - pBuf = &writer.aWriter[0].buf; + pBuf = &writer.writer.buf; fts5BufferGrow(&p->rc, pBuf, pgsz + 20); /* Begin scanning through hash table entries. This loop runs once for each @@ -4008,9 +3943,9 @@ static void fts5FlushOneHash(Fts5Index *p){ /* Decide if the term will fit on the current leaf. If it will not, ** flush the leaf to disk here. */ - if( (pBuf->n + nTerm + 2) > pgsz ){ + if( pBuf->n>4 && (pBuf->n + nTerm + 2) > pgsz ){ fts5WriteFlushLeaf(p, &writer); - pBuf = &writer.aWriter[0].buf; + pBuf = &writer.writer.buf; if( (nTerm + 32) > pBuf->nSpace ){ fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n); if( p->rc ) break; @@ -4027,10 +3962,10 @@ static void fts5FlushOneHash(Fts5Index *p){ }else{ fts5PutU16(&pBuf->p[2], pBuf->n); writer.bFirstTermInPage = 0; - if( writer.aWriter[0].pgno!=1 ){ + if( writer.writer.pgno!=1 ){ int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm); fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm); - pBuf = &writer.aWriter[0].buf; + pBuf = &writer.writer.buf; assert( nPre0 && writer.aDlidx[0].buf.n==0 ); - writer.aDlidx[0].pgno = writer.aWriter[0].pgno; + writer.aDlidx[0].pgno = writer.writer.pgno; if( pgsz>=(pBuf->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ @@ -4067,7 +4002,7 @@ static void fts5FlushOneHash(Fts5Index *p){ iRowid += iDelta; if( writer.bFirstRowidInPage ){ - fts5PutU16(&pBuf->p[0], pBuf->n); /* first docid on page */ + fts5PutU16(&pBuf->p[0], pBuf->n); /* first rowid on page */ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); writer.bFirstRowidInPage = 0; fts5WriteDlidxAppend(p, &writer, iRowid); @@ -4099,7 +4034,7 @@ static void fts5FlushOneHash(Fts5Index *p){ iPos += n; if( pBuf->n>=pgsz ){ fts5WriteFlushLeaf(p, &writer); - pBuf = &writer.aWriter[0].buf; + pBuf = &writer.writer.buf; } if( iPos>=nCopy ) break; } @@ -4133,7 +4068,6 @@ static void fts5FlushOneHash(Fts5Index *p){ fts5StructurePromote(p, 0, pStruct); } - fts5IndexAutomerge(p, &pStruct, pgnoLast); fts5IndexCrisismerge(p, &pStruct); fts5StructureWrite(p, pStruct); @@ -4561,6 +4495,12 @@ int sqlite3Fts5IndexOpen( rc = sqlite3Fts5CreateTable( pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5CreateTable(pConfig, "idx", + "segid, term, pgno, PRIMARY KEY(segid, term)", + 1, pzErr + ); + } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexReinit(p); } @@ -4584,6 +4524,9 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ assert( p->pReader==0 ); sqlite3_finalize(p->pWriter); sqlite3_finalize(p->pDeleter); + sqlite3_finalize(p->pIdxWriter); + sqlite3_finalize(p->pIdxDeleter); + sqlite3_finalize(p->pIdxSelect); sqlite3Fts5HashFree(p->pHash); sqlite3Fts5BufferFree(&p->scratch); sqlite3_free(p->zDataTbl); @@ -4665,7 +4608,7 @@ int sqlite3Fts5IndexWrite( } /* -** Open a new iterator to iterate though all docids that match the +** Open a new iterator to iterate though all rowid that match the ** specified token or token prefix. */ int sqlite3Fts5IndexQuery( @@ -4933,92 +4876,6 @@ static u64 fts5IndexEntryCksum( return ret; } -static void fts5BtreeIterInit( - Fts5Index *p, - Fts5StructureSegment *pSeg, - Fts5BtreeIter *pIter -){ - int nByte; - int i; - nByte = sizeof(pIter->aLvl[0]) * (pSeg->nHeight-1); - memset(pIter, 0, sizeof(*pIter)); - if( nByte ){ - pIter->aLvl = (Fts5BtreeIterLevel*)fts5IdxMalloc(p, nByte); - } - if( p->rc==SQLITE_OK ){ - pIter->nLvl = pSeg->nHeight-1; - pIter->p = p; - pIter->pSeg = pSeg; - } - for(i=0; p->rc==SQLITE_OK && inLvl; i++){ - i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, i+1, 1); - Fts5Data *pData; - pIter->aLvl[i].pData = pData = fts5DataRead(p, iRowid); - if( pData ){ - fts5NodeIterInit(pData->p, pData->n, &pIter->aLvl[i].s); - } - } - - if( pIter->nLvl==0 || p->rc ){ - pIter->bEof = 1; - pIter->iLeaf = pSeg->pgnoLast; - }else{ - pIter->nEmpty = pIter->aLvl[0].s.nEmpty; - pIter->iLeaf = pIter->aLvl[0].s.iChild; - pIter->bDlidx = pIter->aLvl[0].s.bDlidx; - } -} - -static void fts5BtreeIterNext(Fts5BtreeIter *pIter){ - Fts5Index *p = pIter->p; - int i; - - assert( pIter->bEof==0 && pIter->aLvl[0].s.aData ); - for(i=0; inLvl && p->rc==SQLITE_OK; i++){ - Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i]; - fts5NodeIterNext(&p->rc, &pLvl->s); - if( pLvl->s.aData ){ - fts5BufferSet(&p->rc, &pIter->term, pLvl->s.term.n, pLvl->s.term.p); - break; - }else{ - fts5NodeIterFree(&pLvl->s); - fts5DataRelease(pLvl->pData); - pLvl->pData = 0; - } - } - if( i==pIter->nLvl || p->rc ){ - pIter->bEof = 1; - }else{ - int iSegid = pIter->pSeg->iSegid; - for(i--; i>=0; i--){ - Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i]; - i64 iRowid = FTS5_SEGMENT_ROWID(iSegid, i+1, pLvl[1].s.iChild); - pLvl->pData = fts5DataRead(p, iRowid); - if( pLvl->pData ){ - fts5NodeIterInit(pLvl->pData->p, pLvl->pData->n, &pLvl->s); - } - } - } - - pIter->nEmpty = pIter->aLvl[0].s.nEmpty; - pIter->bDlidx = pIter->aLvl[0].s.bDlidx; - pIter->iLeaf = pIter->aLvl[0].s.iChild; -} - -static void fts5BtreeIterFree(Fts5BtreeIter *pIter){ - int i; - for(i=0; inLvl; i++){ - Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i]; - fts5NodeIterFree(&pLvl->s); - if( pLvl->pData ){ - fts5DataRelease(pLvl->pData); - pLvl->pData = 0; - } - } - sqlite3_free(pIter->aLvl); - fts5BufferFree(&pIter->term); -} - #ifdef SQLITE_DEBUG /* ** This function is purely an internal test. It does not contribute to @@ -5166,33 +5023,73 @@ static void fts5TestTerm( # define fts5TestTerm(u,v,w,x,y,z) #endif +/* +** Check that: +** +** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and +** contain zero terms. +** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and +** contain zero rowids. +*/ +static void fts5IndexIntegrityCheckEmpty( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to check internal consistency */ + int iFirst, + int iNoRowid, + int iLast +){ + int i; + + /* Now check that the iter.nEmpty leaves following the current leaf + ** (a) exist and (b) contain no terms. */ + for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ + Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, i)); + if( pLeaf ){ + if( 0!=fts5GetU16(&pLeaf->p[2]) ) p->rc = FTS5_CORRUPT; + if( i>=iNoRowid && 0!=fts5GetU16(&pLeaf->p[0]) ) p->rc = FTS5_CORRUPT; + } + fts5DataRelease(pLeaf); + if( p->rc ) break; + } +} + static void fts5IndexIntegrityCheckSegment( Fts5Index *p, /* FTS5 backend object */ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ - Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */ + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pStmt = 0; + int rc2; + int iIdxPrevLeaf = pSeg->pgnoFirst-1; + int iDlidxPrevLeaf = pSeg->pgnoLast; if( pSeg->pgnoFirst==0 ) return; + fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( + "SELECT segid, term, (pgno>>1), (pgno & 1) FROM '%q'.'%q_idx' WHERE segid=%d", + pConfig->zDb, pConfig->zName, pSeg->iSegid + )); + /* Iterate through the b-tree hierarchy. */ - for(fts5BtreeIterInit(p, pSeg, &iter); - p->rc==SQLITE_OK && iter.bEof==0; - fts5BtreeIterNext(&iter) - ){ + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ i64 iRow; /* Rowid for this leaf */ Fts5Data *pLeaf; /* Data for this leaf */ int iOff; /* Offset of first term on leaf */ - int i; /* Used to iterate through empty leaves */ + + int nIdxTerm = sqlite3_column_bytes(pStmt, 1); + const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1); + int iIdxLeaf = sqlite3_column_int(pStmt, 2); + int bIdxDlidx = sqlite3_column_int(pStmt, 3); /* If the leaf in question has already been trimmed from the segment, ** ignore this b-tree entry. Otherwise, load it into memory. */ - if( iter.iLeafpgnoFirst ) continue; - iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iter.iLeaf); + if( iIdxLeafpgnoFirst ) continue; + iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iIdxLeaf); pLeaf = fts5DataRead(p, iRow); if( pLeaf==0 ) break; /* Check that the leaf contains at least one term, and that it is equal - ** to or larger than the split-key in iter.term. Also check that if there + ** to or larger than the split-key in zIdxTerm. Also check that if there ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ iOff = fts5GetU16(&pLeaf->p[2]); @@ -5208,8 +5105,8 @@ static void fts5IndexIntegrityCheckSegment( p->rc = FTS5_CORRUPT; }else{ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); - res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n)); - if( res==0 ) res = nTerm - iter.term.n; + res = memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); + if( res==0 ) res = nTerm - nIdxTerm; if( res<0 ) p->rc = FTS5_CORRUPT; } } @@ -5219,23 +5116,20 @@ static void fts5IndexIntegrityCheckSegment( /* Now check that the iter.nEmpty leaves following the current leaf ** (a) exist and (b) contain no terms. */ - for(i=1; p->rc==SQLITE_OK && i<=iter.nEmpty; i++){ - pLeaf = fts5DataRead(p, iRow+i); - if( pLeaf && 0!=fts5GetU16(&pLeaf->p[2]) ){ - p->rc = FTS5_CORRUPT; - } - fts5DataRelease(pLeaf); - } + fts5IndexIntegrityCheckEmpty( + p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1 + ); + if( p->rc ) break; /* If there is a doclist-index, check that it looks right. */ - if( iter.bDlidx ){ + if( bIdxDlidx ){ Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */ - int iPrevLeaf = iter.iLeaf; + int iPrevLeaf = iIdxLeaf; int iSegid = pSeg->iSegid; - int iPg; + int iPg = 0; i64 iKey; - for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iter.iLeaf); + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf); fts5DlidxIterEof(p, pDlidx)==0; fts5DlidxIterNext(p, pDlidx) ){ @@ -5268,26 +5162,26 @@ static void fts5IndexIntegrityCheckSegment( } } - for(iPg=iPrevLeaf+1; iPg<=(iter.iLeaf + iter.nEmpty); iPg++){ - iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg); - pLeaf = fts5DataRead(p, iKey); - if( pLeaf ){ - if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT; - fts5DataRelease(pLeaf); - } - } - + iDlidxPrevLeaf = iPg; fts5DlidxIterFree(pDlidx); - fts5TestDlidxReverse(p, iSegid, iter.iLeaf); + fts5TestDlidxReverse(p, iSegid, iIdxLeaf); + }else{ + iDlidxPrevLeaf = pSeg->pgnoLast; + /* TODO: Check there is no doclist index */ } + + iIdxPrevLeaf = iIdxLeaf; } + rc2 = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */ +#if 0 if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){ p->rc = FTS5_CORRUPT; } - - fts5BtreeIterFree(&iter); +#endif } diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 7afe0653e1..1f42716514 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -27,7 +27,6 @@ int sqlite3_fts5_may_be_corrupt = 1; typedef struct Fts5Table Fts5Table; typedef struct Fts5Cursor Fts5Cursor; -typedef struct Fts5Global Fts5Global; typedef struct Fts5Auxiliary Fts5Auxiliary; typedef struct Fts5Auxdata Fts5Auxdata; @@ -170,6 +169,11 @@ struct Fts5Sorter { */ struct Fts5Cursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ + int *aColumnSize; /* Values for xColumnSize() */ + i64 iCsrId; /* Cursor id */ + + /* Zero from this point onwards on cursor reset */ int ePlan; /* FTS5_PLAN_XXX value */ int bDesc; /* True for "ORDER BY rowid DESC" queries */ i64 iFirstRowid; /* Return no rowids earlier than this */ @@ -178,7 +182,6 @@ struct Fts5Cursor { Fts5Expr *pExpr; /* Expression for MATCH queries */ Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */ int csrflags; /* Mask of cursor flags (see below) */ - Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ i64 iSpecial; /* Result of special query */ /* "rank" function. Populated on demand from vtab.xColumn(). */ @@ -189,13 +192,13 @@ struct Fts5Cursor { sqlite3_value **apRankArg; /* Array of trailing arguments */ sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */ - /* Variables used by auxiliary functions */ - i64 iCsrId; /* Cursor id */ + /* Auxiliary data storage */ Fts5Auxiliary *pAux; /* Currently executing extension function */ Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ - int *aColumnSize; /* Values for xColumnSize() */ /* Cache used by auxiliary functions xInst() and xInstCount() */ + Fts5PoslistReader *aInstIter; /* One for each phrase */ + int nInstAlloc; /* Size of aInst[] array (entries / 3) */ int nInstCount; /* Number of phrase instances */ int *aInst; /* 3 integers per phrase instance */ }; @@ -430,12 +433,12 @@ static int fts5CreateMethod( /* ** The different query plans. */ -#define FTS5_PLAN_MATCH 0 /* ( MATCH ?) */ -#define FTS5_PLAN_SOURCE 1 /* A source cursor for SORTED_MATCH */ -#define FTS5_PLAN_SPECIAL 2 /* An internal query */ -#define FTS5_PLAN_SORTED_MATCH 3 /* ( MATCH ? ORDER BY rank) */ -#define FTS5_PLAN_SCAN 4 /* No usable constraint */ -#define FTS5_PLAN_ROWID 5 /* (rowid = ?) */ +#define FTS5_PLAN_MATCH 1 /* ( MATCH ?) */ +#define FTS5_PLAN_SOURCE 2 /* A source cursor for SORTED_MATCH */ +#define FTS5_PLAN_SPECIAL 3 /* An internal query */ +#define FTS5_PLAN_SORTED_MATCH 4 /* ( MATCH ? ORDER BY rank) */ +#define FTS5_PLAN_SCAN 5 /* No usable constraint */ +#define FTS5_PLAN_ROWID 6 /* (rowid = ?) */ /* ** Implementation of the xBestIndex method for FTS5 tables. Within the @@ -611,6 +614,45 @@ static void fts5CsrNewrow(Fts5Cursor *pCsr){ ); } +static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + Fts5Auxdata *pData; + Fts5Auxdata *pNext; + + sqlite3_free(pCsr->aInstIter); + sqlite3_free(pCsr->aInst); + if( pCsr->pStmt ){ + int eStmt = fts5StmtType(pCsr); + sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); + } + if( pCsr->pSorter ){ + Fts5Sorter *pSorter = pCsr->pSorter; + sqlite3_finalize(pSorter->pStmt); + sqlite3_free(pSorter); + } + + if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){ + sqlite3Fts5ExprFree(pCsr->pExpr); + } + + for(pData=pCsr->pAuxdata; pData; pData=pNext){ + pNext = pData->pNext; + if( pData->xDelete ) pData->xDelete(pData->pPtr); + sqlite3_free(pData); + } + + sqlite3_finalize(pCsr->pRankArgStmt); + sqlite3_free(pCsr->apRankArg); + + if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ + sqlite3_free(pCsr->zRank); + sqlite3_free(pCsr->zRankArgs); + } + + memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr)); +} + + /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. @@ -620,41 +662,12 @@ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; Fts5Cursor **pp; - Fts5Auxdata *pData; - Fts5Auxdata *pNext; - - sqlite3_free(pCsr->aInst); - if( pCsr->pStmt ){ - int eStmt = fts5StmtType(pCsr); - sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); - } - if( pCsr->pSorter ){ - Fts5Sorter *pSorter = pCsr->pSorter; - sqlite3_finalize(pSorter->pStmt); - sqlite3_free(pSorter); - } - - if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){ - sqlite3Fts5ExprFree(pCsr->pExpr); - } - - for(pData=pCsr->pAuxdata; pData; pData=pNext){ - pNext = pData->pNext; - if( pData->xDelete ) pData->xDelete(pData->pPtr); - sqlite3_free(pData); - } + fts5FreeCursorComponents(pCsr); /* Remove the cursor from the Fts5Global.pCsr list */ for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); *pp = pCsr->pNext; - sqlite3_finalize(pCsr->pRankArgStmt); - sqlite3_free(pCsr->apRankArg); - - if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ - sqlite3_free(pCsr->zRank); - sqlite3_free(pCsr->zRankArgs); - } sqlite3_free(pCsr); } return SQLITE_OK; @@ -758,11 +771,11 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; - assert( (pCsr->ePlan<2)== + assert( (pCsr->ePlan<3)== (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) ); - if( pCsr->ePlan<2 ){ + if( pCsr->ePlan<3 ){ int bSkip = 0; if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid); @@ -1043,6 +1056,11 @@ static int fts5FilterMethod( sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ char **pzErrmsg = pConfig->pzErrmsg; + if( pCsr->ePlan ){ + fts5FreeCursorComponents(pCsr); + memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); + } + assert( pCsr->pStmt==0 ); assert( pCsr->pExpr==0 ); assert( pCsr->csrflags==0 ); @@ -1174,7 +1192,7 @@ static i64 fts5CursorRowid(Fts5Cursor *pCsr){ /* ** This is the xRowid method. The SQLite core calls this routine to ** retrieve the rowid for the current row of the result set. fts5 -** exposes %_content.docid as the rowid for the virtual table. The +** exposes %_content.rowid as the rowid for the virtual table. The ** rowid should be written to *pRowid. */ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ @@ -1517,62 +1535,71 @@ static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ */ static int fts5CacheInstArray(Fts5Cursor *pCsr){ int rc = SQLITE_OK; - if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST) ){ - Fts5PoslistReader *aIter; /* One iterator for each phrase */ - int nIter; /* Number of iterators/phrases */ - int nByte; - - nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); - nByte = sizeof(Fts5PoslistReader) * nIter; - aIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte); - if( aIter ){ - Fts5Buffer buf = {0, 0, 0}; /* Build up aInst[] here */ - int nInst = 0; /* Number instances seen so far */ - int i; + Fts5PoslistReader *aIter; /* One iterator for each phrase */ + int nIter; /* Number of iterators/phrases */ + + nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); + if( pCsr->aInstIter==0 ){ + int nByte = sizeof(Fts5PoslistReader) * nIter; + pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte); + } + aIter = pCsr->aInstIter; - /* Initialize all iterators */ - for(i=0; iaInst); - pCsr->aInst = (int*)buf.p; - pCsr->nInstCount = nInst; - sqlite3_free(aIter); - CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); + /* Initialize all iterators */ + for(i=0; i=pCsr->nInstAlloc ){ + pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; + aInst = (int*)sqlite3_realloc( + pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 + ); + if( aInst ){ + pCsr->aInst = aInst; + }else{ + rc = SQLITE_NOMEM; + break; + } + } + + aInst = &pCsr->aInst[3 * (nInst-1)]; + aInst[0] = iBest; + aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); + aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); + sqlite3Fts5PoslistReaderNext(&aIter[iBest]); + } + + pCsr->nInstCount = nInst; + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); } return rc; } static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - int rc; - if( SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ *pnInst = pCsr->nInstCount; } return rc; @@ -1586,8 +1613,10 @@ static int fts5ApiInst( int *piOff ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - int rc; - if( SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) + ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; }else{ @@ -1742,12 +1771,47 @@ static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){ return pRet; } +static void fts5ApiPhraseNext( + Fts5Context *pCtx, + Fts5PhraseIter *pIter, + int *piCol, int *piOff +){ + if( pIter->a>=pIter->b ){ + *piCol = -1; + *piOff = -1; + }else{ + int iVal; + pIter->a += fts5GetVarint32(pIter->a, iVal); + if( iVal==1 ){ + pIter->a += fts5GetVarint32(pIter->a, iVal); + *piCol = iVal; + *piOff = 0; + pIter->a += fts5GetVarint32(pIter->a, iVal); + } + *piOff += (iVal-2); + } +} + +static void fts5ApiPhraseFirst( + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, + int *piCol, int *piOff +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a); + pIter->b = &pIter->a[n]; + *piCol = 0; + *piOff = 0; + fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); +} + static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); static const Fts5ExtensionApi sFts5Api = { - 1, /* iVersion */ + 2, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, @@ -1763,6 +1827,8 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiQueryPhrase, fts5ApiSetAuxdata, fts5ApiGetAuxdata, + fts5ApiPhraseFirst, + fts5ApiPhraseNext, }; diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index da822ffad2..f09b7d9158 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -180,8 +180,10 @@ static int fts5ExecPrintf( int sqlite3Fts5DropAll(Fts5Config *pConfig){ int rc = fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_data';" + "DROP TABLE IF EXISTS %Q.'%q_idx';" "DROP TABLE IF EXISTS %Q.'%q_config';", pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ @@ -218,6 +220,7 @@ int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){ int rc = sqlite3Fts5StorageSync(pStorage, 1); fts5StorageRenameOne(pConfig, &rc, "data", zName); + fts5StorageRenameOne(pConfig, &rc, "idx", zName); fts5StorageRenameOne(pConfig, &rc, "config", zName); if( pConfig->bColumnsize ){ fts5StorageRenameOne(pConfig, &rc, "docsize", zName); @@ -601,7 +604,9 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ /* Delete the contents of the %_data and %_docsize tables. */ rc = fts5ExecPrintf(pConfig->db, 0, - "DELETE FROM %Q.'%q_data';", + "DELETE FROM %Q.'%q_data';" + "DELETE FROM %Q.'%q_idx';", + pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 6ff91cdceb..82f3e0390d 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -23,6 +23,7 @@ #include extern int sqlite3_fts5_may_be_corrupt; +extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *); /************************************************************************* ** This is a copy of the first part of the SqliteDb structure in @@ -946,6 +947,31 @@ static int f5tTokenHash( return TCL_OK; } +static int f5tRegisterMatchinfo( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + sqlite3 *db = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( f5tDbPointer(interp, objv[1], &db) ){ + return TCL_ERROR; + } + + rc = sqlite3Fts5TestRegisterMatchinfo(db); + if( rc!=SQLITE_OK ){ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + return TCL_ERROR; + } + return TCL_OK; +} + /* ** Entry point. */ @@ -955,12 +981,13 @@ int Fts5tcl_Init(Tcl_Interp *interp){ Tcl_ObjCmdProc *xProc; int bTokenizeCtx; } aCmd[] = { - { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, - { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, - { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, - { "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, - { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, - { "sqlite3_fts5_token_hash", f5tTokenHash, 0 } + { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, + { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, + { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, + { "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, + { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, + { "sqlite3_fts5_token_hash", f5tTokenHash, 0 }, + { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 } }; int i; F5tTokenizerContext *pContext; diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c new file mode 100644 index 0000000000..8ebf5f5077 --- /dev/null +++ b/ext/fts5/fts5_test_mi.c @@ -0,0 +1,406 @@ +/* +** 2015 Aug 04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains test code only, it is not included in release +** versions of FTS5. It contains the implementation of an FTS5 auxiliary +** function very similar to the FTS4 function matchinfo(): +** +** https://www.sqlite.org/fts3.html#matchinfo +** +** Known differences are that: +** +** 1) this function uses the FTS5 definition of "matchable phrase", which +** excludes any phrases that are part of an expression sub-tree that +** does not match the current row. This comes up for MATCH queries +** such as: +** +** "a OR (b AND c)" +** +** In FTS4, if a single row contains instances of tokens "a" and "c", +** but not "b", all instances of "c" are considered matches. In FTS5, +** they are not (as the "b AND c" sub-tree does not match the current +** row. +** +** 2) For the values returned by 'x' that apply to all rows of the table, +** NEAR constraints are not considered. But for the number of hits in +** the current row, they are. +** +** This file exports a single function that may be called to register the +** matchinfo() implementation with a database handle: +** +** int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db); +*/ + + +#ifdef SQLITE_TEST +#ifdef SQLITE_ENABLE_FTS5 + +#include "fts5.h" +#include +#include +#include + +typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx; +typedef unsigned int u32; + +struct Fts5MatchinfoCtx { + int nCol; /* Number of cols in FTS5 table */ + int nPhrase; /* Number of phrases in FTS5 query */ + char *zArg; /* nul-term'd copy of 2nd arg */ + int nRet; /* Number of elements in aRet[] */ + u32 *aRet; /* Array of 32-bit unsigned ints to return */ +}; + + + +/* +** Return a pointer to the fts5_api pointer for database connection db. +** If an error occurs, return NULL and leave an error in the database +** handle (accessible using sqlite3_errcode()/errmsg()). +*/ +static fts5_api *fts5_api_from_db(sqlite3 *db){ + fts5_api *pRet = 0; + sqlite3_stmt *pStmt = 0; + + if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0) + && SQLITE_ROW==sqlite3_step(pStmt) + && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0) + ){ + memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet)); + } + sqlite3_finalize(pStmt); + return pRet; +} + + +/* +** Argument f should be a flag accepted by matchinfo() (a valid character +** in the string passed as the second argument). If it is not, -1 is +** returned. Otherwise, if f is a valid matchinfo flag, the value returned +** is the number of 32-bit integers added to the output array if the +** table has nCol columns and the query nPhrase phrases. +*/ +static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){ + int ret = -1; + switch( f ){ + case 'p': ret = 1; break; + case 'c': ret = 1; break; + case 'x': ret = 3 * nCol * nPhrase; break; + case 'y': ret = nCol * nPhrase; break; + case 'b': ret = ((nCol + 31) / 32) * nPhrase; break; + case 'n': ret = 1; break; + case 'a': ret = nCol; break; + case 'l': ret = nCol; break; + case 's': ret = nCol; break; + } + return ret; +} + +static int fts5MatchinfoIter( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + Fts5MatchinfoCtx *p, + int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*) +){ + int i; + int n = 0; + int rc = SQLITE_OK; + char f; + for(i=0; (f = p->zArg[i]); i++){ + rc = x(pApi, pFts, p, f, &p->aRet[n]); + if( rc!=SQLITE_OK ) break; + n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f); + } + return rc; +} + +static int fts5MatchinfoXCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + void *pUserData +){ + Fts5PhraseIter iter; + int iCol, iOff; + u32 *aOut = (u32*)pUserData; + int iPrev = -1; + + for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); + iOff>=0; + pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) + ){ + aOut[iCol*3+1]++; + if( iCol!=iPrev ) aOut[iCol*3 + 2]++; + iPrev = iCol; + } + + return SQLITE_OK; +} + +static int fts5MatchinfoGlobalCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5MatchinfoCtx *p, + char f, + u32 *aOut +){ + int rc = SQLITE_OK; + switch( f ){ + case 'p': + aOut[0] = p->nPhrase; + break; + + case 'c': + aOut[0] = p->nCol; + break; + + case 'x': { + int i; + for(i=0; inPhrase && rc==SQLITE_OK; i++){ + void *pPtr = (void*)&aOut[i * p->nCol * 3]; + rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb); + } + break; + } + + case 'n': { + sqlite3_int64 nRow; + rc = pApi->xRowCount(pFts, &nRow); + aOut[0] = (u32)nRow; + break; + } + + case 'a': { + sqlite3_int64 nRow = 0; + rc = pApi->xRowCount(pFts, &nRow); + if( nRow==0 ){ + memset(aOut, 0, sizeof(u32) * p->nCol); + }else{ + int i; + for(i=0; rc==SQLITE_OK && inCol; i++){ + sqlite3_int64 nToken; + rc = pApi->xColumnTotalSize(pFts, i, &nToken); + if( rc==SQLITE_OK){ + aOut[i] = (u32)((2*nToken + nRow) / (2*nRow)); + } + } + } + break; + } + + } + return rc; +} + +static int fts5MatchinfoLocalCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5MatchinfoCtx *p, + char f, + u32 *aOut +){ + int i; + int rc = SQLITE_OK; + + switch( f ){ + case 'b': + case 'x': + case 'y': { + int nMul = (f=='x' ? 3 : 1); + int iPhrase; + + if( f=='b' ){ + int nInt = ((p->nCol + 31) / 32) * p->nPhrase; + for(i=0; inCol*p->nPhrase); i++) aOut[i*nMul] = 0; + } + + for(iPhrase=0; iPhrasenPhrase; iPhrase++){ + Fts5PhraseIter iter; + int iOff, iCol; + for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); + iOff>=0; + pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) + ){ + if( f=='b' ){ + aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); + }else{ + aOut[nMul * (iCol + iPhrase * p->nCol)]++; + } + } + } + + break; + } + + case 'l': { + for(i=0; rc==SQLITE_OK && inCol; i++){ + int nToken; + rc = pApi->xColumnSize(pFts, i, &nToken); + aOut[i] = (u32)nToken; + } + break; + } + + case 's': { + int nInst; + + memset(aOut, 0, sizeof(u32) * p->nCol); + + rc = pApi->xInstCount(pFts, &nInst); + for(i=0; rc==SQLITE_OK && ixInst(pFts, i, &iPhrase, &iCol, &iOff); + iNextPhrase = iPhrase+1; + iNextOff = iOff+pApi->xPhraseSize(pFts, 0); + for(j=i+1; rc==SQLITE_OK && jxInst(pFts, j, &ip, &ic, &io); + if( ic!=iCol || io>iNextOff ) break; + if( ip==iNextPhrase && io==iNextOff ){ + nSeq++; + iNextPhrase = ip+1; + iNextOff = io + pApi->xPhraseSize(pFts, ip); + } + } + + if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq; + } + + break; + } + } + return rc; +} + +static Fts5MatchinfoCtx *fts5MatchinfoNew( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning error message */ + const char *zArg /* Matchinfo flag string */ +){ + Fts5MatchinfoCtx *p; + int nCol; + int nPhrase; + int i; + int nInt; + int nByte; + int rc; + + nCol = pApi->xColumnCount(pFts); + nPhrase = pApi->xPhraseCount(pFts); + + nInt = 0; + for(i=0; zArg[i]; i++){ + int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]); + if( n<0 ){ + char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return 0; + } + nInt += n; + } + + nByte = sizeof(Fts5MatchinfoCtx) /* The struct itself */ + + sizeof(u32) * nInt /* The p->aRet[] array */ + + (i+1); /* The p->zArg string */ + p = (Fts5MatchinfoCtx*)sqlite3_malloc(nByte); + if( p==0 ){ + sqlite3_result_error_nomem(pCtx); + return 0; + } + memset(p, 0, nByte); + + p->nCol = nCol; + p->nPhrase = nPhrase; + p->aRet = (u32*)&p[1]; + p->nRet = nInt; + p->zArg = (char*)&p->aRet[nInt]; + memcpy(p->zArg, zArg, i); + + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(p); + p = 0; + } + + return p; +} + +static void fts5MatchinfoFunc( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + const char *zArg; + Fts5MatchinfoCtx *p; + int rc; + + if( nVal>0 ){ + zArg = (const char*)sqlite3_value_text(apVal[0]); + }else{ + zArg = "pcx"; + } + + p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0); + if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){ + p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg); + pApi->xSetAuxdata(pFts, p, sqlite3_free); + if( p==0 ) return; + } + + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + }else{ + /* No errors has occured, so return a copy of the array of integers. */ + int nByte = p->nRet * sizeof(u32); + sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT); + } +} + +int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ + int rc; /* Return code */ + fts5_api *pApi; /* FTS5 API functions */ + + /* Extract the FTS5 API pointer from the database handle. The + ** fts5_api_from_db() function above is copied verbatim from the + ** FTS5 documentation. Refer there for details. */ + pApi = fts5_api_from_db(db); + + /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered + ** with this database handle, or an error (OOM perhaps?) has occurred. + ** + ** Also check that the fts5_api object is version 2 or newer. + */ + if( pApi==0 || pApi->iVersion<1 ){ + return SQLITE_ERROR; + } + + /* Register the implementation of matchinfo() */ + rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0); + + return rc; +} + +#endif /* SQLITE_ENABLE_FTS5 */ +#endif /* SQLITE_TEST */ + diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index afaa3034ce..426e35551b 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -537,7 +537,9 @@ static int fts5PorterCreate( rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ - rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer); + int nArg2 = (nArg>0 ? nArg-1 : 0); + const char **azArg2 = (nArg2 ? &azArg[1] : 0); + rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer); } if( rc!=SQLITE_OK ){ diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 609beb0b30..bdf2e36b63 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -442,8 +442,7 @@ static int fts5VocabColumnMethod( /* ** This is the xRowid method. The SQLite core calls this routine to -** retrieve the rowid for the current row of the result set. fts5 -** exposes %_content.docid as the rowid for the virtual table. The +** retrieve the rowid for the current row of the result set. The ** rowid should be written to *pRowid. */ static int fts5VocabRowidMethod( diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 22aa3187fe..59058a8462 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -93,6 +93,10 @@ proc fts5_test_queryphrase {cmd} { set res } +proc fts5_test_phrasecount {cmd} { + $cmd xPhraseCount +} + proc fts5_test_all {cmd} { set res [list] lappend res columnsize [fts5_test_columnsize $cmd] @@ -115,6 +119,7 @@ proc fts5_aux_test_functions {db} { fts5_test_all fts5_test_queryphrase + fts5_test_phrasecount } { sqlite3_fts5_create_function $db $f $f } diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 70e086e8c9..daa535cd9b 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -27,6 +27,7 @@ do_execsql_test 1.0 { } { t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)} t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)} + t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID} t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)} t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID} @@ -47,9 +48,10 @@ do_execsql_test 2.0 { do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); } + do_test 2.2 { execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } -} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=1 leaves=1..1}}}/} +} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=0 leaves=1..1}}}/} foreach w {a b c d e f} { do_execsql_test 2.3.$w.asc { diff --git a/ext/fts5/test/fts5ae.test b/ext/fts5/test/fts5ae.test index e32e1a3b6e..ded73d472f 100644 --- a/ext/fts5/test/fts5ae.test +++ b/ext/fts5/test/fts5ae.test @@ -277,5 +277,28 @@ foreach {tn q res} { } $res } +#------------------------------------------------------------------------- +# Test xPhraseCount() for some different queries. +# +do_test 9.1 { + execsql { CREATE VIRTUAL TABLE t9 USING fts5(x) } + foreach x { + "a b c" "d e f" + } { + execsql { INSERT INTO t9 VALUES($x) } + } +} {} + +foreach {tn q cnt} { + 1 {a AND b} 2 + 2 {a OR b} 2 + 3 {a OR b OR c} 3 + 4 {NEAR(a b)} 2 +} { + do_execsql_test 9.2.$tn { + SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1 + } $cnt +} + finish_test diff --git a/ext/fts5/test/fts5al.test b/ext/fts5/test/fts5al.test index cc6435bb0e..99dfeb357b 100644 --- a/ext/fts5/test/fts5al.test +++ b/ext/fts5/test/fts5al.test @@ -26,17 +26,17 @@ ifcapable !fts5 { do_execsql_test 1.1 { CREATE VIRTUAL TABLE ft1 USING fts5(x); SELECT * FROM ft1_config; -} {version 2} +} {version 3} do_execsql_test 1.2 { INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32); SELECT * FROM ft1_config; -} {pgsz 32 version 2} +} {pgsz 32 version 3} do_execsql_test 1.3 { INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); SELECT * FROM ft1_config; -} {pgsz 64 version 2} +} {pgsz 64 version 3} #-------------------------------------------------------------------------- # Test the logic for parsing the rank() function definition. diff --git a/ext/fts5/test/fts5content.test b/ext/fts5/test/fts5content.test index f87aa3d947..69e66a54f8 100644 --- a/ext/fts5/test/fts5content.test +++ b/ext/fts5/test/fts5content.test @@ -247,7 +247,7 @@ reset_db do_execsql_test 6.1 { CREATE VIRTUAL TABLE xx USING fts5(x, y, content=""); SELECT name FROM sqlite_master; -} {xx xx_data xx_docsize xx_config} +} {xx xx_data xx_idx xx_docsize xx_config} do_execsql_test 6.2 { DROP TABLE xx; SELECT name FROM sqlite_master; diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index 987cb5ef19..352e1b4a17 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -46,6 +46,13 @@ foreach {tn expr res} { do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] } +do_catchsql_test 2.1 { + SELECT fts5_expr() +} {1 {wrong number of arguments to function fts5_expr}} + +do_catchsql_test 2.1 { + SELECT fts5_expr_tcl() +} {1 {wrong number of arguments to function fts5_expr_tcl}} finish_test diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test new file mode 100644 index 0000000000..359702eff6 --- /dev/null +++ b/ext/fts5/test/fts5matchinfo.test @@ -0,0 +1,457 @@ +# 2015 August 05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5matchinfo + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { finish_test ; return } + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + +sqlite3_fts5_register_matchinfo db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(content); +} + +do_execsql_test 1.1 { + INSERT INTO t1(content) VALUES('I wandered lonely as a cloud'); + INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,'); + INSERT INTO t1(content) VALUES('When all at once I saw a crowd,'); + INSERT INTO t1(content) VALUES('A host, of golden daffodils,'); + SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I'; +} {{1 1 1 2 2} {1 1 1 2 2}} + +# Now create an FTS4 table that does not specify matchinfo=fts3. +# +do_execsql_test 1.2 { + CREATE VIRTUAL TABLE t2 USING fts5(content); + INSERT INTO t2 SELECT * FROM t1; + SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I'; +} {{1 1 1 2 2} {1 1 1 2 2}} + + +#-------------------------------------------------------------------------- +# Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function. +# +# The first argument - $tn - is a test identifier. This may be either a +# full identifier (i.e. "fts3matchinfo-1.1") or, if global var $testprefix +# is set, just the numeric component (i.e. "1.1"). +# +# The second argument is the name of an FTSX table. The third is the +# full text of a WHERE/MATCH expression to query the table for +# (i.e. "t1 MATCH 'abc'"). The final argument - $results - should be a +# key-value list (serialized array) with matchinfo() format specifiers +# as keys, and the results of executing the statement: +# +# SELECT matchinfo($tbl, '$key') FROM $tbl WHERE $expr +# +# For example: +# +# CREATE VIRTUAL TABLE t1 USING fts4; +# INSERT INTO t1 VALUES('abc'); +# INSERT INTO t1 VALUES('def'); +# INSERT INTO t1 VALUES('abc abc'); +# +# do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" { +# n {3 3} +# p {1 1} +# c {1 1} +# x {{1 3 2} {2 3 2}} +# } +# +# If the $results list contains keys mapped to "-" instead of a matchinfo() +# result, then this command computes the expected results based on other +# mappings to test the matchinfo() function. For example, the command above +# could be changed to: +# +# do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" { +# n {3 3} p {1 1} c {1 1} x {{1 3 2} {2 3 2}} +# pcx - +# } +# +# And this command would compute the expected results for matchinfo(t1, 'pcx') +# based on the results of matchinfo(t1, 'p'), matchinfo(t1, 'c') and +# matchinfo(t1, 'x') in order to test 'pcx'. +# +proc do_matchinfo_test {tn tbl expr results} { + + foreach {fmt res} $results { + if {$res == "-"} continue + set resarray($fmt) $res + } + + set nRow 0 + foreach {fmt res} [array get resarray] { + if {[llength $res]>$nRow} { set nRow [llength $res] } + } + + # Construct expected results for any formats for which the caller + # supplied result is "-". + # + foreach {fmt res} $results { + if {$res == "-"} { + set res [list] + for {set iRow 0} {$iRow<$nRow} {incr iRow} { + set rowres [list] + foreach c [split $fmt ""] { + set rowres [concat $rowres [lindex $resarray($c) $iRow]] + } + lappend res $rowres + } + set resarray($fmt) $res + } + } + + # Test each matchinfo() request individually. + # + foreach {fmt res} [array get resarray] { + set sql "SELECT mit(matchinfo($tbl, '$fmt')) FROM $tbl WHERE $expr" + do_execsql_test $tn.$fmt $sql [normalize2 $res] + } + + # Test them all executed together (multiple invocations of matchinfo()). + # + set exprlist [list] + foreach {format res} [array get resarray] { + lappend exprlist "mit(matchinfo($tbl, '$format'))" + } + set allres [list] + for {set iRow 0} {$iRow<$nRow} {incr iRow} { + foreach {format res} [array get resarray] { + lappend allres [lindex $res $iRow] + } + } + set sql "SELECT [join $exprlist ,] FROM $tbl WHERE $expr" + do_execsql_test $tn.multi $sql [normalize2 $allres] +} +proc normalize2 {list_of_lists} { + set res [list] + foreach elem $list_of_lists { + lappend res [list {*}$elem] + } + return $res +} + + +do_execsql_test 4.1.0 { + CREATE VIRTUAL TABLE t4 USING fts5(x, y); + INSERT INTO t4 VALUES('a b c d e', 'f g h i j'); + INSERT INTO t4 VALUES('f g h i j', 'a b c d e'); +} + +do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { + s {{3 0} {0 3}} +} + +do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { + p {3 3} + x { + {1 1 1 0 1 1 1 1 1 0 1 1 1 1 1 0 1 1} + {0 1 1 1 1 1 0 1 1 1 1 1 0 1 1 1 1 1} + } +} + +do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { + p {3 3} + c {2 2} + x { + {1 1 1 0 1 1 1 1 1 0 1 1 1 1 1 0 1 1} + {0 1 1 1 1 1 0 1 1 1 1 1 0 1 1 1 1 1} + } + n {2 2} + l {{5 5} {5 5}} + a {{5 5} {5 5}} + + s {{3 0} {0 3}} + + xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - + xpxsscplax - +} + +do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} { + p {1 1} + c {2 2} + x { + {0 1 1 1 1 1} + {1 1 1 0 1 1} + } + n {2 2} + l {{5 5} {5 5}} + a {{5 5} {5 5}} + + s {{0 1} {1 0}} + + xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - + sxsxs - +} + +do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} } +do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } +do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } +do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} } +do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} { + x { + {0 1 1 1 1 1 0 0 0 0 0 0} + {1 1 1 0 1 1 0 0 0 0 0 0} + } +} +do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f NOT abcd'} { + x { + {0 1 1 1 1 1 0 0 0 0 0 0} + {1 1 1 0 1 1 0 0 0 0 0 0} + } +} + +do_execsql_test 4.2.0 { + CREATE VIRTUAL TABLE t5 USING fts5(content); + INSERT INTO t5 VALUES('a a a a a'); + INSERT INTO t5 VALUES('a b a b a'); + INSERT INTO t5 VALUES('c b c b c'); + INSERT INTO t5 VALUES('x x x x x'); +} +do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'} { + x {{5 8 2 5 8 2} {3 8 2 3 8 2}} + s {2 1} +} +do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'} { s {2} } +do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'} { s {3} } +do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'} { s {3 1} } +do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } +do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} } + +do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')"; + +# It used to be that the second 'a' token would be deferred. That doesn't +# work any longer. +if 0 { + do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { + x {{5 8 2 5 5 5} {3 8 2 3 5 5}} + s {2 1} + } +} + +do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} } +do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'} { s {3} } +do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'} { s {3 1} } +do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } +do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1 1} } + +do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') } + +do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } +do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'} { s {2 1} } +do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } +do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'} { s {3} } +do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'} { s {3 1} } +do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } + +do_execsql_test 4.5.0 { + CREATE VIRTUAL TABLE t6 USING fts5(a, b, c); + INSERT INTO t6 VALUES('a', 'b', 'c'); +} +do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} } + + +#------------------------------------------------------------------------- +# Test the outcome of matchinfo() when used within a query that does not +# use the full-text index (i.e. lookup by rowid or full-table scan). +# +do_execsql_test 7.1 { + CREATE VIRTUAL TABLE t10 USING fts5(content); + INSERT INTO t10 VALUES('first record'); + INSERT INTO t10 VALUES('second record'); +} +do_execsql_test 7.2 { + SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10; +} {blob 8 blob 8} +do_execsql_test 7.3 { + SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10 WHERE rowid=1; +} {blob 8} +do_execsql_test 7.4 { + SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) + FROM t10 WHERE t10 MATCH 'record' +} {blob 20 blob 20} + +#------------------------------------------------------------------------- +# Test a special case - matchinfo('nxa') with many zero length documents. +# Special because "x" internally uses a statement used by both "n" and "a". +# This was causing a problem at one point in the obscure case where the +# total number of bytes of data stored in an fts3 table was greater than +# the number of rows. i.e. when the following query returns true: +# +# SELECT sum(length(content)) < count(*) FROM fts4table; +# +do_execsql_test 8.1 { + CREATE VIRTUAL TABLE t11 USING fts5(content); + INSERT INTO t11(t11, rank) VALUES('pgsz', 32); + INSERT INTO t11 VALUES('quitealongstringoftext'); + INSERT INTO t11 VALUES('anotherquitealongstringoftext'); + INSERT INTO t11 VALUES('athirdlongstringoftext'); + INSERT INTO t11 VALUES('andonemoreforgoodluck'); +} +do_test 8.2 { + for {set i 0} {$i < 200} {incr i} { + execsql { INSERT INTO t11 VALUES('') } + } + execsql { INSERT INTO t11(t11) VALUES('optimize') } +} {} +do_execsql_test 8.3 { + SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*' +} {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}} + +#------------------------------------------------------------------------- + +do_execsql_test 9.1 { + CREATE VIRTUAL TABLE t12 USING fts5(content); + INSERT INTO t12 VALUES('a b c d'); + SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; +} {{0 1 1 0 1 1 1 1 1}} +do_execsql_test 9.2 { + INSERT INTO t12 VALUES('a d c d'); + SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; +} { + {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2} +} +do_execsql_test 9.3 { + INSERT INTO t12 VALUES('a d d a'); + SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; +} { + {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3} +} + +#--------------------------------------------------------------------------- +# Test for a memory leak +# +do_execsql_test 10.1 { + DROP TABLE t10; + CREATE VIRTUAL TABLE t10 USING fts5(idx, value); + INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three'); + SELECT t10.rowid, t10.* + FROM t10 + JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x + WHERE t10 MATCH x.idx + AND matchinfo(t10) not null + GROUP BY t10.rowid + ORDER BY 1; +} {1 1 one 2 2 two 3 3 three} + +#--------------------------------------------------------------------------- +# Test the 'y' matchinfo flag +# +set sqlite_fts3_enable_parentheses 1 +reset_db +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE tt USING fts3(x, y); + INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1 + INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2 + INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3 + INSERT INTO tt VALUES('a c f f g d', 'd b f d e g'); -- 4 + INSERT INTO tt VALUES('g a c f c f', 'd g g b c c'); -- 5 + INSERT INTO tt VALUES('g a c e b b', 'd b f b g g'); -- 6 + INSERT INTO tt VALUES('f d a a f c', 'e e a d c f'); -- 7 + INSERT INTO tt VALUES('a c b b g f', 'a b a e d f'); -- 8 + INSERT INTO tt VALUES('b a f e c c', 'f d b b a b'); -- 9 + INSERT INTO tt VALUES('f d c e a c', 'f a f a a f'); -- 10 +} + +db func mit mit +foreach {tn expr res} { + 1 "a" { + 1 {1 2} 2 {1 0} 3 {0 1} 4 {1 0} 5 {1 0} + 6 {1 0} 7 {2 1} 8 {1 2} 9 {1 1} 10 {1 3} + } + + 2 "b" { + 1 {0 1} 2 {1 0} 3 {1 2} 4 {0 1} 5 {0 1} + 6 {2 2} 8 {2 1} 9 {1 3} + } + + 3 "y:a" { + 1 {0 2} 3 {0 1} + 7 {0 1} 8 {0 2} 9 {0 1} 10 {0 3} + } + + 4 "x:a" { + 1 {1 0} 2 {1 0} 4 {1 0} 5 {1 0} + 6 {1 0} 7 {2 0} 8 {1 0} 9 {1 0} 10 {1 0} + } + + 5 "a OR b" { + 1 {1 2 0 1} 2 {1 0 1 0} 3 {0 1 1 2} 4 {1 0 0 1} 5 {1 0 0 1} + 6 {1 0 2 2} 7 {2 1 0 0} 8 {1 2 2 1} 9 {1 1 1 3} 10 {1 3 0 0} + } + + 6 "a AND b" { + 1 {1 2 0 1} 2 {1 0 1 0} 3 {0 1 1 2} 4 {1 0 0 1} 5 {1 0 0 1} + 6 {1 0 2 2} 8 {1 2 2 1} 9 {1 1 1 3} + } + + 7 "a OR (a AND b)" { + 1 {1 2 1 2 0 1} 2 {1 0 1 0 1 0} 3 {0 1 0 1 1 2} 4 {1 0 1 0 0 1} + 5 {1 0 1 0 0 1} 6 {1 0 1 0 2 2} 7 {2 1 0 0 0 0} 8 {1 2 1 2 2 1} + 9 {1 1 1 1 1 3} 10 {1 3 0 0 0 0} + } + +} { + do_execsql_test 11.1.$tn.1 { + SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr + } $res + + set r2 [list] + foreach {rowid L} $res { + lappend r2 $rowid + set M [list] + foreach {a b} $L { + lappend M [expr ($a ? 1 : 0) + ($b ? 2 : 0)] + } + lappend r2 $M + } + + do_execsql_test 11.1.$tn.2 { + SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr + } $r2 + + do_execsql_test 11.1.$tn.2 { + SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr + } $r2 +} +set sqlite_fts3_enable_parentheses 0 + +#--------------------------------------------------------------------------- +# Test the 'b' matchinfo flag +# +set sqlite_fts3_enable_parentheses 1 +reset_db +db func mit mit + +do_test 12.0 { + set cols [list] + for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } + execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])" +} {} + +do_execsql_test 12.1 { + INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc'); + SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; +} [list [list [expr 1<<4] [expr 1<<(45-32)]]] + +set sqlite_fts3_enable_parentheses 0 +finish_test + diff --git a/ext/fts5/test/fts5tokenizer.test b/ext/fts5/test/fts5tokenizer.test index 0246419572..9316d3c234 100644 --- a/ext/fts5/test/fts5tokenizer.test +++ b/ext/fts5/test/fts5tokenizer.test @@ -247,5 +247,20 @@ do_execsql_test 8.2 [subst { brown dog fox jumped lazy over quick the \u0E08 \u0E09 }] +# Test that the porter tokenizer correctly passes arguments through to +# its parent tokenizer. +do_execsql_test 8.3 { + BEGIN; + CREATE VIRTUAL TABLE e6 USING fts5(x, + tokenize="porter unicode61 separators ABCDEFGHIJKLMNOPQRSTUVWXYZ" + ); + INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog'); + CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row'); + SELECT term FROM e7; + ROLLBACK; +} { + brown dog fox jump lazi over quick the +} + finish_test diff --git a/ext/fts5/test/fts5version.test b/ext/fts5/test/fts5version.test index 1ae75fd2ae..8c5a772146 100644 --- a/ext/fts5/test/fts5version.test +++ b/ext/fts5/test/fts5version.test @@ -30,34 +30,34 @@ do_execsql_test 1.1 { do_execsql_test 1.2 { SELECT * FROM t1_config WHERE k='version' -} {version 2} +} {version 3} do_execsql_test 1.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'a'; } {1} do_execsql_test 1.4 { - UPDATE t1_config set v=3 WHERE k='version'; + UPDATE t1_config set v=4 WHERE k='version'; } do_test 1.5 { db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } -} {1 {invalid fts5 file format (found 3, expected 2) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 4, expected 3) - run 'rebuild'}} do_test 1.6 { db close sqlite3 db test.db catchsql { INSERT INTO t1 VALUES('x y z') } -} {1 {invalid fts5 file format (found 3, expected 2) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 4, expected 3) - run 'rebuild'}} do_test 1.7 { execsql { DELETE FROM t1_config WHERE k='version' } db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } -} {1 {invalid fts5 file format (found 0, expected 2) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 0, expected 3) - run 'rebuild'}} finish_test diff --git a/ext/misc/series.c b/ext/misc/series.c new file mode 100644 index 0000000000..21f95ccb74 --- /dev/null +++ b/ext/misc/series.c @@ -0,0 +1,403 @@ +/* +** 2015-08-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates how to create a table-valued-function using +** a virtual table. This demo implements the generate_series() function +** which gives similar results to the eponymous function in PostgreSQL. +** Examples: +** +** SELECT * FROM generate_series(0,100,5); +** +** The query above returns integers from 0 through 100 counting by steps +** of 5. +** +** SELECT * FROM generate_series(0,100); +** +** Integers from 0 through 100 with a step size of 1. +** +** SELECT * FROM generate_series(20) LIMIT 10; +** +** Integers 20 through 29. +** +** HOW IT WORKS +** +** The generate_series "function" is really a virtual table with the +** following schema: +** +** CREATE FUNCTION generate_series( +** value, +** start HIDDEN, +** stop HIDDEN, +** step HIDDEN +** ); +** +** Function arguments in queries against this virtual table are translated +** into equality constraints against successive hidden columns. In other +** words, the following pairs of queries are equivalent to each other: +** +** SELECT * FROM generate_series(0,100,5); +** SELECT * FROM generate_series WHERE start=0 AND stop=100 AND step=5; +** +** SELECT * FROM generate_series(0,100); +** SELECT * FROM generate_series WHERE start=0 AND stop=100; +** +** SELECT * FROM generate_series(20) LIMIT 10; +** SELECT * FROM generate_series WHERE start=20 LIMIT 10; +** +** The generate_series virtual table implementation leaves the xCreate method +** set to NULL. This means that it is not possible to do a CREATE VIRTUAL +** TABLE command with "generate_series" as the USING argument. Instead, there +** is a single generate_series virtual table that is always available without +** having to be created first. +** +** The xBestIndex method looks for equality constraints against the hidden +** start, stop, and step columns, and if present, it uses those constraints +** to bound the sequence of generated values. If the equality constraints +** are missing, it uses 0 for start, 4294967295 for stop, and 1 for step. +** xBestIndex returns a small cost when both start and stop are available, +** and a very large cost if either start or stop are unavailable. This +** encourages the query planner to order joins such that the bounds of the +** series are well-defined. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + + +/* series_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct series_cursor series_cursor; +struct series_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + int isDesc; /* True to count down rather than up */ + sqlite3_int64 iRowid; /* The rowid */ + sqlite3_int64 iValue; /* Current value ("value") */ + sqlite3_int64 mnValue; /* Mimimum value ("start") */ + sqlite3_int64 mxValue; /* Maximum value ("stop") */ + sqlite3_int64 iStep; /* Increment ("step") */ +}; + +/* +** The seriesConnect() method is invoked to create a new +** series_vtab that describes the generate_series virtual table. +** +** Think of this routine as the constructor for series_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the series_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against generate_series will look like. +*/ +static int seriesConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define SERIES_COLUMN_VALUE 0 +#define SERIES_COLUMN_START 1 +#define SERIES_COLUMN_STOP 2 +#define SERIES_COLUMN_STEP 3 + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* +** This method is the destructor for series_cursor objects. +*/ +static int seriesDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new series_cursor object. +*/ +static int seriesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + series_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a series_cursor. +*/ +static int seriesClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a series_cursor to its next row of output. +*/ +static int seriesNext(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + if( pCur->isDesc ){ + pCur->iValue -= pCur->iStep; + }else{ + pCur->iValue += pCur->iStep; + } + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int seriesColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + series_cursor *pCur = (series_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case SERIES_COLUMN_START: x = pCur->mnValue; break; + case SERIES_COLUMN_STOP: x = pCur->mxValue; break; + case SERIES_COLUMN_STEP: x = pCur->iStep; break; + default: x = pCur->iValue; break; + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + series_cursor *pCur = (series_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int seriesEof(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + if( pCur->isDesc ){ + return pCur->iValue < pCur->mnValue; + }else{ + return pCur->iValue > pCur->mxValue; + } +} + +/* +** This method is called to "rewind" the series_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to seriesColumn() or seriesRowid() or +** seriesEof(). +** +** The query plan selected by seriesBestIndex is passed in the idxNum +** parameter. (idxStr is not used in this implementation.) idxNum +** is a bitmask showing which constraints are available: +** +** 1: start=VALUE +** 2: stop=VALUE +** 4: step=VALUE +** +** Also, if bit 8 is set, that means that the series should be output +** in descending order rather than in ascending order. +** +** This routine should initialize the cursor and position it so that it +** is pointing at the first row, or pointing off the end of the table +** (so that seriesEof() will return true) if the table is empty. +*/ +static int seriesFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + series_cursor *pCur = (series_cursor *)pVtabCursor; + int i = 0; + if( idxNum & 1 ){ + pCur->mnValue = sqlite3_value_int64(argv[i++]); + }else{ + pCur->mnValue = 0; + } + if( idxNum & 2 ){ + pCur->mxValue = sqlite3_value_int64(argv[i++]); + }else{ + pCur->mxValue = 0xffffffff; + } + if( idxNum & 4 ){ + pCur->iStep = sqlite3_value_int64(argv[i++]); + if( pCur->iStep<1 ) pCur->iStep = 1; + }else{ + pCur->iStep = 1; + } + if( idxNum & 8 ){ + pCur->isDesc = 1; + pCur->iValue = pCur->mxValue; + if( pCur->iStep>0 ){ + pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep; + } + }else{ + pCur->isDesc = 0; + pCur->iValue = pCur->mnValue; + } + pCur->iRowid = 1; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by bits in idxNum: +** +** (1) start = $value -- constraint exists +** (2) stop = $value -- constraint exists +** (4) step = $value -- constraint exists +** (8) output in descending order +*/ +static int seriesBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int idxNum = 0; /* The query plan bitmask */ + int startIdx = -1; /* Index of the start= constraint, or -1 if none */ + int stopIdx = -1; /* Index of the stop= constraint, or -1 if none */ + int stepIdx = -1; /* Index of the step= constraint, or -1 if none */ + int nArg = 0; /* Number of arguments that seriesFilter() expects */ + + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case SERIES_COLUMN_START: + startIdx = i; + idxNum |= 1; + break; + case SERIES_COLUMN_STOP: + stopIdx = i; + idxNum |= 2; + break; + case SERIES_COLUMN_STEP: + stepIdx = i; + idxNum |= 4; + break; + } + } + if( startIdx>=0 ){ + pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[startIdx].omit = 1; + } + if( stopIdx>=0 ){ + pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[stopIdx].omit = 1; + } + if( stepIdx>=0 ){ + pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[stepIdx].omit = 1; + } + if( pIdxInfo->nOrderBy==1 ){ + if( pIdxInfo->aOrderBy[0].desc ) idxNum |= 8; + pIdxInfo->orderByConsumed = 1; + } + if( (idxNum & 3)==3 ){ + /* Both start= and stop= boundaries are available. This is the + ** the preferred case */ + pIdxInfo->estimatedCost = (double)1; + }else{ + /* If either boundary is missing, we have to generate a huge span + ** of numbers. Make this case very expensive so that the query + ** planner will work hard to avoid it. */ + pIdxInfo->estimatedCost = (double)2000000000; + } + pIdxInfo->idxNum = idxNum; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** generate_series virtual table. +*/ +static sqlite3_module seriesModule = { + 0, /* iVersion */ + 0, /* xCreate */ + seriesConnect, /* xConnect */ + seriesBestIndex, /* xBestIndex */ + seriesDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + seriesOpen, /* xOpen - open a cursor */ + seriesClose, /* xClose - close a cursor */ + seriesFilter, /* xFilter - configure scan constraints */ + seriesNext, /* xNext - advance a cursor */ + seriesEof, /* xEof - check for end of scan */ + seriesColumn, /* xColumn - read data */ + seriesRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_series_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3_libversion_number()<3008012 ){ + *pzErrMsg = sqlite3_mprintf( + "generate_series() requires SQLite 3.8.12 or later"); + return SQLITE_ERROR; + } + rc = sqlite3_create_module(db, "generate_series", &seriesModule, 0); +#endif + return rc; +} diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index b9514427cf..336203433e 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -1770,6 +1770,7 @@ struct spellfix1_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ spellfix1_vtab *pVTab; /* The table to which this cursor belongs */ char *zPattern; /* rhs of MATCH clause */ + int idxNum; /* idxNum value passed to xFilter() */ int nRow; /* Number of rows of content */ int nAlloc; /* Number of allocated rows */ int iRow; /* Current row of content */ @@ -2040,26 +2041,19 @@ static int spellfix1Close(sqlite3_vtab_cursor *cur){ return SQLITE_OK; } +#define SPELLFIX_IDXNUM_MATCH 0x01 /* word MATCH $str */ +#define SPELLFIX_IDXNUM_LANGID 0x02 /* langid == $langid */ +#define SPELLFIX_IDXNUM_TOP 0x04 /* top = $top */ +#define SPELLFIX_IDXNUM_SCOPE 0x08 /* scope = $scope */ +#define SPELLFIX_IDXNUM_DISTLT 0x10 /* distance < $distance */ +#define SPELLFIX_IDXNUM_DISTLE 0x20 /* distance <= $distance */ +#define SPELLFIX_IDXNUM_ROWID 0x40 /* rowid = $rowid */ +#define SPELLFIX_IDXNUM_DIST (0x10|0x20) /* DISTLT and DISTLE */ + /* -** Search for terms of these forms: ** -** (A) word MATCH $str -** (B) langid == $langid -** (C) top = $top -** (D) scope = $scope -** (E) distance < $distance -** (F) distance <= $distance -** (G) rowid = $rowid -** -** The plan number is a bit mask formed with these bits: -** -** 0x01 (A) is found -** 0x02 (B) is found -** 0x04 (C) is found -** 0x08 (D) is found -** 0x10 (E) is found -** 0x20 (F) is found -** 0x40 (G) is found +** The plan number is a bitmask of the SPELLFIX_IDXNUM_* values defined +** above. ** ** filter.argv[*] values contains $str, $langid, $top, $scope and $rowid ** if specified and in that order. @@ -2078,62 +2072,66 @@ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ if( pConstraint->usable==0 ) continue; /* Terms of the form: word MATCH $str */ - if( (iPlan & 1)==0 + if( (iPlan & SPELLFIX_IDXNUM_MATCH)==0 && pConstraint->iColumn==SPELLFIX_COL_WORD && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ - iPlan |= 1; + iPlan |= SPELLFIX_IDXNUM_MATCH; pIdxInfo->aConstraintUsage[i].argvIndex = 1; pIdxInfo->aConstraintUsage[i].omit = 1; } /* Terms of the form: langid = $langid */ - if( (iPlan & 2)==0 + if( (iPlan & SPELLFIX_IDXNUM_LANGID)==0 && pConstraint->iColumn==SPELLFIX_COL_LANGID && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - iPlan |= 2; + iPlan |= SPELLFIX_IDXNUM_LANGID; iLangTerm = i; } /* Terms of the form: top = $top */ - if( (iPlan & 4)==0 + if( (iPlan & SPELLFIX_IDXNUM_TOP)==0 && pConstraint->iColumn==SPELLFIX_COL_TOP && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - iPlan |= 4; + iPlan |= SPELLFIX_IDXNUM_TOP; iTopTerm = i; } /* Terms of the form: scope = $scope */ - if( (iPlan & 8)==0 + if( (iPlan & SPELLFIX_IDXNUM_SCOPE)==0 && pConstraint->iColumn==SPELLFIX_COL_SCOPE && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - iPlan |= 8; + iPlan |= SPELLFIX_IDXNUM_SCOPE; iScopeTerm = i; } /* Terms of the form: distance < $dist or distance <= $dist */ - if( (iPlan & (16|32))==0 + if( (iPlan & SPELLFIX_IDXNUM_DIST)==0 && pConstraint->iColumn==SPELLFIX_COL_DISTANCE && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) ){ - iPlan |= pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ? 16 : 32; + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ){ + iPlan |= SPELLFIX_IDXNUM_DISTLT; + }else{ + iPlan |= SPELLFIX_IDXNUM_DISTLE; + } iDistTerm = i; } /* Terms of the form: distance < $dist or distance <= $dist */ - if( (iPlan & 64)==0 + if( (iPlan & SPELLFIX_IDXNUM_ROWID)==0 && pConstraint->iColumn<0 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - iPlan |= 64; + iPlan |= SPELLFIX_IDXNUM_ROWID; iRowidTerm = i; } } - if( iPlan&1 ){ + if( iPlan&SPELLFIX_IDXNUM_MATCH ){ int idx = 2; pIdxInfo->idxNum = iPlan; if( pIdxInfo->nOrderBy==1 @@ -2142,25 +2140,25 @@ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ pIdxInfo->orderByConsumed = 1; /* Default order by iScore */ } - if( iPlan&2 ){ + if( iPlan&SPELLFIX_IDXNUM_LANGID ){ pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx++; pIdxInfo->aConstraintUsage[iLangTerm].omit = 1; } - if( iPlan&4 ){ + if( iPlan&SPELLFIX_IDXNUM_TOP ){ pIdxInfo->aConstraintUsage[iTopTerm].argvIndex = idx++; pIdxInfo->aConstraintUsage[iTopTerm].omit = 1; } - if( iPlan&8 ){ + if( iPlan&SPELLFIX_IDXNUM_SCOPE ){ pIdxInfo->aConstraintUsage[iScopeTerm].argvIndex = idx++; pIdxInfo->aConstraintUsage[iScopeTerm].omit = 1; } - if( iPlan&(16|32) ){ + if( iPlan&SPELLFIX_IDXNUM_DIST ){ pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = idx++; pIdxInfo->aConstraintUsage[iDistTerm].omit = 1; } pIdxInfo->estimatedCost = 1e5; - }else if( (iPlan & 64) ){ - pIdxInfo->idxNum = 64; + }else if( (iPlan & SPELLFIX_IDXNUM_ROWID) ){ + pIdxInfo->idxNum = SPELLFIX_IDXNUM_ROWID; pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; pIdxInfo->estimatedCost = 5; @@ -2311,15 +2309,24 @@ static void spellfix1RunQuery(MatchQuery *p, const char *zQuery, int nQuery){ break; } pCur->nSearch++; - iScore = spellfix1Score(iDist,iRank); + + /* If there is a "distance < $dist" or "distance <= $dist" constraint, + ** check if this row meets it. If not, jump back up to the top of the + ** loop to process the next row. Otherwise, if the row does match the + ** distance constraint, check if the pCur->a[] array is already full. + ** If it is and no explicit "top = ?" constraint was present in the + ** query, grow the array to ensure there is room for the new entry. */ + assert( (p->iMaxDist>=0)==((pCur->idxNum & SPELLFIX_IDXNUM_DIST) ? 1 : 0) ); if( p->iMaxDist>=0 ){ if( iDist>p->iMaxDist ) continue; - if( pCur->nRow>=pCur->nAlloc-1 ){ + if( pCur->nRow>=pCur->nAlloc && (pCur->idxNum & SPELLFIX_IDXNUM_TOP)==0 ){ spellfix1ResizeCursor(pCur, pCur->nAlloc*2 + 10); if( pCur->a==0 ) break; } - idx = pCur->nRow; - }else if( pCur->nRownAlloc ){ + } + + iScore = spellfix1Score(iDist,iRank); + if( pCur->nRownAlloc ){ idx = pCur->nRow; }else if( iScorea[idx].zWord = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); if( pCur->a[idx].zWord==0 ){ p->rc = SQLITE_NOMEM; @@ -2361,10 +2369,10 @@ static void spellfix1RunQuery(MatchQuery *p, const char *zQuery, int nQuery){ */ static int spellfix1FilterForMatch( spellfix1_cursor *pCur, - int idxNum, int argc, sqlite3_value **argv ){ + int idxNum = pCur->idxNum; const unsigned char *zMatchThis; /* RHS of the MATCH operator */ EditDist3FromString *pMatchStr3 = 0; /* zMatchThis as an editdist string */ char *zPattern; /* Transliteration of zMatchThis */ @@ -2476,11 +2484,11 @@ filter_exit: */ static int spellfix1FilterForFullScan( spellfix1_cursor *pCur, - int idxNum, int argc, sqlite3_value **argv ){ int rc = SQLITE_OK; + int idxNum = pCur->idxNum; char *zSql; spellfix1_vtab *pVTab = pCur->pVTab; spellfix1ResetCursor(pCur); @@ -2521,10 +2529,11 @@ static int spellfix1Filter( ){ spellfix1_cursor *pCur = (spellfix1_cursor *)cur; int rc; + pCur->idxNum = idxNum; if( idxNum & 1 ){ - rc = spellfix1FilterForMatch(pCur, idxNum, argc, argv); + rc = spellfix1FilterForMatch(pCur, argc, argv); }else{ - rc = spellfix1FilterForFullScan(pCur, idxNum, argc, argv); + rc = spellfix1FilterForFullScan(pCur, argc, argv); } return rc; } diff --git a/ext/rbu/rbuB.test b/ext/rbu/rbuB.test new file mode 100644 index 0000000000..c007fb7d9a --- /dev/null +++ b/ext/rbu/rbuB.test @@ -0,0 +1,62 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuB + +db close +sqlite3_shutdown +test_sqlite3_log xLog +reset_db + +proc xLog {args} { } + +set db_sql { + CREATE TABLE t1(a PRIMARY KEY, b, c); +} +set rbu_sql { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(4, 5, 6, 0); + INSERT INTO data_t1 VALUES(7, 8, 9, 0); +} + +do_test 1.1 { + forcedelete rbu.db + sqlite3 rbu rbu.db + rbu eval $rbu_sql + rbu close + + db eval $db_sql +} {} + +set ::errlog [list] +proc xLog {err msg} { lappend ::errlog $err } +do_test 1.2 { + run_rbu test.db rbu.db +} {SQLITE_DONE} + +do_test 1.3 { + set ::errlog +} {SQLITE_NOTICE_RECOVER_WAL SQLITE_INTERNAL} + +do_execsql_test 1.4 { + SELECT * FROM t1 +} {1 2 3 4 5 6 7 8 9} + +db close +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize +finish_test + diff --git a/ext/rbu/rbu_common.tcl b/ext/rbu/rbu_common.tcl new file mode 100644 index 0000000000..8190021baf --- /dev/null +++ b/ext/rbu/rbu_common.tcl @@ -0,0 +1,38 @@ +# 2015 Aug 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl + +# Run the RBU in file $rbu on target database $target until completion. +# +proc run_rbu {target rbu} { + sqlite3rbu rbu $target $rbu + while 1 { + set rc [rbu step] + if {$rc!="SQLITE_OK"} break + } + rbu close +} + +proc step_rbu {target rbu} { + while 1 { + sqlite3rbu rbu $target $rbu + set rc [rbu step] + rbu close + if {$rc != "SQLITE_OK"} break + } + set rc +} + diff --git a/ext/rbu/rbudiff.test b/ext/rbu/rbudiff.test new file mode 100644 index 0000000000..10be9e0d53 --- /dev/null +++ b/ext/rbu/rbudiff.test @@ -0,0 +1,150 @@ +# 2015-07-31 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the [sqldiff --rbu] command. +# +# +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set testprefix rbudiff + +if {$tcl_platform(platform)=="windows"} { + set PROG "sqldiff.exe" +} else { + set PROG "./sqldiff" +} +if {![file exe $PROG]} { + puts "rbudiff.test cannot run because $PROG is not available" + finish_test + return +} +db close + +proc get_rbudiff_sql {db1 db2} { + exec $::PROG --rbu $db1 $db2 +} + +proc step_rbu {target rbu} { + while 1 { + sqlite3rbu rbu $target $rbu + set rc [rbu step] + rbu close + if {$rc != "SQLITE_OK"} break + } + set rc +} + +proc apply_rbudiff {sql target} { + forcedelete rbu.db + sqlite3 rbudb rbu.db + rbudb eval $sql + rbudb close + step_rbu $target rbu.db +} + +proc rbudiff_cksum {db1} { + set txt "" + + sqlite3 dbtmp $db1 + foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] { + set cols [list] + dbtmp eval "PRAGMA table_info = $tbl" { lappend cols "quote( $name )" } + append txt [dbtmp eval \ + "SELECT [join $cols {||'.'||}] FROM $tbl ORDER BY 1" + ] + } + dbtmp close + + md5 $txt +} + +foreach {tn init mod} { + 1 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + + CREATE TABLE t2(a, b, c, PRIMARY KEY(b, c)); + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(4, 5, 6); + } { + INSERT INTO t1 VALUES(7, 8, 9); + DELETE FROM t1 WHERE a=4; + UPDATE t1 SET c = 11 WHERE a = 1; + + INSERT INTO t2 VALUES(7, 8, 9); + DELETE FROM t2 WHERE a=4; + UPDATE t2 SET c = 11 WHERE a = 1; + } + + 2 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b, c)); + INSERT INTO t1 VALUES('u', 'v', 'w'); + INSERT INTO t1 VALUES('x', 'y', 'z'); + } { + DELETE FROM t1 WHERE a='u'; + INSERT INTO t1 VALUES('a', 'b', 'c'); + } + + 3 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, x); + INSERT INTO t1 VALUES(1, + X'0000000000000000111111111111111122222222222222223333333333333333' + ); + CREATE TABLE t2(y INTEGER PRIMARY KEY, x); + INSERT INTO t2 VALUES(1, + X'0000000000000000111111111111111122222222222222223333333333333333' + ); + } { + DELETE FROM t1; + INSERT INTO t1 VALUES(1, + X'0000000000000000111111111111111122222555555552223333333333333333' + ); + DELETE FROM t2; + INSERT INTO t2 VALUES(1, + X'0000000000000000111111111111111122222222222222223333333FFF333333' + ); + } + +} { + catch { db close } + + forcedelete test.db test.db2 + sqlite3 db test.db + db eval "$init" + sqlite3 db test.db2 + db eval "$init ; $mod" + db close + + do_test 1.$tn.2 { + set sql [get_rbudiff_sql test.db test.db2] + apply_rbudiff $sql test.db + } {SQLITE_DONE} + do_test 1.$tn.3 { rbudiff_cksum test.db } [rbudiff_cksum test.db2] + + forcedelete test.db test.db2 + sqlite3 db test.db + db eval "$init ; $mod" + sqlite3 db test.db2 + db eval "$init" + db close + + do_test 1.$tn.4 { + set sql [get_rbudiff_sql test.db test.db2] + apply_rbudiff $sql test.db + } {SQLITE_DONE} + do_test 1.$tn.5 { rbudiff_cksum test.db } [rbudiff_cksum test.db2] +} + +finish_test + diff --git a/ext/rbu/rbufts.test b/ext/rbu/rbufts.test new file mode 100644 index 0000000000..d5c9fe5609 --- /dev/null +++ b/ext/rbu/rbufts.test @@ -0,0 +1,134 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that RBU works with FTS tables. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix rbufts + +ifcapable !fts3 { + finish_test + return +} + +proc step_rbu {target rbu} { + while 1 { + sqlite3rbu rbu $target $rbu + set rc [rbu step] + rbu close + if {$rc != "SQLITE_OK"} break + } + set rc +} + +proc apply_rbu_update {target sql} { + forcedelete rbu.db + sqlite3 dbrbu rbu.db + execsql $sql dbrbu + dbrbu close + + step_rbu $target rbu.db +} + +do_execsql_test 1.1.0 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b); + CREATE VIRTUAL TABLE xx USING fts4(content=t1, a, b); + INSERT INTO t1(rowid, a, b) VALUES(10, 'a b c', 'c b a'); + INSERT INTO t1(rowid, a, b) VALUES(20, 'a b c', 'd e f'); + INSERT INTO t1(rowid, a, b) VALUES(30, 'd e f', 'a b c'); + INSERT INTO t1(rowid, a, b) VALUES(40, 'd e f', 'd e f'); +} + +do_execsql_test 1.1.1 { + INSERT INTO xx(xx) VALUES('rebuild'); + INSERT INTO xx(xx) VALUES('integrity-check'); +} + +do_test 1.1.2 { + apply_rbu_update test.db { + CREATE TABLE data_t1(i, a, b, rbu_control); + INSERT INTO data_t1 VALUES(20, NULL, NULL, 1); -- delete + INSERT INTO data_t1 VALUES(30, 'x y z', NULL, '.x.'); -- update + INSERT INTO data_t1 VALUES(50, '1 2 3', 'x y z', 0); -- insert + + CREATE VIEW data0_xx AS + SELECT i AS rbu_rowid, a, b, + CASE WHEN rbu_control IN (0, 1) + THEN rbu_control ELSE substr(rbu_control, 2) END AS rbu_control + FROM data_t1; + } +} {SQLITE_DONE} + +do_execsql_test 1.1.3 { + INSERT INTO xx(xx) VALUES('integrity-check'); +} + +reset_db +do_execsql_test 1.2.1 { + CREATE TABLE ccc(addr, text); + CREATE VIRTUAL TABLE ccc_fts USING fts4(addr, text, content=ccc); + INSERT INTO ccc VALUES('a b c', 'd e f'); + INSERT INTO ccc VALUES('a b c', 'd e f'); + INSERT INTO ccc_fts(ccc_fts) VALUES('rebuild'); + INSERT INTO ccc_fts(ccc_fts) VALUES('integrity-check'); +} + +do_test 1.2.2 { + apply_rbu_update test.db { + CREATE TABLE data_ccc(addr, text, rbu_rowid, rbu_control); + CREATE VIEW data0_ccc_fts AS SELECT * FROM data_ccc; + INSERT INTO data_ccc VALUES(NULL, NULL, 1, 1); + INSERT INTO data_ccc VALUES('x y z', NULL, 2, 'x.'); + INSERT INTO data_ccc VALUES('y y y', '1 1 1', 3, 0); + } +} {SQLITE_DONE} + +do_execsql_test 1.2.3 { + INSERT INTO ccc_fts(ccc_fts) VALUES('integrity-check'); +} +do_execsql_test 1.2.4 { + SELECT rowid, * FROM ccc_fts; +} {2 {x y z} {d e f} 3 {y y y} {1 1 1}} + +#------------------------------------------------------------------------- +# Test the outcome of attempting to delete or update a row within a +# contentless FTS table using RBU. An error. +# +reset_db +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE ft USING fts4(x, content=); + INSERT INTO ft(rowid, x) VALUES(1, '1 2 3'); + INSERT INTO ft(rowid, x) VALUES(2, '4 5 6'); +} + +do_test 3.2 { + list [catch { apply_rbu_update test.db { + CREATE TABLE data_ft(x, rbu_rowid, rbu_control); + INSERT INTO data_ft VALUES(NULL, 2, 1); + } } msg] $msg] +} {1 {SQLITE_ERROR - SQL logic error or missing database]}} + +do_test 3.3 { + list [catch { apply_rbu_update test.db { + CREATE TABLE data_ft(x, rbu_rowid, rbu_control); + INSERT INTO data_ft VALUES('7 8 9', 1, 'x'); + } } msg] $msg] +} {1 {SQLITE_ERROR - SQL logic error or missing database]}} + + + +finish_test + diff --git a/ext/rbu/rbusave.test b/ext/rbu/rbusave.test new file mode 100644 index 0000000000..1bb71a8e05 --- /dev/null +++ b/ext/rbu/rbusave.test @@ -0,0 +1,105 @@ +# 2015 August 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix rbusave + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t2(c, b); + + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + + INSERT INTO t2 VALUES(1, 1, 1); + INSERT INTO t2 VALUES(2, 2, 2); + INSERT INTO t2 VALUES(3, 3, 3); +} + +do_test 1.1 { + forcedelete test.db2 + sqlite3 db2 test.db2 + db2 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(1, 'one', NULL, '.x.'); + + CREATE TABLE data_t2(a, b, c, rbu_control); + INSERT INTO data_t2 VALUES(4, 4, 4, 0); + INSERT INTO data_t2 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t2 VALUES(1, 'one', NULL, '.x.'); + } +} {} + +proc test_to_bak {} { + foreach f { + test.db test.db-wal test.db-oal test.db-journal + test.db2 test.db2-wal test.db2-oal test.db2-journal + } { + set t [string map {test bak} $f] + forcedelete $t + if {[file exists $f]} { forcecopy $f $t } + } +} + +do_test 1.2 { + test_to_bak + sqlite3rbu rrr bak.db bak.db2 + set nStep 0 + while {[rrr step]=="SQLITE_OK"} {incr nStep} + set res2 [rrr close] +} {SQLITE_DONE} + + +sqlite3rbu rbu test.db test.db2 +set res "SQLITE_OK" +for {set i 1} {$res=="SQLITE_OK"} {incr i} { + set res [rbu step] + + do_test 1.3.$i.1 { + rbu savestate + test_to_bak + sqlite3rbu rrr bak.db bak.db2 + set nRem 0 + while {[rrr step]=="SQLITE_OK"} {incr nRem} + set res2 [rrr close] + } {SQLITE_DONE} + + do_test 1.3.$i.3 { expr $nRem+$i } [expr {$nStep + ($res=="SQLITE_DONE")}] + + do_test 1.3.$i.3 { + sqlite3 bak bak.db + bak eval { + SELECT * FROM t1; + SELECT * FROM t2; + } + } {1 one 1 3 3 3 4 4 4 1 one 1 3 3 3 4 4 4} + + bak close +} + +do_test 1.4 { rbu close } {SQLITE_DONE} + +do_execsql_test 1.5 { + SELECT * FROM t1; + SELECT * FROM t2; +} {1 one 1 3 3 3 4 4 4 1 one 1 3 3 3 4 4 4} + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 491313ad9a..7c7480bcfc 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -84,10 +84,6 @@ #include #include -#if !defined(_WIN32) -# include -#endif - #include "sqlite3.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) @@ -239,6 +235,7 @@ struct RbuObjIter { /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ const char *zTbl; /* Name of target db table */ + const char *zDataTbl; /* Name of rbu db table (or null) */ const char *zIdx; /* Name of target db index (or null) */ int iTnum; /* Root page of current object */ int iPkTnum; /* If eType==EXTERNAL, root of PK index */ @@ -249,7 +246,7 @@ struct RbuObjIter { sqlite3_stmt *pSelect; /* Source data */ sqlite3_stmt *pInsert; /* Statement for INSERT operations */ sqlite3_stmt *pDelete; /* Statement for DELETE ops */ - sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zTbl */ + sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ /* Last UPDATE used (for PK b-tree updates only), or NULL. */ RbuUpdateStmt *pRbuUpdate; @@ -360,6 +357,252 @@ struct rbu_file { }; +/************************************************************************* +** The following three functions, found below: +** +** rbuDeltaGetInt() +** rbuDeltaChecksum() +** rbuDeltaApply() +** +** are lifted from the fossil source code (http://fossil-scm.org). They +** are used to implement the scalar SQL function rbu_fossil_delta(). +*/ + +/* +** Read bytes from *pz and convert them into a positive integer. When +** finished, leave *pz pointing to the first character past the end of +** the integer. The *pLen parameter holds the length of the string +** in *pz and is decremented once for each character in the integer. +*/ +static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + unsigned int v = 0; + int c; + unsigned char *z = (unsigned char*)*pz; + unsigned char *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} + +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} + +/* +** Apply a delta. +** +** The output buffer should be big enough to hold the whole output +** file and a NUL terminator at the end. The delta_output_size() +** routine will determine this size for you. +** +** The delta string should be null-terminated. But the delta string +** may contain embedded NUL characters (if the input and output are +** binary files) so we also have to pass in the length of the delta in +** the lenDelta parameter. +** +** This function returns the size of the output file in bytes (excluding +** the final NUL terminator character). Except, if the delta string is +** malformed or intended for use with a source file other than zSrc, +** then this routine returns -1. +** +** Refer to the delta_create() documentation above for a description +** of the delta file format. +*/ +static int rbuDeltaApply( + const char *zSrc, /* The source or pattern file */ + int lenSrc, /* Length of the source file */ + const char *zDelta, /* Delta to apply to the pattern */ + int lenDelta, /* Length of the delta */ + char *zOut /* Write the output into this preallocated buffer */ +){ + unsigned int limit; + unsigned int total = 0; +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST + char *zOrigOut = zOut; +#endif + + limit = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + zDelta++; lenDelta--; + while( *zDelta && lenDelta>0 ){ + unsigned int cnt, ofst; + cnt = rbuDeltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = rbuDeltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; + } + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( ofst+cnt > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; + } + if( cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST + if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; + } +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; + } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; + } + } + } + /* ERROR: unterminated delta */ + return -1; +} + +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){ + int size; + size = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + return size; +} + +/* +** End of code taken from fossil. +*************************************************************************/ + +/* +** Implementation of SQL scalar function rbu_fossil_delta(). +** +** This function applies a fossil delta patch to a blob. Exactly two +** arguments must be passed to this function. The first is the blob to +** patch and the second the patch to apply. If no error occurs, this +** function returns the patched blob. +*/ +static void rbuFossilDeltaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aDelta; + int nDelta; + const char *aOrig; + int nOrig; + + int nOut; + int nOut2; + char *aOut; + + assert( argc==2 ); + + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); + + /* Figure out the size of the output */ + nOut = rbuDeltaOutputSize(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + } + + aOut = sqlite3_malloc(nOut+1); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); + if( nOut2!=nOut ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + } + } +} + + /* ** Prepare the SQL statement in buffer zSql against database handle db. ** If successful, set *ppStmt to point to the new statement and return @@ -526,7 +769,8 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ pIter->zTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); - rc = pIter->zTbl ? SQLITE_OK : SQLITE_NOMEM; + pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); + rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; } }else{ if( pIter->zIdx==0 ){ @@ -557,6 +801,40 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ return rc; } + +/* +** The implementation of the rbu_target_name() SQL function. This function +** accepts one argument - the name of a table in the RBU database. If the +** table name matches the pattern: +** +** data[0-9]_ +** +** where is any sequence of 1 or more characters, is returned. +** Otherwise, if the only argument does not match the above pattern, an SQL +** NULL is returned. +** +** "data_t1" -> "t1" +** "data0123_t2" -> "t2" +** "dataAB_t3" -> NULL +*/ +static void rbuTargetNameFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zIn; + assert( argc==1 ); + + zIn = (const char*)sqlite3_value_text(argv[0]); + if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC); + } + } +} + /* ** Initialize the iterator structure passed as the second argument. ** @@ -570,8 +848,9 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ memset(pIter, 0, sizeof(RbuObjIter)); rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, - "SELECT substr(name, 6) FROM sqlite_master " - "WHERE type IN ('table', 'view') AND name LIKE 'data_%'" + "SELECT rbu_target_name(name) AS target, name FROM sqlite_master " + "WHERE type IN ('table', 'view') AND target IS NOT NULL " + "ORDER BY name" ); if( rc==SQLITE_OK ){ @@ -917,7 +1196,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ ** of the input table. Ignore any input table columns that begin with ** "rbu_". */ p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT * FROM 'data_%q'", pIter->zTbl) + sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) ); if( p->rc==SQLITE_OK ){ nCol = sqlite3_column_count(pStmt); @@ -942,7 +1221,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ ){ p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf( - "table data_%q %s rbu_rowid column", pIter->zTbl, + "table %q %s rbu_rowid column", pIter->zDataTbl, (bRbuRowid ? "may not have" : "requires") ); } @@ -963,8 +1242,8 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ } if( i==pIter->nTblCol ){ p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("column missing from data_%q: %s", - pIter->zTbl, zName + p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", + pIter->zDataTbl, zName ); }else{ int iPk = sqlite3_column_int(pStmt, 5); @@ -1263,12 +1542,18 @@ static char *rbuObjIterGetSetlist( ); zSep = ", "; } - if( c=='d' ){ + else if( c=='d' ){ zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 ); zSep = ", "; } + else if( c=='f' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } } } } @@ -1519,7 +1804,7 @@ static void rbuObjIterPrepareTmpInsert( p->rc = prepareFreeAndCollectError( p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", - p->zStateDb, pIter->zTbl, zCollist, zRbuRowid, zBind + p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind )); } } @@ -1615,18 +1900,18 @@ static int rbuObjIterPrepareAll( if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", - zCollist, p->zStateDb, pIter->zTbl, + zCollist, p->zStateDb, pIter->zDataTbl, zCollist, zLimit ); }else{ zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM 'data_%q' " + "SELECT %s, rbu_control FROM '%q' " "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " "UNION ALL " "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " "ORDER BY %s%s", - zCollist, pIter->zTbl, - zCollist, p->zStateDb, pIter->zTbl, + zCollist, pIter->zDataTbl, + zCollist, p->zStateDb, pIter->zDataTbl, zCollist, zLimit ); } @@ -1650,16 +1935,6 @@ static int rbuObjIterPrepareAll( zCollist = rbuObjIterGetCollist(p, pIter); pIter->nCol = pIter->nTblCol; - /* Create the SELECT statement to read keys from data_xxx */ - if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, - sqlite3_mprintf( - "SELECT %s, rbu_control%s FROM 'data_%q'%s", - zCollist, (bRbuRowid ? ", rbu_rowid" : ""), zTbl, zLimit - ) - ); - } - /* Create the imposter table or tables (if required). */ rbuCreateImposterTable(p, pIter); rbuCreateImposterTable2(p, pIter); @@ -1693,10 +1968,10 @@ static int rbuObjIterPrepareAll( /* Create the rbu_tmp_xxx table and the triggers to populate it. */ rbuMPrintfExec(p, p->dbRbu, "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " - "SELECT *%s FROM 'data_%q' WHERE 0;" - , p->zStateDb - , zTbl, (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") - , zTbl + "SELECT *%s FROM '%q' WHERE 0;" + , p->zStateDb, pIter->zDataTbl + , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") + , pIter->zDataTbl ); rbuMPrintfExec(p, p->dbMain, @@ -1732,6 +2007,17 @@ static int rbuObjIterPrepareAll( rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid); } + /* Create the SELECT statement to read keys from data_xxx */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s, rbu_control%s FROM '%q'%s", + zCollist, (bRbuRowid ? ", rbu_rowid" : ""), + pIter->zDataTbl, zLimit + ) + ); + } + sqlite3_free(zWhere); sqlite3_free(zOldlist); sqlite3_free(zNewlist); @@ -1862,6 +2148,18 @@ static void rbuOpenDatabase(sqlite3rbu *p){ ); } + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + ); + } + if( p->rc==SQLITE_OK ){ p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); } @@ -2291,7 +2589,7 @@ static int rbuStep(sqlite3rbu *p){ for(i=0; p->rc==SQLITE_OK && inCol; i++){ char c = zMask[pIter->aiSrcOrder[i]]; pVal = sqlite3_column_value(pIter->pSelect, i); - if( pIter->abTblPk[i] || c=='x' || c=='d' ){ + if( pIter->abTblPk[i] || c!='.' ){ p->rc = sqlite3_bind_value(pUpdate, i+1, pVal); } } @@ -2403,7 +2701,7 @@ int sqlite3rbu_step(sqlite3rbu *p){ ** But the contents can be deleted. */ if( pIter->abIndexed ){ rbuMPrintfExec(p, p->dbRbu, - "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zTbl + "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl ); } }else{ @@ -2626,10 +2924,13 @@ static void rbuSetupOal(sqlite3rbu *p, RbuState *pState){ ** leave an error code and error message in the rbu handle. */ static void rbuDeleteOalFile(sqlite3rbu *p){ - char *zOal = sqlite3_mprintf("%s-oal", p->zTarget); - assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); - unlink(zOal); - sqlite3_free(zOal); + char *zOal = rbuMPrintf(p, "%s-oal", p->zTarget); + if( zOal ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + assert( pVfs && p->rc==SQLITE_OK && p->zErrmsg==0 ); + pVfs->xDelete(pVfs, zOal, 0); + sqlite3_free(zOal); + } } /* @@ -2742,14 +3043,25 @@ sqlite3rbu *sqlite3rbu_open( if( p->rc==SQLITE_OK ){ if( p->eStage==RBU_STAGE_OAL ){ + sqlite3 *db = p->dbMain; /* Open transactions both databases. The *-oal file is opened or ** created at this point. */ - p->rc = sqlite3_exec(p->dbMain, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); if( p->rc==SQLITE_OK ){ p->rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); } - + + /* Check if the main database is a zipvfs db. If it is, set the upper + ** level pager to use "journal_mode=off". This prevents it from + ** generating a large journal using a temp file. */ + if( p->rc==SQLITE_OK ){ + int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0); + if( frc==SQLITE_OK ){ + p->rc = sqlite3_exec(db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg); + } + } + /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ p->rc = rbuObjIterFirst(p, &p->objiter); @@ -2863,6 +3175,32 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){ return pRbu->nProgress; } +int sqlite3rbu_savestate(sqlite3rbu *p){ + int rc = p->rc; + + if( rc==SQLITE_DONE ) return SQLITE_OK; + + assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE ); + if( p->eStage==RBU_STAGE_OAL ){ + assert( rc!=SQLITE_DONE ); + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0); + } + + p->rc = rc; + rbuSaveState(p, p->eStage); + rc = p->rc; + + if( p->eStage==RBU_STAGE_OAL ){ + assert( rc!=SQLITE_DONE ); + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, 0); + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "BEGIN IMMEDIATE", 0, 0,0); + } + + p->rc = rc; + return rc; +} + /************************************************************************** ** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour ** of a standard VFS in the following ways: diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index c3e32f9410..b6cbf4d8b9 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -97,6 +97,18 @@ ** ** The order of the columns in the data_% table does not matter. ** +** Instead of a regular table, the RBU database may also contain virtual +** tables or view named using the data_ naming scheme. +** +** Instead of the plain data_ naming scheme, RBU database tables +** may also be named data_, where is any sequence +** of zero or more numeric characters (0-9). This can be significant because +** tables within the RBU database are always processed in order sorted by +** name. By judicious selection of the the portion of the names +** of the RBU tables the user can therefore control the order in which they +** are processed. This can be useful, for example, to ensure that "external +** content" FTS4 tables are updated before their underlying content tables. +** ** If the target database table is a virtual table or a table that has no ** PRIMARY KEY declaration, the data_% table must also contain a column ** named "rbu_rowid". This column is mapped to the tables implicit primary @@ -177,6 +189,14 @@ ** ** UPDATE t1 SET c = rbu_delta(c, 'usa') WHERE a = 4; ** +** Finally, if an 'f' character appears in place of a 'd' or 's' in an +** ota_control string, the contents of the data_xxx table column is assumed +** to be a "fossil delta" - a patch to be applied to a blob value in the +** format used by the fossil source-code management system. In this case +** the existing value within the target database table must be of type BLOB. +** It is replaced by the result of applying the specified fossil delta to +** itself. +** ** If the target database table is a virtual table or a table with no PRIMARY ** KEY, the rbu_control value should not include a character corresponding ** to the rbu_rowid value. For example, this: @@ -334,6 +354,18 @@ sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu); */ int sqlite3rbu_step(sqlite3rbu *pRbu); +/* +** Force RBU to save its state to disk. +** +** If a power failure or application crash occurs during an update, following +** system recovery RBU may resume the update from the point at which the state +** was last saved. In other words, from the most recent successful call to +** sqlite3rbu_close() or this function. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +int sqlite3rbu_savestate(sqlite3rbu *pRbu); + /* ** Close an RBU handle. ** diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index fbaea16268..6648f28e8b 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -56,7 +56,9 @@ static int test_sqlite3rbu_cmd( ){ int ret = TCL_OK; sqlite3rbu *pRbu = (sqlite3rbu*)clientData; - const char *azMethod[] = { "step", "close", "create_rbu_delta", 0 }; + const char *azMethod[] = { + "step", "close", "create_rbu_delta", "savestate", 0 + }; int iMethod; if( objc!=2 ){ @@ -103,6 +105,13 @@ static int test_sqlite3rbu_cmd( break; } + case 3: /* savestate */ { + int rc = sqlite3rbu_savestate(pRbu); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + ret = (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); + break; + } + default: /* seems unlikely */ assert( !"cannot happen" ); break; diff --git a/main.mk b/main.mk index e8c1cdd34a..4e4e226c53 100644 --- a/main.mk +++ b/main.mk @@ -307,11 +307,13 @@ TESTSRC += \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ $(TOP)/ext/misc/wholenumber.c \ $(TOP)/ext/misc/vfslog.c \ $(TOP)/ext/fts5/fts5_tcl.c \ + $(TOP)/ext/fts5/fts5_test_mi.c \ fts5.c diff --git a/manifest b/manifest index f8c7b41f33..72c7b00892 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Merge\stest\simprovements\sand\sminor\sfixes\sfrom\strunk. -D 2015-07-24T22:21:01.450 +C Merge\srecent\strunk\senhancements,\sinclude\stable-valued\sfunctions. +D 2015-08-20T23:54:25.007 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in bb0577f51dd8eda80a4d14bcf830f9696e31dcce +F Makefile.in 7c77beda9c1e956ff754ff12141b4b2ed033686b F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 60f6a05fe6da8ec23596df656b3fa6c2e052cc20 +F Makefile.msc e978ec57e8eb6e2f5b8a5114fa8cb320626337ca F Makefile.vxworks e1b65dea203f054e71653415bd8f96dcaed47858 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 -F VERSION ce0ae95abd7121c534f6917c1c8f2b70d9acd4db +F VERSION ccfc4d1576dbfdeece0a4372a2e6a2e37d3e7975 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 42b71ad3fe21c9e88fa59e8458ca1a6bc72eb0c0 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 17bd8dc3e35c718df68d04f53bf7dacf2b639732 x +F configure 2f61915a1bdfbc589244334401cf97d3401e6a39 x F configure.ac 713de38000413e469188db2cb85bed759b56f322 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -46,7 +46,7 @@ F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 -F ext/async/sqlite3async.c b5a3e30f538a9ffe81538b3063b4d5963f9bb422 +F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b @@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c d2f7981f4d7dfeb76aac82a15c7f37f425329c0f +F ext/fts3/fts3.c b04b0c57761fdba2ae562d9d9ba50c7c4a95d9ea F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 601743955ac43a0e82e6828a931c07bb3b0c95ff F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1 @@ -105,36 +105,37 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 -F ext/fts5/fts5.h 81d1a92fc2b4bd477af7e4e0b38b456f3e199fba -F ext/fts5/fts5Int.h 8d9bce1847a10df2e4ed9492ea4f3868276748fb +F ext/fts5/fts5.h 1950ec0544de667a24c1d8af9b2fde5db7db3bc9 +F ext/fts5/fts5Int.h 45f2ceb3c030f70e2cc4c199e9f700c2f2367f77 F ext/fts5/fts5_aux.c 044cb176a815f4388308738437f6e130aa384fb0 F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 -F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90 -F ext/fts5/fts5_expr.c 56dcbcbdc9029dd76a31360de664559839f4be41 -F ext/fts5/fts5_hash.c ff07722c73587c12781213133edbdb22cd156378 -F ext/fts5/fts5_index.c 892c13a7f44b68e962a91af62f2078f23fabb241 -F ext/fts5/fts5_main.c 0de7ba81488d2c502c8e794eaf7983d468e4c6e9 -F ext/fts5/fts5_storage.c 1c35a38a564ee9cadcbd7ae0b13a806bdda722bd -F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20 -F ext/fts5/fts5_tokenize.c 30f97a8c74683797b4cd233790444fbefb3b0708 +F ext/fts5/fts5_config.c fdfa63ae8e527ecfaa50f94063c610429cc887cf +F ext/fts5/fts5_expr.c d075d36c84975a1cfcf070442d28e28027b61c25 +F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 +F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 +F ext/fts5/fts5_main.c fc47ad734dfb55765b7542a345cee981170e7caa +F ext/fts5/fts5_storage.c 22ec9b5d35a39e2b5b65daf4ba7cd47fbb2d0df5 +F ext/fts5/fts5_tcl.c 96a3b9e982c4a64a242eefd752fa6669cd405a67 +F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 +F ext/fts5/fts5_tokenize.c 2836f6728bd74c7efac7487f5d9c27ca3e1b509c F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 -F ext/fts5/fts5_vocab.c 4e268a3fcbc099e50e335a1135be985a41ff6f7f +F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl e0b4a846a7670f6232a644ece69ef25a5c19c0e8 -F ext/fts5/test/fts5aa.test 4e896b9154764fed48179a87ba0bdf3650d7f49d +F ext/fts5/test/fts5_common.tcl 3338968de1880ca12b0451ae8f9b8b12d14e0ba7 +F ext/fts5/test/fts5aa.test c6e680a0d1b6c2616a382f1006d5d91eca697bd0 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d -F ext/fts5/test/fts5ae.test ddc558e3e3b52db0101f7541b2e3849b77052c92 +F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 F ext/fts5/test/fts5ah.test b9e78fa986a7bd564ebadfb244de02c84d7ac3ae F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37 F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592 -F ext/fts5/test/fts5al.test fc60ebeac9d8e366e71309d4c31fa72199d711d7 +F ext/fts5/test/fts5al.test 440d77c0b39ba73bad2ceb8986c2bb1093570735 F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04 F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f @@ -142,14 +143,14 @@ F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb F ext/fts5/test/fts5columnsize.test 97dc6bd66c91009d00407aa078dd5e9e8eb22f99 F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b -F ext/fts5/test/fts5content.test d0d90a45f0bcf07d75d474500d81f941b45e2021 +F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test 928c9c91d40690d301f943a7ed0ffc19e0d0e7b6 F ext/fts5/test/fts5corrupt2.test 1a830ccd6dbe1b601c7e3f5bbc1cf77bd8c8803b F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3 F ext/fts5/test/fts5dlidx.test 59b80bbe34169a082c575d9c26f0a7019a7b79c1 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test 451bb37310ee6df8ef72e4354fda5621b3b51448 -F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e +F ext/fts5/test/fts5eb.test 46f49497edc25ef3b2bff9fb6d75b6d201e2b39e F ext/fts5/test/fts5fault1.test 7a562367cb4a735b57b410dbdb62dcc8d971faec F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 @@ -159,6 +160,7 @@ F ext/fts5/test/fts5fault6.test 234dc6355f8d3f8b5be2763f30699d770247c215 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 +F ext/fts5/test/fts5matchinfo.test ee6e7b130096c708c12049fa9c1ceb628954c4f9 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 @@ -170,12 +172,12 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5tokenizer.test 83e7e01a21ec7fdf814d51f6184cc26bb77d7695 +F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 -F ext/fts5/test/fts5version.test c54a708236642bcc850d2aedc6f505fef1d9f9f1 +F ext/fts5/test/fts5version.test 205beb2a67d9496af64df959e6a19238f69b83e8 F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c F ext/fts5/tool/loadfts5.tcl 95edf0b6b92a09f9ed85595038b1108127987556 F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51 @@ -194,8 +196,9 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a +F ext/misc/series.c 6f94daf590d0668187631dee2a4d7e1d8f3095c3 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c de9181ec188294dd2a1087b329ca55cfaa76a29d +F ext/misc/spellfix.c 86998fb73aefb7b5dc346ba8a58912f312da4996 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e @@ -214,12 +217,17 @@ F ext/rbu/rbu7.test fd025d5ba440fcfe151fbb0e3835e1e7fe964fa1 F ext/rbu/rbu8.test 3bbf2c35d71a843c463efe93946f14ad10c3ede0 F ext/rbu/rbu9.test 0806d1772c9f4981774ff028de6656e4183082af F ext/rbu/rbuA.test c1a7b3e2d926b8f8448bb3b4ae787e314ee4b2b3 +F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2 +F ext/rbu/rbu_common.tcl 0398545fed614f807d5f0ba55a85a51f08ba8f1a F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 +F ext/rbu/rbudiff.test 6cc806dc36389292f2a8f5842d0103721df4a07d F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 -F ext/rbu/sqlite3rbu.c dbd7e4b31821398dcdeb21492970401ff1027881 -F ext/rbu/sqlite3rbu.h 6a280298e9eeb8ef59841a620f07f4f844651545 -F ext/rbu/test_rbu.c f99698956cc9158d6bf865e461e2d15876538ac1 +F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda +F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 +F ext/rbu/sqlite3rbu.c 1650e682b3568db0ed97ff2c7ba5d1c8ea060a84 +F ext/rbu/sqlite3rbu.h 5357f070cd8c0bcad459b620651ec1656859e4d0 +F ext/rbu/test_rbu.c 2a3652241fa45d5eaa141775e4ae68c1d3582c03 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0f9b595bd0debcbedf1d7a63d0e0678d619e6c9c F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -268,7 +276,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 0918e34121c7d5120d34a27653cb4bcd16d532fe +F main.mk 2d090a509da78d5dde9da7062ff0c104a4bb3023 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -291,15 +299,15 @@ F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 -F src/btreeInt.h 2ad754dd4528baa8d0946a593cc373b890bf859e -F src/build.c b3f15255d5b16e42dafeaa638fd4f8a47c94ed70 +F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 +F src/build.c 5eb5d055a1d1cdaaea25e01b12607aa894bc0911 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c 8ec787fed4929d8ccdf6b1bc360fccc3e1d2ca58 F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a F src/delete.c b998fbc3c55e8331a5f40aa7ff80972254de8de1 -F src/expr.c c5c58e4d01c7ceb2266791d8d877f1b23a88e316 +F src/expr.c 9b9fa7f825290dee945007edc9fe8fdd9b8ce49e F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c c9b63a217d86582c22121699a47f22f524608869 F src/func.c 824bea430d3a2b7dbc62806ad54da8fdb8ed9e3f @@ -307,12 +315,12 @@ F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c a81d4454051c92d058d79cd77099e700e36a74f6 +F src/insert.c 43e48fa63c9b9b5f195a13c40ddf7fd112370bb2 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712 F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2 -F src/main.c 6dd8bf94f181925a50a1f494042e50a64d5ecaca +F src/main.c 9e2e596c97401a7b99f2f500f5e2c725d53eabe2 F src/malloc.c 19461e159bccf0e2cf06a50e867963d0a7b124a8 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987 @@ -336,28 +344,28 @@ F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c aa916ca28606ccf4b6877dfc2b643ccbca86589f F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 -F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8 +F src/parse.y ad9af8552f6f340bd646577ca63356a6f82b6a7e F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c d08939800abf3031bd0affd5a13fbc4d7ba3fb68 -F src/pragma.c e52084b37a08a88f258830518461e94627af2621 +F src/pragma.c 669bc0fdb3fb5554e18330e8dd9743319bac16f4 F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c 2d47554370de8de6dd5be060cef9559eec315005 +F src/resolve.c 7a67cd2aebc9a9eeecd1d104eb6a9237388eb452 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 57ef3d98c4400b93eea318813be41b2af2da2217 -F src/shell.c e4ad9031072a6d679b2c69a780014d30db62dc7f -F src/sqlite.h.in 52dce8a4cf52ae3201507a8f0f0c68603fc925fc +F src/select.c c46de38c1b66355f02a839bb72eb13f277e6d19c +F src/shell.c 534f332318f1cff781e07d82a952f22e644e8f6f +F src/sqlite.h.in a25d949fd0714136d574e18738defa893baf14bb F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h a0b948ebc89bac13941254641326a6aa248c2cc4 -F src/sqliteInt.h d6389d1902f97533210c443f1cc77f83bd4bb73e +F src/sqliteInt.h 198917e2122c33b43412efedc2b055a82a212225 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e F src/tclsqlite.c e2344bee0d192397f555a24ef3fab26f2ed93bcc -F src/test1.c ab312f4160fdd9c82e65e8b123e35ef17e879c07 +F src/test1.c c12ed85c22ac95f87f79de2ec9553334d115f71e F src/test2.c 577961fe48961b2f2e5c8b56ee50c3f459d3359d F src/test3.c 64d2afdd68feac1bb5e2ffb8226c8c639f798622 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e @@ -404,30 +412,30 @@ F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 F src/tokenize.c 57cb3720f53f84d811def2069c2b169b6be539a5 -F src/treeview.c c84b1a8ebc7f1d00cd76ce4958eeb3ae1021beed +F src/treeview.c c15df00728034549ff92d78ae851b44952736d3b F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/update.c 24dd6a45b8b3470e62702128ebf11be1f2693145 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c -F src/util.c 46358a204b35971a839341cf64599d65b151ba88 +F src/util.c bc9dd64b5db544218b871b66243871c202b2781f F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 -F src/vdbe.c 8dbffc354f8b8bfb388d56c2e470a589fba0543c +F src/vdbe.c 76304f37d020d217ee6cfd3ac632fb4e6fcddb26 F src/vdbe.h d0f8ab919146109d080cde4b0840af9b5fafad4b F src/vdbeInt.h 963c87c4bf8040c0a316ca3e58f8a4888e1fa3c4 -F src/vdbeapi.c 7813b9725069c8b32e6da727bcf3e73f321384a4 -F src/vdbeaux.c 2d86fc5411e4e659c9181ef642c63dff602b3684 +F src/vdbeapi.c c58b0f5aab970e190b1047c4c6b74769381dcd04 +F src/vdbeaux.c 634a480aaec26e170b4fcfab91697f3cc6788ee8 F src/vdbeblob.c ab33f9b57cfce7dddb23853090186da614be4846 F src/vdbemem.c 6c9e261d135fc175da2f34e46d60243a19fffb9f F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 -F src/vtab.c 082b35a25a26e3d36f365ca8cd73c1922532f05e +F src/vtab.c d31174e4c8f592febab3fa7f69e18320b4fd657a F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 -F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 909eba3b6db984eb2adfbca9de2c237ee7056adb -F src/whereInt.h 5f87e3c4b0551747d119730dfebddd3c54f04047 -F src/wherecode.c 5da5049224f12db314931ae7e0919b4914a2a0b1 -F src/whereexpr.c 9ce1c9cfedbf80c93c7d899497025ec8256ce652 +F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba +F src/where.c 66518a14a1238611aa0744d6980b6b7f544f4816 +F src/whereInt.h 880a8599226ac1c00203490d934f3ed79b292572 +F src/wherecode.c 69f19535a6de0cceb10e16b31a3a03463e31bc24 +F src/whereexpr.c f9dbd159127452150c92b558e184827ecb8f9229 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -514,7 +522,7 @@ F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 F test/capi3.test bf6f0308bbbba1e770dac13aa08e5c2ac61c7324 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test fdc0d67a2cb8e8fc400d5b7735e330161ea057a2 -F test/capi3d.test a82b6321c50a1cfc848e386fa2c851893606f68c +F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/check.test 5831ddb6f2c687782eaf2e1a07b6e17f24c4f763 @@ -787,7 +795,7 @@ F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test b6ec456cf3b81d9a32123fe7e449bde434db338b F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 -F test/index6.test fbf45ceb39eb8a01b837d22623b93b208e6509ef +F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c F test/indexedby.test 5f527a78bae74c61b8046ae3037f9dfb0bf0c353 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d @@ -867,7 +875,7 @@ F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 F test/memsubsys1.test d2b2d6ca37890b26703a2258df8fd66f9869da02 -F test/memsubsys2.test 3a1c1a9de48e5726faa85108b02459fae8cb9ee9 +F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08 F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 @@ -942,12 +950,12 @@ F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl 8eb718bca8bca71c0c0145384f9915bb7e53fca3 +F test/releasetest.tcl cd2de2749aab7f45b2fe91b4a05431fc08e1692a F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 F test/rollbackfault.test 6a004f71087cc399296cffbb5429ea6da655ae65 -F test/rowallock.test f7f834125f11ff62f6e1ae7d0b07fd9228f2d5a2 +F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1004,7 +1012,7 @@ F test/shell4.test ddf0a99044e2245a87fc17423e3aaa1445b3243b F test/shell5.test c04e9f9f948305706b88377c464c7f08ce7479f9 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 -F test/shrink.test 06deac10d591186017466ce67d10645150bfdeec +F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test d37a75b4be4eb9dedeb69b4f38b1d0a74b5021d7 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a @@ -1029,6 +1037,7 @@ F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b F test/speedtest1.c 857439869d1cb4db35e1c720ee9c2756eb9ea2a0 F test/spellfix.test 0597065ff57042df1f138e6a2611ae19c2698135 +F test/spellfix2.test e5f2bc1dae046dbdd8008f2a84ed7749ff9b325e F test/sqldiff1.test 8f6bc7c6a5b3585d350d779c6078869ba402f8f5 F test/sqllimits1.test e05786eaed7950ff6a2d00031d001d8a26131e68 F test/stat.test 8de91498c99f5298b303f70f1d1f3b9557af91bf @@ -1042,6 +1051,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 +F test/tabfunc01.test d556af2def6af10b0a759b2f8a8f41135c2b634e F test/table.test 33bf0d1fd07f304582695184b8e6feb017303816 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -1059,7 +1069,7 @@ F test/thread1.test df115faa10a4ba1d456e9d4d9ec165016903eae4 F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b -F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 +F test/threadtest2.c a70a8e94bef23339d34226eb9521015ef99f4df8 F test/threadtest3.c 9ab4b168681c3a6f70f6c833ba08e0d48dd4af9b F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c @@ -1246,7 +1256,7 @@ F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 -F test/view.test f311691d696a5cc27e3c1b875cec1b0866b4ccd9 +F test/view.test f44014f78d7650fb4bfb8ef96a5e4dc8f25eb083 F test/vtab1.test 6210e076997f176bedc300a87ad6404651b601dd F test/vtab2.test f8cd1bb9aba7143eba97812d9617880a36d247ad F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -1265,9 +1275,9 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 -F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc +F test/wal.test dbfc482e10c7263298833bb1fc60b3ac9d6340a1 F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada -F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 +F test/wal3.test 2b5445e5da44780b9b44712f5a38523f7aeb0941 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 88b5d9a6a3d1532497ee9f4296f010d66f07e33c F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e @@ -1323,14 +1333,14 @@ F test/with2.test ee227a663586aa09771cafd4fa269c5217eaf775 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test 1a7b9bd51b899928d327052df9741d2fe8dbe701 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 -F test/without_rowid3.test 1081aabf60a1e1123b7f9a8f6ae19954351843b0 +F test/without_rowid3.test aad4f9d383e199349b6c7e508a778f7dff5dff79 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a F test/without_rowid5.test 61256715b686359df48ca1742db50cc7e3e7b862 F test/without_rowid6.test db0dbf03c49030aa3c1ba5f618620334bd2baf5f F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda -F test/zeroblob.test c54bc7a95df5fb2d463e00822e1377384954c161 +F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac -F tool/build-all-msvc.bat 9058bd90a3c078a3d8c17d40e853aaa0f47885f4 x +F tool/build-all-msvc.bat 60dbf6021d3de0a98575f6dfe4e12bd80b3edcf0 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -1354,39 +1364,39 @@ F tool/mksqlite3c-noext.tcl 87240b09c20042999b41d5fabe091b7111287835 F tool/mksqlite3c.tcl 038ed0aee939e6207c462727088a08d0bdf49a8c F tool/mksqlite3h.tcl 96d92fcac21c6037d9db20c7cb2e06b534b550ac F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b -F tool/mkvsix.tcl 3b58b9398f91c7dbf18d49eb87cefeee9efdbce1 +F tool/mkvsix.tcl bbe57cd9ae11c6cc70319241101ef8d2b8c3765b F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 34d7ac01fe4fd18e3637f64abe12c40eca0f6b97 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c 3b5d335d537e4dc44d0c86967023819453c87dc6 +F tool/showdb.c b1e16174385d5bd0815823a7fda1ecc82ed6088b F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68 F tool/showstat4.c 9515faa8ec176599d4a8288293ba8ec61f7b728a F tool/showwal.c 85cb36d4fe3e93e2fbd63e786e0d1ce42d0c4fad F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b -F tool/spaceanal.tcl 713c587a057334de42c41ad9566f10e255d3ad27 +F tool/spaceanal.tcl 63a415385a66fdbf736bfd204a31c6d851ed8da6 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/sqldiff.c 0748c0daed08f31e5a8eab6de98ca57500e61ecf +F tool/sqldiff.c b318efc2eaf7a7fac4d281a0ce736193cb2506df F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh c5a617b8c61a0926747a56c65f5671ef8ac0e148 -F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 +F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7f0ee77062d2fcb014942c7c62c163ccc801f21b cca79fdc3dff65907c2a59369057265b4512058f -R afa7eed8a3a47b08d78c1d46aa2aac25 +P 0298a9a780695b666e7c683700d9f2f889d6f826 64d13339d44d1b7ec6768a33421f2138cb7872d8 +R 68454f4230fc116d30b90969f8a680d7 U drh -Z d3e71694cf71fedeea79c377e4fa2c08 +Z c9e6b5cac5d4ca7f98b94a1f656dcb53 diff --git a/manifest.uuid b/manifest.uuid index 6be0992b22..7c89602550 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0298a9a780695b666e7c683700d9f2f889d6f826 \ No newline at end of file +e9196d566690de0e9815f8cd85be7844322b5a79 \ No newline at end of file diff --git a/src/btreeInt.h b/src/btreeInt.h index cbf6c99847..e52130cc39 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -700,9 +700,11 @@ struct IntegrityCk { */ #if SQLITE_BYTEORDER==4321 # define get2byteAligned(x) (*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000 +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && GCC_VERSION>=4008000 # define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 # define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) #else # define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) diff --git a/src/build.c b/src/build.c index 2936805365..e45908dc3b 100644 --- a/src/build.c +++ b/src/build.c @@ -356,6 +356,15 @@ Table *sqlite3LocateTable( p = sqlite3FindTable(pParse->db, zName, zDbase); if( p==0 ){ const char *zMsg = isView ? "no such view" : "no such table"; +#ifndef SQLITE_OMIT_VIRTUAL_TABLE + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); + if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ + return pMod->pEpoTab; + } +#endif if( zDbase ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); }else{ @@ -560,7 +569,7 @@ void sqlite3CommitInternalChanges(sqlite3 *db){ ** Delete memory allocated for the column names of a table or view (the ** Table.aCol[] array). */ -static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ +void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); @@ -627,7 +636,7 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Delete the Table structure itself. */ - sqliteDeleteColumnNames(db, pTable); + sqlite3DeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); @@ -1816,9 +1825,10 @@ void sqlite3EndTable( int iDb; /* Database in which the table lives */ Index *pIdx; /* An implied index of the table */ - if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ + if( pEnd==0 && pSelect==0 ){ return; } + assert( !db->mallocFailed ); p = pParse->pNewTable; if( p==0 ) return; @@ -2088,9 +2098,6 @@ void sqlite3CreateView( if( db->mallocFailed ){ return; } - if( !db->init.busy ){ - sqlite3ViewGetColumnNames(pParse, p); - } /* Locate the end of the CREATE VIEW statement. Make sEnd point to ** the end. @@ -2220,7 +2227,7 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); if( pTab->pSelect ){ - sqliteDeleteColumnNames(db, pTab); + sqlite3DeleteColumnNames(db, pTab); pTab->aCol = 0; pTab->nCol = 0; } @@ -3702,7 +3709,8 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbFree(db, pItem->zDatabase); sqlite3DbFree(db, pItem->zName); sqlite3DbFree(db, pItem->zAlias); - sqlite3DbFree(db, pItem->zIndexedBy); + if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); + if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pTab); sqlite3SelectDelete(db, pItem->pSelect); sqlite3ExprDelete(db, pItem->pOn); @@ -3775,17 +3783,37 @@ void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ assert( pIndexedBy!=0 ); if( p && ALWAYS(p->nSrc>0) ){ struct SrcList_item *pItem = &p->a[p->nSrc-1]; - assert( pItem->notIndexed==0 && pItem->zIndexedBy==0 ); + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); if( pIndexedBy->n==1 && !pIndexedBy->z ){ /* A "NOT INDEXED" clause was supplied. See parse.y ** construct "indexed_opt" for details. */ - pItem->notIndexed = 1; + pItem->fg.notIndexed = 1; }else{ - pItem->zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->fg.isIndexedBy = (pItem->u1.zIndexedBy!=0); } } } +/* +** Add the list of function arguments to the SrcList entry for a +** table-valued-function. +*/ +void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){ + if( p && pList ){ + struct SrcList_item *pItem = &p->a[p->nSrc-1]; + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); + pItem->u1.pFuncArg = pList; + pItem->fg.isTabFunc = 1; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + /* ** When building up a FROM clause in the parser, the join operator ** is initially attached to the left operand. But the code generator @@ -3805,9 +3833,9 @@ void sqlite3SrcListShiftJoinType(SrcList *p){ if( p ){ int i; for(i=p->nSrc-1; i>0; i--){ - p->a[i].jointype = p->a[i-1].jointype; + p->a[i].fg.jointype = p->a[i-1].fg.jointype; } - p->a[0].jointype = 0; + p->a[0].fg.jointype = 0; } } diff --git a/src/expr.c b/src/expr.c index 5acb909667..1062733cb9 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1034,16 +1034,18 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->jointype = pOldItem->jointype; + pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; - pNewItem->isCorrelated = pOldItem->isCorrelated; - pNewItem->viaCoroutine = pOldItem->viaCoroutine; - pNewItem->isRecursive = pOldItem->isRecursive; - pNewItem->zIndexedBy = sqlite3DbStrDup(db, pOldItem->zIndexedBy); - pNewItem->notIndexed = pOldItem->notIndexed; - pNewItem->pIndex = pOldItem->pIndex; + if( pNewItem->fg.isIndexedBy ){ + pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + } + pNewItem->pIBIndex = pOldItem->pIBIndex; + if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + } pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ pTab->nRef++; diff --git a/src/insert.c b/src/insert.c index e1b4344b7f..d9d88cab6a 100644 --- a/src/insert.c +++ b/src/insert.c @@ -2008,7 +2008,7 @@ static int xferOptimization( sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName); } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ - u8 useSeekResult = 0; + u8 idxInsFlags = 0; for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; } @@ -2043,12 +2043,15 @@ static int xferOptimization( if( sqlite3_stricmp("BINARY", zColl) ) break; } if( i==pSrcIdx->nColumn ){ - useSeekResult = OPFLAG_USESEEKRESULT; + idxInsFlags = OPFLAG_USESEEKRESULT; sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); } } + if( !HasRowid(pSrc) && pDestIdx->idxType==2 ){ + idxInsFlags |= OPFLAG_NCHANGE; + } sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); - sqlite3VdbeChangeP5(v, useSeekResult); + sqlite3VdbeChangeP5(v, idxInsFlags); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); diff --git a/src/main.c b/src/main.c index d0545ccf16..fbb7da437e 100644 --- a/src/main.c +++ b/src/main.c @@ -932,17 +932,23 @@ static void functionDestroy(sqlite3 *db, FuncDef *p){ static void disconnectAllVtab(sqlite3 *db){ #ifndef SQLITE_OMIT_VIRTUALTABLE int i; + HashElem *p; sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ Schema *pSchema = db->aDb[i].pSchema; if( db->aDb[i].pSchema ){ - HashElem *p; for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ Table *pTab = (Table *)sqliteHashData(p); if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); } } } + for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){ + Module *pMod = (Module *)sqliteHashData(p); + if( pMod->pEpoTab ){ + sqlite3VtabDisconnect(db, pMod->pEpoTab); + } + } sqlite3VtabUnlockList(db); sqlite3BtreeLeaveAll(db); #else @@ -1120,6 +1126,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ if( pMod->xDestroy ){ pMod->xDestroy(pMod->pAux); } + sqlite3VtabEponymousTableClear(db, pMod); sqlite3DbFree(db, pMod); } sqlite3HashClear(&db->aModule); diff --git a/src/parse.y b/src/parse.y index d7aa763683..3d186b28aa 100644 --- a/src/parse.y +++ b/src/parse.y @@ -586,7 +586,7 @@ from(A) ::= FROM seltablist(X). { // stl_prefix(A) ::= seltablist(X) joinop(Y). { A = X; - if( ALWAYS(A && A->nSrc>0) ) A->a[A->nSrc-1].jointype = (u8)Y; + if( ALWAYS(A && A->nSrc>0) ) A->a[A->nSrc-1].fg.jointype = (u8)Y; } stl_prefix(A) ::= . {A = 0;} seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) @@ -594,6 +594,11 @@ seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) A = sqlite3SrcListAppendFromTerm(pParse,X,&Y,&D,&Z,0,N,U); sqlite3SrcListIndexedBy(pParse, A, &I); } +seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) + on_opt(N) using_opt(U). { + A = sqlite3SrcListAppendFromTerm(pParse,X,&Y,&D,&Z,0,N,U); + sqlite3SrcListFuncArgs(pParse, A, E); +} %ifndef SQLITE_OMIT_SUBQUERY seltablist(A) ::= stl_prefix(X) LP select(S) RP as(Z) on_opt(N) using_opt(U). { diff --git a/src/pragma.c b/src/pragma.c index ffa0685727..96ff136c25 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -727,6 +727,7 @@ void sqlite3Pragma( int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + if( sqlite3ReadSchema(pParse) ) goto pragma_out; } break; } diff --git a/src/resolve.c b/src/resolve.c index fd57fd7028..c859e886a7 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -306,7 +306,7 @@ static int lookupName( ** USING clause, then skip this match. */ if( cnt==1 ){ - if( pItem->jointype & JT_NATURAL ) continue; + if( pItem->fg.jointype & JT_NATURAL ) continue; if( nameInUsingClause(pItem->pUsing, zCol) ) continue; } cnt++; @@ -321,8 +321,8 @@ static int lookupName( pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; /* RIGHT JOIN not (yet) supported */ - assert( (pMatch->jointype & JT_RIGHT)==0 ); - if( (pMatch->jointype & JT_LEFT)!=0 ){ + assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); + if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } pSchema = pExpr->pTab->pSchema; @@ -1142,7 +1142,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ int isCompound; /* True if p is a compound select */ int nCompound; /* Number of compound terms processed so far */ Parse *pParse; /* Parsing context */ - ExprList *pEList; /* Result set expression list */ int i; /* Loop counter */ ExprList *pGroupBy; /* The GROUP BY clause */ Select *pLeftmost; /* Left-most of SELECT of a compound */ @@ -1215,7 +1214,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** parent contexts. After resolving references to expressions in ** pItem->pSelect, check if this value has changed. If so, then ** SELECT statement pItem->pSelect must be correlated. Set the - ** pItem->isCorrelated flag if this is the case. */ + ** pItem->fg.isCorrelated flag if this is the case. */ for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; if( pItem->zName ) pParse->zAuthContext = pItem->zName; @@ -1224,8 +1223,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pParse->nErr || db->mallocFailed ) return WRC_Abort; for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; - assert( pItem->isCorrelated==0 && nRef<=0 ); - pItem->isCorrelated = (nRef!=0); + assert( pItem->fg.isCorrelated==0 && nRef<=0 ); + pItem->fg.isCorrelated = (nRef!=0); } } @@ -1237,14 +1236,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sNC.pNext = pOuterNC; /* Resolve names in the result set. */ - pEList = p->pEList; - assert( pEList!=0 ); - for(i=0; inExpr; i++){ - Expr *pX = pEList->a[i].pExpr; - if( sqlite3ResolveExprNames(&sNC, pX) ){ - return WRC_Abort; - } - } + if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. @@ -1277,6 +1269,16 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + /* Resolve names in table-valued-function arguments */ + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + if( pItem->fg.isTabFunc + && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } + } + /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries */ @@ -1440,6 +1442,22 @@ int sqlite3ResolveExprNames( return ExprHasProperty(pExpr, EP_Error); } +/* +** Resolve all names for all expression in an expression list. This is +** just like sqlite3ResolveExprNames() except that it works for an expression +** list rather than a single expression. +*/ +int sqlite3ResolveExprListNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + ExprList *pList /* The expression list to be analyzed. */ +){ + int i; + assert( pList!=0 ); + for(i=0; inExpr; i++){ + if( sqlite3ResolveExprNames(pNC, pList->a[i].pExpr) ) return WRC_Abort; + } + return WRC_Continue; +} /* ** Resolve all names in all expressions of a SELECT and in all diff --git a/src/select.c b/src/select.c index 8ac98f1759..acc4c88a57 100644 --- a/src/select.c +++ b/src/select.c @@ -406,12 +406,12 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ int isOuter; if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; - isOuter = (pRight->jointype & JT_OUTER)!=0; + isOuter = (pRight->fg.jointype & JT_OUTER)!=0; /* When the NATURAL keyword is present, add WHERE clause terms for ** every column that the two tables have in common. */ - if( pRight->jointype & JT_NATURAL ){ + if( pRight->fg.jointype & JT_NATURAL ){ if( pRight->pOn || pRight->pUsing ){ sqlite3ErrorMsg(pParse, "a NATURAL join may not have " "an ON or USING clause", 0); @@ -1933,7 +1933,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ ** ** ** There is exactly one reference to the recursive-table in the FROM clause -** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** of recursive-query, marked with the SrcList->a[].fg.isRecursive flag. ** ** The setup-query runs once to generate an initial set of rows that go ** into a Queue table. Rows are extracted from the Queue table one by @@ -1998,7 +1998,7 @@ static void generateWithRecursiveQuery( /* Locate the cursor number of the Current table */ for(i=0; ALWAYS(inSrc); i++){ - if( pSrc->a[i].isRecursive ){ + if( pSrc->a[i].fg.isRecursive ){ iCurrent = pSrc->a[i].iCursor; break; } @@ -3413,7 +3413,7 @@ static int flattenSubquery( ** is fraught with danger. Best to avoid the whole thing. If the ** subquery is the right term of a LEFT JOIN, then do not flatten. */ - if( (pSubitem->jointype & JT_OUTER)!=0 ){ + if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ return 0; } @@ -3584,7 +3584,7 @@ static int flattenSubquery( if( pSrc ){ assert( pParent==p ); /* First time through the loop */ - jointype = pSubitem->jointype; + jointype = pSubitem->fg.jointype; }else{ assert( pParent!=p ); /* 2nd and subsequent times through the loop */ pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); @@ -3624,7 +3624,7 @@ static int flattenSubquery( pSrc->a[i+iFrom] = pSubSrc->a[i]; memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].jointype = jointype; + pSrc->a[iFrom].fg.jointype = jointype; /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. @@ -3875,9 +3875,9 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ ** pFrom->pIndex and return SQLITE_OK. */ int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ - if( pFrom->pTab && pFrom->zIndexedBy ){ + if( pFrom->pTab && pFrom->fg.isIndexedBy ){ Table *pTab = pFrom->pTab; - char *zIndexedBy = pFrom->zIndexedBy; + char *zIndexedBy = pFrom->u1.zIndexedBy; Index *pIdx; for(pIdx=pTab->pIndex; pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); @@ -3888,7 +3888,7 @@ int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ pParse->checkSchema = 1; return SQLITE_ERROR; } - pFrom->pIndex = pIdx; + pFrom->pIBIndex = pIdx; } return SQLITE_OK; } @@ -4083,7 +4083,7 @@ static int withExpand( && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ pItem->pTab = pTab; - pItem->isRecursive = 1; + pItem->fg.isRecursive = 1; pTab->nRef++; pSel->selFlags |= SF_Recursive; } @@ -4213,8 +4213,8 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->isRecursive==0 || pFrom->pTab ); - if( pFrom->isRecursive ) continue; + assert( pFrom->fg.isRecursive==0 || pFrom->pTab ); + if( pFrom->fg.isRecursive ) continue; if( pFrom->pTab!=0 ){ /* This statement has already been prepared. There is no need ** to go further. */ @@ -4377,7 +4377,7 @@ static int selectExpander(Walker *pWalker, Select *p){ tableSeen = 1; if( i>0 && zTName==0 ){ - if( (pFrom->jointype & JT_NATURAL)!=0 + if( (pFrom->fg.jointype & JT_NATURAL)!=0 && tableAndColumnIndex(pTabList, i, zName, 0, 0) ){ /* In a NATURAL join, omit the join columns from the @@ -4904,7 +4904,7 @@ int sqlite3Select( ** is sufficient, though the subroutine to manifest the view does need ** to be invoked again. */ if( pItem->addrFillSub ){ - if( pItem->viaCoroutine==0 ){ + if( pItem->fg.viaCoroutine==0 ){ sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); } continue; @@ -4922,7 +4922,7 @@ int sqlite3Select( /* Make copies of constant WHERE-clause terms in the outer query down ** inside the subquery. This can help the subquery to run more efficiently. */ - if( (pItem->jointype & JT_OUTER)==0 + if( (pItem->fg.jointype & JT_OUTER)==0 && pushDownWhereTerms(db, pSub, p->pWhere, pItem->iCursor) ){ #if SELECTTRACE_ENABLED @@ -4951,7 +4951,7 @@ int sqlite3Select( explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); - pItem->viaCoroutine = 1; + pItem->fg.viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); @@ -4969,7 +4969,7 @@ int sqlite3Select( pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; - if( pItem->isCorrelated==0 ){ + if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ diff --git a/src/shell.c b/src/shell.c index 175f849b62..f10a6932e0 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1336,6 +1336,8 @@ static int display_stats( fprintf(pArg->out, "Virtual Machine Steps: %d\n", iCur); } + /* Do not remove this machine readable comment: extra-stats-output-here */ + return 0; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 85fe1abaf8..cefda6f99b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3373,7 +3373,8 @@ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); ** ** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the ** [prepared statement] S has been stepped at least once using -** [sqlite3_step(S)] but has not run to completion and/or has not +** [sqlite3_step(S)] but has neither run to completion (returned +** [SQLITE_DONE] from [sqlite3_step(S)]) nor ** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) ** interface returns false if S is a NULL pointer. If S is not a ** NULL pointer and is not a pointer to a valid [prepared statement] @@ -4535,9 +4536,9 @@ typedef void (*sqlite3_destructor_type)(void*); ** to by the second parameter and which is N bytes long where N is the ** third parameter. ** -** ^The sqlite3_result_zeroblob() and zeroblob64() interfaces set the result -** of the application-defined function to be a BLOB containing all zero -** bytes and N bytes in size, where N is the value of the 2nd parameter. +** ^The sqlite3_result_zeroblob(C,N) and sqlite3_result_zeroblob64(C,N) +** interfaces set the result of the application-defined function to be +** a BLOB containing all zero bytes and N bytes in size. ** ** ^The sqlite3_result_double() interface sets the result from ** an application-defined function to be a floating point value specified diff --git a/src/sqliteInt.h b/src/sqliteInt.h index aa6b63bffe..2c912bf503 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -187,15 +187,18 @@ /* ** Make sure that the compiler intrinsics we desire are enabled when -** compiling with an appropriate version of MSVC. +** compiling with an appropriate version of MSVC unless prevented by +** the SQLITE_DISABLE_INTRINSIC define. */ -#if defined(_MSC_VER) && _MSC_VER>=1300 -# if !defined(_WIN32_WCE) -# include -# pragma intrinsic(_byteswap_ushort) -# pragma intrinsic(_byteswap_ulong) -# else -# include +#if !defined(SQLITE_DISABLE_INTRINSIC) +# if defined(_MSC_VER) && _MSC_VER>=1300 +# if !defined(_WIN32_WCE) +# include +# pragma intrinsic(_byteswap_ushort) +# pragma intrinsic(_byteswap_ulong) +# else +# include +# endif # endif #endif @@ -1484,6 +1487,7 @@ struct Module { const char *zName; /* Name passed to create_module() */ void *pAux; /* pAux passed to create_module() */ void (*xDestroy)(void *); /* Module destructor function */ + Table *pEpoTab; /* Eponymous table for this module */ }; /* @@ -1654,7 +1658,7 @@ struct Table { #endif #ifndef SQLITE_OMIT_VIRTUALTABLE int nModuleArg; /* Number of arguments to the module */ - char **azModuleArg; /* Text of all module args. [0] is module name */ + char **azModuleArg; /* 0: module 1: schema 2: vtab name 3...: args */ VTable *pVTable; /* List of VTable objects. */ #endif Trigger *pTrigger; /* List of triggers stored in pSchema */ @@ -2289,11 +2293,15 @@ struct SrcList { int addrFillSub; /* Address of subroutine to manifest a subquery */ int regReturn; /* Register holding return address of addrFillSub */ int regResult; /* Registers holding results of a co-routine */ - u8 jointype; /* Type of join between this able and the previous */ - unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ - unsigned isCorrelated :1; /* True if sub-query is correlated */ - unsigned viaCoroutine :1; /* Implemented as a co-routine */ - unsigned isRecursive :1; /* True for recursive reference in WITH */ + struct { + u8 jointype; /* Type of join between this able and the previous */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ + unsigned isTabFunc :1; /* True if table-valued-function syntax */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ + } fg; #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif @@ -2301,8 +2309,11 @@ struct SrcList { Expr *pOn; /* The ON clause of a join */ IdList *pUsing; /* The USING clause of a join */ Bitmask colUsed; /* Bit N (1<" clause */ - Index *pIndex; /* Index structure corresponding to zIndex, if any */ + union { + char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ + ExprList *pFuncArg; /* Arguments to table-valued-function */ + } u1; + Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ } a[1]; /* One entry for each identifier on the list */ }; @@ -3257,6 +3268,7 @@ void sqlite3ResetOneSchema(sqlite3*,int); void sqlite3CollapseDatabaseArray(sqlite3*); void sqlite3BeginParse(Parse*,int); void sqlite3CommitInternalChanges(sqlite3*); +void sqlite3DeleteColumnNames(sqlite3*,Table*); Table *sqlite3ResultSetOfSelect(Parse*,Select*); void sqlite3OpenMasterTable(Parse *, int); Index *sqlite3PrimaryKeyIndex(Table*); @@ -3328,6 +3340,7 @@ SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*); SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, Expr*, IdList*); void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); +void sqlite3SrcListFuncArgs(Parse*, SrcList*, ExprList*); int sqlite3IndexedByLookup(Parse *, struct SrcList_item *); void sqlite3SrcListShiftJoinType(SrcList*); void sqlite3SrcListAssignCursors(Parse*, SrcList*); @@ -3619,6 +3632,7 @@ void sqlite3SelectPrep(Parse*, Select*, NameContext*); void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); int sqlite3ResolveExprNames(NameContext*, Expr*); +int sqlite3ResolveExprListNames(NameContext*, ExprList*); void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); @@ -3727,6 +3741,8 @@ void sqlite3AutoLoadExtensions(sqlite3*); VTable *sqlite3GetVTable(sqlite3*, Table*); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif +int sqlite3VtabEponymousTableInit(Parse*,Module*); +void sqlite3VtabEponymousTableClear(sqlite3*,Module*); void sqlite3VtabMakeWritable(Parse*,Table*); void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int); void sqlite3VtabFinishParse(Parse*, Token*); diff --git a/src/test1.c b/src/test1.c index ceccf10db8..539c674d9a 100644 --- a/src/test1.c +++ b/src/test1.c @@ -338,7 +338,7 @@ static int test_exec_hex( int rc, i, j; char *zErr = 0; char *zHex; - char zSql[500]; + char zSql[501]; char zBuf[30]; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], @@ -347,7 +347,7 @@ static int test_exec_hex( } if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; zHex = argv[2]; - for(i=j=0; izAlias ){ sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); } - if( pItem->jointype & JT_LEFT ){ + if( pItem->fg.jointype & JT_LEFT ){ sqlite3XPrintf(&x, 0, " LEFT-JOIN"); } sqlite3StrAccumFinish(&x); @@ -128,6 +128,9 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ if( pItem->pSelect ){ sqlite3TreeViewSelect(pView, pItem->pSelect, 0); } + if( pItem->fg.isTabFunc ){ + sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + } sqlite3TreeViewPop(pView); } sqlite3TreeViewPop(pView); diff --git a/src/util.c b/src/util.c index 091481d921..38072a35cb 100644 --- a/src/util.c +++ b/src/util.c @@ -1082,11 +1082,13 @@ u32 sqlite3Get4byte(const u8 *p){ u32 x; memcpy(&x,p,4); return x; -#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__) && GCC_VERSION>=4003000 +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(__GNUC__) && GCC_VERSION>=4003000 u32 x; memcpy(&x,p,4); return __builtin_bswap32(x); -#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 u32 x; memcpy(&x,p,4); return _byteswap_ulong(x); diff --git a/src/vdbe.c b/src/vdbe.c index 3a960ccf5c..75d9630980 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -581,7 +581,7 @@ int sqlite3VdbeExec( ** sqlite3_column_text16() failed. */ goto no_mem; } - assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY ); + assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); assert( p->bIsReader || p->readOnly!=0 ); p->rc = SQLITE_OK; p->iCurrentTime = 0; @@ -3018,12 +3018,12 @@ case OP_AutoCommit: { goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; - if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ - p->pc = (int)(pOp - aOp); - db->autoCommit = (u8)(1-desiredAutoCommit); - p->rc = rc = SQLITE_BUSY; - goto vdbe_return; - } + } + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + db->autoCommit = (u8)(1-desiredAutoCommit); + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; } assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); @@ -3095,9 +3095,11 @@ case OP_Transaction: { if( pBt ){ rc = sqlite3BtreeBeginTrans(pBt, pOp->p2); - if( rc==SQLITE_BUSY ){ + testcase( rc==SQLITE_BUSY_SNAPSHOT ); + testcase( rc==SQLITE_BUSY_RECOVERY ); + if( (rc&0xff)==SQLITE_BUSY ){ p->pc = (int)(pOp - aOp); - p->rc = rc = SQLITE_BUSY; + p->rc = rc; goto vdbe_return; } if( rc!=SQLITE_OK ){ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 2c75bbf2ab..c9d1cb830d 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -611,7 +611,7 @@ end_of_step: ** were called on statement p. */ assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR - || rc==SQLITE_BUSY || rc==SQLITE_MISUSE + || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE ); assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 5c4f31b55d..a8dfd9ea13 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1095,8 +1095,9 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ zColl = "B"; n = 1; } - if( i+n>nTemp-6 ){ + if( i+n>nTemp-7 ){ memcpy(&zTemp[i],",...",4); + i += 4; break; } zTemp[i++] = ','; diff --git a/src/vtab.c b/src/vtab.c index 2ae861e67f..1675ca2e31 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -58,6 +58,7 @@ static int createModule( pMod->pModule = pModule; pMod->pAux = pAux; pMod->xDestroy = xDestroy; + pMod->pEpoTab = 0; pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); assert( pDel==0 || pDel==pMod ); if( pDel ){ @@ -285,23 +286,17 @@ void sqlite3VtabClear(sqlite3 *db, Table *p){ ** deleted. */ static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int i = pTable->nModuleArg++; - int nBytes = sizeof(char *)*(1+pTable->nModuleArg); + int nBytes = sizeof(char *)*(2+pTable->nModuleArg); char **azModuleArg; azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); if( azModuleArg==0 ){ - int j; - for(j=0; jazModuleArg[j]); - } sqlite3DbFree(db, zArg); - sqlite3DbFree(db, pTable->azModuleArg); - pTable->nModuleArg = 0; }else{ + int i = pTable->nModuleArg++; azModuleArg[i] = zArg; azModuleArg[i+1] = 0; + pTable->azModuleArg = azModuleArg; } - pTable->azModuleArg = azModuleArg; } /* @@ -704,7 +699,7 @@ int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){ ** invoke it now. If the module has not been registered, return an ** error. Otherwise, do nothing. */ - if( !pMod ){ + if( pMod==0 || pMod->pModule->xCreate==0 || pMod->pModule->xDestroy==0 ){ *pzErr = sqlite3MPrintf(db, "no such module: %s", zMod); rc = SQLITE_ERROR; }else{ @@ -806,6 +801,7 @@ int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ VTable *p; + int (*xDestroy)(sqlite3_vtab *); for(p=pTab->pVTable; p; p=p->pNext){ assert( p->pVtab ); if( p->pVtab->nRef>0 ){ @@ -813,7 +809,9 @@ int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ } } p = vtabDisconnectAll(db, pTab); - rc = p->pMod->pModule->xDestroy(p->pVtab); + xDestroy = p->pMod->pModule->xDestroy; + assert( xDestroy!=0 ); /* Checked before the virtual table is created */ + rc = xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ assert( pTab->pVTable==p && p->pNext==0 ); @@ -1092,6 +1090,67 @@ void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ } } +/* +** Check to see if virtual tale module pMod can be have an eponymous +** virtual table instance. If it can, create one if one does not already +** exist. Return non-zero if the eponymous virtual table instance exists +** when this routine returns, and return zero if it does not exist. +** +** An eponymous virtual table instance is one that is named after its +** module, and more importantly, does not require a CREATE VIRTUAL TABLE +** statement in order to come into existance. Eponymous virtual table +** instances always exist. They cannot be DROP-ed. +** +** Any virtual table module for which xConnect and xCreate are the same +** method can have an eponymous virtual table instance. +*/ +int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ + const sqlite3_module *pModule = pMod->pModule; + Table *pTab; + char *zErr = 0; + int nName; + int rc; + sqlite3 *db = pParse->db; + if( pMod->pEpoTab ) return 1; + if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0; + nName = sqlite3Strlen30(pMod->zName) + 1; + pTab = sqlite3DbMallocZero(db, sizeof(Table) + nName); + if( pTab==0 ) return 0; + pMod->pEpoTab = pTab; + pTab->zName = (char*)&pTab[1]; + memcpy(pTab->zName, pMod->zName, nName); + pTab->nRef = 1; + pTab->pSchema = db->aDb[0].pSchema; + pTab->tabFlags |= TF_Virtual; + pTab->nModuleArg = 0; + pTab->iPKey = -1; + addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(db, pTab, 0); + addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); + if( rc ){ + sqlite3ErrorMsg(pParse, "%s", zErr); + sqlite3DbFree(db, zErr); + sqlite3VtabEponymousTableClear(db, pMod); + return 0; + } + return 1; +} + +/* +** Erase the eponymous virtual table instance associated with +** virtual table module pMod, if it exists. +*/ +void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ + Table *pTab = pMod->pEpoTab; + if( (pTab = pMod->pEpoTab)!=0 ){ + sqlite3DeleteColumnNames(db, pTab); + sqlite3VtabClear(db, pTab); + sqlite3DbFree(db, pTab); + pMod->pEpoTab = 0; + } +} + /* ** Return the ON CONFLICT resolution mode in effect for the virtual ** table update operation currently in progress. diff --git a/src/walker.c b/src/walker.c index e30bb60b5a..81e0f2cd60 100644 --- a/src/walker.c +++ b/src/walker.c @@ -105,6 +105,11 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ if( sqlite3WalkSelect(pWalker, pItem->pSelect) ){ return WRC_Abort; } + if( pItem->fg.isTabFunc + && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } } } return WRC_Continue; diff --git a/src/where.c b/src/where.c index 5be830e7ed..bdcfeaa9aa 100644 --- a/src/where.c +++ b/src/where.c @@ -709,7 +709,7 @@ static void constructAutomaticIndex( /* Fill the automatic index with content */ sqlite3ExprCachePush(pParse); pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; - if( pTabItem->viaCoroutine ){ + if( pTabItem->fg.viaCoroutine ){ int regYield = pTabItem->regReturn; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); @@ -728,10 +728,10 @@ static void constructAutomaticIndex( sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); - if( pTabItem->viaCoroutine ){ + if( pTabItem->fg.viaCoroutine ){ translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); - pTabItem->viaCoroutine = 0; + pTabItem->fg.viaCoroutine = 0; }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); } @@ -2128,7 +2128,7 @@ static int whereLoopAddBtreeIndex( assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); if( pNew->wsFlags & WHERE_BTM_LIMIT ){ opMask = WO_LT|WO_LE; - }else if( /*pProbe->tnum<=0 ||*/ (pSrc->jointype & JT_LEFT)!=0 ){ + }else if( /*pProbe->tnum<=0 ||*/ (pSrc->fg.jointype & JT_LEFT)!=0 ){ opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE; }else{ opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; @@ -2423,6 +2423,10 @@ static Bitmask columnsInIndex(Index *pIdx){ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ int i; WhereTerm *pTerm; + while( pWhere->op==TK_AND ){ + if( !whereUsablePartialIndex(iTab,pWC,pWhere->pLeft) ) return 0; + pWhere = pWhere->pRight; + } for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ Expr *pExpr = pTerm->pExpr; if( sqlite3ExprImpliesExpr(pExpr, pWhere, iTab) @@ -2498,9 +2502,9 @@ static int whereLoopAddBtree( pWC = pBuilder->pWC; assert( !IsVirtual(pSrc->pTab) ); - if( pSrc->pIndex ){ + if( pSrc->pIBIndex ){ /* An INDEXED BY clause specifies a particular index to use */ - pProbe = pSrc->pIndex; + pProbe = pSrc->pIBIndex; }else if( !HasRowid(pTab) ){ pProbe = pTab->pIndex; }else{ @@ -2520,7 +2524,7 @@ static int whereLoopAddBtree( aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; - if( pSrc->notIndexed==0 ){ + if( pSrc->fg.notIndexed==0 ){ /* The real indices of the table are only considered if the ** NOT INDEXED qualifier is omitted from the FROM clause */ sPk.pNext = pFirst; @@ -2532,14 +2536,14 @@ static int whereLoopAddBtree( #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* Automatic indexes */ - if( !pBuilder->pOrSet /* Not part of an OR optimization */ + if( !pBuilder->pOrSet /* Not part of an OR optimization */ && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 - && pSrc->pIndex==0 /* Has no INDEXED BY clause */ - && !pSrc->notIndexed /* Has no NOT INDEXED clause */ - && HasRowid(pTab) /* Is not a WITHOUT ROWID table. (FIXME: Why not?) */ - && !pSrc->isCorrelated /* Not a correlated subquery */ - && !pSrc->isRecursive /* Not a recursive common table expression. */ + && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */ + && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ + && HasRowid(pTab) /* Is not a WITHOUT ROWID table. (FIXME: Why not?) */ + && !pSrc->fg.isCorrelated /* Not a correlated subquery */ + && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ ){ /* Generate auto-index WhereLoops */ WhereTerm *pTerm; @@ -2660,7 +2664,7 @@ static int whereLoopAddBtree( /* If there was an INDEXED BY clause, then only that one index is ** considered. */ - if( pSrc->pIndex ) break; + if( pSrc->pIBIndex ) break; } return rc; } @@ -3006,16 +3010,16 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ Bitmask mUnusable = 0; pNew->iTab = iTab; pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); - if( ((pItem->jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){ + if( ((pItem->fg.jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){ /* This condition is true when pItem is the FROM clause term on the ** right-hand-side of a LEFT or CROSS JOIN. */ mExtra = mPrior; } - priorJointype = pItem->jointype; + priorJointype = pItem->fg.jointype; if( IsVirtual(pItem->pTab) ){ struct SrcList_item *p; for(p=&pItem[1]; pjointype & (JT_LEFT|JT_CROSS)) ){ + if( mUnusable || (p->fg.jointype & (JT_LEFT|JT_CROSS)) ){ mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); } } @@ -3745,7 +3749,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pItem = pWInfo->pTabList->a; pTab = pItem->pTab; if( IsVirtual(pTab) ) return 0; - if( pItem->zIndexedBy ) return 0; + if( pItem->fg.isIndexedBy ) return 0; iCur = pItem->iCursor; pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; @@ -4026,6 +4030,7 @@ WhereInfo *sqlite3WhereBegin( */ for(ii=0; iinSrc; ii++){ createMask(pMaskSet, pTabList->a[ii].iCursor); + sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); } #ifndef NDEBUG { @@ -4132,7 +4137,7 @@ WhereInfo *sqlite3WhereBegin( while( pWInfo->nLevel>=2 ){ WhereTerm *pTerm, *pEnd; pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; - if( (pWInfo->pTabList->a[pLoop->iTab].jointype & JT_LEFT)==0 ) break; + if( (pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 ) break; if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 && (pLoop->wsFlags & WHERE_ONEROW)==0 ){ @@ -4425,7 +4430,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ ** the co-routine into OP_Copy of result contained in a register. ** OP_Rowid becomes OP_Null. */ - if( pTabItem->viaCoroutine && !db->mallocFailed ){ + if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){ translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur, pTabItem->regResult); continue; diff --git a/src/whereInt.h b/src/whereInt.h index 8929d8c4be..76b665d971 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -475,6 +475,7 @@ void sqlite3WhereSplit(WhereClause*,Expr*,u8); Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); +void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); diff --git a/src/wherecode.c b/src/wherecode.c index 9747f7f375..eb23d8f1a2 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -646,14 +646,14 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** initialize a memory cell that records if this table matches any ** row of the left table of the join. */ - if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){ + if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ pLevel->iLeftJoin = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); VdbeComment((v, "init LEFT JOIN no-match flag")); } /* Special case of a FROM clause subquery implemented as a co-routine */ - if( pTabItem->viaCoroutine ){ + if( pTabItem->fg.viaCoroutine ){ int regYield = pTabItem->regReturn; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); @@ -1395,7 +1395,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( static const u8 aStep[] = { OP_Next, OP_Prev }; static const u8 aStart[] = { OP_Rewind, OP_Last }; assert( bRev==0 || bRev==1 ); - if( pTabItem->isRecursive ){ + if( pTabItem->fg.isRecursive ){ /* Tables marked isRecursive have only a single row that is stored in ** a pseudo-cursor. No need to Rewind or Next such cursors. */ pLevel->op = OP_Noop; diff --git a/src/whereexpr.c b/src/whereexpr.c index 3607ef5352..d6f94b3e1a 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1247,3 +1247,42 @@ void sqlite3WhereExprAnalyze( exprAnalyze(pTabList, pWC, i); } } + +/* +** For table-valued-functions, transform the function arguments into +** new WHERE clause terms. +** +** Each function argument translates into an equality constraint against +** a HIDDEN column in the table. +*/ +void sqlite3WhereTabFuncArgs( + Parse *pParse, /* Parsing context */ + struct SrcList_item *pItem, /* The FROM clause term to process */ + WhereClause *pWC /* Xfer function arguments to here */ +){ + Table *pTab; + int j, k; + ExprList *pArgs; + Expr *pColRef; + Expr *pTerm; + if( pItem->fg.isTabFunc==0 ) return; + pTab = pItem->pTab; + assert( pTab!=0 ); + pArgs = pItem->u1.pFuncArg; + assert( pArgs!=0 ); + for(j=k=0; jnExpr; j++){ + while( knCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){ k++; } + if( k>=pTab->nCol ){ + sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", + pTab->zName, j); + return; + } + pColRef = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); + if( pColRef==0 ) return; + pColRef->iTable = pItem->iCursor; + pColRef->iColumn = k++; + pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, + sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); + whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); + } +} diff --git a/test/capi3d.test b/test/capi3d.test index 1459c5abfe..3b9b8375d1 100644 --- a/test/capi3d.test +++ b/test/capi3d.test @@ -161,11 +161,11 @@ do_test capi3d-4.2.1 { do_test capi3d-4.2.2 { sqlite3_stmt_busy $::s1 -} {1} +} {0} do_catchsql_test capi3d-4.2.3 { VACUUM -} {1 {cannot VACUUM - SQL statements in progress}} +} {0 {}} do_test capi3d-4.2.4 { sqlite3_reset $::s1 diff --git a/test/index6.test b/test/index6.test index e15820290e..33ae3d1cb1 100644 --- a/test/index6.test +++ b/test/index6.test @@ -345,5 +345,35 @@ do_execsql_test index6-9.2 { SELECT a,b,c,'|' FROM t9 ORDER BY a; } {1 1 9 | 10 35 35 | 11 15 82 | 20 5 5 |} +# AND-connected terms in the WHERE clause of a partial index +# +do_execsql_test index6-10.1 { + CREATE TABLE t10(a,b,c,d,e INTEGER PRIMARY KEY); + INSERT INTO t10 VALUES + (1,2,3,4,5), + (2,3,4,5,6), + (3,4,5,6,7), + (1,2,3,8,9); + CREATE INDEX t10x ON t10(d) WHERE a=1 AND b=2 AND c=3; + SELECT e FROM t10 WHERE a=1 AND b=2 AND c=3 ORDER BY d; +} {5 9} +do_execsql_test index6-10.1eqp { + EXPLAIN QUERY PLAN + SELECT e FROM t10 WHERE a=1 AND b=2 AND c=3 ORDER BY d; +} {/USING INDEX t10x/} +do_execsql_test index6-10.2 { + SELECT e FROM t10 WHERE c=3 AND 2=b AND a=1 ORDER BY d DESC; +} {9 5} +do_execsql_test index6-10.2eqp { + EXPLAIN QUERY PLAN + SELECT e FROM t10 WHERE c=3 AND 2=b AND a=1 ORDER BY d DESC; +} {/USING INDEX t10x/} +do_execsql_test index6-10.3 { + SELECT e FROM t10 WHERE a=1 AND b=2 ORDER BY d DESC; +} {9 5} +do_execsql_test index6-10.3eqp { + EXPLAIN QUERY PLAN + SELECT e FROM t10 WHERE a=1 AND b=2 ORDER BY d DESC; +} {~/USING INDEX t10x/} finish_test diff --git a/test/memsubsys2.test b/test/memsubsys2.test index a40f4e4f03..68e4e22d32 100644 --- a/test/memsubsys2.test +++ b/test/memsubsys2.test @@ -80,8 +80,11 @@ do_test memsubsys2-2.1 { sqlite3_free [set x [sqlite3_malloc 100000]] expr {$x!="0"} } {1} -do_test memsubsys2-2.2 { - expr {[sqlite3_memory_highwater 0]>=[sqlite3_memory_used]+$highwater} +do_test memsubsys2-2.2.1 { + expr {[sqlite3_memory_highwater 0]>=[sqlite3_memory_used]+100000} +} {1} +do_test memsubsys2-2.2.2 { + expr {[sqlite3_memory_highwater 0]>=$highwater+50000} } {1} # Test 3: Verify that turning of memstatus disables the statistics diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 43473e2a0d..eadf69ce81 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -286,6 +286,22 @@ foreach {key value} [array get ::Platforms] { } } +# Output log +# +set LOG [open releasetest-out.txt w] +proc PUTS {args} { + if {[llength $args]==2} { + puts [lindex $args 0] [lindex $args 1] + puts [lindex $args 0] $::LOG [lindex $args 1] + } else { + puts [lindex $args 0] + puts $::LOG [lindex $args 0] + } +} +puts $LOG "$argv0 $argv" +set tm0 [clock format [clock seconds] -format {%Y-%m-%d %H:%M:%S} -gmt 1] +puts $LOG "start-time: $tm0 UTC" + # Open the file $logfile and look for a report on the number of errors # and the number of test cases run. Add these values to the global # $::NERRCASE and $::NTESTCASE variables. @@ -310,10 +326,15 @@ proc count_tests_and_errors {logfile rcVar errmsgVar} { } } if {[regexp {runtime error: +(.*)} $line all msg]} { - incr ::NERRCASE - if {$rc==0} { - set rc 1 - set errmsg $msg + # skip over "value is outside range" errors + if {[regexp {value .* is outside the range of representable} $line]} { + # noop + } else { + incr ::NERRCASE + if {$rc==0} { + set rc 1 + set errmsg $msg + } } } if {[regexp {fatal error +(.*)} $line all msg]} { @@ -403,7 +424,7 @@ proc run_test_suite {name testtarget config} { if {!$::TRACE} { set n [string length $title] - puts -nonewline "${title}[string repeat . [expr {63-$n}]]" + PUTS -nonewline "${title}[string repeat . [expr {63-$n}]]" flush stdout } @@ -428,12 +449,12 @@ proc run_test_suite {name testtarget config} { set seconds [expr {($tm2-$tm1)%60}] set tm [format (%02d:%02d:%02d) $hours $minutes $seconds] if {$rc} { - puts " FAIL $tm" + PUTS " FAIL $tm" incr ::NERR } else { - puts " Ok $tm" + PUTS " Ok $tm" } - if {$errmsg!=""} {puts " $errmsg"} + if {$errmsg!=""} {PUTS " $errmsg"} } } @@ -475,7 +496,7 @@ proc makeCommand { targets cflags opts } { # proc trace_cmd {args} { if {$::TRACE} { - puts $args + PUTS $args } if {!$::DRYRUN} { uplevel 1 $args @@ -543,25 +564,25 @@ proc process_options {argv} { } -info { - puts "Command-line Options:" - puts " --srcdir $::SRCDIR" - puts " --platform [list $platform]" - puts " --config [list $config]" + PUTS "Command-line Options:" + PUTS " --srcdir $::SRCDIR" + PUTS " --platform [list $platform]" + PUTS " --config [list $config]" if {$::QUICK} { - if {$::QUICK==1} {puts " --quick"} - if {$::QUICK==2} {puts " --veryquick"} + if {$::QUICK==1} {PUTS " --quick"} + if {$::QUICK==2} {PUTS " --veryquick"} } - if {$::MSVC} {puts " --msvc"} - if {$::BUILDONLY} {puts " --buildonly"} - if {$::DRYRUN} {puts " --dryrun"} - if {$::TRACE} {puts " --trace"} - puts "\nAvailable --platform options:" + if {$::MSVC} {PUTS " --msvc"} + if {$::BUILDONLY} {PUTS " --buildonly"} + if {$::DRYRUN} {PUTS " --dryrun"} + if {$::TRACE} {PUTS " --trace"} + PUTS "\nAvailable --platform options:" foreach y [lsort [array names ::Platforms]] { - puts " [list $y]" + PUTS " [list $y]" } - puts "\nAvailable --config options:" + PUTS "\nAvailable --config options:" foreach y [lsort [array names ::Configs]] { - puts " [list $y]" + PUTS " [list $y]" } exit } @@ -587,22 +608,22 @@ proc process_options {argv} { } default { - puts stderr "" - puts stderr [string trim $::USAGE_MESSAGE] + PUTS stderr "" + PUTS stderr [string trim $::USAGE_MESSAGE] exit -1 } } } if {0==[info exists ::Platforms($platform)]} { - puts "Unknown platform: $platform" - puts -nonewline "Set the -platform option to " + PUTS "Unknown platform: $platform" + PUTS -nonewline "Set the -platform option to " set print [list] foreach p [array names ::Platforms] { lappend print "\"$p\"" } lset print end "or [lindex $print end]" - puts "[join $print {, }]." + PUTS "[join $print {, }]." exit } @@ -612,17 +633,17 @@ proc process_options {argv} { } else { set ::CONFIGLIST $::Platforms($platform) } - puts "Running the following test configurations for $platform:" - puts " [string trim $::CONFIGLIST]" - puts -nonewline "Flags:" - if {$::DRYRUN} {puts -nonewline " --dryrun"} - if {$::BUILDONLY} {puts -nonewline " --buildonly"} - if {$::MSVC} {puts -nonewline " --msvc"} + PUTS "Running the following test configurations for $platform:" + PUTS " [string trim $::CONFIGLIST]" + PUTS -nonewline "Flags:" + if {$::DRYRUN} {PUTS -nonewline " --dryrun"} + if {$::BUILDONLY} {PUTS -nonewline " --buildonly"} + if {$::MSVC} {PUTS -nonewline " --msvc"} switch -- $::QUICK { - 1 {puts -nonewline " --quick"} - 2 {puts -nonewline " --veryquick"} + 1 {PUTS -nonewline " --quick"} + 2 {PUTS -nonewline " --veryquick"} } - puts "" + PUTS "" } # Main routine. @@ -632,7 +653,7 @@ proc main {argv} { # Process any command line options. set ::EXTRACONFIG {} process_options $argv - puts [string repeat * 79] + PUTS [string repeat * 79] set ::NERR 0 set ::NTEST 0 @@ -643,7 +664,7 @@ proc main {argv} { foreach {zConfig target} $::CONFIGLIST { if {$::MSVC && ($zConfig eq "Sanitize" || "checksymbols" in $target || "valgrindtest" in $target)} { - puts "Skipping $zConfig / $target for MSVC..." + PUTS "Skipping $zConfig / $target for MSVC..." continue } if {$target ne "checksymbols"} { @@ -688,10 +709,10 @@ proc main {argv} { set min [expr {($elapsetime/60)%60}] set sec [expr {$elapsetime%60}] set etime [format (%02d:%02d:%02d) $hr $min $sec] - puts [string repeat * 79] - puts "$::NERRCASE failures out of $::NTESTCASE tests in $etime" + PUTS [string repeat * 79] + PUTS "$::NERRCASE failures out of $::NTESTCASE tests in $etime" if {$::SQLITE_VERSION ne ""} { - puts "SQLite $::SQLITE_VERSION" + PUTS "SQLite $::SQLITE_VERSION" } } diff --git a/test/rowallock.test b/test/rowallock.test index 57dbbb2c2a..a1cab8d327 100644 --- a/test/rowallock.test +++ b/test/rowallock.test @@ -17,6 +17,11 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl set testprefix rowallock +set mmap_res 1000000 +ifcapable !mmap { + set mmap_res 0 +} + do_multiclient_test tn { code2 { db2 close } code3 { db3 close } @@ -36,7 +41,7 @@ do_multiclient_test tn { do_execsql_test 1.$tn.2 { PRAGMA mmap_size = 1000000; - } {1000000} + } $mmap_res do_execsql_test 1.$tn.2.1 { SELECT * FROM t1; } {1 2 3 4} diff --git a/test/shrink.test b/test/shrink.test index 6cc0786d28..7c9bed08b0 100644 --- a/test/shrink.test +++ b/test/shrink.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +test_set_config_pagecache 0 0 unset -nocomplain baseline do_test shrink-1.1 { @@ -43,4 +44,5 @@ do_test shrink-1.3 { expr {$::baseline > [sqlite3_memory_used]+500000} } {1} +test_restore_config_pagecache finish_test diff --git a/test/spellfix2.test b/test/spellfix2.test new file mode 100644 index 0000000000..b4614a9e1f --- /dev/null +++ b/test/spellfix2.test @@ -0,0 +1,114 @@ +# 2012 July 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix spellfix2 + +ifcapable !vtab { finish_test ; return } +load_static_extension db spellfix nextchar + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE demo USING spellfix1; + INSERT INTO demo(word) VALUES ('amsterdam'); + INSERT INTO demo(word) VALUES ('amsterdammetje'); + INSERT INTO demo(word) VALUES ('amsterdamania'); + INSERT INTO demo(word) VALUES ('amsterdamweg'); + INSERT INTO demo(word) VALUES ('amsterdamsestraat'); + INSERT INTO demo(word) VALUES ('amsterdamlaan'); +} + +do_execsql_test 1.1 { + SELECT word, distance, matchlen FROM demo + WHERE word MATCH 'amstedam*' AND top=3; +} { + amsterdam 100 9 + amsterdammetje 100 9 + amsterdamania 100 9 +} + +do_execsql_test 1.2 { + SELECT word, distance, matchlen FROM demo WHERE + word MATCH 'amstedam*' AND top=3 AND distance <= 100; +} { + amsterdam 100 9 + amsterdammetje 100 9 + amsterdamania 100 9 +} + +do_execsql_test 1.3 { + SELECT word, distance, matchlen FROM demo WHERE + word MATCH 'amstedam*' AND distance <= 100; +} { + amsterdam 100 9 + amsterdammetje 100 9 + amsterdamania 100 9 + amsterdamweg 100 9 + amsterdamsestraat 100 9 + amsterdamlaan 100 9 +} + +do_test 1.4 { + foreach l {a b c d e f g h i j k l m n o p q r s t u v w x y z} { + execsql { INSERT INTO demo(word) VALUES ('amsterdam' || $l) } + } +} {} + +do_execsql_test 1.5 { + SELECT count(*) FROM demo WHERE word MATCH 'amstedam*' AND distance <= 100; + SELECT count(*) FROM demo + WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20; +} { + 32 20 +} + +do_execsql_test 1.6 { + SELECT word, distance, matchlen FROM demo + WHERE word MATCH 'amstedam*' AND distance <= 100; +} { + amsterdam 100 9 amsterdamh 100 9 + amsterdamm 100 9 amsterdamn 100 9 + amsterdama 100 9 amsterdame 100 9 + amsterdami 100 9 amsterdamo 100 9 + amsterdamu 100 9 amsterdamy 100 9 + amsterdammetje 100 9 amsterdamania 100 9 + amsterdamb 100 9 amsterdamf 100 9 + amsterdamp 100 9 amsterdamv 100 9 + amsterdamw 100 9 amsterdamweg 100 9 + amsterdamc 100 9 amsterdamg 100 9 + amsterdamj 100 9 amsterdamk 100 9 + amsterdamq 100 9 amsterdams 100 9 + amsterdamx 100 9 amsterdamz 100 9 + amsterdamsestraat 100 9 amsterdamd 100 9 + amsterdamt 100 9 amsterdaml 100 9 + amsterdamlaan 100 9 amsterdamr 100 9 +} + +do_execsql_test 1.7 { + SELECT word, distance, matchlen FROM demo + WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20; +} { + amsterdam 100 9 amsterdamh 100 9 + amsterdamm 100 9 amsterdamn 100 9 + amsterdama 100 9 amsterdame 100 9 + amsterdami 100 9 amsterdamo 100 9 + amsterdamu 100 9 amsterdamy 100 9 + amsterdammetje 100 9 amsterdamania 100 9 + amsterdamb 100 9 amsterdamf 100 9 + amsterdamp 100 9 amsterdamv 100 9 + amsterdamw 100 9 amsterdamweg 100 9 + amsterdamc 100 9 amsterdamg 100 9 +} + + +finish_test + diff --git a/test/tabfunc01.test b/test/tabfunc01.test new file mode 100644 index 0000000000..39264d2f05 --- /dev/null +++ b/test/tabfunc01.test @@ -0,0 +1,69 @@ +# 2015-08-19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests for table-valued-functions implemented using +# eponymous virtual tables. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix tabfunc01 + +ifcapable !vtab { + finish_test + return +} +load_static_extension db series + +do_execsql_test tabfunc01-1.1 { + SELECT *, '|' FROM generate_series WHERE start=1 AND stop=9 AND step=2; +} {1 | 3 | 5 | 7 | 9 |} +do_execsql_test tabfunc01-1.2 { + SELECT *, '|' FROM generate_series LIMIT 5; +} {0 | 1 | 2 | 3 | 4 |} +do_catchsql_test tabfunc01-1.3 { + CREATE VIRTUAL TABLE t1 USING generate_series; +} {1 {no such module: generate_series}} +do_execsql_test tabfunc01-1.4 { + SELECT * FROM generate_series(1,9,2); +} {1 3 5 7 9} +do_execsql_test tabfunc01-1.5 { + SELECT * FROM generate_series(1,9); +} {1 2 3 4 5 6 7 8 9} +do_execsql_test tabfunc01-1.6 { + SELECT * FROM generate_series(1,10) WHERE step=3; +} {1 4 7 10} +do_catchsql_test tabfunc01-1.7 { + SELECT * FROM generate_series(1,9,2,11); +} {1 {too many arguments on generate_series() - max 3}} + +do_execsql_test tabfunc01-1.8 { + SELECT * FROM generate_series(0,32,5) ORDER BY rowid DESC; +} {30 25 20 15 10 5 0} +do_execsql_test tabfunc01-1.9 { + SELECT rowid, * FROM generate_series(0,32,5) ORDER BY value DESC; +} {1 30 2 25 3 20 4 15 5 10 6 5 7 0} +do_execsql_test tabfunc01-1.10 { + SELECT rowid, * FROM generate_series(0,32,5) ORDER BY +value DESC; +} {7 30 6 25 5 20 4 15 3 10 2 5 1 0} + +do_execsql_test tabfunc01-2.1 { + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES(2),(3); + SELECT *, '|' FROM t1, generate_series(1,x) ORDER BY 1, 2 +} {2 1 | 2 2 | 3 1 | 3 2 | 3 3 |} + +do_execsql_test tabfunc01-2.2 { + SELECT * FROM generate_series() LIMIT 5; +} {0 1 2 3 4} + + +finish_test diff --git a/test/threadtest2.c b/test/threadtest2.c index 25b1d908d2..b7a67c579b 100644 --- a/test/threadtest2.c +++ b/test/threadtest2.c @@ -43,7 +43,7 @@ int check_callback(void *pid, int argc, char **argv, char **notUsed2){ int id = (int)pid; if( strcmp(argv[0],"ok") ){ all_stop = 1; - fprintf(stderr,"id: %s\n", id, argv[0]); + fprintf(stderr,"%d: %s\n", id, argv[0]); }else{ /* fprintf(stderr,"%d: OK\n", id); */ } diff --git a/test/view.test b/test/view.test index 3ba6c0b465..a5fe85e648 100644 --- a/test/view.test +++ b/test/view.test @@ -599,7 +599,7 @@ do_test view-21.1 { CREATE VIEW v8192 AS SELECT * FROM v4096 UNION SELECT * FROM v4096; CREATE VIEW v16384 AS SELECT * FROM v8192 UNION SELECT * FROM v8192; CREATE VIEW v32768 AS SELECT * FROM v16384 UNION SELECT * FROM v16384; - CREATE VIEW vx AS SELECT * FROM v32768 UNION SELECT * FROM v32768; + SELECT * FROM v32768 UNION SELECT * FROM v32768; } } {1 {too many references to "v1": max 65535}} ifcapable progress { diff --git a/test/wal.test b/test/wal.test index 675be73791..bfe3634577 100644 --- a/test/wal.test +++ b/test/wal.test @@ -22,6 +22,7 @@ source $testdir/wal_common.tcl set testprefix wal ifcapable !wal {finish_test ; return } +test_set_config_pagecache 0 0 proc reopen_db {} { catch { db close } @@ -1587,4 +1588,5 @@ foreach mode {OFF MEMORY PERSIST DELETE TRUNCATE WAL} { db close } +test_restore_config_pagecache finish_test diff --git a/test/wal3.test b/test/wal3.test index 18e6075a4a..33bb285c53 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -34,6 +34,10 @@ db func a_string a_string # of test cases tests that nothing appears to go wrong when this is # done. # +set ans 4056 +if {[info exists G(perm:name)] && $G(perm:name)=="memsubsys1"} { + set ans 4251 +} do_test wal3-1.0 { execsql { PRAGMA cache_size = 2000; @@ -61,7 +65,7 @@ do_test wal3-1.0 { PRAGMA cache_size = 10; } wal_frame_count test.db-wal 1024 -} 4056 +} $ans for {set i 1} {$i < 50} {incr i} { diff --git a/test/without_rowid3.test b/test/without_rowid3.test index a0dc76d3f9..2af43a9191 100644 --- a/test/without_rowid3.test +++ b/test/without_rowid3.test @@ -2081,4 +2081,20 @@ do_test without_rowid3-ce7c13.1.6 { } } {1 {FOREIGN KEY constraint failed}} +# Confirm that changes() works on WITHOUT ROWID tables that use the +# xfer optimization. +# +db close +sqlite3 db :memory: +do_execsql_test without_rowid3-30.1 { + CREATE TABLE t1(a,b,PRIMARY KEY(a,b)) WITHOUT ROWID; + CREATE TABLE t2(a,b,PRIMARY KEY(a,b)) WITHOUT ROWID; + INSERT INTO t1 VALUES(1,2),(3,4),(5,6); + SELECT changes(); +} {3} +do_execsql_test without_rowid3-30.2 { + INSERT INTO t2 SELECT * FROM t1; + SELECT changes(); +} {3} + finish_test diff --git a/test/zeroblob.test b/test/zeroblob.test index c45be2b09c..0514644a28 100644 --- a/test/zeroblob.test +++ b/test/zeroblob.test @@ -276,10 +276,10 @@ do_execsql_test 11.0 { SELECT length(zeroblob(-1444444444444444)); } {0} do_catchsql_test 11.1 { - SELECT zeroblob(1025 * 1024 * 1024); + SELECT zeroblob(5000 * 1024 * 1024); } {1 {string or blob too big}} do_catchsql_test 11.2 { - SELECT quote(zeroblob(1025 * 1024 * 1024)); + SELECT quote(zeroblob(5000 * 1024 * 1024)); } {1 {string or blob too big}} do_catchsql_test 11.3 { SELECT quote(zeroblob(-1444444444444444)); @@ -305,7 +305,7 @@ do_test 12.2 { bind_and_run $stmt 0 } 0 do_test 12.3 { bind_and_run $stmt 1000 } 1000 do_test 12.4 { - list [catch { bind_and_run $stmt [expr 1500 * 1024 * 1024] } msg] $msg + list [catch { bind_and_run $stmt [expr 5000 * 1024 * 1024] } msg] $msg } {1 SQLITE_TOOBIG} do_test 12.5 { sqlite3_step $stmt diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index 9f5176db9d..25435838a3 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -312,6 +312,13 @@ IF "%VisualStudioVersion%" == "11.0" ( ) ) +REM +REM NOTE: This is the name of the sub-directory where the Windows 10.0 SDK +REM libraries may be found. It is only used when compiling with the +REM Windows 10.0 SDK. +REM +SET WIN10LIBDIR=10.0.10240.0 + REM REM NOTE: Check if this is the Windows Phone SDK. If so, a different batch REM file is necessary to setup the build environment. Since the variable @@ -482,9 +489,9 @@ FOR %%P IN (%PLATFORMS%) DO ( REM different directory naming conventions. REM IF DEFINED USE_WINV100_NSDKLIBPATH ( - CALL :fn_AppendVariable NSDKLIBPATH \..\10\lib\10.0.10030.0\um\x86 + CALL :fn_AppendVariable NSDKLIBPATH \..\10\lib\%WIN10LIBDIR%\um\x86 CALL :fn_CopyVariable UniversalCRTSdkDir PSDKLIBPATH - CALL :fn_AppendVariable PSDKLIBPATH Lib\10.0.10030.0\um\%%D + CALL :fn_AppendVariable PSDKLIBPATH Lib\%WIN10LIBDIR%\um\%%D ) ELSE IF DEFINED USE_WINV63_NSDKLIBPATH ( CALL :fn_AppendVariable NSDKLIBPATH \lib\winv6.3\um\x86 ) ELSE IF "%VisualStudioVersion%" == "12.0" ( @@ -507,7 +514,7 @@ FOR %%P IN (%PLATFORMS%) DO ( IF DEFINED SET_NUCRTLIBPATH ( IF DEFINED UniversalCRTSdkDir ( CALL :fn_CopyVariable UniversalCRTSdkDir NUCRTLIBPATH - CALL :fn_AppendVariable NUCRTLIBPATH \lib\winv10.0\ucrt\x86 + CALL :fn_AppendVariable NUCRTLIBPATH \lib\%WIN10LIBDIR%\ucrt\x86 ) ) diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl index 15ae2b0d3a..b7c5983dbe 100644 --- a/tool/mkvsix.tcl +++ b/tool/mkvsix.tcl @@ -261,12 +261,9 @@ proc getExtraFileListXmlChunk { packageFlavor vsVersion } { "\r\n " {DependsOn="Microsoft.VCLibs, version=12.0"}] } 2015 { - # - # TODO: Is the ".AppLocal" suffix always needed here? - # return [appendArgs \ "\r\n " AppliesTo=\" $appliesTo \" \ - "\r\n " {DependsOn="Microsoft.VCLibs.AppLocal, version=14.0"}] + "\r\n " {DependsOn="Microsoft.VCLibs, version=14.0"}] } default { return "" diff --git a/tool/showdb.c b/tool/showdb.c index c90f410d95..892cacd552 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -15,13 +15,20 @@ #include #include +#include #include "sqlite3.h" -static int pagesize = 1024; /* Size of a database page */ -static int db = -1; /* File descriptor for reading the DB */ -static int mxPage = 0; /* Last page number */ -static int perLine = 16; /* HEX elements to print per line */ +static struct GlobalData { + int pagesize; /* Size of a database page */ + int dbfd; /* File descriptor for reading the DB */ + int mxPage; /* Last page number */ + int perLine; /* HEX elements to print per line */ + int bRaw; /* True to access db file via OS APIs */ + sqlite3_file *pFd; /* File descriptor for non-raw mode */ + sqlite3 *pDb; /* Database handle that owns pFd */ +} g = {1024, -1, 0, 16, 0, 0, 0}; + typedef long long int i64; /* Datatype for 64-bit integers */ @@ -56,24 +63,122 @@ static void out_of_memory(void){ exit(1); } +/* +** Open a database connection. +*/ +static sqlite3 *openDatabase(const char *zPrg, const char *zName){ + sqlite3 *db = 0; + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI; + int rc = sqlite3_open_v2(zName, &db, flags, 0); + if( rc!=SQLITE_OK ){ + const char *zErr = sqlite3_errmsg(db); + fprintf(stderr, "%s: can't open %s (%s)\n", zPrg, zName, zErr); + sqlite3_close(db); + exit(1); + } + return db; +} + +/************************************************************************** +** Beginning of low-level file access functions. +** +** All low-level access to the database file read by this program is +** performed using the following four functions: +** +** fileOpen() - open the db file +** fileClose() - close the db file +** fileRead() - read raw data from the db file +** fileGetsize() - return the size of the db file in bytes +*/ + +/* +** Open the database file. +*/ +static void fileOpen(const char *zPrg, const char *zName){ + assert( g.dbfd<0 ); + if( g.bRaw==0 ){ + int rc; + void *pArg = (void *)(&g.pFd); + g.pDb = openDatabase(zPrg, zName); + rc = sqlite3_file_control(g.pDb, "main", SQLITE_FCNTL_FILE_POINTER, pArg); + if( rc!=SQLITE_OK ){ + fprintf(stderr, + "%s: failed to obtain fd for %s (SQLite too old?)\n", zPrg, zName + ); + exit(1); + } + }else{ + g.dbfd = open(zName, O_RDONLY); + if( g.dbfd<0 ){ + fprintf(stderr,"%s: can't open %s\n", zPrg, zName); + exit(1); + } + } +} + +/* +** Close the database file opened by fileOpen() +*/ +static void fileClose(){ + if( g.bRaw==0 ){ + sqlite3_close(g.pDb); + g.pDb = 0; + g.pFd = 0; + }else{ + close(g.dbfd); + g.dbfd = -1; + } +} + /* ** Read content from the file. ** -** Space to hold the content is obtained from malloc() and needs to be -** freed by the caller. +** Space to hold the content is obtained from sqlite3_malloc() and needs +** to be freed by the caller. */ -static unsigned char *getContent(int ofst, int nByte){ +static unsigned char *fileRead(sqlite3_int64 ofst, int nByte){ unsigned char *aData; int got; - aData = malloc(nByte+32); + aData = sqlite3_malloc(nByte+32); if( aData==0 ) out_of_memory(); memset(aData, 0, nByte+32); - lseek(db, ofst, SEEK_SET); - got = read(db, aData, nByte); - if( got>0 && gotpMethods->xRead(g.pFd, (void*)aData, nByte, ofst); + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ + fprintf(stderr, "error in xRead() - %d\n", rc); + exit(1); + } + }else{ + lseek(g.dbfd, ofst, SEEK_SET); + got = read(g.dbfd, aData, nByte); + if( got>0 && gotpMethods->xFileSize(g.pFd, &res); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "error in xFileSize() - %d\n", rc); + exit(1); + } + }else{ + struct stat sbuf; + fstat(g.dbfd, &sbuf); + res = (sqlite3_int64)(sbuf.st_size); + } + return res; +} + +/* +** End of low-level file access functions. +**************************************************************************/ + /* ** Print a range of bytes as hex and as ascii. */ @@ -98,17 +203,17 @@ static unsigned char *print_byte_range( zOfstFmt = " %08x: "; } - aData = getContent(ofst, nByte); - for(i=0; inByte ){ fprintf(stdout, " "); }else{ fprintf(stdout,"%02x ", aData[i+j]); } } - for(j=0; jnByte ){ fprintf(stdout, " "); }else{ @@ -126,11 +231,11 @@ static unsigned char *print_byte_range( static void print_page(int iPg){ int iStart; unsigned char *aData; - iStart = (iPg-1)*pagesize; + iStart = (iPg-1)*g.pagesize; fprintf(stdout, "Page %d: (offsets 0x%x..0x%x)\n", - iPg, iStart, iStart+pagesize-1); - aData = print_byte_range(iStart, pagesize, 0); - free(aData); + iPg, iStart, iStart+g.pagesize-1); + aData = print_byte_range(iStart, g.pagesize, 0); + sqlite3_free(aData); } @@ -267,14 +372,14 @@ static i64 localPayload(i64 nPayload, char cType){ i64 nLocal; if( cType==13 ){ /* Table leaf */ - maxLocal = pagesize-35; - minLocal = (pagesize-12)*32/255-23; + maxLocal = g.pagesize-35; + minLocal = (g.pagesize-12)*32/255-23; }else{ - maxLocal = (pagesize-12)*64/255-23; - minLocal = (pagesize-12)*32/255-23; + maxLocal = (g.pagesize-12)*64/255-23; + minLocal = (g.pagesize-12)*32/255-23; } if( nPayload>maxLocal ){ - surplus = minLocal + (nPayload-minLocal)%(pagesize-4); + surplus = minLocal + (nPayload-minLocal)%(g.pagesize-4); if( surplus<=maxLocal ){ nLocal = surplus; }else{ @@ -581,8 +686,8 @@ static void decode_btree_page( printf(" key: lx=left-child n=payload-size r=rowid\n"); } if( showMap ){ - zMap = malloc(pagesize); - memset(zMap, '.', pagesize); + zMap = sqlite3_malloc(g.pagesize); + memset(zMap, '.', g.pagesize); memset(zMap, '1', hdrSize); memset(&zMap[hdrSize], 'H', iCellPtr); memset(&zMap[hdrSize+iCellPtr], 'P', 2*nCell); @@ -611,10 +716,10 @@ static void decode_btree_page( } if( showMap ){ printf("Page map: (H=header P=cell-index 1=page-1-header .=free-space)\n"); - for(i=0; i0 ){ - a = getContent((pgno-1)*pagesize, pagesize); + a = fileRead((pgno-1)*g.pagesize, g.pagesize); printf("Decode of freelist trunk page %d:\n", pgno); print_decode_line(a, 0, 4, "Next freelist trunk page"); print_decode_line(a, 4, 4, "Number of entries on this page"); @@ -650,7 +754,7 @@ static void decode_trunk_page( }else{ pgno = (int)decodeInt32(&a[0]); } - free(a); + sqlite3_free(a); } } @@ -669,9 +773,9 @@ static void page_usage_msg(int pgno, const char *zFormat, ...){ va_start(ap, zFormat); zMsg = sqlite3_vmprintf(zFormat, ap); va_end(ap); - if( pgno<=0 || pgno>mxPage ){ + if( pgno<=0 || pgno>g.mxPage ){ printf("ERROR: page %d out of range 1..%d: %s\n", - pgno, mxPage, zMsg); + pgno, g.mxPage, zMsg); sqlite3_free(zMsg); return; } @@ -719,12 +823,12 @@ static void page_usage_cell( if( nLocalmxPage ) return; - a = getContent((pgno-1)*pagesize, pagesize); + if( pgno<=0 || pgno>g.mxPage ) return; + a = fileRead((pgno-1)*g.pagesize, g.pagesize); switch( a[hdr] ){ case 2: zType = "interior node of index"; break; case 5: zType = "interior node of table"; break; @@ -783,7 +887,7 @@ static void page_usage_btree( page_usage_cell(a[hdr], a+ofst, pgno, i); } } - free(a); + sqlite3_free(a); } /* @@ -797,9 +901,9 @@ static void page_usage_freelist(int pgno){ int iNext; int parent = 1; - while( pgno>0 && pgno<=mxPage && (cnt++)0 && pgno<=g.mxPage && (cnt++)1 ){ + if( sqlite3_stricmp("-raw", azArg[1])==0 + || sqlite3_stricmp("--raw", azArg[1])==0 + ){ + g.bRaw = 1; + azArg++; + nArg--; + } + } + + if( nArg<2 ){ + usage(zPrg); exit(1); } - db = open(argv[1], O_RDONLY); - if( db<0 ){ - fprintf(stderr,"%s: can't open %s\n", argv[0], argv[1]); - exit(1); - } - zPgSz[0] = 0; - zPgSz[1] = 0; - lseek(db, 16, SEEK_SET); - if( read(db, zPgSz, 2)<2 ) memset(zPgSz, 0, 2); - pagesize = zPgSz[0]*256 + zPgSz[1]*65536; - if( pagesize==0 ) pagesize = 1024; - printf("Pagesize: %d\n", pagesize); - fstat(db, &sbuf); - mxPage = (sbuf.st_size+pagesize-1)/pagesize; - printf("Available pages: 1..%d\n", mxPage); - if( argc==2 ){ + + fileOpen(zPrg, azArg[1]); + szFile = fileGetsize(); + + zPgSz = fileRead(16, 2); + g.pagesize = zPgSz[0]*256 + zPgSz[1]*65536; + if( g.pagesize==0 ) g.pagesize = 1024; + sqlite3_free(zPgSz); + + printf("Pagesize: %d\n", g.pagesize); + g.mxPage = (szFile+g.pagesize-1)/g.pagesize; + + printf("Available pages: 1..%d\n", g.mxPage); + if( nArg==2 ){ int i; - for(i=1; i<=mxPage; i++) print_page(i); + for(i=1; i<=g.mxPage; i++) print_page(i); }else{ int i; - for(i=2; imxPage ){ + if( iStart<1 || iEndg.mxPage ){ fprintf(stderr, "Page argument should be LOWER?..UPPER?. Range 1 to %d\n", - mxPage); + g.mxPage); exit(1); } while( iStart<=iEnd ){ @@ -1058,6 +1172,6 @@ int main(int argc, char **argv){ } } } - close(db); + fileClose(); return 0; } diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl index cd3785bd7d..dea2723397 100644 --- a/tool/spaceanal.tcl +++ b/tool/spaceanal.tcl @@ -26,7 +26,7 @@ proc is_without_rowid {tname} { # proc usage {} { set argv0 [file rootname [file tail [info nameofexecutable]]] - puts stderr "Usage: $argv0 database-name" + puts stderr "Usage: $argv0 \[--pageinfo] \[--stats] database-name" exit 1 } set file_to_analyze {} @@ -142,6 +142,7 @@ set tabledef {CREATE TABLE space_used( is_index boolean, -- TRUE if it is an index, false for a table nentry int, -- Number of entries in the BTree leaf_entries int, -- Number of leaf entries + depth int, -- Depth of the b-tree payload int, -- Total amount of data stored in this table or index ovfl_payload int, -- Total amount of data stored on overflow pages ovfl_cnt int, -- Number of entries that use overflow @@ -164,22 +165,9 @@ db eval {CREATE TEMP TABLE dbstat AS SELECT * FROM temp.stat ORDER BY name, path} db eval {DROP TABLE temp.stat} -proc isleaf {pagetype is_index} { - return [expr {$pagetype == "leaf" || ($pagetype == "internal" && $is_index)}] -} -proc isoverflow {pagetype is_index} { - return [expr {$pagetype == "overflow"}] -} -proc isinternal {pagetype is_index} { - return [expr {$pagetype == "internal" && $is_index==0}] -} - -db func isleaf isleaf -db func isinternal isinternal -db func isoverflow isoverflow - set isCompressed 0 set compressOverhead 0 +set depth 0 set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 } foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] { @@ -188,18 +176,20 @@ foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] { db eval { SELECT sum(ncell) AS nentry, - sum(isleaf(pagetype, $idx_btree) * ncell) AS leaf_entries, + sum((pagetype=='leaf')*ncell) AS leaf_entries, sum(payload) AS payload, - sum(isoverflow(pagetype, $idx_btree) * payload) AS ovfl_payload, + sum((pagetype=='overflow') * payload) AS ovfl_payload, sum(path LIKE '%+000000') AS ovfl_cnt, max(mx_payload) AS mx_payload, - sum(isinternal(pagetype, $idx_btree)) AS int_pages, - sum(isleaf(pagetype, $idx_btree)) AS leaf_pages, - sum(isoverflow(pagetype, $idx_btree)) AS ovfl_pages, - sum(isinternal(pagetype, $idx_btree) * unused) AS int_unused, - sum(isleaf(pagetype, $idx_btree) * unused) AS leaf_unused, - sum(isoverflow(pagetype, $idx_btree) * unused) AS ovfl_unused, - sum(pgsize) AS compressed_size + sum(pagetype=='internal') AS int_pages, + sum(pagetype=='leaf') AS leaf_pages, + sum(pagetype=='overflow') AS ovfl_pages, + sum((pagetype=='internal') * unused) AS int_unused, + sum((pagetype=='leaf') * unused) AS leaf_unused, + sum((pagetype=='overflow') * unused) AS ovfl_unused, + sum(pgsize) AS compressed_size, + max((length(CASE WHEN path LIKE '%+%' THEN '' ELSE path END)+3)/4) + AS depth FROM temp.dbstat WHERE name = $name } break @@ -235,6 +225,7 @@ foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] { $is_index, $nentry, $leaf_entries, + $depth, $payload, $ovfl_payload, $ovfl_cnt, @@ -344,7 +335,9 @@ proc subreport {title where showFrag} { int(sum(int_unused)) AS int_unused, int(sum(ovfl_unused)) AS ovfl_unused, int(sum(gap_cnt)) AS gap_cnt, - int(sum(compressed_size)) AS compressed_size + int(sum(compressed_size)) AS compressed_size, + int(max(depth)) AS depth, + count(*) AS cnt FROM space_used WHERE $where" {} {} # Output the sub-report title, nicely decorated with * characters. @@ -381,7 +374,7 @@ proc subreport {title where showFrag} { "] set avg_fanout [mem eval " SELECT (sum(leaf_pages+int_pages)-$nTab)/sum(int_pages) FROM space_used - WHERE $where AND is_index = 0 + WHERE $where "] set avg_fanout [format %.2f $avg_fanout] } @@ -399,6 +392,7 @@ proc subreport {title where showFrag} { statline {Bytes used after compression} $compressed_size $pct } statline {Bytes of payload} $payload $payload_percent + if {$cnt==1} {statline {B-tree depth} $depth} statline {Average payload per entry} $avg_payload statline {Average unused bytes per entry} $avg_unused if {[info exists avg_fanout]} { diff --git a/tool/sqldiff.c b/tool/sqldiff.c index 6d72303545..9f0b705c40 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "sqlite3.h" /* @@ -259,7 +260,12 @@ static void namelistFree(char **az){ ** CREATE TABLE t5(rowid,_rowid_,oid); ** az = 0 // The rowid is not accessible */ -static char **columnNames(const char *zDb, const char *zTab, int *pnPKey){ +static char **columnNames( + const char *zDb, /* Database ("main" or "aux") to query */ + const char *zTab, /* Name of table to return details of */ + int *pnPKey, /* OUT: Number of PK columns */ + int *pbRowid /* OUT: True if PK is an implicit rowid */ +){ char **az = 0; /* List of column names to be returned */ int naz = 0; /* Number of entries in az[] */ sqlite3_stmt *pStmt; /* SQL statement being run */ @@ -338,6 +344,15 @@ static char **columnNames(const char *zDb, const char *zTab, int *pnPKey){ } sqlite3_finalize(pStmt); if( az ) az[naz] = 0; + + /* If it is non-NULL, set *pbRowid to indicate whether or not the PK of + ** this table is an implicit rowid (*pbRowid==1) or not (*pbRowid==0). */ + if( pbRowid ) *pbRowid = (az[0]==0); + + /* If this table has an implicit rowid for a PK, figure out how to refer + ** to it. There are three options - "rowid", "_rowid_" and "oid". Any + ** of these will work, unless the table has an explicit column of the + ** same name. */ if( az[0]==0 ){ const char *azRowid[] = { "rowid", "_rowid_", "oid" }; for(i=0; iz[i] = z[i]; + } + pHash->a = a & 0xffff; + pHash->b = b & 0xffff; + pHash->i = 0; +} + +/* +** Advance the rolling hash by a single character "c" +*/ +static void hash_next(hash *pHash, int c){ + u16 old = pHash->z[pHash->i]; + pHash->z[pHash->i] = (char)c; + pHash->i = (pHash->i+1)&(NHASH-1); + pHash->a = pHash->a - old + (char)c; + pHash->b = pHash->b - NHASH*old + pHash->a; +} + +/* +** Return a 32-bit hash value +*/ +static u32 hash_32bit(hash *pHash){ + return (pHash->a & 0xffff) | (((u32)(pHash->b & 0xffff))<<16); +} + +/* +** Write an base-64 integer into the given buffer. +*/ +static void putInt(unsigned int v, char **pz){ + static const char zDigits[] = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~"; + /* 123456789 123456789 123456789 123456789 123456789 123456789 123 */ + int i, j; + char zBuf[20]; + if( v==0 ){ + *(*pz)++ = '0'; + return; + } + for(i=0; v>0; i++, v>>=6){ + zBuf[i] = zDigits[v&0x3f]; + } + for(j=i-1; j>=0; j--){ + *(*pz)++ = zBuf[j]; + } +} + +/* +** Return the number digits in the base-64 representation of a positive integer +*/ +static int digit_count(int v){ + unsigned int i, x; + for(i=1, x=64; (unsigned int)v>=x; i++, x <<= 6){} + return i; +} + +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int checksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} + +/* +** Create a new delta. +** +** The delta is written into a preallocated buffer, zDelta, which +** should be at least 60 bytes longer than the target file, zOut. +** The delta string will be NUL-terminated, but it might also contain +** embedded NUL characters if either the zSrc or zOut files are +** binary. This function returns the length of the delta string +** in bytes, excluding the final NUL terminator character. +** +** Output Format: +** +** The delta begins with a base64 number followed by a newline. This +** number is the number of bytes in the TARGET file. Thus, given a +** delta file z, a program can compute the size of the output file +** simply by reading the first line and decoding the base-64 number +** found there. The delta_output_size() routine does exactly this. +** +** After the initial size number, the delta consists of a series of +** literal text segments and commands to copy from the SOURCE file. +** A copy command looks like this: +** +** NNN@MMM, +** +** where NNN is the number of bytes to be copied and MMM is the offset +** into the source file of the first byte (both base-64). If NNN is 0 +** it means copy the rest of the input file. Literal text is like this: +** +** NNN:TTTTT +** +** where NNN is the number of bytes of text (base-64) and TTTTT is the text. +** +** The last term is of the form +** +** NNN; +** +** In this case, NNN is a 32-bit bigendian checksum of the output file +** that can be used to verify that the delta applied correctly. All +** numbers are in base-64. +** +** Pure text files generate a pure text delta. Binary files generate a +** delta that may contain some binary data. +** +** Algorithm: +** +** The encoder first builds a hash table to help it find matching +** patterns in the source file. 16-byte chunks of the source file +** sampled at evenly spaced intervals are used to populate the hash +** table. +** +** Next we begin scanning the target file using a sliding 16-byte +** window. The hash of the 16-byte window in the target is used to +** search for a matching section in the source file. When a match +** is found, a copy command is added to the delta. An effort is +** made to extend the matching section to regions that come before +** and after the 16-byte hash window. A copy command is only issued +** if the result would use less space that just quoting the text +** literally. Literal text is added to the delta for sections that +** do not match or which can not be encoded efficiently using copy +** commands. +*/ +static int rbuDeltaCreate( + const char *zSrc, /* The source or pattern file */ + unsigned int lenSrc, /* Length of the source file */ + const char *zOut, /* The target file */ + unsigned int lenOut, /* Length of the target file */ + char *zDelta /* Write the delta into this buffer */ +){ + unsigned int i, base; + char *zOrigDelta = zDelta; + hash h; + int nHash; /* Number of hash table entries */ + int *landmark; /* Primary hash table */ + int *collide; /* Collision chain */ + int lastRead = -1; /* Last byte of zSrc read by a COPY command */ + + /* Add the target file size to the beginning of the delta + */ + putInt(lenOut, &zDelta); + *(zDelta++) = '\n'; + + /* If the source file is very small, it means that we have no + ** chance of ever doing a copy command. Just output a single + ** literal segment for the entire target and exit. + */ + if( lenSrc<=NHASH ){ + putInt(lenOut, &zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, zOut, lenOut); + zDelta += lenOut; + putInt(checksum(zOut, lenOut), &zDelta); + *(zDelta++) = ';'; + return zDelta - zOrigDelta; + } + + /* Compute the hash table used to locate matching sections in the + ** source file. + */ + nHash = lenSrc/NHASH; + collide = sqlite3_malloc( nHash*2*sizeof(int) ); + landmark = &collide[nHash]; + memset(landmark, -1, nHash*sizeof(int)); + memset(collide, -1, nHash*sizeof(int)); + for(i=0; i=0 && (limit--)>0 ){ + /* + ** The hash window has identified a potential match against + ** landmark block iBlock. But we need to investigate further. + ** + ** Look for a region in zOut that matches zSrc. Anchor the search + ** at zSrc[iSrc] and zOut[base+i]. Do not include anything prior to + ** zOut[base] or after zOut[outLen] nor anything after zSrc[srcLen]. + ** + ** Set cnt equal to the length of the match and set ofst so that + ** zSrc[ofst] is the first element of the match. litsz is the number + ** of characters between zOut[base] and the beginning of the match. + ** sz will be the overhead (in bytes) needed to encode the copy + ** command. Only generate copy command if the overhead of the + ** copy command is less than the amount of literal text to be copied. + */ + int cnt, ofst, litsz; + int j, k, x, y; + int sz; + + /* Beginning at iSrc, match forwards as far as we can. j counts + ** the number of characters that match */ + iSrc = iBlock*NHASH; + for( + j=0, x=iSrc, y=base+i; + (unsigned int)x=sz && cnt>bestCnt ){ + /* Remember this match only if it is the best so far and it + ** does not increase the file size */ + bestCnt = cnt; + bestOfst = iSrc-k; + bestLitsz = litsz; + } + + /* Check the next matching block */ + iBlock = collide[iBlock]; + } + + /* We have a copy command that does not cause the delta to be larger + ** than a literal insert. So add the copy command to the delta. + */ + if( bestCnt>0 ){ + if( bestLitsz>0 ){ + /* Add an insert command before the copy */ + putInt(bestLitsz,&zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, &zOut[base], bestLitsz); + zDelta += bestLitsz; + base += bestLitsz; + } + base += bestCnt; + putInt(bestCnt, &zDelta); + *(zDelta++) = '@'; + putInt(bestOfst, &zDelta); + *(zDelta++) = ','; + if( bestOfst + bestCnt -1 > lastRead ){ + lastRead = bestOfst + bestCnt - 1; + } + bestCnt = 0; + break; + } + + /* If we reach this point, it means no match is found so far */ + if( base+i+NHASH>=lenOut ){ + /* We have reached the end of the file and have not found any + ** matches. Do an "insert" for everything that does not match */ + putInt(lenOut-base, &zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, &zOut[base], lenOut-base); + zDelta += lenOut-base; + base = lenOut; + break; + } + + /* Advance the hash by one character. Keep looking for a match */ + hash_next(&h, zOut[base+i+NHASH]); + i++; + } + } + /* Output a final "insert" record to get all the text at the end of + ** the file that does not match anything in the source file. + */ + if( base1)?", ":""), i); +} + +static void rbudiff_one_table(const char *zTab, FILE *out){ + int bOtaRowid; /* True to use an ota_rowid column */ + int nPK; /* Number of primary key columns in table */ + char **azCol; /* NULL terminated array of col names */ + int i; + int nCol; + Str ct = {0, 0, 0}; /* The "CREATE TABLE data_xxx" statement */ + Str sql = {0, 0, 0}; /* Query to find differences */ + Str insert = {0, 0, 0}; /* First part of output INSERT statement */ + sqlite3_stmt *pStmt = 0; + + /* --rbu mode must use real primary keys. */ + g.bSchemaPK = 1; + + /* Check that the schemas of the two tables match. Exit early otherwise. */ + checkSchemasMatch(zTab); + + /* Grab the column names and PK details for the table(s). If no usable PK + ** columns are found, bail out early. */ + azCol = columnNames("main", zTab, &nPK, &bOtaRowid); + if( azCol==0 ){ + runtimeError("table %s has no usable PK columns", zTab); + } + for(nCol=0; azCol[nCol]; nCol++); + + /* Build and output the CREATE TABLE statement for the data_xxx table */ + strPrintf(&ct, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab); + if( bOtaRowid ) strPrintf(&ct, "rbu_rowid, "); + strPrintfArray(&ct, ", ", "%s", &azCol[bOtaRowid], -1); + strPrintf(&ct, ", rbu_control);"); + + /* Get the SQL for the query to retrieve data from the two databases */ + getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, &sql); + + /* Build the first part of the INSERT statement output for each row + ** in the data_xxx table. */ + strPrintf(&insert, "INSERT INTO 'data_%q' (", zTab); + if( bOtaRowid ) strPrintf(&insert, "rbu_rowid, "); + strPrintfArray(&insert, ", ", "%s", &azCol[bOtaRowid], -1); + strPrintf(&insert, ", rbu_control) VALUES("); + + pStmt = db_prepare("%s", sql.z); + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + + /* If this is the first row output, print out the CREATE TABLE + ** statement first. And then set ct.z to NULL so that it is not + ** printed again. */ + if( ct.z ){ + fprintf(out, "%s\n", ct.z); + strFree(&ct); + } + + /* Output the first part of the INSERT statement */ + fprintf(out, "%s", insert.z); + + if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){ + for(i=0; i<=nCol; i++){ + if( i>0 ) fprintf(out, ", "); + printQuoted(out, sqlite3_column_value(pStmt, i)); + } + }else{ + char *zOtaControl; + int nOtaControl = sqlite3_column_bytes(pStmt, nCol); + + zOtaControl = (char*)sqlite3_malloc(nOtaControl); + memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1); + + for(i=0; i=nPK + && sqlite3_column_type(pStmt, i)==SQLITE_BLOB + && sqlite3_column_type(pStmt, nCol+1+i)==SQLITE_BLOB + ){ + const char *aSrc = sqlite3_column_blob(pStmt, nCol+1+i); + int nSrc = sqlite3_column_bytes(pStmt, nCol+1+i); + const char *aFinal = sqlite3_column_blob(pStmt, i); + int nFinal = sqlite3_column_bytes(pStmt, i); + char *aDelta; + int nDelta; + + aDelta = sqlite3_malloc(nFinal + 60); + nDelta = rbuDeltaCreate(aSrc, nSrc, aFinal, nFinal, aDelta); + if( nDelta