diff --git a/Makefile.in b/Makefile.in index 6a5132ca7c..a8e356c102 100644 --- a/Makefile.in +++ b/Makefile.in @@ -449,6 +449,7 @@ TESTSRC += \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/prefixes.c \ $(TOP)/ext/misc/qpvtab.c \ + $(TOP)/ext/misc/randomjson.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ @@ -603,6 +604,7 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC +SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 FUZZERSHELL_OPT = FUZZCHECK_OPT += -I$(TOP)/test FUZZCHECK_OPT += -I$(TOP)/ext/recover @@ -633,7 +635,9 @@ FUZZCHECK_OPT += \ -DSQLITE_MAX_MMAP_SIZE=0 \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_PRINTF_PRECISION_LIMIT=1000 \ - -DSQLITE_PRIVATE="" + -DSQLITE_PRIVATE="" \ + -DSQLITE_STRICT_SUBTYPE=1 \ + -DSQLITE_STATIC_RANDOMJSON FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c @@ -641,6 +645,7 @@ FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c FUZZCHECK_SRC += $(TOP)/test/vt02.c +FUZZCHECK_SRC += $(TOP)/ext/misc/randomjson.c DBFUZZ_OPT = ST_OPT = -DSQLITE_OS_KV_OPTIONAL @@ -1294,6 +1299,8 @@ TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_BYTECODE_VTAB TESTFIXTURE_FLAGS += -DSQLITE_CKSUMVFS_STATIC +TESTFIXTURE_FLAGS += -DSQLITE_STATIC_RANDOMJSON +TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la TESTFIXTURE_SRC1 = sqlite3.c diff --git a/Makefile.msc b/Makefile.msc index 14e32bdca1..28fcdbeb23 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1585,6 +1585,7 @@ TESTEXT = \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\prefixes.c \ $(TOP)\ext\misc\qpvtab.c \ + $(TOP)\ext\misc\randomjson.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\remember.c \ $(TOP)\ext\misc\series.c \ @@ -1693,6 +1694,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF # <> @@ -1729,6 +1731,8 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MMAP_SIZE=0 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE="" +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STRICT_SUBTYPE=1 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STATIC_RANDOMJSON FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 @@ -1745,6 +1749,7 @@ FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\randomjson.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION @@ -2434,6 +2439,8 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CKSUMVFS_STATIC=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STATIC_RANDOMJSON +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 280bb95deb..45a07a9f31 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -990,6 +990,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF diff --git a/doc/jsonb.md b/doc/jsonb.md new file mode 100644 index 0000000000..5beed1631d --- /dev/null +++ b/doc/jsonb.md @@ -0,0 +1,290 @@ +# The JSONB Format + +This document describes SQLite's JSONB binary encoding of +JSON. + +## 1.0 What Is JSONB? + +Beginning with version 3.45.0 (circa 2024-01-01), SQLite supports an +alternative binary encoding of JSON which we call "JSONB". JSONB is +a binary format that stored as a BLOB. + +The advantage of JSONB over ordinary text RFC 8259 JSON is that JSONB +is both slightly smaller (by between 5% and 10% in most cases) and +can be processed in less than half the number of CPU cycles. The built-in +[JSON SQL functions] of SQLite can accept either ordinary text JSON +or the binary JSONB encoding for any of their JSON inputs. + +The "JSONB" name is inspired by [PostgreSQL](https://postgresql.org), but the +on-disk format for SQLite's JSONB is not the same as PostgreSQL's. +The two formats have the same name, but they have wildly different internal +representations and are not in any way binary compatible. + +The central idea behind this JSONB specification is that each element +begins with a header that includes the size and type of that element. +The header takes the place of punctuation such as double-quotes, +curly-brackes, square-brackets, commas, and colons. Since the size +and type of each element is contained in its header, the element can +be read faster since it is no longer necessary to carefully scan forward +looking for the closing delimiter. The payload of JSONB is the same +as for corresponding text JSON. The same payload bytes occur in the +same order. The only real difference between JSONB and ordinary text +JSON is that JSONB includes a binary header on +each element and omits delimiter and separator punctuation. + +### 1.1 Internal Use Only + +The details of the JSONB are not intended to be visible to application +developers. Application developers should look at JSONB as an opaque BLOB +used internally by SQLite. Nevertheless, we want the format to be backwards +compatible across all future versions of SQLite. To that end, the format +is documented by this file in the source tree. But this file should be +used only by SQLite core developers, not by developers of applications +that only use SQLite. + +## 2.0 The Purpose Of This Document + +JSONB is not intended as an external format to be used by +applications. JSONB is designed for internal use by SQLite only. +Programmers do not need to understand the JSONB format in order to +use it effectively. +Applications should access JSONB only through the [JSON SQL functions], +not by looking at individual bytes of the BLOB. + +However, JSONB is intended to be portable and backwards compatible +for all future versions of SQLite. In other words, you should not have +to export and reimport your SQLite database files when you upgrade to +a newer SQLite version. For that reason, the JSONB format needs to +be well-defined. + +This document is therefore similar in purpose to the +[SQLite database file format] document that describes the on-disk +format of an SQLite database file. Applications are not expected +to directly read and write the bits and bytes of SQLite database files. +The SQLite database file format is carefully documented so that it +can be stable and enduring. In the same way, the JSONB representation +of JSON is documented here so that it too can be stable and enduring, +not so that applications can read or writes individual bytes. + +## 3.0 Encoding + +JSONB is a direct translation of the underlying text JSON. The difference +is that JSONB uses a binary encoding that is faster to parse compared to +the detailed syntax of text JSON. + +Each JSON element is encoded as a header and a payload. The header +determines type of element (string, numeric, boolean, null, object, or +array) and the size of the payload. The header can be between 1 and +9 bytes in size. The payload can be any size from zero bytes up to the +maximum allowed BLOB size. + +### 3.1 Payload Size + +The upper four bits of the first byte of the header determine size of the +header and possibly also the size of the payload. +If the upper four bits have a value between 0 and 11, then the header is +exactly one byte in size and the payload size is determined by those +upper four bits. If the upper four bits have a value between 12 and 15, +that means that the total header size is 2, 3, 5, or 9 bytes and the +payload size is unsigned big-endian integer that is contained in the +subsequent bytes. The size integer is the one byte that following the +initial header byte if the upper four bits +are 12, two bytes if the upper bits are 13, four bytes if the upper bits +are 14, and eight bytes if the upper bits are 15. The current design +of SQLite does not support BLOB values larger than 2GiB, so the eight-byte +variant of the payload size integer will never be used by the current code. +The eight-byte payload size integer is included in the specification +to allow for future expansion. + +The header for an element does *not* need to be in its simplest +form. For example, consider the JSON numeric value "`1`". +That element can be encode in five different ways: + + * `0x13 0x31` + * `0xc3 0x01 0x31` + * `0xd3 0x00 0x01 0x31` + * `0xe3 0x00 0x00 0x00 0x01 0x31` + * `0xf3 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x01 0x31` + +The shortest encoding is preferred, of course, and usually happens with +primitive elements such as numbers. However the total size of an array +or object might not be known exactly when the header of the element is +first generated. It is convenient to reserve space for the largest +possible header and then go back and fill in the correct payload size +at the end. This technique can result in array or object headers that +are larger than absolutely necessary. + +### 3.2 Element Type + +The least-significant four bits of the first byte of the header (the first +byte masked against 0x0f) determine element type. The following codes are +used: + +
    +

  1. NULL → +The element is a JSON "null". The payload size for a true JSON NULL must +must be zero. Future versions of SQLite might extend the JSONB format +with elements that have a zero element type but a non-zero size. In that +way, legacy versions of SQLite will interpret the element as a NULL +for backwards compatibility while newer versions will interpret the +element in some other way. + +

  2. TRUE → +The element is a JSON "true". The payload size must be zero for a actual +"true" value. Elements with type 1 and a non-zero payload size are +reserved for future expansion. Legacy implementations that see an element +type of 1 with a non-zero payload size should continue to interpret that +element as "true" for compatibility. + +

  3. FALSE → +The element is a JSON "false". The payload size must be zero for a actual +"false" value. Elements with type 2 and a non-zero payload size are +reserved for future expansion. Legacy implementations that see an element +type of 2 with a non-zero payload size should continue to interpret that +element as "false" for compatibility. + +

  4. INT → +The element is a JSON integer value in the canonical +RFC 8259 format, without extensions. The payload is the ASCII +text representation of that numeric value. + +

  5. INT5 → +The element is a JSON integer value that is not in the +canonical format. The payload is the ASCII +text representation of that numeric value. Because the payload is in a +non-standard format, it will need to be translated when the JSONB is +converted into RFC 8259 text JSON. + +

  6. FLOAT → +The element is a JSON floating-point value in the canonical +RFC 8259 format, without extensions. The payload is the ASCII +text representation of that numeric value. + +

  7. FLOAT5 → +The element is a JSON floating-point value that is not in the +canonical format. The payload is the ASCII +text representation of that numeric value. Because the payload is in a +non-standard format, it will need to be translated when the JSONB is +converted into RFC 8259 text JSON. + +

  8. TEXT → +The element is a JSON string value that does not contain +any escapes nor any characters that need to be escaped for either SQL or +JSON. The payload is the UTF8 text representation of the string value. +The payload does not include string delimiters. + +

  9. TEXTJ → +The element is a JSON string value that contains +RFC 8259 character escapes (such as "\n" or "\u0020"). +Those escapes will need to be translated into actual UTF8 if this element +is [json_extract|extracted] into SQL. +The payload is the UTF8 text representation of the escaped string value. +The payload does not include string delimiters. + +

  10. TEXT5 → +The element is a JSON string value that contains +character escapes, including some character escapes that part of JSON5 +and which are not found in the canonical RFC 8259 spec. +Those escapes will need to be translated into standard JSON prior to +rendering the JSON as text, or into their actual UTF8 characters if this +element is [json_extract|extracted] into SQL. +The payload is the UTF8 text representation of the escaped string value. +The payload does not include string delimiters. + +

  11. TEXTRAW → +The element is a JSON string value that contains +UTF8 characters that need to be escaped if this string is rendered into +standard JSON text. +The payload does not include string delimiters. + +

  12. ARRAY → +The element is a JSON array. The payload contains +JSONB elements that comprise values contained within the array. + +

  13. OBJECT → +The element is a JSON object. The payload contains +pairs of JSONB elements that comprise entries for the JSON object. +The first element in each pair must be a string (types 7 through 10). +The second element of each pair may be any types, including nested +arrays or objects. + +

  14. RESERVED-13 → +Reserved for future expansion. Legacy implements that encounter this +element type should raise an error. + +

  15. RESERVED-14 → +Reserved for future expansion. Legacy implements that encounter this +element type should raise an error. + +

  16. RESERVED-15 → +Reserved for future expansion. Legacy implements that encounter this +element type should raise an error. +

+ +Element types outside the range of 0 to 12 are reserved for future +expansion. The current implement raises an error if see an element type +other than those listed above. However, future versions of SQLite might +use of the three remaining element types to implement indexing or similar +optimizations, to speed up lookup against large JSON arrays and/or objects. + +### 3.3 Design Rationale For Element Types + +A key goal of JSONB is that it should be quick to translate +to and from text JSON and/or be constructed from SQL values. +When converting from text into JSONB, we do not want the +converter subroutine to burn CPU cycles converting elements +values into some standard format which might never be used. +Format conversion is "lazy" - it is deferred until actually +needed. This has implications for the JSONB format design: + + 1. Numeric values are stored as text, not a numbers. The values are + a direct copy of the text JSON values from which they are derived. + + 2. There are multiple element types depending on the details of value + formats. For example, INT is used for pure RFC-8259 integer + literals and INT5 exists for JSON5 extensions such as hexadecimal + notation. FLOAT is used for pure RFC-8259 floating point literals + and FLOAT5 is used for JSON5 extensions. There are four different + representations of strings, depending on where the string came from + and how special characters within the string are escaped. + +A second goal of JSONB is that it should be capable of serving as the +"parse tree" for JSON when a JSON value is being processed by the +various [JSON SQL functions] built into SQLite. Before JSONB was +developed, operations such [json_replace()] and [json_patch()] +and similar worked in three stages: + + + 1. Translate the text JSON into a internal format that is + easier to scan and edit. + 2. Perform the requested operation on the JSON. + 3. Translate the internal format back into text. + +JSONB seeks to serve as the internal format directly - bypassing +the first and third stages of that process. Since most of the CPU +cycles are spent on the first and third stages, that suggests that +JSONB processing will be much faster than text JSON processing. + +So when processing JSONB, only the second stage of the three-stage +process is required. But when processing text JSON, it is still necessary +to do stages one and three. If JSONB is to be used as the internal +binary representation, this is yet another reason to store numeric +values as text. Storing numbers as text minimizes the amount of +conversion work needed for stages one and three. This is also why +there are four different representations of text in JSONB. Different +text representations are used for text coming from different sources +(RFC-8259 JSON, JSON5, or SQL string values) and conversions only +happen if and when they are actually needed. + +### 3.4 Valid JSONB BLOBs + +A valid JSONB BLOB consists of a single JSON element. The element must +exactly fill the BLOB. This one element is often a JSON object or array +and those usually contain additional elements as its payload, but the +element can be a primite value such a string, number, boolean, or null. + +When the built-in JSON functions are attempting to determine if a BLOB +argument is a JSONB or just a random BLOB, they look at the header of +the outer element to see that it is well-formed and that the element +completely fills the BLOB. If these conditions are met, then the BLOB +is accepted as a JSONB value. diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index c7a5e6b7de..250d2ee7e3 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -88,8 +88,11 @@ struct Fts5PhraseIter { ** created with the "columnsize=0" option. ** ** xColumnText: -** This function attempts to retrieve the text of column iCol of the -** current document. If successful, (*pz) is set to point to a buffer +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the text of column iCol of +** the current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values @@ -99,8 +102,10 @@ struct Fts5PhraseIter { ** Returns the number of phrases in the current query expression. ** ** xPhraseSize: -** Returns the number of tokens in phrase iPhrase of the query. Phrases -** are numbered starting from zero. +** If parameter iCol is less than zero, or greater than or equal to the +** number of phrases in the current query, as returned by xPhraseCount, +** 0 is returned. Otherwise, this function returns the number of tokens in +** phrase iPhrase of the query. Phrases are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within @@ -116,12 +121,13 @@ struct Fts5PhraseIter { ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value -** output by xInstCount(). +** output by xInstCount(). If iIdx is less than zero or greater than +** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned. ** -** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. Returns SQLITE_OK if successful, or an error -** code (i.e. SQLITE_NOMEM) if an error occurs. +** first token of the phrase. SQLITE_OK is returned if successful, or an +** error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. @@ -147,6 +153,10 @@ struct Fts5PhraseIter { ** Invoking Api.xUserData() returns a copy of the pointer passed as ** the third argument to pUserData. ** +** If parameter iPhrase is less than zero, or greater than or equal to +** the number of phrases in the query, as returned by xPhraseCount(), +** this function returns SQLITE_RANGE. +** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. @@ -268,17 +278,25 @@ struct Fts5PhraseIter { ** to a buffer containing the requested token, and *pnToken to the ** size of this buffer in bytes. ** +** If iPhrase or iToken are less than zero, or if iPhrase is greater than +** or equal to the number of phrases in the query as reported by +** xPhraseCount(), or if iToken is equal to or greater than the number of +** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken + are both zeroed. +** ** The output text is not a copy of the query text that specified the ** token. It is the output of the tokenizer module. For tokendata=1 ** tables, this includes any embedded 0x00 and trailing data. ** ** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) ** This is used to access token iToken of phrase hit iIdx within the -** current row. Output variable (*ppToken) is set to point to a buffer -** containing the matching document token, and (*pnToken) to the size -** of that buffer in bytes. This API is not available if the specified -** token matches a prefix query term. In that case both output variables -** are always set to 0. +** current row. If iIdx is less than zero or greater than or equal to the +** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, +** output variable (*ppToken) is set to point to a buffer containing the +** matching document token, and (*pnToken) to the size of that buffer in +** bytes. This API is not available if the specified token matches a +** prefix query term. In that case both output variables are always set +** to 0. ** ** The output text is not a copy of the document text that was tokenized. ** It is the output of the tokenizer module. For tokendata=1 tables, this diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index b6919fc9cc..30101fbe25 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -252,8 +252,10 @@ static void fts5HighlightFunction( ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); ctx.iRangeEnd = -1; rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); - - if( ctx.zIn ){ + if( rc==SQLITE_RANGE ){ + sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); + rc = SQLITE_OK; + }else if( ctx.zIn ){ if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); } diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index b9614e1290..891ef0203a 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -68,6 +68,7 @@ void sqlite3Fts5BufferAppendBlob( ){ if( nData ){ if( fts5BufferGrow(pRc, pBuf, nData) ) return; + assert( pBuf->p!=0 ); memcpy(&pBuf->p[pBuf->n], pData, nData); pBuf->n += nData; } @@ -169,6 +170,7 @@ int sqlite3Fts5PoslistNext64( i64 *piOff /* IN/OUT: Current offset */ ){ int i = *pi; + assert( a!=0 || i==0 ); if( i>=n ){ /* EOF */ *piOff = -1; @@ -176,6 +178,7 @@ int sqlite3Fts5PoslistNext64( }else{ i64 iOff = *piOff; u32 iVal; + assert( a!=0 ); fts5FastGetVarint32(a, i, iVal); if( iVal<=1 ){ if( iVal==0 ){ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index bc7c9741ee..3a7e1d5e82 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1740,7 +1740,9 @@ static int fts5ParseTokenize( memset(pSyn, 0, (size_t)nByte); pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); pSyn->nFullTerm = pSyn->nQueryTerm = nToken; - if( pCtx->pConfig->bTokendata ) pSyn->nQueryTerm = strlen(pSyn->pTerm); + if( pCtx->pConfig->bTokendata ){ + pSyn->nQueryTerm = (int)strlen(pSyn->pTerm); + } memcpy(pSyn->pTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; @@ -1769,7 +1771,7 @@ static int fts5ParseTokenize( pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); pTerm->nFullTerm = pTerm->nQueryTerm = nToken; if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ - pTerm->nQueryTerm = strlen(pTerm->pTerm); + pTerm->nQueryTerm = (int)strlen(pTerm->pTerm); } } } @@ -1888,8 +1890,12 @@ int sqlite3Fts5ExprClonePhrase( Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */ - pOrig = pExpr->apExprPhrase[iPhrase]; - pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + rc = SQLITE_RANGE; + }else{ + pOrig = pExpr->apExprPhrase[iPhrase]; + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + } if( rc==SQLITE_OK ){ pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase*)); @@ -1916,25 +1922,27 @@ int sqlite3Fts5ExprClonePhrase( } } - if( pOrig->nTerm ){ - int i; /* Used to iterate through phrase terms */ - sCtx.pConfig = pExpr->pConfig; - for(i=0; rc==SQLITE_OK && inTerm; i++){ - int tflags = 0; - Fts5ExprTerm *p; - for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ - rc = fts5ParseTokenize((void*)&sCtx, tflags, p->pTerm,p->nFullTerm,0,0); - tflags = FTS5_TOKEN_COLOCATED; - } - if( rc==SQLITE_OK ){ - sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; - sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + if( rc==SQLITE_OK ){ + if( pOrig->nTerm ){ + int i; /* Used to iterate through phrase terms */ + sCtx.pConfig = pExpr->pConfig; + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + } } + }else{ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); } - }else{ - /* This happens when parsing a token or quoted phrase that contains - ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); } if( rc==SQLITE_OK && ALWAYS(sCtx.pPhrase) ){ @@ -3247,4 +3255,3 @@ void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){ } } } - diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 7ec1976d33..8eb8f328f6 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -360,6 +360,7 @@ struct Fts5Index { /* Error state. */ int rc; /* Current error code */ + int flushRc; /* State used by the fts5DataXXX() functions. */ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ @@ -1546,9 +1547,9 @@ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ } if( iOffnn ){ - i64 iVal; + u64 iVal; pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; - iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal); + iOff += fts5GetVarint(&pData->p[iOff], &iVal); pLvl->iRowid += iVal; pLvl->iOff = iOff; }else{ @@ -4120,6 +4121,7 @@ static void fts5IndexDiscardData(Fts5Index *p){ sqlite3Fts5HashClear(p->pHash); p->nPendingData = 0; p->nPendingRow = 0; + p->flushRc = SQLITE_OK; } p->nContentlessDelete = 0; } @@ -5701,6 +5703,10 @@ static void fts5FlushOneHash(Fts5Index *p){ */ static void fts5IndexFlush(Fts5Index *p){ /* Unless it is empty, flush the hash table to disk */ + if( p->flushRc ){ + p->rc = p->flushRc; + return; + } if( p->nPendingData || p->nContentlessDelete ){ assert( p->pHash ); fts5FlushOneHash(p); @@ -5709,6 +5715,8 @@ static void fts5IndexFlush(Fts5Index *p){ p->nPendingData = 0; p->nPendingRow = 0; p->nContentlessDelete = 0; + }else if( p->nPendingData || p->nContentlessDelete ){ + p->flushRc = p->rc; } } } @@ -6879,7 +6887,7 @@ static Fts5Iter *fts5SetupTokendataIter( fts5IndexFlush(p); pStruct = fts5StructureRead(p); - while( 1 ){ + while( p->rc==SQLITE_OK ){ Fts5Iter *pPrev = pSet ? pSet->apIter[pSet->nIter-1] : 0; Fts5Iter *pNew = 0; Fts5SegIter *pNewIter = 0; @@ -6888,14 +6896,16 @@ static Fts5Iter *fts5SetupTokendataIter( int iLvl, iSeg, ii; pNew = fts5MultiIterAlloc(p, pStruct->nSegment); - if( pNew==0 ) break; - if( pSmall ){ fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p); fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0"); }else{ fts5BufferSet(&p->rc, &bSeek, nToken, pToken); } + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + break; + } pNewIter = &pNew->aSeg[0]; pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0); @@ -6930,6 +6940,7 @@ static Fts5Iter *fts5SetupTokendataIter( pNewIter++; if( pPrevIter ) pPrevIter++; + if( p->rc ) break; } } fts5TokendataSetTermIfEof(pPrev, pSmall); @@ -7021,6 +7032,10 @@ int sqlite3Fts5IndexQuery( int bTokendata = pConfig->bTokendata; if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); + if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){ + bTokendata = 0; + } + /* Figure out which index to search and set iIdx accordingly. If this ** is a prefix query for which there is no prefix index, set iIdx to ** greater than pConfig->nPrefix to indicate that the query will be @@ -7032,7 +7047,6 @@ int sqlite3Fts5IndexQuery( ** for internal sanity checking by the integrity-check in debug ** mode only. */ #ifdef SQLITE_DEBUG - if( flags & FTS5INDEX_QUERY_NOTOKENDATA ) bTokendata = 0; if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; @@ -7211,7 +7225,7 @@ int sqlite3Fts5IterToken( */ void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - if( pIter->pTokenDataIter ){ + if( pIter && pIter->pTokenDataIter ){ pIter->pTokenDataIter->nMap = 0; } } @@ -7930,7 +7944,7 @@ static void fts5IndexIntegrityCheckEmpty( } static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ - int iTermOff = 0; + i64 iTermOff = 0; int ii; Fts5Buffer buf1 = {0,0,0}; @@ -7939,7 +7953,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ ii = pLeaf->szLeaf; while( iinn && p->rc==SQLITE_OK ){ int res; - int iOff; + i64 iOff; int nIncr; ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index d35e998da0..05946d8b64 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -118,7 +118,7 @@ struct Fts5FullTable { Fts5Global *pGlobal; /* Global (connection wide) data */ Fts5Cursor *pSortCsr; /* Sort data from this cursor */ int iSavepoint; /* Successful xSavepoint()+1 */ - int bInSavepoint; + #ifdef SQLITE_DEBUG struct Fts5TransactionState ts; #endif @@ -1911,7 +1911,10 @@ static int fts5ApiColumnText( ){ int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + if( iCol<0 || iCol>=pTab->pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) || pCsr->ePlan==FTS5_PLAN_SPECIAL ){ *pz = 0; @@ -1936,8 +1939,9 @@ static int fts5CsrPoslist( int rc = SQLITE_OK; int bLive = (pCsr->pSorter==0); - if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ - + if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){ + rc = SQLITE_RANGE; + }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; @@ -1961,15 +1965,21 @@ static int fts5CsrPoslist( CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); } - if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ - Fts5Sorter *pSorter = pCsr->pSorter; - int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); - *pn = pSorter->aIdx[iPhrase] - i1; - *pa = &pSorter->aPoslist[i1]; + if( rc==SQLITE_OK ){ + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + *pn = pSorter->aIdx[iPhrase] - i1; + *pa = &pSorter->aPoslist[i1]; + }else{ + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + } }else{ - *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + *pa = 0; + *pn = 0; } + return rc; } @@ -2667,9 +2677,7 @@ static int fts5RenameMethod( ){ int rc; Fts5FullTable *pTab = (Fts5FullTable*)pVtab; - pTab->bInSavepoint = 1; rc = sqlite3Fts5StorageRename(pTab->pStorage, zName); - pTab->bInSavepoint = 0; return rc; } @@ -2686,26 +2694,12 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; int rc = SQLITE_OK; - char *zSql = 0; + fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); - - if( pTab->bInSavepoint==0 ){ - zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')", - pTab->p.pConfig->zDb, pTab->p.pConfig->zName, pTab->p.pConfig->zName - ); - if( zSql ){ - pTab->bInSavepoint = 1; - rc = sqlite3_exec(pTab->p.pConfig->db, zSql, 0, 0, 0); - pTab->bInSavepoint = 0; - sqlite3_free(zSql); - }else{ - rc = SQLITE_NOMEM; - } - if( rc==SQLITE_OK ){ - pTab->iSavepoint = iSavepoint+1; - } + rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; } - return rc; } @@ -2966,7 +2960,7 @@ static int fts5ShadowName(const char *zName){ ** if anything is found amiss. Return a NULL pointer if everything is ** OK. */ -static int fts5Integrity( +static int fts5IntegrityMethod( sqlite3_vtab *pVtab, /* the FTS5 virtual table to check */ const char *zSchema, /* Name of schema in which this table lives */ const char *zTabname, /* Name of the table itself */ @@ -3024,7 +3018,7 @@ static int fts5Init(sqlite3 *db){ /* xRelease */ fts5ReleaseMethod, /* xRollbackTo */ fts5RollbackToMethod, /* xShadowName */ fts5ShadowName, - /* xIntegrity */ fts5Integrity + /* xIntegrity */ fts5IntegrityMethod }; int rc; diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 9480da7c52..a04b152fb0 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -673,7 +673,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ } if( rc==SQLITE_OK ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg); } while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index d738ada311..4782d0fb94 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -629,7 +629,7 @@ static int fts5VocabFilterMethod( if( pEq ){ zTerm = (const char *)sqlite3_value_text(pEq); nTerm = sqlite3_value_bytes(pEq); - f = 0; + f = FTS5INDEX_QUERY_NOTOKENDATA; }else{ if( pGe ){ zTerm = (const char *)sqlite3_value_text(pGe); diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 001cad1de2..fda388a6fb 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -61,6 +61,12 @@ proc fts5_test_collist {cmd} { set res } +proc fts5_collist {cmd iPhrase} { + set res [list] + $cmd xPhraseColumnForeach $iPhrase c { lappend res $c } + set res +} + proc fts5_test_columnsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -69,6 +75,10 @@ proc fts5_test_columnsize {cmd} { set res } +proc fts5_columntext {cmd iCol} { + $cmd xColumnText $iCol +} + proc fts5_test_columntext {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -125,6 +135,13 @@ proc fts5_test_queryphrase {cmd} { set res } +proc fts5_queryphrase {cmd iPhrase} { + set cnt [list] + for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 } + $cmd xQueryPhrase $iPhrase [list test_queryphrase_cb cnt] + set cnt +} + proc fts5_test_phrasecount {cmd} { $cmd xPhraseCount } @@ -154,6 +171,9 @@ proc fts5_aux_test_functions {db} { fts5_test_queryphrase fts5_test_phrasecount + fts5_columntext + fts5_queryphrase + fts5_collist } { sqlite3_fts5_create_function $db $f $f } diff --git a/ext/fts5/test/fts5aux.test b/ext/fts5/test/fts5aux.test index b5a13aea1a..5569f48cf3 100644 --- a/ext/fts5/test/fts5aux.test +++ b/ext/fts5/test/fts5aux.test @@ -334,4 +334,47 @@ do_execsql_test 11.2 { SELECT fts5_hitcount(x1) FROM x1('one') LIMIT 1; } {5} +#------------------------------------------------------------------------- +# Test that xColumnText returns SQLITE_RANGE when it should. +# +reset_db +fts5_aux_test_functions db +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + INSERT INTO t1 VALUES('one', 'two', 'three'); + INSERT INTO t1 VALUES('one', 'one', 'one'); + INSERT INTO t1 VALUES('two', 'two', 'two'); + INSERT INTO t1 VALUES('three', 'three', 'three'); +} + +do_catchsql_test 12.1.1 { + SELECT fts5_columntext(t1, -1) FROM t1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.1.2 { + SELECT fts5_columntext(t1, 3) FROM t1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.1.2 { + SELECT fts5_columntext(t1, 1) FROM t1('one AND two'); +} {0 two} + +do_catchsql_test 12.2.1 { + SELECT fts5_queryphrase(t1, -1) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.2.2 { + SELECT fts5_queryphrase(t1, 2) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.2.3 { + SELECT fts5_queryphrase(t1, 1) FROM t1('one AND two'); +} {0 {{1 2 1}}} + +do_catchsql_test 12.3.1 { + SELECT fts5_collist(t1, -1) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.3.2 { + SELECT fts5_collist(t1, 2) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.3.3 { + SELECT fts5_collist(t1, 1) FROM t1('one AND two'); +} {0 1} + finish_test diff --git a/ext/fts5/test/fts5content.test b/ext/fts5/test/fts5content.test index ca2726a902..986d7f3311 100644 --- a/ext/fts5/test/fts5content.test +++ b/ext/fts5/test/fts5content.test @@ -293,4 +293,39 @@ do_catchsql_test 7.2.5 { SELECT * FROM t1('abc') ORDER BY rank; } {1 {recursively defined fts5 content table}} +#--------------------------------------------------------------------------- +# Check that if the content table is a view, and that view contains an +# error, a reasonable error message is returned if the user tries to +# read from the view via the fts5 table. +# +reset_db +do_execsql_test 8.1 { + CREATE VIEW a1 AS + SELECT 1 AS r, text_value(1) AS t + UNION ALL + SELECT 2 AS r, text_value(2) AS t; + + CREATE VIRTUAL TABLE t1 USING fts5(t, content='a1', content_rowid='r'); +} + +foreach {tn sql} { + 1 "SELECT * FROM t1" + 2 "INSERT INTO t1(t1) VALUES('rebuild')" + 3 "SELECT * FROM t1 WHERE rowid=1" +} { + do_catchsql_test 8.2.$tn $sql {1 {no such function: text_value}} +} + +proc text_value {i} { + if {$i==1} { return "one" } + if {$i==2} { return "two" } + return "many" +} +db func text_value text_value + +do_execsql_test 8.3.1 { SELECT * FROM t1 } {one two} +do_execsql_test 8.3.2 { INSERT INTO t1(t1) VALUES('rebuild') } +do_execsql_test 8.3.3 { SELECT * FROM t1 WHERE rowid=1 } {one} +do_execsql_test 8.3.4 { SELECT rowid FROM t1('two') } {2} + finish_test diff --git a/ext/fts5/test/fts5corrupt5.test b/ext/fts5/test/fts5corrupt5.test index 60910dce7f..a56467376b 100644 --- a/ext/fts5/test/fts5corrupt5.test +++ b/ext/fts5/test/fts5corrupt5.test @@ -966,6 +966,262 @@ do_catchsql_test 6.2 { UPDATE t1 SET content=randomblob(500) WHERE t1; } {1 {constraint failed}} +#------------------------------------------------------------------------- +reset_db +do_test 7.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 40960 pagesize 4096 filename crash-d8b4a99207c10b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 62 00 0f 97 0f 40 ..........b....@ +| 112: 0e d5 0e 75 0e 18 0d c0 0d 66 0d 0f 0c a4 0c 44 ...u.....f.....D +| 128: 0b ec 0b a7 0b 62 00 00 00 00 00 00 00 00 00 00 .....b.......... +| 2912: 00 00 43 0d 06 17 11 11 08 75 74 61 62 6c 65 74 ..C......utablet +| 2928: 34 74 34 43 52 45 41 54 45 20 56 49 52 54 55 41 4t4CREATE VIRTUA +| 2944: 4c 20 54 41 42 4c 45 20 74 34 20 55 53 49 4e 47 L TABLE t4 USING +| 2960: 20 66 74 73 35 76 6f 63 61 62 28 27 74 32 27 2c fts5vocab('t2', +| 2976: 20 27 72 6f 77 27 29 43 0c 06 17 11 11 08 75 74 'row')C......ut +| 2992: 61 62 6c 65 74 33 74 33 43 52 45 41 54 45 20 56 ablet3t3CREATE V +| 3008: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 33 20 IRTUAL TABLE t3 +| 3024: 55 53 49 4e 47 20 66 74 73 35 76 6f 63 61 62 28 USING fts5vocab( +| 3040: 27 74 31 27 2c 20 27 72 6f 77 27 29 56 0b 06 17 't1', 'row')V... +| 3056: 1f 1f 01 7d 74 61 62 6c 65 74 32 5f 63 6f 6e 66 ....tablet2_conf +| 3072: 69 67 74 32 5f 63 6f 6e 66 69 67 0a 43 52 45 41 igt2_config.CREA +| 3088: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 63 6f 6e TE TABLE 't2_con +| 3104: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3120: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3136: 4f 57 49 44 5e 0a 07 17 21 21 01 81 07 74 61 62 OWID^...!!...tab +| 3152: 6c 65 74 32 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 let2_contentt2_c +| 3168: 6f 6e 74 65 6e 74 09 43 52 45 41 54 45 20 54 41 ontent.CREATE TA +| 3184: 42 4c 45 20 27 74 32 5f 63 6f 6e 74 65 6e 74 27 BLE 't2_content' +| 3200: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3216: 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c ARY KEY, c0, c1, +| 3232: 20 63 32 29 69 09 07 17 19 19 01 81 2d 74 61 62 c2)i.......-tab +| 3248: 6c 65 74 32 5f 69 64 78 74 32 5f 69 64 78 08 43 let2_idxt2_idx.C +| 3264: 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 32 5f REATE TABLE 't2_ +| 3280: 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d idx'(segid, term +| 3296: 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 , pgno, PRIMARY +| 3312: 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 KEY(segid, term) +| 3328: 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 ) WITHOUT ROWIDU +| 3344: 08 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 32 5f ........tablet2_ +| 3360: 64 61 74 61 74 32 5f 64 61 74 61 07 43 52 45 41 datat2_data.CREA +| 3376: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 64 61 74 TE TABLE 't2_dat +| 3392: 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 a'(id INTEGER PR +| 3408: 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b IMARY KEY, block +| 3424: 20 42 4c 4f 42 29 58 07 07 17 11 11 08 81 1d 74 BLOB)X........t +| 3440: 61 62 6c 65 74 32 74 32 43 52 45 41 54 45 20 56 ablet2t2CREATE V +| 3456: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 32 20 IRTUAL TABLE t2 +| 3472: 55 53 49 4e 47 20 66 74 73 35 28 27 61 27 2c 5b USING fts5('a',[ +| 3488: 62 5d 2c 22 63 22 2c 64 65 74 61 69 6c 3d 6e 6f b],.c.,detail=no +| 3504: 6e 65 2c 63 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 ne,columnsize=0) +| 3520: 56 06 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f V.......tablet1_ +| 3536: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 06 configt1_config. +| 3552: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3568: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41 _config'(k PRIMA +| 3584: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f RY KEY, v) WITHO +| 3600: 55 54 20 52 4f 57 49 44 5b 05 07 17 21 21 01 81 UT ROWID[...!!.. +| 3616: 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 .tablet1_docsize +| 3632: 74 31 5f 64 6f 63 73 69 7a 65 05 43 52 45 41 54 t1_docsize.CREAT +| 3648: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs +| 3664: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20 ize'(id INTEGER +| 3680: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20 PRIMARY KEY, sz +| 3696: 42 4c 4f 42 29 5e 04 07 17 21 21 01 81 07 74 61 BLOB)^...!!...ta +| 3712: 62 6c 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f blet1_contentt1_ +| 3728: 63 6f 6e 74 65 6e 74 04 43 52 45 41 54 45 20 54 content.CREATE T +| 3744: 41 42 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74 ABLE 't1_content +| 3760: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 '(id INTEGER PRI +| 3776: 4d 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 MARY KEY, c0, c1 +| 3792: 2c 20 63 32 29 69 03 07 17 19 19 01 81 2d 74 61 , c2)i.......-ta +| 3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03 blet1_idxt1_idx. +| 3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 _idx'(segid, ter +| 3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 m, pgno, PRIMARY +| 3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d KEY(segid, term +| 3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 )) WITHOUT ROWID +| 3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 U........tablet1 +| 3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45 _datat1_data.CRE +| 3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 ATE TABLE 't1_da +| 3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 ta'(id INTEGER P +| 3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 RIMARY KEY, bloc +| 3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b k BLOB)g.......; +| 4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE +| 4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 62 20 USING fts5(a,b +| 4048: 75 6e 69 6e 64 65 78 65 64 2c 63 2c 74 6f 6b 65 unindexed,c,toke +| 4064: 6e 69 7a 65 3d 22 70 6f 72 74 65 72 20 61 73 63 nize=.porter asc +| 4080: 69 69 22 2c 74 6f 6b 65 6e 64 61 74 61 3d 31 29 ii.,tokendata=1) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 7f f1 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-d8b4a99207c10b.db +}]} {} + +do_catchsql_test 7.1 { + SELECT snippet(t1, -1, '.', '..', '[', ']'), + highlight(t1, 2, '[', ']') + FROM t1('g + h') + WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 8.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 20480 pagesize 4096 filename crash-d57c01958e48ab.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 05 0e 10 00 0f 97 0f 40 ...............@ +| 112: 0e d5 0e 68 0e 10 01 00 00 00 00 00 00 00 00 00 ...h............ +| 3600: 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f V.......tablet1_ +| 3616: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 05 configt1_config. +| 3632: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3648: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41 _config'(k PRIMA +| 3664: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f RY KEY, v) WITHO +| 3680: 55 54 20 52 4f 57 49 44 6b 04 07 17 21 21 01 81 UT ROWIDk...!!.. +| 3696: 21 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 !tablet1_docsize +| 3712: 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 45 41 54 t1_docsize.CREAT +| 3728: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs +| 3744: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20 ize'(id INTEGER +| 3760: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20 PRIMARY KEY, sz +| 3776: 42 4c 4f 42 2c 20 6f 72 69 67 69 6e 20 49 4e 54 BLOB, origin INT +| 3792: 45 47 45 52 29 69 03 07 17 19 19 01 81 2d 74 61 EGER)i.......-ta +| 3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03 blet1_idxt1_idx. +| 3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 _idx'(segid, ter +| 3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 m, pgno, PRIMARY +| 3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d KEY(segid, term +| 3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 )) WITHOUT ROWID +| 3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 U........tablet1 +| 3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45 _datat1_data.CRE +| 3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 ATE TABLE 't1_da +| 3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 ta'(id INTEGER P +| 3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 RIMARY KEY, bloc +| 3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b k BLOB)g.......; +| 4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE +| 4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62 USING fts5(a, b +| 4048: 2c 20 63 6f 6e 74 65 6e 74 3d 27 27 2c 20 63 6f , content='', co +| 4064: 6e 74 65 6e 74 6c 65 73 73 5f 64 65 6c 65 74 65 ntentless_delete +| 4080: 3d 31 2c 20 74 6f 6b 65 6e 64 61 74 61 3d 31 29 =1, tokendata=1) +| page 2 offset 4096 +| 0: 0d 0f eb 00 03 0e 17 00 0f e2 0e 17 0e 31 00 00 .............1.. +| 16: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3600: 00 00 00 00 00 00 00 18 0a 03 00 36 00 00 00 00 ...........6.... +| 3616: ff 00 00 01 01 01 01 00 01 01 01 01 01 01 00 00 ................ +| 3632: 07 83 29 84 80 80 80 80 01 04 00 86 56 00 00 01 ..).........V... +| 3648: 96 04 30 61 61 61 01 02 02 01 04 02 04 01 08 02 ..0aaa.......... +| 3664: 04 04 04 01 10 02 04 04 04 04 04 04 04 01 20 02 .............. . +| 3680: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 01 ................ +| 3696: 40 02 04 04 04 04 04 04 04 04 04 04 04 04 04 04 @............... +| 3712: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3728: 04 01 81 00 02 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3744: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3760: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3776: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3792: 04 04 04 04 02 02 62 63 01 06 01 01 02 01 03 62 ......bc.......b +| 3808: 62 62 02 02 03 01 04 03 06 01 08 03 06 06 06 01 bb.............. +| 3824: 10 03 06 06 06 06 06 06 06 01 20 03 06 06 06 06 .......... ..... +| 3840: 06 06 06 06 06 06 06 06 06 06 06 01 40 03 06 06 ............@... +| 3856: 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 ................ +| 3872: 06 06 06 06 06 06 06 06 06 06 16 06 06 02 02 63 ...............c +| 3888: 64 02 06 01 01 02 01 03 63 63 63 03 02 05 01 04 d.......ccc..... +| 3904: 05 0a 01 08 05 0a 0a 0a 01 10 05 0a 0a 0a 0a 0a ................ +| 3920: 0a 0a 01 20 05 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a ... ............ +| 3936: 0a 0a 0a 0a 02 02 64 65 03 06 01 01 02 01 03 64 ......de.......d +| 3952: 64 64 04 02 09 01 04 09 12 01 08 09 12 12 12 01 dd.............. +| 3968: 10 09 12 12 12 12 12 12 12 02 02 65 66 04 06 01 ...........ef... +| 3984: 01 02 01 03 65 65 65 05 02 11 01 04 11 22 01 08 ....eee......... +| 4000: 11 22 22 22 02 02 66 67 05 06 01 01 02 01 03 66 ......fg.......f +| 4016: 56 66 06 02 21 01 04 21 42 02 02 67 68 06 06 01 Vf..!..!B..gh... +| 4032: 01 02 cb 03 67 67 67 07 02 41 02 02 68 69 07 06 ....ggg..A..hi.. +| 4048: 01 01 02 04 81 13 09 50 09 2e 09 1c 09 12 09 0c .......P........ +| 4064: 09 08 07 01 03 00 14 07 81 77 07 00 00 00 15 22 .........w...... +| 4080: 00 00 00 00 ff 00 00 01 00 00 00 00 00 00 05 0c ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 07 0f c8 00 0f f8 0f f0 0f e8 0f e0 ................ +| 16: 0f d8 0f d0 0f c8 00 00 00 00 00 00 00 00 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 06 07 04 00 10 09 7f 01 ................ +| 4048: 06 06 04 00 10 09 3f 01 06 05 04 00 10 09 1f 01 ......?......... +| 4064: 06 04 04 00 10 09 0f 01 06 03 04 00 10 09 07 01 ................ +| 4080: 06 02 04 00 10 09 03 01 06 01 04 00 10 09 01 01 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-d57c01958e48ab.db +}]} {} + +do_catchsql_test 8.1 { + SELECT rowid FROM t1('a* NOT ý‘') ; +} {0 {1 2 3 4 5 6 7}} + sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/ext/fts5/test/fts5faultH.test b/ext/fts5/test/fts5faultH.test index 9dd4cac0d6..540b889f37 100644 --- a/ext/fts5/test/fts5faultH.test +++ b/ext/fts5/test/fts5faultH.test @@ -88,6 +88,54 @@ do_faultsim_test 2 -faults oom* -prep { faultsim_test_result {0 {10 24}} } +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + + INSERT INTO t1(rowid, x) VALUES(9, 'bbb Bbb BBB'); + BEGIN; + INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(11, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(12, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(13, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(14, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(15, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(16, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(17, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(18, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(19, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(20, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(21, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(22, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(23, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(24, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(25, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(26, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(27, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(28, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(29, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(30, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(31, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(32, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(33, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB'); + COMMIT; +} + +do_faultsim_test 3 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1('BBB AND AAA'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {10 35}} +} finish_test diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index d67d79e29f..221f3996a7 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -91,7 +91,6 @@ do_execsql_test 2.2.1 { INSERT INTO vt0(c0) VALUES ('xyz'); } -breakpoint do_execsql_test 2.2.2 { ALTER TABLE t0 RENAME TO t1; } @@ -500,6 +499,21 @@ do_execsql_test 17.5 { SELECT c0 FROM t0 WHERE c0 GLOB '*faul*'; } {assertionfaultproblem} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 18.0 { + BEGIN; + CREATE VIRTUAL TABLE t1 USING fts5(text); + ALTER TABLE t1 RENAME TO t2; +} + +do_execsql_test 18.1 { + DROP TABLE t2; +} + +do_execsql_test 18.2 { + COMMIT; +} finish_test diff --git a/ext/fts5/test/fts5origintext4.test b/ext/fts5/test/fts5origintext4.test index 8973a24b05..b9a889a514 100644 --- a/ext/fts5/test/fts5origintext4.test +++ b/ext/fts5/test/fts5origintext4.test @@ -21,6 +21,19 @@ ifcapable !fts5 { return } +# The tests below verify that a doclist-index is used to limit the number +# of pages loaded into the cache. It does this by querying sqlite3_db_status() +# for the amount of memory used by the pager cache. +# +# memsubsys1 effectively limits the page-cache to 24 pages. Which masks +# the effect tested by the tests in this file. And "mmap" prevents the +# cache from being used, also preventing these tests from working. +# +if {[permutation]=="memsubsys1" || [permutation]=="mmap"} { + finish_test + return +} + sqlite3_fts5_register_origintext db do_execsql_test 1.0 { CREATE VIRTUAL TABLE ft USING fts5( diff --git a/ext/fts5/test/fts5secure3.test b/ext/fts5/test/fts5secure3.test index bc56e08200..49347144f3 100644 --- a/ext/fts5/test/fts5secure3.test +++ b/ext/fts5/test/fts5secure3.test @@ -86,71 +86,76 @@ do_execsql_test 2.8 { # Tests with large/small rowid values. # -reset_db - -expr srand(0) - -set vocab { - Popper Poppins Popsicle Porfirio Porrima Porsche - Porter Portia Portland Portsmouth Portugal Portuguese - Poseidon Post PostgreSQL Potemkin Potomac Potsdam - Pottawatomie Potter Potts Pound Poussin Powell - PowerPC PowerPoint Powers Powhatan Poznan Prada - Prado Praetorian Prague Praia Prakrit Pratchett - Pratt Pravda Praxiteles Preakness Precambrian Preminger - Premyslid Prensa Prentice Pres Presbyterian Presbyterianism -} -proc newdoc {} { - for {set i 0} {$i<8} {incr i} { - lappend ret [lindex $::vocab [expr int(abs(rand()) * [llength $::vocab])]] +foreach {tn cfg} { + 1 "" + 2 "INSERT INTO fff(fff, rank) VALUES('secure-delete', 1)" +} { + reset_db + + expr srand(0) + + set vocab { + Popper Poppins Popsicle Porfirio Porrima Porsche + Porter Portia Portland Portsmouth Portugal Portuguese + Poseidon Post PostgreSQL Potemkin Potomac Potsdam + Pottawatomie Potter Potts Pound Poussin Powell + PowerPC PowerPoint Powers Powhatan Poznan Prada + Prado Praetorian Prague Praia Prakrit Pratchett + Pratt Pravda Praxiteles Preakness Precambrian Preminger + Premyslid Prensa Prentice Pres Presbyterian Presbyterianism } - set ret -} -db func newdoc newdoc - -do_execsql_test 3.0 { - CREATE VIRTUAL TABLE fff USING fts5(y); - INSERT INTO fff(fff, rank) VALUES('pgsz', 64); - - WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) - INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; - - WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) - INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; - - WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) - INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; - - INSERT INTO fff(fff, rank) VALUES('secure-delete', 1); -} - -proc lshuffle {in} { - set out [list] - while {[llength $in]>0} { - set idx [expr int(abs(rand()) * [llength $in])] - lappend out [lindex $in $idx] - set in [lreplace $in $idx $idx] - } - set out -} - -#dump fff - -set iTest 1 -foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] { - #if {$iTest==1} { dump fff } - #if {$iTest==1} { breakpoint } - do_execsql_test 3.1.$iTest.$ii { - DELETE FROM fff WHERE rowid=$ii; - } - #if {$iTest==1} { dump fff } - if {($iTest % 20)==0} { - do_execsql_test 3.1.$iTest.$ii.ic { - INSERT INTO fff(fff) VALUES('integrity-check'); + proc newdoc {} { + for {set i 0} {$i<8} {incr i} { + lappend ret [lindex $::vocab [expr int(abs(rand()) * [llength $::vocab])]] } + set ret + } + db func newdoc newdoc + + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE fff USING fts5(y); + INSERT INTO fff(fff, rank) VALUES('pgsz', 64); + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + } + + execsql $cfg + + proc lshuffle {in} { + set out [list] + while {[llength $in]>0} { + set idx [expr int(abs(rand()) * [llength $in])] + lappend out [lindex $in $idx] + set in [lreplace $in $idx $idx] + } + set out + } + + #dump fff + + set iTest 1 + foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] { + #if {$iTest==1} { dump fff } + #if {$iTest==1} { breakpoint } + do_execsql_test 3.$tn.1.$iTest.$ii { + DELETE FROM fff WHERE rowid=$ii; + } + #if {$iTest==1} { dump fff } + if {($iTest % 20)==0} { + do_execsql_test 3.$tn.1.$iTest.$ii.ic { + INSERT INTO fff(fff) VALUES('integrity-check'); + } + } + #if {$iTest==1} { break } + incr iTest } - #if {$iTest==1} { break } - incr iTest } #execsql_pp { SELECT rowid FROM fff('post') ORDER BY rowid ASC } diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test index 2c2705e34f..9fdf757693 100644 --- a/ext/fts5/test/fts5synonym2.test +++ b/ext/fts5/test/fts5synonym2.test @@ -42,7 +42,7 @@ proc fts5_test_bothlist {cmd} { } sqlite3_fts5_create_function db fts5_test_bothlist fts5_test_bothlist -proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] } +proc fts5_rowid {cmd} { expr [$cmd xRowid] } sqlite3_fts5_create_function db fts5_rowid fts5_rowid do_execsql_test 1.$tok.0.1 " diff --git a/ext/fts5/test/fts5vocab2.test b/ext/fts5/test/fts5vocab2.test index 6f7aad329c..ecacc50dab 100644 --- a/ext/fts5/test/fts5vocab2.test +++ b/ext/fts5/test/fts5vocab2.test @@ -280,6 +280,30 @@ do_catchsql_test 5.2 { INSERT INTO t1 SELECT randomblob(3000) FROM v1 } {1 {query aborted}} +#------------------------------------------------------------------------- +reset_db +sqlite3_fts5_may_be_corrupt 1 + +do_execsql_test 6.0 { + BEGIN TRANSACTION; + CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1); + REPLACE INTO t1_data VALUES(1,X'03090009'); + REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101'); + REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808'); + REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909'); + REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808'); + COMMIT; +} + +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row'); +} + +do_catchsql_test 6.2 { + SELECT * FROM t3; +} {1 {database disk image is malformed}} + +sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index 14c447acd5..5deff19ef1 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -107,6 +107,12 @@ # undef SQLITE_OMIT_UTF16 1 #endif +/**********************************************************************/ +/* SQLITE_S... */ +#ifndef SQLITE_STRICT_SUBTYPE +# define SQLITE_STRICT_SUBTYPE 1 +#endif + /**********************************************************************/ /* SQLITE_T... */ #ifndef SQLITE_TEMP_STORE diff --git a/ext/misc/randomjson.c b/ext/misc/randomjson.c index 3a6f545fe6..cacc4feb68 100644 --- a/ext/misc/randomjson.c +++ b/ext/misc/randomjson.c @@ -26,9 +26,14 @@ ** ** .load ./randomjson ** SELECT random_json(1); +** SELECT random_json5(1); */ -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 +#ifdef SQLITE_STATIC_RANDOMJSON +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif #include #include #include @@ -51,17 +56,18 @@ static unsigned int prngInt(Prng *p){ return p->x ^ p->y; } -static const char *azJsonAtoms[] = { - /* JSON /* JSON-5 */ +static char *azJsonAtoms[] = { + /* JSON JSON-5 */ "0", "0", "1", "1", "-1", "-1", "2", "+2", - "3", "3", - "2.5", "2.5", + "3DDDD", "3DDDD", + "2.5DD", "2.5DD", "0.75", ".75", "-4.0e2", "-4.e2", "5.0e-3", "+5e-3", + "6.DDe+0DD", "6.DDe+0DD", "0", "0x0", "512", "0x200", "256", "+0x100", @@ -73,12 +79,14 @@ static const char *azJsonAtoms[] = { "-9.0e999", "-Infinity", "9.0e999", "+Infinity", "null", "NaN", - "-0.0005123", "-0.0005123", + "-0.0005DD", "-0.0005DD", "4.35e-3", "+4.35e-3", "\"gem\\\"hay\"", "\"gem\\\"hay\"", "\"icy'joy\"", "'icy\\'joy\'", "\"keylog\"", "\"key\\\nlog\"", "\"mix\\\\\\tnet\"", "\"mix\\\\\\tnet\"", + "\"oat\\r\\n\"", "\"oat\\r\\n\"", + "\"\\fpan\\b\"", "\"\\fpan\\b\"", "{}", "{}", "[]", "[]", "[]", "[/*empty*/]", @@ -89,19 +97,20 @@ static const char *azJsonAtoms[] = { "\"day\"", "\"day\"", "\"end\"", "'end'", "\"fly\"", "\"fly\"", + "\"\\u00XX\\u00XX\"", "\"\\xXX\\xXX\"", + "\"y\\uXXXXz\"", "\"y\\uXXXXz\"", "\"\"", "\"\"", }; -static const char *azJsonTemplate[] = { +static char *azJsonTemplate[] = { /* JSON JSON-5 */ - "{\"a\":%,\"b\":%,\"c\":%}", "{a:%,b:%,c:%}", + "{\"a\":%,\"b\":%,\"cDD\":%}", "{a:%,b:%,cDD:%}", "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"e\":%}", "{a:%,b:%,c:%,d:%,e:%}", - "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}", "{a:%,b:%,c:%,d:%,\"\":%}", + "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}", "{a:%,b:%,c:%,d:%,'':%}", "{\"d\":%}", "{d:%}", "{\"eeee\":%, \"ffff\":%}", "{eeee:% /*and*/, ffff:%}", - "{\"$g\":%,\"_h_\":%}", "{$g:%,_h_:%,}", + "{\"$g\":%,\"_h_\":%,\"a b c d\":%}", "{$g:%,_h_:%,\"a b c d\":%}", "{\"x\":%,\n \"y\":%}", "{\"x\":%,\n \"y\":%}", - "{\"a b c d\":%,\"e\":%,\"f\":%,\"x\":%,\"y\":%}", - "{\"a b c d\":%,e:%,f:%,x:%,y:%}", + "{\"\\u00XX\":%,\"\\uXXXX\":%}", "{\"\\xXX\":%,\"\\uXXXX\":%}", "{\"Z\":%}", "{Z:%,}", "[%]", "[%,]", "[%,%]", "[%,%]", @@ -122,15 +131,13 @@ static void jsonExpand( unsigned int r /* Growth probability 0..1000. 0 means no growth */ ){ unsigned int i, j, k; - const char *z; + char *z; + char *zX; size_t n; + char zBuf[200]; j = 0; - if( zSrc==0 ){ - k = prngInt(p)%(count(azJsonTemplate)/2); - k = k*2 + eType; - zSrc = azJsonTemplate[k]; - } + if( zSrc==0 ) zSrc = "%"; if( strlen(zSrc)>=STRSZ/10 ) r = 0; for(i=0; zSrc[i]; i++){ if( zSrc[i]!='%' ){ @@ -149,9 +156,36 @@ static void jsonExpand( z = azJsonTemplate[k]; } n = strlen(z); + if( (zX = strstr(z,"XX"))!=0 ){ + unsigned int y = prngInt(p); + if( (y&0xff)==((y>>8)&0xff) ) y += 0x100; + while( (y&0xff)==((y>>16)&0xff) || ((y>>8)&0xff)==((y>>16)&0xff) ){ + y += 0x10000; + } + memcpy(zBuf, z, n+1); + z = zBuf; + zX = strstr(z,"XX"); + while( zX!=0 ){ + zX[0] = "0123456789abcdef"[y%16]; y /= 16; + zX[1] = "0123456789abcdef"[y%16]; y /= 16; + zX = strstr(zX, "XX"); + } + }else if( (zX = strstr(z,"DD"))!=0 ){ + unsigned int y = prngInt(p); + memcpy(zBuf, z, n+1); + z = zBuf; + zX = strstr(z,"DD"); + while( zX!=0 ){ + zX[0] = "0123456789"[y%10]; y /= 10; + zX[1] = "0123456789"[y%10]; y /= 10; + zX = strstr(zX, "DD"); + } + } + assert( strstr(z, "XX")==0 ); + assert( strstr(z, "DD")==0 ); if( j+ny.pTab==0 || /* Reference to column of index on expression */ + NEVER(p->y.pTab==0) || /* Reference to column of index on expr */ (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ + && ALWAYS(p->iColumny.pTab->nCol) && p->y.pTab->aCol[p->iColumn].notNull==0); default: return 1; @@ -5324,8 +5325,10 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); if( inReg!=target ){ u8 op; - if( ALWAYS(pExpr) - && (ExprHasProperty(pExpr,EP_Subquery) || pExpr->op==TK_REGISTER) + Expr *pX = sqlite3ExprSkipCollateAndLikely(pExpr); + testcase( pX!=pExpr ); + if( ALWAYS(pX) + && (ExprHasProperty(pX,EP_Subquery) || pX->op==TK_REGISTER) ){ op = OP_Copy; }else{ @@ -6821,6 +6824,8 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ }else{ pItem->bOBPayload = 1; } + pItem->bUseSubtype = + (pItem->pFunc->funcFlags & SQLITE_SUBTYPE)!=0; }else{ pItem->iOBTab = -1; } diff --git a/src/global.c b/src/global.c index 60cd13e2ab..7f27d91d15 100644 --- a/src/global.c +++ b/src/global.c @@ -244,6 +244,9 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* bSmallMalloc */ 1, /* bExtraSchemaChecks */ sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */ +#ifdef SQLITE_DEBUG + 0, /* bJsonSelfcheck */ +#endif 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ diff --git a/src/json.c b/src/json.c index 46d7a3fa8c..42d6155fb6 100644 --- a/src/json.c +++ b/src/json.c @@ -20,9 +20,9 @@ ** All generated JSON text still conforms strictly to RFC-8259, but text ** with JSON-5 extensions is accepted as input. ** -** Beginning with version 3.45.0 (pending), these routines also accept -** BLOB values that have JSON encoded using a binary representation we -** call JSONB. The name JSONB comes from PostgreSQL, however the on-disk +** Beginning with version 3.45.0 (circa 2024-01-01), these routines also +** accept BLOB values that have JSON encoded using a binary representation +** called "JSONB". The name JSONB comes from PostgreSQL, however the on-disk ** format SQLite JSONB is completely different and incompatible with ** PostgreSQL JSONB. ** @@ -113,6 +113,9 @@ ** checks are true, the BLOB is assumed to be JSONB and processing continues. ** Errors are only raised if some other miscoding is discovered during ** processing. +** +** Additional information can be found in the doc/jsonb.md file of the +** canonical SQLite source tree. */ #ifndef SQLITE_OMIT_JSON #include "sqliteInt.h" @@ -176,7 +179,9 @@ static const char jsonSpaces[] = "\011\012\015\040"; /* ** Characters that are special to JSON. Control characters, -** '"' and '\\'. +** '"' and '\\' and '\''. Actually, '\'' is not special to +** canonical JSON, but it is special in JSON-5, so we include +** it in the set of special characters. */ static const char jsonIsOk[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -209,6 +214,12 @@ typedef struct JsonParse JsonParse; #define JSON_CACHE_ID (-429938) /* Cache entry */ #define JSON_CACHE_SIZE 4 /* Max number of cache entries */ +/* +** jsonUnescapeOneChar() returns this invalid code point if it encounters +** a syntax error. +*/ +#define JSON_INVALID_CHAR 0x99999 + /* A cache mapping JSON text into JSONB blobs. ** ** Each cache entry is a JsonParse object with the following restrictions: @@ -218,6 +229,8 @@ typedef struct JsonParse JsonParse; ** * The aBlob[] array must be owned by the JsonParse object. In other ** words, nBlobAlloc must be non-zero. ** +** * eEdit and delta must be zero. +** ** * zJson must be an RCStr. In other words bJsonIsRCStr must be true. */ struct JsonCache { @@ -277,8 +290,8 @@ struct JsonString { ** json_replace() or json_patch() or similar). ** ** 4. New JSON text is generated from the aBlob[] for output. This step -** is skipped the function is one of the jsonb_* functions that returns -** JSONB instead of text JSON. +** is skipped if the function is one of the jsonb_* functions that +** returns JSONB instead of text JSON. */ struct JsonParse { u8 *aBlob; /* JSONB representation of JSON value */ @@ -286,14 +299,14 @@ struct JsonParse { u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ char *zJson; /* Json text used for parsing */ int nJson; /* Length of the zJson string in bytes */ + u32 nJPRef; /* Number of references to this object */ + u32 iErr; /* Error location in zJson[] */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ u8 bJsonIsRCStr; /* True if zJson is an RCStr */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 bReadOnly; /* Do not modify. */ - u32 nJPRef; /* Number of references to this object */ - u32 iErr; /* Error location in zJson[] */ /* Search and edit information. See jsonLookupStep() */ u8 eEdit; /* Edit operation to apply */ int delta; /* Size change due to the edit */ @@ -332,10 +345,12 @@ struct JsonParse { **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); -static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); +static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static void jsonParseFree(JsonParse*); +static u32 jsonbPayloadSize(const JsonParse*, u32, u32*); +static u32 jsonUnescapeOneChar(const char*, u32, u32*); /************************************************************************** ** Utility routines for dealing with JsonCache objects @@ -370,6 +385,7 @@ static int jsonCacheInsert( assert( pParse->zJson!=0 ); assert( pParse->bJsonIsRCStr ); + assert( pParse->delta==0 ); p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); if( p==0 ){ sqlite3 *db = sqlite3_context_db_handle(ctx); @@ -442,6 +458,7 @@ static JsonParse *jsonCacheSearch( p->a[p->nUsed-1] = tmp; i = p->nUsed - 1; } + assert( p->a[i]->delta==0 ); return p->a[i]; }else{ return 0; @@ -703,7 +720,7 @@ static void jsonAppendSqlValue( memset(&px, 0, sizeof(px)); px.aBlob = (u8*)sqlite3_value_blob(pValue); px.nBlob = sqlite3_value_bytes(pValue); - jsonXlateBlobToText(&px, 0, p); + jsonTranslateBlobToText(&px, 0, p); }else if( p->eErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); p->eErr = JSTRING_ERR; @@ -1220,14 +1237,211 @@ static int jsonBlobChangePayloadSize( */ static int jsonIs4HexB(const char *z, int *pOp){ if( z[0]!='u' ) return 0; - if( !sqlite3Isxdigit(z[1]) ) return 0; - if( !sqlite3Isxdigit(z[2]) ) return 0; - if( !sqlite3Isxdigit(z[3]) ) return 0; - if( !sqlite3Isxdigit(z[4]) ) return 0; + if( !jsonIs4Hex(&z[1]) ) return 0; *pOp = JSONB_TEXTJ; return 1; } +/* +** Check a single element of the JSONB in pParse for validity. +** +** The element to be checked starts at offset i and must end at on the +** last byte before iEnd. +** +** Return 0 if everything is correct. Return the 1-based byte offset of the +** error if a problem is detected. (In other words, if the error is at offset +** 0, return 1). +*/ +static u32 jsonbValidityCheck( + const JsonParse *pParse, /* Input JSONB. Only aBlob and nBlob are used */ + u32 i, /* Start of element as pParse->aBlob[i] */ + u32 iEnd, /* One more than the last byte of the element */ + u32 iDepth /* Current nesting depth */ +){ + u32 n, sz, j, k; + const u8 *z; + u8 x; + if( iDepth>JSON_MAX_DEPTH ) return i+1; + sz = 0; + n = jsonbPayloadSize(pParse, i, &sz); + if( NEVER(n==0) ) return i+1; /* Checked by caller */ + if( NEVER(i+n+sz!=iEnd) ) return i+1; /* Checked by caller */ + z = pParse->aBlob; + x = z[i] & 0x0f; + switch( x ){ + case JSONB_NULL: + case JSONB_TRUE: + case JSONB_FALSE: { + return n+sz==1 ? 0 : i+1; + } + case JSONB_INT: { + if( sz<1 ) return i+1; + j = i+n; + if( z[j]=='-' ){ + j++; + if( sz<2 ) return i+1; + } + k = i+n+sz; + while( jk ) return j+1; + if( z[j+1]!='.' && z[j+1]!='e' && z[j+1]!='E' ) return j+1; + j++; + } + for(; j0 ) return j+1; + if( x==JSONB_FLOAT && (j==k-1 || !sqlite3Isdigit(z[j+1])) ){ + return j+1; + } + seen = 1; + continue; + } + if( z[j]=='e' || z[j]=='E' ){ + if( seen==2 ) return j+1; + if( j==k-1 ) return j+1; + if( z[j+1]=='+' || z[j+1]=='-' ){ + j++; + if( j==k-1 ) return j+1; + } + seen = 2; + continue; + } + return j+1; + } + if( seen==0 ) return i+1; + return 0; + } + case JSONB_TEXT: { + j = i+n; + k = j+sz; + while( j=k ){ + return j+1; + }else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){ + j++; + }else if( z[j+1]=='u' ){ + if( j+5>=k ) return j+1; + if( !jsonIs4Hex((const char*)&z[j+2]) ) return j+1; + j++; + }else if( x!=JSONB_TEXT5 ){ + return j+1; + }else{ + u32 c = 0; + u32 szC = jsonUnescapeOneChar((const char*)&z[j], k-j, &c); + if( c==JSON_INVALID_CHAR ) return j+1; + j += szC - 1; + } + } + j++; + } + return 0; + } + case JSONB_TEXTRAW: { + return 0; + } + case JSONB_ARRAY: { + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1); + if( sub ) return sub; + j += n + sz; + } + assert( j==k ); + return 0; + } + case JSONB_OBJECT: { + u32 cnt = 0; + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + if( (cnt & 1)==0 ){ + x = z[j] & 0x0f; + if( xJSONB_TEXTRAW ) return j+1; + } + sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1); + if( sub ) return sub; + cnt++; + j += n + sz; + } + assert( j==k ); + if( (cnt & 1)!=0 ) return j+1; + return 0; + } + default: { + return i+1; + } + } +} + /* ** Translate a single element of JSON text at pParse->zJson[i] into ** its equivalent binary JSONB representation. Append the translation into @@ -1244,7 +1458,7 @@ static int jsonIs4HexB(const char *z, int *pOp){ ** -4 ',' seen / the index in zJson[] of the seen character ** -5 ':' seen / */ -static int jsonXlateTextToBlob(JsonParse *pParse, u32 i){ +static int jsonTranslateTextToBlob(JsonParse *pParse, u32 i){ char c; u32 j; u32 iThis, iStart; @@ -1264,7 +1478,7 @@ json_parse_restart: iStart = pParse->nBlob; for(j=i+1;;j++){ u32 iBlob = pParse->nBlob; - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x<=0 ){ int op; if( x==(-2) ){ @@ -1310,7 +1524,7 @@ json_parse_restart: goto parse_object_value; } } - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x!=(-5) ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1318,7 +1532,7 @@ json_parse_restart: j = pParse->iErr+1; } parse_object_value: - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x<=0 ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1330,14 +1544,14 @@ json_parse_restart: break; }else{ if( jsonIsspace(z[j]) ){ - j += 1 + strspn(&z[j+1], jsonSpaces); + j += 1 + (u32)strspn(&z[j+1], jsonSpaces); if( z[j]==',' ){ continue; }else if( z[j]=='}' ){ break; } } - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1365,7 +1579,7 @@ json_parse_restart: return -1; } for(j=i+1;;j++){ - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; @@ -1382,14 +1596,14 @@ json_parse_restart: break; }else{ if( jsonIsspace(z[j]) ){ - j += 1 + strspn(&z[j+1], jsonSpaces); + j += 1 + (u32)strspn(&z[j+1], jsonSpaces); if( z[j]==',' ){ continue; }else if( z[j]==']' ){ break; } } - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1456,6 +1670,8 @@ json_parse_restart: /* Control characters are not allowed in strings */ pParse->iErr = j; return -1; + }else if( c=='"' ){ + opcode = JSONB_TEXT5; } j++; } @@ -1643,7 +1859,7 @@ json_parse_restart: case 0x0a: case 0x0d: case 0x20: { - i += 1 + strspn(&z[i+1], jsonSpaces); + i += 1 + (u32)strspn(&z[i+1], jsonSpaces); goto json_parse_restart; } case 0x0b: @@ -1710,10 +1926,15 @@ static int jsonConvertTextToBlob( ){ int i; const char *zJson = pParse->zJson; - i = jsonXlateTextToBlob(pParse, 0); + i = jsonTranslateTextToBlob(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ +#ifdef SQLITE_DEBUG assert( pParse->iDepth==0 ); + if( sqlite3Config.bJsonSelfcheck ){ + assert( jsonbValidityCheck(pParse, 0, pParse->nBlob, 0)==0 ); + } +#endif while( jsonIsspace(zJson[i]) ) i++; if( zJson[i] ){ i += json5Whitespace(&zJson[i]); @@ -1750,7 +1971,7 @@ static void jsonReturnStringAsBlob(JsonString *pStr){ jsonStringTerminate(pStr); px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; - (void)jsonXlateTextToBlob(&px, 0); + (void)jsonTranslateTextToBlob(&px, 0); if( px.oom ){ sqlite3_free(px.aBlob); sqlite3_result_error_nomem(pStr->pCtx); @@ -1797,7 +2018,7 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ *pSz = 0; return 0; } - sz = (pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + + sz = ((u32)pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; n = 5; }else{ @@ -1838,7 +2059,7 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ ** ** The pOut->eErr JSTRING_OOM flag is set on a OOM. */ -static u32 jsonXlateBlobToText( +static u32 jsonTranslateBlobToText( const JsonParse *pParse, /* the complete parse of the JSON */ u32 i, /* Start rendering at this index */ JsonString *pOut /* Write JSON here */ @@ -1910,13 +2131,13 @@ static u32 jsonXlateBlobToText( } break; } + case JSONB_TEXT: case JSONB_TEXTJ: { jsonAppendChar(pOut, '"'); jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); jsonAppendChar(pOut, '"'); break; } - case JSONB_TEXT: case JSONB_TEXT5: { const char *zIn; u32 k; @@ -2009,7 +2230,7 @@ static u32 jsonXlateBlobToText( j = i+n; iEnd = j+sz; while( j0 ) pOut->nUsed--; @@ -2022,7 +2243,7 @@ static u32 jsonXlateBlobToText( j = i+n; iEnd = j+sz; while( jeErr |= JSTRING_MALFORMED; @@ -2175,19 +2396,23 @@ static u32 jsonBytesToBypass(const char *z, u32 n){ ** Input z[0..n] defines JSON escape sequence including the leading '\\'. ** Decode that escape sequence into a single character. Write that ** character into *piOut. Return the number of bytes in the escape sequence. +** +** If there is a syntax error of some kind (for example too few characters +** after the '\\' to complete the encoding) then *piOut is set to +** JSON_INVALID_CHAR. */ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ assert( n>0 ); assert( z[0]=='\\' ); if( n<2 ){ - *piOut = 0xFFFD; + *piOut = JSON_INVALID_CHAR; return n; } switch( (u8)z[1] ){ case 'u': { u32 v, vlo; if( n<6 ){ - *piOut = 0xFFFD; + *piOut = JSON_INVALID_CHAR; return n; } v = jsonHexToInt4(&z[2]); @@ -2217,7 +2442,7 @@ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ case '\\':{ *piOut = z[1]; return 2; } case 'x': { if( n<4 ){ - *piOut = 0xFFFD; + *piOut = JSON_INVALID_CHAR; return n; } *piOut = (jsonHexToInt(z[2])<<4) | jsonHexToInt(z[3]); @@ -2228,7 +2453,7 @@ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ case '\n': { u32 nSkip = jsonBytesToBypass(z, n); if( nSkip==0 ){ - *piOut = 0xFFFD; + *piOut = JSON_INVALID_CHAR; return n; }else if( nSkip==n ){ *piOut = 0; @@ -2236,12 +2461,12 @@ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ }else if( z[nSkip]=='\\' ){ return nSkip + jsonUnescapeOneChar(&z[nSkip], n-nSkip, piOut); }else{ - *piOut = z[nSkip]; - return nSkip+1; + int sz = sqlite3Utf8ReadLimited((u8*)&z[nSkip], n-nSkip, piOut); + return nSkip + sz; } } default: { - *piOut = 0xFFFD; + *piOut = JSON_INVALID_CHAR; return 2; } } @@ -2265,21 +2490,37 @@ static SQLITE_NOINLINE int jsonLabelCompareEscaped( ){ u32 cLeft, cRight; assert( rawLeft==0 || rawRight==0 ); - while( nLeft>0 && nRight>0 ){ - if( rawLeft || zLeft[0]!='\\' ){ + while( 1 /*exit-by-return*/ ){ + if( nLeft==0 ){ + cLeft = 0; + }else if( rawLeft || zLeft[0]!='\\' ){ cLeft = ((u8*)zLeft)[0]; - zLeft++; - nLeft--; + if( cLeft>=0xc0 ){ + int sz = sqlite3Utf8ReadLimited((u8*)zLeft, nLeft, &cLeft); + zLeft += sz; + nLeft -= sz; + }else{ + zLeft++; + nLeft--; + } }else{ u32 n = jsonUnescapeOneChar(zLeft, nLeft, &cLeft); zLeft += n; assert( n<=nLeft ); nLeft -= n; } - if( rawRight || zRight[0]!='\\' ){ + if( nRight==0 ){ + cRight = 0; + }else if( rawRight || zRight[0]!='\\' ){ cRight = ((u8*)zRight)[0]; - zRight++; - nRight--; + if( cRight>=0xc0 ){ + int sz = sqlite3Utf8ReadLimited((u8*)zRight, nRight, &cRight); + zRight += sz; + nRight -= sz; + }else{ + zRight++; + nRight--; + } }else{ u32 n = jsonUnescapeOneChar(zRight, nRight, &cRight); zRight += n; @@ -2287,8 +2528,8 @@ static SQLITE_NOINLINE int jsonLabelCompareEscaped( nRight -= n; } if( cLeft!=cRight ) return 0; + if( cLeft==0 ) return 1; } - return nLeft==0 && nRight==0; } /* @@ -2588,7 +2829,7 @@ static void jsonReturnTextJsonFromBlob( x.aBlob = (u8*)aBlob; x.nBlob = nBlob; jsonStringInit(&s, ctx); - jsonXlateBlobToText(&x, 0, &s); + jsonTranslateBlobToText(&x, 0, &s); jsonReturnString(&s, 0, 0); } @@ -2618,14 +2859,17 @@ static void jsonReturnFromBlob( } switch( pParse->aBlob[i] & 0x0f ){ case JSONB_NULL: { + if( sz ) goto returnfromblob_malformed; sqlite3_result_null(pCtx); break; } case JSONB_TRUE: { + if( sz ) goto returnfromblob_malformed; sqlite3_result_int(pCtx, 1); break; } case JSONB_FALSE: { + if( sz ) goto returnfromblob_malformed; sqlite3_result_int(pCtx, 0); break; } @@ -2634,16 +2878,25 @@ static void jsonReturnFromBlob( sqlite3_int64 iRes = 0; char *z; int bNeg = 0; - char x = (char)pParse->aBlob[i+n]; - if( x=='-' && ALWAYS(sz>0) ){ n++; sz--; bNeg = 1; } + char x; + if( sz==0 ) goto returnfromblob_malformed; + x = (char)pParse->aBlob[i+n]; + if( x=='-' ){ + if( sz<2 ) goto returnfromblob_malformed; + n++; + sz--; + bNeg = 1; + } z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); - if( z==0 ) return; + if( z==0 ) goto returnfromblob_oom; rc = sqlite3DecOrHexToI64(z, &iRes); sqlite3DbFree(db, z); - if( rc<=1 ){ + if( rc==0 ){ sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); }else if( rc==3 && bNeg ){ sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else if( rc==1 ){ + goto returnfromblob_malformed; }else{ if( bNeg ){ n--; sz++; } goto to_double; @@ -2654,11 +2907,13 @@ static void jsonReturnFromBlob( case JSONB_FLOAT: { double r; char *z; + if( sz==0 ) goto returnfromblob_malformed; to_double: z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); - if( z==0 ) return; - sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + if( z==0 ) goto returnfromblob_oom; + rc = sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); sqlite3DbFree(db, z); + if( rc<=0 ) goto returnfromblob_malformed; sqlite3_result_double(pCtx, r); break; } @@ -2677,10 +2932,7 @@ static void jsonReturnFromBlob( u32 nOut = sz; z = (const char*)&pParse->aBlob[i+n]; zOut = sqlite3_malloc( nOut+1 ); - if( zOut==0 ){ - sqlite3_result_error_nomem(pCtx); - break; - } + if( zOut==0 ) goto returnfromblob_oom; for(iIn=iOut=0; iIn=2 ); zOut[iOut++] = (char)(0xc0 | (v>>6)); zOut[iOut++] = 0x80 | (v&0x3f); }else if( v<0x10000 ){ + assert( szEscape>=3 ); zOut[iOut++] = 0xe0 | (v>>12); zOut[iOut++] = 0x80 | ((v>>6)&0x3f); zOut[iOut++] = 0x80 | (v&0x3f); + }else if( v==JSON_INVALID_CHAR ){ + /* Silently ignore illegal unicode */ }else{ + assert( szEscape>=4 ); zOut[iOut++] = 0xf0 | (v>>18); zOut[iOut++] = 0x80 | ((v>>12)&0x3f); zOut[iOut++] = 0x80 | ((v>>6)&0x3f); @@ -2706,6 +2963,7 @@ static void jsonReturnFromBlob( zOut[iOut++] = c; } } /* end for() */ + assert( iOut<=nOut ); zOut[iOut] = 0; sqlite3_result_text(pCtx, zOut, iOut, sqlite3_free); break; @@ -2721,10 +2979,18 @@ static void jsonReturnFromBlob( break; } default: { - sqlite3_result_error(pCtx, "malformed JSON", -1); - break; + goto returnfromblob_malformed; } } + return; + +returnfromblob_oom: + sqlite3_result_error_nomem(pCtx); + return; + +returnfromblob_malformed: + sqlite3_result_error(pCtx, "malformed JSON", -1); + return; } /* @@ -2786,13 +3052,29 @@ static int jsonFunctionArgToBlob( } break; } - case SQLITE_FLOAT: + case SQLITE_FLOAT: { + double r = sqlite3_value_double(pArg); + if( NEVER(sqlite3IsNaN(r)) ){ + jsonBlobAppendNode(pParse, JSONB_NULL, 0, 0); + }else{ + int n = sqlite3_value_bytes(pArg); + const char *z = (const char*)sqlite3_value_text(pArg); + if( z==0 ) return 1; + if( z[0]=='I' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); + }else if( z[0]=='-' && z[1]=='I' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 6, "-9e999"); + }else{ + jsonBlobAppendNode(pParse, JSONB_FLOAT, n, z); + } + } + break; + } case SQLITE_INTEGER: { int n = sqlite3_value_bytes(pArg); const char *z = (const char*)sqlite3_value_text(pArg); - int e = eType==SQLITE_INTEGER ? JSONB_INT : JSONB_FLOAT; if( z==0 ) return 1; - jsonBlobAppendNode(pParse, e, n, z); + jsonBlobAppendNode(pParse, JSONB_INT, n, z); break; } } @@ -2894,11 +3176,6 @@ jsonInsertIntoBlob_patherror: return; } -/* -** Make a copy of a JsonParse object. The copy will be editable. -*/ - - /* ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, ** from the SQL function argument pArg. Return a pointer to the new @@ -3053,7 +3330,8 @@ static void jsonReturnParse( }else{ JsonString s; jsonStringInit(&s, ctx); - jsonXlateBlobToText(p, 0, &s); + p->delta = 0; + jsonTranslateBlobToText(p, 0, &s); jsonReturnString(&s, p, ctx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -3381,7 +3659,7 @@ static void jsonExtractFunc( if( argc==2 ){ if( flags & JSON_JSON ){ jsonStringInit(&jx, ctx); - jsonXlateBlobToText(p, j, &jx); + jsonTranslateBlobToText(p, j, &jx); jsonReturnString(&jx, 0, 0); jsonStringReset(&jx); assert( (flags & JSON_BLOB)==0 ); @@ -3396,7 +3674,7 @@ static void jsonExtractFunc( } }else{ jsonAppendSeparator(&jx); - jsonXlateBlobToText(p, j, &jx); + jsonTranslateBlobToText(p, j, &jx); } }else if( j==JSON_LOOKUP_NOTFOUND ){ if( argc==2 ){ @@ -3935,37 +4213,19 @@ static void jsonValidFunc( case SQLITE_BLOB: { if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){ if( flags & 0x04 ){ - /* Superficial checking only - accomplisehd by the + /* Superficial checking only - accomplished by the ** jsonFuncArgMightBeBinary() call above. */ res = 1; }else{ /* Strict checking. Check by translating BLOB->TEXT->BLOB. If ** no errors occur, call that a "strict check". */ JsonParse px; - JsonString sx; - u8 oom = 0; + u32 iErr; memset(&px, 0, sizeof(px)); px.aBlob = (u8*)sqlite3_value_blob(argv[0]); px.nBlob = sqlite3_value_bytes(argv[0]); - jsonStringInit(&sx, 0); - jsonXlateBlobToText(&px, 0, &sx); - jsonParseReset(&px); - if( sx.eErr & JSTRING_OOM ) oom = 1; - if( sx.eErr==0 ){ - memset(&px, 0, sizeof(px)); - px.zJson = sx.zBuf; - px.nJson = sx.nUsed; - if( jsonXlateTextToBlob(&px, 0)==px.nJson ){ - res = 1; - } - oom |= px.oom; - jsonParseReset(&px); - } - jsonStringReset(&sx); - if( oom ){ - sqlite3_result_error_nomem(ctx); - return; - } + iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); + res = iErr==0; } } break; @@ -3999,9 +4259,9 @@ static void jsonValidFunc( ** ** If the argument is NULL, return NULL ** -** If the argument is BLOB, do a fast validity check and return non-zero -** if the check fails. The returned value does not indicate where in the -** BLOB the error occurs. +** If the argument is BLOB, do a full validity check and return non-zero +** if the check fails. The return value is the approximate 1-based offset +** to the byte of the element that contains the first error. ** ** Otherwise interpret the argument is TEXT (even if it is numeric) and ** return the 1-based character position for where the parser first recognized @@ -4020,15 +4280,9 @@ static void jsonErrorFunc( UNUSED_PARAMETER(argc); memset(&s, 0, sizeof(s)); if( jsonFuncArgMightBeBinary(argv[0]) ){ - JsonString out; - jsonStringInit(&out, 0); s.aBlob = (u8*)sqlite3_value_blob(argv[0]); s.nBlob = sqlite3_value_bytes(argv[0]); - jsonXlateBlobToText(&s, 0, &out); - if( out.eErr ){ - iErrPos = (out.eErr & JSTRING_MALFORMED)!=0 ? 1 : -1; - } - jsonStringReset(&out); + iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 1); }else{ s.zJson = (char*)sqlite3_value_text(argv[0]); if( s.zJson==0 ) return; /* NULL input or OOM */ @@ -4096,7 +4350,7 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); if( isFinal ){ - sqlite3RCStrUnref(pStr->zBuf); + if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); }else{ pStr->nUsed--; } @@ -4216,7 +4470,7 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); if( isFinal ){ - sqlite3RCStrUnref(pStr->zBuf); + if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); }else{ pStr->nUsed--; } @@ -4502,23 +4756,20 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ */ static int jsonEachPathLength(JsonEachCursor *p){ u32 n = p->path.nUsed; - const char *z = p->path.zBuf; - if( p->iRowid==0 && p->bRecursive && n>1 ){ - if( z[n-1]==']' ){ - do{ - assert( n>1 ); - n--; - }while( z[n]!='[' ); - }else if( z[n-1]=='"' ){ - do{ - assert( n>1 ); - n--; - }while( z[n]!='.' || z[n+1]!='"' ); - }else{ - do{ - assert( n>1 ); - n--; - }while( z[n]!='.' ); + char *z = p->path.zBuf; + if( p->iRowid==0 && p->bRecursive && n>=2 ){ + while( n>1 ){ + n--; + if( z[n]=='[' || z[n]=='.' ){ + u32 x, sz = 0; + char cSaved = z[n]; + z[n] = 0; + assert( p->sParse.eEdit==0 ); + x = jsonLookupStep(&p->sParse, 0, z+1, 0); + z[n] = cSaved; + if( JSON_LOOKUP_ISERROR(x) ) continue; + if( x + jsonbPayloadSize(&p->sParse, x, &sz) == p->i ) break; + } } } return n; diff --git a/src/main.c b/src/main.c index 82a149fda6..41e3caa2da 100644 --- a/src/main.c +++ b/src/main.c @@ -4687,6 +4687,28 @@ int sqlite3_test_control(int op, ...){ break; } #endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &onOff); + ** + ** Activate or deactivate validation of JSONB that is generated from + ** text. Off by default, as the validation is slow. Validation is + ** only available if compiled using SQLITE_DEBUG. + ** + ** If onOff is initially 1, then turn it on. If onOff is initially + ** off, turn it off. If onOff is initially -1, then change onOff + ** to be the current setting. + */ + case SQLITE_TESTCTRL_JSON_SELFCHECK: { +#if defined(SQLITE_DEBUG) + int *pOnOff = va_arg(ap, int*); + if( *pOnOff<0 ){ + *pOnOff = sqlite3Config.bJsonSelfcheck; + }else{ + sqlite3Config.bJsonSelfcheck = (u8)((*pOnOff)&0xff); + } +#endif + break; + } } va_end(ap); #endif /* SQLITE_UNTESTABLE */ diff --git a/src/os_unix.c b/src/os_unix.c index 7362a13206..21bbd97694 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4434,9 +4434,15 @@ static int unixShmSystemLock( pShmNode = pFile->pInode->pShmNode; - /* Assert that the correct mutex or mutexes are held. */ - if( pShmNode->nRef==0 ){ - assert( ofst==UNIX_SHM_DMS && n==1 && unixMutexHeld() ); + /* Assert that the parameters are within expected range and that the + ** correct mutex or mutexes are held. */ + assert( pShmNode->nRef>=0 ); + assert( (ofst==UNIX_SHM_DMS && n==1) + || (ofst>=UNIX_SHM_BASE && ofst+n<=(UNIX_SHM_BASE+SQLITE_SHM_NLOCK)) + ); + if( ofst==UNIX_SHM_DMS ){ + assert( pShmNode->nRef>0 || unixMutexHeld() ); + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); }else{ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT int ii; diff --git a/src/pager.c b/src/pager.c index 4687ab0f19..f779d74573 100644 --- a/src/pager.c +++ b/src/pager.c @@ -688,7 +688,7 @@ struct Pager { char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ - int aStat[4]; /* Total cache hits, misses, writes, spills */ + u32 aStat[4]; /* Total cache hits, misses, writes, spills */ #ifdef SQLITE_TEST int nRead; /* Database pages read */ #endif @@ -6832,11 +6832,11 @@ int *sqlite3PagerStats(Pager *pPager){ a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; - a[6] = pPager->aStat[PAGER_STAT_HIT]; - a[7] = pPager->aStat[PAGER_STAT_MISS]; + a[6] = (int)pPager->aStat[PAGER_STAT_HIT] & 0x7fffffff; + a[7] = (int)pPager->aStat[PAGER_STAT_MISS] & 0x7fffffff; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; - a[10] = pPager->aStat[PAGER_STAT_WRITE]; + a[10] = (int)pPager->aStat[PAGER_STAT_WRITE] & 0x7fffffff; return a; } #endif @@ -6852,7 +6852,7 @@ int *sqlite3PagerStats(Pager *pPager){ ** reset parameter is non-zero, the cache hit or miss count is zeroed before ** returning. */ -void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ +void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, u64 *pnVal){ assert( eStat==SQLITE_DBSTATUS_CACHE_HIT || eStat==SQLITE_DBSTATUS_CACHE_MISS @@ -7792,7 +7792,7 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif -#ifdef SQLITE_USE_SEH +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) int sqlite3PagerWalSystemErrno(Pager *pPager){ return sqlite3WalSystemErrno(pPager->pWal); } diff --git a/src/pager.h b/src/pager.h index 044c2573e6..7ef9a237ae 100644 --- a/src/pager.h +++ b/src/pager.h @@ -216,7 +216,7 @@ sqlite3_file *sqlite3PagerJrnlFile(Pager*); const char *sqlite3PagerJournalname(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); -void sqlite3PagerCacheStat(Pager *, int, int, int *); +void sqlite3PagerCacheStat(Pager *, int, int, u64*); void sqlite3PagerClearCache(Pager*); int sqlite3SectorSize(sqlite3_file *); diff --git a/src/resolve.c b/src/resolve.c index 5f675c1d2d..b4f03fe7e6 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -182,6 +182,7 @@ Bitmask sqlite3ExprColUsed(Expr *pExpr){ assert( ExprUseYTab(pExpr) ); pExTab = pExpr->y.pTab; assert( pExTab!=0 ); + assert( n < pExTab->nCol ); if( (pExTab->tabFlags & TF_HasGenerated)!=0 && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 ){ @@ -758,6 +759,7 @@ static int lookupName( sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); pParse->checkSchema = 1; pTopNC->nNcErr++; + eNewExprOp = TK_NULL; } assert( pFJMatch==0 ); @@ -784,7 +786,7 @@ static int lookupName( ** If a generated column is referenced, set bits for every column ** of the table. */ - if( pExpr->iColumn>=0 && pMatch!=0 ){ + if( pExpr->iColumn>=0 && cnt==1 && pMatch!=0 ){ pMatch->colUsed |= sqlite3ExprColUsed(pExpr); } diff --git a/src/select.c b/src/select.c index f61b34f5fe..1215727791 100644 --- a/src/select.c +++ b/src/select.c @@ -185,7 +185,7 @@ void sqlite3SelectDelete(sqlite3 *db, Select *p){ if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1); } void sqlite3SelectDeleteGeneric(sqlite3 *db, void *p){ - if( p ) clearSelect(db, (Select*)p, 1); + if( ALWAYS(p) ) clearSelect(db, (Select*)p, 1); } /* @@ -6659,6 +6659,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ assert( pFunc->pFExpr->pLeft!=0 ); assert( pFunc->pFExpr->pLeft->op==TK_ORDER ); assert( ExprUseXList(pFunc->pFExpr->pLeft) ); + assert( pFunc->pFunc!=0 ); pOBList = pFunc->pFExpr->pLeft->x.pList; if( !pFunc->bOBUnique ){ nExtra++; /* One extra column for the OP_Sequence */ @@ -6668,6 +6669,9 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ assert( ExprUseXList(pFunc->pFExpr) ); nExtra += pFunc->pFExpr->x.pList->nExpr; } + if( pFunc->bUseSubtype ){ + nExtra += pFunc->pFExpr->x.pList->nExpr; + } pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOBList, 0, nExtra); if( !pFunc->bOBUnique && pParse->nErr==0 ){ pKeyInfo->nKeyField++; @@ -6694,16 +6698,17 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; if( pF->iOBTab>=0 ){ - /* For an ORDER BY aggregate, calls to OP_AggStep where deferred and - ** all content was stored in emphermal table pF->iOBTab. Extract that - ** content now (in ORDER BY order) and make all calls to OP_AggStep + /* For an ORDER BY aggregate, calls to OP_AggStep were deferred. Inputs + ** were stored in emphermal table pF->iOBTab. Here, we extract those + ** inputs (in ORDER BY order) and make all calls to OP_AggStep ** before doing the OP_AggFinal call. */ int iTop; /* Start of loop for extracting columns */ int nArg; /* Number of columns to extract */ int nKey; /* Key columns to be skipped */ int regAgg; /* Extract into this array */ int j; /* Loop counter */ - + + assert( pF->pFunc!=0 ); nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); @@ -6720,6 +6725,15 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ for(j=nArg-1; j>=0; j--){ sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, nKey+j, regAgg+j); } + if( pF->bUseSubtype ){ + int regSubtype = sqlite3GetTempReg(pParse); + int iBaseCol = nKey + nArg + (pF->bOBPayload==0 && pF->bOBUnique==0); + for(j=nArg-1; j>=0; j--){ + sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, iBaseCol+j, regSubtype); + sqlite3VdbeAddOp2(v, OP_SetSubtype, regSubtype, regAgg+j); + } + sqlite3ReleaseTempReg(pParse, regSubtype); + } sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); @@ -6774,6 +6788,7 @@ static void updateAccumulator( ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); assert( !IsWindowFunc(pF->pFExpr) ); + assert( pF->pFunc!=0 ); pList = pF->pFExpr->x.pList; if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){ Expr *pFilter = pF->pFExpr->y.pWin->pFilter; @@ -6818,6 +6833,9 @@ static void updateAccumulator( if( pF->bOBPayload ){ regAggSz += nArg; } + if( pF->bUseSubtype ){ + regAggSz += nArg; + } regAggSz++; /* One extra register to hold result of MakeRecord */ regAgg = sqlite3GetTempRange(pParse, regAggSz); regDistinct = regAgg; @@ -6830,6 +6848,14 @@ static void updateAccumulator( if( pF->bOBPayload ){ regDistinct = regAgg+jj; sqlite3ExprCodeExprList(pParse, pList, regDistinct, 0, SQLITE_ECEL_DUP); + jj += nArg; + } + if( pF->bUseSubtype ){ + int kk; + int regBase = pF->bOBPayload ? regDistinct : regAgg; + for(kk=0; kknExpr; @@ -7109,7 +7135,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ pSub->selFlags |= SF_Aggregate; pSub->selFlags &= ~SF_Compound; pSub->nSelectRow = 0; - sqlite3ExprListDelete(db, pSub->pEList); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pSub->pEList); pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount; pSub->pEList = sqlite3ExprListAppend(pParse, 0, pTerm); pTerm = sqlite3PExpr(pParse, TK_SELECT, 0, 0); diff --git a/src/shell.c.in b/src/shell.c.in index 5290a18a87..0530a572b7 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -736,14 +736,14 @@ static int strlenChar(const char *z){ */ static FILE * openChrSource(const char *zFile){ #if defined(_WIN32) || defined(WIN32) - struct _stat x = {0}; + struct __stat64 x = {0}; # define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) /* On Windows, open first, then check the stream nature. This order ** is necessary because _stat() and sibs, when checking a named pipe, ** effectively break the pipe as its supplier sees it. */ FILE *rv = fopen(zFile, "rb"); if( rv==0 ) return 0; - if( _fstat(_fileno(rv), &x) != 0 + if( _fstat64(_fileno(rv), &x) != 0 || !STAT_CHR_SRC(x.st_mode)){ fclose(rv); rv = 0; @@ -5331,7 +5331,6 @@ static void open_db(ShellState *p, int openFlags){ break; } } - globalDb = p->db; if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ eputf("Error: unable to open database \"%s\": %s\n", zDbFilename, sqlite3_errmsg(p->db)); @@ -5348,6 +5347,7 @@ static void open_db(ShellState *p, int openFlags){ zDbFilename); } } + globalDb = p->db; sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0); /* Reflect the use or absence of --unsafe-testing invocation. */ @@ -11044,6 +11044,7 @@ static int do_meta_command(char *zLine, ShellState *p){ {"fk_no_action", SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN" }, {"imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"}, {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" }, + {"json_selfcheck", SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN" }, {"localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN" }, {"never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN" }, {"optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK" }, @@ -11262,6 +11263,16 @@ static int do_meta_command(char *zLine, ShellState *p){ isOk = 3; } break; + case SQLITE_TESTCTRL_JSON_SELFCHECK: + if( nArg==2 ){ + rc2 = -1; + isOk = 1; + }else{ + rc2 = booleanValue(azArg[2]); + isOk = 3; + } + sqlite3_test_control(testctrl, &rc2); + break; } } if( isOk==0 && iCtrl>=0 ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0ebd73cfbb..5d5f0d4be3 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8040,9 +8040,11 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable -** behavior.)^ +** will always return SQLITE_BUSY. In most cases the SQLite core only uses +** sqlite3_mutex_try() as an optimization, so this is acceptable +** behavior. The exceptions are unix builds that set the +** SQLITE_ENABLE_SETLK_TIMEOUT build option. In that case a working +** sqlite3_mutex_try() is required.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was ** previously entered by the same thread. The behavior @@ -8301,6 +8303,7 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ +#define SQLITE_TESTCTRL_JSON_SELFCHECK 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 35164a1f77..64e752b9cf 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2886,6 +2886,7 @@ struct AggInfo { int iOBTab; /* Ephemeral table to implement ORDER BY */ u8 bOBPayload; /* iOBTab has payload columns separate from key */ u8 bOBUnique; /* Enforce uniqueness on iOBTab keys */ + u8 bUseSubtype; /* Transfer subtype info through sorter */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ @@ -4201,6 +4202,9 @@ struct Sqlite3Config { u8 bSmallMalloc; /* Avoid large memory allocations if true */ u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ u8 bUseLongDouble; /* Make use of long double */ +#ifdef SQLITE_DEBUG + u8 bJsonSelfcheck; /* Double-check JSON parsing */ +#endif int mxStrlen; /* Maximum string length */ int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ @@ -5192,6 +5196,7 @@ int sqlite3Utf16ByteLen(const void *pData, int nChar); #endif int sqlite3Utf8CharLen(const char *pData, int nByte); u32 sqlite3Utf8Read(const u8**); +int sqlite3Utf8ReadLimited(const u8*, int, u32*); LogEst sqlite3LogEst(u64); LogEst sqlite3LogEstAdd(LogEst,LogEst); LogEst sqlite3LogEstFromDouble(double); diff --git a/src/status.c b/src/status.c index b253d43d5d..d658da4cb4 100644 --- a/src/status.c +++ b/src/status.c @@ -382,7 +382,7 @@ int sqlite3_db_status( case SQLITE_DBSTATUS_CACHE_MISS: case SQLITE_DBSTATUS_CACHE_WRITE:{ int i; - int nRet = 0; + u64 nRet = 0; assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); @@ -395,7 +395,7 @@ int sqlite3_db_status( *pHighwater = 0; /* IMP: R-42420-56072 */ /* IMP: R-54100-20147 */ /* IMP: R-29431-39229 */ - *pCurrent = nRet; + *pCurrent = (int)nRet & 0x7fffffff; break; } diff --git a/src/test1.c b/src/test1.c index 55be0596b0..9c28259b41 100644 --- a/src/test1.c +++ b/src/test1.c @@ -8120,6 +8120,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd( extern int sqlite3_prefixes_init(sqlite3*,char**,const sqlite3_api_routines*); #endif extern int sqlite3_qpvtab_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_randomjson_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_remember_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_series_init(sqlite3*,char**,const sqlite3_api_routines*); @@ -8152,6 +8153,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd( { "prefixes", sqlite3_prefixes_init }, #endif { "qpvtab", sqlite3_qpvtab_init }, + { "randomjson", sqlite3_randomjson_init }, { "regexp", sqlite3_regexp_init }, { "remember", sqlite3_remember_init }, { "series", sqlite3_series_init }, diff --git a/src/test_func.c b/src/test_func.c index fa52438261..80df488282 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -694,7 +694,8 @@ static int registerTestFunctions( { "test_extract", 2, SQLITE_UTF8, test_extract}, { "test_zeroblob", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, test_zeroblob}, { "test_getsubtype", 1, SQLITE_UTF8, test_getsubtype}, - { "test_setsubtype", 2, SQLITE_UTF8, test_setsubtype}, + { "test_setsubtype", 2, SQLITE_UTF8|SQLITE_RESULT_SUBTYPE, + test_setsubtype}, { "test_frombind", -1, SQLITE_UTF8, test_frombind}, }; int i; diff --git a/src/utf.c b/src/utf.c index 5f27babdfc..216864f5c7 100644 --- a/src/utf.c +++ b/src/utf.c @@ -164,7 +164,38 @@ u32 sqlite3Utf8Read( return c; } - +/* +** Read a single UTF8 character out of buffer z[], but reading no +** more than n characters from the buffer. z[] is not zero-terminated. +** +** Return the number of bytes used to construct the character. +** +** Invalid UTF8 might generate a strange result. No effort is made +** to detect invalid UTF8. +** +** At most 4 bytes will be read out of z[]. The return value will always +** be between 1 and 4. +*/ +int sqlite3Utf8ReadLimited( + const u8 *z, + int n, + u32 *piOut +){ + u32 c; + int i = 1; + assert( n>0 ); + c = z[0]; + if( c>=0xc0 ){ + c = sqlite3Utf8Trans1[c-0xc0]; + if( n>4 ) n = 4; + while( ip1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Subtype ){ + sqlite3VdbeMemSetInt64(pOut, pIn1->eSubtype); + }else{ + sqlite3VdbeMemSetNull(pOut); + } + break; +} + +/* Opcode: SetSubtype P1 P2 * * * +** Synopsis: r[P2].subtype = r[P1] +** +** Set the subtype value of register P2 to the integer from register P1. +** If P1 is NULL, clear the subtype from p2. +*/ +case OP_SetSubtype: { /* in1 out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Null ){ + pOut->flags &= ~MEM_Subtype; + }else{ + assert( pIn1->flags & MEM_Int ); + pOut->flags |= MEM_Subtype; + pOut->eSubtype = (u8)(pIn1->u.i & 0xff); + } + break; +} + /* Opcode: FilterAdd P1 * P3 P4 * ** Synopsis: filter(P1) += key(P3@P4) ** diff --git a/src/vtab.c b/src/vtab.c index 60d8f2e523..9fd6010126 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -333,7 +333,6 @@ void sqlite3VtabUnlockList(sqlite3 *db){ if( p ){ db->pDisconnect = 0; - sqlite3ExpirePreparedStatements(db, 0); do { VTable *pNext = p->pNext; sqlite3VtabUnlock(p); diff --git a/src/where.c b/src/where.c index 8885f0a583..7c49fe2723 100644 --- a/src/where.c +++ b/src/where.c @@ -2020,7 +2020,8 @@ static int whereRangeScanEst( ** sample, then assume they are 4x more selective. This brings ** the estimated selectivity more in line with what it would be ** if estimated without the use of STAT4 tables. */ - if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); + if( iLwrIdx==iUprIdx ){ nNew -= 20; } + assert( 20==sqlite3LogEst(4) ); }else{ nNew = 10; assert( 10==sqlite3LogEst(2) ); } @@ -2244,17 +2245,34 @@ void sqlite3WhereClausePrint(WhereClause *pWC){ #ifdef WHERETRACE_ENABLED /* ** Print a WhereLoop object for debugging purposes +** +** Format example: +** +** .--- Position in WHERE clause rSetup, rRun, nOut ---. +** | | +** | .--- selfMask nTerm ------. | +** | | | | +** | | .-- prereq Idx wsFlags----. | | +** | | | Name | | | +** | | | __|__ nEq ---. ___|__ | __|__ +** | / \ / \ / \ | / \ / \ / \ +** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31 */ -void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ - WhereInfo *pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+3)/4; - SrcItem *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pTab; - Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; - sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, - p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); - sqlite3DebugPrintf(" %12s", - pItem->zAlias ? pItem->zAlias : pTab->zName); +void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ + if( pWC ){ + WhereInfo *pWInfo = pWC->pWInfo; + int nb = 1+(pWInfo->pTabList->nSrc+3)/4; + SrcItem *pItem = pWInfo->pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); + sqlite3DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + }else{ + sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d", + p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab); + } if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ const char *zName; if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ @@ -2291,6 +2309,15 @@ void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ } } } +void sqlite3ShowWhereLoop(const WhereLoop *p){ + if( p ) sqlite3WhereLoopPrint(p, 0); +} +void sqlite3ShowWhereLoopList(const WhereLoop *p){ + while( p ){ + sqlite3ShowWhereLoop(p); + p = p->pNextLoop; + } +} #endif /* @@ -2403,46 +2430,60 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Return TRUE if all of the following are true: +** Return TRUE if X is a proper subset of Y but is of equal or less cost. +** In other words, return true if all constraints of X are also part of Y +** and Y has additional constraints that might speed the search that X lacks +** but the cost of running X is not more than the cost of running Y. ** -** (1) X has the same or lower cost, or returns the same or fewer rows, -** than Y. -** (2) X uses fewer WHERE clause terms than Y -** (3) Every WHERE clause term used by X is also used by Y -** (4) X skips at least as many columns as Y -** (5) If X is a covering index, than Y is too +** In other words, return true if the cost relationwship between X and Y +** is inverted and needs to be adjusted. ** -** Conditions (2) and (3) mean that X is a "proper subset" of Y. -** If X is a proper subset of Y then Y is a better choice and ought -** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. Constraint (4) -** was added because if X uses skip-scan less than Y it still might -** deserve a lower cost even if it is a proper subset of Y. Constraint (5) -** was added because a covering index probably deserves to have a lower cost -** than a non-covering index even if it is a proper subset. +** Case 1: +** +** (1a) X and Y use the same index. +** (1b) X has fewer == terms than Y +** (1c) Neither X nor Y use skip-scan +** (1d) X does not have a a greater cost than Y +** +** Case 2: +** +** (2a) X has the same or lower cost, or returns the same or fewer rows, +** than Y. +** (2b) X uses fewer WHERE clause terms than Y +** (2c) Every WHERE clause term used by X is also used by Y +** (2d) X skips at least as many columns as Y +** (2e) If X is a covering index, than Y is too */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; - if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ - return 0; /* X is not a subset of Y */ + if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; /* (1d) and (2a) */ + assert( (pX->wsFlags & WHERE_VIRTUALTABLE)==0 ); + assert( (pY->wsFlags & WHERE_VIRTUALTABLE)==0 ); + if( pX->u.btree.nEq < pY->u.btree.nEq /* (1b) */ + && pX->u.btree.pIndex==pY->u.btree.pIndex /* (1a) */ + && pX->nSkip==0 && pY->nSkip==0 /* (1c) */ + ){ + return 1; /* Case 1 is true */ } - if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; - if( pY->nSkip > pX->nSkip ) return 0; + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* (2b) */ + } + if( pY->nSkip > pX->nSkip ) return 0; /* (2d) */ for(i=pX->nLTerm-1; i>=0; i--){ if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } - if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + if( j<0 ) return 0; /* (2c) */ } if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){ - return 0; /* Constraint (5) */ + return 0; /* (2e) */ } - return 1; /* All conditions meet */ + return 1; /* Case 2 is true */ } /* @@ -6037,7 +6078,10 @@ WhereInfo *sqlite3WhereBegin( ** field (type Bitmask) it must be aligned on an 8-byte boundary on ** some architectures. Hence the ROUND8() below. */ - nByteWInfo = ROUND8P(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); + nByteWInfo = ROUND8P(sizeof(WhereInfo)); + if( nTabList>1 ){ + nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel)); + } pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); @@ -6599,6 +6643,11 @@ whereBeginError: pParse->nQueryLoop = pWInfo->savedNQueryLoop; whereInfoFree(db, pWInfo); } +#ifdef WHERETRACE_ENABLED + /* Prevent harmless compiler warnings about debugging routines + ** being declared but never used */ + sqlite3ShowWhereLoopList(0); +#endif /* WHERETRACE_ENABLED */ return 0; } diff --git a/src/whereInt.h b/src/whereInt.h index 343aca9f36..f3cc5776a0 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -502,7 +502,7 @@ Bitmask sqlite3WhereGetMask(WhereMaskSet*,int); #ifdef WHERETRACE_ENABLED void sqlite3WhereClausePrint(WhereClause *pWC); void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm); -void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC); +void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC); #endif WhereTerm *sqlite3WhereFindTerm( WhereClause *pWC, /* The WHERE clause to be searched */ diff --git a/test/aggnested.test b/test/aggnested.test index ad6e208ddf..f3539076bd 100644 --- a/test/aggnested.test +++ b/test/aggnested.test @@ -472,6 +472,16 @@ do_execsql_test 9.5 { FROM t1; } {6} - +# 2023-12-16 +# New test case for check-in [4470f657d2069972] from 2023-11-02 +# https://bugs.chromium.org/p/chromium/issues/detail?id=1511689 +# +do_execsql_test 10.1 { + DROP TABLE IF EXISTS t0; + DROP TABLE IF EXISTS t1; + CREATE TABLE t0(c1, c2); INSERT INTO t0 VALUES(1,2); + CREATE TABLE t1(c3, c4); INSERT INTO t1 VALUES(3,4); + SELECT * FROM t0 WHERE EXISTS (SELECT 1 FROM t1 GROUP BY c3 HAVING ( SELECT count(*) FROM (SELECT 1 UNION ALL SELECT sum(DISTINCT c1) ) ) ) BETWEEN 1 AND 1; +} {1 2} finish_test diff --git a/test/aggorderby.test b/test/aggorderby.test index f94c5ec898..eed1f83a7e 100644 --- a/test/aggorderby.test +++ b/test/aggorderby.test @@ -118,5 +118,45 @@ do_execsql_test aggorderby-8.2 { SELECT sum(DISTINCT x ORDER BY y) FROM c; } 6 +# Subtype information is transfered through the sorter for aggregates +# that make use of subtype info. +# +do_execsql_test aggorderby-9.0 { + WITH c(x,y) AS (VALUES + ('{a:3}', 3), + ('[1,1]', 1), + ('[4,4]', 4), + ('{x:2}', 2)) + SELECT json_group_array(json(x) ORDER BY y) FROM c; +} {{[[1,1],{"x":2},{"a":3},[4,4]]}} +do_execsql_test aggorderby-9.1 { + WITH c(x,y) AS (VALUES + ('[4,4]', 4), + ('{a:3}', 3), + ('[4,4]', 4), + ('[1,1]', 1), + ('[4,4]', 4), + ('{x:2}', 2)) + SELECT json_group_array(DISTINCT json(x) ORDER BY y) FROM c; +} {{[[1,1],{"x":2},{"a":3},[4,4]]}} +do_execsql_test aggorderby-9.2 { + WITH c(x,y) AS (VALUES + ('{a:3}', 3), + ('[1,1]', 1), + ('[4,4]', 4), + ('{x:2}', 2)) + SELECT json_group_array(json(x) ORDER BY json(x)) FROM c; +} {{[[1,1],[4,4],{"a":3},{"x":2}]}} +do_execsql_test aggorderby-9.3 { + WITH c(x,y) AS (VALUES + ('[4,4]', 4), + ('{a:3}', 3), + ('[4,4]', 4), + ('[1,1]', 1), + ('[4,4]', 4), + ('{x:2}', 2)) + SELECT json_group_array(DISTINCT json(x) ORDER BY json(x)) FROM c; +} {{[[1,1],[4,4],{"a":3},{"x":2}]}} + finish_test diff --git a/test/fts3integrity.test b/test/fts3integrity.test new file mode 100644 index 0000000000..bcbc49dc33 --- /dev/null +++ b/test/fts3integrity.test @@ -0,0 +1,42 @@ +# 2023 December 16 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file runs all tests. +# +# $Id: fts3.test,v 1.2 2008/07/23 18:17:32 drh Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix fts3integrity + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts3(x); + INSERT INTO t1 VALUES('first row'); + INSERT INTO t1 VALUES('second row'); + + CREATE TABLE t2(x PRIMARY KEY); + INSERT INTO t2 VALUES('first row'); + INSERT INTO t2 VALUES('second row'); +} + +sqlite3 db2 test.db + +do_execsql_test -db db2 1.1 { + CREATE TABLE t3(x, y); +} + +do_execsql_test 1.2 { + PRAGMA integrity_check; +} {ok} + +finish_test diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 23200a5f03..dd49120115 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -159,9 +159,10 @@ static struct GlobalVars { } g; /* -** Include the external vt02.c module. +** Include the external vt02.c and randomjson.c modules. */ extern int sqlite3_vt02_init(sqlite3*,char***,void*); +extern int sqlite3_randomjson_init(sqlite3*,char***,void*); /* @@ -1267,6 +1268,8 @@ int runCombinedDbSqlInput( } sqlite3_limit(cx.db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 100); sqlite3_hard_heap_limit64(heapLimit); + rc = 1; + sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &rc); if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){ aDb[18] = aDb[19] = 1; @@ -1295,6 +1298,9 @@ int runCombinedDbSqlInput( /* Add the vt02 virtual table */ sqlite3_vt02_init(cx.db, 0, 0); + /* Add the random_json() and random_json5() functions */ + sqlite3_randomjson_init(cx.db, 0, 0); + /* Add support for sqlite_dbdata and sqlite_dbptr virtual tables used ** by the recovery API */ sqlite3_dbdata_init(cx.db, 0, 0); diff --git a/test/json105.test b/test/json105.test index 5592191c22..509db94e11 100644 --- a/test/json105.test +++ b/test/json105.test @@ -13,7 +13,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix json104 +set testprefix json105 # This is the example from pages 2 and 3 of RFC-7396 db eval { diff --git a/test/json106.test b/test/json106.test new file mode 100644 index 0000000000..f52d2b6386 --- /dev/null +++ b/test/json106.test @@ -0,0 +1,70 @@ +# 2023-12-18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Invariant tests for JSON built around the randomjson extension +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json106 + +load_static_extension db randomjson +db eval { + CREATE TEMP TABLE t1(j0,j5,p); + CREATE TEMP TABLE kv(n,key,val); +} +unset -nocomplain ii +for {set ii 1} {$ii<=5000} {incr ii} { + do_execsql_test $ii.1 { + DELETE FROM t1; + INSERT INTO t1(j0,j5) VALUES(random_json($ii),random_json5($ii)); + SELECT json_valid(j0), json_valid(j5,2) FROM t1; + } {1 1} + do_execsql_test $ii.2 { + SELECT count(*) + FROM t1, json_tree(j0) AS rt + WHERE rt.type NOT IN ('object','array') + AND rt.atom IS NOT (j0 ->> rt.fullkey); + } 0 + do_execsql_test $ii.3 { + SELECT count(*) + FROM t1, json_tree(j5) AS rt + WHERE rt.type NOT IN ('object','array') + AND rt.atom IS NOT (j0 ->> rt.fullkey); + } 0 + do_execsql_test $ii.4 { + DELETE FROM kv; + INSERT INTO kv + SELECT rt.rowid, rt.fullkey, rt.atom + FROM t1, json_tree(j0) AS rt + WHERE rt.type NOT IN ('object','array'); + } + do_execsql_test $ii.5 { + SELECT count(*) + FROM t1, kv + WHERE key NOT LIKE '%]' + AND json_remove(j5,key)->>key IS NOT NULL + } 0 + do_execsql_test $ii.6 { + SELECT count(*) + FROM t1, kv + WHERE key NOT LIKE '%]' + AND json_insert(json_remove(j5,key),key,val)->>key IS NOT val + } 0 + do_execsql_test $ii.7 { + UPDATE t1 SET p=json_patch(j0,j5); + SELECT count(*) + FROM t1, kv + WHERE p->>key IS NOT val + } 0 +} + + +finish_test diff --git a/test/json502.test b/test/json502.test index 48c372c4f8..cc14b8cb71 100644 --- a/test/json502.test +++ b/test/json502.test @@ -47,7 +47,7 @@ do_execsql_test 3.2 { db null null do_execsql_test 3.3 { - DROP TABLE t1; + DROP TABLE IF EXISTS t1; CREATE TABLE t1(x); INSERT INTO t1 VALUES(json_insert('{}','$.a\',111,'$."b\\"',222)); INSERT INTO t1 VALUES(jsonb_insert('{}','$.a\',111,'$."b\\"',222)); @@ -58,8 +58,10 @@ do_execsql_test 3.4 { SELECT json_patch('{"a\x62c":123}','{"ab\x63":456}') ->> 'abc'; } 456 -do_execsql_test 4.1 { - SELECT * FROM json_tree('{"\u0017":1}','$."\x17"'); -} {{\x17} 1 integer 1 1 null {$."\x17"} {$}} +ifcapable vtab { + do_execsql_test 4.1 { + SELECT * FROM json_tree('{"\u0017":1}','$."\x17"'); + } {{\x17} 1 integer 1 1 null {$."\x17"} {$}} +} finish_test diff --git a/test/testrunner.tcl b/test/testrunner.tcl index edefdcd156..0c704daf21 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -58,6 +58,8 @@ Usage: $a0 status where SWITCHES are: + --buildonly + --dryrun --jobs NUMBER-OF-JOBS --zipvfs ZIPVFS-SOURCE-DIR diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index c4e24c4382..984c6d8272 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -30,7 +30,7 @@ namespace eval trd { set tcltest(osx.Locking-Style) veryquick set tcltest(osx.Have-Not) veryquick - set tcltest(osx.Apple) all + set tcltest(osx.Apple) all_less_no_mutex_try set tcltest(win.Stdcall) veryquick set tcltest(win.Have-Not) veryquick @@ -100,11 +100,11 @@ namespace eval trd { } set build(All-Sanitize) { -DSQLITE_OMIT_LOOKASIDE=1 - --enable-all -fsanitize=address,undefined + --enable-all -fsanitize=address,undefined -fno-sanitize-recover=undefined } set build(Sanitize) { - CC=clang -fsanitize=address,undefined + CC=clang -fsanitize=address,undefined -fno-sanitize-recover=undefined -DSQLITE_ENABLE_STAT4 -DSQLITE_OMIT_LOOKASIDE=1 -DCONFIG_SLOWDOWN_FACTOR=5.0 @@ -367,6 +367,9 @@ proc trd_configs {platform bld} { set clist $all_configs } elseif {$clist=="all_plus_autovacuum_crash"} { set clist [concat $all_configs autovacuum_crash] + } elseif {$clist=="all_less_no_mutex_try"} { + set idx [lsearch $all_configs no_mutex_try] + set clist [lreplace $all_configs $idx $idx] } } diff --git a/test/unionall.test b/test/unionall.test index 32fc76543a..1c580d2912 100644 --- a/test/unionall.test +++ b/test/unionall.test @@ -346,6 +346,11 @@ do_execsql_test 5.10 { do_execsql_test 5.20 { SELECT *, '+' FROM t1 LEFT JOIN t3 ON (a NOT IN(SELECT v FROM t1 LEFT JOIN t2 ON (a=k))=k); } {0 {} {} {} + 1 one {} {} + 2 two {} {} + 5 five {} {} + 3 three {} {} + 6 six {} {} +} +do_catchsql_test 5.30 { + SELECT * FROM (t1 NATURAL JOIN pragma_table_xinfo('t1_a') NATURAL JOIN t3) t1 + NATURAL JOIN t2 NATURAL JOIN t3 + WHERE rowid ISNULL>0 AND 0%y; +} {1 {no such column: rowid}} reset_db do_execsql_test 6.0 { diff --git a/test/where3.test b/test/where3.test index 20af995cfe..f574c0d55f 100644 --- a/test/where3.test +++ b/test/where3.test @@ -492,5 +492,27 @@ foreach disabled_opt {none omit-noop-join all} { } {123} } +# 2023-12-23 +# https://sqlite.org/forum/forumpost/2568d1f6e6 +# +# Index usage should be "x=? and y=?" - equality on both values. +# Not: "x=? AND y>?" - inequality on "y" +# +reset_db +do_execsql_test where3-8.1 { + CREATE TABLE t1(a,b,c,d); INSERT INTO t1 VALUES(1,2,3,4); + CREATE TABLE t2(x,y); INSERT INTO t2 VALUES(3,4); + CREATE INDEX t2xy ON t2(x,y); + SELECT 1 FROM t1 JOIN t2 ON x=c AND y=d WHERE d>0; +} 1 +do_eqp_test where3-8.2 { + SELECT 1 FROM t1 JOIN t2 ON x=c AND y=d WHERE d>0; +} { + QUERY PLAN + |--SCAN t1 + `--SEARCH t2 USING COVERING INDEX t2xy (x=? AND y=?) +} + + finish_test