From 20728a599e52fecad15f6e2d8658ec47f38d0225 Mon Sep 17 00:00:00 2001 From: larrybr Date: Wed, 10 Jan 2024 05:52:02 +0000 Subject: [PATCH 01/99] For CLI shell and other utilities, optionally avoid C runtime file I/O on Windows in favor of WIN32 calls, while nominally preserving FILE* API interfaces. (a WIP, awaiting build and testing) FossilOrigin-Name: 6be68be17927e72b1f1c4b93e781b7e9235876b3ad9512fe168e5b9d0b7b763a --- ext/consio/console_io.c | 84 ++++++++++++++++++++++++++++++++++++++++- ext/consio/console_io.h | 15 ++++++-- manifest | 21 ++++++----- manifest.uuid | 2 +- src/shell.c.in | 3 ++ 5 files changed, 109 insertions(+), 16 deletions(-) diff --git a/ext/consio/console_io.c b/ext/consio/console_io.c index 3acb0daa27..0630e90e16 100755 --- a/ext/consio/console_io.c +++ b/ext/consio/console_io.c @@ -593,6 +593,86 @@ oPutbUtf8(const char *cBuf, int nAccept){ # endif } +/* +** Flush the given output stream. Return non-zero for success, else 0. +*/ +#if !defined(SQLITE_CIO_NO_FLUSH) && !defined(SQLITE_CIO_NO_SETMODE) +SQLITE_INTERNAL_LINKAGE int +fFlushBuffer(FILE *pfOut){ +# if CIO_WIN_WC_XLATE && !defined(SHELL_OMIT_FIO_DUPE) + return FlushFileBuffers(handleOfFile(pfOut))? 1 : 0; +# else + return fflush(pfOut); +# endif +} +#endif + +# if CIO_WIN_WC_XLATE && !defined(SHELL_OMIT_FIO_DUPE) +static struct FileAltIds { + int fd; + HANDLE fh; +} altIdsOfFile(FILE *pf){ + struct FileAltIds rv = { _fileno(pf) }; + union { intptr_t osfh; HANDLE fh; } fid = { + (rv.fd>=0)? _get_osfhandle(rv.fd) : (intptr_t)INVALID_HANDLE_VALUE + }; + rv.fh = fid.fh; + return rv; +} + +SQLITE_INTERNAL_LINKAGE size_t +cfWrite(const void *buf, size_t osz, size_t ocnt, FILE *pf){ + size_t rv = 0; + struct FileAltIds fai = altIdsOfFile(pf); + int fmode = _setmode(fai.fd, _O_BINARY); + _setmode(fai.fd, fmode); + while( rv < ocnt ){ + size_t nbo = osz; + while( nbo > 0 ){ + DWORD dwno = (nbo>(1L<<24))? 1L<<24 : (DWORD)nbo; + BOOL wrc = TRUE; + BOOL genCR = (fmode & _O_TEXT)!=0; + if( genCR ){ + const char *pnl = (const char*)memchr(buf, '\n', nbo); + if( pnl ) nbo = pnl - (const char*)buf; + else genCR = 0; + } + if( dwno>0 ) wrc = WriteFile(fai.hf, buf, dwno, 0,0); + if( genCR && wrc ){ + wrc = WriteFile(fai.hf, "\r\n", 2, 0,0); + ++dwno; /* Skip over the LF */ + } + if( !wrc ) return rv; + buf = (const char*)buf + dwno; + nbo += dwno; + } + ++rv; + } + return rv; +} + +SQLITE_INTERNAL_LINKAGE char * +cfGets(char *cBuf, int n, FILE *pf){ + int nci = 0; + struct FileAltIds fai = altIdsOfFile(pf); + int fmode = _setmode(fai.fd, _O_BINARY); + BOOL eatCR = (fmode & _O_TEXT)!=0; + _setmode(fai.fd, fmode); + while( nci < n-1 ){ + char cin; + DWORD nr; + if( !ReadFile(fai.hf, cBuf+nci, 1, &nr, 0) || nr==0 ) break; + if( eatCR && cin=='\r' ) continue; + cBuf[nci++] = cin; + } + if( nci < n ) cBuf[nci] = 0; + return (nci>0)? cBuf : 0; +} +# else +# define cfWrite(b,os,no,f) fwrite(b,os,no,f) +# define cfGets(b,n,f) fgets(b,n,f) +# endif + # ifdef CONSIO_EPUTB SQLITE_INTERNAL_LINKAGE int ePutbUtf8(const char *cBuf, int nAccept){ @@ -604,7 +684,7 @@ ePutbUtf8(const char *cBuf, int nAccept){ return conZstrEmit(ppst, cBuf, nAccept); }else { # endif - return (int)fwrite(cBuf, 1, nAccept, pfErr); + return (int)cfWrite(cBuf, 1, nAccept, pfErr); # if CIO_WIN_WC_XLATE } # endif @@ -670,7 +750,7 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ # endif }else{ # endif - return fgets(cBuf, ncMax, pfIn); + return cfGets(cBuf, ncMax, pfIn); # if CIO_WIN_WC_XLATE } # endif diff --git a/ext/consio/console_io.h b/ext/consio/console_io.h index 26fd7dd946..1affa15bad 100644 --- a/ext/consio/console_io.h +++ b/ext/consio/console_io.h @@ -176,12 +176,19 @@ SQLITE_INTERNAL_LINKAGE int ePutbUtf8(const char *cBuf, int nAccept); #endif +/* +** Flush the given output stream. Return non-zero for success, else 0. +*/ +#if !defined(SQLITE_CIO_NO_FLUSH) && !defined(SQLITE_CIO_NO_SETMODE) +SQLITE_INTERNAL_LINKAGE int +fFlushBuffer(FILE *pfOut); +#endif + /* ** Collect input like fgets(...) with special provisions for input -** from the console on platforms that require same. Defers to the -** C library fgets() when input is not from the console. Newline -** translation may be done as set by set{Binary,Text}Mode(). As a -** convenience, pfIn==NULL is treated as stdin. +** from the console on such platforms as require same. Newline +** translation may be done as set by set{Binary,Text}Mode(). +** As a convenience, pfIn==NULL is treated as stdin. */ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn); /* Like fGetsUtf8 except stream is always the designated input. */ diff --git a/manifest b/manifest index 467ba30c52..1c2810a843 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sresolution\sof\sunqualified\snames\sin\sthe\sREINDEX\scommand.\n[forum:/info/74cd0ceabd|Forum\sthread\s74cd0ceabd]. -D 2024-01-09T12:28:51.969 +C For\sCLI\sshell\sand\sother\sutilities,\soptionally\savoid\sC\sruntime\sfile\sI/O\son\sWindows\sin\sfavor\sof\sWIN32\scalls,\swhile\snominally\spreserving\sFILE*\sAPI\sinterfaces.\s(a\sWIP,\sawaiting\sbuild\sand\stesting) +D 2024-01-10T05:52:02.536 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -51,8 +51,8 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94 F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a -F ext/consio/console_io.c e1be639e79e54264b3ae97ca291728987a9aa82e6a4526458e6400f5e083e524 x -F ext/consio/console_io.h 0548b83d7c4b7270ad544a67f2bb90cebc519637fa39b1838df4744cf0d87646 +F ext/consio/console_io.c 23c4ce0448a93803eb93e47ed46e4ffd5efb9001d7e18230737082612b1573de x +F ext/consio/console_io.h b5ebe34aa15b357621ebbea3d3f2e2b24750d4280b5802516409e23947fd9ee5 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 F ext/expert/expert1.test 0dd5cb096d66bed593e33053a3b364f6ef52ed72064bf5cf298364636dbf3cd6 @@ -738,7 +738,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c e25f51a473a5f30a0d978e4df2aaa98aeec84eac29ecae1ad4708a6c3e669345 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c f1a81ff4f8e9e76c224e2ab3a4baa799add0db22158c7fcede65d8cc4a6fa2da -F src/shell.c.in 3d19abd924ed1cec9c9908d5a10cb1580b8ca30df24c26bfe80efa0c00f664d8 +F src/shell.c.in aff59dbd24dcee008dc64a3b0aa8b58d5fa43c5fd0fd9893a81ed6fd538ba6ac F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2157,8 +2157,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cd016f26bb61549a304f2148035e050f76a8f4a35cdb7131bba2f5fc5d09f49e -R 79992c442c5c0c9ae7a8d7762db4a7c0 -U drh -Z 0f0f16e628a0bab1003ad4c460a1aa8e +P 97709ce2a1f5ae05495e412ca27108048e5b8a63a1e3bca4be13933f7527da7b +R 1d946ec61d5c9d01acdd2c0e650e9027 +T *branch * win-dupe-crt-fio +T *sym-win-dupe-crt-fio * +T -sym-trunk * +U larrybr +Z 0b0925e801b21e9b8c7d51a5b0a1c236 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cb966c2175..cc336ddfe9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -97709ce2a1f5ae05495e412ca27108048e5b8a63a1e3bca4be13933f7527da7b \ No newline at end of file +6be68be17927e72b1f1c4b93e781b7e9235876b3ad9512fe168e5b9d0b7b763a \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 19574dc79c..33f2ea9a76 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -246,6 +246,7 @@ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); # define SQLITE_CIO_NO_CLASSIFY # define SQLITE_CIO_NO_TRANSLATE # define SQLITE_CIO_NO_SETMODE +# define SQLITE_CIO_NO_FLUSH #endif INCLUDE ../ext/consio/console_io.h INCLUDE ../ext/consio/console_io.c @@ -276,6 +277,7 @@ INCLUDE ../ext/consio/console_io.c # define eputz(z) ePutsUtf8(z) # define eputf ePrintfUtf8 # define oputb(buf,na) oPutbUtf8(buf,na) +# define fflush(s) fFlushBuffer(s); #else /* For Fiddle, all console handling and emit redirection is omitted. */ @@ -286,6 +288,7 @@ INCLUDE ../ext/consio/console_io.c # define eputz(z) fputs(z,stderr) # define eputf(fmt, ...) fprintf(stderr,fmt,__VA_ARGS__) # define oputb(buf,na) fwrite(buf,1,na,stdout) +# undef fflush #endif /* True if the timer is enabled */ From 3a39c44c21a801927f98aa3cdb2240f00e701b8a Mon Sep 17 00:00:00 2001 From: larrybr Date: Wed, 10 Jan 2024 20:13:38 +0000 Subject: [PATCH 02/99] Fix a few compiler gripes. FossilOrigin-Name: 2e74a938f771a889dd8c49648b5a125ff748835ba1925388cc6b379a0abde9ac --- ext/consio/console_io.c | 10 ++++------ manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/ext/consio/console_io.c b/ext/consio/console_io.c index 0630e90e16..8ecd97f0b2 100755 --- a/ext/consio/console_io.c +++ b/ext/consio/console_io.c @@ -637,9 +637,9 @@ cfWrite(const void *buf, size_t osz, size_t ocnt, FILE *pf){ if( pnl ) nbo = pnl - (const char*)buf; else genCR = 0; } - if( dwno>0 ) wrc = WriteFile(fai.hf, buf, dwno, 0,0); + if( dwno>0 ) wrc = WriteFile(fai.fh, buf, dwno, 0,0); if( genCR && wrc ){ - wrc = WriteFile(fai.hf, "\r\n", 2, 0,0); + wrc = WriteFile(fai.fh, "\r\n", 2, 0,0); ++dwno; /* Skip over the LF */ } if( !wrc ) return rv; @@ -659,11 +659,9 @@ cfGets(char *cBuf, int n, FILE *pf){ BOOL eatCR = (fmode & _O_TEXT)!=0; _setmode(fai.fd, fmode); while( nci < n-1 ){ - char cin; DWORD nr; - if( !ReadFile(fai.hf, cBuf+nci, 1, &nr, 0) || nr==0 ) break; - if( eatCR && cin=='\r' ) continue; - cBuf[nci++] = cin; + if( !ReadFile(fai.fh, cBuf+nci, 1, &nr, 0) || nr==0 ) break; + if( nr>0 && (!eatCR || cBuf[nci]!='\r') ) nci += nr; } if( nci < n ) cBuf[nci] = 0; return (nci>0)? cBuf : 0; diff --git a/manifest b/manifest index 1c2810a843..05fe008c4e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sCLI\sshell\sand\sother\sutilities,\soptionally\savoid\sC\sruntime\sfile\sI/O\son\sWindows\sin\sfavor\sof\sWIN32\scalls,\swhile\snominally\spreserving\sFILE*\sAPI\sinterfaces.\s(a\sWIP,\sawaiting\sbuild\sand\stesting) -D 2024-01-10T05:52:02.536 +C Fix\sa\sfew\scompiler\sgripes. +D 2024-01-10T20:13:38.259 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -51,7 +51,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94 F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a -F ext/consio/console_io.c 23c4ce0448a93803eb93e47ed46e4ffd5efb9001d7e18230737082612b1573de x +F ext/consio/console_io.c 3f635b6dd2ddbc722aeaa738d415768c9a40dedff53894975816dcbd7711145f x F ext/consio/console_io.h b5ebe34aa15b357621ebbea3d3f2e2b24750d4280b5802516409e23947fd9ee5 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 @@ -2157,11 +2157,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 97709ce2a1f5ae05495e412ca27108048e5b8a63a1e3bca4be13933f7527da7b -R 1d946ec61d5c9d01acdd2c0e650e9027 -T *branch * win-dupe-crt-fio -T *sym-win-dupe-crt-fio * -T -sym-trunk * +P 6be68be17927e72b1f1c4b93e781b7e9235876b3ad9512fe168e5b9d0b7b763a +R 2f77944a93e4bc36153f9450a52f3c42 U larrybr -Z 0b0925e801b21e9b8c7d51a5b0a1c236 +Z 5b64f41a30f93cac02d6a6331bee0468 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cc336ddfe9..9db802f7c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6be68be17927e72b1f1c4b93e781b7e9235876b3ad9512fe168e5b9d0b7b763a \ No newline at end of file +2e74a938f771a889dd8c49648b5a125ff748835ba1925388cc6b379a0abde9ac \ No newline at end of file From 2ec78c0e4bab11ce288a830c3a13fe34a9224c4a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Jul 2024 20:50:33 +0000 Subject: [PATCH 03/99] Add the fts5_locale() function, and begin adding the related functionality to fts5. FossilOrigin-Name: 8839ef7cfb49239e7f1c4812a53a93a672827c88d6921408b1d5062b352c87cc --- ext/fts5/fts5.h | 44 ++++- ext/fts5/fts5Int.h | 16 +- ext/fts5/fts5_config.c | 32 ++++ ext/fts5/fts5_main.c | 326 ++++++++++++++++++++++++++++++---- ext/fts5/fts5_storage.c | 105 +++++++---- ext/fts5/fts5_tcl.c | 99 ++++++++++- ext/fts5/test/fts5locale.test | 176 ++++++++++++++++++ manifest | 28 +-- manifest.uuid | 2 +- 9 files changed, 730 insertions(+), 98 deletions(-) create mode 100644 ext/fts5/test/fts5locale.test diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index d3042fcb8c..ccdebbe1ab 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -570,6 +570,29 @@ struct fts5_tokenizer { ); }; +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); + + int (*xSetLocale)(Fts5Tokenizer*, const char *pLocale, int nLocale); +}; + /* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 @@ -589,7 +612,7 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently 3, was once 2 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( @@ -616,6 +639,25 @@ struct fts5_api { fts5_extension_function xFunction, void (*xDestroy)(void*) ); + + /* APIs below this point are only available if iVersion>=3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer_v2 *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer + ); }; /* diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 4311faceb5..e8fdd8a4fa 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -162,7 +162,7 @@ typedef struct Fts5TokenizerConfig Fts5TokenizerConfig; struct Fts5TokenizerConfig { Fts5Tokenizer *pTok; - fts5_tokenizer *pTokApi; + fts5_tokenizer_v2 *pTokApi; const char **azArg; int nArg; int ePattern; /* FTS_PATTERN_XXX constant */ @@ -223,6 +223,7 @@ struct Fts5Config { char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ int bTokendata; /* "tokendata=" option value (dflt==0) */ + int bLocale; /* "locale=" option value (dflt==0) */ int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; Fts5TokenizerConfig t; @@ -292,6 +293,8 @@ int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*); int sqlite3Fts5ConfigParseRank(const char*, char**, char**); +void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...); + /* ** End of interface to code in fts5_config.c. **************************************************************************/ @@ -627,6 +630,17 @@ Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64); int sqlite3Fts5FlushToDisk(Fts5Table*); +int sqlite3Fts5ExtractText( + Fts5Config *pConfig, + int bContent, /* Loaded from content table */ + sqlite3_value *pVal, /* Value to extract text from */ + int *pbResetTokenizer, /* OUT: True if xSetLocale(NULL) required */ + const char **ppText, /* OUT: Pointer to text buffer */ + int *pnText /* OUT: Size of (*ppText) in bytes */ +); + +void sqlite3Fts5ClearLocale(Fts5Config *pConfig); + /* ** End of interface to code in fts5.c. **************************************************************************/ diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 01f40455a0..3736f8685f 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -380,6 +380,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed locale=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bLocale = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ const Fts5Enum aDetail[] = { { "none", FTS5_DETAIL_NONE }, @@ -605,6 +615,11 @@ int sqlite3Fts5ConfigParse( sqlite3_free(zTwo); } + /* If this is not an FTS5_CONTENT_NORMAL table, set bLocale */ + if( pRet->eContent!=FTS5_CONTENT_NORMAL ){ + pRet->bLocale = 1; + } + /* We only allow contentless_delete=1 if the table is indeed contentless. */ if( rc==SQLITE_OK && pRet->bContentlessDelete @@ -1027,3 +1042,20 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ } return rc; } + +void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ + va_list ap; /* ... printf arguments */ + char *zMsg = 0; + + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + if( pConfig->pzErrmsg ){ + *pConfig->pzErrmsg = zMsg; + }else{ + sqlite3_free(zMsg); + } + + va_end(ap); +} + + diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index c6e7e346aa..06bbd820c6 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -107,7 +107,7 @@ struct Fts5Auxiliary { struct Fts5TokenizerModule { char *zName; /* Name of tokenizer */ void *pUserData; /* User pointer passed to xCreate() */ - fts5_tokenizer x; /* Tokenizer functions */ + fts5_tokenizer_v2 x; /* Tokenizer functions */ void (*xDestroy)(void*); /* Destructor function */ Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ }; @@ -230,6 +230,8 @@ struct Fts5Cursor { #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) +#define FTS5_LOCALE_SUBTYPE ((unsigned int)'L') + /* ** Macros to Set(), Clear() and Test() cursor flags. @@ -1229,6 +1231,116 @@ static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ } +static int fts5SetLocale( + Fts5Config *pConfig, + const char *zLocale, + int nLocale +){ + Fts5TokenizerConfig *pT = &pConfig->t; + int rc = SQLITE_OK; + if( pT->pTokApi->xSetLocale ){ + rc = pT->pTokApi->xSetLocale(pT->pTok, zLocale, nLocale); + } + return rc; +} + +void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ + fts5SetLocale(pConfig, 0, 0); +} + +int sqlite3Fts5ExtractText( + Fts5Config *pConfig, + int bContent, + sqlite3_value *pVal, /* Value to extract text from */ + int *pbResetTokenizer, /* OUT: True if xSetLocale(NULL) required */ + const char **ppText, /* OUT: Pointer to text buffer */ + int *pnText /* OUT: Size of (*ppText) in bytes */ +){ + const char *pText = 0; + int nText = 0; + int bResetTokenizer = 0; + int rc = SQLITE_OK; + + int bDecodeBlob = 0; + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE + || (bContent && pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL) + ){ + bDecodeBlob = 1; + } + } + + if( bDecodeBlob ){ + const u8 *pBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + int nLocale = 0; + + for(nLocale=0; nLocalep.pConfig, 0, pVal, &bReset, &zText,&nText); + if( rc==SQLITE_OK ){ + if( bReset ){ + *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText); + if( rc!=SQLITE_OK ){ + sqlite3Fts5ClearLocale(pTab->p.pConfig); + }else{ + *pbFreeAndReset = 1; + } + }else{ + *pzText = (char*)zText; + } + } + + return rc; +} + + /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional @@ -1293,8 +1405,13 @@ static int fts5FilterMethod( pRank = apVal[i]; break; case 'M': { - const char *zText = (const char*)sqlite3_value_text(apVal[i]); + char *zText = 0; + int bFreeAndReset = 0; + + rc = fts5ExtractExprText(pTab, apVal[i], &zText, &bFreeAndReset); + if( rc!=SQLITE_OK ) goto filter_out; if( zText==0 ) zText = ""; + iCol = 0; do{ iCol = iCol*10 + (idxStr[iIdxStr]-'0'); @@ -1306,7 +1423,6 @@ static int fts5FilterMethod( ** indicates that the MATCH expression is not a full text query, ** but a request for an internal parameter. */ rc = fts5SpecialMatch(pTab, pCsr, &zText[1]); - goto filter_out; }else{ char **pzErr = &pTab->p.base.zErrMsg; rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr); @@ -1314,9 +1430,15 @@ static int fts5FilterMethod( rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); pExpr = 0; } - if( rc!=SQLITE_OK ) goto filter_out; } + if( bFreeAndReset ){ + sqlite3_free(zText); + sqlite3Fts5ClearLocale(pConfig); + } + + if( zText[0]=='*' || rc!=SQLITE_OK ) goto filter_out; + break; } case 'L': @@ -2596,6 +2718,35 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ return rc; } +static void fts5ExtractValueFromColumn( + sqlite3_context *pCtx, + Fts5Config *pConfig, + sqlite3_value *pVal +){ + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE + || (pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL) + ){ + const u8 *pBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + + if( nBlob>=4 && memcmp(pBlob, "\0\0\0\0", 4)==0 ){ + sqlite3_result_blob(pCtx, &pBlob[4], nBlob-4, SQLITE_TRANSIENT); + }else{ + int ii; + for(ii=0; iipzErrmsg = &pTab->p.base.zErrMsg; rc = fts5SeekCursor(pCsr, 1); if( rc==SQLITE_OK ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + fts5ExtractValueFromColumn(pCtx, pConfig, pVal); } pConfig->pzErrmsg = 0; }else if( pConfig->bContentlessDelete && sqlite3_vtab_nochange(pCtx) ){ @@ -2791,7 +2943,47 @@ static int fts5CreateAux( /* ** Register a new tokenizer. This is the implementation of the -** fts5_api.xCreateTokenizer() method. +** fts5_api.xCreateTokenizer_v2() method. +*/ +static int fts5CreateTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer_v2 *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + int rc = SQLITE_OK; + + if( pTokenizer->iVersion>2 ){ + rc = SQLITE_ERROR; + }else{ + Fts5TokenizerModule *pNew; + sqlite3_int64 nName; /* Size of zName and its \0 terminator */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + + nName = strlen(zName) + 1; + nByte = sizeof(Fts5TokenizerModule) + nName; + pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte); + if( pNew ){ + pNew->zName = (char*)&pNew[1]; + memcpy(pNew->zName, zName, nName); + pNew->pUserData = pUserData; + pNew->x = *pTokenizer; + pNew->xDestroy = xDestroy; + pNew->pNext = pGlobal->pTok; + pGlobal->pTok = pNew; + if( pNew->pNext==0 ){ + pGlobal->pDfltTok = pNew; + } + } + } + + return rc; +} + +/* +** The fts5_api.xCreateTokenizer() method. */ static int fts5CreateTokenizer( fts5_api *pApi, /* Global context (one per db handle) */ @@ -2800,32 +2992,15 @@ static int fts5CreateTokenizer( fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ void(*xDestroy)(void*) /* Destructor for pUserData */ ){ - Fts5Global *pGlobal = (Fts5Global*)pApi; - Fts5TokenizerModule *pNew; - sqlite3_int64 nName; /* Size of zName and its \0 terminator */ - sqlite3_int64 nByte; /* Bytes of space to allocate */ - int rc = SQLITE_OK; + fts5_tokenizer_v2 tok; - nName = strlen(zName) + 1; - nByte = sizeof(Fts5TokenizerModule) + nName; - pNew = (Fts5TokenizerModule*)sqlite3_malloc64(nByte); - if( pNew ){ - memset(pNew, 0, (size_t)nByte); - pNew->zName = (char*)&pNew[1]; - memcpy(pNew->zName, zName, nName); - pNew->pUserData = pUserData; - pNew->x = *pTokenizer; - pNew->xDestroy = xDestroy; - pNew->pNext = pGlobal->pTok; - pGlobal->pTok = pNew; - if( pNew->pNext==0 ){ - pGlobal->pDfltTok = pNew; - } - }else{ - rc = SQLITE_NOMEM; - } + memset(&tok, 0, sizeof(tok)); + tok.iVersion = 2; + tok.xCreate = pTokenizer->xCreate; + tok.xTokenize = pTokenizer->xTokenize; + tok.xDelete = pTokenizer->xDelete; - return rc; + return fts5CreateTokenizer_v2(pApi, zName, pUserData, &tok, xDestroy); } static Fts5TokenizerModule *fts5LocateTokenizer( @@ -2845,6 +3020,32 @@ static Fts5TokenizerModule *fts5LocateTokenizer( return pMod; } +/* +** Find a tokenizer. This is the implementation of the +** fts5_api.xFindTokenizer_v2() method. +*/ +static int fts5FindTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of tokenizer */ + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer /* Populate this object */ +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pMod; + + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + *ppTokenizer = &pMod->x; + *ppUserData = pMod->pUserData; + }else{ + *ppTokenizer = 0; + *ppUserData = 0; + rc = SQLITE_ERROR; + } + + return rc; +} + /* ** Find a tokenizer. This is the implementation of the ** fts5_api.xFindTokenizer() method. @@ -2855,16 +3056,14 @@ static int fts5FindTokenizer( void **ppUserData, fts5_tokenizer *pTokenizer /* Populate this object */ ){ + fts5_tokenizer_v2 *pV2 = 0; int rc = SQLITE_OK; - Fts5TokenizerModule *pMod; - pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); - if( pMod ){ - *pTokenizer = pMod->x; - *ppUserData = pMod->pUserData; - }else{ - memset(pTokenizer, 0, sizeof(fts5_tokenizer)); - rc = SQLITE_ERROR; + rc = fts5FindTokenizer_v2(pApi, zName, ppUserData, &pV2); + if( rc==SQLITE_OK ){ + pTokenizer->xCreate = pV2->xCreate; + pTokenizer->xDelete = pV2->xDelete; + pTokenizer->xTokenize = pV2->xTokenize; } return rc; @@ -2966,6 +3165,46 @@ static void fts5SourceIdFunc( sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT); } +/* +** Implementation of fts5_locale() function. +*/ +static void fts5LocaleFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + u8 *pBlob = 0; + int nBlob = 0; + + const char *zLocale = 0; + int nLocale = 0; + const char *zText = 0; + int nText = 0; + + assert( nArg==2 ); + UNUSED_PARAM(nArg); + + zLocale = (const char*)sqlite3_value_text(apArg[0]); + nLocale = sqlite3_value_bytes(apArg[0]); + + zText = (const char*)sqlite3_value_text(apArg[1]); + nText = sqlite3_value_bytes(apArg[1]); + + nBlob = nLocale + 1 + nText; + pBlob = (u8*)sqlite3_malloc(nBlob); + if( pBlob==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + + if( zLocale ) memcpy(pBlob, zLocale, nLocale); + pBlob[nLocale] = 0x00; + if( zText ) memcpy(&pBlob[nLocale+1], zText, nText); + + sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free); + sqlite3_result_subtype(pCtx, FTS5_LOCALE_SUBTYPE); +} + /* ** Return true if zName is the extension on one of the shadow tables used ** by this module. @@ -3058,10 +3297,12 @@ static int fts5Init(sqlite3 *db){ void *p = (void*)pGlobal; memset(pGlobal, 0, sizeof(Fts5Global)); pGlobal->db = db; - pGlobal->api.iVersion = 2; + pGlobal->api.iVersion = 3; pGlobal->api.xCreateFunction = fts5CreateAux; pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; pGlobal->api.xFindTokenizer = fts5FindTokenizer; + pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2; + pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2; rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); @@ -3080,6 +3321,13 @@ static int fts5Init(sqlite3 *db){ p, fts5SourceIdFunc, 0, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_locale", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE, + p, fts5LocaleFunc, 0, 0 + ); + } } /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 0b676e6b4f..5d7f3f055e 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -429,26 +429,30 @@ static int fts5StorageDeleteFromIndex( ctx.iCol = -1; for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1]==0 ){ - const char *zText; - int nText; + sqlite3_value *pVal = 0; + const char *pText = 0; + int nText = 0; + int bReset = 0; + assert( pSeek==0 || apVal==0 ); assert( pSeek!=0 || apVal!=0 ); if( pSeek ){ - zText = (const char*)sqlite3_column_text(pSeek, iCol); - nText = sqlite3_column_bytes(pSeek, iCol); - }else if( ALWAYS(apVal) ){ - zText = (const char*)sqlite3_value_text(apVal[iCol-1]); - nText = sqlite3_value_bytes(apVal[iCol-1]); + pVal = sqlite3_column_value(pSeek, iCol); }else{ - continue; + pVal = apVal[iCol-1]; } - ctx.szCol = 0; - rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, - zText, nText, (void*)&ctx, fts5StorageInsertCallback - ); - p->aTotalSize[iCol-1] -= (i64)ctx.szCol; - if( p->aTotalSize[iCol-1]<0 && rc==SQLITE_OK ){ - rc = FTS5_CORRUPT; + + rc = sqlite3Fts5ExtractText(pConfig,pSeek!=0,pVal,&bReset,&pText,&nText); + if( rc==SQLITE_OK ){ + ctx.szCol = 0; + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, + pText, nText, (void*)&ctx, fts5StorageInsertCallback + ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; + if( p->aTotalSize[iCol-1]<0 && rc==SQLITE_OK ){ + rc = FTS5_CORRUPT; + } + if( bReset ) sqlite3Fts5ClearLocale(pConfig); } } } @@ -684,14 +688,22 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1); - int nText = sqlite3_column_bytes(pScan, ctx.iCol+1); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageInsertCallback + int bReset = 0; + int nText = 0; + const char *pText = 0; + rc = sqlite3Fts5ExtractText(pConfig, 1, + sqlite3_column_value(pScan, ctx.iCol+1), &bReset, &pText, &nText ); + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageInsertCallback + ); + if( bReset ) sqlite3Fts5ClearLocale(pConfig); + } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; @@ -810,14 +822,26 @@ int sqlite3Fts5StorageIndexInsert( for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]); - int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageInsertCallback + int bReset = 0; + int nText = 0; + const char *pText = 0; + rc = sqlite3Fts5ExtractText( + pConfig, 0, apVal[ctx.iCol+2], &bReset, &pText, &nText ); + if( rc==SQLITE_OK ){ + if( bReset && pConfig->bLocale==0 ){ + rc = SQLITE_ERROR; + sqlite3Fts5ConfigErrmsg(pConfig, + "fts5_locale() may not be used without locale=1" + ); + }else{ + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, + fts5StorageInsertCallback + ); + } + if( bReset ) sqlite3Fts5ClearLocale(pConfig); + } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; @@ -988,14 +1012,23 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } if( rc==SQLITE_OK ){ - const char *zText = (const char*)sqlite3_column_text(pScan, i+1); - int nText = sqlite3_column_bytes(pScan, i+1); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageIntegrityCallback + const char *pText = 0; + int nText = 0; + int bReset = 0; + + rc = sqlite3Fts5ExtractText(pConfig, 1, + sqlite3_column_value(pScan, i+1), &bReset, &pText, &nText ); + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageIntegrityCallback + ); + if( bReset ) sqlite3Fts5ClearLocale(pConfig); + } } if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ rc = FTS5_CORRUPT; diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index c5b5f41f83..c1fc7f82ae 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -801,6 +801,7 @@ typedef struct F5tTokenizerInstance F5tTokenizerInstance; struct F5tTokenizerContext { void *pCtx; int (*xToken)(void*, int, const char*, int, int, int); + F5tTokenizerInstance *pInst; }; struct F5tTokenizerModule { @@ -809,12 +810,37 @@ struct F5tTokenizerModule { F5tTokenizerContext *pContext; }; +/* +** zLocale: +** Buffer zLocale contains the current locale, as configured by the most +** recent call to xSetLocale(). A NULL (default) locale is represented as +** a 0 byte string - "\0". +** +** This can be retrieved by a Tcl tokenize script using [sqlite3_fts5_locale]. +*/ struct F5tTokenizerInstance { Tcl_Interp *interp; Tcl_Obj *pScript; F5tTokenizerContext *pContext; + char zLocale[128]; }; +static int f5tTokenizerSetLocale( + Fts5Tokenizer *pTokenizer, + const char *pLocale, + int nLocale +){ + F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)pTokenizer; + if( nLocale>=sizeof(pInst->zLocale) ){ + return SQLITE_ERROR; + } + + memset(pInst->zLocale, 0, sizeof(pInst->zLocale)); + memcpy(pInst->zLocale, pLocale, nLocale); + + return SQLITE_OK; +} + static int f5tTokenizerCreate( void *pCtx, const char **azArg, @@ -867,6 +893,7 @@ static int f5tTokenizerTokenize( int (*xToken)(void*, int, const char*, int, int, int) ){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + F5tTokenizerInstance *pOldInst = 0; void *pOldCtx; int (*xOldToken)(void*, int, const char*, int, int, int); Tcl_Obj *pEval; @@ -875,9 +902,11 @@ static int f5tTokenizerTokenize( pOldCtx = pInst->pContext->pCtx; xOldToken = pInst->pContext->xToken; + pOldInst = pInst->pContext->pInst; pInst->pContext->pCtx = pCtx; pInst->pContext->xToken = xToken; + pInst->pContext->pInst = pInst; assert( flags==FTS5_TOKENIZE_DOCUMENT @@ -913,9 +942,37 @@ static int f5tTokenizerTokenize( pInst->pContext->pCtx = pOldCtx; pInst->pContext->xToken = xOldToken; + pInst->pContext->pInst = pOldInst; return rc; } +/* +** sqlite3_fts5_locale +*/ +static int SQLITE_TCLAPI f5tTokenizerLocale( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + F5tTokenizerContext *p = (F5tTokenizerContext*)clientData; + + if( objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + + if( p->xToken==0 ){ + Tcl_AppendResult(interp, + "sqlite3_fts5_locale may only be used by tokenizer callback", 0 + ); + return TCL_ERROR; + } + + Tcl_SetObjResult(interp, Tcl_NewStringObj(p->pInst->zLocale, -1)); + return TCL_OK; +} + /* ** sqlite3_fts5_token ?-colocated? TEXT START END */ @@ -1001,12 +1058,21 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( fts5_api *pApi; char *zName; Tcl_Obj *pScript; - fts5_tokenizer t; F5tTokenizerModule *pMod; int rc; + int bV2 = 0; /* True to use _v2 API */ + + if( objc==5 ){ + const char *zArg = Tcl_GetString(objv[1]); + if( 0==strcmp(zArg, "-v2") ){ + objv++; + objc--; + bV2 = 1; + } + } if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT"); + Tcl_WrongNumArgs(interp, 1, objv, "?-v2? DB NAME SCRIPT"); return TCL_ERROR; } if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){ @@ -1015,18 +1081,34 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( zName = Tcl_GetString(objv[2]); pScript = objv[3]; - t.xCreate = f5tTokenizerCreate; - t.xTokenize = f5tTokenizerTokenize; - t.xDelete = f5tTokenizerDelete; - pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule)); pMod->interp = interp; pMod->pScript = pScript; pMod->pContext = pContext; Tcl_IncrRefCount(pScript); - rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer); + + if( bV2==0 ){ + fts5_tokenizer t; + t.xCreate = f5tTokenizerCreate; + t.xTokenize = f5tTokenizerTokenize; + t.xDelete = f5tTokenizerDelete; + rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer); + }else{ + fts5_tokenizer_v2 t2; + memset(&t2, 0, sizeof(t2)); + t2.iVersion = 2; + t2.xCreate = f5tTokenizerCreate; + t2.xTokenize = f5tTokenizerTokenize; + t2.xDelete = f5tTokenizerDelete; + t2.xSetLocale = f5tTokenizerSetLocale; + rc = pApi->xCreateTokenizer_v2(pApi, zName,(void*)pMod,&t2,f5tDelTokenizer); + } + if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0); + Tcl_AppendResult(interp, ( + bV2 ? "error in fts5_api.xCreateTokenizer_v2()" + : "error in fts5_api.xCreateTokenizer()" + ), 0); return TCL_ERROR; } @@ -1333,6 +1415,7 @@ int Fts5tcl_Init(Tcl_Interp *interp){ } aCmd[] = { { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, + { "sqlite3_fts5_locale", f5tTokenizerLocale, 1 }, { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, { "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test new file mode 100644 index 0000000000..923990f5a9 --- /dev/null +++ b/ext/fts5/test/fts5locale.test @@ -0,0 +1,176 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5locale + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc transform_token {locale token} { + switch -- $locale { + reverse { + set ret "" + foreach c [split $token ""] { + set ret "$c$ret" + } + set token $ret + } + + default { + # no-op + } + } + + set token +} + +proc tcl_create {args} { return "tcl_tokenize" } +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + set w [transform_token [sqlite3_fts5_locale] $w] + sqlite3_fts5_token $w $iStart $iEnd + } +} + +#------------------------------------------------------------------------- +# Check that queries can have a locale attached to them. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl); + INSERT INTO t1 VALUES('abc'); + INSERT INTO t1 VALUES('cba'); +} {} + +do_execsql_test 1.1 { + SELECT rowid, a FROM t1( fts5_locale('en_US', 'abc') ); +} {1 abc} + +do_execsql_test 1.2 { + SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') ); +} {2 cba} + +#------------------------------------------------------------------------- +# Test that the locale= option exists and seems to accept values. And +# that fts5_locale() values may only be inserted into an internal-content +# table if the locale=1 option was specified. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE b1 USING fts5(x, y, locale=1, tokenize=tcl); + CREATE VIRTUAL TABLE b2 USING fts5(x, y, locale=0, tokenize=tcl); + + CREATE VIRTUAL TABLE ttt USING fts5vocab('b1', instance); +} + +do_catchsql_test 2.2 { + CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=2); +} {1 {malformed locale=... directive}} + +do_catchsql_test 2.3 { + INSERT INTO b1(b1, rank) VALUES('locale', 0); +} {1 {SQL logic error}} + +do_execsql_test 2.4 { + INSERT INTO b1 VALUES('abc', 'one two three'); + INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six')); +} + +do_execsql_test 2.5 { + INSERT INTO b2 VALUES('abc', 'one two three'); +} + +do_catchsql_test 2.6 { + INSERT INTO b2 VALUES('def', fts5_locale('reverse', 'four five six')); +} {1 {fts5_locale() may not be used without locale=1}} + +do_execsql_test 2.7 { SELECT rowid FROM b1('one') } {1} +do_execsql_test 2.8 { SELECT rowid FROM b1('four') } {} +do_execsql_test 2.9 { SELECT rowid FROM b1('ruof') } 2 +do_execsql_test 2.10 { SELECT rowid FROM b1(fts5_locale('reverse', 'five'))} 2 + +do_execsql_test 2.11 { + SELECT x, quote(y) FROM b1 +} { + abc {'one two three'} + def {'four five six'} +} + +do_execsql_test 2.12 { SELECT quote(y) FROM b1('ruof') } { + {'four five six'} +} + +do_execsql_test 2.13 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} +do_execsql_test 2.14 { + INSERT INTO b1(b1) VALUES('rebuild'); +} +do_execsql_test 2.15 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +do_execsql_test 2.16 { + DELETE FROM b1 WHERE rowid=2 +} +do_execsql_test 2.17 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# Test the 'delete' command with contentless tables. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl); + CREATE VIRTUAL TABLE c2 USING fts5vocab('c1', instance); + + INSERT INTO c1 VALUES('hello world'); + INSERT INTO c1 VALUES( fts5_locale('reverse', 'one two three') ); +} + +do_execsql_test 3.2 { + SELECT DISTINCT term FROM c2 ORDER BY 1 +} { + eerht eno hello owt world +} + +do_execsql_test 3.3 { + INSERT INTO c1(c1, rowid, x) + VALUES('delete', 2, fts5_locale('reverse', 'one two three') ); +} + +do_execsql_test 3.4 { + SELECT DISTINCT term FROM c2 ORDER BY 1 +} { + hello world +} + + + +# execsql_pp { SELECT * FROM ttt } + +finish_test + + diff --git a/manifest b/manifest index bccfb3c0a2..6f223ff085 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\spercentile\sextension\sto\sthe\sCLI\sby\sdefault. -D 2024-07-24T13:53:51.649 +C Add\sthe\sfts5_locale()\sfunction,\sand\sbegin\sadding\sthe\srelated\sfunctionality\sto\sfts5. +D 2024-07-26T20:50:33.303 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,17 +92,17 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h 6b49ce6eb2e395e7fd84557b21d32f5de8041f2fada4c617e481e99427e24b6e -F ext/fts5/fts5Int.h 41fb3a2dd40e818cc96c6f4176dbdf2aaa8f57043cfc9a8f2676e7e6a72ad764 +F ext/fts5/fts5.h 38a9553791828b3cf677b9347735fc531d54015ce4f5229d5cf1e2a5c1d3955a +F ext/fts5/fts5Int.h b4a5ed934cb3da55737c4d75cb5f26a39b17470fca67c06c7fe6878992998c99 F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 -F ext/fts5/fts5_config.c 68cb87a49215f8e7028000b681df4057c430a4a6afbd676463886da94c9e1c37 +F ext/fts5/fts5_config.c 0c96490fbad746b3780174f38b2ee5e3d719f2f81ee6b58ca828772871e0f680 F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 77fefb37e7931095a5ff271a28fbe4f73ec46d5492ef1f35d405d98e137ad8ed -F ext/fts5/fts5_storage.c 1d7e08d4331da2f3f7e78e70eef2ed6a013d91ba16175c651adbc5ad672235aa -F ext/fts5/fts5_tcl.c fdf7e2bb9a9186cfcaf2d2ce11d338309342b7a7593c2812bc54455db53da5d2 +F ext/fts5/fts5_main.c 5b6f85aae5f25ee4e8762f26eb8c998c9c53443bb56483ebf712aca591bcb41e +F ext/fts5/fts5_storage.c 1d7b358af3d4a7a4c5a7258a847229ca54c1b26d4f1b9e971ea5f2539631c3d4 +F ext/fts5/fts5_tcl.c a1c307785bb505735a8d914fff7d08881e64ba28c40c406b218c591010d1bc9e F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c fa5493075101540270f572038fc1723d44fcc97bfbf237c8530013b8a27860be @@ -184,6 +184,7 @@ F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fd F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c +F ext/fts5/test/fts5locale.test 92c6ae79df0aa57b379c50e400151f4a9a36d292819beefc31019c749249844a F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -2195,8 +2196,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 095c22e62248f8ef50cd8531171827f50a7bdd4fc1128bf0e616a3eb2dce980e -R 76e7f4761efc98a28727d2a1442d8c13 -U drh -Z 956d236c4ff2c748d10dde2ade86c2ae +P bcc31846964102385d5a21eb5e85d7db153b155e76b4e2847c9453d3d0e1af04 +R f9f51b6d625a93fd57b76dfee17ab828 +T *branch * fts5-locale +T *sym-fts5-locale * +T -sym-trunk * +U dan +Z 2b42cc25153434047e86f4740a226c4e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c7d267a281..64398355c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bcc31846964102385d5a21eb5e85d7db153b155e76b4e2847c9453d3d0e1af04 +8839ef7cfb49239e7f1c4812a53a93a672827c88d6921408b1d5062b352c87cc From 53bef48fbed08188030db5265f01ff504b6985f2 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Jul 2024 19:46:10 +0000 Subject: [PATCH 04/99] Ensure that an fts5 UPDATE does not strip the locale from columns that it does not write to. FossilOrigin-Name: 69205264debd829573b1c777a5a493cfeb6083c4cdec106b1f819989f859ac75 --- ext/fts5/fts5Int.h | 5 +- ext/fts5/fts5_main.c | 78 ++++++++++++++++--------- ext/fts5/fts5_storage.c | 105 ++++++++++++++++++++++++++-------- ext/fts5/test/fts5locale.test | 47 ++++++++++++++- ext/fts5/test/fts5simple.test | 1 + manifest | 23 ++++---- manifest.uuid | 2 +- 7 files changed, 193 insertions(+), 68 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index e8fdd8a4fa..1cc25ae7e0 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -720,7 +720,7 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); int sqlite3Fts5DropAll(Fts5Config*); int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**); +int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int); int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); @@ -746,6 +746,9 @@ int sqlite3Fts5StorageOptimize(Fts5Storage *p); int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); int sqlite3Fts5StorageReset(Fts5Storage *p); +void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*); +int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel); + /* ** End of interface to code in fts5_storage.c. **************************************************************************/ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 06bbd820c6..611c08a57e 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1746,7 +1746,7 @@ static int fts5SpecialDelete( int eType1 = sqlite3_value_type(apVal[1]); if( eType1==SQLITE_INTEGER ){ sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2], 0); } return rc; } @@ -1870,7 +1870,7 @@ static int fts5UpdateMethod( /* DELETE */ else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0, 0); bUpdateOrDelete = 1; } @@ -1878,16 +1878,18 @@ static int fts5UpdateMethod( else{ int eType1 = sqlite3_value_numeric_type(apVal[1]); - if( eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL ){ + if( (eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL) + || (eType0==SQLITE_INTEGER && eType1==SQLITE_NULL) + ){ rc = SQLITE_MISMATCH; } - else if( eType0!=SQLITE_INTEGER ){ + else if( eType0!=SQLITE_INTEGER ){ /* An INSERT statement. If the conflict-mode is REPLACE, first remove ** the current entry (if any). */ if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0); bUpdateOrDelete = 1; } fts5StorageInsert(&rc, pTab, apVal, pRowid); @@ -1897,28 +1899,34 @@ static int fts5UpdateMethod( else{ i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ - if( eType1==SQLITE_INTEGER && iOld!=iNew ){ + assert( eType1==SQLITE_INTEGER ); + if( iOld!=iNew ){ if( eConflict==SQLITE_REPLACE ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0, 1); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0); } fts5StorageInsert(&rc, pTab, apVal, pRowid); }else{ - rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); + rc = sqlite3Fts5StorageFindDeleteRow(pTab->pStorage, iOld); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage,apVal,pRowid); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0, 1); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal,*pRowid); } } }else{ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0, 1); fts5StorageInsert(&rc, pTab, apVal, pRowid); } bUpdateOrDelete = 1; + sqlite3Fts5StorageReleaseDeleteRow(pTab->pStorage); } + } } @@ -2606,6 +2614,16 @@ static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ return pCsr; } +static void fts5ResultError(sqlite3_context *pCtx, const char *zFmt, ...){ + char *zErr = 0; + va_list ap; + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + va_end(ap); +} + static void fts5ApiCallback( sqlite3_context *context, int argc, @@ -2622,9 +2640,7 @@ static void fts5ApiCallback( pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); if( pCsr==0 || pCsr->ePlan==0 ){ - char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); + fts5ResultError(context, "no such cursor: %lld", iCsrId); }else{ sqlite3_vtab *pTab = pCsr->base.pVtab; fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); @@ -2776,8 +2792,8 @@ static int fts5ColumnMethod( ** auxiliary function. */ sqlite3_result_int64(pCtx, pCsr->iCsrId); }else if( iCol==pConfig->nCol+1 ){ - /* The value of the "rank" column. */ + if( pCsr->ePlan==FTS5_PLAN_SOURCE ){ fts5PoslistBlob(pCtx, pCsr); }else if( @@ -2788,21 +2804,27 @@ static int fts5ColumnMethod( fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); } } - }else if( !fts5IsContentless(pTab) ){ - pConfig->pzErrmsg = &pTab->p.base.zErrMsg; - rc = fts5SeekCursor(pCsr, 1); - if( rc==SQLITE_OK ){ - sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); - fts5ExtractValueFromColumn(pCtx, pConfig, pVal); + }else{ + /* A column created by the user containing values. */ + int bNochange = sqlite3_vtab_nochange(pCtx); + + if( fts5IsContentless(pTab) ){ + if( bNochange && pConfig->bContentlessDelete ){ + fts5ResultError(pCtx, "cannot UPDATE a subset of " + "columns on fts5 contentless-delete table: %s", pConfig->zName + ); + } + }else if( bNochange==0 || pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + rc = fts5SeekCursor(pCsr, 1); + if( rc==SQLITE_OK ){ + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + fts5ExtractValueFromColumn(pCtx, pConfig, pVal); + } + pConfig->pzErrmsg = 0; } - pConfig->pzErrmsg = 0; - }else if( pConfig->bContentlessDelete && sqlite3_vtab_nochange(pCtx) ){ - char *zErr = sqlite3_mprintf("cannot UPDATE a subset of " - "columns on fts5 contentless-delete table: %s", pConfig->zName - ); - sqlite3_result_error(pCtx, zErr, -1); - sqlite3_free(zErr); } + return rc; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 5d7f3f055e..f94c2ea908 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -22,7 +22,8 @@ struct Fts5Storage { int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */ i64 nTotalRow; /* Total number of rows in FTS table */ i64 *aTotalSize; /* Total sizes of each column */ - sqlite3_stmt *aStmt[11]; + sqlite3_stmt *pSavedRow; + sqlite3_stmt *aStmt[12]; }; @@ -36,14 +37,15 @@ struct Fts5Storage { # error "FTS5_STMT_LOOKUP mismatch" #endif -#define FTS5_STMT_INSERT_CONTENT 3 -#define FTS5_STMT_REPLACE_CONTENT 4 -#define FTS5_STMT_DELETE_CONTENT 5 -#define FTS5_STMT_REPLACE_DOCSIZE 6 -#define FTS5_STMT_DELETE_DOCSIZE 7 -#define FTS5_STMT_LOOKUP_DOCSIZE 8 -#define FTS5_STMT_REPLACE_CONFIG 9 -#define FTS5_STMT_SCAN 10 +#define FTS5_STMT_LOOKUP2 3 +#define FTS5_STMT_INSERT_CONTENT 4 +#define FTS5_STMT_REPLACE_CONTENT 5 +#define FTS5_STMT_DELETE_CONTENT 6 +#define FTS5_STMT_REPLACE_DOCSIZE 7 +#define FTS5_STMT_DELETE_DOCSIZE 8 +#define FTS5_STMT_LOOKUP_DOCSIZE 9 +#define FTS5_STMT_REPLACE_CONFIG 10 +#define FTS5_STMT_SCAN 11 /* ** Prepare the two insert statements - Fts5Storage.pInsertContent and @@ -73,6 +75,7 @@ static int fts5StorageGetStmt( "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC", "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC", "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ + "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ @@ -88,6 +91,8 @@ static int fts5StorageGetStmt( Fts5Config *pC = p->pConfig; char *zSql = 0; + assert( ArraySize(azStmt)==ArraySize(p->aStmt) ); + switch( eStmt ){ case FTS5_STMT_SCAN: zSql = sqlite3_mprintf(azStmt[eStmt], @@ -104,6 +109,7 @@ static int fts5StorageGetStmt( break; case FTS5_STMT_LOOKUP: + case FTS5_STMT_LOOKUP2: zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, pC->zContent, pC->zContentRowid ); @@ -150,7 +156,7 @@ static int fts5StorageGetStmt( rc = SQLITE_NOMEM; }else{ int f = SQLITE_PREPARE_PERSISTENT; - if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB; + if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB; p->pConfig->bLock++; rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0); p->pConfig->bLock--; @@ -399,6 +405,24 @@ static int fts5StorageInsertCallback( return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); } +int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){ + int rc = SQLITE_OK; + sqlite3_stmt *pSeek = 0; + + assert( p->pSavedRow==0 ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + rc = sqlite3_reset(pSeek); + }else{ + p->pSavedRow = pSeek; + } + } + + return rc; +} + /* ** If a row with rowid iDel is present in the %_content table, add the ** delete-markers to the FTS index necessary to delete it. Do not actually @@ -407,7 +431,8 @@ static int fts5StorageInsertCallback( static int fts5StorageDeleteFromIndex( Fts5Storage *p, i64 iDel, - sqlite3_value **apVal + sqlite3_value **apVal, + int bSaveRow ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ @@ -416,12 +441,21 @@ static int fts5StorageDeleteFromIndex( int iCol; Fts5InsertCtx ctx; + assert( bSaveRow==0 || apVal==0 ); + assert( bSaveRow==0 || bSaveRow==1 ); + assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 ); + if( apVal==0 ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int64(pSeek, 1, iDel); - if( sqlite3_step(pSeek)!=SQLITE_ROW ){ - return sqlite3_reset(pSeek); + if( p->pSavedRow && bSaveRow ){ + pSeek = p->pSavedRow; + p->pSavedRow = 0; + }else{ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + return sqlite3_reset(pSeek); + } } } @@ -462,11 +496,21 @@ static int fts5StorageDeleteFromIndex( p->nTotalRow--; } - rc2 = sqlite3_reset(pSeek); - if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bSaveRow ){ + assert( p->pSavedRow==0 ); + p->pSavedRow = pSeek; + }else{ + rc2 = sqlite3_reset(pSeek); + if( rc==SQLITE_OK ) rc = rc2; + } return rc; } +void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){ + sqlite3_reset(pStorage->pSavedRow); + pStorage->pSavedRow = 0; +} + /* ** This function is called to process a DELETE on a contentless_delete=1 ** table. It adds the tombstone required to delete the entry with rowid @@ -582,7 +626,12 @@ static int fts5StorageSaveTotals(Fts5Storage *p){ /* ** Remove a row from the FTS table. */ -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ +int sqlite3Fts5StorageDelete( + Fts5Storage *p, /* Storage object */ + i64 iDel, /* Rowid to delete from table */ + sqlite3_value **apVal, /* Optional - values to remove from index */ + int bSaveRow +){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel = 0; @@ -599,7 +648,7 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ if( p->pConfig->bContentlessDelete ){ rc = fts5StorageContentlessDelete(p, iDel); }else{ - rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow); } } @@ -787,7 +836,11 @@ int sqlite3Fts5StorageContentInsert( int i; /* Counter variable */ rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ - rc = sqlite3_bind_value(pInsert, i, apVal[i]); + sqlite3_value *pVal = apVal[i]; + if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ + pVal = sqlite3_column_value(p->pSavedRow, i-1); + } + rc = sqlite3_bind_value(pInsert, i, pVal); } if( rc==SQLITE_OK ){ sqlite3_step(pInsert); @@ -825,9 +878,13 @@ int sqlite3Fts5StorageIndexInsert( int bReset = 0; int nText = 0; const char *pText = 0; - rc = sqlite3Fts5ExtractText( - pConfig, 0, apVal[ctx.iCol+2], &bReset, &pText, &nText - ); + sqlite3_value *pVal = apVal[ctx.iCol+2]; + int bDisk = 0; + if( p->pSavedRow && sqlite3_value_nochange(pVal) ){ + pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1); + bDisk = 1; + } + rc = sqlite3Fts5ExtractText(pConfig, bDisk, pVal, &bReset,&pText,&nText); if( rc==SQLITE_OK ){ if( bReset && pConfig->bLocale==0 ){ rc = SQLITE_ERROR; diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index 923990f5a9..f508ea1335 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -167,9 +167,54 @@ do_execsql_test 3.4 { hello world } +#------------------------------------------------------------------------- +# Test that an UPDATE that updates a subset of the columns does not +# magically discard the locale from those columns not updated. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE d1 USING fts5(x, y, locale=1, tokenize=tcl); + CREATE VIRTUAL TABLE d2 USING fts5vocab('d1', instance); -# execsql_pp { SELECT * FROM ttt } + INSERT INTO d1(rowid, x, y) VALUES(1, 'abc', 'def'); + INSERT INTO d1(rowid, x, y) VALUES(2, 'ghi', fts5_locale('reverse', 'hello')); +} + +do_execsql_test 4.2 { + SELECT DISTINCT term FROM d2 ORDER BY 1 +} { + abc def ghi olleh +} + +do_execsql_test 4.3 { + UPDATE d1 SET x='jkl' WHERE rowid=2; +} + +do_execsql_test 4.4 { + SELECT DISTINCT term FROM d2 ORDER BY 1 +} { + abc def jkl olleh +} + +do_execsql_test 4.5 { + SELECT rowid, * FROM d1 +} { + 1 abc def + 2 jkl hello +} + +do_execsql_test 4.6 { + UPDATE d1 SET rowid=4 WHERE rowid=2 +} + +do_execsql_test 4.7 { + SELECT rowid, * FROM d1 +} { + 1 abc def + 4 jkl hello +} finish_test diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 6384095067..050509fb37 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -312,6 +312,7 @@ do_execsql_test 13.1 { INSERT INTO xy(rowid, x) VALUES(3, '3 4 5'); } +breakpoint do_execsql_test 13.2 { UPDATE OR REPLACE xy SET rowid=3 WHERE rowid = 2; SELECT rowid, x FROM xy; diff --git a/manifest b/manifest index 6f223ff085..9ff1a30afb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sfts5_locale()\sfunction,\sand\sbegin\sadding\sthe\srelated\sfunctionality\sto\sfts5. -D 2024-07-26T20:50:33.303 +C Ensure\sthat\san\sfts5\sUPDATE\sdoes\snot\sstrip\sthe\slocale\sfrom\scolumns\sthat\sit\sdoes\snot\swrite\sto. +D 2024-07-27T19:46:10.539 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -93,15 +93,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e F ext/fts5/fts5.h 38a9553791828b3cf677b9347735fc531d54015ce4f5229d5cf1e2a5c1d3955a -F ext/fts5/fts5Int.h b4a5ed934cb3da55737c4d75cb5f26a39b17470fca67c06c7fe6878992998c99 +F ext/fts5/fts5Int.h 330b1e2dad9ea9cccc9fa93817062fa21e89f00e7eac9a84be440f7e93bf7c3c F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 0c96490fbad746b3780174f38b2ee5e3d719f2f81ee6b58ca828772871e0f680 F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 5b6f85aae5f25ee4e8762f26eb8c998c9c53443bb56483ebf712aca591bcb41e -F ext/fts5/fts5_storage.c 1d7b358af3d4a7a4c5a7258a847229ca54c1b26d4f1b9e971ea5f2539631c3d4 +F ext/fts5/fts5_main.c f08525b3378b9131f76b415b44b882aa9f0a861fb135caaf3faf6a9ba5a409a1 +F ext/fts5/fts5_storage.c 6beefaa9229193e85fe9b920aa62f46a79cac867feb8a5af8ce6df28dec90e8d F ext/fts5/fts5_tcl.c a1c307785bb505735a8d914fff7d08881e64ba28c40c406b218c591010d1bc9e F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -184,7 +184,7 @@ F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fd F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 92c6ae79df0aa57b379c50e400151f4a9a36d292819beefc31019c749249844a +F ext/fts5/test/fts5locale.test 60217c6f67331e2b3218b4da3f96f54b856a341b23e7328c9f30efe9ca343130 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -222,7 +222,7 @@ F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a F ext/fts5/test/fts5secure8.test e68c0ac4447f415ff3e4e82531e99548289286f9f3a29c8cd53036113fe28602 F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5 -F ext/fts5/test/fts5simple.test 847fb828262328744733847dc76d6b5d4a6bd4c5d9b282cb819f6504340e061a +F ext/fts5/test/fts5simple.test 453b2348193911c2d51c208bfe247a09b936c4ddd9a836160495a5554340c256 F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc F ext/fts5/test/fts5simple3.test 146ec3dc8f5763d6212641c9f0a2f1cba41679353d2add7b963beceb115dc7f4 F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988202ce67638d2c6cf6 @@ -2196,11 +2196,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bcc31846964102385d5a21eb5e85d7db153b155e76b4e2847c9453d3d0e1af04 -R f9f51b6d625a93fd57b76dfee17ab828 -T *branch * fts5-locale -T *sym-fts5-locale * -T -sym-trunk * +P 8839ef7cfb49239e7f1c4812a53a93a672827c88d6921408b1d5062b352c87cc +R 8e79393df1bdcc86ce7609abeba8cbda U dan -Z 2b42cc25153434047e86f4740a226c4e +Z 68a8dc4d8e1906054989de2a0038cabe # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 64398355c9..26be2f42c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8839ef7cfb49239e7f1c4812a53a93a672827c88d6921408b1d5062b352c87cc +69205264debd829573b1c777a5a493cfeb6083c4cdec106b1f819989f859ac75 From b4afecc626c8db8a2d540279384a960c76cc6cf9 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Jul 2024 15:00:11 +0000 Subject: [PATCH 05/99] Fix a couple of asan problems on this branch. FossilOrigin-Name: 4e3e529313c86b42d5f1ba113d0714d1545bc6641e09e4537d622057f0352989 --- ext/fts5/fts5_main.c | 4 +++- ext/fts5/fts5_tcl.c | 4 +++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 611c08a57e..7fb08fd6ed 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1407,6 +1407,7 @@ static int fts5FilterMethod( case 'M': { char *zText = 0; int bFreeAndReset = 0; + int bInternal = 0; rc = fts5ExtractExprText(pTab, apVal[i], &zText, &bFreeAndReset); if( rc!=SQLITE_OK ) goto filter_out; @@ -1423,6 +1424,7 @@ static int fts5FilterMethod( ** indicates that the MATCH expression is not a full text query, ** but a request for an internal parameter. */ rc = fts5SpecialMatch(pTab, pCsr, &zText[1]); + bInternal = 1; }else{ char **pzErr = &pTab->p.base.zErrMsg; rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr); @@ -1437,7 +1439,7 @@ static int fts5FilterMethod( sqlite3Fts5ClearLocale(pConfig); } - if( zText[0]=='*' || rc!=SQLITE_OK ) goto filter_out; + if( bInternal || rc!=SQLITE_OK ) goto filter_out; break; } diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index c1fc7f82ae..a0d951526d 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -836,7 +836,9 @@ static int f5tTokenizerSetLocale( } memset(pInst->zLocale, 0, sizeof(pInst->zLocale)); - memcpy(pInst->zLocale, pLocale, nLocale); + if( nLocale>0 ){ + memcpy(pInst->zLocale, pLocale, nLocale); + } return SQLITE_OK; } diff --git a/manifest b/manifest index 9ff1a30afb..594ebdb848 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\san\sfts5\sUPDATE\sdoes\snot\sstrip\sthe\slocale\sfrom\scolumns\sthat\sit\sdoes\snot\swrite\sto. -D 2024-07-27T19:46:10.539 +C Fix\sa\scouple\sof\sasan\sproblems\son\sthis\sbranch. +D 2024-07-29T15:00:11.833 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,9 +100,9 @@ F ext/fts5/fts5_config.c 0c96490fbad746b3780174f38b2ee5e3d719f2f81ee6b58ca828772 F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c f08525b3378b9131f76b415b44b882aa9f0a861fb135caaf3faf6a9ba5a409a1 +F ext/fts5/fts5_main.c 71ea9fe3aba423ad325a7ac600b83bb79e2a640f62094f6df356f978245c78db F ext/fts5/fts5_storage.c 6beefaa9229193e85fe9b920aa62f46a79cac867feb8a5af8ce6df28dec90e8d -F ext/fts5/fts5_tcl.c a1c307785bb505735a8d914fff7d08881e64ba28c40c406b218c591010d1bc9e +F ext/fts5/fts5_tcl.c e22e5791076fdfcbcd2dd5e45aff9d3615f7840bae5d984088ce2c24cfdf77b4 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c fa5493075101540270f572038fc1723d44fcc97bfbf237c8530013b8a27860be @@ -2196,8 +2196,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8839ef7cfb49239e7f1c4812a53a93a672827c88d6921408b1d5062b352c87cc -R 8e79393df1bdcc86ce7609abeba8cbda +P 69205264debd829573b1c777a5a493cfeb6083c4cdec106b1f819989f859ac75 +R c565bdc34845dfb3802fedc964dd8edd U dan -Z 68a8dc4d8e1906054989de2a0038cabe +Z 92c34b492aef5c04b9af8f99ed4665b1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 26be2f42c2..d52188542b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69205264debd829573b1c777a5a493cfeb6083c4cdec106b1f819989f859ac75 +4e3e529313c86b42d5f1ba113d0714d1545bc6641e09e4537d622057f0352989 From 8f14c4519782942cd659df0ad8acd2b115b420a5 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Jul 2024 20:31:17 +0000 Subject: [PATCH 06/99] Fix xColumnText() and xColumnSize() APIs. Add xColumnLocale() and xTokenizeSetLocale(). FossilOrigin-Name: 14006711d83d098e665c540b978b0e29aa8f458da1c2c8e9c2baf2ad5ebd502c --- ext/fts5/fts5.h | 40 +++++++++- ext/fts5/fts5_main.c | 134 ++++++++++++++++++++++++++++------ ext/fts5/fts5_storage.c | 12 +++ ext/fts5/fts5_tcl.c | 15 ++++ ext/fts5/test/fts5_common.tcl | 13 ++++ ext/fts5/test/fts5locale.test | 87 ++++++++++++++++++++++ manifest | 22 +++--- manifest.uuid | 2 +- 8 files changed, 289 insertions(+), 36 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index ccdebbe1ab..ecdb944f68 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -308,9 +308,42 @@ struct Fts5PhraseIter { ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. +** +** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the locale associated +** with column iCol of the current row. Usually, there is no associated +** locale, and output parameters (*pzLocale) and (*pnLocale) are set +** to NULL and 0, respectively. However, if the fts5_locale() function +** was used to associated a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a buffer +** containing the name of the locale in utf-8 encoding. (*pnLocale) is +** set to the size in bytes of the buffer. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an +** SQLite error code is returned. The final value of the output parameters +** is undefined in this case. +** +** xTokenizeSetLocale(pFts5, pLocale, nLocale) +** This API function is used to invoke the fts5_tokenizer_v2.xSetLocale() +** method of the fts5 table's tokenizer, in the same way as xTokenize() is +** used to invoke fts5_tokenizer_v2.xTokenize(). +** +** Parameters pLocale and nLocale may both be 0, in which case the tokenizer +** is configured to use its default locale. Otherwise, pLocale should point +** to a buffer containing the name of the locale to use encoded as utf-8. +** It does not have to be nul-terminated. nLocale must be passed the size +** of the text in bytes. The buffer indicated by pLocale must remain valid +** for the duration of any calls made to xTokenize() by the auxiliary +** function call up until the next invocation of xTokenizeSetLocale(), if +** any. +** +** SQLITE_OK is returned on success, or an SQLite error code otherwise. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 4 */ void *(*xUserData)(Fts5Context*); @@ -352,6 +385,11 @@ struct Fts5ExtensionApi { const char **ppToken, int *pnToken ); int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + + /* Below this point are iVersion>=4 only */ + int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); + + int (*xTokenizeSetLocale)(Fts5Context*, const char *z, int n); }; /* diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 7fb08fd6ed..b57864d8b0 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -195,11 +195,15 @@ struct Fts5Cursor { Fts5Auxiliary *pAux; /* Currently executing extension function */ Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ - /* Cache used by auxiliary functions xInst() and xInstCount() */ + /* Cache used by auxiliary API functions xInst() and xInstCount() */ Fts5PoslistReader *aInstIter; /* One for each phrase */ int nInstAlloc; /* Size of aInst[] array (entries / 3) */ int nInstCount; /* Number of phrase instances */ int *aInst; /* 3 integers per phrase instance */ + + /* Values set by xTokenizeSetLocale() */ + const char *pLocale; + int nLocale; }; /* @@ -1258,16 +1262,17 @@ int sqlite3Fts5ExtractText( ){ const char *pText = 0; int nText = 0; - int bResetTokenizer = 0; int rc = SQLITE_OK; - int bDecodeBlob = 0; + + assert( pbResetTokenizer==0 || *pbResetTokenizer==0 ); + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ - if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE - || (bContent && pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL) - ){ - bDecodeBlob = 1; - } + if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE + || (bContent && pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL) + ){ + bDecodeBlob = 1; + } } if( bDecodeBlob ){ @@ -1283,8 +1288,10 @@ int sqlite3Fts5ExtractText( pText = (const char*)&pBlob[nLocale+1]; nText = nBlob-nLocale-1; - rc = fts5SetLocale(pConfig, (const char*)pBlob, nLocale); - bResetTokenizer = 1; + if( pbResetTokenizer ){ + rc = fts5SetLocale(pConfig, (const char*)pBlob, nLocale); + *pbResetTokenizer = 1; + } }else{ pText = (const char*)sqlite3_value_text(pVal); @@ -1293,7 +1300,6 @@ int sqlite3Fts5ExtractText( *ppText = pText; *pnText = nText; - *pbResetTokenizer = bResetTokenizer; return rc; } @@ -2030,9 +2036,20 @@ static int fts5ApiTokenize( ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - return sqlite3Fts5Tokenize( - pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken - ); + int rc = SQLITE_OK; + const char *pLocale = pCsr->pLocale; + if( pLocale ){ + rc = fts5SetLocale(pTab->pConfig, pLocale, pCsr->nLocale); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize( + pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + ); + } + if( pLocale ){ + sqlite3Fts5ClearLocale(pTab->pConfig); + } + return rc; } static int fts5ApiPhraseCount(Fts5Context *pCtx){ @@ -2064,8 +2081,13 @@ static int fts5ApiColumnText( }else{ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + Fts5Config *pConfig = pTab->pConfig; + int bContent = ( + pConfig->bLocale && pConfig->abUnindexed[iCol]==0 && + pConfig->eContent==FTS5_CONTENT_NORMAL + ); + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + sqlite3Fts5ExtractText(pConfig, bContent, pVal, 0, pz, pn); } } return rc; @@ -2277,16 +2299,22 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ } }else{ int i; + rc = fts5SeekCursor(pCsr, 0); for(i=0; rc==SQLITE_OK && inCol; i++){ if( pConfig->abUnindexed[i]==0 ){ - const char *z; int n; - void *p = (void*)(&pCsr->aColumnSize[i]); + const int bContent = (pConfig->eContent==FTS5_CONTENT_NORMAL); + const char *z = 0; + int n = 0; + int bReset = 0; + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1); + pCsr->aColumnSize[i] = 0; - rc = fts5ApiColumnText(pCtx, i, &z, &n); + rc = sqlite3Fts5ExtractText(pConfig, bContent, pVal, &bReset, &z, &n); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5Tokenize( - pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, + z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb ); + if( bReset ) sqlite3Fts5ClearLocale(pConfig); } } } @@ -2530,8 +2558,64 @@ static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); +static int fts5ApiColumnLocale( + Fts5Context *pCtx, + int iCol, + const char **pzLocale, + int *pnLocale +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + *pzLocale = 0; + *pnLocale = 0; + + if( iCol<0 || iCol>=pConfig->nCol ){ + rc = SQLITE_RANGE; + }else{ + int bNormal = (pConfig->eContent==FTS5_CONTENT_NORMAL); + if( pConfig->abUnindexed[iCol]==0 + && pCsr->ePlan!=FTS5_PLAN_SPECIAL + && pConfig->eContent!=FTS5_CONTENT_NONE + && (bNormal==0 || pConfig->bLocale) + ){ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + if( sqlite3_value_type(pVal)==SQLITE_BLOB + && (bNormal || sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE) + ){ + const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + int nLocale = 0; + for(nLocale=0; nLocalepLocale = pLocale; + pCsr->nLocale = nLocale; + return rc; +} + static const Fts5ExtensionApi sFts5Api = { - 3, /* iVersion */ + 4, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, @@ -2552,7 +2636,9 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, fts5ApiQueryToken, - fts5ApiInstToken + fts5ApiInstToken, + fts5ApiColumnLocale, + fts5ApiTokenizeSetLocale }; /* @@ -2606,6 +2692,8 @@ static void fts5ApiInvoke( pCsr->pAux = pAux; pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); pCsr->pAux = 0; + pCsr->pLocale = 0; + pCsr->nLocale = 0; } static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index f94c2ea908..72a136f421 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -839,7 +839,19 @@ int sqlite3Fts5StorageContentInsert( sqlite3_value *pVal = apVal[i]; if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ pVal = sqlite3_column_value(p->pSavedRow, i-1); + }else if( i>1 && pConfig->abUnindexed[i-2] + && pConfig->bLocale + && sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE + ){ + /* At attempt to insert an fts5_locale() value into an UNINDEXED + ** column. Strip the locale away and just bind the text. */ + const char *pText = 0; + int nText = 0; + rc = sqlite3Fts5ExtractText(pConfig, 0, pVal, 0, &pText, &nText); + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + continue; } + rc = sqlite3_bind_value(pInsert, i, pVal); } if( rc==SQLITE_OK ){ diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index a0d951526d..fb951291a6 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -247,6 +247,7 @@ static int SQLITE_TCLAPI xF5tApi( { "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */ { "xInstToken", 2, "IDX ITERM" }, /* 19 */ + { "xColumnLocale", 1, "COL" }, /* 20 */ { 0, 0, 0} }; @@ -535,6 +536,20 @@ static int SQLITE_TCLAPI xF5tApi( break; } + CASE(20, "xColumnLocale") { + const char *z = 0; + int n = 0; + int iCol; + if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){ + return TCL_ERROR; + } + rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n); + if( rc==SQLITE_OK && z ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n)); + } + break; + } + default: assert( 0 ); break; diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 7076a52bbf..b78064e8d8 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -78,6 +78,9 @@ proc fts5_test_columnsize {cmd} { proc fts5_columntext {cmd iCol} { $cmd xColumnText $iCol } +proc fts5_columnlocale {cmd iCol} { + $cmd xColumnLocale $iCol +} proc fts5_test_columntext {cmd} { set res [list] @@ -87,6 +90,14 @@ proc fts5_test_columntext {cmd} { set res } +proc fts5_test_columnlocale {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { + lappend res [$cmd xColumnLocale $i] + } + set res +} + proc fts5_test_columntotalsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -165,6 +176,7 @@ proc fts5_aux_test_functions {db} { foreach f { fts5_test_columnsize fts5_test_columntext + fts5_test_columnlocale fts5_test_columntotalsize fts5_test_poslist fts5_test_poslist2 @@ -177,6 +189,7 @@ proc fts5_aux_test_functions {db} { fts5_test_queryphrase fts5_test_phrasecount fts5_columntext + fts5_columnlocale fts5_queryphrase fts5_collist } { diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index f508ea1335..71a76677ae 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -41,7 +41,13 @@ proc transform_token {locale token} { proc tcl_create {args} { return "tcl_tokenize" } proc tcl_tokenize {tflags text} { + set iToken 1 + set bSkip 0 + if {[sqlite3_fts5_locale]=="second"} { set bSkip 1 } foreach {w iStart iEnd} [fts5_tokenize_split $text] { + incr iToken + if {(($iToken) % ($bSkip + 1))} continue + set w [transform_token [sqlite3_fts5_locale] $w] sqlite3_fts5_token $w $iStart $iEnd } @@ -216,6 +222,87 @@ do_execsql_test 4.7 { 4 jkl hello } +fts5_aux_test_functions db + +do_execsql_test 4.8.1 { + SELECT fts5_test_columntext(d1) FROM d1('jkl') +} {{jkl hello}} +do_execsql_test 4.8.2 { + SELECT fts5_test_columntext(d1) FROM d1(fts5_locale('reverse', 'hello')) +} {{jkl hello}} + +do_execsql_test 4.9 { + SELECT fts5_test_columnlocale(d1) FROM d1(fts5_locale('reverse', 'hello')) +} {{{} reverse}} + +do_execsql_test 4.10 { + SELECT fts5_test_columnlocale(d1) FROM d1 +} { + {{} {}} + {{} reverse} +} + +#------------------------------------------------------------------------- +# Test that if an fts5_locale() value is written to an UNINDEXED +# column it is stored as text. This is so that blobs and other values +# can also be stored as is. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, y UNINDEXED, locale=1, tokenize=tcl + ); + + INSERT INTO t1(rowid, x, y) VALUES(111, + fts5_locale('reverse', 'one two three'), + fts5_locale('reverse', 'four five six') + ); +} + +do_execsql_test 5.2 { + SELECT rowid, x, y FROM t1 +} { + 111 {one two three} {four five six} +} + +do_execsql_test 5.3 { + SELECT typeof(c0), typeof(c1) FROM t1_content +} { + blob text +} + +#------------------------------------------------------------------------- + +foreach {tn opt} { + 1 {} + 2 {, columnsize=0} +} { + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + do_execsql_test 6.$tn.1 " + CREATE VIRTUAL TABLE y1 USING fts5(t, locale=1, tokenize=tcl $opt); + " + + do_execsql_test 6.$tn.2 { + INSERT INTO y1(rowid, t) VALUES + (1, fts5_locale('second', 'the city of London')), + (2, fts5_locale('second', 'shall have all the old')), + (3, fts5_locale('second', 'Liberties and Customs')), + (4, fts5_locale('second', 'which it hath been used to have')); + } + + fts5_aux_test_functions db + + do_execsql_test 5.$tn.3 { + SELECT fts5_test_columnsize(y1) FROM y1 + } { + 2 3 2 4 + } +} + + finish_test diff --git a/manifest b/manifest index 594ebdb848..8fb2030236 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\sasan\sproblems\son\sthis\sbranch. -D 2024-07-29T15:00:11.833 +C Fix\sxColumnText()\sand\sxColumnSize()\sAPIs.\sAdd\sxColumnLocale()\sand\sxTokenizeSetLocale(). +D 2024-07-29T20:31:17.644 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h 38a9553791828b3cf677b9347735fc531d54015ce4f5229d5cf1e2a5c1d3955a +F ext/fts5/fts5.h 7cd47e5dfe68d798e667caa76722374f0c909f2db05bb4d42b1ec5300d18e658 F ext/fts5/fts5Int.h 330b1e2dad9ea9cccc9fa93817062fa21e89f00e7eac9a84be440f7e93bf7c3c F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 @@ -100,9 +100,9 @@ F ext/fts5/fts5_config.c 0c96490fbad746b3780174f38b2ee5e3d719f2f81ee6b58ca828772 F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 71ea9fe3aba423ad325a7ac600b83bb79e2a640f62094f6df356f978245c78db -F ext/fts5/fts5_storage.c 6beefaa9229193e85fe9b920aa62f46a79cac867feb8a5af8ce6df28dec90e8d -F ext/fts5/fts5_tcl.c e22e5791076fdfcbcd2dd5e45aff9d3615f7840bae5d984088ce2c24cfdf77b4 +F ext/fts5/fts5_main.c ee4bba42d8d093cd870c55a342ab85a647ec39dc79985e2b1084514d990de82d +F ext/fts5/fts5_storage.c 2118a5bd2ae617367259f6cf2412b6ff26bc1cd1cc29bf64a6249edcc1572f19 +F ext/fts5/fts5_tcl.c 93b705cb87633574983161edc5234f9b91ba03f9fecfbd2c5d401a1da6f93aa5 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c fa5493075101540270f572038fc1723d44fcc97bfbf237c8530013b8a27860be @@ -111,7 +111,7 @@ F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d0988 F ext/fts5/fts5_vocab.c e4830b00809e5da53bc10f93adc59e321407b0f801c7f4167c0e47f5552267e0 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl bc33c6cc65e5d390f28a68aeeb3a526dadd2c3a947d2466ee1986c1a4002df56 +F ext/fts5/test/fts5_common.tcl 48c0855d235a2f141e5519d3170bad7cddcc53ad3a16ac8ef952ba61048c2958 F ext/fts5/test/fts5aa.test 4db81519863244a3cab35795fe65ab6b592e7970c7409eba098b23ebbfc08d95 F ext/fts5/test/fts5ab.test 4bdb619fee409e11417e8827e320b857e42e926a01a0408fc9f143ec870a6ced F ext/fts5/test/fts5ac.test 4a73626de86f3d17c95738034880c4f0de8d54741fb943d819b528373657e59b @@ -184,7 +184,7 @@ F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fd F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 60217c6f67331e2b3218b4da3f96f54b856a341b23e7328c9f30efe9ca343130 +F ext/fts5/test/fts5locale.test 14e46de0fbab3a01e16d8f61143260a7865cbe9d1759a564927635a1b08e9310 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -2196,8 +2196,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 69205264debd829573b1c777a5a493cfeb6083c4cdec106b1f819989f859ac75 -R c565bdc34845dfb3802fedc964dd8edd +P 4e3e529313c86b42d5f1ba113d0714d1545bc6641e09e4537d622057f0352989 +R 76149d45da5064496ccc1cbc5f33d6bf U dan -Z 92c34b492aef5c04b9af8f99ed4665b1 +Z 4addb1a2e7bbe8797df686b1934d0dc8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d52188542b..eb3a346d11 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e3e529313c86b42d5f1ba113d0714d1545bc6641e09e4537d622057f0352989 +14006711d83d098e665c540b978b0e29aa8f458da1c2c8e9c2baf2ad5ebd502c From 8174305e175bf352d47f6b3023b4b664abeb5ef6 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 30 Jul 2024 15:55:51 +0000 Subject: [PATCH 07/99] Update the highlight() and snippet() functions to use locales when available. FossilOrigin-Name: 569ae56a673a2e101fac7003a1cb41c7a02a515adf55bd1e1f2a03c19eb2b085 --- ext/fts5/fts5_aux.c | 29 +++++++++++++++++++++++++++++ ext/fts5/test/fts5locale.test | 34 ++++++++++++++++++++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 30101fbe25..e2226e3f03 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -226,6 +226,27 @@ static int fts5HighlightCb( return rc; } +/* +** Use xTokenizeSetLocale() to configure the tokenizer to use the locale +** associated with column iCol of the current row. +*/ +static int fts5ConfigureTokenizer( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + int iCol +){ + int rc = SQLITE_OK; + const char *zLocale = 0; + int nLocale = 0; + + rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale); + if( rc==SQLITE_OK ){ + rc = pApi->xTokenizeSetLocale(pFts, zLocale, nLocale); + } + return rc; +} + + /* ** Implementation of highlight() function. */ @@ -260,6 +281,9 @@ static void fts5HighlightFunction( rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); } + if( rc==SQLITE_OK ){ + rc = fts5ConfigureTokenizer(pApi, pFts, iCol); + } if( rc==SQLITE_OK ){ rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); } @@ -465,6 +489,8 @@ static void fts5SnippetFunction( sFinder.nFirst = 0; rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc); if( rc!=SQLITE_OK ) break; + rc = fts5ConfigureTokenizer(pApi, pFts, i); + if( rc!=SQLITE_OK ) break; rc = pApi->xTokenize(pFts, sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb ); @@ -541,6 +567,9 @@ static void fts5SnippetFunction( rc = fts5CInstIterNext(&ctx.iter); } + if( rc==SQLITE_OK ){ + rc = fts5ConfigureTokenizer(pApi, pFts, iBestCol); + } if( rc==SQLITE_OK ){ rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); } diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index 71a76677ae..1124a44436 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -300,6 +300,40 @@ foreach {tn opt} { } { 2 3 2 4 } + + do_execsql_test 5.$tn.4 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); + } { + 2 3 + } + + do_execsql_test 5.$tn.5 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); + } { + 2 3 + } + + do_execsql_test 5.$tn.6 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('have'); + } { + 4 4 + } + + do_execsql_test 5.$tn.7 { + SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have'); + } { + 4 {which it hath been used to [have]} + } + + do_execsql_test 5.$tn.8 { + SELECT rowid, + highlight(y1, 0, '[', ']'), + snippet(y1, 0, '[', ']', '...', 10) + FROM y1('Liberties + Customs'); + } { + 3 {[Liberties and Customs]} + {[Liberties and Customs]} + } } diff --git a/manifest b/manifest index 8fb2030236..89acd2b99f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sxColumnText()\sand\sxColumnSize()\sAPIs.\sAdd\sxColumnLocale()\sand\sxTokenizeSetLocale(). -D 2024-07-29T20:31:17.644 +C Update\sthe\shighlight()\sand\ssnippet()\sfunctions\sto\suse\slocales\swhen\savailable. +D 2024-07-30T15:55:51.884 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -94,7 +94,7 @@ F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a0 F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e F ext/fts5/fts5.h 7cd47e5dfe68d798e667caa76722374f0c909f2db05bb4d42b1ec5300d18e658 F ext/fts5/fts5Int.h 330b1e2dad9ea9cccc9fa93817062fa21e89f00e7eac9a84be440f7e93bf7c3c -F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1 +F ext/fts5/fts5_aux.c 652f839dc0c77431295f10b08f268631560bb5630e65fd701de7a58744428a82 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 0c96490fbad746b3780174f38b2ee5e3d719f2f81ee6b58ca828772871e0f680 F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad @@ -184,7 +184,7 @@ F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fd F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 14e46de0fbab3a01e16d8f61143260a7865cbe9d1759a564927635a1b08e9310 +F ext/fts5/test/fts5locale.test 7c332f882080a69ed4be9790d99ee49747ca44cf241d45f762298e17fea6e7a7 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -2196,8 +2196,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4e3e529313c86b42d5f1ba113d0714d1545bc6641e09e4537d622057f0352989 -R 76149d45da5064496ccc1cbc5f33d6bf +P 14006711d83d098e665c540b978b0e29aa8f458da1c2c8e9c2baf2ad5ebd502c +R 8ced4a5b525d31957945a9ad2a3de4d9 U dan -Z 4addb1a2e7bbe8797df686b1934d0dc8 +Z 395467bf290b92744f082b81bcb8dffe # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index eb3a346d11..634c3661ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14006711d83d098e665c540b978b0e29aa8f458da1c2c8e9c2baf2ad5ebd502c +569ae56a673a2e101fac7003a1cb41c7a02a515adf55bd1e1f2a03c19eb2b085 From 9015b8822df620e23ca692d6dfbad3825aac14b2 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 30 Jul 2024 20:39:58 +0000 Subject: [PATCH 08/99] Fix cases surrounding blob values being stored in locale=1 fts5 tables. FossilOrigin-Name: ae435aff1785d5832821c19dc88ccf6c496a7ff55be80276b31e5c9abee723db --- ext/fts5/fts5Int.h | 2 + ext/fts5/fts5_main.c | 92 ++++++++++++++++++++++++++--------- ext/fts5/fts5_storage.c | 35 +++++++++---- ext/fts5/test/fts5blob.test | 77 +++++++++++++++++++++++++++++ ext/fts5/test/fts5locale.test | 13 +++++ manifest | 19 ++++---- manifest.uuid | 2 +- 7 files changed, 198 insertions(+), 42 deletions(-) create mode 100644 ext/fts5/test/fts5blob.test diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 1cc25ae7e0..14140658b2 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -228,6 +228,8 @@ struct Fts5Config { char *zContentExprlist; Fts5TokenizerConfig t; int bLock; /* True when table is preparing statement */ + int eEnc; /* An FTS5_ENCODING_XXX constant */ + /* Values loaded from the %_config table */ int iVersion; /* fts5 file format 'version' */ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index b57864d8b0..673a44d139 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -118,12 +118,16 @@ struct Fts5FullTable { Fts5Global *pGlobal; /* Global (connection wide) data */ Fts5Cursor *pSortCsr; /* Sort data from this cursor */ int iSavepoint; /* Successful xSavepoint()+1 */ - + #ifdef SQLITE_DEBUG struct Fts5TransactionState ts; #endif }; +#define FTS5_ENCODING_UNKNOWN 0 +#define FTS5_ENCODING_UTF8 1 +#define FTS5_ENCODING_UTF16 2 + struct Fts5MatchPhrase { Fts5Buffer *pPoslist; /* Pointer to current poslist */ int nTerm; /* Size of phrase in terms */ @@ -1252,6 +1256,30 @@ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ fts5SetLocale(pConfig, 0, 0); } +static int fts5IsUtf16(Fts5Config *pConfig, int *pbIs){ + if( pConfig->eEnc==FTS5_ENCODING_UNKNOWN ){ + sqlite3_stmt *pPragma = 0; + int rc = fts5PrepareStatement(&pPragma, pConfig, + "SELECT (encoding LIKE '%%16%%') FROM pragma_encoding" + ); + if( rc==SQLITE_OK ){ + int val; + sqlite3_step(pPragma); + val = sqlite3_column_int(pPragma, 0); + rc = sqlite3_finalize(pPragma); + if( rc!=SQLITE_OK ) return rc; + if( val ){ + pConfig->eEnc = FTS5_ENCODING_UTF16; + }else{ + pConfig->eEnc = FTS5_ENCODING_UTF8; + } + } + } + + *pbIs = (pConfig->eEnc==FTS5_ENCODING_UTF16); + return SQLITE_OK; +} + int sqlite3Fts5ExtractText( Fts5Config *pConfig, int bContent, @@ -1280,17 +1308,33 @@ int sqlite3Fts5ExtractText( int nBlob = sqlite3_value_bytes(pVal); int nLocale = 0; - for(nLocale=0; nLocale=4 && memcmp(pBlob, "\0\0\0\0", 4)==0 ){ + int bIs16 = 0; + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + rc = fts5IsUtf16(pConfig, &bIs16); + + if( bIs16 ){ + pText += 2; + nText -= 2; + }else{ + pText += 4; + nText -= 4; + } - if( nLocale==nBlob ) return SQLITE_ERROR; - pText = (const char*)&pBlob[nLocale+1]; - nText = nBlob-nLocale-1; + }else{ + for(nLocale=0; nLocalepSavedRow ){ pVal = sqlite3_column_value(p->pSavedRow, i-1); - }else if( i>1 && pConfig->abUnindexed[i-2] - && pConfig->bLocale - && sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE - ){ - /* At attempt to insert an fts5_locale() value into an UNINDEXED - ** column. Strip the locale away and just bind the text. */ - const char *pText = 0; - int nText = 0; - rc = sqlite3Fts5ExtractText(pConfig, 0, pVal, 0, &pText, &nText); - sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){ + if( pConfig->bLocale==0 ){ + sqlite3Fts5ConfigErrmsg(pConfig, + "fts5_locale() may not be used without locale=1" + ); + rc = SQLITE_ERROR; + break; + }else if( i>1 && pConfig->abUnindexed[i-2] ){ + /* At attempt to insert an fts5_locale() value into an UNINDEXED + ** column. Strip the locale away and just bind the text. */ + const char *pText = 0; + int nText = 0; + rc = sqlite3Fts5ExtractText(pConfig, 0, pVal, 0, &pText, &nText); + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + continue; + } + }else if( pConfig->bLocale && sqlite3_value_type(pVal)==SQLITE_BLOB ){ + /* Inserting a blob into a normal content table with locale=1. */ + int n = sqlite3_value_bytes(pVal); + u8 *pBlob = sqlite3Fts5MallocZero(&rc, n+4); + if( pBlob ){ + memcpy(&pBlob[4], sqlite3_value_blob(pVal), n); + rc = sqlite3_bind_blob(pInsert, i, pBlob, n+4, SQLITE_TRANSIENT); + sqlite3_free(pBlob); + } continue; } diff --git a/ext/fts5/test/fts5blob.test b/ext/fts5/test/fts5blob.test new file mode 100644 index 0000000000..9c838b75a6 --- /dev/null +++ b/ext/fts5/test/fts5blob.test @@ -0,0 +1,77 @@ +# 2024 July 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5blob + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach {tn enc locale} { + 1 utf8 0 + 2 utf8 1 + 3 utf16 0 + 4 utf16 1 +} { + reset_db + fts5_aux_test_functions db + + execsql "PRAGMA encoding = $enc" + + if {$tn==3 || $tn==4} breakpoint + + execsql " + CREATE VIRTUAL TABLE t1 USING fts5(x, y, locale=$locale); + " + do_execsql_test 1.$tn.0 { + CREATE VIRTUAL TABLE tt USING fts5vocab('t1', 'instance'); + INSERT INTO t1(rowid, x, y) VALUES(1, 555, X'0000000041424320444546'); + INSERT INTO t1(rowid, x, y) VALUES(2, 666, X'41424300444546'); + INSERT INTO t1(rowid, x, y) VALUES(3, 777, 'xyz'); + } + + do_execsql_test 1.$tn.1 { + SELECT rowid, quote(x), quote(y) FROM t1 + } { + 1 555 X'0000000041424320444546' + 2 666 X'41424300444546' + 3 777 'xyz' + } + + set T($enc,$locale) [execsql { SELECT * FROM tt }] + set U($enc,$locale) [execsql { SELECT fts5_test_columntext(t1) FROM t1 }] + + do_execsql_test 1.$tn.2 { + DELETE FROM t1 WHERE rowid=2; + DELETE FROM t1 WHERE rowid=1; + } + + do_execsql_test 1.$tn.3 { + PRAGMA integrity_check; + } {ok} +} + +do_test 1.5.1 { set T(utf8,1) } $T(utf8,0) +do_test 1.5.2 { set T(utf16,1) } $T(utf16,0) + +do_test 1.6.1 { set U(utf8,1) } $U(utf8,0) +do_test 1.6.2 { set U(utf16,1) } $U(utf16,0) + + + +finish_test + + diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index 1124a44436..d3fd9989c0 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -336,6 +336,19 @@ foreach {tn opt} { } } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} +do_catchsql_test 6.1 { + INSERT INTO x1(rowid, x) VALUES(123, fts5_locale('en_AU', 'hello world')); +} {1 {fts5_locale() may not be used without locale=1}} + +do_execsql_test 6.2 { + SELECT typeof( fts5_locale(NULL, 'xyz') ), typeof( fts5_locale('', 'abc') ); +} {text text} + finish_test diff --git a/manifest b/manifest index 89acd2b99f..156f1a64a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\shighlight()\sand\ssnippet()\sfunctions\sto\suse\slocales\swhen\savailable. -D 2024-07-30T15:55:51.884 +C Fix\scases\ssurrounding\sblob\svalues\sbeing\sstored\sin\slocale=1\sfts5\stables. +D 2024-07-30T20:39:58.636 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -93,15 +93,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e F ext/fts5/fts5.h 7cd47e5dfe68d798e667caa76722374f0c909f2db05bb4d42b1ec5300d18e658 -F ext/fts5/fts5Int.h 330b1e2dad9ea9cccc9fa93817062fa21e89f00e7eac9a84be440f7e93bf7c3c +F ext/fts5/fts5Int.h 833a2fe729f926ebcde47e21e495d141b99ede9a188fc577873f24bea0f0bfa2 F ext/fts5/fts5_aux.c 652f839dc0c77431295f10b08f268631560bb5630e65fd701de7a58744428a82 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 0c96490fbad746b3780174f38b2ee5e3d719f2f81ee6b58ca828772871e0f680 F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c ee4bba42d8d093cd870c55a342ab85a647ec39dc79985e2b1084514d990de82d -F ext/fts5/fts5_storage.c 2118a5bd2ae617367259f6cf2412b6ff26bc1cd1cc29bf64a6249edcc1572f19 +F ext/fts5/fts5_main.c d9a3fef86887e373027d48ab8216ab8caca95f2316e05d953d8885633162a9c5 +F ext/fts5/fts5_storage.c cc6173bb755d668573169c038034a9ec8deadd10b3c10c145adbdf04ab5f889c F ext/fts5/fts5_tcl.c 93b705cb87633574983161edc5234f9b91ba03f9fecfbd2c5d401a1da6f93aa5 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -132,6 +132,7 @@ F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06c F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36 +F ext/fts5/test/fts5blob.test a16160688e181a212bcb4968325c57ef2864a0bcae0794d6a1e16185007f00b9 F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c2be25568cc33a F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62 F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897 @@ -184,7 +185,7 @@ F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fd F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 7c332f882080a69ed4be9790d99ee49747ca44cf241d45f762298e17fea6e7a7 +F ext/fts5/test/fts5locale.test bfd8704f9bea963314fcbcf810f08a357ac8035bcb80a2d6170c1e57fa6ad52a F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -2196,8 +2197,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14006711d83d098e665c540b978b0e29aa8f458da1c2c8e9c2baf2ad5ebd502c -R 8ced4a5b525d31957945a9ad2a3de4d9 +P 569ae56a673a2e101fac7003a1cb41c7a02a515adf55bd1e1f2a03c19eb2b085 +R 937929d817b727d57bac9fb974f93247 U dan -Z 395467bf290b92744f082b81bcb8dffe +Z 807f8f93a084e88be44a4f6a2009b39e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 634c3661ab..dfceffa3c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -569ae56a673a2e101fac7003a1cb41c7a02a515adf55bd1e1f2a03c19eb2b085 +ae435aff1785d5832821c19dc88ccf6c496a7ff55be80276b31e5c9abee723db From 8e3042b5657fc41ab1c01881b80754c71b17aa95 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 Jul 2024 15:46:41 +0000 Subject: [PATCH 09/99] Add missing documentation for new features to fts5.h. FossilOrigin-Name: fa0da3b28e411affd45a918d6b7faba49f744ca8b4adf3cd5ce6609bb630499e --- ext/fts5/fts5.h | 94 +++++++++++++++++++++++++++++++++---------------- manifest | 14 ++++---- manifest.uuid | 2 +- 3 files changed, 72 insertions(+), 38 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index ecdb944f68..2f6c387760 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -317,10 +317,11 @@ struct Fts5PhraseIter { ** with column iCol of the current row. Usually, there is no associated ** locale, and output parameters (*pzLocale) and (*pnLocale) are set ** to NULL and 0, respectively. However, if the fts5_locale() function -** was used to associated a locale with the value when it was inserted -** into the fts5 table, then (*pzLocale) is set to point to a buffer -** containing the name of the locale in utf-8 encoding. (*pnLocale) is -** set to the size in bytes of the buffer. +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. ** ** If successful, SQLITE_OK is returned. Or, if an error occurs, an ** SQLite error code is returned. The final value of the output parameters @@ -333,7 +334,7 @@ struct Fts5PhraseIter { ** ** Parameters pLocale and nLocale may both be 0, in which case the tokenizer ** is configured to use its default locale. Otherwise, pLocale should point -** to a buffer containing the name of the locale to use encoded as utf-8. +** to a buffer containing the utf-8 encoded name of the locale to use. ** It does not have to be nul-terminated. nLocale must be passed the size ** of the text in bytes. The buffer indicated by pLocale must remain valid ** for the duration of any calls made to xTokenize() by the auxiliary @@ -388,8 +389,7 @@ struct Fts5ExtensionApi { /* Below this point are iVersion>=4 only */ int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); - - int (*xTokenizeSetLocale)(Fts5Context*, const char *z, int n); + int (*xTokenizeSetLocale)(Fts5Context*, const char *p, int n); }; /* @@ -401,16 +401,17 @@ struct Fts5ExtensionApi { ** ** Applications may also register custom tokenizer types. A tokenizer ** is registered by providing fts5 with a populated instance of the -** following structure. All structure methods must be defined, setting -** any member of the fts5_tokenizer struct to NULL leads to undefined -** behaviour. The structure methods are expected to function as follows: +** following structure. Of the three structure methods, xCreate, xDelete and +** xTokenize must be supplied, any fo these three members of the +** fts5_tokenizer_v2 struct to NULL leads to undefined behaviour. The +** structure methods are expected to function as follows: ** ** xCreate: ** This function is used to allocate and initialize a tokenizer instance. ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) -** pointer provided by the application when the fts5_tokenizer object +** pointer provided by the application when the fts5_tokenizer_v2 object ** was registered with FTS5 (the third argument to xCreateTokenizer()). ** The second and third arguments are an array of nul-terminated strings ** containing the tokenizer arguments, if any, specified following the @@ -481,6 +482,33 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** xSetLocale: +** This function is invoked by FTS5 to configure the locale to use for +** subsequent calls to xTokenize. The second argument is a pointer to +** a nul-terminated buffer containing the utf-8 encoded name of the locale +** to use. The third argument is the size of the buffer in bytes, not +** including the nul-terminator character. This function may also be +** invoked with the second and third parameters set to 0 - instructing +** the tokenizer to use its default locale. +** +** FTS5 guarantees that any buffer passed to xSetLocale() will remain +** valid until either the next call to xSetLocale() or xDelete() on the +** same tokenizer object. +** +** This function should return SQLITE_OK if successful, or an SQLite +** error code if an error occurs. If an error does occur and an error +** code is returned, execution of the current statement is abandoned +** and FTS5 returns the error code to the caller. +** +** Often, this function is not required and is never invoked. It is only +** ever invoked when processing a value that has had a locale associated +** with it using SQL function fts5_locale(). +** +** It is not necessary to supply an implementation of this method when +** registering a tokenizer. If fts5_tokenizer_v2.xSetLocale is set to NULL, +** then no attempt is made to pass locale information through to the +** tokenizer. +** ** SYNONYM SUPPORT ** ** Custom tokenizers may also support synonyms. Consider a case in which a @@ -589,25 +617,6 @@ struct Fts5ExtensionApi { ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; -typedef struct fts5_tokenizer fts5_tokenizer; -struct fts5_tokenizer { - int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); - void (*xDelete)(Fts5Tokenizer*); - int (*xTokenize)(Fts5Tokenizer*, - void *pCtx, - int flags, /* Mask of FTS5_TOKENIZE_* flags */ - const char *pText, int nText, - int (*xToken)( - void *pCtx, /* Copy of 2nd argument to xTokenize() */ - int tflags, /* Mask of FTS5_TOKEN_* flags */ - const char *pToken, /* Pointer to buffer containing token */ - int nToken, /* Size of token in bytes */ - int iStart, /* Byte offset of token within input text */ - int iEnd /* Byte offset of end of token within input text */ - ) - ); -}; - typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; struct fts5_tokenizer_v2 { int iVersion; /* Currently always 2 */ @@ -631,6 +640,31 @@ struct fts5_tokenizer_v2 { int (*xSetLocale)(Fts5Tokenizer*, const char *pLocale, int nLocale); }; +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + + /* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 diff --git a/manifest b/manifest index a08c7ec6c7..a18841d7c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\schanges\sfrom\strunk\sinto\sthis\sbranch. -D 2024-07-31T10:52:38.878 +C Add\smissing\sdocumentation\sfor\snew\sfeatures\sto\sfts5.h. +D 2024-07-31T15:46:41.319 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h 7cd47e5dfe68d798e667caa76722374f0c909f2db05bb4d42b1ec5300d18e658 +F ext/fts5/fts5.h f5451da088d0004c4be7314e2fdb41fda16ce682ce2aa3b54b9474ebe9a013d5 F ext/fts5/fts5Int.h 833a2fe729f926ebcde47e21e495d141b99ede9a188fc577873f24bea0f0bfa2 F ext/fts5/fts5_aux.c 652f839dc0c77431295f10b08f268631560bb5630e65fd701de7a58744428a82 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 @@ -604,7 +604,7 @@ F ext/wasm/SQLTester/SQLTester.run.mjs c72b7fe2072d05992f7a3d8c6a1d34e95712513ce F ext/wasm/SQLTester/index.html 3f8a016df0776be76605abf20e815ecaafbe055abac0e1fe5ea080e7846b760d F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth 7ac80cc3b6a6d52e041bb295e85555ce797be78c15ef2008a64ae58815014080 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core 400213eb52a7e5ad5f448053d375cacf4dac2cf45d134f3edfe485ae4a49a183 w ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core 400213eb52a7e5ad5f448053d375cacf4dac2cf45d134f3edfe485ae4a49a183 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate d1d62a2212099f2c0782d730beb8cb84a7a52d99c15ead2cb9b1411fff5fd6b1 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session 213b6c04267cb9bd760172db011eb1650732805fb3d01f9395478a8ceec18eb0 @@ -2201,8 +2201,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ae435aff1785d5832821c19dc88ccf6c496a7ff55be80276b31e5c9abee723db 86de4e755e37dc1cbcbd59018927aa87ff49fc15f706a36187631d8f14075c12 -R 854c24bb1cc78c16c421b7985fead9f3 +P ff64939ec8399949393f1029fa6d514892fbf2bf1498545300cc5e120b9622a5 +R f7dbcf0c98ed562d42b896f7c143383b U dan -Z 7ee036ea576e2778dfe1b42f432458d4 +Z 344ab71a9372cfa91d638c7c0c606714 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2864ea1b96..a1e4d24ea7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff64939ec8399949393f1029fa6d514892fbf2bf1498545300cc5e120b9622a5 +fa0da3b28e411affd45a918d6b7faba49f744ca8b4adf3cd5ce6609bb630499e From 5bd8cc7dd53be6c9516c9060a4af9c8bfd6dec9e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 Jul 2024 20:49:00 +0000 Subject: [PATCH 10/99] Fix various problems with the code on this branch. FossilOrigin-Name: 8bd4ae7e95c7b6ce34db5ea705dc136e742a22f333d0e7370b485ebd736b5ec2 --- ext/fts5/fts5Int.h | 16 ++- ext/fts5/fts5_aux.c | 2 + ext/fts5/fts5_config.c | 12 +- ext/fts5/fts5_main.c | 221 ++++++++++++++++++++++++++++-------- ext/fts5/fts5_storage.c | 19 ++-- ext/fts5/test/fts5blob.test | 16 ++- manifest | 22 ++-- manifest.uuid | 2 +- 8 files changed, 235 insertions(+), 75 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 14140658b2..53e269df93 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -206,6 +206,14 @@ struct Fts5TokenizerConfig { ** ** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); ** +** bLocale: +** Set to true if locale=1 was specified when the table was created. +** +** eEnc: +** Set to either FTS5_ENCODING_UNKNOWN, ENCODING_UTF8, or ENCODING_UTF16, +** to indicate the encoding used by the database handle. This is initially +** set to UNKNOWN, then to one of the other two values the first time it +** is required. */ struct Fts5Config { sqlite3 *db; /* Database handle */ @@ -270,6 +278,10 @@ struct Fts5Config { #define FTS5_PATTERN_LIKE 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */ #define FTS5_PATTERN_GLOB 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */ +#define FTS5_ENCODING_UNKNOWN 0 +#define FTS5_ENCODING_UTF8 1 +#define FTS5_ENCODING_UTF16 2 + int sqlite3Fts5ConfigParse( Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** ); @@ -634,9 +646,9 @@ int sqlite3Fts5FlushToDisk(Fts5Table*); int sqlite3Fts5ExtractText( Fts5Config *pConfig, - int bContent, /* Loaded from content table */ sqlite3_value *pVal, /* Value to extract text from */ - int *pbResetTokenizer, /* OUT: True if xSetLocale(NULL) required */ + int bContent, /* Loaded from content table */ + int *pbResetTokenizer, /* OUT: True if ClearLocale() required */ const char **ppText, /* OUT: Pointer to text buffer */ int *pnText /* OUT: Size of (*ppText) in bytes */ ); diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index e2226e3f03..8695369f5e 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -239,6 +239,8 @@ static int fts5ConfigureTokenizer( const char *zLocale = 0; int nLocale = 0; + assert( pApi->iVersion>=4 ); /* Ensure xColumnLocale() is available */ + rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale); if( rc==SQLITE_OK ){ rc = pApi->xTokenizeSetLocale(pFts, zLocale, nLocale); diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 3736f8685f..1312f459db 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -615,7 +615,9 @@ int sqlite3Fts5ConfigParse( sqlite3_free(zTwo); } - /* If this is not an FTS5_CONTENT_NORMAL table, set bLocale */ + /* If this is not an FTS5_CONTENT_NORMAL table, set bLocale. There are + ** no restrictions on using fts5_locale() with external-content or + ** contentless tables. */ if( pRet->eContent!=FTS5_CONTENT_NORMAL ){ pRet->bLocale = 1; } @@ -1028,7 +1030,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ rc = SQLITE_ERROR; if( pConfig->pzErrmsg ){ assert( 0==*pConfig->pzErrmsg ); - *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format " + sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format " "(found %d, expected %d or %d) - run 'rebuild'", iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE ); @@ -1043,6 +1045,11 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ return rc; } +/* +** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer +** containing the error message created using printf() style formatting +** string zFmt and its trailing arguments. +*/ void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ va_list ap; /* ... printf arguments */ char *zMsg = 0; @@ -1050,6 +1057,7 @@ void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ va_start(ap, zFmt); zMsg = sqlite3_vmprintf(zFmt, ap); if( pConfig->pzErrmsg ){ + assert( *pConfig->pzErrmsg==0 ); *pConfig->pzErrmsg = zMsg; }else{ sqlite3_free(zMsg); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 673a44d139..c9c01aafea 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -124,10 +124,6 @@ struct Fts5FullTable { #endif }; -#define FTS5_ENCODING_UNKNOWN 0 -#define FTS5_ENCODING_UTF8 1 -#define FTS5_ENCODING_UTF16 2 - struct Fts5MatchPhrase { Fts5Buffer *pPoslist; /* Pointer to current poslist */ int nTerm; /* Size of phrase in terms */ @@ -169,6 +165,12 @@ struct Fts5Sorter { ** If the cursor iterates in descending order of rowid, iFirstRowid ** is the upper limit (i.e. the "first" rowid visited) and iLastRowid ** the lower. +** +** pLocale, nLocale: +** These are set by API method xTokenizeSetLocale(). xTokenizeSetLocale() +** does not actually configure the tokenizer, it just stores the values +** it is passed in these variables. The fts5_tokenizer_v2.xSetLocale() +** method is called from within the xTokenize() API method if required. */ struct Fts5Cursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ @@ -238,6 +240,9 @@ struct Fts5Cursor { #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) +/* +** The subtype values returned by fts5_locale() are tagged with. +*/ #define FTS5_LOCALE_SUBTYPE ((unsigned int)'L') @@ -1003,7 +1008,7 @@ static int fts5PrepareStatement( rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, SQLITE_PREPARE_PERSISTENT, &pRet, 0); if( rc!=SQLITE_OK ){ - *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); + sqlite3Fts5ConfigErrmsg(pConfig, "%s", sqlite3_errmsg(pConfig->db)); } sqlite3_free(zSql); } @@ -1238,7 +1243,11 @@ static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ va_end(ap); } - +/* +** Configure the tokenizer to use the locale specified by nLocale byte +** buffer zLocale. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +*/ static int fts5SetLocale( Fts5Config *pConfig, const char *zLocale, @@ -1252,11 +1261,27 @@ static int fts5SetLocale( return rc; } +/* +** Reset the locale of the tokenizer to its default. +*/ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ fts5SetLocale(pConfig, 0, 0); } -static int fts5IsUtf16(Fts5Config *pConfig, int *pbIs){ +/* +** This function is used to determine if the database handle uses utf-8 or +** a utf-16 encoding. If it uses utf-8, then output parameter (*pbIs) is set +** to 0. Otherwise, if it uses utf-16, then the output parameter is set +** to 1. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. If an error code is returned, the final value of (*pbIs) is +** undefined. +*/ +static int fts5IsUtf16( + Fts5Config *pConfig, /* Configuration object */ + int *pbIs /* OUT: True if utf-16, false if utf-8 */ +){ if( pConfig->eEnc==FTS5_ENCODING_UNKNOWN ){ sqlite3_stmt *pPragma = 0; int rc = fts5PrepareStatement(&pPragma, pConfig, @@ -1280,10 +1305,57 @@ static int fts5IsUtf16(Fts5Config *pConfig, int *pbIs){ return SQLITE_OK; } +/* +** This function is used to extract utf-8 text from an sqlite3_value. This +** is usually done in order to tokenize it. For example, when: +** +** * a value is written to an fts5 table, +** * a value is deleted from an FTS5_CONTENT_NORMAL table, +** * a value containing a query expression is passed to xFilter() +** +** and so on. +** +** This function handles 3 cases: +** +** 1) Ordinary values. The text can be extracted from these using +** sqlite3_value_text(). +** +** 2) Blobs tagged with sub-type FTS5_LOCALE_SUBTYPE, or those read from +** the content table of an FTS5_CONTENT_NORMAL table with locale=1 +** set that do not begin with 0x00. +** +** In these cases the value is a blob formatted by fts5_locale() that +** contains both a locale and a text value. The locale is first, as +** utf-8, followed by a single 0x00 byte, followed by the text value, +** also as utf-8. There is no nul-terminator for the text value. +** +** 3) Blobs read from the content table of an FTS5_CONTENT_NORMAL table +** with locale=1 set that do begin with 0x00. These are used to +** store actual SQLITE_BLOB values written to the fts5 table by the +** user. They begin with 4 0x00 bytes, followed by the blob data as +** specified by the user. +** +** If successful, SQLITE_OK is returned and output parameters (*ppText) +** and (*pnText) are set to point to a buffer containing the extracted utf-8 +** text and its length in bytes, respectively. The buffer is not +** nul-terminated. It has the same lifetime as the sqlite3_value object +** from which it is extracted. +** +** Parameter bContent must be true if the value was read from an indexed +** column (i.e. not UNINDEXED) of the on disk content. In this case, if +** the table is FTS5_CONTENT_NORMAL and locale=1 was specified, special blob +** cases (2) and (3) above will apply. +** +** If pbResetTokenizer is not NULL and if case (2) is used, then the +** tokenizer is configured to use the locale. In this case (*pbResetTokenizer) +** is set to true before returning, to indicate that the caller must +** call sqlite3Fts5ClearLocale() to reset the tokenizer after tokenizing +** the text. +*/ int sqlite3Fts5ExtractText( Fts5Config *pConfig, - int bContent, sqlite3_value *pVal, /* Value to extract text from */ + int bContent, /* True if indexed table content */ int *pbResetTokenizer, /* OUT: True if xSetLocale(NULL) required */ const char **ppText, /* OUT: Pointer to text buffer */ int *pnText /* OUT: Size of (*ppText) in bytes */ @@ -1309,11 +1381,23 @@ int sqlite3Fts5ExtractText( int nLocale = 0; if( nBlob>=4 && memcmp(pBlob, "\0\0\0\0", 4)==0 ){ - int bIs16 = 0; + /* Extract the text from a blob stored in a locale=1 table. The + ** value consists of 4 0x00 bytes followed by the blob specified + ** by the user. The extracted text has to match the text extracted + ** when the blob was inserted - by calling sqlite3_value_text() on + ** the value without the 4 0x00 byte header. + ** + ** The tricky bit here is that the exact text extracted from a blob + ** depends on the encoding of the database. To avoid reimplementing + ** SQLite's blob-to-text conversion code here, we call + ** sqlite3_value_text() on the blob with the header, then trim off the + ** leading utf-8 characters that the 4 byte header was converted to. In + ** practice this is 4 0x00 bytes for a utf-8 database, or 2 0x00 bytes + ** for a utf-16 database. */ + int bIs16 = 0; /* True for utf-16 database */ pText = (const char*)sqlite3_value_text(pVal); nText = sqlite3_value_bytes(pVal); rc = fts5IsUtf16(pConfig, &bIs16); - if( bIs16 ){ pText += 2; nText -= 2; @@ -1321,7 +1405,6 @@ int sqlite3Fts5ExtractText( pText += 4; nText -= 4; } - }else{ for(nLocale=0; nLocalep.pConfig, 0, pVal, &bReset, &zText,&nText); + rc = sqlite3Fts5ExtractText(pTab->p.pConfig, pVal, 0, &bReset, &zText,&nText); if( rc==SQLITE_OK ){ if( bReset ){ *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText); @@ -1930,16 +2012,12 @@ static int fts5UpdateMethod( else{ int eType1 = sqlite3_value_numeric_type(apVal[1]); - if( (eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL) - || (eType0==SQLITE_INTEGER && eType1==SQLITE_NULL) - ){ - rc = SQLITE_MISMATCH; - } - - else if( eType0!=SQLITE_INTEGER ){ + if( eType0!=SQLITE_INTEGER ){ /* An INSERT statement. If the conflict-mode is REPLACE, first remove ** the current entry (if any). */ - if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ + if( eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL ){ + rc = SQLITE_MISMATCH; + }else if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0); bUpdateOrDelete = 1; @@ -1951,8 +2029,9 @@ static int fts5UpdateMethod( else{ i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ - assert( eType1==SQLITE_INTEGER ); - if( iOld!=iNew ){ + if( eType1!=SQLITE_INTEGER ){ + rc = SQLITE_MISMATCH; + }else if( iOld!=iNew ){ if( eConflict==SQLITE_REPLACE ){ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0, 1); if( rc==SQLITE_OK ){ @@ -2126,12 +2205,9 @@ static int fts5ApiColumnText( rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ Fts5Config *pConfig = pTab->pConfig; - int bContent = ( - pConfig->bLocale && pConfig->abUnindexed[iCol]==0 && - pConfig->eContent==FTS5_CONTENT_NORMAL - ); + int bContent = (pConfig->abUnindexed[iCol]==0); sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); - sqlite3Fts5ExtractText(pConfig, bContent, pVal, 0, pz, pn); + sqlite3Fts5ExtractText(pConfig, pVal, bContent, 0, pz, pn); } } return rc; @@ -2346,14 +2422,13 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ rc = fts5SeekCursor(pCsr, 0); for(i=0; rc==SQLITE_OK && inCol; i++){ if( pConfig->abUnindexed[i]==0 ){ - const int bContent = (pConfig->eContent==FTS5_CONTENT_NORMAL); const char *z = 0; int n = 0; int bReset = 0; sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1); pCsr->aColumnSize[i] = 0; - rc = sqlite3Fts5ExtractText(pConfig, bContent, pVal, &bReset, &z, &n); + rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb @@ -2602,6 +2677,9 @@ static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); +/* +** The xColumnLocale() API. +*/ static int fts5ApiColumnLocale( Fts5Context *pCtx, int iCol, @@ -2617,24 +2695,34 @@ static int fts5ApiColumnLocale( if( iCol<0 || iCol>=pConfig->nCol ){ rc = SQLITE_RANGE; - }else{ - int bNormal = (pConfig->eContent==FTS5_CONTENT_NORMAL); - if( pConfig->abUnindexed[iCol]==0 - && pCsr->ePlan!=FTS5_PLAN_SPECIAL - && pConfig->eContent!=FTS5_CONTENT_NONE - && (bNormal==0 || pConfig->bLocale) - ){ - rc = fts5SeekCursor(pCsr, 0); - if( rc==SQLITE_OK ){ - sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); - if( sqlite3_value_type(pVal)==SQLITE_BLOB - && (bNormal || sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE) + }else if( + pConfig->abUnindexed[iCol]==0 + && pCsr->ePlan!=FTS5_PLAN_SPECIAL + && pConfig->eContent!=FTS5_CONTENT_NONE + && pConfig->bLocale + ){ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + /* Load the value into pVal. pVal is a locale/text pair iff: + ** + ** 1) It is an SQLITE_BLOB, and + ** 2) Either the subtype is FTS5_LOCALE_SUBTYPE, or else the + ** value was loaded from an FTS5_CONTENT_NORMAL table, and + ** 3) It does not begin with an 0x00 byte. + */ + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){ const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); int nBlob = sqlite3_value_bytes(pVal); int nLocale = 0; for(nLocale=0; nLocalebLocale + && sqlite3_value_type(pVal)==SQLITE_BLOB + && pConfig->abUnindexed[iCol]==0 + ){ if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE - || (pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL) + || pConfig->eContent==FTS5_CONTENT_NORMAL ){ const u8 *pBlob = sqlite3_value_blob(pVal); int nBlob = sqlite3_value_bytes(pVal); @@ -2953,7 +3066,7 @@ static int fts5ColumnMethod( rc = fts5SeekCursor(pCsr, 1); if( rc==SQLITE_OK ){ sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); - fts5ExtractValueFromColumn(pCtx, pConfig, pVal); + fts5ExtractValueFromColumn(pCtx, pConfig, iCol, pVal); } pConfig->pzErrmsg = 0; } @@ -3322,7 +3435,17 @@ static void fts5SourceIdFunc( } /* -** Implementation of fts5_locale() function. +** Implementation of fts5_locale(LOCALE, TEXT) function. +** +** If parameter LOCALE is NULL, or a zero-length string, then a copy of +** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as +** text, and the value returned is a blob consisting of: +** +** * The LOCALE, as utf-8 text, followed by +** * 0x00, followed by +** * The TEXT, as utf-8 text. +** +** There is no final nul-terminator following the TEXT value. */ static void fts5LocaleFunc( sqlite3_context *pCtx, /* Function call context */ diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index ea5b61a4b6..e55e2b833b 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -476,7 +476,7 @@ static int fts5StorageDeleteFromIndex( pVal = apVal[iCol-1]; } - rc = sqlite3Fts5ExtractText(pConfig,pSeek!=0,pVal,&bReset,&pText,&nText); + rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText); if( rc==SQLITE_OK ){ ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, @@ -740,8 +740,8 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ int bReset = 0; int nText = 0; const char *pText = 0; - rc = sqlite3Fts5ExtractText(pConfig, 1, - sqlite3_column_value(pScan, ctx.iCol+1), &bReset, &pText, &nText + rc = sqlite3Fts5ExtractText(pConfig, + sqlite3_column_value(pScan, ctx.iCol+1), 1, &bReset, &pText, &nText ); if( rc==SQLITE_OK ){ @@ -851,11 +851,14 @@ int sqlite3Fts5StorageContentInsert( ** column. Strip the locale away and just bind the text. */ const char *pText = 0; int nText = 0; - rc = sqlite3Fts5ExtractText(pConfig, 0, pVal, 0, &pText, &nText); + rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText); sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); continue; } - }else if( pConfig->bLocale && sqlite3_value_type(pVal)==SQLITE_BLOB ){ + }else if( pConfig->bLocale + && sqlite3_value_type(pVal)==SQLITE_BLOB + && i>=2 && pConfig->abUnindexed[i-2]==0 + ){ /* Inserting a blob into a normal content table with locale=1. */ int n = sqlite3_value_bytes(pVal); u8 *pBlob = sqlite3Fts5MallocZero(&rc, n+4); @@ -911,7 +914,7 @@ int sqlite3Fts5StorageIndexInsert( pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1); bDisk = 1; } - rc = sqlite3Fts5ExtractText(pConfig, bDisk, pVal, &bReset,&pText,&nText); + rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset,&pText,&nText); if( rc==SQLITE_OK ){ if( bReset && pConfig->bLocale==0 ){ rc = SQLITE_ERROR; @@ -1100,8 +1103,8 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ int nText = 0; int bReset = 0; - rc = sqlite3Fts5ExtractText(pConfig, 1, - sqlite3_column_value(pScan, i+1), &bReset, &pText, &nText + rc = sqlite3Fts5ExtractText(pConfig, + sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText ); if( rc==SQLITE_OK ){ diff --git a/ext/fts5/test/fts5blob.test b/ext/fts5/test/fts5blob.test index 9c838b75a6..56a571b9c4 100644 --- a/ext/fts5/test/fts5blob.test +++ b/ext/fts5/test/fts5blob.test @@ -31,8 +31,6 @@ foreach {tn enc locale} { execsql "PRAGMA encoding = $enc" - if {$tn==3 || $tn==4} breakpoint - execsql " CREATE VIRTUAL TABLE t1 USING fts5(x, y, locale=$locale); " @@ -71,6 +69,20 @@ do_test 1.6.1 { set U(utf8,1) } $U(utf8,0) do_test 1.6.2 { set U(utf16,1) } $U(utf16,0) +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1); + INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546'); +} + +breakpoint +do_execsql_test 2.1 { + select rowid, x, quote(y) FROM t1 +} { + 12 twelve X'0000000041424320444546' +} + finish_test diff --git a/manifest b/manifest index a18841d7c4..9a70fc55a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smissing\sdocumentation\sfor\snew\sfeatures\sto\sfts5.h. -D 2024-07-31T15:46:41.319 +C Fix\svarious\sproblems\swith\sthe\scode\son\sthis\sbranch. +D 2024-07-31T20:49:00.656 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -93,15 +93,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e F ext/fts5/fts5.h f5451da088d0004c4be7314e2fdb41fda16ce682ce2aa3b54b9474ebe9a013d5 -F ext/fts5/fts5Int.h 833a2fe729f926ebcde47e21e495d141b99ede9a188fc577873f24bea0f0bfa2 -F ext/fts5/fts5_aux.c 652f839dc0c77431295f10b08f268631560bb5630e65fd701de7a58744428a82 +F ext/fts5/fts5Int.h 2eab38c52d12bfa0a1cedb742276035e05634c6365f8defb599f023bb8743559 +F ext/fts5/fts5_aux.c 598c80fc0faabab91c833cdda99f8e36387bd907f4acb0480a19b612a4add93e F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 -F ext/fts5/fts5_config.c 0c96490fbad746b3780174f38b2ee5e3d719f2f81ee6b58ca828772871e0f680 +F ext/fts5/fts5_config.c 31267fc68f9d16feaa5b5f7efb902a3410f0caa708cc2372c90a77d75890b2a8 F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c d9a3fef86887e373027d48ab8216ab8caca95f2316e05d953d8885633162a9c5 -F ext/fts5/fts5_storage.c cc6173bb755d668573169c038034a9ec8deadd10b3c10c145adbdf04ab5f889c +F ext/fts5/fts5_main.c bda8d421024191376343a571370a7b2f92fdf5303e32c6dc0d7456bef9ffedd8 +F ext/fts5/fts5_storage.c f94b924db1bc164af3feadcc3f08f0d8f5da5cd45e4909313637aeee85d0d13c F ext/fts5/fts5_tcl.c 93b705cb87633574983161edc5234f9b91ba03f9fecfbd2c5d401a1da6f93aa5 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -132,7 +132,7 @@ F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06c F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36 -F ext/fts5/test/fts5blob.test a16160688e181a212bcb4968325c57ef2864a0bcae0794d6a1e16185007f00b9 +F ext/fts5/test/fts5blob.test 6d1cf0c5ba2e6f8f9b4e915d3677c16ead3a79c22dcb386c0b21169a9349718d F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c2be25568cc33a F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62 F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897 @@ -2201,8 +2201,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff64939ec8399949393f1029fa6d514892fbf2bf1498545300cc5e120b9622a5 -R f7dbcf0c98ed562d42b896f7c143383b +P fa0da3b28e411affd45a918d6b7faba49f744ca8b4adf3cd5ce6609bb630499e +R 4281c2d6e2e80158745c4320fdfed09d U dan -Z 344ab71a9372cfa91d638c7c0c606714 +Z 7ae6a6ddbc4056daf1c0903e2b7934cf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a1e4d24ea7..dee0563fc6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa0da3b28e411affd45a918d6b7faba49f744ca8b4adf3cd5ce6609bb630499e +8bd4ae7e95c7b6ce34db5ea705dc136e742a22f333d0e7370b485ebd736b5ec2 From f971c15b1f48bd986da68c40602158e9669914ba Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 Aug 2024 17:15:17 +0000 Subject: [PATCH 11/99] Clarify the role of Fts5Storage.pSavedRow in the new feature on this branch. FossilOrigin-Name: e8a61d5c48073fdd4d99d0b6fc70469b37af009f281336a44e3789e7eeed820d --- ext/fts5/fts5_storage.c | 106 ++++++++++++++++++++++++++++------------ manifest | 12 ++--- manifest.uuid | 2 +- 3 files changed, 82 insertions(+), 38 deletions(-) diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index e55e2b833b..0a655d160f 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -16,6 +16,32 @@ #include "fts5Int.h" +/* +** pSavedRow: +** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it +** does a by-rowid lookup to retrieve a single row from the %_content +** table or equivalent external-content table/view. +** +** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original +** values for a row being UPDATEd. In that case, the SQL statement is +** not reset and pSavedRow is set to point at it. This is so that the +** insert operation that follows the delete may access the original +** row values for any new values for which sqlite3_value_nochange() returns +** true. i.e. if the user executes: +** +** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1); +** ... +** UPDATE fts SET a=?, b=? WHERE rowid=?; +** +** then the value passed to the xUpdate() method of this table as the +** new.c value is an sqlite3_value_nochange() value. So in this case it +** must be read from the saved row stored in Fts5Storage.pSavedRow. +** +** This is necessary - using sqlite3_value_nochange() instead of just having +** SQLite pass the original value back via xUpdate() - so as not to discard +** any locale information associated with such values. +** +*/ struct Fts5Storage { Fts5Config *pConfig; Fts5Index *pIndex; @@ -37,15 +63,15 @@ struct Fts5Storage { # error "FTS5_STMT_LOOKUP mismatch" #endif -#define FTS5_STMT_LOOKUP2 3 +#define FTS5_STMT_LOOKUP2 3 #define FTS5_STMT_INSERT_CONTENT 4 #define FTS5_STMT_REPLACE_CONTENT 5 #define FTS5_STMT_DELETE_CONTENT 6 -#define FTS5_STMT_REPLACE_DOCSIZE 7 +#define FTS5_STMT_REPLACE_DOCSIZE 7 #define FTS5_STMT_DELETE_DOCSIZE 8 #define FTS5_STMT_LOOKUP_DOCSIZE 9 #define FTS5_STMT_REPLACE_CONFIG 10 -#define FTS5_STMT_SCAN 11 +#define FTS5_STMT_SCAN 11 /* ** Prepare the two insert statements - Fts5Storage.pInsertContent and @@ -405,6 +431,16 @@ static int fts5StorageInsertCallback( return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); } +/* +** This function is used as part of an UPDATE statement that modifies the +** rowid of a row. In that case, this function is called first to set +** Fts5Storage.pSavedRow to point to a statement that may be used to +** access the original values of the row being deleted - iDel. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** It is not considered an error if row iDel does not exist. In this case +** pSavedRow is not set and SQLITE_OK returned. +*/ int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){ int rc = SQLITE_OK; sqlite3_stmt *pSeek = 0; @@ -427,12 +463,17 @@ int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){ ** If a row with rowid iDel is present in the %_content table, add the ** delete-markers to the FTS index necessary to delete it. Do not actually ** remove the %_content row at this time though. +** +** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left +** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access +** the original values of the row being deleted. This is used by UPDATE +** statements. */ static int fts5StorageDeleteFromIndex( Fts5Storage *p, i64 iDel, sqlite3_value **apVal, - int bSaveRow + int bSaveRow /* True to set pSavedRow */ ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ @@ -506,7 +547,15 @@ static int fts5StorageDeleteFromIndex( return rc; } +/* +** Reset any saved statement pSavedRow. Zero pSavedRow as well. This +** should be called by the xUpdate() method of the fts5 table before +** returning from any operation that may have set Fts5Storage.pSavedRow. +*/ void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){ + assert( pStorage->pSavedRow==0 + || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2] + ); sqlite3_reset(pStorage->pSavedRow); pStorage->pSavedRow = 0; } @@ -630,7 +679,7 @@ int sqlite3Fts5StorageDelete( Fts5Storage *p, /* Storage object */ i64 iDel, /* Rowid to delete from table */ sqlite3_value **apVal, /* Optional - values to remove from index */ - int bSaveRow + int bSaveRow /* If true, set pSavedRow for deleted row */ ){ Fts5Config *pConfig = p->pConfig; int rc; @@ -737,13 +786,12 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - int bReset = 0; - int nText = 0; - const char *pText = 0; - rc = sqlite3Fts5ExtractText(pConfig, - sqlite3_column_value(pScan, ctx.iCol+1), 1, &bReset, &pText, &nText - ); + int bReset = 0; /* True if tokenizer locale must be reset */ + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1); + rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText); if( rc==SQLITE_OK ){ rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, @@ -838,6 +886,8 @@ int sqlite3Fts5StorageContentInsert( for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ sqlite3_value *pVal = apVal[i]; if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ + /* This is an UPDATE statement, and column (i-2) was not modified. + ** Retrieve the value from Fts5Storage.pSavedRow instead. */ pVal = sqlite3_column_value(p->pSavedRow, i-1); }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){ if( pConfig->bLocale==0 ){ @@ -859,7 +909,8 @@ int sqlite3Fts5StorageContentInsert( && sqlite3_value_type(pVal)==SQLITE_BLOB && i>=2 && pConfig->abUnindexed[i-2]==0 ){ - /* Inserting a blob into a normal content table with locale=1. */ + /* Inserting a blob into a normal content table with locale=1. + ** Add the 4 0x00 byte header. */ int n = sqlite3_value_bytes(pVal); u8 *pBlob = sqlite3Fts5MallocZero(&rc, n+4); if( pBlob ){ @@ -905,28 +956,22 @@ int sqlite3Fts5StorageIndexInsert( for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - int bReset = 0; - int nText = 0; - const char *pText = 0; + int bReset = 0; /* True if tokenizer locale must be reset */ + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ sqlite3_value *pVal = apVal[ctx.iCol+2]; int bDisk = 0; if( p->pSavedRow && sqlite3_value_nochange(pVal) ){ pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1); bDisk = 1; } - rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset,&pText,&nText); + rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset, &pText,&nText); if( rc==SQLITE_OK ){ - if( bReset && pConfig->bLocale==0 ){ - rc = SQLITE_ERROR; - sqlite3Fts5ConfigErrmsg(pConfig, - "fts5_locale() may not be used without locale=1" - ); - }else{ - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, - fts5StorageInsertCallback - ); - } + assert( bReset==0 || pConfig->bLocale ); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, + fts5StorageInsertCallback + ); if( bReset ) sqlite3Fts5ClearLocale(pConfig); } } @@ -1099,14 +1144,13 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } if( rc==SQLITE_OK ){ - const char *pText = 0; - int nText = 0; - int bReset = 0; + int bReset = 0; /* True if tokenizer locale must be reset */ + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ rc = sqlite3Fts5ExtractText(pConfig, sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText ); - if( rc==SQLITE_OK ){ rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, diff --git a/manifest b/manifest index 9a70fc55a7..fc335fa46f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\svarious\sproblems\swith\sthe\scode\son\sthis\sbranch. -D 2024-07-31T20:49:00.656 +C Clarify\sthe\srole\sof\sFts5Storage.pSavedRow\sin\sthe\snew\sfeature\son\sthis\sbranch. +D 2024-08-01T17:15:17.507 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -101,7 +101,7 @@ F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac62232 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe F ext/fts5/fts5_main.c bda8d421024191376343a571370a7b2f92fdf5303e32c6dc0d7456bef9ffedd8 -F ext/fts5/fts5_storage.c f94b924db1bc164af3feadcc3f08f0d8f5da5cd45e4909313637aeee85d0d13c +F ext/fts5/fts5_storage.c 784c6c4edf676c831ffbaef7b0484dcafe925d3d39049b8ee6b6eeaadae1b042 F ext/fts5/fts5_tcl.c 93b705cb87633574983161edc5234f9b91ba03f9fecfbd2c5d401a1da6f93aa5 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -2201,8 +2201,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fa0da3b28e411affd45a918d6b7faba49f744ca8b4adf3cd5ce6609bb630499e -R 4281c2d6e2e80158745c4320fdfed09d +P 8bd4ae7e95c7b6ce34db5ea705dc136e742a22f333d0e7370b485ebd736b5ec2 +R 65a2f69590c259e395cd85b8f6d4acad U dan -Z 7ae6a6ddbc4056daf1c0903e2b7934cf +Z cbe99cb860ae582ece42e3b6610aa855 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dee0563fc6..0c89f035e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8bd4ae7e95c7b6ce34db5ea705dc136e742a22f333d0e7370b485ebd736b5ec2 +e8a61d5c48073fdd4d99d0b6fc70469b37af009f281336a44e3789e7eeed820d From e317e7f4b4238bb16f33aa6831eb09376adf5115 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Aug 2024 21:06:13 +0000 Subject: [PATCH 12/99] Change things so that locale=1 is required to write fts5_locale() values to an fts5 table, and so that blobs may not be stored in indexed (i.e. not UNINDEXED) columns of these tables. FossilOrigin-Name: c98ccc12169419b8b27ead89ef0665de40320277c5daa748b80869337419e43e --- ext/fts5/fts5Int.h | 11 -- ext/fts5/fts5_config.c | 7 -- ext/fts5/fts5_main.c | 220 +++++++++++++++------------------- ext/fts5/fts5_storage.c | 46 +++---- ext/fts5/test/fts5blob.test | 94 ++++++++++++--- ext/fts5/test/fts5locale.test | 52 +++++++- manifest | 22 ++-- manifest.uuid | 2 +- 8 files changed, 254 insertions(+), 200 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 53e269df93..5f10d5773e 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -208,12 +208,6 @@ struct Fts5TokenizerConfig { ** ** bLocale: ** Set to true if locale=1 was specified when the table was created. -** -** eEnc: -** Set to either FTS5_ENCODING_UNKNOWN, ENCODING_UTF8, or ENCODING_UTF16, -** to indicate the encoding used by the database handle. This is initially -** set to UNKNOWN, then to one of the other two values the first time it -** is required. */ struct Fts5Config { sqlite3 *db; /* Database handle */ @@ -236,7 +230,6 @@ struct Fts5Config { char *zContentExprlist; Fts5TokenizerConfig t; int bLock; /* True when table is preparing statement */ - int eEnc; /* An FTS5_ENCODING_XXX constant */ /* Values loaded from the %_config table */ @@ -278,10 +271,6 @@ struct Fts5Config { #define FTS5_PATTERN_LIKE 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */ #define FTS5_PATTERN_GLOB 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */ -#define FTS5_ENCODING_UNKNOWN 0 -#define FTS5_ENCODING_UTF8 1 -#define FTS5_ENCODING_UTF16 2 - int sqlite3Fts5ConfigParse( Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** ); diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 1312f459db..0d86d81622 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -615,13 +615,6 @@ int sqlite3Fts5ConfigParse( sqlite3_free(zTwo); } - /* If this is not an FTS5_CONTENT_NORMAL table, set bLocale. There are - ** no restrictions on using fts5_locale() with external-content or - ** contentless tables. */ - if( pRet->eContent!=FTS5_CONTENT_NORMAL ){ - pRet->bLocale = 1; - } - /* We only allow contentless_delete=1 if the table is indeed contentless. */ if( rc==SQLITE_OK && pRet->bContentlessDelete diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index c9c01aafea..6f2e8de88c 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -244,6 +244,7 @@ struct Fts5Cursor { ** The subtype values returned by fts5_locale() are tagged with. */ #define FTS5_LOCALE_SUBTYPE ((unsigned int)'L') +#define FTS5_LOCALE_HEADER "\x00\xE0\xB2\xEB" /* @@ -1268,43 +1269,6 @@ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ fts5SetLocale(pConfig, 0, 0); } -/* -** This function is used to determine if the database handle uses utf-8 or -** a utf-16 encoding. If it uses utf-8, then output parameter (*pbIs) is set -** to 0. Otherwise, if it uses utf-16, then the output parameter is set -** to 1. -** -** This function returns SQLITE_OK if successful, or an SQLite error code -** otherwise. If an error code is returned, the final value of (*pbIs) is -** undefined. -*/ -static int fts5IsUtf16( - Fts5Config *pConfig, /* Configuration object */ - int *pbIs /* OUT: True if utf-16, false if utf-8 */ -){ - if( pConfig->eEnc==FTS5_ENCODING_UNKNOWN ){ - sqlite3_stmt *pPragma = 0; - int rc = fts5PrepareStatement(&pPragma, pConfig, - "SELECT (encoding LIKE '%%16%%') FROM pragma_encoding" - ); - if( rc==SQLITE_OK ){ - int val; - sqlite3_step(pPragma); - val = sqlite3_column_int(pPragma, 0); - rc = sqlite3_finalize(pPragma); - if( rc!=SQLITE_OK ) return rc; - if( val ){ - pConfig->eEnc = FTS5_ENCODING_UTF16; - }else{ - pConfig->eEnc = FTS5_ENCODING_UTF8; - } - } - } - - *pbIs = (pConfig->eEnc==FTS5_ENCODING_UTF16); - return SQLITE_OK; -} - /* ** This function is used to extract utf-8 text from an sqlite3_value. This ** is usually done in order to tokenize it. For example, when: @@ -1315,25 +1279,14 @@ static int fts5IsUtf16( ** ** and so on. ** -** This function handles 3 cases: +** This function handles 2 cases: ** ** 1) Ordinary values. The text can be extracted from these using ** sqlite3_value_text(). ** ** 2) Blobs tagged with sub-type FTS5_LOCALE_SUBTYPE, or those read from -** the content table of an FTS5_CONTENT_NORMAL table with locale=1 -** set that do not begin with 0x00. -** -** In these cases the value is a blob formatted by fts5_locale() that -** contains both a locale and a text value. The locale is first, as -** utf-8, followed by a single 0x00 byte, followed by the text value, -** also as utf-8. There is no nul-terminator for the text value. -** -** 3) Blobs read from the content table of an FTS5_CONTENT_NORMAL table -** with locale=1 set that do begin with 0x00. These are used to -** store actual SQLITE_BLOB values written to the fts5 table by the -** user. They begin with 4 0x00 bytes, followed by the blob data as -** specified by the user. +** the content table of a normal content or external-conten table +** with locale=1 set. ** ** If successful, SQLITE_OK is returned and output parameters (*ppText) ** and (*pnText) are set to point to a buffer containing the extracted utf-8 @@ -1342,9 +1295,7 @@ static int fts5IsUtf16( ** from which it is extracted. ** ** Parameter bContent must be true if the value was read from an indexed -** column (i.e. not UNINDEXED) of the on disk content. In this case, if -** the table is FTS5_CONTENT_NORMAL and locale=1 was specified, special blob -** cases (2) and (3) above will apply. +** column (i.e. not UNINDEXED) of the on disk content. ** ** If pbResetTokenizer is not NULL and if case (2) is used, then the ** tokenizer is configured to use the locale. In this case (*pbResetTokenizer) @@ -1366,57 +1317,50 @@ int sqlite3Fts5ExtractText( int bDecodeBlob = 0; assert( pbResetTokenizer==0 || *pbResetTokenizer==0 ); + assert( bContent==0 || pConfig->eContent!=FTS5_CONTENT_NONE ); + assert( bContent==0 || sqlite3_value_subtype(pVal)==0 ); if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE - || (bContent && pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL) + || (bContent && pConfig->bLocale) ){ bDecodeBlob = 1; } } if( bDecodeBlob ){ + const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; const u8 *pBlob = sqlite3_value_blob(pVal); int nBlob = sqlite3_value_bytes(pVal); - int nLocale = 0; - if( nBlob>=4 && memcmp(pBlob, "\0\0\0\0", 4)==0 ){ - /* Extract the text from a blob stored in a locale=1 table. The - ** value consists of 4 0x00 bytes followed by the blob specified - ** by the user. The extracted text has to match the text extracted - ** when the blob was inserted - by calling sqlite3_value_text() on - ** the value without the 4 0x00 byte header. - ** - ** The tricky bit here is that the exact text extracted from a blob - ** depends on the encoding of the database. To avoid reimplementing - ** SQLite's blob-to-text conversion code here, we call - ** sqlite3_value_text() on the blob with the header, then trim off the - ** leading utf-8 characters that the 4 byte header was converted to. In - ** practice this is 4 0x00 bytes for a utf-8 database, or 2 0x00 bytes - ** for a utf-16 database. */ - int bIs16 = 0; /* True for utf-16 database */ - pText = (const char*)sqlite3_value_text(pVal); - nText = sqlite3_value_bytes(pVal); - rc = fts5IsUtf16(pConfig, &bIs16); - if( bIs16 ){ - pText += 2; - nText -= 2; + /* Unless this blob was read from the %_content table of an + ** FTS5_CONTENT_NORMAL table, it should have the 4 byte fts5_locale() + ** header. Check for this. If it is not found, return an error. */ + if( (!bContent || pConfig->eContent!=FTS5_CONTENT_NORMAL) ){ + if( nBlobnCol; ii++){ + if( sqlite3_value_type(apVal[ii+2])==SQLITE_BLOB ){ + int bSub = (sqlite3_value_subtype(apVal[ii+2])==FTS5_LOCALE_SUBTYPE); + if( (pConfig->bLocale && !bSub && pConfig->abUnindexed[ii]==0) + || (pConfig->bLocale==0 && bSub) + ){ + if( pConfig->bLocale==0 ){ + fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); + } + rc = SQLITE_MISMATCH; + goto update_out; + } + } + } + if( eType0!=SQLITE_INTEGER ){ /* An INSERT statement. If the conflict-mode is REPLACE, first remove ** the current entry (if any). */ @@ -2074,6 +2037,7 @@ static int fts5UpdateMethod( } } + update_out: pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -2712,11 +2676,17 @@ static int fts5ApiColumnLocale( */ sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ - if( pConfig->eContent==FTS5_CONTENT_NORMAL - || sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE - ){ - const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); + const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ + const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; + if( nBlobeContent!=FTS5_CONTENT_NONE ); + if( pConfig->bLocale && sqlite3_value_type(pVal)==SQLITE_BLOB && pConfig->abUnindexed[iCol]==0 ){ - if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE - || pConfig->eContent==FTS5_CONTENT_NORMAL - ){ - const u8 *pBlob = sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); + const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; + const u8 *pBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + int ii; - if( nBlob>=4 && memcmp(pBlob, "\0\0\0\0", 4)==0 ){ - sqlite3_result_blob(pCtx, &pBlob[4], nBlob-4, SQLITE_TRANSIENT); + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ + if( nBlobpSavedRow, i-1); }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){ - if( pConfig->bLocale==0 ){ - sqlite3Fts5ConfigErrmsg(pConfig, - "fts5_locale() may not be used without locale=1" - ); - rc = SQLITE_ERROR; - break; - }else if( i>1 && pConfig->abUnindexed[i-2] ){ - /* At attempt to insert an fts5_locale() value into an UNINDEXED - ** column. Strip the locale away and just bind the text. */ - const char *pText = 0; - int nText = 0; - rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText); - sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + assert( pConfig->bLocale ); + if( i>1 ){ + if( pConfig->abUnindexed[i-2] ){ + /* At attempt to insert an fts5_locale() value into an UNINDEXED + ** column. Strip the locale away and just bind the text. */ + const char *pText = 0; + int nText = 0; + rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText); + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + }else{ + const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + assert( nBlob>4 ); + sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT); + } continue; } - }else if( pConfig->bLocale - && sqlite3_value_type(pVal)==SQLITE_BLOB - && i>=2 && pConfig->abUnindexed[i-2]==0 - ){ - /* Inserting a blob into a normal content table with locale=1. - ** Add the 4 0x00 byte header. */ - int n = sqlite3_value_bytes(pVal); - u8 *pBlob = sqlite3Fts5MallocZero(&rc, n+4); - if( pBlob ){ - memcpy(&pBlob[4], sqlite3_value_blob(pVal), n); - rc = sqlite3_bind_blob(pInsert, i, pBlob, n+4, SQLITE_TRANSIENT); - sqlite3_free(pBlob); - } - continue; } rc = sqlite3_bind_value(pInsert, i, pVal); diff --git a/ext/fts5/test/fts5blob.test b/ext/fts5/test/fts5blob.test index 56a571b9c4..985fe2888c 100644 --- a/ext/fts5/test/fts5blob.test +++ b/ext/fts5/test/fts5blob.test @@ -9,6 +9,19 @@ # #*********************************************************************** # +# This file verifies that: +# +# * blob values may be written to locale=0 tables. +# +# * blob values - other than fts5_locale() values - may not be written +# to locale=0 tables. This is an SQLITE_MISMATCH error +# +# * blob values may be returned by queries on the external-content table +# of a locale=0 table. +# +# * blob values not may be returned by queries on the external-content +# table of a locale=1 table, apart from fts5_locale() blobs. This is an +# SQLITE_MISMATCH error. # source [file join [file dirname [info script]] fts5_common.tcl] @@ -20,11 +33,11 @@ ifcapable !fts5 { return } -foreach {tn enc locale} { - 1 utf8 0 - 2 utf8 1 - 3 utf16 0 - 4 utf16 1 +# Test that blobs may be stored in normal locale=0 tables. +# +foreach {tn enc} { + 1 utf8 + 2 utf16 } { reset_db fts5_aux_test_functions db @@ -32,7 +45,7 @@ foreach {tn enc locale} { execsql "PRAGMA encoding = $enc" execsql " - CREATE VIRTUAL TABLE t1 USING fts5(x, y, locale=$locale); + CREATE VIRTUAL TABLE t1 USING fts5(x, y); " do_execsql_test 1.$tn.0 { CREATE VIRTUAL TABLE tt USING fts5vocab('t1', 'instance'); @@ -49,9 +62,6 @@ foreach {tn enc locale} { 3 777 'xyz' } - set T($enc,$locale) [execsql { SELECT * FROM tt }] - set U($enc,$locale) [execsql { SELECT fts5_test_columntext(t1) FROM t1 }] - do_execsql_test 1.$tn.2 { DELETE FROM t1 WHERE rowid=2; DELETE FROM t1 WHERE rowid=1; @@ -62,27 +72,77 @@ foreach {tn enc locale} { } {ok} } -do_test 1.5.1 { set T(utf8,1) } $T(utf8,0) -do_test 1.5.2 { set T(utf16,1) } $T(utf16,0) - -do_test 1.6.1 { set U(utf8,1) } $U(utf8,0) -do_test 1.6.2 { set U(utf16,1) } $U(utf16,0) - - #-------------------------------------------------------------------------- +# Test that a blob may be stored and retrieved in an unindexed column of +# a regular table with locale=1. +# reset_db do_execsql_test 2.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1); INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546'); } -breakpoint do_execsql_test 2.1 { select rowid, x, quote(y) FROM t1 } { 12 twelve X'0000000041424320444546' } +#-------------------------------------------------------------------------- +# Test that blobs may not be written to any type of table with locale=1 +# set. Except, they may be written to UNINDEXED columns. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b); + + CREATE VIRTUAL TABLE x1 USING fts5(a, b, locale=1); + CREATE VIRTUAL TABLE x2 USING fts5(a, b, locale=1, content=t2); + CREATE VIRTUAL TABLE x3 USING fts5(a, b, locale=1, content=); +} + +do_catchsql_test 3.1 { + INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {1 {datatype mismatch}} +do_catchsql_test 3.2 { + INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {1 {datatype mismatch}} +do_catchsql_test 3.3 { + INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {1 {datatype mismatch}} + + +#-------------------------------------------------------------------------- +# Test that fts5_locale() values may not be written to any type of table +# without locale=1 set. Even to an UNINDEXED column. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b); + + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + CREATE VIRTUAL TABLE x2 USING fts5(a, b, content=t2); + CREATE VIRTUAL TABLE x3 USING fts5(a, b, content=); + + CREATE VIRTUAL TABLE x4 USING fts5(a, b, c UNINDEXED); +} + +do_catchsql_test 3.1 { + INSERT INTO x1(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.2 { + INSERT INTO x2(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.3 { + INSERT INTO x3(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.4 { + INSERT INTO x4(rowid, a, b, c) + VALUES(113, 'hello world', 'yesno', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} finish_test diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index d3fd9989c0..40ed28243c 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -107,7 +107,7 @@ do_execsql_test 2.5 { do_catchsql_test 2.6 { INSERT INTO b2 VALUES('def', fts5_locale('reverse', 'four five six')); -} {1 {fts5_locale() may not be used without locale=1}} +} {1 {fts5_locale() requires locale=1}} do_execsql_test 2.7 { SELECT rowid FROM b1('one') } {1} do_execsql_test 2.8 { SELECT rowid FROM b1('four') } {} @@ -149,7 +149,7 @@ reset_db sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create do_execsql_test 3.1 { - CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl); + CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl, locale=1); CREATE VIRTUAL TABLE c2 USING fts5vocab('c1', instance); INSERT INTO c1 VALUES('hello world'); @@ -343,12 +343,58 @@ do_execsql_test 6.0 { } do_catchsql_test 6.1 { INSERT INTO x1(rowid, x) VALUES(123, fts5_locale('en_AU', 'hello world')); -} {1 {fts5_locale() may not be used without locale=1}} +} {1 {fts5_locale() requires locale=1}} do_execsql_test 6.2 { SELECT typeof( fts5_locale(NULL, 'xyz') ), typeof( fts5_locale('', 'abc') ); } {text text} +#-------------------------------------------------------------------------- +# Test that fts5_locale() works with virtual tables. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create +# optimization_control db query-flattener 0 + +do_execsql_test 7.1 { + CREATE TABLE t1(ii INTEGER PRIMARY KEY, bb BLOB, tt TEXT, locale TEXT); + CREATE VIEW v1 AS + SELECT ii AS rowid, bb, fts5_locale(locale, tt) AS tt FROM t1; + + CREATE VIRTUAL TABLE ft USING fts5( + bb, tt, locale=1, tokenize=tcl, content=v1 + ); + + INSERT INTO t1 VALUES(1, NULL, 'one two three', NULL); + INSERT INTO t1 VALUES(2, '7800616263', 'four five six', 'reverse'); + INSERT INTO t1 VALUES(3, '000000007800616263', 'seven eight nine', 'second'); +} + +do_execsql_test 7.2 { + INSERT INTO ft(ft) VALUES('rebuild'); + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 7.3 { + SELECT rowid, quote(bb), quote(tt) FROM ft +} { + 1 NULL {'one two three'} + 2 '7800616263' {'four five six'} + 3 '000000007800616263' {'seven eight nine'} +} + +do_execsql_test 7.4 { SELECT rowid FROM ft('six'); } +do_execsql_test 7.5 { SELECT rowid FROM ft(fts5_locale('reverse','six')); } 2 + +fts5_aux_test_functions db + +do_execsql_test 7.6 { + SELECT fts5_test_columnlocale(ft) FROM ft; +} { + {{} {}} {{} reverse} {{} second} +} + + finish_test diff --git a/manifest b/manifest index fc335fa46f..fdb1dfd115 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarify\sthe\srole\sof\sFts5Storage.pSavedRow\sin\sthe\snew\sfeature\son\sthis\sbranch. -D 2024-08-01T17:15:17.507 +C Change\sthings\sso\sthat\slocale=1\sis\srequired\sto\swrite\sfts5_locale()\svalues\sto\san\sfts5\stable,\sand\sso\sthat\sblobs\smay\snot\sbe\sstored\sin\sindexed\s(i.e.\snot\sUNINDEXED)\scolumns\sof\sthese\stables. +D 2024-08-02T21:06:13.360 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -93,15 +93,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e F ext/fts5/fts5.h f5451da088d0004c4be7314e2fdb41fda16ce682ce2aa3b54b9474ebe9a013d5 -F ext/fts5/fts5Int.h 2eab38c52d12bfa0a1cedb742276035e05634c6365f8defb599f023bb8743559 +F ext/fts5/fts5Int.h 64ccc862cbf3be08103ed9435b8670017019f5bd620db4e4317936170008b2b1 F ext/fts5/fts5_aux.c 598c80fc0faabab91c833cdda99f8e36387bd907f4acb0480a19b612a4add93e F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 -F ext/fts5/fts5_config.c 31267fc68f9d16feaa5b5f7efb902a3410f0caa708cc2372c90a77d75890b2a8 +F ext/fts5/fts5_config.c b7924194b5dc035d9f0bd82c74564e133ff38aaa2edf86d8a95c23c9c82ba17f F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c bda8d421024191376343a571370a7b2f92fdf5303e32c6dc0d7456bef9ffedd8 -F ext/fts5/fts5_storage.c 784c6c4edf676c831ffbaef7b0484dcafe925d3d39049b8ee6b6eeaadae1b042 +F ext/fts5/fts5_main.c 79b1d6099d8140afbd930a4e280759728099f584f91e39a8e81f2df0a0e0e839 +F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 F ext/fts5/fts5_tcl.c 93b705cb87633574983161edc5234f9b91ba03f9fecfbd2c5d401a1da6f93aa5 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -132,7 +132,7 @@ F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06c F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36 -F ext/fts5/test/fts5blob.test 6d1cf0c5ba2e6f8f9b4e915d3677c16ead3a79c22dcb386c0b21169a9349718d +F ext/fts5/test/fts5blob.test 2f84ef3591dd3eb0efa686a3615e9ed5c5005462c663c7c72c350692b99f91f5 F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c2be25568cc33a F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62 F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897 @@ -185,7 +185,7 @@ F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fd F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test bfd8704f9bea963314fcbcf810f08a357ac8035bcb80a2d6170c1e57fa6ad52a +F ext/fts5/test/fts5locale.test 57f4effee98b8c3b8b63f80ce08cc424e68a4ef1b7ce74a91c1e64b2d213053e F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -2201,8 +2201,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8bd4ae7e95c7b6ce34db5ea705dc136e742a22f333d0e7370b485ebd736b5ec2 -R 65a2f69590c259e395cd85b8f6d4acad +P e8a61d5c48073fdd4d99d0b6fc70469b37af009f281336a44e3789e7eeed820d +R 1b40b0a91a44b0bf6782847dce5b9e1d U dan -Z cbe99cb860ae582ece42e3b6610aa855 +Z 06efbaab9999ac759a0d226937b631dd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0c89f035e2..09bba22cd4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e8a61d5c48073fdd4d99d0b6fc70469b37af009f281336a44e3789e7eeed820d +c98ccc12169419b8b27ead89ef0665de40320277c5daa748b80869337419e43e From 5eba4281b029900a5a4518e5259f52da67a45bad Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 9 Aug 2024 20:59:50 +0000 Subject: [PATCH 13/99] Ensure tokenizers registered with xCreateTokenizer_v2() can be accessed using xFindTokenizer(), and that those registered with xCreateTokenizer() work with xFindTokenizer_v2(). FossilOrigin-Name: a3ef7d47b582a66a85ebe802ab40114f77289d6b44a4b375ea7858c60f59b27f --- ext/fts5/fts5_tcl.c | 170 ++++++++++++++++++++++++------ ext/fts5/test/fts5tokenizer3.test | 77 ++++++++++++++ manifest | 13 +-- manifest.uuid | 2 +- 4 files changed, 225 insertions(+), 37 deletions(-) create mode 100644 ext/fts5/test/fts5tokenizer3.test diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index fb951291a6..8b676f1325 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -822,6 +822,9 @@ struct F5tTokenizerContext { struct F5tTokenizerModule { Tcl_Interp *interp; Tcl_Obj *pScript; + void *pParentCtx; + fts5_tokenizer_v2 parent_v2; + fts5_tokenizer parent; F5tTokenizerContext *pContext; }; @@ -836,6 +839,8 @@ struct F5tTokenizerModule { struct F5tTokenizerInstance { Tcl_Interp *interp; Tcl_Obj *pScript; + F5tTokenizerModule *pModule; + Fts5Tokenizer *pParent; F5tTokenizerContext *pContext; char zLocale[128]; }; @@ -864,11 +869,20 @@ static int f5tTokenizerCreate( int nArg, Fts5Tokenizer **ppOut ){ + Fts5Tokenizer *pParent = 0; F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx; Tcl_Obj *pEval; int rc = TCL_OK; int i; + assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 ); + if( pMod->parent_v2.xCreate ){ + rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent); + } + if( pMod->parent.xCreate ){ + rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent); + } + pEval = Tcl_DuplicateObj(pMod->pScript); Tcl_IncrRefCount(pEval); for(i=0; rc==TCL_OK && iinterp = pMod->interp; pInst->pScript = Tcl_GetObjResult(pMod->interp); pInst->pContext = pMod->pContext; + pInst->pParent = pParent; + pInst->pModule = pMod; Tcl_IncrRefCount(pInst->pScript); *ppOut = (Fts5Tokenizer*)pInst; } @@ -898,11 +914,19 @@ static int f5tTokenizerCreate( static void f5tTokenizerDelete(Fts5Tokenizer *p){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + if( pInst->pParent ){ + if( pInst->pModule->parent_v2.xDelete ){ + pInst->pModule->parent_v2.xDelete(pInst->pParent); + }else{ + pInst->pModule->parent.xDelete(pInst->pParent); + } + } Tcl_DecrRefCount(pInst->pScript); ckfree((char *)pInst); } -static int f5tTokenizerTokenize( + +static int f5tTokenizerReallyTokenize( Fts5Tokenizer *p, void *pCtx, int flags, @@ -963,6 +987,54 @@ static int f5tTokenizerTokenize( return rc; } +typedef struct CallbackCtx CallbackCtx; +struct CallbackCtx { + Fts5Tokenizer *p; + void *pCtx; + int flags; + int (*xToken)(void*, int, const char*, int, int, int); +}; + +static int f5tTokenizeCallback( + void *pCtx, + int tflags, + const char *z, int n, + int iStart, int iEnd +){ + CallbackCtx *p = (CallbackCtx*)pCtx; + return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken); +} + +static int f5tTokenizerTokenize( + Fts5Tokenizer *p, + void *pCtx, + int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + int rc = SQLITE_OK; + F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + if( pInst->pParent ){ + CallbackCtx ctx; + ctx.p = p; + ctx.pCtx = pCtx; + ctx.flags = flags; + ctx.xToken = xToken; + if( pInst->pModule->parent_v2.xTokenize ){ + rc = pInst->pModule->parent_v2.xTokenize( + pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback + ); + }else{ + rc = pInst->pModule->parent.xTokenize( + pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback + ); + } + }else{ + rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken); + } + return rc; +} + /* ** sqlite3_fts5_locale */ @@ -1078,47 +1150,85 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( F5tTokenizerModule *pMod; int rc; int bV2 = 0; /* True to use _v2 API */ + const char *zParent = 0; /* Name of parent tokenizer, if any */ + int ii = 0; - if( objc==5 ){ - const char *zArg = Tcl_GetString(objv[1]); - if( 0==strcmp(zArg, "-v2") ){ - objv++; - objc--; - bV2 = 1; + if( objc<4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT"); + return TCL_ERROR; + } + + /* Parse any options. Set stack variables bV2 and zParent. */ + for(ii=1; iiinterp = interp; pMod->pScript = pScript; - pMod->pContext = pContext; Tcl_IncrRefCount(pScript); + pMod->pContext = pContext; + if( zParent ){ + if( bV2 ){ + fts5_tokenizer_v2 *pParent = 0; + rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent); + if( rc==SQLITE_OK ){ + memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2)); + } + }else{ + rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent); + } + } - if( bV2==0 ){ - fts5_tokenizer t; - t.xCreate = f5tTokenizerCreate; - t.xTokenize = f5tTokenizerTokenize; - t.xDelete = f5tTokenizerDelete; - rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer); - }else{ - fts5_tokenizer_v2 t2; - memset(&t2, 0, sizeof(t2)); - t2.iVersion = 2; - t2.xCreate = f5tTokenizerCreate; - t2.xTokenize = f5tTokenizerTokenize; - t2.xDelete = f5tTokenizerDelete; - t2.xSetLocale = f5tTokenizerSetLocale; - rc = pApi->xCreateTokenizer_v2(pApi, zName,(void*)pMod,&t2,f5tDelTokenizer); + if( rc==SQLITE_OK ){ + void *pModCtx = (void*)pMod; + if( bV2==0 ){ + fts5_tokenizer t; + t.xCreate = f5tTokenizerCreate; + t.xTokenize = f5tTokenizerTokenize; + t.xDelete = f5tTokenizerDelete; + rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer); + }else{ + fts5_tokenizer_v2 t2; + memset(&t2, 0, sizeof(t2)); + t2.iVersion = 2; + t2.xCreate = f5tTokenizerCreate; + t2.xTokenize = f5tTokenizerTokenize; + t2.xDelete = f5tTokenizerDelete; + t2.xSetLocale = f5tTokenizerSetLocale; + rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer); + } } if( rc!=SQLITE_OK ){ diff --git a/ext/fts5/test/fts5tokenizer3.test b/ext/fts5/test/fts5tokenizer3.test new file mode 100644 index 0000000000..5cdab743c2 --- /dev/null +++ b/ext/fts5/test/fts5tokenizer3.test @@ -0,0 +1,77 @@ +# 2024 Aug 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5tokenizer3 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +proc get_sod {args} { return "split_on_dot" } +proc get_lowercase {args} { return "lowercase" } + +proc lowercase {flags txt} { + set n [string length $txt] + sqlite3_fts5_token [string tolower $txt] 0 $n + return 0 +} + +proc split_on_dot {flags txt} { + set iOff 0 + foreach t [split $txt "."] { + set n [string length $txt] + sqlite3_fts5_token $t $iOff [expr $iOff+$n] + incr iOff [expr {$n+1}] + } + return "" +} + +foreach {tn script} { + 1 { + sqlite3_fts5_create_tokenizer db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod + } + 2 { + sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod + } + 3 { + sqlite3_fts5_create_tokenizer db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod + } + 4 { + sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod + } +} { + reset_db + eval $script + + do_execsql_test 1.$tn.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=split_on_dot); + CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, instance); + INSERT INTO t1 VALUES('ABC.Def.ghi'); + } + + do_execsql_test 1.$tn.1 { + SELECT term FROM t1vocab ORDER BY 1 + } {abc def ghi} +} + + +finish_test diff --git a/manifest b/manifest index fdb1dfd115..a95f78f7fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthings\sso\sthat\slocale=1\sis\srequired\sto\swrite\sfts5_locale()\svalues\sto\san\sfts5\stable,\sand\sso\sthat\sblobs\smay\snot\sbe\sstored\sin\sindexed\s(i.e.\snot\sUNINDEXED)\scolumns\sof\sthese\stables. -D 2024-08-02T21:06:13.360 +C Ensure\stokenizers\sregistered\swith\sxCreateTokenizer_v2()\scan\sbe\saccessed\susing\sxFindTokenizer(),\sand\sthat\sthose\sregistered\swith\sxCreateTokenizer()\swork\swith\sxFindTokenizer_v2(). +D 2024-08-09T20:59:50.030 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -102,7 +102,7 @@ F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe F ext/fts5/fts5_main.c 79b1d6099d8140afbd930a4e280759728099f584f91e39a8e81f2df0a0e0e839 F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 -F ext/fts5/fts5_tcl.c 93b705cb87633574983161edc5234f9b91ba03f9fecfbd2c5d401a1da6f93aa5 +F ext/fts5/fts5_tcl.c 20bb08b43f6eeff34f12ba25988f46e3a2500bc441e9885a509d5f3932bc1cdb F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c fa5493075101540270f572038fc1723d44fcc97bfbf237c8530013b8a27860be @@ -232,6 +232,7 @@ F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98 F ext/fts5/test/fts5tokenizer2.test 9c1ad8ef0465076cbc9ff5c764782594329b3bce3e0f6a931a026902d006f495 +F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3 F ext/fts5/test/fts5trigram.test 6c4e37864f3e7d90673db5563d9736d7e40080ab94d10ebdffa94c1b77941da0 F ext/fts5/test/fts5trigram2.test c91f0a94f7e1ff859682228646abeab4c0eba2effc46af2cbc8f0f48b05a0566 F ext/fts5/test/fts5ubsan.test 9a2dcf399dc8d0e0de661f0d93884d1d27e5b7f0693cfceb97dd24d818df5dd2 @@ -2201,8 +2202,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e8a61d5c48073fdd4d99d0b6fc70469b37af009f281336a44e3789e7eeed820d -R 1b40b0a91a44b0bf6782847dce5b9e1d +P c98ccc12169419b8b27ead89ef0665de40320277c5daa748b80869337419e43e +R bfbea59aa50b425cf741e16058f3ea26 U dan -Z 06efbaab9999ac759a0d226937b631dd +Z 43db44ab1958e51c418293959d627cfb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 09bba22cd4..f82f7047f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c98ccc12169419b8b27ead89ef0665de40320277c5daa748b80869337419e43e +a3ef7d47b582a66a85ebe802ab40114f77289d6b44a4b375ea7858c60f59b27f From ff6ab9dd2b8437f98b19d8abd7219ca054546735 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 10 Aug 2024 18:59:36 +0000 Subject: [PATCH 14/99] Update the fts5_tokenizer_v2 API so that the locale is passed as parameter to xTokenize(), instead of via a separate call to xSetLocale(). FossilOrigin-Name: 03e63ed24e7a08817341e59b075ae2d4e3f7a5e5b37e0e6138359d5fd29a5e9e --- ext/fts5/fts5.h | 3 +- ext/fts5/fts5Int.h | 5 +- ext/fts5/fts5_config.c | 18 +++- ext/fts5/fts5_expr.c | 4 +- ext/fts5/fts5_main.c | 215 ++++++++++++++++++++++++++++++++--------- ext/fts5/fts5_tcl.c | 27 +++++- manifest | 22 ++--- manifest.uuid | 2 +- 8 files changed, 227 insertions(+), 69 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 2f6c387760..f87b03a5c7 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -627,6 +627,7 @@ struct fts5_tokenizer_v2 { void *pCtx, int flags, /* Mask of FTS5_TOKENIZE_* flags */ const char *pText, int nText, + const char *pLocale, int nLocale, int (*xToken)( void *pCtx, /* Copy of 2nd argument to xTokenize() */ int tflags, /* Mask of FTS5_TOKEN_* flags */ @@ -636,8 +637,6 @@ struct fts5_tokenizer_v2 { int iEnd /* Byte offset of end of token within input text */ ) ); - - int (*xSetLocale)(Fts5Tokenizer*, const char *pLocale, int nLocale); }; /* diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 5f10d5773e..9ed15fb3ac 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -162,10 +162,13 @@ typedef struct Fts5TokenizerConfig Fts5TokenizerConfig; struct Fts5TokenizerConfig { Fts5Tokenizer *pTok; - fts5_tokenizer_v2 *pTokApi; + fts5_tokenizer_v2 *pApi2; + fts5_tokenizer *pApi1; const char **azArg; int nArg; int ePattern; /* FTS_PATTERN_XXX constant */ + const char *pLocale; /* Current locale to use */ + int nLocale; }; /* diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 0d86d81622..b11193ec23 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -679,7 +679,11 @@ void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ if( pConfig ){ int i; if( pConfig->t.pTok ){ - pConfig->t.pTokApi->xDelete(pConfig->t.pTok); + if( pConfig->t.pApi1 ){ + pConfig->t.pApi1->xDelete(pConfig->t.pTok); + }else{ + pConfig->t.pApi2->xDelete(pConfig->t.pTok); + } } sqlite3_free((char*)pConfig->t.azArg); sqlite3_free(pConfig->zDb); @@ -762,9 +766,15 @@ int sqlite3Fts5Tokenize( rc = sqlite3Fts5LoadTokenizer(pConfig); } if( rc==SQLITE_OK ){ - rc = pConfig->t.pTokApi->xTokenize( - pConfig->t.pTok, pCtx, flags, pText, nText, xToken - ); + if( pConfig->t.pApi1 ){ + rc = pConfig->t.pApi1->xTokenize( + pConfig->t.pTok, pCtx, flags, pText, nText, xToken + ); + }else{ + rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags, + pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken + ); + } } } return rc; diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 732d4cf893..6a83f63224 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3084,8 +3084,8 @@ int sqlite3Fts5ExprPopulatePoslists( } } - return sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb + return sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, z, n, + (void*)&sCtx, fts5ExprPopulatePoslistsCb ); } diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 6f2e8de88c..9707d21317 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -103,11 +103,25 @@ struct Fts5Auxiliary { ** Each tokenizer module registered with the FTS5 module is represented ** by an object of the following type. All such objects are stored as part ** of the Fts5Global.pTok list. +** +** bV2Native: +** True if the tokenizer was registered using xCreateTokenizer_v2(), false +** for xCreateTokenizer(). If this variable is true, then x2 is populated +** with the routines as supplied by the caller and x1 contains synthesized +** wrapper routines. In this case the user-data pointer passed to +** x1.xCreate should be a pointer to the Fts5TokenizerModule structure, +** not a copy of pUserData. +** +** Of course, if bV2Native is false, then x1 contains the real routines and +** x2 the synthesized ones. In this case a pointer to the Fts5TokenizerModule +** object should be passed to x2.xCreate. */ struct Fts5TokenizerModule { char *zName; /* Name of tokenizer */ void *pUserData; /* User pointer passed to xCreate() */ - fts5_tokenizer_v2 x; /* Tokenizer functions */ + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* Tokenizer functions */ void (*xDestroy)(void*); /* Destructor function */ Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ }; @@ -1254,11 +1268,10 @@ static int fts5SetLocale( const char *zLocale, int nLocale ){ - Fts5TokenizerConfig *pT = &pConfig->t; int rc = SQLITE_OK; - if( pT->pTokApi->xSetLocale ){ - rc = pT->pTokApi->xSetLocale(pT->pTok, zLocale, nLocale); - } + Fts5TokenizerConfig *pT = &pConfig->t; + pT->pLocale = zLocale; + pT->nLocale = nLocale; return rc; } @@ -3177,6 +3190,105 @@ static int fts5CreateAux( return rc; } +static int fts5NewTokenizerModule( + Fts5Global *pGlobal, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + void(*xDestroy)(void*), /* Destructor for pUserData */ + Fts5TokenizerModule **ppNew +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pNew; + sqlite3_int64 nName; /* Size of zName and its \0 terminator */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + + nName = strlen(zName) + 1; + nByte = sizeof(Fts5TokenizerModule) + nName; + *ppNew = pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte); + if( pNew ){ + pNew->zName = (char*)&pNew[1]; + memcpy(pNew->zName, zName, nName); + pNew->pUserData = pUserData; + pNew->xDestroy = xDestroy; + pNew->pNext = pGlobal->pTok; + pGlobal->pTok = pNew; + if( pNew->pNext==0 ){ + pGlobal->pDfltTok = pNew; + } + } + + return rc; +} + +typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer; +struct Fts5VtoVTokenizer { + Fts5TokenizerModule *pMod; + Fts5Tokenizer *pReal; +}; + +static int fts5VtoVCreate( + void *pCtx, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + Fts5TokenizerModule *pMod = (Fts5TokenizerModule*)pCtx; + Fts5VtoVTokenizer *pNew = 0; + int rc = SQLITE_OK; + + pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + if( rc==SQLITE_OK ){ + pNew->pMod = pMod; + if( pMod->bV2Native ){ + rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + }else{ + rc = pMod->x1.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew); + pNew = 0; + } + } + + *ppOut = (Fts5Tokenizer*)pNew; + return rc; +} +static void fts5VtoVDelete(Fts5Tokenizer *pTok){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + if( p ){ + Fts5TokenizerModule *pMod = p->pMod; + if( pMod->bV2Native ){ + pMod->x2.xDelete(p->pReal); + }else{ + pMod->x1.xDelete(p->pReal); + } + sqlite3_free(p); + } +} +static int fts5V1toV2Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + Fts5TokenizerModule *pMod = p->pMod; + assert( pMod->bV2Native ); + return pMod->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken); +} +static int fts5V2toV1Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + Fts5TokenizerModule *pMod = p->pMod; + assert( pMod->bV2Native==0 ); + return pMod->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken); +} + /* ** Register a new tokenizer. This is the implementation of the ** fts5_api.xCreateTokenizer_v2() method. @@ -3194,24 +3306,14 @@ static int fts5CreateTokenizer_v2( if( pTokenizer->iVersion>2 ){ rc = SQLITE_ERROR; }else{ - Fts5TokenizerModule *pNew; - sqlite3_int64 nName; /* Size of zName and its \0 terminator */ - sqlite3_int64 nByte; /* Bytes of space to allocate */ - - nName = strlen(zName) + 1; - nByte = sizeof(Fts5TokenizerModule) + nName; - pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte); + Fts5TokenizerModule *pNew = 0; + rc = fts5NewTokenizerModule(pGlobal, zName, pUserData, xDestroy, &pNew); if( pNew ){ - pNew->zName = (char*)&pNew[1]; - memcpy(pNew->zName, zName, nName); - pNew->pUserData = pUserData; - pNew->x = *pTokenizer; - pNew->xDestroy = xDestroy; - pNew->pNext = pGlobal->pTok; - pGlobal->pTok = pNew; - if( pNew->pNext==0 ){ - pGlobal->pDfltTok = pNew; - } + pNew->x2 = *pTokenizer; + pNew->bV2Native = 1; + pNew->x1.xCreate = fts5VtoVCreate; + pNew->x1.xTokenize = fts5V1toV2Tokenize; + pNew->x1.xDelete = fts5VtoVDelete; } } @@ -3228,15 +3330,19 @@ static int fts5CreateTokenizer( fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ void(*xDestroy)(void*) /* Destructor for pUserData */ ){ - fts5_tokenizer_v2 tok; + Fts5TokenizerModule *pNew = 0; + int rc = SQLITE_OK; - memset(&tok, 0, sizeof(tok)); - tok.iVersion = 2; - tok.xCreate = pTokenizer->xCreate; - tok.xTokenize = pTokenizer->xTokenize; - tok.xDelete = pTokenizer->xDelete; - - return fts5CreateTokenizer_v2(pApi, zName, pUserData, &tok, xDestroy); + rc = fts5NewTokenizerModule( + (Fts5Global*)pApi, zName, pUserData, xDestroy, &pNew + ); + if( pNew ){ + pNew->x1 = *pTokenizer; + pNew->x2.xCreate = fts5VtoVCreate; + pNew->x2.xTokenize = fts5V2toV1Tokenize; + pNew->x2.xDelete = fts5VtoVDelete; + } + return rc; } static Fts5TokenizerModule *fts5LocateTokenizer( @@ -3271,8 +3377,12 @@ static int fts5FindTokenizer_v2( pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); if( pMod ){ - *ppTokenizer = &pMod->x; - *ppUserData = pMod->pUserData; + if( pMod->bV2Native ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *ppTokenizer = &pMod->x2; }else{ *ppTokenizer = 0; *ppUserData = 0; @@ -3292,14 +3402,21 @@ static int fts5FindTokenizer( void **ppUserData, fts5_tokenizer *pTokenizer /* Populate this object */ ){ - fts5_tokenizer_v2 *pV2 = 0; int rc = SQLITE_OK; + Fts5TokenizerModule *pMod; - rc = fts5FindTokenizer_v2(pApi, zName, ppUserData, &pV2); - if( rc==SQLITE_OK ){ - pTokenizer->xCreate = pV2->xCreate; - pTokenizer->xDelete = pV2->xDelete; - pTokenizer->xTokenize = pV2->xTokenize; + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + if( pMod->bV2Native==0 ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *pTokenizer = pMod->x1; + }else{ + memset(pTokenizer, 0, sizeof(*pTokenizer)); + *ppUserData = 0; + rc = SQLITE_ERROR; } return rc; @@ -3321,23 +3438,33 @@ int fts5GetTokenizer( rc = SQLITE_ERROR; if( pzErr ) *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); }else{ - rc = pMod->x.xCreate( - pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**) = 0; + if( pMod->bV2Native ){ + xCreate = pMod->x2.xCreate; + pConfig->t.pApi2 = &pMod->x2; + }else{ + pConfig->t.pApi1 = &pMod->x1; + xCreate = pMod->x1.xCreate; + } + + rc = xCreate(pMod->pUserData, + (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok ); - pConfig->t.pTokApi = &pMod->x; + if( rc!=SQLITE_OK ){ if( pzErr && rc!=SQLITE_NOMEM ){ *pzErr = sqlite3_mprintf("error in tokenizer constructor"); } - }else{ + }else if( pMod->bV2Native==0 ){ pConfig->t.ePattern = sqlite3Fts5TokenizerPattern( - pMod->x.xCreate, pConfig->t.pTok + pMod->x1.xCreate, pConfig->t.pTok ); } } if( rc!=SQLITE_OK ){ - pConfig->t.pTokApi = 0; + pConfig->t.pApi1 = 0; + pConfig->t.pApi2 = 0; pConfig->t.pTok = 0; } diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 8b676f1325..516c8b88aa 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -1005,15 +1005,22 @@ static int f5tTokenizeCallback( return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken); } -static int f5tTokenizerTokenize( +static int f5tTokenizerTokenize_v2( Fts5Tokenizer *p, void *pCtx, int flags, const char *pText, int nText, + const char *pLoc, int nLoc, int (*xToken)(void*, int, const char*, int, int, int) ){ int rc = SQLITE_OK; F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + + memset(pInst->zLocale, 0, sizeof(pInst->zLocale)); + if( pLoc && nLoczLocale) ){ + memcpy(pInst->zLocale, pLoc, nLoc); + } + if( pInst->pParent ){ CallbackCtx ctx; ctx.p = p; @@ -1022,7 +1029,8 @@ static int f5tTokenizerTokenize( ctx.xToken = xToken; if( pInst->pModule->parent_v2.xTokenize ){ rc = pInst->pModule->parent_v2.xTokenize( - pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback + pInst->pParent, (void*)&ctx, flags, pText, nText, + pLoc, nLoc, f5tTokenizeCallback ); }else{ rc = pInst->pModule->parent.xTokenize( @@ -1032,8 +1040,19 @@ static int f5tTokenizerTokenize( }else{ rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken); } + + memset(pInst->zLocale, 0, sizeof(pInst->zLocale)); return rc; } +static int f5tTokenizerTokenize( + Fts5Tokenizer *p, + void *pCtx, + int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken); +} /* ** sqlite3_fts5_locale @@ -1224,9 +1243,9 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( memset(&t2, 0, sizeof(t2)); t2.iVersion = 2; t2.xCreate = f5tTokenizerCreate; - t2.xTokenize = f5tTokenizerTokenize; + t2.xTokenize = f5tTokenizerTokenize_v2; t2.xDelete = f5tTokenizerDelete; - t2.xSetLocale = f5tTokenizerSetLocale; + // t2.xSetLocale = f5tTokenizerSetLocale; rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer); } } diff --git a/manifest b/manifest index a95f78f7fb..83397e4981 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\stokenizers\sregistered\swith\sxCreateTokenizer_v2()\scan\sbe\saccessed\susing\sxFindTokenizer(),\sand\sthat\sthose\sregistered\swith\sxCreateTokenizer()\swork\swith\sxFindTokenizer_v2(). -D 2024-08-09T20:59:50.030 +C Update\sthe\sfts5_tokenizer_v2\sAPI\sso\sthat\sthe\slocale\sis\spassed\sas\sparameter\sto\sxTokenize(),\sinstead\sof\svia\sa\sseparate\scall\sto\sxSetLocale(). +D 2024-08-10T18:59:36.297 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,17 +92,17 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h f5451da088d0004c4be7314e2fdb41fda16ce682ce2aa3b54b9474ebe9a013d5 -F ext/fts5/fts5Int.h 64ccc862cbf3be08103ed9435b8670017019f5bd620db4e4317936170008b2b1 +F ext/fts5/fts5.h 1bdf3cde2b00e80f62925931471b6a00da38546eed6f2d9fe6a1d5999cbe3ba6 +F ext/fts5/fts5Int.h b40bb0bd54aaa4ac4712b6c5763b2167764614aaef204dbae81638b4548bca5d F ext/fts5/fts5_aux.c 598c80fc0faabab91c833cdda99f8e36387bd907f4acb0480a19b612a4add93e F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 -F ext/fts5/fts5_config.c b7924194b5dc035d9f0bd82c74564e133ff38aaa2edf86d8a95c23c9c82ba17f -F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad +F ext/fts5/fts5_config.c 187f7ffa5eddd6539ffa592de85e95b18be951728491390121bb215549a24a2a +F ext/fts5/fts5_expr.c 16fdd35ebfd9c21d799ad6c7542a36bb3d25ba5fb97bc2e1092d3d86394267b8 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 79b1d6099d8140afbd930a4e280759728099f584f91e39a8e81f2df0a0e0e839 +F ext/fts5/fts5_main.c 6248ba625b6a24c4c5f4706883aa3939b06e7f8e0266e0e8af24f5516e599879 F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 -F ext/fts5/fts5_tcl.c 20bb08b43f6eeff34f12ba25988f46e3a2500bc441e9885a509d5f3932bc1cdb +F ext/fts5/fts5_tcl.c f901f32a1be49dc76c6da63d08224e4d971657db0a0783ebc9796ef0757a57a9 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c fa5493075101540270f572038fc1723d44fcc97bfbf237c8530013b8a27860be @@ -2202,8 +2202,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c98ccc12169419b8b27ead89ef0665de40320277c5daa748b80869337419e43e -R bfbea59aa50b425cf741e16058f3ea26 +P a3ef7d47b582a66a85ebe802ab40114f77289d6b44a4b375ea7858c60f59b27f +R c78b035ad007a6bbcb5c77edceb19899 U dan -Z 43db44ab1958e51c418293959d627cfb +Z 3bd05c86feeacf019b7ab5899d9e19c7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f82f7047f7..4c5b9b3363 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a3ef7d47b582a66a85ebe802ab40114f77289d6b44a4b375ea7858c60f59b27f +03e63ed24e7a08817341e59b075ae2d4e3f7a5e5b37e0e6138359d5fd29a5e9e From 7978910f85327807ac1f7178a97be91bdc108ceb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 10 Aug 2024 19:18:19 +0000 Subject: [PATCH 15/99] Fix an uninitialized variable in fts5_tcl.c. FossilOrigin-Name: 924d3467cebd71ac8dfef8225965d71eda82ef1f0b43508af5b7ea78e2099cf3 --- ext/fts5/fts5_tcl.c | 3 +-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 516c8b88aa..1e1243a337 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -1167,7 +1167,7 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( char *zName; Tcl_Obj *pScript; F5tTokenizerModule *pMod; - int rc; + int rc = SQLITE_OK; int bV2 = 0; /* True to use _v2 API */ const char *zParent = 0; /* Name of parent tokenizer, if any */ int ii = 0; @@ -1245,7 +1245,6 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( t2.xCreate = f5tTokenizerCreate; t2.xTokenize = f5tTokenizerTokenize_v2; t2.xDelete = f5tTokenizerDelete; - // t2.xSetLocale = f5tTokenizerSetLocale; rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer); } } diff --git a/manifest b/manifest index 83397e4981..fbf7eb7280 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sfts5_tokenizer_v2\sAPI\sso\sthat\sthe\slocale\sis\spassed\sas\sparameter\sto\sxTokenize(),\sinstead\sof\svia\sa\sseparate\scall\sto\sxSetLocale(). -D 2024-08-10T18:59:36.297 +C Fix\san\suninitialized\svariable\sin\sfts5_tcl.c. +D 2024-08-10T19:18:19.197 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -102,7 +102,7 @@ F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe F ext/fts5/fts5_main.c 6248ba625b6a24c4c5f4706883aa3939b06e7f8e0266e0e8af24f5516e599879 F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 -F ext/fts5/fts5_tcl.c f901f32a1be49dc76c6da63d08224e4d971657db0a0783ebc9796ef0757a57a9 +F ext/fts5/fts5_tcl.c af5817de028f5b1210a27dd99b3592fda55f438b3053d6f84145f32fdd265533 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c fa5493075101540270f572038fc1723d44fcc97bfbf237c8530013b8a27860be @@ -2202,8 +2202,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a3ef7d47b582a66a85ebe802ab40114f77289d6b44a4b375ea7858c60f59b27f -R c78b035ad007a6bbcb5c77edceb19899 +P 03e63ed24e7a08817341e59b075ae2d4e3f7a5e5b37e0e6138359d5fd29a5e9e +R c027b27c24a454af656d09d633573aa9 U dan -Z 3bd05c86feeacf019b7ab5899d9e19c7 +Z d725700101283648380ab327f25111bf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4c5b9b3363..f9d9639f86 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03e63ed24e7a08817341e59b075ae2d4e3f7a5e5b37e0e6138359d5fd29a5e9e +924d3467cebd71ac8dfef8225965d71eda82ef1f0b43508af5b7ea78e2099cf3 From 12866777738884e44d561e27f0d3e17ecae04a4e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Aug 2024 11:13:56 +0000 Subject: [PATCH 16/99] Update the auxiliary function API to include xTokenize_x2() instead of xSetLocale(). FossilOrigin-Name: f7d56a1f2149f0da117167db62e2c28ec337e8da3403873b64cdfc6a951e2e8e --- ext/fts5/fts5.h | 59 +++++++++++++------------------------------- ext/fts5/fts5_aux.c | 47 ++++++++++++++--------------------- ext/fts5/fts5_main.c | 53 ++++++++++++++------------------------- ext/fts5/fts5_tcl.c | 42 +++++++++---------------------- manifest | 18 +++++++------- manifest.uuid | 2 +- 6 files changed, 75 insertions(+), 146 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index f87b03a5c7..784511de26 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -327,21 +327,10 @@ struct Fts5PhraseIter { ** SQLite error code is returned. The final value of the output parameters ** is undefined in this case. ** -** xTokenizeSetLocale(pFts5, pLocale, nLocale) -** This API function is used to invoke the fts5_tokenizer_v2.xSetLocale() -** method of the fts5 table's tokenizer, in the same way as xTokenize() is -** used to invoke fts5_tokenizer_v2.xTokenize(). -** -** Parameters pLocale and nLocale may both be 0, in which case the tokenizer -** is configured to use its default locale. Otherwise, pLocale should point -** to a buffer containing the utf-8 encoded name of the locale to use. -** It does not have to be nul-terminated. nLocale must be passed the size -** of the text in bytes. The buffer indicated by pLocale must remain valid -** for the duration of any calls made to xTokenize() by the auxiliary -** function call up until the next invocation of xTokenizeSetLocale(), if -** any. -** -** SQLITE_OK is returned on success, or an SQLite error code otherwise. +** xTokenize_v2: +** Tokenize text using the tokenizer belonging to the FTS5 table. This +** API is the same as the xTokenize() API, except that it allows a tokenizer +** locale to be specified. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 4 */ @@ -389,7 +378,12 @@ struct Fts5ExtensionApi { /* Below this point are iVersion>=4 only */ int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); - int (*xTokenizeSetLocale)(Fts5Context*, const char *p, int n); + int (*xTokenize_v2)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + const char *pLoc, int nLoc, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); }; /* @@ -482,32 +476,13 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** -** xSetLocale: -** This function is invoked by FTS5 to configure the locale to use for -** subsequent calls to xTokenize. The second argument is a pointer to -** a nul-terminated buffer containing the utf-8 encoded name of the locale -** to use. The third argument is the size of the buffer in bytes, not -** including the nul-terminator character. This function may also be -** invoked with the second and third parameters set to 0 - instructing -** the tokenizer to use its default locale. -** -** FTS5 guarantees that any buffer passed to xSetLocale() will remain -** valid until either the next call to xSetLocale() or xDelete() on the -** same tokenizer object. -** -** This function should return SQLITE_OK if successful, or an SQLite -** error code if an error occurs. If an error does occur and an error -** code is returned, execution of the current statement is abandoned -** and FTS5 returns the error code to the caller. -** -** Often, this function is not required and is never invoked. It is only -** ever invoked when processing a value that has had a locale associated -** with it using SQL function fts5_locale(). -** -** It is not necessary to supply an implementation of this method when -** registering a tokenizer. If fts5_tokenizer_v2.xSetLocale is set to NULL, -** then no attempt is made to pass locale information through to the -** tokenizer. +** If the tokenizer is registered using an fts5_tokenizer_v2 object, +** then the xTokenize() method has two additional arguments - pLocale +** and nLocale. These specify the locale that the tokenizer should use +** for the current request. If pLocale and nLocale are both 0, then the +** tokenizer should use its default locale. Otherwise, pLocale points to +** a buffer containing the name of the locale to use as utf-8 text. nLocale +** contains the number of bytes in pLocale. pLocale is not nul-terminated. ** ** SYNONYM SUPPORT ** diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 8695369f5e..fb177d561e 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -226,28 +226,6 @@ static int fts5HighlightCb( return rc; } -/* -** Use xTokenizeSetLocale() to configure the tokenizer to use the locale -** associated with column iCol of the current row. -*/ -static int fts5ConfigureTokenizer( - const Fts5ExtensionApi *pApi, - Fts5Context *pFts, - int iCol -){ - int rc = SQLITE_OK; - const char *zLocale = 0; - int nLocale = 0; - - assert( pApi->iVersion>=4 ); /* Ensure xColumnLocale() is available */ - - rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale); - if( rc==SQLITE_OK ){ - rc = pApi->xTokenizeSetLocale(pFts, zLocale, nLocale); - } - return rc; -} - /* ** Implementation of highlight() function. @@ -279,15 +257,19 @@ static void fts5HighlightFunction( sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); rc = SQLITE_OK; }else if( ctx.zIn ){ + const char *pLoc = 0; + int nLoc = 0; if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); } if( rc==SQLITE_OK ){ - rc = fts5ConfigureTokenizer(pApi, pFts, iCol); + rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc); } if( rc==SQLITE_OK ){ - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb + ); } if( ctx.bOpen ){ fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); @@ -484,6 +466,8 @@ static void fts5SnippetFunction( memset(&sFinder, 0, sizeof(Fts5SFinder)); for(i=0; ixColumnText(pFts, i, &sFinder.zDoc, &nDoc); if( rc!=SQLITE_OK ) break; - rc = fts5ConfigureTokenizer(pApi, pFts, i); + rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc); if( rc!=SQLITE_OK ) break; - rc = pApi->xTokenize(pFts, - sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb + rc = pApi->xTokenize_v2(pFts, + sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb ); if( rc!=SQLITE_OK ) break; rc = pApi->xColumnSize(pFts, i, &nDocsize); @@ -552,6 +536,9 @@ static void fts5SnippetFunction( rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); } if( ctx.zIn ){ + const char *pLoc = 0; /* Locale to tokenize in */ + int nLoc = 0; /* Bytes in pLoc */ + if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); } @@ -570,10 +557,12 @@ static void fts5SnippetFunction( } if( rc==SQLITE_OK ){ - rc = fts5ConfigureTokenizer(pApi, pFts, iBestCol); + rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc); } if( rc==SQLITE_OK ){ - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb + ); } if( ctx.bOpen ){ fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 0aa80093bc..9cc7b5cbf3 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -220,10 +220,6 @@ struct Fts5Cursor { int nInstAlloc; /* Size of aInst[] array (entries / 3) */ int nInstCount; /* Number of phrase instances */ int *aInst; /* 3 integers per phrase instance */ - - /* Values set by xTokenizeSetLocale() */ - const char *pLocale; - int nLocale; }; /* @@ -2128,29 +2124,33 @@ static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); } -static int fts5ApiTokenize( +static int fts5ApiTokenize_v2( Fts5Context *pCtx, const char *pText, int nText, + const char *pLoc, int nLoc, void *pUserData, int (*xToken)(void*, int, const char*, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); int rc = SQLITE_OK; - const char *pLocale = pCsr->pLocale; - if( pLocale ){ - rc = fts5SetLocale(pTab->pConfig, pLocale, pCsr->nLocale); - } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts5Tokenize( - pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken - ); - } - if( pLocale ){ - sqlite3Fts5ClearLocale(pTab->pConfig); - } + + fts5SetLocale(pTab->pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pTab->pConfig, + FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + ); + fts5SetLocale(pTab->pConfig, 0, 0); + return rc; } +static int fts5ApiTokenize( + Fts5Context *pCtx, + const char *pText, int nText, + void *pUserData, + int (*xToken)(void*, int, const char*, int, int, int) +){ + return fts5ApiTokenize_v2(pCtx, pText, nText, 0, 0, pUserData, xToken); +} static int fts5ApiPhraseCount(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; @@ -2720,21 +2720,6 @@ static int fts5ApiColumnLocale( return rc; } -/* -** The xTokenizeSetLocale() API. -*/ -static int fts5ApiTokenizeSetLocale( - Fts5Context *pCtx, - const char *pLocale, - int nLocale -){ - int rc = SQLITE_OK; - Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - pCsr->pLocale = pLocale; - pCsr->nLocale = nLocale; - return rc; -} - static const Fts5ExtensionApi sFts5Api = { 4, /* iVersion */ fts5ApiUserData, @@ -2759,7 +2744,7 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiQueryToken, fts5ApiInstToken, fts5ApiColumnLocale, - fts5ApiTokenizeSetLocale + fts5ApiTokenize_v2 }; /* @@ -2813,8 +2798,6 @@ static void fts5ApiInvoke( pCsr->pAux = pAux; pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); pCsr->pAux = 0; - pCsr->pLocale = 0; - pCsr->nLocale = 0; } static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 2fcc3815fd..60b3f0e9ef 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -825,11 +825,9 @@ struct F5tTokenizerModule { /* ** zLocale: -** Buffer zLocale contains the current locale, as configured by the most -** recent call to xSetLocale(). A NULL (default) locale is represented as -** a 0 byte string - "\0". -** -** This can be retrieved by a Tcl tokenize script using [sqlite3_fts5_locale]. +** Within a call to xTokenize_v2(), pLocale/nLocale store the locale +** passed to the call by fts5. This can be retrieved by a Tcl tokenize +** script using [sqlite3_fts5_locale]. */ struct F5tTokenizerInstance { Tcl_Interp *interp; @@ -837,27 +835,10 @@ struct F5tTokenizerInstance { F5tTokenizerModule *pModule; Fts5Tokenizer *pParent; F5tTokenizerContext *pContext; - char zLocale[128]; + const char *pLocale; + int nLocale; }; -static int f5tTokenizerSetLocale( - Fts5Tokenizer *pTokenizer, - const char *pLocale, - int nLocale -){ - F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)pTokenizer; - if( nLocale>=sizeof(pInst->zLocale) ){ - return SQLITE_ERROR; - } - - memset(pInst->zLocale, 0, sizeof(pInst->zLocale)); - if( nLocale>0 ){ - memcpy(pInst->zLocale, pLocale, nLocale); - } - - return SQLITE_OK; -} - static int f5tTokenizerCreate( void *pCtx, const char **azArg, @@ -1011,10 +992,8 @@ static int f5tTokenizerTokenize_v2( int rc = SQLITE_OK; F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; - memset(pInst->zLocale, 0, sizeof(pInst->zLocale)); - if( pLoc && nLoczLocale) ){ - memcpy(pInst->zLocale, pLoc, nLoc); - } + pInst->pLocale = pLoc; + pInst->nLocale = nLoc; if( pInst->pParent ){ CallbackCtx ctx; @@ -1036,7 +1015,8 @@ static int f5tTokenizerTokenize_v2( rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken); } - memset(pInst->zLocale, 0, sizeof(pInst->zLocale)); + pInst->pLocale = 0; + pInst->nLocale = 0; return rc; } static int f5tTokenizerTokenize( @@ -1072,7 +1052,9 @@ static int SQLITE_TCLAPI f5tTokenizerLocale( return TCL_ERROR; } - Tcl_SetObjResult(interp, Tcl_NewStringObj(p->pInst->zLocale, -1)); + Tcl_SetObjResult(interp, + Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale) + ); return TCL_OK; } diff --git a/manifest b/manifest index a8ab8d86ed..d3ebedd919 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2024-08-10T20:03:01.148 +C Update\sthe\sauxiliary\sfunction\sAPI\sto\sinclude\sxTokenize_x2()\sinstead\sof\sxSetLocale(). +D 2024-08-12T11:13:56.109 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,17 +92,17 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h 1bdf3cde2b00e80f62925931471b6a00da38546eed6f2d9fe6a1d5999cbe3ba6 +F ext/fts5/fts5.h 7f1197009fc0e9822a8a584aa1f90591bdbf04f4503ecfe06949f3afe7a1fe06 F ext/fts5/fts5Int.h b40bb0bd54aaa4ac4712b6c5763b2167764614aaef204dbae81638b4548bca5d -F ext/fts5/fts5_aux.c 598c80fc0faabab91c833cdda99f8e36387bd907f4acb0480a19b612a4add93e +F ext/fts5/fts5_aux.c 0d0ee62dfebe93ccf6b293edb0b21ebe5c8bdc85e962a001745f2d13ea3e79d2 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 187f7ffa5eddd6539ffa592de85e95b18be951728491390121bb215549a24a2a F ext/fts5/fts5_expr.c ee1949c5c20901cbaca0885902f1d0c136679262dee71b457a34a92e1d16ddac F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c d6afe9ddaa995d52009d6a3a06b1f4d7481610d7d34b1af6f05bcd4756f7b99a +F ext/fts5/fts5_main.c cd61abbfd02f0f22e3c124ae2ad10c2a51cdf8acf38177410d44e134c1d1364b F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 -F ext/fts5/fts5_tcl.c 7bd4bcd4557814df4fe8412edac34a309da83e90780cdf1d0b6bf5da18e127dd +F ext/fts5/fts5_tcl.c 50c7e16753fde0c4d80d8abd00a4ed2b0e998d5d3899a484510d01923c5da43b F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c 63ebe9057ed3f4dfc49944bc4aee3d3b745cc2faff73bc152ed3554ed3bf9cf4 @@ -2207,8 +2207,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 924d3467cebd71ac8dfef8225965d71eda82ef1f0b43508af5b7ea78e2099cf3 d4014c87ba9b011a6a04c2bf85879b668dc762ebcbbfb50a2f8a417ce594ef88 -R f5c4cc25ad182e75985989cfb0d3dcb3 +P b243007525a825b3daf8aa9bb2d3088efb853bd8b7c9ea3c0924fde193eb5c44 +R ca9e80b1fa88da85d78ec3898ab25337 U dan -Z c7a930832d2066471c172888c15e20f1 +Z 8ccf549b58c37ea212fa9c3ab517ca5f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2a2534e013..68e7d56fa7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b243007525a825b3daf8aa9bb2d3088efb853bd8b7c9ea3c0924fde193eb5c44 +f7d56a1f2149f0da117167db62e2c28ec337e8da3403873b64cdfc6a951e2e8e From 29d5e43ad4834f011ccf87a78ad1c75ed8f2da24 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Aug 2024 11:46:09 +0000 Subject: [PATCH 17/99] Update the porter tokenizer to use locales. FossilOrigin-Name: 3291ce3a3359a80e51e4546a3d7a187cbe4c7530fca6632f0bb2728525efe212 --- ext/fts5/fts5_tokenize.c | 33 ++++++++++++++++++++++++--------- ext/fts5/test/fts5locale.test | 23 ++++++++++++++++++++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index 9f5cd24c3c..7a36b750a1 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -556,7 +556,7 @@ static int fts5UnicodeTokenize( typedef struct PorterTokenizer PorterTokenizer; struct PorterTokenizer { - fts5_tokenizer tokenizer; /* Parent tokenizer module */ + fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */ Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */ char aBuf[FTS5_PORTER_MAX_TOKEN + 64]; }; @@ -568,7 +568,7 @@ static void fts5PorterDelete(Fts5Tokenizer *pTok){ if( pTok ){ PorterTokenizer *p = (PorterTokenizer*)pTok; if( p->pTokenizer ){ - p->tokenizer.xDelete(p->pTokenizer); + p->tokenizer_v2.xDelete(p->pTokenizer); } sqlite3_free(p); } @@ -587,6 +587,7 @@ static int fts5PorterCreate( PorterTokenizer *pRet; void *pUserdata = 0; const char *zBase = "unicode61"; + fts5_tokenizer_v2 *pV2 = 0; if( nArg>0 ){ zBase = azArg[0]; @@ -595,14 +596,15 @@ static int fts5PorterCreate( pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); - rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer); + rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); }else{ rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ int nArg2 = (nArg>0 ? nArg-1 : 0); - const char **azArg2 = (nArg2 ? &azArg[1] : 0); - rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer); + const char **az2 = (nArg2 ? &azArg[1] : 0); + memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2)); + rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer); } if( rc!=SQLITE_OK ){ @@ -1253,6 +1255,7 @@ static int fts5PorterTokenize( void *pCtx, int flags, const char *pText, int nText, + const char *pLoc, int nLoc, int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; @@ -1260,8 +1263,8 @@ static int fts5PorterTokenize( sCtx.xToken = xToken; sCtx.pCtx = pCtx; sCtx.aBuf = p->aBuf; - return p->tokenizer.xTokenize( - p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb + return p->tokenizer_v2.xTokenize( + p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb ); } @@ -1450,7 +1453,6 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ } aBuiltin[] = { { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}}, { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }}, - { "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }}, { "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}}, }; @@ -1465,6 +1467,19 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ 0 ); } - + if( rc==SQLITE_OK ){ + fts5_tokenizer_v2 sPorter = { + 2, + fts5PorterCreate, + fts5PorterDelete, + fts5PorterTokenize + }; + rc = pApi->xCreateTokenizer_v2(pApi, + "porter", + (void*)pApi, + &sPorter, + 0 + ); + } return rc; } diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index 40ed28243c..2d5f2a00dc 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -350,11 +350,10 @@ do_execsql_test 6.2 { } {text text} #-------------------------------------------------------------------------- -# Test that fts5_locale() works with virtual tables. +# Test that fts5_locale() works with external-content tables. # reset_db sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create -# optimization_control db query-flattener 0 do_execsql_test 7.1 { CREATE TABLE t1(ii INTEGER PRIMARY KEY, bb BLOB, tt TEXT, locale TEXT); @@ -394,8 +393,26 @@ do_execsql_test 7.6 { {{} {}} {{} reverse} {{} second} } +#------------------------------------------------------------------------- +# Test that the porter tokenizer works with locales. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create +do_execsql_test 8.1 { + CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize="porter tcl"); + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance); + + INSERT INTO ft(rowid, tt) VALUES + (111, fts5_locale('second', 'the porter tokenizer is a wrapper tokenizer')), + (222, fts5_locale('reverse', 'This value may also be set')); +} + +do_execsql_test 8.1 { + SELECT DISTINCT term FROM vocab ORDER BY 1 +} { + a eb eulav osla sihT te the token yam +} finish_test - diff --git a/manifest b/manifest index d3ebedd919..ce9f94a445 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sauxiliary\sfunction\sAPI\sto\sinclude\sxTokenize_x2()\sinstead\sof\sxSetLocale(). -D 2024-08-12T11:13:56.109 +C Update\sthe\sporter\stokenizer\sto\suse\slocales. +D 2024-08-12T11:46:09.154 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -105,7 +105,7 @@ F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f F ext/fts5/fts5_tcl.c 50c7e16753fde0c4d80d8abd00a4ed2b0e998d5d3899a484510d01923c5da43b F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b -F ext/fts5/fts5_tokenize.c 63ebe9057ed3f4dfc49944bc4aee3d3b745cc2faff73bc152ed3554ed3bf9cf4 +F ext/fts5/fts5_tokenize.c 96efa85a21a85276680ce3fb19dc5dd8d6b6541b2c37f953ee55bc15092262e1 F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80 F ext/fts5/fts5_vocab.c e4830b00809e5da53bc10f93adc59e321407b0f801c7f4167c0e47f5552267e0 @@ -186,7 +186,7 @@ F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba104 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 57f4effee98b8c3b8b63f80ce08cc424e68a4ef1b7ce74a91c1e64b2d213053e +F ext/fts5/test/fts5locale.test 8e893b5a764d181260f5f862dc529fcdb42315b2d683317043d4609f13f88a02 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -2207,8 +2207,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b243007525a825b3daf8aa9bb2d3088efb853bd8b7c9ea3c0924fde193eb5c44 -R ca9e80b1fa88da85d78ec3898ab25337 +P f7d56a1f2149f0da117167db62e2c28ec337e8da3403873b64cdfc6a951e2e8e +R 7151af5ed6816182b47b60322cc8dcba U dan -Z 8ccf549b58c37ea212fa9c3ab517ca5f +Z a17240af0068f64d6da9a8176108962c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 68e7d56fa7..6edb67265e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7d56a1f2149f0da117167db62e2c28ec337e8da3403873b64cdfc6a951e2e8e +3291ce3a3359a80e51e4546a3d7a187cbe4c7530fca6632f0bb2728525efe212 From 10e54e365a5f9b2a2ea4ab103f762171e733a903 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Aug 2024 17:03:37 +0000 Subject: [PATCH 18/99] Fix further issues to do with fts5 locale support. FossilOrigin-Name: e626123580065986f7df50b6140f00048944becce179b9391fbf09f97ac55485 --- ext/fts5/fts5.h | 14 ++--- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_aux.c | 10 ++-- ext/fts5/fts5_expr.c | 4 +- ext/fts5/fts5_main.c | 123 +++++++++++++++++++++++++++++-------------- manifest | 20 +++---- manifest.uuid | 2 +- 7 files changed, 109 insertions(+), 66 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 784511de26..dfa075f0a9 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -395,10 +395,10 @@ struct Fts5ExtensionApi { ** ** Applications may also register custom tokenizer types. A tokenizer ** is registered by providing fts5 with a populated instance of the -** following structure. Of the three structure methods, xCreate, xDelete and -** xTokenize must be supplied, any fo these three members of the -** fts5_tokenizer_v2 struct to NULL leads to undefined behaviour. The -** structure methods are expected to function as follows: +** following structure. All structure methods must be defined, setting +** +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: ** ** xCreate: ** This function is used to allocate and initialize a tokenizer instance. @@ -481,8 +481,8 @@ struct Fts5ExtensionApi { ** and nLocale. These specify the locale that the tokenizer should use ** for the current request. If pLocale and nLocale are both 0, then the ** tokenizer should use its default locale. Otherwise, pLocale points to -** a buffer containing the name of the locale to use as utf-8 text. nLocale -** contains the number of bytes in pLocale. pLocale is not nul-terminated. +** an nLocale byte buffer containing the name of the locale to use as utf-8 +** text. pLocale is not nul-terminated. ** ** SYNONYM SUPPORT ** @@ -658,7 +658,7 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently 3, was once 2 */ + int iVersion; /* Currently always set to 3 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 9ed15fb3ac..7e42ea82c1 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -168,7 +168,7 @@ struct Fts5TokenizerConfig { int nArg; int ePattern; /* FTS_PATTERN_XXX constant */ const char *pLocale; /* Current locale to use */ - int nLocale; + int nLocale; /* Size of pLocale in bytes */ }; /* diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index fb177d561e..eb3f7e359d 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -257,8 +257,8 @@ static void fts5HighlightFunction( sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); rc = SQLITE_OK; }else if( ctx.zIn ){ - const char *pLoc = 0; - int nLoc = 0; + const char *pLoc = 0; /* Locale of column iCol */ + int nLoc = 0; /* Size of pLoc in bytes */ if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); } @@ -466,8 +466,8 @@ static void fts5SnippetFunction( memset(&sFinder, 0, sizeof(Fts5SFinder)); for(i=0; ixColumnSize(pFts, iBestCol, &nColSize); } if( ctx.zIn ){ - const char *pLoc = 0; /* Locale to tokenize in */ + const char *pLoc = 0; /* Locale of column iBestCol */ int nLoc = 0; /* Bytes in pLoc */ if( rc==SQLITE_OK ){ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index b1a0f48209..a39cc16bbf 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3092,8 +3092,8 @@ int sqlite3Fts5ExprPopulatePoslists( } } - return sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, z, n, - (void*)&sCtx, fts5ExprPopulatePoslistsCb + return sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb ); } diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 9cc7b5cbf3..b4ccd04028 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -115,13 +115,16 @@ struct Fts5Auxiliary { ** Of course, if bV2Native is false, then x1 contains the real routines and ** x2 the synthesized ones. In this case a pointer to the Fts5TokenizerModule ** object should be passed to x2.xCreate. +** +** The synthesized wrapper routines are necessary for xFindTokenizer(_v2) +** calls. */ struct Fts5TokenizerModule { char *zName; /* Name of tokenizer */ void *pUserData; /* User pointer passed to xCreate() */ int bV2Native; /* True if v2 native tokenizer */ fts5_tokenizer x1; /* Tokenizer functions */ - fts5_tokenizer_v2 x2; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ void (*xDestroy)(void*); /* Destructor function */ Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ }; @@ -179,12 +182,6 @@ struct Fts5Sorter { ** If the cursor iterates in descending order of rowid, iFirstRowid ** is the upper limit (i.e. the "first" rowid visited) and iLastRowid ** the lower. -** -** pLocale, nLocale: -** These are set by API method xTokenizeSetLocale(). xTokenizeSetLocale() -** does not actually configure the tokenizer, it just stores the values -** it is passed in these variables. The fts5_tokenizer_v2.xSetLocale() -** method is called from within the xTokenize() API method if required. */ struct Fts5Cursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ @@ -251,7 +248,7 @@ struct Fts5Cursor { #define BitFlagTest(x,y) (((x) & (y))!=0) /* -** The subtype values returned by fts5_locale() are tagged with. +** The subtype value and header bytes used by fts5_locale(). */ #define FTS5_LOCALE_SUBTYPE ((unsigned int)'L') #define FTS5_LOCALE_HEADER "\x00\xE0\xB2\xEB" @@ -1255,24 +1252,24 @@ static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ } /* -** Configure the tokenizer to use the locale specified by nLocale byte -** buffer zLocale. Return SQLITE_OK if successful, or an SQLite error -** code otherwise. +** Arrange for subsequent calls to sqlite3Fts5Tokenize() to use the locale +** specified by pLocale/nLocale. The buffer indicated by pLocale must remain +** valid until after the final call to sqlite3Fts5Tokenize() that will use +** the locale. */ -static int fts5SetLocale( +static void fts5SetLocale( Fts5Config *pConfig, const char *zLocale, int nLocale ){ - int rc = SQLITE_OK; Fts5TokenizerConfig *pT = &pConfig->t; pT->pLocale = zLocale; pT->nLocale = nLocale; - return rc; } /* -** Reset the locale of the tokenizer to its default. +** Clear any locale configured by an earlier call to fts5SetLocale() or +** sqlite3Fts5ExtractText(). */ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ fts5SetLocale(pConfig, 0, 0); @@ -1293,9 +1290,20 @@ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ ** 1) Ordinary values. The text can be extracted from these using ** sqlite3_value_text(). ** -** 2) Blobs tagged with sub-type FTS5_LOCALE_SUBTYPE, or those read from -** the content table of a normal content or external-conten table -** with locale=1 set. +** 2) Combination text/locale blobs created by fts5_locale(). There +** are several cases for these: +** +** * Blobs tagged with FTS5_LOCALE_SUBTYPE. +** * Blobs read from the content table of a locale=1 external-content +** table, and +** * Blobs read from the content table of a locale=1 regular +** content table. +** +** The first two cases above should have the 4 byte FTS5_LOCALE_HEADER +** header. It is an error if a blob with the subtype or a blob read +** from the content table of an external content table does not have +** the required header. A blob read from the content table of a regular +** locale=1 table does not have the header. This is to save space. ** ** If successful, SQLITE_OK is returned and output parameters (*ppText) ** and (*pnText) are set to point to a buffer containing the extracted utf-8 @@ -1306,11 +1314,11 @@ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ ** Parameter bContent must be true if the value was read from an indexed ** column (i.e. not UNINDEXED) of the on disk content. ** -** If pbResetTokenizer is not NULL and if case (2) is used, then the -** tokenizer is configured to use the locale. In this case (*pbResetTokenizer) -** is set to true before returning, to indicate that the caller must -** call sqlite3Fts5ClearLocale() to reset the tokenizer after tokenizing -** the text. +** If pbResetTokenizer is not NULL and if case (2) is used, then +** fts5SetLocale() is called to ensure subsequent sqlite3Fts5Tokenize() calls +** use the locale. In this case (*pbResetTokenizer) is set to true before +** returning, to indicate that the caller must call sqlite3Fts5ClearLocale() +** to clear the locale after tokenizing the text. */ int sqlite3Fts5ExtractText( Fts5Config *pConfig, @@ -1367,7 +1375,7 @@ int sqlite3Fts5ExtractText( nText = nBlob-nLocale-1; if( pbResetTokenizer ){ - rc = fts5SetLocale(pConfig, (const char*)pBlob, nLocale); + fts5SetLocale(pConfig, (const char*)pBlob, nLocale); *pbResetTokenizer = 1; } } @@ -1389,18 +1397,18 @@ int sqlite3Fts5ExtractText( ** the text of the expression, and sets output variable (*pzText) to ** point to a nul-terminated buffer containing the expression. ** -** If pVal was an fts5_locale() value, then the tokenizer has been -** configured to us the required locale. +** If pVal was an fts5_locale() value, then fts5SetLocale() is called to +** set the tokenizer to use the specified locale. ** ** If output variable (*pbFreeAndReset) is set to true, then the caller ** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer ** locale, and (b) call sqlite3_free() to free (*pzText). */ static int fts5ExtractExprText( - Fts5FullTable *pTab, - sqlite3_value *pVal, - char **pzText, - int *pbFreeAndReset + Fts5Config *pConfig, /* Fts5 configuration */ + sqlite3_value *pVal, /* Value to extract expression text from */ + char **pzText, /* OUT: nul-terminated buffer of text */ + int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */ ){ const char *zText = 0; int nText = 0; @@ -1408,12 +1416,12 @@ static int fts5ExtractExprText( int bReset = 0; *pbFreeAndReset = 0; - rc = sqlite3Fts5ExtractText(pTab->p.pConfig, pVal, 0, &bReset, &zText,&nText); + rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, &bReset, &zText, &nText); if( rc==SQLITE_OK ){ if( bReset ){ *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText); if( rc!=SQLITE_OK ){ - sqlite3Fts5ClearLocale(pTab->p.pConfig); + sqlite3Fts5ClearLocale(pConfig); }else{ *pbFreeAndReset = 1; } @@ -1494,7 +1502,7 @@ static int fts5FilterMethod( int bFreeAndReset = 0; int bInternal = 0; - rc = fts5ExtractExprText(pTab, apVal[i], &zText, &bFreeAndReset); + rc = fts5ExtractExprText(pConfig, apVal[i], &zText, &bFreeAndReset); if( rc!=SQLITE_OK ) goto filter_out; if( zText==0 ) zText = ""; @@ -2124,6 +2132,9 @@ static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); } +/* +** Implementation of xTokenize_v2() API. +*/ static int fts5ApiTokenize_v2( Fts5Context *pCtx, const char *pText, int nText, @@ -2143,6 +2154,11 @@ static int fts5ApiTokenize_v2( return rc; } + +/* +** Implementation of xTokenize() API. This is just xTokenize_v2() with NULL/0 +** passed as the locale. +*/ static int fts5ApiTokenize( Fts5Context *pCtx, const char *pText, int nText, @@ -2190,11 +2206,18 @@ static int fts5ApiColumnText( return rc; } +/* +** This is called by various API functions - xInst, xPhraseFirst, +** xPhraseFirstColumn etc. - to obtain the position list for phrase iPhrase +** of the current row. This function works for both detail=full tables (in +** which case the position-list was read from the fts index) or for other +** detail= modes if the row content is available. +*/ static int fts5CsrPoslist( - Fts5Cursor *pCsr, - int iPhrase, - const u8 **pa, - int *pn + Fts5Cursor *pCsr, /* Fts5 cursor object */ + int iPhrase, /* Phrase to find position list for */ + const u8 **pa, /* OUT: Pointer to position list buffer */ + int *pn /* OUT: Size of (*pa) in bytes */ ){ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; int rc = SQLITE_OK; @@ -2240,7 +2263,6 @@ static int fts5CsrPoslist( *pn = 0; } - return rc; } @@ -2808,6 +2830,11 @@ static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ return pCsr; } +/* +** Parameter zFmt is a printf() style formatting string. This function +** formats it using the trailing arguments and returns the result as +** an error message to the context passed as the first argument. +*/ static void fts5ResultError(sqlite3_context *pCtx, const char *zFmt, ...){ char *zErr = 0; va_list ap; @@ -2931,12 +2958,13 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ /* ** Value pVal was read from column iCol of the FTS5 table. This function ** returns it to the owner of pCtx via a call to an sqlite3_result_xxx() -** function. This function deals with the same 3 cases as +** function. This function deals with the same cases as ** sqlite3Fts5ExtractText(): ** ** 1) Ordinary values. These can be returned using sqlite3_result_value(). ** -** 2) Blobs from fts5_locale(). +** 2) Blobs from fts5_locale(). The text is extracted from these and +** returned via sqlite3_result_text(). The locale is discarded. */ static void fts5ExtractValueFromColumn( sqlite3_context *pCtx, @@ -3176,6 +3204,21 @@ static int fts5CreateAux( return rc; } +/* +** This function is used by xCreateTokenizer_v2() and xCreateTokenizer(). +** It allocates and partially populates a new Fts5TokenizerModule object. +** The new object is already linked into the Fts5Global context before +** returning. +** +** If successful, SQLITE_OK is returned and a pointer to the new +** Fts5TokenizerModule object returned via output parameter (*ppNew). All +** that is required is for the caller to fill in the methods in +** Fts5TokenizerModule.x1 and x2, and to set Fts5TokenizerModule.bV2Native +** as appropriate. +** +** If an error occurs, an SQLite error code is returned and the final value +** of (*ppNew) undefined. +*/ static int fts5NewTokenizerModule( Fts5Global *pGlobal, /* Global context (one per db handle) */ const char *zName, /* Name of new function */ diff --git a/manifest b/manifest index ce9f94a445..6750b90740 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sporter\stokenizer\sto\suse\slocales. -D 2024-08-12T11:46:09.154 +C Fix\sfurther\sissues\sto\sdo\swith\sfts5\slocale\ssupport. +D 2024-08-12T17:03:37.726 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,15 +92,15 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h 7f1197009fc0e9822a8a584aa1f90591bdbf04f4503ecfe06949f3afe7a1fe06 -F ext/fts5/fts5Int.h b40bb0bd54aaa4ac4712b6c5763b2167764614aaef204dbae81638b4548bca5d -F ext/fts5/fts5_aux.c 0d0ee62dfebe93ccf6b293edb0b21ebe5c8bdc85e962a001745f2d13ea3e79d2 +F ext/fts5/fts5.h 4c6998c6186268b4dbe9baef2c0d2ab974bd90996d61d4dbe801367249be6de4 +F ext/fts5/fts5Int.h 776b21159eef8d30379e5bc4627eae9e841d36e43f19dc8908c786e62aaf9e38 +F ext/fts5/fts5_aux.c 12cd2512f869217c38b70c31de5b5f741812734fafa80f55b32ea9bbd96e2152 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 187f7ffa5eddd6539ffa592de85e95b18be951728491390121bb215549a24a2a -F ext/fts5/fts5_expr.c ee1949c5c20901cbaca0885902f1d0c136679262dee71b457a34a92e1d16ddac +F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a35da6f6 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c cd61abbfd02f0f22e3c124ae2ad10c2a51cdf8acf38177410d44e134c1d1364b +F ext/fts5/fts5_main.c 4fe8349b812a9fde8e44ac5568f19d713ccc4790eb3ecb692f6551729c481b2b F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 F ext/fts5/fts5_tcl.c 50c7e16753fde0c4d80d8abd00a4ed2b0e998d5d3899a484510d01923c5da43b F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -2207,8 +2207,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f7d56a1f2149f0da117167db62e2c28ec337e8da3403873b64cdfc6a951e2e8e -R 7151af5ed6816182b47b60322cc8dcba +P 3291ce3a3359a80e51e4546a3d7a187cbe4c7530fca6632f0bb2728525efe212 +R 5bdde041363e74c68796cadade4d8480 U dan -Z a17240af0068f64d6da9a8176108962c +Z 098e7ed7a851f6658bf54618988e8ebd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6edb67265e..902e51759c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3291ce3a3359a80e51e4546a3d7a187cbe4c7530fca6632f0bb2728525efe212 +e626123580065986f7df50b6140f00048944becce179b9391fbf09f97ac55485 From a4a8e9102226f16cccd952e7902a20ee882c08c3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Aug 2024 17:28:13 +0000 Subject: [PATCH 19/99] Fix problem with some fts5 aux function APIs with (locale=, detail=none/column) tables. FossilOrigin-Name: bf116dec6fb20cce69099cb8c031b0de511195482919a99ccfb3c498e9bf8125 --- ext/fts5/fts5_main.c | 11 +++++++++-- ext/fts5/test/fts5locale.test | 30 ++++++++++++++++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index b4ccd04028..8af3dd0515 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2231,14 +2231,21 @@ static int fts5CsrPoslist( int i; aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); if( aPopulator==0 ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + rc = fts5SeekCursor(pCsr, 0); + } for(i=0; inCol && rc==SQLITE_OK; i++){ - int n; const char *z; - rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1); + const char *z = 0; + int n = 0; + int bReset = 0; + rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprPopulatePoslists( pConfig, pCsr->pExpr, aPopulator, i, z, n ); } + if( bReset ) sqlite3Fts5ClearLocale(pConfig); } sqlite3_free(aPopulator); diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index 2d5f2a00dc..a8ea31e221 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -414,5 +414,35 @@ do_execsql_test 8.1 { a eb eulav osla sihT te the token yam } +#------------------------------------------------------------------------- +# Test that position-lists (used by xInst, xPhraseFirst etc.) work with +# locales and modes other than detail=full. +# + +foreach {tn detail} { + 1 detail=full + 2 detail=none + 3 detail=column +} { + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + do_execsql_test 9.$tn.0 " + CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize=tcl, $detail); + " + do_execsql_test 9.$tn.1 { + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance); + INSERT INTO ft(rowid, tt) VALUES + (-1, fts5_locale('second', 'it is an ancient mariner')); + } + + do_execsql_test 9.$tn.2 { + SELECT DISTINCT term FROM vocab + } {an it mariner} + + do_execsql_test 9.$tn.3 { + SELECT highlight(ft, 0, '[', ']') FROM ft('mariner') + } {{it is an ancient [mariner]}} +} + finish_test diff --git a/manifest b/manifest index 6750b90740..c845da3de9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sfurther\sissues\sto\sdo\swith\sfts5\slocale\ssupport. -D 2024-08-12T17:03:37.726 +C Fix\sproblem\swith\ssome\sfts5\saux\sfunction\sAPIs\swith\s(locale=,\sdetail=none/column)\stables. +D 2024-08-12T17:28:13.218 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 187f7ffa5eddd6539ffa592de85e95b18be951728491390121bb215 F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a35da6f6 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 4fe8349b812a9fde8e44ac5568f19d713ccc4790eb3ecb692f6551729c481b2b +F ext/fts5/fts5_main.c b3194ee180859067fc8651d3c4e40d7f24877eb9cd820618d938d1a6d0974493 F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 F ext/fts5/fts5_tcl.c 50c7e16753fde0c4d80d8abd00a4ed2b0e998d5d3899a484510d01923c5da43b F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -186,7 +186,7 @@ F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba104 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 8e893b5a764d181260f5f862dc529fcdb42315b2d683317043d4609f13f88a02 +F ext/fts5/test/fts5locale.test 1f08fb39af3c0cb49ce48ec1212308db4b0105a38ff1a6864c8ef73abffb5431 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -2207,8 +2207,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3291ce3a3359a80e51e4546a3d7a187cbe4c7530fca6632f0bb2728525efe212 -R 5bdde041363e74c68796cadade4d8480 +P e626123580065986f7df50b6140f00048944becce179b9391fbf09f97ac55485 +R 7d41bee88044e0297d8070928920af0c U dan -Z 098e7ed7a851f6658bf54618988e8ebd +Z 30446f621f4abd678975f659d5345d71 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 902e51759c..0fc1e84a71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e626123580065986f7df50b6140f00048944becce179b9391fbf09f97ac55485 +bf116dec6fb20cce69099cb8c031b0de511195482919a99ccfb3c498e9bf8125 From 3e38aa08218c8a854916a513d54b456960a4f49e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Aug 2024 18:39:13 +0000 Subject: [PATCH 20/99] Add extra documentation comments to fts5_main.c. FossilOrigin-Name: 1a7a9bd3f6a2b9347720f7f0386399e7f14b428500acf07d7c55b9dde1ec233a --- ext/fts5/fts5_main.c | 39 +++++++++++++++++++++++++++++++++++++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 8af3dd0515..18582eb54e 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -3256,12 +3256,22 @@ static int fts5NewTokenizerModule( return rc; } +/* +** An instance of this type is used as the Fts5Tokenizer object for +** wrapper tokenizers - those that provide access to a v1 tokenizer via +** the fts5_tokenizer_v2 API, and those that provide access to a v2 tokenizer +** via the fts5_tokenizer API. +*/ typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer; struct Fts5VtoVTokenizer { Fts5TokenizerModule *pMod; Fts5Tokenizer *pReal; }; +/* +** Create a wrapper tokenizer. The context argument pCtx points to the +** Fts5TokenizerModule object. +*/ static int fts5VtoVCreate( void *pCtx, const char **azArg, @@ -3289,6 +3299,10 @@ static int fts5VtoVCreate( *ppOut = (Fts5Tokenizer*)pNew; return rc; } + +/* +** Delete an Fts5VtoVTokenizer wrapper tokenizer. +*/ static void fts5VtoVDelete(Fts5Tokenizer *pTok){ Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; if( p ){ @@ -3301,6 +3315,12 @@ static void fts5VtoVDelete(Fts5Tokenizer *pTok){ sqlite3_free(p); } } + + +/* +** xTokenizer method for a wrapper tokenizer that offers the v1 interface +** (no support for locales). +*/ static int fts5V1toV2Tokenize( Fts5Tokenizer *pTok, void *pCtx, int flags, @@ -3312,6 +3332,11 @@ static int fts5V1toV2Tokenize( assert( pMod->bV2Native ); return pMod->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken); } + +/* +** xTokenizer method for a wrapper tokenizer that offers the v2 interface +** (with locale support). +*/ static int fts5V2toV1Tokenize( Fts5Tokenizer *pTok, void *pCtx, int flags, @@ -3381,9 +3406,14 @@ static int fts5CreateTokenizer( return rc; } +/* +** Search the global context passed as the first argument for a tokenizer +** module named zName. If found, return a pointer to the Fts5TokenizerModule +** object. Otherwise, return NULL. +*/ static Fts5TokenizerModule *fts5LocateTokenizer( - Fts5Global *pGlobal, - const char *zName + Fts5Global *pGlobal, /* Global (one per db handle) object */ + const char *zName /* Name of tokenizer module to find */ ){ Fts5TokenizerModule *pMod = 0; @@ -3518,6 +3548,11 @@ int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){ } +/* +** xDestroy callback passed to sqlite3_create_module(). This is invoked +** when the db handle is being closed. Free memory associated with +** tokenizers and aux functions registered with this db handle. +*/ static void fts5ModuleDestroy(void *pCtx){ Fts5TokenizerModule *pTok, *pNextTok; Fts5Auxiliary *pAux, *pNextAux; diff --git a/manifest b/manifest index c845da3de9..238fedebb5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblem\swith\ssome\sfts5\saux\sfunction\sAPIs\swith\s(locale=,\sdetail=none/column)\stables. -D 2024-08-12T17:28:13.218 +C Add\sextra\sdocumentation\scomments\sto\sfts5_main.c. +D 2024-08-12T18:39:13.026 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 187f7ffa5eddd6539ffa592de85e95b18be951728491390121bb215 F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a35da6f6 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c b3194ee180859067fc8651d3c4e40d7f24877eb9cd820618d938d1a6d0974493 +F ext/fts5/fts5_main.c 93468b6f48aa7600e46e940918186b74c79388487ea85790ebab8dab133e8468 F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 F ext/fts5/fts5_tcl.c 50c7e16753fde0c4d80d8abd00a4ed2b0e998d5d3899a484510d01923c5da43b F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -2207,8 +2207,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e626123580065986f7df50b6140f00048944becce179b9391fbf09f97ac55485 -R 7d41bee88044e0297d8070928920af0c +P bf116dec6fb20cce69099cb8c031b0de511195482919a99ccfb3c498e9bf8125 +R bb64d9fe4ea841aa0ab73938efab7522 U dan -Z 30446f621f4abd678975f659d5345d71 +Z 8f8f9d2cf7c267c2be98bbe165feb5ff # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0fc1e84a71..5f0fdf183c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf116dec6fb20cce69099cb8c031b0de511195482919a99ccfb3c498e9bf8125 +1a7a9bd3f6a2b9347720f7f0386399e7f14b428500acf07d7c55b9dde1ec233a From 7dad9b4297bfc1e66c69aa39ea5673b66755602d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Aug 2024 15:04:00 +0000 Subject: [PATCH 21/99] Add tests and fix minor issues with the code on this branch. FossilOrigin-Name: def5243a7b4f1d50427ba607d31332db2d6ac34b181b0cbd2ed4911064f5637a --- ext/fts5/fts5_main.c | 37 ++++++++++++++--------------------- ext/fts5/test/fts5simple.test | 1 - manifest | 14 ++++++------- manifest.uuid | 2 +- 4 files changed, 23 insertions(+), 31 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 18582eb54e..a70dad4b20 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -3488,21 +3488,20 @@ static int fts5FindTokenizer( return rc; } -int fts5GetTokenizer( - Fts5Global *pGlobal, - const char **azArg, - int nArg, - Fts5Config *pConfig, - char **pzErr -){ - Fts5TokenizerModule *pMod; +/* +** Attempt to instantiate the tokenizer. +*/ +int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){ + const char **azArg = pConfig->t.azArg; + const int nArg = pConfig->t.nArg; + Fts5TokenizerModule *pMod = 0; int rc = SQLITE_OK; - pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); + pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]); if( pMod==0 ){ assert( nArg>0 ); rc = SQLITE_ERROR; - if( pzErr ) *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + sqlite3Fts5ConfigErrmsg(pConfig, "no such tokenizer: %s", azArg[0]); }else{ int (*xCreate)(void*, const char**, int, Fts5Tokenizer**) = 0; if( pMod->bV2Native ){ @@ -3518,8 +3517,8 @@ int fts5GetTokenizer( ); if( rc!=SQLITE_OK ){ - if( pzErr && rc!=SQLITE_NOMEM ){ - *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + if( rc!=SQLITE_NOMEM ){ + sqlite3Fts5ConfigErrmsg(pConfig, "error in tokenizer constructor"); } }else if( pMod->bV2Native==0 ){ pConfig->t.ePattern = sqlite3Fts5TokenizerPattern( @@ -3537,16 +3536,6 @@ int fts5GetTokenizer( return rc; } -/* -** Attempt to instantiate the tokenizer. -*/ -int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){ - return fts5GetTokenizer( - pConfig->pGlobal, pConfig->t.azArg, pConfig->t.nArg, - pConfig, pConfig->pzErrmsg - ); -} - /* ** xDestroy callback passed to sqlite3_create_module(). This is invoked @@ -3573,6 +3562,10 @@ static void fts5ModuleDestroy(void *pCtx){ sqlite3_free(pGlobal); } +/* +** Implementation of the fts5() function used by clients to obtain the +** API pointer. +*/ static void fts5Fts5Func( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args */ diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 050509fb37..6384095067 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -312,7 +312,6 @@ do_execsql_test 13.1 { INSERT INTO xy(rowid, x) VALUES(3, '3 4 5'); } -breakpoint do_execsql_test 13.2 { UPDATE OR REPLACE xy SET rowid=3 WHERE rowid = 2; SELECT rowid, x FROM xy; diff --git a/manifest b/manifest index 238fedebb5..8e8fa479e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\sdocumentation\scomments\sto\sfts5_main.c. -D 2024-08-12T18:39:13.026 +C Add\stests\sand\sfix\sminor\sissues\swith\sthe\scode\son\sthis\sbranch. +D 2024-08-13T15:04:00.350 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 187f7ffa5eddd6539ffa592de85e95b18be951728491390121bb215 F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a35da6f6 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 93468b6f48aa7600e46e940918186b74c79388487ea85790ebab8dab133e8468 +F ext/fts5/fts5_main.c 42860180de19a159dd821e83450916ddfae92a6aeb8cd357c8c93ef10d3dec0e F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 F ext/fts5/fts5_tcl.c 50c7e16753fde0c4d80d8abd00a4ed2b0e998d5d3899a484510d01923c5da43b F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -224,7 +224,7 @@ F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a F ext/fts5/test/fts5secure8.test e68c0ac4447f415ff3e4e82531e99548289286f9f3a29c8cd53036113fe28602 F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5 -F ext/fts5/test/fts5simple.test 453b2348193911c2d51c208bfe247a09b936c4ddd9a836160495a5554340c256 +F ext/fts5/test/fts5simple.test 847fb828262328744733847dc76d6b5d4a6bd4c5d9b282cb819f6504340e061a F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc F ext/fts5/test/fts5simple3.test 146ec3dc8f5763d6212641c9f0a2f1cba41679353d2add7b963beceb115dc7f4 F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988202ce67638d2c6cf6 @@ -2207,8 +2207,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bf116dec6fb20cce69099cb8c031b0de511195482919a99ccfb3c498e9bf8125 -R bb64d9fe4ea841aa0ab73938efab7522 +P 1a7a9bd3f6a2b9347720f7f0386399e7f14b428500acf07d7c55b9dde1ec233a +R 3ca8b15875339dd4f641808ef826812c U dan -Z 8f8f9d2cf7c267c2be98bbe165feb5ff +Z 64c2a3f3ef9fc7e9fed1ba5b27ea297b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5f0fdf183c..80e74cf5f3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a7a9bd3f6a2b9347720f7f0386399e7f14b428500acf07d7c55b9dde1ec233a +def5243a7b4f1d50427ba607d31332db2d6ac34b181b0cbd2ed4911064f5637a From a7384e0de2df5371e069b37f8705259159fe418b Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Aug 2024 21:15:43 +0000 Subject: [PATCH 22/99] More tests for the new code on this branch. FossilOrigin-Name: 00792e807f1dde750e6ac9f9b0095cf60b9fa6ff7cf4b14440600a21de2ee61a --- ext/fts5/fts5_main.c | 19 ++++---- ext/fts5/test/fts5blob.test | 17 ++++++++ ext/fts5/test/fts5faultI.test | 67 +++++++++++++++++++++++++++++ ext/fts5/test/fts5locale.test | 81 ++++++++++++++++++++++++++++++++++- ext/fts5/test/fts5misc.test | 5 +++ manifest | 19 ++++---- manifest.uuid | 2 +- 7 files changed, 189 insertions(+), 21 deletions(-) create mode 100644 ext/fts5/test/fts5faultI.test diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index a70dad4b20..80fe73a2fa 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1992,12 +1992,11 @@ static int fts5UpdateMethod( } } + assert( eType1==SQLITE_INTEGER || eType1==SQLITE_NULL ); if( eType0!=SQLITE_INTEGER ){ /* An INSERT statement. If the conflict-mode is REPLACE, first remove ** the current entry (if any). */ - if( eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL ){ - rc = SQLITE_MISMATCH; - }else if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ + if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0); bUpdateOrDelete = 1; @@ -2726,7 +2725,7 @@ static int fts5ApiColumnLocale( if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; if( nBlobeContent==FTS5_CONTENT_EXTERNAL ){ if( nBlob', '') FROM t1('*reads')); } {1 {no such cursor: 1}} +fts5_aux_test_functions db +do_catchsql_test 1.3.4 { + SELECT fts5_columntext(t1) FROM t1('*reads'); +} {1 {no such cursor: 1}} + #------------------------------------------------------------------------- reset_db do_execsql_test 2.0 { diff --git a/manifest b/manifest index 8e8fa479e7..318eb8554f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sand\sfix\sminor\sissues\swith\sthe\scode\son\sthis\sbranch. -D 2024-08-13T15:04:00.350 +C More\stests\sfor\sthe\snew\scode\son\sthis\sbranch. +D 2024-08-13T21:15:43.313 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 187f7ffa5eddd6539ffa592de85e95b18be951728491390121bb215 F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a35da6f6 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 42860180de19a159dd821e83450916ddfae92a6aeb8cd357c8c93ef10d3dec0e +F ext/fts5/fts5_main.c f8eae223ce93aad7c503a76872267e0e6d288b54ccb4d9bfe894c4ed365f8d3c F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 F ext/fts5/fts5_tcl.c 50c7e16753fde0c4d80d8abd00a4ed2b0e998d5d3899a484510d01923c5da43b F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -132,7 +132,7 @@ F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06c F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36 -F ext/fts5/test/fts5blob.test 2f84ef3591dd3eb0efa686a3615e9ed5c5005462c663c7c72c350692b99f91f5 +F ext/fts5/test/fts5blob.test caa33369e93e99ff494cd1103506ae34c5afbc0bcc369ed5e58e135144e33689 F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c2be25568cc33a F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62 F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897 @@ -177,6 +177,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d +F ext/fts5/test/fts5faultI.test bfaf9b94888ef604bd8034a3edd1bbdeb626c07337376b9b66d6171b28ee7931 F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -186,11 +187,11 @@ F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba104 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 1f08fb39af3c0cb49ce48ec1212308db4b0105a38ff1a6864c8ef73abffb5431 +F ext/fts5/test/fts5locale.test f58ac6ab539d3813556646f7d4b1e19f7e3b1a56ca4439349eb8099d6a821ff6 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test b88aa28ff20238b394495b0795cddca1a62b98fa09b99e462a8abc572d04ee88 +F ext/fts5/test/fts5misc.test 946abba0b45298e3d9d7a98e19eb4ac5384e8f2848ce1888c7ab3f62645efd2f F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74 @@ -2207,8 +2208,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a7a9bd3f6a2b9347720f7f0386399e7f14b428500acf07d7c55b9dde1ec233a -R 3ca8b15875339dd4f641808ef826812c +P def5243a7b4f1d50427ba607d31332db2d6ac34b181b0cbd2ed4911064f5637a +R 4b120525083c8e765eb88ddcda2d97e2 U dan -Z 64c2a3f3ef9fc7e9fed1ba5b27ea297b +Z aad3d9cc9413a3fb75b2e684b7b3f795 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 80e74cf5f3..154fd54b3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -def5243a7b4f1d50427ba607d31332db2d6ac34b181b0cbd2ed4911064f5637a +00792e807f1dde750e6ac9f9b0095cf60b9fa6ff7cf4b14440600a21de2ee61a From 189c41221d499fb97bd0bac6d1e1f955b8a1afab Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Aug 2024 18:50:13 +0000 Subject: [PATCH 23/99] Further tests and fixes for this branch. FossilOrigin-Name: d27985245a0e8c0d6b04323c98b26b6a8fb4e489fa8f5f3234252c7c198f23c8 --- ext/fts5/fts5_main.c | 22 +++++++----------- ext/fts5/test/fts5ah.test | 11 +++++++++ ext/fts5/test/fts5al.test | 10 +++++++++ ext/fts5/test/fts5faultI.test | 41 +++++++++++++++++++++++++++++++++- ext/fts5/test/fts5misc.test | 8 +++---- ext/fts5/test/fts5simple.test | 29 ++++++++++++++++++++++++ ext/fts5/test/fts5trigram.test | 31 +++++++++++++++++++++++++ manifest | 24 ++++++++++---------- manifest.uuid | 2 +- 9 files changed, 146 insertions(+), 32 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 80fe73a2fa..d03bd2513e 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -630,7 +630,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ if( bSeenRank ) continue; idxStr[iIdxStr++] = 'r'; bSeenRank = 1; - }else if( iCol>=0 ){ + }else{ nSeenMatch++; idxStr[iIdxStr++] = 'M'; sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); @@ -1468,13 +1468,7 @@ static int fts5FilterMethod( int iIdxStr = 0; Fts5Expr *pExpr = 0; - if( pConfig->bLock ){ - pTab->p.base.zErrMsg = sqlite3_mprintf( - "recursively defined fts5 content table" - ); - return SQLITE_ERROR; - } - + assert( pConfig->bLock==0 ); if( pCsr->ePlan ){ fts5FreeCursorComponents(pCsr); memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); @@ -1992,7 +1986,6 @@ static int fts5UpdateMethod( } } - assert( eType1==SQLITE_INTEGER || eType1==SQLITE_NULL ); if( eType0!=SQLITE_INTEGER ){ /* An INSERT statement. If the conflict-mode is REPLACE, first remove ** the current entry (if any). */ @@ -2186,11 +2179,11 @@ static int fts5ApiColumnText( int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); if( iCol<0 || iCol>=pTab->pConfig->nCol ){ rc = SQLITE_RANGE; - }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) - || pCsr->ePlan==FTS5_PLAN_SPECIAL - ){ + }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) ){ *pz = 0; *pn = 0; }else{ @@ -2701,11 +2694,11 @@ static int fts5ApiColumnLocale( *pzLocale = 0; *pnLocale = 0; + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); if( iCol<0 || iCol>=pConfig->nCol ){ rc = SQLITE_RANGE; }else if( pConfig->abUnindexed[iCol]==0 - && pCsr->ePlan!=FTS5_PLAN_SPECIAL && pConfig->eContent!=FTS5_CONTENT_NONE && pConfig->bLocale ){ @@ -2823,6 +2816,7 @@ static void fts5ApiInvoke( sqlite3_value **argv ){ assert( pCsr->pAux==0 ); + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); pCsr->pAux = pAux; pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); pCsr->pAux = 0; @@ -2866,7 +2860,7 @@ static void fts5ApiCallback( iCsrId = sqlite3_value_int64(argv[0]); pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); - if( pCsr==0 || pCsr->ePlan==0 ){ + if( pCsr==0 || (pCsr->ePlan==0 || pCsr->ePlan==FTS5_PLAN_SPECIAL) ){ fts5ResultError(context, "no such cursor: %lld", iCsrId); }else{ sqlite3_vtab *pTab = pCsr->base.pVtab; diff --git a/ext/fts5/test/fts5ah.test b/ext/fts5/test/fts5ah.test index bc80057833..bf9c9e9dbc 100644 --- a/ext/fts5/test/fts5ah.test +++ b/ext/fts5/test/fts5ah.test @@ -163,6 +163,17 @@ do_execsql_test 1.8.2 { SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text'; } {10000} +do_execsql_test 1.8.3 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid<5000 AND rowid < 'text'; +} {4999} +do_execsql_test 1.8.4 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid>5000 AND rowid > 'text'; +} {0} + +do_catchsql_test 1.9 { + SELECT * FROM t1('*xy'); +} {1 {unknown special query: xy}} + } ;# foreach_detail_mode #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} diff --git a/ext/fts5/test/fts5al.test b/ext/fts5/test/fts5al.test index b344e0d2ea..7187ad67c7 100644 --- a/ext/fts5/test/fts5al.test +++ b/ext/fts5/test/fts5al.test @@ -293,6 +293,16 @@ do_catchsql_test 4.4.4 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL } {1 {parse error in rank function: }} +# Check that the second and subsequent rank= constraints are ignored. +# +do_catchsql_test 4.3.3 { + SELECT *, rank FROM t3 + WHERE t3 MATCH 'a' AND + rank MATCH 'nosuch()' AND + rank MATCH 'rowidmod(3)' + ORDER BY rank ASC +} {1 {unable to use function MATCH in the requested context}} + } ;# foreach_detail_mode diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test index 0d5ef4a54a..b64f27c1fc 100644 --- a/ext/fts5/test/fts5faultI.test +++ b/ext/fts5/test/fts5faultI.test @@ -60,7 +60,46 @@ do_faultsim_test 3 -faults oom* -prep { SELECT fts5_columnlocale(t1, 0) FROM t1('unicode*'); } } -test { - faultsim_test_result {0 {{}}} + faultsim_test_result {0 {{}}} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); +} +faultsim_save_and_close + +do_faultsim_test 4 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + INSERT INTO w1 VALUES('token token token'); + } +} -body { + execsql { + INSERT INTO w1(w1, rank) VALUES('rank', 'bm25()'); + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 5 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + INSERT INTO w1 VALUES('one'); + SAVEPOINT one; + INSERT INTO w1 VALUES('two'); + ROLLBACK TO one; + } + +} -body { + execsql { + INSERT INTO w1 VALUES('string'); + } +} -test { + faultsim_test_result {0 {}} } finish_test diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index a9806ff8bc..e5f43d69c5 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -35,21 +35,21 @@ do_catchsql_test 1.1.2 { do_catchsql_test 1.2.1 { SELECT highlight(t1, 4, '', '') FROM t1('*id'); -} {0 {{}}} +} {1 {no such cursor: 4}} do_catchsql_test 1.2.2 { SELECT a FROM t1 WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*id')); -} {0 {}} +} {1 {no such cursor: 6}} do_catchsql_test 1.3.1 { SELECT highlight(t1, 4, '', '') FROM t1('*reads'); -} {1 {no such cursor: 2}} +} {1 {no such cursor: 1}} do_catchsql_test 1.3.2 { SELECT a FROM t1 WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*reads')); -} {1 {no such cursor: 2}} +} {1 {no such cursor: 1}} db close sqlite3 db test.db diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 6384095067..60ccb5a9c5 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -480,4 +480,33 @@ do_execsql_test 22.0 { do_catchsql_test 22.1 {SELECT * FROM x1('')} {1 {fts5: syntax error near ""}} do_catchsql_test 22.2 {SELECT * FROM x1(NULL)} {1 {fts5: syntax error near ""}} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 23.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + SELECT count(*) FROM x1_data; +} {2} + +do_execsql_test 23.1 { + BEGIN; + INSERT INTO x1 VALUES('a b c d'); + INSERT INTO x1 VALUES('a b c d'); + INSERT INTO x1 VALUES('a b c d'); +} + +do_execsql_test 23.2 { + SELECT count(*) FROM x1_data; +} {2} + +do_execsql_test 23.3 { + INSERT INTO x1(x1) VALUES('flush'); + SELECT count(*) FROM x1_data; +} {3} + +do_execsql_test 23.4 { + ROLLBACK; + SELECT count(*) FROM x1_data; +} {2} + + finish_test diff --git a/ext/fts5/test/fts5trigram.test b/ext/fts5/test/fts5trigram.test index 752686620c..b76dbda063 100644 --- a/ext/fts5/test/fts5trigram.test +++ b/ext/fts5/test/fts5trigram.test @@ -200,6 +200,12 @@ do_eqp_test 6.3 { do_eqp_test 6.4 { SELECT * FROM ci1 WHERE x GLOB ? } {VIRTUAL TABLE INDEX 0:G0} +do_eqp_test 6.5 { + SELECT * FROM ci1 WHERE x < ? +} {{SCAN ci1 VIRTUAL TABLE INDEX 0:}} +do_eqp_test 6.6 { + SELECT * FROM ci0 WHERE x < ? +} {{SCAN ci0 VIRTUAL TABLE INDEX 0:}} reset_db do_execsql_test 7.0 { @@ -256,4 +262,29 @@ do_execsql_test 8.3 { {[abcde]} } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, + tokenize=trigram + ); + + INSERT INTO t1(rowid, a12) VALUES(111, 'thats a tricky case though'); + INSERT INTO t1(rowid, a12) VALUES(222, 'the query planner cannot do'); +} + +do_execsql_test 9.1 { + SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%' +} {111} + +do_execsql_test 9.2 { + SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%' AND a12 LIKE '%case%' +} {111} + +do_execsql_test 9.3 { + SELECT rowid FROM t1 WHERE a12 LIKE NULL +} {} + + finish_test diff --git a/manifest b/manifest index 318eb8554f..4071c7a84b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\stests\sfor\sthe\snew\scode\son\sthis\sbranch. -D 2024-08-13T21:15:43.313 +C Further\stests\sand\sfixes\sfor\sthis\sbranch. +D 2024-08-15T18:50:13.581 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 187f7ffa5eddd6539ffa592de85e95b18be951728491390121bb215 F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a35da6f6 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c f8eae223ce93aad7c503a76872267e0e6d288b54ccb4d9bfe894c4ed365f8d3c +F ext/fts5/fts5_main.c 21160a392addf6921e380090984aed88bbb6d88c33a7d03187997c9c8f666e49 F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 F ext/fts5/fts5_tcl.c 50c7e16753fde0c4d80d8abd00a4ed2b0e998d5d3899a484510d01923c5da43b F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -119,11 +119,11 @@ F ext/fts5/test/fts5ad.test 058e616612964e61d19f70295f0e6eaedceb4b29b1fbf4f85961 F ext/fts5/test/fts5ae.test 3d49edbd50bb0684199a2e7568aeb30d1d29718f5c0f61751983740fa836d15f F ext/fts5/test/fts5af.test ae81f08b8da4c5f9b3ec1ef538a4ab6b7c278e92fa9058d6dc5d842c5d9771b9 F ext/fts5/test/fts5ag.test 6667807b5d3fbf460892e756763fbe3d87a2fffe345a06514ba010ca6f6641f7 -F ext/fts5/test/fts5ah.test ac327281c8910cf9b85738a2655003fd0c6a8a76189ef34f3d086b8f9e54263b +F ext/fts5/test/fts5ah.test e1f01314b35745a30e1b494b46045b82005d71cae74f1ebd9f1338566b77f9fc F ext/fts5/test/fts5ai.test cbe26d78030998f535bc103f37915350b137a822c71a9db439a077d7666a3539 F ext/fts5/test/fts5aj.test 53c8508dab4acca3e691a4c51eca4b3b018319ab8635e540103d5bbdc91543c9 F ext/fts5/test/fts5ak.test 25e2f8afdcff30d98ca9dee8c5cacca2f26db17501c9401f16d99ee036f70e8d -F ext/fts5/test/fts5al.test 842c50fd9b287e3fa988dfcab436b27c432866f7406a56aaf3c67f316952cc02 +F ext/fts5/test/fts5al.test f0e655606771b2b5dbaf70e7f0044d560257cf3531d5eea40df58d0d7add8c39 F ext/fts5/test/fts5alter.test ebbee06419c2d3cee5ef7ebb5ba6a9996f1aa374035361c0acd37368cc5f64f3 F ext/fts5/test/fts5auto.test 2278de323172ced485d2844cb1357d00036ac1665f27e70fa1a48ce57bf31c2c F ext/fts5/test/fts5aux.test 27210687338133b1e9bc0dd669322fca59fd432439f40b126895e2d7c2f899d6 @@ -177,7 +177,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d -F ext/fts5/test/fts5faultI.test bfaf9b94888ef604bd8034a3edd1bbdeb626c07337376b9b66d6171b28ee7931 +F ext/fts5/test/fts5faultI.test 6aa7c42a9c14f00b90e7b33412d98153c0ca1534c5ab11db25e39b4d63f88795 F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -191,7 +191,7 @@ F ext/fts5/test/fts5locale.test f58ac6ab539d3813556646f7d4b1e19f7e3b1a56ca443934 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test 946abba0b45298e3d9d7a98e19eb4ac5384e8f2848ce1888c7ab3f62645efd2f +F ext/fts5/test/fts5misc.test 281a55fc3b41543c4a7fc30b0d4c703a819ce659f04553a4840a5dc8ec277397 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74 @@ -225,7 +225,7 @@ F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a F ext/fts5/test/fts5secure8.test e68c0ac4447f415ff3e4e82531e99548289286f9f3a29c8cd53036113fe28602 F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5 -F ext/fts5/test/fts5simple.test 847fb828262328744733847dc76d6b5d4a6bd4c5d9b282cb819f6504340e061a +F ext/fts5/test/fts5simple.test ed7c3815c9fa1c16166258cb98edb2e014c63c7589958d76c5487df0df913d61 F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc F ext/fts5/test/fts5simple3.test 146ec3dc8f5763d6212641c9f0a2f1cba41679353d2add7b963beceb115dc7f4 F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988202ce67638d2c6cf6 @@ -235,7 +235,7 @@ F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98 F ext/fts5/test/fts5tokenizer2.test ddb8b10fbe4b84b2a75812671f127774c1d2e3e2bf82d2e0e4f0bb1cd8a2b2d6 F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3 -F ext/fts5/test/fts5trigram.test be914555deb8504dde682bd5aa343d00c4da37dfad20709a5bac30d5f97f2ef5 +F ext/fts5/test/fts5trigram.test f99fc0235e967826e556de9aee959d3cb8eae2e9abff37b5fe2732fd7052dac2 F ext/fts5/test/fts5trigram2.test 6fde9de7f63a6b4aa18dc731be56dbd6be4e755c9b13dcd55479e200d1df0e61 F ext/fts5/test/fts5ubsan.test 9a2dcf399dc8d0e0de661f0d93884d1d27e5b7f0693cfceb97dd24d818df5dd2 F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602 @@ -2208,8 +2208,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P def5243a7b4f1d50427ba607d31332db2d6ac34b181b0cbd2ed4911064f5637a -R 4b120525083c8e765eb88ddcda2d97e2 +P 00792e807f1dde750e6ac9f9b0095cf60b9fa6ff7cf4b14440600a21de2ee61a +R 26644a6a4d254f87e6d210a1b5d5866f U dan -Z aad3d9cc9413a3fb75b2e684b7b3f795 +Z 43505ba0ff6d8eb4d2d57750e91b3ee5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 154fd54b3f..4334ad1f96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00792e807f1dde750e6ac9f9b0095cf60b9fa6ff7cf4b14440600a21de2ee61a +d27985245a0e8c0d6b04323c98b26b6a8fb4e489fa8f5f3234252c7c198f23c8 From 41ebf965cd1e80d72dfb4535e9c0143fed0435c7 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Aug 2024 20:33:05 +0000 Subject: [PATCH 24/99] Extra tests. FossilOrigin-Name: fb257540d32ced4a5fe0759d3ad9c1b36fd2b7d64d93cb0a0ecdb48355a68b7a --- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_main.c | 3 ++- ext/fts5/test/fts5_common.tcl | 5 ++++ ext/fts5/test/fts5faultI.test | 44 +++++++++++++++++++++++++++++++ ext/fts5/test/fts5misc.test | 20 ++++++++++++++ ext/fts5/test/fts5origintext.test | 9 +++++++ manifest | 22 ++++++++-------- manifest.uuid | 2 +- 8 files changed, 93 insertions(+), 14 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 7e42ea82c1..7e41119572 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -345,7 +345,7 @@ char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); void sqlite3Fts5Put32(u8*, int); int sqlite3Fts5Get32(const u8*); -#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32) +#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF) #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF) typedef struct Fts5PoslistReader Fts5PoslistReader; diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index d03bd2513e..06833a98b2 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2330,7 +2330,8 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ aInst[0] = iBest; aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); - if( aInst[1]<0 || aInst[1]>=nCol ){ + assert( aInst[1]>=0 ); + if( aInst[1]>=nCol ){ rc = FTS5_CORRUPT; break; } diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index b78064e8d8..8ea87dbdd1 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -51,6 +51,10 @@ proc fts5_test_poslist2 {cmd} { sort_poslist $res } +proc fts5_test_insttoken {cmd iInst iToken} { + $cmd xInstToken $iInst $iToken +} + proc fts5_test_collist {cmd} { set res [list] @@ -181,6 +185,7 @@ proc fts5_aux_test_functions {db} { fts5_test_poslist fts5_test_poslist2 fts5_test_collist + fts5_test_insttoken fts5_test_tokenize fts5_test_rowcount fts5_test_rowid diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test index b64f27c1fc..33c93c63af 100644 --- a/ext/fts5/test/fts5faultI.test +++ b/ext/fts5/test/fts5faultI.test @@ -102,5 +102,49 @@ do_faultsim_test 5 -faults oom* -prep { faultsim_test_result {0 {}} } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); +} +fts5_aux_test_functions db + +do_faultsim_test 5 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_test_insttoken(w1, 0, 0) FROM w1('two'); + } +} -test { + faultsim_test_result {0 two} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); +} +fts5_aux_test_functions db +faultsim_save_and_close + +do_faultsim_test 6 -faults oom* -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + INSERT INTO w1 VALUES('four five six'); + SAVEPOINT abc; + INSERT INTO w1 VALUES('seven eight nine'); + SAVEPOINT def; + INSERT INTO w1 VALUES('ten eleven twelve'); + } +} -body { + execsql { + RELEASE abc; + } +} -test { + faultsim_test_result {0 {}} +} + finish_test diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index e5f43d69c5..4b943c3488 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -571,5 +571,25 @@ do_execsql_test 20.5 { SELECT rowid FROM x1 WHERE x1 MATCH 'z' OR (x1 MATCH 'a' AND x1 MATCH 'd'); } {3 1} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 21.0 { + CREATE TABLE t1(ii INTEGER, x TEXT, y TEXT); + CREATE VIRTUAL TABLE xyz USING fts5(content_rowid=ii, content=t1, x, y); + INSERT INTO t1 VALUES(1, 'one', 'i'); + INSERT INTO t1 VALUES(2, 'two', 'ii'); + INSERT INTO t1 VALUES(3, 'tree', 'iii'); + INSERT INTO xyz(xyz) VALUES('rebuild'); +} + +do_execsql_test 21.1 { + UPDATE xyz SET y='TWO' WHERE rowid=2; + UPDATE t1 SET y='TWO' WHERE ii=2; +} + +do_execsql_test 21.2 { + PRAGMA integrity_check +} {ok} + finish_test diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index cc9d99e2d9..1612b0d7f4 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -291,6 +291,15 @@ do_execsql_test 6.3 { SELECT rowid, tokens(ft) FROM ft('Three*'); } {1 {{}} 2 {{}}} +fts5_aux_test_functions db +do_catchsql_test 6.4 { + SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one'); +} {1 SQLITE_RANGE} + +do_catchsql_test 6.5 { + SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one'); +} {1 SQLITE_RANGE} + } finish_test diff --git a/manifest b/manifest index 4071c7a84b..28ab5391d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stests\sand\sfixes\sfor\sthis\sbranch. -D 2024-08-15T18:50:13.581 +C Extra\stests. +D 2024-08-15T20:33:05.956 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -93,14 +93,14 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e F ext/fts5/fts5.h 4c6998c6186268b4dbe9baef2c0d2ab974bd90996d61d4dbe801367249be6de4 -F ext/fts5/fts5Int.h 776b21159eef8d30379e5bc4627eae9e841d36e43f19dc8908c786e62aaf9e38 +F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d805655360 F ext/fts5/fts5_aux.c 12cd2512f869217c38b70c31de5b5f741812734fafa80f55b32ea9bbd96e2152 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 187f7ffa5eddd6539ffa592de85e95b18be951728491390121bb215549a24a2a F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a35da6f6 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 21160a392addf6921e380090984aed88bbb6d88c33a7d03187997c9c8f666e49 +F ext/fts5/fts5_main.c 23029229021240dc21a69eb749305316103d183375413e2090c064b74b03b7f8 F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 F ext/fts5/fts5_tcl.c 50c7e16753fde0c4d80d8abd00a4ed2b0e998d5d3899a484510d01923c5da43b F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -111,7 +111,7 @@ F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d0988 F ext/fts5/fts5_vocab.c e4830b00809e5da53bc10f93adc59e321407b0f801c7f4167c0e47f5552267e0 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 48c0855d235a2f141e5519d3170bad7cddcc53ad3a16ac8ef952ba61048c2958 +F ext/fts5/test/fts5_common.tcl c5aa7cf7148b6dcffb5b61520ae18212baf169936af734ab265143f59db328fe F ext/fts5/test/fts5aa.test 015c81b84d53bfcedd77d624202c8b02e9f0cbbb4b51688e3a9c9f90bccbb4ac F ext/fts5/test/fts5ab.test 4bdb619fee409e11417e8827e320b857e42e926a01a0408fc9f143ec870a6ced F ext/fts5/test/fts5ac.test 4a73626de86f3d17c95738034880c4f0de8d54741fb943d819b528373657e59b @@ -177,7 +177,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d -F ext/fts5/test/fts5faultI.test 6aa7c42a9c14f00b90e7b33412d98153c0ca1534c5ab11db25e39b4d63f88795 +F ext/fts5/test/fts5faultI.test d9e8c4b0b42910fa447a0e148236def618e7decbf3a334c7933a95e36410f772 F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -191,7 +191,7 @@ F ext/fts5/test/fts5locale.test f58ac6ab539d3813556646f7d4b1e19f7e3b1a56ca443934 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test 281a55fc3b41543c4a7fc30b0d4c703a819ce659f04553a4840a5dc8ec277397 +F ext/fts5/test/fts5misc.test b49a3afbda6de02970560c1cd3b804a4896b9f30ce5d75a5f215fa972537b26b F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74 @@ -199,7 +199,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 264b9101721c17d06d1d174feb743fda3ddc89fad41dee980fef821428258e47 F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9be0cfe22064fc71a44ed F ext/fts5/test/fts5optimize3.test 1653029284e10e0715246819893ba30565c4ead0d0fc470adae92c353ea857d3 -F ext/fts5/test/fts5origintext.test 87c34c78f201b1e22ac93ac6bbe6196dde59f0a7266b1aeb938604a0eb9d5552 +F ext/fts5/test/fts5origintext.test 002abba00b334cb4b3f957f940f75b9ab53f4caae97aa753e19da285c245e463 F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b37f29cad3a8b8d5ac6f9ba F ext/fts5/test/fts5origintext3.test 45c33cf0c91a9ca0e36d298462db3edc7c8fe45fd185649a9dbfd66bb670058b F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8 @@ -2208,8 +2208,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 00792e807f1dde750e6ac9f9b0095cf60b9fa6ff7cf4b14440600a21de2ee61a -R 26644a6a4d254f87e6d210a1b5d5866f +P d27985245a0e8c0d6b04323c98b26b6a8fb4e489fa8f5f3234252c7c198f23c8 +R c260c68d60eebc40b0c96509388ca9e3 U dan -Z 43505ba0ff6d8eb4d2d57750e91b3ee5 +Z d7c15a75ca8b075cfcf78184db374c6a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4334ad1f96..83c523a401 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d27985245a0e8c0d6b04323c98b26b6a8fb4e489fa8f5f3234252c7c198f23c8 +fb257540d32ced4a5fe0759d3ad9c1b36fd2b7d64d93cb0a0ecdb48355a68b7a From db79dd1f88e27fda30b27f52e6512897e6c98c72 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Aug 2024 16:20:43 +0000 Subject: [PATCH 25/99] Add tests to restore coverage of fts5_main.c. FossilOrigin-Name: 681edd08ab7024ddcd6447a64d8aad581b47b039481b697d91d26e6c6660f373 --- ext/fts5/fts5_tcl.c | 38 +++++++++++++++++------ ext/fts5/test/fts5faultI.test | 58 ++++++++++++++++++++++++++++++++++- ext/fts5/test/fts5locale.test | 9 ++++++ ext/fts5/test/fts5misc.test | 22 +++++++++++++ manifest | 18 +++++------ manifest.uuid | 2 +- 6 files changed, 127 insertions(+), 20 deletions(-) diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 60b3f0e9ef..1e9f7bbb60 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -890,15 +890,17 @@ static int f5tTokenizerCreate( static void f5tTokenizerDelete(Fts5Tokenizer *p){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; - if( pInst->pParent ){ - if( pInst->pModule->parent_v2.xDelete ){ - pInst->pModule->parent_v2.xDelete(pInst->pParent); - }else{ - pInst->pModule->parent.xDelete(pInst->pParent); + if( pInst ){ + if( pInst->pParent ){ + if( pInst->pModule->parent_v2.xDelete ){ + pInst->pModule->parent_v2.xDelete(pInst->pParent); + }else{ + pInst->pModule->parent.xDelete(pInst->pParent); + } } + Tcl_DecrRefCount(pInst->pScript); + ckfree((char *)pInst); } - Tcl_DecrRefCount(pInst->pScript); - ckfree((char *)pInst); } @@ -1146,6 +1148,7 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( F5tTokenizerModule *pMod; int rc = SQLITE_OK; int bV2 = 0; /* True to use _v2 API */ + int iVersion = 2; /* Value for _v2.iVersion */ const char *zParent = 0; /* Name of parent tokenizer, if any */ int ii = 0; @@ -1157,7 +1160,7 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( /* Parse any options. Set stack variables bV2 and zParent. */ for(ii=1; iixFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent); if( rc==SQLITE_OK ){ memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2)); + pMod->parent_v2.xDelete(0); } }else{ rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent); + if( rc==SQLITE_OK ){ + pMod->parent.xDelete(0); + } } } @@ -1218,7 +1238,7 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( }else{ fts5_tokenizer_v2 t2; memset(&t2, 0, sizeof(t2)); - t2.iVersion = 2; + t2.iVersion = iVersion; t2.xCreate = f5tTokenizerCreate; t2.xTokenize = f5tTokenizerTokenize_v2; t2.xDelete = f5tTokenizerDelete; diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test index 33c93c63af..ab641b6514 100644 --- a/ext/fts5/test/fts5faultI.test +++ b/ext/fts5/test/fts5faultI.test @@ -20,7 +20,9 @@ ifcapable !fts5 { return } -set ::testprefix fts5faultH +set ::testprefix fts5faultI + +if 0 { do_execsql_test 1.0 { PRAGMA encoding = utf16; @@ -146,5 +148,59 @@ do_faultsim_test 6 -faults oom* -prep { faultsim_test_result {0 {}} } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); + INSERT INTO w1 VALUES('three two one'); + DELETE FROM w1_content WHERE rowid=1; +} + +faultsim_save_and_close + +do_faultsim_test 7 -faults oom* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM w1 } +} -body { + execsql { + PRAGMA integrity_check; + } +} -test { +} +} +#------------------------------------------------------------------------- +reset_db +fts5_tclnum_register db +fts5_aux_test_functions db + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize = "tclnum query", detail=columns + ); + INSERT INTO ft VALUES('one two three i ii iii'); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +do_faultsim_test 8.1 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_test_collist (ft) FROM ft('one two'); + } +} -test { + faultsim_test_result {0 {{0.0 1.0}}} {1 {SQL logic error}} {1 SQLITE_NOMEM} +} + +do_faultsim_test 8.2 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM ft('one two') ORDER BY rank; + } +} -test { + faultsim_test_result {0 1} {1 {SQL logic error}} {1 SQLITE_NOMEM} +} + + finish_test diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index a07f4ccfa8..f647317c3b 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -521,5 +521,14 @@ do_catchsql_test 11.2 { SELECT fts5_columnlocale(x1, 1) FROM x1('two'); } {1 SQLITE_RANGE} +#------------------------------------------------------------------------- +# +reset_db +do_test 12.0 { + list [catch { + sqlite3_fts5_create_tokenizer -v2 -version 3 db tcl tcl_create + } msg] $msg +} {1 {error in fts5_api.xCreateTokenizer_v2()}} + finish_test diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index 4b943c3488..c7f4a53e38 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -591,5 +591,27 @@ do_execsql_test 21.2 { PRAGMA integrity_check } {ok} +breakpoint +sqlite3_db_config db DEFENSIVE 1 +do_execsql_test 21.3 { + CREATE TABLE xyz_notashadow(x, y); + DROP TABLE xyz_notashadow; +} +sqlite3_db_config db DEFENSIVE 0 + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 22.0 { + SELECT fts5(NULL); +} {{}} +do_execsql_test 22.1 { + SELECT count(*) FROM ( + SELECT fts5_source_id() + ) +} {1} +execsql_pp { + SELECT fts5_source_id() +} + finish_test diff --git a/manifest b/manifest index 28ab5391d0..5847c8b75f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\stests. -D 2024-08-15T20:33:05.956 +C Add\stests\sto\srestore\scoverage\sof\sfts5_main.c. +D 2024-08-16T16:20:43.666 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -102,7 +102,7 @@ F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe F ext/fts5/fts5_main.c 23029229021240dc21a69eb749305316103d183375413e2090c064b74b03b7f8 F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 -F ext/fts5/fts5_tcl.c 50c7e16753fde0c4d80d8abd00a4ed2b0e998d5d3899a484510d01923c5da43b +F ext/fts5/fts5_tcl.c 1dcf08028141c40a32634bdcf2d5601622ce4edc48f82ac4ce0cbe0a92a6961d F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c 96efa85a21a85276680ce3fb19dc5dd8d6b6541b2c37f953ee55bc15092262e1 @@ -177,7 +177,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d -F ext/fts5/test/fts5faultI.test d9e8c4b0b42910fa447a0e148236def618e7decbf3a334c7933a95e36410f772 +F ext/fts5/test/fts5faultI.test eb0910ec0fa57b4e06241c58252124155e689770b4dd2d8f4f3fc74185be934e F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -187,11 +187,11 @@ F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba104 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test f58ac6ab539d3813556646f7d4b1e19f7e3b1a56ca4439349eb8099d6a821ff6 +F ext/fts5/test/fts5locale.test 5ad9f1b66db4d698c67284f3495007be94f8c6ed183688e64defed72a169b6c0 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test b49a3afbda6de02970560c1cd3b804a4896b9f30ce5d75a5f215fa972537b26b +F ext/fts5/test/fts5misc.test c860b75721d162d7a4f2856322e55e814c2cf0bc45f93ef4388551b54e61a0a0 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74 @@ -2208,8 +2208,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d27985245a0e8c0d6b04323c98b26b6a8fb4e489fa8f5f3234252c7c198f23c8 -R c260c68d60eebc40b0c96509388ca9e3 +P fb257540d32ced4a5fe0759d3ad9c1b36fd2b7d64d93cb0a0ecdb48355a68b7a +R 067a05b128f4dcdbb34743e9c0788abc U dan -Z d7c15a75ca8b075cfcf78184db374c6a +Z 1b380833fd7dd34e177f4e045cee1c46 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 83c523a401..e93e9346cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb257540d32ced4a5fe0759d3ad9c1b36fd2b7d64d93cb0a0ecdb48355a68b7a +681edd08ab7024ddcd6447a64d8aad581b47b039481b697d91d26e6c6660f373 From f0f240c0d83903f4222d39131df4d9719272a4a0 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Aug 2024 17:54:00 +0000 Subject: [PATCH 26/99] Add tests to restore coverage of fts5_storage.c. FossilOrigin-Name: 38841881a08f4d75c2dfcbc1b2b45e311d7dda1287780f8ba1c8388f7adc1629 --- ext/fts5/fts5_storage.c | 46 +++++++++++++++++----------------- ext/fts5/test/fts5corrupt.test | 21 ++++++++++++++++ ext/fts5/test/fts5faultI.test | 28 ++++++++++++++++++--- ext/fts5/test/fts5locale.test | 42 +++++++++++++++++++++++++++++++ manifest | 18 ++++++------- manifest.uuid | 2 +- 6 files changed, 120 insertions(+), 37 deletions(-) diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 76f57499df..03af105a75 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -526,7 +526,7 @@ static int fts5StorageDeleteFromIndex( pText, nText, (void*)&ctx, fts5StorageInsertCallback ); p->aTotalSize[iCol-1] -= (i64)ctx.szCol; - if( p->aTotalSize[iCol-1]<0 && rc==SQLITE_OK ){ + if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){ rc = FTS5_CORRUPT; } if( bReset ) sqlite3Fts5ClearLocale(pConfig); @@ -618,12 +618,12 @@ static int fts5StorageInsertDocsize( rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin); sqlite3_bind_int64(pReplace, 3, iOrigin); } - if( rc==SQLITE_OK ){ - sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); - sqlite3_step(pReplace); - rc = sqlite3_reset(pReplace); - sqlite3_bind_null(pReplace, 2); - } + } + if( rc==SQLITE_OK ){ + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); } } return rc; @@ -887,28 +887,28 @@ int sqlite3Fts5StorageContentInsert( rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ sqlite3_value *pVal = apVal[i]; - if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ + if( sqlite3_value_nochange(pVal) ){ /* This is an UPDATE statement, and column (i-2) was not modified. ** Retrieve the value from Fts5Storage.pSavedRow instead. */ + assert( p->pSavedRow ); pVal = sqlite3_column_value(p->pSavedRow, i-1); }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){ assert( pConfig->bLocale ); - if( i>1 ){ - if( pConfig->abUnindexed[i-2] ){ - /* At attempt to insert an fts5_locale() value into an UNINDEXED - ** column. Strip the locale away and just bind the text. */ - const char *pText = 0; - int nText = 0; - rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText); - sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); - }else{ - const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); - assert( nBlob>4 ); - sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT); - } - continue; + assert( i>1 ); + if( pConfig->abUnindexed[i-2] ){ + /* At attempt to insert an fts5_locale() value into an UNINDEXED + ** column. Strip the locale away and just bind the text. */ + const char *pText = 0; + int nText = 0; + rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText); + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + }else{ + const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + assert( nBlob>4 ); + sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT); } + continue; } rc = sqlite3_bind_value(pInsert, i, pVal); diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test index ae07383b23..0abd8b86de 100644 --- a/ext/fts5/test/fts5corrupt.test +++ b/ext/fts5/test/fts5corrupt.test @@ -101,4 +101,25 @@ do_catchsql_test 3.1 { SELECT * FROM t3 WHERE t3 MATCH 'o'; } {1 {fts5: missing row 3 from content table 'main'.'t3_content'}} +#-------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2 VALUES('one two three'); + INSERT INTO t2 VALUES('four five six'); + INSERT INTO t2 VALUES('seven eight nine'); + INSERT INTO t2 VALUES('ten eleven twelve'); +} +do_execsql_test 4.1 { + SELECT hex(block) FROM t2_data WHERE id=1; +} {040C} +do_execsql_test 4.2 { + UPDATE t2_data SET block = X'0402' WHERE id=1 +} +breakpoint +do_catchsql_test 4.3 { + DELETE FROM t2 WHERE rowid=3 +} {1 {database disk image is malformed}} + finish_test diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test index ab641b6514..d35cdc3800 100644 --- a/ext/fts5/test/fts5faultI.test +++ b/ext/fts5/test/fts5faultI.test @@ -22,8 +22,6 @@ ifcapable !fts5 { set ::testprefix fts5faultI -if 0 { - do_execsql_test 1.0 { PRAGMA encoding = utf16; CREATE VIRTUAL TABLE t1 USING fts5(x, locale=1); @@ -168,13 +166,13 @@ do_faultsim_test 7 -faults oom* -prep { } } -test { } -} + #------------------------------------------------------------------------- reset_db fts5_tclnum_register db fts5_aux_test_functions db -do_execsql_test 2.0 { +do_execsql_test 8.0 { CREATE VIRTUAL TABLE ft USING fts5( x, tokenize = "tclnum query", detail=columns ); @@ -201,6 +199,28 @@ do_faultsim_test 8.2 -faults oom* -prep { faultsim_test_result {0 1} {1 {SQL logic error}} {1 SQLITE_NOMEM} } +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); + INSERT INTO ft VALUES('one two three i ii iii'); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +faultsim_save_and_close + +do_faultsim_test 9 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + UPDATE ft SET rowid=4 WHERE rowid=1 + } +} -test { + faultsim_test_result {0 {}} +} + finish_test diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index f647317c3b..d64df18491 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -145,6 +145,13 @@ do_execsql_test 2.17 { INSERT INTO b1(b1) VALUES('integrity-check'); } +do_execsql_test 2.18 { + INSERT INTO b1(rowid, x, y) VALUES( + test_setsubtype(45, 76), 'abc def', 'def abc' + ); + INSERT INTO b1(b1) VALUES('integrity-check'); +} + #------------------------------------------------------------------------- # Test the 'delete' command with contentless tables. # @@ -459,6 +466,10 @@ foreach_detail_mode $::testprefix { CREATE VIRTUAL TABLE ft USING fts5(x, content=x1, content_rowid=ii, locale=1, detail=%DETAIL%, columnsize=0 ); + + CREATE VIRTUAL TABLE ft2 USING fts5( + x, locale=1, detail=%DETAIL%, columnsize=0 + ); } foreach {tn v} { @@ -500,6 +511,37 @@ foreach_detail_mode $::testprefix { do_catchsql_test 10.2.$tn.8 { SELECT * FROM ft('hello') } {1 {SQL logic error}} + + do_catchsql_test 10.2.$tn.9 { + PRAGMA integrity_check; + } {0 ok} + + do_execsql_test 10.2.$tn.10 { + DELETE FROM x1; + INSERT INTO x1(ii, x) VALUES(1, 'hello world'); + } + + do_catchsql_test 10.2.$tn.11 " + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) ) + " {1 {SQL logic error}} + + do_catchsql_test 10.2.$tn.12 " + INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) ) + " {1 {SQL logic error}} + + do_execsql_test 10.2.$tn.13 { + INSERT INTO ft2(rowid, x) VALUES(1, 'hello world'); + } + do_execsql_test 10.2.$tn.14 "UPDATE ft2_content SET c0=$v" + + do_catchsql_test 10.2.$tn.15 { + PRAGMA integrity_check; + } {1 {SQL logic error}} + + do_execsql_test 10.2.$tn.16 { + DELETE FROM ft2_content; + INSERT INTO ft2(ft2) VALUES('rebuild'); + } } } diff --git a/manifest b/manifest index 5847c8b75f..1c1eedf452 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\srestore\scoverage\sof\sfts5_main.c. -D 2024-08-16T16:20:43.666 +C Add\stests\sto\srestore\scoverage\sof\sfts5_storage.c. +D 2024-08-16T17:54:00.409 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -101,7 +101,7 @@ F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe F ext/fts5/fts5_main.c 23029229021240dc21a69eb749305316103d183375413e2090c064b74b03b7f8 -F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992 +F ext/fts5/fts5_storage.c acece8abf1b4b2012bdad93aa2d2cf6fc0d1da919763479c7055f699c2cde96c F ext/fts5/fts5_tcl.c 1dcf08028141c40a32634bdcf2d5601622ce4edc48f82ac4ce0cbe0a92a6961d F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -146,7 +146,7 @@ F ext/fts5/test/fts5contentless2.test 70ffe6c611d8f278240da56734df8a77948f04e273 F ext/fts5/test/fts5contentless3.test 75eaae5ad6b284ee447788943974d323228f27cc35a1681da997135cff95bc6a F ext/fts5/test/fts5contentless4.test ec34dc69ef474ca9997dae6d91e072906e0e9a5a4b05ea89964c863833b6eff8 F ext/fts5/test/fts5contentless5.test 40cdcb4fe751672450829c5a96bd32c25fc2f6076279dd2ce5c58ac9a390132a -F ext/fts5/test/fts5corrupt.test a9bda1ded5112ebf1ee85c5381bd1fe8974952e2523cede4d5072804d2011503 +F ext/fts5/test/fts5corrupt.test 6485f721b88ba355ca5d701e7ee87a4efa3ea578d8e6adb26f51ef956c8328bd F ext/fts5/test/fts5corrupt2.test 335911e3f68b9625d850325f9e29a128db3f4276a8c9d4e32134580da8f924c4 F ext/fts5/test/fts5corrupt3.test b5f35d72af85b1d5a092b3d5e437f7944d142dd0b0c87b928fd0436a0aec6987 F ext/fts5/test/fts5corrupt4.test dc08d19f5b8943e95a7778a7d8da592042504faf18dd93f68f7d7a0d7d7dd733 @@ -177,7 +177,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d -F ext/fts5/test/fts5faultI.test eb0910ec0fa57b4e06241c58252124155e689770b4dd2d8f4f3fc74185be934e +F ext/fts5/test/fts5faultI.test e57fa07b771565fcb7aa2b2c349a5104812b4edd86d09ca531afd9b247cf36dd F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -187,7 +187,7 @@ F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba104 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 5ad9f1b66db4d698c67284f3495007be94f8c6ed183688e64defed72a169b6c0 +F ext/fts5/test/fts5locale.test 79cbd3000ae269de50826f6061c81f7c9fdb21dd9954c0b7f9485263482c539d F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -2208,8 +2208,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb257540d32ced4a5fe0759d3ad9c1b36fd2b7d64d93cb0a0ecdb48355a68b7a -R 067a05b128f4dcdbb34743e9c0788abc +P 681edd08ab7024ddcd6447a64d8aad581b47b039481b697d91d26e6c6660f373 +R 39ee8d76619b01513c02abf781a1c1d5 U dan -Z 1b380833fd7dd34e177f4e045cee1c46 +Z 17e6bca1c1b4a8da8fc2bb9198c32b95 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e93e9346cb..9ad6542d4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -681edd08ab7024ddcd6447a64d8aad581b47b039481b697d91d26e6c6660f373 +38841881a08f4d75c2dfcbc1b2b45e311d7dda1287780f8ba1c8388f7adc1629 From 342984075b04c9edc5f719091bd187c56a07089e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Aug 2024 19:05:47 +0000 Subject: [PATCH 27/99] Add tests to restore coverage of fts5_config.c. FossilOrigin-Name: 9d971b31df7ad4d68eb348f95d8f996071cf87d41c47033bde3fcc4dba732e06 --- ext/fts5/fts5_config.c | 11 ++++------- ext/fts5/test/fts5contentless.test | 20 ++++++++++++++++++++ ext/fts5/test/fts5origintext.test | 11 +++++++++-- ext/fts5/test/fts5secure8.test | 4 ++++ manifest | 18 +++++++++--------- manifest.uuid | 2 +- 6 files changed, 47 insertions(+), 19 deletions(-) diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index b11193ec23..3cb1bd3bea 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -1031,13 +1031,10 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE ){ rc = SQLITE_ERROR; - if( pConfig->pzErrmsg ){ - assert( 0==*pConfig->pzErrmsg ); - sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format " - "(found %d, expected %d or %d) - run 'rebuild'", - iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE - ); - } + sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE + ); }else{ pConfig->iVersion = iVersion; } diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test index eb6b928ab8..991e9888fc 100644 --- a/ext/fts5/test/fts5contentless.test +++ b/ext/fts5/test/fts5contentless.test @@ -267,4 +267,24 @@ do_execsql_test 8.2 { SELECT rowid FROM ft('four'); } {} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=0); + INSERT INTO ft VALUES('hello world'); + INSERT INTO ft VALUES('one two three'); +} + +do_catchsql_test 9.1 { + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world'); +} {0 {}} + +do_catchsql_test 9.2 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=2); +} {1 {malformed contentless_delete=... directive}} + +do_catchsql_test 9.3 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=11); +} {1 {malformed contentless_delete=... directive}} + finish_test diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 1612b0d7f4..8e975fa17c 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -261,8 +261,8 @@ do_execsql_test 5.3 { reset_db sqlite3_fts5_register_origintext db do_execsql_test 6.0 { - CREATE VIRTUAL TABLE ft USING fts5( - x, y, tokenize='origintext unicode61', detail=%DETAIL% + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0 ); INSERT INTO ft VALUES('One Two', 'Three two'); @@ -300,6 +300,13 @@ do_catchsql_test 6.5 { SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one'); } {1 SQLITE_RANGE} +do_catchsql_test 6.6 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2); +} {1 {malformed tokendata=... directive}} +do_catchsql_test 6.7 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11); +} {1 {malformed tokendata=... directive}} + } finish_test diff --git a/ext/fts5/test/fts5secure8.test b/ext/fts5/test/fts5secure8.test index 0216bb6ea1..8b65b7c59f 100644 --- a/ext/fts5/test/fts5secure8.test +++ b/ext/fts5/test/fts5secure8.test @@ -58,6 +58,10 @@ do_execsql_test 2.1 { pragma quick_check; } {ok} +do_catchsql_test 2.2 { + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 'hello world'); +} {1 {SQL logic error}} + diff --git a/manifest b/manifest index 1c1eedf452..8e78f14e63 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\srestore\scoverage\sof\sfts5_storage.c. -D 2024-08-16T17:54:00.409 +C Add\stests\sto\srestore\scoverage\sof\sfts5_config.c. +D 2024-08-16T19:05:47.160 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5.h 4c6998c6186268b4dbe9baef2c0d2ab974bd90996d61d4dbe801367249be6d F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d805655360 F ext/fts5/fts5_aux.c 12cd2512f869217c38b70c31de5b5f741812734fafa80f55b32ea9bbd96e2152 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 -F ext/fts5/fts5_config.c 187f7ffa5eddd6539ffa592de85e95b18be951728491390121bb215549a24a2a +F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81cbefc109ea F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a35da6f6 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe @@ -141,7 +141,7 @@ F ext/fts5/test/fts5config.test 017daf10d2642496e97402baa0134de8b5b46b9c37e53c22 F ext/fts5/test/fts5conflict.test bf6030a77dbb1bedfcc42e589ed7980846c995765d77460551e448b56d741244 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4 F ext/fts5/test/fts5content.test d5c0c2142e64cb305f0968de70c01f8e59dbc3ecc56520c22e739e5dd99ea3bb -F ext/fts5/test/fts5contentless.test b107465f8cd27dde6313b9c60b61d7158a7753b9c663c5c553695f826bb3c0a5 +F ext/fts5/test/fts5contentless.test 606f063b29ba0f46d4b79aa36cdd1ef4dab5de53eae8c881d731af75a4894aca F ext/fts5/test/fts5contentless2.test 70ffe6c611d8f278240da56734df8a77948f04e2739b358439e9bdcf56ced35f F ext/fts5/test/fts5contentless3.test 75eaae5ad6b284ee447788943974d323228f27cc35a1681da997135cff95bc6a F ext/fts5/test/fts5contentless4.test ec34dc69ef474ca9997dae6d91e072906e0e9a5a4b05ea89964c863833b6eff8 @@ -199,7 +199,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 264b9101721c17d06d1d174feb743fda3ddc89fad41dee980fef821428258e47 F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9be0cfe22064fc71a44ed F ext/fts5/test/fts5optimize3.test 1653029284e10e0715246819893ba30565c4ead0d0fc470adae92c353ea857d3 -F ext/fts5/test/fts5origintext.test 002abba00b334cb4b3f957f940f75b9ab53f4caae97aa753e19da285c245e463 +F ext/fts5/test/fts5origintext.test 2015f69bc8abd111152a8e66211fd2d45026378001e07c054159aa4f84e6691d F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b37f29cad3a8b8d5ac6f9ba F ext/fts5/test/fts5origintext3.test 45c33cf0c91a9ca0e36d298462db3edc7c8fe45fd185649a9dbfd66bb670058b F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8 @@ -223,7 +223,7 @@ F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b98 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf6be82c88a0ac857fd F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a -F ext/fts5/test/fts5secure8.test e68c0ac4447f415ff3e4e82531e99548289286f9f3a29c8cd53036113fe28602 +F ext/fts5/test/fts5secure8.test 808ade9d172ed07b24b85c57dd53b6d2b1aba018b4e634d267ce572221de80e0 F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5 F ext/fts5/test/fts5simple.test ed7c3815c9fa1c16166258cb98edb2e014c63c7589958d76c5487df0df913d61 F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc @@ -2208,8 +2208,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 681edd08ab7024ddcd6447a64d8aad581b47b039481b697d91d26e6c6660f373 -R 39ee8d76619b01513c02abf781a1c1d5 +P 38841881a08f4d75c2dfcbc1b2b45e311d7dda1287780f8ba1c8388f7adc1629 +R d19b3ba3504e0807fa9fc6488171b084 U dan -Z 17e6bca1c1b4a8da8fc2bb9198c32b95 +Z 92294a85664ab83f8a52ccaa818e0f8f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9ad6542d4f..cd18c0e49e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38841881a08f4d75c2dfcbc1b2b45e311d7dda1287780f8ba1c8388f7adc1629 +9d971b31df7ad4d68eb348f95d8f996071cf87d41c47033bde3fcc4dba732e06 From 52d03102562a4a141357cc3a48f4b903df3aa3bb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Aug 2024 14:59:26 +0000 Subject: [PATCH 28/99] Fix a typo in an "ifdef" that broke SQLITE_OMIT_ANALYZE builds. FossilOrigin-Name: c09da26cb6f772a18c595ff0eb55d8345a377c3ef5ed02a5e539ffdbe9d07f85 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index dd7e947eb5..704cb20843 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\ssubquery\shas\san\sORDER\sBY\sclause\sand\sthat\sordering\sis\shelpfile\sin\nsatisfying\sthe\sORDER\sBY\sor\sGROUP\sBY\sof\sthe\souter\squery\swithout\sdoing\san\nextra\ssort,\sthen\somit\sor\sreduce\sthe\ssort\sin\sthe\souter\squery.\s\sCall\nthis\sthe\s"order-by-subquery\soptimization". -D 2024-08-16T18:51:46.975 +C Fix\sa\stypo\sin\san\s"ifdef"\sthat\sbroke\sSQLITE_OMIT_ANALYZE\sbuilds. +D 2024-08-17T14:59:26.789 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -831,7 +831,7 @@ F src/upsert.c 2e60567a0e9e8520c18671b30712a88dc73534474304af94f32bb5f3ef65ac65 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c 5d1a0134cf4240648d1c6bb5cc8efaca0ea2b5d5c840985aec7e947271f04375 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c de13de572eccb688b2b7cf50e2f9005c44bf9ae89e35245ef8eadfc60dfd2764 +F src/vdbe.c be5f58bc29f60252e041a618eae59e8d57d460ba136c5403cf0abf955560c457 F src/vdbe.h c2549a215898a390de6669cfa32adba56f0d7e17ba5a7f7b14506d6fd5f0c36a F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df @@ -2205,9 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a1915cbdeccac2ed49fb8a9733abf2c6239396e3174fed2ff275f66b606c48b6 41a41c173a9d15d94f23d73a5c04bfb1616cb9223bc81d41808f9b4d00817fbf -R a24a84e9d7ceff04c159b78242edac9d -T +closed 41a41c173a9d15d94f23d73a5c04bfb1616cb9223bc81d41808f9b4d00817fbf -U drh -Z 169fa841b16225f819f8996d6765647e +P 7a0cdc7edb704a88a77b748cd28f6e00c49849cc2c1af838b95b34232ecc21f9 +R 501e32d0617414047f040d4ae2e8bb39 +U dan +Z 4ff808b9d2897218daeb3b1363a873d1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d32d2498e4..a4764edcfc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a0cdc7edb704a88a77b748cd28f6e00c49849cc2c1af838b95b34232ecc21f9 +c09da26cb6f772a18c595ff0eb55d8345a377c3ef5ed02a5e539ffdbe9d07f85 diff --git a/src/vdbe.c b/src/vdbe.c index d097bfd8b9..ea69c759c2 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2096,7 +2096,7 @@ case OP_RealAffinity: { /* in1 */ } #endif -#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_ANALYZE) +#if !defined(SQLITE_OMIT_CAST) || !defined(SQLITE_OMIT_ANALYZE) /* Opcode: Cast P1 P2 * * * ** Synopsis: affinity(r[P1]) ** From f87a54ab41de3d7a276059c19f47aa1f9c93ecff Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 17 Aug 2024 15:52:21 +0000 Subject: [PATCH 29/99] Adjust the "configure" script so that it just disables the ability to run tests rather than erroring out if tclsh is unable to recommend a suitable tclConfig.sh. FossilOrigin-Name: 793ff83d09bea165c8cedc978afd3fcfb1a4cee894c5914aa1d92416afce963d --- configure | 8 +++++++- configure.ac | 7 ++++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/configure b/configure index 50d9e2b3d0..5f60385c28 100755 --- a/configure +++ b/configure @@ -10330,6 +10330,7 @@ else use_tcl=yes fi +original_use_tcl=${use_tcl} if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then for ac_prog in tclsh8.6 tclsh tclsh9.0 do @@ -10388,6 +10389,7 @@ $as_echo "$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&5 $as_echo "$as_me: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&2;} + use_tcl=no fi fi fi @@ -10438,10 +10440,14 @@ $as_echo "loading TCL configuration from ${tclconfig}" >&6; } # tclConfig.sh file and that could be included here. But as of right now, # TCL_LIB_SPEC is the only what that the Makefile uses. HAVE_TCL=1 -else +elif test x"${original_use_tcl}" = "xno"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because of --disable-tcl" >&5 $as_echo "unable to run tests because of --disable-tcl" >&6; } HAVE_TCL=0 +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because no tclConfig.sh file could be located" >&5 +$as_echo "unable to run tests because no tclConfig.sh file could be located" >&6; } + HAVE_TCL=0 fi if test x"$TCLSH_CMD" == x; then diff --git a/configure.ac b/configure.ac index 4f95a5c08c..2936c4ab5a 100644 --- a/configure.ac +++ b/configure.ac @@ -123,6 +123,7 @@ AC_ARG_WITH(tclsh, AS_HELP_STRING([--with-tclsh=PATHNAME],[full pathname of a tc AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing (tclConfig.sh)])) AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[omit building accessory programs that require TCL-dev]), [use_tcl=$enableval],[use_tcl=yes]) +original_use_tcl=${use_tcl} if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh tclsh9.0],none) with_tclsh=${TCLSH_CMD} @@ -136,6 +137,7 @@ if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then AC_MSG_RESULT([$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}]) else AC_MSG_WARN([$TCLSH_CMD is unable to recommend a tclConfig.sh]) + use_tcl=no fi fi fi @@ -185,9 +187,12 @@ if test x"${use_tcl}" = "xyes"; then # tclConfig.sh file and that could be included here. But as of right now, # TCL_LIB_SPEC is the only what that the Makefile uses. HAVE_TCL=1 -else +elif test x"${original_use_tcl}" = "xno"; then AC_MSG_RESULT([unable to run tests because of --disable-tcl]) HAVE_TCL=0 +else + AC_MSG_RESULT([unable to run tests because no tclConfig.sh file could be located]) + HAVE_TCL=0 fi AC_SUBST(HAVE_TCL) if test x"$TCLSH_CMD" == x; then diff --git a/manifest b/manifest index 704cb20843..8b5e68507a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\san\s"ifdef"\sthat\sbroke\sSQLITE_OMIT_ANALYZE\sbuilds. -D 2024-08-17T14:59:26.789 +C Adjust\sthe\s"configure"\sscript\sso\sthat\sit\sjust\sdisables\sthe\sability\sto\nrun\stests\srather\sthan\serroring\sout\sif\stclsh\sis\sunable\sto\srecommend\sa\nsuitable\stclConfig.sh. +D 2024-08-17T15:52:21.666 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -35,8 +35,8 @@ F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea0034 F autoconf/tea/win/rules.vc 7b3bb2ef32ade0f3f14d951231811678722725e3bca240dd9727ae0dfe10f6a5 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure 2e5058c2ba2c6b50a01cf50c158180c4ec969b2068c2e361dfbfc875ec3f9d1c x -F configure.ac b7e26a699a2ffc5eb7a25636f32755c27bbb665777b68ef35a90c1ee9bd9d469 +F configure 956e132a3e2481e5016e9ecaa0a2bdd9df95c1d923fa4fb2aa2cf9f0b3a54ad9 x +F configure.ac fd9ff064b142682964971fec3d867235e51e1f6de30a031e33f683b7699338ff F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd F doc/compile-for-windows.md e8635eea9153dcd6a51fd2740666ebc4492b3813cb1ac31cd8e99150df91762d @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7a0cdc7edb704a88a77b748cd28f6e00c49849cc2c1af838b95b34232ecc21f9 -R 501e32d0617414047f040d4ae2e8bb39 -U dan -Z 4ff808b9d2897218daeb3b1363a873d1 +P c09da26cb6f772a18c595ff0eb55d8345a377c3ef5ed02a5e539ffdbe9d07f85 +R 8082c14d3791ff31a3c70ef600eeef0f +U drh +Z 2bdc019c3dd756a5d860815c6dd9a72b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a4764edcfc..1e3ae45cbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c09da26cb6f772a18c595ff0eb55d8345a377c3ef5ed02a5e539ffdbe9d07f85 +793ff83d09bea165c8cedc978afd3fcfb1a4cee894c5914aa1d92416afce963d From 21363ac78df6751655c33372a7277512531b9570 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 17 Aug 2024 16:17:16 +0000 Subject: [PATCH 30/99] Adjust Makefile.in so that it outputs a slightly better error message on an attempt to build something that requires TCL libraries when HAVE_TCL is false. FossilOrigin-Name: a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3 --- Makefile.in | 14 +++++++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Makefile.in b/Makefile.in index 7ff6e901e4..44665184fc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -796,6 +796,10 @@ has_tclsh85: sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD) touch has_tclsh85 +has_tclconfig: + @ if test x"$(HAVE_TCL)" != "x1"; then echo 'ERROR: Requires access to "tclConfig.sh" which "configure" was not able to locate'; exit 1; fi + touch has_tclconfig + # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to @@ -1118,7 +1122,7 @@ tclsqlite-shell.lo: $(TOP)/src/tclsqlite.c $(HDR) tclsqlite-stubs.lo: $(TOP)/src/tclsqlite.c $(HDR) $(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c -tclsqlite3$(TEXE): tclsqlite-shell.lo libsqlite3.la +tclsqlite3$(TEXE): has_tclconfig tclsqlite-shell.lo libsqlite3.la $(LTLINK) -o $@ tclsqlite-shell.lo \ libsqlite3.la $(LIBTCL) @@ -1305,7 +1309,7 @@ TESTFIXTURE_SRC1 = sqlite3.c TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) -testfixture$(TEXE): has_tclsh85 $(TESTFIXTURE_SRC) +testfixture$(TEXE): has_tclconfig has_tclsh85 $(TESTFIXTURE_SRC) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \ -o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS) @@ -1404,13 +1408,13 @@ shelltest: $(TESTPROGS) sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85 $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c -sqlite3_analyzer$(TEXE): sqlite3_analyzer.c +sqlite3_analyzer$(TEXE): has_tclconfig sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in has_tclsh85 $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c -sqltclsh$(TEXE): sqltclsh.c +sqltclsh$(TEXE): has_tclconfig sqltclsh.c $(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS) sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c @@ -1429,7 +1433,7 @@ CHECKER_DEPS =\ sqlite3_checker.c: $(CHECKER_DEPS) has_tclsh85 $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ -sqlite3_checker$(TEXE): sqlite3_checker.c +sqlite3_checker$(TEXE): has_tclconfig sqlite3_checker.c $(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS) dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo diff --git a/manifest b/manifest index 8b5e68507a..bd4f4cbe1c 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Adjust\sthe\s"configure"\sscript\sso\sthat\sit\sjust\sdisables\sthe\sability\sto\nrun\stests\srather\sthan\serroring\sout\sif\stclsh\sis\sunable\sto\srecommend\sa\nsuitable\stclConfig.sh. -D 2024-08-17T15:52:21.666 +C Adjust\sMakefile.in\sso\sthat\sit\soutputs\sa\sslightly\sbetter\serror\smessage\son\san\nattempt\sto\sbuild\ssomething\sthat\srequires\sTCL\slibraries\swhen\sHAVE_TCL\sis\sfalse. +D 2024-08-17T16:17:16.045 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in cf96aa7d2682650dec56dc2f03cfe463feb101a1be594e4929bcd38662bc2ea8 +F Makefile.in 209e9c64edf499b0ea7a15448699101c01d15828af35a3439d32fc25fdf1b787 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc 6c3fe8b6ce60e73f34a148c957d78b4648745c8d30e792423aa1a8d8bf12d065 F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3 @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c09da26cb6f772a18c595ff0eb55d8345a377c3ef5ed02a5e539ffdbe9d07f85 -R 8082c14d3791ff31a3c70ef600eeef0f +P 793ff83d09bea165c8cedc978afd3fcfb1a4cee894c5914aa1d92416afce963d +R be5ece8b231f01bec02cbe9ef4eda884 U drh -Z 2bdc019c3dd756a5d860815c6dd9a72b +Z 6d429e685f531219ca0d0afba9760eb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1e3ae45cbc..a5034c9d32 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -793ff83d09bea165c8cedc978afd3fcfb1a4cee894c5914aa1d92416afce963d +a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3 From b651084713e8bb9a6a7a0399a00677604d157b2f Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Aug 2024 17:22:49 +0000 Subject: [PATCH 31/99] Add tests to restore coverage of fts5_tokenizer.c. FossilOrigin-Name: 8f9257361b05e368bf433e56d0698923b0f97d12e7c0ad7760aaab6746c0e467 --- ext/fts3/unicode/mkunicode.tcl | 3 ++ ext/fts5/fts5_tokenize.c | 68 +++++++++++++++++---------------- ext/fts5/fts5_unicode2.c | 3 ++ ext/fts5/test/fts5cat.test | 17 +++++++++ ext/fts5/test/fts5trigram.test | 56 +++++++++++++++++++++++++++ ext/fts5/test/fts5unicode2.test | 20 ++++++++++ manifest | 22 +++++------ manifest.uuid | 2 +- 8 files changed, 146 insertions(+), 45 deletions(-) diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 58d90c68c7..1306629da8 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -628,6 +628,9 @@ proc print_categories {lMap} { $caseP $caseS $caseZ + + default: + return 1; } return 0; } diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index 7a36b750a1..f92529b840 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -79,7 +79,7 @@ static int fts5AsciiCreate( int i; memset(p, 0, sizeof(AsciiTokenizer)); memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); - for(i=0; rc==SQLITE_OK && ibFold = 1; - pNew->iFoldParam = 0; - for(i=0; rc==SQLITE_OK && ibFold = 1; + pNew->iFoldParam = 0; + + for(i=0; rc==SQLITE_OK && ibFold = (zArg[0]=='0'); + } + }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ + if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){ + rc = SQLITE_ERROR; + }else{ + pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0; + } }else{ - pNew->bFold = (zArg[0]=='0'); - } - }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ - if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){ rc = SQLITE_ERROR; - }else{ - pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0; } - }else{ + } + + if( pNew->iFoldParam!=0 && pNew->bFold==0 ){ rc = SQLITE_ERROR; } - } - if( iiFoldParam!=0 && pNew->bFold==0 ){ - rc = SQLITE_ERROR; - } - - if( rc!=SQLITE_OK ){ - fts5TriDelete((Fts5Tokenizer*)pNew); - pNew = 0; + + if( rc!=SQLITE_OK ){ + fts5TriDelete((Fts5Tokenizer*)pNew); + pNew = 0; + } } } *ppOut = (Fts5Tokenizer*)pNew; diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c index 3e97264fa8..cc164a4569 100644 --- a/ext/fts5/fts5_unicode2.c +++ b/ext/fts5/fts5_unicode2.c @@ -364,6 +364,9 @@ int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ default: return 1; } break; + + default: + return 1; } return 0; } diff --git a/ext/fts5/test/fts5cat.test b/ext/fts5/test/fts5cat.test index 483f64bfef..71e2abe3ae 100644 --- a/ext/fts5/test/fts5cat.test +++ b/ext/fts5/test/fts5cat.test @@ -55,5 +55,22 @@ do_execsql_test 1.5 { SELECT * FROM t4t } {สนามกีฬา 1 1} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 " + CREATE VIRTUAL TABLE x1 USING fts5(c, + tokenize=\"unicode61 categories ' \t'\"); +" + +do_catchsql_test 2.1 " + CREATE VIRTUAL TABLE x2 USING fts5(c, + tokenize=\"unicode61 categories 'N*\t\tMYZ'\"); +" {1 {error in tokenizer constructor}} + +do_catchsql_test 2.2 " + CREATE VIRTUAL TABLE x2 USING fts5(c, + tokenize=\"unicode61 categories 'N*\t\tXYZ'\"); +" {1 {error in tokenizer constructor}} + finish_test diff --git a/ext/fts5/test/fts5trigram.test b/ext/fts5/test/fts5trigram.test index b76dbda063..b06d35d6d1 100644 --- a/ext/fts5/test/fts5trigram.test +++ b/ext/fts5/test/fts5trigram.test @@ -286,5 +286,61 @@ do_execsql_test 9.3 { SELECT rowid FROM t1 WHERE a12 LIKE NULL } {} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=trigram); +} + +do_test 10.1 { + foreach {val} { + "abc \UFFjkl\UFF" + "abc \UFFFjkl\UFFF" + "abc \UFFFFjkl\UFFFF" + "abc \UFFFFFjkl\UFFFFF" + "\UFFjkl\UFF abc" + "\UFFFjkl\UFFF abc" + "\UFFFFjkl\UFFFF abc" + "\UFFFFFjkl\UFFFFF abc" + "\U10001jkl\U10001 abc" + } { + execsql { INSERT INTO t1 VALUES( $val ) } + } +} {} + +do_test 10.2 { + foreach {val} { + X'E18000626320646566' + X'61EDA0806320646566' + X'61EDA0806320646566' + X'61EFBFBE6320646566' + X'76686920E18000626320646566' + X'7668692061EDA0806320646566' + X'7668692061EDA0806320646566' + X'7668692061EFBFBE6320646566' + } { + execsql " INSERT INTO t1 VALUES( $val ) " + } +} {} + +do_test 10.3 { + set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}] + set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}] + set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] + set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] + execsql { + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + + INSERT INTO t1 VALUES('abcd' || $a); + INSERT INTO t1 VALUES('abcd' || $b); + INSERT INTO t1 VALUES('abcd' || $c); + INSERT INTO t1 VALUES('abcd' || $d); + } +} {} + + finish_test diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test index 3fc1f673a4..7a49a1d83f 100644 --- a/ext/fts5/test/fts5unicode2.test +++ b/ext/fts5/test/fts5unicode2.test @@ -470,4 +470,24 @@ do_execsql_test 8.2.3 { SELECT rowid FROM t4 WHERE t4 MATCH 'a' ORDER BY rowid ASC; } {2 4} +#------------------------------------------------------------------------- + +foreach {tn val bErr} { + 1 0 0 + 2 1 0 + 3 2 0 + 4 3 1 + 5 11 1 +} { + reset_db + set aRes(0) {0 {}} + set aRes(1) {1 {error in tokenizer constructor}} + set res $aRes($bErr) + do_catchsql_test 9.1.$tn " + CREATE VIRTUAL TABLE bl USING fts5( + s, tokenize='trigram remove_diacritics $val' + ); + " $res +} + finish_test diff --git a/manifest b/manifest index 8e78f14e63..4a616cffcb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\srestore\scoverage\sof\sfts5_config.c. -D 2024-08-16T19:05:47.160 +C Add\stests\sto\srestore\scoverage\sof\sfts5_tokenizer.c. +D 2024-08-17T17:22:49.665 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -89,7 +89,7 @@ F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87 F ext/fts3/tool/fts3view.c 413c346399159df81f86c4928b7c4a455caab73bfbc8cd68f950f632e5751674 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 +F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7ede551eac3ead62baa F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e F ext/fts5/fts5.h 4c6998c6186268b4dbe9baef2c0d2ab974bd90996d61d4dbe801367249be6de4 @@ -105,8 +105,8 @@ F ext/fts5/fts5_storage.c acece8abf1b4b2012bdad93aa2d2cf6fc0d1da919763479c7055f6 F ext/fts5/fts5_tcl.c 1dcf08028141c40a32634bdcf2d5601622ce4edc48f82ac4ce0cbe0a92a6961d F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b -F ext/fts5/fts5_tokenize.c 96efa85a21a85276680ce3fb19dc5dd8d6b6541b2c37f953ee55bc15092262e1 -F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00 +F ext/fts5/fts5_tokenize.c ae9c4fa93174ef06ffc138bd4280a1c37f7e13624d3d2706aad4b80573f23c41 +F ext/fts5/fts5_unicode2.c 6f9b0fb79a8facaed76628ffd4eb9c16d7f2b84b52872784f617cf3422a9b043 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80 F ext/fts5/fts5_vocab.c e4830b00809e5da53bc10f93adc59e321407b0f801c7f4167c0e47f5552267e0 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 @@ -133,7 +133,7 @@ F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5 F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36 F ext/fts5/test/fts5blob.test caa33369e93e99ff494cd1103506ae34c5afbc0bcc369ed5e58e135144e33689 -F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c2be25568cc33a +F ext/fts5/test/fts5cat.test bf67dd335f964482ee658287521b81e2b88697b45eb7f73933e15f198ed447cb F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62 F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897 F ext/fts5/test/fts5columnsize.test 0af91d63985afdf663455d4b572b935238380140d74079eac362760866d3297b @@ -235,12 +235,12 @@ F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98 F ext/fts5/test/fts5tokenizer2.test ddb8b10fbe4b84b2a75812671f127774c1d2e3e2bf82d2e0e4f0bb1cd8a2b2d6 F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3 -F ext/fts5/test/fts5trigram.test f99fc0235e967826e556de9aee959d3cb8eae2e9abff37b5fe2732fd7052dac2 +F ext/fts5/test/fts5trigram.test fcb6b93510b917add63aee8f363c466ac406528822951e76e0980ce6f5170978 F ext/fts5/test/fts5trigram2.test 6fde9de7f63a6b4aa18dc731be56dbd6be4e755c9b13dcd55479e200d1df0e61 F ext/fts5/test/fts5ubsan.test 9a2dcf399dc8d0e0de661f0d93884d1d27e5b7f0693cfceb97dd24d818df5dd2 F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602 F ext/fts5/test/fts5unicode.test 41898f7e476e6515cd4b737c02a442cda5a580a74509788aa9072a2074948e0e -F ext/fts5/test/fts5unicode2.test a5c38179b311a188b24376772309389b073c996f52b79bb9ca760a19e62043ea +F ext/fts5/test/fts5unicode2.test 3bbd30152f9f760bf13886e5b1e5ec23ff62f56758ddda5d9c775a6082fb4c7c F ext/fts5/test/fts5unicode3.test f4891a3dac3b49c3d7c0fdb29566e9eb0ecff35263370c89f9661b1952b20818 F ext/fts5/test/fts5unicode4.test 728c8f0caafb05567f524ad313d9f8b780fa45987b8a8df04eff87923c74b4d0 F ext/fts5/test/fts5unindexed.test 168838d2c385e131120bbf5b516d2432a5fabc4caa2259c932e1d49ae209a4ae @@ -2208,8 +2208,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 38841881a08f4d75c2dfcbc1b2b45e311d7dda1287780f8ba1c8388f7adc1629 -R d19b3ba3504e0807fa9fc6488171b084 +P 9d971b31df7ad4d68eb348f95d8f996071cf87d41c47033bde3fcc4dba732e06 +R 00d1f889322bc8b1a7560fd99767ae0b U dan -Z 92294a85664ab83f8a52ccaa818e0f8f +Z 653dde5241b23832e0da59a6216ceb6f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cd18c0e49e..c8caf8375f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d971b31df7ad4d68eb348f95d8f996071cf87d41c47033bde3fcc4dba732e06 +8f9257361b05e368bf433e56d0698923b0f97d12e7c0ad7760aaab6746c0e467 From ef2401f66978d62187859c1039dc7e385dbb51c6 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Aug 2024 19:07:13 +0000 Subject: [PATCH 32/99] Tests to improve coverage of fts5_expr.c. FossilOrigin-Name: f4b839e5265700b1a89066d1b6e0d0d010852a69c5da3d75d2c41624dbf3c0af --- ext/fts5/fts5_expr.c | 18 ++++-------------- ext/fts5/test/fts5expr.test | 4 ++++ ext/fts5/test/fts5faultI.test | 13 ++++++++++++- ext/fts5/test/fts5misc.test | 21 +++++++++++++++++++++ ext/fts5/test/fts5trigram.test | 1 + manifest | 20 ++++++++++---------- manifest.uuid | 2 +- 7 files changed, 53 insertions(+), 26 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a39cc16bbf..d3b6598172 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -282,11 +282,12 @@ int sqlite3Fts5ExprNew( }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + assert( sParse.pExpr || sParse.rc!=SQLITE_OK ); assert_expr_depth_ok(sParse.rc, sParse.pExpr); /* If the LHS of the MATCH expression was a user column, apply the ** implicit column-filter. */ - if( iColnCol && sParse.pExpr && sParse.rc==SQLITE_OK ){ + if( sParse.rc==SQLITE_OK && iColnCol ){ int n = sizeof(Fts5Colset); Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n); if( pColset ){ @@ -303,15 +304,7 @@ int sqlite3Fts5ExprNew( sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); }else{ - if( !sParse.pExpr ){ - const int nByte = sizeof(Fts5ExprNode); - pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte); - if( pNew->pRoot ){ - pNew->pRoot->bEof = 1; - } - }else{ - pNew->pRoot = sParse.pExpr; - } + pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; pNew->pConfig = pConfig; pNew->apExprPhrase = sParse.apPhrase; @@ -1129,7 +1122,7 @@ static int fts5ExprNodeTest_STRING( } }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - if( pIter->iRowid==iLast || pIter->bEof ) continue; + if( pIter->iRowid==iLast ) continue; bMatch = 0; if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ return rc; @@ -1651,9 +1644,6 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( Fts5ExprNearset *pRet = 0; if( pParse->rc==SQLITE_OK ){ - if( pPhrase==0 ){ - return pNear; - } if( pNear==0 ){ sqlite3_int64 nByte; nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); diff --git a/ext/fts5/test/fts5expr.test b/ext/fts5/test/fts5expr.test index e3938beb0f..49be61d9c4 100644 --- a/ext/fts5/test/fts5expr.test +++ b/ext/fts5/test/fts5expr.test @@ -44,5 +44,9 @@ for {set ii 0} {$ii < 300} {incr ii} { } $res } +do_execsql_test 1.2 { + SELECT rowid FROM x1 WHERE a MATCH '"..."' +} {} + finish_test diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test index d35cdc3800..08a6bf056b 100644 --- a/ext/fts5/test/fts5faultI.test +++ b/ext/fts5/test/fts5faultI.test @@ -211,7 +211,7 @@ do_execsql_test 9.0 { faultsim_save_and_close -do_faultsim_test 9 -faults oom* -prep { +do_faultsim_test 9.1 -faults oom* -prep { faultsim_restore_and_reopen } -body { execsql { @@ -221,6 +221,17 @@ do_faultsim_test 9 -faults oom* -prep { faultsim_test_result {0 {}} } +do_faultsim_test 9.2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + SELECT rowid FROM ft WHERE x MATCH 'one AND two AND three' + } +} -test { + faultsim_test_result {0 1} +} + + finish_test diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index c7f4a53e38..71ad421c45 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -613,5 +613,26 @@ execsql_pp { SELECT fts5_source_id() } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 23.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1 VALUES('one + two + three'); + INSERT INTO x1 VALUES('one + xyz + three'); + INSERT INTO x1 VALUES('xyz + two + xyz'); +} +do_execsql_test 23.1 { + SELECT rowid FROM x1('one + two + three'); +} {1} + +do_execsql_test 23.2 { + SELECT rowid FROM x1('^".." AND one'); +} {} + +do_execsql_test 23.3 { + SELECT rowid FROM x1('abc NEAR ".." NEAR def'); +} {} + + finish_test diff --git a/ext/fts5/test/fts5trigram.test b/ext/fts5/test/fts5trigram.test index b06d35d6d1..3742c647f0 100644 --- a/ext/fts5/test/fts5trigram.test +++ b/ext/fts5/test/fts5trigram.test @@ -56,6 +56,7 @@ foreach {tn like res} { 7 {ABCDEFG%} 1 8 {%รุงเ%} 2 9 {%งเ%} 2 + 10 {%"งเ"%} {} } { do_execsql_test 1.3.$tn { SELECT rowid FROM t1 WHERE y LIKE $like diff --git a/manifest b/manifest index 4a616cffcb..46056ef32c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\srestore\scoverage\sof\sfts5_tokenizer.c. -D 2024-08-17T17:22:49.665 +C Tests\sto\simprove\scoverage\sof\sfts5_expr.c. +D 2024-08-17T19:07:13.921 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -97,7 +97,7 @@ F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d8056 F ext/fts5/fts5_aux.c 12cd2512f869217c38b70c31de5b5f741812734fafa80f55b32ea9bbd96e2152 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81cbefc109ea -F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a35da6f6 +F ext/fts5/fts5_expr.c 2667e822e90da6c41485b13ad8aa81e7e8768ff9018743a96fac6ddd76ecd0b9 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe F ext/fts5/fts5_main.c 23029229021240dc21a69eb749305316103d183375413e2090c064b74b03b7f8 @@ -160,7 +160,7 @@ F ext/fts5/test/fts5dlidx.test a7c42b0a74dc7c8aa1a46d586e0aadda4b6cc42c24450f8d3 F ext/fts5/test/fts5doclist.test b7cb84758504519746957802db9cd31187bb4e0028b89d9087ba06e26cc4155f F ext/fts5/test/fts5ea.test cefdf66024550fa7920c03395c71ce5046235ed1a1a7a469d79b19e7aad5afb5 F ext/fts5/test/fts5eb.test 401f756fdb77083aeba8b696c1e0ad4d834c39dbd6f17e492bb55a2ad64b4296 -F ext/fts5/test/fts5expr.test 7e1b2d075b63b727a624a378c2c09f94296a93dc4ae968aad67f8d9f3810c266 +F ext/fts5/test/fts5expr.test c7e208813df7a90badc856fde3796da79569b39382e0fdb43042127f3b8e06a7 F ext/fts5/test/fts5fault1.test d28a65caee75db6897c3cf1358c5230d3bb2a3bf7fb31062c19c7e5382b3d2bd F ext/fts5/test/fts5fault2.test 69c8fdbef830cd0d450908d4504d5bb86609e255af99c421c20a0756251fe344 F ext/fts5/test/fts5fault3.test da2f9e3e56ff5740d68ebdd6877c97089e7ed28ddff28a0da87a6afea27e5522 @@ -177,7 +177,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d -F ext/fts5/test/fts5faultI.test e57fa07b771565fcb7aa2b2c349a5104812b4edd86d09ca531afd9b247cf36dd +F ext/fts5/test/fts5faultI.test fbc65a64944fb747f6d3fb30628a807d5cce1bca43c11df40e7770ad7a7ed593 F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -191,7 +191,7 @@ F ext/fts5/test/fts5locale.test 79cbd3000ae269de50826f6061c81f7c9fdb21dd9954c0b7 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test c860b75721d162d7a4f2856322e55e814c2cf0bc45f93ef4388551b54e61a0a0 +F ext/fts5/test/fts5misc.test 558bc33c7b0b987b95e16c927f8a21e42f9a625eafad806b37f44008f5bf6314 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74 @@ -235,7 +235,7 @@ F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98 F ext/fts5/test/fts5tokenizer2.test ddb8b10fbe4b84b2a75812671f127774c1d2e3e2bf82d2e0e4f0bb1cd8a2b2d6 F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3 -F ext/fts5/test/fts5trigram.test fcb6b93510b917add63aee8f363c466ac406528822951e76e0980ce6f5170978 +F ext/fts5/test/fts5trigram.test fb9ee982edd76280ce979905a2251081cd04ae4c470248bd5d391b2d096430ab F ext/fts5/test/fts5trigram2.test 6fde9de7f63a6b4aa18dc731be56dbd6be4e755c9b13dcd55479e200d1df0e61 F ext/fts5/test/fts5ubsan.test 9a2dcf399dc8d0e0de661f0d93884d1d27e5b7f0693cfceb97dd24d818df5dd2 F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602 @@ -2208,8 +2208,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9d971b31df7ad4d68eb348f95d8f996071cf87d41c47033bde3fcc4dba732e06 -R 00d1f889322bc8b1a7560fd99767ae0b +P 8f9257361b05e368bf433e56d0698923b0f97d12e7c0ad7760aaab6746c0e467 +R f8d6b944dbf347c8074787f443d7147c U dan -Z 653dde5241b23832e0da59a6216ceb6f +Z d2bdd09fc432aeea5dfb2a1dda9df33e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c8caf8375f..50b3e4e2a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f9257361b05e368bf433e56d0698923b0f97d12e7c0ad7760aaab6746c0e467 +f4b839e5265700b1a89066d1b6e0d0d010852a69c5da3d75d2c41624dbf3c0af From 6c796631fc6371b19fa506eef86810453eb14b80 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Aug 2024 19:40:15 +0000 Subject: [PATCH 33/99] Fix a broken assert() in fts5_storage.c. FossilOrigin-Name: 01116e32797e1b127ba79059d890fbe5075eee2b6e3ce4468e82862b0fc3aa46 --- ext/fts5/fts5_storage.c | 3 +- ext/fts5/test/fts5corrupt3.test | 154 ++++++++++++++++++++++++++++++++ manifest | 14 +-- manifest.uuid | 2 +- 4 files changed, 163 insertions(+), 10 deletions(-) diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 03af105a75..cf25eb361e 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -887,10 +887,9 @@ int sqlite3Fts5StorageContentInsert( rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ sqlite3_value *pVal = apVal[i]; - if( sqlite3_value_nochange(pVal) ){ + if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ /* This is an UPDATE statement, and column (i-2) was not modified. ** Retrieve the value from Fts5Storage.pSavedRow instead. */ - assert( p->pSavedRow ); pVal = sqlite3_column_value(p->pSavedRow, i-1); }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){ assert( pConfig->bLocale ); diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index cfe1438eda..c5faaa87b1 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -15524,6 +15524,160 @@ do_catchsql_test 80.1 { SELECT snippet(rowid, -1, '.', '..', '[', '(]'),snippet(rowid, -1, '.', '.', '', '(]'), highlight(t1, 29, 1 , '') FROM t1('g+ h') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY NOT (SELECT 1 FROM t1('g+ æ') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY rank); } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +reset_db +do_test 81.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 40960 pagesize 4096 filename crash-44e8035a976422.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 8d 02 03 01 01 6a 42 .....h........jB +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 12 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 05 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: e6 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 21 69 14 02 05 00 17 17 17 67 20 68 20 69 h!i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-44e8035a976422.db +}]} {} + +do_catchsql_test 81.2 { + UPDATE t1 SET b=zeroblob(299); +} {1 {database disk image is malformed}} + sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/manifest b/manifest index 6ad1c9d6c6..a12f78027e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2024-08-17T19:11:09.556 +C Fix\sa\sbroken\sassert()\sin\sfts5_storage.c. +D 2024-08-17T19:40:15.498 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -101,7 +101,7 @@ F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe F ext/fts5/fts5_main.c 23029229021240dc21a69eb749305316103d183375413e2090c064b74b03b7f8 -F ext/fts5/fts5_storage.c acece8abf1b4b2012bdad93aa2d2cf6fc0d1da919763479c7055f699c2cde96c +F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612 F ext/fts5/fts5_tcl.c 1dcf08028141c40a32634bdcf2d5601622ce4edc48f82ac4ce0cbe0a92a6961d F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -148,7 +148,7 @@ F ext/fts5/test/fts5contentless4.test ec34dc69ef474ca9997dae6d91e072906e0e9a5a4b F ext/fts5/test/fts5contentless5.test 40cdcb4fe751672450829c5a96bd32c25fc2f6076279dd2ce5c58ac9a390132a F ext/fts5/test/fts5corrupt.test 6485f721b88ba355ca5d701e7ee87a4efa3ea578d8e6adb26f51ef956c8328bd F ext/fts5/test/fts5corrupt2.test 335911e3f68b9625d850325f9e29a128db3f4276a8c9d4e32134580da8f924c4 -F ext/fts5/test/fts5corrupt3.test b5f35d72af85b1d5a092b3d5e437f7944d142dd0b0c87b928fd0436a0aec6987 +F ext/fts5/test/fts5corrupt3.test 621e9bca3e7299f487e1b29ff4179d9fc9560f5847dfc5b50a16010c9d2a0e5f F ext/fts5/test/fts5corrupt4.test dc08d19f5b8943e95a7778a7d8da592042504faf18dd93f68f7d7a0d7d7dd733 F ext/fts5/test/fts5corrupt5.test 11b47126f5772cc37b67e3e8b2ed05895c4d07c05338bc07e4eea225bfe32c76 F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 @@ -2209,8 +2209,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f4b839e5265700b1a89066d1b6e0d0d010852a69c5da3d75d2c41624dbf3c0af a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3 -R 49e782c5b107501b250d5b7832f0211a +P 6a8ff9ba5e71d817489093d8dff0a8d77365b4222773b941accbd58558d24379 +R 9367527e69e79bca7a5a92aed8bd4950 U dan -Z 89224bb8e19ae56a7b37235b096e6728 +Z 275daf935ae7d8dde2bbfa29a46fdb3e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b7bf2f3e64..0408009dda 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a8ff9ba5e71d817489093d8dff0a8d77365b4222773b941accbd58558d24379 +01116e32797e1b127ba79059d890fbe5075eee2b6e3ce4468e82862b0fc3aa46 From 8797bd695f97db094bf10e546170d7b1e266867c Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 17 Aug 2024 19:46:49 +0000 Subject: [PATCH 34/99] Reduce the size of the SrcItem object by combining fields into a union. FossilOrigin-Name: a4c59ac3c6ec979c25b544d29e47b8e39f6439c098eed8f84b3bd506c9adf047 --- manifest | 35 +++++++++++++++++++---------------- manifest.uuid | 2 +- src/attach.c | 13 +++++++------ src/build.c | 24 ++++++++++++++---------- src/delete.c | 3 ++- src/expr.c | 7 +++++-- src/fkey.c | 3 ++- src/parse.y | 5 +++-- src/printf.c | 4 ++-- src/select.c | 23 +++++++++++++++++------ src/sqliteInt.h | 18 +++++++++++------- src/trigger.c | 13 ++++++++----- test/with2.test | 1 + 13 files changed, 92 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index bd4f4cbe1c..6e192bd300 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sMakefile.in\sso\sthat\sit\soutputs\sa\sslightly\sbetter\serror\smessage\son\san\nattempt\sto\sbuild\ssomething\sthat\srequires\sTCL\slibraries\swhen\sHAVE_TCL\sis\sfalse. -D 2024-08-17T16:17:16.045 +C Reduce\sthe\ssize\sof\sthe\sSrcItem\sobject\sby\scombining\sfields\sinto\sa\sunion. +D 2024-08-17T19:46:49.038 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -694,7 +694,7 @@ F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c bb663fddf1fe0e2e6d8758b2b7fb6374e7c057a6ca3955f37a48986806029765 F src/analyze.c 30bf40ec4208ead9e977bec017bccc8a9681820936e38ca5a4a7443100a6d5c5 -F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 +F src/attach.c d778bacb2ee5bc6a74c81a17376064e6a16ee447d82ee4ad1f7717a320d5a232 F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 @@ -702,17 +702,17 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c 8b42fc7d9efdb2df05c30e8f91ff6cfbd979724ae24bf90269028468b7a13333 F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 -F src/build.c 237ccc0290d131d646be722f418e92ee0a38043aee25e7dfdc75f8ce5b3abe4e +F src/build.c 8d853f7900dee57d9b3281d71b772f80c45798a3dd782e5e0b6e732ddfe31fe4 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 F src/date.c 13dd752847afb32ed70510ad7345a5b9c841f51ad904dba5d010f1fa3a6a324e F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 -F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c fe958028b36af640b70b2174354c044f75b8c4a4645c921592122aa2a022083a +F src/delete.c 9999f697346ae1bf4e31ed5d5049ab31d023f290f7f9e86df550f8c269904f4d +F src/expr.c 737704d4ce25bb1f0724702dfa52a5850f28792a5631c7476ef2204e2bee2985 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c 852f93c0ef995e0c2b8983059a2b97151c194cc8259e21f5bc2b7ac508348c2a +F src/fkey.c d27603c782215c4e001745768dc6abb2c6394e9f3f47909c9e00502177462ac3 F src/func.c 1f61e32e7a357e615b5d2e774bee563761fce4f2fd97ecb0f72c33e62a2ada5f F src/global.c 61a419dd9e993b9be0f91de4c4ccf322b053eb829868e089f0321dd669be3b90 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 @@ -749,23 +749,23 @@ F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c b08600ebf0db90b6d1e9b8b6577c6fa3877cbe1a100bd0b2899e4c6e9adad4b3 F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a -F src/parse.y 318ef86fbe358b1a93262a42e152f37b97b3fddae8d319dffbd24ce2300f6c88 +F src/parse.y 9eac03d271c6b4a626125de5ce15fdb62e99f4542c1c93bf04cbc57b91d2b2b1 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 49516ad7718a3626f28f710fa7448ef1fce3c07fd169acbb4817341950264319 F src/pragma.c 52bfbf6dfd668b69b5eb9bd1186e3a67367c8453807150d6e75239229924f684 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c d99931f45416652895e502328ca49fe782cfc4e1ebdcda13b3736d991ebf42ce -F src/printf.c 8b250972305e14b365561be5117ed0fd364e4fd58968776df1ce64c6280b90f9 +F src/printf.c 3820e4fbb96a76ad5799ed4d949f53a1eb61b5e029a172740ee98538f0e57406 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 0aee8a2e5340ba95a966917305dfaff5147fcad78d0839cd364b16e4746b8bcb F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 6a95a2bffa6c09584dea99db5a7ae10c813305c09c92920ffc54f6eae2ba399e +F src/select.c 18325f3c24d3b75ce102694f3eb96d6d8dd74ac9b0d10525758ed673c7728bee F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab67704 F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 128b9004698cc79993d5369d7d1763deaf8bbf26e5e8931ec540707e5a7238df +F src/sqliteInt.h dab52ce59c82852105ac40e058c0097368c6a6dce47da2ef68471312b7643ad2 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -825,7 +825,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68 F src/treeview.c 774838df4e25956ca34ff79bef150266412cfc2640620d04e22d5c8a55a98992 -F src/trigger.c 0858f75818ed1580332db274f1032bcc5effe567cb132df5c5be8b1d800ca97f +F src/trigger.c 423b50d4eea6b6e59bfadcaa0e3806b10fc21a55e5a5d2f73094ba4a76a70ab8 F src/update.c 732404a04d1737ef14bb6ec6b84f74edf28b3c102a92ae46b4855438a710efe7 F src/upsert.c 2e60567a0e9e8520c18671b30712a88dc73534474304af94f32bb5f3ef65ac65 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e @@ -2070,7 +2070,7 @@ F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3c F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c F test/windowpushd.test c420e2265f0e09a0e798d0513a660d71b51602088d81b3dbd038918ee1339dcc F test/with1.test b93833890e5d2a368e78747f124503a0159aa029b98e9ed4795ebf630b2efd3d -F test/with2.test a1df41b987198383b9b70bf5e5fda390582e46398653858dbc6ceb24253b28df +F test/with2.test 5885f0ef250504096ea4c0c6350ef2f2ab09b9e2dc2bc691e182f773db3aabb2 F test/with3.test e30369ea27aa27eb1bda4c5e510c8a9f782c8afd2ab99d1a02b8a7f25a5d3e65 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/with5.test 6248213c41fab36290b5b73aa3f937309dfba337004d9d8434c3fabc8c7d4be8 @@ -2205,8 +2205,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 793ff83d09bea165c8cedc978afd3fcfb1a4cee894c5914aa1d92416afce963d -R be5ece8b231f01bec02cbe9ef4eda884 +P a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3 +R d143d0c065164b192322cbf1e4e8b378 +T *branch * srcitem-opt +T *sym-srcitem-opt * +T -sym-trunk * U drh -Z 6d429e685f531219ca0d0afba9760eb7 +Z c1a523f650613fd8f9b88c0821d4d359 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a5034c9d32..2ba642c078 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3 +a4c59ac3c6ec979c25b544d29e47b8e39f6439c098eed8f84b3bd506c9adf047 diff --git a/src/attach.c b/src/attach.c index 4a6a25bf04..65b492474e 100644 --- a/src/attach.c +++ b/src/attach.c @@ -480,19 +480,20 @@ static int fixSelectCb(Walker *p, Select *pSelect){ if( NEVER(pList==0) ) return WRC_Continue; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ if( pFix->bTemp==0 ){ - if( pItem->zDatabase ){ - if( iDb!=sqlite3FindDbName(db, pItem->zDatabase) ){ + if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + if( iDb!=sqlite3FindDbName(db, pItem->u4.zDatabase) ){ sqlite3ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", - pFix->zType, pFix->pName, pItem->zDatabase); + pFix->zType, pFix->pName, pItem->u4.zDatabase); return WRC_Abort; } - sqlite3DbFree(db, pItem->zDatabase); - pItem->zDatabase = 0; + sqlite3DbFree(db, pItem->u4.zDatabase); pItem->fg.notCte = 1; + pItem->fg.hadSchema = 1; } - pItem->pSchema = pFix->pSchema; + pItem->u4.pSchema = pFix->pSchema; pItem->fg.fromDDL = 1; + pItem->fg.fixedSchema = 1; } #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) if( pList->a[i].fg.isUsing==0 diff --git a/src/build.c b/src/build.c index 9747810e82..a0c4b493f0 100644 --- a/src/build.c +++ b/src/build.c @@ -497,12 +497,11 @@ Table *sqlite3LocateTableItem( SrcItem *p ){ const char *zDb; - assert( p->pSchema==0 || p->zDatabase==0 ); - if( p->pSchema ){ - int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + if( p->fg.fixedSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->u4.pSchema); zDb = pParse->db->aDb[iDb].zDbSName; }else{ - zDb = p->zDatabase; + zDb = p->u4.zDatabase; } return sqlite3LocateTable(pParse, flags, p->zName, zDb); } @@ -3487,6 +3486,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ } assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); + assert( pName->a[0].fg.fixedSchema==0 ); if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; assert( isView==0 || isView==LOCATE_VIEW ); @@ -3495,7 +3495,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ if( pTab==0 ){ if( noErr ){ - sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].u4.zDatabase); sqlite3ForceNotReadOnly(pParse); } goto exit_drop_table; @@ -4586,15 +4586,16 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ } assert( pParse->nErr==0 ); /* Never called with prior non-OOM errors */ assert( pName->nSrc==1 ); + assert( pName->a[0].fg.fixedSchema==0 ); if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto exit_drop_index; } - pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].zDatabase); + pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].u4.zDatabase); if( pIndex==0 ){ if( !ifExists ){ sqlite3ErrorMsg(pParse, "no such index: %S", pName->a); }else{ - sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].u4.zDatabase); sqlite3ForceNotReadOnly(pParse); } pParse->checkSchema = 1; @@ -4891,12 +4892,13 @@ SrcList *sqlite3SrcListAppend( if( pDatabase && pDatabase->z==0 ){ pDatabase = 0; } + assert( pItem->fg.fixedSchema==0 ); if( pDatabase ){ pItem->zName = sqlite3NameFromToken(db, pDatabase); - pItem->zDatabase = sqlite3NameFromToken(db, pTable); + pItem->u4.zDatabase = sqlite3NameFromToken(db, pTable); }else{ pItem->zName = sqlite3NameFromToken(db, pTable); - pItem->zDatabase = 0; + pItem->u4.zDatabase = 0; } return pList; } @@ -4928,9 +4930,11 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ assert( db!=0 ); if( pList==0 ) return; for(pItem=pList->a, i=0; inSrc; i++, pItem++){ - if( pItem->zDatabase ) sqlite3DbNNFreeNN(db, pItem->zDatabase); if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); + if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + sqlite3DbNNFreeNN(db, pItem->u4.zDatabase); + } if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pTab); diff --git a/src/delete.c b/src/delete.c index 2baff5b3d4..bc7a7a3f53 100644 --- a/src/delete.c +++ b/src/delete.c @@ -156,7 +156,8 @@ void sqlite3MaterializeView( if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pFrom->a[0].fg.fixedSchema==0 ); + pFrom->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); assert( pFrom->a[0].fg.isUsing==0 ); assert( pFrom->a[0].u3.pOn==0 ); } diff --git a/src/expr.c b/src/expr.c index 53b0170ab4..d0fa4b5007 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1877,8 +1877,11 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ SrcItem *pNewItem = &pNew->a[i]; const SrcItem *pOldItem = &p->a[i]; Table *pTab; - pNewItem->pSchema = pOldItem->pSchema; - pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); + if( pOldItem->fg.fixedSchema ){ + pNewItem->u4.pSchema = pOldItem->u4.pSchema; + }else{ + pNewItem->u4.zDatabase = sqlite3DbStrDup(db, pOldItem->u4.zDatabase); + } pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); pNewItem->fg = pOldItem->fg; diff --git a/src/fkey.c b/src/fkey.c index 1efdc0bba1..c68037a654 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -1337,7 +1337,8 @@ static Trigger *fkActionTrigger( if( pSrc ){ assert( pSrc->nSrc==1 ); pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); - pSrc->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pSrc->a[0].fg.fixedSchema==0 ); + pSrc->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), diff --git a/src/parse.y b/src/parse.y index 68c00f0a7f..a605ad6912 100644 --- a/src/parse.y +++ b/src/parse.y @@ -740,8 +740,9 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N if( A ){ SrcItem *pNew = &A->a[A->nSrc-1]; SrcItem *pOld = F->a; + assert( pOld->fg.fixedSchema==0 ); pNew->zName = pOld->zName; - pNew->zDatabase = pOld->zDatabase; + pNew->u4.zDatabase = pOld->u4.zDatabase; pNew->pSelect = pOld->pSelect; if( pNew->pSelect && (pNew->pSelect->selFlags & SF_NestedFrom)!=0 ){ pNew->fg.isNestedFrom = 1; @@ -752,7 +753,7 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N pOld->fg.isTabFunc = 0; pNew->fg.isTabFunc = 1; } - pOld->zName = pOld->zDatabase = 0; + pOld->zName = pOld->u4.zDatabase = 0; pOld->pSelect = 0; } sqlite3SrcListDelete(pParse->db, F); diff --git a/src/printf.c b/src/printf.c index c0dcc5d0fa..12dd91a781 100644 --- a/src/printf.c +++ b/src/printf.c @@ -848,8 +848,8 @@ void sqlite3_str_vappendf( if( pItem->zAlias && !flag_altform2 ){ sqlite3_str_appendall(pAccum, pItem->zAlias); }else if( pItem->zName ){ - if( pItem->zDatabase ){ - sqlite3_str_appendall(pAccum, pItem->zDatabase); + if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + sqlite3_str_appendall(pAccum, pItem->u4.zDatabase); sqlite3_str_append(pAccum, ".", 1); } sqlite3_str_appendall(pAccum, pItem->zName); diff --git a/src/select.c b/src/select.c index 4b0b554295..634f06e301 100644 --- a/src/select.c +++ b/src/select.c @@ -4506,10 +4506,12 @@ static int flattenSubquery( /* Delete the transient structures associated with the subquery */ pSub1 = pSubitem->pSelect; - sqlite3DbFree(db, pSubitem->zDatabase); + if( pSubitem->fg.fixedSchema==0 ){ + sqlite3DbFree(db, pSubitem->u4.zDatabase); + pSubitem->u4.zDatabase = 0; + } sqlite3DbFree(db, pSubitem->zName); sqlite3DbFree(db, pSubitem->zAlias); - pSubitem->zDatabase = 0; pSubitem->zName = 0; pSubitem->zAlias = 0; pSubitem->pSelect = 0; @@ -5637,7 +5639,7 @@ static struct Cte *searchWith( ){ const char *zName = pItem->zName; With *p; - assert( pItem->zDatabase==0 ); + assert( pItem->fg.fixedSchema || pItem->u4.zDatabase==0 ); assert( zName!=0 ); for(p=pWith; p; p=p->pOuter){ int i; @@ -5717,7 +5719,8 @@ static int resolveFromTermToCte( ** go no further. */ return 0; } - if( pFrom->zDatabase!=0 ){ + assert( pFrom->fg.hadSchema==0 || pFrom->fg.notCte!=0 ); + if( pFrom->fg.fixedSchema==0 && pFrom->u4.zDatabase!=0 ){ /* The FROM term contains a schema qualifier (ex: main.t1) and so ** it cannot possibly be a CTE reference. */ return 0; @@ -5794,7 +5797,8 @@ static int resolveFromTermToCte( assert( pRecTerm->pPrior!=0 ); for(i=0; inSrc; i++){ SrcItem *pItem = &pSrc->a[i]; - if( pItem->zDatabase==0 + if( ((pItem->fg.fixedSchema==0 && pItem->u4.zDatabase==0) + || pItem->fg.hadSchema==0) && pItem->zName!=0 && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ @@ -7681,7 +7685,14 @@ int sqlite3Select( ** string for the fake column name seems safer. */ if( pItem->colUsed==0 && pItem->zName!=0 ){ - sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase); + const char *zDb; + if( pItem->fg.fixedSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pItem->u4.pSchema); + zDb = db->aDb[iDb].zDbSName; + }else{ + zDb = pItem->u4.zDatabase; + } + sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", zDb); } #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ebd0f01b26..75e1f61201 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3297,13 +3297,11 @@ struct IdList { ** u2.pCteUse fg.isCte && !fg.isIndexedBy */ struct SrcItem { - Schema *pSchema; /* Schema to which this item is fixed */ - char *zDatabase; /* Name of database holding this table */ char *zName; /* Name of the table */ char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ Table *pTab; /* An SQL table corresponding to zName */ Select *pSelect; /* A SELECT statement used in place of a table name */ - int addrFillSub; /* Address of subroutine to manifest a subquery */ + int addrFillSub; /* Address of subroutine to initialize a subquery */ int regReturn; /* Register holding return address of addrFillSub */ int regResult; /* Registers holding results of a co-routine */ struct { @@ -3323,12 +3321,10 @@ struct SrcItem { unsigned isSynthUsing :1; /* u3.pUsing is synthesized from NATURAL */ unsigned isNestedFrom :1; /* pSelect is a SF_NestedFrom subquery */ unsigned rowidUsed :1; /* The ROWID of this table is referenced */ + unsigned fixedSchema :1; /* Uses u4.pSchema, not u4.zDatabase */ + unsigned hadSchema :1; /* Has u4.zDatabase before u4.pSchema */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ - union { - Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */ - IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ - } u3; Bitmask colUsed; /* Bit N set if column N used. Details above for N>62 */ union { char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ @@ -3339,6 +3335,14 @@ struct SrcItem { Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ CteUse *pCteUse; /* CTE Usage info when fg.isCte is true */ } u2; + union { + Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */ + IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ + } u3; + union { + Schema *pSchema; /* Schema to which this item is fixed */ + char *zDatabase; /* Name of database holding this table */ + } u4; }; /* diff --git a/src/trigger.c b/src/trigger.c index 98e8da58c1..a5a57abf16 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -151,9 +151,9 @@ void sqlite3BeginTrigger( ** To maintain backwards compatibility, ignore the database ** name on pTableName if we are reparsing out of the schema table */ - if( db->init.busy && iDb!=1 ){ - sqlite3DbFree(db, pTableName->a[0].zDatabase); - pTableName->a[0].zDatabase = 0; + if( db->init.busy && iDb!=1 && ALWAYS(pTableName->a[0].fg.fixedSchema==0) ){ + sqlite3DbFree(db, pTableName->a[0].u4.zDatabase); + pTableName->a[0].u4.zDatabase = 0; } /* If the trigger name was unqualified, and the table is a temp table, @@ -631,7 +631,8 @@ void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ } assert( pName->nSrc==1 ); - zDb = pName->a[0].zDatabase; + assert( pName->a[0].fg.fixedSchema==0 ); + zDb = pName->a[0].u4.zDatabase; zName = pName->a[0].zName; assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ @@ -868,7 +869,9 @@ SrcList *sqlite3TriggerStepSrc( Schema *pSchema = pStep->pTrig->pSchema; pSrc->a[0].zName = zName; if( pSchema!=db->aDb[1].pSchema ){ - pSrc->a[0].pSchema = pSchema; + assert( pSrc->a[0].fg.fixedSchema || pSrc->a[0].u4.zDatabase==0 ); + pSrc->a[0].u4.pSchema = pSchema; + pSrc->a[0].fg.fixedSchema = 1; } if( pStep->pFrom ){ SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0); diff --git a/test/with2.test b/test/with2.test index 660df52b77..ad1753d659 100644 --- a/test/with2.test +++ b/test/with2.test @@ -146,6 +146,7 @@ do_execsql_test 1.14 { SELECT * FROM t4; } {4 5 6 7 8 9 10} +exit do_execsql_test 1.15 { WITH t4(x) AS ( From b204b6aa7bd94252c8043f53701f314ee433bafd Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 17 Aug 2024 23:23:23 +0000 Subject: [PATCH 35/99] Give unique names to fields in the SrcItem object, to facilitate analysis of how those fields are used using "grep". FossilOrigin-Name: 9f5aeef3cbc2c95267c8f7bf60d5c66971a76789669fb0e8f853273ff6f616f2 --- manifest | 55 +++++++------ manifest.uuid | 2 +- src/alter.c | 8 +- src/auth.c | 2 +- src/build.c | 19 +++-- src/delete.c | 4 +- src/expr.c | 16 ++-- src/fkey.c | 4 +- src/insert.c | 33 ++++---- src/parse.y | 6 +- src/printf.c | 2 +- src/resolve.c | 39 +++++----- src/select.c | 199 ++++++++++++++++++++++++------------------------ src/sqliteInt.h | 18 +++-- src/treeview.c | 16 ++-- src/trigger.c | 4 +- src/update.c | 6 +- src/upsert.c | 4 +- src/walker.c | 2 +- src/where.c | 94 +++++++++++------------ src/wherecode.c | 27 +++---- src/whereexpr.c | 10 +-- src/window.c | 8 +- 23 files changed, 299 insertions(+), 279 deletions(-) diff --git a/manifest b/manifest index 6e192bd300..ef77b8894d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\ssize\sof\sthe\sSrcItem\sobject\sby\scombining\sfields\sinto\sa\sunion. -D 2024-08-17T19:46:49.038 +C Give\sunique\snames\sto\sfields\sin\sthe\sSrcItem\sobject,\sto\sfacilitate\sanalysis\sof\nhow\sthose\sfields\sare\sused\susing\s"grep". +D 2024-08-17T23:23:23.271 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -692,34 +692,34 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 -F src/alter.c bb663fddf1fe0e2e6d8758b2b7fb6374e7c057a6ca3955f37a48986806029765 +F src/alter.c b73964bd6bc3122e884993c0666bae96d4e1e50a94b6f9008df32e6a18e99cd0 F src/analyze.c 30bf40ec4208ead9e977bec017bccc8a9681820936e38ca5a4a7443100a6d5c5 F src/attach.c d778bacb2ee5bc6a74c81a17376064e6a16ee447d82ee4ad1f7717a320d5a232 -F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 +F src/auth.c 4c1ea890e0069ad73bead5d17a5b12c34cfa4f1a24175c8147ea439b64be271c F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c 8b42fc7d9efdb2df05c30e8f91ff6cfbd979724ae24bf90269028468b7a13333 F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 -F src/build.c 8d853f7900dee57d9b3281d71b772f80c45798a3dd782e5e0b6e732ddfe31fe4 +F src/build.c 664804cbefaf43c13fcb369474609cf8befa4d89933804dc276d0a15fbd66c34 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 F src/date.c 13dd752847afb32ed70510ad7345a5b9c841f51ad904dba5d010f1fa3a6a324e F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 -F src/delete.c 9999f697346ae1bf4e31ed5d5049ab31d023f290f7f9e86df550f8c269904f4d -F src/expr.c 737704d4ce25bb1f0724702dfa52a5850f28792a5631c7476ef2204e2bee2985 +F src/delete.c 8aaf9d05d6ff6349a7a6001cbf90c7a46b19b9f52ef2e47aa0415a529d5307a1 +F src/expr.c 835b637da89fe20a1ff90bd372904fd421ff92d1dfb9bbea9c81bbffbea46715 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c d27603c782215c4e001745768dc6abb2c6394e9f3f47909c9e00502177462ac3 +F src/fkey.c 849049c74a4c68961154c124087b5c67d277217a2e642570d0c1bd6336859940 F src/func.c 1f61e32e7a357e615b5d2e774bee563761fce4f2fd97ecb0f72c33e62a2ada5f F src/global.c 61a419dd9e993b9be0f91de4c4ccf322b053eb829868e089f0321dd669be3b90 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 8ff11e9e54c5fc1fe89707b3d41cf44ad2822f712bd3b5da68338ea42518847e +F src/insert.c 2152b159ac214519e778b6996a85b9073ace054f5c9d430b375852e343a5ec92 F src/json.c 5b6a1d6015997b9ee848a32948720bdb26a0ef2de5a2127ebf7355ce66dbdc0d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 @@ -749,23 +749,23 @@ F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c b08600ebf0db90b6d1e9b8b6577c6fa3877cbe1a100bd0b2899e4c6e9adad4b3 F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a -F src/parse.y 9eac03d271c6b4a626125de5ce15fdb62e99f4542c1c93bf04cbc57b91d2b2b1 +F src/parse.y bc8df5d36afd06d50c1828072e4f3120d892af685436db3b8e776288803535e0 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 49516ad7718a3626f28f710fa7448ef1fce3c07fd169acbb4817341950264319 F src/pragma.c 52bfbf6dfd668b69b5eb9bd1186e3a67367c8453807150d6e75239229924f684 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c d99931f45416652895e502328ca49fe782cfc4e1ebdcda13b3736d991ebf42ce -F src/printf.c 3820e4fbb96a76ad5799ed4d949f53a1eb61b5e029a172740ee98538f0e57406 +F src/printf.c efa3973bdcacdbc8cf0ff3b3a58e16a105319eac8819823f5807579dc2fa4138 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c 0aee8a2e5340ba95a966917305dfaff5147fcad78d0839cd364b16e4746b8bcb +F src/resolve.c d25fc99cd54d2acf4c655f942dd48ee60e3ada94d73c8b992487515f1295a3bc F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 18325f3c24d3b75ce102694f3eb96d6d8dd74ac9b0d10525758ed673c7728bee +F src/select.c 10ff02352f4c6eb7bc8d5711f0f74199b94ae9adc346f10391931f5c55d1c68f F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab67704 F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h dab52ce59c82852105ac40e058c0097368c6a6dce47da2ef68471312b7643ad2 +F src/sqliteInt.h 95186ebaf79560fa44b1a05bd7b4b14846e037a285177ca65082a9ae952a6c29 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -824,10 +824,10 @@ F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68 -F src/treeview.c 774838df4e25956ca34ff79bef150266412cfc2640620d04e22d5c8a55a98992 -F src/trigger.c 423b50d4eea6b6e59bfadcaa0e3806b10fc21a55e5a5d2f73094ba4a76a70ab8 -F src/update.c 732404a04d1737ef14bb6ec6b84f74edf28b3c102a92ae46b4855438a710efe7 -F src/upsert.c 2e60567a0e9e8520c18671b30712a88dc73534474304af94f32bb5f3ef65ac65 +F src/treeview.c a337ab329f94e627c9a5efee0326d3359e352c4f0c0bb5c8f4b686fe03e6e7e8 +F src/trigger.c 68d849ea6ccab67beb450d1a0d065c6118f0884a779388b07f0d9a27dbda38c9 +F src/update.c 0e01aa6a3edf9ec112b33eb714b9016a81241497b1fb7c3e74332f4f71756508 +F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c 5d1a0134cf4240648d1c6bb5cc8efaca0ea2b5d5c840985aec7e947271f04375 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 @@ -845,12 +845,12 @@ F src/vtab.c 5fb499d20494b7eecaadb7584634af9afcb374cb0524912b475fcb1712458a1b F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 -F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c f5be664f3379c9f930696e339ec4ef4c1187af860cca727411156101fae6b677 +F src/walker.c e5898234c5920481d962e29b96ecf403b992b12ae243ad0e53659386a1b3e554 +F src/where.c d0140395a35fbc451e9d07c9816895d399f02ae76cbcd255e8ce1069548b02c8 F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e -F src/wherecode.c c9cac0b0b8e809c5e7e79d7796918907fb685ad99be2aaa9737f9787aa47349c -F src/whereexpr.c 7d0d34b42b9edfd8e8ca66beb3a6ef63fe211c001af54caf2ccbcd989b783290 -F src/window.c 1e40ffc509bae21e466f6106382d238e91eb73edd4ba10e66ca4fd7af2b96896 +F src/wherecode.c 3e130abafc185a29c8dbdcc33e6150dff2aa884f75483e159feef170961af65d +F src/whereexpr.c 784f641ba3d6f1fdb4f65e3891daa50067a4c51f1acd11b461397f7d4c622907 +F src/window.c ffbf65ba62ce329e2fd68b35f11e920121fac45dab0f9a4bbc486125ccc98b6c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9 @@ -2205,11 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3 -R d143d0c065164b192322cbf1e4e8b378 -T *branch * srcitem-opt -T *sym-srcitem-opt * -T -sym-trunk * +P a4c59ac3c6ec979c25b544d29e47b8e39f6439c098eed8f84b3bd506c9adf047 +R 1d81e98efd997ac91c2ceea4c803b7e6 U drh -Z c1a523f650613fd8f9b88c0821d4d359 +Z 8ec986f4842c563a443c2f1a23450bf4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2ba642c078..0e51ef0919 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4c59ac3c6ec979c25b544d29e47b8e39f6439c098eed8f84b3bd506c9adf047 +9f5aeef3cbc2c95267c8f7bf60d5c66971a76789669fb0e8f853273ff6f616f2 diff --git a/src/alter.c b/src/alter.c index a8556d115e..41146f54e9 100644 --- a/src/alter.c +++ b/src/alter.c @@ -1366,8 +1366,8 @@ static int renameResolveTrigger(Parse *pParse){ int i; for(i=0; ipFrom->nSrc && rc==SQLITE_OK; i++){ SrcItem *p = &pStep->pFrom->a[i]; - if( p->pSelect ){ - sqlite3SelectPrep(pParse, p->pSelect, 0); + if( p->sq.pSelect ){ + sqlite3SelectPrep(pParse, p->sq.pSelect, 0); } } } @@ -1436,7 +1436,7 @@ static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ if( pStep->pFrom ){ int i; for(i=0; ipFrom->nSrc; i++){ - sqlite3WalkSelect(pWalker, pStep->pFrom->a[i].pSelect); + sqlite3WalkSelect(pWalker, pStep->pFrom->a[i].sq.pSelect); } } } @@ -1683,7 +1683,7 @@ static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ } for(i=0; inSrc; i++){ SrcItem *pItem = &pSrc->a[i]; - if( pItem->pTab==p->pTab ){ + if( pItem->pSTab==p->pTab ){ renameTokenFind(pWalker->pParse, p, pItem->zName); } } diff --git a/src/auth.c b/src/auth.c index fa6c82d85d..fba2c09905 100644 --- a/src/auth.c +++ b/src/auth.c @@ -165,7 +165,7 @@ void sqlite3AuthRead( assert( pTabList ); for(iSrc=0; iSrcnSrc; iSrc++){ if( pExpr->iTable==pTabList->a[iSrc].iCursor ){ - pTab = pTabList->a[iSrc].pTab; + pTab = pTabList->a[iSrc].pSTab; break; } } diff --git a/src/build.c b/src/build.c index a0c4b493f0..ff798087af 100644 --- a/src/build.c +++ b/src/build.c @@ -4914,8 +4914,8 @@ void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ for(i=0, pItem=pList->a; inSrc; i++, pItem++){ if( pItem->iCursor>=0 ) continue; pItem->iCursor = pParse->nTab++; - if( pItem->pSelect ){ - sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); + if( pItem->sq.pSelect ){ + sqlite3SrcListAssignCursors(pParse, pItem->sq.pSelect->pSrc); } } } @@ -4930,6 +4930,15 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ assert( db!=0 ); if( pList==0 ) return; for(pItem=pList->a, i=0; inSrc; i++, pItem++){ + + /* Check invariants on SrcItem */ + assert( pItem->sq.pSelect==0 + || (pItem->sq.pSelect->selFlags & SF_MultiValue)==0 + || (pItem->fg.hadSchema==0 + && (pItem->fg.fixedSchema || pItem->u4.zDatabase==0)) ); + assert( pItem->fg.hadSchema==0 || pItem->fg.fixedSchema==1 ); + + if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ @@ -4937,8 +4946,8 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ } if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); - sqlite3DeleteTable(db, pItem->pTab); - if( pItem->pSelect ) sqlite3SelectDelete(db, pItem->pSelect); + sqlite3DeleteTable(db, pItem->pSTab); + if( pItem->sq.pSelect ) sqlite3SelectDelete(db, pItem->sq.pSelect); if( pItem->fg.isUsing ){ sqlite3IdListDelete(db, pItem->u3.pUsing); }else if( pItem->u3.pOn ){ @@ -4998,7 +5007,7 @@ SrcList *sqlite3SrcListAppendFromTerm( pItem->zAlias = sqlite3NameFromToken(db, pAlias); } if( pSubquery ){ - pItem->pSelect = pSubquery; + pItem->sq.pSelect = pSubquery; if( pSubquery->selFlags & SF_NestedFrom ){ pItem->fg.isNestedFrom = 1; } diff --git a/src/delete.c b/src/delete.c index bc7a7a3f53..010a900191 100644 --- a/src/delete.c +++ b/src/delete.c @@ -33,8 +33,8 @@ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ Table *pTab; assert( pItem && pSrc->nSrc>=1 ); pTab = sqlite3LocateTableItem(pParse, 0, pItem); - if( pItem->pTab ) sqlite3DeleteTable(pParse->db, pItem->pTab); - pItem->pTab = pTab; + if( pItem->pSTab ) sqlite3DeleteTable(pParse->db, pItem->pSTab); + pItem->pSTab = pTab; pItem->fg.notCte = 1; if( pTab ){ pTab->nTabRef++; diff --git a/src/expr.c b/src/expr.c index d0fa4b5007..6ad789a3d8 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1886,9 +1886,9 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; - pNewItem->addrFillSub = pOldItem->addrFillSub; - pNewItem->regReturn = pOldItem->regReturn; - pNewItem->regResult = pOldItem->regResult; + pNewItem->sq.addrFillSub = pOldItem->sq.addrFillSub; + pNewItem->sq.regReturn = pOldItem->sq.regReturn; + pNewItem->sq.regResult = pOldItem->sq.regResult; if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); }else if( pNewItem->fg.isTabFunc ){ @@ -1901,11 +1901,11 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ if( pNewItem->fg.isCte ){ pNewItem->u2.pCteUse->nUse++; } - pTab = pNewItem->pTab = pOldItem->pTab; + pTab = pNewItem->pSTab = pOldItem->pSTab; if( pTab ){ pTab->nTabRef++; } - pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); + pNewItem->sq.pSelect = sqlite3SelectDup(db, pOldItem->sq.pSelect, flags); if( pOldItem->fg.isUsing ){ assert( pNewItem->fg.isUsing ); pNewItem->u3.pUsing = sqlite3IdListDup(db, pOldItem->u3.pUsing); @@ -2999,8 +2999,8 @@ static Select *isCandidateForInOpt(const Expr *pX){ pSrc = p->pSrc; assert( pSrc!=0 ); if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ - if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ - pTab = pSrc->a[0].pTab; + if( pSrc->a[0].sq.pSelect ) return 0; /* FROM is not a subquery or view */ + pTab = pSrc->a[0].pSTab; assert( pTab!=0 ); assert( !IsView(pTab) ); /* FROM clause is not a view */ if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ @@ -3183,7 +3183,7 @@ int sqlite3FindInIndex( assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ - pTab = p->pSrc->a[0].pTab; + pTab = p->pSrc->a[0].pSTab; /* Code an OP_Transaction and OP_TableLock for . */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); diff --git a/src/fkey.c b/src/fkey.c index c68037a654..cd1f88502e 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -1043,9 +1043,9 @@ void sqlite3FkCheck( pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pSrc ){ SrcItem *pItem = pSrc->a; - pItem->pTab = pFKey->pFrom; + pItem->pSTab = pFKey->pFrom; pItem->zName = pFKey->pFrom->zName; - pItem->pTab->nTabRef++; + pItem->pSTab->nTabRef++; pItem->iCursor = pParse->nTab++; if( regNew!=0 ){ diff --git a/src/insert.c b/src/insert.c index a7e94420bf..4447cdc22e 100644 --- a/src/insert.c +++ b/src/insert.c @@ -585,8 +585,8 @@ void sqlite3AutoincrementEnd(Parse *pParse){ void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ if( ALWAYS(pVal) && pVal->pSrc->nSrc>0 ){ SrcItem *pItem = &pVal->pSrc->a[0]; - sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->regReturn); - sqlite3VdbeJumpHere(pParse->pVdbe, pItem->addrFillSub - 1); + sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->sq.regReturn); + sqlite3VdbeJumpHere(pParse->pVdbe, pItem->sq.addrFillSub - 1); } } @@ -723,14 +723,15 @@ Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ assert( pLeft->pNext==0 ); assert( pRet->pNext==0 ); p = &pRet->pSrc->a[0]; - p->pSelect = pLeft; + p->sq.pSelect = pLeft; p->fg.viaCoroutine = 1; - p->addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; - p->regReturn = ++pParse->nMem; + p->sq.addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; + p->sq.regReturn = ++pParse->nMem; p->iCursor = -1; p->u1.nRow = 2; - sqlite3VdbeAddOp3(v,OP_InitCoroutine,p->regReturn,0,p->addrFillSub); - sqlite3SelectDestInit(&dest, SRT_Coroutine, p->regReturn); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, + p->sq.regReturn, 0, p->sq.addrFillSub); + sqlite3SelectDestInit(&dest, SRT_Coroutine, p->sq.regReturn); /* Allocate registers for the output of the co-routine. Do so so ** that there are two unused registers immediately before those @@ -743,7 +744,7 @@ Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ pLeft->selFlags |= SF_MultiValue; sqlite3Select(pParse, pLeft, &dest); - p->regResult = dest.iSdst; + p->sq.regResult = dest.iSdst; assert( pParse->nErr || dest.iSdst>0 ); pLeft = pRet; } @@ -755,11 +756,11 @@ Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ if( pParse->nErr==0 ){ assert( p!=0 ); - if( p->pSelect->pEList->nExpr!=pRow->nExpr ){ - sqlite3SelectWrongNumTermsError(pParse, p->pSelect); + if( p->sq.pSelect->pEList->nExpr!=pRow->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, p->sq.pSelect); }else{ - sqlite3ExprCodeExprList(pParse, pRow, p->regResult, 0, 0); - sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, p->regReturn); + sqlite3ExprCodeExprList(pParse, pRow, p->sq.regResult, 0, 0); + sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, p->sq.regReturn); } } sqlite3ExprListDelete(pParse->db, pRow); @@ -1110,9 +1111,9 @@ void sqlite3Insert( && pSelect->pPrior==0 ){ SrcItem *pItem = &pSelect->pSrc->a[0]; - dest.iSDParm = pItem->regReturn; - regFromSelect = pItem->regResult; - nColumn = pItem->pSelect->pEList->nExpr; + dest.iSDParm = pItem->sq.regReturn; + regFromSelect = pItem->sq.regResult; + nColumn = pItem->sq.pSelect->pEList->nExpr; ExplainQueryPlan((pParse, 0, "SCAN %S", pItem)); if( bIdListInOrder && nColumn==pTab->nCol ){ regData = regFromSelect; @@ -3032,7 +3033,7 @@ static int xferOptimization( if( pSelect->pSrc->nSrc!=1 ){ return 0; /* FROM clause must have exactly one term */ } - if( pSelect->pSrc->a[0].pSelect ){ + if( pSelect->pSrc->a[0].sq.pSelect ){ return 0; /* FROM clause cannot contain a subquery */ } if( pSelect->pWhere ){ diff --git a/src/parse.y b/src/parse.y index a605ad6912..9e656d0bc7 100644 --- a/src/parse.y +++ b/src/parse.y @@ -743,8 +743,8 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N assert( pOld->fg.fixedSchema==0 ); pNew->zName = pOld->zName; pNew->u4.zDatabase = pOld->u4.zDatabase; - pNew->pSelect = pOld->pSelect; - if( pNew->pSelect && (pNew->pSelect->selFlags & SF_NestedFrom)!=0 ){ + pNew->sq.pSelect = pOld->sq.pSelect; + if( pNew->sq.pSelect && (pNew->sq.pSelect->selFlags & SF_NestedFrom)!=0 ){ pNew->fg.isNestedFrom = 1; } if( pOld->fg.isTabFunc ){ @@ -754,7 +754,7 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N pNew->fg.isTabFunc = 1; } pOld->zName = pOld->u4.zDatabase = 0; - pOld->pSelect = 0; + pOld->sq.pSelect = 0; } sqlite3SrcListDelete(pParse->db, F); }else{ diff --git a/src/printf.c b/src/printf.c index 12dd91a781..027905d26f 100644 --- a/src/printf.c +++ b/src/printf.c @@ -856,7 +856,7 @@ void sqlite3_str_vappendf( }else if( pItem->zAlias ){ sqlite3_str_appendall(pAccum, pItem->zAlias); }else{ - Select *pSel = pItem->pSelect; + Select *pSel = pItem->sq.pSelect; assert( pSel!=0 ); /* Because of tag-20240424-1 */ if( pSel->selFlags & SF_NestedFrom ){ sqlite3_str_appendf(pAccum, "(join-%u)", pSel->selId); diff --git a/src/resolve.c b/src/resolve.c index dbb54198cb..02a48d6993 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -215,7 +215,7 @@ static void extendFJMatch( if( pNew ){ pNew->iTable = pMatch->iCursor; pNew->iColumn = iColumn; - pNew->y.pTab = pMatch->pTab; + pNew->y.pTab = pMatch->pSTab; assert( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ); ExprSetProperty(pNew, EP_CanBeNull); *ppList = sqlite3ExprListAppend(pParse, *ppList, pNew); @@ -346,10 +346,10 @@ static int lookupName( if( pSrcList ){ for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ u8 hCol; - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 && pTab->zName!=0 ); assert( pTab->nCol>0 || pParse->nErr ); - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->sq.pSelect)); if( pItem->fg.isNestedFrom ){ /* In this case, pItem is a subquery that has been formed from a ** parenthesized subset of the FROM clause terms. Example: @@ -358,8 +358,8 @@ static int lookupName( ** This pItem -------------^ */ int hit = 0; - assert( pItem->pSelect!=0 ); - pEList = pItem->pSelect->pEList; + assert( pItem->sq.pSelect!=0 ); + pEList = pItem->sq.pSelect->pEList; assert( pEList!=0 ); assert( pEList->nExpr==pTab->nCol ); for(j=0; jnExpr; j++){ @@ -483,8 +483,8 @@ static int lookupName( if( cntTab==0 || (cntTab==1 && ALWAYS(pMatch!=0) - && ALWAYS(pMatch->pTab!=0) - && (pMatch->pTab->tabFlags & TF_Ephemeral)!=0 + && ALWAYS(pMatch->pSTab!=0) + && (pMatch->pSTab->tabFlags & TF_Ephemeral)!=0 && (pTab->tabFlags & TF_Ephemeral)==0) ){ cntTab = 1; @@ -505,7 +505,7 @@ static int lookupName( if( pMatch ){ pExpr->iTable = pMatch->iCursor; assert( ExprUseYTab(pExpr) ); - pExpr->y.pTab = pMatch->pTab; + pExpr->y.pTab = pMatch->pSTab; if( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } @@ -547,7 +547,7 @@ static int lookupName( if( (pNC->ncFlags & NC_UUpsert)!=0 && zTab!=0 ){ Upsert *pUpsert = pNC->uNC.pUpsert; if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){ - pTab = pUpsert->pUpsertSrc->a[0].pTab; + pTab = pUpsert->pUpsertSrc->a[0].pSTab; pExpr->iTable = EXCLUDED_TABLE_NUMBER; } } @@ -630,11 +630,11 @@ static int lookupName( && pMatch && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) - && ALWAYS(VisibleRowid(pMatch->pTab) || pMatch->fg.isNestedFrom) + && ALWAYS(VisibleRowid(pMatch->pSTab) || pMatch->fg.isNestedFrom) ){ cnt = cntTab; #if SQLITE_ALLOW_ROWID_IN_VIEW+0==2 - if( pMatch->pTab!=0 && IsView(pMatch->pTab) ){ + if( pMatch->pSTab!=0 && IsView(pMatch->pSTab) ){ eNewExprOp = TK_NULL; } #endif @@ -871,7 +871,7 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ SrcItem *pItem = &pSrc->a[iSrc]; Table *pTab; assert( ExprUseYTab(p) ); - pTab = p->y.pTab = pItem->pTab; + pTab = p->y.pTab = pItem->pSTab; p->iTable = pItem->iCursor; if( p->y.pTab->iPKey==iCol ){ p->iColumn = -1; @@ -990,7 +990,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pItem = pSrcList->a; pExpr->op = TK_COLUMN; assert( ExprUseYTab(pExpr) ); - pExpr->y.pTab = pItem->pTab; + pExpr->y.pTab = pItem->pSTab; pExpr->iTable = pItem->iCursor; pExpr->iColumn--; pExpr->affExpr = SQLITE_AFF_INTEGER; @@ -1880,7 +1880,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** moves the pOrderBy down to the sub-query. It will be moved back ** after the names have been resolved. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].pSelect; + Select *pSub = p->pSrc->a[0].sq.pSelect; assert( p->pSrc->nSrc==1 && p->pOrderBy ); assert( pSub->pPrior && pSub->pOrderBy==0 ); pSub->pOrderBy = p->pOrderBy; @@ -1892,13 +1892,14 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pOuterNC ) pOuterNC->nNestedSelect++; for(i=0; ipSrc->nSrc; i++){ SrcItem *pItem = &p->pSrc->a[i]; - assert( pItem->zName!=0 || pItem->pSelect!=0 );/* Test of tag-20240424-1*/ - if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ + assert( pItem->zName!=0 + || pItem->sq.pSelect!=0 ); /* Test of tag-20240424-1*/ + if( pItem->sq.pSelect && (pItem->sq.pSelect->selFlags & SF_Resolved)==0 ){ int nRef = pOuterNC ? pOuterNC->nRef : 0; const char *zSavedContext = pParse->zAuthContext; if( pItem->zName ) pParse->zAuthContext = pItem->zName; - sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); + sqlite3ResolveSelectNames(pParse, pItem->sq.pSelect, pOuterNC); pParse->zAuthContext = zSavedContext; if( pParse->nErr ) return WRC_Abort; assert( db->mallocFailed==0 ); @@ -2000,7 +2001,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** These integers will be replaced by copies of the corresponding result ** set expressions by the call to resolveOrderGroupBy() below. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].pSelect; + Select *pSub = p->pSrc->a[0].sq.pSelect; p->pOrderBy = pSub->pOrderBy; pSub->pOrderBy = 0; } @@ -2267,7 +2268,7 @@ int sqlite3ResolveSelfReference( if( pTab ){ sSrc.nSrc = 1; sSrc.a[0].zName = pTab->zName; - sSrc.a[0].pTab = pTab; + sSrc.a[0].pSTab = pTab; sSrc.a[0].iCursor = -1; if( pTab->pSchema!=pParse->db->aDb[1].pSchema ){ /* Cause EP_FromDDL to be set on TK_FUNCTION nodes of non-TEMP diff --git a/src/select.c b/src/select.c index 634f06e301..5f55b402cf 100644 --- a/src/select.c +++ b/src/select.c @@ -332,11 +332,11 @@ int sqlite3ColumnIndex(Table *pTab, const char *zCol){ */ void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){ assert( pItem!=0 ); - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->sq.pSelect) ); if( pItem->fg.isNestedFrom ){ ExprList *pResults; - assert( pItem->pSelect!=0 ); - pResults = pItem->pSelect->pEList; + assert( pItem->sq.pSelect!=0 ); + pResults = pItem->sq.pSelect->pEList; assert( pResults!=0 ); assert( iCol>=0 && iColnExpr ); pResults->a[iCol].fg.bUsed = 1; @@ -370,9 +370,9 @@ static int tableAndColumnIndex( assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ for(i=iStart; i<=iEnd; i++){ - iCol = sqlite3ColumnIndex(pSrc->a[i].pTab, zCol); + iCol = sqlite3ColumnIndex(pSrc->a[i].pSTab, zCol); if( iCol>=0 - && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0) + && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pSTab->aCol[iCol])==0) ){ if( piTab ){ sqlite3SrcItemColumnUsed(&pSrc->a[i], iCol); @@ -501,10 +501,10 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ pLeft = &pSrc->a[0]; pRight = &pLeft[1]; for(i=0; inSrc-1; i++, pRight++, pLeft++){ - Table *pRightTab = pRight->pTab; + Table *pRightTab = pRight->pSTab; u32 joinType; - if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue; + if( NEVER(pLeft->pSTab==0 || pRightTab==0) ) continue; joinType = (pRight->fg.jointype & JT_OUTER)!=0 ? EP_OuterON : EP_InnerON; /* If this is a NATURAL join, synthesize an appropriate USING clause @@ -1930,8 +1930,8 @@ static const char *columnTypeImpl( SrcList *pTabList = pNC->pSrcList; for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); if( jnSrc ){ - pTab = pTabList->a[j].pTab; - pS = pTabList->a[j].pSelect; + pTab = pTabList->a[j].pSTab; + pS = pTabList->a[j].sq.pSelect; }else{ pNC = pNC->pNext; } @@ -3983,7 +3983,7 @@ static void substSelect( pSrc = p->pSrc; assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(pSubst, pItem->pSelect, 1); + substSelect(pSubst, pItem->sq.pSelect, 1); if( pItem->fg.isTabFunc ){ substExprList(pSubst, pItem->u1.pFuncArg); } @@ -4014,7 +4014,7 @@ static void recomputeColumnsUsed( SrcItem *pSrcItem /* Which FROM clause item to recompute */ ){ Walker w; - if( NEVER(pSrcItem->pTab==0) ) return; + if( NEVER(pSrcItem->pSTab==0) ) return; memset(&w, 0, sizeof(w)); w.xExprCallback = recomputeColumnsUsedExpr; w.xSelectCallback = sqlite3SelectWalkNoop; @@ -4054,7 +4054,7 @@ static void srclistRenumberCursors( aCsrMap[pItem->iCursor+1] = pParse->nTab++; } pItem->iCursor = aCsrMap[pItem->iCursor+1]; - for(p=pItem->pSelect; p; p=p->pPrior){ + for(p=pItem->sq.pSelect; p; p=p->pPrior){ srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); } } @@ -4366,7 +4366,7 @@ static int flattenSubquery( assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; iParent = pSubitem->iCursor; - pSub = pSubitem->pSelect; + pSub = pSubitem->sq.pSelect; assert( pSub!=0 ); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -4419,7 +4419,7 @@ static int flattenSubquery( */ if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ if( pSubSrc->nSrc>1 /* (3a) */ - || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */ + || IsVirtual(pSubSrc->a[0].pSTab) /* (3b) */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ ){ @@ -4505,7 +4505,7 @@ static int flattenSubquery( pParse->zAuthContext = zSavedAuthContext; /* Delete the transient structures associated with the subquery */ - pSub1 = pSubitem->pSelect; + pSub1 = pSubitem->sq.pSelect; if( pSubitem->fg.fixedSchema==0 ){ sqlite3DbFree(db, pSubitem->u4.zDatabase); pSubitem->u4.zDatabase = 0; @@ -4514,7 +4514,7 @@ static int flattenSubquery( sqlite3DbFree(db, pSubitem->zAlias); pSubitem->zName = 0; pSubitem->zAlias = 0; - pSubitem->pSelect = 0; + pSubitem->sq.pSelect = 0; assert( pSubitem->fg.isUsing!=0 || pSubitem->u3.pOn==0 ); /* If the sub-query is a compound SELECT statement, then (by restrictions @@ -4555,8 +4555,8 @@ static int flattenSubquery( ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; Select *pPrior = p->pPrior; - Table *pItemTab = pSubitem->pTab; - pSubitem->pTab = 0; + Table *pItemTab = pSubitem->pSTab; + pSubitem->pSTab = 0; p->pOrderBy = 0; p->pPrior = 0; p->pLimit = 0; @@ -4564,7 +4564,7 @@ static int flattenSubquery( p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->op = TK_ALL; - pSubitem->pTab = pItemTab; + pSubitem->pSTab = pItemTab; if( pNew==0 ){ p->pPrior = pPrior; }else{ @@ -4579,11 +4579,11 @@ static int flattenSubquery( TREETRACE(0x4,pParse,p,("compound-subquery flattener" " creates %u as peer\n",pNew->selId)); } - assert( pSubitem->pSelect==0 ); + assert( pSubitem->sq.pSelect==0 ); } sqlite3DbFree(db, aCsrMap); if( db->mallocFailed ){ - pSubitem->pSelect = pSub1; + pSubitem->sq.pSelect = pSub1; return 1; } @@ -4594,8 +4594,8 @@ static int flattenSubquery( ** ** pSubitem->pTab is always non-NULL by test restrictions and tests above. */ - if( ALWAYS(pSubitem->pTab!=0) ){ - Table *pTabToDel = pSubitem->pTab; + if( ALWAYS(pSubitem->pSTab!=0) ){ + Table *pTabToDel = pSubitem->pSTab; if( pTabToDel->nTabRef==1 ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel); @@ -4603,7 +4603,7 @@ static int flattenSubquery( }else{ pTabToDel->nTabRef--; } - pSubitem->pTab = 0; + pSubitem->pSTab = 0; } /* The following loop runs once for each term in a compound-subquery @@ -5344,10 +5344,10 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ if( pItem->fg.isCorrelated || pItem->fg.isCte ){ return 0; } - assert( pItem->pTab!=0 ); - pTab = pItem->pTab; - assert( pItem->pSelect!=0 ); - pSub = pItem->pSelect; + assert( pItem->pSTab!=0 ); + pTab = pItem->pSTab; + assert( pItem->sq.pSelect!=0 ); + pSub = pItem->sq.pSelect; assert( pSub->pEList->nExpr==pTab->nCol ); for(pX=pSub; pX; pX=pX->pPrior){ if( (pX->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ @@ -5476,13 +5476,13 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ if( p->pWhere || p->pEList->nExpr!=1 || p->pSrc->nSrc!=1 - || p->pSrc->a[0].pSelect + || p->pSrc->a[0].sq.pSelect || pAggInfo->nFunc!=1 || p->pHaving ){ return 0; } - pTab = p->pSrc->a[0].pTab; + pTab = p->pSrc->a[0].pSTab; assert( pTab!=0 ); assert( !IsView(pTab) ); if( !IsOrdinaryTable(pTab) ) return 0; @@ -5507,7 +5507,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ ** pFrom->pIndex and return SQLITE_OK. */ int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ - Table *pTab = pFrom->pTab; + Table *pTab = pFrom->pSTab; char *zIndexedBy = pFrom->u1.zIndexedBy; Index *pIdx; assert( pTab!=0 ); @@ -5709,7 +5709,7 @@ static int resolveFromTermToCte( Cte *pCte; /* Matched CTE (or NULL if no match) */ With *pWith; /* The matching WITH */ - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); if( pParse->pWith==0 ){ /* There are no WITH clauses in the stack. No match is possible */ return 0; @@ -5756,7 +5756,7 @@ static int resolveFromTermToCte( } if( cannotBeFunction(pParse, pFrom) ) return 2; - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return 2; pCteUse = pCte->pUse; @@ -5770,16 +5770,16 @@ static int resolveFromTermToCte( } pCteUse->eM10d = pCte->eM10d; } - pFrom->pTab = pTab; + pFrom->pSTab = pTab; pTab->nTabRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; - pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + pFrom->sq.pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return 2; - pFrom->pSelect->selFlags |= SF_CopyCte; - assert( pFrom->pSelect ); + assert( pFrom->sq.pSelect ); + pFrom->sq.pSelect->selFlags |= SF_CopyCte; if( pFrom->fg.isIndexedBy ){ sqlite3ErrorMsg(pParse, "no such index: \"%s\"", pFrom->u1.zIndexedBy); return 2; @@ -5789,7 +5789,7 @@ static int resolveFromTermToCte( pCteUse->nUse++; /* Check if this is a recursive CTE. */ - pRecTerm = pSel = pFrom->pSelect; + pRecTerm = pSel = pFrom->sq.pSelect; bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); while( bMayRecursive && pRecTerm->op==pSel->op ){ int i; @@ -5802,7 +5802,7 @@ static int resolveFromTermToCte( && pItem->zName!=0 && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ - pItem->pTab = pTab; + pItem->pSTab = pTab; pTab->nTabRef++; pItem->fg.isRecursive = 1; if( pRecTerm->selFlags & SF_Recursive ){ @@ -5904,11 +5904,11 @@ void sqlite3SelectPopWith(Walker *pWalker, Select *p){ ** SQLITE_NOMEM. */ int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ - Select *pSel = pFrom->pSelect; + Select *pSel = pFrom->sq.pSelect; Table *pTab; assert( pSel ); - pFrom->pTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); + pFrom->pSTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); if( pTab==0 ) return SQLITE_NOMEM; pTab->nTabRef = 1; if( pFrom->zAlias ){ @@ -6028,33 +6028,33 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); - if( pFrom->pTab ) continue; + assert( pFrom->fg.isRecursive==0 || pFrom->pSTab!=0 ); + if( pFrom->pSTab ) continue; assert( pFrom->fg.isRecursive==0 ); if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY - Select *pSel = pFrom->pSelect; + Select *pSel = pFrom->sq.pSelect; /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort; #endif #ifndef SQLITE_OMIT_CTE }else if( (rc = resolveFromTermToCte(pParse, pWalker, pFrom))!=0 ){ if( rc>1 ) return WRC_Abort; - pTab = pFrom->pTab; + pTab = pFrom->pSTab; assert( pTab!=0 ); #endif }else{ /* An ordinary table or view name in the FROM clause */ - assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); + assert( pFrom->pSTab==0 ); + pFrom->pSTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; if( pTab->nTabRef>=0xffff ){ sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", pTab->zName); - pFrom->pTab = 0; + pFrom->pSTab = 0; return WRC_Abort; } pTab->nTabRef++; @@ -6066,7 +6066,7 @@ static int selectExpander(Walker *pWalker, Select *p){ i16 nCol; u8 eCodeOrig = pWalker->eCode; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; - assert( pFrom->pSelect==0 ); + assert( pFrom->sq.pSelect==0 ); if( IsView(pTab) ){ if( (db->flags & SQLITE_EnableView)==0 && pTab->pSchema!=db->aDb[1].pSchema @@ -6074,7 +6074,7 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited", pTab->zName); } - pFrom->pSelect = sqlite3SelectDup(db, pTab->u.view.pSelect, 0); + pFrom->sq.pSelect = sqlite3SelectDup(db, pTab->u.view.pSelect, 0); } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( ALWAYS(IsVirtual(pTab)) @@ -6090,7 +6090,7 @@ static int selectExpander(Walker *pWalker, Select *p){ nCol = pTab->nCol; pTab->nCol = -1; pWalker->eCode = 1; /* Turn on Select.selId renumbering */ - sqlite3WalkSelect(pWalker, pFrom->pSelect); + sqlite3WalkSelect(pWalker, pFrom->sq.pSelect); pWalker->eCode = eCodeOrig; pTab->nCol = nCol; } @@ -6177,7 +6177,7 @@ static int selectExpander(Walker *pWalker, Select *p){ } for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ int nAdd; /* Number of cols including rowid */ - Table *pTab = pFrom->pTab; /* Table for this data source */ + Table *pTab = pFrom->pSTab; /* Table for this data source */ ExprList *pNestedFrom; /* Result-set of a nested FROM clause */ char *zTabName; /* AS name for this data source */ const char *zSchemaName = 0; /* Schema name for this data source */ @@ -6188,10 +6188,11 @@ static int selectExpander(Walker *pWalker, Select *p){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) ); + assert( (int)pFrom->fg.isNestedFrom == + IsNestedFrom(pFrom->sq.pSelect) ); if( pFrom->fg.isNestedFrom ){ - assert( pFrom->pSelect!=0 ); - pNestedFrom = pFrom->pSelect->pEList; + assert( pFrom->sq.pSelect!=0 ); + pNestedFrom = pFrom->sq.pSelect->pEList; assert( pNestedFrom!=0 ); assert( pNestedFrom->nExpr==pTab->nCol ); assert( VisibleRowid(pTab)==0 || ViewCanHaveRowid ); @@ -6430,11 +6431,11 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ assert( (p->selFlags & SF_Resolved) ); pTabList = p->pSrc; for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; + Table *pTab = pFrom->pSTab; assert( pTab!=0 ); if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; + Select *pSel = pFrom->sq.pSelect; if( pSel ){ sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); } @@ -7084,25 +7085,25 @@ static SrcItem *isSelfJoinView( int iFirst, int iEnd /* Range of FROM-clause entries to search. */ ){ SrcItem *pItem; - assert( pThis->pSelect!=0 ); - if( pThis->pSelect->selFlags & SF_PushDown ) return 0; + assert( pThis->sq.pSelect!=0 ); + if( pThis->sq.pSelect->selFlags & SF_PushDown ) return 0; while( iFirsta[iFirst++]; - if( pItem->pSelect==0 ) continue; + if( pItem->sq.pSelect==0 ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; - assert( pItem->pTab!=0 ); - assert( pThis->pTab!=0 ); - if( pItem->pTab->pSchema!=pThis->pTab->pSchema ) continue; + assert( pItem->pSTab!=0 ); + assert( pThis->pSTab!=0 ); + if( pItem->pSTab->pSchema!=pThis->pSTab->pSchema ) continue; if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; - pS1 = pItem->pSelect; - if( pItem->pTab->pSchema==0 && pThis->pSelect->selId!=pS1->selId ){ + pS1 = pItem->sq.pSelect; + if( pItem->pSTab->pSchema==0 && pThis->sq.pSelect->selId!=pS1->selId ){ /* The query flattener left two different CTE tables with identical ** names in the same FROM clause. */ continue; } - if( pItem->pSelect->selFlags & SF_PushDown ){ + if( pItem->sq.pSelect->selFlags & SF_PushDown ){ /* The view was modified by some other optimization such as ** pushDownWhereTerms() */ continue; @@ -7160,7 +7161,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */ if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */ if( ExprHasProperty(pExpr, EP_WinFunc) ) return 0;/* Not a window function */ - pSub = p->pSrc->a[0].pSelect; + pSub = p->pSrc->a[0].sq.pSelect; if( pSub==0 ) return 0; /* The FROM is a subquery */ if( pSub->pPrior==0 ) return 0; /* Must be a compound */ if( pSub->selFlags & SF_CopyCte ) return 0; /* Not a CTE */ @@ -7178,8 +7179,8 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ db = pParse->db; pCount = pExpr; pExpr = 0; - pSub = p->pSrc->a[0].pSelect; - p->pSrc->a[0].pSelect = 0; + pSub = p->pSrc->a[0].sq.pSelect; + p->pSrc->a[0].sq.pSelect = 0; sqlite3SrcListDelete(db, p->pSrc); p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc)); while( pSub ){ @@ -7224,12 +7225,12 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ for(i=0; inSrc; i++){ SrcItem *p1 = &pSrc->a[i]; if( p1==p0 ) continue; - if( p0->pTab==p1->pTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ + if( p0->pSTab==p1->pSTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ return 1; } - if( p1->pSelect - && (p1->pSelect->selFlags & SF_NestedFrom)!=0 - && sameSrcAlias(p0, p1->pSelect->pSrc) + if( p1->sq.pSelect + && (p1->sq.pSelect->selFlags & SF_NestedFrom)!=0 + && sameSrcAlias(p0, p1->sq.pSelect->pSrc) ){ return 1; } @@ -7294,7 +7295,7 @@ static int fromClauseTermCanBeCoroutine( if( i==0 ) break; i--; pItem--; - if( pItem->pSelect!=0 ) return 0; /* (1c-i) */ + if( pItem->sq.pSelect!=0 ) return 0; /* (1c-i) */ } return 1; } @@ -7405,7 +7406,7 @@ int sqlite3Select( if( sameSrcAlias(p0, p->pSrc) ){ sqlite3ErrorMsg(pParse, "target object/alias may not appear in FROM clause: %s", - p0->zAlias ? p0->zAlias : p0->pTab->zName + p0->zAlias ? p0->zAlias : p0->pSTab->zName ); goto select_end; } @@ -7444,8 +7445,8 @@ int sqlite3Select( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ SrcItem *pItem = &pTabList->a[i]; - Select *pSub = pItem->pSelect; - Table *pTab = pItem->pTab; + Select *pSub = pItem->sq.pSelect; + Table *pTab = pItem->pSTab; /* The expander should have already created transient Table objects ** even for FROM clause elements such as subqueries that do not correspond @@ -7698,11 +7699,11 @@ int sqlite3Select( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Generate code for all sub-queries in the FROM clause */ - pSub = pItem->pSelect; - if( pSub==0 || pItem->addrFillSub!=0 ) continue; + pSub = pItem->sq.pSelect; + if( pSub==0 || pItem->sq.addrFillSub!=0 ) continue; /* The code for a subquery should only be generated once. */ - assert( pItem->addrFillSub==0 ); + assert( pItem->sq.addrFillSub==0 ); /* Increment Parse.nHeight by the height of the largest expression ** tree referred to by this, the parent select. The child select @@ -7728,7 +7729,8 @@ int sqlite3Select( sqlite3TreeViewSelect(0, p, 0); } #endif - assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); + assert( pItem->sq.pSelect + && (pItem->sq.pSelect->selFlags & SF_PushDown)!=0 ); }else{ TREETRACE(0x4000,pParse,p,("WHERE-lcause push-down not possible\n")); } @@ -7760,17 +7762,17 @@ int sqlite3Select( */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; - pItem->regReturn = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); + pItem->sq.regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->sq.regReturn, 0, addrTop); VdbeComment((v, "%!S", pItem)); - pItem->addrFillSub = addrTop; - sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + pItem->sq.addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->sq.regReturn); ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pSTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; - pItem->regResult = dest.iSdst; - sqlite3VdbeEndCoroutine(v, pItem->regReturn); + pItem->sq.regResult = dest.iSdst; + sqlite3VdbeEndCoroutine(v, pItem->sq.regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ @@ -7788,11 +7790,12 @@ int sqlite3Select( }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ /* This view has already been materialized by a prior entry in ** this same FROM clause. Reuse it. */ - if( pPrior->addrFillSub ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub); + if( pPrior->sq.addrFillSub ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->sq.regReturn, + pPrior->sq.addrFillSub); } sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); - pSub->nSelectRow = pPrior->pSelect->nSelectRow; + pSub->nSelectRow = pPrior->sq.pSelect->nSelectRow; }else{ /* Materialize the view. If the view is not correlated, generate a ** subroutine to do the materialization so that subsequent uses of @@ -7803,9 +7806,9 @@ int sqlite3Select( int addrExplain; #endif - pItem->regReturn = ++pParse->nMem; + pItem->sq.regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp0(v, OP_Goto); - pItem->addrFillSub = topAddr+1; + pItem->sq.addrFillSub = topAddr+1; pItem->fg.isMaterialized = 1; if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of @@ -7820,17 +7823,17 @@ int sqlite3Select( ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pSTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); - sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); + sqlite3VdbeAddOp2(v, OP_Return, pItem->sq.regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ CteUse *pCteUse = pItem->u2.pCteUse; - pCteUse->addrM9e = pItem->addrFillSub; - pCteUse->regRtn = pItem->regReturn; + pCteUse->addrM9e = pItem->sq.addrFillSub; + pCteUse->regRtn = pItem->sq.regReturn; pCteUse->iCur = pItem->iCursor; pCteUse->nRowEst = pSub->nSelectRow; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 75e1f61201..54e4f2db73 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3295,15 +3295,23 @@ struct IdList { ** ** u2.pIBIndex fg.isIndexedBy && !fg.isCte ** u2.pCteUse fg.isCte && !fg.isIndexedBy +** +** u3.pOn fg.isUsing==0 +** u3.pUsing fg.isUsing==1 +** +** u4.zDatabase fg.fixedSchema==0 +** u4.pSchema fg.fixedSchema==1 */ struct SrcItem { char *zName; /* Name of the table */ char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ - Table *pTab; /* An SQL table corresponding to zName */ - Select *pSelect; /* A SELECT statement used in place of a table name */ - int addrFillSub; /* Address of subroutine to initialize a subquery */ - int regReturn; /* Register holding return address of addrFillSub */ - int regResult; /* Registers holding results of a co-routine */ + Table *pSTab; /* Table object for zName. Mnemonic: Srcitem-TABle */ + struct SrcItemSubquery { + Select *pSelect; /* A SELECT statement used in place of a table name */ + int addrFillSub; /* Address of subroutine to initialize a subquery */ + int regReturn; /* Register holding return address of addrFillSub */ + int regResult; /* Registers holding results of a co-routine */ + } sq; struct { u8 jointype; /* Type of join between this table and the previous */ unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ diff --git a/src/treeview.c b/src/treeview.c index 3960d2859e..0c9fdc5266 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -193,9 +193,9 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); x.printfFlags |= SQLITE_PRINTF_INTERNAL; sqlite3_str_appendf(&x, "{%d:*} %!S", pItem->iCursor, pItem); - if( pItem->pTab ){ + if( pItem->pSTab ){ sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx%s", - pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, + pItem->pSTab->zName, pItem->pSTab->nCol, pItem->pSTab, pItem->colUsed, pItem->fg.rowidUsed ? "+rowid" : ""); } @@ -230,19 +230,19 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ sqlite3StrAccumFinish(&x); sqlite3TreeViewItem(pView, zLine, inSrc-1); n = 0; - if( pItem->pSelect ) n++; + if( pItem->sq.pSelect ) n++; if( pItem->fg.isTabFunc ) n++; if( pItem->fg.isUsing ) n++; if( pItem->fg.isUsing ){ sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); } - if( pItem->pSelect ){ - if( pItem->pTab ){ - Table *pTab = pItem->pTab; + if( pItem->sq.pSelect ){ + if( pItem->pSTab ){ + Table *pTab = pItem->pSTab; sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); } - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); - sqlite3TreeViewSelect(pView, pItem->pSelect, (--n)>0); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->sq.pSelect) ); + sqlite3TreeViewSelect(pView, pItem->sq.pSelect, (--n)>0); } if( pItem->fg.isTabFunc ){ sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); diff --git a/src/trigger.c b/src/trigger.c index a5a57abf16..170f094eab 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -984,7 +984,7 @@ static int sqlite3ReturningSubqueryCorrelated(Walker *pWalker, Select *pSelect){ pSrc = pSelect->pSrc; assert( pSrc!=0 ); for(i=0; inSrc; i++){ - if( pSrc->a[i].pTab==pWalker->u.pTab ){ + if( pSrc->a[i].pSTab==pWalker->u.pTab ){ testcase( pSelect->selFlags & SF_Correlated ); pSelect->selFlags |= SF_Correlated; pWalker->eCode = 1; @@ -1055,7 +1055,7 @@ static void codeReturningTrigger( sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); sSelect.pSrc = &sFrom; sFrom.nSrc = 1; - sFrom.a[0].pTab = pTab; + sFrom.a[0].pSTab = pTab; sFrom.a[0].zName = pTab->zName; /* tag-20240424-1 */ sFrom.a[0].iCursor = -1; sqlite3SelectPrep(pParse, &sSelect, 0); diff --git a/src/update.c b/src/update.c index b6068caa79..a8e7f77803 100644 --- a/src/update.c +++ b/src/update.c @@ -202,7 +202,7 @@ static void updateFromSelect( Expr *pLimit2 = 0; ExprList *pOrderBy2 = 0; sqlite3 *db = pParse->db; - Table *pTab = pTabList->a[0].pTab; + Table *pTab = pTabList->a[0].pSTab; SrcList *pSrc; Expr *pWhere2; int eDest; @@ -226,8 +226,8 @@ static void updateFromSelect( if( pSrc ){ assert( pSrc->a[0].fg.notCte ); pSrc->a[0].iCursor = -1; - pSrc->a[0].pTab->nTabRef--; - pSrc->a[0].pTab = 0; + pSrc->a[0].pSTab->nTabRef--; + pSrc->a[0].pSTab = 0; } if( pPk ){ for(i=0; inKeyCol; i++){ diff --git a/src/upsert.c b/src/upsert.c index f74d4fabf5..82295d52ae 100644 --- a/src/upsert.c +++ b/src/upsert.c @@ -104,7 +104,7 @@ int sqlite3UpsertAnalyzeTarget( int nClause = 0; /* Counter of ON CONFLICT clauses */ assert( pTabList->nSrc==1 ); - assert( pTabList->a[0].pTab!=0 ); + assert( pTabList->a[0].pSTab!=0 ); assert( pUpsert!=0 ); assert( pUpsert->pUpsertTarget!=0 ); @@ -123,7 +123,7 @@ int sqlite3UpsertAnalyzeTarget( if( rc ) return rc; /* Check to see if the conflict target matches the rowid. */ - pTab = pTabList->a[0].pTab; + pTab = pTabList->a[0].pSTab; pTarget = pUpsert->pUpsertTarget; iCursor = pTabList->a[0].iCursor; if( HasRowid(pTab) diff --git a/src/walker.c b/src/walker.c index 0fe4a1d379..27963c2234 100644 --- a/src/walker.c +++ b/src/walker.c @@ -171,7 +171,7 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ pSrc = p->pSrc; if( ALWAYS(pSrc) ){ for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){ + if( pItem->sq.pSelect && sqlite3WalkSelect(pWalker, pItem->sq.pSelect) ){ return WRC_Abort; } if( pItem->fg.isTabFunc diff --git a/src/where.c b/src/where.c index 631b7581fb..ded5f06816 100644 --- a/src/where.c +++ b/src/where.c @@ -644,7 +644,7 @@ static int isDistinctRedundant( ** clause is redundant. */ if( pTabList->nSrc!=1 ) return 0; iBase = pTabList->a[0].iCursor; - pTab = pTabList->a[0].pTab; + pTab = pTabList->a[0].pSTab; /* If any of the expressions is an IPK column on table iBase, then return ** true. Note: The (p->iTable==iBase) part of this test may be false if the @@ -908,10 +908,10 @@ static int termCanDriveIndex( assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); leftCol = pTerm->u.x.leftColumn; if( leftCol<0 ) return 0; - aff = pSrc->pTab->aCol[leftCol].affinity; + aff = pSrc->pSTab->aCol[leftCol].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; testcase( pTerm->pExpr->op==TK_IS ); - return columnIsGoodIndexCandidate(pSrc->pTab, leftCol); + return columnIsGoodIndexCandidate(pSrc->pSTab, leftCol); } #endif @@ -1019,7 +1019,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( nKeyCol = 0; pTabList = pWC->pWInfo->pTabList; pSrc = &pTabList->a[pLevel->iFrom]; - pTable = pSrc->pTab; + pTable = pSrc->pSTab; pWCEnd = &pWC->a[pWC->nTerm]; pLoop = pLevel->pWLoop; idxCols = 0; @@ -1161,12 +1161,12 @@ static SQLITE_NOINLINE void constructAutomaticIndex( /* Fill the automatic index with content */ assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] ); if( pSrc->fg.viaCoroutine ){ - int regYield = pSrc->regReturn; + int regYield = pSrc->sq.regReturn; addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->sq.addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); - VdbeComment((v, "next row of %s", pSrc->pTab->zName)); + VdbeComment((v, "next row of %s", pSrc->pSTab->zName)); }else{ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); } @@ -1192,7 +1192,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( testcase( pParse->db->mallocFailed ); assert( pLevel->iIdxCur>0 ); translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, - pSrc->regResult, pLevel->iIdxCur); + pSrc->sq.regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); pSrc->fg.viaCoroutine = 0; }else{ @@ -1283,7 +1283,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( iSrc = pLevel->iFrom; pItem = &pTabList->a[iSrc]; assert( pItem!=0 ); - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 ); sz = sqlite3LogEstToInt(pTab->nRowLogEst); if( sz<10000 ){ @@ -1314,7 +1314,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( int r1 = sqlite3GetTempRange(pParse, n); int jj; for(jj=0; jjpTable==pItem->pTab ); + assert( pIdx->pTable==pItem->pSTab ); sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj); } sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); @@ -1395,7 +1395,7 @@ static sqlite3_index_info *allocateIndexInfo( WhereClause *p; assert( pSrc!=0 ); - pTab = pSrc->pTab; + pTab = pSrc->pSTab; assert( pTab!=0 ); assert( IsVirtual(pTab) ); @@ -2403,7 +2403,7 @@ void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ WhereInfo *pWInfo = pWC->pWInfo; int nb = 1+(pWInfo->pTabList->nSrc+3)/4; SrcItem *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pTab; + Table *pTab = pItem->pSTab; Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); @@ -3391,7 +3391,7 @@ static int whereLoopAddBtreeIndex( ** 2. Stepping forward in the index pNew->nOut times to find all ** additional matching entries. */ - assert( pSrc->pTab->szTabRow>0 ); + assert( pSrc->pSTab->szTabRow>0 ); if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ /* The pProbe->szIdxRow is low for an IPK table since the interior ** pages are small. Thus szIdxRow gives a good estimate of seek cost. @@ -3399,7 +3399,7 @@ static int whereLoopAddBtreeIndex( ** under-estimate the scanning cost. */ rCostIdx = pNew->nOut + 16; }else{ - rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pSTab->szTabRow; } rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx); @@ -3864,9 +3864,9 @@ static int whereLoopAddBtree( pWInfo = pBuilder->pWInfo; pTabList = pWInfo->pTabList; pSrc = pTabList->a + pNew->iTab; - pTab = pSrc->pTab; + pTab = pSrc->pSTab; pWC = pBuilder->pWC; - assert( !IsVirtual(pSrc->pTab) ); + assert( !IsVirtual(pSrc->pSTab) ); if( pSrc->fg.isIndexedBy ){ assert( pSrc->fg.isCte==0 ); @@ -3891,7 +3891,7 @@ static int whereLoopAddBtree( sPk.idxType = SQLITE_IDXTYPE_IPK; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; - pFirst = pSrc->pTab->pIndex; + pFirst = pSrc->pSTab->pIndex; if( pSrc->fg.notIndexed==0 ){ /* The real indices of the table are only considered if the ** NOT INDEXED qualifier is omitted from the FROM clause */ @@ -4010,9 +4010,9 @@ static int whereLoopAddBtree( #endif ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); - if( pSrc->pSelect ){ + if( pSrc->sq.pSelect ){ if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; - pNew->u.btree.pOrderBy = pSrc->pSelect->pOrderBy; + pNew->u.btree.pOrderBy = pSrc->sq.pSelect->pOrderBy; } rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4238,7 +4238,7 @@ static int whereLoopAddVirtualOne( pHidden->mHandleIn = 0; /* Invoke the virtual table xBestIndex() method */ - rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); + rc = vtabBestIndex(pParse, pSrc->pSTab, pIdxInfo); if( rc ){ if( rc==SQLITE_CONSTRAINT ){ /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means @@ -4268,7 +4268,7 @@ static int whereLoopAddVirtualOne( || pNew->aLTerm[iTerm]!=0 || pIdxCons->usable==0 ){ - sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName); freeIdxStr(pIdxInfo); return SQLITE_ERROR; } @@ -4331,7 +4331,7 @@ static int whereLoopAddVirtualOne( if( pNew->aLTerm[i]==0 ){ /* The non-zero argvIdx values must be contiguous. Raise an ** error if they are not */ - sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName); freeIdxStr(pIdxInfo); return SQLITE_ERROR; } @@ -4533,7 +4533,7 @@ static int whereLoopAddVirtual( pWC = pBuilder->pWC; pNew = pBuilder->pNew; pSrc = &pWInfo->pTabList->a[pNew->iTab]; - assert( IsVirtual(pSrc->pTab) ); + assert( IsVirtual(pSrc->pSTab) ); p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit); if( p==0 ) return SQLITE_NOMEM_BKPT; pNew->rSetup = 0; @@ -4547,7 +4547,7 @@ static int whereLoopAddVirtual( } /* First call xBestIndex() with all constraints usable. */ - WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName)); + WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pSTab->zName)); WHERETRACE(0x800, (" VirtualOne: all usable\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry @@ -4629,7 +4629,7 @@ static int whereLoopAddVirtual( } freeIndexInfo(pParse->db, p); - WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc)); + WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pSTab->zName, rc)); return rc; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -4701,7 +4701,7 @@ static int whereLoopAddOr( } #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pItem->pTab) ){ + if( IsVirtual(pItem->pSTab) ){ rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable); }else #endif @@ -4815,7 +4815,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ mPrereq = 0; } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pItem->pTab) ){ + if( IsVirtual(pItem->pSTab) ){ SrcItem *p; for(p=&pItem[1]; pfg.jointype & (JT_OUTER|JT_CROSS)) ){ @@ -5451,7 +5451,7 @@ static int computeMxChoice(WhereInfo *pWInfo, LogEst nRowEst){ if( sqlite3WhereTrace&0x4 ){ SrcItem *pItem = pWInfo->pTabList->a + iLoop; sqlite3DebugPrintf("Fact-table %s: %d dimensions, cost reduced %d\n", - pItem->zAlias ? pItem->zAlias : pItem->pTab->zName, + pItem->zAlias ? pItem->zAlias : pItem->pSTab->zName, nDep, rDelta); } #endif @@ -6001,7 +6001,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0; assert( pWInfo->pTabList->nSrc>=1 ); pItem = pWInfo->pTabList->a; - pTab = pItem->pTab; + pTab = pItem->pSTab; if( IsVirtual(pTab) ) return 0; if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){ testcase( pItem->fg.isIndexedBy ); @@ -6264,7 +6264,7 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( WhereLoop *pLoop = pWInfo->a[i].pWLoop; const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ); SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; - Table *pTab = pItem->pTab; + Table *pTab = pItem->pSTab; if( (pTab->tabFlags & TF_HasStat1)==0 ) break; pTab->tabFlags |= TF_MaybeReanalyze; if( i>=1 @@ -6421,8 +6421,8 @@ static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){ SrcItem *pItem = &pWInfo->pTabList->a[ii]; if( !pItem->fg.isCte || pItem->u2.pCteUse->eM10d!=M10d_Yes - || NEVER(pItem->pSelect==0) - || pItem->pSelect->pOrderBy==0 + || NEVER(pItem->sq.pSelect==0) + || pItem->sq.pSelect->pOrderBy==0 ){ pWInfo->revMask |= MASKBIT(ii); } @@ -6912,15 +6912,15 @@ WhereInfo *sqlite3WhereBegin( if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ int wsFlags = pWInfo->a[0].pWLoop->wsFlags; int bOnerow = (wsFlags & WHERE_ONEROW)!=0; - assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) ); + assert( !(wsFlags&WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pSTab) ); if( bOnerow || ( 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) - && !IsVirtual(pTabList->a[0].pTab) + && !IsVirtual(pTabList->a[0].pSTab) && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) && OptimizationEnabled(db, SQLITE_OnePass) )){ pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; - if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ + if( HasRowid(pTabList->a[0].pSTab) && (wsFlags & WHERE_IDX_ONLY) ){ if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ bFordelete = OPFLAG_FORDELETE; } @@ -6938,7 +6938,7 @@ WhereInfo *sqlite3WhereBegin( SrcItem *pTabItem; pTabItem = &pTabList->a[pLevel->iFrom]; - pTab = pTabItem->pTab; + pTab = pTabItem->pSTab; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pLoop = pLevel->pWLoop; if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){ @@ -7009,7 +7009,7 @@ WhereInfo *sqlite3WhereBegin( iIndexCur = pLevel->iTabCur; op = 0; }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ - Index *pJ = pTabItem->pTab->pIndex; + Index *pJ = pTabItem->pSTab->pIndex; iIndexCur = iAuxArg; assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); while( ALWAYS(pJ) && pJ!=pIx ){ @@ -7076,7 +7076,7 @@ WhereInfo *sqlite3WhereBegin( sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom); pRJ->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn); - assert( pTab==pTabItem->pTab ); + assert( pTab==pTabItem->pSTab ); if( HasRowid(pTab) ){ KeyInfo *pInfo; sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1); @@ -7116,10 +7116,10 @@ WhereInfo *sqlite3WhereBegin( pSrc = &pTabList->a[pLevel->iFrom]; if( pSrc->fg.isMaterialized ){ if( pSrc->fg.isCorrelated ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); + sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->sq.regReturn,pSrc->sq.addrFillSub); }else{ int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); + sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->sq.regReturn,pSrc->sq.addrFillSub); sqlite3VdbeJumpHere(v, iOnce); } } @@ -7334,9 +7334,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ assert( pLevel->iTabCur==pSrc->iCursor ); if( pSrc->fg.viaCoroutine ){ int m, n; - n = pSrc->regResult; - assert( pSrc->pTab!=0 ); - m = pSrc->pTab->nCol; + n = pSrc->sq.regResult; + assert( pSrc->pSTab!=0 ); + m = pSrc->pSTab->nCol; sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1); } sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); @@ -7360,7 +7360,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeJumpHere(v, addr); } VdbeModuleComment((v, "End WHERE-loop%d: %s", i, - pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); + pWInfo->pTabList->a[pLevel->iFrom].pSTab->zName)); } assert( pWInfo->nLevel<=pTabList->nSrc ); @@ -7369,7 +7369,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeOp *pOp, *pLastOp; Index *pIdx = 0; SrcItem *pTabItem = &pTabList->a[pLevel->iFrom]; - Table *pTab = pTabItem->pTab; + Table *pTab = pTabItem->pSTab; assert( pTab!=0 ); pLoop = pLevel->pWLoop; @@ -7388,9 +7388,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ */ if( pTabItem->fg.viaCoroutine ){ testcase( pParse->db->mallocFailed ); - assert( pTabItem->regResult>=0 ); + assert( pTabItem->sq.regResult>=0 ); translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, - pTabItem->regResult, 0); + pTabItem->sq.regResult, 0); continue; } diff --git a/src/wherecode.c b/src/wherecode.c index 098af7375e..ebc71d1102 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -157,7 +157,7 @@ int sqlite3WhereExplainOneScan( assert( pLoop->u.btree.pIndex!=0 ); pIdx = pLoop->u.btree.pIndex; assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); - if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + if( !HasRowid(pItem->pSTab) && IsPrimaryKeyIndex(pIdx) ){ if( isSearch ){ zFmt = "PRIMARY KEY"; } @@ -254,7 +254,7 @@ int sqlite3WhereExplainBloomFilter( sqlite3_str_appendf(&str, "BLOOM FILTER ON %S (", pItem); pLoop = pLevel->pWLoop; if( pLoop->wsFlags & WHERE_IPK ){ - const Table *pTab = pItem->pTab; + const Table *pTab = pItem->pSTab; if( pTab->iPKey>=0 ){ sqlite3_str_appendf(&str, "%s=?", pTab->aCol[pTab->iPKey].zCnName); }else{ @@ -317,7 +317,7 @@ void sqlite3WhereAddScanStatus( sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); } }else{ - int addr = pSrclist->a[pLvl->iFrom].addrFillSub; + int addr = pSrclist->a[pLvl->iFrom].sq.addrFillSub; VdbeOp *pOp = sqlite3VdbeGetOp(v, addr-1); assert( sqlite3VdbeDb(v)->mallocFailed || pOp->opcode==OP_InitCoroutine ); assert( sqlite3VdbeDb(v)->mallocFailed || pOp->p2>addr ); @@ -1454,7 +1454,8 @@ Bitmask sqlite3WhereCodeOneLoopStart( iCur = pTabItem->iCursor; pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); bRev = (pWInfo->revMask>>iLevel)&1; - VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); + VdbeModuleComment((v, "Begin WHERE-loop%d: %s", + iLevel, pTabItem->pSTab->zName)); #if WHERETRACE_ENABLED /* 0x4001 */ if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("Coding level %d of %d: notReady=%llx iFrom=%d\n", @@ -1509,11 +1510,11 @@ Bitmask sqlite3WhereCodeOneLoopStart( /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->regReturn; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + int regYield = pTabItem->sq.regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield,0,pTabItem->sq.addrFillSub); pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); VdbeCoverage(v); - VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pTabItem->pSTab->zName)); pLevel->op = OP_Goto; }else @@ -2242,7 +2243,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - Table *pTab = pTabItem->pTab; + Table *pTab = pTabItem->pSTab; pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); @@ -2701,7 +2702,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** least once. This is accomplished by storing the PK for the row in ** both the iMatch index and the regBloom Bloom filter. */ - pTab = pWInfo->pTabList->a[pLevel->iFrom].pTab; + pTab = pWInfo->pTabList->a[pLevel->iFrom].pSTab; if( HasRowid(pTab) ){ r = sqlite3GetTempRange(pParse, 2); sqlite3ExprCodeGetColumnOfTable(v, pTab, pLevel->iTabCur, -1, r+1); @@ -2808,7 +2809,7 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( Bitmask mAll = 0; int k; - ExplainQueryPlan((pParse, 1, "RIGHT-JOIN %s", pTabItem->pTab->zName)); + ExplainQueryPlan((pParse, 1, "RIGHT-JOIN %s", pTabItem->pSTab->zName)); sqlite3VdbeNoJumpsOutsideSubrtn(v, pRJ->addrSubrtn, pRJ->endSubrtn, pRJ->regReturn); for(k=0; ka[k].pWLoop->maskSelf; if( pRight->fg.viaCoroutine ){ sqlite3VdbeAddOp3( - v, OP_Null, 0, pRight->regResult, - pRight->regResult + pRight->pSelect->pEList->nExpr-1 + v, OP_Null, 0, pRight->sq.regResult, + pRight->sq.regResult + pRight->sq.pSelect->pEList->nExpr-1 ); } sqlite3VdbeAddOp1(v, OP_NullRow, pWInfo->a[k].iTabCur); @@ -2858,7 +2859,7 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( int nPk; int jmp; int addrCont = sqlite3WhereContinueLabel(pSubWInfo); - Table *pTab = pTabItem->pTab; + Table *pTab = pTabItem->pSTab; if( HasRowid(pTab) ){ sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, r); nPk = 1; diff --git a/src/whereexpr.c b/src/whereexpr.c index dcda75d264..766754828d 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -958,7 +958,7 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ if( ALWAYS(pSrc!=0) ){ int i; for(i=0; inSrc; i++){ - mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); + mask |= exprSelectUsage(pMaskSet, pSrc->a[i].sq.pSelect); if( pSrc->a[i].fg.isUsing==0 ){ mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].u3.pOn); } @@ -996,7 +996,7 @@ static SQLITE_NOINLINE int exprMightBeIndexed2( int iCur; do{ iCur = pFrom->a[j].iCursor; - for(pIdx=pFrom->a[j].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + for(pIdx=pFrom->a[j].pSTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->aColExpr==0 ) continue; for(i=0; inKeyCol; i++){ if( pIdx->aiColumn[i]!=XN_EXPR ) continue; @@ -1040,7 +1040,7 @@ static int exprMightBeIndexed( for(i=0; inSrc; i++){ Index *pIdx; - for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + for(pIdx=pFrom->a[i].pSTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->aColExpr ){ return exprMightBeIndexed2(pFrom,aiCurCol,pExpr,i); } @@ -1628,7 +1628,7 @@ void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ assert( p!=0 && p->pLimit!=0 ); /* 1 -- checked by caller */ if( p->pGroupBy==0 && (p->selFlags & (SF_Distinct|SF_Aggregate))==0 /* 2 */ - && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pTab)) /* 3 */ + && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pSTab)) /* 3 */ ){ ExprList *pOrderBy = p->pOrderBy; int iCsr = p->pSrc->a[0].iCursor; @@ -1849,7 +1849,7 @@ void sqlite3WhereTabFuncArgs( Expr *pColRef; Expr *pTerm; if( pItem->fg.isTabFunc==0 ) return; - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 ); pArgs = pItem->u1.pFuncArg; if( pArgs==0 ) return; diff --git a/src/window.c b/src/window.c index e5a78e37d2..9829d1740b 100644 --- a/src/window.c +++ b/src/window.c @@ -1079,7 +1079,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ ** sqlite3SrcListAppend() */ if( p->pSrc ){ Table *pTab2; - p->pSrc->a[0].pSelect = pSub; + p->pSrc->a[0].sq.pSelect = pSub; p->pSrc->a[0].fg.isCorrelated = 1; sqlite3SrcListAssignCursors(pParse, p->pSrc); pSub->selFlags |= SF_Expanded|SF_OrderByReqd; @@ -1093,7 +1093,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ }else{ memcpy(pTab, pTab2, sizeof(Table)); pTab->tabFlags |= TF_Ephemeral; - p->pSrc->a[0].pTab = pTab; + p->pSrc->a[0].pSTab = pTab; pTab = pTab2; memset(&w, 0, sizeof(w)); w.xExprCallback = sqlite3WindowExtraAggFuncDepth; @@ -1389,7 +1389,7 @@ int sqlite3WindowCompare( ** and initialize registers and cursors used by sqlite3WindowCodeStep(). */ void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){ - int nEphExpr = pSelect->pSrc->a[0].pSelect->pEList->nExpr; + int nEphExpr = pSelect->pSrc->a[0].sq.pSelect->pEList->nExpr; Window *pMWin = pSelect->pWin; Window *pWin; Vdbe *v = sqlite3GetVdbe(pParse); @@ -2789,7 +2789,7 @@ void sqlite3WindowCodeStep( Vdbe *v = sqlite3GetVdbe(pParse); int csrWrite; /* Cursor used to write to eph. table */ int csrInput = p->pSrc->a[0].iCursor; /* Cursor of sub-select */ - int nInput = p->pSrc->a[0].pTab->nCol; /* Number of cols returned by sub */ + int nInput = p->pSrc->a[0].pSTab->nCol; /* Number of cols returned by sub */ int iInput; /* To iterate through sub cols */ int addrNe; /* Address of OP_Ne */ int addrGosubFlush = 0; /* Address of OP_Gosub to flush: */ From 901b6714555f90ac38421cc5b97e532367e9bccb Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 18 Aug 2024 09:53:37 +0000 Subject: [PATCH 36/99] Automatically disable directory fsync when compiling for AIX. FossilOrigin-Name: 8d170e07e20c21a5bb97fdc8c2d01d92178f5fe740e60d203e4621747cfce33e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index bd4f4cbe1c..15c9bb1203 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sMakefile.in\sso\sthat\sit\soutputs\sa\sslightly\sbetter\serror\smessage\son\san\nattempt\sto\sbuild\ssomething\sthat\srequires\sTCL\slibraries\swhen\sHAVE_TCL\sis\sfalse. -D 2024-08-17T16:17:16.045 +C Automatically\sdisable\sdirectory\sfsync\swhen\scompiling\sfor\sAIX. +D 2024-08-18T09:53:37.962 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -744,7 +744,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c 2ea8d3ed496b8d1f9332a9505653424e5464fd797ea9d91f8e2e62f9dd0298d0 +F src/os_unix.c 6e3e4fc75904ff85184091dbab996e6e35c1799e771788961cc3b4fcbe8f852c F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c b08600ebf0db90b6d1e9b8b6577c6fa3877cbe1a100bd0b2899e4c6e9adad4b3 @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 793ff83d09bea165c8cedc978afd3fcfb1a4cee894c5914aa1d92416afce963d -R be5ece8b231f01bec02cbe9ef4eda884 +P a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3 +R cd57173ec450b44036cf8242b7c2fe1f U drh -Z 6d429e685f531219ca0d0afba9760eb7 +Z a06494827a8461d8ba3867ea1f761ae1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a5034c9d32..1dca61d1bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3 +8d170e07e20c21a5bb97fdc8c2d01d92178f5fe740e60d203e4621747cfce33e diff --git a/src/os_unix.c b/src/os_unix.c index c94c0c111f..5d1dc9ac6b 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -322,7 +322,7 @@ static pid_t randomnessPid = 0; #define UNIXFILE_EXCL 0x01 /* Connections from one process only */ #define UNIXFILE_RDONLY 0x02 /* Connection is read only */ #define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ -#ifndef SQLITE_DISABLE_DIRSYNC +#if !defined(SQLITE_DISABLE_DIRSYNC) && !defined(_AIX) # define UNIXFILE_DIRSYNC 0x08 /* Directory sync needed */ #else # define UNIXFILE_DIRSYNC 0x00 From dd9a7e45409f9f94a59165ddfd9379c5e1a14350 Mon Sep 17 00:00:00 2001 From: dan Date: Sun, 18 Aug 2024 14:46:13 +0000 Subject: [PATCH 37/99] Fix a problem with detail=none, content='' tables. FossilOrigin-Name: 4b8070816ada3f269d53c9cee20669f59e19a23c31c8509fc397c21dd097ec9f --- ext/fts5/fts5_main.c | 7 +++++++ ext/fts5/test/fts5misc.test | 13 +++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 06833a98b2..e271402ecf 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2217,10 +2217,17 @@ static int fts5CsrPoslist( if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){ rc = SQLITE_RANGE; + }else if( pConfig->eDetail!=FTS5_DETAIL_FULL + && pConfig->eContent==FTS5_CONTENT_NONE + ){ + *pa = 0; + *pn = 0; + return SQLITE_OK; }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); if( aPopulator==0 ) rc = SQLITE_NOMEM; if( rc==SQLITE_OK ){ diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index 0da071192d..534c42fff6 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -652,5 +652,18 @@ do_execsql_test 24.4 { SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') ); } {-1e-06} +#------------------------------------------------------------------------- +reset_db +fts5_aux_test_functions db + +do_execsql_test 25.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none', content=''); + INSERT INTO t1(a) VALUES('a b c'); +} + +do_execsql_test 25.0 { + SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; +} {{}} + finish_test diff --git a/manifest b/manifest index a12f78027e..9d907ad392 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbroken\sassert()\sin\sfts5_storage.c. -D 2024-08-17T19:40:15.498 +C Fix\sa\sproblem\swith\sdetail=none,\scontent=''\stables. +D 2024-08-18T14:46:13.208 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81c F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 23029229021240dc21a69eb749305316103d183375413e2090c064b74b03b7f8 +F ext/fts5/fts5_main.c 5ea7ab0c9967594e73b7dd0ad737595922a14f175aa4b486dc2992a3e3138b68 F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612 F ext/fts5/fts5_tcl.c 1dcf08028141c40a32634bdcf2d5601622ce4edc48f82ac4ce0cbe0a92a6961d F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -191,7 +191,7 @@ F ext/fts5/test/fts5locale.test 79cbd3000ae269de50826f6061c81f7c9fdb21dd9954c0b7 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test c0521e173b16d40b5a37e7e436459028403be1e6d3a5f4dfb2ef515d4486124b +F ext/fts5/test/fts5misc.test 60bb2be4a2d83d7a45047c1812781e2e337a27efa539d86356ef7f4acaf08eab F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74 @@ -2209,8 +2209,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6a8ff9ba5e71d817489093d8dff0a8d77365b4222773b941accbd58558d24379 -R 9367527e69e79bca7a5a92aed8bd4950 +P 01116e32797e1b127ba79059d890fbe5075eee2b6e3ce4468e82862b0fc3aa46 +R 502de970004bd002cb48fa6eef375d2c U dan -Z 275daf935ae7d8dde2bbfa29a46fdb3e +Z 7efc0e935f060687a4e12ab8572c886e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0408009dda..43ca2661de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01116e32797e1b127ba79059d890fbe5075eee2b6e3ce4468e82862b0fc3aa46 +4b8070816ada3f269d53c9cee20669f59e19a23c31c8509fc397c21dd097ec9f From aa2258e35b3df3dcd49220f2fed53725b150ac2b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 19 Aug 2024 12:54:12 +0000 Subject: [PATCH 38/99] Attempt to fix date/time calculations for days earlier than 0400-03-01. See [forum:/forumpost/eaa0a09786c6368b|forum thread eaa0a09786c6368b]. FossilOrigin-Name: 00cae11fffaf50e2133915e851d41098d00b6b1458e1438bcb6664e1411d8498 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/date.c | 10 +++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 15c9bb1203..7e115d9526 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Automatically\sdisable\sdirectory\sfsync\swhen\scompiling\sfor\sAIX. -D 2024-08-18T09:53:37.962 +C Attempt\sto\sfix\sdate/time\scalculations\sfor\sdays\searlier\sthan\s0400-03-01.\nSee\s[forum:/forumpost/eaa0a09786c6368b|forum\sthread\seaa0a09786c6368b]. +D 2024-08-19T12:54:12.528 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/build.c 237ccc0290d131d646be722f418e92ee0a38043aee25e7dfdc75f8ce5b3abe4e F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 -F src/date.c 13dd752847afb32ed70510ad7345a5b9c841f51ad904dba5d010f1fa3a6a324e +F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 @@ -2205,8 +2205,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3 -R cd57173ec450b44036cf8242b7c2fe1f +P 8d170e07e20c21a5bb97fdc8c2d01d92178f5fe740e60d203e4621747cfce33e +R 63b5eb42616efb9e489a03771c227de0 +T *branch * ancient-date-fix +T *sym-ancient-date-fix * +T -sym-trunk * U drh -Z a06494827a8461d8ba3867ea1f761ae1 +Z baa6c6f17edc1a9ecec4f4cec4c126b5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1dca61d1bd..358b550bd1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d170e07e20c21a5bb97fdc8c2d01d92178f5fe740e60d203e4621747cfce33e +00cae11fffaf50e2133915e851d41098d00b6b1458e1438bcb6664e1411d8498 diff --git a/src/date.c b/src/date.c index 8a609ae3cc..8c48a81fa5 100644 --- a/src/date.c +++ b/src/date.c @@ -271,8 +271,8 @@ static void computeJD(DateTime *p){ Y--; M += 12; } - A = Y/100; - B = 2 - A + (A/4); + A = (Y+4800)/100; + B = 38 - A + (A/4); X1 = 36525*(Y+4716)/100; X2 = 306001*(M+1)/10000; p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); @@ -456,7 +456,7 @@ static int validJulianDay(sqlite3_int64 iJD){ ** Compute the Year, Month, and Day from the julian day number. */ static void computeYMD(DateTime *p){ - int Z, A, B, C, D, E, X1; + int Z, alpha, A, B, C, D, E, X1; if( p->validYMD ) return; if( !p->validJD ){ p->Y = 2000; @@ -467,8 +467,8 @@ static void computeYMD(DateTime *p){ return; }else{ Z = (int)((p->iJD + 43200000)/86400000); - A = (int)((Z - 1867216.25)/36524.25); - A = Z + 1 + A - (A/4); + alpha = (int)((Z + 32044.75)/36524.25) - 52; + A = Z + 1 + alpha - ((alpha+100)/4) + 25; B = A + 1524; C = (int)((B - 122.1)/365.25); D = (36525*(C&32767))/100; From 5a18c1f9cd79150dbf263295467fde646d74f2b9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 19 Aug 2024 13:53:49 +0000 Subject: [PATCH 39/99] New date/time test cases to validate ancient dates. FossilOrigin-Name: 82719074f090d9d0a0b9baea0dee7b1dd40a272e3f3e45043d4a640c07989d5d --- manifest | 14 ++++---- manifest.uuid | 2 +- test/date5.test | 86 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 test/date5.test diff --git a/manifest b/manifest index 7e115d9526..912b177c84 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\sfix\sdate/time\scalculations\sfor\sdays\searlier\sthan\s0400-03-01.\nSee\s[forum:/forumpost/eaa0a09786c6368b|forum\sthread\seaa0a09786c6368b]. -D 2024-08-19T12:54:12.528 +C New\sdate/time\stest\scases\sto\svalidate\sancient\sdates. +D 2024-08-19T13:53:49.904 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1060,6 +1060,7 @@ F test/date.test c8ff835023f2107b57ce7a45c92265d51c98a23fc93231e998f12d850831aad F test/date2.test 7e12ec14aaf4d5e6294b4ba140445b0eca06ea50062a9c3a69c4ee13d0b6f8b1 F test/date3.test a1b77abf05c6772fe5ca2337cac1398892f2a41e62bce7e6be0f4a08a0e64ae5 F test/date4.test 75dc8401e8c0639a228cd26a6eaa4ff5ea8ccda912b9853d1c9462c476670e17 +F test/date5.test 14ba189bc4d03efc371dd5302e035764f6633355a3e13acb4a45e7b33530231e F test/dbdata.test 042f49acff3438f940eeba5868d3af080ae64ddf26ae78f80c92bec3ca7d8603 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test 6c9a4622029d69dc38926f115864b055cb2f39badd25ec22cbfb130c8ba8e9c3 @@ -2205,11 +2206,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d170e07e20c21a5bb97fdc8c2d01d92178f5fe740e60d203e4621747cfce33e -R 63b5eb42616efb9e489a03771c227de0 -T *branch * ancient-date-fix -T *sym-ancient-date-fix * -T -sym-trunk * +P 00cae11fffaf50e2133915e851d41098d00b6b1458e1438bcb6664e1411d8498 +R 020e143aeb047149147ee2aa5a7ba85f U drh -Z baa6c6f17edc1a9ecec4f4cec4c126b5 +Z d05bb26a1492de783006f86a1565084e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 358b550bd1..40201f610b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00cae11fffaf50e2133915e851d41098d00b6b1458e1438bcb6664e1411d8498 +82719074f090d9d0a0b9baea0dee7b1dd40a272e3f3e45043d4a640c07989d5d diff --git a/test/date5.test b/test/date5.test new file mode 100644 index 0000000000..688f84d0f1 --- /dev/null +++ b/test/date5.test @@ -0,0 +1,86 @@ +# 2024-08-19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# https://sqlite.org/forum/forumpost/eaa0a09786c6368b +# +# Apparently SQLite has been miscomputing leap-year dates before +# the year 0400. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Skip this whole file if date and time functions are omitted +# at compile-time +# +ifcapable {!datetime} { + finish_test + return +} + +# Data sources: +# 1-10 https://ssd.jpl.nasa.gov/tools/jdc/#/cd +# 11 Jean Meeus, Astronomical Algorithms, ISBN 0-943396-61-1, p.59 +# 12 https://en.wikipedia.org/wiki/Julian_day +# +# ID YEAR MONTH DAY JD +set date5data { + 1 2024 2 29 2460369.5 + 2 2024 3 1 2460370.5 + 3 2023 2 28 2460003.5 + 4 2023 3 1 2460004.5 + 5 2000 2 29 2451603.5 + 6 2000 3 1 2451604.5 + 7 1900 2 28 2415078.5 + 8 1900 3 1 2415079.5 + 9 1712 2 29 2346413.5 + 10 1712 3 1 2346414.5 + 11 1977 4 26 2443259.5 + 12 2013 1 1 2456293.5 +} + +foreach {id y m d jd} $date5data { + set date [format %04d-%02d-%02d $y $m $d] + do_execsql_test date5-jd$jd { + SELECT date($::jd); + } $date + do_execsql_test date5-cal/$date { + SELECT julianday($::date); + } $jd + for {set i 1} {$y+400*$i<=9999} {incr i} { + set y2 [expr {$y+400*$i}] + set date2 [format %04d-%02d-%02d $y2 $m $d] + set jd2 [expr {$jd+146097*$i}] + do_execsql_test date5-jd$jd2 { + SELECT date($::jd2); + } $date2 + do_execsql_test date5-cal/$date2 { + SELECT julianday($::date2); + } $jd2 + } + for {set i 1} {$y-400*$i>=-4712} {incr i} { + set y2 [expr {$y-400*$i}] + if {$y2<0} { + set date2 [format -%04d-%02d-%02d [expr {-$y2}] $m $d] + } else { + set date2 [format %04d-%02d-%02d $y2 $m $d] + } + set jd2 [expr {$jd-146097*$i}] + do_execsql_test date5-jd$jd2 { + SELECT date($::jd2); + } $date2 + do_execsql_test date5-cal/$date2 { + SELECT julianday($::date2); + } $jd2 + } +} + +finish_test From 127812e64534ea80b0e2281f234370d9b02fa0b2 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 19 Aug 2024 20:35:30 +0000 Subject: [PATCH 40/99] Update some comments that are processed into the fts5.html webpage. FossilOrigin-Name: 3e06ab218bbd1ed75a24afb44e8df9ce84e9fc24701428cb8b3459760f44006d --- ext/fts5/extract_api_docs.tcl | 5 ++++- ext/fts5/fts5.h | 24 +++++++++++++++++++++++- manifest | 15 +++++++-------- manifest.uuid | 2 +- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl index 6762a036d5..634dc70cbd 100644 --- a/ext/fts5/extract_api_docs.tcl +++ b/ext/fts5/extract_api_docs.tcl @@ -108,8 +108,11 @@ proc get_tokenizer_docs {data} { append res "
$line

\n" continue } + if {[regexp {FTS5_TOKENIZER} $line]} { + set line

+ } if {[regexp {SYNONYM SUPPORT} $line]} { - set line "

Synonym Support

" + set line "

Synonym Support

" } if {[string trim $line] == ""} { append res "

\n" diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index dfa075f0a9..682a8da386 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -429,7 +429,7 @@ struct Fts5ExtensionApi { ** argument passed to this function is a pointer to an Fts5Tokenizer object ** returned by an earlier call to xCreate(). ** -** The second argument indicates the reason that FTS5 is requesting +** The third argument indicates the reason that FTS5 is requesting ** tokenization of the supplied text. This is always one of the following ** four values: ** @@ -453,6 +453,13 @@ struct Fts5ExtensionApi { ** on a columnsize=0 database. ** ** +** The sixth and seventh arguments passed to xTokenize() - pLocale and +** nLocale - are a pointer to a buffer containing the locale to use for +** tokenization (e.g. "en_US") and its size in bytes, respectively. The +** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in +** which case nLocale is always 0) to indicate that the tokenizer should +** use its default locale. +** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer ** passed as the second argument to xTokenize(). The third and fourth @@ -484,6 +491,21 @@ struct Fts5ExtensionApi { ** an nLocale byte buffer containing the name of the locale to use as utf-8 ** text. pLocale is not nul-terminated. ** +** FTS5_TOKENIZER +** +** There is also an fts5_tokenizer object. This is an older version of +** fts5_tokenizer_v2. It is similar except that: +** +**

    +**
  • There is no "iVersion" field, and +**
  • The xTokenize() method does not take a locale argument. +**
+** +** fts5_tokenizer tokenizers should be registered with the xCreateTokenizer() +** function, instead of xCreateTokenizer_v2(). Tokenizers implementations +** registered using either API may be retrieved using both xFindTokenizer() +** and xFindTokenizer_v2(). +** ** SYNONYM SUPPORT ** ** Custom tokenizers may also support synonyms. Consider a case in which a diff --git a/manifest b/manifest index 72992c9562..f8108ac374 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sfts5_locale()\sfunction\sand\srelated\sfunctionality\sto\sfts5. -D 2024-08-19T14:33:39.681 +C Update\ssome\scomments\sthat\sare\sprocessed\sinto\sthe\sfts5.html\swebpage. +D 2024-08-19T20:35:30.209 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -91,8 +91,8 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7ede551eac3ead62baa F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb -F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h 4c6998c6186268b4dbe9baef2c0d2ab974bd90996d61d4dbe801367249be6de4 +F ext/fts5/extract_api_docs.tcl 1db7f85f4d84b7b6f33336155d5053fafc3c8debd074422d8003c8f7fa4d0fdb +F ext/fts5/fts5.h c65fc7799a4cd6774628da4fa9408955623e504d7369ab5b89c4413fdfe11eb5 F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d805655360 F ext/fts5/fts5_aux.c 12cd2512f869217c38b70c31de5b5f741812734fafa80f55b32ea9bbd96e2152 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 @@ -2210,9 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 474b7e34b83bc5e85854bc3b386f31ff39b390549d89b94921f33bbc5b658d1d 6787603bd0d8117e41e6b572c0d7cada0f6d7266a4ff2ad79f9471b9beb28a9e -R 2c3030846c904496873c37b37c82be1d -T +closed 6787603bd0d8117e41e6b572c0d7cada0f6d7266a4ff2ad79f9471b9beb28a9e +P b9632900100bdbc913f83bfb03b32585cf07a192b9a7f26b9bebc7d91e63a610 +R ea532ddb7482adbca514ac43836cd1cf U dan -Z fdb5bbd69f79c041555a00a73e5efcdd +Z 0d4360caeae428f14a1f60694bd440b2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 43438a3211..39ef92242c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9632900100bdbc913f83bfb03b32585cf07a192b9a7f26b9bebc7d91e63a610 +3e06ab218bbd1ed75a24afb44e8df9ce84e9fc24701428cb8b3459760f44006d From 1521ca4c20c7ade39c671291b2214096c74cacd0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 19 Aug 2024 22:48:30 +0000 Subject: [PATCH 41/99] Refactor the SrcItem object so that information about subqueries is stored in a separately allocated Subquery object. This reduces the memory requirements for SrcItem and makes the code run faster. It also provides an expansion path for subquery processing that does not burden simple queries. The current checking mostly works, but there are still issues that need to be tracked down and fixed. FossilOrigin-Name: 8ff5dda8448d7e1a533d7f27db2573ce68fa9956b9d9847ced45e83c1f06bab0 --- manifest | 42 ++++++------ manifest.uuid | 2 +- src/alter.c | 13 ++-- src/attach.c | 2 +- src/build.c | 96 ++++++++++++++++++++++---- src/expr.c | 26 ++++--- src/insert.c | 71 +++++++++++-------- src/parse.y | 21 ++++-- src/printf.c | 13 ++-- src/resolve.c | 29 +++++--- src/select.c | 179 +++++++++++++++++++++++++++++------------------- src/sqliteInt.h | 40 ++++++++--- src/treeview.c | 8 +-- src/walker.c | 4 +- src/where.c | 35 ++++++---- src/wherecode.c | 20 ++++-- src/whereexpr.c | 4 +- src/window.c | 18 +++-- 18 files changed, 419 insertions(+), 204 deletions(-) diff --git a/manifest b/manifest index ef77b8894d..30b1edeab1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Give\sunique\snames\sto\sfields\sin\sthe\sSrcItem\sobject,\sto\sfacilitate\sanalysis\sof\nhow\sthose\sfields\sare\sused\susing\s"grep". -D 2024-08-17T23:23:23.271 +C Refactor\sthe\sSrcItem\sobject\sso\sthat\sinformation\sabout\ssubqueries\sis\sstored\nin\sa\sseparately\sallocated\sSubquery\sobject.\s\sThis\sreduces\sthe\smemory\srequirements\nfor\sSrcItem\sand\smakes\sthe\scode\srun\sfaster.\s\sIt\salso\sprovides\san\sexpansion\spath\nfor\ssubquery\sprocessing\sthat\sdoes\snot\sburden\ssimple\squeries.\s\sThe\scurrent\nchecking\smostly\sworks,\sbut\sthere\sare\sstill\sissues\sthat\sneed\sto\sbe\stracked\ndown\sand\sfixed. +D 2024-08-19T22:48:30.539 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -692,9 +692,9 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 -F src/alter.c b73964bd6bc3122e884993c0666bae96d4e1e50a94b6f9008df32e6a18e99cd0 +F src/alter.c aa93e37e4a36a0525bbb2a2aeda20d2018f0aa995542c7dc658e031375e3f532 F src/analyze.c 30bf40ec4208ead9e977bec017bccc8a9681820936e38ca5a4a7443100a6d5c5 -F src/attach.c d778bacb2ee5bc6a74c81a17376064e6a16ee447d82ee4ad1f7717a320d5a232 +F src/attach.c 08235ab62ed5ccc93c22bf36e640d19effcd632319615851bccf724ec9341333 F src/auth.c 4c1ea890e0069ad73bead5d17a5b12c34cfa4f1a24175c8147ea439b64be271c F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 @@ -702,7 +702,7 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c 8b42fc7d9efdb2df05c30e8f91ff6cfbd979724ae24bf90269028468b7a13333 F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 -F src/build.c 664804cbefaf43c13fcb369474609cf8befa4d89933804dc276d0a15fbd66c34 +F src/build.c d243a17292e027c1d662e992fbd395123aacc4b74a4a9419d0b010088ba318dd F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 @@ -710,7 +710,7 @@ F src/date.c 13dd752847afb32ed70510ad7345a5b9c841f51ad904dba5d010f1fa3a6a324e F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c 8aaf9d05d6ff6349a7a6001cbf90c7a46b19b9f52ef2e47aa0415a529d5307a1 -F src/expr.c 835b637da89fe20a1ff90bd372904fd421ff92d1dfb9bbea9c81bbffbea46715 +F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 849049c74a4c68961154c124087b5c67d277217a2e642570d0c1bd6336859940 F src/func.c 1f61e32e7a357e615b5d2e774bee563761fce4f2fd97ecb0f72c33e62a2ada5f @@ -719,7 +719,7 @@ F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 2152b159ac214519e778b6996a85b9073ace054f5c9d430b375852e343a5ec92 +F src/insert.c b07865339e67869ff12d2eee3a1417f4779c742e2a51ae3d14ad45e5168f6f41 F src/json.c 5b6a1d6015997b9ee848a32948720bdb26a0ef2de5a2127ebf7355ce66dbdc0d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 @@ -749,23 +749,23 @@ F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c b08600ebf0db90b6d1e9b8b6577c6fa3877cbe1a100bd0b2899e4c6e9adad4b3 F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a -F src/parse.y bc8df5d36afd06d50c1828072e4f3120d892af685436db3b8e776288803535e0 +F src/parse.y 967e0a712de8bee4b2cf82bd9efa36b68afd03cc486d4ea77a945feb49b057dc F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 49516ad7718a3626f28f710fa7448ef1fce3c07fd169acbb4817341950264319 F src/pragma.c 52bfbf6dfd668b69b5eb9bd1186e3a67367c8453807150d6e75239229924f684 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c d99931f45416652895e502328ca49fe782cfc4e1ebdcda13b3736d991ebf42ce -F src/printf.c efa3973bdcacdbc8cf0ff3b3a58e16a105319eac8819823f5807579dc2fa4138 +F src/printf.c 17054fb94ffcf7a28362e9a5af4a0f813bd0c52200ae408eeebddc81feed9274 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c d25fc99cd54d2acf4c655f942dd48ee60e3ada94d73c8b992487515f1295a3bc +F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 10ff02352f4c6eb7bc8d5711f0f74199b94ae9adc346f10391931f5c55d1c68f +F src/select.c 3e284895745f57de3b37b17fce9eb0e26bc54fccc94ea1ff5342be4360e3674b F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab67704 F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 95186ebaf79560fa44b1a05bd7b4b14846e037a285177ca65082a9ae952a6c29 +F src/sqliteInt.h 630e356b7d94df1dcbc32d4f7d4569725a12c6de014fe064311219c22d9cbd54 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -824,7 +824,7 @@ F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68 -F src/treeview.c a337ab329f94e627c9a5efee0326d3359e352c4f0c0bb5c8f4b686fe03e6e7e8 +F src/treeview.c e98dbc8068d27f3d0bd2a4cebfce6a7c21c776edc23d80e12c67a70cd2ba3fe5 F src/trigger.c 68d849ea6ccab67beb450d1a0d065c6118f0884a779388b07f0d9a27dbda38c9 F src/update.c 0e01aa6a3edf9ec112b33eb714b9016a81241497b1fb7c3e74332f4f71756508 F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 @@ -845,12 +845,12 @@ F src/vtab.c 5fb499d20494b7eecaadb7584634af9afcb374cb0524912b475fcb1712458a1b F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 -F src/walker.c e5898234c5920481d962e29b96ecf403b992b12ae243ad0e53659386a1b3e554 -F src/where.c d0140395a35fbc451e9d07c9816895d399f02ae76cbcd255e8ce1069548b02c8 +F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 +F src/where.c b6f79b189fcba459bb80420d3b4102f42a399be36ff29a3deff4ae4888fee46d F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e -F src/wherecode.c 3e130abafc185a29c8dbdcc33e6150dff2aa884f75483e159feef170961af65d -F src/whereexpr.c 784f641ba3d6f1fdb4f65e3891daa50067a4c51f1acd11b461397f7d4c622907 -F src/window.c ffbf65ba62ce329e2fd68b35f11e920121fac45dab0f9a4bbc486125ccc98b6c +F src/wherecode.c 137797b0de9ddf1ff43e5b0edffcc76fb05184ed651fc4f5a0a01a45c0b89d04 +F src/whereexpr.c 6a72cf607548765a262f216e87373fd675a4646f9cc4278fc519b66cf03dbc13 +F src/window.c 499d48f315a09242dc68f2fac635ed27dcf6bbb0d9ab9084857898c64489e975 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9 @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4c59ac3c6ec979c25b544d29e47b8e39f6439c098eed8f84b3bd506c9adf047 -R 1d81e98efd997ac91c2ceea4c803b7e6 +P 9f5aeef3cbc2c95267c8f7bf60d5c66971a76789669fb0e8f853273ff6f616f2 +R 0dac77664f5c62d0dcc882ea3148d6f4 U drh -Z 8ec986f4842c563a443c2f1a23450bf4 +Z a11cd0d49c593b1144d9510fd31aa2a7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0e51ef0919..ca5d536722 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f5aeef3cbc2c95267c8f7bf60d5c66971a76789669fb0e8f853273ff6f616f2 +8ff5dda8448d7e1a533d7f27db2573ce68fa9956b9d9847ced45e83c1f06bab0 diff --git a/src/alter.c b/src/alter.c index 41146f54e9..ff20757589 100644 --- a/src/alter.c +++ b/src/alter.c @@ -1366,8 +1366,9 @@ static int renameResolveTrigger(Parse *pParse){ int i; for(i=0; ipFrom->nSrc && rc==SQLITE_OK; i++){ SrcItem *p = &pStep->pFrom->a[i]; - if( p->sq.pSelect ){ - sqlite3SelectPrep(pParse, p->sq.pSelect, 0); + if( p->fg.isSubquery ){ + assert( p->u4.pSubq!=0 ); + sqlite3SelectPrep(pParse, p->u4.pSubq->pSelect, 0); } } } @@ -1435,8 +1436,12 @@ static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ } if( pStep->pFrom ){ int i; - for(i=0; ipFrom->nSrc; i++){ - sqlite3WalkSelect(pWalker, pStep->pFrom->a[i].sq.pSelect); + SrcList *pFrom = pStep->pFrom; + for(i=0; inSrc; i++){ + if( pFrom->a[i].fg.isSubquery ){ + assert( pFrom->a[i].u4.pSubq!=0 ); + sqlite3WalkSelect(pWalker, pFrom->a[i].u4.pSubq->pSelect); + } } } } diff --git a/src/attach.c b/src/attach.c index 65b492474e..76476685fb 100644 --- a/src/attach.c +++ b/src/attach.c @@ -479,7 +479,7 @@ static int fixSelectCb(Walker *p, Select *pSelect){ if( NEVER(pList==0) ) return WRC_Continue; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pFix->bTemp==0 ){ + if( pFix->bTemp==0 && pItem->fg.isSubquery==0 ){ if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ if( iDb!=sqlite3FindDbName(db, pItem->u4.zDatabase) ){ sqlite3ErrorMsg(pFix->pParse, diff --git a/src/build.c b/src/build.c index ff798087af..b585c19719 100644 --- a/src/build.c +++ b/src/build.c @@ -4914,13 +4914,40 @@ void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ for(i=0, pItem=pList->a; inSrc; i++, pItem++){ if( pItem->iCursor>=0 ) continue; pItem->iCursor = pParse->nTab++; - if( pItem->sq.pSelect ){ - sqlite3SrcListAssignCursors(pParse, pItem->sq.pSelect->pSrc); + if( pItem->fg.isSubquery ){ + assert( pItem->u4.pSubq!=0 ); + assert( pItem->u4.pSubq->pSelect!=0 ); + assert( pItem->u4.pSubq->pSelect->pSrc!=0 ); + sqlite3SrcListAssignCursors(pParse, pItem->u4.pSubq->pSelect->pSrc); } } } } +/* +** Delete a Subquery object and its substructure. +*/ +void sqlite3SubqueryDelete(sqlite3 *db, Subquery *pSubq){ + assert( pSubq!=0 && pSubq->pSelect!=0 ); + sqlite3SelectDelete(db, pSubq->pSelect); + sqlite3DbFree(db, pSubq); +} + +/* +** Remove a Subquery from a SrcItem. Return the associated Select object. +** The returned Select becomes the responsibility of the caller. +*/ +Select *sqlite3SubqueryDetach(sqlite3 *db, SrcItem *pItem){ + Select *pSel; + assert( pItem!=0 ); + assert( pItem->fg.isSubquery ); + pSel = pItem->u4.pSubq->pSelect; + sqlite3DbFree(db, pItem->u4.pSubq); + pItem->u4.pSubq = 0; + pItem->fg.isSubquery = 0; + return pSel; +} + /* ** Delete an entire SrcList including all its substructure. */ @@ -4932,22 +4959,19 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ for(pItem=pList->a, i=0; inSrc; i++, pItem++){ /* Check invariants on SrcItem */ - assert( pItem->sq.pSelect==0 - || (pItem->sq.pSelect->selFlags & SF_MultiValue)==0 - || (pItem->fg.hadSchema==0 - && (pItem->fg.fixedSchema || pItem->u4.zDatabase==0)) ); + assert( !pItem->fg.hadSchema || !pItem->fg.isSubquery ); assert( pItem->fg.hadSchema==0 || pItem->fg.fixedSchema==1 ); - if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); - if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + if( pItem->fg.isSubquery ){ + sqlite3SubqueryDelete(db, pItem->u4.pSubq); + }else if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ sqlite3DbNNFreeNN(db, pItem->u4.zDatabase); } if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pSTab); - if( pItem->sq.pSelect ) sqlite3SelectDelete(db, pItem->sq.pSelect); if( pItem->fg.isUsing ){ sqlite3IdListDelete(db, pItem->u3.pUsing); }else if( pItem->u3.pOn ){ @@ -4957,6 +4981,52 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbNNFreeNN(db, pList); } +/* +** Attach a Subquery object to pItem->uv.pSubq. Set the +** pSelect value but leave all the other values initialized +** to zero. +** +** A copy of the Select object is made if dupSelect is true, and the +** SrcItem takes responsibility for deleting the copy. If dupSelect is +** false, ownership of the Select passes to the SrcItem. Either way, +** the SrcItem will take responsibility for deleting the Select. +** +** When dupSelect is zero, that means the Select might get deleted right +** away if there is an OOM error. Beware. +** +** Return non-zero on success. Return zero on an OOM error. +*/ +int sqlite3SrcItemAttachSubquery( + Parse *pParse, /* Parsing context */ + SrcItem *pItem, /* Item to which the subquery is to be attached */ + Select *pSelect, /* The subquery SELECT. Must be non-NULL */ + int dupSelect /* If true, attach a copy of pSelect, not pSelect itself.*/ +){ + Subquery *p; + if( pSelect==0 ) return SQLITE_OK; + assert( pItem->fg.isSubquery==0 ); + if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + sqlite3DbFree(pParse->db, pItem->u4.zDatabase); + pItem->u4.zDatabase = 0; + } + pItem->fg.fixedSchema = 0; + if( dupSelect ){ + pSelect = sqlite3SelectDup(pParse->db, pSelect, 0); + if( pSelect==0 ) return 0; + } + p = pItem->u4.pSubq = sqlite3DbMallocRawNN(pParse->db, sizeof(Subquery)); + if( p==0 ){ + sqlite3SelectDelete(pParse->db, pSelect); + return 0; + } + pItem->fg.isSubquery = 1; + p->pSelect = pSelect; + assert( offsetof(Subquery, pSelect)==0 ); + memset(((char*)p)+sizeof(p->pSelect), 0, sizeof(*p)-sizeof(p->pSelect)); + return 1; +} + + /* ** This routine is called by the parser to add a new term to the ** end of a growing FROM clause. The "p" parameter is the part of @@ -5006,10 +5076,12 @@ SrcList *sqlite3SrcListAppendFromTerm( if( pAlias->n ){ pItem->zAlias = sqlite3NameFromToken(db, pAlias); } + assert( pSubquery==0 || pDatabase==0 ); if( pSubquery ){ - pItem->sq.pSelect = pSubquery; - if( pSubquery->selFlags & SF_NestedFrom ){ - pItem->fg.isNestedFrom = 1; + if( sqlite3SrcItemAttachSubquery(pParse, pItem, pSubquery, 0) ){ + if( pSubquery->selFlags & SF_NestedFrom ){ + pItem->fg.isNestedFrom = 1; + } } } assert( pOnUsing==0 || pOnUsing->pOn==0 || pOnUsing->pUsing==0 ); diff --git a/src/expr.c b/src/expr.c index 6ad789a3d8..1b18828dd6 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1877,18 +1877,30 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ SrcItem *pNewItem = &pNew->a[i]; const SrcItem *pOldItem = &p->a[i]; Table *pTab; - if( pOldItem->fg.fixedSchema ){ + pNewItem->fg = pOldItem->fg; + if( pOldItem->fg.isSubquery ){ + Subquery *pNewSubq = sqlite3DbMallocRaw(db, sizeof(Subquery)); + if( pNewSubq==0 ){ + assert( db->mallocFailed ); + pNewItem->fg.isSubquery = 0; + }else{ + memcpy(pNewSubq, pOldItem->u4.pSubq, sizeof(*pNewSubq)); + pNewSubq->pSelect = sqlite3SelectDup(db, pNewSubq->pSelect, flags); + if( pNewSubq->pSelect==0 ){ + sqlite3DbFree(db, pNewSubq); + pNewSubq = 0; + pNewItem->fg.isSubquery = 0; + } + } + pNewItem->u4.pSubq = pNewSubq; + }else if( pOldItem->fg.fixedSchema ){ pNewItem->u4.pSchema = pOldItem->u4.pSchema; }else{ pNewItem->u4.zDatabase = sqlite3DbStrDup(db, pOldItem->u4.zDatabase); } pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; - pNewItem->sq.addrFillSub = pOldItem->sq.addrFillSub; - pNewItem->sq.regReturn = pOldItem->sq.regReturn; - pNewItem->sq.regResult = pOldItem->sq.regResult; if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); }else if( pNewItem->fg.isTabFunc ){ @@ -1905,7 +1917,6 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ if( pTab ){ pTab->nTabRef++; } - pNewItem->sq.pSelect = sqlite3SelectDup(db, pOldItem->sq.pSelect, flags); if( pOldItem->fg.isUsing ){ assert( pNewItem->fg.isUsing ); pNewItem->u3.pUsing = sqlite3IdListDup(db, pOldItem->u3.pUsing); @@ -1979,7 +1990,6 @@ Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int flags){ pp = &pNew->pPrior; pNext = pNew; } - return pRet; } #else @@ -2999,7 +3009,7 @@ static Select *isCandidateForInOpt(const Expr *pX){ pSrc = p->pSrc; assert( pSrc!=0 ); if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ - if( pSrc->a[0].sq.pSelect ) return 0; /* FROM is not a subquery or view */ + if( pSrc->a[0].fg.isSubquery) return 0;/* FROM is not a subquery or view */ pTab = pSrc->a[0].pSTab; assert( pTab!=0 ); assert( !IsView(pTab) ); /* FROM clause is not a view */ diff --git a/src/insert.c b/src/insert.c index 4447cdc22e..d65ead2a0b 100644 --- a/src/insert.c +++ b/src/insert.c @@ -585,8 +585,9 @@ void sqlite3AutoincrementEnd(Parse *pParse){ void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ if( ALWAYS(pVal) && pVal->pSrc->nSrc>0 ){ SrcItem *pItem = &pVal->pSrc->a[0]; - sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->sq.regReturn); - sqlite3VdbeJumpHere(pParse->pVdbe, pItem->sq.addrFillSub - 1); + assert( pItem->fg.isSubquery && pItem->u4.pSubq!=0 ); + sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->u4.pSubq->regReturn); + sqlite3VdbeJumpHere(pParse->pVdbe, pItem->u4.pSubq->addrFillSub - 1); } } @@ -714,6 +715,7 @@ Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ if( pRet ){ SelectDest dest; + Subquery *pSubq; pRet->pSrc->nSrc = 1; pRet->pPrior = pLeft->pPrior; pRet->op = pLeft->op; @@ -723,29 +725,31 @@ Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ assert( pLeft->pNext==0 ); assert( pRet->pNext==0 ); p = &pRet->pSrc->a[0]; - p->sq.pSelect = pLeft; p->fg.viaCoroutine = 1; - p->sq.addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; - p->sq.regReturn = ++pParse->nMem; p->iCursor = -1; p->u1.nRow = 2; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, - p->sq.regReturn, 0, p->sq.addrFillSub); - sqlite3SelectDestInit(&dest, SRT_Coroutine, p->sq.regReturn); + if( sqlite3SrcItemAttachSubquery(pParse, p, pLeft, 0) ){ + pSubq = p->u4.pSubq; + pSubq->addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; + pSubq->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, + pSubq->regReturn, 0, pSubq->addrFillSub); + sqlite3SelectDestInit(&dest, SRT_Coroutine, pSubq->regReturn); - /* Allocate registers for the output of the co-routine. Do so so - ** that there are two unused registers immediately before those - ** used by the co-routine. This allows the code in sqlite3Insert() - ** to use these registers directly, instead of copying the output - ** of the co-routine to a separate array for processing. */ - dest.iSdst = pParse->nMem + 3; - dest.nSdst = pLeft->pEList->nExpr; - pParse->nMem += 2 + dest.nSdst; + /* Allocate registers for the output of the co-routine. Do so so + ** that there are two unused registers immediately before those + ** used by the co-routine. This allows the code in sqlite3Insert() + ** to use these registers directly, instead of copying the output + ** of the co-routine to a separate array for processing. */ + dest.iSdst = pParse->nMem + 3; + dest.nSdst = pLeft->pEList->nExpr; + pParse->nMem += 2 + dest.nSdst; - pLeft->selFlags |= SF_MultiValue; - sqlite3Select(pParse, pLeft, &dest); - p->sq.regResult = dest.iSdst; - assert( pParse->nErr || dest.iSdst>0 ); + pLeft->selFlags |= SF_MultiValue; + sqlite3Select(pParse, pLeft, &dest); + pSubq->regResult = dest.iSdst; + assert( pParse->nErr || dest.iSdst>0 ); + } pLeft = pRet; } }else{ @@ -755,12 +759,18 @@ Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ } if( pParse->nErr==0 ){ + Subquery *pSubq; assert( p!=0 ); - if( p->sq.pSelect->pEList->nExpr!=pRow->nExpr ){ - sqlite3SelectWrongNumTermsError(pParse, p->sq.pSelect); + assert( p->fg.isSubquery ); + pSubq = p->u4.pSubq; + assert( pSubq!=0 ); + assert( pSubq->pSelect!=0 ); + assert( pSubq->pSelect->pEList!=0 ); + if( pSubq->pSelect->pEList->nExpr!=pRow->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, pSubq->pSelect); }else{ - sqlite3ExprCodeExprList(pParse, pRow, p->sq.regResult, 0, 0); - sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, p->sq.regReturn); + sqlite3ExprCodeExprList(pParse, pRow, pSubq->regResult, 0, 0); + sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, pSubq->regReturn); } } sqlite3ExprListDelete(pParse->db, pRow); @@ -1111,9 +1121,14 @@ void sqlite3Insert( && pSelect->pPrior==0 ){ SrcItem *pItem = &pSelect->pSrc->a[0]; - dest.iSDParm = pItem->sq.regReturn; - regFromSelect = pItem->sq.regResult; - nColumn = pItem->sq.pSelect->pEList->nExpr; + Subquery *pSubq; + assert( pItem->fg.isSubquery ); + pSubq = pItem->u4.pSubq; + dest.iSDParm = pSubq->regReturn; + regFromSelect = pSubq->regResult; + assert( pSubq->pSelect!=0 ); + assert( pSubq->pSelect->pEList!=0 ); + nColumn = pSubq->pSelect->pEList->nExpr; ExplainQueryPlan((pParse, 0, "SCAN %S", pItem)); if( bIdListInOrder && nColumn==pTab->nCol ){ regData = regFromSelect; @@ -3033,7 +3048,7 @@ static int xferOptimization( if( pSelect->pSrc->nSrc!=1 ){ return 0; /* FROM clause must have exactly one term */ } - if( pSelect->pSrc->a[0].sq.pSelect ){ + if( pSelect->pSrc->a[0].fg.isSubquery ){ return 0; /* FROM clause cannot contain a subquery */ } if( pSelect->pWhere ){ diff --git a/src/parse.y b/src/parse.y index 9e656d0bc7..a47c587547 100644 --- a/src/parse.y +++ b/src/parse.y @@ -742,10 +742,20 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N SrcItem *pOld = F->a; assert( pOld->fg.fixedSchema==0 ); pNew->zName = pOld->zName; - pNew->u4.zDatabase = pOld->u4.zDatabase; - pNew->sq.pSelect = pOld->sq.pSelect; - if( pNew->sq.pSelect && (pNew->sq.pSelect->selFlags & SF_NestedFrom)!=0 ){ - pNew->fg.isNestedFrom = 1; + assert( pOld->fg.fixedSchema==0 ); + if( pOld->fg.isSubquery ){ + pNew->fg.isSubquery = 1; + pNew->u4.pSubq = pOld->u4.pSubq; + pOld->u4.pSubq = 0; + pOld->fg.isSubquery = 0; + if( pNew->u4.pSubq->pSelect + && (pNew->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 + ){ + pNew->fg.isNestedFrom = 1; + } + }else{ + pNew->u4.zDatabase = pOld->u4.zDatabase; + pOld->u4.zDatabase = 0; } if( pOld->fg.isTabFunc ){ pNew->u1.pFuncArg = pOld->u1.pFuncArg; @@ -753,8 +763,7 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N pOld->fg.isTabFunc = 0; pNew->fg.isTabFunc = 1; } - pOld->zName = pOld->u4.zDatabase = 0; - pOld->sq.pSelect = 0; + pOld->zName = 0; } sqlite3SrcListDelete(pParse->db, F); }else{ diff --git a/src/printf.c b/src/printf.c index 027905d26f..be7ffc8c76 100644 --- a/src/printf.c +++ b/src/printf.c @@ -848,16 +848,19 @@ void sqlite3_str_vappendf( if( pItem->zAlias && !flag_altform2 ){ sqlite3_str_appendall(pAccum, pItem->zAlias); }else if( pItem->zName ){ - if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + if( pItem->fg.fixedSchema==0 + && pItem->fg.isSubquery==0 + && pItem->u4.zDatabase!=0 + ){ sqlite3_str_appendall(pAccum, pItem->u4.zDatabase); sqlite3_str_append(pAccum, ".", 1); } sqlite3_str_appendall(pAccum, pItem->zName); }else if( pItem->zAlias ){ sqlite3_str_appendall(pAccum, pItem->zAlias); - }else{ - Select *pSel = pItem->sq.pSelect; - assert( pSel!=0 ); /* Because of tag-20240424-1 */ + }else if( pItem->fg.isSubquery ){/* Because of tag-20240424-1 */ + Select *pSel = pItem->u4.pSubq->pSelect; + assert( pSel!=0 ); if( pSel->selFlags & SF_NestedFrom ){ sqlite3_str_appendf(pAccum, "(join-%u)", pSel->selId); }else if( pSel->selFlags & SF_MultiValue ){ @@ -867,6 +870,8 @@ void sqlite3_str_vappendf( }else{ sqlite3_str_appendf(pAccum, "(subquery-%u)", pSel->selId); } + }else{ + sqlite3_str_appendf(pAccum, "(unknown-data-source-%p)", pItem); } length = width = 0; break; diff --git a/src/resolve.c b/src/resolve.c index 02a48d6993..6dcb4e7322 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -349,7 +349,7 @@ static int lookupName( pTab = pItem->pSTab; assert( pTab!=0 && pTab->zName!=0 ); assert( pTab->nCol>0 || pParse->nErr ); - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->sq.pSelect)); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem)); if( pItem->fg.isNestedFrom ){ /* In this case, pItem is a subquery that has been formed from a ** parenthesized subset of the FROM clause terms. Example: @@ -358,8 +358,12 @@ static int lookupName( ** This pItem -------------^ */ int hit = 0; - assert( pItem->sq.pSelect!=0 ); - pEList = pItem->sq.pSelect->pEList; + Select *pSel; + assert( pItem->fg.isSubquery ); + assert( pItem->u4.pSubq!=0 ); + pSel = pItem->u4.pSubq->pSelect; + assert( pSel!=0 ); + pEList = pSel->pEList; assert( pEList!=0 ); assert( pEList->nExpr==pTab->nCol ); for(j=0; jnExpr; j++){ @@ -1880,7 +1884,11 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** moves the pOrderBy down to the sub-query. It will be moved back ** after the names have been resolved. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].sq.pSelect; + Select *pSub; + assert( p->pSrc->a[0].fg.isSubquery ); + assert( p->pSrc->a[0].u4.pSubq!=0 ); + pSub = p->pSrc->a[0].u4.pSubq->pSelect; + assert( pSub!=0 ); assert( p->pSrc->nSrc==1 && p->pOrderBy ); assert( pSub->pPrior && pSub->pOrderBy==0 ); pSub->pOrderBy = p->pOrderBy; @@ -1893,13 +1901,15 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ for(i=0; ipSrc->nSrc; i++){ SrcItem *pItem = &p->pSrc->a[i]; assert( pItem->zName!=0 - || pItem->sq.pSelect!=0 ); /* Test of tag-20240424-1*/ - if( pItem->sq.pSelect && (pItem->sq.pSelect->selFlags & SF_Resolved)==0 ){ + || pItem->fg.isSubquery ); /* Test of tag-20240424-1*/ + if( pItem->fg.isSubquery + && (pItem->u4.pSubq->pSelect->selFlags & SF_Resolved)==0 + ){ int nRef = pOuterNC ? pOuterNC->nRef : 0; const char *zSavedContext = pParse->zAuthContext; if( pItem->zName ) pParse->zAuthContext = pItem->zName; - sqlite3ResolveSelectNames(pParse, pItem->sq.pSelect, pOuterNC); + sqlite3ResolveSelectNames(pParse, pItem->u4.pSubq->pSelect, pOuterNC); pParse->zAuthContext = zSavedContext; if( pParse->nErr ) return WRC_Abort; assert( db->mallocFailed==0 ); @@ -2001,7 +2011,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** These integers will be replaced by copies of the corresponding result ** set expressions by the call to resolveOrderGroupBy() below. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].sq.pSelect; + Select *pSub; + assert( p->pSrc->a[0].fg.isSubquery ); + pSub = p->pSrc->a[0].u4.pSubq->pSelect; + assert( pSub!=0 ); p->pOrderBy = pSub->pOrderBy; pSub->pOrderBy = 0; } diff --git a/src/select.c b/src/select.c index 5f55b402cf..8bddc03bb4 100644 --- a/src/select.c +++ b/src/select.c @@ -332,11 +332,13 @@ int sqlite3ColumnIndex(Table *pTab, const char *zCol){ */ void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){ assert( pItem!=0 ); - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->sq.pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) ); if( pItem->fg.isNestedFrom ){ ExprList *pResults; - assert( pItem->sq.pSelect!=0 ); - pResults = pItem->sq.pSelect->pEList; + assert( pItem->fg.isSubquery ); + assert( pItem->u4.pSubq!=0 ); + assert( pItem->u4.pSubq->pSelect!=0 ); + pResults = pItem->u4.pSubq->pSelect->pEList; assert( pResults!=0 ); assert( iCol>=0 && iColnExpr ); pResults->a[iCol].fg.bUsed = 1; @@ -1931,7 +1933,11 @@ static const char *columnTypeImpl( for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); if( jnSrc ){ pTab = pTabList->a[j].pSTab; - pS = pTabList->a[j].sq.pSelect; + if( pTabList->a[j].fg.isSubquery ){ + pS = pTabList->a[j].u4.pSubq->pSelect; + }else{ + pS = 0; + } }else{ pNC = pNC->pNext; } @@ -3983,7 +3989,9 @@ static void substSelect( pSrc = p->pSrc; assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(pSubst, pItem->sq.pSelect, 1); + if( pItem->fg.isSubquery ){ + substSelect(pSubst, pItem->u4.pSubq->pSelect, 1); + } if( pItem->fg.isTabFunc ){ substExprList(pSubst, pItem->u1.pFuncArg); } @@ -4054,8 +4062,10 @@ static void srclistRenumberCursors( aCsrMap[pItem->iCursor+1] = pParse->nTab++; } pItem->iCursor = aCsrMap[pItem->iCursor+1]; - for(p=pItem->sq.pSelect; p; p=p->pPrior){ - srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + if( pItem->fg.isSubquery ){ + for(p=pItem->u4.pSubq->pSelect; p; p=p->pPrior){ + srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + } } } } @@ -4366,7 +4376,8 @@ static int flattenSubquery( assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; iParent = pSubitem->iCursor; - pSub = pSubitem->sq.pSelect; + assert( pSubitem->fg.isSubquery ); + pSub = pSubitem->u4.pSubq->pSelect; assert( pSub!=0 ); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -4505,7 +4516,12 @@ static int flattenSubquery( pParse->zAuthContext = zSavedAuthContext; /* Delete the transient structures associated with the subquery */ - pSub1 = pSubitem->sq.pSelect; + + if( ALWAYS(pSubitem->fg.isSubquery) ){ + pSub1 = sqlite3SubqueryDetach(db, pSubitem); + }else{ + pSub1 = 0; + } if( pSubitem->fg.fixedSchema==0 ){ sqlite3DbFree(db, pSubitem->u4.zDatabase); pSubitem->u4.zDatabase = 0; @@ -4514,7 +4530,6 @@ static int flattenSubquery( sqlite3DbFree(db, pSubitem->zAlias); pSubitem->zName = 0; pSubitem->zAlias = 0; - pSubitem->sq.pSelect = 0; assert( pSubitem->fg.isUsing!=0 || pSubitem->u3.pOn==0 ); /* If the sub-query is a compound SELECT statement, then (by restrictions @@ -4579,11 +4594,14 @@ static int flattenSubquery( TREETRACE(0x4,pParse,p,("compound-subquery flattener" " creates %u as peer\n",pNew->selId)); } - assert( pSubitem->sq.pSelect==0 ); + assert( pSubitem->fg.isSubquery==0 ); } sqlite3DbFree(db, aCsrMap); if( db->mallocFailed ){ - pSubitem->sq.pSelect = pSub1; + assert( pSubitem->fg.fixedSchema==0 ); + assert( pSubitem->fg.isSubquery==0 ); + assert( pSubitem->u4.zDatabase==0 ); + sqlite3SrcItemAttachSubquery(pParse, pSubitem, pSub1, 0); return 1; } @@ -4659,8 +4677,11 @@ static int flattenSubquery( */ for(i=0; ia[i+iFrom]; - if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); assert( pItem->fg.isTabFunc==0 ); + assert( pItem->fg.isSubquery + || pItem->fg.fixedSchema + || pItem->u4.zDatabase==0 ); + if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); *pItem = pSubSrc->a[i]; pItem->fg.jointype |= ltorj; iNewParent = pSubSrc->a[i].iCursor; @@ -5346,8 +5367,8 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ } assert( pItem->pSTab!=0 ); pTab = pItem->pSTab; - assert( pItem->sq.pSelect!=0 ); - pSub = pItem->sq.pSelect; + assert( pItem->fg.isSubquery ); + pSub = pItem->u4.pSubq->pSelect; assert( pSub->pEList->nExpr==pTab->nCol ); for(pX=pSub; pX; pX=pX->pPrior){ if( (pX->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ @@ -5476,7 +5497,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ if( p->pWhere || p->pEList->nExpr!=1 || p->pSrc->nSrc!=1 - || p->pSrc->a[0].sq.pSelect + || p->pSrc->a[0].fg.isSubquery || pAggInfo->nFunc!=1 || p->pHaving ){ @@ -5776,10 +5797,12 @@ static int resolveFromTermToCte( pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; - pFrom->sq.pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + sqlite3SrcItemAttachSubquery(pParse, pFrom, pCte->pSelect, 1); if( db->mallocFailed ) return 2; - assert( pFrom->sq.pSelect ); - pFrom->sq.pSelect->selFlags |= SF_CopyCte; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq ); + pSel = pFrom->u4.pSubq->pSelect; + assert( pSel!=0 ); + pSel->selFlags |= SF_CopyCte; if( pFrom->fg.isIndexedBy ){ sqlite3ErrorMsg(pParse, "no such index: \"%s\"", pFrom->u1.zIndexedBy); return 2; @@ -5789,7 +5812,7 @@ static int resolveFromTermToCte( pCteUse->nUse++; /* Check if this is a recursive CTE. */ - pRecTerm = pSel = pFrom->sq.pSelect; + pRecTerm = pSel; bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); while( bMayRecursive && pRecTerm->op==pSel->op ){ int i; @@ -5904,9 +5927,12 @@ void sqlite3SelectPopWith(Walker *pWalker, Select *p){ ** SQLITE_NOMEM. */ int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ - Select *pSel = pFrom->sq.pSelect; + Select *pSel; Table *pTab; + assert( pFrom->fg.isSubquery ); + assert( pFrom->u4.pSubq!=0 ); + pSel = pFrom->u4.pSubq->pSelect; assert( pSel ); pFrom->pSTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); if( pTab==0 ) return SQLITE_NOMEM; @@ -6033,7 +6059,9 @@ static int selectExpander(Walker *pWalker, Select *p){ assert( pFrom->fg.isRecursive==0 ); if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY - Select *pSel = pFrom->sq.pSelect; + Select *pSel; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq!=0 ); + pSel = pFrom->u4.pSubq->pSelect; /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); assert( pFrom->pSTab==0 ); @@ -6066,7 +6094,7 @@ static int selectExpander(Walker *pWalker, Select *p){ i16 nCol; u8 eCodeOrig = pWalker->eCode; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; - assert( pFrom->sq.pSelect==0 ); + assert( pFrom->fg.isSubquery==0 ); if( IsView(pTab) ){ if( (db->flags & SQLITE_EnableView)==0 && pTab->pSchema!=db->aDb[1].pSchema @@ -6074,7 +6102,7 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited", pTab->zName); } - pFrom->sq.pSelect = sqlite3SelectDup(db, pTab->u.view.pSelect, 0); + sqlite3SrcItemAttachSubquery(pParse, pFrom, pTab->u.view.pSelect, 1); } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( ALWAYS(IsVirtual(pTab)) @@ -6090,7 +6118,9 @@ static int selectExpander(Walker *pWalker, Select *p){ nCol = pTab->nCol; pTab->nCol = -1; pWalker->eCode = 1; /* Turn on Select.selId renumbering */ - sqlite3WalkSelect(pWalker, pFrom->sq.pSelect); + if( pFrom->fg.isSubquery ){ + sqlite3WalkSelect(pWalker, pFrom->u4.pSubq->pSelect); + } pWalker->eCode = eCodeOrig; pTab->nCol = nCol; } @@ -6188,11 +6218,11 @@ static int selectExpander(Walker *pWalker, Select *p){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - assert( (int)pFrom->fg.isNestedFrom == - IsNestedFrom(pFrom->sq.pSelect) ); + assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom) ); if( pFrom->fg.isNestedFrom ){ - assert( pFrom->sq.pSelect!=0 ); - pNestedFrom = pFrom->sq.pSelect->pEList; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq ); + assert( pFrom->u4.pSubq->pSelect!=0 ); + pNestedFrom = pFrom->u4.pSubq->pSelect->pEList; assert( pNestedFrom!=0 ); assert( pNestedFrom->nExpr==pTab->nCol ); assert( VisibleRowid(pTab)==0 || ViewCanHaveRowid ); @@ -6433,12 +6463,10 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab = pFrom->pSTab; assert( pTab!=0 ); - if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + if( (pTab->tabFlags & TF_Ephemeral)!=0 && pFrom->fg.isSubquery ){ /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->sq.pSelect; - if( pSel ){ - sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); - } + Select *pSel = pFrom->u4.pSubq->pSelect; + sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); } } } @@ -7085,25 +7113,28 @@ static SrcItem *isSelfJoinView( int iFirst, int iEnd /* Range of FROM-clause entries to search. */ ){ SrcItem *pItem; - assert( pThis->sq.pSelect!=0 ); - if( pThis->sq.pSelect->selFlags & SF_PushDown ) return 0; + Select *pSel; + assert( pThis->fg.isSubquery ); + pSel = pThis->u4.pSubq->pSelect; + assert( pSel!=0 ); + if( pSel->selFlags & SF_PushDown ) return 0; while( iFirsta[iFirst++]; - if( pItem->sq.pSelect==0 ) continue; + if( !pItem->fg.isSubquery ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; assert( pItem->pSTab!=0 ); assert( pThis->pSTab!=0 ); if( pItem->pSTab->pSchema!=pThis->pSTab->pSchema ) continue; if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; - pS1 = pItem->sq.pSelect; - if( pItem->pSTab->pSchema==0 && pThis->sq.pSelect->selId!=pS1->selId ){ + pS1 = pItem->u4.pSubq->pSelect; + if( pItem->pSTab->pSchema==0 && pSel->selId!=pS1->selId ){ /* The query flattener left two different CTE tables with identical ** names in the same FROM clause. */ continue; } - if( pItem->sq.pSelect->selFlags & SF_PushDown ){ + if( pS1->selFlags & SF_PushDown ){ /* The view was modified by some other optimization such as ** pushDownWhereTerms() */ continue; @@ -7147,6 +7178,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ Expr *pExpr; Expr *pCount; sqlite3 *db; + SrcItem *pFrom; if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ if( p->pWhere ) return 0; @@ -7161,8 +7193,9 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */ if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */ if( ExprHasProperty(pExpr, EP_WinFunc) ) return 0;/* Not a window function */ - pSub = p->pSrc->a[0].sq.pSelect; - if( pSub==0 ) return 0; /* The FROM is a subquery */ + pFrom = p->pSrc->a; + if( pFrom->fg.isSubquery==0 ) return 0; /* FROM is a subquery */ + pSub = pFrom->u4.pSubq->pSelect; if( pSub->pPrior==0 ) return 0; /* Must be a compound */ if( pSub->selFlags & SF_CopyCte ) return 0; /* Not a CTE */ do{ @@ -7171,7 +7204,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( pSub->pLimit ) return 0; /* No LIMIT clause */ if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ assert( pSub->pHaving==0 ); /* Due to the previous */ - pSub = pSub->pPrior; /* Repeat over compound */ + pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); /* If we reach this point then it is OK to perform the transformation */ @@ -7179,8 +7212,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ db = pParse->db; pCount = pExpr; pExpr = 0; - pSub = p->pSrc->a[0].sq.pSelect; - p->pSrc->a[0].sq.pSelect = 0; + pSub = sqlite3SubqueryDetach(db, pFrom); sqlite3SrcListDelete(db, p->pSrc); p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc)); while( pSub ){ @@ -7228,9 +7260,9 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ if( p0->pSTab==p1->pSTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ return 1; } - if( p1->sq.pSelect - && (p1->sq.pSelect->selFlags & SF_NestedFrom)!=0 - && sameSrcAlias(p0, p1->sq.pSelect->pSrc) + if( p1->fg.isSubquery + && (p1->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 + && sameSrcAlias(p0, p1->u4.pSubq->pSelect->pSrc) ){ return 1; } @@ -7295,7 +7327,7 @@ static int fromClauseTermCanBeCoroutine( if( i==0 ) break; i--; pItem--; - if( pItem->sq.pSelect!=0 ) return 0; /* (1c-i) */ + if( pItem->fg.isSubquery ) return 0; /* (1c-i) */ } return 1; } @@ -7445,7 +7477,7 @@ int sqlite3Select( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ SrcItem *pItem = &pTabList->a[i]; - Select *pSub = pItem->sq.pSelect; + Select *pSub = pItem->fg.isSubquery ? pItem->u4.pSubq->pSelect : 0; Table *pTab = pItem->pSTab; /* The expander should have already created transient Table objects @@ -7664,6 +7696,7 @@ int sqlite3Select( SrcItem *pItem = &pTabList->a[i]; SrcItem *pPrior; SelectDest dest; + Subquery *pSubq; Select *pSub; #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) const char *zSavedAuthContext; @@ -7699,11 +7732,14 @@ int sqlite3Select( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Generate code for all sub-queries in the FROM clause */ - pSub = pItem->sq.pSelect; - if( pSub==0 || pItem->sq.addrFillSub!=0 ) continue; + if( pItem->fg.isSubquery==0 ) continue; + pSubq = pItem->u4.pSubq; + assert( pSubq!=0 ); + pSub = pSubq->pSelect; + if( pSubq->addrFillSub!=0 ) continue; /* The code for a subquery should only be generated once. */ - assert( pItem->sq.addrFillSub==0 ); + assert( pSubq->addrFillSub==0 ); /* Increment Parse.nHeight by the height of the largest expression ** tree referred to by this, the parent select. The child select @@ -7729,8 +7765,7 @@ int sqlite3Select( sqlite3TreeViewSelect(0, p, 0); } #endif - assert( pItem->sq.pSelect - && (pItem->sq.pSelect->selFlags & SF_PushDown)!=0 ); + assert( pSubq->pSelect && (pSub->selFlags & SF_PushDown)!=0 ); }else{ TREETRACE(0x4000,pParse,p,("WHERE-lcause push-down not possible\n")); } @@ -7762,17 +7797,17 @@ int sqlite3Select( */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; - pItem->sq.regReturn = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->sq.regReturn, 0, addrTop); + pSubq->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pSubq->regReturn, 0, addrTop); VdbeComment((v, "%!S", pItem)); - pItem->sq.addrFillSub = addrTop; - sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->sq.regReturn); + pSubq->addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pSubq->regReturn); ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); pItem->pSTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; - pItem->sq.regResult = dest.iSdst; - sqlite3VdbeEndCoroutine(v, pItem->sq.regReturn); + pSubq->regResult = dest.iSdst; + sqlite3VdbeEndCoroutine(v, pSubq->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ @@ -7790,12 +7825,16 @@ int sqlite3Select( }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ /* This view has already been materialized by a prior entry in ** this same FROM clause. Reuse it. */ - if( pPrior->sq.addrFillSub ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->sq.regReturn, - pPrior->sq.addrFillSub); + Subquery *pPriorSubq; + assert( pPrior->fg.isSubquery ); + pPriorSubq = pPrior->u4.pSubq; + assert( pPriorSubq!=0 ); + if( pPriorSubq->addrFillSub ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pPriorSubq->regReturn, + pPriorSubq->addrFillSub); } sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); - pSub->nSelectRow = pPrior->sq.pSelect->nSelectRow; + pSub->nSelectRow = pPriorSubq->pSelect->nSelectRow; }else{ /* Materialize the view. If the view is not correlated, generate a ** subroutine to do the materialization so that subsequent uses of @@ -7806,9 +7845,9 @@ int sqlite3Select( int addrExplain; #endif - pItem->sq.regReturn = ++pParse->nMem; + pSubq->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp0(v, OP_Goto); - pItem->sq.addrFillSub = topAddr+1; + pSubq->addrFillSub = topAddr+1; pItem->fg.isMaterialized = 1; if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of @@ -7825,15 +7864,15 @@ int sqlite3Select( sqlite3Select(pParse, pSub, &dest); pItem->pSTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); - sqlite3VdbeAddOp2(v, OP_Return, pItem->sq.regReturn, topAddr+1); + sqlite3VdbeAddOp2(v, OP_Return, pSubq->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ CteUse *pCteUse = pItem->u2.pCteUse; - pCteUse->addrM9e = pItem->sq.addrFillSub; - pCteUse->regRtn = pItem->sq.regReturn; + pCteUse->addrM9e = pSubq->addrFillSub; + pCteUse->regRtn = pSubq->regReturn; pCteUse->iCur = pItem->iCursor; pCteUse->nRowEst = pSub->nSelectRow; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 54e4f2db73..c282023de4 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1335,6 +1335,7 @@ typedef struct Savepoint Savepoint; typedef struct Select Select; typedef struct SQLiteThread SQLiteThread; typedef struct SelectDest SelectDest; +typedef struct Subquery Subquery; typedef struct SrcItem SrcItem; typedef struct SrcList SrcList; typedef struct sqlite3_str StrAccum; /* Internal alias for sqlite3_str */ @@ -3275,6 +3276,16 @@ struct IdList { #define EU4_IDX 1 /* Uses IdList.a.u4.idx */ #define EU4_EXPR 2 /* Uses IdList.a.u4.pExpr -- NOT CURRENTLY USED */ +/* +** Details of the implementation of a subquery. +*/ +struct Subquery { + Select *pSelect; /* A SELECT statement used in place of a table name */ + int addrFillSub; /* Address of subroutine to initialize a subquery */ + int regReturn; /* Register holding return address of addrFillSub */ + int regResult; /* Registers holding results of a co-routine */ +}; + /* ** The SrcItem object represents a single term in the FROM clause of a query. ** The SrcList object is mostly an array of SrcItems. @@ -3287,10 +3298,15 @@ struct IdList { ** In the colUsed field, the high-order bit (bit 63) is set if the table ** contains more than 63 columns and the 64-th or later column is used. ** +** Intenstive use of "union" helps keep the size of the object small. This +** has been shown to boost performance due to less time spend initializing +** fields to zero when a new instance of this object is allocated. The unions +** also help SrcItem, and hence SrcList and Select, use less memory. +** ** Union member validity: ** -** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc -** u1.pFuncArg fg.isTabFunc && !fg.isIndexedBy +** u1.zIndexedBy fg.isIndexedBy +** u1.pFuncArg fg.isTabFunc ** u1.nRow !fg.isTabFunc && !fg.isIndexedBy ** ** u2.pIBIndex fg.isIndexedBy && !fg.isCte @@ -3299,23 +3315,19 @@ struct IdList { ** u3.pOn fg.isUsing==0 ** u3.pUsing fg.isUsing==1 ** -** u4.zDatabase fg.fixedSchema==0 +** u4.zDatabase fg.fixedSchema==0 && !fg.isSubquery ** u4.pSchema fg.fixedSchema==1 +** u4.pSubq fg.isSubquery */ struct SrcItem { char *zName; /* Name of the table */ char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ Table *pSTab; /* Table object for zName. Mnemonic: Srcitem-TABle */ - struct SrcItemSubquery { - Select *pSelect; /* A SELECT statement used in place of a table name */ - int addrFillSub; /* Address of subroutine to initialize a subquery */ - int regReturn; /* Register holding return address of addrFillSub */ - int regResult; /* Registers holding results of a co-routine */ - } sq; struct { u8 jointype; /* Type of join between this table and the previous */ unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ + unsigned isSubquery :1; /* True if this term is a subquery */ unsigned isTabFunc :1; /* True if table-valued-function syntax */ unsigned isCorrelated :1; /* True if sub-query is correlated */ unsigned isMaterialized:1; /* This is a materialized view */ @@ -3350,6 +3362,7 @@ struct SrcItem { union { Schema *pSchema; /* Schema to which this item is fixed */ char *zDatabase; /* Name of database holding this table */ + Subquery *pSubq; /* Description of a subquery */ } u4; }; @@ -3610,8 +3623,10 @@ struct Select { #define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ #define SF_Correlated 0x20000000 /* True if references the outer context */ -/* True if S exists and has SF_NestedFrom */ -#define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0) +/* True if SrcList item X is a subquery that has SF_NestedFrom */ +#define IsNestedFrom(X) \ + ((X)->fg.isSubquery && \ + ((X)->u4.pSubq->pSelect->selFlags&SF_NestedFrom)!=0) /* ** The results of a SELECT can be distributed in several ways, as defined @@ -5003,6 +5018,9 @@ int sqlite3IdListIndex(IdList*,const char*); SrcList *sqlite3SrcListEnlarge(Parse*, SrcList*, int, int); SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2); SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*); +void sqlite3SubqueryDelete(sqlite3*,Subquery*); +Select *sqlite3SubqueryDetach(sqlite3*,SrcItem*); +int sqlite3SrcItemAttachSubquery(Parse*, SrcItem*, Select*, int); SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, OnOrUsing*); void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); diff --git a/src/treeview.c b/src/treeview.c index 0c9fdc5266..dbd54b3466 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -230,19 +230,19 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ sqlite3StrAccumFinish(&x); sqlite3TreeViewItem(pView, zLine, inSrc-1); n = 0; - if( pItem->sq.pSelect ) n++; + if( pItem->fg.isSubquery ) n++; if( pItem->fg.isTabFunc ) n++; if( pItem->fg.isUsing ) n++; if( pItem->fg.isUsing ){ sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); } - if( pItem->sq.pSelect ){ + if( pItem->fg.isSubquery ){ if( pItem->pSTab ){ Table *pTab = pItem->pSTab; sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); } - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->sq.pSelect) ); - sqlite3TreeViewSelect(pView, pItem->sq.pSelect, (--n)>0); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) ); + sqlite3TreeViewSelect(pView, pItem->u4.pSubq->pSelect, (--n)>0); } if( pItem->fg.isTabFunc ){ sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); diff --git a/src/walker.c b/src/walker.c index 27963c2234..c8735e39b8 100644 --- a/src/walker.c +++ b/src/walker.c @@ -171,7 +171,9 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ pSrc = p->pSrc; if( ALWAYS(pSrc) ){ for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - if( pItem->sq.pSelect && sqlite3WalkSelect(pWalker, pItem->sq.pSelect) ){ + if( pItem->fg.isSubquery + && sqlite3WalkSelect(pWalker, pItem->u4.pSubq->pSelect) + ){ return WRC_Abort; } if( pItem->fg.isTabFunc diff --git a/src/where.c b/src/where.c index ded5f06816..c9f6b97c94 100644 --- a/src/where.c +++ b/src/where.c @@ -1161,9 +1161,14 @@ static SQLITE_NOINLINE void constructAutomaticIndex( /* Fill the automatic index with content */ assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] ); if( pSrc->fg.viaCoroutine ){ - int regYield = pSrc->sq.regReturn; + int regYield; + Subquery *pSubq; + assert( pSrc->fg.isSubquery ); + pSubq = pSrc->u4.pSubq; + assert( pSubq!=0 ); + regYield = pSubq->regReturn; addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->sq.addrFillSub); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); VdbeComment((v, "next row of %s", pSrc->pSTab->zName)); @@ -1188,11 +1193,12 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); if( pSrc->fg.viaCoroutine ){ + assert( pSrc->fg.isSubquery && pSrc->u4.pSubq!=0 ); sqlite3VdbeChangeP2(v, addrCounter, regBase+n); testcase( pParse->db->mallocFailed ); assert( pLevel->iIdxCur>0 ); translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, - pSrc->sq.regResult, pLevel->iIdxCur); + pSrc->u4.pSubq->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); pSrc->fg.viaCoroutine = 0; }else{ @@ -4010,9 +4016,9 @@ static int whereLoopAddBtree( #endif ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); - if( pSrc->sq.pSelect ){ + if( pSrc->fg.isSubquery ){ if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; - pNew->u.btree.pOrderBy = pSrc->sq.pSelect->pOrderBy; + pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; } rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -6421,8 +6427,8 @@ static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){ SrcItem *pItem = &pWInfo->pTabList->a[ii]; if( !pItem->fg.isCte || pItem->u2.pCteUse->eM10d!=M10d_Yes - || NEVER(pItem->sq.pSelect==0) - || pItem->sq.pSelect->pOrderBy==0 + || NEVER(pItem->fg.isSubquery==0) + || pItem->u4.pSubq->pSelect->pOrderBy==0 ){ pWInfo->revMask |= MASKBIT(ii); } @@ -7115,11 +7121,14 @@ WhereInfo *sqlite3WhereBegin( wsFlags = pLevel->pWLoop->wsFlags; pSrc = &pTabList->a[pLevel->iFrom]; if( pSrc->fg.isMaterialized ){ + Subquery *pSubq; + assert( pSrc->fg.isSubquery ); + pSubq = pSrc->u4.pSubq; if( pSrc->fg.isCorrelated ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->sq.regReturn,pSrc->sq.addrFillSub); + sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub); }else{ int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->sq.regReturn,pSrc->sq.addrFillSub); + sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub); sqlite3VdbeJumpHere(v, iOnce); } } @@ -7334,7 +7343,8 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ assert( pLevel->iTabCur==pSrc->iCursor ); if( pSrc->fg.viaCoroutine ){ int m, n; - n = pSrc->sq.regResult; + assert( pSrc->fg.isSubquery ); + n = pSrc->u4.pSubq->regResult; assert( pSrc->pSTab!=0 ); m = pSrc->pSTab->nCol; sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1); @@ -7388,9 +7398,10 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ */ if( pTabItem->fg.viaCoroutine ){ testcase( pParse->db->mallocFailed ); - assert( pTabItem->sq.regResult>=0 ); + assert( pTabItem->fg.isSubquery ); + assert( pTabItem->u4.pSubq->regResult>=0 ); translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, - pTabItem->sq.regResult, 0); + pTabItem->u4.pSubq->regResult, 0); continue; } diff --git a/src/wherecode.c b/src/wherecode.c index ebc71d1102..b0c0d9aedd 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -317,7 +317,9 @@ void sqlite3WhereAddScanStatus( sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); } }else{ - int addr = pSrclist->a[pLvl->iFrom].sq.addrFillSub; + int addr; + assert( pSrclist->a[pLvl->iFrom].fg.isSubquery ); + addr = pSrclist->a[pLvl->iFrom].u4.pSubq->addrFillSub; VdbeOp *pOp = sqlite3VdbeGetOp(v, addr-1); assert( sqlite3VdbeDb(v)->mallocFailed || pOp->opcode==OP_InitCoroutine ); assert( sqlite3VdbeDb(v)->mallocFailed || pOp->p2>addr ); @@ -1510,8 +1512,12 @@ Bitmask sqlite3WhereCodeOneLoopStart( /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->sq.regReturn; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield,0,pTabItem->sq.addrFillSub); + int regYield; + Subquery *pSubq; + assert( pTabItem->fg.isSubquery && pTabItem->u4.pSubq!=0 ); + pSubq = pTabItem->u4.pSubq; + regYield = pSubq->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub); pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); VdbeCoverage(v); VdbeComment((v, "next row of %s", pTabItem->pSTab->zName)); @@ -2819,9 +2825,13 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( pRight = &pWInfo->pTabList->a[pWInfo->a[k].iFrom]; mAll |= pWInfo->a[k].pWLoop->maskSelf; if( pRight->fg.viaCoroutine ){ + Subquery *pSubq; + assert( pRight->fg.isSubquery && pRight->u4.pSubq!=0 ); + pSubq = pRight->u4.pSubq; + assert( pSubq->pSelect!=0 && pSubq->pSelect->pEList!=0 ); sqlite3VdbeAddOp3( - v, OP_Null, 0, pRight->sq.regResult, - pRight->sq.regResult + pRight->sq.pSelect->pEList->nExpr-1 + v, OP_Null, 0, pSubq->regResult, + pSubq->regResult + pSubq->pSelect->pEList->nExpr-1 ); } sqlite3VdbeAddOp1(v, OP_NullRow, pWInfo->a[k].iTabCur); diff --git a/src/whereexpr.c b/src/whereexpr.c index 766754828d..ae26e85d25 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -958,7 +958,9 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ if( ALWAYS(pSrc!=0) ){ int i; for(i=0; inSrc; i++){ - mask |= exprSelectUsage(pMaskSet, pSrc->a[i].sq.pSelect); + if( pSrc->a[i].fg.isSubquery ){ + mask |= exprSelectUsage(pMaskSet, pSrc->a[i].u4.pSubq->pSelect); + } if( pSrc->a[i].fg.isUsing==0 ){ mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].u3.pOn); } diff --git a/src/window.c b/src/window.c index 9829d1740b..d4083beeb3 100644 --- a/src/window.c +++ b/src/window.c @@ -1077,9 +1077,10 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ assert( pSub!=0 || p->pSrc==0 ); /* Due to db->mallocFailed test inside ** of sqlite3DbMallocRawNN() called from ** sqlite3SrcListAppend() */ - if( p->pSrc ){ + if( p->pSrc==0 ){ + sqlite3SelectDelete(db, pSub); + }else if( sqlite3SrcItemAttachSubquery(pParse, &p->pSrc->a[0], pSub, 0) ){ Table *pTab2; - p->pSrc->a[0].sq.pSelect = pSub; p->pSrc->a[0].fg.isCorrelated = 1; sqlite3SrcListAssignCursors(pParse, p->pSrc); pSub->selFlags |= SF_Expanded|SF_OrderByReqd; @@ -1101,8 +1102,6 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ w.xSelectCallback2 = sqlite3WalkerDepthDecrease; sqlite3WalkSelect(&w, pSub); } - }else{ - sqlite3SelectDelete(db, pSub); } if( db->mallocFailed ) rc = SQLITE_NOMEM; @@ -1389,10 +1388,15 @@ int sqlite3WindowCompare( ** and initialize registers and cursors used by sqlite3WindowCodeStep(). */ void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){ - int nEphExpr = pSelect->pSrc->a[0].sq.pSelect->pEList->nExpr; - Window *pMWin = pSelect->pWin; Window *pWin; - Vdbe *v = sqlite3GetVdbe(pParse); + int nEphExpr; + Window *pMWin; + Vdbe *v; + + assert( pSelect->pSrc->a[0].fg.isSubquery ); + nEphExpr = pSelect->pSrc->a[0].u4.pSubq->pSelect->pEList->nExpr; + pMWin = pSelect->pWin; + v = sqlite3GetVdbe(pParse); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, nEphExpr); sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+1, pMWin->iEphCsr); From 1c8ed5fa43a0d588874ab9006faf584e88303714 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 19 Aug 2024 23:43:08 +0000 Subject: [PATCH 42/99] Earlier error detection while processing complex aggregate queries. dbsqlfuzz 5242c2f07f4aa031aa3c80461f18e9b7619ede9b FossilOrigin-Name: 70f4973078ffc72f4ff247234e6f8e695b40803c3e7c9ed12050d97195728352 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 3 +++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f8108ac374..07c8d04347 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\ssome\scomments\sthat\sare\sprocessed\sinto\sthe\sfts5.html\swebpage. -D 2024-08-19T20:35:30.209 +C Earlier\serror\sdetection\swhile\sprocessing\scomplex\saggregate\nqueries.\s\sdbsqlfuzz\s5242c2f07f4aa031aa3c80461f18e9b7619ede9b +D 2024-08-19T23:43:08.285 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -764,7 +764,7 @@ F src/printf.c 8b250972305e14b365561be5117ed0fd364e4fd58968776df1ce64c6280b90f9 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 0aee8a2e5340ba95a966917305dfaff5147fcad78d0839cd364b16e4746b8bcb F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 6a95a2bffa6c09584dea99db5a7ae10c813305c09c92920ffc54f6eae2ba399e +F src/select.c 7e907830b01c7fd34968514067aed1eac13e47da4bc1a936ef2020c6b866cc08 F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab67704 F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b9632900100bdbc913f83bfb03b32585cf07a192b9a7f26b9bebc7d91e63a610 -R ea532ddb7482adbca514ac43836cd1cf -U dan -Z 0d4360caeae428f14a1f60694bd440b2 +P 3e06ab218bbd1ed75a24afb44e8df9ce84e9fc24701428cb8b3459760f44006d +R 2c7a18e25ae50797bdba35997432ce72 +U drh +Z 62ca2500b12eeccb94ffabbf52380d6e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 39ef92242c..ea720a0f9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e06ab218bbd1ed75a24afb44e8df9ce84e9fc24701428cb8b3459760f44006d +70f4973078ffc72f4ff247234e6f8e695b40803c3e7c9ed12050d97195728352 diff --git a/src/select.c b/src/select.c index 4b0b554295..c1246d3c2a 100644 --- a/src/select.c +++ b/src/select.c @@ -6747,6 +6747,7 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); + if( pParse->nErr ) return; pList = pF->pFExpr->x.pList; if( pF->iOBTab>=0 ){ /* For an ORDER BY aggregate, calls to OP_AggStep were deferred. Inputs @@ -6956,6 +6957,7 @@ static void updateAccumulator( if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); } + if( pParse->nErr ) return; } if( regHit==0 && pAggInfo->nAccumulator ){ regHit = regAcc; @@ -6965,6 +6967,7 @@ static void updateAccumulator( } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); + if( pParse->nErr ) return; } pAggInfo->directMode = 0; From 6ad4e9fd2a45bd5204b55c73179c6e06689312eb Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Aug 2024 12:09:55 +0000 Subject: [PATCH 43/99] Fix for the UTF-16LE problem reporte by [forum:/forumpost/dc7e1b7527e84343|forum post dc7e1b7527e84343]. Because of the unexpected ordering of characters using the default collation (memcmp() order) in UTF-16LE, the LIKE/GLOB optimization restricts its attention to the pattern prefix that is all ASCII, which is the common case. FossilOrigin-Name: a5797ebdea423afc3d2d3bd8adaf1575d33a01f594c0c315afcb1499f1718e9b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/whereexpr.c | 13 +++++++++++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 07c8d04347..a5207a4320 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Earlier\serror\sdetection\swhile\sprocessing\scomplex\saggregate\nqueries.\s\sdbsqlfuzz\s5242c2f07f4aa031aa3c80461f18e9b7619ede9b -D 2024-08-19T23:43:08.285 +C Fix\sfor\sthe\sUTF-16LE\sproblem\sreporte\sby\n[forum:/forumpost/dc7e1b7527e84343|forum\spost\sdc7e1b7527e84343].\s\sBecause\sof\nthe\sunexpected\sordering\sof\scharacters\susing\sthe\sdefault\scollation\s(memcmp()\norder)\sin\sUTF-16LE,\sthe\sLIKE/GLOB\soptimization\srestricts\sits\sattention\sto\nthe\spattern\sprefix\sthat\sis\sall\sASCII,\swhich\sis\sthe\scommon\scase. +D 2024-08-20T12:09:55.604 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -853,7 +853,7 @@ F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 F src/where.c f5be664f3379c9f930696e339ec4ef4c1187af860cca727411156101fae6b677 F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e F src/wherecode.c c9cac0b0b8e809c5e7e79d7796918907fb685ad99be2aaa9737f9787aa47349c -F src/whereexpr.c 7d0d34b42b9edfd8e8ca66beb3a6ef63fe211c001af54caf2ccbcd989b783290 +F src/whereexpr.c f8a4fa8846aa65289be2077f66c58487ec3a00579426117170438501b5d8a7df F src/window.c 1e40ffc509bae21e466f6106382d238e91eb73edd4ba10e66ca4fd7af2b96896 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3e06ab218bbd1ed75a24afb44e8df9ce84e9fc24701428cb8b3459760f44006d -R 2c7a18e25ae50797bdba35997432ce72 +P 70f4973078ffc72f4ff247234e6f8e695b40803c3e7c9ed12050d97195728352 +R 300d8d51d39fe8e4da3c8738b6efbd9d U drh -Z 62ca2500b12eeccb94ffabbf52380d6e +Z bb9edd492b1f36638d01451d6aaeeec1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ea720a0f9c..d71d3e2329 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -70f4973078ffc72f4ff247234e6f8e695b40803c3e7c9ed12050d97195728352 +a5797ebdea423afc3d2d3bd8adaf1575d33a01f594c0c315afcb1499f1718e9b diff --git a/src/whereexpr.c b/src/whereexpr.c index dcda75d264..4a7cdb3ed7 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -220,11 +220,20 @@ static int isLikeOrGlob( } if( z ){ - /* Count the number of prefix characters prior to the first wildcard */ + /* Count the number of prefix characters prior to the first wildcard. + ** If the underlying database has a UTF16LE encoding, then only consider + ** ASCII characters. Note that the encoding of z[] is UTF8 - we are + ** dealing with only UTF8 here in this code, but the database engine + ** itself might be processing content using a different encoding. */ cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; - if( c==wc[3] && z[cnt]!=0 ) cnt++; + if( c==wc[3] && z[cnt]!=0 ){ + cnt++; + }else if( c>=0x80 && ENC(db)==SQLITE_UTF16LE ){ + cnt--; + break; + } } /* The optimization is possible only if (1) the pattern does not begin From 781163aa9dce91608ba33af479564f83dda1bd40 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Aug 2024 14:12:16 +0000 Subject: [PATCH 44/99] When the database encoding is UTF-16LE and the GLOB optimization is used, it is ok to use the range search over an index, but it is not ok to disable the actual GLOB function call. FossilOrigin-Name: 3399698376761ab8c422f8ea02bfa2759afb606f08bedbd1cf7eee834229a9aa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/whereexpr.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a5207a4320..e5de88085a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sfor\sthe\sUTF-16LE\sproblem\sreporte\sby\n[forum:/forumpost/dc7e1b7527e84343|forum\spost\sdc7e1b7527e84343].\s\sBecause\sof\nthe\sunexpected\sordering\sof\scharacters\susing\sthe\sdefault\scollation\s(memcmp()\norder)\sin\sUTF-16LE,\sthe\sLIKE/GLOB\soptimization\srestricts\sits\sattention\sto\nthe\spattern\sprefix\sthat\sis\sall\sASCII,\swhich\sis\sthe\scommon\scase. -D 2024-08-20T12:09:55.604 +C When\sthe\sdatabase\sencoding\sis\sUTF-16LE\sand\sthe\sGLOB\soptimization\sis\sused,\nit\sis\sok\sto\suse\sthe\srange\ssearch\sover\san\sindex,\sbut\sit\sis\snot\sok\sto\ndisable\sthe\sactual\sGLOB\sfunction\scall. +D 2024-08-20T14:12:16.713 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -853,7 +853,7 @@ F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 F src/where.c f5be664f3379c9f930696e339ec4ef4c1187af860cca727411156101fae6b677 F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e F src/wherecode.c c9cac0b0b8e809c5e7e79d7796918907fb685ad99be2aaa9737f9787aa47349c -F src/whereexpr.c f8a4fa8846aa65289be2077f66c58487ec3a00579426117170438501b5d8a7df +F src/whereexpr.c 26a5a798b3f676447ae54a27584c43b32abfebb8bbcaf6faffbb7ee1da486f2d F src/window.c 1e40ffc509bae21e466f6106382d238e91eb73edd4ba10e66ca4fd7af2b96896 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 70f4973078ffc72f4ff247234e6f8e695b40803c3e7c9ed12050d97195728352 -R 300d8d51d39fe8e4da3c8738b6efbd9d +P a5797ebdea423afc3d2d3bd8adaf1575d33a01f594c0c315afcb1499f1718e9b +R 0a328206336eacdda27ba4526bd5a010 U drh -Z bb9edd492b1f36638d01451d6aaeeec1 +Z 4c7bd06b974e78d392c3b37a21911656 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d71d3e2329..7fb9914a05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a5797ebdea423afc3d2d3bd8adaf1575d33a01f594c0c315afcb1499f1718e9b +3399698376761ab8c422f8ea02bfa2759afb606f08bedbd1cf7eee834229a9aa diff --git a/src/whereexpr.c b/src/whereexpr.c index 4a7cdb3ed7..d44085a783 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -248,7 +248,7 @@ static int isLikeOrGlob( Expr *pPrefix; /* A "complete" match if the pattern ends with "*" or "%" */ - *pisComplete = c==wc[0] && z[cnt+1]==0; + *pisComplete = c==wc[0] && z[cnt+1]==0 && ENC(db)!=SQLITE_UTF16LE; /* Get the pattern prefix. Remove all escapes from the prefix. */ pPrefix = sqlite3Expr(db, TK_STRING, (char*)z); From ff4ad29d05156598d70b8328cbfb352114446bc3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Aug 2024 16:50:21 +0000 Subject: [PATCH 45/99] Memory issues fixes so that mdevtest now passes. FossilOrigin-Name: b249516f6a6650808e035ec219e762bf24dcb7c32a6235984b995d08d07d35d8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 6 ++++-- src/insert.c | 8 +++++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 30b1edeab1..06a26110e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\sSrcItem\sobject\sso\sthat\sinformation\sabout\ssubqueries\sis\sstored\nin\sa\sseparately\sallocated\sSubquery\sobject.\s\sThis\sreduces\sthe\smemory\srequirements\nfor\sSrcItem\sand\smakes\sthe\scode\srun\sfaster.\s\sIt\salso\sprovides\san\sexpansion\spath\nfor\ssubquery\sprocessing\sthat\sdoes\snot\sburden\ssimple\squeries.\s\sThe\scurrent\nchecking\smostly\sworks,\sbut\sthere\sare\sstill\sissues\sthat\sneed\sto\sbe\stracked\ndown\sand\sfixed. -D 2024-08-19T22:48:30.539 +C Memory\sissues\sfixes\sso\sthat\smdevtest\snow\spasses. +D 2024-08-20T16:50:21.429 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -702,7 +702,7 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c 8b42fc7d9efdb2df05c30e8f91ff6cfbd979724ae24bf90269028468b7a13333 F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 -F src/build.c d243a17292e027c1d662e992fbd395123aacc4b74a4a9419d0b010088ba318dd +F src/build.c 29ebc26d90a9d6e860a3018a6d61fb667c80ec8e80e3798c4164d2349ba28650 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 @@ -719,7 +719,7 @@ F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c b07865339e67869ff12d2eee3a1417f4779c742e2a51ae3d14ad45e5168f6f41 +F src/insert.c 37c0c12c6eea72c4dad77699b963be370ca6cbd7d18270ff04f5fe9d6031622d F src/json.c 5b6a1d6015997b9ee848a32948720bdb26a0ef2de5a2127ebf7355ce66dbdc0d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9f5aeef3cbc2c95267c8f7bf60d5c66971a76789669fb0e8f853273ff6f616f2 -R 0dac77664f5c62d0dcc882ea3148d6f4 +P 8ff5dda8448d7e1a533d7f27db2573ce68fa9956b9d9847ced45e83c1f06bab0 +R c585d13f1a80cb4428e40ac20e746454 U drh -Z a11cd0d49c593b1144d9510fd31aa2a7 +Z ce4831b3d2660999f30841c855773e8f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ca5d536722..90e67e8d3a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8ff5dda8448d7e1a533d7f27db2573ce68fa9956b9d9847ced45e83c1f06bab0 +b249516f6a6650808e035ec219e762bf24dcb7c32a6235984b995d08d07d35d8 diff --git a/src/build.c b/src/build.c index b585c19719..9cbedf1ec4 100644 --- a/src/build.c +++ b/src/build.c @@ -5005,11 +5005,13 @@ int sqlite3SrcItemAttachSubquery( Subquery *p; if( pSelect==0 ) return SQLITE_OK; assert( pItem->fg.isSubquery==0 ); - if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + if( pItem->fg.fixedSchema ){ + pItem->u4.pSchema = 0; + pItem->fg.fixedSchema = 0; + }else if( pItem->u4.zDatabase!=0 ){ sqlite3DbFree(pParse->db, pItem->u4.zDatabase); pItem->u4.zDatabase = 0; } - pItem->fg.fixedSchema = 0; if( dupSelect ){ pSelect = sqlite3SelectDup(pParse->db, pSelect, 0); if( pSelect==0 ) return 0; diff --git a/src/insert.c b/src/insert.c index d65ead2a0b..8214cd9c5c 100644 --- a/src/insert.c +++ b/src/insert.c @@ -585,9 +585,11 @@ void sqlite3AutoincrementEnd(Parse *pParse){ void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ if( ALWAYS(pVal) && pVal->pSrc->nSrc>0 ){ SrcItem *pItem = &pVal->pSrc->a[0]; - assert( pItem->fg.isSubquery && pItem->u4.pSubq!=0 ); - sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->u4.pSubq->regReturn); - sqlite3VdbeJumpHere(pParse->pVdbe, pItem->u4.pSubq->addrFillSub - 1); + assert( (pItem->fg.isSubquery && pItem->u4.pSubq!=0) || pParse->nErr ); + if( pItem->fg.isSubquery ){ + sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->u4.pSubq->regReturn); + sqlite3VdbeJumpHere(pParse->pVdbe, pItem->u4.pSubq->addrFillSub - 1); + } } } From 27c680cc69f181a548d942168e45e37b498684a5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Aug 2024 17:24:42 +0000 Subject: [PATCH 46/99] Fix a memory leak so that now all dev tests (including TH3) are passing. FossilOrigin-Name: 3995c45fff0e4bd10bcf49cc16eb1050216426723c63ba56d3ae5ba738c34019 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 06a26110e0..a976440679 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Memory\sissues\sfixes\sso\sthat\smdevtest\snow\spasses. -D 2024-08-20T16:50:21.429 +C Fix\sa\smemory\sleak\sso\sthat\snow\sall\sdev\stests\s(including\sTH3)\sare\spassing. +D 2024-08-20T17:24:42.323 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -760,7 +760,7 @@ F src/printf.c 17054fb94ffcf7a28362e9a5af4a0f813bd0c52200ae408eeebddc81feed9274 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 3e284895745f57de3b37b17fce9eb0e26bc54fccc94ea1ff5342be4360e3674b +F src/select.c 9aa44e562f2fbfa33e600e3f935369dd9a07e4beb89b344dfc7187dc2adc19ce F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab67704 F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8ff5dda8448d7e1a533d7f27db2573ce68fa9956b9d9847ced45e83c1f06bab0 -R c585d13f1a80cb4428e40ac20e746454 +P b249516f6a6650808e035ec219e762bf24dcb7c32a6235984b995d08d07d35d8 +R 5330111931e3ecff89186b5f2d6b0982 U drh -Z ce4831b3d2660999f30841c855773e8f +Z d58fa1c081ceed524dfd09ccfa304c4e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 90e67e8d3a..0ffd5cf0fb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b249516f6a6650808e035ec219e762bf24dcb7c32a6235984b995d08d07d35d8 +3995c45fff0e4bd10bcf49cc16eb1050216426723c63ba56d3ae5ba738c34019 diff --git a/src/select.c b/src/select.c index 8bddc03bb4..0c10550fac 100644 --- a/src/select.c +++ b/src/select.c @@ -5600,12 +5600,17 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ /* If we reach this point, that means the transformation is required. */ pParse = pWalker->pParse; + if( pParse->nErr ) return WRC_Abort; db = pParse->db; pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); if( pNew==0 ) return WRC_Abort; memset(&dummy, 0, sizeof(dummy)); pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0); - if( pNewSrc==0 ) return WRC_Abort; + assert( pNewSrc!=0 || pParse->nErr ); + if( pParse->nErr ){ + sqlite3SrcListDelete(db, pNewSrc); + return WRC_Abort; + } *pNew = *p; p->pSrc = pNewSrc; p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); From ff176231da8496e9fec72fe147eb60316f9c7f60 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Aug 2024 17:41:44 +0000 Subject: [PATCH 47/99] Fix some stray SrcItem field name fixes. FossilOrigin-Name: bc5f5ce59e1e4323422bda7d002310923f927a03b4fd42749bf04f6c9853956b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/delete.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a976440679..614e098984 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sso\sthat\snow\sall\sdev\stests\s(including\sTH3)\sare\spassing. -D 2024-08-20T17:24:42.323 +C Fix\ssome\sstray\sSrcItem\sfield\sname\sfixes. +D 2024-08-20T17:41:44.355 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -709,7 +709,7 @@ F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 F src/date.c 13dd752847afb32ed70510ad7345a5b9c841f51ad904dba5d010f1fa3a6a324e F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 -F src/delete.c 8aaf9d05d6ff6349a7a6001cbf90c7a46b19b9f52ef2e47aa0415a529d5307a1 +F src/delete.c 3638367ea4ed5d422ee1728a74f0b7a2973af9d7eef9b8b959c9c1f6e761ab8d F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 849049c74a4c68961154c124087b5c67d277217a2e642570d0c1bd6336859940 @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b249516f6a6650808e035ec219e762bf24dcb7c32a6235984b995d08d07d35d8 -R 5330111931e3ecff89186b5f2d6b0982 +P 3995c45fff0e4bd10bcf49cc16eb1050216426723c63ba56d3ae5ba738c34019 +R 652ce342076e266e8f982c0969e3c6bc U drh -Z d58fa1c081ceed524dfd09ccfa304c4e +Z ae5f81dac74fb395c307223c199b02f0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0ffd5cf0fb..fa4bad2218 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3995c45fff0e4bd10bcf49cc16eb1050216426723c63ba56d3ae5ba738c34019 +bc5f5ce59e1e4323422bda7d002310923f927a03b4fd42749bf04f6c9853956b diff --git a/src/delete.c b/src/delete.c index 010a900191..1aa67c6a62 100644 --- a/src/delete.c +++ b/src/delete.c @@ -24,7 +24,7 @@ ** ** The following fields are initialized appropriate in pSrc: ** -** pSrc->a[0].pTab Pointer to the Table object +** pSrc->a[0].spTab Pointer to the Table object ** pSrc->a[0].pIndex Pointer to the INDEXED BY index, if there is one ** */ @@ -219,7 +219,7 @@ Expr *sqlite3LimitWhere( ** ); */ - pTab = pSrc->a[0].pTab; + pTab = pSrc->a[0].pSTab; if( HasRowid(pTab) ){ pLhs = sqlite3PExpr(pParse, TK_ROW, 0, 0); pEList = sqlite3ExprListAppend( @@ -252,9 +252,9 @@ Expr *sqlite3LimitWhere( /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ - pSrc->a[0].pTab = 0; + pSrc->a[0].pSTab = 0; pSelectSrc = sqlite3SrcListDup(db, pSrc, 0); - pSrc->a[0].pTab = pTab; + pSrc->a[0].pSTab = pTab; if( pSrc->a[0].fg.isIndexedBy ){ assert( pSrc->a[0].fg.isCte==0 ); pSrc->a[0].u2.pIBIndex = 0; From 692c160360c688c4bee3dd13d5293f2c416ad92f Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Aug 2024 19:09:59 +0000 Subject: [PATCH 48/99] Tighter checking of access constraints on union members in SrcItem. Improved invariant checking. FossilOrigin-Name: fd72d3400a8fe5747f494eee81654698acee350bb95b9db269e87d857af03492 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/build.c | 11 ++++++++++- src/delete.c | 2 +- src/fkey.c | 2 +- src/insert.c | 1 + src/select.c | 4 ++++ src/sqliteInt.h | 30 +++++++++++++++++------------- src/trigger.c | 6 ++++-- 9 files changed, 51 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 614e098984..298d9f0ab3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sstray\sSrcItem\sfield\sname\sfixes. -D 2024-08-20T17:41:44.355 +C Tighter\schecking\sof\saccess\sconstraints\son\sunion\smembers\sin\sSrcItem.\nImproved\sinvariant\schecking. +D 2024-08-20T19:09:59.313 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -702,24 +702,24 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c 8b42fc7d9efdb2df05c30e8f91ff6cfbd979724ae24bf90269028468b7a13333 F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 -F src/build.c 29ebc26d90a9d6e860a3018a6d61fb667c80ec8e80e3798c4164d2349ba28650 +F src/build.c 69d79a255a6a0547f5e99a1726c3cbd363fa221c4f0458425abc7ba59efb9f83 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 F src/date.c 13dd752847afb32ed70510ad7345a5b9c841f51ad904dba5d010f1fa3a6a324e F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 -F src/delete.c 3638367ea4ed5d422ee1728a74f0b7a2973af9d7eef9b8b959c9c1f6e761ab8d +F src/delete.c cfe1a09cd67e3c8aef51ef928c69ba2824e6e2439f451e990afbb89e774add01 F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c 849049c74a4c68961154c124087b5c67d277217a2e642570d0c1bd6336859940 +F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f F src/func.c 1f61e32e7a357e615b5d2e774bee563761fce4f2fd97ecb0f72c33e62a2ada5f F src/global.c 61a419dd9e993b9be0f91de4c4ccf322b053eb829868e089f0321dd669be3b90 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 37c0c12c6eea72c4dad77699b963be370ca6cbd7d18270ff04f5fe9d6031622d +F src/insert.c f8d1a0f8ee258411009c6b7f2d93170e351bd19f5ad89d57e1180644297cbe70 F src/json.c 5b6a1d6015997b9ee848a32948720bdb26a0ef2de5a2127ebf7355ce66dbdc0d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 @@ -760,12 +760,12 @@ F src/printf.c 17054fb94ffcf7a28362e9a5af4a0f813bd0c52200ae408eeebddc81feed9274 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 9aa44e562f2fbfa33e600e3f935369dd9a07e4beb89b344dfc7187dc2adc19ce +F src/select.c 7addcd53d8f9bef2d47a7e47faeb1f6e401dba02f1f338f532298a24657ea4e6 F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab67704 F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 630e356b7d94df1dcbc32d4f7d4569725a12c6de014fe064311219c22d9cbd54 +F src/sqliteInt.h 521a9abd808e2b6c2e377cfc85b5ad5508202c33f878f929dccf60d6aca13034 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -825,7 +825,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68 F src/treeview.c e98dbc8068d27f3d0bd2a4cebfce6a7c21c776edc23d80e12c67a70cd2ba3fe5 -F src/trigger.c 68d849ea6ccab67beb450d1a0d065c6118f0884a779388b07f0d9a27dbda38c9 +F src/trigger.c 0bb986a5b96047fd597c6aac28588853df56064e576e6b81ba777ef2ccaac461 F src/update.c 0e01aa6a3edf9ec112b33eb714b9016a81241497b1fb7c3e74332f4f71756508 F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3995c45fff0e4bd10bcf49cc16eb1050216426723c63ba56d3ae5ba738c34019 -R 652ce342076e266e8f982c0969e3c6bc +P bc5f5ce59e1e4323422bda7d002310923f927a03b4fd42749bf04f6c9853956b +R ee08ce1ecc132f8a5f905e65ae160cc1 U drh -Z ae5f81dac74fb395c307223c199b02f0 +Z 01685802d904a82a17f4e5855c564d7e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fa4bad2218..06a9d2670a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bc5f5ce59e1e4323422bda7d002310923f927a03b4fd42749bf04f6c9853956b +fd72d3400a8fe5747f494eee81654698acee350bb95b9db269e87d857af03492 diff --git a/src/build.c b/src/build.c index 9cbedf1ec4..b2386035d8 100644 --- a/src/build.c +++ b/src/build.c @@ -501,6 +501,7 @@ Table *sqlite3LocateTableItem( int iDb = sqlite3SchemaToIndex(pParse->db, p->u4.pSchema); zDb = pParse->db->aDb[iDb].zDbSName; }else{ + assert( !p->fg.isSubquery ); zDb = p->u4.zDatabase; } return sqlite3LocateTable(pParse, flags, p->zName, zDb); @@ -3487,6 +3488,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.isSubquery==0 ); if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; assert( isView==0 || isView==LOCATE_VIEW ); @@ -4587,6 +4589,7 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ assert( pParse->nErr==0 ); /* Never called with prior non-OOM errors */ assert( pName->nSrc==1 ); assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.isSubquery==0 ); if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto exit_drop_index; } @@ -4893,6 +4896,7 @@ SrcList *sqlite3SrcListAppend( pDatabase = 0; } assert( pItem->fg.fixedSchema==0 ); + assert( pItem->fg.isSubquery==0 ); if( pDatabase ){ pItem->zName = sqlite3NameFromToken(db, pDatabase); pItem->u4.zDatabase = sqlite3NameFromToken(db, pTable); @@ -4959,8 +4963,13 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ for(pItem=pList->a, i=0; inSrc; i++, pItem++){ /* Check invariants on SrcItem */ + assert( !pItem->fg.isIndexedBy || !pItem->fg.isTabFunc ); + assert( !pItem->fg.isCte || !pItem->fg.isIndexedBy ); assert( !pItem->fg.hadSchema || !pItem->fg.isSubquery ); - assert( pItem->fg.hadSchema==0 || pItem->fg.fixedSchema==1 ); + assert( !pItem->fg.hadSchema || pItem->fg.fixedSchema ); + assert( !pItem->fg.fixedSchema || !pItem->fg.isSubquery ); + assert( !pItem->fg.isSubquery || (pItem->u4.pSubq!=0 && + pItem->u4.pSubq->pSelect!=0) ); if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); diff --git a/src/delete.c b/src/delete.c index 1aa67c6a62..1d7ba66a22 100644 --- a/src/delete.c +++ b/src/delete.c @@ -156,7 +156,7 @@ void sqlite3MaterializeView( if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); - assert( pFrom->a[0].fg.fixedSchema==0 ); + assert( pFrom->a[0].fg.fixedSchema==0 && pFrom->a[0].fg.isSubquery==0 ); pFrom->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); assert( pFrom->a[0].fg.isUsing==0 ); assert( pFrom->a[0].u3.pOn==0 ); diff --git a/src/fkey.c b/src/fkey.c index cd1f88502e..f1117a8845 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -1337,7 +1337,7 @@ static Trigger *fkActionTrigger( if( pSrc ){ assert( pSrc->nSrc==1 ); pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); - assert( pSrc->a[0].fg.fixedSchema==0 ); + assert( pSrc->a[0].fg.fixedSchema==0 && pSrc->a[0].fg.isSubquery==0 ); pSrc->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); } pSelect = sqlite3SelectNew(pParse, diff --git a/src/insert.c b/src/insert.c index 8214cd9c5c..d380281bed 100644 --- a/src/insert.c +++ b/src/insert.c @@ -729,6 +729,7 @@ Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ p = &pRet->pSrc->a[0]; p->fg.viaCoroutine = 1; p->iCursor = -1; + assert( !p->fg.isIndexedBy && !p->fg.isTabFunc ); p->u1.nRow = 2; if( sqlite3SrcItemAttachSubquery(pParse, p, pLeft, 0) ){ pSubq = p->u4.pSubq; diff --git a/src/select.c b/src/select.c index 0c10550fac..39955dd60a 100644 --- a/src/select.c +++ b/src/select.c @@ -4522,6 +4522,7 @@ static int flattenSubquery( }else{ pSub1 = 0; } + assert( pSubitem->fg.isSubquery==0 ); if( pSubitem->fg.fixedSchema==0 ){ sqlite3DbFree(db, pSubitem->u4.zDatabase); pSubitem->u4.zDatabase = 0; @@ -5812,6 +5813,7 @@ static int resolveFromTermToCte( sqlite3ErrorMsg(pParse, "no such index: \"%s\"", pFrom->u1.zIndexedBy); return 2; } + assert( !pFrom->fg.isIndexedBy ); pFrom->fg.isCte = 1; pFrom->u2.pCteUse = pCteUse; pCteUse->nUse++; @@ -7728,6 +7730,8 @@ int sqlite3Select( if( pItem->fg.fixedSchema ){ int iDb = sqlite3SchemaToIndex(pParse->db, pItem->u4.pSchema); zDb = db->aDb[iDb].zDbSName; + }else if( pItem->fg.isSubquery ){ + zDb = 0; }else{ zDb = pItem->u4.zDatabase; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c282023de4..9e0faf9cc1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3298,26 +3298,30 @@ struct Subquery { ** In the colUsed field, the high-order bit (bit 63) is set if the table ** contains more than 63 columns and the 64-th or later column is used. ** -** Intenstive use of "union" helps keep the size of the object small. This -** has been shown to boost performance due to less time spend initializing -** fields to zero when a new instance of this object is allocated. The unions -** also help SrcItem, and hence SrcList and Select, use less memory. -** -** Union member validity: +** Aggressive use of "union" helps keep the size of the object small. This +** has been shown to boost performance, in addition to saving memory. +** Access to union elements is gated by the following rules which should +** always be checked, either by an if-statement or by an assert(). ** +** Field Only access if this is true +** --------------- ----------------------------------- ** u1.zIndexedBy fg.isIndexedBy ** u1.pFuncArg fg.isTabFunc ** u1.nRow !fg.isTabFunc && !fg.isIndexedBy ** -** u2.pIBIndex fg.isIndexedBy && !fg.isCte -** u2.pCteUse fg.isCte && !fg.isIndexedBy +** u2.pIBIndex fg.isIndexedBy +** u2.pCteUse fg.isCte ** -** u3.pOn fg.isUsing==0 -** u3.pUsing fg.isUsing==1 +** u3.pOn !fg.isUsing +** u3.pUsing fg.isUsing ** -** u4.zDatabase fg.fixedSchema==0 && !fg.isSubquery -** u4.pSchema fg.fixedSchema==1 +** u4.zDatabase !fg.fixedSchema && !fg.isSubquery +** u4.pSchema fg.fixedSchema ** u4.pSubq fg.isSubquery +** +** See also the sqlite3SrcListDelete() routine for assert() statements that +** check invariants on the fields of this object, especially the flags +** inside the fg struct. */ struct SrcItem { char *zName; /* Name of the table */ @@ -3342,7 +3346,7 @@ struct SrcItem { unsigned isNestedFrom :1; /* pSelect is a SF_NestedFrom subquery */ unsigned rowidUsed :1; /* The ROWID of this table is referenced */ unsigned fixedSchema :1; /* Uses u4.pSchema, not u4.zDatabase */ - unsigned hadSchema :1; /* Has u4.zDatabase before u4.pSchema */ + unsigned hadSchema :1; /* Had u4.zDatabase before u4.pSchema */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ Bitmask colUsed; /* Bit N set if column N used. Details above for N>62 */ diff --git a/src/trigger.c b/src/trigger.c index 170f094eab..ff2df82cbc 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -151,7 +151,9 @@ void sqlite3BeginTrigger( ** To maintain backwards compatibility, ignore the database ** name on pTableName if we are reparsing out of the schema table */ - if( db->init.busy && iDb!=1 && ALWAYS(pTableName->a[0].fg.fixedSchema==0) ){ + if( db->init.busy && iDb!=1 ){ + assert( pTableName->a[0].fg.fixedSchema==0 ); + assert( pTableName->a[0].fg.isSubquery==0 ); sqlite3DbFree(db, pTableName->a[0].u4.zDatabase); pTableName->a[0].u4.zDatabase = 0; } @@ -631,7 +633,7 @@ void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ } assert( pName->nSrc==1 ); - assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.fixedSchema==0 && pName->a[0].fg.isSubquery==0 ); zDb = pName->a[0].u4.zDatabase; zName = pName->a[0].zName; assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); From 0766cbf51157671e39e9109303bf315c661616ac Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Aug 2024 20:01:21 +0000 Subject: [PATCH 49/99] Remove unreachable code. FossilOrigin-Name: 17699329aa8e234c6793a38db5ca05def9395b287bcc8bc05493b824327edb59 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/build.c | 2 +- src/parse.y | 5 ++--- src/printf.c | 4 +--- src/select.c | 6 +----- 6 files changed, 15 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 298d9f0ab3..347a5b8ce1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tighter\schecking\sof\saccess\sconstraints\son\sunion\smembers\sin\sSrcItem.\nImproved\sinvariant\schecking. -D 2024-08-20T19:09:59.313 +C Remove\sunreachable\scode. +D 2024-08-20T20:01:21.013 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -702,7 +702,7 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c 8b42fc7d9efdb2df05c30e8f91ff6cfbd979724ae24bf90269028468b7a13333 F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 -F src/build.c 69d79a255a6a0547f5e99a1726c3cbd363fa221c4f0458425abc7ba59efb9f83 +F src/build.c c5522b5faf8128227678e194275cefaeb4d063f55dbe70bcff745f1b63a187cf F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 @@ -749,18 +749,18 @@ F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c b08600ebf0db90b6d1e9b8b6577c6fa3877cbe1a100bd0b2899e4c6e9adad4b3 F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a -F src/parse.y 967e0a712de8bee4b2cf82bd9efa36b68afd03cc486d4ea77a945feb49b057dc +F src/parse.y 5972b7d00af4c8d96fdad781af1ea1d5d51fc3b907ad61bda60e49503274e5ed F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 49516ad7718a3626f28f710fa7448ef1fce3c07fd169acbb4817341950264319 F src/pragma.c 52bfbf6dfd668b69b5eb9bd1186e3a67367c8453807150d6e75239229924f684 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c d99931f45416652895e502328ca49fe782cfc4e1ebdcda13b3736d991ebf42ce -F src/printf.c 17054fb94ffcf7a28362e9a5af4a0f813bd0c52200ae408eeebddc81feed9274 +F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 7addcd53d8f9bef2d47a7e47faeb1f6e401dba02f1f338f532298a24657ea4e6 +F src/select.c 6f1651ca254a6856c2ec5b8a0188bc226e7a454d6741d3f71ebcd631ce147807 F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab67704 F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bc5f5ce59e1e4323422bda7d002310923f927a03b4fd42749bf04f6c9853956b -R ee08ce1ecc132f8a5f905e65ae160cc1 +P fd72d3400a8fe5747f494eee81654698acee350bb95b9db269e87d857af03492 +R ed1ab5c9cb5cf9505f2201c6b302f119 U drh -Z 01685802d904a82a17f4e5855c564d7e +Z 40ddef7af9ce2e2b1803942ebd50a97a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 06a9d2670a..33f57517e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd72d3400a8fe5747f494eee81654698acee350bb95b9db269e87d857af03492 +17699329aa8e234c6793a38db5ca05def9395b287bcc8bc05493b824327edb59 diff --git a/src/build.c b/src/build.c index b2386035d8..74577ed0d5 100644 --- a/src/build.c +++ b/src/build.c @@ -5012,7 +5012,7 @@ int sqlite3SrcItemAttachSubquery( int dupSelect /* If true, attach a copy of pSelect, not pSelect itself.*/ ){ Subquery *p; - if( pSelect==0 ) return SQLITE_OK; + assert( pSelect!=0 ); assert( pItem->fg.isSubquery==0 ); if( pItem->fg.fixedSchema ){ pItem->u4.pSchema = 0; diff --git a/src/parse.y b/src/parse.y index a47c587547..926dd6e7d1 100644 --- a/src/parse.y +++ b/src/parse.y @@ -748,9 +748,8 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N pNew->u4.pSubq = pOld->u4.pSubq; pOld->u4.pSubq = 0; pOld->fg.isSubquery = 0; - if( pNew->u4.pSubq->pSelect - && (pNew->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 - ){ + assert( pNew->u4.pSubq!=0 && pNew->u4.pSubq->pSelect!=0 ); + if( (pNew->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 ){ pNew->fg.isNestedFrom = 1; } }else{ diff --git a/src/printf.c b/src/printf.c index be7ffc8c76..a140565146 100644 --- a/src/printf.c +++ b/src/printf.c @@ -858,7 +858,7 @@ void sqlite3_str_vappendf( sqlite3_str_appendall(pAccum, pItem->zName); }else if( pItem->zAlias ){ sqlite3_str_appendall(pAccum, pItem->zAlias); - }else if( pItem->fg.isSubquery ){/* Because of tag-20240424-1 */ + }else if( ALWAYS(pItem->fg.isSubquery) ){/* Because of tag-20240424-1 */ Select *pSel = pItem->u4.pSubq->pSelect; assert( pSel!=0 ); if( pSel->selFlags & SF_NestedFrom ){ @@ -870,8 +870,6 @@ void sqlite3_str_vappendf( }else{ sqlite3_str_appendf(pAccum, "(subquery-%u)", pSel->selId); } - }else{ - sqlite3_str_appendf(pAccum, "(unknown-data-source-%p)", pItem); } length = width = 0; break; diff --git a/src/select.c b/src/select.c index 39955dd60a..655221b067 100644 --- a/src/select.c +++ b/src/select.c @@ -4523,10 +4523,7 @@ static int flattenSubquery( pSub1 = 0; } assert( pSubitem->fg.isSubquery==0 ); - if( pSubitem->fg.fixedSchema==0 ){ - sqlite3DbFree(db, pSubitem->u4.zDatabase); - pSubitem->u4.zDatabase = 0; - } + assert( pSubitem->fg.fixedSchema==0 ); sqlite3DbFree(db, pSubitem->zName); sqlite3DbFree(db, pSubitem->zAlias); pSubitem->zName = 0; @@ -5601,7 +5598,6 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ /* If we reach this point, that means the transformation is required. */ pParse = pWalker->pParse; - if( pParse->nErr ) return WRC_Abort; db = pParse->db; pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); if( pNew==0 ) return WRC_Abort; From bb36d55e638fb7f8ba1c1ce787b835f398fc98d3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Aug 2024 22:05:01 +0000 Subject: [PATCH 50/99] Fix minor comment issues. FossilOrigin-Name: c1bbed19f4348ec31c3cb4723eaa8f1554bb58ef4de7feb3a0f39612785f13d5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/delete.c | 4 ++-- src/sqliteInt.h | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 347a5b8ce1..72fecdb486 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunreachable\scode. -D 2024-08-20T20:01:21.013 +C Fix\sminor\scomment\sissues. +D 2024-08-20T22:05:01.213 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -709,7 +709,7 @@ F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 F src/date.c 13dd752847afb32ed70510ad7345a5b9c841f51ad904dba5d010f1fa3a6a324e F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 -F src/delete.c cfe1a09cd67e3c8aef51ef928c69ba2824e6e2439f451e990afbb89e774add01 +F src/delete.c 444c4d1eaac40103461e3b6f0881846dd3aafc1cec1dd169d3482fa331667da7 F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f @@ -765,7 +765,7 @@ F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab6770 F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 521a9abd808e2b6c2e377cfc85b5ad5508202c33f878f929dccf60d6aca13034 +F src/sqliteInt.h 28c878bdf528879afefe1994ac007d094f8061f2fdacdc55d6055d7e9341151e F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fd72d3400a8fe5747f494eee81654698acee350bb95b9db269e87d857af03492 -R ed1ab5c9cb5cf9505f2201c6b302f119 +P 17699329aa8e234c6793a38db5ca05def9395b287bcc8bc05493b824327edb59 +R 0e51c42305c6bc1152097a45ebccdb0d U drh -Z 40ddef7af9ce2e2b1803942ebd50a97a +Z a6fa404293297d6b35cfa7a7af20f07b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 33f57517e4..b77c8e18bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17699329aa8e234c6793a38db5ca05def9395b287bcc8bc05493b824327edb59 +c1bbed19f4348ec31c3cb4723eaa8f1554bb58ef4de7feb3a0f39612785f13d5 diff --git a/src/delete.c b/src/delete.c index 1d7ba66a22..4cdb3946e3 100644 --- a/src/delete.c +++ b/src/delete.c @@ -24,8 +24,8 @@ ** ** The following fields are initialized appropriate in pSrc: ** -** pSrc->a[0].spTab Pointer to the Table object -** pSrc->a[0].pIndex Pointer to the INDEXED BY index, if there is one +** pSrc->a[0].spTab Pointer to the Table object +** pSrc->a[0].u2.pIBIndex Pointer to the INDEXED BY index, if there is one ** */ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9e0faf9cc1..d1ffce6d87 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3627,7 +3627,7 @@ struct Select { #define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ #define SF_Correlated 0x20000000 /* True if references the outer context */ -/* True if SrcList item X is a subquery that has SF_NestedFrom */ +/* True if SrcItem X is a subquery that has SF_NestedFrom */ #define IsNestedFrom(X) \ ((X)->fg.isSubquery && \ ((X)->u4.pSubq->pSelect->selFlags&SF_NestedFrom)!=0) From a0651b370706e58f9014453c5f36124ca8a738c9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Aug 2024 22:44:40 +0000 Subject: [PATCH 51/99] Fix a name resolution issue with CTEs. FossilOrigin-Name: 4fa8235dd59cd683d6c6c97bfe181a9637be7c054d435323c903b9dbd74aff02 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 7 ++++--- test/with2.test | 11 ++++++++++- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 72fecdb486..7ee60fbf49 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\scomment\sissues. -D 2024-08-20T22:05:01.213 +C Fix\sa\sname\sresolution\sissue\swith\sCTEs. +D 2024-08-20T22:44:40.653 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -760,7 +760,7 @@ F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 6f1651ca254a6856c2ec5b8a0188bc226e7a454d6741d3f71ebcd631ce147807 +F src/select.c c1c28650d3ea5dc0670dd658600b963c29a5c31b685941a5df583b02631d04ff F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab67704 F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2070,7 +2070,7 @@ F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3c F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c F test/windowpushd.test c420e2265f0e09a0e798d0513a660d71b51602088d81b3dbd038918ee1339dcc F test/with1.test b93833890e5d2a368e78747f124503a0159aa029b98e9ed4795ebf630b2efd3d -F test/with2.test 5885f0ef250504096ea4c0c6350ef2f2ab09b9e2dc2bc691e182f773db3aabb2 +F test/with2.test 181674a6cc86a601ca2ac052741cdfad5b529e07e870435d2f6cdb92d589ff17 F test/with3.test e30369ea27aa27eb1bda4c5e510c8a9f782c8afd2ab99d1a02b8a7f25a5d3e65 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/with5.test 6248213c41fab36290b5b73aa3f937309dfba337004d9d8434c3fabc8c7d4be8 @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 17699329aa8e234c6793a38db5ca05def9395b287bcc8bc05493b824327edb59 -R 0e51c42305c6bc1152097a45ebccdb0d +P c1bbed19f4348ec31c3cb4723eaa8f1554bb58ef4de7feb3a0f39612785f13d5 +R 1c80ace1a0877505f5ca5ebb5b98295b U drh -Z a6fa404293297d6b35cfa7a7af20f07b +Z ef82496d97c0311a270f63ff1146b092 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b77c8e18bf..53632b3479 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1bbed19f4348ec31c3cb4723eaa8f1554bb58ef4de7feb3a0f39612785f13d5 +4fa8235dd59cd683d6c6c97bfe181a9637be7c054d435323c903b9dbd74aff02 diff --git a/src/select.c b/src/select.c index 655221b067..b76e42b74c 100644 --- a/src/select.c +++ b/src/select.c @@ -5823,9 +5823,10 @@ static int resolveFromTermToCte( assert( pRecTerm->pPrior!=0 ); for(i=0; inSrc; i++){ SrcItem *pItem = &pSrc->a[i]; - if( ((pItem->fg.fixedSchema==0 && pItem->u4.zDatabase==0) - || pItem->fg.hadSchema==0) - && pItem->zName!=0 + if( pItem->zName!=0 + && !pItem->fg.hadSchema + && ALWAYS( !pItem->fg.isSubquery ) + && (pItem->fg.fixedSchema || pItem->u4.zDatabase==0) && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ pItem->pSTab = pTab; diff --git a/test/with2.test b/test/with2.test index ad1753d659..68790fe860 100644 --- a/test/with2.test +++ b/test/with2.test @@ -146,7 +146,6 @@ do_execsql_test 1.14 { SELECT * FROM t4; } {4 5 6 7 8 9 10} -exit do_execsql_test 1.15 { WITH t4(x) AS ( @@ -157,6 +156,16 @@ do_execsql_test 1.15 { SELECT * FROM t4; } {4 5} +do_execsql_test 1.15.2 { + WITH + t4(x) AS ( + VALUES(4) + UNION ALL + SELECT x+1 FROM (SELECT * FROM main.t4) WHERE x<10 + ) + SELECT * FROM t4; +} {4 5} + do_catchsql_test 1.16 { WITH t4(x) AS ( From 8c58fd78c39baca9820ed4c6d6a74ad16e08c9d6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 21 Aug 2024 07:38:14 +0000 Subject: [PATCH 52/99] Remove incorrect assert() statements in the SrcItem invariant validator. dbsqlfuzz 7f908865436ce531b0ace55c97a1c71d05c4e76c. FossilOrigin-Name: 81a1ede38823b8d50e221ecb6ce52a05d82a856099002a71d9f1ac561a587f05 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/build.c | 2 -- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index de163306d5..0d53a2be4d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\sSrcItem\sobject\sto\smove\sfields\sassociated\swith\ssubqueries\sout\ninto\sa\sseparate\sobject\snamed\sSubquery.\s\sThis\sreduces\sthe\ssize\sof\sthe\sSrcItem\nobject\sby\sabout\s1/3rd\sand\sprovides\simproved\sperformance. -D 2024-08-20T23:11:28.443 +C Remove\sincorrect\sassert()\sstatements\sin\sthe\sSrcItem\sinvariant\svalidator.\ndbsqlfuzz\s7f908865436ce531b0ace55c97a1c71d05c4e76c. +D 2024-08-21T07:38:14.508 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c 8b42fc7d9efdb2df05c30e8f91ff6cfbd979724ae24bf90269028468b7a13333 F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 -F src/build.c c5522b5faf8128227678e194275cefaeb4d063f55dbe70bcff745f1b63a187cf +F src/build.c 3a1840d9d171ce2d24f4c1f7acda7266ab796c664290c1acba65ff98ce2bd01e F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3399698376761ab8c422f8ea02bfa2759afb606f08bedbd1cf7eee834229a9aa 4fa8235dd59cd683d6c6c97bfe181a9637be7c054d435323c903b9dbd74aff02 -R 82a6023b6446e680173d0de603e0e98e +P 484bcd75bc95491d8540c791c1c4d40d996cb465839564662e14f98739699bf1 +R 136086d644c5a2efc92647bc4c586910 U drh -Z cc2792cc7e3d54684569210f9c97c9df +Z 029346cb8d0ee71c8f522bf954942df0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c96f758867..326f56a63a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -484bcd75bc95491d8540c791c1c4d40d996cb465839564662e14f98739699bf1 +81a1ede38823b8d50e221ecb6ce52a05d82a856099002a71d9f1ac561a587f05 diff --git a/src/build.c b/src/build.c index 74577ed0d5..943d862e97 100644 --- a/src/build.c +++ b/src/build.c @@ -4965,8 +4965,6 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ /* Check invariants on SrcItem */ assert( !pItem->fg.isIndexedBy || !pItem->fg.isTabFunc ); assert( !pItem->fg.isCte || !pItem->fg.isIndexedBy ); - assert( !pItem->fg.hadSchema || !pItem->fg.isSubquery ); - assert( !pItem->fg.hadSchema || pItem->fg.fixedSchema ); assert( !pItem->fg.fixedSchema || !pItem->fg.isSubquery ); assert( !pItem->fg.isSubquery || (pItem->u4.pSubq!=0 && pItem->u4.pSubq->pSelect!=0) ); From fb7d0c3c01ce94e2704e6e9878bde247daa156d8 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 21 Aug 2024 09:56:45 +0000 Subject: [PATCH 53/99] De-duplicate some error-output code in shell.c.in as part of tracking down the cause of [forum:5647ca2af1|forum post 5647ca2af1]. FossilOrigin-Name: fd5904495e0c775174f9380bc6877fb9def3d30dec1c9315979ad4503a18d70b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 35 ++++++++++++++++++++--------------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 0d53a2be4d..3c85ec36eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sincorrect\sassert()\sstatements\sin\sthe\sSrcItem\sinvariant\svalidator.\ndbsqlfuzz\s7f908865436ce531b0ace55c97a1c71d05c4e76c. -D 2024-08-21T07:38:14.508 +C De-duplicate\ssome\serror-output\scode\sin\sshell.c.in\sas\spart\sof\stracking\sdown\sthe\scause\sof\s[forum:5647ca2af1|forum\spost\s5647ca2af1]. +D 2024-08-21T09:56:45.002 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -765,7 +765,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 44d135bbea93872a7318f048d9d6e566b1c3eaa92d6dabe06e4741aa78d1c1ec -F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab67704 +F src/shell.c.in fb7fe0a39473e7a3ad5ad7076ff49d87aa8ed7fdd98a1595345747ac5d78ac2b F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 484bcd75bc95491d8540c791c1c4d40d996cb465839564662e14f98739699bf1 -R 136086d644c5a2efc92647bc4c586910 -U drh -Z 029346cb8d0ee71c8f522bf954942df0 +P 81a1ede38823b8d50e221ecb6ce52a05d82a856099002a71d9f1ac561a587f05 +R 7dfeb7387454a33b7099aec64d81efae +U stephan +Z b55d6e3262271daffaa830513ab83e17 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 326f56a63a..f487a539aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -81a1ede38823b8d50e221ecb6ce52a05d82a856099002a71d9f1ac561a587f05 +fd5904495e0c775174f9380bc6877fb9def3d30dec1c9315979ad4503a18d70b diff --git a/src/shell.c.in b/src/shell.c.in index 5b0a1a84b0..8c4fcc4e12 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6343,12 +6343,17 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ } #endif /* SQLITE_SHELL_HAVE_RECOVER */ +/* +** Print the given string as an error message. +*/ +static void shellEmitError(const char *zErr){ + eputf("Error: %s\n", zErr); +} /* ** Print the current sqlite3_errmsg() value to stderr and return 1. */ static int shellDatabaseError(sqlite3 *db){ - const char *zErr = sqlite3_errmsg(db); - eputf("Error: %s\n", zErr); + shellEmitError(sqlite3_errmsg(db)); return 1; } @@ -6893,7 +6898,7 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ va_start(ap, zFmt); z = sqlite3_vmprintf(zFmt, ap); va_end(ap); - eputf("Error: %s\n", z); + shellEmitError(z); if( pAr->fromCmdLine ){ eputz("Use \"-A\" for more help\n"); }else{ @@ -8124,7 +8129,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb); if( pBackup==0 ){ - eputf("Error: %s\n", sqlite3_errmsg(pDest)); + shellDatabaseError(pDest); close_db(pDest); return 1; } @@ -8133,7 +8138,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( rc==SQLITE_DONE ){ rc = 0; }else{ - eputf("Error: %s\n", sqlite3_errmsg(pDest)); + shellDatabaseError(pDest); rc = 1; } close_db(pDest); @@ -8309,7 +8314,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); if( rc ){ - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); rc = 1; }else{ while( sqlite3_step(pStmt)==SQLITE_ROW ){ @@ -9005,7 +9010,7 @@ static int do_meta_command(char *zLine, ShellState *p){ zSql = 0; if( rc ){ if (pStmt) sqlite3_finalize(pStmt); - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); import_cleanup(&sCtx); rc = 1; goto meta_command_exit; @@ -9049,7 +9054,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zSql); zSql = 0; if( rc ){ - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); if (pStmt) sqlite3_finalize(pStmt); import_cleanup(&sCtx); rc = 1; @@ -9343,7 +9348,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg); if( rc!=SQLITE_OK ){ - eputf("Error: %s\n", zErrMsg); + shellEmitError(zErrMsg); sqlite3_free(zErrMsg); rc = 1; } @@ -9965,7 +9970,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main"); if( pBackup==0 ){ - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); close_db(pSrc); return 1; } @@ -9983,7 +9988,7 @@ static int do_meta_command(char *zLine, ShellState *p){ eputz("Error: source database is busy\n"); rc = 1; }else{ - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); rc = 1; } close_db(pSrc); @@ -10080,7 +10085,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list", -1, &pStmt, 0); if( rc ){ - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); sqlite3_finalize(pStmt); rc = 1; goto meta_command_exit; @@ -10149,7 +10154,7 @@ static int do_meta_command(char *zLine, ShellState *p){ freeText(&sSelect); } if( zErrMsg ){ - eputf("Error: %s\n", zErrMsg); + shellEmitError(zErrMsg); sqlite3_free(zErrMsg); rc = 1; }else if( rc != SQLITE_OK ){ @@ -12728,7 +12733,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ open_db(&data, 0); rc = shell_exec(&data, z, &zErrMsg); if( zErrMsg!=0 ){ - eputf("Error: %s\n", zErrMsg); + shellEmitError(zErrMsg); if( bail_on_error ) return rc!=0 ? rc : 1; }else if( rc!=0 ){ eputf("Error: unable to process SQL \"%s\"\n", z); @@ -12782,7 +12787,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ rc = shell_exec(&data, azCmd[i], &zErrMsg); if( zErrMsg || rc ){ if( zErrMsg!=0 ){ - eputf("Error: %s\n", zErrMsg); + shellEmitError(zErrMsg); }else{ eputf("Error: unable to process SQL: %s\n", azCmd[i]); } From 01972f55fe9e73b0cc20d71d156aae3473af3d06 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 21 Aug 2024 10:32:00 +0000 Subject: [PATCH 54/99] Improvements to treetrace output for subqueries. Debug code only. FossilOrigin-Name: 0a835d18d9063b17a1d506fd989ebcf7f0d76593978a3bd5981be59618335f21 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/treeview.c | 9 ++++++++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3c85ec36eb..e8e708addf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C De-duplicate\ssome\serror-output\scode\sin\sshell.c.in\sas\spart\sof\stracking\sdown\sthe\scause\sof\s[forum:5647ca2af1|forum\spost\s5647ca2af1]. -D 2024-08-21T09:56:45.002 +C Improvements\sto\streetrace\soutput\sfor\ssubqueries.\s\sDebug\scode\sonly. +D 2024-08-21T10:32:00.693 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -828,7 +828,7 @@ F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68 -F src/treeview.c e98dbc8068d27f3d0bd2a4cebfce6a7c21c776edc23d80e12c67a70cd2ba3fe5 +F src/treeview.c 88aa39b754f5ef7214385c1bbbdd2f3dc20efafeed0cf590e8d1199b9c6e44aa F src/trigger.c 0bb986a5b96047fd597c6aac28588853df56064e576e6b81ba777ef2ccaac461 F src/update.c 0e01aa6a3edf9ec112b33eb714b9016a81241497b1fb7c3e74332f4f71756508 F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 81a1ede38823b8d50e221ecb6ce52a05d82a856099002a71d9f1ac561a587f05 -R 7dfeb7387454a33b7099aec64d81efae -U stephan -Z b55d6e3262271daffaa830513ab83e17 +P fd5904495e0c775174f9380bc6877fb9def3d30dec1c9315979ad4503a18d70b +R 9b61fbb833ee0abcfc996b0de63e571b +U drh +Z 56e8c0285d76008f8c95bfad3d078586 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f487a539aa..0090278b24 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd5904495e0c775174f9380bc6877fb9def3d30dec1c9315979ad4503a18d70b +0a835d18d9063b17a1d506fd989ebcf7f0d76593978a3bd5981be59618335f21 diff --git a/src/treeview.c b/src/treeview.c index dbd54b3466..de67161229 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -226,6 +226,9 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ if( pItem->fg.viaCoroutine ) sqlite3_str_appendf(&x, " viaCoroutine"); if( pItem->fg.notCte ) sqlite3_str_appendf(&x, " notCte"); if( pItem->fg.isNestedFrom ) sqlite3_str_appendf(&x, " isNestedFrom"); + if( pItem->fg.fixedSchema ) sqlite3_str_appendf(&x, " fixedSchema"); + if( pItem->fg.hadSchema ) sqlite3_str_appendf(&x, " hadSchema"); + if( pItem->fg.isSubquery ) sqlite3_str_appendf(&x, " isSubquery"); sqlite3StrAccumFinish(&x); sqlite3TreeViewItem(pView, zLine, inSrc-1); @@ -237,12 +240,16 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); } if( pItem->fg.isSubquery ){ + assert( n==1 ); if( pItem->pSTab ){ Table *pTab = pItem->pSTab; sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); } assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) ); - sqlite3TreeViewSelect(pView, pItem->u4.pSubq->pSelect, (--n)>0); + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, "SUBQUERY"); + sqlite3TreeViewPop(&pView); + sqlite3TreeViewSelect(pView, pItem->u4.pSubq->pSelect, 0); } if( pItem->fg.isTabFunc ){ sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); From 91937a6961e2277f11b029aacf6a45bb52a1a0d5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 21 Aug 2024 10:37:59 +0000 Subject: [PATCH 55/99] Fix a minor typo in a comment. FossilOrigin-Name: 92c80af1129051c9eded8df170730ad6366b4f7715dead34c4364c8149d0dce9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e8e708addf..dccf0d0c1f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\streetrace\soutput\sfor\ssubqueries.\s\sDebug\scode\sonly. -D 2024-08-21T10:32:00.693 +C Fix\sa\sminor\stypo\sin\sa\scomment. +D 2024-08-21T10:37:59.175 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -764,7 +764,7 @@ F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 44d135bbea93872a7318f048d9d6e566b1c3eaa92d6dabe06e4741aa78d1c1ec +F src/select.c 534eccfb8e3879cc54d93a55ca5118cf72ed9c5337456c217cc0424da33b87a7 F src/shell.c.in fb7fe0a39473e7a3ad5ad7076ff49d87aa8ed7fdd98a1595345747ac5d78ac2b F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fd5904495e0c775174f9380bc6877fb9def3d30dec1c9315979ad4503a18d70b -R 9b61fbb833ee0abcfc996b0de63e571b +P 0a835d18d9063b17a1d506fd989ebcf7f0d76593978a3bd5981be59618335f21 +R 3a629ede216b591a2dde1a6c7535f80d U drh -Z 56e8c0285d76008f8c95bfad3d078586 +Z d8ab38fee62be945af061484336e5eb9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0090278b24..ba3b844e79 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a835d18d9063b17a1d506fd989ebcf7f0d76593978a3bd5981be59618335f21 +92c80af1129051c9eded8df170730ad6366b4f7715dead34c4364c8149d0dce9 diff --git a/src/select.c b/src/select.c index 87bf7fac2f..18fc9d3be6 100644 --- a/src/select.c +++ b/src/select.c @@ -7696,7 +7696,7 @@ int sqlite3Select( } /* For each term in the FROM clause, do two things: - ** (1) Authorized unreferenced tables + ** (1) Authorize unreferenced tables ** (2) Generate code for all sub-queries */ for(i=0; inSrc; i++){ From 4cc86b6e9246c4b85cbad7161f48b4c5c2b9cf43 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 21 Aug 2024 12:01:46 +0000 Subject: [PATCH 56/99] Add code comments for a "table-of-contents" and various milestone marks in the 1300+ line long sqlite3Select() function, to help improve readiability and maintainability. Comment changes only - no functional changes. FossilOrigin-Name: 00cfbb9fa5136e6a7847da4e4ab30d320ca79c22acda9db2030e558d59b8c744 --- manifest | 12 +++--- manifest.uuid | 2 +- src/select.c | 109 +++++++++++++++++++++++++++++++++++++------------- 3 files changed, 89 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index dccf0d0c1f..ed35eeaa1c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sminor\stypo\sin\sa\scomment. -D 2024-08-21T10:37:59.175 +C Add\scode\scomments\sfor\sa\s"table-of-contents"\sand\svarious\smilestone\smarks\sin\nthe\s1300+\sline\slong\ssqlite3Select()\sfunction,\sto\shelp\simprove\sreadiability\nand\smaintainability.\s\sComment\schanges\sonly\s-\sno\sfunctional\schanges. +D 2024-08-21T12:01:46.057 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -764,7 +764,7 @@ F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 534eccfb8e3879cc54d93a55ca5118cf72ed9c5337456c217cc0424da33b87a7 +F src/select.c 26bd0ce6e8816f8968f50a377dc4255da4b392793e87ae27acd7276b56a447ce F src/shell.c.in fb7fe0a39473e7a3ad5ad7076ff49d87aa8ed7fdd98a1595345747ac5d78ac2b F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0a835d18d9063b17a1d506fd989ebcf7f0d76593978a3bd5981be59618335f21 -R 3a629ede216b591a2dde1a6c7535f80d +P 92c80af1129051c9eded8df170730ad6366b4f7715dead34c4364c8149d0dce9 +R 96f2b1d6b34bcd20452ccf075e5a2b9a U drh -Z d8ab38fee62be945af061484336e5eb9 +Z 790a1d73ddaf8f8dea50d658062e6edf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ba3b844e79..a4857cf3eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92c80af1129051c9eded8df170730ad6366b4f7715dead34c4364c8149d0dce9 +00cfbb9fa5136e6a7847da4e4ab30d320ca79c22acda9db2030e558d59b8c744 diff --git a/src/select.c b/src/select.c index 18fc9d3be6..0bb033785c 100644 --- a/src/select.c +++ b/src/select.c @@ -7340,7 +7340,7 @@ static int fromClauseTermCanBeCoroutine( } /* -** Generate code for the SELECT statement given in the p argument. +** Generate byte-code for the SELECT statement given in the p argument. ** ** The results are returned according to the SelectDest structure. ** See comments in sqliteInt.h for further information. @@ -7351,6 +7351,40 @@ static int fromClauseTermCanBeCoroutine( ** ** This routine does NOT free the Select structure passed in. The ** calling function needs to do that. +** +** This is a long function. The following is an outline of the processing +** steps, with tags referencing various milestones: +** +** * Resolve names and similar preparation tag-select-0100 +** * Scan of the FROM clause tag-select-0200 +** + OUTER JOIN strength reduction tag-select-0220 +** + Sub-query ORDER BY removal tag-select-0230 +** + Query flattening tag-select-0240 +** * Separate subroutine for compound-SELECT tag-select-0300 +** * WHERE-clause constant propagation tag-select-0330 +** * Count()-of-VIEW optimization tag-select-0350 +** * Scan of the FROM clause again tag-select-0400 +** + Authorize unreferenced tables tag-select-0410 +** + Predicate push-down optimization tag-select-0420 +** + Omit unused subquery columns optimization tag-select-0440 +** + Generate code to implement subqueries tag-select-0480 +** - Co-routines tag-select-0482 +** - Reuse previously computed CTE tag-select-0484 +** - REuse previously computed VIEW tag-select-0486 +** - Materialize a VIEW or CTE tag-select-0488 +** * DISTINCT ORDER BY -> GROUP BY optimization tag-select-0500 +** * Set up for ORDER BY tag-select-0600 +** * Create output table tag-select-0630 +** * Prepare registers for LIMIT tag-select-0650 +** * Setup for DISTINCT tag-select-0680 +** * Generate code for non-aggregate and non-GROUP BY tag-select-0700 +** * Generate code for aggregate and/or GROUP BY tag-select-0800 +** + GROUP BY queries tag-select-0810 +** + non-GROUP BY queries tag-select-0820 +** - Special case of count() w/o GROUP BY tag-select-0821 +** - General case of non-GROUP BY aggregates tag-select-0822 +** * Sort results, as needed tag-select-0900 +** * Internal self-checks tag-select-1000 */ int sqlite3Select( Parse *pParse, /* The parser context */ @@ -7394,6 +7428,7 @@ int sqlite3Select( } #endif + /* tag-select-0100 */ assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); @@ -7480,6 +7515,7 @@ int sqlite3Select( /* Try to do various optimizations (flattening subqueries, and strength ** reduction of join operators) in the FROM clause up into the main query + ** tag-select-0200 */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ @@ -7502,6 +7538,7 @@ int sqlite3Select( ** way that the i-th table cannot be the NULL row of a join, then ** perform the appropriate simplification. This is called ** "OUTER JOIN strength reduction" in the SQLite documentation. + ** tag-select-0220 */ if( (pItem->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor, @@ -7572,7 +7609,8 @@ int sqlite3Select( if( (pSub->selFlags & SF_Aggregate)!=0 ) continue; assert( pSub->pGroupBy==0 ); - /* If a FROM-clause subquery has an ORDER BY clause that is not + /* tag-select-0230: + ** If a FROM-clause subquery has an ORDER BY clause that is not ** really doing anything, then delete it now so that it does not ** interfere with query flattening. See the discussion at ** https://sqlite.org/forum/forumpost/2d76f2bcf65d256a @@ -7638,6 +7676,7 @@ int sqlite3Select( continue; } + /* tag-select-0240 */ if( flattenSubquery(pParse, p, i, isAgg) ){ if( pParse->nErr ) goto select_end; /* This subquery can be absorbed into its parent. */ @@ -7653,7 +7692,7 @@ int sqlite3Select( #ifndef SQLITE_OMIT_COMPOUND_SELECT /* Handle compound SELECT statements using the separate multiSelect() - ** procedure. + ** procedure. tag-select-0300 */ if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); @@ -7669,9 +7708,9 @@ int sqlite3Select( #endif /* Do the WHERE-clause constant propagation optimization if this is - ** a join. No need to speed time on this operation for non-join queries + ** a join. No need to spend time on this operation for non-join queries ** as the equivalent optimization will be handled by query planner in - ** sqlite3WhereBegin(). + ** sqlite3WhereBegin(). tag-select-0330 */ if( p->pWhere!=0 && p->pWhere->op==TK_AND @@ -7688,6 +7727,7 @@ int sqlite3Select( TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } + /* tag-select-0350 */ if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) && countOfViewOptimization(pParse, p) ){ @@ -7695,9 +7735,12 @@ int sqlite3Select( pTabList = p->pSrc; } - /* For each term in the FROM clause, do two things: - ** (1) Authorize unreferenced tables - ** (2) Generate code for all sub-queries + /* Loop over all terms in the FROM clause and do two things for each term: + ** + ** (1) Authorize unreferenced tables + ** (2) Generate code for all sub-queries + ** + ** tag-select-0400 */ for(i=0; inSrc; i++){ SrcItem *pItem = &pTabList->a[i]; @@ -7709,7 +7752,9 @@ int sqlite3Select( const char *zSavedAuthContext; #endif - /* Issue SQLITE_READ authorizations with a fake column name for any + /* Authorized unreferenced tables. tag-select-0410 + ** + ** Issue SQLITE_READ authorizations with a fake column name for any ** tables that are referenced but from which no values are extracted. ** Examples of where these kinds of null SQLITE_READ authorizations ** would occur: @@ -7745,10 +7790,9 @@ int sqlite3Select( pSubq = pItem->u4.pSubq; assert( pSubq!=0 ); pSub = pSubq->pSelect; - if( pSubq->addrFillSub!=0 ) continue; /* The code for a subquery should only be generated once. */ - assert( pSubq->addrFillSub==0 ); + if( pSubq->addrFillSub!=0 ) continue; /* Increment Parse.nHeight by the height of the largest expression ** tree referred to by this, the parent select. The child select @@ -7761,6 +7805,7 @@ int sqlite3Select( /* Make copies of constant WHERE-clause terms in the outer query down ** inside the subquery. This can help the subquery to run more efficiently. + ** This is the "predicate push-down optimization". tag-select-0420 */ if( OptimizationEnabled(db, SQLITE_PushDown) && (pItem->fg.isCte==0 @@ -7781,6 +7826,7 @@ int sqlite3Select( /* Convert unused result columns of the subquery into simple NULL ** expressions, to avoid unneeded searching and computation. + ** tag-select-0440 */ if( OptimizationEnabled(db, SQLITE_NullUnusedCols) && disableUnusedSubqueryResultColumns(pItem) @@ -7798,11 +7844,11 @@ int sqlite3Select( zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; - /* Generate code to implement the subquery + /* Generate byte-code to implement the subquery tag-select-0480 */ if( fromClauseTermCanBeCoroutine(pParse, pTabList, i, p->selFlags) ){ /* Implement a co-routine that will return a single row of the result - ** set on each invocation. + ** set on each invocation. tag-select-0482 */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; @@ -7822,8 +7868,8 @@ int sqlite3Select( }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ /* This is a CTE for which materialization code has already been ** generated. Invoke the subroutine to compute the materialization, - ** the make the pItem->iCursor be a copy of the ephemeral table that - ** holds the result of the materialization. */ + ** then make the pItem->iCursor be a copy of the ephemeral table that + ** holds the result of the materialization. tag-select-0484 */ CteUse *pCteUse = pItem->u2.pCteUse; sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e); if( pItem->iCursor!=pCteUse->iCur ){ @@ -7833,7 +7879,7 @@ int sqlite3Select( pSub->nSelectRow = pCteUse->nRowEst; }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ /* This view has already been materialized by a prior entry in - ** this same FROM clause. Reuse it. */ + ** this same FROM clause. Reuse it. tag-select-0486 */ Subquery *pPriorSubq; assert( pPrior->fg.isSubquery ); pPriorSubq = pPrior->u4.pSubq; @@ -7847,7 +7893,7 @@ int sqlite3Select( }else{ /* Materialize the view. If the view is not correlated, generate a ** subroutine to do the materialization so that subsequent uses of - ** the same view can reuse the materialization. */ + ** the same view can reuse the materialization. tag-select-0488 */ int topAddr; int onceAddr = 0; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS @@ -7907,7 +7953,9 @@ int sqlite3Select( } #endif - /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + /* tag-select-0500 + ** + ** If the query is DISTINCT with an ORDER BY but is not an aggregate, and ** if the select-list is the same as the ORDER BY list, then this query ** can be rewritten as a GROUP BY. In other words, this: ** @@ -7957,7 +8005,7 @@ int sqlite3Select( ** If that is the case, then the OP_OpenEphemeral instruction will be ** changed to an OP_Noop once we figure out that the sorting index is ** not needed. The sSort.addrSortIndex variable is used to facilitate - ** that change. + ** that change. tag-select-0600 */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; @@ -7974,6 +8022,7 @@ int sqlite3Select( } /* If the output is destined for a temporary table, open that table. + ** tag-select-0630 */ if( pDest->eDest==SRT_EphemTab ){ sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); @@ -7991,7 +8040,7 @@ int sqlite3Select( } } - /* Set the limiter. + /* Set the limiter. tag-select-0650 */ iEnd = sqlite3VdbeMakeLabel(pParse); if( (p->selFlags & SF_FixedLimit)==0 ){ @@ -8003,7 +8052,7 @@ int sqlite3Select( sSort.sortFlags |= SORTFLAG_UseSorter; } - /* Open an ephemeral index to use for the distinct set. + /* Open an ephemeral index to use for the distinct set. tag-select-0680 */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; @@ -8018,7 +8067,7 @@ int sqlite3Select( } if( !isAgg && pGroupBy==0 ){ - /* No aggregate functions and no GROUP BY clause */ + /* No aggregate functions and no GROUP BY clause. tag-select-0700 */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0) | (p->selFlags & SF_FixedLimit); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -8091,8 +8140,8 @@ int sqlite3Select( sqlite3WhereEnd(pWInfo); } }else{ - /* This case when there exist aggregate functions or a GROUP BY clause - ** or both */ + /* This case is for when there exist aggregate functions or a GROUP BY + ** clause or both. tag-select-0800 */ NameContext sNC; /* Name context for processing aggregate information */ int iAMem; /* First Mem address for storing current GROUP BY */ int iBMem; /* First Mem address for previous GROUP BY */ @@ -8211,7 +8260,7 @@ int sqlite3Select( /* Processing for aggregates with GROUP BY is very different and - ** much more complex than aggregates without a GROUP BY. + ** much more complex than aggregates without a GROUP BY. tag-select-0810 */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ @@ -8508,9 +8557,12 @@ int sqlite3Select( } } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { + /* Aggregate functions without GROUP BY. tag-select-0820 */ Table *pTab; if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){ - /* If isSimpleCount() returns a pointer to a Table structure, then + /* tag-select-0821 + ** + ** If isSimpleCount() returns a pointer to a Table structure, then ** the SQL statement is of the form: ** ** SELECT count(*) FROM @@ -8569,6 +8621,8 @@ int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ + /* The general case of an aggregate query without GROUP BY + ** tag-select-0822 */ int regAcc = 0; /* "populate accumulators" flag */ ExprList *pDistinct = 0; u16 distFlag = 0; @@ -8657,7 +8711,7 @@ int sqlite3Select( } /* If there is an ORDER BY clause, then we need to sort the results - ** and send them to the callback one by one. + ** and send them to the callback one by one. tag-select-0900 */ if( sSort.pOrderBy ){ assert( p->pEList==pEList ); @@ -8680,6 +8734,7 @@ select_end: assert( db->mallocFailed==0 || pParse->nErr!=0 ); sqlite3ExprListDelete(db, pMinMaxOrderBy); #ifdef SQLITE_DEBUG + /* Internal self-checks. tag-select-1000 */ if( pAggInfo && !db->mallocFailed ){ #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x20 ){ From 59b171172f55351bb35f5954007395f3464a7ab1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 21 Aug 2024 13:44:40 +0000 Subject: [PATCH 57/99] Updates and enhancements to the README.md file. No code changes. FossilOrigin-Name: 3161b8028916bff9a13a04bfb1996bce7e1274a3c403fc58f4d798afd30c528f --- README.md | 153 +++++++++++++++++++++++++++++++++----------------- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 107 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 0975a32d9a..3d8b5379b0 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,19 @@

SQLite Source Repository

This repository contains the complete source code for the -[SQLite database engine](https://sqlite.org/). Some test scripts -are also included. However, many other test scripts +[SQLite database engine](https://sqlite.org/), including +many test scripts. However, other test scripts and most of the documentation are managed separately. +See the [on-line documentation](https://sqlite.org/) for more information +about what SQLite is and how it works from a user's perspective. This +README file is about the source code that goes into building SQLite, +not about how SQLite is used. + ## Version Control SQLite sources are managed using -[Fossil](https://www.fossil-scm.org/), a distributed version control system +[Fossil](https://fossil-scm.org/), a distributed version control system that was specifically designed and written to support SQLite development. The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext. @@ -68,12 +73,11 @@ archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows: then click on the "Tarball" or "ZIP Archive" links on the information page. -If you do want to use Fossil to check out the source tree, -first install Fossil version 2.0 or later. -(Source tarballs and precompiled binaries available -[here](https://www.fossil-scm.org/fossil/uv/download.html). Fossil is +To access sources directly using Fossil, first install Fossil version 2.0 or later. +Source tarballs and precompiled binaries available +[here](https://www.fossil-scm.org/fossil/uv/download.html. Fossil is a stand-alone program. To install, simply download or build the single -executable file and put that file someplace on your $PATH.) +executable file and put that file someplace on your $PATH. Then run commands like this: mkdir -p ~/sqlite ~/Fossils @@ -81,8 +85,8 @@ Then run commands like this: fossil clone https://www.sqlite.org/src ~/Fossils/sqlite.fossil fossil open ~/Fossils/sqlite.fossil -After setting up a repository using the steps above, you can always -update to the latest version using: +After setting up a repository using the steps above, you can do +bandwidth-efficient updates to the latest version using: fossil update trunk ;# latest trunk check-in fossil update release ;# latest official release @@ -99,15 +103,30 @@ script found at the root of the source tree. Then run "make". For example: - tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build will occur in a sibling directory - cd bld ;# Change to the build directory - ../sqlite/configure ;# Run the configure script - make ;# Builds the "sqlite3" command-line tool - make sqlite3.c ;# Build the "amalgamation" source file - make devtest ;# Run some tests (requires Tcl) + tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite/configure ;# Run the configure script + make sqlite3 ;# Builds the "sqlite3" command-line tool + make sqlite3.c ;# Build the "amalgamation" source file + make mdevtest ;# Run development tests (requires tcl-dev) + make releasetest ;# Run full release tests (requires tcl-dev) + make sqldiff ;# Builds the "sqldiff" command-line tool + make sqlite3_analyzer ;# Builds the "sqlite3_analyzer" tool (requires tcl-dev) + make tclextension-install ;# Build and install the SQLite TCL extension -See the makefile for additional targets. +See the makefile for additional targets. For debugging builds, the +core developers typically run "configure" with options like this: + + ../sqlite/configure --enable-all --enable-debug CFLAGS='-O0 -g' + +For release builds, the core developers usually do: + + ../sqlite/configure --enable-all + +Almost all makefile targets require a "tclsh" TCL interpreter +version 8.6 or later. The targets marked with "(requires tcl-dev)" also require +the TCL development libraries. The configure script uses autoconf 2.61 and libtool. If the configure script does not work out for you, there is a generic makefile named @@ -128,48 +147,61 @@ TCL library, using a command like this: set TCLDIR=c:\Tcl -SQLite uses "tclsh.exe" as part of the build process, and so that utility -program will need to be somewhere on your %PATH%. The finished SQLite library -does not contain any TCL code, but it does use TCL to help with the build process -and to run tests. +SQLite uses "tclsh.exe" as part of the build process, and so that +program will need to be somewhere on your %PATH%. SQLite itself +does not contain any TCL code, but it does use TCL to help with the +build process and to run tests. Build using Makefile.msc. Example: - nmake /f Makefile.msc + nmake /f Makefile.msc sqlite3.exe nmake /f Makefile.msc sqlite3.c - nmake /f Makefile.msc devtest + nmake /f Makefile.msc mdevtest nmake /f Makefile.msc releasetest + nmake /f Makefile.msc tclextension-install There are many other makefile targets. See comments in Makefile.msc for details. -## Source Code Tour +## Source Tree Map -Most of the core source files are in the **src/** subdirectory. The -**src/** folder also contains files used to build the "testfixture" test -harness. The names of the source files used by "testfixture" all begin -with "test". -The **src/** also contains the "shell.c" file -which is the main program for the "sqlite3.exe" -[command-line shell](https://sqlite.org/cli.html) and -the "tclsqlite.c" file which implements the -[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite. -(Historical note: SQLite began as a Tcl -extension and only later escaped to the wild as an independent library.) + * **src/** - This directory contains the primary source code for the + SQLite core. For historical reasons, C-code used for testing is + also found here. Source files intended for testing begin with "`test`". + The `tclsqlite3.c` and `tclsqlite3.h` files are the TCL interface + for SQLite and are also not part of the core. -Test scripts and programs are found in the **test/** subdirectory. -Additional test code is found in other source repositories. -See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for -additional information. + * **test/** - This directory and its subdirectories contains code used + for testing. Files that end in "`.test`" are TCL scripts that run + tests using an augmented TCL interpreter named "testfixture". Use + a command like "`make testfixture`" (unix) or + "`nmake /f Makefile.msc testfixture.exe`" (windows) to build that + augmented TCL interpreter, then run individual tests using commands like + "`testfixture test/main.test`". This test/ subdirectory also contains + additional C code modules and scripts for other kinds of testing. -The **ext/** subdirectory contains code for extensions. The -Full-text search engine is in **ext/fts3**. The R-Tree engine is in -**ext/rtree**. The **ext/misc** subdirectory contains a number of -smaller, single-file extensions, such as a REGEXP operator. + * **tool/** - This directory contains programs and scripts used to + build some of the machine-generated code that goes into the SQLite + core, as well as to build and run tests and perform diagnostics. + The source code to [the Lemon parser generator](./doc/lemon.html) is + found here. There are also TCL scripts used to build and/or transform + source code files. For example, the tool/mksqlite3h.tcl script reads + the src/sqlite.h.in file and uses it as a template to construct + the deliverable "sqlite3.h" file that defines the SQLite interface. -The **tool/** subdirectory contains various scripts and programs used -for building generated source code files or for testing or for generating -accessory programs such as "sqlite3_analyzer(.exe)". + * **ext/** - Various extensions to SQLite are found under this + directory. For example, the FTS5 subsystem is in "ext/fts5/". + Some of these extensions (ex: FTS3/4, FTS5, RTREE) might get built + into the SQLite amalgamation, but not all of them. The + "ext/misc/" subdirectory contains an assortment of one-file extensions, + many of which are omitted from the SQLite core, but which are included + in the [SQLite CLI](https://sqlite.org/cli.html). + + * **doc/** - Some documentation files about SQLite internals are found + here. Note, however, that the primary documentation designed for + application developers and users of SQLite is in a completely separate + repository. Note also that the primary API documentation is derived + from specially constructed comments in the src/sqlite.h.in file. ### Generated Source Code Files @@ -252,31 +284,37 @@ individual source file exceeds 32K lines in length. SQLite is modular in design. See the [architectural description](https://www.sqlite.org/arch.html) for details. Other documents that are useful in -(helping to understand how SQLite works include the +helping to understand how SQLite works include the [file format](https://www.sqlite.org/fileformat2.html) description, the [virtual machine](https://www.sqlite.org/opcode.html) that runs prepared statements, the description of [how transactions work](https://www.sqlite.org/atomiccommit.html), and the [overview of the query planner](https://www.sqlite.org/optoverview.html). -Years of effort have gone into optimizing SQLite, both +Decades of effort have gone into optimizing SQLite, both for small size and high performance. And optimizations tend to result in complex code. So there is a lot of complexity in the current SQLite implementation. It will not be the easiest library in the world to hack. -Key files: +### Key source code files * **sqlite.h.in** - This file defines the public interface to the SQLite library. Readers will need to be familiar with this interface before - trying to understand how the library works internally. + trying to understand how the library works internally. This file is + really a template that is transformed into the "sqlite3.h" deliverable + using a script invoked by the makefile. * **sqliteInt.h** - this header file defines many of the data objects used internally by SQLite. In addition to "sqliteInt.h", some - subsystems have their own header files. + subsystems inside of sQLite have their own header files. These internal + interfaces are not for use by applications. They can and do change + from one release of SQLite to the next. * **parse.y** - This file describes the LALR(1) grammar that SQLite uses to parse SQL statements, and the actions that are taken at each step - in the parsing process. + in the parsing process. The file is processed by the + [Lemon Parser Generator](./doc/lemon.html) to produce the actual C code + used for parsing. * **vdbe.c** - This file implements the virtual machine that runs prepared statements. There are various helper files whose names @@ -319,6 +357,11 @@ Key files: (and some other test programs too) is built and run when you type "make test". + * **VERSION**, **manifest**, and **manifest.uuid** - These files define + the current SQLite version number. The "VERSION" file is human generated, + but the "manifest" and "manifest.uuid" files are automatically generated + by the [Fossil version control system](https://fossil-scm/). + There are many other source files. Each has a succinct header comment that describes its purpose and role within the larger system. @@ -357,3 +400,7 @@ The main SQLite website is [https://sqlite.org/](https://sqlite.org/) with geographically distributed backups at [https://www2.sqlite.org/](https://www2.sqlite.org) and [https://www3.sqlite.org/](https://www3.sqlite.org). + +Contact the SQLite developers through the +[SQLite Forum](https://sqlite.org/forum/). In an emergency, you +can send private email to the lead developer at drh at sqlite dot org. diff --git a/manifest b/manifest index ed35eeaa1c..a806c63448 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Add\scode\scomments\sfor\sa\s"table-of-contents"\sand\svarious\smilestone\smarks\sin\nthe\s1300+\sline\slong\ssqlite3Select()\sfunction,\sto\shelp\simprove\sreadiability\nand\smaintainability.\s\sComment\schanges\sonly\s-\sno\sfunctional\schanges. -D 2024-08-21T12:01:46.057 +C Updates\sand\senhancements\sto\sthe\sREADME.md\sfile.\s\sNo\scode\schanges. +D 2024-08-21T13:44:40.490 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 209e9c64edf499b0ea7a15448699101c01d15828af35a3439d32fc25fdf1b787 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc 6c3fe8b6ce60e73f34a148c957d78b4648745c8d30e792423aa1a8d8bf12d065 -F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3 +F README.md 0f063e8688bf418c131b10b738cb0cb565188718152cc1ab59fa764396f0221d F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 92c80af1129051c9eded8df170730ad6366b4f7715dead34c4364c8149d0dce9 -R 96f2b1d6b34bcd20452ccf075e5a2b9a +P 00cfbb9fa5136e6a7847da4e4ab30d320ca79c22acda9db2030e558d59b8c744 +R 4f9e7c4307cf2bdaee18c0795e924886 U drh -Z 790a1d73ddaf8f8dea50d658062e6edf +Z 75d46cf029113e1403bee9c72a50cf8b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a4857cf3eb..5fe891c876 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00cfbb9fa5136e6a7847da4e4ab30d320ca79c22acda9db2030e558d59b8c744 +3161b8028916bff9a13a04bfb1996bce7e1274a3c403fc58f4d798afd30c528f From 5b5625f679aa130d5a16204bc06ce7e99b2f869f Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 21 Aug 2024 13:57:30 +0000 Subject: [PATCH 58/99] README.md: a markup link fix and use the canonical URL for Fossil. FossilOrigin-Name: c49bfee6b878bd96adfc852fdc76e98bf81e2fb82be8325672fbc7af04e7dffa --- README.md | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3d8b5379b0..4cddb47752 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows: To access sources directly using Fossil, first install Fossil version 2.0 or later. Source tarballs and precompiled binaries available -[here](https://www.fossil-scm.org/fossil/uv/download.html. Fossil is +[here](https://www.fossil-scm.org/home/uv/download.html). Fossil is a stand-alone program. To install, simply download or build the single executable file and put that file someplace on your $PATH. Then run commands like this: @@ -379,7 +379,7 @@ The `manifest.uuid` file should contain the SHA3-256 hash of the `manifest` file. If all of the above hash comparisons are correct, then you can be confident that your source tree is authentic and unadulterated. Details on the format for the `manifest` files are available -[on the Fossil website](https://fossil-scm.org/fossil/doc/trunk/www/fileformat.wiki#manifest). +[on the Fossil website](https://fossil-scm.org/home/doc/trunk/www/fileformat.wiki#manifest). The process of checking source code authenticity is automated by the makefile: diff --git a/manifest b/manifest index a806c63448..e6ceef9686 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Updates\sand\senhancements\sto\sthe\sREADME.md\sfile.\s\sNo\scode\schanges. -D 2024-08-21T13:44:40.490 +C README.md:\sa\smarkup\slink\sfix\sand\suse\sthe\scanonical\sURL\sfor\sFossil. +D 2024-08-21T13:57:30.455 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 209e9c64edf499b0ea7a15448699101c01d15828af35a3439d32fc25fdf1b787 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc 6c3fe8b6ce60e73f34a148c957d78b4648745c8d30e792423aa1a8d8bf12d065 -F README.md 0f063e8688bf418c131b10b738cb0cb565188718152cc1ab59fa764396f0221d +F README.md 3d47ef5758e2de29d6f4e1aca714d51424baba86c8b561fef330f6ef2dc9282c F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 00cfbb9fa5136e6a7847da4e4ab30d320ca79c22acda9db2030e558d59b8c744 -R 4f9e7c4307cf2bdaee18c0795e924886 -U drh -Z 75d46cf029113e1403bee9c72a50cf8b +P 3161b8028916bff9a13a04bfb1996bce7e1274a3c403fc58f4d798afd30c528f +R 2e449f153f783008445322155cf77dd3 +U stephan +Z fedf47a9f8fa7c0c7d99b3ea8b72d3a3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5fe891c876..04eae5dbf7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3161b8028916bff9a13a04bfb1996bce7e1274a3c403fc58f4d798afd30c528f +c49bfee6b878bd96adfc852fdc76e98bf81e2fb82be8325672fbc7af04e7dffa From 9904d824b890b9b30c514c14f2c5fcbc4118f456 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 21 Aug 2024 17:38:50 +0000 Subject: [PATCH 59/99] Improved comments on byte-code subroutine calls to materialize subqueries. Should not affect production builds. FossilOrigin-Name: 6b024c85ef856e26cddac8ad862e16909b5149c3e3efb4e73ba54c97291f1cc4 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 1 + src/where.c | 12 +++++++----- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index e6ceef9686..eba71f8794 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C README.md:\sa\smarkup\slink\sfix\sand\suse\sthe\scanonical\sURL\sfor\sFossil. -D 2024-08-21T13:57:30.455 +C Improved\scomments\son\sbyte-code\ssubroutine\scalls\sto\smaterialize\ssubqueries.\nShould\snot\saffect\sproduction\sbuilds. +D 2024-08-21T17:38:50.489 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -764,7 +764,7 @@ F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 26bd0ce6e8816f8968f50a377dc4255da4b392793e87ae27acd7276b56a447ce +F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe F src/shell.c.in fb7fe0a39473e7a3ad5ad7076ff49d87aa8ed7fdd98a1595345747ac5d78ac2b F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -850,7 +850,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c b6f79b189fcba459bb80420d3b4102f42a399be36ff29a3deff4ae4888fee46d +F src/where.c 61fea6c34c9c2350b114a4f64dce51ec073c2d8c50ca0734a0882ba6ab531805 F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e F src/wherecode.c 137797b0de9ddf1ff43e5b0edffcc76fb05184ed651fc4f5a0a01a45c0b89d04 F src/whereexpr.c 44f41ae554c7572e1de1485b3169b233ee04d464b2ee5881687ede3bf07cacfa @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3161b8028916bff9a13a04bfb1996bce7e1274a3c403fc58f4d798afd30c528f -R 2e449f153f783008445322155cf77dd3 -U stephan -Z fedf47a9f8fa7c0c7d99b3ea8b72d3a3 +P c49bfee6b878bd96adfc852fdc76e98bf81e2fb82be8325672fbc7af04e7dffa +R 862ab59c7e7e7a7369184a7d27a7302b +U drh +Z 51170964494b840c9c6cd6315a6727db # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 04eae5dbf7..2c55f6f94e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c49bfee6b878bd96adfc852fdc76e98bf81e2fb82be8325672fbc7af04e7dffa +6b024c85ef856e26cddac8ad862e16909b5149c3e3efb4e73ba54c97291f1cc4 diff --git a/src/select.c b/src/select.c index 0bb033785c..9fcf30ff4a 100644 --- a/src/select.c +++ b/src/select.c @@ -7863,6 +7863,7 @@ int sqlite3Select( pItem->fg.viaCoroutine = 1; pSubq->regResult = dest.iSdst; sqlite3VdbeEndCoroutine(v, pSubq->regReturn); + VdbeComment((v, "end %!S", pItem)); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ diff --git a/src/where.c b/src/where.c index c9f6b97c94..6e494850c3 100644 --- a/src/where.c +++ b/src/where.c @@ -7122,15 +7122,17 @@ WhereInfo *sqlite3WhereBegin( pSrc = &pTabList->a[pLevel->iFrom]; if( pSrc->fg.isMaterialized ){ Subquery *pSubq; + int iOnce = 0; assert( pSrc->fg.isSubquery ); pSubq = pSrc->u4.pSubq; - if( pSrc->fg.isCorrelated ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub); + if( pSrc->fg.isCorrelated==0 ){ + iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); }else{ - int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub); - sqlite3VdbeJumpHere(v, iOnce); + iOnce = 0; } + sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub); + VdbeComment((v, "materialize %!S", pSrc)); + if( iOnce ) sqlite3VdbeJumpHere(v, iOnce); } assert( pTabList == pWInfo->pTabList ); if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){ From be1fdbd272588cc61359dd3baf755bf8e857cf54 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 21 Aug 2024 18:57:55 +0000 Subject: [PATCH 60/99] Add the new SQLITE_TESTCTRL_OPTGET that retrieves the current optimization setting. Use this in the CLI to provide symbolic fine-grain control over optimization settings using the ".testctrl opt" dot-command. FossilOrigin-Name: 0ea4e1e8fc689cb1af3a8d21dd6af9d483115412b414e85e8a42480f50d65af2 --- manifest | 19 ++++---- manifest.uuid | 2 +- src/main.c | 12 +++++ src/shell.c.in | 114 +++++++++++++++++++++++++++++++++++++++++++++++- src/sqlite.h.in | 1 + 5 files changed, 137 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index eba71f8794..a206e45d7b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\son\sbyte-code\ssubroutine\scalls\sto\smaterialize\ssubqueries.\nShould\snot\saffect\sproduction\sbuilds. -D 2024-08-21T17:38:50.489 +C Add\sthe\snew\sSQLITE_TESTCTRL_OPTGET\sthat\sretrieves\sthe\scurrent\soptimization\nsetting.\s\sUse\sthis\sin\sthe\sCLI\sto\sprovide\ssymbolic\sfine-grain\scontrol\nover\soptimization\ssettings\susing\sthe\s".testctrl\sopt"\sdot-command. +D 2024-08-21T18:57:55.594 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -727,7 +727,7 @@ F src/insert.c f8d1a0f8ee258411009c6b7f2d93170e351bd19f5ad89d57e1180644297cbe70 F src/json.c 5b6a1d6015997b9ee848a32948720bdb26a0ef2de5a2127ebf7355ce66dbdc0d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 -F src/main.c 8a59d297ec77e6b78550433bfccb95a1b26f2fb69aaaf233206e21579a1cfcc1 +F src/main.c a520c325c7400b249242945f2c602acb19662c40f5db1c04d88664cfaa8b85ac F src/malloc.c 410e570b30c26cc36e3372577df50f7a96ee3eed5b2b161c6b6b48773c650c5e F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 @@ -765,8 +765,8 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe -F src/shell.c.in fb7fe0a39473e7a3ad5ad7076ff49d87aa8ed7fdd98a1595345747ac5d78ac2b -F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 +F src/shell.c.in 9a45cd618ed3cd09927e04bb0ccfd929fb5efa03a7166daec4645d230b7267f3 +F src/sqlite.h.in c6512f60f9fe06de46e49161e3f8fca72f3875f67dad8eaf469622567f434e48 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 F src/sqliteInt.h 28c878bdf528879afefe1994ac007d094f8061f2fdacdc55d6055d7e9341151e @@ -2210,8 +2210,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c49bfee6b878bd96adfc852fdc76e98bf81e2fb82be8325672fbc7af04e7dffa -R 862ab59c7e7e7a7369184a7d27a7302b +P 6b024c85ef856e26cddac8ad862e16909b5149c3e3efb4e73ba54c97291f1cc4 +R 5688f40bf117ffce0b629c4446a727b9 +T *branch * optimization-onoff +T *sym-optimization-onoff * +T -sym-trunk * U drh -Z 51170964494b840c9c6cd6315a6727db +Z b26803088924fd26f08434065f39c8bf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2c55f6f94e..6e30e87c81 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b024c85ef856e26cddac8ad862e16909b5149c3e3efb4e73ba54c97291f1cc4 +0ea4e1e8fc689cb1af3a8d21dd6af9d483115412b414e85e8a42480f50d65af2 diff --git a/src/main.c b/src/main.c index bff801a87d..5d6212208e 100644 --- a/src/main.c +++ b/src/main.c @@ -4393,6 +4393,18 @@ int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, sqlite3 *db, int *N) + ** + ** Write the current optimization settings into *N. A zero bit means that + ** the optimization is on, and a 1 bit means that the optimization is off. + */ + case SQLITE_TESTCTRL_GETOPT: { + sqlite3 *db = va_arg(ap, sqlite3*); + int *pN = va_arg(ap, int*); + *pN = db->dbOptFlags; + break; + } + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, onoff, xAlt); ** ** If parameter onoff is 1, subsequent calls to localtime() fail. diff --git a/src/shell.c.in b/src/shell.c.in index 8c4fcc4e12..a75b511903 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10925,7 +10925,7 @@ static int do_meta_command(char *zLine, ShellState *p){ {"json_selfcheck", SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN" }, {"localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN" }, {"never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN" }, - {"optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK" }, + {"optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK ..."}, #ifdef YYCOVERAGE {"parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE,0,"" }, #endif @@ -10989,8 +10989,118 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ switch(testctrl){ + /* Special processing for .testctrl opt MASK ... + ** Each MASK argument can be one of: + ** + ** +LABEL Enable the named optimization + ** + ** -LABEL Disable the named optimization + ** + ** INTEGER Mask of optimizations to disable + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + static const struct { + unsigned int mask; /* Mask for this optimization */ + unsigned int bDsply; /* Display this on output */ + const char *zLabel; /* Name of optimization */ + } aLabel[] = { + { 0x00000001, 1, "QueryFlattener" }, + { 0x00000001, 0, "Flatten" }, + { 0x00000002, 1, "WindowFunc" }, + { 0x00000004, 1, "GroupByOrder" }, + { 0x00000008, 1, "FactorOutConst" }, + { 0x00000010, 1, "DistinctOpt" }, + { 0x00000020, 1, "CoverIdxScan" }, + { 0x00000040, 1, "OrderByIdxJoin" }, + { 0x00000080, 1, "Transitive" }, + { 0x00000100, 1, "OmitNoopJoin" }, + { 0x00000200, 1, "CountOfView" }, + { 0x00000400, 1, "CurosrHints" }, + { 0x00000800, 1, "Stat4" }, + { 0x00001000, 1, "PushDown" }, + { 0x00002000, 1, "SimplifyJoin" }, + { 0x00004000, 1, "SkipScan" }, + { 0x00008000, 1, "PropagateConst" }, + { 0x00010000, 1, "MinMaxOpt" }, + { 0x00020000, 1, "SeekScan" }, + { 0x00040000, 1, "OmitOrderBy" }, + { 0x00080000, 1, "BloomFilter" }, + { 0x00100000, 1, "BloomPulldown" }, + { 0x00200000, 1, "BalancedMerge" }, + { 0x00400000, 1, "ReleaseReg" }, + { 0x00800000, 1, "FlttnUnionAll" }, + { 0x01000000, 1, "IndexedEXpr" }, + { 0x02000000, 1, "Coroutines" }, + { 0x04000000, 1, "NullUnusedCols" }, + { 0x08000000, 1, "OnePass" }, + { 0x10000000, 1, "OrderBySubq" }, + { 0xffffffff, 0, "All" }, + }; + unsigned int curOpt; + unsigned int newOpt; + int ii; + sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, p->db, &curOpt); + newOpt = curOpt; + for(ii=2; ii=ArraySize(aLabel) ){ + eputf("Error: no such optimization: \"%s\"\n", zLabel); + eputf("Should be one of:"); + for(jj=0; jjdb,newOpt); + }else if( nArg<3 ){ + curOpt = ~newOpt; + } + if( newOpt==0 ){ + oputf("+All\n"); + }else if( newOpt==0xffffffff ){ + oputf("-All\n"); + }else{ + int jj; + for(jj=0; jj Date: Thu, 22 Aug 2024 16:22:08 +0000 Subject: [PATCH 61/99] Add the SQLITE_INDEX_SCAN_HEX bit to the sqlite3_index_info.idxFlags bitmask. When set, this bit causes the EXPLAIN QUERY PLAN output to show the idxNum value in hex rather than in decimal. This is purely a debugging aid. FossilOrigin-Name: 6c00e88ebdb41d6317bb8758825521614dedc2e6e6289ff415c5f0406eed815b --- manifest | 21 ++++++++++----------- manifest.uuid | 2 +- src/dbstat.c | 1 + src/sqlite.h.in | 12 ++++++++---- src/where.c | 1 + src/whereInt.h | 1 + src/wherecode.c | 4 +++- 7 files changed, 25 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index e659774b46..1cf656fb1c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_TESTCTRL_GETOPT.\s\sUse\sit\sto\simplement\simprovements\sto\sthe\n".testctrl\sopt"\sdot-command\sin\sthe\sCLI,\sand\ssimilar\senhancements\sto\sTH3. -D 2024-08-21T20:25:31.840 +C Add\sthe\sSQLITE_INDEX_SCAN_HEX\sbit\sto\sthe\ssqlite3_index_info.idxFlags\sbitmask.\nWhen\sset,\sthis\sbit\scauses\sthe\sEXPLAIN\sQUERY\sPLAN\soutput\sto\sshow\sthe\sidxNum\nvalue\sin\shex\srather\sthan\sin\sdecimal.\s\sThis\sis\spurely\sa\sdebugging\said. +D 2024-08-22T16:22:08.900 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -712,7 +712,7 @@ F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 -F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 +F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 444c4d1eaac40103461e3b6f0881846dd3aafc1cec1dd169d3482fa331667da7 F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 @@ -766,7 +766,7 @@ F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe F src/shell.c.in 9a45cd618ed3cd09927e04bb0ccfd929fb5efa03a7166daec4645d230b7267f3 -F src/sqlite.h.in c6512f60f9fe06de46e49161e3f8fca72f3875f67dad8eaf469622567f434e48 +F src/sqlite.h.in f07bff4225a1244efd604a0ffef81ed69f29d3dbaed7e22f906f26229ba3ca9e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 F src/sqliteInt.h 28c878bdf528879afefe1994ac007d094f8061f2fdacdc55d6055d7e9341151e @@ -850,9 +850,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 61fea6c34c9c2350b114a4f64dce51ec073c2d8c50ca0734a0882ba6ab531805 -F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e -F src/wherecode.c 137797b0de9ddf1ff43e5b0edffcc76fb05184ed651fc4f5a0a01a45c0b89d04 +F src/where.c 70392f65e2c76a27ab1faed05892c2402ce547e574074a4f10282b3bc1ffe4eb +F src/whereInt.h a5d079c346a658b7a6e9e47bb943d021e02fa1e6aed3b964ca112112a4892192 +F src/wherecode.c 5172d647798134e7c92536ddffe7e530c393d79b5dedd648b88faf2646c65baf F src/whereexpr.c 44f41ae554c7572e1de1485b3169b233ee04d464b2ee5881687ede3bf07cacfa F src/window.c 499d48f315a09242dc68f2fac635ed27dcf6bbb0d9ab9084857898c64489e975 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -2210,9 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6b024c85ef856e26cddac8ad862e16909b5149c3e3efb4e73ba54c97291f1cc4 0ea4e1e8fc689cb1af3a8d21dd6af9d483115412b414e85e8a42480f50d65af2 -R 5688f40bf117ffce0b629c4446a727b9 -T +closed 0ea4e1e8fc689cb1af3a8d21dd6af9d483115412b414e85e8a42480f50d65af2 +P 6500baa9914b08ce19c361a69f19a80e5d800fcb82579d1c1e03601c7a664338 +R 7b7b9fd0e23c278e007dcb15333f71d2 U drh -Z 0dda3a273baa063a0338cbb5653c54ad +Z e92224536d6b88aa809210ea20e1cd7b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b34147cbe2..a3815e38d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6500baa9914b08ce19c361a69f19a80e5d800fcb82579d1c1e03601c7a664338 +6c00e88ebdb41d6317bb8758825521614dedc2e6e6289ff415c5f0406eed815b diff --git a/src/dbstat.c b/src/dbstat.c index c70d806370..d635a82975 100644 --- a/src/dbstat.c +++ b/src/dbstat.c @@ -279,6 +279,7 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->orderByConsumed = 1; pIdxInfo->idxNum |= 0x08; } + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_HEX; return SQLITE_OK; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 3c98e95d9f..810ccecc96 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7427,9 +7427,11 @@ struct sqlite3_module { ** will be returned by the strategy. ** ** The xBestIndex method may optionally populate the idxFlags field with a -** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag - -** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite -** assumes that the strategy may visit at most one row. +** mask of SQLITE_INDEX_SCAN_* flags. One such flag is +** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN] +** output to show the idxNum has hex instead of as decimal. Another flag is +** SQLITE_INDEX_SCAN_UNIQUE, which if set indicates that the query plan will +** return at most one row. ** ** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then ** SQLite also assumes that if a call to the xUpdate() method is made as @@ -7493,7 +7495,9 @@ struct sqlite3_index_info { ** [sqlite3_index_info].idxFlags field to some combination of ** these bits. */ -#define SQLITE_INDEX_SCAN_UNIQUE 1 /* Scan visits at most 1 row */ +#define SQLITE_INDEX_SCAN_UNIQUE 0x00000001 /* Scan visits at most 1 row */ +#define SQLITE_INDEX_SCAN_HEX 0x00000002 /* Display idxNum as hex */ + /* in EXPLAIN QUERY PLAN */ /* ** CAPI3REF: Virtual Table Constraint Operator Codes diff --git a/src/where.c b/src/where.c index 6e494850c3..a82f0305b4 100644 --- a/src/where.c +++ b/src/where.c @@ -4349,6 +4349,7 @@ static int whereLoopAddVirtualOne( pNew->u.vtab.idxStr = pIdxInfo->idxStr; pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? pIdxInfo->nOrderBy : 0); + pNew->u.vtab.bIdxNumHex = (pIdxInfo->idxFlags&SQLITE_INDEX_SCAN_HEX)!=0; pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); diff --git a/src/whereInt.h b/src/whereInt.h index c9afed267b..8247528a93 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -149,6 +149,7 @@ struct WhereLoop { int idxNum; /* Index number */ u32 needFree : 1; /* True if sqlite3_free(idxStr) is needed */ u32 bOmitOffset : 1; /* True to let virtual table handle offset */ + u32 bIdxNumHex : 1; /* Show idxNum as hex in EXPLAIN QUERY PLAN */ i8 isOrdered; /* True if satisfies ORDER BY */ u16 omitMask; /* Terms that may be omitted */ char *idxStr; /* Index identifier string */ diff --git a/src/wherecode.c b/src/wherecode.c index b0c0d9aedd..0951e5e204 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -200,7 +200,9 @@ int sqlite3WhereExplainOneScan( } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ - sqlite3_str_appendf(&str, " VIRTUAL TABLE INDEX %d:%s", + sqlite3_str_appendall(&str, " VIRTUAL TABLE INDEX "); + sqlite3_str_appendf(&str, + pLoop->u.vtab.bIdxNumHex ? "0x%x:%s" : "%d:%s", pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif From 5804da02182927dafae24a83bfe4176bb1746118 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 22 Aug 2024 18:12:10 +0000 Subject: [PATCH 62/99] Enhance the generate_series() table-valued function such that it is able to recognize equality and inequality constraints on the "value" column and optimize its operating accordingly. FossilOrigin-Name: d50b784807333c5461a2d027778c746c799285b95bb1952f142b317ea2846460 --- ext/misc/series.c | 254 ++++++++++++++++++++++++++++++++++++++------ manifest | 18 ++-- manifest.uuid | 2 +- test/merge1.test | 24 ++--- test/tabfunc01.test | 12 +-- test/tester.tcl | 2 +- 6 files changed, 252 insertions(+), 60 deletions(-) diff --git a/ext/misc/series.c b/ext/misc/series.c index 0dfed181f6..f2ca75c253 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -90,6 +90,26 @@ ** and a very large cost if either start or stop are unavailable. This ** encourages the query planner to order joins such that the bounds of the ** series are well-defined. +** +** Update on 2024-08-22: +** xBestIndex now also looks for equality and inequality constraints against +** the value column and uses those constraints as additional bounds against +** the sequence range. Thus, a query like this: +** +** SELECT value FROM generate_series($SA,$EA) +** WHERE value BETWEEN $SB AND $EB; +** +** Is logically the same as: +** +** SELECT value FROM generate_series(max($SA,$SB),min($EA,$EB)); +** +** Constraints on the value column can server as substitutes for constraints +** on the hidden start and stop columns. So, the following two queries +** are equivalent: +** +** SELECT value FROM generate_series($S,$E); +** SELECT value FROM generate_series WHERE value BETWEEN $S and $E; +** */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -131,8 +151,10 @@ static sqlite3_int64 genSeqMember( typedef unsigned char u8; typedef struct SequenceSpec { - sqlite3_int64 iBase; /* Starting value ("start") */ - sqlite3_int64 iTerm; /* Given terminal value ("stop") */ + sqlite3_int64 iOBase; /* Original starting value ("start") */ + sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ + sqlite3_int64 iBase; /* Starting value to actually use */ + sqlite3_int64 iTerm; /* Terminal value to actually use */ sqlite3_int64 iStep; /* Increment ("step") */ sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ @@ -325,9 +347,9 @@ static int seriesColumn( series_cursor *pCur = (series_cursor*)cur; sqlite3_int64 x = 0; switch( i ){ - case SERIES_COLUMN_START: x = pCur->ss.iBase; break; - case SERIES_COLUMN_STOP: x = pCur->ss.iTerm; break; - case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; + case SERIES_COLUMN_START: x = pCur->ss.iOBase; break; + case SERIES_COLUMN_STOP: x = pCur->ss.iOTerm; break; + case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; default: x = pCur->ss.iValueNow; break; } sqlite3_result_int64(ctx, x); @@ -335,7 +357,9 @@ static int seriesColumn( } #ifndef LARGEST_UINT64 +#define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) #define LARGEST_UINT64 (0xffffffff|(((sqlite3_uint64)0xffffffff)<<32)) +#define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) #endif /* @@ -376,13 +400,18 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ ** parameter. (idxStr is not used in this implementation.) idxNum ** is a bitmask showing which constraints are available: ** -** 0x01: start=VALUE -** 0x02: stop=VALUE -** 0x04: step=VALUE -** 0x08: descending order -** 0x10: ascending order -** 0x20: LIMIT VALUE -** 0x40: OFFSET VALUE +** 0x0001: start=VALUE +** 0x0002: stop=VALUE +** 0x0004: step=VALUE +** 0x0008: descending order +** 0x0010: ascending order +** 0x0020: LIMIT VALUE +** 0x0040: OFFSET VALUE +** 0x0080: value=VALUE +** 0x0100: value>=VALUE +** 0x0200: value>VALUE +** 0x1000: value<=VALUE +** 0x2000: valuess.iBase = sqlite3_value_int64(argv[i++]); @@ -416,16 +451,98 @@ static int seriesFilter( }else{ pCur->ss.iStep = 1; } + + /* If there are constraints on the value column but there are + ** no constraints on the start, stop, and step columns, then + ** initialize the default range to be the entire range of 64-bit signed + ** integers. This range will contracted by the value column constraints + ** further below. + */ + if( (idxNum & 0x05)==0 && (idxNum & 0x0380)!=0 ){ + pCur->ss.iBase = SMALLEST_INT64; + } + if( (idxNum & 0x06)==0 && (idxNum & 0x3080)!=0 ){ + pCur->ss.iTerm = LARGEST_INT64; + } + pCur->ss.iOBase = pCur->ss.iBase; + pCur->ss.iOTerm = pCur->ss.iTerm; + + /* Extract the LIMIT and OFFSET values, but do not apply them yet. + ** The range must first be constrained by the limits on value. + */ if( idxNum & 0x20 ){ - sqlite3_int64 iLimit = sqlite3_value_int64(argv[i++]); - sqlite3_int64 iTerm; + iLimit = sqlite3_value_int64(argv[i++]); if( idxNum & 0x40 ){ - sqlite3_int64 iOffset = sqlite3_value_int64(argv[i++]); - if( iOffset>0 ){ - pCur->ss.iBase += pCur->ss.iStep*iOffset; + iOffset = sqlite3_value_int64(argv[i++]); + } + } + + if( idxNum & 0x3380 ){ + /* Extract the maximum range of output values determined by + ** constraints on the "value" column. + */ + if( idxNum & 0x0080 ){ + iMin = iMax = sqlite3_value_int64(argv[i++]); + }else{ + if( idxNum & 0x0300 ){ + iMin = sqlite3_value_int64(argv[i++]); + if( idxNum & 0x0200 ){ + if( iMin==LARGEST_INT64 ){ + returnNoRows = 1; + }else{ + iMin++; + } + } + } + if( idxNum & 0x3000 ){ + iMax = sqlite3_value_int64(argv[i++]); + if( idxNum & 0x2000 ){ + if( iMax==SMALLEST_INT64 ){ + returnNoRows = 1; + }else{ + iMax--; + } + } + } + if( iMin>iMax ){ + returnNoRows = 1; } } + + /* Try to reduce the range of values to be generated based on + ** constraints on the "value" column. + */ + if( pCur->ss.iStep>0 ){ + sqlite3_int64 szStep = pCur->ss.iStep; + if( pCur->ss.iBasess.iBase; + pCur->ss.iBase += ((d+szStep-1)/szStep)*szStep; + } + if( pCur->ss.iTerm>iMax ){ + sqlite3_uint64 d = pCur->ss.iTerm - iMax; + pCur->ss.iTerm -= ((d+szStep-1)/szStep)*szStep; + } + }else{ + sqlite3_int64 szStep = -pCur->ss.iStep; + assert( szStep>0 ); + if( pCur->ss.iBase>iMax ){ + sqlite3_uint64 d = pCur->ss.iBase - iMax; + pCur->ss.iBase -= ((d+szStep-1)/szStep)*szStep; + } + if( pCur->ss.iTermss.iTerm; + pCur->ss.iTerm += ((d+szStep-1)/szStep)*szStep; + } + } + } + + /* Apply LIMIT and OFFSET constraints, if any */ + if( idxNum & 0x20 ){ + if( iOffset>0 ){ + pCur->ss.iBase += pCur->ss.iStep*iOffset; + } if( iLimit>=0 ){ + sqlite3_int64 iTerm; iTerm = pCur->ss.iBase + (iLimit - 1)*pCur->ss.iStep; if( pCur->ss.iStep<0 ){ if( iTerm>pCur->ss.iTerm ) pCur->ss.iTerm = iTerm; @@ -434,16 +551,21 @@ static int seriesFilter( } } } + + for(i=0; iss.iBase = 1; - pCur->ss.iTerm = 0; - pCur->ss.iStep = 1; + returnNoRows = 1; break; } } + if( returnNoRows ){ + pCur->ss.iBase = 1; + pCur->ss.iTerm = 0; + pCur->ss.iStep = 1; + } if( idxNum & 0x08 ){ pCur->ss.isReversing = pCur->ss.iStep > 0; }else{ @@ -464,13 +586,35 @@ static int seriesFilter( ** ** The query plan is represented by bits in idxNum: ** -** 0x01 start = $value -- constraint exists -** 0x02 stop = $value -- constraint exists -** 0x04 step = $value -- constraint exists -** 0x08 output is in descending order -** 0x10 output is in ascending order -** 0x20 LIMIT $value -- constraint exists -** 0x40 OFFSET $value -- constraint exists +** 0x0001 start = $num +** 0x0002 stop = $num +** 0x0004 step = $num +** 0x0008 output is in descending order +** 0x0010 output is in ascending order +** 0x0020 LIMIT $num +** 0x0040 OFFSET $num +** 0x0080 value = $num +** 0x0100 value >= $num +** 0x0200 value > $num +** 0x1000 value <= $num +** 0x2000 value < $num +** +** Only one of 0x0100 or 0x0200 will be returned. Similarly, only +** one of 0x1000 or 0x2000 will be returned. If the 0x0080 is set, then +** none of the 0xff00 bits will be set. +** +** The order of parameters passed to xFilter is as follows: +** +** * The argument to start= if bit 0x0001 is in the idxNum mask +** * The argument to stop= if bit 0x0002 is in the idxNum mask +** * The argument to step= if bit 0x0004 is in the idxNum mask +** * The argument to LIMIT if bit 0x0020 is in the idxNum mask +** * The argument to OFFSET if bit 0x0040 is in the idxNum mask +** * The argument to value=, or value>= or value> if any of +** bits 0x0380 are in the idxNum mask +** * The argument to value<= or value< if either of bits 0x3000 +** are in the mask +** */ static int seriesBestIndex( sqlite3_vtab *pVTab, @@ -483,7 +627,9 @@ static int seriesBestIndex( #endif int unusableMask = 0; /* Mask of unusable constraints */ int nArg = 0; /* Number of arguments that seriesFilter() expects */ - int aIdx[5]; /* Constraints on start, stop, step, LIMIT, OFFSET */ + int aIdx[7]; /* Constraints on start, stop, step, LIMIT, OFFSET, + ** and value. aIdx[5] covers value=, value>=, and + ** value>, aIdx[6] covers value<= and value< */ const struct sqlite3_index_constraint *pConstraint; /* This implementation assumes that the start, stop, and step columns @@ -491,7 +637,7 @@ static int seriesBestIndex( assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 ); assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 ); - aIdx[0] = aIdx[1] = aIdx[2] = aIdx[3] = aIdx[4] = -1; + aIdx[0] = aIdx[1] = aIdx[2] = aIdx[3] = aIdx[4] = aIdx[5] = aIdx[6] = -1; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ int iCol; /* 0 for start, 1 for stop, 2 for step */ @@ -512,7 +658,50 @@ static int seriesBestIndex( } continue; } - if( pConstraint->iColumniColumn==SERIES_COLUMN_VALUE ){ + switch( op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: + case SQLITE_INDEX_CONSTRAINT_IS: { + idxNum |= 0x0080; + idxNum &= ~0x3300; + aIdx[5] = i; + aIdx[6] = -1; + bStartSeen = 1; + break; + } + case SQLITE_INDEX_CONSTRAINT_GE: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x0100; + idxNum &= ~0x0200; + aIdx[5] = i; + bStartSeen = 1; + break; + } + case SQLITE_INDEX_CONSTRAINT_GT: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x0200; + idxNum &= ~0x0100; + aIdx[5] = i; + bStartSeen = 1; + break; + } + case SQLITE_INDEX_CONSTRAINT_LE: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x1000; + idxNum &= ~0x2000; + aIdx[6] = i; + break; + } + case SQLITE_INDEX_CONSTRAINT_LT: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x2000; + idxNum &= ~0x1000; + aIdx[6] = i; + break; + } + } + continue; + } iCol = pConstraint->iColumn - SERIES_COLUMN_START; assert( iCol>=0 && iCol<=2 ); iMask = 1 << iCol; @@ -534,7 +723,7 @@ static int seriesBestIndex( idxNum &= ~0x60; aIdx[4] = 0; } - for(i=0; i<5; i++){ + for(i=0; i<7; i++){ if( (j = aIdx[i])>=0 ){ pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg; pIdxInfo->aConstraintUsage[j].omit = @@ -582,6 +771,9 @@ static int seriesBestIndex( pIdxInfo->estimatedRows = 2147483647; } pIdxInfo->idxNum = idxNum; +#ifdef SQLITE_INDEX_SCAN_HEX + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_HEX; +#endif return SQLITE_OK; } diff --git a/manifest b/manifest index 1cf656fb1c..2c5b73a1b1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_INDEX_SCAN_HEX\sbit\sto\sthe\ssqlite3_index_info.idxFlags\sbitmask.\nWhen\sset,\sthis\sbit\scauses\sthe\sEXPLAIN\sQUERY\sPLAN\soutput\sto\sshow\sthe\sidxNum\nvalue\sin\shex\srather\sthan\sin\sdecimal.\s\sThis\sis\spurely\sa\sdebugging\said. -D 2024-08-22T16:22:08.900 +C Enhance\sthe\sgenerate_series()\stable-valued\sfunction\ssuch\sthat\sit\sis\sable\sto\nrecognize\sequality\sand\sinequality\sconstraints\son\sthe\s"value"\scolumn\sand\noptimize\sits\soperating\saccordingly. +D 2024-08-22T18:12:10.402 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -417,7 +417,7 @@ F ext/misc/regexp.c 4bdd0045912f81c84908bd535ec5ad3b1c8540b4287c70ab840709636240 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 -F ext/misc/series.c d96e5aac21658c6b5d54f918ac140460ec7197734c1a4fba806950831a7b1e7a +F ext/misc/series.c a6089b5e8e3002bd1e5d9877cee6aead0b9a6426e406c09a399817db9e9ae823 F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d F ext/misc/shathree.c 1821d90a0040c9accdbe3e3527d378d30569475d758aa70f6848924c0b430e8c F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 @@ -1441,7 +1441,7 @@ F test/memjournal2.test dbc2c5cb5f7b38950f4f6dc3e73fcecf0fcbed3fc32c7ce913bba164 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 F test/memsubsys1.test 86b8158752af9188ed5b32a30674a1ef71183e6bc4e6808e815cd658ca9058a6 F test/memsubsys2.test 774b93cb09ca50d1b759bb7c645baa2a9ce172edc3a3da67d5150a26a9fc2a08 -F test/merge1.test 2de6d6ef8d25402764b1aab49d8f9d7f89208c89a6674e437f76de4c812157b8 +F test/merge1.test 7dd9dc6838bcd0623a069485fe3a8dd498a051c16e1877cf84f506c0d6a29b43 F test/minmax.test fe638b55d77d2375531a8f549b338eafcd9adfbd2f72df37ed77d9b26ca0a71a F test/minmax2.test cf9311babb6f0518d04e42fd6a42c619531c4309a9dd790a2c4e9b3bc595e0de F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 @@ -1700,7 +1700,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test f150d206294471d20f50029e6b46b76b87a7a010b16dc57eb44245c76dd02802 +F test/tabfunc01.test 6002a5f37b76355f173c75c2b3b03173b19d6a8b078c5baaa4c78bbcd0fa6323 F test/table.test 7862a00b58b5541511a26757ea9c5c7c3f8298766e98aa099deec703d9c0a8e0 F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -1712,7 +1712,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl e88c498c369cff6bf0898db6d04088685066730be51821ef775ef13fd2b1d077 +F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 F test/testrunner.tcl 5d02deeba7a53baeadae6aa7641d90aac58fdfa3a7bcac85cfcfd752b1aab87c F test/testrunner_data.tcl c5ae2b1f9a99210b0600d002fb3af1fee350997cee9416551e83b93501360ebf F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6500baa9914b08ce19c361a69f19a80e5d800fcb82579d1c1e03601c7a664338 -R 7b7b9fd0e23c278e007dcb15333f71d2 +P 6c00e88ebdb41d6317bb8758825521614dedc2e6e6289ff415c5f0406eed815b +R c06569ccdd3e5d2b3b096a4379f23ac8 U drh -Z e92224536d6b88aa809210ea20e1cd7b +Z e410ad1cfdbbb6e926b30bd406bc8816 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a3815e38d9..b90e36fb5a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c00e88ebdb41d6317bb8758825521614dedc2e6e6289ff415c5f0406eed815b +d50b784807333c5461a2d027778c746c799285b95bb1952f142b317ea2846460 diff --git a/test/merge1.test b/test/merge1.test index 7ec4dab108..686271648a 100644 --- a/test/merge1.test +++ b/test/merge1.test @@ -64,21 +64,21 @@ do_eqp_test 101 { | |--LEFT | | `--MERGE (UNION ALL) | | |--LEFT - | | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | | `--RIGHT - | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | `--RIGHT - | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: `--RIGHT `--MERGE (UNION ALL) |--LEFT | `--MERGE (UNION ALL) | |--LEFT - | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | `--RIGHT - | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: `--RIGHT - `--SCAN generate_series VIRTUAL TABLE INDEX 23: + `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: } # Same test with the blanced-merge optimization @@ -129,17 +129,17 @@ do_eqp_test 111 { | | | |--LEFT | | | | `--MERGE (UNION ALL) | | | | |--LEFT - | | | | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | | | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | | | | `--RIGHT - | | | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | | | `--RIGHT - | | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | | `--RIGHT - | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | `--RIGHT - | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: `--RIGHT - `--SCAN generate_series VIRTUAL TABLE INDEX 23: + `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: } finish_test diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 3a62b81f99..8c24198e1d 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -121,26 +121,26 @@ do_eqp_test tabfunc01-3.10 { SELECT value FROM generate_series(1,10) ORDER BY value; } { QUERY PLAN - `--SCAN generate_series VIRTUAL TABLE INDEX 19: + `--SCAN generate_series VIRTUAL TABLE INDEX 0x13: } do_eqp_test tabfunc01-3.11 { SELECT value FROM generate_series(1,10) ORDER BY +value; } { QUERY PLAN - |--SCAN generate_series VIRTUAL TABLE INDEX 3: + |--SCAN generate_series VIRTUAL TABLE INDEX 0x3: `--USE TEMP B-TREE FOR ORDER BY } do_eqp_test tabfunc01-3.12 { SELECT value FROM generate_series(1,10) ORDER BY value, stop; } { QUERY PLAN - `--SCAN generate_series VIRTUAL TABLE INDEX 19: + `--SCAN generate_series VIRTUAL TABLE INDEX 0x13: } do_eqp_test tabfunc01-3.13 { SELECT value FROM generate_series(1,10) ORDER BY stop, value; } { QUERY PLAN - |--SCAN generate_series VIRTUAL TABLE INDEX 3: + |--SCAN generate_series VIRTUAL TABLE INDEX 0x3: `--USE TEMP B-TREE FOR ORDER BY } @@ -156,9 +156,9 @@ do_eqp_test tabfunc01-3.20 { QUERY PLAN `--MERGE (UNION ALL) |--LEFT - | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: `--RIGHT - `--SCAN generate_series VIRTUAL TABLE INDEX 23: + `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: } diff --git a/test/tester.tcl b/test/tester.tcl index 43f7c7a047..164ee47f42 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -985,7 +985,7 @@ proc query_plan_graph {sql} { } set a "\n QUERY PLAN\n" append a [append_graph " " dx cx 0] - regsub -all { 0x[A-F0-9]+\y} $a { xxxxxx} a + regsub -all {SUBQUERY 0x[A-F0-9]+\y} $a {SUBQUERY xxxxxx} a regsub -all {(MATERIALIZE|CO-ROUTINE|SUBQUERY) \d+\y} $a {\1 xxxxxx} a regsub -all {\((join|subquery)-\d+\)} $a {(\1-xxxxxx)} a return $a From 8d4edd0d67db8048f86426378ff8f7bba3c9e67b Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Aug 2024 15:18:02 +0000 Subject: [PATCH 63/99] Add fts5 auxiliary function fts5_get_locale(). For querying the locale of a stored value. FossilOrigin-Name: 396f720f36a937145ac727c2750acfd26e4eda350b46d648d82a7e9985545a9f --- ext/fts5/fts5_aux.c | 54 +++++++++++++++++++-- ext/fts5/test/fts5faultI.test | 24 +++++++++ ext/fts5/test/fts5locale.test | 91 +++++++++++++++++++++++++++++++++++ manifest | 18 +++---- manifest.uuid | 2 +- 5 files changed, 176 insertions(+), 13 deletions(-) diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index eb3f7e359d..ad578156da 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -746,6 +746,53 @@ static void fts5Bm25Function( } } +/* +** Implementation of fts5_get_locale() function. +*/ +static void fts5GetLocaleFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + int iCol = 0; + int eType = 0; + int rc = SQLITE_OK; + const char *zLocale = 0; + int nLocale = 0; + + /* xColumnLocale() must be available */ + assert( pApi->iVersion>=4 ); + + if( nVal!=1 ){ + const char *z = "wrong number of arguments to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + eType = sqlite3_value_numeric_type(apVal[0]); + if( eType!=SQLITE_INTEGER ){ + const char *z = "non-integer argument passed to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + iCol = sqlite3_value_int(apVal[0]); + if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){ + sqlite3_result_error_code(pCtx, SQLITE_RANGE); + return; + } + + rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + return; + } + + sqlite3_result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT); +} + int sqlite3Fts5AuxInit(fts5_api *pApi){ struct Builtin { const char *zFunc; /* Function name (nul-terminated) */ @@ -753,9 +800,10 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){ fts5_extension_function xFunc;/* Callback function */ void (*xDestroy)(void*); /* Destructor function */ } aBuiltin [] = { - { "snippet", 0, fts5SnippetFunction, 0 }, - { "highlight", 0, fts5HighlightFunction, 0 }, - { "bm25", 0, fts5Bm25Function, 0 }, + { "snippet", 0, fts5SnippetFunction, 0 }, + { "highlight", 0, fts5HighlightFunction, 0 }, + { "bm25", 0, fts5Bm25Function, 0 }, + { "fts5_get_locale", 0, fts5GetLocaleFunction, 0 }, }; int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test index 08a6bf056b..63bfdd68ac 100644 --- a/ext/fts5/test/fts5faultI.test +++ b/ext/fts5/test/fts5faultI.test @@ -231,6 +231,30 @@ do_faultsim_test 9.2 -faults oom* -prep { faultsim_test_result {0 1} } +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, locale=1); + INSERT INTO ft VALUES(fts5_locale('hello', 'one two three i ii iii')); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +do_execsql_test 10.1 { + SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three' +} {hello} + +faultsim_save_and_close +do_faultsim_test 10 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three' + } +} -test { + faultsim_test_result {0 hello} +} finish_test diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index d64df18491..684dcecd80 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -572,5 +572,96 @@ do_test 12.0 { } msg] $msg } {1 {error in fts5_api.xCreateTokenizer_v2()}} +#------------------------------------------------------------------------- +# Tests for auxiliary function fts5_get_locale(). +# +reset_db + +# Check that if the table does not support locale=1, fts5_get_locale() +# always returns NULL. +do_execsql_test 13.1.0 { + CREATE VIRTUAL TABLE nolocale USING fts5(a, b); + INSERT INTO nolocale VALUES('one two three', 'four five six'); + INSERT INTO nolocale VALUES('three two one', 'seven eight nine'); +} +do_execsql_test 13.1.1 { + SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale; +} {1 1} +do_execsql_test 13.1.2 { + SELECT fts5_get_locale(nolocale, 1) IS NULL FROM nolocale('one + two'); +} {1} +do_execsql_test 13.1.3 { + SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale('one AND two'); +} {1 1} +do_execsql_test 13.1.4 { + SELECT + fts5_get_locale(nolocale, 1) IS NULL + FROM nolocale('three AND two') ORDER BY rank +} {1 1} +do_catchsql_test 13.1.5 { + SELECT fts5_get_locale(nolocale, 2) IS NULL FROM nolocale('three AND two'); +} {1 {column index out of range}} +do_catchsql_test 13.1.6 { + SELECT fts5_get_locale(nolocale, -1) IS NULL FROM nolocale('three AND two'); +} {1 {column index out of range}} +do_catchsql_test 13.1.7 { + SELECT fts5_get_locale(nolocale) IS NULL FROM nolocale('three AND two'); +} {1 {wrong number of arguments to function fts5_get_locale()}} +do_catchsql_test 13.1.8 { + SELECT fts5_get_locale(nolocale, 0, 0) IS NULL FROM nolocale('three AND two'); +} {1 {wrong number of arguments to function fts5_get_locale()}} +do_catchsql_test 13.1.9 { + SELECT fts5_get_locale(nolocale, 'text') FROM nolocale('three AND two'); +} {1 {non-integer argument passed to function fts5_get_locale()}} + + +# Check that if the table does support locale=1, fts5_get_locale() +# returns the locale of the identified row/column. +do_execsql_test 13.2.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1); + INSERT INTO ft VALUES( + fts5_locale('th_TH', 'one two three'), 'four five six seven' + ); + INSERT INTO ft VALUES( + 'three two one', fts5_locale('en_AU', 'seven eight nine') + ); +} + +do_execsql_test 13.2.1 { + SELECT quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) FROM ft +} { 'th_TH' NULL NULL 'en_AU' } +do_execsql_test 13.2.2 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') +} { 'th_TH' NULL NULL 'en_AU' } +do_execsql_test 13.2.3 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rank +} { NULL 'en_AU' 'th_TH' NULL } +do_execsql_test 13.2.4 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} { 'th_TH' NULL NULL 'en_AU' } + +do_execsql_test 13.2.5 { + SELECT + quote(fts5_get_locale(ft, '0')), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} { 'th_TH' NULL NULL 'en_AU' } + +do_catchsql_test 13.2.6 { + SELECT + quote(fts5_get_locale(ft, '0.0')), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} {1 {non-integer argument passed to function fts5_get_locale()}} +do_catchsql_test 13.2.7 { + SELECT + quote(fts5_get_locale(ft, 0.0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} {1 {non-integer argument passed to function fts5_get_locale()}} + finish_test diff --git a/manifest b/manifest index 2c5b73a1b1..3cfdd034e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sgenerate_series()\stable-valued\sfunction\ssuch\sthat\sit\sis\sable\sto\nrecognize\sequality\sand\sinequality\sconstraints\son\sthe\s"value"\scolumn\sand\noptimize\sits\soperating\saccordingly. -D 2024-08-22T18:12:10.402 +C Add\sfts5\sauxiliary\sfunction\sfts5_get_locale().\sFor\squerying\sthe\slocale\sof\sa\sstored\svalue. +D 2024-08-23T15:18:02.618 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -94,7 +94,7 @@ F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a0 F ext/fts5/extract_api_docs.tcl 1db7f85f4d84b7b6f33336155d5053fafc3c8debd074422d8003c8f7fa4d0fdb F ext/fts5/fts5.h c65fc7799a4cd6774628da4fa9408955623e504d7369ab5b89c4413fdfe11eb5 F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d805655360 -F ext/fts5/fts5_aux.c 12cd2512f869217c38b70c31de5b5f741812734fafa80f55b32ea9bbd96e2152 +F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81cbefc109ea F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 @@ -177,7 +177,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d -F ext/fts5/test/fts5faultI.test fbc65a64944fb747f6d3fb30628a807d5cce1bca43c11df40e7770ad7a7ed593 +F ext/fts5/test/fts5faultI.test e1e8a927122a8c3be8614a59717e838cad0505ba0a6f404983da8430e7aeb14d F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -187,7 +187,7 @@ F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba104 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 79cbd3000ae269de50826f6061c81f7c9fdb21dd9954c0b7f9485263482c539d +F ext/fts5/test/fts5locale.test 797cf6f5e017462ab11313ce884b9f1df8ff063811e74ef42190cd19ed6b600b F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6c00e88ebdb41d6317bb8758825521614dedc2e6e6289ff415c5f0406eed815b -R c06569ccdd3e5d2b3b096a4379f23ac8 -U drh -Z e410ad1cfdbbb6e926b30bd406bc8816 +P d50b784807333c5461a2d027778c746c799285b95bb1952f142b317ea2846460 +R 7170ba769edcf90a031dd78df3acb3d2 +U dan +Z 6e0f0933950fea8415d62eb627301612 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b90e36fb5a..9c9dac50bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d50b784807333c5461a2d027778c746c799285b95bb1952f142b317ea2846460 +396f720f36a937145ac727c2750acfd26e4eda350b46d648d82a7e9985545a9f From 789da358487944d96eb3290e76d1c848b9c8faea Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Aug 2024 17:40:29 +0000 Subject: [PATCH 64/99] Fix a problem in the ext/fts5/extract_api_docs.tcl script. FossilOrigin-Name: 9a9d0f6301faefe324261f03543023ffb6a90823349c6946abb0df2f69b31f96 --- ext/fts5/extract_api_docs.tcl | 2 +- ext/fts5/fts5.h | 6 +++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl index 634dc70cbd..6ee71c262c 100644 --- a/ext/fts5/extract_api_docs.tcl +++ b/ext/fts5/extract_api_docs.tcl @@ -82,7 +82,7 @@ proc get_struct_docs {data names} { set current_doc "" } set subject n/a - regexp {^ *([[:alpha:]]*)} $line -> subject + regexp {^ *([[:alnum:]_]*)} $line -> subject if {[lsearch $names $subject]>=0} { set current_header $subject } else { diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 682a8da386..67adb40ac6 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -379,9 +379,9 @@ struct Fts5ExtensionApi { /* Below this point are iVersion>=4 only */ int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); int (*xTokenize_v2)(Fts5Context*, - const char *pText, int nText, /* Text to tokenize */ - const char *pLoc, int nLoc, /* Locale to pass to tokenizer */ - void *pCtx, /* Context passed to xToken() */ + const char *pText, int nText, /* Text to tokenize */ + const char *pLocale, int nLocale, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ); }; diff --git a/manifest b/manifest index 3cfdd034e0..bb205d7a92 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfts5\sauxiliary\sfunction\sfts5_get_locale().\sFor\squerying\sthe\slocale\sof\sa\sstored\svalue. -D 2024-08-23T15:18:02.618 +C Fix\sa\sproblem\sin\sthe\sext/fts5/extract_api_docs.tcl\sscript. +D 2024-08-23T17:40:29.728 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -91,8 +91,8 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7ede551eac3ead62baa F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb -F ext/fts5/extract_api_docs.tcl 1db7f85f4d84b7b6f33336155d5053fafc3c8debd074422d8003c8f7fa4d0fdb -F ext/fts5/fts5.h c65fc7799a4cd6774628da4fa9408955623e504d7369ab5b89c4413fdfe11eb5 +F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 +F ext/fts5/fts5.h c72759fd15179934f10b71ae51ee999e8558f696335e4ec2d049243b764354ee F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d805655360 F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d50b784807333c5461a2d027778c746c799285b95bb1952f142b317ea2846460 -R 7170ba769edcf90a031dd78df3acb3d2 +P 396f720f36a937145ac727c2750acfd26e4eda350b46d648d82a7e9985545a9f +R 9811f59e13ff374b57741662af2d5cd0 U dan -Z 6e0f0933950fea8415d62eb627301612 +Z b44a5154d84ee4e53cef82c304d55edf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9c9dac50bd..be09817682 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -396f720f36a937145ac727c2750acfd26e4eda350b46d648d82a7e9985545a9f +9a9d0f6301faefe324261f03543023ffb6a90823349c6946abb0df2f69b31f96 From 7766c9943ebecbaa3320efcefa3afbec431f2b38 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 23 Aug 2024 21:20:50 +0000 Subject: [PATCH 65/99] Eliminate use of strcpy() in shell.c.in to squelch an unwarranted link-time warning on OpenBSD. FossilOrigin-Name: 32e9bf7f972144eacbb2580151220bc4fc1cbcc8bcf586ffd8e10762abf75c7e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 10 +++++++++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bb205d7a92..5736796277 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\sext/fts5/extract_api_docs.tcl\sscript. -D 2024-08-23T17:40:29.728 +C Eliminate\suse\sof\sstrcpy()\sin\sshell.c.in\sto\ssquelch\san\sunwarranted\slink-time\swarning\son\sOpenBSD. +D 2024-08-23T21:20:50.763 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -765,7 +765,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe -F src/shell.c.in 9a45cd618ed3cd09927e04bb0ccfd929fb5efa03a7166daec4645d230b7267f3 +F src/shell.c.in 990e77ca805273edd55071c71327d202f1854e19c2e9437c039d91ff3192d7c4 F src/sqlite.h.in f07bff4225a1244efd604a0ffef81ed69f29d3dbaed7e22f906f26229ba3ca9e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 396f720f36a937145ac727c2750acfd26e4eda350b46d648d82a7e9985545a9f -R 9811f59e13ff374b57741662af2d5cd0 -U dan -Z b44a5154d84ee4e53cef82c304d55edf +P 9a9d0f6301faefe324261f03543023ffb6a90823349c6946abb0df2f69b31f96 +R 0473ef71b70c85fedef9e64071463927 +U stephan +Z 10e164d89e2d1a1189bcd188933ef2aa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index be09817682..d54c88a271 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a9d0f6301faefe324261f03543023ffb6a90823349c6946abb0df2f69b31f96 +32e9bf7f972144eacbb2580151220bc4fc1cbcc8bcf586ffd8e10762abf75c7e diff --git a/src/shell.c.in b/src/shell.c.in index a75b511903..1a1d357749 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -534,6 +534,14 @@ static char *shell_strncpy(char *dest, const char *src, size_t n){ return dest; } +/* +** strcpy() workalike to squelch an unwarranted link-time warning +** from OpenBSD. +*/ +static void shell_strcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} +} + /* ** Optionally disable dynamic continuation prompt. ** Unless disabled, the continuation prompt shows open SQL lexemes if any, @@ -599,7 +607,7 @@ static char *dynamicContinuePrompt(void){ size_t ncp = strlen(continuePrompt); size_t ndp = strlen(dynPrompt.zScannerAwaits); if( ndp > ncp-3 ) return continuePrompt; - strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); + shell_strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); From 27095fc4a5ed21df5023c4fb35ebeef10d7894fc Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 24 Aug 2024 11:48:19 +0000 Subject: [PATCH 66/99] Ensure that the WhereLoop.u.btree.pOrderBy field added by the order-by-subquery optimization [7a0cdc7edb704a88] is correctly initialized. dbsqlfuzz 437be171a98687082a31ea6efc2fd5c5d64481c8. FossilOrigin-Name: b75f8b2252aefe7bb96fa237205a1a9dc88c46d9ec09bca7407e6c636b203e7e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5736796277..3751b90e1b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Eliminate\suse\sof\sstrcpy()\sin\sshell.c.in\sto\ssquelch\san\sunwarranted\slink-time\swarning\son\sOpenBSD. -D 2024-08-23T21:20:50.763 +C Ensure\sthat\sthe\sWhereLoop.u.btree.pOrderBy\sfield\sadded\sby\sthe\norder-by-subquery\soptimization\s[7a0cdc7edb704a88]\sis\scorrectly\ninitialized.\s\sdbsqlfuzz\s437be171a98687082a31ea6efc2fd5c5d64481c8. +D 2024-08-24T11:48:19.350 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -850,7 +850,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 70392f65e2c76a27ab1faed05892c2402ce547e574074a4f10282b3bc1ffe4eb +F src/where.c c046dd58c3410f7b7528e1e6317cb876398557bad346d568ed8562321a7d002d F src/whereInt.h a5d079c346a658b7a6e9e47bb943d021e02fa1e6aed3b964ca112112a4892192 F src/wherecode.c 5172d647798134e7c92536ddffe7e530c393d79b5dedd648b88faf2646c65baf F src/whereexpr.c 44f41ae554c7572e1de1485b3169b233ee04d464b2ee5881687ede3bf07cacfa @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9a9d0f6301faefe324261f03543023ffb6a90823349c6946abb0df2f69b31f96 -R 0473ef71b70c85fedef9e64071463927 -U stephan -Z 10e164d89e2d1a1189bcd188933ef2aa +P 32e9bf7f972144eacbb2580151220bc4fc1cbcc8bcf586ffd8e10762abf75c7e +R c767a4091aaf30f3cd810bce15db7a5b +U drh +Z 6904df1088a904060173307e81bc10aa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d54c88a271..b7454fff9f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32e9bf7f972144eacbb2580151220bc4fc1cbcc8bcf586ffd8e10762abf75c7e +b75f8b2252aefe7bb96fa237205a1a9dc88c46d9ec09bca7407e6c636b203e7e diff --git a/src/where.c b/src/where.c index a82f0305b4..2b835009db 100644 --- a/src/where.c +++ b/src/where.c @@ -3987,6 +3987,7 @@ static int whereLoopAddBtree( pNew->prereq = mPrereq; pNew->nOut = rSize; pNew->u.btree.pIndex = pProbe; + pNew->u.btree.pOrderBy = 0; b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ From 58b08d77029b63a80cc7bb920819b177c06bdfe6 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Aug 2024 15:54:15 +0000 Subject: [PATCH 67/99] Avoid a stack overflow that could be caused by a recursively defined WINDOW() with a strategically embedded error. FossilOrigin-Name: bada54bd6bf54190e40aa721b77081015957d204c7b6a9fdbe8c67bcf20798f8 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/resolve.c | 2 +- test/windowE.test | 20 ++++++++++++++++++++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3751b90e1b..d040579c44 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\sWhereLoop.u.btree.pOrderBy\sfield\sadded\sby\sthe\norder-by-subquery\soptimization\s[7a0cdc7edb704a88]\sis\scorrectly\ninitialized.\s\sdbsqlfuzz\s437be171a98687082a31ea6efc2fd5c5d64481c8. -D 2024-08-24T11:48:19.350 +C Avoid\sa\sstack\soverflow\sthat\scould\sbe\scaused\sby\sa\srecursively\sdefined\sWINDOW()\swith\sa\sstrategically\sembedded\serror. +D 2024-08-24T15:54:15.509 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -762,7 +762,7 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c d99931f45416652895e502328ca49fe782cfc4e1ebdcda13b3736d991ebf42ce F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab +F src/resolve.c fc033f60ce022b1a18d441882965c77042a184d5ea14f8995b6984d38d8bb323 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe F src/shell.c.in 990e77ca805273edd55071c71327d202f1854e19c2e9437c039d91ff3192d7c4 @@ -2069,7 +2069,7 @@ F test/windowA.test 6d63dc1260daa17141a55007600581778523a8b420629f1282d2acfc36af F test/windowB.test aad7c31739999f68a98a813cfd78390918fc70f56d2d925317a1523cab548ecf F test/windowC.test 6fd75f5bb2f1343d34e470e36e68f0ff638d8a42f6aa7d99471261b31a0d42f2 F test/windowD.test 65cf5a765fb8072450e8a0de2979ce7f09a38d87724fe1280c6444073e3da49b -F test/windowE.test 6ba0c8048e4cc02b942e56640f8fcd50fd7ca72c876656c40f6baf42e316684c +F test/windowE.test c98507e0b0d95980ad25845db758557757be2d7054198ccf522c7d277057a3df F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387019e0 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 32e9bf7f972144eacbb2580151220bc4fc1cbcc8bcf586ffd8e10762abf75c7e -R c767a4091aaf30f3cd810bce15db7a5b -U drh -Z 6904df1088a904060173307e81bc10aa +P b75f8b2252aefe7bb96fa237205a1a9dc88c46d9ec09bca7407e6c636b203e7e +R 0795580169838cc9e02bb8801025ded0 +U dan +Z 355e8a98a035830f16c29e0d1fb54ccb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b7454fff9f..e2adcf6c6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b75f8b2252aefe7bb96fa237205a1a9dc88c46d9ec09bca7407e6c636b203e7e +bada54bd6bf54190e40aa721b77081015957d204c7b6a9fdbe8c67bcf20798f8 diff --git a/src/resolve.c b/src/resolve.c index 6dcb4e7322..202e87db95 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1300,7 +1300,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ sqlite3WalkExprList(pWalker, pExpr->pLeft->x.pList); } #ifndef SQLITE_OMIT_WINDOWFUNC - if( pWin ){ + if( pWin && pParse->nErr==0 ){ Select *pSel = pNC->pWinSelect; assert( pWin==0 || (ExprUseYWin(pExpr) && pWin==pExpr->y.pWin) ); if( IN_RENAME_OBJECT==0 ){ diff --git a/test/windowE.test b/test/windowE.test index f20bcdaaa8..9128468b3c 100644 --- a/test/windowE.test +++ b/test/windowE.test @@ -54,5 +54,25 @@ do_execsql_test 1.3 { 5 5,4 5,4,1 5,4,1,6 5,4,1,6,3 5,4,1,6,3,2 } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(x); +} + +sqlite3_create_aggregate db + +breakpoint +do_catchsql_test 2.1 { + SELECT min(x) OVER w1 FROM t1 + WINDOW w1 AS (PARTITION BY x_count(x) OVER w1); +} {1 {x_count() may not be used as a window function}} + +do_catchsql_test 2.2 { + SELECT min(x) FILTER (WHERE x_count(x) OVER w1) OVER w1 FROM t1 + WINDOW w1 AS (PARTITION BY x OVER w1); +} {1 {near "OVER": syntax error}} + + finish_test From f9eff9052d3574ee275823347d46792ada12f4be Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 24 Aug 2024 18:42:39 +0000 Subject: [PATCH 68/99] Remove a term from an assert() that is no longer relevant. FossilOrigin-Name: 8b4766252b80075354a6b1406c5430fe999a6490aa3e079893fcc682d88734f7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/resolve.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d040579c44..064ca3e7dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\sstack\soverflow\sthat\scould\sbe\scaused\sby\sa\srecursively\sdefined\sWINDOW()\swith\sa\sstrategically\sembedded\serror. -D 2024-08-24T15:54:15.509 +C Remove\sa\sterm\sfrom\san\sassert()\sthat\sis\sno\slonger\srelevant. +D 2024-08-24T18:42:39.760 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -762,7 +762,7 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c d99931f45416652895e502328ca49fe782cfc4e1ebdcda13b3736d991ebf42ce F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c fc033f60ce022b1a18d441882965c77042a184d5ea14f8995b6984d38d8bb323 +F src/resolve.c 2c127880c0634962837f16f2f48a295e514357af959330cc038de73015d5b5e8 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe F src/shell.c.in 990e77ca805273edd55071c71327d202f1854e19c2e9437c039d91ff3192d7c4 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b75f8b2252aefe7bb96fa237205a1a9dc88c46d9ec09bca7407e6c636b203e7e -R 0795580169838cc9e02bb8801025ded0 -U dan -Z 355e8a98a035830f16c29e0d1fb54ccb +P bada54bd6bf54190e40aa721b77081015957d204c7b6a9fdbe8c67bcf20798f8 +R 00dd14ae6fd191eb1e24d275e84ccb9b +U drh +Z 4d530e51c1fc3bb001032c5e8c4a226a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e2adcf6c6d..803a4f07f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bada54bd6bf54190e40aa721b77081015957d204c7b6a9fdbe8c67bcf20798f8 +8b4766252b80075354a6b1406c5430fe999a6490aa3e079893fcc682d88734f7 diff --git a/src/resolve.c b/src/resolve.c index 202e87db95..b755cc8646 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1302,7 +1302,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #ifndef SQLITE_OMIT_WINDOWFUNC if( pWin && pParse->nErr==0 ){ Select *pSel = pNC->pWinSelect; - assert( pWin==0 || (ExprUseYWin(pExpr) && pWin==pExpr->y.pWin) ); + assert( ExprUseYWin(pExpr) && pWin==pExpr->y.pWin ); if( IN_RENAME_OBJECT==0 ){ sqlite3WindowUpdate(pParse, pSel ? pSel->pWinDefn : 0, pWin, pDef); if( pParse->db->mallocFailed ) break; From 1d5a12e6a59263982cfcc3e8326c82a998e6c583 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 24 Aug 2024 19:02:33 +0000 Subject: [PATCH 69/99] New ./configure option --enable-test-status causes testrunner.tcl to be run with the --status option for targets like mdevtest and releasetest. FossilOrigin-Name: 9ca8513c26e54c97f2d3c6105c42144100044727f6fb4c28048f7cd7270dd35e --- Makefile.in | 11 ++++++++--- configure | 49 +++++++++++++++++++++++++++++++++++-------------- configure.ac | 12 ++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 64 insertions(+), 26 deletions(-) diff --git a/Makefile.in b/Makefile.in index 44665184fc..b8149f846f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -117,6 +117,11 @@ HAVE_TCL = @HAVE_TCL@ # TCLSH_CMD = @TCLSH_CMD@ +# Additional options when running tests using testrunner.tcl +# This is usually either blank, or else --status +# +TSTRNNR_OPTS = @TSTRNNR_OPTS@ + # Where do we want to install the tcl plugin # TCLLIBDIR = @TCLLIBDIR@ @@ -1363,10 +1368,10 @@ testrunner: testfixture$(TEXE) devtest: srctree-check testfixture$(TEXE) fuzztest testrunner mdevtest: srctree-check has_tclsh85 - $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS) sdevtest: has_tclsh85 - $(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS) # Validate that various generated files in the source tree # are up-to-date. @@ -1377,7 +1382,7 @@ srctree-check: $(TOP)/tool/srctree-check.tcl # Testing for a release # releasetest: srctree-check testfixture$(TEXE) - ./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release + ./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS) # Minimal testing that runs in less than 3 minutes # diff --git a/configure b/configure index 5f60385c28..12f96ceba4 100755 --- a/configure +++ b/configure @@ -793,6 +793,7 @@ HAVE_WASI_SDK RELEASE VERSION program_prefix +TSTRNNR_OPTS TCLLIBDIR HAVE_TCL TCL_STUB_LIB_SPEC @@ -889,6 +890,7 @@ enable_largefile with_tclsh with_tcl enable_tcl +enable_test_status with_wasi_sdk enable_threadsafe enable_releasemode @@ -1546,6 +1548,8 @@ Optional Features: --disable-largefile omit support for large files --disable-tcl omit building accessory programs that require TCL-dev + --enable-test-status show status of testing targets using VT100 cursor + movment --disable-threadsafe Disable mutexing --enable-releasemode Support libtool link to release mode --enable-tempstore Use an in-ram database for temporary tables @@ -3939,13 +3943,13 @@ if ${lt_cv_nm_interface+:} false; then : else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3942: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3946: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3945: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3949: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3948: output\"" >&5) + (eval echo "\"\$as_me:3952: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5151,7 +5155,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5154 "configure"' > conftest.$ac_ext + echo '#line 5158 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -6676,11 +6680,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6679: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6683: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6683: \$? = $ac_status" >&5 + echo "$as_me:6687: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7015,11 +7019,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7018: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7022: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7022: \$? = $ac_status" >&5 + echo "$as_me:7026: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7120,11 +7124,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7123: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7127: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7127: \$? = $ac_status" >&5 + echo "$as_me:7131: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -7175,11 +7179,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7178: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7182: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7182: \$? = $ac_status" >&5 + echo "$as_me:7186: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9555,7 +9559,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9558 "configure" +#line 9562 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -9651,7 +9655,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9654 "configure" +#line 9658 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -10484,6 +10488,23 @@ if test "x${TCLLIBDIR+set}" != "xset" ; then TCLLIBDIR="${TCLLIBDIR}/sqlite3" fi +######### +# Set up options for running tests. +# +# Check whether --enable-test-status was given. +if test "${enable_test_status+set}" = set; then : + enableval=$enable_test_status; use_vt100=$enableval +else + use_vt100=no +fi + +if test $use_vt100; then + TSTRNNR_OPTS=--status +else + TSTRNNR_OPTS= +fi + + ######### # Set up an appropriate program prefix diff --git a/configure.ac b/configure.ac index 2936c4ab5a..6df65b9f13 100644 --- a/configure.ac +++ b/configure.ac @@ -227,6 +227,18 @@ if test "x${TCLLIBDIR+set}" != "xset" ; then TCLLIBDIR="${TCLLIBDIR}/sqlite3" fi +######### +# Set up options for running tests. +# +AC_ARG_ENABLE(test-status, AS_HELP_STRING([--enable-test-status],[show status of testing targets using VT100 cursor movment]), + [use_vt100=$enableval],[use_vt100=no]) +if test $use_vt100; then + TSTRNNR_OPTS=--status +else + TSTRNNR_OPTS= +fi +AC_SUBST(TSTRNNR_OPTS) + ######### # Set up an appropriate program prefix diff --git a/manifest b/manifest index 064ca3e7dd..18db7ded47 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Remove\sa\sterm\sfrom\san\sassert()\sthat\sis\sno\slonger\srelevant. -D 2024-08-24T18:42:39.760 +C New\s./configure\soption\s--enable-test-status\scauses\stestrunner.tcl\sto\sbe\nrun\swith\sthe\s--status\soption\sfor\stargets\slike\smdevtest\sand\sreleasetest. +D 2024-08-24T19:02:33.721 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 209e9c64edf499b0ea7a15448699101c01d15828af35a3439d32fc25fdf1b787 +F Makefile.in 101cb90f75c16ca3f8e7088364322441f9b01d6fdcef4a9c7f5fdcf78a3601a4 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc 6c3fe8b6ce60e73f34a148c957d78b4648745c8d30e792423aa1a8d8bf12d065 F README.md 3d47ef5758e2de29d6f4e1aca714d51424baba86c8b561fef330f6ef2dc9282c @@ -35,8 +35,8 @@ F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea0034 F autoconf/tea/win/rules.vc 7b3bb2ef32ade0f3f14d951231811678722725e3bca240dd9727ae0dfe10f6a5 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure 956e132a3e2481e5016e9ecaa0a2bdd9df95c1d923fa4fb2aa2cf9f0b3a54ad9 x -F configure.ac fd9ff064b142682964971fec3d867235e51e1f6de30a031e33f683b7699338ff +F configure b4c3cbde74d7bf7a3c046e2b9641c2784f67b0d3a8e56235a107e0a230fb26b0 x +F configure.ac f084a0765a98bc76513d0fba69264164d72d20cbfbc84d85f0a4e8400425b8d5 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd F doc/compile-for-windows.md e8635eea9153dcd6a51fd2740666ebc4492b3813cb1ac31cd8e99150df91762d @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bada54bd6bf54190e40aa721b77081015957d204c7b6a9fdbe8c67bcf20798f8 -R 00dd14ae6fd191eb1e24d275e84ccb9b +P 8b4766252b80075354a6b1406c5430fe999a6490aa3e079893fcc682d88734f7 +R 624c0fde9a976c048fe809e58bc6f948 U drh -Z 4d530e51c1fc3bb001032c5e8c4a226a +Z a7b786c85627cbad9eafd537f720c473 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 803a4f07f2..af222d498c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b4766252b80075354a6b1406c5430fe999a6490aa3e079893fcc682d88734f7 +9ca8513c26e54c97f2d3c6105c42144100044727f6fb4c28048f7cd7270dd35e From 838d2d3078b9c746da75f76b20ab36ff40d3fd1d Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 24 Aug 2024 19:06:52 +0000 Subject: [PATCH 70/99] Improved help message for the --enable-test-status configuration option. Fix the build for when that option is omitted. FossilOrigin-Name: b404a5fe3f0532f6d56b5e65c44a59379c17d175c21ac69a1f2ba8aadf453313 --- configure | 33 ++++++++++++++++----------------- configure.ac | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/configure b/configure index 12f96ceba4..22b961e554 100755 --- a/configure +++ b/configure @@ -1548,8 +1548,7 @@ Optional Features: --disable-largefile omit support for large files --disable-tcl omit building accessory programs that require TCL-dev - --enable-test-status show status of testing targets using VT100 cursor - movment + --enable-test-status Full-screen status of tests --disable-threadsafe Disable mutexing --enable-releasemode Support libtool link to release mode --enable-tempstore Use an in-ram database for temporary tables @@ -3943,13 +3942,13 @@ if ${lt_cv_nm_interface+:} false; then : else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3946: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3945: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3949: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3948: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3952: output\"" >&5) + (eval echo "\"\$as_me:3951: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5155,7 +5154,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5158 "configure"' > conftest.$ac_ext + echo '#line 5157 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -6680,11 +6679,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6683: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6682: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6687: \$? = $ac_status" >&5 + echo "$as_me:6686: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7019,11 +7018,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7022: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7021: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7026: \$? = $ac_status" >&5 + echo "$as_me:7025: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7124,11 +7123,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7127: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7126: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7131: \$? = $ac_status" >&5 + echo "$as_me:7130: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -7179,11 +7178,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7182: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7181: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7186: \$? = $ac_status" >&5 + echo "$as_me:7185: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9559,7 +9558,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9562 "configure" +#line 9561 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -9655,7 +9654,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9658 "configure" +#line 9657 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -10498,7 +10497,7 @@ else use_vt100=no fi -if test $use_vt100; then +if test $use_vt100 != no; then TSTRNNR_OPTS=--status else TSTRNNR_OPTS= diff --git a/configure.ac b/configure.ac index 6df65b9f13..b3a0dd299f 100644 --- a/configure.ac +++ b/configure.ac @@ -230,9 +230,9 @@ fi ######### # Set up options for running tests. # -AC_ARG_ENABLE(test-status, AS_HELP_STRING([--enable-test-status],[show status of testing targets using VT100 cursor movment]), +AC_ARG_ENABLE(test-status, AS_HELP_STRING([--enable-test-status],[Full-screen status of tests]), [use_vt100=$enableval],[use_vt100=no]) -if test $use_vt100; then +if test $use_vt100 != no; then TSTRNNR_OPTS=--status else TSTRNNR_OPTS= diff --git a/manifest b/manifest index 18db7ded47..da0380bd95 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\s./configure\soption\s--enable-test-status\scauses\stestrunner.tcl\sto\sbe\nrun\swith\sthe\s--status\soption\sfor\stargets\slike\smdevtest\sand\sreleasetest. -D 2024-08-24T19:02:33.721 +C Improved\shelp\smessage\sfor\sthe\s--enable-test-status\sconfiguration\soption.\nFix\sthe\sbuild\sfor\swhen\sthat\soption\sis\somitted. +D 2024-08-24T19:06:52.229 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -35,8 +35,8 @@ F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea0034 F autoconf/tea/win/rules.vc 7b3bb2ef32ade0f3f14d951231811678722725e3bca240dd9727ae0dfe10f6a5 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure b4c3cbde74d7bf7a3c046e2b9641c2784f67b0d3a8e56235a107e0a230fb26b0 x -F configure.ac f084a0765a98bc76513d0fba69264164d72d20cbfbc84d85f0a4e8400425b8d5 +F configure 49523f0a070b583cea040d26eff53a65fb0893eca4663b1343a4d5a9a964da53 x +F configure.ac a100ebf7a07f5dedd319ef547dd467d1676ed059b85a7877aa9c44ac309f7000 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd F doc/compile-for-windows.md e8635eea9153dcd6a51fd2740666ebc4492b3813cb1ac31cd8e99150df91762d @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8b4766252b80075354a6b1406c5430fe999a6490aa3e079893fcc682d88734f7 -R 624c0fde9a976c048fe809e58bc6f948 +P 9ca8513c26e54c97f2d3c6105c42144100044727f6fb4c28048f7cd7270dd35e +R af6e572379558bf7f71bf69d4df3ecc4 U drh -Z a7b786c85627cbad9eafd537f720c473 +Z 30edae94264472f64c3131f2153cb3fe # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index af222d498c..f9406ce800 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ca8513c26e54c97f2d3c6105c42144100044727f6fb4c28048f7cd7270dd35e +b404a5fe3f0532f6d56b5e65c44a59379c17d175c21ac69a1f2ba8aadf453313 From 306a8dcf3e5b4e94156b208eb9f601990ac8ce99 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 24 Aug 2024 20:01:05 +0000 Subject: [PATCH 71/99] Fix compiler warnings in the CLI detected by MSVC. FossilOrigin-Name: 23ae505cbfde6dfd1dbb2216cf76c1e316b49f2ca84491a1aae28ad33f7777a9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index da0380bd95..ba33622d87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\shelp\smessage\sfor\sthe\s--enable-test-status\sconfiguration\soption.\nFix\sthe\sbuild\sfor\swhen\sthat\soption\sis\somitted. -D 2024-08-24T19:06:52.229 +C Fix\scompiler\swarnings\sin\sthe\sCLI\sdetected\sby\sMSVC. +D 2024-08-24T20:01:05.217 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -765,7 +765,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 2c127880c0634962837f16f2f48a295e514357af959330cc038de73015d5b5e8 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe -F src/shell.c.in 990e77ca805273edd55071c71327d202f1854e19c2e9437c039d91ff3192d7c4 +F src/shell.c.in fa98789df3e106eac287b001decd28ef7fe703c119282f91294af32d5c18ce10 F src/sqlite.h.in f07bff4225a1244efd604a0ffef81ed69f29d3dbaed7e22f906f26229ba3ca9e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ca8513c26e54c97f2d3c6105c42144100044727f6fb4c28048f7cd7270dd35e -R af6e572379558bf7f71bf69d4df3ecc4 +P b404a5fe3f0532f6d56b5e65c44a59379c17d175c21ac69a1f2ba8aadf453313 +R 1dc458f3efda36153cdb87bbe3ce0cfd U drh -Z 30edae94264472f64c3131f2153cb3fe +Z e6cd8d3cc158f5893ff38516ff50820a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f9406ce800..5d64c9e710 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b404a5fe3f0532f6d56b5e65c44a59379c17d175c21ac69a1f2ba8aadf453313 +23ae505cbfde6dfd1dbb2216cf76c1e316b49f2ca84491a1aae28ad33f7777a9 diff --git a/src/shell.c.in b/src/shell.c.in index 1a1d357749..ab17f46e14 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3556,10 +3556,11 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ }else if( strncmp(zVar, "$int_", 5)==0 ){ sqlite3_bind_int(pStmt, i, atoi(&zVar[5])); }else if( strncmp(zVar, "$text_", 6)==0 ){ - char *zBuf = sqlite3_malloc64( strlen(zVar)-5 ); + size_t szVar = strlen(zVar); + char *zBuf = sqlite3_malloc64( szVar-5 ); if( zBuf ){ - memcpy(zBuf, &zVar[6], strlen(zVar)-5); - sqlite3_bind_text64(pStmt, i, zBuf, -1, sqlite3_free, SQLITE_UTF8); + memcpy(zBuf, &zVar[6], szVar-5); + sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8); } }else{ sqlite3_bind_null(pStmt, i); @@ -11052,7 +11053,7 @@ static int do_meta_command(char *zLine, ShellState *p){ for(ii=2; ii Date: Sat, 24 Aug 2024 20:05:59 +0000 Subject: [PATCH 72/99] Fix harmless compiler warnings in fts5V2toV1Tokenize(). FossilOrigin-Name: df65d00f104f31741056686f2ba41ecb192c552012bffb17c2a8b5d4db058328 --- ext/fts5/fts5_main.c | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index e271402ecf..4addc07294 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -3350,6 +3350,7 @@ static int fts5V2toV1Tokenize( Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; Fts5TokenizerModule *pMod = p->pMod; assert( pMod->bV2Native==0 ); + UNUSED_PARAM2(pLocale,nLocale); return pMod->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken); } diff --git a/manifest b/manifest index ba33622d87..7dd86f3553 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\sin\sthe\sCLI\sdetected\sby\sMSVC. -D 2024-08-24T20:01:05.217 +C Fix\sharmless\scompiler\swarnings\sin\sfts5V2toV1Tokenize(). +D 2024-08-24T20:05:59.988 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81c F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 5ea7ab0c9967594e73b7dd0ad737595922a14f175aa4b486dc2992a3e3138b68 +F ext/fts5/fts5_main.c 3d8b778f65fe5be218f6a8e4019048f092c0c09621b035ce3e8804093036b578 F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612 F ext/fts5/fts5_tcl.c 1dcf08028141c40a32634bdcf2d5601622ce4edc48f82ac4ce0cbe0a92a6961d F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b404a5fe3f0532f6d56b5e65c44a59379c17d175c21ac69a1f2ba8aadf453313 -R 1dc458f3efda36153cdb87bbe3ce0cfd +P 23ae505cbfde6dfd1dbb2216cf76c1e316b49f2ca84491a1aae28ad33f7777a9 +R 3d0d5a942eeee701f351d015af5828aa U drh -Z e6cd8d3cc158f5893ff38516ff50820a +Z b6c5eb8f043c1d2348c0c60d14f2a5f6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5d64c9e710..fc66c305e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23ae505cbfde6dfd1dbb2216cf76c1e316b49f2ca84491a1aae28ad33f7777a9 +df65d00f104f31741056686f2ba41ecb192c552012bffb17c2a8b5d4db058328 From 87ebadbf4e0349b1987d89a2c8a9b3638a88667c Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 25 Aug 2024 11:59:29 +0000 Subject: [PATCH 73/99] shell.c.in: use eputz/oputz() instead of eputf/oputf() where appropriate to avoid compilation errors in -std=c99 mode (namely wasm builds). FossilOrigin-Name: 9ef8317faebc29d016bdf2e8c678fb21ca2cfa95272f1f18b461fcaf0e220ac5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 7dd86f3553..1153795db6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sfts5V2toV1Tokenize(). -D 2024-08-24T20:05:59.988 +C shell.c.in:\suse\seputz/oputz()\sinstead\sof\seputf/oputf()\swhere\sappropriate\sto\savoid\scompilation\serrors\sin\s-std=c99\smode\s(namely\swasm\sbuilds). +D 2024-08-25T11:59:29.477 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -765,7 +765,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 2c127880c0634962837f16f2f48a295e514357af959330cc038de73015d5b5e8 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe -F src/shell.c.in fa98789df3e106eac287b001decd28ef7fe703c119282f91294af32d5c18ce10 +F src/shell.c.in 5a05e1a1d187ab916afe9a628d7e9b20b5997273b595bd748db324814e0ccca9 F src/sqlite.h.in f07bff4225a1244efd604a0ffef81ed69f29d3dbaed7e22f906f26229ba3ca9e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 23ae505cbfde6dfd1dbb2216cf76c1e316b49f2ca84491a1aae28ad33f7777a9 -R 3d0d5a942eeee701f351d015af5828aa -U drh -Z b6c5eb8f043c1d2348c0c60d14f2a5f6 +P df65d00f104f31741056686f2ba41ecb192c552012bffb17c2a8b5d4db058328 +R 8d4b348b6bf673b94cab6e107b5bbb73 +U stephan +Z 5120e8419efd39e01384da30ed409ec4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fc66c305e9..4e98a04790 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df65d00f104f31741056686f2ba41ecb192c552012bffb17c2a8b5d4db058328 +9ef8317faebc29d016bdf2e8c678fb21ca2cfa95272f1f18b461fcaf0e220ac5 diff --git a/src/shell.c.in b/src/shell.c.in index ab17f46e14..db10d21b8e 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -283,7 +283,7 @@ INCLUDE ../ext/consio/console_io.c #else /* For Fiddle, all console handling and emit redirection is omitted. */ /* These next 3 macros are for emitting formatted output. When complaints - * from the WASM build are issued for non-formatted output, (when a mere + * from the WASM build are issued for non-formatted output, when a mere * string literal is to be emitted, the ?putz(z) forms should be used. * (This permits compile-time checking of format string / argument mismatch.) */ @@ -11070,11 +11070,11 @@ static int do_meta_command(char *zLine, ShellState *p){ } if( jj>=ArraySize(aLabel) ){ eputf("Error: no such optimization: \"%s\"\n", zLabel); - eputf("Should be one of:"); + eputz("Should be one of:"); for(jj=0; jj Date: Mon, 26 Aug 2024 17:35:32 +0000 Subject: [PATCH 74/99] Randomize the name of the attached database used to do VACUUM. FossilOrigin-Name: 0f4e65d73a2b95932f36768c0d9925a2d265d9b395b886d8e492d1d519b2093c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vacuum.c | 23 ++++++++++++++--------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 1153795db6..9b018b0b17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C shell.c.in:\suse\seputz/oputz()\sinstead\sof\seputf/oputf()\swhere\sappropriate\sto\savoid\scompilation\serrors\sin\s-std=c99\smode\s(namely\swasm\sbuilds). -D 2024-08-25T11:59:29.477 +C Randomize\sthe\sname\sof\sthe\sattached\sdatabase\sused\sto\sdo\sVACUUM. +D 2024-08-26T17:35:32.270 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -834,7 +834,7 @@ F src/update.c 0e01aa6a3edf9ec112b33eb714b9016a81241497b1fb7c3e74332f4f71756508 F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c 5d1a0134cf4240648d1c6bb5cc8efaca0ea2b5d5c840985aec7e947271f04375 -F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 +F src/vacuum.c b763b6457bd058d2072ef9364832351fd8d11e8abf70cbb349657360f7d55c40 F src/vdbe.c be5f58bc29f60252e041a618eae59e8d57d460ba136c5403cf0abf955560c457 F src/vdbe.h c2549a215898a390de6669cfa32adba56f0d7e17ba5a7f7b14506d6fd5f0c36a F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P df65d00f104f31741056686f2ba41ecb192c552012bffb17c2a8b5d4db058328 -R 8d4b348b6bf673b94cab6e107b5bbb73 -U stephan -Z 5120e8419efd39e01384da30ed409ec4 +P 9ef8317faebc29d016bdf2e8c678fb21ca2cfa95272f1f18b461fcaf0e220ac5 +R f1ecfb99c58e61a39b55af180b76bb09 +U drh +Z 1c617d8c3be95e126399c709f0b8f264 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4e98a04790..dfaf3a1f10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ef8317faebc29d016bdf2e8c678fb21ca2cfa95272f1f18b461fcaf0e220ac5 +0f4e65d73a2b95932f36768c0d9925a2d265d9b395b886d8e492d1d519b2093c diff --git a/src/vacuum.c b/src/vacuum.c index c0ae4bc1e1..e203f68c65 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -162,6 +162,9 @@ SQLITE_NOINLINE int sqlite3RunVacuum( const char *zDbMain; /* Schema name of database to vacuum */ const char *zOut; /* Name of output file */ u32 pgflags = PAGER_SYNCHRONOUS_OFF; /* sync flags for output db */ + u64 iRandom; /* Random value used for zDbVacuum[] */ + char zDbVacuum[42]; /* Name of the ATTACH-ed database used for vacuum */ + if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); @@ -202,27 +205,29 @@ SQLITE_NOINLINE int sqlite3RunVacuum( pMain = db->aDb[iDb].pBt; isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); - /* Attach the temporary database as 'vacuum_db'. The synchronous pragma + /* Attach the temporary database as 'vacuum_XXXXXX'. The synchronous pragma ** can be set to 'off' for this file, as it is not recovered if a crash ** occurs anyway. The integrity of the database is maintained by a ** (possibly synchronous) transaction opened on the main database before ** sqlite3BtreeCopyFile() is called. ** ** An optimization would be to use a non-journaled pager. - ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but + ** (Later:) I tried setting "PRAGMA vacuum_XXXXXX.journal_mode=OFF" but ** that actually made the VACUUM run slower. Very little journalling ** actually occurs when doing a vacuum since the vacuum_db is initially ** empty. Only the journal header is written. Apparently it takes more ** time to parse and run the PRAGMA to turn journalling off than it does ** to write the journal header file. */ + sqlite3_randomness(sizeof(iRandom),&iRandom); + sqlite3_snprintf(sizeof(zDbVacuum), zDbVacuum, "vacuum_%016llx", iRandom); nDb = db->nDb; - rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut); + rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS %s", zOut, zDbVacuum); db->openFlags = saved_openFlags; if( rc!=SQLITE_OK ) goto end_of_vacuum; assert( (db->nDb-1)==nDb ); pDb = &db->aDb[nDb]; - assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); + assert( strcmp(pDb->zDbSName,zDbVacuum)==0 ); pTemp = pDb->pBt; if( pOut ){ sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); @@ -299,11 +304,11 @@ SQLITE_NOINLINE int sqlite3RunVacuum( ** the contents to the temporary database. */ rc = execSqlF(db, pzErrMsg, - "SELECT'INSERT INTO vacuum_db.'||quote(name)" + "SELECT'INSERT INTO %s.'||quote(name)" "||' SELECT*FROM\"%w\".'||quote(name)" - "FROM vacuum_db.sqlite_schema " + "FROM %s.sqlite_schema " "WHERE type='table'AND coalesce(rootpage,1)>0", - zDbMain + zDbVacuum, zDbMain, zDbVacuum ); assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 ); db->mDbFlags &= ~DBFLAG_Vacuum; @@ -315,11 +320,11 @@ SQLITE_NOINLINE int sqlite3RunVacuum( ** from the schema table. */ rc = execSqlF(db, pzErrMsg, - "INSERT INTO vacuum_db.sqlite_schema" + "INSERT INTO %s.sqlite_schema" " SELECT*FROM \"%w\".sqlite_schema" " WHERE type IN('view','trigger')" " OR(type='table'AND rootpage=0)", - zDbMain + zDbVacuum, zDbMain ); if( rc ) goto end_of_vacuum; From c013fa50db60dad905aff1c2202edfa7223bdd67 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Aug 2024 19:08:54 +0000 Subject: [PATCH 75/99] Changes to documentation comments in fts5.h. FossilOrigin-Name: 886545a8d0e9cbf4911ecf747da0ea8aad702d79cb065dad01a04aa4f51da12b --- ext/fts5/fts5.h | 13 +++++++------ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 67adb40ac6..ea69f8f023 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -493,18 +493,19 @@ struct Fts5ExtensionApi { ** ** FTS5_TOKENIZER ** -** There is also an fts5_tokenizer object. This is an older version of -** fts5_tokenizer_v2. It is similar except that: +** There is also an fts5_tokenizer object. This is an older, deprecated, +** version of fts5_tokenizer_v2. It is similar except that: ** **
    **
  • There is no "iVersion" field, and **
  • The xTokenize() method does not take a locale argument. **
** -** fts5_tokenizer tokenizers should be registered with the xCreateTokenizer() -** function, instead of xCreateTokenizer_v2(). Tokenizers implementations -** registered using either API may be retrieved using both xFindTokenizer() -** and xFindTokenizer_v2(). +** Legacy fts5_tokenizer tokenizers must be registered using the +** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2(). +** +** Tokenizer implementations registered using either API may be retrieved +** using both xFindTokenizer() and xFindTokenizer_v2(). ** ** SYNONYM SUPPORT ** diff --git a/manifest b/manifest index 9b018b0b17..178aa88497 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Randomize\sthe\sname\sof\sthe\sattached\sdatabase\sused\sto\sdo\sVACUUM. -D 2024-08-26T17:35:32.270 +C Changes\sto\sdocumentation\scomments\sin\sfts5.h. +D 2024-08-26T19:08:54.811 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7ede551eac3ead62baa F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 -F ext/fts5/fts5.h c72759fd15179934f10b71ae51ee999e8558f696335e4ec2d049243b764354ee +F ext/fts5/fts5.h efaaac0df3d3bc740383044c144b582f47921aafa21d7b10eb98f42c24c740b0 F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d805655360 F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ef8317faebc29d016bdf2e8c678fb21ca2cfa95272f1f18b461fcaf0e220ac5 -R f1ecfb99c58e61a39b55af180b76bb09 -U drh -Z 1c617d8c3be95e126399c709f0b8f264 +P 0f4e65d73a2b95932f36768c0d9925a2d265d9b395b886d8e492d1d519b2093c +R 5eb525d2c75add4a5d571ba8012dc6b9 +U dan +Z dd80b3a4edb71c561dd200b24c9de74c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dfaf3a1f10..7c55e8d5d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f4e65d73a2b95932f36768c0d9925a2d265d9b395b886d8e492d1d519b2093c +886545a8d0e9cbf4911ecf747da0ea8aad702d79cb065dad01a04aa4f51da12b From 8aa9135f2cea08b1e8e6fb56dfe0fd6c3491605c Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 27 Aug 2024 14:25:52 +0000 Subject: [PATCH 76/99] Only avoid all C-runtime I/O if the SQLITE_USE_ONLY_WIN32 flag is defined. FossilOrigin-Name: 164b1e1962aa1e16bdf52e9e86d4cf9c9e09220c0821932ac8e390e82074185f --- ext/consio/console_io.c | 4 +++- manifest | 26 +++++++++++++------------- manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/ext/consio/console_io.c b/ext/consio/console_io.c index 40989e77b6..6b837cbafc 100755 --- a/ext/consio/console_io.c +++ b/ext/consio/console_io.c @@ -609,7 +609,9 @@ fFlushBuffer(FILE *pfOut){ } #endif -# if CIO_WIN_WC_XLATE && !defined(SHELL_OMIT_FIO_DUPE) +#if CIO_WIN_WC_XLATE \ + && !defined(SHELL_OMIT_FIO_DUPE) \ + && defined(SQLITE_USE_ONLY_WIN32) static struct FileAltIds { int fd; HANDLE fh; diff --git a/manifest b/manifest index 1e8d9272f2..ddb1f8bfb9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sthe\slatests\strunk\senhancements\sinto\sthe\swin-dupe-crt-fio\sbranch. -D 2024-08-27T13:57:43.910 +C Only\savoid\sall\sC-runtime\sI/O\sif\sthe\sSQLITE_USE_ONLY_WIN32\sflag\sis\sdefined. +D 2024-08-27T14:25:52.592 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -53,7 +53,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94 F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a -F ext/consio/console_io.c 7f3cb6490ea6e3bd51b606bd3a37cd1d26d0bf07ccec21dd75e20bf4ee1a2a59 x +F ext/consio/console_io.c 6e02dea912a49f55785b0027fe77960aafee6c236307c23f82ec86a69f1a2001 x F ext/consio/console_io.h b5ebe34aa15b357621ebbea3d3f2e2b24750d4280b5802516409e23947fd9ee5 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 @@ -607,7 +607,7 @@ F ext/wasm/SQLTester/SQLTester.run.mjs c72b7fe2072d05992f7a3d8c6a1d34e95712513ce F ext/wasm/SQLTester/index.html 3f8a016df0776be76605abf20e815ecaafbe055abac0e1fe5ea080e7846b760d F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth 7ac80cc3b6a6d52e041bb295e85555ce797be78c15ef2008a64ae58815014080 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core 400213eb52a7e5ad5f448053d375cacf4dac2cf45d134f3edfe485ae4a49a183 w ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core 400213eb52a7e5ad5f448053d375cacf4dac2cf45d134f3edfe485ae4a49a183 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate d1d62a2212099f2c0782d730beb8cb84a7a52d99c15ead2cb9b1411fff5fd6b1 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session 213b6c04267cb9bd760172db011eb1650732805fb3d01f9395478a8ceec18eb0 @@ -620,13 +620,13 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 04dc12c3edd666b64a1b4ef3b6690c88dcc653f26451fd4734472d8e29c1c122 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e -F ext/wasm/api/sqlite3-api-glue.c-pp.js 54b32b5321105a72d6f3d3e8b77f28f162d0367b08c63184263d3f85f3d7dbed w ext/wasm/api/sqlite3-api-glue.js -F ext/wasm/api/sqlite3-api-oo1.c-pp.js f3a8e2004c6625d17946c11f2fb32008be78bc5207bf746fc77d59848813225f w ext/wasm/api/sqlite3-api-oo1.js +F ext/wasm/api/sqlite3-api-glue.c-pp.js 54b32b5321105a72d6f3d3e8b77f28f162d0367b08c63184263d3f85f3d7dbed +F ext/wasm/api/sqlite3-api-oo1.c-pp.js f3a8e2004c6625d17946c11f2fb32008be78bc5207bf746fc77d59848813225f F ext/wasm/api/sqlite3-api-prologue.js 6f1257e04885632ed9f44d43aba200b86e0bc16709ffdba29abbbeb1bc8e8b76 -F ext/wasm/api/sqlite3-api-worker1.c-pp.js 5cc22a3c0d52828cb32aad8691488719f47d27567e63e8bc8b832d74371c352d w ext/wasm/api/sqlite3-api-worker1.js +F ext/wasm/api/sqlite3-api-worker1.c-pp.js 5cc22a3c0d52828cb32aad8691488719f47d27567e63e8bc8b832d74371c352d F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js e8f1df56e97a29004a95a2eddd26778f52c33b3e797d32d4b1b668a38e6493dd -F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d w ext/wasm/api/sqlite3-v-helper.js +F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js e529a99b7d5a088284821e2902b20d3404b561126969876997d5a73a656c9199 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e99e3d99f736937914527070f00ab13e9391d3f1cef884ab99a64cbcbee8d675 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js e809739d71e8b35dfe1b55d24d91f02d04239e6aef7ca1ea92a15a29e704f616 @@ -647,8 +647,8 @@ F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b823 F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8 -F ext/wasm/demo-worker1-promiser.c-pp.html 635cf90685805e21772a5f7a35d1ace80f98a9ef7c42ff04d7a125ddca7e5db8 w ext/wasm/demo-worker1-promiser.html -F ext/wasm/demo-worker1-promiser.c-pp.js fcc628cb42fcfaf07d250477801de1e6deb1e319d003976612a0db8d76b9fccc w ext/wasm/demo-worker1-promiser.js +F ext/wasm/demo-worker1-promiser.c-pp.html 635cf90685805e21772a5f7a35d1ace80f98a9ef7c42ff04d7a125ddca7e5db8 +F ext/wasm/demo-worker1-promiser.c-pp.js fcc628cb42fcfaf07d250477801de1e6deb1e319d003976612a0db8d76b9fccc F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js 836bece8615b17b1b572584f7b15912236a5947fe8c68b98d2737d7e287447ef F ext/wasm/dist.make 653e212c1e84aa3be168d62a10616ccea45ee9585b0192745d2706707a5248ce @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2e74a938f771a889dd8c49648b5a125ff748835ba1925388cc6b379a0abde9ac 886545a8d0e9cbf4911ecf747da0ea8aad702d79cb065dad01a04aa4f51da12b -R 79479d743cc98e2dac5001e42c2a12ce +P 2d52db98f47fbcda0622c034b21c2fb19bf4345b88c5c565ae9e6f7128642e43 +R fe2336ed930d3088aea807bfa7c9e690 U drh -Z d5c2cd4bb2868922ff61d70f26a944a8 +Z dffceaa70fb90b81619d76d49bca0c7c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 179264e0ec..5430ab55f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d52db98f47fbcda0622c034b21c2fb19bf4345b88c5c565ae9e6f7128642e43 +164b1e1962aa1e16bdf52e9e86d4cf9c9e09220c0821932ac8e390e82074185f From dac0d0b3102e3b0c1ba3ad41bbbd0b137bb82011 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 27 Aug 2024 17:38:26 +0000 Subject: [PATCH 77/99] Updates to testrunner: (1) Omit all testing of User-Auth. (2) Automatically add the "*" wildcard before and after all pattern arguments. (3) Build the sqlite3 CLI for release tests. FossilOrigin-Name: d03d35eebaf82709414c87cfa6abc9d2baf8d7e64c2627bad0fd5bbda3e78d60 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- test/testrunner.tcl | 33 +++++++++++++++++---------------- test/testrunner_data.tcl | 7 +------ 4 files changed, 26 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index fe2f58deba..3acb861a2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sSQLITE_USE_ONLY_WIN32\sflag\sfor\sconsio\sinto\strunk. -D 2024-08-27T14:35:54.105 +C Updates\sto\stestrunner:\s\s(1)\sOmit\sall\stesting\sof\sUser-Auth.\n(2)\sAutomatically\sadd\sthe\s"*"\swildcard\sbefore\sand\safter\sall\spattern\narguments.\s\s(3)\sBuild\sthe\ssqlite3\sCLI\sfor\srelease\stests. +D 2024-08-27T17:38:26.497 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1713,8 +1713,8 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl 5d02deeba7a53baeadae6aa7641d90aac58fdfa3a7bcac85cfcfd752b1aab87c -F test/testrunner_data.tcl c5ae2b1f9a99210b0600d002fb3af1fee350997cee9416551e83b93501360ebf +F test/testrunner.tcl 74b569b553a85a10677e0f9b2f6d22d90d9f07255fbec545ec9438bed677e8e8 +F test/testrunner_data.tcl b281136c9a4a3cb2e6772a87c2e94efad0c12ea3e7d0710521a3cad964f879a1 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -2210,9 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 886545a8d0e9cbf4911ecf747da0ea8aad702d79cb065dad01a04aa4f51da12b 164b1e1962aa1e16bdf52e9e86d4cf9c9e09220c0821932ac8e390e82074185f -R fe2336ed930d3088aea807bfa7c9e690 -T +closed 164b1e1962aa1e16bdf52e9e86d4cf9c9e09220c0821932ac8e390e82074185f +P e9b03b082dcf141695140a6c2c50bf0dc577c3d64ab76f81dd22f0084eca26d6 +R 4a4a4d707423caa7361c65011daf0aec U drh -Z d17b3af74b4bff5f111775dd3cc3c9cb +Z 9d66ab840eaa6125582be7511985f53f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b2918befc9..0b87dcb672 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9b03b082dcf141695140a6c2c50bf0dc577c3d64ab76f81dd22f0084eca26d6 +d03d35eebaf82709414c87cfa6abc9d2baf8d7e64c2627bad0fd5bbda3e78d60 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index d3a2c1f4c9..821ac6a951 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -89,7 +89,9 @@ Other PERMUTATION arguments must be run using testfixture, not tclsh: If no PATTERN arguments are present, all tests specified by the PERMUTATION are run. Otherwise, each pattern is interpreted as a glob pattern. Only those tcl tests for which the final component of the filename matches at -least one specified pattern are run. +least one specified pattern are run. The glob wildcard '*' is prepended +to the pattern if it does not start with '^' and appended to every +pattern that does not end with '$'. If no PATTERN arguments are present, then various fuzztest, threadtest and other tests are run as part of the "release" permutation. These are @@ -842,6 +844,17 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { if {[llength $patternlist]>0} { set bMatch 0 foreach p $patternlist { + set p [string trim $p *] + if {[string index $p 0]=="^"} { + set p [string range $p 1 end] + } else { + set p "*$p" + } + if {[string index $p end]=="\$"} { + set p [string range $p 0 end-1] + } else { + set p "$p*" + } if {[string match $p [file tail $f]]} { set bMatch 1 break @@ -1062,25 +1075,13 @@ proc add_jobs_from_cmdline {patternlist} { } release { - set patternlist [lrange $patternlist 1 end] + set config_set {} foreach b [trd_builds $TRG(platform)] { if {$TRG(config)!="" && ![regexp "\\y$b\\y" $TRG(config)]} continue if {[regexp "\\y$b\\y" $TRG(omitconfig)]} continue - set bld [add_build_job $b $TRG(testfixture)] - foreach c [trd_configs $TRG(platform) $b] { - add_tcl_jobs $bld $c $patternlist - } - - if {$patternlist==""} { - foreach e [trd_extras $TRG(platform) $b] { - if {$e=="fuzztest"} { - add_fuzztest_jobs $b - } else { - add_make_job $bld $e - } - } - } + lappend config_set $b } + add_devtest_jobs $config_set [lrange $patternlist 1 end] } list { diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index af480fc4cf..5ea03f0c00 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -16,7 +16,6 @@ namespace eval trd { set tcltest(linux.Have-Not) veryquick set tcltest(linux.Secure-Delete) veryquick set tcltest(linux.Unlock-Notify) veryquick - set tcltest(linux.User-Auth) veryquick set tcltest(linux.Update-Delete-Limit) veryquick set tcltest(linux.Extra-Robustness) veryquick set tcltest(linux.Device-Two) veryquick @@ -139,10 +138,6 @@ namespace eval trd { -DSQLITE_THREADSAFE -DSQLITE_TCL_DEFAULT_FULLMUTEX=1 } - set build(User-Auth) { - -O2 - -DSQLITE_USER_AUTHENTICATION=1 - } set build(Secure-Delete) { -O2 -DSQLITE_SECURE_DELETE=1 @@ -222,7 +217,7 @@ namespace eval trd { -DSQLITE_MAX_PAGE_SIZE=4096 -DSQLITE_OMIT_LOAD_EXTENSION=1 -DSQLITE_OMIT_PROGRESS_CALLBACK=1 - -DSQLITE_OMIT_VIRTUALTABLE=1 + -DSQLITE_OMIT_VIRTUALTABLEx=1 -DSQLITE_ENABLE_HIDDEN_COLUMNS -DSQLITE_TEMP_STORE=3 } From 3c39f1f5d6ebe8dcd13d1f513f045163926be5bb Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Aug 2024 18:25:04 +0000 Subject: [PATCH 78/99] Add test code to drop an fts5 table with corrupt records in its shadow tables. FossilOrigin-Name: ca21c942c30a3dbff0e7d118e105b847d80b5388c74d19c2eeea71581f8f40b8 --- ext/fts5/fts5_tcl.c | 111 +++++++++++++++++++++++++++----- ext/fts5/test/fts5corrupt8.test | 103 +++++++++++++++++++++++++++++ manifest | 15 +++-- manifest.uuid | 2 +- 4 files changed, 208 insertions(+), 23 deletions(-) create mode 100644 ext/fts5/test/fts5corrupt8.test diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 1e9f7bbb60..a8ab44096b 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -96,14 +96,14 @@ static int SQLITE_TCLAPI f5tDbAndApi( rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0); return TCL_ERROR; } sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0); sqlite3_step(pStmt); if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0); return TCL_ERROR; } @@ -392,7 +392,7 @@ static int SQLITE_TCLAPI xF5tApi( CASE(12, "xSetAuxdata") { F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData)); if( pData==0 ){ - Tcl_AppendResult(interp, "out of memory", 0); + Tcl_AppendResult(interp, "out of memory", (char*)0); return TCL_ERROR; } pData->pObj = objv[2]; @@ -452,7 +452,7 @@ static int SQLITE_TCLAPI xF5tApi( rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0); return TCL_ERROR; } for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){ @@ -683,7 +683,7 @@ static int SQLITE_TCLAPI f5tCreateFunction( pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy ); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0); return TCL_ERROR; } @@ -750,7 +750,7 @@ static int SQLITE_TCLAPI f5tTokenize( if( objc==5 ){ char *zOpt = Tcl_GetString(objv[1]); if( strcmp("-subst", zOpt) ){ - Tcl_AppendResult(interp, "unrecognized option: ", zOpt, 0); + Tcl_AppendResult(interp, "unrecognized option: ", zOpt, (char*)0); return TCL_ERROR; } } @@ -759,7 +759,7 @@ static int SQLITE_TCLAPI f5tTokenize( return TCL_ERROR; } if( nArg==0 ){ - Tcl_AppendResult(interp, "no such tokenizer: ", 0); + Tcl_AppendResult(interp, "no such tokenizer: ", (char*)0); Tcl_Free((void*)azArg); return TCL_ERROR; } @@ -767,13 +767,13 @@ static int SQLITE_TCLAPI f5tTokenize( rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], 0); + Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], (char*)0); return TCL_ERROR; } rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0); + Tcl_AppendResult(interp, "error in tokenizer.xCreate()", (char*)0); return TCL_ERROR; } @@ -787,7 +787,7 @@ static int SQLITE_TCLAPI f5tTokenize( ); tokenizer.xDelete(pTok); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0); + Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", (char*)0); Tcl_DecrRefCount(pRet); return TCL_ERROR; } @@ -1049,7 +1049,7 @@ static int SQLITE_TCLAPI f5tTokenizerLocale( if( p->xToken==0 ){ Tcl_AppendResult(interp, - "sqlite3_fts5_locale may only be used by tokenizer callback", 0 + "sqlite3_fts5_locale may only be used by tokenizer callback", (char*)0 ); return TCL_ERROR; } @@ -1098,7 +1098,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn( if( p->xToken==0 ){ Tcl_AppendResult(interp, - "sqlite3_fts5_token may only be used by tokenizer callback", 0 + "sqlite3_fts5_token may only be used by tokenizer callback", (char*)0 ); return TCL_ERROR; } @@ -1250,7 +1250,7 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( Tcl_AppendResult(interp, ( bV2 ? "error in fts5_api.xCreateTokenizer_v2()" : "error in fts5_api.xCreateTokenizer()" - ), 0); + ), (char*)0); return TCL_ERROR; } @@ -1540,12 +1540,92 @@ static int SQLITE_TCLAPI f5tRegisterOriginText( Tcl_ResetResult(interp); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0); return TCL_ERROR; } return TCL_OK; } +/* +** This function is used to DROP an fts5 table. It works even if the data +** structures fts5 stores within the database are corrupt, which sometimes +** prevents a straight "DROP TABLE" command from succeeding. +** +** The first parameter is the database handle to use for the DROP TABLE +** operation. The second is the name of the database to drop the fts5 table +** from (i.e. "main", "temp" or the name of an attached database). The +** third parameter is the name of the fts5 table to drop. +** +** SQLITE_OK is returned if the table is successfully dropped. Or, if an +** error occurs, an SQLite error code. +*/ +static int sqlite3_fts5_drop_corrupt_table( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + const char *zTab /* Name of fts5 table to drop */ +){ + int rc = SQLITE_OK; + int bDef = 0; + + rc = sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDef); + if( rc==SQLITE_OK ){ + char *zScript = sqlite3_mprintf( + "DELETE FROM %Q.'%q_data';" + "DELETE FROM %Q.'%q_config';" + "INSERT INTO %Q.'%q_data' VALUES(10, X'0000000000');" + "INSERT INTO %Q.'%q_config' VALUES('version', 4);" + "DROP TABLE %Q.'%q';", + zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab + ); + + if( zScript==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); + rc = sqlite3_exec(db, zScript, 0, 0, 0); + if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0); + sqlite3_free(zScript); + } + } + + return rc; +} + +/* +** sqlite3_fts5_drop_corrupt_table DB DATABASE TABLE +** +** Description... +*/ +static int SQLITE_TCLAPI f5tDropCorruptTable( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + const char *zDb = 0; + const char *zTab = 0; + int rc = SQLITE_OK; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DATABASE TABLE"); + return TCL_ERROR; + } + if( f5tDbPointer(interp, objv[1], &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + zTab = Tcl_GetString(objv[3]); + + rc = sqlite3_fts5_drop_corrupt_table(db, zDb, zTab); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0); + return TCL_ERROR; + } + + return TCL_OK; +} + /* ** Entry point. */ @@ -1564,7 +1644,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){ { "sqlite3_fts5_token_hash", f5tTokenHash, 0 }, { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }, { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }, - { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 } + { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }, + { "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 } }; int i; F5tTokenizerContext *pContext; diff --git a/ext/fts5/test/fts5corrupt8.test b/ext/fts5/test/fts5corrupt8.test new file mode 100644 index 0000000000..d3f0e3d514 --- /dev/null +++ b/ext/fts5/test/fts5corrupt8.test @@ -0,0 +1,103 @@ +# 2024 Aug 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt8 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} + +do_execsql_test 1.1 { + UPDATE t1_data SET block='hello world' WHERE id=10 +} + +db close +sqlite3 db test.db + +do_catchsql_test 1.2 { + SELECT * FROM t1 +} {1 {vtable constructor failed: t1}} +do_catchsql_test 1.3 { + DROP TABLE t1 +} {1 {vtable constructor failed: t1}} +do_test 1.4 { + sqlite3_db_config db DEFENSIVE 0 +} {0} +do_test 1.5 { + sqlite3_fts5_drop_corrupt_table db main t1 +} {} +do_test 1.6 { + sqlite3_db_config db DEFENSIVE -1 +} {0} +do_execsql_test 1.7 { + SELECT * FROM sqlite_schema +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} +do_execsql_test 2.1 { + UPDATE t1_config SET v=555 WHERE k='version' +} +db close +sqlite3 db test.db +do_catchsql_test 2.2 { + SELECT * FROM t1 +} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}} +do_catchsql_test 2.3 { + DROP TABLE t1 +} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}} +do_test 2.4 { + sqlite3_fts5_drop_corrupt_table db main t1 +} {} +do_execsql_test 2.5 { + SELECT * FROM sqlite_schema +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} +do_execsql_test 3.1 { + DELETE FROM t1_config; +} +db close +sqlite3 db test.db +do_catchsql_test 3.2 { + SELECT * FROM t1 +} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}} +do_catchsql_test 3.3 { + DROP TABLE t1 +} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}} + + +do_test 3.4 { + sqlite3_db_config db DEFENSIVE 1 +} {1} +do_test 3.5 { + sqlite3_fts5_drop_corrupt_table db main t1 +} {} +do_test 3.6 { + sqlite3_db_config db DEFENSIVE -1 +} {1} +do_execsql_test 3.7 { + SELECT * FROM sqlite_schema +} + +finish_test + diff --git a/manifest b/manifest index 3acb861a2b..acecee9702 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\stestrunner:\s\s(1)\sOmit\sall\stesting\sof\sUser-Auth.\n(2)\sAutomatically\sadd\sthe\s"*"\swildcard\sbefore\sand\safter\sall\spattern\narguments.\s\s(3)\sBuild\sthe\ssqlite3\sCLI\sfor\srelease\stests. -D 2024-08-27T17:38:26.497 +C Add\stest\scode\sto\sdrop\san\sfts5\stable\swith\scorrupt\srecords\sin\sits\sshadow\stables. +D 2024-08-27T18:25:04.935 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -102,7 +102,7 @@ F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe F ext/fts5/fts5_main.c 3d8b778f65fe5be218f6a8e4019048f092c0c09621b035ce3e8804093036b578 F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612 -F ext/fts5/fts5_tcl.c 1dcf08028141c40a32634bdcf2d5601622ce4edc48f82ac4ce0cbe0a92a6961d +F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c ae9c4fa93174ef06ffc138bd4280a1c37f7e13624d3d2706aad4b80573f23c41 @@ -153,6 +153,7 @@ F ext/fts5/test/fts5corrupt4.test dc08d19f5b8943e95a7778a7d8da592042504faf18dd93 F ext/fts5/test/fts5corrupt5.test 11b47126f5772cc37b67e3e8b2ed05895c4d07c05338bc07e4eea225bfe32c76 F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 F ext/fts5/test/fts5corrupt7.test 4e830875c33b9ea3c4cf1ba71e692b63893cbb4faae8c69b1071889dc26e211c +F ext/fts5/test/fts5corrupt8.test f78b6de9e4327e16ba7a7958b1ce576f9b5d886b4b97690f90b7f3cf654da458 F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4 F ext/fts5/test/fts5determin.test 1b77879b2ae818b5b71c859e534ee334dac088b7cf3ff3bf76a2c82b1c788d11 @@ -2210,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e9b03b082dcf141695140a6c2c50bf0dc577c3d64ab76f81dd22f0084eca26d6 -R 4a4a4d707423caa7361c65011daf0aec -U drh -Z 9d66ab840eaa6125582be7511985f53f +P d03d35eebaf82709414c87cfa6abc9d2baf8d7e64c2627bad0fd5bbda3e78d60 +R eae9b17364855941c00e92c86d398a4e +U dan +Z d07b6efd95ad4c9798aced1dad6099da # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0b87dcb672..6d952d91c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d03d35eebaf82709414c87cfa6abc9d2baf8d7e64c2627bad0fd5bbda3e78d60 +ca21c942c30a3dbff0e7d118e105b847d80b5388c74d19c2eeea71581f8f40b8 From d9bc2bb638ba4d50c8cea1d5bdad94cb6c0aa479 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 27 Aug 2024 19:17:29 +0000 Subject: [PATCH 79/99] Improvements to testrunner.tcl: Show elapse time in MM:SS or HH:MM:SS. Keep track of the total number of tests and the total number of errors and report those values on a summary line at the end. FossilOrigin-Name: 40b232924c973f8d94605946fff75acbe120d90634a2eaf38bd31649e9f1390b --- manifest | 14 +++++----- manifest.uuid | 2 +- test/testrunner.tcl | 63 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 61 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index acecee9702..e776cdf849 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scode\sto\sdrop\san\sfts5\stable\swith\scorrupt\srecords\sin\sits\sshadow\stables. -D 2024-08-27T18:25:04.935 +C Improvements\sto\stestrunner.tcl:\s\sShow\selapse\stime\sin\sMM:SS\sor\sHH:MM:SS.\nKeep\strack\sof\sthe\stotal\snumber\sof\stests\sand\sthe\stotal\snumber\sof\serrors\nand\sreport\sthose\svalues\son\sa\ssummary\sline\sat\sthe\send. +D 2024-08-27T19:17:29.271 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1714,7 +1714,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl 74b569b553a85a10677e0f9b2f6d22d90d9f07255fbec545ec9438bed677e8e8 +F test/testrunner.tcl 21449521b796376fb4341d84bdf9dae6befc409e2273f795b4bfda78644da22c F test/testrunner_data.tcl b281136c9a4a3cb2e6772a87c2e94efad0c12ea3e7d0710521a3cad964f879a1 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d03d35eebaf82709414c87cfa6abc9d2baf8d7e64c2627bad0fd5bbda3e78d60 -R eae9b17364855941c00e92c86d398a4e -U dan -Z d07b6efd95ad4c9798aced1dad6099da +P ca21c942c30a3dbff0e7d118e105b847d80b5388c74d19c2eeea71581f8f40b8 +R e9d7eae7ed3949bc9ba3747c7b003f26 +U drh +Z 1e89338cdc3d7df9980ff860413d15b9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6d952d91c5..2d3f302a52 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca21c942c30a3dbff0e7d118e105b847d80b5388c74d19c2eeea71581f8f40b8 +40b232924c973f8d94605946fff75acbe120d90634a2eaf38bd31649e9f1390b diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 821ac6a951..88acae3c0e 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -279,10 +279,12 @@ set TRG(schema) { priority INTEGER NOT NULL, -- higher priority jobs may run earlier /* Fields updated as jobs run */ - starttime INTEGER, - endtime INTEGER, + starttime INTEGER, -- Start time (milliseconds since 1970) + endtime INTEGER, -- End time state TEXT CHECK( state IN ('','ready','running','done','failed','omit') ), - output TEXT + ntest INT, -- Number of test cases run + nerr INT, -- Number of errors reported + output TEXT -- test output ); CREATE TABLE config( @@ -391,14 +393,34 @@ if {[string compare -nocase script [lindex $argv 0]]==0} { exit } +# Compute an elapse time string MM:SS or HH:MM:SS based on the +# number of milliseconds in the argument. +# +proc elapsetime {ms} { + set s [expr {int(($ms+500.0)*0.001)}] + set hr [expr {$s/3600}] + set mn [expr {($s/60)%60}] + set sc [expr {$s%60}] + if {$hr>0} { + return [format %02d:%02d:%02d $hr $mn $sc] + } else { + return [format %02d:%02d $mn $sc] + } +} + # Helper routine for show_status # proc display_job {jobdict {tm ""}} { array set job $jobdict - set dfname [format %-60s $job(displayname)] + if {[string length $job(displayname)]>65} { + set dfname [format %.65s... $job(displayname)] + } else { + set dfname [format %-68s $job(displayname)] + } set dtm "" if {$tm!=""} { - set dtm [format %-10s "\[[expr {$tm-$job(starttime)}]ms\]"] + set dtm [expr {$tm-$job(starttime)}] + set dtm [format %8s [elapsetime $dtm]] } puts " $dfname $dtm" } @@ -449,10 +471,10 @@ proc show_status {db cls} { if {$S(failed)>0} { set f "$S(failed) FAILED, " } - puts "Command line: \[testrunner.tcl$cmdline\]$clreol" - puts "Jobs: $nJob " - puts "Summary: ${tm}ms, ($fin/$total) finished,\ - ${f}$S(running) running " + puts [format %-79s "Command line: \[testrunner.tcl$cmdline\]$clreol"] + puts [format %-79s "Jobs: $nJob"] + puts [format %-79s "Summary: [elapsetime $tm], ($fin/$total) finished,\ + ${f}$S(running) running "] set srcdir [file dirname [file dirname $TRG(info_script)]] if {$S(running)>0} { @@ -1121,15 +1143,25 @@ proc make_new_testset {} { } proc mark_job_as_finished {jobid output state endtm} { + set ntest 1 + set nerr 0 + if {$endtm>0} { + if {[regexp {\y(\d+) errors out of (\d+) tests} $output all a b]} { + set nerr $a + set ntest $b + } + } r_write_db { if {$state=="failed"} { set childstate omit + if {$nerr<=0} {set nerr 1} } else { set childstate ready } trdb eval { UPDATE jobs - SET output=$output, state=$state, endtime=$endtm + SET output=$output, state=$state, endtime=$endtm, + ntest=$ntest, nerr=$nerr WHERE jobid=$jobid; UPDATE jobs SET state=$childstate WHERE depid=$jobid; } @@ -1372,6 +1404,17 @@ proc run_testset {} { puts "\nTest database is $TRG(dbname)" puts "Test log is $TRG(logname)" + trdb eval { + SELECT sum(ntest) AS totaltest, + sum(nerr) AS totalerr + FROM jobs + } break + trdb eval { + SELECT max(endtime)-min(starttime) AS totaltime + FROM jobs WHERE endtime>0 + } break; + set et [elapsetime $totaltime] + puts "$totalerr errors out of $totaltest tests in about $et" } # Handle the --buildonly option, if it was specified. From c8edf07ca73b6f85235e1df09ea3650e6688d402 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Aug 2024 19:27:40 +0000 Subject: [PATCH 80/99] Modify the implementation of "wrapper" tokenizers to make them more robust in the case where the database connection is closed before the tokenizers are deleted. FossilOrigin-Name: 7c0001d6eb43f89144eb84d9e30f575a3feed401d685a0d1f260692e419b2df2 --- ext/fts5/fts5_main.c | 25 +++++++++++++------------ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 4addc07294..5a97930ef7 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -3267,7 +3267,9 @@ static int fts5NewTokenizerModule( */ typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer; struct Fts5VtoVTokenizer { - Fts5TokenizerModule *pMod; + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ Fts5Tokenizer *pReal; }; @@ -3287,7 +3289,9 @@ static int fts5VtoVCreate( pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); if( rc==SQLITE_OK ){ - pNew->pMod = pMod; + pNew->x1 = pMod->x1; + pNew->x2 = pMod->x2; + pNew->bV2Native = pMod->bV2Native; if( pMod->bV2Native ){ rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); }else{ @@ -3309,11 +3313,10 @@ static int fts5VtoVCreate( static void fts5VtoVDelete(Fts5Tokenizer *pTok){ Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; if( p ){ - Fts5TokenizerModule *pMod = p->pMod; - if( pMod->bV2Native ){ - pMod->x2.xDelete(p->pReal); + if( p->bV2Native ){ + p->x2.xDelete(p->pReal); }else{ - pMod->x1.xDelete(p->pReal); + p->x1.xDelete(p->pReal); } sqlite3_free(p); } @@ -3331,9 +3334,8 @@ static int fts5V1toV2Tokenize( int (*xToken)(void*, int, const char*, int, int, int) ){ Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; - Fts5TokenizerModule *pMod = p->pMod; - assert( pMod->bV2Native ); - return pMod->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken); + assert( p->bV2Native ); + return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken); } /* @@ -3348,10 +3350,9 @@ static int fts5V2toV1Tokenize( int (*xToken)(void*, int, const char*, int, int, int) ){ Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; - Fts5TokenizerModule *pMod = p->pMod; - assert( pMod->bV2Native==0 ); + assert( p->bV2Native==0 ); UNUSED_PARAM2(pLocale,nLocale); - return pMod->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken); + return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken); } /* diff --git a/manifest b/manifest index e776cdf849..95f0b78d9e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\stestrunner.tcl:\s\sShow\selapse\stime\sin\sMM:SS\sor\sHH:MM:SS.\nKeep\strack\sof\sthe\stotal\snumber\sof\stests\sand\sthe\stotal\snumber\sof\serrors\nand\sreport\sthose\svalues\son\sa\ssummary\sline\sat\sthe\send. -D 2024-08-27T19:17:29.271 +C Modify\sthe\simplementation\sof\s"wrapper"\stokenizers\sto\smake\sthem\smore\srobust\sin\sthe\scase\swhere\sthe\sdatabase\sconnection\sis\sclosed\sbefore\sthe\stokenizers\sare\sdeleted. +D 2024-08-27T19:27:40.626 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81c F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 3d8b778f65fe5be218f6a8e4019048f092c0c09621b035ce3e8804093036b578 +F ext/fts5/fts5_main.c 0dbe241c50b62e24a229a176d66d580e359b3c32cb0970c39bb6dd6b86e125d5 F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ca21c942c30a3dbff0e7d118e105b847d80b5388c74d19c2eeea71581f8f40b8 -R e9d7eae7ed3949bc9ba3747c7b003f26 -U drh -Z 1e89338cdc3d7df9980ff860413d15b9 +P 40b232924c973f8d94605946fff75acbe120d90634a2eaf38bd31649e9f1390b +R 37747ff8c02c47392327f3909e12ba2c +U dan +Z 559c141ac65916995897dbbf28b9ea88 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2d3f302a52..b2a520b2f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -40b232924c973f8d94605946fff75acbe120d90634a2eaf38bd31649e9f1390b +7c0001d6eb43f89144eb84d9e30f575a3feed401d685a0d1f260692e419b2df2 From 71475478a1f3c00043144164858802c65b4004c1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 27 Aug 2024 19:28:21 +0000 Subject: [PATCH 81/99] Further improvement to the status display in testrunner.tcl. Show the number of errors and the number of test cases on the status summary. FossilOrigin-Name: dbc1ac8692feccde271eaca78f2833c8bb7bf620088c8f975692d6ae2c8ba53a --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/testrunner.tcl | 15 ++++++++------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 95f0b78d9e..e4f301cbb5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\simplementation\sof\s"wrapper"\stokenizers\sto\smake\sthem\smore\srobust\sin\sthe\scase\swhere\sthe\sdatabase\sconnection\sis\sclosed\sbefore\sthe\stokenizers\sare\sdeleted. -D 2024-08-27T19:27:40.626 +C Further\simprovement\sto\sthe\sstatus\sdisplay\sin\stestrunner.tcl.\s\sShow\sthe\nnumber\sof\serrors\sand\sthe\snumber\sof\stest\scases\son\sthe\sstatus\ssummary. +D 2024-08-27T19:28:21.475 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1714,7 +1714,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl 21449521b796376fb4341d84bdf9dae6befc409e2273f795b4bfda78644da22c +F test/testrunner.tcl dfb6049b07a44559a2cba909589d56bfe2d9f5ce1828ad3537e350183be3416f F test/testrunner_data.tcl b281136c9a4a3cb2e6772a87c2e94efad0c12ea3e7d0710521a3cad964f879a1 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 40b232924c973f8d94605946fff75acbe120d90634a2eaf38bd31649e9f1390b -R 37747ff8c02c47392327f3909e12ba2c -U dan -Z 559c141ac65916995897dbbf28b9ea88 +P 7c0001d6eb43f89144eb84d9e30f575a3feed401d685a0d1f260692e419b2df2 +R 2b16d26e239f800d0fd65b874c223ff3 +U drh +Z 3e4827c72f9a5e7f0a58061fb0e5d912 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b2a520b2f8..f88cc1fe3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c0001d6eb43f89144eb84d9e30f575a3feed401d685a0d1f260692e419b2df2 +dbc1ac8692feccde271eaca78f2833c8bb7bf620088c8f975692d6ae2c8ba53a diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 88acae3c0e..bf9e46d934 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -455,6 +455,11 @@ proc show_status {db cls} { incr S($state) $cnt incr total $cnt } + set nt 0 + set ne 0 + $db eval { + SELECT sum(ntest) AS nt, sum(nerr) AS ne FROM jobs HAVING nt>0 + } break set fin [expr $S(done)+$S(failed)] if {$cmdline!=""} {set cmdline " $cmdline"} @@ -467,14 +472,10 @@ proc show_status {db cls} { } else { set clreol "" } - set f "" - if {$S(failed)>0} { - set f "$S(failed) FAILED, " - } puts [format %-79s "Command line: \[testrunner.tcl$cmdline\]$clreol"] - puts [format %-79s "Jobs: $nJob"] - puts [format %-79s "Summary: [elapsetime $tm], ($fin/$total) finished,\ - ${f}$S(running) running "] + puts [format %-79s "Cores: $nJob max $S(running) active"] + puts [format %-79s "Summary: [elapsetime $tm], $fin/$total tasks,\ + $ne errors, $nt tests"] set srcdir [file dirname [file dirname $TRG(info_script)]] if {$S(running)>0} { From db05c66a16370cb336de662a597edee2c0b814f2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 27 Aug 2024 19:43:21 +0000 Subject: [PATCH 82/99] In testrunner, when doing the --status updates to the screen, be sure to overwrite dead text at the end of each line with spaces. FossilOrigin-Name: 7c195f132cb48ee44124eb4af3532937493429f7c45f6ac611f7a9b128799ad7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e4f301cbb5..9a3d8d14ca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\simprovement\sto\sthe\sstatus\sdisplay\sin\stestrunner.tcl.\s\sShow\sthe\nnumber\sof\serrors\sand\sthe\snumber\sof\stest\scases\son\sthe\sstatus\ssummary. -D 2024-08-27T19:28:21.475 +C In\stestrunner,\swhen\sdoing\sthe\s--status\supdates\sto\sthe\sscreen,\sbe\ssure\sto\noverwrite\sdead\stext\sat\sthe\send\sof\seach\sline\swith\sspaces. +D 2024-08-27T19:43:21.041 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1714,7 +1714,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl dfb6049b07a44559a2cba909589d56bfe2d9f5ce1828ad3537e350183be3416f +F test/testrunner.tcl caa8c5d6cd61f0656b16fbd3e3207814d4aa1b320eedf0bdf1ac516963c7b8ae F test/testrunner_data.tcl b281136c9a4a3cb2e6772a87c2e94efad0c12ea3e7d0710521a3cad964f879a1 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7c0001d6eb43f89144eb84d9e30f575a3feed401d685a0d1f260692e419b2df2 -R 2b16d26e239f800d0fd65b874c223ff3 +P dbc1ac8692feccde271eaca78f2833c8bb7bf620088c8f975692d6ae2c8ba53a +R ae9b21fe337c43816e809f3a8dafca48 U drh -Z 3e4827c72f9a5e7f0a58061fb0e5d912 +Z 306e9d99d5e1930ae9fd911a1e781c28 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f88cc1fe3b..f1d875d829 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dbc1ac8692feccde271eaca78f2833c8bb7bf620088c8f975692d6ae2c8ba53a +7c195f132cb48ee44124eb4af3532937493429f7c45f6ac611f7a9b128799ad7 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index bf9e46d934..53dead6bc9 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -421,6 +421,8 @@ proc display_job {jobdict {tm ""}} { if {$tm!=""} { set dtm [expr {$tm-$job(starttime)}] set dtm [format %8s [elapsetime $dtm]] + } else{ + set dtm [format %8s ""] } puts " $dfname $dtm" } @@ -495,7 +497,7 @@ proc show_status {db cls} { } set nOmit [$db one {SELECT count(*) FROM jobs WHERE state='omit'}] if {$nOmit} { - puts "$nOmit jobs omitted due to failures$clreol" + puts [format %-79s "$nOmit jobs omitted due to failures$clreol"] } } if {$cls} { From 1a7cfbebb01d3d34a388bed5617a28b9f993ebee Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 27 Aug 2024 19:48:19 +0000 Subject: [PATCH 83/99] Fix typo in the previous check-in. Ensure that --status header lines do not overflow and wrap. FossilOrigin-Name: 2eaea67495fd9961b7145d829789889bf1d8fa7a834d40d1be43e7c74c233bc2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/testrunner.tcl | 10 +++++----- test/testrunner_data.tcl | 1 + 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 9a3d8d14ca..000260da0a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\stestrunner,\swhen\sdoing\sthe\s--status\supdates\sto\sthe\sscreen,\sbe\ssure\sto\noverwrite\sdead\stext\sat\sthe\send\sof\seach\sline\swith\sspaces. -D 2024-08-27T19:43:21.041 +C Fix\stypo\sin\sthe\sprevious\scheck-in.\s\sEnsure\sthat\s--status\sheader\slines\sdo\snot\noverflow\sand\swrap. +D 2024-08-27T19:48:19.164 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1714,8 +1714,8 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl caa8c5d6cd61f0656b16fbd3e3207814d4aa1b320eedf0bdf1ac516963c7b8ae -F test/testrunner_data.tcl b281136c9a4a3cb2e6772a87c2e94efad0c12ea3e7d0710521a3cad964f879a1 +F test/testrunner.tcl c27905c57f54450411be922037d227fee65b08928d09715d784380a398b0e758 +F test/testrunner_data.tcl 6a0facfbff26f2f8573ebaeb209426bbb59809d5c0f6925f983bd030ab5248cd F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dbc1ac8692feccde271eaca78f2833c8bb7bf620088c8f975692d6ae2c8ba53a -R ae9b21fe337c43816e809f3a8dafca48 +P 7c195f132cb48ee44124eb4af3532937493429f7c45f6ac611f7a9b128799ad7 +R 15868716c90b102057036239341d8066 U drh -Z 306e9d99d5e1930ae9fd911a1e781c28 +Z fde036223228b7af985cf94952c295a1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f1d875d829..f21f71e7f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c195f132cb48ee44124eb4af3532937493429f7c45f6ac611f7a9b128799ad7 +2eaea67495fd9961b7145d829789889bf1d8fa7a834d40d1be43e7c74c233bc2 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 53dead6bc9..8d81469c02 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -421,7 +421,7 @@ proc display_job {jobdict {tm ""}} { if {$tm!=""} { set dtm [expr {$tm-$job(starttime)}] set dtm [format %8s [elapsetime $dtm]] - } else{ + } else { set dtm [format %8s ""] } puts " $dfname $dtm" @@ -474,10 +474,10 @@ proc show_status {db cls} { } else { set clreol "" } - puts [format %-79s "Command line: \[testrunner.tcl$cmdline\]$clreol"] - puts [format %-79s "Cores: $nJob max $S(running) active"] - puts [format %-79s "Summary: [elapsetime $tm], $fin/$total tasks,\ - $ne errors, $nt tests"] + puts [format %-79.79s "Command: \[testrunner.tcl$cmdline\]$clreol"] + puts [format %-79.79s "Cores: $nJob max $S(running) active"] + puts [format %-79.79s "Summary: [elapsetime $tm], $fin/$total tasks,\ + $ne errors, $nt tests"] set srcdir [file dirname [file dirname $TRG(info_script)]] if {$S(running)>0} { diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index 5ea03f0c00..48484a3e24 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -115,6 +115,7 @@ namespace eval trd { } set build(Stdcall) { -DUSE_STDCALL=1 + -DSQLITE_USE_ONLY_WIN32=1 -O2 } From f5a9b58c8bc6b159cb942dbdb2c71a402c0ce3bd Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Aug 2024 20:37:30 +0000 Subject: [PATCH 84/99] Avoid reading the structure record from within the fts5 xConnect method. FossilOrigin-Name: 6a6ce343b249e269229867b2c2f107a49ac8a8f24ad66801718ef01159655319 --- ext/fts5/fts5_main.c | 3 +-- ext/fts5/test/fts5corrupt3.test | 14 +++++++------- ext/fts5/test/fts5corrupt8.test | 15 +++------------ ext/fts5/test/fts5fault4.test | 2 +- ext/fts5/test/fts5misc.test | 8 ++++---- ext/fts5/test/fts5simple.test | 2 +- manifest | 25 ++++++++++++------------- manifest.uuid | 2 +- 8 files changed, 30 insertions(+), 41 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 5a97930ef7..565b1ec305 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -427,8 +427,7 @@ static int fts5InitVtab( /* Load the initial configuration */ if( rc==SQLITE_OK ){ - rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); - sqlite3Fts5IndexRollback(pTab->p.pIndex); + rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie-1); } if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index c5faaa87b1..3e8b0377cc 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -680,11 +680,11 @@ do_test 12.0 { do_catchsql_test 11.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {vtable constructor failed: t1}} +} {1 {database disk image is malformed}} do_catchsql_test 11.2 { INSERT INTO t1(t1, rank) VALUES('merge', 500); -} {1 {vtable constructor failed: t1}} +} {1 {database disk image is malformed}} #------------------------------------------------------------------------- # @@ -1040,7 +1040,7 @@ do_test 16.0 { do_catchsql_test 16.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {vtable constructor failed: t1}} +} {1 {database disk image is malformed}} #-------------------------------------------------------------------------- reset_db @@ -1126,7 +1126,7 @@ do_test 17.0 { do_catchsql_test 17.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {vtable constructor failed: t1}} +} {1 {database disk image is malformed}} #-------------------------------------------------------------------------- reset_db @@ -1630,7 +1630,7 @@ do_test 20.0 { do_catchsql_test 20.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {vtable constructor failed: t1}} +} {1 {database disk image is malformed}} #------------------------------------------------------------------------- reset_db @@ -2100,7 +2100,7 @@ do_test 22.0 { do_catchsql_test 22.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {vtable constructor failed: t1}} +} {1 {database disk image is malformed}} #-------------------------------------------------------------------------- reset_db @@ -3700,7 +3700,7 @@ do_catchsql_test 32.1 { highlight(t1, 2, '[', ']') FROM t1('g + h') WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank; -} {1 {vtable constructor failed: t1}} +} {1 {database disk image is malformed}} do_catchsql_test 32.2 { SELECT * FROM t3; diff --git a/ext/fts5/test/fts5corrupt8.test b/ext/fts5/test/fts5corrupt8.test index d3f0e3d514..d642920e45 100644 --- a/ext/fts5/test/fts5corrupt8.test +++ b/ext/fts5/test/fts5corrupt8.test @@ -32,20 +32,11 @@ sqlite3 db test.db do_catchsql_test 1.2 { SELECT * FROM t1 -} {1 {vtable constructor failed: t1}} +} {1 {database disk image is malformed}} do_catchsql_test 1.3 { DROP TABLE t1 -} {1 {vtable constructor failed: t1}} -do_test 1.4 { - sqlite3_db_config db DEFENSIVE 0 -} {0} -do_test 1.5 { - sqlite3_fts5_drop_corrupt_table db main t1 -} {} -do_test 1.6 { - sqlite3_db_config db DEFENSIVE -1 -} {0} -do_execsql_test 1.7 { +} {0 {}} +do_execsql_test 1.4 { SELECT * FROM sqlite_schema } diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test index 1d0d5c9b7c..2b4f6c4d2a 100644 --- a/ext/fts5/test/fts5fault4.test +++ b/ext/fts5/test/fts5fault4.test @@ -90,7 +90,7 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}] do_faultsim_test 4 -faults oom-* -body { db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'} } -test { - faultsim_test_result {0 {0 {} 3}} + faultsim_test_result {0 {0 {} 2}} } #------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index 534c42fff6..c2e580c564 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -44,12 +44,12 @@ do_catchsql_test 1.2.2 { do_catchsql_test 1.3.1 { SELECT highlight(t1, 4, '', '') FROM t1('*reads'); -} {1 {no such cursor: 1}} +} {1 {no such cursor: 0}} do_catchsql_test 1.3.2 { SELECT a FROM t1 WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*reads')); -} {1 {no such cursor: 1}} +} {1 {no such cursor: 0}} db close sqlite3 db test.db @@ -57,12 +57,12 @@ sqlite3 db test.db do_catchsql_test 1.3.3 { SELECT a FROM t1 WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*reads')); -} {1 {no such cursor: 1}} +} {1 {no such cursor: 0}} fts5_aux_test_functions db do_catchsql_test 1.3.4 { SELECT fts5_columntext(t1) FROM t1('*reads'); -} {1 {no such cursor: 1}} +} {1 {no such cursor: 0}} #------------------------------------------------------------------------- reset_db diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 60ccb5a9c5..ad59bf0d9e 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -350,7 +350,7 @@ do_execsql_test 14.3 { do_execsql_test 14.4 { SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' -} {0 {} 3} +} {0 {} 2} #------------------------------------------------------------------------- reset_db diff --git a/manifest b/manifest index 129a212ab1..a00129dc3c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sminor\simprovements\sand\stweaks\sto\sthe\stestrunner\s--status\sdisplay. -D 2024-08-27T19:49:51.307 +C Avoid\sreading\sthe\sstructure\srecord\sfrom\swithin\sthe\sfts5\sxConnect\smethod. +D 2024-08-27T20:37:30.982 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81c F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c 0dbe241c50b62e24a229a176d66d580e359b3c32cb0970c39bb6dd6b86e125d5 +F ext/fts5/fts5_main.c bd41fc81751fc7cb666de7dee52cf9bb97c04ce06a8122e7e41bb7527bde54fc F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -148,12 +148,12 @@ F ext/fts5/test/fts5contentless4.test ec34dc69ef474ca9997dae6d91e072906e0e9a5a4b F ext/fts5/test/fts5contentless5.test 40cdcb4fe751672450829c5a96bd32c25fc2f6076279dd2ce5c58ac9a390132a F ext/fts5/test/fts5corrupt.test 6485f721b88ba355ca5d701e7ee87a4efa3ea578d8e6adb26f51ef956c8328bd F ext/fts5/test/fts5corrupt2.test 335911e3f68b9625d850325f9e29a128db3f4276a8c9d4e32134580da8f924c4 -F ext/fts5/test/fts5corrupt3.test 621e9bca3e7299f487e1b29ff4179d9fc9560f5847dfc5b50a16010c9d2a0e5f +F ext/fts5/test/fts5corrupt3.test 4fc3bf129f1616bea00884a23fd9d7b0e46d01791d2b57fe8d68ac36e8d3ff7c F ext/fts5/test/fts5corrupt4.test dc08d19f5b8943e95a7778a7d8da592042504faf18dd93f68f7d7a0d7d7dd733 F ext/fts5/test/fts5corrupt5.test 11b47126f5772cc37b67e3e8b2ed05895c4d07c05338bc07e4eea225bfe32c76 F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 F ext/fts5/test/fts5corrupt7.test 4e830875c33b9ea3c4cf1ba71e692b63893cbb4faae8c69b1071889dc26e211c -F ext/fts5/test/fts5corrupt8.test f78b6de9e4327e16ba7a7958b1ce576f9b5d886b4b97690f90b7f3cf654da458 +F ext/fts5/test/fts5corrupt8.test b81d802e41631e98100f49a1aadeeffef860e30a62d6ed7d743c2797c477239e F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4 F ext/fts5/test/fts5determin.test 1b77879b2ae818b5b71c859e534ee334dac088b7cf3ff3bf76a2c82b1c788d11 @@ -165,7 +165,7 @@ F ext/fts5/test/fts5expr.test c7e208813df7a90badc856fde3796da79569b39382e0fdb430 F ext/fts5/test/fts5fault1.test d28a65caee75db6897c3cf1358c5230d3bb2a3bf7fb31062c19c7e5382b3d2bd F ext/fts5/test/fts5fault2.test 69c8fdbef830cd0d450908d4504d5bb86609e255af99c421c20a0756251fe344 F ext/fts5/test/fts5fault3.test da2f9e3e56ff5740d68ebdd6877c97089e7ed28ddff28a0da87a6afea27e5522 -F ext/fts5/test/fts5fault4.test 87a10d0caee57da587c7588b0c8d25d2930197399b4812ad1e4d574c75324cee +F ext/fts5/test/fts5fault4.test a5c0e849127c24e1751bc453a817f09a1b8d460e75f9ae4764017e216a870db3 F ext/fts5/test/fts5fault5.test a336e4e11847de24c9497f80cce18e00bb3fab7fb11f97d04eb9af898900a762 F ext/fts5/test/fts5fault6.test 40f49976c6ca8927bf7d65d0b8df46009d7ea172e1d4050b294610e7ea0a2979 F ext/fts5/test/fts5fault7.test 0acbec416edb24b8881f154e99c31e9ccf73f539cfcd164090be139e9e97ed4c @@ -192,7 +192,7 @@ F ext/fts5/test/fts5locale.test 797cf6f5e017462ab11313ce884b9f1df8ff063811e74ef4 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test 60bb2be4a2d83d7a45047c1812781e2e337a27efa539d86356ef7f4acaf08eab +F ext/fts5/test/fts5misc.test 8c3cc771f773dc4bb4973620c51e7729e324ca2cc80eb8894f1c2c605e361f0b F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74 @@ -226,7 +226,7 @@ F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a F ext/fts5/test/fts5secure8.test 808ade9d172ed07b24b85c57dd53b6d2b1aba018b4e634d267ce572221de80e0 F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5 -F ext/fts5/test/fts5simple.test ed7c3815c9fa1c16166258cb98edb2e014c63c7589958d76c5487df0df913d61 +F ext/fts5/test/fts5simple.test 302cdb4f8a3350b091f4f1bccd82d05610428657f6f9e81c17703ba48267ec40 F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc F ext/fts5/test/fts5simple3.test 146ec3dc8f5763d6212641c9f0a2f1cba41679353d2add7b963beceb115dc7f4 F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988202ce67638d2c6cf6 @@ -2211,9 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dbc1ac8692feccde271eaca78f2833c8bb7bf620088c8f975692d6ae2c8ba53a 2eaea67495fd9961b7145d829789889bf1d8fa7a834d40d1be43e7c74c233bc2 -R 15868716c90b102057036239341d8066 -T +closed 2eaea67495fd9961b7145d829789889bf1d8fa7a834d40d1be43e7c74c233bc2 -U drh -Z c190467c6ca0a01405f49df20c5de973 +P 7d1ccc6dfc9abe053baaf39f68e9fb14001f5032a52b85fe1e4b752e48317536 +R b2f3edb080a36a6c22913345d9099db8 +U dan +Z ae75f308e945276fc7d3f4e6bfacd7cd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bdd3c3e43e..7394734f08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7d1ccc6dfc9abe053baaf39f68e9fb14001f5032a52b85fe1e4b752e48317536 +6a6ce343b249e269229867b2c2f107a49ac8a8f24ad66801718ef01159655319 From c3c645ef465767abc71b33533c1f576c35f84338 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 27 Aug 2024 21:44:45 +0000 Subject: [PATCH 85/99] Improvements to --status overwrite in testrunner. FossilOrigin-Name: 26372762ab1451eb95d29367f5b2534bde616a37e5e757e72ba2cfe42571d180 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/testrunner.tcl | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a00129dc3c..d025a4f329 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sreading\sthe\sstructure\srecord\sfrom\swithin\sthe\sfts5\sxConnect\smethod. -D 2024-08-27T20:37:30.982 +C Improvements\sto\s--status\soverwrite\sin\stestrunner. +D 2024-08-27T21:44:45.108 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1714,7 +1714,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl c27905c57f54450411be922037d227fee65b08928d09715d784380a398b0e758 +F test/testrunner.tcl 0a9e76679856e5782d520f8e4329c18ba655b6bbd19a154f44a4fb5e350be531 F test/testrunner_data.tcl 6a0facfbff26f2f8573ebaeb209426bbb59809d5c0f6925f983bd030ab5248cd F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7d1ccc6dfc9abe053baaf39f68e9fb14001f5032a52b85fe1e4b752e48317536 -R b2f3edb080a36a6c22913345d9099db8 -U dan -Z ae75f308e945276fc7d3f4e6bfacd7cd +P 6a6ce343b249e269229867b2c2f107a49ac8a8f24ad66801718ef01159655319 +R f5da9843604e1033edc0cf6e244622b0 +U drh +Z fa568405ca45a71e0343a4bd7c4efb05 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7394734f08..13a472ba51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a6ce343b249e269229867b2c2f107a49ac8a8f24ad66801718ef01159655319 +26372762ab1451eb95d29367f5b2534bde616a37e5e757e72ba2cfe42571d180 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 8d81469c02..6344d0b0e2 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -481,7 +481,7 @@ proc show_status {db cls} { set srcdir [file dirname [file dirname $TRG(info_script)]] if {$S(running)>0} { - puts "Running: " + puts [format %-79s "Running:"] $db eval { SELECT * FROM jobs WHERE state='running' ORDER BY starttime } job { @@ -489,7 +489,7 @@ proc show_status {db cls} { } } if {$S(failed)>0} { - puts "Failures: " + puts [format %-79s "Failures:"] $db eval { SELECT * FROM jobs WHERE state='failed' ORDER BY starttime } job { From b5b4ca2a11f513d802c1eccea24fd5d128a324e8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 27 Aug 2024 22:28:03 +0000 Subject: [PATCH 86/99] Changes to the way "release" tests were constructed in [d03d35eebaf82709] were incorrect. Fixed here. Also enhance --explain to chose permutation configurations and so that PATTERNS on the the command line can match against permutation configurations. FossilOrigin-Name: 1f962ffc9e956a9d6311ed9c02fea13c0fe056283ee2c650a2312a258808b3b3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 40 ++++++++++++++++++++++++++++++++++------ 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index d025a4f329..de8374a5ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\s--status\soverwrite\sin\stestrunner. -D 2024-08-27T21:44:45.108 +C Changes\sto\sthe\sway\s"release"\stests\swere\sconstructed\sin\s[d03d35eebaf82709]\nwere\sincorrect.\s\sFixed\shere.\s\sAlso\senhance\s--explain\sto\schose\spermutation\nconfigurations\sand\sso\sthat\sPATTERNS\son\sthe\sthe\scommand\sline\scan\smatch\nagainst\spermutation\sconfigurations. +D 2024-08-27T22:28:03.935 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1714,7 +1714,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl 0a9e76679856e5782d520f8e4329c18ba655b6bbd19a154f44a4fb5e350be531 +F test/testrunner.tcl de5deab405d25cdb0f85c4b75cc63dc1fa0571b099621b585f083647d7f6d75a F test/testrunner_data.tcl 6a0facfbff26f2f8573ebaeb209426bbb59809d5c0f6925f983bd030ab5248cd F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6a6ce343b249e269229867b2c2f107a49ac8a8f24ad66801718ef01159655319 -R f5da9843604e1033edc0cf6e244622b0 +P 26372762ab1451eb95d29367f5b2534bde616a37e5e757e72ba2cfe42571d180 +R 66a023125402741fa708637568f51009 U drh -Z fa568405ca45a71e0343a4bd7c4efb05 +Z f1950a098efb328b04fe6e5b9690c497 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 13a472ba51..df0f6a1846 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -26372762ab1451eb95d29367f5b2534bde616a37e5e757e72ba2cfe42571d180 +1f962ffc9e956a9d6311ed9c02fea13c0fe056283ee2c650a2312a258808b3b3 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 6344d0b0e2..fcb46a9214 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -475,7 +475,7 @@ proc show_status {db cls} { set clreol "" } puts [format %-79.79s "Command: \[testrunner.tcl$cmdline\]$clreol"] - puts [format %-79.79s "Cores: $nJob max $S(running) active"] + puts [format %-79.79s "Jobs: $nJob max $S(running) active"] puts [format %-79.79s "Summary: [elapsetime $tm], $fin/$total tasks,\ $ne errors, $nt tests"] @@ -880,7 +880,7 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { } else { set p "$p*" } - if {[string match $p [file tail $f]]} { + if {[string match $p "$config [file tail $f]"]} { set bMatch 1 break } @@ -1100,13 +1100,34 @@ proc add_jobs_from_cmdline {patternlist} { } release { - set config_set {} + set patternlist [lrange $patternlist 1 end] foreach b [trd_builds $TRG(platform)] { if {$TRG(config)!="" && ![regexp "\\y$b\\y" $TRG(config)]} continue if {[regexp "\\y$b\\y" $TRG(omitconfig)]} continue - lappend config_set $b + set bld [add_build_job $b $TRG(testfixture)] + foreach c [trd_configs $TRG(platform) $b] { + add_tcl_jobs $bld $c $patternlist SHELL + } + + if {$patternlist==""} { + foreach e [trd_extras $TRG(platform) $b] { + if {$e=="fuzztest"} { + add_fuzztest_jobs $b + } else { + add_make_job $bld $e + } + } + } + + if {[trdb one "SELECT EXISTS(SELECT 1 + FROM jobs WHERE depid='SHELL')"]} { + set sbld [add_shell_build_job $b [lindex $bld 1] [lindex $bld 0]] + set sbldid [lindex $sbld 0] + trdb eval { + UPDATE jobs SET depid=$sbldid WHERE depid='SHELL' + } + } } - add_devtest_jobs $config_set [lrange $patternlist 1 end] } list { @@ -1448,7 +1469,14 @@ proc explain_layer {indent depid} { puts "${indent}$displayname in $dirname" explain_layer "${indent} " $jobid } elseif {$showtests} { - puts "${indent}[lindex $displayname end]" + set tail [lindex $displayname end] + set e1 [lindex $displayname 1] + if {[string match config=* $e1]} { + set cfg [string range $e1 7 end] + puts "${indent}($cfg) $tail" + } else { + puts "${indent}$tail" + } } } } From af43eddc232e2da4ab5c4d6f9fa084de1cd8fb8c Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 28 Aug 2024 09:31:17 +0000 Subject: [PATCH 87/99] Reenable SQLITE_OMIT_VIRTUALTABLE on the Device-One alignment for testrunner. FossilOrigin-Name: 8f2cb357634ec0b5aef14f3d967e76db236f8899f3201efc80c264f548cc1b0b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner_data.tcl | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index de8374a5ce..a21b754312 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sthe\sway\s"release"\stests\swere\sconstructed\sin\s[d03d35eebaf82709]\nwere\sincorrect.\s\sFixed\shere.\s\sAlso\senhance\s--explain\sto\schose\spermutation\nconfigurations\sand\sso\sthat\sPATTERNS\son\sthe\sthe\scommand\sline\scan\smatch\nagainst\spermutation\sconfigurations. -D 2024-08-27T22:28:03.935 +C Reenable\sSQLITE_OMIT_VIRTUALTABLE\son\sthe\sDevice-One\salignment\sfor\stestrunner. +D 2024-08-28T09:31:17.887 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1715,7 +1715,7 @@ F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 F test/testrunner.tcl de5deab405d25cdb0f85c4b75cc63dc1fa0571b099621b585f083647d7f6d75a -F test/testrunner_data.tcl 6a0facfbff26f2f8573ebaeb209426bbb59809d5c0f6925f983bd030ab5248cd +F test/testrunner_data.tcl f1cbff53fe42087cac3d43ca02f9574bd212c842307442e2b6fff2183f5ccbfe F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 26372762ab1451eb95d29367f5b2534bde616a37e5e757e72ba2cfe42571d180 -R 66a023125402741fa708637568f51009 +P 1f962ffc9e956a9d6311ed9c02fea13c0fe056283ee2c650a2312a258808b3b3 +R 8e623cc4a65f73a6029848f64a42525f U drh -Z f1950a098efb328b04fe6e5b9690c497 +Z 1895efedce008ae727a6effa6ee80029 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index df0f6a1846..59d2cc678e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f962ffc9e956a9d6311ed9c02fea13c0fe056283ee2c650a2312a258808b3b3 +8f2cb357634ec0b5aef14f3d967e76db236f8899f3201efc80c264f548cc1b0b diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index 48484a3e24..71a42ab404 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -218,7 +218,7 @@ namespace eval trd { -DSQLITE_MAX_PAGE_SIZE=4096 -DSQLITE_OMIT_LOAD_EXTENSION=1 -DSQLITE_OMIT_PROGRESS_CALLBACK=1 - -DSQLITE_OMIT_VIRTUALTABLEx=1 + -DSQLITE_OMIT_VIRTUALTABLE=1 -DSQLITE_ENABLE_HIDDEN_COLUMNS -DSQLITE_TEMP_STORE=3 } From a88d61824dfa5d896cf13eed7c984a09d1d5bd0c Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 28 Aug 2024 09:47:29 +0000 Subject: [PATCH 88/99] Disable shell test cases that require virtual tables when testfixture is built using SQLITE_OMIT_VIRTUALTABLE. FossilOrigin-Name: 8c73d54fd1e250fcd7f30741cfbd169af9aaecc2096c0c8a9486abaa064d69af --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/shell2.test | 4 ++++ test/shell5.test | 8 ++++++++ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a21b754312..60890f8056 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reenable\sSQLITE_OMIT_VIRTUALTABLE\son\sthe\sDevice-One\salignment\sfor\stestrunner. -D 2024-08-28T09:31:17.887 +C Disable\sshell\stest\scases\sthat\srequire\svirtual\stables\swhen\stestfixture\sis\nbuilt\susing\sSQLITE_OMIT_VIRTUALTABLE. +D 2024-08-28T09:47:29.176 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1626,10 +1626,10 @@ F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8e F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test 490bf9d0c7c9564fea318c46d49369f4690b825b584c9a544dbdccf61bc0babc -F test/shell2.test 56da24128304c9ab67da2964cc80beff7b35761c446ec6e6e98bff2775b15026 +F test/shell2.test 01a01f76ed98088ce598794fbf5b359e148271541a8ddbf79d21cc353cc67a24 F test/shell3.test db1953a8e59d08e9240b7cc5948878e184f7eb2623591587f8fd1f1a5bd536d8 F test/shell4.test 522fdc628c55eff697b061504fb0a9e4e6dfc5d9087a633ab0f3dd11bcc4f807 -F test/shell5.test bafa4c0b67b7a8027e729970a625c9225cb7ef854acc4e52624c45074faaaddf +F test/shell5.test 0e5f8ce08206b9998a778cfe1989e20e47839153c05af2da29198150172e22fc F test/shell6.test e3b883b61d4916b6906678a35f9d19054861123ad91b856461e0a456273bdbb8 F test/shell7.test 43fd8e511c533bab5232e95c7b4be93b243451709e89582600d4b6e67693d5c3 F test/shell8.test aea51ecbcd4494c746b096aeff51d841d04d5f0dc4b62eb42427f16109b87acd @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1f962ffc9e956a9d6311ed9c02fea13c0fe056283ee2c650a2312a258808b3b3 -R 8e623cc4a65f73a6029848f64a42525f +P 8f2cb357634ec0b5aef14f3d967e76db236f8899f3201efc80c264f548cc1b0b +R 6d8d08b44812eef81a87de61d7679ec8 U drh -Z 1895efedce008ae727a6effa6ee80029 +Z 9d65cc1858f9f46a1f30863c1abce3fa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 59d2cc678e..fb2482f0ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f2cb357634ec0b5aef14f3d967e76db236f8899f3201efc80c264f548cc1b0b +8c73d54fd1e250fcd7f30741cfbd169af9aaecc2096c0c8a9486abaa064d69af diff --git a/test/shell2.test b/test/shell2.test index c6c27d2165..ee5ae4bdd9 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -217,6 +217,7 @@ do_test shell2-1.4.9 { done 2}} +ifcapable vtab { # Verify that generate_series stays sane near 64-bit range boundaries. # See overflow report at https://sqlite.org/forum/forumpost/5d34ce5280 do_test shell2-1.4.10 { @@ -247,7 +248,9 @@ do_test shell2-1.4.10 { 0 1 2}} +} ;# ifcapable vtab +ifcapable vtab { # Bug discovered while messing around, .import hangs with # bit 7 set in column separator. do_test shell2-1.4.11 { @@ -262,6 +265,7 @@ do_test shell2-1.4.11 { .import dummy.csv t SELECT count(*) FROM t;}]] } {0 1} +} ;# ifcapable vtab # Bug from forum post 7cbe081746dd3803 # Keywords as column names were producing an error message. diff --git a/test/shell5.test b/test/shell5.test index 31d5449fdc..8eb905974b 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -84,6 +84,14 @@ do_test shell5-1.4.1 { .import FOO t1}] } {1 {Error: cannot open "FOO"}} +# the remainder of these test cases require virtual tables. +# +ifcapable !vtab { + puts "Skipping subsequent tests due to SQLITE_OMIT_VIRTUALTABLE" + finish_test + return +} + # empty import file do_test shell5-1.4.2 { forcedelete shell5.csv From 4a90f81a6d17af33dc689e4d32ee0091620b8cd9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 28 Aug 2024 10:25:44 +0000 Subject: [PATCH 89/99] In the testrunner status display, limit the number of reported failed jobs to avoid overflowing the terminal when there are many failures. FossilOrigin-Name: ffeaa4d5d73871cbdf1ef70b9845d921ebdb96e964d232661a5048cab7d744ed --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 25 +++++++++++++++++++++---- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 60890f8056..b36a4ba20f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sshell\stest\scases\sthat\srequire\svirtual\stables\swhen\stestfixture\sis\nbuilt\susing\sSQLITE_OMIT_VIRTUALTABLE. -D 2024-08-28T09:47:29.176 +C In\sthe\stestrunner\sstatus\sdisplay,\slimit\sthe\snumber\sof\sreported\sfailed\sjobs\nto\savoid\soverflowing\sthe\sterminal\swhen\sthere\sare\smany\sfailures. +D 2024-08-28T10:25:44.140 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1714,7 +1714,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl de5deab405d25cdb0f85c4b75cc63dc1fa0571b099621b585f083647d7f6d75a +F test/testrunner.tcl c3f6e70f02bc0257d2d66ac1fff7d26d76b1c4fcdd5e4b4d90faf2a126623587 F test/testrunner_data.tcl f1cbff53fe42087cac3d43ca02f9574bd212c842307442e2b6fff2183f5ccbfe F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8f2cb357634ec0b5aef14f3d967e76db236f8899f3201efc80c264f548cc1b0b -R 6d8d08b44812eef81a87de61d7679ec8 +P 8c73d54fd1e250fcd7f30741cfbd169af9aaecc2096c0c8a9486abaa064d69af +R c92552ba898da06e8effb71b87a3f9eb U drh -Z 9d65cc1858f9f46a1f30863c1abce3fa +Z 9fabe008bd6ec45672f53069445b0fd8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fb2482f0ae..c41d9ba427 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c73d54fd1e250fcd7f30741cfbd169af9aaecc2096c0c8a9486abaa064d69af +ffeaa4d5d73871cbdf1ef70b9845d921ebdb96e964d232661a5048cab7d744ed diff --git a/test/testrunner.tcl b/test/testrunner.tcl index fcb46a9214..a97983fac5 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -480,20 +480,37 @@ proc show_status {db cls} { $ne errors, $nt tests"] set srcdir [file dirname [file dirname $TRG(info_script)]] + set nrun 0 if {$S(running)>0} { - puts [format %-79s "Running:"] + puts [format %-79s "$S(running) Running:"] $db eval { SELECT * FROM jobs WHERE state='running' ORDER BY starttime } job { + incr nrun display_job [array get job] $now } } if {$S(failed)>0} { - puts [format %-79s "Failures:"] + puts [format %-79s "$S(failed) Failed:"] + set nfail 0 + + # $mxtoshow tries to limit the number of "Failures:" reported so that + # the status display does not overflow a 24-line terminal. But it will + # always show at least the most recent 4 failures, even if an overflow + # is needed. + set mxtoshow [expr {16-$nrun}] + if {$mxtoshow<4} {set mxtoshow 4} + $db eval { - SELECT * FROM jobs WHERE state='failed' ORDER BY starttime + SELECT * FROM jobs WHERE state='failed' ORDER BY endtime DESC } job { - display_job [array get job] + incr nfail + if {$nfail<=$mxtoshow} { + display_job [array get job] + } + } + if {$nfail>$mxtoshow} { + puts [format %-79s " ... plus [expr {$nfail-$mxtoshow}] more"] } set nOmit [$db one {SELECT count(*) FROM jobs WHERE state='omit'}] if {$nOmit} { From 242e06497855d5d4e3bf7376ac10bda024d3db47 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 28 Aug 2024 10:36:18 +0000 Subject: [PATCH 90/99] Only limit the number of failures reported by testrunner status if using VT100 cursor movement. For a straight-up "status" command, show everything. FossilOrigin-Name: 791237e0f4b67f0197ae11e966554edeade0c443289d2ade00470d23bf5e8ec8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 10 +++++++--- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b36a4ba20f..5f2f3faa01 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\stestrunner\sstatus\sdisplay,\slimit\sthe\snumber\sof\sreported\sfailed\sjobs\nto\savoid\soverflowing\sthe\sterminal\swhen\sthere\sare\smany\sfailures. -D 2024-08-28T10:25:44.140 +C Only\slimit\sthe\snumber\sof\sfailures\sreported\sby\stestrunner\sstatus\sif\susing\nVT100\scursor\smovement.\s\sFor\sa\sstraight-up\s"status"\scommand,\sshow\severything. +D 2024-08-28T10:36:18.262 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1714,7 +1714,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl c3f6e70f02bc0257d2d66ac1fff7d26d76b1c4fcdd5e4b4d90faf2a126623587 +F test/testrunner.tcl cfe0f1dd8ce1da41ebc2238eabd2f9562e48162505eb6c4495eb9221e00a0c93 F test/testrunner_data.tcl f1cbff53fe42087cac3d43ca02f9574bd212c842307442e2b6fff2183f5ccbfe F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8c73d54fd1e250fcd7f30741cfbd169af9aaecc2096c0c8a9486abaa064d69af -R c92552ba898da06e8effb71b87a3f9eb +P ffeaa4d5d73871cbdf1ef70b9845d921ebdb96e964d232661a5048cab7d744ed +R 8085922607334118c1b73e04568394c4 U drh -Z 9fabe008bd6ec45672f53069445b0fd8 +Z 328eda1f95b36f3977ef4f1d3bbc3083 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c41d9ba427..6bcef0ddb0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffeaa4d5d73871cbdf1ef70b9845d921ebdb96e964d232661a5048cab7d744ed +791237e0f4b67f0197ae11e966554edeade0c443289d2ade00470d23bf5e8ec8 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index a97983fac5..1744af2ea7 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -497,9 +497,13 @@ proc show_status {db cls} { # $mxtoshow tries to limit the number of "Failures:" reported so that # the status display does not overflow a 24-line terminal. But it will # always show at least the most recent 4 failures, even if an overflow - # is needed. - set mxtoshow [expr {16-$nrun}] - if {$mxtoshow<4} {set mxtoshow 4} + # is needed. But, do not limit the length of the output of $cls is false. + if {$cls} { + set mxtoshow [expr {16-$nrun}] + if {$mxtoshow<4} {set mxtoshow 4} + } else { + set mxtoshow 9999999 + } $db eval { SELECT * FROM jobs WHERE state='failed' ORDER BY endtime DESC From 4f2f91bf08cb47ae7ac492f92b0cd2ffb894f3f1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 28 Aug 2024 11:03:58 +0000 Subject: [PATCH 91/99] Further improvements to the status display in order to maximum the amount of information shown on the limited screen realestate available when running via "watch" or in continuous monitoring mode. All details are still shown when doing a stand-alone "status". FossilOrigin-Name: 557a2c65f5ae1d8cb1858c17da0fe64fbfba83d787cfb7f2f21582960a8b9eac --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 46 +++++++++++++++++++-------------------------- 3 files changed, 26 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 5f2f3faa01..b367857848 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\slimit\sthe\snumber\sof\sfailures\sreported\sby\stestrunner\sstatus\sif\susing\nVT100\scursor\smovement.\s\sFor\sa\sstraight-up\s"status"\scommand,\sshow\severything. -D 2024-08-28T10:36:18.262 +C Further\simprovements\sto\sthe\sstatus\sdisplay\sin\sorder\sto\smaximum\sthe\samount\sof\ninformation\sshown\son\sthe\slimited\sscreen\srealestate\savailable\swhen\srunning\nvia\s"watch"\sor\sin\scontinuous\smonitoring\smode.\s\sAll\sdetails\sare\sstill\sshown\nwhen\sdoing\sa\sstand-alone\s"status". +D 2024-08-28T11:03:58.055 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1714,7 +1714,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl cfe0f1dd8ce1da41ebc2238eabd2f9562e48162505eb6c4495eb9221e00a0c93 +F test/testrunner.tcl 2345d50bca7a48fa3454cf088a2ec6978d8a0042feadc8e7336ac5d0dfe53b07 F test/testrunner_data.tcl f1cbff53fe42087cac3d43ca02f9574bd212c842307442e2b6fff2183f5ccbfe F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ffeaa4d5d73871cbdf1ef70b9845d921ebdb96e964d232661a5048cab7d744ed -R 8085922607334118c1b73e04568394c4 +P 791237e0f4b67f0197ae11e966554edeade0c443289d2ade00470d23bf5e8ec8 +R d373eef32be36327da73bfcab1a028e6 U drh -Z 328eda1f95b36f3977ef4f1d3bbc3083 +Z 38527615900da2d45d6401569f849a97 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6bcef0ddb0..7e1aedcbf9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -791237e0f4b67f0197ae11e966554edeade0c443289d2ade00470d23bf5e8ec8 +557a2c65f5ae1d8cb1858c17da0fe64fbfba83d787cfb7f2f21582960a8b9eac diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 1744af2ea7..5bddeee610 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -470,19 +470,15 @@ proc show_status {db cls} { # overwrite. puts -nonewline "\033\[H" flush stdout - set clreol "\033\[K" - } else { - set clreol "" } - puts [format %-79.79s "Command: \[testrunner.tcl$cmdline\]$clreol"] - puts [format %-79.79s "Jobs: $nJob max $S(running) active"] - puts [format %-79.79s "Summary: [elapsetime $tm], $fin/$total tasks,\ + puts [format %-79.79s "Command: \[testrunner.tcl$cmdline\]"] + puts [format %-79.79s "Summary: [elapsetime $tm], $fin/$total jobs,\ $ne errors, $nt tests"] set srcdir [file dirname [file dirname $TRG(info_script)]] set nrun 0 + puts [format %-79s "Running: $S(running) (max: $nJob)"] if {$S(running)>0} { - puts [format %-79s "$S(running) Running:"] $db eval { SELECT * FROM jobs WHERE state='running' ORDER BY starttime } job { @@ -491,34 +487,30 @@ proc show_status {db cls} { } } if {$S(failed)>0} { - puts [format %-79s "$S(failed) Failed:"] - set nfail 0 - - # $mxtoshow tries to limit the number of "Failures:" reported so that - # the status display does not overflow a 24-line terminal. But it will + # $toshow is the number of failures to report. In $cls mode, + # status tries to limit the number of failure reported so that + # the status display does not overflow a 24-line terminal. It will # always show at least the most recent 4 failures, even if an overflow - # is needed. But, do not limit the length of the output of $cls is false. - if {$cls} { - set mxtoshow [expr {16-$nrun}] - if {$mxtoshow<4} {set mxtoshow 4} + # is needed. No limit is imposed for a status within $cls. + # + if {$cls && $S(failed)>18-$S(running)} { + set toshow [expr {18-$S(running)}] + if {$toshow<4} {set toshow 4} + set shown " (must recent $toshow shown)" } else { - set mxtoshow 9999999 + set toshow $S(failed) + set shown "" } - + puts [format %-79s "Failed: $S(failed) $shown"] $db eval { - SELECT * FROM jobs WHERE state='failed' ORDER BY endtime DESC + SELECT * FROM jobs WHERE state='failed' + ORDER BY endtime DESC LIMIT $toshow } job { - incr nfail - if {$nfail<=$mxtoshow} { - display_job [array get job] - } - } - if {$nfail>$mxtoshow} { - puts [format %-79s " ... plus [expr {$nfail-$mxtoshow}] more"] + display_job [array get job] } set nOmit [$db one {SELECT count(*) FROM jobs WHERE state='omit'}] if {$nOmit} { - puts [format %-79s "$nOmit jobs omitted due to failures$clreol"] + puts [format %-79s " ... $nOmit jobs omitted due to failures"] } } if {$cls} { From d6b56d65a8c330c7310611f78d281cdc050e654b Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 28 Aug 2024 12:10:51 +0000 Subject: [PATCH 92/99] Update the "make test" target so that it is really just "mdevtest" with "srctree-check" and "sourcetest". FossilOrigin-Name: 7420b13238da4e6bfb1589eaf3cfd842fe3c4b0f3b6a3b49c24cd70589bf4809 --- Makefile.in | 14 ++++++-------- Makefile.msc | 13 ++++++++++--- main.mk | 12 +++++++++--- manifest | 18 +++++++++--------- manifest.uuid | 2 +- tool/mkctimec.tcl | 1 + 6 files changed, 36 insertions(+), 24 deletions(-) diff --git a/Makefile.in b/Makefile.in index b8149f846f..d17d6830d0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1381,18 +1381,16 @@ srctree-check: $(TOP)/tool/srctree-check.tcl # Testing for a release # -releasetest: srctree-check testfixture$(TEXE) - ./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS) +releasetest: srctree-check has_tclsh85 + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS) # Minimal testing that runs in less than 3 minutes # quicktest: ./testfixture$(TEXE) ./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS) -# This is the common case. Run many tests that do not take too long, -# including fuzzcheck, sqlite3_analyzer, and sqldiff tests. -# -test: srctree-check fuzztest sourcetest $(TESTPROGS) tcltest +test: srctree-check sourcetest + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS) # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. @@ -1407,8 +1405,8 @@ valgrindtest: $(TESTPROGS) valgrindfuzz smoketest: $(TESTPROGS) fuzzcheck$(TEXE) ./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS) -shelltest: $(TESTPROGS) - ./testfixture$(TEXT) $(TOP)/test/permutations.test shell +shelltest: + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl release shell sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85 $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c diff --git a/Makefile.msc b/Makefile.msc index e31aee0899..e4420b1dea 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2544,7 +2544,8 @@ quicktest: testfixture.exe sourcetest # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # -test: $(TESTPROGS) sourcetest fuzztest tcltest +test: srctree-check sourcetest + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest # The veryquick.test TCL tests. # @@ -2565,10 +2566,16 @@ devtest: testfixture.exe fuzztest testrunner mdevtest: $(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest +# Validate that various generated files in the source tree +# are up-to-date. +# +srctree-check: $(TOP)\tool\srctree-check.tcl + $(TCLSH_CMD) $(TOP)\tool\srctree-check.tcl + # Testing for a release # -releasetest: testfixture.exe - testfixture.exe $(TOP)\test\testrunner.tcl release +releasetest: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl release smoketest: $(TESTPROGS) diff --git a/main.mk b/main.mk index baf8260417..ea6b83d059 100644 --- a/main.mk +++ b/main.mk @@ -971,9 +971,15 @@ mdevtest: quicktest: ./testfixture$(EXE) ./testfixture$(EXE) $(TOP)/test/extraquick.test $(TESTOPTS) -# The default test case. Runs most of the faster standard TCL tests, -# and fuzz tests, and sqlite3_analyzer and sqldiff tests. -test: fuzztest sourcetest $(TESTPROGS) tcltest +# Validate that various generated files in the source tree +# are up-to-date. +# +srctree-check: $(TOP)/tool/srctree-check.tcl + tclsh $(TOP)/tool/srctree-check.tcl + +# The default test case. +test: srctree-check sourcetest + tclsh $(TOP)/test/testrunner.tcl mdevtest # Run a test using valgrind. This can take a really long time diff --git a/manifest b/manifest index b367857848..b4f5e7e97f 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Further\simprovements\sto\sthe\sstatus\sdisplay\sin\sorder\sto\smaximum\sthe\samount\sof\ninformation\sshown\son\sthe\slimited\sscreen\srealestate\savailable\swhen\srunning\nvia\s"watch"\sor\sin\scontinuous\smonitoring\smode.\s\sAll\sdetails\sare\sstill\sshown\nwhen\sdoing\sa\sstand-alone\s"status". -D 2024-08-28T11:03:58.055 +C Update\sthe\s"make\stest"\starget\sso\sthat\sit\sis\sreally\sjust\s"mdevtest"\swith\n"srctree-check"\sand\s"sourcetest". +D 2024-08-28T12:10:51.091 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 101cb90f75c16ca3f8e7088364322441f9b01d6fdcef4a9c7f5fdcf78a3601a4 +F Makefile.in d51de7cfc0500e54b7a8961218ce9d7fa115610f5353f98fd9b407a1a69dfdcc F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 6c3fe8b6ce60e73f34a148c957d78b4648745c8d30e792423aa1a8d8bf12d065 +F Makefile.msc fa422ec289537f61c5fe660e85ff4cab9389c3095a9071b5a0487aa521e657ba F README.md 3d47ef5758e2de29d6f4e1aca714d51424baba86c8b561fef330f6ef2dc9282c F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -685,7 +685,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 9541ffdce424ddddb463e2480c3f1cb4067bbd3a6d2c84b3f083cb128a2fa721 +F main.mk 7a70a20e224bf9cdf518cafe7738000ef7b2aec7b7de19d3e49700301d6ff130 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2131,7 +2131,7 @@ F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 F tool/mkautoconfamal.sh cbdcf993fa83dccbef7fb77b39cdeb31ef9f77d9d88c9e343b58d35ca3898a6a F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x -F tool/mkctimec.tcl 060e9785e9503bf51f8b1b11b542bdeef90fd0ceb0738154f6762acec0c61e5f x +F tool/mkctimec.tcl 48ca8eefa9e615cb9057ce6485b9c9ae5801381f24690d7d60b3b2dc8e6b7457 x F tool/mkkeywordhash.c b9faa0ae7e14e4dbbcd951cddd786bf46b8a65bb07b129ba8c0cfade723aaffd F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 791237e0f4b67f0197ae11e966554edeade0c443289d2ade00470d23bf5e8ec8 -R d373eef32be36327da73bfcab1a028e6 +P 557a2c65f5ae1d8cb1858c17da0fe64fbfba83d787cfb7f2f21582960a8b9eac +R 810ec2efa964f89a1c1304eac241e2a3 U drh -Z 38527615900da2d45d6401569f849a97 +Z a85e1bb31ced99882e1388879ef7a06f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7e1aedcbf9..4a6b864348 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -557a2c65f5ae1d8cb1858c17da0fe64fbfba83d787cfb7f2f21582960a8b9eac +7420b13238da4e6bfb1589eaf3cfd842fe3c4b0f3b6a3b49c24cd70589bf4809 diff --git a/tool/mkctimec.tcl b/tool/mkctimec.tcl index 135164e3d1..7ca664a37e 100755 --- a/tool/mkctimec.tcl +++ b/tool/mkctimec.tcl @@ -448,6 +448,7 @@ if {[catch {set cfd [open $destfile w]}]!=0} { puts stderr "File '$destfile' unwritable." exit 1; } +fconfigure $cfd -translation binary puts $cfd $::headWarning; puts $cfd $::headCode; From 3341f5ce85e9ad392c9828105e6d5a5cf5afb6fc Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 28 Aug 2024 13:34:20 +0000 Subject: [PATCH 93/99] Minor wording change on testrunner output. FossilOrigin-Name: 115e68597957a9ee80de7f554ee3768c992051687c8f07eaf3a9fcaab7efd44e --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b4f5e7e97f..cf14534781 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\s"make\stest"\starget\sso\sthat\sit\sis\sreally\sjust\s"mdevtest"\swith\n"srctree-check"\sand\s"sourcetest". -D 2024-08-28T12:10:51.091 +C Minor\swording\schange\son\stestrunner\soutput. +D 2024-08-28T13:34:20.794 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1714,7 +1714,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl 2345d50bca7a48fa3454cf088a2ec6978d8a0042feadc8e7336ac5d0dfe53b07 +F test/testrunner.tcl ff5f42f683a9c868fe01a855d81b9f08e1afb031edc5c340e5cf0fe5deaa0041 F test/testrunner_data.tcl f1cbff53fe42087cac3d43ca02f9574bd212c842307442e2b6fff2183f5ccbfe F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 557a2c65f5ae1d8cb1858c17da0fe64fbfba83d787cfb7f2f21582960a8b9eac -R 810ec2efa964f89a1c1304eac241e2a3 +P 7420b13238da4e6bfb1589eaf3cfd842fe3c4b0f3b6a3b49c24cd70589bf4809 +R 203b7adbe6856a41e47a582ad4a2ff84 U drh -Z a85e1bb31ced99882e1388879ef7a06f +Z ec83570cfd83b688d421f55742cc81eb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4a6b864348..507c4e1143 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7420b13238da4e6bfb1589eaf3cfd842fe3c4b0f3b6a3b49c24cd70589bf4809 +115e68597957a9ee80de7f554ee3768c992051687c8f07eaf3a9fcaab7efd44e diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 5bddeee610..58c718fe47 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1504,7 +1504,7 @@ if {$TRG(explain)} { explain_tests } else { if {$TRG(nJob)>1} { - puts "splitting work across $TRG(nJob) jobs" + puts "splitting work across $TRG(nJob) cores" } puts "built testset in [expr $tm/1000]ms.." handle_buildonly From ba5994b1818fea883217f0a930b0c819d0cfbed8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 28 Aug 2024 13:51:51 +0000 Subject: [PATCH 94/99] Maybe [7420b13238da4e6b] was not such a good idea. Restore "make test" so that it works as before - running a single-threaded test over the source tree with options specified by ./configure and similar. Instead, change the "make devtest" target so that runs the automated multi-threaded tests on a standard configuration. FossilOrigin-Name: dc9fee980d4f4b5a569628702e52cef9e5d8e6c69c2b84546dce47e3e514775e --- Makefile.in | 18 ++++++++++++++---- Makefile.msc | 17 +++++++++++++---- main.mk | 20 ++++++++++++++------ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 50 insertions(+), 23 deletions(-) diff --git a/Makefile.in b/Makefile.in index d17d6830d0..6f41c9bf81 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1363,9 +1363,14 @@ tcltest: ./testfixture$(TEXE) testrunner: testfixture$(TEXE) ./testfixture$(TEXE) $(TOP)/test/testrunner.tcl -# Runs both fuzztest and testrunner, consecutively. +# This is the testing target preferred by the core SQLite developers. +# It runs tests under a standard configuration, regardless of how +# ./configure was run. The devs run "make devtest" prior to each +# check-in, at a minimum. Probably other tests too, but at least this +# one. # -devtest: srctree-check testfixture$(TEXE) fuzztest testrunner +devtest: srctree-check sourcetest + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS) mdevtest: srctree-check has_tclsh85 $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS) @@ -1389,8 +1394,13 @@ releasetest: srctree-check has_tclsh85 quicktest: ./testfixture$(TEXE) ./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS) -test: srctree-check sourcetest - $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS) +# Try to run tests on whatever options are specified by the +# ./configure. The developers seldom use this target. Instead +# they use "make devtest" which runs tests on a standard set of +# options regardless of how SQLite is configured. This "test" +# target is provided for legacy only. +# +test: srctree-check fuzztest sourcetest $(TESTPROGS) tcltest # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. diff --git a/Makefile.msc b/Makefile.msc index e4420b1dea..64b63a704e 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2535,6 +2535,13 @@ queryplantest: testfixture.exe shell fuzztest: fuzzcheck.exe .\fuzzcheck.exe $(FUZZDATA) +# Legacy testing target for third-party integrators. The SQLite +# developers seldom use this target themselves. Instead +# they use "nmake /f Makefile.msc devtest" which runs tests on +# a standard set of options +# +test: $(TESTPROGS) sourcetest fuzztest tcltest + # Minimal testing that runs in less than 3 minutes (on a fast machine) # quicktest: testfixture.exe sourcetest @@ -2544,8 +2551,6 @@ quicktest: testfixture.exe sourcetest # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # -test: srctree-check sourcetest - $(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest # The veryquick.test TCL tests. # @@ -2559,9 +2564,13 @@ tcltest: testfixture.exe testrunner: testfixture.exe .\testfixture.exe $(TOP)\test\testrunner.tcl -# Runs both fuzztest and testrunner, consecutively. +# This is the testing target preferred by the core SQLite developers. +# It runs tests under a standard configuration. The devs run +# "nmake /f Makefile.msc devtest" prior to each check-in, at a minimum. +# Probably other tests too, but at least this one. # -devtest: testfixture.exe fuzztest testrunner +devtest: srctree-check sourcetest + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest mdevtest: $(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest diff --git a/main.mk b/main.mk index ea6b83d059..7926dd0501 100644 --- a/main.mk +++ b/main.mk @@ -958,9 +958,14 @@ tcltest: ./testfixture$(EXE) testrunner: testfixture$(EXE) ./testfixture$(EXE) $(TOP)/test/testrunner.tcl -# Runs both fuzztest and testrunner, consecutively. +# This is the testing target preferred by the core SQLite developers. +# It runs tests under a standard configuration, regardless of how +# ./configure was run. The devs run "make devtest" prior to each +# check-in, at a minimum. Probably other tests too, but at least this +# one. # -devtest: testfixture$(EXE) fuzztest testrunner +devtest: srctree-check sourcetest + tclsh $(TOP)/test/testrunner.tcl mdevtest mdevtest: tclsh $(TOP)/test/testrunner.tcl mdevtest @@ -977,10 +982,13 @@ quicktest: ./testfixture$(EXE) srctree-check: $(TOP)/tool/srctree-check.tcl tclsh $(TOP)/tool/srctree-check.tcl -# The default test case. -test: srctree-check sourcetest - tclsh $(TOP)/test/testrunner.tcl mdevtest - +# Try to run tests on whatever options are specified by the +# environment variables. The SQLite developers seldom use this target. +# Instead# they use "make devtest" which runs tests on a standard set of +# options regardless of how SQLite is configured. This "test" +# target is provided for legacy only. +# +test: fuzztest sourcetest $(TESTPROGS) tcltest # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. diff --git a/manifest b/manifest index cf14534781..a66a4cd72d 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Minor\swording\schange\son\stestrunner\soutput. -D 2024-08-28T13:34:20.794 +C Maybe\s[7420b13238da4e6b]\swas\snot\ssuch\sa\sgood\sidea.\s\sRestore\s"make\stest"\sso\nthat\sit\sworks\sas\sbefore\s-\srunning\sa\ssingle-threaded\stest\sover\sthe\ssource\stree\nwith\soptions\sspecified\sby\s./configure\sand\ssimilar.\s\sInstead,\schange\sthe\n"make\sdevtest"\starget\sso\sthat\sruns\sthe\sautomated\smulti-threaded\stests\son\s\na\sstandard\sconfiguration. +D 2024-08-28T13:51:51.631 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in d51de7cfc0500e54b7a8961218ce9d7fa115610f5353f98fd9b407a1a69dfdcc +F Makefile.in 100a8ba2aee96203e57f0dfeb7c75befa7db0d9d17bdb5a7f814eaa60b974e16 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc fa422ec289537f61c5fe660e85ff4cab9389c3095a9071b5a0487aa521e657ba +F Makefile.msc 75946777a32696b6e894e686ba663f9dcdecea4730a827717e1b2be3b262234d F README.md 3d47ef5758e2de29d6f4e1aca714d51424baba86c8b561fef330f6ef2dc9282c F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -685,7 +685,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 7a70a20e224bf9cdf518cafe7738000ef7b2aec7b7de19d3e49700301d6ff130 +F main.mk 70366119ceca019c618ab9acb1b4e8cb1ab5d342a6c4884a9d514f264bc0f2ca F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7420b13238da4e6bfb1589eaf3cfd842fe3c4b0f3b6a3b49c24cd70589bf4809 -R 203b7adbe6856a41e47a582ad4a2ff84 +P 115e68597957a9ee80de7f554ee3768c992051687c8f07eaf3a9fcaab7efd44e +R 4897791cd3fac7134f1184e69605f11d U drh -Z ec83570cfd83b688d421f55742cc81eb +Z f3181ad81fdede715ff6716653ae5e00 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 507c4e1143..402ccd1247 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -115e68597957a9ee80de7f554ee3768c992051687c8f07eaf3a9fcaab7efd44e +dc9fee980d4f4b5a569628702e52cef9e5d8e6c69c2b84546dce47e3e514775e From 8db3fc548881727f5dac450e32222fe0ba116b17 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 28 Aug 2024 13:55:46 +0000 Subject: [PATCH 95/99] Update the README.md file to recommend running "devtest" instead of "mdevtest". FossilOrigin-Name: eb016f66ea2c7ebacf8c57495843db3414602d062e4a430f2cc603c88b10c3f9 --- README.md | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4cddb47752..ca8f5a7bd2 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ For example: ../sqlite/configure ;# Run the configure script make sqlite3 ;# Builds the "sqlite3" command-line tool make sqlite3.c ;# Build the "amalgamation" source file - make mdevtest ;# Run development tests (requires tcl-dev) + make devtest ;# Run development tests (requires tcl-dev) make releasetest ;# Run full release tests (requires tcl-dev) make sqldiff ;# Builds the "sqldiff" command-line tool make sqlite3_analyzer ;# Builds the "sqlite3_analyzer" tool (requires tcl-dev) @@ -156,7 +156,7 @@ Build using Makefile.msc. Example: nmake /f Makefile.msc sqlite3.exe nmake /f Makefile.msc sqlite3.c - nmake /f Makefile.msc mdevtest + nmake /f Makefile.msc devtest nmake /f Makefile.msc releasetest nmake /f Makefile.msc tclextension-install diff --git a/manifest b/manifest index a66a4cd72d..e4d95bda60 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Maybe\s[7420b13238da4e6b]\swas\snot\ssuch\sa\sgood\sidea.\s\sRestore\s"make\stest"\sso\nthat\sit\sworks\sas\sbefore\s-\srunning\sa\ssingle-threaded\stest\sover\sthe\ssource\stree\nwith\soptions\sspecified\sby\s./configure\sand\ssimilar.\s\sInstead,\schange\sthe\n"make\sdevtest"\starget\sso\sthat\sruns\sthe\sautomated\smulti-threaded\stests\son\s\na\sstandard\sconfiguration. -D 2024-08-28T13:51:51.631 +C Update\sthe\sREADME.md\sfile\sto\srecommend\srunning\s"devtest"\sinstead\sof\s"mdevtest". +D 2024-08-28T13:55:46.396 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 100a8ba2aee96203e57f0dfeb7c75befa7db0d9d17bdb5a7f814eaa60b974e16 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc 75946777a32696b6e894e686ba663f9dcdecea4730a827717e1b2be3b262234d -F README.md 3d47ef5758e2de29d6f4e1aca714d51424baba86c8b561fef330f6ef2dc9282c +F README.md 83335813752d41cd445c727b0918597d99e18e7de0e05024536f555a5f34cc56 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 115e68597957a9ee80de7f554ee3768c992051687c8f07eaf3a9fcaab7efd44e -R 4897791cd3fac7134f1184e69605f11d +P dc9fee980d4f4b5a569628702e52cef9e5d8e6c69c2b84546dce47e3e514775e +R 1aff88f40958f7cfa960a88f6231afc4 U drh -Z f3181ad81fdede715ff6716653ae5e00 +Z e14775f88501d5236fe68a1292e9a48b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 402ccd1247..fb81145c12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc9fee980d4f4b5a569628702e52cef9e5d8e6c69c2b84546dce47e3e514775e +eb016f66ea2c7ebacf8c57495843db3414602d062e4a430f2cc603c88b10c3f9 From 50ca8af6f2ff18c1e6e77cd5d0b8c60e26331b4c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 28 Aug 2024 15:54:46 +0000 Subject: [PATCH 96/99] Fix a dropped error code in fts5. FossilOrigin-Name: df55502e4f412e5b1daccf82f11fa4eb932047d9972dcd16e36be00cf09f78e1 --- ext/fts5/fts5_main.c | 8 +++++--- ext/fts5/test/fts5faultI.test | 21 +++++++++++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 565b1ec305..03c1bb83fa 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2067,9 +2067,11 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){ ** Implementation of xBegin() method. */ static int fts5BeginMethod(sqlite3_vtab *pVtab){ - fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0); - fts5NewTransaction((Fts5FullTable*)pVtab); - return SQLITE_OK; + int rc = fts5NewTransaction((Fts5FullTable*)pVtab); + if( rc==SQLITE_OK ){ + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0); + } + return rc; } /* diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test index 63bfdd68ac..e74162ee35 100644 --- a/ext/fts5/test/fts5faultI.test +++ b/ext/fts5/test/fts5faultI.test @@ -256,6 +256,27 @@ do_faultsim_test 10 -faults oom* -prep { faultsim_test_result {0 hello} } +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE f1 USING fts5(content); + CREATE TABLE g1(id, content); + INSERT INTO g1 VALUES(30000, 'a b c'); + INSERT INTO g1 VALUES(40000, 'd e f'); +} + +faultsim_save_and_close + +do_faultsim_test 11 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + INSERT INTO f1(rowid, content) SELECT id, content FROM g1; + } +} -test { + faultsim_test_result {0 {}} +} finish_test diff --git a/manifest b/manifest index e4d95bda60..717ed83794 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sREADME.md\sfile\sto\srecommend\srunning\s"devtest"\sinstead\sof\s"mdevtest". -D 2024-08-28T13:55:46.396 +C Fix\sa\sdropped\serror\scode\sin\sfts5. +D 2024-08-28T15:54:46.677 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81c F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe -F ext/fts5/fts5_main.c bd41fc81751fc7cb666de7dee52cf9bb97c04ce06a8122e7e41bb7527bde54fc +F ext/fts5/fts5_main.c 1fddb53f495425d9314c74b30c5848a9dd254be0e5f445bfe38292d5ab21c288 F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -178,7 +178,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d -F ext/fts5/test/fts5faultI.test e1e8a927122a8c3be8614a59717e838cad0505ba0a6f404983da8430e7aeb14d +F ext/fts5/test/fts5faultI.test ae4b83ac953200bd7b66d53038f7d6a4fc29cd64831b8e1795538babcea7c638 F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc9fee980d4f4b5a569628702e52cef9e5d8e6c69c2b84546dce47e3e514775e -R 1aff88f40958f7cfa960a88f6231afc4 -U drh -Z e14775f88501d5236fe68a1292e9a48b +P eb016f66ea2c7ebacf8c57495843db3414602d062e4a430f2cc603c88b10c3f9 +R 78d4e3d98d20e086c6528776d2d5c051 +U dan +Z 1a3069fd43dc36b1a74436c6f020d339 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fb81145c12..183857bc4a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb016f66ea2c7ebacf8c57495843db3414602d062e4a430f2cc603c88b10c3f9 +df55502e4f412e5b1daccf82f11fa4eb932047d9972dcd16e36be00cf09f78e1 From c1547d1f7acfdf7502f743529c6c9a1b7aefcf60 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 28 Aug 2024 19:13:16 +0000 Subject: [PATCH 97/99] Do not allow sqlite3_blob_open() to work on a any table that contains generated columns, even columns of the table which are not generated themselves, because such columns might be part of the expression of a STORED column. This restriction could be relaxed some, but that would be a lot of code for something that nobody ever uses. FossilOrigin-Name: 6e84947123339f0ea2b7dabc6134a621eed09317f81647f15870ed50a0f7d1bc --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeblob.c | 5 +++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 717ed83794..d7b2d80921 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sdropped\serror\scode\sin\sfts5. -D 2024-08-28T15:54:46.677 +C Do\snot\sallow\ssqlite3_blob_open()\sto\swork\son\sa\sany\stable\sthat\scontains\ngenerated\scolumns,\seven\scolumns\sof\sthe\stable\swhich\sare\snot\sgenerated\sthemselves,\nbecause\ssuch\scolumns\smight\sbe\spart\sof\sthe\sexpression\sof\sa\sSTORED\scolumn.\nThis\srestriction\scould\sbe\srelaxed\ssome,\sbut\sthat\swould\sbe\sa\slot\sof\scode\nfor\ssomething\sthat\snobody\sever\suses. +D 2024-08-28T19:13:16.654 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -841,7 +841,7 @@ F src/vdbe.h c2549a215898a390de6669cfa32adba56f0d7e17ba5a7f7b14506d6fd5f0c36a F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df F src/vdbeaux.c 25d685cafe119ff890c94345e884ea558a6b5d823bfa52ba708eb8ff3c70aa71 -F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5 +F src/vdbeblob.c 255be187436da38b01f276c02e6a08103489bbe2a7c6c21537b7aecbe0e1f797 F src/vdbemem.c 831a244831eaa45335f9ae276b50a7a82ee10d8c46c2c72492d4eb8c98d94d89 F src/vdbesort.c d0a3c7056c081703c8b6d91ad60f17da5e062a5c64bf568ed0fa1b5f4cae311f F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eb016f66ea2c7ebacf8c57495843db3414602d062e4a430f2cc603c88b10c3f9 -R 78d4e3d98d20e086c6528776d2d5c051 -U dan -Z 1a3069fd43dc36b1a74436c6f020d339 +P df55502e4f412e5b1daccf82f11fa4eb932047d9972dcd16e36be00cf09f78e1 +R e1af11959b2bec73273fe58897c14052 +U drh +Z 762ce3d8ddc624dbf22ecdf6a6f25bea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 183857bc4a..aec5cf28d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df55502e4f412e5b1daccf82f11fa4eb932047d9972dcd16e36be00cf09f78e1 +6e84947123339f0ea2b7dabc6134a621eed09317f81647f15870ed50a0f7d1bc diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 522447dbc1..6cb36da37a 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -167,6 +167,11 @@ int sqlite3_blob_open( pTab = 0; sqlite3ErrorMsg(&sParse, "cannot open table without rowid: %s", zTable); } + if( pTab && (pTab->tabFlags&TF_HasGenerated)!=0 ){ + pTab = 0; + sqlite3ErrorMsg(&sParse, "cannot open table with generated columns: %s", + zTable); + } #ifndef SQLITE_OMIT_VIEW if( pTab && IsView(pTab) ){ pTab = 0; From ccd34894e1a9e7895004867c3f3f4d0897b0dc30 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 29 Aug 2024 16:27:57 +0000 Subject: [PATCH 98/99] Fix a case where the BTCF_AtLast flag was being left set incorrectly. This was causing obscure window function queries to fail. FossilOrigin-Name: e042eb024738a83eed92cd2dfac3d2a50d2589715a3d81b1a494564dd7d8e7fa --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 2 +- test/windowE.test | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d7b2d80921..c76c28e768 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\ssqlite3_blob_open()\sto\swork\son\sa\sany\stable\sthat\scontains\ngenerated\scolumns,\seven\scolumns\sof\sthe\stable\swhich\sare\snot\sgenerated\sthemselves,\nbecause\ssuch\scolumns\smight\sbe\spart\sof\sthe\sexpression\sof\sa\sSTORED\scolumn.\nThis\srestriction\scould\sbe\srelaxed\ssome,\sbut\sthat\swould\sbe\sa\slot\sof\scode\nfor\ssomething\sthat\snobody\sever\suses. -D 2024-08-28T19:13:16.654 +C Fix\sa\scase\swhere\sthe\sBTCF_AtLast\sflag\swas\sbeing\sleft\sset\sincorrectly.\sThis\swas\scausing\sobscure\swindow\sfunction\squeries\sto\sfail. +D 2024-08-29T16:27:57.817 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -704,7 +704,7 @@ F src/auth.c 4c1ea890e0069ad73bead5d17a5b12c34cfa4f1a24175c8147ea439b64be271c F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 -F src/btree.c 8b42fc7d9efdb2df05c30e8f91ff6cfbd979724ae24bf90269028468b7a13333 +F src/btree.c 8c5592c618741c5fc9733e7efe3927bfafad3e999d15b0a0f3f1d3f3e17b919e F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 F src/build.c 3a1840d9d171ce2d24f4c1f7acda7266ab796c664290c1acba65ff98ce2bd01e @@ -2070,7 +2070,7 @@ F test/windowA.test 6d63dc1260daa17141a55007600581778523a8b420629f1282d2acfc36af F test/windowB.test aad7c31739999f68a98a813cfd78390918fc70f56d2d925317a1523cab548ecf F test/windowC.test 6fd75f5bb2f1343d34e470e36e68f0ff638d8a42f6aa7d99471261b31a0d42f2 F test/windowD.test 65cf5a765fb8072450e8a0de2979ce7f09a38d87724fe1280c6444073e3da49b -F test/windowE.test c98507e0b0d95980ad25845db758557757be2d7054198ccf522c7d277057a3df +F test/windowE.test d045a5fbaaf50ecac9483e1249dd317ba4f9d189c405a730ba6effdefb87b94f F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387019e0 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P df55502e4f412e5b1daccf82f11fa4eb932047d9972dcd16e36be00cf09f78e1 -R e1af11959b2bec73273fe58897c14052 -U drh -Z 762ce3d8ddc624dbf22ecdf6a6f25bea +P 6e84947123339f0ea2b7dabc6134a621eed09317f81647f15870ed50a0f7d1bc +R 842b0bd8861928ef2186779e003e3d2c +U dan +Z 8acde417655a8746f3e46d7a85e6d010 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aec5cf28d8..53067bc77e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6e84947123339f0ea2b7dabc6134a621eed09317f81647f15870ed50a0f7d1bc +e042eb024738a83eed92cd2dfac3d2a50d2589715a3d81b1a494564dd7d8e7fa diff --git a/src/btree.c b/src/btree.c index c752b0771b..f985ce3405 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5989,7 +5989,7 @@ int sqlite3BtreeIndexMoveto( && indexCellCompare(pCur, 0, pIdxKey, xRecordCompare)<=0 && pIdxKey->errCode==SQLITE_OK ){ - pCur->curFlags &= ~BTCF_ValidOvfl; + pCur->curFlags &= ~(BTCF_ValidOvfl|BTCF_AtLast); if( !pCur->pPage->isInit ){ return SQLITE_CORRUPT_BKPT; } diff --git a/test/windowE.test b/test/windowE.test index 9128468b3c..1cb67f56b0 100644 --- a/test/windowE.test +++ b/test/windowE.test @@ -73,6 +73,38 @@ do_catchsql_test 2.2 { WINDOW w1 AS (PARTITION BY x OVER w1); } {1 {near "OVER": syntax error}} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + BEGIN TRANSACTION; + CREATE TABLE t2(c1 INT, c2 REAL); + INSERT INTO t2 VALUES + (447,0.0), (448,0.0), (449,0.0), (452,0.0), (453,0.0), (454,0.0), (455,0.0), + (456,0.0), (459,0.0), (460,0.0), (462,0.0), (463,0.0), (466,0.0), (467,0.0), + (468,0.0), (469,0.0), (470,0.0), (473,0.0), (474,0.0), (475,0.0), (476,0.0), + (477,0.0), (480,0.0), (481,0.0), (482,0.0), (483,0.0), (484,0.0), (487,0.0), + (488,0.0), (489,0.0), (490,0.0), (491,0.0), (494,0.0), (495,0.0), (496,0.0), + (497,0.0), (498,0.0), (501,0.0), (502,0.0), (503,0.0), (504,0.0), (505,0.0), + (508,0.0), (509,0.0), (510,0.0), (511,0.0), (512,0.0), (515,0.0), (516,0.0), + (517,0.0), (518,0.0), (519,0.0), (522,0.0), (523,0.0), (524,0.0), (525,0.0), + (526,0.0), (529,0.0), (530,0.0), (531,0.0), (532,0.0), (533,0.0), (536,0.0), + (537,1.0), (538,0.0), (539,0.0), (540,0.0), (543,0.0), (544,0.0); + COMMIT; +} + +do_execsql_test 3.1 { + select c1, max(c2) over (order by c1 range 366.0 preceding) from t2; +} { + 447 0.0 448 0.0 449 0.0 452 0.0 453 0.0 454 0.0 455 0.0 456 0.0 459 0.0 + 460 0.0 462 0.0 463 0.0 466 0.0 467 0.0 468 0.0 469 0.0 470 0.0 473 0.0 + 474 0.0 475 0.0 476 0.0 477 0.0 480 0.0 481 0.0 482 0.0 483 0.0 484 0.0 + 487 0.0 488 0.0 489 0.0 490 0.0 491 0.0 494 0.0 495 0.0 496 0.0 497 0.0 + 498 0.0 501 0.0 502 0.0 503 0.0 504 0.0 505 0.0 508 0.0 509 0.0 510 0.0 + 511 0.0 512 0.0 515 0.0 516 0.0 517 0.0 518 0.0 519 0.0 522 0.0 523 0.0 + 524 0.0 525 0.0 526 0.0 529 0.0 530 0.0 531 0.0 532 0.0 533 0.0 536 0.0 + 537 1.0 538 1.0 539 1.0 540 1.0 543 1.0 544 1.0 +} + finish_test From 6a18c9e06a770357a74703f56dc26ab445bd7ccd Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 29 Aug 2024 18:32:56 +0000 Subject: [PATCH 99/99] All makefiles are responsive to OPTIONS=... command-line arguments, and add the RHS value of OPTIONS= to builds and to lemon and other build steps that might respond to -D or -U options. FossilOrigin-Name: 854b3776ee1fcaa5931e3a0ed104978ca350d218e553586d1c40c2420e1be498 --- Makefile.in | 8 ++++++-- Makefile.msc | 9 +++++++++ README.md | 12 ++++++++++++ main.mk | 4 ++++ manifest | 20 ++++++++++---------- manifest.uuid | 2 +- 6 files changed, 42 insertions(+), 13 deletions(-) diff --git a/Makefile.in b/Makefile.in index 6f41c9bf81..d230a40acb 100644 --- a/Makefile.in +++ b/Makefile.in @@ -83,9 +83,13 @@ TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@ # based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). # The same set of OMIT and ENABLE flags should be passed to the # LEMON parser generator and the mkkeywordhash tool as well. -OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ +# +# Add OPTIONS=... on the command line to append additional options +# to the OPT_FEATURE_FLAGS. +# +OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ $(OPTIONS) -TCC += $(OPT_FEATURE_FLAGS) +TCC += $(OPT_FEATURE_FLAGS) # Add in any optional parameters specified on the make commane line # ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". diff --git a/Makefile.msc b/Makefile.msc index 64b63a704e..bf707edfd6 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -379,6 +379,7 @@ SQLITE_TCL_DEP = # the Windows platform. # !IFNDEF OPT_FEATURE_FLAGS +OPT_FEATURE_FLAGS = $(OPT_XTRA) !IF $(MINIMAL_AMALGAMATION)==0 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1 @@ -392,6 +393,14 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF +# Additional feature-options above and beyond what are normally used can be +# be added using OPTIONS=.... on the command-line. These values are +# appended to the OPT_FEATURE_FLAGS variable. +# +!IFDEF OPTIONS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) $(OPTIONS) +!ENDIF + # Should the session extension be enabled? If so, add compilation options # to enable it. # diff --git a/README.md b/README.md index ca8f5a7bd2..c208f11ea3 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,13 @@ Almost all makefile targets require a "tclsh" TCL interpreter version 8.6 or later. The targets marked with "(requires tcl-dev)" also require the TCL development libraries. +On "make" command-lines, one can add "OPTIONS=..." to specify additional +compile-time options over and above those set by ./configure. For example, +to compile with the SQLITE_OMIT_DEPRECATED compile-time option, one could say: + + ./configure --enable-all + make OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3 + The configure script uses autoconf 2.61 and libtool. If the configure script does not work out for you, there is a generic makefile named "Makefile.linux-gcc" in the top directory of the source tree that you @@ -163,6 +170,11 @@ Build using Makefile.msc. Example: There are many other makefile targets. See comments in Makefile.msc for details. +As with the unix Makefile, the OPTIONS=... argument can be passed on the nmake +command-line to enable new compile-time options. For example: + + nmake /f Makefile.msc OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3.exe + ## Source Tree Map * **src/** - This directory contains the primary source code for the diff --git a/main.mk b/main.mk index 7926dd0501..0c86c77181 100644 --- a/main.mk +++ b/main.mk @@ -42,6 +42,10 @@ # build the SQLite library and testing tools. ################################################################################ +# If OPTIONS... is specified on the command-line, append its value to OPTS +# +OPTS += $(OPTIONS) + # This is how we compile # TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) diff --git a/manifest b/manifest index c76c28e768..1a8810cd58 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Fix\sa\scase\swhere\sthe\sBTCF_AtLast\sflag\swas\sbeing\sleft\sset\sincorrectly.\sThis\swas\scausing\sobscure\swindow\sfunction\squeries\sto\sfail. -D 2024-08-29T16:27:57.817 +C All\smakefiles\sare\sresponsive\sto\sOPTIONS=...\scommand-line\sarguments,\sand\sadd\nthe\sRHS\svalue\sof\sOPTIONS=\sto\sbuilds\sand\sto\slemon\sand\sother\sbuild\ssteps\sthat\nmight\srespond\sto\s-D\sor\s-U\soptions. +D 2024-08-29T18:32:56.249 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 100a8ba2aee96203e57f0dfeb7c75befa7db0d9d17bdb5a7f814eaa60b974e16 +F Makefile.in f246a6d8a8a77c35d50d5494ee9bcd8fb6f90118eff46169427f9f404057b56e F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 75946777a32696b6e894e686ba663f9dcdecea4730a827717e1b2be3b262234d -F README.md 83335813752d41cd445c727b0918597d99e18e7de0e05024536f555a5f34cc56 +F Makefile.msc 47dfd26a401c5a33e237ecfe126c996977f34ac44e44a1a1ebf2204abb89d483 +F README.md 5b678e264236788390d11991f2c0052bd73f19790173883fc56d638bcb849154 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 @@ -685,7 +685,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 70366119ceca019c618ab9acb1b4e8cb1ab5d342a6c4884a9d514f264bc0f2ca +F main.mk 715c833f9733fd10b15d8773c59dd7535067d9d0f1a20073182069538f523bc7 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6e84947123339f0ea2b7dabc6134a621eed09317f81647f15870ed50a0f7d1bc -R 842b0bd8861928ef2186779e003e3d2c -U dan -Z 8acde417655a8746f3e46d7a85e6d010 +P e042eb024738a83eed92cd2dfac3d2a50d2589715a3d81b1a494564dd7d8e7fa +R f410afede94c7e996193f1f14e81b108 +U drh +Z d182fc30f4d0c921d3b91364d1eaed1e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 53067bc77e..21e3bcb9c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e042eb024738a83eed92cd2dfac3d2a50d2589715a3d81b1a494564dd7d8e7fa +854b3776ee1fcaa5931e3a0ed104978ca350d218e553586d1c40c2420e1be498