From 43e862723ec680542ca6f608f9963c0993dd7324 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Apr 2020 15:31:22 +0000 Subject: [PATCH 01/31] When compiling the shell for WinRT, avoid using Win32 APIs that are unavailable. FossilOrigin-Name: 85d3dc8c50d8dbb8eac1956e8976e861d3b671e03355ca9257060fa3dca51cc4 --- ext/misc/fileio.c | 2 ++ manifest | 16 ++++++------- manifest.uuid | 2 +- src/shell.c.in | 57 ++++++++++++++++++++++++++++++++++------------- 4 files changed, 52 insertions(+), 25 deletions(-) diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 1335229f9e..d977d41ddc 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -394,6 +394,7 @@ static int writeFile( if( mtime>=0 ){ #if defined(_WIN32) +#if !SQLITE_OS_WINRT /* Windows */ FILETIME lastAccess; FILETIME lastWrite; @@ -424,6 +425,7 @@ static int writeFile( }else{ return 1; } +#endif #elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ /* Recent unix */ struct timespec times[2]; diff --git a/manifest b/manifest index 43e422bbaa..ce27245686 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Limit\sLIKE/GLOB\spattern\slength\sto\s100\sbytes\s(default\sis\s50K)\swhen\srunning\ndbsql\scases\sin\sthe\sfuzzcheck\sutility. -D 2020-04-07T15:07:11.446 +C When\scompiling\sthe\sshell\sfor\sWinRT,\savoid\susing\sWin32\sAPIs\sthat\sare\sunavailable. +D 2020-04-09T15:31:22.553 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -293,7 +293,7 @@ F ext/misc/dbdata.c e316fba936571584e55abd5b974a32a191727a6b746053a0c9d439bd2cf9 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f -F ext/misc/fileio.c bfa11a207da4eed8e5f84a1e3954608492f25f8850f9f00d0d2076f4648d7608 +F ext/misc/fileio.c 9b69e25da3b51d4a1d905a464ccb96709792ad627a742ba09215bc0d1447e7bd F ext/misc/fossildelta.c 1240b2d3e52eab1d50c160c7fe1902a9bd210e052dc209200a750bbf885402d5 F ext/misc/fuzzer.c eae560134f66333e9e1ca4c8ffea75df42056e2ce8456734565dbe1c2a92bf3d F ext/misc/ieee754.c eaffd9b364d7c8371727e9c43fc8bec38cdacc4d11fc26beffaa3ca05a0ea9d6 @@ -532,7 +532,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 -F src/shell.c.in 759bb4a283651955ff2ddb104541b1805b1fff915017083bdd39975cd4e223aa +F src/shell.c.in cf7d6140c33859a86188aa52093dfa5d4e4d9ce32ecf1d588a127cb0a8f6f96f F src/sqlite.h.in cc7d0949ac32bb68ed97acdb3e7ae91cd413a24d32d6ff049ef8308d620a4367 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee @@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b1eae2686f03a6e20a49ca2b3a654b3019506d4941708ee3919c339cd093a57d -R b12213b0df5dc8fc33fac9cdcf090d4e -U drh -Z 6587ee9cfa3092e47e4718062b1b99b1 +P 10306118e8591e727af477a1a15d136852d21170e645bd0e75f7c88346b037d7 +R 979d0103f7b3acef2369409e370d6fa7 +U mistachkin +Z 484824f16820465db78cbfdb9a5d496b diff --git a/manifest.uuid b/manifest.uuid index 7e3d8e0200..fb32b5bfff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -10306118e8591e727af477a1a15d136852d21170e645bd0e75f7c88346b037d7 \ No newline at end of file +85d3dc8c50d8dbb8eac1956e8976e861d3b671e03355ca9257060fa3dca51cc4 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index a35e863a71..2bc6b35b36 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -17,6 +17,14 @@ #define _CRT_SECURE_NO_WARNINGS #endif +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + /* ** Warning pragmas copied from msvc.h in the core. */ @@ -129,22 +137,26 @@ typedef unsigned char u8; #if defined(_WIN32) || defined(WIN32) -# include -# include -# define isatty(h) _isatty(h) -# ifndef access -# define access(f,m) _access((f),(m)) +# if SQLITE_OS_WINRT +# define SQLITE_OMIT_POPEN 1 +# else +# include +# include +# define isatty(h) _isatty(h) +# ifndef access +# define access(f,m) _access((f),(m)) +# endif +# ifndef unlink +# define unlink _unlink +# endif +# ifndef strdup +# define strdup _strdup +# endif +# undef popen +# define popen _popen +# undef pclose +# define pclose _pclose # endif -# ifndef unlink -# define unlink _unlink -# endif -# ifndef strdup -# define strdup _strdup -# endif -# undef popen -# define popen _popen -# undef pclose -# define pclose _pclose #else /* Make sure isatty() has a prototype. */ extern int isatty(int); @@ -173,6 +185,9 @@ typedef unsigned char u8; #define ToLower(X) (char)tolower((unsigned char)X) #if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT +#include +#endif #include /* string conversion routines only needed on Win32 */ @@ -188,7 +203,7 @@ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); ** rendering quoted strings that contain \n characters). The following ** routines take care of that. */ -#if defined(_WIN32) || defined(WIN32) +#if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT static void setBinaryMode(FILE *file, int isOutput){ if( isOutput ) fflush(file); _setmode(_fileno(file), _O_BINARY); @@ -292,6 +307,7 @@ static int hasTimer(void){ if( getProcessTimesAddr ){ return 1; } else { +#if !SQLITE_OS_WINRT /* GetProcessTimes() isn't supported in WIN95 and some other Windows ** versions. See if the version we are running on has it, and if it ** does, save off a pointer to it and the current process handle. @@ -308,6 +324,7 @@ static int hasTimer(void){ FreeLibrary(hinstLib); } } +#endif } return 0; } @@ -10226,14 +10243,18 @@ static void main_init(ShellState *data) { */ #ifdef _WIN32 static void printBold(const char *zText){ +#if !SQLITE_OS_WINRT HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo; GetConsoleScreenBufferInfo(out, &defaultScreenInfo); SetConsoleTextAttribute(out, FOREGROUND_RED|FOREGROUND_INTENSITY ); +#endif printf("%s", zText); +#if !SQLITE_OS_WINRT SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); +#endif } #else static void printBold(const char *zText){ @@ -10301,7 +10322,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ fgetc(stdin); }else{ #if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT + __debugbreak(); +#else DebugBreak(); +#endif #elif defined(SIGTRAP) raise(SIGTRAP); #endif From e6a85962e754fded78939a826093baaf7869af4d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Apr 2020 15:48:55 +0000 Subject: [PATCH 02/31] Add the UINT collating sequence extension. The implementation is copied out of the "natsort" branch. FossilOrigin-Name: 6f46c6e3e3c471ca864d7596e0211ee90316b784c8fe22c7ae177c9d29731dc7 --- ext/misc/uint.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++++ manifest | 13 +++---- manifest.uuid | 2 +- 3 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 ext/misc/uint.c diff --git a/ext/misc/uint.c b/ext/misc/uint.c new file mode 100644 index 0000000000..12982c2a20 --- /dev/null +++ b/ext/misc/uint.c @@ -0,0 +1,91 @@ +/* +** 2020-04-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements the UINT collating sequence. +** +** UINT works like BINARY for text, except that embedded strings +** of digits compare in numeric order. +** +** * Leading zeros are handled properly, in the sense that +** they do not mess of the maginitude comparison of embedded +** strings of digits. "x00123y" is equal to "x123y". +** +** * Only unsigned integers are recognized. Plus and minus +** signs are ignored. Decimal points and exponential notation +** are ignored. +** +** * Embedded integers can be of arbitrary length. Comparison +** is *not* limited integers that can be expressed as a +** 64-bit machine integer. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/* +** Compare text in lexicographic order, except strings of digits +** compare in numeric order. +*/ +static int uintCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + int i=0, j=0, x; + while( i Date: Tue, 14 Apr 2020 15:53:58 +0000 Subject: [PATCH 03/31] Build the UINT collating sequence extension into the CLI. FossilOrigin-Name: 2b8c6b035a276029850de02651712a5fd69f4dfee45083d24b9d1f998004829b --- Makefile.in | 1 + Makefile.msc | 1 + main.mk | 1 + manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/shell.c.in | 2 ++ 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index e4b824365d..01e3bed6e5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1069,6 +1069,7 @@ SHELL_SRC = \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/completion.c \ $(TOP)/ext/misc/sqlar.c \ + $(TOP)/ext/misc/uint.c \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ diff --git a/Makefile.msc b/Makefile.msc index d0b7860d90..90944e845c 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2182,6 +2182,7 @@ SHELL_SRC = \ $(TOP)\ext\misc\shathree.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\completion.c \ + $(TOP)\ext\misc\uint.c \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\sqlite3expert.h \ $(TOP)\ext\misc\memtrace.c \ diff --git a/main.mk b/main.mk index cbf14f065c..bb51b44575 100644 --- a/main.mk +++ b/main.mk @@ -733,6 +733,7 @@ SHELL_SRC = \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/completion.c \ $(TOP)/ext/misc/sqlar.c \ + $(TOP)/ext/misc/uint.c \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ diff --git a/manifest b/manifest index 07721fa1c5..3f838de2fc 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Add\sthe\sUINT\scollating\ssequence\sextension.\s\sThe\simplementation\sis\scopied\sout\nof\sthe\s"natsort"\sbranch. -D 2020-04-14T15:48:55.103 +C Build\sthe\sUINT\scollating\ssequence\sextension\sinto\sthe\sCLI. +D 2020-04-14T15:53:58.512 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 9dfc7936f675785309b74d09202bb656732325e65df889e5aaa18cc8932e5b0c +F Makefile.in f6ab6e307d31e09d82aa4eca87ccb21ce66f486279e204b54b3149ff0d71616f F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc fab23c6b10cb6f06a7e9c407fc2e35cb184a6d653f1b793bda87fcee2eafa4f6 +F Makefile.msc 46c338b1151fbeed42c82416c6bc4da7ff9a10f851bf379f8592280fd53d6efa F README.md 1514a365ffca3c138e00c5cc839906108a01011a6b082bad19b09781e3aa498a F VERSION 980d78a2ce04a1fd0ebefbaabd665f7f9186563820629ee29c6e350e96f19b52 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -454,7 +454,7 @@ F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 7ce055f3df31a4f7d21e38f493f907c21db1f673863a573e231f55e2ab005023 +F main.mk f3a972451ed68fa1101d81606a2a1b0a2600b85547059fe89ccb2380b983e50c F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -533,7 +533,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 -F src/shell.c.in cf7d6140c33859a86188aa52093dfa5d4e4d9ce32ecf1d588a127cb0a8f6f96f +F src/shell.c.in 792b901ae19c7cf8cbac06b3c58ef860328d0ca6d20149bdc469e5c4fc550a1d F src/sqlite.h.in cc7d0949ac32bb68ed97acdb3e7ae91cd413a24d32d6ff049ef8308d620a4367 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 85d3dc8c50d8dbb8eac1956e8976e861d3b671e03355ca9257060fa3dca51cc4 -R 90abc1adacfe98d7a7e36bcfbb02dfec +P 6f46c6e3e3c471ca864d7596e0211ee90316b784c8fe22c7ae177c9d29731dc7 +R 55c0a7e7e6364ecba4d144377c4424a0 U drh -Z 3c9766781f32af8c6294daca79b779f0 +Z 63a1ce21bf796da3b1d1d32831983a3d diff --git a/manifest.uuid b/manifest.uuid index d6a630a3aa..52c41554f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f46c6e3e3c471ca864d7596e0211ee90316b784c8fe22c7ae177c9d29731dc7 \ No newline at end of file +2b8c6b035a276029850de02651712a5fd69f4dfee45083d24b9d1f998004829b \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 2bc6b35b36..b589cb4238 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1010,6 +1010,7 @@ INCLUDE ../ext/misc/fileio.c INCLUDE ../ext/misc/completion.c INCLUDE ../ext/misc/appendvfs.c INCLUDE ../ext/misc/memtrace.c +INCLUDE ../ext/misc/uint.c #ifdef SQLITE_HAVE_ZLIB INCLUDE ../ext/misc/zipfile.c INCLUDE ../ext/misc/sqlar.c @@ -4252,6 +4253,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); + sqlite3_uint_init(p->db, 0, 0); #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) sqlite3_dbdata_init(p->db, 0, 0); #endif From c39b121a95c10b4ba36cbde89606037aab8dfc9f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Apr 2020 17:39:39 +0000 Subject: [PATCH 04/31] Clarification of the byte-order determination for UTF16 inputs to routines like sqlite3_bind_text16() and sqlite3_result_text16() and others that accept UTF16 input strings. FossilOrigin-Name: a42fdcf54bcbd72a301dad4a040346dc48e67cacab43479ec618f5c32108c55f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 44 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3f838de2fc..4f7d83de2d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Build\sthe\sUINT\scollating\ssequence\sextension\sinto\sthe\sCLI. -D 2020-04-14T15:53:58.512 +C Clarification\sof\sthe\sbyte-order\sdetermination\sfor\sUTF16\sinputs\sto\sroutines\nlike\ssqlite3_bind_text16()\sand\ssqlite3_result_text16()\sand\sothers\sthat\naccept\sUTF16\sinput\sstrings. +D 2020-04-15T17:39:39.862 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -534,7 +534,7 @@ F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 F src/shell.c.in 792b901ae19c7cf8cbac06b3c58ef860328d0ca6d20149bdc469e5c4fc550a1d -F src/sqlite.h.in cc7d0949ac32bb68ed97acdb3e7ae91cd413a24d32d6ff049ef8308d620a4367 +F src/sqlite.h.in 4276f461ed8405630e3089b682dad77ae7a65c345d8daebcee52a13be8cd880c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee F src/sqliteInt.h 0f3848c46310d197246003f052985b72d1cdbfc0b31e069db76cb5231062fa1d @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6f46c6e3e3c471ca864d7596e0211ee90316b784c8fe22c7ae177c9d29731dc7 -R 55c0a7e7e6364ecba4d144377c4424a0 +P 2b8c6b035a276029850de02651712a5fd69f4dfee45083d24b9d1f998004829b +R 7af907123cde70d72ab14d313300d9ef U drh -Z 63a1ce21bf796da3b1d1d32831983a3d +Z e055c082ef6d50bf436bbdcfa90b28f7 diff --git a/manifest.uuid b/manifest.uuid index 52c41554f5..eebee4beb8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b8c6b035a276029850de02651712a5fd69f4dfee45083d24b9d1f998004829b \ No newline at end of file +a42fdcf54bcbd72a301dad4a040346dc48e67cacab43479ec618f5c32108c55f \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 86c3223801..f93fab1c59 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4260,6 +4260,24 @@ typedef struct sqlite3_context sqlite3_context; ** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() ** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter ** is ignored and the end result is the same as sqlite3_bind_null(). +** ^If the third parameter to sqlite3_bind_text() is not NULL, then +** it should be a pointer to well-formed UTF8 text. +** ^If the third parameter to sqlite3_bind_text16() is not NULL, then +** it should be a pointer to well-formed UTF16 text. +** ^If the third parameter to sqlite3_bind_text64() is not NULL, then +** it should be a pointer to a well-formed unicode string that is +** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 +** otherwise. +** +** [[byte-order determination rules]] ^The byte-order of +** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) +** found in first character, which is removed, or in the absence of a BOM +** the byte order is the native byte order of the host +** machine for sqlite3_bind_text16() or the byte order specified in +** the 6th parameter for sqlite3_bind_text64().)^ +** ^If UTF16 input text contains invalid unicode +** characters, then SQLite might change those invalid characters +** into the unicode replacement character: U+FFFD. ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the @@ -4273,7 +4291,7 @@ typedef struct sqlite3_context sqlite3_context; ** or sqlite3_bind_text16() or sqlite3_bind_text64() then ** that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL -** terminated. If any NUL characters occur at byte offsets less than +** terminated. If any NUL characters occurs at byte offsets less than ** the value of the fourth parameter then the resulting string value will ** contain embedded NULs. The result of expressions involving strings ** with embedded NULs is undefined. @@ -5598,8 +5616,9 @@ typedef void (*sqlite3_destructor_type)(void*); ** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() ** as the text of an error message. ^SQLite interprets the error ** message string from sqlite3_result_error() as UTF-8. ^SQLite -** interprets the string from sqlite3_result_error16() as UTF-16 in native -** byte order. ^If the third parameter to sqlite3_result_error() +** interprets the string from sqlite3_result_error16() as UTF-16 using +** the same [byte-order determination rules] as [sqlite3_bind_text16()]. +** ^If the third parameter to sqlite3_result_error() ** or sqlite3_result_error16() is negative then SQLite takes as the error ** message all text up through the first zero character. ** ^If the third parameter to sqlite3_result_error() or @@ -5667,6 +5686,25 @@ typedef void (*sqlite3_destructor_type)(void*); ** then SQLite makes a copy of the result into space obtained ** from [sqlite3_malloc()] before it returns. ** +** ^For the sqlite3_result_text16(), sqlite3_result_text16le(), and +** sqlite3_result_text16be() routines, and for sqlite3_result_text64() +** when the encoding is not UTF8, if the input UTF16 begins with a +** byte-order mark (BOM, U+FEFF) then the BOM is removed from the +** string and the rest of the string is interpreted according to the +** byte-order specified by the BOM. ^The byte-order specified by +** the BOM at the beginning of the text overrides the byte-order +** specified by the interface procedure. ^So, for example, if +** sqlite3_result_text16le() is invoked with text that begins +** with bytes 0xfe, 0xff (a big-endian byte-order mark) then the +** first two bytes of input are skipped and the remaining input +** is interpreted as UTF16BE text. +** +** ^For UTF16 input text to the sqlite3_result_text16(), +** sqlite3_result_text16be(), sqlite3_result_text16le(), and +** sqlite3_result_text64() routines, if the text contains invalid +** UTF16 characters, the invalid characters might be converted +** into the unicode replacement character, U+FFFD. +** ** ^The sqlite3_result_value() interface sets the result of ** the application-defined function to be a copy of the ** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The From 7576a68c8c9281288ab6ddc25f202d0f3c0ee05e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Apr 2020 11:35:27 +0000 Subject: [PATCH 05/31] Improve corruption detection in fts3 shadow tables earlier in order to prevent an assert() from failing. FossilOrigin-Name: a9ec8c8f80a59badabb0afdb4189f0fd2934f936530d4151de395b3a7e7c1f1f --- ext/fts3/fts3_write.c | 1 + manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/fts3corrupt4.test | 13 +++++++++++++ test/fts4aa.test | 4 ++-- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index f5114eca42..9a052a8f26 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1415,6 +1415,7 @@ static int fts3SegReaderNext( */ if( pReader->nDoclist > pReader->nNode-(pReader->aDoclist-pReader->aNode) || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) + || pReader->nDoclist==0 ){ return FTS_CORRUPT_VTAB; } diff --git a/manifest b/manifest index ce27245686..03e7a37f31 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\scompiling\sthe\sshell\sfor\sWinRT,\savoid\susing\sWin32\sAPIs\sthat\sare\sunavailable. -D 2020-04-09T15:31:22.553 +C Improve\scorruption\sdetection\sin\sfts3\sshadow\stables\searlier\sin\sorder\sto\sprevent\san\sassert()\sfrom\sfailing. +D 2020-04-16T11:35:27.558 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c eb5c0184762a310003762db4ebd5ddcce097d9bb08804279ba33a42907f847a6 +F ext/fts3/fts3_write.c d5da5f010b2e2c1523f0e359ec43858bb724f608d3805d0e2a82ca2b466eb22e F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 @@ -945,7 +945,7 @@ F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f F test/fts3corrupt.test 79a32ffdcd5254e2f7fa121d9656e61949ad049c3c6554229911b7ceac37c9c6 F test/fts3corrupt2.test bf55c3fa0b0dc8ea1c0fe5543623bd27714585da6a129038fd6999fe3b0d25f3 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f -F test/fts3corrupt4.test e8ad49403179cbf714b6b669d2e0f9234ae95f4ca258a253b0f29ce28c1b027c +F test/fts3corrupt4.test 6a1331bb51dc27acc063d26e0f1b610863b56a833ee54fa32767d17cf96bda4a F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf F test/fts3d.test 2bd8c97bcb9975f2334147173b4872505b6a41359a4f9068960a36afe07a679f @@ -980,7 +980,7 @@ F test/fts3sort.test ed34c716a11cc2009a35210e84ad5f9c102362ca F test/fts3tok1.test a663f4cac22a9505400bc22aacb818d7055240409c28729669ea7d4cc2120d15 F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d F test/fts3varint.test 0b84a3fd4eba8a39f3687523804d18f3b322e6d4539a55bf342079c3614f2ada -F test/fts4aa.test 9857f939f5aeb6cc4c143e74cf1cfe07ac87972240ac26d3a0100e36d02f4359 +F test/fts4aa.test 0e6bfd6a81695a39b23e448dda25d864e63dda75bde6949c45ddc95426c6c3f5 F test/fts4check.test 6259f856604445d7b684c9b306b2efb6346834c3f50e8fc4a59a2ca6d5319ad0 F test/fts4content.test 73bbb123420d2c46ef2fb3b24761e9acdb78b0877179d3a5d7d57aada08066f6 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 @@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 10306118e8591e727af477a1a15d136852d21170e645bd0e75f7c88346b037d7 -R 979d0103f7b3acef2369409e370d6fa7 -U mistachkin -Z 484824f16820465db78cbfdb9a5d496b +P 85d3dc8c50d8dbb8eac1956e8976e861d3b671e03355ca9257060fa3dca51cc4 +R 6df0bde47e9f675523e9d30a8887c1df +U dan +Z 4f76143b71cc5a7c4e3cc701b84eb415 diff --git a/manifest.uuid b/manifest.uuid index fb32b5bfff..84d9dbc14d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -85d3dc8c50d8dbb8eac1956e8976e861d3b671e03355ca9257060fa3dca51cc4 \ No newline at end of file +a9ec8c8f80a59badabb0afdb4189f0fd2934f936530d4151de395b3a7e7c1f1f \ No newline at end of file diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test index b11d19c7f1..89853833cb 100644 --- a/test/fts3corrupt4.test +++ b/test/fts3corrupt4.test @@ -5834,4 +5834,17 @@ do_catchsql_test 36.1 { INSERT INTO f(f) VALUES ('merge=59,59'); } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 37.0 { + CREATE VIRTUAL TABLE f USING fts3(a,b); + INSERT INTO f_segdir VALUES (28,0,0,0,'0 0',x'00'); + INSERT INTO f_segdir VALUES (0,241,0,0,'0 0',x'0001000030310000f1'); +} + +do_catchsql_test 37.1 { + INSERT INTO f VALUES (0,x'00'); +} {1 {database disk image is malformed}} + finish_test diff --git a/test/fts4aa.test b/test/fts4aa.test index 112d60ab4b..0c6c0b972f 100644 --- a/test/fts4aa.test +++ b/test/fts4aa.test @@ -234,13 +234,13 @@ if {$tcl_platform(byteOrder)=="littleEndian"} { } else { set res {X'0000000200000000000000000000000E0000000E00000001000000010000000100000001'} } -do_execsql_test fts4aa-6.10 { +do_catchsql_test fts4aa-6.10 { CREATE VIRTUAL TABLE f USING fts4(); INSERT INTO f_segdir VALUES (77,91,0,0,'255 77',x'0001308000004d5c4ddddddd4d4d7b4d4d4d614d8019ff4d05000001204d4d2e4d6e4d4d4d4b4d6c4d004d4d4d4d4d4d3d000000004d5d4d4d645d4d004d4d4d4d4d4d4d4d4d454d6910004d05ffff054d646c4d004d5d4d4d4d4d3d000000004d4d4d4d4d4d4d4d4d4d4d69624d4d4d04004d4d4d4d4d604d4ce1404d554d45'); INSERT INTO f_segdir VALUES (77,108,0,0,'255 77',x'0001310000fa64004d4d4d3c5d4d654d4d4d614d8000ff4d05000001204d4d2e4d6e4d4d4dff4d4d4d4d4d4d00104d4d4d4d000000004d4d4d0400311d4d4d4d4d4d4d4d4d4d684d6910004d05ffff054d4d6c4d004d4d4d4d4d4d3d000000004d4d4d4d644d4d4d4d4d4d69624d4d4d03ed4d4d4d4d4d604d4ce1404d550080'); INSERT INTO f_stat VALUES (0,x'80808080100000000064004d4d4d3c4d4d654d4d4d614d8000ff4df6ff1a00204d4d2e4d6e4d4d4d104d4d4d4d4d4d00104d4d4d4d4d4d69574d4d4d000031044d4d4d3e4d4d4c4d05004d6910'); SELECT quote(matchinfo(f,'pnax')) from f where f match '0 1'; -} $res +} {1 {database disk image is malformed}} # 2019-11-18 Detect infinite loop in fts3SelectLeaf() db close From 1d9ea27f7e7e053d99745b58b9109e063b4c03ad Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Apr 2020 23:46:54 +0000 Subject: [PATCH 06/31] Fix the ".excel" command and the ".open -x" and ".open -e" command so that they work better when running from an in-memory database and on Windows and when running from a script. FossilOrigin-Name: 07752164c2bf00b6885808533bbdb2cefbf1bf281a887b0b4f6316649a6cb810 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 20 ++++++++++++++++++-- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 32654b3149..d7bcffe5bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\saccidentally\screated\sfork. -D 2020-04-16T15:56:03.261 +C Fix\sthe\s".excel"\scommand\sand\sthe\s".open\s-x"\sand\s".open\s-e"\scommand\sso\sthat\nthey\swork\sbetter\swhen\srunning\sfrom\san\sin-memory\sdatabase\sand\son\sWindows\nand\swhen\srunning\sfrom\sa\sscript. +D 2020-04-17T23:46:54.073 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -533,7 +533,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 -F src/shell.c.in 792b901ae19c7cf8cbac06b3c58ef860328d0ca6d20149bdc469e5c4fc550a1d +F src/shell.c.in d615184e9a5e24da419db6bb9e60905306f45a6ceed760bdbbe656c4a684a18b F src/sqlite.h.in 4276f461ed8405630e3089b682dad77ae7a65c345d8daebcee52a13be8cd880c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a42fdcf54bcbd72a301dad4a040346dc48e67cacab43479ec618f5c32108c55f a9ec8c8f80a59badabb0afdb4189f0fd2934f936530d4151de395b3a7e7c1f1f -R f504f25559e67d24d7bac4ba4fbb1261 -U dan -Z bb44f1eb19ef7e1708ecd20763bc065a +P cb772b7a8fb53694cb267e74c11f49d2b9fd6920821c4e232f90ec35739c8904 +R e921db661ca4e205b542b92ef8a31312 +U drh +Z 880b12b2b5c0e130871dd9c9bbfa0760 diff --git a/manifest.uuid b/manifest.uuid index 587bd3fd29..226a3f3ebc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb772b7a8fb53694cb267e74c11f49d2b9fd6920821c4e232f90ec35739c8904 \ No newline at end of file +07752164c2bf00b6885808533bbdb2cefbf1bf281a887b0b4f6316649a6cb810 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index b589cb4238..f492285139 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4973,11 +4973,15 @@ static void output_reset(ShellState *p){ zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); if( system(zCmd) ){ utf8_printf(stderr, "Failed: [%s]\n", zCmd); + }else{ + /* Give the start/open/xdg-open command some time to get + ** going before we continue, and potential delete the + ** p->zTempFile data file out from under it */ + sqlite3_sleep(2000); } sqlite3_free(zCmd); outputModePop(p); p->doXdgOpen = 0; - sqlite3_sleep(100); } #endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ } @@ -5267,9 +5271,21 @@ static void newTempFile(ShellState *p, const char *zSuffix){ sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile); } if( p->zTempFile==0 ){ + /* If p->db is an in-memory database then the TEMPFILENAME file-control + ** will not work and we will need to fallback to guessing */ + char *zTemp; sqlite3_uint64 r; sqlite3_randomness(sizeof(r), &r); - p->zTempFile = sqlite3_mprintf("temp%llx.%s", r, zSuffix); + zTemp = getenv("TEMP"); + if( zTemp==0 ) zTemp = getenv("TMP"); + if( zTemp==0 ){ +#ifdef _WIN32 + zTemp = "\\tmp"; +#else + zTemp = "/tmp"; +#endif + } + p->zTempFile = sqlite3_mprintf("%s/temp%llx.%s", zTemp, r, zSuffix); }else{ p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix); } From 7a43100afbdb0abd50ccf5b1e8fdb552616f7b2b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Apr 2020 14:12:00 +0000 Subject: [PATCH 07/31] Add the --bom option to the ".excel", ".once", and ".output" commands of the CLI. Also fix the "--all" option on ".help" so that it works with two dashes in addition to just one. FossilOrigin-Name: d5b0def96ba6d90f47bc96fab1ccf9c501d84885d086744035b16fd96f3e248c --- manifest | 12 ++++---- manifest.uuid | 2 +- src/shell.c.in | 81 ++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 65 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index d7bcffe5bc..74a0e6e8a3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s".excel"\scommand\sand\sthe\s".open\s-x"\sand\s".open\s-e"\scommand\sso\sthat\nthey\swork\sbetter\swhen\srunning\sfrom\san\sin-memory\sdatabase\sand\son\sWindows\nand\swhen\srunning\sfrom\sa\sscript. -D 2020-04-17T23:46:54.073 +C Add\sthe\s--bom\soption\sto\sthe\s".excel",\s".once",\sand\s".output"\scommands\nof\sthe\sCLI.\s\sAlso\sfix\sthe\s"--all"\soption\son\s".help"\sso\sthat\sit\sworks\swith\ntwo\sdashes\sin\saddition\sto\sjust\sone. +D 2020-04-18T14:12:00.030 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -533,7 +533,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 -F src/shell.c.in d615184e9a5e24da419db6bb9e60905306f45a6ceed760bdbbe656c4a684a18b +F src/shell.c.in d48577e0282bc71f0692c3585dfeb59acacf28f721520814b0229cb7ef8972cb F src/sqlite.h.in 4276f461ed8405630e3089b682dad77ae7a65c345d8daebcee52a13be8cd880c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb772b7a8fb53694cb267e74c11f49d2b9fd6920821c4e232f90ec35739c8904 -R e921db661ca4e205b542b92ef8a31312 +P 07752164c2bf00b6885808533bbdb2cefbf1bf281a887b0b4f6316649a6cb810 +R f789048a883ae9edd5940e9d7edbaa7f U drh -Z 880b12b2b5c0e130871dd9c9bbfa0760 +Z 993c0c14d705afacb01cbe8f69aff20c diff --git a/manifest.uuid b/manifest.uuid index 226a3f3ebc..72cf879c71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07752164c2bf00b6885808533bbdb2cefbf1bf281a887b0b4f6316649a6cb810 \ No newline at end of file +d5b0def96ba6d90f47bc96fab1ccf9c501d84885d086744035b16fd96f3e248c \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index f492285139..e8517fb7ec 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1107,6 +1107,7 @@ struct ShellState { unsigned mxProgress; /* Maximum progress callbacks before failing */ unsigned flgProgress; /* Flags for the progress callback */ unsigned shellFlgs; /* Various flags */ + unsigned priorShFlgs; /* Saved copy of flags */ sqlite3_int64 szMax; /* --maxsize argument to .open */ char *zDestTable; /* Name of destination table when MODE_Insert */ char *zTempFile; /* Temporary file that might need deleting */ @@ -1407,11 +1408,13 @@ edit_func_end: */ static void outputModePush(ShellState *p){ p->modePrior = p->mode; + p->priorShFlgs = p->shellFlgs; memcpy(p->colSepPrior, p->colSeparator, sizeof(p->colSeparator)); memcpy(p->rowSepPrior, p->rowSeparator, sizeof(p->rowSeparator)); } static void outputModePop(ShellState *p){ p->mode = p->modePrior; + p->shellFlgs = p->priorShFlgs; memcpy(p->colSeparator, p->colSepPrior, sizeof(p->colSeparator)); memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator)); } @@ -3580,6 +3583,7 @@ static const char *(azHelp[]) = { #endif " trigger Like \"full\" but also show trigger bytecode", ".excel Display the output of next command in spreadsheet", + " --bom Put a UTF8 byte-order mark on intermediate file", ".exit ?CODE? Exit this program with return-code CODE", ".expert EXPERIMENTAL. Suggest indexes for queries", ".explain ?on|off|auto? Change the EXPLAIN formatting mode. Default: auto", @@ -3631,11 +3635,11 @@ static const char *(azHelp[]) = { " tabs Tab-separated values", " tcl TCL list elements", ".nullvalue STRING Use STRING in place of NULL values", - ".once (-e|-x|FILE) Output for the next SQL command only to FILE", + ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", " If FILE begins with '|' then open as a pipe", - " Other options:", - " -e Invoke system text editor", - " -x Open in a spreadsheet", + " --bom Put a UTF8 byte-order mark at the beginning", + " -e Send output to the system text editor", + " -x Send output as CSV to a spreadsheet (same as \".excel\")", #ifdef SQLITE_DEBUG ".oom [--repeat M] [N] Simulate an OOM error on the N-th allocation", #endif @@ -3652,7 +3656,11 @@ static const char *(azHelp[]) = { " --readonly Open FILE readonly", " --zip FILE is a ZIP archive", ".output ?FILE? Send output to FILE or stdout if FILE is omitted", - " If FILE begins with '|' then open it as a pipe.", + " If FILE begins with '|' then open it as a pipe.", + " Options:", + " --bom Prefix output with a UTF8 byte-order mark", + " -e Send output to the system text editor", + " -x Send output as CSV to a spreadsheet", ".parameter CMD ... Manage SQL parameter bindings", " clear Erase all bindings", " init Initialize the TEMP table that holds bindings", @@ -3772,6 +3780,7 @@ static int showHelp(FILE *out, const char *zPattern){ || zPattern[0]=='0' || strcmp(zPattern,"-a")==0 || strcmp(zPattern,"-all")==0 + || strcmp(zPattern,"--all")==0 ){ /* Show all commands, but only one line per command */ if( zPattern==0 ) zPattern = ""; @@ -8307,42 +8316,66 @@ static int do_meta_command(char *zLine, ShellState *p){ && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) ){ - const char *zFile = nArg>=2 ? azArg[1] : "stdout"; + const char *zFile = 0; int bTxtMode = 0; - if( azArg[0][0]=='e' ){ - /* Transform the ".excel" command into ".once -x" */ - nArg = 2; - azArg[0] = "once"; - zFile = azArg[1] = "-x"; - n = 4; + int i; + int eMode = 0; + int bBOM = 0; + int bOnce; + + if( c=='e' ){ + eMode = 'x'; + bOnce = 2; + }else if( strncmp(azArg[0],"once",n)==0 ){ + bOnce = 1; } - if( nArg>2 ){ - utf8_printf(stderr, "Usage: .%s [-e|-x|FILE]\n", azArg[0]); - rc = 1; - goto meta_command_exit; - } - if( n>1 && strncmp(azArg[0], "once", n)==0 ){ - if( nArg<2 ){ - raw_printf(stderr, "Usage: .once (-e|-x|FILE)\n"); + for(i=1; iout, "ERROR: unknown option: \"%s\". Usage:\n", + azArg[i]); + showHelp(p->out, azArg[0]); + rc = 1; + goto meta_command_exit; + } + }else if( zFile==0 ){ + zFile = z; + }else{ + utf8_printf(p->out,"ERROR: extra parameter: \"%s\". Usage:\n", + azArg[i]); + showHelp(p->out, azArg[0]); rc = 1; goto meta_command_exit; } + } + if( zFile==0 ) zFile = "stdout"; + if( bOnce ){ p->outCount = 2; }else{ p->outCount = 0; } output_reset(p); - if( zFile[0]=='-' && zFile[1]=='-' ) zFile++; #ifndef SQLITE_NOHAVE_SYSTEM - if( strcmp(zFile, "-e")==0 || strcmp(zFile, "-x")==0 ){ + if( eMode=='e' || eMode=='x' ){ p->doXdgOpen = 1; outputModePush(p); - if( zFile[1]=='x' ){ + if( eMode=='x' ){ + /* spreadsheet mode. Output as CSV. */ newTempFile(p, "csv"); + ShellClearFlag(p, SHFLG_Echo); p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); }else{ + /* text editor mode */ newTempFile(p, "txt"); bTxtMode = 1; } @@ -8361,6 +8394,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->out = stdout; rc = 1; }else{ + if( bBOM ) fprintf(p->out,"\357\273\277"); sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); } #endif @@ -8373,6 +8407,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->out = stdout; rc = 1; } else { + if( bBOM ) fprintf(p->out,"\357\273\277"); sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); } } From 871f45441cd99398b2daec14bc141649db21a06f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Apr 2020 14:05:54 +0000 Subject: [PATCH 08/31] Remove an obsolete comment. No changes to code. FossilOrigin-Name: 4135cb024456288d9c85aef5fb572dbb591527dd33d9a60ca5946b712c269941 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 4 ---- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 74a0e6e8a3..856ee8ceea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--bom\soption\sto\sthe\s".excel",\s".once",\sand\s".output"\scommands\nof\sthe\sCLI.\s\sAlso\sfix\sthe\s"--all"\soption\son\s".help"\sso\sthat\sit\sworks\swith\ntwo\sdashes\sin\saddition\sto\sjust\sone. -D 2020-04-18T14:12:00.030 +C Remove\san\sobsolete\scomment.\s\sNo\schanges\sto\scode. +D 2020-04-20T14:05:54.955 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -473,7 +473,7 @@ F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 79ce96ab39fd2fc21ff00d03913587d5a08280a9eb081a08d0ffa9fa26f4f6fb +F src/btree.c 671be07873797f106cf59a81471196523bb88dacc4fcd6030d1d415fff3f7b2e F src/btree.h 6111552f19ed7a40f029cf4b33badc6fef9880314fffd80a945f0b7f43ab7471 F src/btreeInt.h dee1a1d0c621524e006bb260bd6b66d5d1867da6fe38cba9ad7b6a9bb9c0c175 F src/build.c ec6c0bda1e43ef55e5f5121a77ba19fac51fc6585f95ce2da795bcedcf6e8f36 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07752164c2bf00b6885808533bbdb2cefbf1bf281a887b0b4f6316649a6cb810 -R f789048a883ae9edd5940e9d7edbaa7f +P d5b0def96ba6d90f47bc96fab1ccf9c501d84885d086744035b16fd96f3e248c +R 4a09aaf85f0fed3dd0723d494ffedd7d U drh -Z 993c0c14d705afacb01cbe8f69aff20c +Z 504e3b736c67f90426ab20d53647f809 diff --git a/manifest.uuid b/manifest.uuid index 72cf879c71..7a60b45cea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5b0def96ba6d90f47bc96fab1ccf9c501d84885d086744035b16fd96f3e248c \ No newline at end of file +4135cb024456288d9c85aef5fb572dbb591527dd33d9a60ca5946b712c269941 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 20bb7c7d9b..f325ee8699 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2910,10 +2910,6 @@ int sqlite3BtreeGetReserveNoMutex(Btree *p){ ** Return the number of bytes of space at the end of every page that ** are intentually left unused. This is the "reserved" space that is ** sometimes used by extensions. -** -** If SQLITE_HAS_MUTEX is defined then the number returned is the -** greater of the current reserved space and the maximum requested -** reserve space. */ int sqlite3BtreeGetOptimalReserve(Btree *p){ int n; From 45248de39acbf885d5580bf53f23a4e769374957 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Apr 2020 15:18:43 +0000 Subject: [PATCH 09/31] The SQLITE_TESTCTRL_RESERVE operator is removed. In its place is the more generate SQLITE_FCNTL_RESERVE_BYTES which is an API and which can operator on more than just the main schema. FossilOrigin-Name: abc1aad74f7b6a1e72fb09936239f2224aa942d16296c6a3de0b8daef4bc8471 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/btree.c | 10 +++++++--- src/btree.h | 2 +- src/btreeInt.h | 1 + src/main.c | 21 +++++++-------------- src/shell.c.in | 2 -- src/sqlite.h.in | 3 ++- src/vacuum.c | 2 +- 9 files changed, 32 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index 856ee8ceea..ac9b8a0f5d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sobsolete\scomment.\s\sNo\schanges\sto\scode. -D 2020-04-20T14:05:54.955 +C The\sSQLITE_TESTCTRL_RESERVE\soperator\sis\sremoved.\s\sIn\sits\splace\sis\sthe\nmore\sgenerate\sSQLITE_FCNTL_RESERVE_BYTES\swhich\sis\san\sAPI\sand\swhich\scan\noperator\son\smore\sthan\sjust\sthe\smain\sschema. +D 2020-04-20T15:18:43.124 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -473,9 +473,9 @@ F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 671be07873797f106cf59a81471196523bb88dacc4fcd6030d1d415fff3f7b2e -F src/btree.h 6111552f19ed7a40f029cf4b33badc6fef9880314fffd80a945f0b7f43ab7471 -F src/btreeInt.h dee1a1d0c621524e006bb260bd6b66d5d1867da6fe38cba9ad7b6a9bb9c0c175 +F src/btree.c 97673c54a3023845e7320cfa9e0de67fd6ca34b37a17a420981bb6e767601310 +F src/btree.h 32672fa1aa74a7e9ab3aae822f94ffc8e732b1eb005988dc2283f91dc7573398 +F src/btreeInt.h 887cdd2ea7f4a65143074a8a7c8928b0546f8c18dda3c06a408ce7992cbab0c0 F src/build.c ec6c0bda1e43ef55e5f5121a77ba19fac51fc6585f95ce2da795bcedcf6e8f36 F src/callback.c d0b853dd413255d2e337b34545e54d888ea02f20da5ad0e63585b389624c4a6c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e @@ -496,7 +496,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 8e4211d04eb460c0694d486c6ba1c068d468c6f653c3f237869a802ad82854de F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c b179df50e6e8bb0c36c149e95d958d49bd8c6c7469e59c01b53d164360bc6c32 -F src/main.c 2e076b6dc1f8ab69c7fc604e8af88ab138a64c0616826361e254cee55bcba4e8 +F src/main.c 652a782cd7b6c6ddf7419fcaf06b8aa9440b7c815857241171c9bdf03ab6544c F src/malloc.c cabfef0d725e04c8abfe0231a556ed8b78bf329dcc3fddbf903f6cdcd53cf4e6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -533,8 +533,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 -F src/shell.c.in d48577e0282bc71f0692c3585dfeb59acacf28f721520814b0229cb7ef8972cb -F src/sqlite.h.in 4276f461ed8405630e3089b682dad77ae7a65c345d8daebcee52a13be8cd880c +F src/shell.c.in c21ce88a16c04c46741ee04373992ad9982bd2fe83217e9c77012e63af0dde73 +F src/sqlite.h.in d4430b302155f757f0fbce33c5afe136525827ddfbe532a79fd7fe85b127ea29 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee F src/sqliteInt.h 0f3848c46310d197246003f052985b72d1cdbfc0b31e069db76cb5231062fa1d @@ -603,7 +603,7 @@ F src/update.c 3eb778c42155d944377a4ee5e440b04520f07094804ed6ce63d2528f619614d9 F src/upsert.c 2920de71b20f04fe25eb00b655d086f0ba60ea133c59d7fa3325c49838818e78 F src/utf.c 95fb6e03a5ca679045c5adccd05380f0addccabef5911abddcb06af069500ab7 F src/util.c 3b6cedf7a0c69bd6e1acce832873952d416212d6293b18d03064e07d7a9b5118 -F src/vacuum.c 813b510ba887fee6492bcb11f2bf77d7eb58b232b83649136372e0a2fc17f4b9 +F src/vacuum.c f25d3b39681595e321a8a4e7fca3e5971cb14a401213d0742c9bf7d4ce8c2a1a F src/vdbe.c 972999395eee88702091fb5d50cf4effd07889c371807d222a7f517388e6378e F src/vdbe.h 51282fbe819ee0e8eeeaab176240860d334c20a12b14f3b363e7f1a4e05d60b9 F src/vdbeInt.h 0b728ee662862a38b1912af741e2ac64f524de3c77aa86cf4306c42bdcd9de59 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d5b0def96ba6d90f47bc96fab1ccf9c501d84885d086744035b16fd96f3e248c -R 4a09aaf85f0fed3dd0723d494ffedd7d +P 4135cb024456288d9c85aef5fb572dbb591527dd33d9a60ca5946b712c269941 +R 1d205cbee16938d3cfb21de74a2e62b1 U drh -Z 504e3b736c67f90426ab20d53647f809 +Z db2ebefa734c86f2ca8a25996eb4de7d diff --git a/manifest.uuid b/manifest.uuid index 7a60b45cea..5d4108268d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4135cb024456288d9c85aef5fb572dbb591527dd33d9a60ca5946b712c269941 \ No newline at end of file +abc1aad74f7b6a1e72fb09936239f2224aa942d16296c6a3de0b8daef4bc8471 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f325ee8699..75cdd25eb3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2857,8 +2857,11 @@ int sqlite3BtreeSetPagerFlags( int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ int rc = SQLITE_OK; BtShared *pBt = p->pBt; - assert( nReserve>=-1 && nReserve<=255 ); + assert( nReserve>=-1 && nReserve<=254 ); sqlite3BtreeEnter(p); + if( nReserve>=0 ){ + pBt->nReserveWanted = nReserve + 1; + } if( pBt->btsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); return SQLITE_READONLY; @@ -2911,10 +2914,11 @@ int sqlite3BtreeGetReserveNoMutex(Btree *p){ ** are intentually left unused. This is the "reserved" space that is ** sometimes used by extensions. */ -int sqlite3BtreeGetOptimalReserve(Btree *p){ +int sqlite3BtreeGetRequestedReserve(Btree *p){ int n; sqlite3BtreeEnter(p); - n = sqlite3BtreeGetReserveNoMutex(p); + n = ((int)p->pBt->nReserveWanted) - 1; + if( n<0 ) n = sqlite3BtreeGetReserveNoMutex(p); sqlite3BtreeLeave(p); return n; } diff --git a/src/btree.h b/src/btree.h index 4bd41f7f37..680742a21d 100644 --- a/src/btree.h +++ b/src/btree.h @@ -74,7 +74,7 @@ int sqlite3BtreeGetPageSize(Btree*); int sqlite3BtreeMaxPageCount(Btree*,int); u32 sqlite3BtreeLastPage(Btree*); int sqlite3BtreeSecureDelete(Btree*,int); -int sqlite3BtreeGetOptimalReserve(Btree*); +int sqlite3BtreeGetRequestedReserve(Btree*); int sqlite3BtreeGetReserveNoMutex(Btree *p); int sqlite3BtreeSetAutoVacuum(Btree *, int); int sqlite3BtreeGetAutoVacuum(Btree *); diff --git a/src/btreeInt.h b/src/btreeInt.h index e5149d97ea..34b33096ba 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -417,6 +417,7 @@ struct BtShared { #endif u8 inTransaction; /* Transaction state */ u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ + u8 nReserveWanted; /* 1 more than desired number of extra bytes per page */ u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ diff --git a/src/main.c b/src/main.c index d693621b1e..f186d6ede2 100644 --- a/src/main.c +++ b/src/main.c @@ -3843,6 +3843,13 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ }else if( op==SQLITE_FCNTL_DATA_VERSION ){ *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESERVE_BYTES ){ + int iNew = *(int*)pArg; + *(int*)pArg = sqlite3BtreeGetRequestedReserve(pBtree); + if( iNew>=0 && iNew<=254 ){ + sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); + } + rc = SQLITE_OK; }else{ rc = sqlite3OsFileControl(fd, op, pArg); } @@ -4059,20 +4066,6 @@ int sqlite3_test_control(int op, ...){ break; } - /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) - ** - ** Set the nReserve size to N for the main database on the database - ** connection db. - */ - case SQLITE_TESTCTRL_RESERVE: { - sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); - sqlite3_mutex_leave(db->mutex); - break; - } - /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) ** ** Enable or disable various optimizations for testing purposes. The diff --git a/src/shell.c.in b/src/shell.c.in index e8517fb7ec..f252cd7bc7 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -9471,7 +9471,6 @@ static int do_meta_command(char *zLine, ShellState *p){ { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE, "" }, { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE, "" }, { "prng_seed", SQLITE_TESTCTRL_PRNG_SEED, "SEED ?db?" }, - { "reserve", SQLITE_TESTCTRL_RESERVE, "BYTES-OF-RESERVE"}, }; int testctrl = -1; int iCtrl = -1; @@ -9524,7 +9523,6 @@ static int do_meta_command(char *zLine, ShellState *p){ /* sqlite3_test_control(int, db, int) */ case SQLITE_TESTCTRL_OPTIMIZATIONS: - case SQLITE_TESTCTRL_RESERVE: if( nArg==3 ){ int opt = (int)strtol(azArg[2], 0, 0); rc2 = sqlite3_test_control(testctrl, p->db, opt); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f93fab1c59..2f05665e96 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1157,6 +1157,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_DATA_VERSION 35 #define SQLITE_FCNTL_SIZE_LIMIT 36 #define SQLITE_FCNTL_CKPT_DONE 37 +#define SQLITE_FCNTL_RESERVE_BYTES 38 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -7652,7 +7653,7 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_PENDING_BYTE 11 #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 -#define SQLITE_TESTCTRL_RESERVE 14 +#define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ diff --git a/src/vacuum.c b/src/vacuum.c index ce9db649d2..5c230fbe14 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -233,7 +233,7 @@ SQLITE_NOINLINE int sqlite3RunVacuum( } db->mDbFlags |= DBFLAG_VacuumInto; } - nRes = sqlite3BtreeGetOptimalReserve(pMain); + nRes = sqlite3BtreeGetRequestedReserve(pMain); sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); From 541ef2c36c3c7a42bb563cf80f73681c460a25d1 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Apr 2020 16:21:30 +0000 Subject: [PATCH 10/31] Enhance the ".filectrl" command in the CLI to support the --schema option. FossilOrigin-Name: 698d40db58b76f4094687d46b5c6292702a62168054f9e6a70ee86650a6635f7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 37 +++++++++++++++++++++++++++++++------ test/shell1.test | 18 ++++++++++++++++-- 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index ac9b8a0f5d..861b547a6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sSQLITE_TESTCTRL_RESERVE\soperator\sis\sremoved.\s\sIn\sits\splace\sis\sthe\nmore\sgenerate\sSQLITE_FCNTL_RESERVE_BYTES\swhich\sis\san\sAPI\sand\swhich\scan\noperator\son\smore\sthan\sjust\sthe\smain\sschema. -D 2020-04-20T15:18:43.124 +C Enhance\sthe\s".filectrl"\scommand\sin\sthe\sCLI\sto\ssupport\sthe\s--schema\soption. +D 2020-04-20T16:21:30.906 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -533,7 +533,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 -F src/shell.c.in c21ce88a16c04c46741ee04373992ad9982bd2fe83217e9c77012e63af0dde73 +F src/shell.c.in 5402d3a7281576c96d788569778483746859c5e47052facfc4a4ccfac025d5da F src/sqlite.h.in d4430b302155f757f0fbce33c5afe136525827ddfbe532a79fd7fe85b127ea29 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee @@ -1334,7 +1334,7 @@ F test/sharedA.test 49d87ec54ab640fbbc3786ee3c01de94aaa482a3a9f834ad3fe92770eb69 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test 1796b7f76d09ffd2b2dc0ff5ad89428e9892f237d4f4e33ef2278f064a2d94a4 +F test/shell1.test 5bd10014ec494744f5e966a1521334e9d612119a0afcfa5251684a4e1f2ffc66 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494 F test/shell4.test 1c6aef11daaa2d6830acaba3ac9cbec93fbc1c3d5530743a637f39b3987d08ce @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4135cb024456288d9c85aef5fb572dbb591527dd33d9a60ca5946b712c269941 -R 1d205cbee16938d3cfb21de74a2e62b1 +P abc1aad74f7b6a1e72fb09936239f2224aa942d16296c6a3de0b8daef4bc8471 +R 0b689458032aceec4d998f84188e0ee8 U drh -Z db2ebefa734c86f2ca8a25996eb4de7d +Z af548dcf210abacdeac04040d9882947 diff --git a/manifest.uuid b/manifest.uuid index 5d4108268d..c04fa12c7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -abc1aad74f7b6a1e72fb09936239f2224aa942d16296c6a3de0b8daef4bc8471 \ No newline at end of file +698d40db58b76f4094687d46b5c6292702a62168054f9e6a70ee86650a6635f7 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index f252cd7bc7..a2351ce876 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3588,7 +3588,8 @@ static const char *(azHelp[]) = { ".expert EXPERIMENTAL. Suggest indexes for queries", ".explain ?on|off|auto? Change the EXPLAIN formatting mode. Default: auto", ".filectrl CMD ... Run various sqlite3_file_control() operations", - " Run \".filectrl\" with no arguments for details", + " --schema SCHEMA Use SCHEMA instead of \"main\"", + " --help Show CMD details", ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", ".headers on|off Turn display of headers on or off", ".help ?-all? ?PATTERN? Show help text for PATTERN", @@ -7489,6 +7490,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { "tempfilename", SQLITE_FCNTL_TEMPFILENAME, "" }, { "has_moved", SQLITE_FCNTL_HAS_MOVED, "" }, { "lock_timeout", SQLITE_FCNTL_LOCK_TIMEOUT, "MILLISEC" }, + { "reserve_bytes", SQLITE_FCNTL_RESERVE_BYTES, "[N]" }, }; int filectrl = -1; int iCtrl = -1; @@ -7496,10 +7498,21 @@ static int do_meta_command(char *zLine, ShellState *p){ int isOk = 0; /* 0: usage 1: %lld 2: no-result */ int n2, i; const char *zCmd = 0; + const char *zSchema = 0; open_db(p, 0); zCmd = nArg>=2 ? azArg[1] : "help"; + if( zCmd[0]=='-' + && (strcmp(zCmd,"--schema")==0 || strcmp(zCmd,"-schema")==0) + && nArg>=4 + ){ + zSchema = azArg[2]; + for(i=3; idb, 0, SQLITE_FCNTL_SIZE_LIMIT, &iRes); + sqlite3_file_control(p->db, zSchema, SQLITE_FCNTL_SIZE_LIMIT, &iRes); isOk = 1; break; } @@ -7550,7 +7563,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int x; if( nArg!=3 ) break; x = (int)integerValue(azArg[2]); - sqlite3_file_control(p->db, 0, filectrl, &x); + sqlite3_file_control(p->db, zSchema, filectrl, &x); isOk = 2; break; } @@ -7559,7 +7572,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int x; if( nArg!=2 && nArg!=3 ) break; x = nArg==3 ? booleanValue(azArg[2]) : -1; - sqlite3_file_control(p->db, 0, filectrl, &x); + sqlite3_file_control(p->db, zSchema, filectrl, &x); iRes = x; isOk = 1; break; @@ -7567,7 +7580,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_FCNTL_HAS_MOVED: { int x; if( nArg!=2 ) break; - sqlite3_file_control(p->db, 0, filectrl, &x); + sqlite3_file_control(p->db, zSchema, filectrl, &x); iRes = x; isOk = 1; break; @@ -7575,7 +7588,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_FCNTL_TEMPFILENAME: { char *z = 0; if( nArg!=2 ) break; - sqlite3_file_control(p->db, 0, filectrl, &z); + sqlite3_file_control(p->db, zSchema, filectrl, &z); if( z ){ utf8_printf(p->out, "%s\n", z); sqlite3_free(z); @@ -7583,6 +7596,18 @@ static int do_meta_command(char *zLine, ShellState *p){ isOk = 2; break; } + case SQLITE_FCNTL_RESERVE_BYTES: { + int x; + if( nArg>=3 ){ + x = atoi(azArg[2]); + sqlite3_file_control(p->db, zSchema, filectrl, &x); + } + x = -1; + sqlite3_file_control(p->db, zSchema, filectrl, &x); + utf8_printf(p->out,"%d\n", x); + isOk = 2; + break; + } } } if( isOk==0 && iCtrl>=0 ){ diff --git a/test/shell1.test b/test/shell1.test index bbe2ad765e..c142ea7241 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -493,7 +493,14 @@ do_test shell1-3.15.2 { do_test shell1-3.15.3 { # too many arguments catchcmd "test.db" ".output FOO BAD" -} {1 {Usage: .output [-e|-x|FILE]}} +} {1 {ERROR: extra parameter: "BAD". Usage: +.output ?FILE? Send output to FILE or stdout if FILE is omitted + If FILE begins with '|' then open it as a pipe. + Options: + --bom Prefix output with a UTF8 byte-order mark + -e Send output to the system text editor + -x Send output as CSV to a spreadsheet +child process exited abnormally}} # .output stdout Send output to the screen do_test shell1-3.16.1 { @@ -502,7 +509,14 @@ do_test shell1-3.16.1 { do_test shell1-3.16.2 { # too many arguments catchcmd "test.db" ".output stdout BAD" -} {1 {Usage: .output [-e|-x|FILE]}} +} {1 {ERROR: extra parameter: "BAD". Usage: +.output ?FILE? Send output to FILE or stdout if FILE is omitted + If FILE begins with '|' then open it as a pipe. + Options: + --bom Prefix output with a UTF8 byte-order mark + -e Send output to the system text editor + -x Send output as CSV to a spreadsheet +child process exited abnormally}} # .prompt MAIN CONTINUE Replace the standard prompts do_test shell1-3.17.1 { From 3b9f154bb7c2f5382a998ce7ca1312654ddea587 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Apr 2020 17:35:32 +0000 Subject: [PATCH 11/31] Do not use O_NOFOLLOW when opening a directory just to call fsync() on that directory. FossilOrigin-Name: 2fc80ef16ce5878311ab88a0c64631813572ffbb71f75363b4619c9667e0926b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 861b547a6b..f57656af3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\s".filectrl"\scommand\sin\sthe\sCLI\sto\ssupport\sthe\s--schema\soption. -D 2020-04-20T16:21:30.906 +C Do\snot\suse\sO_NOFOLLOW\swhen\sopening\sa\sdirectory\sjust\sto\scall\sfsync()\son\nthat\sdirectory. +D 2020-04-20T17:35:32.334 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -516,7 +516,7 @@ F src/os.c 669cc3839cc35d20f81faf0be1ab6d4581cea35e9d8f3a9d48a98d6571f7c285 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 06593ba4bdfae42b30a897b3b4e59abb88a11d50dd0cad39003da955d822b8d1 +F src/os_unix.c 7ef8b60222558a373d89c18d0c3bc44365b45273a528183d40bec5bb76ce23fc F src/os_win.c 035a813cbd17f355bdcad7ab894af214a9c13a1db8aeac902365350b98cd45a7 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c a71ffd145f55e28cbdc1bdabb5e6bef063da428a6c0de3c3a36e9a0c41d4c8c0 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P abc1aad74f7b6a1e72fb09936239f2224aa942d16296c6a3de0b8daef4bc8471 -R 0b689458032aceec4d998f84188e0ee8 +P 698d40db58b76f4094687d46b5c6292702a62168054f9e6a70ee86650a6635f7 +R 1a3f713395bcb50f232d9e1e989d47d0 U drh -Z af548dcf210abacdeac04040d9882947 +Z 3de631e900a36e1c44f2a5a4e6c38aa5 diff --git a/manifest.uuid b/manifest.uuid index c04fa12c7a..567a9e7beb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -698d40db58b76f4094687d46b5c6292702a62168054f9e6a70ee86650a6635f7 \ No newline at end of file +2fc80ef16ce5878311ab88a0c64631813572ffbb71f75363b4619c9667e0926b \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index a4dd2f906c..56e53929ea 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3685,7 +3685,7 @@ static int openDirectory(const char *zFilename, int *pFd){ if( zDirname[0]!='/' ) zDirname[0] = '.'; zDirname[1] = 0; } - fd = robust_open(zDirname, O_RDONLY|O_BINARY|O_NOFOLLOW, 0); + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); if( fd>=0 ){ OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); } From 480620c71340e8dca10d6c842985c45f6ba6cf15 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Apr 2020 01:06:35 +0000 Subject: [PATCH 12/31] Experimental API: sqlite3_database_file_object(). FossilOrigin-Name: ae697b152d22737169892411a0c4d908895ff5fb249cce9bdb1ba0bbe32806f0 --- manifest | 21 ++++++++++++--------- manifest.uuid | 2 +- src/loadext.c | 1 + src/pager.c | 16 ++++++++++++++++ src/sqlite.h.in | 19 +++++++++++++++++++ src/sqlite3ext.h | 1 + 6 files changed, 50 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f57656af3a..35ca686870 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sO_NOFOLLOW\swhen\sopening\sa\sdirectory\sjust\sto\scall\sfsync()\son\nthat\sdirectory. -D 2020-04-20T17:35:32.334 +C Experimental\sAPI:\ssqlite3_database_file_object(). +D 2020-04-21T01:06:35.069 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -495,7 +495,7 @@ F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 8e4211d04eb460c0694d486c6ba1c068d468c6f653c3f237869a802ad82854de F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c b179df50e6e8bb0c36c149e95d958d49bd8c6c7469e59c01b53d164360bc6c32 +F src/loadext.c 4540fb20e8606b5cea7efee1d727cc69b20390ddc2e6a6c5515e20f96dcc205c F src/main.c 652a782cd7b6c6ddf7419fcaf06b8aa9440b7c815857241171c9bdf03ab6544c F src/malloc.c cabfef0d725e04c8abfe0231a556ed8b78bf329dcc3fddbf903f6cdcd53cf4e6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -519,7 +519,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 7ef8b60222558a373d89c18d0c3bc44365b45273a528183d40bec5bb76ce23fc F src/os_win.c 035a813cbd17f355bdcad7ab894af214a9c13a1db8aeac902365350b98cd45a7 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c a71ffd145f55e28cbdc1bdabb5e6bef063da428a6c0de3c3a36e9a0c41d4c8c0 +F src/pager.c 29ee0618daa1f49c24431a9f21b7ac15fa148eb63626c9e0ac145fffbe0b209d F src/pager.h 3b33619a90180e0874c7eca31d6f6ceb464d9322c6fb4e9a7bbb318c8a17bdb3 F src/parse.y c8eff38606f443d5ba245263fa7abc05e4116d95656e050c4b78e9bfbf931add F src/pcache.c 385ff064bca69789d199a98e2169445dc16e4291fa807babd61d4890c3b34177 @@ -534,9 +534,9 @@ F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 F src/shell.c.in 5402d3a7281576c96d788569778483746859c5e47052facfc4a4ccfac025d5da -F src/sqlite.h.in d4430b302155f757f0fbce33c5afe136525827ddfbe532a79fd7fe85b127ea29 +F src/sqlite.h.in fd6fcfe173accab8d9cb9a843856d9e9fb475f893b60a455e01d8739b5076f0e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee +F src/sqlite3ext.h 224bd560533a3479bd38ece4af12799b684751c61d7762a54859efc2cf68c702 F src/sqliteInt.h 0f3848c46310d197246003f052985b72d1cdbfc0b31e069db76cb5231062fa1d F src/sqliteLimit.h 95cb8479ca459496d9c1c6a9f76b38aee12203a56ce1092fe13e50ae2454c032 F src/status.c 9ff2210207c6c3b4d9631a8241a7d45ab1b26a0e9c84cb07a9b5ce2de9a3b278 @@ -1861,7 +1861,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 698d40db58b76f4094687d46b5c6292702a62168054f9e6a70ee86650a6635f7 -R 1a3f713395bcb50f232d9e1e989d47d0 +P 2fc80ef16ce5878311ab88a0c64631813572ffbb71f75363b4619c9667e0926b +R 179a1e5a1fd45f685d8f9f4125f02840 +T *branch * sqlite3_database_file_object +T *sym-sqlite3_database_file_object * +T -sym-trunk * U drh -Z 3de631e900a36e1c44f2a5a4e6c38aa5 +Z 4494bc3e04abebffc2e5f13976ff0914 diff --git a/manifest.uuid b/manifest.uuid index 567a9e7beb..9bb6891bbe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2fc80ef16ce5878311ab88a0c64631813572ffbb71f75363b4619c9667e0926b \ No newline at end of file +ae697b152d22737169892411a0c4d908895ff5fb249cce9bdb1ba0bbe32806f0 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 06297e6bb6..5a661cdc26 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -477,6 +477,7 @@ static const sqlite3_api_routines sqlite3Apis = { /* Version 3.32.0 and later */ sqlite3_create_filename, sqlite3_free_filename, + sqlite3_database_file_object, }; /* diff --git a/src/pager.c b/src/pager.c index 5d2225590e..3377762300 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4742,6 +4742,7 @@ int sqlite3PagerOpen( ** Database file handle (pVfs->szOsFile bytes) ** Sub-journal file handle (journalFileSize bytes) ** Main journal file handle (journalFileSize bytes) + ** Ptr back to the Pager (sizeof(Pager*) bytes) ** \0\0\0\0 database prefix (4 bytes) ** Database file name (nPathname+1 bytes) ** URI query parameters (nUriByte bytes) @@ -4781,6 +4782,7 @@ int sqlite3PagerOpen( ROUND8(pcacheSize) + /* PCache object */ ROUND8(pVfs->szOsFile) + /* The main db file */ journalFileSize * 2 + /* The two journal files */ + sizeof(pPager) + /* Space to hold a pointer */ 4 + /* Database prefix */ nPathname + 1 + /* database filename */ nUriByte + /* query parameters */ @@ -4801,6 +4803,7 @@ int sqlite3PagerOpen( pPager->sjfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; pPager->jfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); + memcpy(pPtr, &pPager, sizeof(pPager)); pPtr += sizeof(pPager); /* Fill in the Pager.zFilename and pPager.zQueryParam fields */ pPtr += 4; /* Skip zero prefix */ @@ -5001,6 +5004,19 @@ act_like_temp_file: return SQLITE_OK; } +/* +** Return the sqlite3_file for the main database given the name +** of the corresonding WAL or Journal name as passed into +** xOpen. +*/ +sqlite3_file *sqlite3_database_file_object(const char *zName){ + Pager *pPager; + while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ + zName--; + } + pPager = *(Pager**)(zName - 4 - sizeof(Pager*)); + return pPager->fd; +} /* diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2f05665e96..a5e8d14974 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3620,6 +3620,25 @@ const char *sqlite3_filename_database(const char*); const char *sqlite3_filename_journal(const char*); const char *sqlite3_filename_wal(const char*); +/* +** CAPI3REF: Database File Corresponding To A Journal +** +** ^If X is the name of a rollback or WAL-mode journal file that is +** passed into the xOpen method of [sqlite3_vfs], then +** sqlite3_database_file_object(X) returns a pointer to the [sqlite3_file] +** object that represents the main database file. +** +** This routine is intended for use in custom [VFS] implementations +** only. It is not a general-purpose interface. +** The argument sqlite3_file_object(X) must be a filename pointer that +** has been passed into [sqlite3_vfs].xOpen method where the +** flags parameter to xOpen contains one of the bits +** [SQLITE_OPEN_MAIN_JOURNAL] or [SQLITE_OPEN_WAL]. Any other use +** of this routine results in undefined and probably undesirable +** behavior. +*/ +sqlite3_file *sqlite3_database_file_object(const char*); + /* ** CAPI3REF: Create and Destroy VFS Filenames ** diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index a7c76a6784..283a679ad4 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -334,6 +334,7 @@ struct sqlite3_api_routines { char *(*create_filename)(const char*,const char*,const char*, int,const char**); void (*free_filename)(char*); + sqlite3_file *(*database_file_object)(const char*); }; /* From 9a1bb3f6166f08e9f520393b2c9c3ce892366922 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Apr 2020 19:27:08 +0000 Subject: [PATCH 13/31] Add the sqlite3_database_file_object() interface to sqlite3ext.h. FossilOrigin-Name: 3cabe06b6e9433cbfb4bf7bd72a425f43b54f57d84a205adbaa6a1147bbe46bf --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/sqlite3ext.h | 1 + 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 35ca686870..c459dd328b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\sAPI:\ssqlite3_database_file_object(). -D 2020-04-21T01:06:35.069 +C Add\sthe\ssqlite3_database_file_object()\sinterface\sto\ssqlite3ext.h. +D 2020-04-21T19:27:08.207 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -536,7 +536,7 @@ F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 F src/shell.c.in 5402d3a7281576c96d788569778483746859c5e47052facfc4a4ccfac025d5da F src/sqlite.h.in fd6fcfe173accab8d9cb9a843856d9e9fb475f893b60a455e01d8739b5076f0e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h 224bd560533a3479bd38ece4af12799b684751c61d7762a54859efc2cf68c702 +F src/sqlite3ext.h 2d1af80082edffd71c6f96f70ad1ce6a4fb46615ad10291fc77fe0dea9ff0197 F src/sqliteInt.h 0f3848c46310d197246003f052985b72d1cdbfc0b31e069db76cb5231062fa1d F src/sqliteLimit.h 95cb8479ca459496d9c1c6a9f76b38aee12203a56ce1092fe13e50ae2454c032 F src/status.c 9ff2210207c6c3b4d9631a8241a7d45ab1b26a0e9c84cb07a9b5ce2de9a3b278 @@ -1861,10 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2fc80ef16ce5878311ab88a0c64631813572ffbb71f75363b4619c9667e0926b -R 179a1e5a1fd45f685d8f9f4125f02840 -T *branch * sqlite3_database_file_object -T *sym-sqlite3_database_file_object * -T -sym-trunk * +P ae697b152d22737169892411a0c4d908895ff5fb249cce9bdb1ba0bbe32806f0 +R cc7f5aa23076777d96573db8bef15752 U drh -Z 4494bc3e04abebffc2e5f13976ff0914 +Z 0fbe741b842883ec4b35af5da437ce6d diff --git a/manifest.uuid b/manifest.uuid index 9bb6891bbe..999a5e365b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae697b152d22737169892411a0c4d908895ff5fb249cce9bdb1ba0bbe32806f0 \ No newline at end of file +3cabe06b6e9433cbfb4bf7bd72a425f43b54f57d84a205adbaa6a1147bbe46bf \ No newline at end of file diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 283a679ad4..78c19a0d10 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -638,6 +638,7 @@ typedef int (*sqlite3_loadext_entry)( /* Version 3.32.0 and later */ #define sqlite3_create_filename sqlite3_api->create_filename #define sqlite3_free_filename sqlite3_api->free_filename +#define sqlite3_database_file_object sqlite3_api->database_file_object #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) From 4d34766f4e537d570de4e280b1d4667244efed15 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Apr 2020 00:50:21 +0000 Subject: [PATCH 14/31] Clarify the comment on the sqlite3BtreeGetRequestedReserve() routine. No changes to code. FossilOrigin-Name: 52a6acca6d5d376308d354c02f4d676d9375c34c3841d7b1941196ee8b4e2511 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/btree.c | 4 ++++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 263d7581cc..c310f11c49 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3_database_file_object()\sinterface. -D 2020-04-21T20:19:25.607 +C Clarify\sthe\scomment\son\sthe\ssqlite3BtreeGetRequestedReserve()\sroutine.\nNo\schanges\sto\scode. +D 2020-04-22T00:50:21.766 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -473,7 +473,7 @@ F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 97673c54a3023845e7320cfa9e0de67fd6ca34b37a17a420981bb6e767601310 +F src/btree.c 02376eb7d49ccf31b53c2504f045ad74687c142a5c15ca837516e59e737867dc F src/btree.h 32672fa1aa74a7e9ab3aae822f94ffc8e732b1eb005988dc2283f91dc7573398 F src/btreeInt.h 887cdd2ea7f4a65143074a8a7c8928b0546f8c18dda3c06a408ce7992cbab0c0 F src/build.c ec6c0bda1e43ef55e5f5121a77ba19fac51fc6585f95ce2da795bcedcf6e8f36 @@ -1861,8 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2fc80ef16ce5878311ab88a0c64631813572ffbb71f75363b4619c9667e0926b 3cabe06b6e9433cbfb4bf7bd72a425f43b54f57d84a205adbaa6a1147bbe46bf -R cc7f5aa23076777d96573db8bef15752 -T +closed 3cabe06b6e9433cbfb4bf7bd72a425f43b54f57d84a205adbaa6a1147bbe46bf +P f534ebeaaf34f825550138f09f9a40221dfa7cd5c6537ef9f86dce5249025ec3 +R 0b277ec515801445e56dff650ba9aa79 U drh -Z c90bcd212b5c6fecb7eed98428e07741 +Z 6f8537667ffe93f8434cdb592b03d84e diff --git a/manifest.uuid b/manifest.uuid index 56d1422dc3..4d78ab2700 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f534ebeaaf34f825550138f09f9a40221dfa7cd5c6537ef9f86dce5249025ec3 \ No newline at end of file +52a6acca6d5d376308d354c02f4d676d9375c34c3841d7b1941196ee8b4e2511 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 75cdd25eb3..abe0b844f3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2913,6 +2913,10 @@ int sqlite3BtreeGetReserveNoMutex(Btree *p){ ** Return the number of bytes of space at the end of every page that ** are intentually left unused. This is the "reserved" space that is ** sometimes used by extensions. +** +** The value returned is the larger of the current reserve size and +** the latest reserve size requested by SQLITE_FILECTRL_RESERVE_BYTES. +** The amount of reserve can only grow - never shrink. */ int sqlite3BtreeGetRequestedReserve(Btree *p){ int n; From fc7f31742de0fea66610ac3211b40e27a467c923 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Apr 2020 11:11:17 +0000 Subject: [PATCH 15/31] Fix an integer overflow in fts3 causing a usan error. FossilOrigin-Name: e256f85289a78e629acdf83e5bf1f8df2a0ffb3d559738eb9e49db6c228dc8c0 --- ext/fts3/fts3.c | 34 ++++++++++++++++++++++++---------- ext/fts3/fts3Int.h | 1 + ext/fts3/fts3_expr.c | 5 +---- ext/fts3/fts3_write.c | 6 +++--- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- test/fts3misc.test | 8 ++++++++ 7 files changed, 49 insertions(+), 29 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index d03f3adf87..841d7448f9 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -962,6 +962,22 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ return zRet; } +/* +** Buffer z contains a positive integer value encoded as utf-8 text. +** Decode this value and store it in *pnOut, returning the number of bytes +** consumed. If an overflow error occurs return a negative value. +*/ +int sqlite3Fts3ReadInt(const char *z, int *pnOut){ + u64 iVal = 0; + int i; + for(i=0; z[i]>='0' && z[i]<='9'; i++){ + iVal = iVal*10 + (z[i] - '0'); + if( iVal>0x7FFFFFFF ) return -1; + } + *pnOut = (int)iVal; + return i; +} + /* ** This function interprets the string at (*pp) as a non-negative integer ** value. It reads the integer and sets *pnOut to the value read, then @@ -977,19 +993,17 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ */ static int fts3GobbleInt(const char **pp, int *pnOut){ const int MAX_NPREFIX = 10000000; - const char *p; /* Iterator pointer */ int nInt = 0; /* Output value */ - - for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ - nInt = nInt * 10 + (p[0] - '0'); - if( nInt>MAX_NPREFIX ){ - nInt = 0; - break; - } + int nByte; + nByte = sqlite3Fts3ReadInt(*pp, &nInt); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + } + if( nByte==0 ){ + return SQLITE_ERROR; } - if( p==*pp ) return SQLITE_ERROR; *pnOut = nInt; - *pp = p; + *pp += nByte; return SQLITE_OK; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 50370a9108..453afcebfd 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -591,6 +591,7 @@ int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); void sqlite3Fts3CreateStatTable(int*, Fts3Table*); int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); +int sqlite3Fts3ReadInt(const char *z, int *pnOut); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 5775fbca3c..e19137a03d 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -446,10 +446,7 @@ static int getNextNode( if( pKey->eType==FTSQUERY_NEAR ){ assert( nKey==4 ); if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ - nNear = 0; - for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ - nNear = nNear * 10 + (zInput[nKey] - '0'); - } + nKey += 1+sqlite3Fts3ReadInt(&zInput[nKey+1], &nNear); } } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 9a052a8f26..b9acc47dc5 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3069,11 +3069,11 @@ static void fts3ReadEndBlockField( if( zText ){ int i; int iMul = 1; - i64 iVal = 0; + u64 iVal = 0; for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } - *piEndBlock = iVal; + *piEndBlock = (i64)iVal; while( zText[i]==' ' ) i++; iVal = 0; if( zText[i]=='-' ){ @@ -3083,7 +3083,7 @@ static void fts3ReadEndBlockField( for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } - *pnByte = (iVal * (i64)iMul); + *pnByte = ((i64)iVal * (i64)iMul); } } diff --git a/manifest b/manifest index c310f11c49..08a28b321b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarify\sthe\scomment\son\sthe\ssqlite3BtreeGetRequestedReserve()\sroutine.\nNo\schanges\sto\scode. -D 2020-04-22T00:50:21.766 +C Fix\san\sinteger\soverflow\sin\sfts3\scausing\sa\susan\serror. +D 2020-04-22T11:11:17.450 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -82,11 +82,11 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 2a9dd452003a143248e68449302da80dd0c43df72195b56577e3562e43c408a0 +F ext/fts3/fts3.c de2cc136ccc6128e948ffd5d74636756014b2430d6237d7002c3bc3ceb1ae3ae F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h f091030b976045e7df91af2337935952b477cdbd9f48058c44c965684484cb50 +F ext/fts3/fts3Int.h 2c59cc46aefde134c1782e89a6a5384710ddcd4e783071337aa5d43d07269be3 F ext/fts3/fts3_aux.c 96708c8b3a7d9b8ca1b68ea2b7e503e283f20e95f145becadedfad096dbd0f34 -F ext/fts3/fts3_expr.c b132af223e90e35b9f9efa9fe63d6ae737d34153a3b6066736086df8abc78a1f +F ext/fts3/fts3_expr.c f081e38da641724cd72c20e23b71db2bf4d0c9517c14637442f6910259f11a34 F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116 @@ -100,7 +100,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c d5da5f010b2e2c1523f0e359ec43858bb724f608d3805d0e2a82ca2b466eb22e +F ext/fts3/fts3_write.c ed869b24d074f2498bdbef915d6db1f88c604ca5811502112061932a0bed5133 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 @@ -967,7 +967,7 @@ F test/fts3fuzz001.test e3c7b0ce9b04cc02281dcc96812a277f02df03cd7dc082055d87e11e F test/fts3join.test 949b4f5ae3ae9cc2423cb865d711e32476bdb205ab2be923fdf48246e4a44166 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 F test/fts3matchinfo.test aa66cc50615578b30f6df9984819ae5b702511cf8a94251ec7c594096a703a4a -F test/fts3misc.test 236f37a57d97fa1b7e0a4303aab7e02da87a9818c106e513ae88af76f25ace4a +F test/fts3misc.test 9ec15e7c0b5831a6353bd4c46bf3acdf1360eda5d9f396f667db4d05bcf92ecf F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905 F test/fts3offsets.test b85fd382abdc78ebce721d8117bd552dfb75094c F test/fts3prefix.test fa794eaab0bdae466494947b0b153d7844478ab2 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f534ebeaaf34f825550138f09f9a40221dfa7cd5c6537ef9f86dce5249025ec3 -R 0b277ec515801445e56dff650ba9aa79 -U drh -Z 6f8537667ffe93f8434cdb592b03d84e +P 52a6acca6d5d376308d354c02f4d676d9375c34c3841d7b1941196ee8b4e2511 +R 239e034db6f2572fb8fec1e8bff6f6f8 +U dan +Z e2a67ef3238147bad4f5c8a4180c4e31 diff --git a/manifest.uuid b/manifest.uuid index 4d78ab2700..96679a2c15 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52a6acca6d5d376308d354c02f4d676d9375c34c3841d7b1941196ee8b4e2511 \ No newline at end of file +e256f85289a78e629acdf83e5bf1f8df2a0ffb3d559738eb9e49db6c228dc8c0 \ No newline at end of file diff --git a/test/fts3misc.test b/test/fts3misc.test index 9becba9af7..a1bec42432 100644 --- a/test/fts3misc.test +++ b/test/fts3misc.test @@ -315,4 +315,12 @@ do_catchsql_test 10.1 { INSERT INTO f(f) VALUES ('merge=69,59'); } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE xyz USING fts3(); +} +do_execsql_test 11.1 { + SELECT * FROM xyz WHERE xyz MATCH 'a NEAR/4294836224 a'; +} + finish_test From 542bd6521e91c49e663123848e5898cd5632d097 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Apr 2020 13:49:25 +0000 Subject: [PATCH 16/31] Fix an off-by-one error in the "calculated" page count output from the sqlite3_analyzer utility for databases that are more than 1GB in size. FossilOrigin-Name: 8789368b91fb5b7477bdba3a953412fc3839b4894443b65186f7b8f79f6369c9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/spaceanal.tcl | 3 +++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 08a28b321b..2c6424a0a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sinteger\soverflow\sin\sfts3\scausing\sa\susan\serror. -D 2020-04-22T11:11:17.450 +C Fix\san\soff-by-one\serror\sin\sthe\s"calculated"\spage\scount\soutput\sfrom\sthe\nsqlite3_analyzer\sutility\sfor\sdatabases\sthat\sare\smore\sthan\s1GB\sin\ssize. +D 2020-04-22T13:49:25.923 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1818,7 +1818,7 @@ F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a80 F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c2a1 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe -F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec +F tool/spaceanal.tcl c161d838825d0242317c7cc13b1eb2126f8cec031950ef31114d42732cb2674e F tool/speed-check.sh 2b042d703a9472f08c3b13be27afac658426f8e4fc87cd2d575953fda86f08d1 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 52a6acca6d5d376308d354c02f4d676d9375c34c3841d7b1941196ee8b4e2511 -R 239e034db6f2572fb8fec1e8bff6f6f8 -U dan -Z e2a67ef3238147bad4f5c8a4180c4e31 +P e256f85289a78e629acdf83e5bf1f8df2a0ffb3d559738eb9e49db6c228dc8c0 +R aa22f4b98699ca6a0df355f11de7b138 +U drh +Z c8995117adf96d65ebda024f30ba4870 diff --git a/manifest.uuid b/manifest.uuid index 96679a2c15..2591c5adff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e256f85289a78e629acdf83e5bf1f8df2a0ffb3d559738eb9e49db6c228dc8c0 \ No newline at end of file +8789368b91fb5b7477bdba3a953412fc3839b4894443b65186f7b8f79f6369c9 \ No newline at end of file diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl index 3e08d3ffa7..3b33f8868e 100644 --- a/tool/spaceanal.tcl +++ b/tool/spaceanal.tcl @@ -587,6 +587,9 @@ set free_percent2 [percent $free_pgcnt2 $file_pgcnt] set file_pgcnt2 [expr {$inuse_pgcnt+$free_pgcnt2+$av_pgcnt}] +# Account for the lockbyte page +if {$file_pgcnt2*$pageSize>1073742335} {incr file_pgcnt2} + set ntable [db eval {SELECT count(*)+1 FROM sqlite_master WHERE type='table'}] set nindex [db eval {SELECT count(*) FROM sqlite_master WHERE type='index'}] set sql {SELECT count(*) FROM sqlite_master WHERE name LIKE 'sqlite_autoindex%'} From 5415ab49b48be86772557e6e03572ec2600398e7 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Apr 2020 20:45:46 +0000 Subject: [PATCH 17/31] Fix an uninitialized variable in the newly enhanced ".output" command of the CLI. FossilOrigin-Name: 65c6c26bb48d5347ce53bb3607de3a03a5a03946b232d35e46a20533f86750f8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2c6424a0a5..f9aed325cb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\serror\sin\sthe\s"calculated"\spage\scount\soutput\sfrom\sthe\nsqlite3_analyzer\sutility\sfor\sdatabases\sthat\sare\smore\sthan\s1GB\sin\ssize. -D 2020-04-22T13:49:25.923 +C Fix\san\suninitialized\svariable\sin\sthe\snewly\senhanced\s".output"\scommand\sof\nthe\sCLI. +D 2020-04-23T20:45:46.822 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -533,7 +533,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 -F src/shell.c.in 5402d3a7281576c96d788569778483746859c5e47052facfc4a4ccfac025d5da +F src/shell.c.in 1fc834b80c72dd37587ea87a4f4167cf5e6d98d12d143184ed2e732f529c0950 F src/sqlite.h.in fd6fcfe173accab8d9cb9a843856d9e9fb475f893b60a455e01d8739b5076f0e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 2d1af80082edffd71c6f96f70ad1ce6a4fb46615ad10291fc77fe0dea9ff0197 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e256f85289a78e629acdf83e5bf1f8df2a0ffb3d559738eb9e49db6c228dc8c0 -R aa22f4b98699ca6a0df355f11de7b138 +P 8789368b91fb5b7477bdba3a953412fc3839b4894443b65186f7b8f79f6369c9 +R 94debbbab259c50b5d510b556b34bbe9 U drh -Z c8995117adf96d65ebda024f30ba4870 +Z adea19ccf682c3a65d10a2e98fd48b5d diff --git a/manifest.uuid b/manifest.uuid index 2591c5adff..52a8c1756d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8789368b91fb5b7477bdba3a953412fc3839b4894443b65186f7b8f79f6369c9 \ No newline at end of file +65c6c26bb48d5347ce53bb3607de3a03a5a03946b232d35e46a20533f86750f8 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index a2351ce876..0218b71876 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -8346,7 +8346,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int i; int eMode = 0; int bBOM = 0; - int bOnce; + int bOnce = 0; /* 0: .output, 1: .once, 2: .excel */ if( c=='e' ){ eMode = 'x'; From 742efb67495c620b95697ffe0b2a224e3c3e4daf Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Apr 2020 17:55:52 +0000 Subject: [PATCH 18/31] New test case for ticket [1dcb4d44964846ad]. FossilOrigin-Name: 9e9f1e96c9aac60fcbbcda6923e01e350ca4dd88acefb9d400979419ba4e1e4d --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/whereL.test | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f9aed325cb..82884fd605 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\suninitialized\svariable\sin\sthe\snewly\senhanced\s".output"\scommand\sof\nthe\sCLI. -D 2020-04-23T20:45:46.822 +C New\stest\scase\sfor\sticket\s[1dcb4d44964846ad]. +D 2020-04-24T17:55:52.244 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1713,7 +1713,7 @@ F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test a2874062140ed4aba9ffae76e6190a3df6fc73d1373fdfa8fd632945082a5364 F test/whereJ.test 88287550f6ee604422403b053455b1ad894eeaa5c35d348532dfa1439286cb9a F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b -F test/whereL.test d19499a39c9e3e5a74460778b009558d328c8a230c0e6825c9996c9adff89058 +F test/whereL.test e05cedc9389c6f09ad55bd5999a3fddccebec90672fb989433c145dcdaf26996 F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864 F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3 F test/wherelimit.test 592081800806d297dd7449b1030c863d2883d6d42901837ccd2e5a9bd962edb0 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8789368b91fb5b7477bdba3a953412fc3839b4894443b65186f7b8f79f6369c9 -R 94debbbab259c50b5d510b556b34bbe9 +P 65c6c26bb48d5347ce53bb3607de3a03a5a03946b232d35e46a20533f86750f8 +R 9ce5e5b106a98ef938e13dec7c4ca6e7 U drh -Z adea19ccf682c3a65d10a2e98fd48b5d +Z ae06a854a112a06df1f524333bda4911 diff --git a/manifest.uuid b/manifest.uuid index 52a8c1756d..d24a56717c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65c6c26bb48d5347ce53bb3607de3a03a5a03946b232d35e46a20533f86750f8 \ No newline at end of file +9e9f1e96c9aac60fcbbcda6923e01e350ca4dd88acefb9d400979419ba4e1e4d \ No newline at end of file diff --git a/test/whereL.test b/test/whereL.test index 0f577e06eb..fbb424e919 100644 --- a/test/whereL.test +++ b/test/whereL.test @@ -157,4 +157,38 @@ do_execsql_test 600 { WHERE x='good' AND y='good'; } {good good} +# 2020-04-24: Another test case for the previous (1dcb4d44964846ad) +# ticket. The test case comes from +# https://stackoverflow.com/questions/61399253/sqlite3-different-result-in-console-compared-to-python-script/ +# Output verified against postgresql. +# +do_execsql_test 610 { + CREATE TABLE tableA( + ID int, + RunYearMonth int + ); + INSERT INTO tableA VALUES(1,202003),(2,202003),(3,202003),(4,202004), + (5,202004),(6,202004),(7,202004),(8,202004); + CREATE TABLE tableB ( + ID int, + RunYearMonth int + ); + INSERT INTO tableB VALUES(1,202004),(2,202004),(3,202004),(4,202004), + (5,202004); + SELECT * + FROM ( + SELECT * + FROM tableA + WHERE RunYearMonth = 202004 + ) AS A + INNER JOIN ( + SELECT * + FROM tableB + WHERE RunYearMonth = 202004 + ) AS B + ON A.ID = B.ID + AND A.RunYearMonth = B.RunYearMonth; +} {4 202004 4 202004 5 202004 5 202004} + + finish_test From ab2172e69f5dfb5017a6a7b230862c0a54f41e7d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Apr 2020 18:20:30 +0000 Subject: [PATCH 19/31] The new sqlite3_database_file_object() interface requires that the pager never invoke xOpen with SQLITE_OPEN_MAIN_JOURNAL unless it is using a pointer to the journal name found in the Pager structure itself. Make this the case when processing a master-journal. FossilOrigin-Name: b4987a5ced0c0f2c606c040e0c1b8ee11175f40ae35a7446308a43e77b1f1db2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 82884fd605..0b463f86f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scase\sfor\sticket\s[1dcb4d44964846ad]. -D 2020-04-24T17:55:52.244 +C The\snew\ssqlite3_database_file_object()\sinterface\srequires\sthat\sthe\spager\nnever\sinvoke\sxOpen\swith\sSQLITE_OPEN_MAIN_JOURNAL\sunless\sit\sis\susing\na\spointer\sto\sthe\sjournal\sname\sfound\sin\sthe\sPager\sstructure\sitself.\s\s\sMake\nthis\sthe\scase\swhen\sprocessing\sa\smaster-journal. +D 2020-04-24T18:20:30.239 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -519,7 +519,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 7ef8b60222558a373d89c18d0c3bc44365b45273a528183d40bec5bb76ce23fc F src/os_win.c 035a813cbd17f355bdcad7ab894af214a9c13a1db8aeac902365350b98cd45a7 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 29ee0618daa1f49c24431a9f21b7ac15fa148eb63626c9e0ac145fffbe0b209d +F src/pager.c 52cee2f72710be47b5b13ff66b339ca3855e5bc48e92a94114d2affedc70041f F src/pager.h 3b33619a90180e0874c7eca31d6f6ceb464d9322c6fb4e9a7bbb318c8a17bdb3 F src/parse.y c8eff38606f443d5ba245263fa7abc05e4116d95656e050c4b78e9bfbf931add F src/pcache.c 385ff064bca69789d199a98e2169445dc16e4291fa807babd61d4890c3b34177 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 65c6c26bb48d5347ce53bb3607de3a03a5a03946b232d35e46a20533f86750f8 -R 9ce5e5b106a98ef938e13dec7c4ca6e7 +P 9e9f1e96c9aac60fcbbcda6923e01e350ca4dd88acefb9d400979419ba4e1e4d +R cf6b32c8532bdb0b49d3f8cc9bb4c572 U drh -Z ae06a854a112a06df1f524333bda4911 +Z d6dc738e7c78a8bb07af2098cfd2f141 diff --git a/manifest.uuid b/manifest.uuid index d24a56717c..6957ea7ceb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9e9f1e96c9aac60fcbbcda6923e01e350ca4dd88acefb9d400979419ba4e1e4d \ No newline at end of file +b4987a5ced0c0f2c606c040e0c1b8ee11175f40ae35a7446308a43e77b1f1db2 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 3377762300..0d08a2dfaa 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2536,9 +2536,12 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ /* One of the journals pointed to by the master journal exists. ** Open it and check if it points at the master journal. If ** so, return without deleting the master journal file. + ** NB: zJournal is really a MAIN_JOURNAL. But call it a + ** MASTER_JOURNAL here so that the VFS will not send the zJournal + ** name into sqlite3_database_file_object(). */ int c; - int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL); + int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); rc = sqlite3OsOpen(pVfs, zJournal, pJournal, flags, 0); if( rc!=SQLITE_OK ){ goto delmaster_out; From 07f9e8f4f3a473e4efac5f421b59b945a79e22a5 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Apr 2020 15:01:53 +0000 Subject: [PATCH 20/31] Ensure affinity is not discarded from a view column if the view appears on the rhs of a LEFT JOIN. Fix for [45f4bf4e]. FossilOrigin-Name: ac31edd3eeafcef46164a4506bbc32c711bb7cd78378aeaa4c9bb12524ac5ea1 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 6 +++--- src/select.c | 1 + test/join2.test | 31 +++++++++++++++++++++++++++++++ 5 files changed, 45 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 0b463f86f3..71664328dc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\snew\ssqlite3_database_file_object()\sinterface\srequires\sthat\sthe\spager\nnever\sinvoke\sxOpen\swith\sSQLITE_OPEN_MAIN_JOURNAL\sunless\sit\sis\susing\na\spointer\sto\sthe\sjournal\sname\sfound\sin\sthe\sPager\sstructure\sitself.\s\s\sMake\nthis\sthe\scase\swhen\sprocessing\sa\smaster-journal. -D 2020-04-24T18:20:30.239 +C Ensure\saffinity\sis\snot\sdiscarded\sfrom\sa\sview\scolumn\sif\sthe\sview\sappears\son\sthe\srhs\sof\sa\sLEFT\sJOIN.\sFix\sfor\s[45f4bf4e]. +D 2020-04-25T15:01:53.394 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F src/date.c b29b349d277e3d579dcc295b24c0a2caed83fd8f090a9f7cbe6070c0fd662384 F src/dbpage.c 8a01e865bf8bc6d7b1844b4314443a6436c07c3efe1d488ed89e81719047833a F src/dbstat.c 793deaf88a0904f88285d93d6713c636d55ede0ffd9f08d10f4ea825531d367f F src/delete.c 11000121c4281c0bce4e41db29addfaea0038eaa127ece02557c9207bc3e541d -F src/expr.c b292bdecd64cd695109ceaa3c810f8b41f202368c75adb9ea680a875df5b0308 +F src/expr.c d1e1d42cbdec08bb867a1ab43a59b401d82ff2bc88bdcb4af20e479a5facb6d8 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 4b575423b0a5d4898b1a7868ce985cf1a8ad91c741c9abbb108ff02536d20f41 F src/func.c f3dcdc0e95509864767c1f0991b19360f969e44177f4e058fd51da9a6154f47e @@ -532,7 +532,7 @@ F src/printf.c 9be6945837c839ba57837b4bc3af349eba630920fa5532aa518816defe42a7d4 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c ab4eb1aee1bd066feea5b6eff264220ae54459019654264e9f688368a7d0c0b5 +F src/select.c c310de94bf67315054587c18a16e7a3e3dc3a98dc79168f0c2b776548d43f6cd F src/shell.c.in 1fc834b80c72dd37587ea87a4f4167cf5e6d98d12d143184ed2e732f529c0950 F src/sqlite.h.in fd6fcfe173accab8d9cb9a843856d9e9fb475f893b60a455e01d8739b5076f0e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1093,7 +1093,7 @@ F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test 75327829744e65cc8700e69340b8e6c192e10e39dfae7ccb0e970d3c4f49090a F test/join.test bca044589e94bb466e4c1e91fb6fecdc3f3326ca6b3f590f555f1958156eb321 -F test/join2.test 659bc6193f5c3fe20fa444dd2c91713db8c33e376b098b860644e175e87b8dbc +F test/join2.test 7d24d095ab88d3910228d53a3b548b7baf2e0e7d8aac6731a273e300e1b34b61 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 3a96dc62f0b45402d7207e22d1993fe0c2fce1c57644a11439891dd62b990eb7 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9e9f1e96c9aac60fcbbcda6923e01e350ca4dd88acefb9d400979419ba4e1e4d -R cf6b32c8532bdb0b49d3f8cc9bb4c572 -U drh -Z d6dc738e7c78a8bb07af2098cfd2f141 +P b4987a5ced0c0f2c606c040e0c1b8ee11175f40ae35a7446308a43e77b1f1db2 +R 71836ca611d61deceeaf6fb2607af00b +U dan +Z f46546e33cf07990942ccd1411251d86 diff --git a/manifest.uuid b/manifest.uuid index 6957ea7ceb..750cea59af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4987a5ced0c0f2c606c040e0c1b8ee11175f40ae35a7446308a43e77b1f1db2 \ No newline at end of file +ac31edd3eeafcef46164a4506bbc32c711bb7cd78378aeaa4c9bb12524ac5ea1 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 7165e4fa14..23e00db2ee 100644 --- a/src/expr.c +++ b/src/expr.c @@ -45,7 +45,7 @@ char sqlite3TableColumnAffinity(Table *pTab, int iCol){ char sqlite3ExprAffinity(const Expr *pExpr){ int op; while( ExprHasProperty(pExpr, EP_Skip) ){ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW ); pExpr = pExpr->pLeft; assert( pExpr!=0 ); } @@ -112,7 +112,7 @@ Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ */ Expr *sqlite3ExprSkipCollate(Expr *pExpr){ while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW ); pExpr = pExpr->pLeft; } return pExpr; @@ -131,7 +131,7 @@ Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; }else{ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW ); pExpr = pExpr->pLeft; } } diff --git a/src/select.c b/src/select.c index 76b9827e8c..4b7ba37f9f 100644 --- a/src/select.c +++ b/src/select.c @@ -3498,6 +3498,7 @@ static Expr *substExpr( ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; ifNullRow.iTable = pSubst->iNewTable; + ifNullRow.flags = EP_Skip; pCopy = &ifNullRow; } testcase( ExprHasProperty(pCopy, EP_Subquery) ); diff --git a/test/join2.test b/test/join2.test index bfcecda29b..82d597c584 100644 --- a/test/join2.test +++ b/test/join2.test @@ -293,5 +293,36 @@ do_execsql_test 8.1 { WHERE (t1.c0 BETWEEN 0 AND 0) > ('' AND t0.c0); } +#------------------------------------------------------------------------- +# Ticket [45f4bf4eb]. +# +reset_db +do_execsql_test 9.0 { + CREATE TABLE t0(c0 INT); + CREATE VIEW v0(c0) AS SELECT CAST(t0.c0 AS INTEGER) FROM t0; + INSERT INTO t0(c0) VALUES (0); +} + +do_execsql_test 9.1 { + SELECT typeof(c0), c0 FROM v0 WHERE c0>='0' +} {integer 0} + +do_execsql_test 9.2 { + SELECT * FROM t0, v0 WHERE v0.c0 >= '0'; +} {0 0} + +do_execsql_test 9.3 { + SELECT * FROM t0 LEFT JOIN v0 WHERE v0.c0 >= '0'; +} {0 0} + +do_execsql_test 9.4 { + SELECT * FROM t0 LEFT JOIN v0 ON v0.c0 >= '0'; +} {0 0} + +do_execsql_test 9.5 { + SELECT * FROM t0 LEFT JOIN v0 ON v0.c0 >= '0' WHERE TRUE + UNION SELECT 0,0 WHERE 0; +} {0 0} + finish_test From 7d9ad75a564986b06f1dc35fc745ebfed3e2fe45 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 25 Apr 2020 21:05:51 +0000 Subject: [PATCH 21/31] Fix the sqlite3_load_extension() interface so that it tolerates backslashes in place of forward-slashes in pathnames on Windows. FossilOrigin-Name: bc3bf7c6681a96bc18a1ed02f0ccced4731d5dab45f60c347dd1841706e6b62a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/loadext.c | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 71664328dc..1eba766207 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\saffinity\sis\snot\sdiscarded\sfrom\sa\sview\scolumn\sif\sthe\sview\sappears\son\sthe\srhs\sof\sa\sLEFT\sJOIN.\sFix\sfor\s[45f4bf4e]. -D 2020-04-25T15:01:53.394 +C Fix\sthe\ssqlite3_load_extension()\sinterface\sso\sthat\sit\stolerates\sbackslashes\nin\splace\sof\sforward-slashes\sin\spathnames\son\sWindows. +D 2020-04-25T21:05:51.928 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -495,7 +495,7 @@ F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 8e4211d04eb460c0694d486c6ba1c068d468c6f653c3f237869a802ad82854de F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c 4540fb20e8606b5cea7efee1d727cc69b20390ddc2e6a6c5515e20f96dcc205c +F src/loadext.c af284e30250a36af92bcf97de743bae57b215c7d3414e3b913b9d8f2e75c5214 F src/main.c 652a782cd7b6c6ddf7419fcaf06b8aa9440b7c815857241171c9bdf03ab6544c F src/malloc.c cabfef0d725e04c8abfe0231a556ed8b78bf329dcc3fddbf903f6cdcd53cf4e6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b4987a5ced0c0f2c606c040e0c1b8ee11175f40ae35a7446308a43e77b1f1db2 -R 71836ca611d61deceeaf6fb2607af00b -U dan -Z f46546e33cf07990942ccd1411251d86 +P ac31edd3eeafcef46164a4506bbc32c711bb7cd78378aeaa4c9bb12524ac5ea1 +R a6c62e244c55732737962b844b2337e6 +U drh +Z fbfd383a8d39b6b6bd4ef76163a1c79c diff --git a/manifest.uuid b/manifest.uuid index 750cea59af..b967c2d591 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac31edd3eeafcef46164a4506bbc32c711bb7cd78378aeaa4c9bb12524ac5ea1 \ No newline at end of file +bc3bf7c6681a96bc18a1ed02f0ccced4731d5dab45f60c347dd1841706e6b62a \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 5a661cdc26..aea06d4a55 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -581,7 +581,11 @@ static int sqlite3LoadExtension( return SQLITE_NOMEM_BKPT; } memcpy(zAltEntry, "sqlite3_", 8); +#if SQLITE_OS_WIN + for(iFile=ncFile-1; iFile>=0 && ((c=zFile[iFile]!='/')||c=='\\'); iFile--){} +#else for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} +#endif iFile++; if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ From ec9a25c7c701aaab2040ae3f500d9008de011c90 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 26 Apr 2020 14:33:54 +0000 Subject: [PATCH 22/31] Fix an issue with check-in [bc3bf7c6681a96bc] when compiling on Windows. FossilOrigin-Name: 57b16d8ca3d1ede3b411389256bec6686433aae716f47bca309ee7c8e5fe3128 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/loadext.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1eba766207..4b2602159b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3_load_extension()\sinterface\sso\sthat\sit\stolerates\sbackslashes\nin\splace\sof\sforward-slashes\sin\spathnames\son\sWindows. -D 2020-04-25T21:05:51.928 +C Fix\san\sissue\swith\scheck-in\s[bc3bf7c6681a96bc]\swhen\scompiling\son\sWindows. +D 2020-04-26T14:33:54.319 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -495,7 +495,7 @@ F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 8e4211d04eb460c0694d486c6ba1c068d468c6f653c3f237869a802ad82854de F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c af284e30250a36af92bcf97de743bae57b215c7d3414e3b913b9d8f2e75c5214 +F src/loadext.c acefcfb00f58b5c9d5763812c40fed82f11ded617ae8cb80a7bab540b429d039 F src/main.c 652a782cd7b6c6ddf7419fcaf06b8aa9440b7c815857241171c9bdf03ab6544c F src/malloc.c cabfef0d725e04c8abfe0231a556ed8b78bf329dcc3fddbf903f6cdcd53cf4e6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac31edd3eeafcef46164a4506bbc32c711bb7cd78378aeaa4c9bb12524ac5ea1 -R a6c62e244c55732737962b844b2337e6 +P bc3bf7c6681a96bc18a1ed02f0ccced4731d5dab45f60c347dd1841706e6b62a +R a1bd7af663ac8f68011b39c502abe2a2 U drh -Z fbfd383a8d39b6b6bd4ef76163a1c79c +Z dcff85cc0c5396a3824fe3ae621cc10a diff --git a/manifest.uuid b/manifest.uuid index b967c2d591..d57e584616 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bc3bf7c6681a96bc18a1ed02f0ccced4731d5dab45f60c347dd1841706e6b62a \ No newline at end of file +57b16d8ca3d1ede3b411389256bec6686433aae716f47bca309ee7c8e5fe3128 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index aea06d4a55..8dc632df85 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -582,7 +582,7 @@ static int sqlite3LoadExtension( } memcpy(zAltEntry, "sqlite3_", 8); #if SQLITE_OS_WIN - for(iFile=ncFile-1; iFile>=0 && ((c=zFile[iFile]!='/')||c=='\\'); iFile--){} + for(iFile=ncFile-1; iFile>=0 && ((c=zFile[iFile]!='/')&&c!='\\'); iFile--){} #else for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} #endif From e238e3141381dc5a6e811e94f0a1ab93c1bb2286 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 26 Apr 2020 22:04:48 +0000 Subject: [PATCH 23/31] Yet another attempt to enhance sqlite3_load_extension() so that it works with Window-style pathnames using a backslash separator character. FossilOrigin-Name: b73d9a7d6f7fec0ffc9640902a849289c305f8651e891388c01255c4da7a6c4b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/loadext.c | 14 +++++++++----- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 4b2602159b..e6fae833bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\swith\scheck-in\s[bc3bf7c6681a96bc]\swhen\scompiling\son\sWindows. -D 2020-04-26T14:33:54.319 +C Yet\sanother\sattempt\sto\senhance\ssqlite3_load_extension()\sso\sthat\sit\sworks\nwith\sWindow-style\spathnames\susing\sa\sbackslash\sseparator\scharacter. +D 2020-04-26T22:04:48.458 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -495,7 +495,7 @@ F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 8e4211d04eb460c0694d486c6ba1c068d468c6f653c3f237869a802ad82854de F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c acefcfb00f58b5c9d5763812c40fed82f11ded617ae8cb80a7bab540b429d039 +F src/loadext.c 421310045bd78afefb772294a99e50f37d87ae578786a6169074e6291e30d969 F src/main.c 652a782cd7b6c6ddf7419fcaf06b8aa9440b7c815857241171c9bdf03ab6544c F src/malloc.c cabfef0d725e04c8abfe0231a556ed8b78bf329dcc3fddbf903f6cdcd53cf4e6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bc3bf7c6681a96bc18a1ed02f0ccced4731d5dab45f60c347dd1841706e6b62a -R a1bd7af663ac8f68011b39c502abe2a2 +P 57b16d8ca3d1ede3b411389256bec6686433aae716f47bca309ee7c8e5fe3128 +R cae58a18aebbf7b95fb40ba841d77f19 U drh -Z dcff85cc0c5396a3824fe3ae621cc10a +Z a013006b51e7e5f259533e9a3800742b diff --git a/manifest.uuid b/manifest.uuid index d57e584616..b3a75f3c82 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -57b16d8ca3d1ede3b411389256bec6686433aae716f47bca309ee7c8e5fe3128 \ No newline at end of file +b73d9a7d6f7fec0ffc9640902a849289c305f8651e891388c01255c4da7a6c4b \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 8dc632df85..067c47c17f 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -480,6 +480,14 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_database_file_object, }; +/* True if x is the directory separator character +*/ +#if SQLITE_OS_WIN +# define DirSep(X) ((X)=='/'||(X)=='\\') +#else +# define DirSep(X) ((X)=='/') +#endif + /* ** Attempt to load an SQLite extension library contained in the file ** zFile. The entry point is zProc. zProc may be 0 in which case a @@ -581,11 +589,7 @@ static int sqlite3LoadExtension( return SQLITE_NOMEM_BKPT; } memcpy(zAltEntry, "sqlite3_", 8); -#if SQLITE_OS_WIN - for(iFile=ncFile-1; iFile>=0 && ((c=zFile[iFile]!='/')&&c!='\\'); iFile--){} -#else - for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} -#endif + for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} iFile++; if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ From 9f603fcefdaa30dd4cd4ce5b9dd83a8abb865acd Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Apr 2020 11:45:41 +0000 Subject: [PATCH 24/31] Use an AtomicLoad() macro in sqlite3HeapNearlyFull(). FossilOrigin-Name: 7556bc632e271d8a1e4fd836ce91e28213768ac09c90857b91171e9cd1009884 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e6fae833bc..b657255abb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Yet\sanother\sattempt\sto\senhance\ssqlite3_load_extension()\sso\sthat\sit\sworks\nwith\sWindow-style\spathnames\susing\sa\sbackslash\sseparator\scharacter. -D 2020-04-26T22:04:48.458 +C Use\san\sAtomicLoad()\smacro\sin\ssqlite3HeapNearlyFull(). +D 2020-04-28T11:45:41.489 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -497,7 +497,7 @@ F src/insert.c 8e4211d04eb460c0694d486c6ba1c068d468c6f653c3f237869a802ad82854de F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 421310045bd78afefb772294a99e50f37d87ae578786a6169074e6291e30d969 F src/main.c 652a782cd7b6c6ddf7419fcaf06b8aa9440b7c815857241171c9bdf03ab6544c -F src/malloc.c cabfef0d725e04c8abfe0231a556ed8b78bf329dcc3fddbf903f6cdcd53cf4e6 +F src/malloc.c 998984783063d4172bb11630ba586e6c6483ddfc8e24a9881a6f495128f23569 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 57b16d8ca3d1ede3b411389256bec6686433aae716f47bca309ee7c8e5fe3128 -R cae58a18aebbf7b95fb40ba841d77f19 +P b73d9a7d6f7fec0ffc9640902a849289c305f8651e891388c01255c4da7a6c4b +R 375d527c1770329817dc7ce201edba48 U drh -Z a013006b51e7e5f259533e9a3800742b +Z 59f2a3e0f3e029b2d7560f165c376f3c diff --git a/manifest.uuid b/manifest.uuid index b3a75f3c82..6ec6c7587e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b73d9a7d6f7fec0ffc9640902a849289c305f8651e891388c01255c4da7a6c4b \ No newline at end of file +7556bc632e271d8a1e4fd836ce91e28213768ac09c90857b91171e9cd1009884 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index a19d8bdfb3..6d149f3f98 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -179,7 +179,7 @@ int sqlite3MallocInit(void){ ** sqlite3_soft_heap_limit(). */ int sqlite3HeapNearlyFull(void){ - return mem0.nearlyFull; + return AtomicLoad(&mem0.nearlyFull); } /* From 23bef340beb7799bdac6d5fd857809139d9f3683 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Apr 2020 14:01:31 +0000 Subject: [PATCH 25/31] Use AtomicStore() when setting the mem0.nearlyFull boolean to avoid harmless TSAN warnings and to forestall doubts about threadsafety. FossilOrigin-Name: ce980af65a9b528f112baa22a95020a98ac5340155a0b53b09c46f99aad9b12b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b657255abb..af2896961d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\san\sAtomicLoad()\smacro\sin\ssqlite3HeapNearlyFull(). -D 2020-04-28T11:45:41.489 +C Use\sAtomicStore()\swhen\ssetting\sthe\smem0.nearlyFull\sboolean\sto\savoid\nharmless\sTSAN\swarnings\sand\sto\sforestall\sdoubts\sabout\sthreadsafety. +D 2020-04-28T14:01:31.572 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -497,7 +497,7 @@ F src/insert.c 8e4211d04eb460c0694d486c6ba1c068d468c6f653c3f237869a802ad82854de F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 421310045bd78afefb772294a99e50f37d87ae578786a6169074e6291e30d969 F src/main.c 652a782cd7b6c6ddf7419fcaf06b8aa9440b7c815857241171c9bdf03ab6544c -F src/malloc.c 998984783063d4172bb11630ba586e6c6483ddfc8e24a9881a6f495128f23569 +F src/malloc.c a3e13b001f988ecec6bdb90c0ea8912c8c786e623724d7098da623d8d01d19b1 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -1861,7 +1861,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b73d9a7d6f7fec0ffc9640902a849289c305f8651e891388c01255c4da7a6c4b -R 375d527c1770329817dc7ce201edba48 +P 7556bc632e271d8a1e4fd836ce91e28213768ac09c90857b91171e9cd1009884 +R 64df80a09bb93b99ef6e199142ecf936 U drh -Z 59f2a3e0f3e029b2d7560f165c376f3c +Z 42d1735aadc2a550c1baa98e0c20e909 diff --git a/manifest.uuid b/manifest.uuid index 6ec6c7587e..e4a586a16b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7556bc632e271d8a1e4fd836ce91e28213768ac09c90857b91171e9cd1009884 \ No newline at end of file +ce980af65a9b528f112baa22a95020a98ac5340155a0b53b09c46f99aad9b12b \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 6d149f3f98..45f3efff9d 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -111,7 +111,7 @@ sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ } mem0.alarmThreshold = n; nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - mem0.nearlyFull = (n>0 && n<=nUsed); + AtomicStore(&mem0.nearlyFull, n>0 && n<=nUsed); sqlite3_mutex_leave(mem0.mutex); excess = sqlite3_memory_used() - n; if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); @@ -243,7 +243,7 @@ static void mallocWithAlarm(int n, void **pp){ if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ - mem0.nearlyFull = 1; + AtomicStore(&mem0.nearlyFull, 1); sqlite3MallocAlarm(nFull); if( mem0.hardLimit ){ nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); @@ -253,7 +253,7 @@ static void mallocWithAlarm(int n, void **pp){ } } }else{ - mem0.nearlyFull = 0; + AtomicStore(&mem0.nearlyFull, 0); } } p = sqlite3GlobalConfig.m.xMalloc(nFull); From 065e4a8d291e4e55a2a18929184549f1245ead99 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Apr 2020 20:47:40 +0000 Subject: [PATCH 26/31] Add the cksumvfs extension. FossilOrigin-Name: 237c10f941cc6cb775693ae87513ff1b816f12b5e9c3d57b057421204d2d02cf --- ext/misc/cksumvfs.c | 769 ++++++++++++++++++++++++++++++++++++++++++++ manifest | 11 +- manifest.uuid | 2 +- 3 files changed, 776 insertions(+), 6 deletions(-) create mode 100644 ext/misc/cksumvfs.c diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c new file mode 100644 index 0000000000..cd4bb45364 --- /dev/null +++ b/ext/misc/cksumvfs.c @@ -0,0 +1,769 @@ +/* +** 2020-04-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a VFS shim that writes a checksum on each page +** of an SQLite database file. When reading pages, the checksum is verified +** and an error is raised if the checksum is incorrect. +** +** COMPILING +** +** This extension requires SQLite 3.32.0 or later. It uses the +** sqlite3_database_file_object() interface which was added in +** version 3.32.0, so it will not link with an earlier version of +** SQLite. +** +** To build this extension as a separately loaded shared library or +** DLL, use compiler command-lines similar to the following: +** +** (linux) gcc -fPIC -shared cksumvfs.c -o cksumvfs.so +** (mac) clang -fPIC -dynamiclib cksumvfs.c -o cksumvfs.dylib +** (windows) cl cksumvfs.c -link -dll -out:cksumvfs.dll +** +** You may want to add additional compiler options, of course, +** according to the needs of your project. +** +** If you want to statically link this extension with your product, +** then compile it like any other C-language module but add the +** "-DSQLITE_CKSUMVFS_STATIC" option so that this module knows that +** it is being statically linked rather than dynamically linked +** +** LOADING +** +** To load this extension as a shared library, you first have to +** bring up a dummy SQLite database connection to use as the argument +** to the sqlite3_load_extension() API call. Then you invoke the +** sqlite3_load_extension() API and shutdown the dummy database +** connection. All subsequent database connections that are opened +** will include this extension. For example: +** +** sqlite3 *db; +** sqlite3_open(":memory:", &db); +** sqlite3_load_extention(db, "./cksumvfs"); +** sqlite3_close(db); +** +** If this extension is compiled with -DSQLITE_CKSUMVFS_STATIC and +** statically linked against the application, initialize it using +** a single API call as follows: +** +** sqlite3_cksumvfs_init(); +** +** Cksumvfs is a VFS Shim. When loaded, "cksmvfs" becomes the new +** default VFS and it uses the prior default VFS as the next VFS +** down in the stack. This is normally what you want. However, it +** complex situations where multiple VFS shims are being loaded, +** it might be important to ensure that cksumvfs is loaded in the +** correct order so that it sequences itself into the default VFS +** Shim stack in the right order. +** +** USING +** +** Open database connections using the sqlite3_open() or +** sqlite3_open_v2() interfaces, as normal. Ordinary database files +** (without a checksum) will operate normally. Databases with +** checksums will return an SQLITE_CORRUPT error if a page is +** encountered that contains an invalid checksum. +** +** Checksumming only works on databases that have a reserve-bytes +** value of exactly 8. The default value for reserve-bytes is 0. +** Hence, newly created database files will omit the checksum by +** default. To create a database that includes a checksum, change +** the reserve-bytes value to 8 by runing: +** +** int n = 8; +** sqlite3_file_control(db, 0, SQLITE_FCNTL_RESERVED_BYTES, &n); +** +** If you do this immediately after creating a new database file, +** before anything else has been written into the file, then that +** might be all that you need to do. Otherwise, the API call +** above should be followed by: +** +** sqlite3_exec(db, "VACUUM", 0, 0, 0); +** +** It never hurts to run the VACUUM, even if you don't need it. +** If the database is in WAL mode, you should shutdown and +** reopen all database connections before continuing. +** +** From the CLI, use the ".filectrl reserve_bytes 8" command, +** followed by "VACUUM;". +** +** Note that SQLite allows the number of reserve-bytes to be +** increased but not decreased. So if a database file already +** has a reserve-bytes value greater than 8, there is no way to +** activate checksumming on that database, other than to dump +** and restore the database file. Note also that other extensions +** might also make use of the reserve-bytes. Checksumming will +** be incompatible with those other extensions. +** +** VERIFICATION OF CHECKSUMS +** +** If any checksum is incorrect, the "PRAGMA quick_check" command +** will find it. To verify that checksums are actually enabled +** and running, use the following query: +** +** SELECT count(*), verify_checksum(data) +** FROM sqlite_dbpage +** GROUP BY 2; +** +** There are three possible outputs form the verify_checksum() +** function: 1, 0, and NULL. 1 is returned if the checksum is +** correct. 0 is returned if the checksum is incorrect. NULL +** is returned if the page is unreadable. If checksumming is +** enabled, the read will fail if the checksum is wrong, so the +** usual result from verify_checksum() on a bad checksum is NULL. +** +** If everything is OK, the query above should return a single +** row where the second column is 1. Any other result indicates +** either that there is a checksum error, or checksum validation +** is disabled. +** +** CONTROLLING CHECKSUM VERIFICATION +** +** The cksumvfs extension implements a new PRAGMA statement that can +** be used to disable, re-enable, or query the status of checksum +** verification: +** +** PRAGMA checksum_verification; -- query status +** PRAGMA checksum_verification=OFF; -- disable verification +** PRAGMA checksum_verification=ON; -- re-enable verification +** +** The "checksum_verification" pragma will return "1" (true) or "0" +** (false) if checksum verification is enabled or disabled, respectively. +** "Verification" in this context means the feature that causes +** SQLITE_CORRUPT errors if a checksum mismatch is detected while +** reading. Checksums are always kept up-to-date as long as the +** reserve-bytes value of the database is 8, regardless of the setting +** of this pragma. Checksum verification can be disabled (for example) +** to do forensic analysis of a database that has previously reported +** a checksum error. +** +** The "checksum_verification" pragma will always respond with "0" if +** the database file does not have a reserve-bytes value of 8. The +** pragma will return no rows at all if the cksumvfs extension is +** not loaded. +** +** IMPLEMENTATION NOTES +** +** The checksum is stored in the last 8 bytes of each page. This +** module only operates if the "bytes of reserved space on each page" +** value at offset 20 the SQLite database header is exactly 8. If +** the reserved-space value is not 8, this module is a no-op. +*/ +#ifdef SQLITE_CKSUMVFS_STATIC +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif +#include +#include + + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs CksmVfs; +typedef struct CksmFile CksmFile; + +/* +** Useful datatype abbreviations +*/ +#if !defined(SQLITE_CORE) + typedef unsigned char u8; + typedef unsigned int u32; +#endif + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((CksmFile*)(p))+1)) + +/* An open file */ +struct CksmFile { + sqlite3_file base; /* IO methods */ + char computeCksm; /* True to compute checksums. + ** Always true if reserve size is 8. */ + char verifyCksm; /* True to verify checksums */ + char isWal; /* True if processing a WAL file */ + CksmFile *pPartner; /* Ptr from WAL to main-db, or from main-db to WAL */ +}; + +/* +** Methods for CksmFile +*/ +static int cksmClose(sqlite3_file*); +static int cksmRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int cksmWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int cksmTruncate(sqlite3_file*, sqlite3_int64 size); +static int cksmSync(sqlite3_file*, int flags); +static int cksmFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int cksmLock(sqlite3_file*, int); +static int cksmUnlock(sqlite3_file*, int); +static int cksmCheckReservedLock(sqlite3_file*, int *pResOut); +static int cksmFileControl(sqlite3_file*, int op, void *pArg); +static int cksmSectorSize(sqlite3_file*); +static int cksmDeviceCharacteristics(sqlite3_file*); +static int cksmShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int cksmShmLock(sqlite3_file*, int offset, int n, int flags); +static void cksmShmBarrier(sqlite3_file*); +static int cksmShmUnmap(sqlite3_file*, int deleteFlag); +static int cksmFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int cksmUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for CksmVfs +*/ +static int cksmOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int cksmDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int cksmAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int cksmFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *cksmDlOpen(sqlite3_vfs*, const char *zFilename); +static void cksmDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void cksmDlClose(sqlite3_vfs*, void*); +static int cksmRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int cksmSleep(sqlite3_vfs*, int microseconds); +static int cksmCurrentTime(sqlite3_vfs*, double*); +static int cksmGetLastError(sqlite3_vfs*, int, char *); +static int cksmCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int cksmSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr cksmGetSystemCall(sqlite3_vfs*, const char *z); +static const char *cksmNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs cksm_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "cksmvfs", /* zName */ + 0, /* pAppData (set when registered) */ + cksmOpen, /* xOpen */ + cksmDelete, /* xDelete */ + cksmAccess, /* xAccess */ + cksmFullPathname, /* xFullPathname */ + cksmDlOpen, /* xDlOpen */ + cksmDlError, /* xDlError */ + cksmDlSym, /* xDlSym */ + cksmDlClose, /* xDlClose */ + cksmRandomness, /* xRandomness */ + cksmSleep, /* xSleep */ + cksmCurrentTime, /* xCurrentTime */ + cksmGetLastError, /* xGetLastError */ + cksmCurrentTimeInt64, /* xCurrentTimeInt64 */ + cksmSetSystemCall, /* xSetSystemCall */ + cksmGetSystemCall, /* xGetSystemCall */ + cksmNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods cksm_io_methods = { + 3, /* iVersion */ + cksmClose, /* xClose */ + cksmRead, /* xRead */ + cksmWrite, /* xWrite */ + cksmTruncate, /* xTruncate */ + cksmSync, /* xSync */ + cksmFileSize, /* xFileSize */ + cksmLock, /* xLock */ + cksmUnlock, /* xUnlock */ + cksmCheckReservedLock, /* xCheckReservedLock */ + cksmFileControl, /* xFileControl */ + cksmSectorSize, /* xSectorSize */ + cksmDeviceCharacteristics, /* xDeviceCharacteristics */ + cksmShmMap, /* xShmMap */ + cksmShmLock, /* xShmLock */ + cksmShmBarrier, /* xShmBarrier */ + cksmShmUnmap, /* xShmUnmap */ + cksmFetch, /* xFetch */ + cksmUnfetch /* xUnfetch */ +}; + +/* Do byte swapping on a unsigned 32-bit integer */ +#define BYTESWAP32(x) ( \ + (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ +) + +/* Compute a checksum on a buffer */ +static void cksmCompute( + u8 *a, /* Content to be checksummed */ + int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */ + u8 *aOut /* OUT: Final 8-byte checksum value output */ +){ + u32 s1 = 0, s2 = 0; + u32 *aData = (u32*)a; + u32 *aEnd = (u32*)&a[nByte]; + u32 x = 1; + + assert( nByte>=8 ); + assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); + + if( 1 == *(u8*)&x ){ + /* Little-endian */ + do { + s1 += *aData++ + s2; + s2 += *aData++ + s1; + }while( aData65536 || (nByte & (nByte-1))!=0 ) return; + cksmCompute(data, nByte-8, cksum); + sqlite3_result_int(context, memcmp(data+nByte-8,cksum,8)==0); +} + +/* +** Close a cksm-file. +*/ +static int cksmClose(sqlite3_file *pFile){ + CksmFile *p = (CksmFile *)pFile; + if( p->pPartner ){ + assert( p->pPartner->pPartner==p ); + p->pPartner->pPartner = 0; + p->pPartner = 0; + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Read data from a cksm-file. +*/ +static int cksmRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK ){ + if( iOfst==0 && iAmt>=100 && memcmp(zBuf,"SQLite format 3",16)==0 ){ + u8 *d = (u8*)zBuf; + char hasCorrectReserveSize = (d[20]==8); + if( hasCorrectReserveSize!=p->computeCksm ){ + p->computeCksm = p->verifyCksm = hasCorrectReserveSize; + } + } + /* Verify the checksum if (1) the size seems appropriate for a database + ** page, and if (2) checksum verification is enabled. But (3) do not + ** verify the checksums on a WAL page if the main database file + ** has subsequently turned off checksums. + */ + if( iAmt>=512 /* (1) */ + && p->verifyCksm /* (2) */ + && (!p->isWal || (p->pPartner!=0 && p->pPartner->verifyCksm)) /* (3) */ + ){ + u8 cksum[8]; + cksmCompute((u8*)zBuf, iAmt-8, cksum); + if( memcmp(zBuf+iAmt-8, cksum, 8)!=0 ){ + rc = SQLITE_CORRUPT; + } + } + } + return rc; +} + +/* +** Write data to a cksm-file. +*/ +static int cksmWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(pFile); + if( iOfst==0 && iAmt>=100 && memcmp(zBuf,"SQLite format 3",16)==0 ){ + u8 *d = (u8*)zBuf; + char hasCorrectReserveSize = (d[20]==8); + if( hasCorrectReserveSize!=p->computeCksm ){ + p->computeCksm = p->verifyCksm = hasCorrectReserveSize; + } + } + /* If the write size is appropriate for a database page and if + ** checksums where ever enabled, then it will be safe to compute + ** the checksums. The reserve byte size might have increased, but + ** it will never decrease. And because it cannot decrease, the + ** checksum will not overwrite anything. + */ + if( iAmt>=512 && p->computeCksm ){ + cksmCompute((u8*)zBuf, iAmt-8, ((u8*)zBuf)+iAmt-8); + } + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst); +} + +/* +** Truncate a cksm-file. +*/ +static int cksmTruncate(sqlite3_file *pFile, sqlite_int64 size){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xTruncate(pFile, size); +} + +/* +** Sync a cksm-file. +*/ +static int cksmSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of a cksm-file. +*/ +static int cksmFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(p); + return pFile->pMethods->xFileSize(pFile, pSize); +} + +/* +** Lock a cksm-file. +*/ +static int cksmLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock a cksm-file. +*/ +static int cksmUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on a cksm-file. +*/ +static int cksmCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on a cksm-file. +*/ +static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){ + int rc; + CksmFile *p = (CksmFile*)pFile; + pFile = ORIGFILE(pFile); + if( op==SQLITE_FCNTL_PRAGMA ){ + char **azArg = (char**)pArg; + assert( azArg[1]!=0 ); + if( strcmp(azArg[1],"checksum_verification")==0 ){ + char *zArg = azArg[2]; + if( zArg!=0 ){ + if( (zArg[0]>='1' && zArg[0]<='9') + || sqlite3_strlike("enable%",zArg,0)==0 + || sqlite3_stricmp("yes",zArg)==0 + || sqlite3_stricmp("on",zArg)==0 + ){ + p->verifyCksm = p->computeCksm; + }else{ + p->verifyCksm = 0; + } + } + azArg[0] = sqlite3_mprintf("%d",p->verifyCksm); + return SQLITE_OK; + } + } + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("cksm/%z", *(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for a cksm-file. +*/ +static int cksmSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by a cksm-file. +*/ +static int cksmDeviceCharacteristics(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xDeviceCharacteristics(pFile); +} + +/* Create a shared memory file mapping */ +static int cksmShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int cksmShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void cksmShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int cksmShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int cksmFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + CksmFile *p = (CksmFile *)pFile; + if( p->computeCksm ){ + *pp = 0; + return SQLITE_OK; + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xFetch(pFile, iOfst, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int cksmUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnfetch(pFile, iOfst, pPage); +} + +/* +** Open a cksm file handle. +*/ +static int cksmOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + CksmFile *p; + sqlite3_file *pSubFile; + sqlite3_vfs *pSubVfs; + int rc; + pSubVfs = ORIGVFS(pVfs); + if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ + return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + } + p = (CksmFile*)pFile; + memset(p, 0, sizeof(*p)); + pSubFile = ORIGFILE(pFile); + p->base.pMethods = &cksm_io_methods; + rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); + if( rc ) goto cksm_open_done; + if( flags & SQLITE_OPEN_WAL ){ + sqlite3_file *pDb = sqlite3_database_file_object(zName); + p->pPartner = (CksmFile*)pDb; + assert( p->pPartner->pPartner==0 ); + p->pPartner->pPartner = p; + p->isWal = 1; + p->computeCksm = p->pPartner->computeCksm; + }else{ + p->isWal = 0; + p->computeCksm = 0; + } +cksm_open_done: + if( rc ) pFile->pMethods = 0; + return rc; +} + +/* +** All other VFS methods are pass-thrus. +*/ +static int cksmDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} +static int cksmAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); +} +static int cksmFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); +} +static void *cksmDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} +static void cksmDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} +static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} +static void cksmDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} +static int cksmRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} +static int cksmSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} +static int cksmCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +static int cksmGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int cksmCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} +static int cksmSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr cksmGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *cksmNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} + +/* Register the verify_checksum() SQL function. +*/ +static int cksmRegisterFunc( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + if( db==0 ) return SQLITE_OK; + rc = sqlite3_create_function(db, "verify_checksum", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, cksmVerifyFunc, 0, 0); + return rc; +} + +/* +** Register the cksum VFS as the default VFS for the system. +** Also make arrangements to automatically register the "verify_checksum()" +** SQL function on each new database connection. +*/ +static int cksmRegisterVfs(void){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig; + if( sqlite3_vfs_find("cksum")!=0 ) return SQLITE_OK; + pOrig = sqlite3_vfs_find(0); + cksm_vfs.iVersion = pOrig->iVersion; + cksm_vfs.pAppData = pOrig; + cksm_vfs.szOsFile = pOrig->szOsFile + sizeof(CksmFile); + rc = sqlite3_vfs_register(&cksm_vfs, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))cksmRegisterFunc); + } + return rc; +} + +#if defined(SQLITE_CKSUMVFS_STATIC) +/* This variant of the initializer runs when the extension is +** statically linked. +*/ +int sqlite3_register_cksumvfs(const char *NotUsed){ + (void)NotUsed; + return cksmRegisterVfs(); +} +#endif /* defined(SQLITE_CKSUMVFS_STATIC */ + +#if !defined(SQLITE_CKSUMVFS_STATIC) +/* This variant of the initializer function is used when the +** extension is shared library to be loaded at run-time. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called by sqlite3_load_extension() when the +** extension is first loaded. +***/ +int sqlite3_cksumvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* not used */ + rc = cksmRegisterFunc(db, 0, 0); + if( rc==SQLITE_OK ){ + + } + if( rc==SQLITE_OK ){ + rc = cksmRegisterVfs(); + } + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} +#endif /* !defined(SQLITE_CKSUMVFS_STATIC) */ diff --git a/manifest b/manifest index af2896961d..95b5c2849e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sAtomicStore()\swhen\ssetting\sthe\smem0.nearlyFull\sboolean\sto\savoid\nharmless\sTSAN\swarnings\sand\sto\sforestall\sdoubts\sabout\sthreadsafety. -D 2020-04-28T14:01:31.572 +C Add\sthe\scksumvfs\sextension. +D 2020-04-28T20:47:40.963 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -285,6 +285,7 @@ F ext/misc/appendvfs.c 3777f22ec1057dc4e5fd89f2fbddcc7a29fbeef1ad038c736c54411bb F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c 26004b7a6be320ec08fc20ca8d0f01fccb00a98cbe0f3197446794ff2a506aa3 F ext/misc/carray.c 91e9a7f512fda934894bed30464552fffa7d3073b5be04189ae0bd0c59f26bfd +F ext/misc/cksumvfs.c fe6f6a9de21ec23d8bc1a27f349cc7f28ae2095c6b666527c5df672209b603f8 F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c85c243 F ext/misc/completion.c a0efe03edfdc4f717c61e6c9b0bfe2708ff7878010dae3174980a68fdf76aabc F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 @@ -1861,7 +1862,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7556bc632e271d8a1e4fd836ce91e28213768ac09c90857b91171e9cd1009884 -R 64df80a09bb93b99ef6e199142ecf936 +P ce980af65a9b528f112baa22a95020a98ac5340155a0b53b09c46f99aad9b12b +R 04c0fbb7c237ebfacc8bf51983c12e78 U drh -Z 42d1735aadc2a550c1baa98e0c20e909 +Z d04720d0bf98441f3ea328cca7d97af5 diff --git a/manifest.uuid b/manifest.uuid index e4a586a16b..e39ca5b808 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ce980af65a9b528f112baa22a95020a98ac5340155a0b53b09c46f99aad9b12b \ No newline at end of file +237c10f941cc6cb775693ae87513ff1b816f12b5e9c3d57b057421204d2d02cf \ No newline at end of file From 451f89a896f01b8e998364e3204c600d9089b802 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Apr 2020 23:09:56 +0000 Subject: [PATCH 27/31] Fix incorrect error message when something goes wrong with the sqlite3_dbpage() table-valued function in the .dbinfo command of the CLI. FossilOrigin-Name: 0dcf002463f5931c3875d2038d2b97298f1800b1cdfa70485d6430ab758f3b25 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 7 +------ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 95b5c2849e..617111c321 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\scksumvfs\sextension. -D 2020-04-28T20:47:40.963 +C Fix\sincorrect\serror\smessage\swhen\ssomething\sgoes\swrong\swith\sthe\nsqlite3_dbpage()\stable-valued\sfunction\sin\sthe\s.dbinfo\scommand\sof\sthe\sCLI. +D 2020-04-28T23:09:56.084 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -534,7 +534,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c310de94bf67315054587c18a16e7a3e3dc3a98dc79168f0c2b776548d43f6cd -F src/shell.c.in 1fc834b80c72dd37587ea87a4f4167cf5e6d98d12d143184ed2e732f529c0950 +F src/shell.c.in 86cd0f0412b9739b769fafdfcad28f731882d522042a95c30ab033a5eba68b03 F src/sqlite.h.in fd6fcfe173accab8d9cb9a843856d9e9fb475f893b60a455e01d8739b5076f0e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 2d1af80082edffd71c6f96f70ad1ce6a4fb46615ad10291fc77fe0dea9ff0197 @@ -1862,7 +1862,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ce980af65a9b528f112baa22a95020a98ac5340155a0b53b09c46f99aad9b12b -R 04c0fbb7c237ebfacc8bf51983c12e78 +P 237c10f941cc6cb775693ae87513ff1b816f12b5e9c3d57b057421204d2d02cf +R 176a5d897ae4a00cb2a086f8e4bc1304 U drh -Z d04720d0bf98441f3ea328cca7d97af5 +Z 9c42efa8d6d984bdc00c683cbdf1d894 diff --git a/manifest.uuid b/manifest.uuid index e39ca5b808..097222261f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -237c10f941cc6cb775693ae87513ff1b816f12b5e9c3d57b057421204d2d02cf \ No newline at end of file +0dcf002463f5931c3875d2038d2b97298f1800b1cdfa70485d6430ab758f3b25 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 0218b71876..f65caaa8ac 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5067,12 +5067,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", -1, &pStmt, 0); if( rc ){ - if( !sqlite3_compileoption_used("ENABLE_DBPAGE_VTAB") ){ - utf8_printf(stderr, "the \".dbinfo\" command requires the " - "-DSQLITE_ENABLE_DBPAGE_VTAB compile-time options\n"); - }else{ - utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); - } + utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); return 1; } From 95063d8c90b8525d3884b598c417c2b281cbeb25 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Apr 2020 01:09:46 +0000 Subject: [PATCH 28/31] Do not allow page_size changes on an active cksumvfs database. FossilOrigin-Name: 2c17cdce26fd935e6d81ff828f4670291fc014013c93b2a1578506598345ee86 --- ext/misc/cksumvfs.c | 6 +++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c index cd4bb45364..8b7f37b9e1 100644 --- a/ext/misc/cksumvfs.c +++ b/ext/misc/cksumvfs.c @@ -494,7 +494,7 @@ static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){ if( op==SQLITE_FCNTL_PRAGMA ){ char **azArg = (char**)pArg; assert( azArg[1]!=0 ); - if( strcmp(azArg[1],"checksum_verification")==0 ){ + if( sqlite3_stricmp(azArg[1],"checksum_verification")==0 ){ char *zArg = azArg[2]; if( zArg!=0 ){ if( (zArg[0]>='1' && zArg[0]<='9') @@ -509,6 +509,10 @@ static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){ } azArg[0] = sqlite3_mprintf("%d",p->verifyCksm); return SQLITE_OK; + }else if( p->computeCksm && azArg[2]!=0 + && sqlite3_stricmp(azArg[1], "page_size")==0 ){ + /* Do not allow page size changes on a checksum database */ + return SQLITE_OK; } } rc = pFile->pMethods->xFileControl(pFile, op, pArg); diff --git a/manifest b/manifest index 617111c321..290630f678 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sincorrect\serror\smessage\swhen\ssomething\sgoes\swrong\swith\sthe\nsqlite3_dbpage()\stable-valued\sfunction\sin\sthe\s.dbinfo\scommand\sof\sthe\sCLI. -D 2020-04-28T23:09:56.084 +C Do\snot\sallow\spage_size\schanges\son\san\sactive\scksumvfs\sdatabase. +D 2020-04-29T01:09:46.288 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -285,7 +285,7 @@ F ext/misc/appendvfs.c 3777f22ec1057dc4e5fd89f2fbddcc7a29fbeef1ad038c736c54411bb F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c 26004b7a6be320ec08fc20ca8d0f01fccb00a98cbe0f3197446794ff2a506aa3 F ext/misc/carray.c 91e9a7f512fda934894bed30464552fffa7d3073b5be04189ae0bd0c59f26bfd -F ext/misc/cksumvfs.c fe6f6a9de21ec23d8bc1a27f349cc7f28ae2095c6b666527c5df672209b603f8 +F ext/misc/cksumvfs.c 452a699066cd2f60224525aa36301a95c9784f230b56fa67de63f76afd6c68d7 F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c85c243 F ext/misc/completion.c a0efe03edfdc4f717c61e6c9b0bfe2708ff7878010dae3174980a68fdf76aabc F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 @@ -1862,7 +1862,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 237c10f941cc6cb775693ae87513ff1b816f12b5e9c3d57b057421204d2d02cf -R 176a5d897ae4a00cb2a086f8e4bc1304 +P 0dcf002463f5931c3875d2038d2b97298f1800b1cdfa70485d6430ab758f3b25 +R 0cacdd864635f1bdd3a6aa2f972db6cd U drh -Z 9c42efa8d6d984bdc00c683cbdf1d894 +Z db05e4f978a71bce6c644e170e98280e diff --git a/manifest.uuid b/manifest.uuid index 097222261f..b3cba08095 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0dcf002463f5931c3875d2038d2b97298f1800b1cdfa70485d6430ab758f3b25 \ No newline at end of file +2c17cdce26fd935e6d81ff828f4670291fc014013c93b2a1578506598345ee86 \ No newline at end of file From cdb6ce980b348ed106e16132c5b2262535b01b46 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 May 2020 11:31:43 +0000 Subject: [PATCH 29/31] Add the new SQLITE_IOERR_DATA result code and use it in cksumvfs. Also enhance cksumvfs to emit an sqlite3_log() message whenever it finds an invalid checksum. FossilOrigin-Name: a094e8bfdef10d9e5d97f5a9f7c15b0fc547358b83367762ba5c2dfd2c9d0117 --- ext/misc/cksumvfs.c | 11 ++++++++--- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 1 + 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c index 8b7f37b9e1..57f91f3c1a 100644 --- a/ext/misc/cksumvfs.c +++ b/ext/misc/cksumvfs.c @@ -69,7 +69,7 @@ ** Open database connections using the sqlite3_open() or ** sqlite3_open_v2() interfaces, as normal. Ordinary database files ** (without a checksum) will operate normally. Databases with -** checksums will return an SQLITE_CORRUPT error if a page is +** checksums will return an SQLITE_IOERR_DATA error if a page is ** encountered that contains an invalid checksum. ** ** Checksumming only works on databases that have a reserve-bytes @@ -138,7 +138,7 @@ ** The "checksum_verification" pragma will return "1" (true) or "0" ** (false) if checksum verification is enabled or disabled, respectively. ** "Verification" in this context means the feature that causes -** SQLITE_CORRUPT errors if a checksum mismatch is detected while +** SQLITE_IOERR_DATA errors if a checksum mismatch is detected while ** reading. Checksums are always kept up-to-date as long as the ** reserve-bytes value of the database is 8, regardless of the setting ** of this pragma. Checksum verification can be disabled (for example) @@ -190,6 +190,7 @@ typedef struct CksmFile CksmFile; /* An open file */ struct CksmFile { sqlite3_file base; /* IO methods */ + const char *zFName; /* Original name of the file */ char computeCksm; /* True to compute checksums. ** Always true if reserve size is 8. */ char verifyCksm; /* True to verify checksums */ @@ -398,7 +399,10 @@ static int cksmRead( u8 cksum[8]; cksmCompute((u8*)zBuf, iAmt-8, cksum); if( memcmp(zBuf+iAmt-8, cksum, 8)!=0 ){ - rc = SQLITE_CORRUPT; + sqlite3_log(SQLITE_IOERR_DATA, + "checksum fault offset %lld of \"%s\"", + iOfst, p->zFName); + rc = SQLITE_IOERR_DATA; } } } @@ -625,6 +629,7 @@ static int cksmOpen( p->isWal = 0; p->computeCksm = 0; } + p->zFName = zName; cksm_open_done: if( rc ) pFile->pMethods = 0; return rc; diff --git a/manifest b/manifest index 290630f678..984be3d25f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\spage_size\schanges\son\san\sactive\scksumvfs\sdatabase. -D 2020-04-29T01:09:46.288 +C Add\sthe\snew\sSQLITE_IOERR_DATA\sresult\scode\sand\suse\sit\sin\scksumvfs.\nAlso\senhance\scksumvfs\sto\semit\san\ssqlite3_log()\smessage\swhenever\sit\sfinds\nan\sinvalid\schecksum. +D 2020-05-01T11:31:43.081 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -285,7 +285,7 @@ F ext/misc/appendvfs.c 3777f22ec1057dc4e5fd89f2fbddcc7a29fbeef1ad038c736c54411bb F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c 26004b7a6be320ec08fc20ca8d0f01fccb00a98cbe0f3197446794ff2a506aa3 F ext/misc/carray.c 91e9a7f512fda934894bed30464552fffa7d3073b5be04189ae0bd0c59f26bfd -F ext/misc/cksumvfs.c 452a699066cd2f60224525aa36301a95c9784f230b56fa67de63f76afd6c68d7 +F ext/misc/cksumvfs.c 755627c9112a000bc89653cb5038e080e69c58fa0a5c8e9ebd747f24e5fc01c9 F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c85c243 F ext/misc/completion.c a0efe03edfdc4f717c61e6c9b0bfe2708ff7878010dae3174980a68fdf76aabc F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 @@ -535,7 +535,7 @@ F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c310de94bf67315054587c18a16e7a3e3dc3a98dc79168f0c2b776548d43f6cd F src/shell.c.in 86cd0f0412b9739b769fafdfcad28f731882d522042a95c30ab033a5eba68b03 -F src/sqlite.h.in fd6fcfe173accab8d9cb9a843856d9e9fb475f893b60a455e01d8739b5076f0e +F src/sqlite.h.in 177084a0cef8c3b1df835bdec9530f102026bf0b6e9492a32349d62c30113a66 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 2d1af80082edffd71c6f96f70ad1ce6a4fb46615ad10291fc77fe0dea9ff0197 F src/sqliteInt.h 0f3848c46310d197246003f052985b72d1cdbfc0b31e069db76cb5231062fa1d @@ -1862,7 +1862,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0dcf002463f5931c3875d2038d2b97298f1800b1cdfa70485d6430ab758f3b25 -R 0cacdd864635f1bdd3a6aa2f972db6cd +P 2c17cdce26fd935e6d81ff828f4670291fc014013c93b2a1578506598345ee86 +R 1e877c1fea766b151c81954ab2c1dc73 U drh -Z db05e4f978a71bce6c644e170e98280e +Z dd4cd34bda503d060e1073825c3c95ba diff --git a/manifest.uuid b/manifest.uuid index b3cba08095..7d94dfd16a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c17cdce26fd935e6d81ff828f4670291fc014013c93b2a1578506598345ee86 \ No newline at end of file +a094e8bfdef10d9e5d97f5a9f7c15b0fc547358b83367762ba5c2dfd2c9d0117 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a5e8d14974..8b36af77bd 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -507,6 +507,7 @@ int sqlite3_exec( #define SQLITE_IOERR_BEGIN_ATOMIC (SQLITE_IOERR | (29<<8)) #define SQLITE_IOERR_COMMIT_ATOMIC (SQLITE_IOERR | (30<<8)) #define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) +#define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) From 8b2d8de3e20367e62c1320958bdedc4b534b87fd Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 May 2020 13:32:19 +0000 Subject: [PATCH 30/31] Update documentation for sqlite3_close_v2() for clarity. No functional changes. FossilOrigin-Name: 80498b69ea489e8816c80a52c9e55a62699116fcbfdcbfd922ef23c2d9938871 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 34 +++++++++++++++------------------- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 984be3d25f..464caade5b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\snew\sSQLITE_IOERR_DATA\sresult\scode\sand\suse\sit\sin\scksumvfs.\nAlso\senhance\scksumvfs\sto\semit\san\ssqlite3_log()\smessage\swhenever\sit\sfinds\nan\sinvalid\schecksum. -D 2020-05-01T11:31:43.081 +C Update\sdocumentation\sfor\ssqlite3_close_v2()\sfor\sclarity.\s\sNo\sfunctional\nchanges. +D 2020-05-01T13:32:19.580 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -535,7 +535,7 @@ F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c310de94bf67315054587c18a16e7a3e3dc3a98dc79168f0c2b776548d43f6cd F src/shell.c.in 86cd0f0412b9739b769fafdfcad28f731882d522042a95c30ab033a5eba68b03 -F src/sqlite.h.in 177084a0cef8c3b1df835bdec9530f102026bf0b6e9492a32349d62c30113a66 +F src/sqlite.h.in dc7fe6a874febed1625fe270b0bbeb9e5018c585639df2ff7a1140d048dbbc15 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 2d1af80082edffd71c6f96f70ad1ce6a4fb46615ad10291fc77fe0dea9ff0197 F src/sqliteInt.h 0f3848c46310d197246003f052985b72d1cdbfc0b31e069db76cb5231062fa1d @@ -1862,7 +1862,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2c17cdce26fd935e6d81ff828f4670291fc014013c93b2a1578506598345ee86 -R 1e877c1fea766b151c81954ab2c1dc73 +P a094e8bfdef10d9e5d97f5a9f7c15b0fc547358b83367762ba5c2dfd2c9d0117 +R 693a622bdbf2543acdb106cd8db3f135 U drh -Z dd4cd34bda503d060e1073825c3c95ba +Z 6c1f5b11653b9fb726bcbb3b288cff5f diff --git a/manifest.uuid b/manifest.uuid index 7d94dfd16a..c6b677f739 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a094e8bfdef10d9e5d97f5a9f7c15b0fc547358b83367762ba5c2dfd2c9d0117 \ No newline at end of file +80498b69ea489e8816c80a52c9e55a62699116fcbfdcbfd922ef23c2d9938871 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 8b36af77bd..9e87ef8c0c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -299,26 +299,22 @@ typedef sqlite_uint64 sqlite3_uint64; ** the [sqlite3] object is successfully destroyed and all associated ** resources are deallocated. ** -** ^If the database connection is associated with unfinalized prepared -** statements or unfinished sqlite3_backup objects then sqlite3_close() -** will leave the database connection open and return [SQLITE_BUSY]. -** ^If sqlite3_close_v2() is called with unfinalized prepared statements -** and/or unfinished sqlite3_backups, then the database connection becomes -** an unusable "zombie" which will automatically be deallocated when the -** last prepared statement is finalized or the last sqlite3_backup is -** finished. The sqlite3_close_v2() interface is intended for use with -** host languages that are garbage collected, and where the order in which -** destructors are called is arbitrary. -** -** Applications should [sqlite3_finalize | finalize] all [prepared statements], -** [sqlite3_blob_close | close] all [BLOB handles], and +** Ideally, applications should [sqlite3_finalize | finalize] all +** [prepared statements], [sqlite3_blob_close | close] all [BLOB handles], and ** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated -** with the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close_v2() is called on a [database connection] that still has -** outstanding [prepared statements], [BLOB handles], and/or -** [sqlite3_backup] objects then it returns [SQLITE_OK] and the deallocation -** of resources is deferred until all [prepared statements], [BLOB handles], -** and [sqlite3_backup] objects are also destroyed. +** with the [sqlite3] object prior to attempting to close the object. +** ^If the database connection is associated with unfinalized prepared +** statements, BLOB handlers, and/or unfinished sqlite3_backup objects then +** sqlite3_close() will leave the database connection open and return +** [SQLITE_BUSY]. ^If sqlite3_close_v2() is called with unfinalized prepared +** statements, unclosed BLOB handlers, and/or unfinished sqlite3_backups, +** it returns [SQLITE_OK] regardless, but instead of deallocating the database +** connection immediately, it marks the database connection as an unusable +** "zombie" and makes arrangements to automatically deallocate the database +** connection after all prepared statements are finalized, all BLOB handles +** are closed, and all backups have finished. The sqlite3_close_v2() interface +** is intended for use with host languages that are garbage collected, and +** where the order in which destructors are called is arbitrary. ** ** ^If an [sqlite3] object is destroyed while a transaction is open, ** the transaction is automatically rolled back. From 50511942700dcfca877b61aa1be1c4e4382daae1 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 May 2020 13:45:12 +0000 Subject: [PATCH 31/31] Clarification to the sqlite3_uri() family of interfaces. Documentation enhancement only - no changes to code. FossilOrigin-Name: 853703cd6d44d6dd48ef5eda6523e374b8ebdf7c338ddaad31c15a40a8b3fd9b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 17 ++++++++++++++--- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 464caade5b..feb2f07b97 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sdocumentation\sfor\ssqlite3_close_v2()\sfor\sclarity.\s\sNo\sfunctional\nchanges. -D 2020-05-01T13:32:19.580 +C Clarification\sto\sthe\ssqlite3_uri()\sfamily\sof\sinterfaces.\s\sDocumentation\nenhancement\sonly\s-\sno\schanges\sto\scode. +D 2020-05-01T13:45:12.131 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -535,7 +535,7 @@ F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c310de94bf67315054587c18a16e7a3e3dc3a98dc79168f0c2b776548d43f6cd F src/shell.c.in 86cd0f0412b9739b769fafdfcad28f731882d522042a95c30ab033a5eba68b03 -F src/sqlite.h.in dc7fe6a874febed1625fe270b0bbeb9e5018c585639df2ff7a1140d048dbbc15 +F src/sqlite.h.in b20d5dc52765ff82f3701395a7e670611ddf138ee0ae84482452ad3a0b5f24b5 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 2d1af80082edffd71c6f96f70ad1ce6a4fb46615ad10291fc77fe0dea9ff0197 F src/sqliteInt.h 0f3848c46310d197246003f052985b72d1cdbfc0b31e069db76cb5231062fa1d @@ -1862,7 +1862,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a094e8bfdef10d9e5d97f5a9f7c15b0fc547358b83367762ba5c2dfd2c9d0117 -R 693a622bdbf2543acdb106cd8db3f135 +P 80498b69ea489e8816c80a52c9e55a62699116fcbfdcbfd922ef23c2d9938871 +R 5d328ccb43b2f01beca1bf0c7121f5fa U drh -Z 6c1f5b11653b9fb726bcbb3b288cff5f +Z 9b72a689cdf698a7000a4037a7a1049c diff --git a/manifest.uuid b/manifest.uuid index c6b677f739..36b7244a29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -80498b69ea489e8816c80a52c9e55a62699116fcbfdcbfd922ef23c2d9938871 \ No newline at end of file +853703cd6d44d6dd48ef5eda6523e374b8ebdf7c338ddaad31c15a40a8b3fd9b \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9e87ef8c0c..f65576a15d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3533,8 +3533,19 @@ int sqlite3_open_v2( ** that check if a database file was a URI that contained a specific query ** parameter, and if so obtains the value of that query parameter. ** -** If F is the database filename pointer passed into the xOpen() method of -** a VFS implementation or it is the return value of [sqlite3_db_filename()] +** The first parameter to these interfaces (hereafter referred to +** as F) must be one of: +**
    +**
  • A database filename pointer created by the SQLite core and +** passed into the xOpen() method of a VFS implemention, or +**
  • A filename obtained from [sqlite3_db_filename()], or +**
  • A new filename constructed using [sqlite3_create_filename()]. +**
+** If the F parameter is not one of the above, then the behavior is +** undefined and probably undesirable. Older versions of SQLite were +** more tolerant of invalid F parameters than newer versions. +** +** If F is a suitable filename (as described in the previous paragraph) ** and if P is the name of the query parameter, then ** sqlite3_uri_parameter(F,P) returns the value of the P ** parameter if it exists or a NULL pointer if P does not appear as a @@ -3670,7 +3681,7 @@ sqlite3_file *sqlite3_database_file_object(const char*); ** ** The sqlite3_free_filename(Y) routine releases a memory allocation ** previously obtained from sqlite3_create_filename(). Invoking -** sqlite3_free_filename(Y) is a NULL pointer is a harmless no-op. +** sqlite3_free_filename(Y) where Y is a NULL pointer is a harmless no-op. ** ** If the Y parameter to sqlite3_free_filename(Y) is anything other ** than a NULL pointer or a pointer previously acquired from