diff --git a/Makefile.in b/Makefile.in index f6b3987adf..a67b0b7498 100644 --- a/Makefile.in +++ b/Makefile.in @@ -696,12 +696,26 @@ sqlite3$(TEXE): shell.c sqlite3.c shell.c sqlite3.c \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" -sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h - $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS) +sqldiff$(TEXE): $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h sqlite3.lo sqlite3.h + $(LTLINK) -I$(TOP)/ext/misc -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS) dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS) +RSYNC_SRC = \ + $(TOP)/tool/sqlite3-rsync.c \ + sqlite3.c + +RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -USQLITE_THREADSAFE \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3-rsync$(TEXE): $(RSYNC_SRC) + $(TCC) -o $@ $(RSYNC_OPT) $(RSYNC_SRC) $(TLIBS) + scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ $(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS) @@ -1175,8 +1189,6 @@ keywordhash.h: $(TOP)/tool/mkkeywordhash.c # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)/src/shell.c.in \ - $(TOP)/ext/consio/console_io.c \ - $(TOP)/ext/consio/console_io.h \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/intck/sqlite3intck.c \ @@ -1193,9 +1205,13 @@ SHELL_DEP = \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/sha1.c \ $(TOP)/ext/misc/shathree.c \ $(TOP)/ext/misc/sqlar.c \ + $(TOP)/ext/misc/sqlite3_stdio.c \ + $(TOP)/ext/misc/sqlite3_stdio.h \ $(TOP)/ext/misc/uint.c \ + $(TOP)/ext/misc/vfstrace.c \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/sqlite3recover.c \ @@ -1639,7 +1655,7 @@ tidy: rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) rm -f wordcount$(TEXE) changeset$(TEXE) version-info$(TEXE) rm -f *.dll *.lib *.exp *.def *.pc *.vsix *.so *.dylib pkgIndex.tcl - rm -f sqlite3_analyzer$(TEXE) + rm -f sqlite3_analyzer$(TEXE) sqlite3-rsync$(TEXE) rm -f mptester$(TEXE) rbu$(TEXE) srcck1$(TEXE) rm -f fuzzershell$(TEXE) fuzzcheck$(TEXE) sqldiff$(TEXE) dbhash$(TEXE) rm -f threadtest5$(TEXE) diff --git a/Makefile.msc b/Makefile.msc index 11b1c7483e..9247358016 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1862,12 +1862,25 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) # <> -sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.h $(TOP)\ext\consio\console_io.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) - $(LTLINK) $(NO_WARN) -I$(TOP)\ext\consio $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) +sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) + $(LTLINK) $(NO_WARN) -I$(TOP)\ext\misc $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +RSYNC_SRC = \ + $(TOP)\tool\sqlite3-rsync.c \ + $(SQLITE3C) + +RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3-rsync.exe: $(RSYNC_SRC) $(LIBRESOBJS) + $(LTLINK) $(RSYNC_OPT) $(NO_WARN) $(RSYNC_SRC) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) + scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -1895,6 +1908,10 @@ fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) fuzzcheck-asan.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) /fsanitize=address $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +run-fuzzcheck: fuzzcheck.exe fuzzcheck-asan.exe + fuzzcheck --spinner $(FUZZDB) + fuzzcheck-asan --spinner $(FUZZDB) + ossshell.exe: $(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2300,8 +2317,6 @@ keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)\src\shell.c.in \ - $(TOP)\ext\consio\console_io.c \ - $(TOP)\ext\consio\console_io.h \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\sqlite3expert.h \ $(TOP)\ext\intck\sqlite3intck.c \ @@ -2318,9 +2333,13 @@ SHELL_DEP = \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\series.c \ + $(TOP)\ext\misc\sha1.c \ $(TOP)\ext\misc\shathree.c \ $(TOP)\ext\misc\sqlar.c \ + $(TOP)\ext\misc\sqlite3_stdio.c \ + $(TOP)\ext\misc\sqlite3_stdio.h \ $(TOP)\ext\misc\uint.c \ + $(TOP)\ext\misc\vfstrace.c \ $(TOP)\ext\misc\zipfile.c \ $(TOP)\ext\recover\dbdata.c \ $(TOP)\ext\recover\sqlite3recover.c \ @@ -2606,7 +2625,7 @@ smoketest: $(TESTPROGS) shelltest: $(TESTPROGS) .\testfixture.exe $(TOP)\test\permutations.test shell -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\consio\console_io.h $(TOP)\ext\consio\console_io.c $(SQLITE_TCL_DEP) +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE_TCL_DEP) $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) @@ -2761,7 +2780,7 @@ clean: del /Q sqlite3.c sqlite3-*.c sqlite3.h 2>NUL del /Q sqlite3rc.h 2>NUL del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL - del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL + del /Q sqlite3_analyzer.exe sqlite3_analyzer.c sqlite3-rsync.exe 2>NUL del /Q sqlite-*-output.vsix 2>NUL del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL del /Q sqltclsh.* 2>NUL diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test index 72c4fd72cd..239450442e 100644 --- a/ext/expert/expert1.test +++ b/ext/expert/expert1.test @@ -428,6 +428,8 @@ do_execsql_test 5.0 { WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s; + + CREATE INDEX i1 ON t1( lower(a) ); } do_candidates_test 5.1 { SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?) @@ -457,6 +459,7 @@ do_execsql_test 5.3 { ANALYZE; SELECT * FROM sqlite_stat1 ORDER BY 1, 2; } { + t1 i1 {100 50} t1 t1_idx_00000061 {100 50} t1 t1_idx_00000062 {100 20} t1 t1_idx_000123a7 {100 50 17} @@ -491,4 +494,17 @@ USE TEMP B-TREE FOR ORDER BY }} } +do_execsql_test 6.0 { + CREATE TABLE x1(a, b, c, d); + CREATE INDEX x1ab ON x1(a, lower(b)); + CREATE INDEX x1dcba ON x1(d, b+c, a); +} + +do_candidates_test 6.1 { + SELECT * FROM x1 WHERE b=? ORDER BY a; +} { + CREATE INDEX x1_idx_0001267f ON x1(b, a); + CREATE INDEX x1_idx_00000062 ON x1(b); +} + finish_test diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c index b59a59728d..3177339584 100644 --- a/ext/expert/sqlite3expert.c +++ b/ext/expert/sqlite3expert.c @@ -1623,6 +1623,12 @@ static int idxPopulateOneStat1( const char *zComma = zCols==0 ? "" : ", "; const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0); const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1); + if( zName==0 ){ + /* This index contains an expression. Ignore it. */ + sqlite3_free(zCols); + sqlite3_free(zOrder); + return sqlite3_reset(pIndexXInfo); + } zCols = idxAppendText(&rc, zCols, "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 7e41119572..b15521f163 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -636,16 +636,13 @@ Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64); int sqlite3Fts5FlushToDisk(Fts5Table*); -int sqlite3Fts5ExtractText( - Fts5Config *pConfig, - sqlite3_value *pVal, /* Value to extract text from */ - int bContent, /* Loaded from content table */ - int *pbResetTokenizer, /* OUT: True if ClearLocale() required */ - const char **ppText, /* OUT: Pointer to text buffer */ - int *pnText /* OUT: Size of (*ppText) in bytes */ -); - void sqlite3Fts5ClearLocale(Fts5Config *pConfig); +void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc); + +int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal); +int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal, + const char **ppText, int *pnText, const char **ppLoc, int *pnLoc +); /* ** End of interface to code in fts5.c. diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 3cb1bd3bea..1fade5f7a8 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -516,6 +516,15 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){ } } } + if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){ + for(i=0; inCol; i++){ + if( p->abUnindexed[i]==0 ){ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL"); + } + } + } assert( p->zContentExprlist==0 ); p->zContentExprlist = (char*)buf.p; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 4363305a56..a51ae19e72 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4889,6 +4889,11 @@ static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){ nBest = nPercent; } } + + /* If pLvl is already the input level to an ongoing merge, look no + ** further for a merge candidate. The caller should be allowed to + ** continue merging from pLvl first. */ + if( pLvl->nMerge ) break; } } return iRet; @@ -8813,7 +8818,7 @@ static int fts5structConnectMethod( /* ** We must have a single struct=? constraint that will be passed through -** into the xFilter method. If there is no valid stmt=? constraint, +** into the xFilter method. If there is no valid struct=? constraint, ** then return an SQLITE_CONSTRAINT error. */ static int fts5structBestIndexMethod( diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 40ccb7e95f..60a41d6973 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -83,8 +83,17 @@ struct Fts5Global { Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */ Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */ Fts5Cursor *pCsr; /* First in list of all open cursors */ + u32 aLocaleHdr[4]; }; +/* +** Size of header on fts5_locale() values. And macro to access a buffer +** containing a copy of the header from an Fts5Config pointer. +*/ +#define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) +#define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr)) + + /* ** Each auxiliary function registered with the FTS5 module is represented ** by an object of the following type. All such objects are stored as part @@ -247,12 +256,6 @@ struct Fts5Cursor { #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) -/* -** The subtype value and header bytes used by fts5_locale(). -*/ -#define FTS5_LOCALE_SUBTYPE ((unsigned int)'L') -#define FTS5_LOCALE_HEADER "\x00\xE0\xB2\xEB" - /* ** Macros to Set(), Clear() and Test() cursor flags. @@ -1256,7 +1259,7 @@ static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ ** valid until after the final call to sqlite3Fts5Tokenize() that will use ** the locale. */ -static void fts5SetLocale( +static void sqlite3Fts5SetLocale( Fts5Config *pConfig, const char *zLocale, int nLocale @@ -1267,127 +1270,74 @@ static void fts5SetLocale( } /* -** Clear any locale configured by an earlier call to fts5SetLocale() or -** sqlite3Fts5ExtractText(). +** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale(). */ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ - fts5SetLocale(pConfig, 0, 0); + sqlite3Fts5SetLocale(pConfig, 0, 0); } /* -** This function is used to extract utf-8 text from an sqlite3_value. This -** is usually done in order to tokenize it. For example, when: -** -** * a value is written to an fts5 table, -** * a value is deleted from an FTS5_CONTENT_NORMAL table, -** * a value containing a query expression is passed to xFilter() -** -** and so on. -** -** This function handles 2 cases: -** -** 1) Ordinary values. The text can be extracted from these using -** sqlite3_value_text(). -** -** 2) Combination text/locale blobs created by fts5_locale(). There -** are several cases for these: -** -** * Blobs tagged with FTS5_LOCALE_SUBTYPE. -** * Blobs read from the content table of a locale=1 external-content -** table, and -** * Blobs read from the content table of a locale=1 regular -** content table. -** -** The first two cases above should have the 4 byte FTS5_LOCALE_HEADER -** header. It is an error if a blob with the subtype or a blob read -** from the content table of an external content table does not have -** the required header. A blob read from the content table of a regular -** locale=1 table does not have the header. This is to save space. -** -** If successful, SQLITE_OK is returned and output parameters (*ppText) -** and (*pnText) are set to point to a buffer containing the extracted utf-8 -** text and its length in bytes, respectively. The buffer is not -** nul-terminated. It has the same lifetime as the sqlite3_value object -** from which it is extracted. -** -** Parameter bContent must be true if the value was read from an indexed -** column (i.e. not UNINDEXED) of the on disk content. -** -** If pbResetTokenizer is not NULL and if case (2) is used, then -** fts5SetLocale() is called to ensure subsequent sqlite3Fts5Tokenize() calls -** use the locale. In this case (*pbResetTokenizer) is set to true before -** returning, to indicate that the caller must call sqlite3Fts5ClearLocale() -** to clear the locale after tokenizing the text. +** Return true if the value passed as the only argument is an +** fts5_locale() value. */ -int sqlite3Fts5ExtractText( - Fts5Config *pConfig, - sqlite3_value *pVal, /* Value to extract text from */ - int bContent, /* True if indexed table content */ - int *pbResetTokenizer, /* OUT: True if xSetLocale(NULL) required */ - const char **ppText, /* OUT: Pointer to text buffer */ - int *pnText /* OUT: Size of (*ppText) in bytes */ -){ - const char *pText = 0; - int nText = 0; - int rc = SQLITE_OK; - int bDecodeBlob = 0; - - assert( pbResetTokenizer==0 || *pbResetTokenizer==0 ); - assert( bContent==0 || pConfig->eContent!=FTS5_CONTENT_NONE ); - assert( bContent==0 || sqlite3_value_subtype(pVal)==0 ); - +int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){ + int ret = 0; if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ - if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE - || (bContent && pConfig->bLocale) - ){ - bDecodeBlob = 1; - } - } - - if( bDecodeBlob ){ - const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; + /* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case. + ** If the blob was created using zeroblob(), then sqlite3_value_blob() + ** may call malloc(). If this malloc() fails, then the values returned + ** by both value_blob() and value_bytes() will be 0. If value_bytes() were + ** called first, then the NULL pointer returned by value_blob() might + ** be dereferenced. */ const u8 *pBlob = sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); - - /* Unless this blob was read from the %_content table of an - ** FTS5_CONTENT_NORMAL table, it should have the 4 byte fts5_locale() - ** header. Check for this. If it is not found, return an error. */ - if( (!bContent || pConfig->eContent!=FTS5_CONTENT_NORMAL) ){ - if( nBlobFTS5_LOCALE_HDR_SIZE + && 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE) + ){ + ret = 1; } - - if( rc==SQLITE_OK ){ - int nLocale = 0; - - for(nLocale=0; nLocaleFTS5_LOCALE_HDR_SIZE ); + + for(nLoc=FTS5_LOCALE_HDR_SIZE; p[nLoc]; nLoc++){ + if( nLoc==(n-1) ){ + return SQLITE_MISMATCH; + } + } + *ppLoc = &p[FTS5_LOCALE_HDR_SIZE]; + *pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE; + + *ppText = &p[nLoc+1]; + *pnText = n - nLoc - 1; + return SQLITE_OK; } /* @@ -1396,8 +1346,8 @@ int sqlite3Fts5ExtractText( ** the text of the expression, and sets output variable (*pzText) to ** point to a nul-terminated buffer containing the expression. ** -** If pVal was an fts5_locale() value, then fts5SetLocale() is called to -** set the tokenizer to use the specified locale. +** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called +** to set the tokenizer to use the specified locale. ** ** If output variable (*pbFreeAndReset) is set to true, then the caller ** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer @@ -1409,24 +1359,22 @@ static int fts5ExtractExprText( char **pzText, /* OUT: nul-terminated buffer of text */ int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */ ){ - const char *zText = 0; - int nText = 0; int rc = SQLITE_OK; - int bReset = 0; - *pbFreeAndReset = 0; - rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, &bReset, &zText, &nText); - if( rc==SQLITE_OK ){ - if( bReset ){ - *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText); - if( rc!=SQLITE_OK ){ - sqlite3Fts5ClearLocale(pConfig); - }else{ - *pbFreeAndReset = 1; - } - }else{ - *pzText = (char*)zText; + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText); + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); } + *pbFreeAndReset = 1; + }else{ + *pzText = (char*)sqlite3_value_text(pVal); + *pbFreeAndReset = 0; } return rc; @@ -1966,19 +1914,14 @@ static int fts5UpdateMethod( else{ int eType1 = sqlite3_value_numeric_type(apVal[1]); - /* Ensure that no fts5_locale() values are written to locale=0 tables. - ** And that no blobs except fts5_locale() blobs are written to indexed - ** (i.e. not UNINDEXED) columns of locale=1 tables. */ - int ii; - for(ii=0; iinCol; ii++){ - if( sqlite3_value_type(apVal[ii+2])==SQLITE_BLOB ){ - int bSub = (sqlite3_value_subtype(apVal[ii+2])==FTS5_LOCALE_SUBTYPE); - if( (pConfig->bLocale && !bSub && pConfig->abUnindexed[ii]==0) - || (pConfig->bLocale==0 && bSub) - ){ - if( pConfig->bLocale==0 ){ - fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); - } + /* It is an error to write an fts5_locale() value to a table without + ** the locale=1 option. */ + if( pConfig->bLocale==0 ){ + int ii; + for(ii=0; iinCol; ii++){ + sqlite3_value *pVal = apVal[ii+2]; + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); rc = SQLITE_MISMATCH; goto update_out; } @@ -2140,11 +2083,11 @@ static int fts5ApiTokenize_v2( Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); int rc = SQLITE_OK; - fts5SetLocale(pTab->pConfig, pLoc, nLoc); + sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc); rc = sqlite3Fts5Tokenize(pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken ); - fts5SetLocale(pTab->pConfig, 0, 0); + sqlite3Fts5SetLocale(pTab->pConfig, 0, 0); return rc; } @@ -2172,6 +2115,49 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } +/* +** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This +** function extracts the text value of column iCol of the current row. +** Additionally, if there is an associated locale, it invokes +** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller +** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point +** after this function returns. +** +** If successful, (*ppText) is set to point to a buffer containing the text +** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that +** buffer in bytes. It is not guaranteed to be nul-terminated. If an error +** occurs, an SQLite error code is returned. The final values of the two +** output parameters are undefined in this case. +*/ +static int fts5TextFromStmt( + Fts5Config *pConfig, + sqlite3_stmt *pStmt, + int iCol, + const char **ppText, + int *pnText +){ + sqlite3_value *pVal = sqlite3_column_value(pStmt, iCol+1); + const char *pLoc = 0; + int nLoc = 0; + int rc = SQLITE_OK; + + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc); + }else{ + *ppText = (const char*)sqlite3_value_text(pVal); + *pnText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + pLoc = (const char*)sqlite3_column_text(pStmt, iCol+1+pConfig->nCol); + nLoc = sqlite3_column_bytes(pStmt, iCol+1+pConfig->nCol); + } + } + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + return rc; +} + static int fts5ApiColumnText( Fts5Context *pCtx, int iCol, @@ -2191,10 +2177,8 @@ static int fts5ApiColumnText( }else{ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ - Fts5Config *pConfig = pTab->pConfig; - int bContent = (pConfig->abUnindexed[iCol]==0); - sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); - sqlite3Fts5ExtractText(pConfig, pVal, bContent, 0, pz, pn); + rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn); + sqlite3Fts5ClearLocale(pTab->pConfig); } } return rc; @@ -2236,17 +2220,15 @@ static int fts5CsrPoslist( rc = fts5SeekCursor(pCsr, 0); } for(i=0; inCol && rc==SQLITE_OK; i++){ - sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1); const char *z = 0; int n = 0; - int bReset = 0; - rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n); + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprPopulatePoslists( pConfig, pCsr->pExpr, aPopulator, i, z, n ); } - if( bReset ) sqlite3Fts5ClearLocale(pConfig); + sqlite3Fts5ClearLocale(pConfig); } sqlite3_free(aPopulator); @@ -2432,17 +2414,14 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ if( pConfig->abUnindexed[i]==0 ){ const char *z = 0; int n = 0; - int bReset = 0; - sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1); - pCsr->aColumnSize[i] = 0; - rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n); + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb ); - if( bReset ) sqlite3Fts5ClearLocale(pConfig); } + sqlite3Fts5ClearLocale(pConfig); } } } @@ -2714,37 +2693,14 @@ static int fts5ApiColumnLocale( ){ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ - /* Load the value into pVal. pVal is a locale/text pair iff: - ** - ** 1) It is an SQLITE_BLOB, and - ** 2) Either the subtype is FTS5_LOCALE_SUBTYPE, or else the - ** value was loaded from an FTS5_CONTENT_NORMAL table, and - ** 3) It does not begin with an 0x00 byte. - */ - sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); - if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ - const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); - if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ - const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; - if( nBlobpStmt, iCol, &zDummy, &nDummy); + if( rc==SQLITE_OK ){ + *pzLocale = pConfig->t.pLocale; + *pnLocale = pConfig->t.nLocale; } + sqlite3Fts5ClearLocale(pConfig); } } @@ -2965,57 +2921,6 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ return rc; } -/* -** Value pVal was read from column iCol of the FTS5 table. This function -** returns it to the owner of pCtx via a call to an sqlite3_result_xxx() -** function. This function deals with the same cases as -** sqlite3Fts5ExtractText(): -** -** 1) Ordinary values. These can be returned using sqlite3_result_value(). -** -** 2) Blobs from fts5_locale(). The text is extracted from these and -** returned via sqlite3_result_text(). The locale is discarded. -*/ -static void fts5ExtractValueFromColumn( - sqlite3_context *pCtx, - Fts5Config *pConfig, - int iCol, - sqlite3_value *pVal -){ - assert( pConfig->eContent!=FTS5_CONTENT_NONE ); - - if( pConfig->bLocale - && sqlite3_value_type(pVal)==SQLITE_BLOB - && pConfig->abUnindexed[iCol]==0 - ){ - const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; - const u8 *pBlob = sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); - int ii; - - if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ - if( nBlobpStmt, iCol+1); - fts5ExtractValueFromColumn(pCtx, pConfig, iCol, pVal); + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, z, n, SQLITE_TRANSIENT); + } + sqlite3Fts5ClearLocale(pConfig); + }else{ + sqlite3_result_value(pCtx, pVal); + } } + pConfig->pzErrmsg = 0; } } @@ -3635,13 +3554,12 @@ static void fts5LocaleFunc( if( zLocale==0 || zLocale[0]=='\0' ){ sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT); }else{ + Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx); u8 *pBlob = 0; u8 *pCsr = 0; int nBlob = 0; - const int nHdr = 4; - assert( sizeof(FTS5_LOCALE_HEADER)==nHdr+1 ); - nBlob = nHdr + nLocale + 1 + nText; + nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText; pBlob = (u8*)sqlite3_malloc(nBlob); if( pBlob==0 ){ sqlite3_result_error_nomem(pCtx); @@ -3649,8 +3567,8 @@ static void fts5LocaleFunc( } pCsr = pBlob; - memcpy(pCsr, FTS5_LOCALE_HEADER, nHdr); - pCsr += nHdr; + memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE); + pCsr += FTS5_LOCALE_HDR_SIZE; memcpy(pCsr, zLocale, nLocale); pCsr += nLocale; (*pCsr++) = 0x00; @@ -3658,7 +3576,6 @@ static void fts5LocaleFunc( assert( &pCsr[nText]==&pBlob[nBlob] ); sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free); - sqlite3_result_subtype(pCtx, FTS5_LOCALE_SUBTYPE); } } @@ -3760,6 +3677,16 @@ static int fts5Init(sqlite3 *db){ pGlobal->api.xFindTokenizer = fts5FindTokenizer; pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2; pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2; + + /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector. + ** The constants below were generated randomly. */ + sqlite3_randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr); + pGlobal->aLocaleHdr[0] ^= 0xF924976D; + pGlobal->aLocaleHdr[1] ^= 0x16596E13; + pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA; + pGlobal->aLocaleHdr[3] ^= 0x9B03A67F; + assert( sizeof(pGlobal->aLocaleHdr)==16 ); + rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index cf25eb361e..e8649c703f 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -73,6 +73,30 @@ struct Fts5Storage { #define FTS5_STMT_REPLACE_CONFIG 10 #define FTS5_STMT_SCAN 11 +/* +** Return a pointer to a buffer obtained from sqlite3_malloc() that contains +** nBind comma-separated question marks. e.g. if nBind is passed 5, this +** function returns "?,?,?,?,?". +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op and +** NULL is returned immediately. Or, if the attempt to malloc a buffer +** fails, then *pRc is set to SQLITE_NOMEM and NULL is returned. Otherwise, +** if it is SQLITE_OK when this function is called and the malloc() succeeds, +** *pRc is left unchanged. +*/ +static char *fts5BindingsList(int *pRc, int nBind){ + char *zBind = sqlite3Fts5MallocZero(pRc, 1 + nBind*2); + if( zBind ){ + int ii; + for(ii=0; iinCol + 1; + case FTS5_STMT_INSERT_CONTENT: { + int nCol = 0; char *zBind; int i; - zBind = sqlite3_malloc64(1 + nCol*2); - if( zBind ){ - for(i=0; inCol; + if( pC->bLocale ){ + for(i=0; inCol; i++){ + if( pC->abUnindexed[i]==0 ) nCol++; } - zBind[i*2-1] = '\0'; + } + + zBind = fts5BindingsList(&rc, nCol); + if( zBind ){ zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind); sqlite3_free(zBind); } @@ -344,7 +369,7 @@ int sqlite3Fts5StorageOpen( if( bCreate ){ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ int nDefn = 32 + pConfig->nCol*10; - char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10); + char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20); if( zDefn==0 ){ rc = SQLITE_NOMEM; }else{ @@ -356,6 +381,14 @@ int sqlite3Fts5StorageOpen( sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); iOff += (int)strlen(&zDefn[iOff]); } + if( pConfig->bLocale ){ + for(i=0; inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + } + } rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); } sqlite3_free(zDefn); @@ -507,7 +540,8 @@ static int fts5StorageDeleteFromIndex( sqlite3_value *pVal = 0; const char *pText = 0; int nText = 0; - int bReset = 0; + const char *pLoc = 0; + int nLoc = 0; assert( pSeek==0 || apVal==0 ); assert( pSeek!=0 || apVal!=0 ); @@ -517,10 +551,19 @@ static int fts5StorageDeleteFromIndex( pVal = apVal[iCol-1]; } - rc = sqlite3Fts5ExtractText( - pConfig, pVal, pSeek!=0, &bReset, &pText, &nText - ); + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pSeek ){ + pLoc = (const char*)sqlite3_column_text(pSeek, iCol + pConfig->nCol); + nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol); + } + } + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, fts5StorageInsertCallback @@ -529,7 +572,7 @@ static int fts5StorageDeleteFromIndex( if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){ rc = FTS5_CORRUPT; } - if( bReset ) sqlite3Fts5ClearLocale(pConfig); + sqlite3Fts5ClearLocale(pConfig); } } } @@ -788,20 +831,35 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - int bReset = 0; /* True if tokenizer locale must be reset */ int nText = 0; /* Size of pText in bytes */ const char *pText = 0; /* Pointer to buffer containing text value */ - sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1); + int nLoc = 0; /* Size of pLoc in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + + sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1); + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + } - rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText); if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, fts5StorageInsertCallback ); - if( bReset ) sqlite3Fts5ClearLocale(pConfig); + sqlite3Fts5ClearLocale(pConfig); } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); @@ -884,29 +942,45 @@ int sqlite3Fts5StorageContentInsert( }else{ sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */ int i; /* Counter variable */ + int nIndexed = 0; /* Number indexed columns seen */ rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); - for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + if( pInsert ) sqlite3_clear_bindings(pInsert); + + /* Bind the rowid value */ + sqlite3_bind_value(pInsert, 1, apVal[1]); + + /* Loop through values for user-defined columns. i=2 is the leftmost + ** user-defined column. As is column 1 of pSavedRow. */ + for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + int bUnindexed = pConfig->abUnindexed[i-2]; sqlite3_value *pVal = apVal[i]; + + nIndexed += !bUnindexed; if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ /* This is an UPDATE statement, and column (i-2) was not modified. ** Retrieve the value from Fts5Storage.pSavedRow instead. */ pVal = sqlite3_column_value(p->pSavedRow, i-1); - }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){ - assert( pConfig->bLocale ); - assert( i>1 ); - if( pConfig->abUnindexed[i-2] ){ - /* At attempt to insert an fts5_locale() value into an UNINDEXED - ** column. Strip the locale away and just bind the text. */ - const char *pText = 0; - int nText = 0; - rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText); - sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); - }else{ - const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); - assert( nBlob>4 ); - sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT); + if( pConfig->bLocale && bUnindexed==0 ){ + sqlite3_bind_value(pInsert, pConfig->nCol + 1 + nIndexed, + sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1) + ); } + }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + const char *pLoc = 0; + int nText = 0; + int nLoc = 0; + assert( pConfig->bLocale ); + + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + if( bUnindexed==0 ){ + int iLoc = pConfig->nCol + 1 + nIndexed; + sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT); + } + } + continue; } @@ -945,23 +1019,37 @@ int sqlite3Fts5StorageIndexInsert( for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - int bReset = 0; /* True if tokenizer locale must be reset */ int nText = 0; /* Size of pText in bytes */ const char *pText = 0; /* Pointer to buffer containing text value */ + int nLoc = 0; /* Size of pText in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + sqlite3_value *pVal = apVal[ctx.iCol+2]; - int bDisk = 0; if( p->pSavedRow && sqlite3_value_nochange(pVal) ){ pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1); - bDisk = 1; + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol); + nLoc = sqlite3_column_bytes(p->pSavedRow, iCol); + } + }else{ + pVal = apVal[ctx.iCol+2]; } - rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset, &pText,&nText); + + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + if( rc==SQLITE_OK ){ - assert( bReset==0 || pConfig->bLocale ); + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, fts5StorageInsertCallback ); - if( bReset ) sqlite3Fts5ClearLocale(pConfig); + sqlite3Fts5ClearLocale(pConfig); } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); @@ -1126,37 +1214,61 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } for(i=0; rc==SQLITE_OK && inCol; i++){ - if( pConfig->abUnindexed[i] ) continue; - ctx.iCol = i; - ctx.szCol = 0; - if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ - rc = sqlite3Fts5TermsetNew(&ctx.pTermset); - } - if( rc==SQLITE_OK ){ - int bReset = 0; /* True if tokenizer locale must be reset */ - int nText = 0; /* Size of pText in bytes */ - const char *pText = 0; /* Pointer to buffer containing text value */ + if( pConfig->abUnindexed[i]==0 ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + sqlite3_value *pVal = sqlite3_column_value(pScan, i+1); + + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue( + pVal, &pText, &nText, &pLoc, &nLoc + ); + }else{ + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = i + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + ctx.iCol = i; + ctx.szCol = 0; + + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } - rc = sqlite3Fts5ExtractText(pConfig, - sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText - ); if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, fts5StorageIntegrityCallback ); - if( bReset ) sqlite3Fts5ClearLocale(pConfig); + sqlite3Fts5ClearLocale(pConfig); + } + + /* If this is not a columnsize=0 database, check that the number + ** of tokens in the value matches the aColSize[] value read from + ** the %_docsize table. */ + if( rc==SQLITE_OK + && pConfig->bColumnsize + && ctx.szCol!=aColSize[i] + ){ + rc = FTS5_CORRUPT; + } + aTotalSize[i] += ctx.szCol; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; } - } - if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ - rc = FTS5_CORRUPT; - } - aTotalSize[i] += ctx.szCol; - if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ - sqlite3Fts5TermsetFree(ctx.pTermset); - ctx.pTermset = 0; } } sqlite3Fts5TermsetFree(ctx.pTermset); diff --git a/ext/fts5/test/fts5blob.test b/ext/fts5/test/fts5blob.test index 4233719fbc..9348554104 100644 --- a/ext/fts5/test/fts5blob.test +++ b/ext/fts5/test/fts5blob.test @@ -103,13 +103,13 @@ do_execsql_test 3.0 { do_catchsql_test 3.1 { INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456'); -} {1 {datatype mismatch}} +} {0 {}} do_catchsql_test 3.2 { INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456'); -} {1 {datatype mismatch}} +} {0 {}} do_catchsql_test 3.3 { INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456'); -} {1 {datatype mismatch}} +} {0 {}} #-------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5contentless5.test b/ext/fts5/test/fts5contentless5.test index 3563678868..86d0753286 100644 --- a/ext/fts5/test/fts5contentless5.test +++ b/ext/fts5/test/fts5contentless5.test @@ -35,8 +35,8 @@ do_execsql_test 1.01 { } # explain_i "UPDATE t1 SET a='a' WHERE t1.rowid=1" -breakpoint -explain_i "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1 AND b IS NULL" +#breakpoint +#explain_i "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1 AND b IS NULL" #breakpoint #explain_i "UPDATE t1 SET a='a' WHERE b IS NULL AND rowid=?" @@ -56,4 +56,56 @@ foreach {tn up err} { do_catchsql_test 1.$tn $up $res($err) } +#------------------------------------------------------------------------- +reset_db + +proc random {n} { expr {abs(int(rand()*$n))} } +proc select_one {list} { + set n [llength $list] + lindex $list [random $n] +} +proc vocab {} { + list abc def ghi jkl mno pqr stu vwx yza +} +proc term {} { + select_one [vocab] +} +proc document {} { + set nTerm [expr [random 3] + 7] + set doc "" + for {set ii 0} {$ii < $nTerm} {incr ii} { + lappend doc [term] + } + set doc +} +db func document document + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, contentless_delete=1, content=''); + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); +} + +do_test 2.1 { + for {set ii 1} {$ii < 12} {incr ii} { + db transaction { + for {set jj 0} {$jj < 10} {incr jj} { + set doc [document] + execsql { INSERT INTO ft VALUES($doc); } + } + } + } +} {} + +do_test 2.2 { + foreach r [db eval {SELECT rowid FROM ft}] { + execsql { DELETE FROM ft WHERE rowid=$r } + } +} {} + +set doc [document] +do_execsql_test 2.3 { + INSERT INTO ft VALUES($doc) +} + + finish_test diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test index e74162ee35..72f25caee1 100644 --- a/ext/fts5/test/fts5faultI.test +++ b/ext/fts5/test/fts5faultI.test @@ -246,7 +246,7 @@ do_execsql_test 10.1 { } {hello} faultsim_save_and_close -do_faultsim_test 10 -faults oom* -prep { +do_faultsim_test 10.1 -faults oom* -prep { faultsim_restore_and_reopen } -body { execsql { @@ -256,6 +256,18 @@ do_faultsim_test 10 -faults oom* -prep { faultsim_test_result {0 hello} } +breakpoint +faultsim_save_and_close +do_faultsim_test 10.2 -faults oom-t* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + INSERT INTO ft VALUES(zeroblob(10000)); + } +} -test { + faultsim_test_result {0 {}} +} + #------------------------------------------------------------------------- reset_db diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index 684dcecd80..e5799fb7fd 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -73,6 +73,7 @@ do_execsql_test 1.2 { SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') ); } {2 cba} + #------------------------------------------------------------------------- # Test that the locale= option exists and seems to accept values. And # that fts5_locale() values may only be inserted into an internal-content @@ -99,8 +100,11 @@ do_catchsql_test 2.3 { INSERT INTO b1(b1, rank) VALUES('locale', 0); } {1 {SQL logic error}} -do_execsql_test 2.4 { +do_execsql_test 2.4.1 { INSERT INTO b1 VALUES('abc', 'one two three'); +} + +do_execsql_test 2.4.2 { INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six')); } @@ -131,6 +135,7 @@ do_execsql_test 2.12 { SELECT quote(y) FROM b1('ruof') } { do_execsql_test 2.13 { INSERT INTO b1(b1) VALUES('integrity-check'); } + do_execsql_test 2.14 { INSERT INTO b1(b1) VALUES('rebuild'); } @@ -149,7 +154,6 @@ do_execsql_test 2.18 { INSERT INTO b1(rowid, x, y) VALUES( test_setsubtype(45, 76), 'abc def', 'def abc' ); - INSERT INTO b1(b1) VALUES('integrity-check'); } #------------------------------------------------------------------------- @@ -278,9 +282,9 @@ do_execsql_test 5.2 { } do_execsql_test 5.3 { - SELECT typeof(c0), typeof(c1) FROM t1_content + SELECT typeof(c0), typeof(c1), typeof(l0) FROM t1_content } { - blob text + text text text } #------------------------------------------------------------------------- @@ -305,37 +309,37 @@ foreach {tn opt} { fts5_aux_test_functions db - do_execsql_test 5.$tn.3 { + do_execsql_test 6.$tn.3 { SELECT fts5_test_columnsize(y1) FROM y1 } { 2 3 2 4 } - do_execsql_test 5.$tn.4 { + do_execsql_test 6.$tn.4 { SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); } { 2 3 } - do_execsql_test 5.$tn.5 { + do_execsql_test 6.$tn.5 { SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); } { 2 3 } - do_execsql_test 5.$tn.6 { + do_execsql_test 6.$tn.6 { SELECT rowid, fts5_test_columnsize(y1) FROM y1('have'); } { 4 4 } - do_execsql_test 5.$tn.7 { + do_execsql_test 6.$tn.7 { SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have'); } { 4 {which it hath been used to [have]} } - do_execsql_test 5.$tn.8 { + do_execsql_test 6.$tn.8 { SELECT rowid, highlight(y1, 0, '[', ']'), snippet(y1, 0, '[', ']', '...', 10) @@ -473,7 +477,7 @@ foreach_detail_mode $::testprefix { } foreach {tn v} { - 1 X'001122' + 1 X'001152' 2 X'0011223344' 3 X'00E0B2EB68656c6c6f' 4 X'00E0B2EB0068656c6c6f' @@ -484,33 +488,33 @@ foreach_detail_mode $::testprefix { do_catchsql_test 10.2.$tn.3 { INSERT INTO ft(ft) VALUES('rebuild'); - } {1 {SQL logic error}} + } {0 {}} do_catchsql_test 10.2.$tn.4 " SELECT * FROM ft( test_setsubtype($v, 76) ); - " {1 {SQL logic error}} + " {1 {fts5: syntax error near ""}} do_execsql_test 10.2.$tn.5 { INSERT INTO ft(rowid, x) VALUES(1, 'hello world'); } - if {"%DETAIL%"!="full"} { - do_catchsql_test 10.2.$tn.6 { + if {"%DETAIL%"=="full"} { + do_execsql_test 10.2.$tn.6 { SELECT fts5_test_poslist(ft) FROM ft('world'); - } {1 SQLITE_ERROR} + } {0.0.1} - do_catchsql_test 10.2.$tn.7 { + do_execsql_test 10.2.$tn.7.1 { SELECT fts5_test_columnsize(ft) FROM ft('world'); - } {1 SQLITE_ERROR} + } {1} - do_catchsql_test 10.2.$tn.7 { + do_execsql_test 10.2.$tn.7.2 { SELECT fts5_test_columnlocale(ft) FROM ft('world'); - } {1 SQLITE_ERROR} + } {{{}}} } do_catchsql_test 10.2.$tn.8 { - SELECT * FROM ft('hello') - } {1 {SQL logic error}} + SELECT count(*) FROM ft('hello') + } {0 1} do_catchsql_test 10.2.$tn.9 { PRAGMA integrity_check; @@ -523,11 +527,11 @@ foreach_detail_mode $::testprefix { do_catchsql_test 10.2.$tn.11 " INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) ) - " {1 {SQL logic error}} + " {0 {}} do_catchsql_test 10.2.$tn.12 " INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) ) - " {1 {SQL logic error}} + " {0 {}} do_execsql_test 10.2.$tn.13 { INSERT INTO ft2(rowid, x) VALUES(1, 'hello world'); @@ -536,7 +540,7 @@ foreach_detail_mode $::testprefix { do_catchsql_test 10.2.$tn.15 { PRAGMA integrity_check; - } {1 {SQL logic error}} + } {0 {{malformed inverted index for FTS5 table main.ft2}}} do_execsql_test 10.2.$tn.16 { DELETE FROM ft2_content; @@ -663,5 +667,82 @@ do_catchsql_test 13.2.7 { FROM ft('one AND three') ORDER BY rowid } {1 {non-integer argument passed to function fts5_get_locale()}} +#------------------------------------------------------------------------- +# Check that UPDATE statements that may affect more than one row work. +# +reset_db +do_execsql_test 14.1 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1); +} + +do_execsql_test 14.2 { + INSERT INTO ft VALUES('hello', 'world'); +} + +do_execsql_test 14.3 { + UPDATE ft SET b = fts5_locale('en_AU', 'world'); +} + +do_execsql_test 14.4 { + INSERT INTO ft VALUES(X'abcd', X'1234'); +} {} + +do_execsql_test 14.5 { + SELECT quote(a), quote(b) FROM ft +} {'hello' 'world' X'ABCD' X'1234'} + +do_execsql_test 14.6 { + DELETE FROM ft; + INSERT INTO ft VALUES(NULL, 'null'); + INSERT INTO ft VALUES(123, 'int'); + INSERT INTO ft VALUES(345.0, 'real'); + INSERT INTO ft VALUES('abc', 'text'); + INSERT INTO ft VALUES(fts5_locale('abc', 'def'), 'text'); + + SELECT a, typeof(a), b FROM ft +} { + {} null null + 123 integer int + 345.0 real real + abc text text + def text text +} + +do_execsql_test 14.7 { + SELECT quote(c0), typeof(c0) FROM ft_content +} { + NULL null + 123 integer + 345.0 real + 'abc' text + 'def' text +} + +#------------------------------------------------------------------------- +# Check that inserting UNINDEXED columns between indexed columns of a +# locale=1 table does not cause a problem. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create +fts5_aux_test_functions db + +do_execsql_test 15.1 { + CREATE VIRTUAL TABLE ft USING fts5(a, b UNINDEXED, c, locale=1, tokenize=tcl); +} + +do_execsql_test 15.2 { + INSERT INTO ft VALUES('one', 'two', 'three'); + INSERT INTO ft VALUES('one', 'two', fts5_locale('loc', 'three')); +} + +do_execsql_test 15.3 { + SELECT c2, l2 FROM ft_content +} {three {} three loc} + +do_execsql_test 15.4 { + SELECT c, fts5_columnlocale(ft, 2) FROM ft +} {three {} three loc} + + finish_test diff --git a/ext/fts5/test/fts5phrase.test b/ext/fts5/test/fts5phrase.test index ea425a4dd6..708cdfd83e 100644 --- a/ext/fts5/test/fts5phrase.test +++ b/ext/fts5/test/fts5phrase.test @@ -93,15 +93,21 @@ foreach {tn cols tokens} { 10 {b} "i e" 11 {a} "i e" } { - set fts "{$cols}:[join $tokens +]" set where [list] foreach c $cols { lappend where "pmatch($c, '$tokens')" } set where [join $where " OR "] - set res [db eval "SELECT rowid FROM t3 WHERE $where"] - do_execsql_test "1.$tn.$fts->([llength $res] rows)" { - SELECT rowid FROM t3($fts) - } $res + foreach fts [list \ + "{$cols}:[join $tokens +]" \ + "{$cols}:NEAR([join $tokens +])" \ + "{$cols}:NEAR([join $tokens +],1)" \ + "{$cols}:NEAR([join $tokens +],111)" \ + ] { + set res [db eval "SELECT rowid FROM t3 WHERE $where"] + do_execsql_test "1.$tn.$fts->([llength $res] rows)" { + SELECT rowid FROM t3($fts) + } $res + } } do_execsql_test 2.0 { diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index ca8090ed2e..483ef0187f 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -40,7 +40,7 @@ ** modification-time of the target file is set to this value before ** returning. ** -** If three or more arguments are passed to this function and an +** If five or more arguments are passed to this function and an ** error is encountered, an exception is raised. ** ** READFILE(FILE): @@ -110,6 +110,13 @@ SQLITE_EXTENSION_INIT1 #include #include +/* When used as part of the CLI, the sqlite3_stdio.h module will have +** been included before this one. In that case use the sqlite3_stdio.h +** #defines. If not, create our own for fopen(). +*/ +#ifndef _SQLITE3_STDIO_H_ +# define sqlite3_fopen fopen +#endif /* ** Structure of the fsdir() table-valued function @@ -142,7 +149,7 @@ static void readFileContents(sqlite3_context *ctx, const char *zName){ sqlite3 *db; int mxBlob; - in = fopen(zName, "rb"); + in = sqlite3_fopen(zName, "rb"); if( in==0 ){ /* File does not exist or is unreadable. Leave the result set to NULL. */ return; @@ -397,7 +404,7 @@ static int writeFile( sqlite3_int64 nWrite = 0; const char *z; int rc = 0; - FILE *out = fopen(zFile, "wb"); + FILE *out = sqlite3_fopen(zFile, "wb"); if( out==0 ) return 1; z = (const char*)sqlite3_value_blob(pData); if( z ){ diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c index 9790a1d877..07d7970609 100644 --- a/ext/misc/sha1.c +++ b/ext/misc/sha1.c @@ -196,7 +196,8 @@ static void hash_step_vformat( ** zOut[]. zOut[] must be at least 41 bytes long. */ static void hash_finish( SHA1Context *p, /* The SHA1 context to finish and render */ - char *zOut /* Store hexadecimal hash here */ + char *zOut, /* Store hex or binary hash here */ + int bAsBinary /* 1 for binary hash, 0 for hex hash */ ){ unsigned int i; unsigned char finalcount[8]; @@ -215,11 +216,15 @@ static void hash_finish( for (i = 0; i < 20; i++){ digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); } - for(i=0; i<20; i++){ - zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; - zOut[i*2+1] = zEncode[digest[i] & 0xf]; + if( bAsBinary ){ + memcpy(zOut, digest, 20); + }else{ + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; } - zOut[i*2]= 0; } /* End of the hashing logic *****************************************************************************/ @@ -251,8 +256,13 @@ static void sha1Func( }else{ hash_step(&cx, sqlite3_value_text(argv[0]), nByte); } - hash_finish(&cx, zOut); - sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); + if( sqlite3_user_data(context)!=0 ){ + hash_finish(&cx, zOut, 1); + sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT); + }else{ + hash_finish(&cx, zOut, 0); + sqlite3_result_blob(context, zOut, 40, SQLITE_TRANSIENT); + } } /* @@ -365,7 +375,7 @@ static void sha1QueryFunc( } sqlite3_finalize(pStmt); } - hash_finish(&cx, zOut); + hash_finish(&cx, zOut, 0); sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); } @@ -379,11 +389,17 @@ int sqlite3_sha_init( const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; + static int one = 1; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ rc = sqlite3_create_function(db, "sha1", 1, SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, - 0, sha1Func, 0, 0); + 0, sha1Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha1b", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + (void*)&one, sha1Func, 0, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "sha1_query", 1, SQLITE_UTF8|SQLITE_DIRECTONLY, 0, diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index a0c5aafd10..6debdf5dbd 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -351,7 +351,7 @@ static int substituteCost(char cPrev, char cFrom, char cTo){ ** Negative values indicate an error: ** -1 One of the inputs is NULL ** -2 Non-ASCII characters on input -** -3 Unable to allocate memory +** -3 Unable to allocate memory ** ** If pnMatch is not NULL, then *pnMatch is set to the number of bytes ** of zB that matched the pattern in zA. If zA does not end with a '*', @@ -360,8 +360,8 @@ static int substituteCost(char cPrev, char cFrom, char cTo){ ** of zB that was deemed to match zA. */ static int editdist1(const char *zA, const char *zB, int *pnMatch){ - int nA, nB; /* Number of characters in zA[] and zB[] */ - int xA, xB; /* Loop counters for zA[] and zB[] */ + unsigned int nA, nB; /* Number of characters in zA[] and zB[] */ + unsigned int xA, xB; /* Loop counters for zA[] and zB[] */ char cA = 0, cB; /* Current character of zA and zB */ char cAprev, cBprev; /* Previous character of zA and zB */ char cAnext, cBnext; /* Next character in zA and zB */ @@ -3021,7 +3021,7 @@ static sqlite3_module spellfix1Module = { */ static int spellfix1Register(sqlite3 *db){ int rc = SQLITE_OK; - int i; + unsigned int i; rc = sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, transliterateSqlFunc, 0, 0); diff --git a/ext/misc/sqlite3_stdio.c b/ext/misc/sqlite3_stdio.c new file mode 100644 index 0000000000..46f12dff36 --- /dev/null +++ b/ext/misc/sqlite3_stdio.c @@ -0,0 +1,211 @@ +/* +** 2024-09-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Implementation of standard I/O interfaces for UTF-8 that are missing +** on Windows. +*/ +#ifdef _WIN32 /* This file is a no-op on all platforms except Windows */ +#ifndef _SQLITE3_STDIO_H_ +#include "sqlite3_stdio.h" +#endif +#undef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#include +#include +#include "sqlite3.h" +#include +#include +#include +#include + +/* +** If the SQLITE_U8TEXT_ONLY option is defined, then only use +** _O_U8TEXT, _O_WTEXT, and similar together with the UTF-16 +** interfaces to the Windows CRT. The use of ANSI-only routines +** like fputs() and ANSI modes like _O_TEXT and _O_BINARY is +** avoided. +** +** The downside of using SQLITE_U8TEXT_ONLY is that it becomes +** impossible to output a bare newline character (0x0a) - that is, +** a newline that is not preceded by a carriage return (0x0d). +** And without that capability, sometimes the output will be slightly +** incorrect, as extra 0x0d characters will have been inserted where +** they do not belong. +** +** The SQLITE_U8TEXT_STDIO compile-time option is a compromise. +** It always enables _O_WTEXT or similar for stdin, stdout, stderr, +** but allows other streams to be _O_TEXT and/or O_BINARY. The +** SQLITE_U8TEXT_STDIO option has the same downside as SQLITE_U8TEXT_ONLY +** in that stray 0x0d characters might appear where they ought not, but +** at least with this option those characters only appear on standard +** I/O streams, and not on new streams that might be created by the +** application using sqlite3_fopen() or sqlite3_popen(). +*/ +#if defined(SQLITE_U8TEXT_ONLY) +# define UseWtextForOutput(fd) 1 +# define UseWtextForInput(fd) 1 +# define IsConsole(fd) _isatty(_fileno(fd)) +#elif defined(SQLITE_U8TEXT_STDIO) +# define UseWtextForOutput(fd) ((fd)==stdout || (fd)==stderr) +# define UseWtextForInput(fd) ((fd)==stdin) +# define IsConsole(fd) _isatty(_fileno(fd)) +#else +# define UseWtextForOutput(fd) _isatty(_fileno(fd)) +# define UseWtextForInput(fd) _isatty(_fileno(fd)) +# define IsConsole(fd) 1 +#endif + +/* +** Work-alike for the fopen() routine from the standard C library. +*/ +FILE *sqlite3_fopen(const char *zFilename, const char *zMode){ + FILE *fp = 0; + wchar_t *b1, *b2; + int sz1, sz2; + + sz1 = (int)strlen(zFilename); + sz2 = (int)strlen(zMode); + b1 = malloc( (sz1+1)*sizeof(b1[0]) ); + b2 = malloc( (sz2+1)*sizeof(b1[0]) ); + if( b1 && b2 ){ + sz1 = MultiByteToWideChar(CP_UTF8, 0, zFilename, sz1, b1, sz1); + b1[sz1] = 0; + sz2 = MultiByteToWideChar(CP_UTF8, 0, zMode, sz2, b2, sz2); + b2[sz2] = 0; + fp = _wfopen(b1, b2); + } + free(b1); + free(b2); + return fp; +} + + +/* +** Work-alike for the popen() routine from the standard C library. +*/ +FILE *sqlite3_popen(const char *zCommand, const char *zMode){ + FILE *fp = 0; + wchar_t *b1, *b2; + int sz1, sz2; + + sz1 = (int)strlen(zCommand); + sz2 = (int)strlen(zMode); + b1 = malloc( (sz1+1)*sizeof(b1[0]) ); + b2 = malloc( (sz2+1)*sizeof(b1[0]) ); + if( b1 && b2 ){ + sz1 = MultiByteToWideChar(CP_UTF8, 0, zCommand, sz1, b1, sz1); + b1[sz1] = 0; + sz2 = MultiByteToWideChar(CP_UTF8, 0, zMode, sz2, b2, sz2); + b2[sz2] = 0; + fp = _wpopen(b1, b2); + } + free(b1); + free(b2); + return fp; +} + +/* +** Work-alike for fgets() from the standard C library. +*/ +char *sqlite3_fgets(char *buf, int sz, FILE *in){ + if( UseWtextForInput(in) ){ + /* When reading from the command-prompt in Windows, it is necessary + ** to use _O_WTEXT input mode to read UTF-16 characters, then translate + ** that into UTF-8. Otherwise, non-ASCII characters all get translated + ** into '?'. + */ + wchar_t *b1 = malloc( sz*sizeof(wchar_t) ); + if( b1==0 ) return 0; + _setmode(_fileno(in), IsConsole(in) ? _O_WTEXT : _O_U8TEXT); + if( fgetws(b1, sz/4, in)==0 ){ + sqlite3_free(b1); + return 0; + } + WideCharToMultiByte(CP_UTF8, 0, b1, -1, buf, sz, 0, 0); + sqlite3_free(b1); + return buf; + }else{ + /* Reading from a file or other input source, just read bytes without + ** any translation. */ + return fgets(buf, sz, in); + } +} + +/* +** Work-alike for fputs() from the standard C library. +*/ +int sqlite3_fputs(const char *z, FILE *out){ + if( UseWtextForOutput(out) ){ + /* When writing to the command-prompt in Windows, it is necessary + ** to use _O_WTEXT input mode and write UTF-16 characters. + */ + int sz = (int)strlen(z); + wchar_t *b1 = malloc( (sz+1)*sizeof(wchar_t) ); + if( b1==0 ) return 0; + sz = MultiByteToWideChar(CP_UTF8, 0, z, sz, b1, sz); + b1[sz] = 0; + _setmode(_fileno(out), _O_U8TEXT); + fputws(b1, out); + sqlite3_free(b1); + return 0; + }else{ + /* Writing to a file or other destination, just write bytes without + ** any translation. */ + return fputs(z, out); + } +} + + +/* +** Work-alike for fprintf() from the standard C library. +*/ +int sqlite3_fprintf(FILE *out, const char *zFormat, ...){ + int rc; + if( UseWtextForOutput(out) ){ + /* When writing to the command-prompt in Windows, it is necessary + ** to use _O_WTEXT input mode and write UTF-16 characters. + */ + char *z; + va_list ap; + + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + sqlite3_fputs(z, out); + rc = (int)strlen(z); + sqlite3_free(z); + }else{ + /* Writing to a file or other destination, just write bytes without + ** any translation. */ + va_list ap; + va_start(ap, zFormat); + rc = vfprintf(out, zFormat, ap); + va_end(ap); + } + return rc; +} + +/* +** Set the mode for an output stream. mode argument is typically _O_BINARY or +** _O_TEXT. +*/ +void sqlite3_fsetmode(FILE *fp, int mode){ + if( !UseWtextForOutput(fp) ){ + fflush(fp); + _setmode(_fileno(fp), mode); + } +} + +#endif /* defined(_WIN32) */ diff --git a/ext/misc/sqlite3_stdio.h b/ext/misc/sqlite3_stdio.h new file mode 100644 index 0000000000..dd0eefad04 --- /dev/null +++ b/ext/misc/sqlite3_stdio.h @@ -0,0 +1,55 @@ +/* +** 2024-09-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This header file contains definitions of interfaces that provide +** cross-platform I/O for UTF-8 content. +** +** On most platforms, the interfaces definitions in this file are +** just #defines. For example sqlite3_fopen() is a macro that resolves +** to the standard fopen() in the C-library. +** +** But Windows does not have a standard C-library, at least not one that +** can handle UTF-8. So for windows build, the interfaces resolve to new +** C-language routines contained in the separate sqlite3_stdio.c source file. +** +** So on all non-Windows platforms, simply #include this header file and +** use the interfaces defined herein. Then to run your application on Windows, +** also link in the accompanying sqlite3_stdio.c source file when compiling +** to get compatible interfaces. +*/ +#ifndef _SQLITE3_STDIO_H_ +#define _SQLITE3_STDIO_H_ 1 +#ifdef _WIN32 +/**** Definitions For Windows ****/ +#include +#include + +FILE *sqlite3_fopen(const char *zFilename, const char *zMode); +FILE *sqlite3_popen(const char *zCommand, const char *type); +char *sqlite3_fgets(char *s, int size, FILE *stream); +int sqlite3_fputs(const char *s, FILE *stream); +int sqlite3_fprintf(FILE *stream, const char *format, ...); +void sqlite3_fsetmode(FILE *stream, int mode); + + +#else +/**** Definitions For All Other Platforms ****/ +#include +#define sqlite3_fopen fopen +#define sqlite3_popen popen +#define sqlite3_fgets fgets +#define sqlite3_fputs fputs +#define sqlite3_fprintf fprintf +#define sqlite3_fsetmode(F,X) /*no-op*/ + +#endif +#endif /* _SQLITE3_STDIO_H_ */ diff --git a/src/test_vfstrace.c b/ext/misc/vfstrace.c similarity index 81% rename from src/test_vfstrace.c rename to ext/misc/vfstrace.c index 8b7d2fec79..b7c8175eeb 100644 --- a/src/test_vfstrace.c +++ b/ext/misc/vfstrace.c @@ -182,7 +182,7 @@ static const char *vfstraceNextSystemCall(sqlite3_vfs*, const char *zName); ** xyzzy.txt -> xyzzy.txt */ static const char *fileTail(const char *z){ - int i; + size_t i; if( z==0 ) return 0; i = strlen(z)-1; while( i>0 && z[i-1]!='/' ){ i--; } @@ -207,36 +207,34 @@ static void vfstrace_printf( } /* -** Convert value rc into a string and print it using zFormat. zFormat -** should have exactly one %s +** Try to convert an error code into a symbolic name for that error code. */ -static void vfstrace_print_errcode( - vfstrace_info *pInfo, - const char *zFormat, - int rc -){ - char zBuf[50]; - char *zVal; +static const char *vfstrace_errcode_name(int rc ){ + const char *zVal = 0; switch( rc ){ - case SQLITE_OK: zVal = "SQLITE_OK"; break; - case SQLITE_ERROR: zVal = "SQLITE_ERROR"; break; - case SQLITE_PERM: zVal = "SQLITE_PERM"; break; - case SQLITE_ABORT: zVal = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zVal = "SQLITE_BUSY"; break; - case SQLITE_NOMEM: zVal = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zVal = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zVal = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zVal = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zVal = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zVal = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zVal = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zVal = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zVal = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zVal = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zVal = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zVal = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zVal = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zVal = "SQLITE_NOLFS"; break; + case SQLITE_OK: zVal = "SQLITE_OK"; break; + case SQLITE_INTERNAL: zVal = "SQLITE_INTERNAL"; break; + case SQLITE_ERROR: zVal = "SQLITE_ERROR"; break; + case SQLITE_PERM: zVal = "SQLITE_PERM"; break; + case SQLITE_ABORT: zVal = "SQLITE_ABORT"; break; + case SQLITE_BUSY: zVal = "SQLITE_BUSY"; break; + case SQLITE_LOCKED: zVal = "SQLITE_LOCKED"; break; + case SQLITE_NOMEM: zVal = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zVal = "SQLITE_READONLY"; break; + case SQLITE_INTERRUPT: zVal = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zVal = "SQLITE_IOERR"; break; + case SQLITE_CORRUPT: zVal = "SQLITE_CORRUPT"; break; + case SQLITE_NOTFOUND: zVal = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zVal = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zVal = "SQLITE_CANTOPEN"; break; + case SQLITE_PROTOCOL: zVal = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zVal = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zVal = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zVal = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zVal = "SQLITE_CONSTRAINT"; break; + case SQLITE_MISMATCH: zVal = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zVal = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zVal = "SQLITE_NOLFS"; break; case SQLITE_IOERR_READ: zVal = "SQLITE_IOERR_READ"; break; case SQLITE_IOERR_SHORT_READ: zVal = "SQLITE_IOERR_SHORT_READ"; break; case SQLITE_IOERR_WRITE: zVal = "SQLITE_IOERR_WRITE"; break; @@ -266,11 +264,30 @@ static void vfstrace_print_errcode( case SQLITE_LOCKED_SHAREDCACHE: zVal = "SQLITE_LOCKED_SHAREDCACHE"; break; case SQLITE_BUSY_RECOVERY: zVal = "SQLITE_BUSY_RECOVERY"; break; case SQLITE_CANTOPEN_NOTEMPDIR: zVal = "SQLITE_CANTOPEN_NOTEMPDIR"; break; - default: { - sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", rc); - zVal = zBuf; - break; + } + return zVal; +} + +/* +** Convert value rc into a string and print it using zFormat. zFormat +** should have exactly one %s +*/ +static void vfstrace_print_errcode( + vfstrace_info *pInfo, + const char *zFormat, + int rc +){ + const char *zVal; + char zBuf[50]; + zVal = vfstrace_errcode_name(rc); + if( zVal==0 ){ + zVal = vfstrace_errcode_name(rc&0xff); + if( zVal ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%s | 0x%x", zVal, rc&0xffff00); + }else{ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%d (0x%x)", rc, rc); } + zVal = zBuf; } vfstrace_printf(pInfo, zFormat, zVal); } @@ -399,7 +416,7 @@ static const char *lockName(int eLock){ const char *azLockNames[] = { "NONE", "SHARED", "RESERVED", "PENDING", "EXCLUSIVE" }; - if( eLock<0 || eLock>=sizeof(azLockNames)/sizeof(azLockNames[0]) ){ + if( eLock<0 || eLock>=(int)(sizeof(azLockNames)/sizeof(azLockNames[0])) ){ return "???"; }else{ return azLockNames[eLock]; @@ -457,12 +474,14 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ vfstrace_info *pInfo = p->pInfo; int rc; char zBuf[100]; + char zBuf2[100]; char *zOp; + char *zRVal = 0; switch( op ){ - case SQLITE_FCNTL_LOCKSTATE: zOp = "LOCKSTATE"; break; - case SQLITE_GET_LOCKPROXYFILE: zOp = "GET_LOCKPROXYFILE"; break; - case SQLITE_SET_LOCKPROXYFILE: zOp = "SET_LOCKPROXYFILE"; break; - case SQLITE_LAST_ERRNO: zOp = "LAST_ERRNO"; break; + case SQLITE_FCNTL_LOCKSTATE: zOp = "LOCKSTATE"; break; + case SQLITE_GET_LOCKPROXYFILE: zOp = "GET_LOCKPROXYFILE"; break; + case SQLITE_SET_LOCKPROXYFILE: zOp = "SET_LOCKPROXYFILE"; break; + case SQLITE_LAST_ERRNO: zOp = "LAST_ERRNO"; break; case SQLITE_FCNTL_SIZE_HINT: { sqlite3_snprintf(sizeof(zBuf), zBuf, "SIZE_HINT,%lld", *(sqlite3_int64*)pArg); @@ -474,20 +493,62 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ zOp = zBuf; break; } - case SQLITE_FCNTL_FILE_POINTER: zOp = "FILE_POINTER"; break; - case SQLITE_FCNTL_SYNC_OMITTED: zOp = "SYNC_OMITTED"; break; - case SQLITE_FCNTL_WIN32_AV_RETRY: zOp = "WIN32_AV_RETRY"; break; - case SQLITE_FCNTL_PERSIST_WAL: zOp = "PERSIST_WAL"; break; - case SQLITE_FCNTL_OVERWRITE: zOp = "OVERWRITE"; break; - case SQLITE_FCNTL_VFSNAME: zOp = "VFSNAME"; break; - case SQLITE_FCNTL_TEMPFILENAME: zOp = "TEMPFILENAME"; break; - case 0xca093fa0: zOp = "DB_UNCHANGED"; break; + case SQLITE_FCNTL_FILE_POINTER: zOp = "FILE_POINTER"; break; + case SQLITE_FCNTL_WIN32_AV_RETRY: zOp = "WIN32_AV_RETRY"; break; + case SQLITE_FCNTL_PERSIST_WAL: { + sqlite3_snprintf(sizeof(zBuf), zBuf, "PERSIST_WAL,%d", *(int*)pArg); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_OVERWRITE: zOp = "OVERWRITE"; break; + case SQLITE_FCNTL_VFSNAME: zOp = "VFSNAME"; break; + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: zOp = "POWERSAFE_OVERWRITE"; break; case SQLITE_FCNTL_PRAGMA: { const char *const* a = (const char*const*)pArg; sqlite3_snprintf(sizeof(zBuf), zBuf, "PRAGMA,[%s,%s]",a[1],a[2]); zOp = zBuf; break; } + case SQLITE_FCNTL_BUSYHANDLER: zOp = "BUSYHANDLER"; break; + case SQLITE_FCNTL_TEMPFILENAME: zOp = "TEMPFILENAME"; break; + case SQLITE_FCNTL_MMAP_SIZE: { + sqlite3_int64 iMMap = *(sqlite3_int64*)pArg; + sqlite3_snprintf(sizeof(zBuf), zBuf, "MMAP_SIZE,%lld",iMMap); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_TRACE: zOp = "TRACE"; break; + case SQLITE_FCNTL_HAS_MOVED: zOp = "HAS_MOVED"; break; + case SQLITE_FCNTL_SYNC: zOp = "SYNC"; break; + case SQLITE_FCNTL_COMMIT_PHASETWO: zOp = "COMMIT_PHASETWO"; break; + case SQLITE_FCNTL_WIN32_SET_HANDLE: zOp = "WIN32_SET_HANDLE"; break; + case SQLITE_FCNTL_WAL_BLOCK: zOp = "WAL_BLOCK"; break; + case SQLITE_FCNTL_ZIPVFS: zOp = "ZIPVFS"; break; + case SQLITE_FCNTL_RBU: zOp = "RBU"; break; + case SQLITE_FCNTL_VFS_POINTER: zOp = "VFS_POINTER"; break; + case SQLITE_FCNTL_JOURNAL_POINTER: zOp = "JOURNAL_POINTER"; break; + case SQLITE_FCNTL_WIN32_GET_HANDLE: zOp = "WIN32_GET_HANDLE"; break; + case SQLITE_FCNTL_PDB: zOp = "PDB"; break; + case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: zOp = "BEGIN_ATOMIC_WRITE"; break; + case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: zOp = "COMMIT_ATOMIC_WRITE"; break; + case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: { + zOp = "ROLLBACK_ATOMIC_WRITE"; + break; + } + case SQLITE_FCNTL_LOCK_TIMEOUT: { + sqlite3_snprintf(sizeof(zBuf), zBuf, "LOCK_TIMEOUT,%d", *(int*)pArg); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_DATA_VERSION: zOp = "DATA_VERSION"; break; + case SQLITE_FCNTL_SIZE_LIMIT: zOp = "SIZE_LIMIT"; break; + case SQLITE_FCNTL_CKPT_DONE: zOp = "CKPT_DONE"; break; + case SQLITE_FCNTL_RESERVE_BYTES: zOp = "RESERVED_BYTES"; break; + case SQLITE_FCNTL_CKPT_START: zOp = "CKPT_START"; break; + case SQLITE_FCNTL_EXTERNAL_READER: zOp = "EXTERNAL_READER"; break; + case SQLITE_FCNTL_CKSM_FILE: zOp = "CKSM_FILE"; break; + case SQLITE_FCNTL_RESET_CACHE: zOp = "RESET_CACHE"; break; + case 0xca093fa0: zOp = "DB_UNCHANGED"; break; default: { sqlite3_snprintf(sizeof zBuf, zBuf, "%d", op); zOp = zBuf; @@ -497,15 +558,37 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ vfstrace_printf(pInfo, "%s.xFileControl(%s,%s)", pInfo->zVfsName, p->zFName, zOp); rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg); - vfstrace_print_errcode(pInfo, " -> %s\n", rc); - if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ - *(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z", + if( rc==SQLITE_OK ){ + switch( op ){ + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z", pInfo->zVfsName, *(char**)pArg); + zRVal = *(char**)pArg; + break; + } + case SQLITE_FCNTL_MMAP_SIZE: { + sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%lld", *(sqlite3_int64*)pArg); + zRVal = zBuf2; + break; + } + case SQLITE_FCNTL_HAS_MOVED: + case SQLITE_FCNTL_PERSIST_WAL: { + sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%d", *(int*)pArg); + zRVal = zBuf2; + break; + } + case SQLITE_FCNTL_PRAGMA: + case SQLITE_FCNTL_TEMPFILENAME: { + zRVal = *(char**)pArg; + break; + } + } } - if( (op==SQLITE_FCNTL_PRAGMA || op==SQLITE_FCNTL_TEMPFILENAME) - && rc==SQLITE_OK && *(char**)pArg ){ - vfstrace_printf(pInfo, "%s.xFileControl(%s,%s) returns %s", - pInfo->zVfsName, p->zFName, zOp, *(char**)pArg); + if( zRVal ){ + vfstrace_print_errcode(pInfo, " -> %s", rc); + vfstrace_printf(pInfo, ", %s\n", zRVal); + }else{ + vfstrace_print_errcode(pInfo, " -> %s\n", rc); } return rc; } @@ -840,14 +923,14 @@ int vfstrace_register( sqlite3_vfs *pNew; sqlite3_vfs *pRoot; vfstrace_info *pInfo; - int nName; - int nByte; + size_t nName; + size_t nByte; pRoot = sqlite3_vfs_find(zOldVfsName); if( pRoot==0 ) return SQLITE_NOTFOUND; nName = strlen(zTraceName); nByte = sizeof(*pNew) + sizeof(*pInfo) + nName + 1; - pNew = sqlite3_malloc( nByte ); + pNew = sqlite3_malloc64( nByte ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, nByte); pInfo = (vfstrace_info*)&pNew[1]; @@ -890,3 +973,15 @@ int vfstrace_register( pInfo->zVfsName, pRoot->zName); return sqlite3_vfs_register(pNew, makeDefault); } + +/* +** Look for the named VFS. If it is a TRACEVFS, then unregister it +** and delete it. +*/ +void vfstrace_unregister(const char *zTraceName){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zTraceName); + if( pVfs==0 ) return; + if( pVfs->xOpen!=vfstraceOpen ) return; + sqlite3_vfs_unregister(pVfs); + sqlite3_free(pVfs); +} diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index fd6d13bc38..2377457dfb 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -35,6 +35,14 @@ SQLITE_EXTENSION_INIT1 #include +/* When used as part of the CLI, the sqlite3_stdio.h module will have +** been included before this one. In that case use the sqlite3_stdio.h +** #defines. If not, create our own for fopen(). +*/ +#ifndef _SQLITE3_STDIO_H_ +# define sqlite3_fopen fopen +#endif + #ifndef SQLITE_OMIT_VIRTUALTABLE #ifndef SQLITE_AMALGAMATION @@ -1291,7 +1299,7 @@ static int zipfileFilter( } if( 0==pTab->pWriteFd && 0==bInMemory ){ - pCsr->pFile = zFile ? fopen(zFile, "rb") : 0; + pCsr->pFile = zFile ? sqlite3_fopen(zFile, "rb") : 0; if( pCsr->pFile==0 ){ zipfileCursorErr(pCsr, "cannot open file: %s", zFile); rc = SQLITE_ERROR; @@ -1481,7 +1489,7 @@ static int zipfileBegin(sqlite3_vtab *pVtab){ ** structure into memory. During the transaction any new file data is ** appended to the archive file, but the central directory is accumulated ** in main-memory until the transaction is committed. */ - pTab->pWriteFd = fopen(pTab->zFile, "ab+"); + pTab->pWriteFd = sqlite3_fopen(pTab->zFile, "ab+"); if( pTab->pWriteFd==0 ){ pTab->base.zErrMsg = sqlite3_mprintf( "zipfile: failed to open file %s for writing", pTab->zFile diff --git a/ext/session/sessionalter.test b/ext/session/sessionalter.test index 34424cf275..bcff7c5357 100644 --- a/ext/session/sessionalter.test +++ b/ext/session/sessionalter.test @@ -202,6 +202,7 @@ foreach {tn sql1 at sql2} { sqlite3changegroup grp grp schema db main + breakpoint grp add $C1 grp add $C2 set T1 [grp output] diff --git a/ext/session/sessionfault3.test b/ext/session/sessionfault3.test index af5a4cdb43..5af9c9ed70 100644 --- a/ext/session/sessionfault3.test +++ b/ext/session/sessionfault3.test @@ -56,4 +56,28 @@ do_faultsim_test 1 -faults oom* -prep { faultsim_test_result {0 {}} {1 SQLITE_NOMEM} } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + ALTER TABLE t1 ADD COLUMN c DEFAULT 'abcdefghijklmnopqrstuvwxyz'; +} +faultsim_save_and_close + +do_faultsim_test 2 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval {SELECT * FROM sqlite_schema} +} -body { + sqlite3session S db main + S attach * + execsql { + DELETE FROM t1 WHERE a = 1; + } +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + catch { S delete } +} + finish_test diff --git a/ext/session/sessionnoact.test b/ext/session/sessionnoact.test index 1274ecb146..aa1cde4749 100644 --- a/ext/session/sessionnoact.test +++ b/ext/session/sessionnoact.test @@ -82,6 +82,7 @@ do_execsql_test 1.5 { UPDATE p1 SET c=12345 WHERE a = 45; } +breakpoint sqlite3changeset_apply_v2 -noaction db $C conflict do_execsql_test 1.6 { SELECT * FROM c1 diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index ba8b6d860c..c104bba28d 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -81,6 +81,10 @@ struct SessionBuffer { ** input data. Input data may be supplied either as a single large buffer ** (e.g. sqlite3changeset_start()) or using a stream function (e.g. ** sqlite3changeset_start_strm()). +** +** bNoDiscard: +** If true, then the only time data is discarded is as a result of explicit +** sessionDiscardData() calls. Not within every sessionInputBuffer() call. */ struct SessionInput { int bNoDiscard; /* If true, do not discard in InputBuffer() */ @@ -1765,16 +1769,19 @@ static void sessionPreupdateOneChange( for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ - TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p); - assert( trc==SQLITE_OK ); + /* This may fail if the column has a non-NULL default and was added + ** using ALTER TABLE ADD COLUMN after this record was created. */ + rc = pSession->hook.xOld(pSession->hook.pCtx, i, &p); }else if( pTab->abPK[i] ){ TESTONLY(int trc = ) pSession->hook.xNew(pSession->hook.pCtx, i, &p); assert( trc==SQLITE_OK ); } - /* This may fail if SQLite value p contains a utf-16 string that must - ** be converted to utf-8 and an OOM error occurs while doing so. */ - rc = sessionSerializeValue(0, p, &nByte); + if( rc==SQLITE_OK ){ + /* This may fail if SQLite value p contains a utf-16 string that must + ** be converted to utf-8 and an OOM error occurs while doing so. */ + rc = sessionSerializeValue(0, p, &nByte); + } if( rc!=SQLITE_OK ) goto error_out; } if( pTab->bRowid ){ @@ -5155,15 +5162,21 @@ static int sessionChangesetApply( int nTab = 0; /* Result of sqlite3Strlen30(zTab) */ SessionApplyCtx sApply; /* changeset_apply() context object */ int bPatchset; + u64 savedFlag = db->flags & SQLITE_FkNoAction; assert( xConflict!=0 ); + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + if( flags & SQLITE_CHANGESETAPPLY_FKNOACTION ){ + db->flags |= ((u64)SQLITE_FkNoAction); + db->aDb[0].pSchema->schema_cookie -= 32; + } + pIter->in.bNoDiscard = 1; memset(&sApply, 0, sizeof(sApply)); sApply.bRebase = (ppRebase && pnRebase); sApply.bInvertConstraints = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); sApply.bIgnoreNoop = !!(flags & SQLITE_CHANGESETAPPLY_IGNORENOOP); - sqlite3_mutex_enter(sqlite3_db_mutex(db)); if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){ rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); } @@ -5325,6 +5338,12 @@ static int sessionChangesetApply( sqlite3_free((char*)sApply.azCol); /* cast works around VC++ bug */ sqlite3_free((char*)sApply.constraints.aBuf); sqlite3_free((char*)sApply.rebase.aBuf); + + if( (flags & SQLITE_CHANGESETAPPLY_FKNOACTION) && savedFlag==0 ){ + assert( db->flags & SQLITE_FkNoAction ); + db->flags &= ~((u64)SQLITE_FkNoAction); + db->aDb[0].pSchema->schema_cookie -= 32; + } sqlite3_mutex_leave(sqlite3_db_mutex(db)); return rc; } @@ -5353,12 +5372,6 @@ int sqlite3changeset_apply_v2( sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ int bInv = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); int rc = sessionChangesetStart(&pIter, 0, 0, nChangeset, pChangeset, bInv, 1); - u64 savedFlag = db->flags & SQLITE_FkNoAction; - - if( flags & SQLITE_CHANGESETAPPLY_FKNOACTION ){ - db->flags |= ((u64)SQLITE_FkNoAction); - db->aDb[0].pSchema->schema_cookie -= 32; - } if( rc==SQLITE_OK ){ rc = sessionChangesetApply( @@ -5366,11 +5379,6 @@ int sqlite3changeset_apply_v2( ); } - if( (flags & SQLITE_CHANGESETAPPLY_FKNOACTION) && savedFlag==0 ){ - assert( db->flags & SQLITE_FkNoAction ); - db->flags &= ~((u64)SQLITE_FkNoAction); - db->aDb[0].pSchema->schema_cookie -= 32; - } return rc; } @@ -5691,6 +5699,9 @@ static int sessionChangesetExtendRecord( sessionAppendBlob(pOut, aRec, nRec, &rc); if( rc==SQLITE_OK && pTab->pDfltStmt==0 ){ rc = sessionPrepareDfltStmt(pGrp->db, pTab, &pTab->pDfltStmt); + if( rc==SQLITE_OK && SQLITE_ROW!=sqlite3_step(pTab->pDfltStmt) ){ + rc = sqlite3_errcode(pGrp->db); + } } for(ii=nCol; rc==SQLITE_OK && iinCol; ii++){ int eType = sqlite3_column_type(pTab->pDfltStmt, ii); @@ -5707,6 +5718,7 @@ static int sessionChangesetExtendRecord( } if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){ sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + pOut->nBuf += 8; } break; } @@ -5846,6 +5858,8 @@ static int sessionOneChangeToHash( u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; + assert( nRec>0 ); + /* Ensure that only changesets, or only patchsets, but not a mixture ** of both, are being combined. It is an error to try to combine a ** changeset and a patchset. */ @@ -5923,6 +5937,7 @@ static int sessionChangesetToHash( int nRec; int rc = SQLITE_OK; + pIter->in.bNoDiscard = 1; while( SQLITE_ROW==(sessionChangesetNext(pIter, &aRec, &nRec, 0)) ){ rc = sessionOneChangeToHash(pGrp, pIter, bRebase); if( rc!=SQLITE_OK ) break; diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 51531627b9..2602e6b792 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -15,8 +15,9 @@ # by the target name. Rebuild is necessary for all components to get # the desired optimization level. # -# quick, q = do just a minimal build (sqlite3.js/wasm, tester1) for -# faster development-mode turnaround. +# quick, q = do just build the essentials for testing +# (sqlite3.js/wasm, tester1) for faster development-mode +# turnaround. # # dist = create end user deliverables. Add dist.build=oX to build # with a specific optimization level, where oX is one of the @@ -35,25 +36,21 @@ # - wasm-strip for release builds: https://github.com/WebAssembly/wabt # - InfoZip for 'dist' zip file ######################################################################## -# -# Significant TODOs for this build include, but are not necessarily -# limited to: -# -# 1) Consolidate the code generation for sqlite3*.*js into a script -# which generates the makefile code, rather than using $(call) and -# $(eval), or at least centralize the setup of the numerous vars -# related to each build variant $(JS_BUILD_MODES). (Update: an -# external script was attempted but generating properly-escaped -# makefile code from within a shell script is even less legible -# than the $(eval) indirection going on in this file.) -# default: all #default: quick -SHELL := $(shell which bash 2>/dev/null) +SHELL := $(firstword $(wildcard /usr/local/bin/bash /usr/bin/bash /bin/bash)) +ifeq (,$(SHELL)) + $(error Cannot find the bash shell) +endif MAKEFILE := $(lastword $(MAKEFILE_LIST)) CLEAN_FILES := -DISTCLEAN_FILES := ./--dummy-- -release: oz +DISTCLEAN_FILES := +MAKING_CLEAN := $(if $(filter %clean,$(MAKECMDGOALS)),1,0) +.PHONY: clean distclean +clean: + -rm -f $(CLEAN_FILES) +distclean: clean + -rm -f $(DISTCLEAN_FILES) ######################################################################## # JS_BUILD_NAMES exists for documentation purposes only. It enumerates @@ -86,46 +83,6 @@ JS_BUILD_NAMES := sqlite3 sqlite3-wasmfs # JS_BUILD_MODES := vanilla esm bunder-friendly node -######################################################################## -# Emscripten SDK home dir and related binaries... -EMSDK_HOME ?= $(word 1,$(wildcard $(HOME)/emsdk $(HOME)/src/emsdk)) -emcc.bin ?= $(word 1,$(wildcard $(EMSDK_HOME)/upstream/emscripten/emcc) $(shell which emcc)) -ifeq (,$(emcc.bin)) - $(error Cannot find emcc.) -endif -emcc.version := $(shell "$(emcc.bin)" --version | sed -n 1p \ - | sed -e 's/^.* \([3-9][^ ]*\) .*$$/\1/;') -ifeq (,$(emcc.version)) - $(warning Cannot determine emcc version. This might unduly impact build flags.) -else - $(info using emcc version [$(emcc.version)]) -endif - -wasm-strip ?= $(shell which wasm-strip 2>/dev/null) -ifeq (,$(filter clean,$(MAKECMDGOALS))) -ifeq (,$(wasm-strip)) - $(info WARNING: *******************************************************************) - $(info WARNING: builds using -O2/-O3/-Os/-Oz will minify WASM-exported names,) - $(info WARNING: breaking _All The Things_. The workaround for that is to build) - $(info WARNING: with -g3 (which explodes the file size) and then strip the debug) - $(info WARNING: info after compilation, using wasm-strip, to shrink the wasm file.) - $(info WARNING: wasm-strip was not found in the PATH so we cannot strip those.) - $(info WARNING: If this build uses any optimization level higher than -O1 then) - $(info WARNING: the ***resulting JS code WILL NOT BE USABLE***.) - $(info WARNING: wasm-strip is part of the wabt package:) - $(info WARNING: https://github.com/WebAssembly/wabt) - $(info WARNING: on Ubuntu-like systems it can be installed with:) - $(info WARNING: sudo apt install wabt) - $(info WARNING: *******************************************************************) -endif -endif # 'make clean' check - -ifeq (,$(wasm-strip)) - maybe-wasm-strip = echo "not wasm-stripping" -else - maybe-wasm-strip = $(wasm-strip) -endif - ######################################################################## # dir.top = the top dir of the canonical build tree, where # sqlite3.[ch] live. @@ -141,41 +98,19 @@ dir.common := common dir.fiddle := fiddle dir.fiddle-debug := fiddle-debug dir.tool := $(dir.top)/tool -CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ $(dir.fiddle)/*~ \ - $(dir.fiddle-debug)/* - -######################################################################## -# dir.dout = output dir for deliverables. -# -# MAINTENANCE REMINDER: the output .js and .wasm files of certain emcc -# buildables must be in _this_ dir, rather than a subdir, or else -# parts of the generated code get confused and cannot load -# property. Specifically, when X.js loads X.wasm, whether or not X.js -# uses the correct path for X.wasm depends on how it's loaded: an HTML -# script tag will resolve it intuitively, whereas a Worker's call to -# importScripts() will not. That's a fundamental incompatibility with -# how URL resolution in JS happens between those two contexts. See: -# -# https://zzz.buzz/2017/03/14/relative-uris-in-web-development/ -# -# We unfortunately have no way, from Worker-initiated code, to -# automatically resolve the path from X.js to X.wasm. -# -# We have an "only slightly unsightly" solution for our main builds -# but it does not work for the WASMFS builds, so those builds have to -# be built to _this_ directory and can only run when the client app is -# loaded from the same directory. +# dir.dout = output dir for deliverables dir.dout := $(dir.wasm)/jswasm # dir.tmp = output dir for intermediary build files, as opposed to # end-user deliverables. dir.tmp := $(dir.wasm)/bld -CLEAN_FILES += $(dir.tmp)/* $(dir.dout)/* -ifeq (,$(wildcard $(dir.dout))) - dir._tmp := $(shell mkdir -p $(dir.dout)) -endif -ifeq (,$(wildcard $(dir.tmp))) - dir._tmp := $(shell mkdir -p $(dir.tmp)) -endif +dir.wasmfs := $(dir.dout) + +MKDIR.bld := $(dir.tmp) +$(MKDIR.bld): + -mkdir -p $@ $(dir.dout) + +CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ $(dir.fiddle)/*~ \ + $(dir.fiddle-debug)/* $(dir.dout)/* $(dir.tmp)/* ######################################################################## # Set up sqlite3.c and sqlite3.h... @@ -197,48 +132,14 @@ endif sqlite3.canonical.c := $(dir.top)/sqlite3.c sqlite3.c ?= $(firstword $(wildcard $(dir.top)/sqlite3-see.c) $(sqlite3.canonical.c)) sqlite3.h := $(dir.top)/sqlite3.h -ifeq (,$(shell grep sqlite3_activate_see $(sqlite3.c) 2>/dev/null)) + +ifneq (1,$(MAKING_CLEAN)) +ifeq (,$(shell grep sqlite3_activate_see $(sqlite3.c))) SQLITE_C_IS_SEE := 0 else SQLITE_C_IS_SEE := 1 - $(info This is an SEE build.) + $(info This is an SEE build) endif -# Most SQLITE_OPT flags are set in sqlite3-wasm.c but we need them -# made explicit here for building speedtest1.c. -SQLITE_OPT = \ - -DSQLITE_ENABLE_FTS5 \ - -DSQLITE_ENABLE_RTREE \ - -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ - -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ - -DSQLITE_ENABLE_STMTVTAB \ - -DSQLITE_ENABLE_DBPAGE_VTAB \ - -DSQLITE_ENABLE_DBSTAT_VTAB \ - -DSQLITE_ENABLE_BYTECODE_VTAB \ - -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ - -DSQLITE_OMIT_LOAD_EXTENSION \ - -DSQLITE_OMIT_DEPRECATED \ - -DSQLITE_OMIT_UTF16 \ - -DSQLITE_OMIT_SHARED_CACHE \ - -DSQLITE_THREADSAFE=0 \ - -DSQLITE_TEMP_STORE=2 \ - -DSQLITE_OS_KV_OPTIONAL=1 \ - '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ - -DSQLITE_USE_URI=1 \ - -DSQLITE_WASM_ENABLE_C_TESTS \ - -DSQLITE_C=$(sqlite3.c) -#SQLITE_OPT += -DSQLITE_DEBUG -# Enabling SQLITE_DEBUG will break sqlite3_wasm_vfs_create_file() -# (and thus sqlite3_js_vfs_create_file()). Those functions are -# deprecated and alternatives are in place, but this crash behavior -# can be used to find errant uses of sqlite3_js_vfs_create_file() -# in client code. - -######################################################################## -# minimal=1 disables all "extraneous" stuff from sqlite3-wasm.c, the -# goal being to create a WASM file with only the core APIs. -minimal ?= 0 -ifeq (1,$(minimal)) - SQLITE_OPT += -DSQLITE_WASM_MINIMAL endif ########################################################################@ @@ -250,21 +151,178 @@ $(sqlite3.h): $(MAKE) -C $(dir.top) sqlite3.c $(sqlite3.c): $(sqlite3.h) -.PHONY: clean distclean -clean: - -rm -f $(CLEAN_FILES) -distclean: clean - -rm -f $(DISTCLEAN_FILES) +######################################################################## +# Special-case builds for which we require certain pre-conditions +# which, if not met, may cause warnings or fatal errors in the build. +# This also affects the default optimization level flags. Note that +# the fiddle targets are in this list because they are used for +# generating sqlite.org/fiddle. +OPTIMIZED_TARGETS := dist snapshot fiddle fiddle.debug -ifeq (release,$(filter release,$(MAKECMDGOALS))) - ifeq (,$(wasm-strip)) - $(error Cannot make release-quality binary because wasm-strip is not available. \ - See notes in the warning above) +ifneq (1,$(MAKING_CLEAN)) + ifeq (,$(filter $(OPTIMIZED_TARGETS),$(MAKECMDGOALS))) + $(info ==============================================================) + $(info == Development build. Make one of (dist, snapshot) for a) + $(info == smaller release build.) + $(info ==============================================================) endif -else - $(info Development build. Use '$(MAKE) release' for a smaller release build.) endif +######################################################################## +# Find emcc (Emscripten compiler)... +ifeq (1,$(MAKING_CLEAN)) + emcc.bin := echo + emcc.version := unknown +else + emcc.bin := $(shell which emcc 2>/dev/null) + ifeq (,$(emcc.bin)) + ifneq (,$(EMSDK_HOME)) + emcc.bin := $(wildcard $(EMSDK_HOME)/upstream/emscripten/emcc) + endif + ifeq (,$(emcc.bin)) + $(error Cannot find emcc in path.) + endif + endif + emcc.version := $(shell $(emcc.bin) --version | sed -n 1p | sed -e 's/^.* \([3-9][^ ]*\) .*$$/\1/;') + $(info using emcc version [$(emcc.version)]) +endif +######################################################################### +# Find wasm-strip, which we need for release builds (see below for +# why) but not strictly for non-release builds. +ifeq (1,$(MAKING_CLEAN)) + wasm-strip-bin := irrelevant +else + wasm-strip.bin ?= $(shell which wasm-strip 2>/dev/null) + ifeq (,$(wasm-strip.bin)) + $(info WARNING: *******************************************************************) + $(info WARNING: Builds using -O2/-O3/-Os/-Oz will minify WASM-exported names,) + $(info WARNING: breaking _All The Things_. The workaround for that is to build) + $(info WARNING: with -g3 (which explodes the file size) and then strip the debug) + $(info WARNING: info after compilation, using wasm-strip, to shrink the wasm file.) + $(info WARNING: wasm-strip was not found in the PATH so we cannot strip those.) + $(info WARNING: If this build uses any optimization level higher than -O1 then) + $(info WARNING: the ***resulting JS code WILL NOT BE USABLE***.) + $(info WARNING: wasm-strip is part of the wabt package:) + $(info WARNING: https://github.com/WebAssembly/wabt) + $(info WARNING: on Ubuntu-like systems it can be installed with:) + $(info WARNING: sudo apt install wabt) + $(info WARNING: *******************************************************************) + ifneq (,$(filter $(OPTIMIZED_TARGETS),$(MAKECMDGOALS))) + $(error Cannot make release-quality binary because wasm-strip is not available.) + endif + wasm-strip.bin := echo "not wasm-stripping" + endif +endif +maybe-wasm-strip := $(wasm-strip.bin) + +######################################################################## +# barebones=1 disables all "extraneous" stuff from sqlite3-wasm.c, the +# goal being to create a WASM file with only the core APIs. +ifeq (1,$(barebones)) + wasm-bare-bones := 1 + $(info ==============================================================) + $(info == This is a bare-bones build. It trades away features for) + $(info == a smaller .wasm file.) + $(info ==============================================================) +else + wasm-bare-bones := 0 +endif +undefine barebones + +# Common options for building sqlite3-wasm.c and speedtest1.c. +# Explicit ENABLEs... +SQLITE_OPT.common := \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_TEMP_STORE=2 \ + -DSQLITE_ENABLE_MATH_FUNCTIONS \ + -DSQLITE_OS_KV_OPTIONAL=1 \ + '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ + -DSQLITE_USE_URI=1 \ + -DSQLITE_C=$(sqlite3.c) \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_UTF16 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_SHARED_CACHE +# ^^^ These particular OMITs are hard-coded in sqlite3-wasm.c and +# removing them from this list will serve only to break the speedtest1 +# builds. + +# Currently always needed but TODO is paring tester1.c-pp.js down +# to be able to run without this: +SQLITE_OPT.common += -DSQLITE_WASM_ENABLE_C_TESTS + +SQLITE_OPT.full-featured := \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_MATH_FUNCTIONS \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_SESSION \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + +ifeq (0,$(wasm-bare-bones)) + # The so-called canonical build is full-featured: + SQLITE_OPT := \ + $(SQLITE_OPT.common) \ + $(SQLITE_OPT.full-featured) +else + # The so-called bare-bones build is exactly that: + SQLITE_OPT := \ + $(SQLITE_OPT.common) \ + -DSQLITE_WASM_BARE_BONES + # SQLITE_WASM_BARE_BONES tells sqlite3-wasm.c to explicitly omit + # a bunch of stuff, in the interest of keeping the wasm file size + # down. As of this writing it equates to: + # + # -USQLITE_ENABLE_DBPAGE_VTAB + # -USQLITE_ENABLE_DBSTAT_VTAB + # -USQLITE_ENABLE_EXPLAIN_COMMENTS + # -USQLITE_ENABLE_FTS5 + # -USQLITE_ENABLE_OFFSET_SQL_FUNC + # -USQLITE_ENABLE_PREUPDATE_HOOK + # -USQLITE_ENABLE_RTREE + # -USQLITE_ENABLE_SESSION + # -USQLITE_ENABLE_STMTVTAB + # -DSQLITE_OMIT_AUTHORIZATION + # -DSQLITE_OMIT_GET_TABLE + # -DSQLITE_OMIT_INCRBLOB + # -DSQLITE_OMIT_INTROSPECTION_PRAGMAS + # -DSQLITE_OMIT_JSON + # -DSQLITE_OMIT_PROGRESS_CALLBACK + # -DSQLITE_OMIT_WAL + # + # There are others we want here but which require explicit OMIT when + # creating their amalgamation, and that step is TODO: + # + # -DSQLITE_OMIT_EXPLAIN + # -DSQLITE_OMIT_TRIGGER + # -DSQLITE_OMIT_VIRTUALTABLE + # -DSQLITE_OMIT_WINDOWFUNC +endif + +#SQLITE_OPT += -DSQLITE_DEBUG +# Enabling SQLITE_DEBUG will break sqlite3_wasm_vfs_create_file() +# (and thus sqlite3_js_vfs_create_file()). Those functions are +# deprecated and alternatives are in place, but this crash behavior +# can be used to find errant uses of sqlite3_js_vfs_create_file() +# in client code. + +######################################################################## +# The following flags are hard-coded into sqlite3-wasm.c and cannot be +# modified via the build process: +# +# SQLITE_ENABLE_API_ARMOR +# SQLITE_OMIT_LOAD_EXTENSION +# SQLITE_OMIT_DEPRECATED +# SQLITE_OMIT_UTF16 +# SQLITE_OMIT_SHARED_CACHE +######################################################################## + + ######################################################################## # Adding custom C code via sqlite3_wasm_extra_init.c: # @@ -321,6 +379,18 @@ $(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) $(CC) -o $@ $< DISTCLEAN_FILES += $(bin.stripccomments) +######################################################################## +# bin.mkwb is used for generating some of the makefile code for the +# various wasm builds. It used to be generated in this makefile via a +# difficult-to-read/maintain block of $(eval)'d code. Attempts were +# made to generate it from tcl and bash (shell) but having to escape +# the $ references in those languages made it just as illegible as the +# native makefile code. Somewhat surprisingly, moving that code generation +# to C makes it slightly less illegible than the previous 3 options. +bin.mkwb := ./mkwasmbuilds +$(bin.mkwb): $(bin.mkwb).c $(MAKEFILE) + $(CC) -o $@ $< +DISTCLEAN_FILES += $(bin.mkwb) ######################################################################## # C-PP.FILTER: a $(call)able to transform $(1) to $(2) via: @@ -360,6 +430,7 @@ $(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_UTF16 \ -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_WAL -DSQLITE_THREADSAFE=0 \ -DSQLITE_TEMP_STORE=3 +DISTCLEAN_FILES += $(bin.c-pp) C-PP.FILTER.global ?= ifeq (1,$(SQLITE_C_IS_SEE)) C-PP.FILTER.global += -Denable-see @@ -371,7 +442,7 @@ define C-PP.FILTER # $3 = optional c-pp -D... flags $(2): $(1) $$(MAKEFILE) $$(bin.c-pp) $$(bin.c-pp) -f $(1) -o $$@ $(3) $(C-PP.FILTER.global) -CLEAN_FILES += $(2) +#CLEAN_FILES += $(2) endef # /end C-PP.FILTER ######################################################################## @@ -386,7 +457,17 @@ emcc.WASM_BIGINT ?= 1 # emcc_opt = optimization-related flags. These are primarily used by # the various oX targets. build times for -O levels higher than 0 are # painful at dev-time. -emcc_opt ?= -O0 +# +# When running any of the $(OPTIMIZED_TARGETS) explicitly, e.g. for a +# release distribution, use a higher optimization level. Experience +# has shown -Oz to produce the smallest deliverables with only a +# roughly 10% performance hit in the resulting WASM file compared to +# -O2 (which consistently creates the fastest-running deliverables). +ifeq (,$(filter $(OPTIMIZED_TARGETS),$(MAKECMDGOALS))) + emcc_opt ?= -O0 +else + emcc_opt ?= -Oz +endif # When passing emcc_opt from the CLI, += and re-assignment have no # effect, so emcc_opt+=-g3 doesn't work. So... @@ -421,24 +502,16 @@ emcc_opt_full := $(emcc_opt) -g3 ######################################################################## # EXPORTED_FUNCTIONS.* = files for use with Emscripten's # -sEXPORTED_FUNCTION flag. -EXPORTED_FUNCTIONS.api.core := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core) +EXPORTED_FUNCTIONS.api.core := $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core EXPORTED_FUNCTIONS.api.in := $(EXPORTED_FUNCTIONS.api.core) ifeq (1,$(SQLITE_C_IS_SEE)) - EXPORTED_FUNCTIONS.api.in += $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see) + EXPORTED_FUNCTIONS.api.in += $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see endif -ifeq (0,$(minimal)) - EXPORTED_FUNCTIONS.api.in += \ - $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-auth) \ - $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-preupdate) \ - $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-session) \ - $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-vtab) -else - $(info ========================================) - $(info This is a minimal-mode build) - $(info ========================================) +ifeq (0,$(wasm-bare-bones)) + EXPORTED_FUNCTIONS.api.in += $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-extras endif EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api -$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) +$(EXPORTED_FUNCTIONS.api): $(MKDIR.bld) $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) cat $(EXPORTED_FUNCTIONS.api.in) > $@ ######################################################################## @@ -470,8 +543,10 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.c-pp.js # sqlite3-vfs-helper = helper APIs for VFSes: sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.c-pp.js -# sqlite3-vtab-helper = helper APIs for VTABLEs: -sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js +ifeq (0,$(wasm-bare-bones)) + # sqlite3-vtab-helper = helper APIs for VTABLEs: + sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js +endif # sqlite3-vfs-opfs = the first OPFS VFS: sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js # sqlite3-vfs-opfs-sahpool = the second OPFS VFS: @@ -520,7 +595,7 @@ $(SOAP.js.bld): $(SOAP.js) # preprocessed. It contains all of $(sqlite3-api.jses) but none of the # Emscripten-specific headers and footers. sqlite3-api.js.in := $(dir.tmp)/sqlite3-api.c-pp.js -$(sqlite3-api.js.in): $(sqlite3-api.jses) $(MAKEFILE) +$(sqlite3-api.js.in): $(MKDIR.bld) $(sqlite3-api.jses) $(MAKEFILE) @echo "Making $@..." @for i in $(sqlite3-api.jses); do \ echo "/* BEGIN FILE: $$i */"; \ @@ -687,7 +762,7 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED ######################################################################## # $(sqlite3-api-build-version.js) injects the build version info into # the bundle in JSON form. -$(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) +$(sqlite3-api-build-version.js): $(MKDIR.bld) $(bin.version-info) $(MAKEFILE) @echo "Making $@..." @{ \ echo 'globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){'; \ @@ -703,7 +778,7 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) # # Maintenance reminder: there are awk binaries out there which do not # support -e SCRIPT. -$(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) \ +$(sqlite3-license-version.js): $(MKDIR.bld) $(sqlite3.h) $(sqlite3-license-version-header.js) \ $(MAKEFILE) @echo "Making $@..."; { \ cat $(sqlite3-license-version-header.js); \ @@ -731,7 +806,7 @@ post-jses.js := \ $(dir.api)/post-js-header.js \ $(sqlite3-api.js.in) \ $(dir.api)/post-js-footer.js -$(post-js.js.in): $(post-jses.js) $(MAKEFILE) +$(post-js.js.in): $(MKDIR.bld) $(post-jses.js) $(MAKEFILE) @echo "Making $@..." @for i in $(post-jses.js); do \ echo "/* BEGIN FILE: $$i */"; \ @@ -740,55 +815,6 @@ $(post-js.js.in): $(post-jses.js) $(MAKEFILE) done > $@ -######################################################################## -# call-make-pre-post is a $(call)able which creates rules for -# pre-js.$(1)-$(2).js. $1 = the base name of the JS file on whose -# behalf this pre-js is for (one of: $(JS_BUILD_NAMES)). $2 is -# the build mode: one of $(JS_BUILD_MODES). This sets up -# --[extern-][pre/post]-js flags in $(pre-post-$(1)-$(2).flags) and -# dependencies in $(pre-post-$(1)-$(2).deps). The resulting files get -# filtered using $(C-PP.FILTER). Any flags necessary for such -# filtering need to be set in $(c-pp.D.$(1)-$(2)) before $(call)ing -# this. -# -# Maintenance note: a shell script was written to generate these rules -# with the hope that it would make them more legible and maintainable, -# but embedding makefile code in another language makes it even less -# legible than having the level of $(eval) indirection which we have -# here. -define call-make-pre-post -pre-post-$(1)-$(2).flags ?= -pre-js.js.$(1)-$(2).intermediary := $$(dir.tmp)/pre-js.$(1)-$(2).intermediary.js -pre-js.js.$(1)-$(2) := $$(dir.tmp)/pre-js.$(1)-$(2).js -#$$(error $$(pre-js.js.$(1)-$(2).intermediary) $$(pre-js.js.$(1)-$(2))) -$$(eval $$(call C-PP.FILTER,$$(pre-js.js.in),$$(pre-js.js.$(1)-$(2).intermediary),$$(c-pp.D.$(1)-$(2)))) -post-js.js.$(1)-$(2) := $$(dir.tmp)/post-js.$(1)-$(2).js -$$(eval $$(call C-PP.FILTER,$$(post-js.js.in),$$(post-js.js.$(1)-$(2)),$$(c-pp.D.$(1)-$(2)))) -extern-post-js.js.$(1)-$(2) := $$(dir.tmp)/extern-post-js.$(1)-$(2).js -$$(eval $$(call C-PP.FILTER,$$(extern-post-js.js.in),$$(extern-post-js.js.$(1)-$(2)),$$(c-pp.D.$(1)-$(2)))) -pre-post-common.flags.$(1)-$(2) := \ - $$(pre-post-common.flags) \ - --post-js=$$(post-js.js.$(1)-$(2)) \ - --extern-post-js=$$(extern-post-js.js.$(1)-$(2)) -pre-post-jses.$(1)-$(2).deps := $$(pre-post-jses.deps.common) \ - $$(post-js.js.$(1)-$(2)) $$(extern-post-js.js.$(1)-$(2)) -$$(pre-js.js.$(1)-$(2)): $$(pre-js.js.$(1)-$(2).intermediary) $$(MAKEFILE) - cp $$(pre-js.js.$(1)-$(2).intermediary) $$@ - @if [ sqlite3-wasmfs = $(1) ]; then \ - echo "delete Module[xNameOfInstantiateWasm] /*for WASMFS build*/;"; \ - elif [ sqlite3 != $(1) ]; then \ - echo "Module[xNameOfInstantiateWasm].uri = '$(1).wasm';"; \ - fi >> $$@ -pre-post-$(1)-$(2).deps := \ - $$(pre-post-jses.$(1)-$(2).deps) \ - $$(dir.tmp)/pre-js.$(1)-$(2).js -pre-post-$(1)-$(2).flags += \ - $$(pre-post-common.flags.$(1)-$(2)) \ - --pre-js=$$(dir.tmp)/pre-js.$(1)-$(2).js -endef -# /post-js and pre-js -######################################################################## - # Undocumented Emscripten feature: if the target file extension is # "mjs", it defaults to ES6 module builds: # https://github.com/emscripten-core/emscripten/issues/14383 @@ -802,11 +828,13 @@ sqlite3-wasmfs.cfiles := $(sqlite3-wasm.cfiles) # difference, so we build all binaries against sqlite3-wasm.c instead # of building a shared copy of sqlite3-wasm.o to link against. ######################################################################## -# SQLITE3.xJS.EXPORT-DEFAULT is part of SQLITE3-WASMFS.xJS.RECIPE and -# SETUP_LIB_BUILD_MODE, factored into a separate piece to avoid code -# duplication. $1 is 1 if the build mode needs this workaround (esm, -# bundler-friendly, node) and 0 if not (vanilla). $2 must be empty for -# all builds except sqlite3-wasmfs.mjs, in which case it must be 1. + +######################################################################## +# SQLITE3.xJS.ESM-EXPORT-DEFAULT is used by mkwasmbuilds.c and the +# wasmfs build. $1 is 1 if the build mode needs this workaround +# (modes: esm, bundler-friendly, node) and 0 if not (vanilla). $2 must +# be 0 for all builds except sqlite3-wasmfs.mjs, in which case it must +# be 1. # # Reminder for ESM builds: even if we use -sEXPORT_ES6=0, emcc _still_ # adds: @@ -825,11 +853,11 @@ sqlite3-wasmfs.cfiles := $(sqlite3-wasm.cfiles) # use awk instead of sed for this. define SQLITE3.xJS.ESM-EXPORT-DEFAULT if [ x1 = x$(1) ]; then \ - echo "Fragile workaround for emscripten/issues/18237. See SQLITE3.xJS.RECIPE."; \ + echo "Fragile workaround for emscripten/issues/18237. See SQLITE3.xJS.ESM-EXPORT-DEFAULT."; \ {\ awk '/^export default/ && !f{f=1; next} 1' $@ > $@.tmp && mv $@.tmp $@; \ } || exit $$?; \ - if [ x != x$(2) ]; then \ + if [ x1 = x$(2) ]; then \ if ! grep -q '^export default' $@; then \ echo "Cannot find export default." 1>&2; \ exit 1; \ @@ -838,78 +866,6 @@ if [ x1 = x$(1) ]; then \ fi endef -######################################################################## -# extern-post-js* and extern-pre-js* are files for use with -# Emscripten's --extern-pre-js and --extern-post-js flags. -extern-pre-js.js := $(dir.api)/extern-pre-js.js -extern-post-js.js.in := $(dir.api)/extern-post-js.c-pp.js -# Emscripten flags for --[extern-][pre|post]-js=... for the -# various builds. -pre-post-common.flags := \ - --extern-pre-js=$(sqlite3-license-version.js) -# pre-post-jses.deps.* = a list of dependencies for the -# --[extern-][pre/post]-js files. -pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) - -######################################################################## -# SETUP_LIB_BUILD_MODE is a $(call)'able which sets up numerous pieces -# for one of the build modes. -# -# $1 = one of: $(JS_BUILD_NAMES) -# $2 = build mode name: one of $(JS_BUILD_MODES) -# $3 = 1 for ESM build mode, else 0 -# $4 = resulting sqlite-api JS/MJS file -# $5 = resulting JS/MJS file -# $6 = -D... flags for $(bin.c-pp) -# $7 = optional extra flags for emcc -# -# Maintenance reminder: be careful not to introduce spaces around args -# ($1, $2), otherwise string concatenation will malfunction. -# -# Before calling this, emcc.environment.$(2) must be set to a value -# for emcc's -sENVIRONMENT flag. -# -# $(cflags.$(1)) and $(cflags.$(1).$(2)) may be defined to append -# CFLAGS to a given build mode. -# -# $(emcc.flags.$(1)) and $(emcc.flags.$(1).$(2)) may be defined to -# append emcc-specific flags to a given build mode. -define SETUP_LIB_BUILD_MODE -$(info Setting up build [$(1)-$(2)]: $(5)) -c-pp.D.$(1)-$(2) := $(6) -$$(eval $$(call call-make-pre-post,$(1),$(2))) -emcc.flags.$(1).$(2) ?= -emcc.flags.$(1).$(2) += $(7) -$$(eval $$(call C-PP.FILTER, $$(sqlite3-api.js.in), $(4), $(6))) -$(5): $(4) $$(MAKEFILE) $$(sqlite3-wasm.cfiles) $$(EXPORTED_FUNCTIONS.api) $$(pre-post-$(1)-$(2).deps) - @echo "Building $$@ ..." - $$(emcc.bin) -o $$@ $$(emcc_opt_full) $$(emcc.flags) \ - $$(emcc.jsflags) \ - -sENVIRONMENT=$$(emcc.environment.$(2)) \ - $$(pre-post-$(1)-$(2).flags) \ - $$(emcc.flags.$(1)) $$(emcc.flags.$(1).$(2)) \ - $$(cflags.common) $$(SQLITE_OPT) \ - $$(cflags.$(1)) $$(cflags.$(1).$(2)) \ - $$(cflags.wasm_extra_init) $$(sqlite3-wasm.cfiles) - @$$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(3)) - @dotwasm=$$(basename $$@).wasm; \ - chmod -x $$$$dotwasm; \ - $(maybe-wasm-strip) $$$$dotwasm; \ - case $(2) in \ - bundler-friendly|node) \ - echo "Patching $$@ for $(1).wasm..."; \ - rm -f $$$$dotwasm; \ - dotwasm=; \ - sed -i -e 's/$(1)-$(2).wasm/$(1).wasm/g' $$@ || exit $$$$?; \ - ;; \ - esac; \ - ls -la $$$$dotwasm $$@ -all: $(5) -#quick: $(5) -CLEAN_FILES += $(4) $(5) -endef -# ^^^ /SETUP_LIB_BUILD_MODE -######################################################################## sqlite3-api.js := $(dir.dout)/sqlite3-api.js sqlite3.js := $(dir.dout)/sqlite3.js sqlite3-api.mjs := $(dir.dout)/sqlite3-api.mjs @@ -918,17 +874,18 @@ sqlite3-api-bundler-friendly.mjs := $(dir.dout)/sqlite3-api-bundler-friendly.mjs sqlite3-bundler-friendly.mjs := $(dir.dout)/sqlite3-bundler-friendly.mjs sqlite3-api-node.mjs := $(dir.dout)/sqlite3-api-node.mjs sqlite3-node.mjs := $(dir.dout)/sqlite3-node.mjs -#$(info $(call SETUP_LIB_BUILD_MODE,sqlite3,vanilla,0, $(sqlite3-api.js), $(sqlite3.js))) -$(eval $(call SETUP_LIB_BUILD_MODE,sqlite3,vanilla,0,\ - $(sqlite3-api.js), $(sqlite3.js))) -$(eval $(call SETUP_LIB_BUILD_MODE,sqlite3,esm,1,\ - $(sqlite3-api.mjs), $(sqlite3.mjs), -Dtarget=es6-module)) -$(eval $(call SETUP_LIB_BUILD_MODE,sqlite3,bundler-friendly,1,\ - $(sqlite3-api-bundler-friendly.mjs),$(sqlite3-bundler-friendly.mjs),\ - $(c-pp.D.sqlite3-esm) -Dtarget=es6-bundler-friendly)) -$(eval $(call SETUP_LIB_BUILD_MODE,sqlite3,node,1,\ - $(sqlite3-api-node.mjs),$(sqlite3-node.mjs),\ - $(c-pp.D.sqlite3-bundler-friendly) -Dtarget=node)) +sqlite3-api-wasmfs.mjs := $(dir.tmp)/sqlite3-api-wasmfs.mjs +sqlite3-wasmfs.mjs := $(dir.wasmfs)/sqlite3-wasmfs.mjs +EXPORTED_FUNCTIONS.fiddle := $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle +ifneq (1,$(MAKING_CLEAN)) +.wasmbuilds.make: $(bin.mkwb) + @rm -f $@ + $(bin.mkwb) > $@ + @chmod -w $@ +-include .wasmbuilds.make +endif +DISTCLEAN_FILES += .wasmbuilds.make + # The various -D... values used by *.c-pp.js include: # # -Dtarget=es6-module: for all ESM module builds @@ -962,7 +919,7 @@ $(sqlite3.wasm): $(sqlite3.js) $(sqlite3.mjs): $(sqlite3.js) $(sqlite3-bundler-friendly.mjs): $(sqlite3.mjs) $(sqlite3-node.mjs): $(sqlite3.mjs) -CLEAN_FILES += $(sqlite3.wasm) +#CLEAN_FILES += $(sqlite3.wasm) ######################################################################## # We need separate copies of certain supplementary JS files for the @@ -1078,15 +1035,13 @@ speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 # -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app # which runs speedtest1 multiple times. -$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api.core) +$(EXPORTED_FUNCTIONS.speedtest1): $(MKDIR.bld) $(EXPORTED_FUNCTIONS.api.core) @echo "Making $@ ..." @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api.core); } > $@ speedtest1.js := $(dir.dout)/speedtest1.js speedtest1.wasm := $(dir.dout)/speedtest1.wasm emcc.flags.speedtest1-vanilla := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM - speedtest1.cfiles := $(speedtest1.c) $(sqlite3-wasm.c) -$(eval $(call call-make-pre-post,speedtest1,vanilla)) $(speedtest1.js): $(MAKEFILE) $(speedtest1.cfiles) \ $(pre-post-speedtest1-vanilla.deps) \ $(EXPORTED_FUNCTIONS.speedtest1) @@ -1096,7 +1051,7 @@ $(speedtest1.js): $(MAKEFILE) $(speedtest1.cfiles) \ $(emcc.speedtest1.common) \ $(emcc.flags.speedtest1-vanilla) $(pre-post-speedtest1-vanilla.flags) \ $(SQLITE_OPT) \ - -USQLITE_WASM_MINIMAL \ + -USQLITE_WASM_BARE_BONES \ -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c) \ $(speedtest1.exit-runtime0) \ -o $@ $(speedtest1.cfiles) -lm @@ -1106,7 +1061,7 @@ $(speedtest1.js): $(MAKEFILE) $(speedtest1.cfiles) \ speedtest1: $(speedtest1.js) all: speedtest1 -CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) +#CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) # end speedtest1.js ######################################################################## @@ -1143,24 +1098,29 @@ quick: $(sqlite3.js) # painful. .PHONY: o0 o1 o2 o3 os oz -o-xtra := -#o-xtra ?= -flto +emcc-opt-extra := +#ifeq (1,$(wasm-bare-bones)) +#emcc-opt-extra += -flto # ^^^^ -flto can have a considerably performance boost at -O0 but # doubles the build time and seems to have negligible, if any, effect # on higher optimization levels. +# +# -flto does ont shrink the size of bare-bones builds by any measurable +# amount. +#endif o0: clean $(MAKE) -e "emcc_opt=-O0" o1: clean - $(MAKE) -e "emcc_opt=-O1 $(o-xtra)" + $(MAKE) -e "emcc_opt=-O1 $(emcc-opt-extra)" o2: clean - $(MAKE) -j2 -e "emcc_opt=-O2 $(o-xtra)" + $(MAKE) -j2 -e "emcc_opt=-O2 $(emcc-opt-extra)" o3: clean - $(MAKE) -e "emcc_opt=-O3 $(o-xtra)" + $(MAKE) -e "emcc_opt=-O3 $(emcc-opt-extra)" os: clean @echo "WARNING: -Os can result in a build with mysteriously missing pieces!" - $(MAKE) -e "emcc_opt=-Os $(o-xtra)" + $(MAKE) -e "emcc_opt=-Os $(emcc-opt-extra)" oz: clean - $(MAKE) -j2 -e "emcc_opt=-Oz $(o-xtra)" + $(MAKE) -j2 -e "emcc_opt=-Oz $(emcc-opt-extra)" ######################################################################## # Sub-makes... @@ -1174,7 +1134,7 @@ wasmfs.enable ?= 1 else # Unconditionally enable wasmfs for [dist]clean so that the wasmfs # sub-make can clean up. -wasmfs.enable ?= $(if $(filter %clean,$(MAKECMDGOALS)),1,0) +wasmfs.enable ?= $(MAKING_CLEAN) endif ifeq (1,$(wasmfs.enable)) # wasmfs build disabled 2022-10-19 per /chat discussion. @@ -1253,4 +1213,3 @@ endif # Run local web server for the test/demo pages. httpd: althttpd -max-age 1 -enable-sab 1 -page index.html - diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth deleted file mode 100644 index 085090821b..0000000000 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth +++ /dev/null @@ -1 +0,0 @@ -_sqlite3_set_authorizer diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras similarity index 68% rename from ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session rename to ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras index 5b7b53f952..e635d93b32 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras @@ -1,3 +1,12 @@ +_sqlite3_create_window_function +_sqlite3_progress_handler +_sqlite3_set_authorizer +_sqlite3_preupdate_blobwrite +_sqlite3_preupdate_count +_sqlite3_preupdate_depth +_sqlite3_preupdate_hook +_sqlite3_preupdate_new +_sqlite3_preupdate_old _sqlite3changegroup_add _sqlite3changegroup_add_strm _sqlite3changegroup_delete @@ -40,3 +49,15 @@ _sqlite3session_object_config _sqlite3session_patchset _sqlite3session_patchset_strm _sqlite3session_table_filter +_sqlite3_create_module +_sqlite3_create_module_v2 +_sqlite3_declare_vtab +_sqlite3_drop_modules +_sqlite3_vtab_collation +_sqlite3_vtab_distinct +_sqlite3_vtab_in +_sqlite3_vtab_in_first +_sqlite3_vtab_in_next +_sqlite3_vtab_nochange +_sqlite3_vtab_on_conflict +_sqlite3_vtab_rhs_value diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate deleted file mode 100644 index 5c57a76b60..0000000000 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate +++ /dev/null @@ -1,6 +0,0 @@ -_sqlite3_preupdate_blobwrite -_sqlite3_preupdate_count -_sqlite3_preupdate_depth -_sqlite3_preupdate_hook -_sqlite3_preupdate_new -_sqlite3_preupdate_old diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-vtab b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-vtab deleted file mode 100644 index 1f6de96823..0000000000 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-vtab +++ /dev/null @@ -1,11 +0,0 @@ -_sqlite3_create_module -_sqlite3_create_module_v2 -_sqlite3_declare_vtab -_sqlite3_vtab_collation -_sqlite3_vtab_distinct -_sqlite3_vtab_in -_sqlite3_vtab_in_first -_sqlite3_vtab_in_next -_sqlite3_vtab_nochange -_sqlite3_vtab_on_conflict -_sqlite3_vtab_rhs_value diff --git a/ext/wasm/api/pre-js.c-pp.js b/ext/wasm/api/pre-js.c-pp.js index 878f3e0546..4045fb11cd 100644 --- a/ext/wasm/api/pre-js.c-pp.js +++ b/ext/wasm/api/pre-js.c-pp.js @@ -48,7 +48,7 @@ Module['locateFile'] = function(path, prefix) { }else{ theFile = prefix + path; } - sqlite3InitModuleState.debugModule( + this.debugModule( "locateFile(",arguments[0], ',', arguments[1],")", 'sqlite3InitModuleState.scriptDir =',this.scriptDir, 'up.entries() =',Array.from(up.entries()), @@ -59,6 +59,7 @@ Module['locateFile'] = function(path, prefix) { }.bind(sqlite3InitModuleState); //#endif ifnot target=es6-bundler-friendly +//#if custom-Module.instantiateModule /** Bug warning: a custom Module.instantiateWasm() does not work in WASMFS builds: @@ -67,7 +68,15 @@ Module['locateFile'] = function(path, prefix) { In such builds we must disable this. */ -const xNameOfInstantiateWasm = false +const xNameOfInstantiateWasm = +//#if wasmfs + false +//#else + true /* This works, but it does not have the testing coverage in the + wild which Emscripten's default impl does, so we'll save + this option until we really need a custom + Module.instantiateWasm() */ +//#endif ? 'instantiateWasm' : 'emscripten-bug-17951'; Module[xNameOfInstantiateWasm] = function callee(imports,onSuccess){ @@ -80,6 +89,7 @@ Module[xNameOfInstantiateWasm] = function callee(imports,onSuccess){ sqlite3InitModuleState.debugModule( "instantiateWasm() uri =", uri ); + //console.warn("Custom instantiateModule",uri); const wfetch = ()=>fetch(uri, {credentials: 'same-origin'}); const loadWasm = WebAssembly.instantiateStreaming ? async ()=>{ @@ -105,6 +115,7 @@ Module[xNameOfInstantiateWasm] = function callee(imports,onSuccess){ scripts. */ Module[xNameOfInstantiateWasm].uri = 'sqlite3.wasm'; +//#endif custom-Module.instantiateModule /* END FILE: api/pre-js.js, noting that the build process may add a line after this one to change the above .uri to a build-specific one. */ diff --git a/ext/wasm/api/sqlite3-api-glue.c-pp.js b/ext/wasm/api/sqlite3-api-glue.c-pp.js index e5eb0cfeb6..680218370a 100644 --- a/ext/wasm/api/sqlite3-api-glue.c-pp.js +++ b/ext/wasm/api/sqlite3-api-glue.c-pp.js @@ -136,20 +136,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_compileoption_used", "int", "string"], ["sqlite3_complete", "int", "string:flexible"], ["sqlite3_context_db_handle", "sqlite3*", "sqlite3_context*"], - + /* sqlite3_create_collation() and sqlite3_create_collation_v2() + use hand-written bindings to simplify passing of the callback + function. */ /* sqlite3_create_function(), sqlite3_create_function_v2(), and sqlite3_create_window_function() use hand-written bindings to simplify handling of their function-type arguments. */ - /* sqlite3_create_collation() and sqlite3_create_collation_v2() - use hand-written bindings to simplify passing of the callback - function. - ["sqlite3_create_collation", "int", - "sqlite3*", "string", "int",//SQLITE_UTF8 is the only legal value - "*", "*"], - ["sqlite3_create_collation_v2", "int", - "sqlite3*", "string", "int",//SQLITE_UTF8 is the only legal value - "*", "*", "*"], - */ ["sqlite3_data_count", "int", "sqlite3_stmt*"], ["sqlite3_db_filename", "string", "sqlite3*", "string"], ["sqlite3_db_handle", "sqlite3*", "sqlite3_stmt*"], @@ -211,14 +203,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ for those, depending on how their SQL argument is provided. */ /* sqlite3_randomness() uses a hand-written wrapper to extend the range of supported argument types. */ - ["sqlite3_progress_handler", undefined, [ - "sqlite3*", "int", new wasm.xWrap.FuncPtrAdapter({ - name: 'xProgressHandler', - signature: 'i(p)', - bindScope: 'context', - contextKey: (argv,argIndex)=>argv[0/* sqlite3* */] - }), "*" - ]], ["sqlite3_realloc", "*","*","int"], ["sqlite3_reset", "int", "sqlite3_stmt*"], /* sqlite3_reset_auto_extension() has a hand-written binding. */ @@ -303,6 +287,19 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_vfs_unregister", "int", "sqlite3_vfs*"] ]/*wasm.bindingSignatures*/; + if( !!wasm.exports.sqlite3_progress_handler ){ + wasm.bindingSignatures.push( + ["sqlite3_progress_handler", undefined, [ + "sqlite3*", "int", new wasm.xWrap.FuncPtrAdapter({ + name: 'xProgressHandler', + signature: 'i(p)', + bindScope: 'context', + contextKey: (argv,argIndex)=>argv[0/* sqlite3* */] + }), "*" + ]] + ); + } + if( !!wasm.exports.sqlite3_stmt_explain ){ wasm.bindingSignatures.push( ["sqlite3_stmt_explain", "int", "sqlite3_stmt*", "int"], @@ -336,7 +333,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/* sqlite3_set_authorizer() */ if(false && wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ - /* ^^^ "the problem" is that this is an option feature and the + /* ^^^ "the problem" is that this is an optional feature and the build-time function-export list does not currently take optional features into account. */ wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); @@ -385,7 +382,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ of this, the canonical builds of sqlite3.wasm/js guarantee that sqlite3.wasm.alloc() and friends use those allocators. Custom builds may not guarantee that, however. */, - ["sqlite3_drop_modules", "int", ["sqlite3*", "**"]], ["sqlite3_last_insert_rowid", "i64", ["sqlite3*"]], ["sqlite3_malloc64", "*","i64"], ["sqlite3_msize", "i64", "*"], @@ -422,6 +418,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_create_module_v2", "int", ["sqlite3*","string","sqlite3_module*","*","*"]], ["sqlite3_declare_vtab", "int", ["sqlite3*", "string:flexible"]], + ["sqlite3_drop_modules", "int", ["sqlite3*", "**"]], ["sqlite3_vtab_collation","string","sqlite3_index_info*","int"], ["sqlite3_vtab_distinct","int", "sqlite3_index_info*"], ["sqlite3_vtab_in","int", "sqlite3_index_info*", "int", "int"], @@ -950,7 +947,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Code duplication reducer for functions which take an encoding argument and require SQLITE_UTF8. Sets the db error code to - SQLITE_FORMAT and returns that code. */ + SQLITE_FORMAT, installs a descriptive error message, + and returns SQLITE_FORMAT. */ const __errEncoding = (pDb)=>{ return util.sqlite3__wasm_db_error( pDb, capi.SQLITE_FORMAT, "SQLITE_UTF8 is the only supported encoding." @@ -1000,11 +998,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ this._addUDF(pDb, name, arity, m.udf); }; - __dbCleanupMap.addWindowFunc = function(pDb, name, arity){ - const m = __dbCleanupMap(pDb, 1); - if(!m.wudf) m.wudf = new Map; - this._addUDF(pDb, name, arity, m.wudf); - }; + if( wasm.exports.sqlite3_create_window_function ){ + __dbCleanupMap.addWindowFunc = function(pDb, name, arity){ + const m = __dbCleanupMap(pDb, 1); + if(!m.wudf) m.wudf = new Map; + this._addUDF(pDb, name, arity, m.wudf); + }; + } /** Intended to be called _only_ from sqlite3_close_v2(), @@ -1273,17 +1273,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ] ); - const __sqlite3CreateWindowFunction = wasm.xWrap( - "sqlite3_create_window_function", "int", [ - "sqlite3*", "string"/*funcName*/, "int"/*nArg*/, - "int"/*eTextRep*/, "*"/*pApp*/, - new wasm.xWrap.FuncPtrAdapter({name: 'xStep', ...__cfProxy.xInverseAndStep}), - new wasm.xWrap.FuncPtrAdapter({name: 'xFinal', ...__cfProxy.xFinalAndValue}), - new wasm.xWrap.FuncPtrAdapter({name: 'xValue', ...__cfProxy.xFinalAndValue}), - new wasm.xWrap.FuncPtrAdapter({name: 'xInverse', ...__cfProxy.xInverseAndStep}), - new wasm.xWrap.FuncPtrAdapter({name: 'xDestroy', ...__cfProxy.xDestroy}) - ] - ); + const __sqlite3CreateWindowFunction = + wasm.exports.sqlite3_create_window_function + ? wasm.xWrap( + "sqlite3_create_window_function", "int", [ + "sqlite3*", "string"/*funcName*/, "int"/*nArg*/, + "int"/*eTextRep*/, "*"/*pApp*/, + new wasm.xWrap.FuncPtrAdapter({name: 'xStep', ...__cfProxy.xInverseAndStep}), + new wasm.xWrap.FuncPtrAdapter({name: 'xFinal', ...__cfProxy.xFinalAndValue}), + new wasm.xWrap.FuncPtrAdapter({name: 'xValue', ...__cfProxy.xFinalAndValue}), + new wasm.xWrap.FuncPtrAdapter({name: 'xInverse', ...__cfProxy.xInverseAndStep}), + new wasm.xWrap.FuncPtrAdapter({name: 'xDestroy', ...__cfProxy.xDestroy}) + ] + ) + : undefined; /* Documented in the api object's initializer. */ capi.sqlite3_create_function_v2 = function f( @@ -1328,61 +1331,71 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; /* Documented in the api object's initializer. */ - capi.sqlite3_create_window_function = function f( - pDb, funcName, nArg, eTextRep, pApp, - xStep, //void (*xStep)(sqlite3_context*,int,sqlite3_value**) - xFinal, //void (*xFinal)(sqlite3_context*) - xValue, //void (*xValue)(sqlite3_context*) - xInverse,//void (*xInverse)(sqlite3_context*,int,sqlite3_value**) - xDestroy //void (*xDestroy)(void*) - ){ - if( f.length!==arguments.length ){ - return __dbArgcMismatch(pDb,"sqlite3_create_window_function",f.length); - }else if( 0 === (eTextRep & 0xf) ){ - eTextRep |= capi.SQLITE_UTF8; - }else if( capi.SQLITE_UTF8 !== (eTextRep & 0xf) ){ - return __errEncoding(pDb); - } - try{ - const rc = __sqlite3CreateWindowFunction(pDb, funcName, nArg, eTextRep, - pApp, xStep, xFinal, xValue, - xInverse, xDestroy); - if(0===rc && (xStep instanceof Function - || xFinal instanceof Function - || xValue instanceof Function - || xInverse instanceof Function - || xDestroy instanceof Function)){ - __dbCleanupMap.addWindowFunc(pDb, funcName, nArg); + if( __sqlite3CreateWindowFunction ){ + capi.sqlite3_create_window_function = function f( + pDb, funcName, nArg, eTextRep, pApp, + xStep, //void (*xStep)(sqlite3_context*,int,sqlite3_value**) + xFinal, //void (*xFinal)(sqlite3_context*) + xValue, //void (*xValue)(sqlite3_context*) + xInverse,//void (*xInverse)(sqlite3_context*,int,sqlite3_value**) + xDestroy //void (*xDestroy)(void*) + ){ + if( f.length!==arguments.length ){ + return __dbArgcMismatch(pDb,"sqlite3_create_window_function",f.length); + }else if( 0 === (eTextRep & 0xf) ){ + eTextRep |= capi.SQLITE_UTF8; + }else if( capi.SQLITE_UTF8 !== (eTextRep & 0xf) ){ + return __errEncoding(pDb); } - return rc; - }catch(e){ - console.error("sqlite3_create_window_function() setup threw:",e); - return util.sqlite3__wasm_db_error(pDb, e, "Creation of UDF threw: "+e); - } - }; + try{ + const rc = __sqlite3CreateWindowFunction(pDb, funcName, nArg, eTextRep, + pApp, xStep, xFinal, xValue, + xInverse, xDestroy); + if(0===rc && (xStep instanceof Function + || xFinal instanceof Function + || xValue instanceof Function + || xInverse instanceof Function + || xDestroy instanceof Function)){ + __dbCleanupMap.addWindowFunc(pDb, funcName, nArg); + } + return rc; + }catch(e){ + console.error("sqlite3_create_window_function() setup threw:",e); + return util.sqlite3__wasm_db_error(pDb, e, "Creation of UDF threw: "+e); + } + }; + }else{ + delete capi.sqlite3_create_window_function; + } /** A _deprecated_ alias for capi.sqlite3_result_js() which predates the addition of that function in the public API. */ capi.sqlite3_create_function_v2.udfSetResult = - capi.sqlite3_create_function.udfSetResult = + capi.sqlite3_create_function.udfSetResult = capi.sqlite3_result_js; + if(capi.sqlite3_create_window_function){ capi.sqlite3_create_window_function.udfSetResult = capi.sqlite3_result_js; + } /** A _deprecated_ alias for capi.sqlite3_values_to_js() which predates the addition of that function in the public API. */ capi.sqlite3_create_function_v2.udfConvertArgs = - capi.sqlite3_create_function.udfConvertArgs = + capi.sqlite3_create_function.udfConvertArgs = capi.sqlite3_values_to_js; + if(capi.sqlite3_create_window_function){ capi.sqlite3_create_window_function.udfConvertArgs = capi.sqlite3_values_to_js; + } /** A _deprecated_ alias for capi.sqlite3_result_error_js() which predates the addition of that function in the public API. */ capi.sqlite3_create_function_v2.udfSetError = - capi.sqlite3_create_function.udfSetError = + capi.sqlite3_create_function.udfSetError = capi.sqlite3_result_error_js; + if(capi.sqlite3_create_window_function){ capi.sqlite3_create_window_function.udfSetError = capi.sqlite3_result_error_js; + } }/*sqlite3_create_function_v2() and sqlite3_create_window_function() proxies*/; diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 7f7e696899..c5dd495e54 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -14,16 +14,17 @@ */ #define SQLITE_WASM #ifdef SQLITE_WASM_ENABLE_C_TESTS +# undef SQLITE_WASM_ENABLE_C_TESTS +# define SQLITE_WASM_ENABLE_C_TESTS 1 /* -** Code blocked off by SQLITE_WASM_TESTS is intended solely for use in -** unit/regression testing. They may be safely omitted from +** Code blocked off by SQLITE_WASM_ENABLE_C_TESTS is intended solely +** for use in unit/regression testing. They may be safely omitted from ** client-side builds. The main unit test script, tester1.js, will ** skip related tests if it doesn't find the corresponding functions ** in the WASM exports. */ -# define SQLITE_WASM_TESTS 1 #else -# define SQLITE_WASM_TESTS 0 +# define SQLITE_WASM_ENABLE_C_TESTS 0 #endif /* @@ -92,60 +93,18 @@ #undef SQLITE_ENABLE_API_ARMOR #define SQLITE_ENABLE_API_ARMOR 1 -#ifndef SQLITE_ENABLE_BYTECODE_VTAB -# define SQLITE_ENABLE_BYTECODE_VTAB 1 -#endif -#ifndef SQLITE_ENABLE_DBPAGE_VTAB -# define SQLITE_ENABLE_DBPAGE_VTAB 1 -#endif -#ifndef SQLITE_ENABLE_DBSTAT_VTAB -# define SQLITE_ENABLE_DBSTAT_VTAB 1 -#endif -#ifndef SQLITE_ENABLE_EXPLAIN_COMMENTS -# define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 -#endif -#ifndef SQLITE_ENABLE_FTS5 -# define SQLITE_ENABLE_FTS5 1 -#endif -#ifndef SQLITE_ENABLE_MATH_FUNCTIONS -# define SQLITE_ENABLE_MATH_FUNCTIONS 1 -#endif -#ifndef SQLITE_ENABLE_OFFSET_SQL_FUNC -# define SQLITE_ENABLE_OFFSET_SQL_FUNC 1 -#endif -#ifndef SQLITE_ENABLE_PREUPDATE_HOOK -# define SQLITE_ENABLE_PREUPDATE_HOOK 1 /*required by session extension*/ -#endif -#ifndef SQLITE_ENABLE_RTREE -# define SQLITE_ENABLE_RTREE 1 -#endif -#ifndef SQLITE_ENABLE_SESSION -# define SQLITE_ENABLE_SESSION 1 -#endif -#ifndef SQLITE_ENABLE_STMTVTAB -# define SQLITE_ENABLE_STMTVTAB 1 -#endif -#ifndef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION -# define SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION -#endif - /**********************************************************************/ /* SQLITE_O... */ -#ifndef SQLITE_OMIT_DEPRECATED -# define SQLITE_OMIT_DEPRECATED 1 -#endif -#ifndef SQLITE_OMIT_LOAD_EXTENSION -# define SQLITE_OMIT_LOAD_EXTENSION 1 -#endif -#ifndef SQLITE_OMIT_SHARED_CACHE -# define SQLITE_OMIT_SHARED_CACHE 1 -#endif -#ifndef SQLITE_OMIT_UTF16 -# define SQLITE_OMIT_UTF16 1 -#endif -#ifndef SQLITE_OS_KV_OPTIONAL -# define SQLITE_OS_KV_OPTIONAL 1 -#endif +#undef SQLITE_OMIT_DEPRECATED +#define SQLITE_OMIT_DEPRECATED 1 +#undef SQLITE_OMIT_LOAD_EXTENSION +#define SQLITE_OMIT_LOAD_EXTENSION 1 +#undef SQLITE_OMIT_SHARED_CACHE +#define SQLITE_OMIT_SHARED_CACHE 1 +#undef SQLITE_OMIT_UTF16 +#define SQLITE_OMIT_UTF16 1 +#undef SQLITE_OS_KV_OPTIONAL +#define SQLITE_OS_KV_OPTIONAL 1 /**********************************************************************/ /* SQLITE_S... */ @@ -173,45 +132,64 @@ #endif /* -** If SQLITE_WASM_MINIMAL is defined, undefine most of the ENABLE +** If SQLITE_WASM_BARE_BONES is defined, undefine most of the ENABLE ** macros. */ -#ifdef SQLITE_WASM_MINIMAL -# undef SQLITE_ENABLE_DBPAGE_VTAB -# undef SQLITE_ENABLE_DBSTAT_VTAB -# undef SQLITE_ENABLE_EXPLAIN_COMMENTS -# undef SQLITE_ENABLE_FTS5 -# undef SQLITE_ENABLE_OFFSET_SQL_FUNC -# undef SQLITE_ENABLE_PREUPDATE_HOOK -# undef SQLITE_ENABLE_RTREE -# undef SQLITE_ENABLE_SESSION -# undef SQLITE_ENABLE_STMTVTAB -# undef SQLITE_OMIT_AUTHORIZATION +#ifdef SQLITE_WASM_BARE_BONES +# undef SQLITE_ENABLE_DBPAGE_VTAB +# undef SQLITE_ENABLE_DBSTAT_VTAB +# undef SQLITE_ENABLE_EXPLAIN_COMMENTS +# undef SQLITE_ENABLE_FTS5 +# undef SQLITE_ENABLE_OFFSET_SQL_FUNC +# undef SQLITE_ENABLE_PREUPDATE_HOOK +# undef SQLITE_ENABLE_RTREE +# undef SQLITE_ENABLE_SESSION +# undef SQLITE_ENABLE_STMTVTAB +# undef SQLITE_OMIT_AUTHORIZATION # define SQLITE_OMIT_AUTHORIZATION -/*Reminder re. custom sqlite3.c: +# undef SQLITE_OMIT_GET_TABLE +# define SQLITE_OMIT_GET_TABLE +# undef SQLITE_OMIT_INCRBLOB +# define SQLITE_OMIT_INCRBLOB +# undef SQLITE_OMIT_INTROSPECTION_PRAGMAS +# define SQLITE_OMIT_INTROSPECTION_PRAGMAS +# undef SQLITE_OMIT_JSON +# define SQLITE_OMIT_JSON +# undef SQLITE_OMIT_PROGRESS_CALLBACK +# define SQLITE_OMIT_PROGRESS_CALLBACK +# undef SQLITE_OMIT_WAL +# define SQLITE_OMIT_WAL +/* + The following OMITs do not work with the standard amalgamation, so + require a custom build: fossil clean -x ./configure - OPTS='-DSQLITE_OMIT_VIRTUALTABLE -DSQLITE_OMIT_EXPLAIN -DSQLITE_OMIT_TRIGGER' make -e sqlite3 -*/ -/*Requires a custom sqlite3.c -# undef SQLITE_OMIT_TRIGGER -# define SQLITE_OMIT_TRIGGER -*/ -/*TODO (requires build tweaks) -# undef SQLITE_OMIT_WINDOWFUNC -# define SQLITE_OMIT_WINDOWFUNC -*/ -/*Requires a custom sqlite3.c + OPTS='...' make -e sqlite3 + + where ... has a -D... for each of the following OMIT flags: + # undef SQLITE_OMIT_EXPLAIN # define SQLITE_OMIT_EXPLAIN -*/ -/*Requires a custom sqlite3.c + +# undef SQLITE_OMIT_TRIGGER +# define SQLITE_OMIT_TRIGGER + # undef SQLITE_OMIT_VIRTUALTABLE # define SQLITE_OMIT_VIRTUALTABLE + +# undef SQLITE_OMIT_WINDOWFUNC +# define SQLITE_OMIT_WINDOWFUNC + + As of this writing (2024-07-25), such a build fails in various ways + for as-yet-unknown reasons. */ -# undef SQLITE_OMIT_JSON -# define SQLITE_OMIT_JSON +#endif + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_WASM_BARE_BONES) +# define SQLITE_WASM_HAS_VTAB 1 +#else +# define SQLITE_WASM_HAS_VTAB 0 #endif #include @@ -264,10 +242,6 @@ #undef INC__STRINGIFY #undef SQLITE_C -#if defined(__EMSCRIPTEN__) -# include -#endif - #if 0 /* ** An EXPERIMENT in implementing a stack-based allocator analog to @@ -412,7 +386,7 @@ int sqlite3__wasm_db_error(sqlite3*db, int err_code, const char *zMsg){ return err_code; } -#if SQLITE_WASM_TESTS +#if SQLITE_WASM_ENABLE_C_TESTS struct WasmTestStruct { int v4; void * ppV; @@ -432,7 +406,7 @@ void sqlite3__wasm_test_struct(WasmTestStruct * s){ } return; } -#endif /* SQLITE_WASM_TESTS */ +#endif /* SQLITE_WASM_ENABLE_C_TESTS */ /* ** This function is NOT part of the sqlite3 public API. It is strictly @@ -967,7 +941,7 @@ const char * sqlite3__wasm_enum_json(void){ } _DefGroup; DefGroup(vtab) { -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_WASM_MINIMAL) +#if SQLITE_WASM_HAS_VTAB DefInt(SQLITE_INDEX_SCAN_UNIQUE); DefInt(SQLITE_INDEX_CONSTRAINT_EQ); DefInt(SQLITE_INDEX_CONSTRAINT_GT); @@ -995,7 +969,7 @@ const char * sqlite3__wasm_enum_json(void){ DefInt(SQLITE_FAIL); //DefInt(SQLITE_ABORT); // Also an error code DefInt(SQLITE_REPLACE); -#endif /*!SQLITE_OMIT_VIRTUALTABLE*/ +#endif /*SQLITE_WASM_HAS_VTAB*/ } _DefGroup; #undef DefGroup @@ -1110,6 +1084,7 @@ const char * sqlite3__wasm_enum_json(void){ #undef CurrentStruct +#if SQLITE_WASM_HAS_VTAB #define CurrentStruct sqlite3_vtab StructBinder { M(pModule, "p"); @@ -1155,7 +1130,6 @@ const char * sqlite3__wasm_enum_json(void){ } _StructBinder; #undef CurrentStruct -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_WASM_MINIMAL) /** ** Workaround: in order to map the various inner structs from ** sqlite3_index_info, we have to uplift those into constructs we @@ -1232,9 +1206,9 @@ const char * sqlite3__wasm_enum_json(void){ } _StructBinder; #undef CurrentStruct -#endif /*!SQLITE_OMIT_VIRTUALTABLE*/ +#endif /*SQLITE_WASM_HAS_VTAB*/ -#if SQLITE_WASM_TESTS +#if SQLITE_WASM_ENABLE_C_TESTS #define CurrentStruct WasmTestStruct StructBinder { M(v4, "i"); @@ -1244,7 +1218,7 @@ const char * sqlite3__wasm_enum_json(void){ M(xFunc, "v(p)"); } _StructBinder; #undef CurrentStruct -#endif +#endif /*SQLITE_WASM_ENABLE_C_TESTS*/ } out( "]"/*structs*/); @@ -1603,7 +1577,7 @@ sqlite3_kvvfs_methods * sqlite3__wasm_kvvfs_methods(void){ return &sqlite3KvvfsMethods; } -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_WASM_MINIMAL) +#if SQLITE_WASM_HAS_VTAB /* ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. @@ -1626,7 +1600,7 @@ int sqlite3__wasm_vtab_config(sqlite3 *pDb, int op, int arg){ return SQLITE_MISUSE; } } -#endif /*!SQLITE_OMIT_VIRTUALTABLE*/ +#endif /*SQLITE_WASM_HAS_VTAB*/ /* ** This function is NOT part of the sqlite3 public API. It is strictly @@ -1750,6 +1724,7 @@ char * sqlite3__wasm_qfmt_token(char *z, int addQuotes){ } #if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS) +#include #include /* @@ -1801,7 +1776,7 @@ int sqlite3__wasm_init_wasmfs(const char *zUnused){ } #endif /* __EMSCRIPTEN__ && SQLITE_ENABLE_WASMFS */ -#if SQLITE_WASM_TESTS +#if SQLITE_WASM_ENABLE_C_TESTS SQLITE_WASM_EXPORT int sqlite3__wasm_test_intptr(int * p){ @@ -1967,6 +1942,9 @@ int sqlite3__wasm_SQLTester_strglob(const char *zGlob, const char *z){ return !sqlite3__wasm_SQLTester_strnotglob(zGlob, z); } -#endif /* SQLITE_WASM_TESTS */ +#endif /* SQLITE_WASM_ENABLE_C_TESTS */ #undef SQLITE_WASM_EXPORT +#undef SQLITE_WASM_HAS_VTAB +#undef SQLITE_WASM_BARE_BONES +#undef SQLITE_WASM_ENABLE_C_TESTS diff --git a/ext/wasm/c-pp.c b/ext/wasm/c-pp.c index c439a0d091..c67881dd3e 100644 --- a/ext/wasm/c-pp.c +++ b/ext/wasm/c-pp.c @@ -1,7 +1,8 @@ /* ** 2022-11-12: ** -** In place of a legal notice, here is a blessing: +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** ** * May you do good and not evil. ** * May you find forgiveness for yourself and forgive others. @@ -1507,7 +1508,7 @@ int main(int argc, char const * const * argv){ } ISFLAG("debug"){ ++g.doDebug; - }else if(!zInfile){ + }else if(!zInfile && '-'!=argv[i][0]){ goto do_infile; }else{ fatal("Unhandled flag: %s", argv[i]); diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 1f0d9e2ad0..df5c7ab7b7 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -7,21 +7,23 @@ MAKEFILE.fiddle := $(lastword $(MAKEFILE_LIST)) ######################################################################## # shell.c and its build flags... -make-np-0 := make -C $(dir.top) -n -p -make-np-1 := sed -e 's/(TOP)/(dir.top)/g' -# Extract SHELL_OPT and SHELL_DEP from the top-most makefile and import -# them as vars here... -$(eval $(shell $(make-np-0) | grep -e '^SHELL_OPT ' | $(make-np-1))) -$(eval $(shell $(make-np-0) | grep -e '^SHELL_DEP ' | $(make-np-1))) -# ^^^ can't do that in 1 invocation b/c newlines get stripped -ifeq (,$(SHELL_OPT)) -$(error Could not parse SHELL_OPT from $(dir.top)/Makefile.) -endif -ifeq (,$(SHELL_DEP)) -$(error Could not parse SHELL_DEP from $(dir.top)/Makefile.) -endif -$(dir.top)/shell.c: $(SHELL_DEP) $(dir.top)/tool/mkshellc.tcl $(sqlite3.c) +ifneq (1,$(MAKING_CLEAN)) + make-np-0 := make -C $(dir.top) -n -p + make-np-1 := sed -e 's/(TOP)/(dir.top)/g' + # Extract SHELL_OPT and SHELL_DEP from the top-most makefile and import + # them as vars here... + $(eval $(shell $(make-np-0) | grep -e '^SHELL_OPT ' | $(make-np-1))) + $(eval $(shell $(make-np-0) | grep -e '^SHELL_DEP ' | $(make-np-1))) + # ^^^ can't do that in 1 invocation b/c newlines get stripped + ifeq (,$(SHELL_OPT)) + $(error Could not parse SHELL_OPT from $(dir.top)/Makefile.) + endif + ifeq (,$(SHELL_DEP)) + $(error Could not parse SHELL_DEP from $(dir.top)/Makefile.) + endif +$(dir.top)/shell.c: $(SHELL_DEP) $(dir.tool)/mkshellc.tcl $(sqlite3.c) $(MAKE) -C $(dir.top) shell.c +endif # /shell.c ######################################################################## @@ -41,10 +43,11 @@ fiddle.emcc-flags = \ $(emcc.exportedRuntimeMethods) \ -sEXPORTED_FUNCTIONS=@$(abspath $(EXPORTED_FUNCTIONS.fiddle)) \ -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory \ - $(SQLITE_OPT) $(SHELL_OPT) \ - -USQLITE_WASM_MINIMAL \ + $(SQLITE_OPT.full-featured) \ + $(SQLITE_OPT.common) \ + $(SHELL_OPT) \ + -USQLITE_WASM_BARE_BONES \ -DSQLITE_SHELL_FIDDLE -# -D_POSIX_C_SOURCE is needed for strdup() with emcc # Flags specifically for debug builds of fiddle. Performance suffers # greatly in debug builds. @@ -55,52 +58,22 @@ fiddle.emcc-flags.debug := $(fiddle.emcc-flags) \ fiddle.EXPORTED_FUNCTIONS.in := \ EXPORTED_FUNCTIONS.fiddle.in \ - $(EXPORTED_FUNCTIONS.api) + $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core \ + $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-extras -$(EXPORTED_FUNCTIONS.fiddle): $(fiddle.EXPORTED_FUNCTIONS.in) $(MAKEFILE.fiddle) +$(EXPORTED_FUNCTIONS.fiddle): $(MKDIR.bld) $(fiddle.EXPORTED_FUNCTIONS.in) \ + $(MAKEFILE.fiddle) sort -u $(fiddle.EXPORTED_FUNCTIONS.in) > $@ fiddle.cses := $(dir.top)/shell.c $(sqlite3-wasm.c) -$(eval $(call call-make-pre-post,fiddle-module,vanilla)) -######################################################################## -# emit rules for one of the two fiddle builds. $1 must be -# either $(dir.fiddle) or $(dir.fiddle-debug). $2 must be empty -# in the former case and .debug in the latter. -define make-fiddle-rules -fiddle-module.js$(2) := $(1)/fiddle-module.js -fiddle-module.wasm$(2) := $$(subst .js,.wasm,$$(fiddle-module.js$(2))) -$$(fiddle-module.js$(2)): $$(MAKEFILE) $$(MAKEFILE.fiddle) \ - $$(EXPORTED_FUNCTIONS.fiddle) \ - $$(fiddle.cses) $$(pre-post-fiddle-module-vanilla.deps) $$(SOAP.js) - @test -d "$$(dir $$@)" || mkdir -p "$$(dir $$@)" - $$(emcc.bin) -o $$@ $$(fiddle.emcc-flags$(2)) \ - $$(pre-post-fiddle-module-vanilla.flags) \ - $$(fiddle.cses) - $$(maybe-wasm-strip) $$(fiddle-module.wasm$(2)) - @cp -p $$(SOAP.js) $$(dir $$@) - @if [[ x.debug = x$(2) ]]; then \ - cp -p $$(dir.fiddle)/index.html \ - $$(dir.fiddle)/fiddle.js \ - $$(dir.fiddle)/fiddle-worker.js \ - $$(dir $$@)/.; \ - fi - @for i in $(1)/*.*js $(1)/*.html $(1)/*.wasm; do \ - test -f $$$${i} || continue; \ - gzip < $$$${i} > $$$${i}.gz; \ - done -fiddle$(2): $$(fiddle-module.js$(2)) $(1)/fiddle.js.gz -endef - -$(eval $(call make-fiddle-rules,$(dir.fiddle))) -$(eval $(call make-fiddle-rules,$(dir.fiddle-debug),.debug)) fiddle: $(fiddle-module.js) $(fiddle-module.js.debug) fiddle.debug: $(fiddle-module.js.debug) clean: clean-fiddle clean-fiddle: - rm -f $(fiddle-module.js) \ - $(fiddle-module.wasm) \ + rm -f $(dir.fiddle)/fiddle-module.js \ + $(dir.fiddle)/*.wasm \ $(dir.fiddle)/sqlite3-opfs-*.js \ $(dir.fiddle)/*.gz \ EXPORTED_FUNCTIONS.fiddle diff --git a/ext/wasm/index.html b/ext/wasm/index.html index d12a3aa03f..a3d41f1a9c 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -125,10 +125,10 @@ the WASMFS build is available on this server (it is not by default) and that this server emits the COOP/COEP headers.
    -
  • scratchpad-wasmfs: + +
  • speedtest1-wasmfs: a variant of speedtest1 built solely for the wasmfs/opfs feature.
    • diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c new file mode 100644 index 0000000000..29b6cafae9 --- /dev/null +++ b/ext/wasm/mkwasmbuilds.c @@ -0,0 +1,284 @@ +/* +** 2024-09-23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This app's single purpose is to emit parts of the Makefile code for +** building sqlite3's WASM build. The main motivation is to generate +** code which "can" be created via GNU Make's eval command but is +** highly illegible when constructed that way. Attempts to write this +** app in Bash and TCL have suffered from the problem that both +** require escaping $ symbols, making the resulting script code as +** illegible as the eval spaghetti we want to get away from. Writing +** it in C is, somewhat surprisingly, _slightly_ less illegible than +** writing it in bash, tcl, or native Make code. +** +** The emitted makefile code is not standalone - it depends on +** variables and $(call)able functions from the main makefile. +** +*/ + +#undef NDEBUG +#define DEBUG 1 +#include +#include +#include + +#define pf printf +#define ps puts +/* Very common printf() args combo. */ +#define zNM zName, zMode + +/* +** Valid names for the zName arguments. +*/ +#define JS_BUILD_NAMES sqlite3 sqlite3-wasmfs +/* +** Valid names for the zMode arguments of the "sqlite3" build. For the +** "sqlite3-wasmfs" build, only "esm" (ES6 Module) is legal. +*/ +#define JS_BUILD_MODES vanilla esm bundler-friendly node +static const char * zBanner = + "\n########################################################################\n"; + +/* +** Emits common vars needed by the rest of the emitted code (but not +** needed by makefile code outside of these generated pieces). +*/ +static void mk_prologue(void){ + pf("%s", zBanner); + ps("# extern-post-js* and extern-pre-js* are files for use with"); + ps("# Emscripten's --extern-pre-js and --extern-post-js flags."); + ps("extern-pre-js.js := $(dir.api)/extern-pre-js.js"); + ps("extern-post-js.js.in := $(dir.api)/extern-post-js.c-pp.js"); + ps("# Emscripten flags for --[extern-][pre|post]-js=... for the"); + ps("# various builds."); + ps("pre-post-common.flags := --extern-pre-js=$(sqlite3-license-version.js)"); + ps("# pre-post-jses.deps.* = a list of dependencies for the"); + ps("# --[extern-][pre/post]-js files."); + ps("pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js)"); +} + +/* +** Emits makefile code for setting up values for the --pre-js=FILE, +** --post-js=FILE, and --extern-post-js=FILE emcc flags, as well as +** populating those files. +*/ +static void mk_pre_post(const char *zName /* build name */, + const char *zMode /* build mode */, + const char *zCmppD /* optional -D flags for c-pp for the + ** --pre/--post-js files. */){ + pf("%s# Begin --pre/--post flags for %s-%s\n", zBanner, zName, zMode); + pf("c-pp.D.%s-%s := %s\n", zNM, zCmppD ? zCmppD : ""); + pf("pre-post-%s-%s.flags ?=\n", zNM); + + /* --pre-js=... */ + pf("pre-js.js.%s-%s := $(dir.tmp)/pre-js.%s-%s.js\n", + zNM, zNM); + pf("$(pre-js.js.%s-%s): $(MAKEFILE)\n", zNM); +#if 1 + pf("$(eval $(call C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.%s-%s)," + "$(c-pp.D.%s-%s)))\n", zNM, zNM); +#else + /* This part is needed if/when we re-enable the custom + ** Module.instantiateModule() impl in api/pre-js.c-pp.js. */ + pf("pre-js.js.%s-%s.intermediary := $(dir.tmp)/pre-js.%s-%s.intermediary.js\n", + zNM, zNM); + pf("$(eval $(call C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.%s-%s.intermediary)," + "$(c-pp.D.%s-%s) -Dcustom-Module.instantiateModule))\n", zNM, zNM); + pf("$(pre-js.js.%s-%s): $(pre-js.js.%s-%s.intermediary)\n", zNM, zNM); + pf("\tcp $(pre-js.js.%s-%s.intermediary) $@\n", zNM); + + /* Amend $(pre-js.js.zName-zMode) for all targets except the plain + ** "sqlite3" build... */ + if( 0!=strcmp("sqlite3-wasmfs", zName) + && 0!=strcmp("sqlite3", zName) ){ + pf("\t@echo 'Module[xNameOfInstantiateWasm].uri = " + "\"%s.wasm\";' >> $@\n", zName); + } +#endif + + /* --post-js=... */ + pf("post-js.js.%s-%s := $(dir.tmp)/post-js.%s-%s.js\n", zNM, zNM); + pf("$(eval $(call C-PP.FILTER,$(post-js.js.in)," + "$(post-js.js.%s-%s),$(c-pp.D.%s-%s)))\n", zNM, zNM); + + /* --extern-post-js=... */ + pf("extern-post-js.js.%s-%s := $(dir.tmp)/extern-post-js.%s-%s.js\n", zNM, zNM); + pf("$(eval $(call C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.%s-%s)," + "$(c-pp.D.%s-%s)))\n", zNM, zNM); + + /* Combine flags for use with emcc... */ + pf("pre-post-common.flags.%s-%s := " + "$(pre-post-common.flags) " + "--post-js=$(post-js.js.%s-%s) " + "--extern-post-js=$(extern-post-js.js.%s-%s)\n", zNM, zNM, zNM); + + pf("pre-post-%s-%s.flags += $(pre-post-common.flags.%s-%s) " + "--pre-js=$(pre-js.js.%s-%s)\n", zNM, zNM, zNM); + + /* Set up deps... */ + pf("pre-post-jses.%s-%s.deps := $(pre-post-jses.deps.common) " + "$(post-js.js.%s-%s) $(extern-post-js.js.%s-%s)\n", + zNM, zNM, zNM); + pf("pre-post-%s-%s.deps := $(pre-post-jses.%s-%s.deps) $(dir.tmp)/pre-js.%s-%s.js\n", + zNM, zNM, zNM); + pf("# End --pre/--post flags for %s-%s%s", zName, zMode, zBanner); +} + +/* +** Emits rules for the fiddle builds. +** +*/ +static void mk_fiddle(){ + int i = 0; + + mk_pre_post("fiddle-module","vanilla", 0); + for( ; i < 2; ++i ){ + const char *zTail = i ? ".debug" : ""; + const char *zDir = i ? "$(dir.fiddle-debug)" : "$(dir.fiddle)"; + + pf("%s# Begin fiddle%s\n", zBanner, zTail); + pf("fiddle-module.js%s := %s/fiddle-module.js\n", zTail, zDir); + pf("fiddle-module.wasm%s := " + "$(subst .js,.wasm,$(fiddle-module.js%s))\n", zTail, zTail); + pf("$(fiddle-module.js%s):%s $(MAKEFILE) $(MAKEFILE.fiddle) " + "$(EXPORTED_FUNCTIONS.fiddle) " + "$(fiddle.cses) $(pre-post-fiddle-module-vanilla.deps) " + "$(SOAP.js)\n", + zTail, (i ? " $(fiddle-module.js)" : "")); + if( 1==i ){/*fiddle.debug*/ + pf(" @test -d \"$(dir $@)\" || mkdir -p \"$(dir $@)\"\n"); + } + pf(" $(emcc.bin) -o $@ $(fiddle.emcc-flags%s) " + "$(pre-post-fiddle-module-vanilla.flags) $(fiddle.cses)\n", + zTail); + pf(" $(maybe-wasm-strip) $(fiddle-module.wasm%s)\n", zTail); + pf(" @cp -p $(SOAP.js) $(dir $@)\n"); + if( 1==i ){/*fiddle.debug*/ + pf(" cp -p $(dir.fiddle)/index.html " + "$(dir.fiddle)/fiddle.js " + "$(dir.fiddle)/fiddle-worker.js " + "$(dir $@)\n"); + } + pf(" @for i in %s/*.*js %s/*.html %s/*.wasm; do \\\n" + " test -f $${i} || continue; \\\n" + " gzip < $${i} > $${i}.gz; \\\n" + " done\n", zDir, zDir, zDir); + if( 0==i ){ + ps("fiddle: $(fiddle-module.js)"); + }else{ + ps("fiddle-debug: $(fiddle-module-debug.js)"); + } + pf("# End fiddle%s%s", zTail, zBanner); + } +} + +/* +** Emits makefile code for one build of the library, primarily defined +** by the combination of zName and zMode, each of which must be values +** from JS_BUILD_NAMES resp. JS_BUILD_MODES. +*/ +static void mk_lib_mode(const char *zName /* build name */, + const char *zMode /* build mode */, + int bIsEsm /* true only for ESM build */, + const char *zApiJsOut /* name of generated sqlite3-api.js/.mjs */, + const char *zJsOut /* name of generated sqlite3.js/.mjs */, + const char *zCmppD /* extra -D flags for c-pp */, + const char *zEmcc /* extra flags for emcc */){ + assert( zName ); + assert( zMode ); + assert( zApiJsOut ); + assert( zJsOut ); + if( !zCmppD ) zCmppD = ""; + if( !zEmcc ) zEmcc = ""; + + pf("%s# Begin build [%s-%s]\n", zBanner, zNM); + pf("ifneq (1,$(MAKING_CLEAN))\n"); + pf("$(info Setting up build [%s-%s]: %s)\n", zNM, zJsOut); + mk_pre_post(zNM, zCmppD); + pf("\nemcc.flags.%s.%s ?=\n", zNM); + if( zEmcc[0] ){ + pf("emcc.flags.%s.%s += %s\n", zNM, zEmcc); + } + pf("$(eval $(call C-PP.FILTER, $(sqlite3-api.js.in), %s, %s))\n", + zApiJsOut, zCmppD); + + /* target zJsOut */ + pf("%s: %s $(MAKEFILE) $(sqlite3-wasm.cfiles) $(EXPORTED_FUNCTIONS.api) " + "$(pre-post-%s-%s.deps)\n", + zJsOut, zApiJsOut, zNM); + pf("\t@echo \"Building $@ ...\"\n"); + pf("\t$(emcc.bin) -o $@ $(emcc_opt_full) $(emcc.flags) \\\n"); + pf("\t\t$(emcc.jsflags) -sENVIRONMENT=$(emcc.environment.%s) \\\n", zMode); + pf("\t\t$(pre-post-%s-%s.flags) \\\n", zNM); + pf("\t\t$(emcc.flags.%s) $(emcc.flags.%s.%s) \\\n", zName, zNM); + pf("\t\t$(cflags.common) $(SQLITE_OPT) \\\n" + "\t\t$(cflags.%s) $(cflags.%s.%s) \\\n" + "\t\t$(cflags.wasm_extra_init) $(sqlite3-wasm.cfiles)\n", zName, zNM); + if( bIsEsm ){ + /* TODO? Replace this CALL with the corresponding makefile code. + ** OTOH, we also use this $(call) in the speedtest1-wasmfs build, + ** which is not part of the rules emitted by this program. */ + pf("\t@$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,1,%d)\n", + 0==strcmp("sqlite3-wasmfs", zName) ? 1 : 0); + } + pf("\t@dotwasm=$(basename $@).wasm; \\\n" + "\tchmod -x $$dotwasm; \\\n" + "\t$(maybe-wasm-strip) $$dotwasm; \\\n"); + /* + ** The above $(emcc.bin) call will write zJsOut and will create a + ** like-named .wasm file. That .wasm file name gets hard-coded into + ** zJsOut so we need to, for some cases, patch zJsOut to use the + ** name sqlite3.wasm instead. Note that the resulting .wasm file is + ** identical for all builds for which zEmcc is empty. + */ + if( 0==strcmp("bundler-friendly", zMode) + || 0==strcmp("node", zMode) ) { + pf("\techo 'Patching $@ for %s.wasm...' \\\n", zName); + pf("\trm -f $$dotwasm; dotwasm=; \\\n" + "\tsed -i -e 's/%s-%s.wasm/%s.wasm/g' $@ || exit $$?; \\\n", + zNM, zName); + } + pf("\tls -la $$dotwasm $@\n"); + if( 0!=strcmp("sqlite3-wasmfs", zName) ){ + /* The sqlite3-wasmfs build is optional and needs to be invoked + ** conditionally using info we don't have here. */ + pf("all: %s\n", zJsOut); + } + ps("endif\n# ^^^ !$(MAKING_CLEAN)"); + pf("# End build [%s-%s]%s", zNM, zBanner); +} + +int main(void){ + int rc = 0; + pf("# What follows was GENERATED by %s. Edit at your own risk.\n", __FILE__); + mk_prologue(); + mk_lib_mode("sqlite3", "vanilla", 0, + "$(sqlite3-api.js)", "$(sqlite3.js)", 0, 0); + mk_lib_mode("sqlite3", "esm", 1, + "$(sqlite3-api.mjs)", "$(sqlite3.mjs)", + "-Dtarget=es6-module", 0); + mk_lib_mode("sqlite3", "bundler-friendly", 1, + "$(sqlite3-api-bundler-friendly.mjs)", "$(sqlite3-bundler-friendly.mjs)", + "$(c-pp.D.sqlite3-esm) -Dtarget=es6-bundler-friendly", 0); + mk_lib_mode("sqlite3" , "node", 1, + "$(sqlite3-api-node.mjs)", "$(sqlite3-node.mjs)", + "$(c-pp.D.sqlite3-bundler-friendly) -Dtarget=node", 0); + mk_lib_mode("sqlite3-wasmfs", "esm" ,1, + "$(sqlite3-api-wasmfs.mjs)", "$(sqlite3-wasmfs.mjs)", + "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs", + "-sEXPORT_ES6 -sUSE_ES6_IMPORT_META"); + + mk_fiddle(); + mk_pre_post("speedtest1","vanilla", 0); + mk_pre_post("speedtest1-wasmfs","esm", "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs"); + return rc; +} diff --git a/ext/wasm/speedtest1-wasmfs.mjs b/ext/wasm/speedtest1-wasmfs.mjs index 2d5ae322aa..aeb37dd7f9 100644 --- a/ext/wasm/speedtest1-wasmfs.mjs +++ b/ext/wasm/speedtest1-wasmfs.mjs @@ -10,14 +10,14 @@ wMsg('log',"speedtest1-wasmfs starting..."); */ const wasmfsDir = function f(wasmUtil,dirName="/opfs"){ if(undefined !== f._) return f._; - if( !self.FileSystemHandle - || !self.FileSystemDirectoryHandle - || !self.FileSystemFileHandle){ + if( !globalThis.FileSystemHandle + || !globalThis.FileSystemDirectoryHandle + || !globalThis.FileSystemFileHandle){ return f._ = ""; } try{ if(0===wasmUtil.xCallWrapped( - 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], dirName + 'sqlite3__wasm_init_wasmfs', 'i32', ['string'], dirName )){ return f._ = dirName; }else{ @@ -36,7 +36,7 @@ const logErr = (...args)=>wMsg('logErr',...args); const runTests = function(sqlite3){ console.log("Module inited.",sqlite3); const wasm = sqlite3.wasm; - const __unlink = wasm.xWrap("sqlite3_wasm_vfs_unlink", "int", ["*","string"]); + const __unlink = wasm.xWrap("sqlite3__wasm_vfs_unlink", "int", ["*","string"]); const unlink = (fn)=>__unlink(0,fn); const pDir = wasmfsDir(wasm); if(pDir) log("Persistent storage:",pDir); @@ -46,7 +46,7 @@ const runTests = function(sqlite3){ } const scope = wasm.scopedAllocPush(); const dbFile = pDir+"/speedtest1.db"; - const urlParams = new URL(self.location.href).searchParams; + const urlParams = new URL(globalThis.location.href).searchParams; const argv = ["speedtest1"]; if(urlParams.has('flags')){ argv.push(...(urlParams.get('flags').split(','))); diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index c49a82d36e..fd67dc01c3 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -848,171 +848,176 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }/*WhWasmUtil*/) //////////////////////////////////////////////////////////////////// - .t('sqlite3.StructBinder (jaccwabyt🐇)', function(sqlite3){ - const S = sqlite3, W = S.wasm; - const MyStructDef = { - sizeof: 16, - members: { - p4: {offset: 0, sizeof: 4, signature: "i"}, - pP: {offset: 4, sizeof: 4, signature: "P"}, - ro: {offset: 8, sizeof: 4, signature: "i", readOnly: true}, - cstr: {offset: 12, sizeof: 4, signature: "s"} - } - }; - if(W.bigIntEnabled){ - const m = MyStructDef; - m.members.p8 = {offset: m.sizeof, sizeof: 8, signature: "j"}; - m.sizeof += m.members.p8.sizeof; - } - const StructType = S.StructBinder.StructType; - const K = S.StructBinder('my_struct',MyStructDef); - T.mustThrowMatching(()=>K(), /via 'new'/). - mustThrowMatching(()=>new K('hi'), /^Invalid pointer/); - const k1 = new K(), k2 = new K(); - try { - T.assert(k1.constructor === K). - assert(K.isA(k1)). - assert(k1 instanceof K). - assert(K.prototype.lookupMember('p4').key === '$p4'). - assert(K.prototype.lookupMember('$p4').name === 'p4'). - mustThrowMatching(()=>K.prototype.lookupMember('nope'), /not a mapped/). - assert(undefined === K.prototype.lookupMember('nope',false)). - assert(k1 instanceof StructType). - assert(StructType.isA(k1)). - mustThrowMatching(()=>k1.$ro = 1, /read-only/); - Object.keys(MyStructDef.members).forEach(function(key){ - key = K.memberKey(key); - T.assert(0 == k1[key], - "Expecting allocation to zero the memory "+ - "for "+key+" but got: "+k1[key]+ - " from "+k1.memoryDump()); - }); - T.assert('number' === typeof k1.pointer). - mustThrowMatching(()=>k1.pointer = 1, /pointer/); - k1.$p4 = 1; k1.$pP = 2; - T.assert(1 === k1.$p4).assert(2 === k1.$pP); - if(MyStructDef.members.$p8){ - k1.$p8 = 1/*must not throw despite not being a BigInt*/; - k1.$p8 = BigInt(Number.MAX_SAFE_INTEGER * 2); - T.assert(BigInt(2 * Number.MAX_SAFE_INTEGER) === k1.$p8); - } - T.assert(!k1.ondispose); - k1.setMemberCString('cstr', "A C-string."); - T.assert(Array.isArray(k1.ondispose)). - assert(k1.ondispose[0] === k1.$cstr). - assert('number' === typeof k1.$cstr). - assert('A C-string.' === k1.memberToJsString('cstr')); - k1.$pP = k2; - T.assert(k1.$pP === k2.pointer); - k1.$pP = null/*null is special-cased to 0.*/; - T.assert(0===k1.$pP); - let ptr = k1.pointer; - k1.dispose(); - T.assert(undefined === k1.pointer). - mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); - }finally{ - k1.dispose(); - k2.dispose(); - } - - if(!W.bigIntEnabled){ - log("Skipping WasmTestStruct tests: BigInt not enabled."); - return; - } - - const WTStructDesc = - W.ctype.structs.filter((e)=>'WasmTestStruct'===e.name)[0]; - const autoResolvePtr = true /* EXPERIMENTAL */; - if(autoResolvePtr){ - WTStructDesc.members.ppV.signature = 'P'; - } - const WTStruct = S.StructBinder(WTStructDesc); - //log(WTStruct.structName, WTStruct.structInfo); - const wts = new WTStruct(); - //log("WTStruct.prototype keys:",Object.keys(WTStruct.prototype)); - try{ - T.assert(wts.constructor === WTStruct). - assert(WTStruct.memberKeys().indexOf('$ppV')>=0). - assert(wts.memberKeys().indexOf('$v8')>=0). - assert(!K.isA(wts)). - assert(WTStruct.isA(wts)). - assert(wts instanceof WTStruct). - assert(wts instanceof StructType). - assert(StructType.isA(wts)). - assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). - assert(0===wts.$ppV).assert(0===wts.$xFunc); - const testFunc = - W.xGet('sqlite3__wasm_test_struct'/*name gets mangled in -O3 builds!*/); - let counter = 0; - //log("wts.pointer =",wts.pointer); - const wtsFunc = function(arg){ - /*log("This from a JS function called from C, "+ - "which itself was called from JS. arg =",arg);*/ - ++counter; - if(3===counter){ - tossQuietly("Testing exception propagation."); + .t({ + name: 'sqlite3.StructBinder (jaccwabyt🐇)', + predicate: (sqlite3)=>!!sqlite3.wasm.exports.sqlite3__wasm_test_struct + || "Built without SQLITE_WASM_ENABLE_C_TESTS", + test: function(sqlite3){ + const S = sqlite3, W = S.wasm; + const MyStructDef = { + sizeof: 16, + members: { + p4: {offset: 0, sizeof: 4, signature: "i"}, + pP: {offset: 4, sizeof: 4, signature: "P"}, + ro: {offset: 8, sizeof: 4, signature: "i", readOnly: true}, + cstr: {offset: 12, sizeof: 4, signature: "s"} } + }; + if(W.bigIntEnabled){ + const m = MyStructDef; + m.members.p8 = {offset: m.sizeof, sizeof: 8, signature: "j"}; + m.sizeof += m.members.p8.sizeof; + } + const StructType = S.StructBinder.StructType; + const K = S.StructBinder('my_struct',MyStructDef); + T.mustThrowMatching(()=>K(), /via 'new'/). + mustThrowMatching(()=>new K('hi'), /^Invalid pointer/); + const k1 = new K(), k2 = new K(); + try { + T.assert(k1.constructor === K). + assert(K.isA(k1)). + assert(k1 instanceof K). + assert(K.prototype.lookupMember('p4').key === '$p4'). + assert(K.prototype.lookupMember('$p4').name === 'p4'). + mustThrowMatching(()=>K.prototype.lookupMember('nope'), /not a mapped/). + assert(undefined === K.prototype.lookupMember('nope',false)). + assert(k1 instanceof StructType). + assert(StructType.isA(k1)). + mustThrowMatching(()=>k1.$ro = 1, /read-only/); + Object.keys(MyStructDef.members).forEach(function(key){ + key = K.memberKey(key); + T.assert(0 == k1[key], + "Expecting allocation to zero the memory "+ + "for "+key+" but got: "+k1[key]+ + " from "+k1.memoryDump()); + }); + T.assert('number' === typeof k1.pointer). + mustThrowMatching(()=>k1.pointer = 1, /pointer/); + k1.$p4 = 1; k1.$pP = 2; + T.assert(1 === k1.$p4).assert(2 === k1.$pP); + if(MyStructDef.members.$p8){ + k1.$p8 = 1/*must not throw despite not being a BigInt*/; + k1.$p8 = BigInt(Number.MAX_SAFE_INTEGER * 2); + T.assert(BigInt(2 * Number.MAX_SAFE_INTEGER) === k1.$p8); + } + T.assert(!k1.ondispose); + k1.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(k1.ondispose)). + assert(k1.ondispose[0] === k1.$cstr). + assert('number' === typeof k1.$cstr). + assert('A C-string.' === k1.memberToJsString('cstr')); + k1.$pP = k2; + T.assert(k1.$pP === k2.pointer); + k1.$pP = null/*null is special-cased to 0.*/; + T.assert(0===k1.$pP); + let ptr = k1.pointer; + k1.dispose(); + T.assert(undefined === k1.pointer). + mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); + }finally{ + k1.dispose(); + k2.dispose(); } - wts.$v4 = 10; wts.$v8 = 20; - wts.$xFunc = W.installFunction(wtsFunc, wts.memberSignature('xFunc')) - T.assert(0===counter).assert(10 === wts.$v4).assert(20n === wts.$v8) - .assert(0 === wts.$ppV).assert('number' === typeof wts.$xFunc) - .assert(0 === wts.$cstr) - .assert(wts.memberIsString('$cstr')) - .assert(!wts.memberIsString('$v4')) - .assert(null === wts.memberToJsString('$cstr')) - .assert(W.functionEntry(wts.$xFunc) instanceof Function); - /* It might seem silly to assert that the values match - what we just set, but recall that all of those property - reads and writes are, via property interceptors, - actually marshaling their data to/from a raw memory - buffer, so merely reading them back is actually part of - testing the struct-wrapping API. */ - testFunc(wts.pointer); - //log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); - T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) - .assert(wts.$ppV === wts.pointer) - .assert('string' === typeof wts.memberToJsString('cstr')) - .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) - .mustThrowMatching(()=>wts.memberToJsString('xFunc'), - /Invalid member type signature for C-string/) - ; - testFunc(wts.pointer); - T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) - .assert(wts.$ppV === wts.pointer); - /** The 3rd call to wtsFunc throw from JS, which is called - from C, which is called from JS. Let's ensure that - that exception propagates back here... */ - T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); - W.uninstallFunction(wts.$xFunc); - wts.$xFunc = 0; - wts.$ppV = 0; - T.assert(!wts.$ppV); - //WTStruct.debugFlags(0x03); - wts.$ppV = wts; - T.assert(wts.pointer === wts.$ppV) - wts.setMemberCString('cstr', "A C-string."); - T.assert(Array.isArray(wts.ondispose)). - assert(wts.ondispose[0] === wts.$cstr). - assert('A C-string.' === wts.memberToJsString('cstr')); - const ptr = wts.pointer; - wts.dispose(); - T.assert(ptr).assert(undefined === wts.pointer); - }finally{ - wts.dispose(); - } + if(!W.bigIntEnabled){ + log("Skipping WasmTestStruct tests: BigInt not enabled."); + return; + } - if(1){ // ondispose of other struct instances - const s1 = new WTStruct, s2 = new WTStruct, s3 = new WTStruct; - T.assert(s1.lookupMember instanceof Function) - .assert(s1.addOnDispose instanceof Function); - s1.addOnDispose(s2,"testing variadic args"); - T.assert(2===s1.ondispose.length); - s2.addOnDispose(s3); - s1.dispose(); - T.assert(!s2.pointer,"Expecting s2 to be ondispose'd by s1."); - T.assert(!s3.pointer,"Expecting s3 to be ondispose'd by s2."); + const WTStructDesc = + W.ctype.structs.filter((e)=>'WasmTestStruct'===e.name)[0]; + const autoResolvePtr = true /* EXPERIMENTAL */; + if(autoResolvePtr){ + WTStructDesc.members.ppV.signature = 'P'; + } + const WTStruct = S.StructBinder(WTStructDesc); + //log(WTStruct.structName, WTStruct.structInfo); + const wts = new WTStruct(); + //log("WTStruct.prototype keys:",Object.keys(WTStruct.prototype)); + try{ + T.assert(wts.constructor === WTStruct). + assert(WTStruct.memberKeys().indexOf('$ppV')>=0). + assert(wts.memberKeys().indexOf('$v8')>=0). + assert(!K.isA(wts)). + assert(WTStruct.isA(wts)). + assert(wts instanceof WTStruct). + assert(wts instanceof StructType). + assert(StructType.isA(wts)). + assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). + assert(0===wts.$ppV).assert(0===wts.$xFunc); + const testFunc = + W.xGet('sqlite3__wasm_test_struct'/*name gets mangled in -O3 builds!*/); + let counter = 0; + //log("wts.pointer =",wts.pointer); + const wtsFunc = function(arg){ + /*log("This from a JS function called from C, "+ + "which itself was called from JS. arg =",arg);*/ + ++counter; + if(3===counter){ + tossQuietly("Testing exception propagation."); + } + } + wts.$v4 = 10; wts.$v8 = 20; + wts.$xFunc = W.installFunction(wtsFunc, wts.memberSignature('xFunc')) + T.assert(0===counter).assert(10 === wts.$v4).assert(20n === wts.$v8) + .assert(0 === wts.$ppV).assert('number' === typeof wts.$xFunc) + .assert(0 === wts.$cstr) + .assert(wts.memberIsString('$cstr')) + .assert(!wts.memberIsString('$v4')) + .assert(null === wts.memberToJsString('$cstr')) + .assert(W.functionEntry(wts.$xFunc) instanceof Function); + /* It might seem silly to assert that the values match + what we just set, but recall that all of those property + reads and writes are, via property interceptors, + actually marshaling their data to/from a raw memory + buffer, so merely reading them back is actually part of + testing the struct-wrapping API. */ + + testFunc(wts.pointer); + //log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); + T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) + .assert(wts.$ppV === wts.pointer) + .assert('string' === typeof wts.memberToJsString('cstr')) + .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) + .mustThrowMatching(()=>wts.memberToJsString('xFunc'), + /Invalid member type signature for C-string/) + ; + testFunc(wts.pointer); + T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) + .assert(wts.$ppV === wts.pointer); + /** The 3rd call to wtsFunc throw from JS, which is called + from C, which is called from JS. Let's ensure that + that exception propagates back here... */ + T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); + W.uninstallFunction(wts.$xFunc); + wts.$xFunc = 0; + wts.$ppV = 0; + T.assert(!wts.$ppV); + //WTStruct.debugFlags(0x03); + wts.$ppV = wts; + T.assert(wts.pointer === wts.$ppV) + wts.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(wts.ondispose)). + assert(wts.ondispose[0] === wts.$cstr). + assert('A C-string.' === wts.memberToJsString('cstr')); + const ptr = wts.pointer; + wts.dispose(); + T.assert(ptr).assert(undefined === wts.pointer); + }finally{ + wts.dispose(); + } + + if(1){ // ondispose of other struct instances + const s1 = new WTStruct, s2 = new WTStruct, s3 = new WTStruct; + T.assert(s1.lookupMember instanceof Function) + .assert(s1.addOnDispose instanceof Function); + s1.addOnDispose(s2,"testing variadic args"); + T.assert(2===s1.ondispose.length); + s2.addOnDispose(s3); + s1.dispose(); + T.assert(!s2.pointer,"Expecting s2 to be ondispose'd by s1."); + T.assert(!s3.pointer,"Expecting s3 to be ondispose'd by s2."); + } } }/*StructBinder*/) @@ -1126,75 +1131,83 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('sqlite3.oo1') - .t('Create db', function(sqlite3){ - const dbFile = '/tester1.db'; - sqlite3.util.sqlite3__wasm_vfs_unlink(0, dbFile); - const db = this.db = new sqlite3.oo1.DB(dbFile, 0 ? 'ct' : 'c'); - db.onclose = { - disposeAfter: [], - disposeBefore: [ - (db)=>{ - //console.debug("db.onclose.before dropping modules"); - //sqlite3.capi.sqlite3_drop_modules(db.pointer, 0); - } - ], - before: function(db){ - while(this.disposeBefore.length){ - const v = this.disposeBefore.shift(); - console.debug("db.onclose.before cleaning up:",v); - if(wasm.isPtr(v)) wasm.dealloc(v); - else if(v instanceof sqlite3.StructBinder.StructType){ - v.dispose(); - }else if(v instanceof Function){ - try{ v(db) } catch(e){ - console.warn("beforeDispose() callback threw:",e); + .t({ + name:'Create db', + //predicate: (sqlite3)=> + test: function(sqlite3){ + const dbFile = '/tester1.db'; + sqlite3.util.sqlite3__wasm_vfs_unlink(0, dbFile); + const db = this.db = new sqlite3.oo1.DB(dbFile, 0 ? 'ct' : 'c'); + db.onclose = { + disposeAfter: [], + disposeBefore: [ + (db)=>{ + //console.debug("db.onclose.before dropping modules"); + //sqlite3.capi.sqlite3_drop_modules(db.pointer, 0); + } + ], + before: function(db){ + while(this.disposeBefore.length){ + const v = this.disposeBefore.shift(); + console.debug("db.onclose.before cleaning up:",v); + if(wasm.isPtr(v)) wasm.dealloc(v); + else if(v instanceof sqlite3.StructBinder.StructType){ + v.dispose(); + }else if(v instanceof Function){ + try{ v(db) } catch(e){ + console.warn("beforeDispose() callback threw:",e); + } + } + } + }, + after: function(){ + while(this.disposeAfter.length){ + const v = this.disposeAfter.shift(); + console.debug("db.onclose.after cleaning up:",v); + if(wasm.isPtr(v)) wasm.dealloc(v); + else if(v instanceof sqlite3.StructBinder.StructType){ + v.dispose(); + }else if(v instanceof Function){ + try{v()} catch(e){/*ignored*/} } } } - }, - after: function(){ - while(this.disposeAfter.length){ - const v = this.disposeAfter.shift(); - console.debug("db.onclose.after cleaning up:",v); - if(wasm.isPtr(v)) wasm.dealloc(v); - else if(v instanceof sqlite3.StructBinder.StructType){ - v.dispose(); - }else if(v instanceof Function){ - try{v()} catch(e){/*ignored*/} - } - } + }; + + T.assert(wasm.isPtr(db.pointer)) + .mustThrowMatching(()=>db.pointer=1, /read-only/) + .assert(0===sqlite3.capi.sqlite3_extended_result_codes(db.pointer,1)) + .assert('main'===db.dbName(0)) + .assert('string' === typeof db.dbVfsName()) + .assert(db.pointer === wasm.xWrap.testConvertArg('sqlite3*',db)); + // Custom db error message handling via sqlite3_prepare_v2/v3() + let rc = capi.sqlite3_prepare_v3(db.pointer, {/*invalid*/}, -1, 0, null, null); + T.assert(capi.SQLITE_MISUSE === rc) + .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")) + .assert(dbFile === db.dbFilename()) + .assert(!db.dbFilename('nope')); + //Sanity check DB.checkRc()... + let ex; + try{db.checkRc(rc)} + catch(e){ex = e} + T.assert(ex instanceof sqlite3.SQLite3Error) + .assert(capi.SQLITE_MISUSE===ex.resultCode) + .assert(0===ex.message.indexOf("SQLITE_MISUSE: sqlite3 result code")) + .assert(ex.message.indexOf("Invalid SQL")>0); + T.assert(db === db.checkRc(0)) + .assert(db === sqlite3.oo1.DB.checkRc(db,0)) + .assert(null === sqlite3.oo1.DB.checkRc(null,0)); + this.progressHandlerCount = 0; + if( wasm.compileOptionUsed('OMIT_PROGRESS_CALLBACK') ){ + T.assert( !capi.sqlite3_progress_handler ); + }else{ + T.assert( !!capi.sqlite3_progress_handler ); + capi.sqlite3_progress_handler(db, 5, (p)=>{ + ++this.progressHandlerCount; + return 0; + }, 0); } - }; - - T.assert(wasm.isPtr(db.pointer)) - .mustThrowMatching(()=>db.pointer=1, /read-only/) - .assert(0===sqlite3.capi.sqlite3_extended_result_codes(db.pointer,1)) - .assert('main'===db.dbName(0)) - .assert('string' === typeof db.dbVfsName()) - .assert(db.pointer === wasm.xWrap.testConvertArg('sqlite3*',db)); - // Custom db error message handling via sqlite3_prepare_v2/v3() - let rc = capi.sqlite3_prepare_v3(db.pointer, {/*invalid*/}, -1, 0, null, null); - T.assert(capi.SQLITE_MISUSE === rc) - .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")) - .assert(dbFile === db.dbFilename()) - .assert(!db.dbFilename('nope')); - //Sanity check DB.checkRc()... - let ex; - try{db.checkRc(rc)} - catch(e){ex = e} - T.assert(ex instanceof sqlite3.SQLite3Error) - .assert(capi.SQLITE_MISUSE===ex.resultCode) - .assert(0===ex.message.indexOf("SQLITE_MISUSE: sqlite3 result code")) - .assert(ex.message.indexOf("Invalid SQL")>0); - T.assert(db === db.checkRc(0)) - .assert(db === sqlite3.oo1.DB.checkRc(db,0)) - .assert(null === sqlite3.oo1.DB.checkRc(null,0)); - - this.progressHandlerCount = 0; - capi.sqlite3_progress_handler(db, 5, (p)=>{ - ++this.progressHandlerCount; - return 0; - }, 0); + } }) //////////////////////////////////////////////////////////////////// .t('sqlite3_db_config() and sqlite3_db_status()', function(sqlite3){ @@ -1236,7 +1249,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; new TextEncoder('utf-8').encode("select 3 as a") ); //debug("statement =",st); - this.progressHandlerCount = 0; + T.assert( !this.progressHandlerCount ); let rc; try { T.assert(wasm.isPtr(st.pointer)) @@ -1278,7 +1291,8 @@ globalThis.sqlite3InitModule = sqlite3InitModule; st, capi.SQLITE_STMTSTATUS_RUN, 0 ) > 0); - T.assert(this.progressHandlerCount > 0, + T.assert(this.progressHandlerCount>0 + || wasm.compileOptionUsed('OMIT_PROGRESS_CALLBACK'), "Expecting progress callback."). assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). @@ -1363,7 +1377,8 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(2 === list.length) .assert('string'===typeof list[1]) .assert(3===db.changes()) - .assert(this.progressHandlerCount > 0, + .assert(this.progressHandlerCount > 0 + || wasm.compileOptionUsed('OMIT_PROGRESS_CALLBACK'), "Expecting progress callback.") if(wasm.bigIntEnabled){ T.assert(3n===db.changes(false,true)); @@ -1904,7 +1919,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////// .t({ name: 'Window UDFs', - //predicate: ()=>false, + predicate: (sqlite3)=>!!sqlite3.wasm.exports.sqlite3_create_window_function + /*!sqlite3.wasm.compileOptionUsed('OMIT_WINDOWFUNC')*/ + || "Missing window functions", test: function(){ /* Example window function, table, and results taken from: https://sqlite.org/windowfunctions.html#udfwinfunc */ @@ -2153,7 +2170,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// .t({ name: 'virtual table #1: eponymous w/ manual exception handling', - predicate: ()=>!!capi.sqlite3_create_module || "Missing vtab support", + predicate: (sqlite3)=>!!sqlite3.capi.sqlite3_vtab || "Missing vtab support", test: function(sqlite3){ const VT = sqlite3.vtab; const tmplCols = Object.assign(Object.create(null),{ @@ -2350,7 +2367,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// .t({ name: 'virtual table #2: non-eponymous w/ automated exception wrapping', - predicate: ()=>!!capi.sqlite3_create_module || "Missing vtab support", + predicate: (sqlite3)=>!!sqlite3.capi.sqlite3_vtab || "Missing vtab support", test: function(sqlite3){ const VT = sqlite3.vtab; const tmplCols = Object.assign(Object.create(null),{ @@ -3132,7 +3149,10 @@ globalThis.sqlite3InitModule = sqlite3InitModule; ]); T.assert(2 === u1.getFileCount() /* one is the journal file */) .assert(3 === db.selectValue('select count(*) from t')) - .assert('wal'===db.selectValue('pragma journal_mode')); + .assert( + 'wal'===db.selectValue('pragma journal_mode') + || wasm.compileOptionUsed('OMIT_WAL') + ); db.close(); T.assert(1 === u1.getFileCount()); db = new u2.OpfsSAHPoolDb(dbName); @@ -3300,17 +3320,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.g('Bug Reports') .t({ name: 'Delete via bound parameter in subquery', - predicate: function(sqlite3){ - const d = new sqlite3.oo1.DB(); - try{ - d.exec("create virtual table f using fts5(x)"); - return true; - }catch(e){ - return "FTS5 is not available"; - }finally{ - d.close(); - } - }, + predicate: ()=>wasm.compileOptionUsed('ENABLE_FTS5') || "Missing FTS5", test: function(sqlite3){ // Testing https://sqlite.org/forum/forumpost/40ce55bdf5 // with the exception that that post uses "external content" diff --git a/ext/wasm/wasmfs.make b/ext/wasm/wasmfs.make index 71c0fe75a8..9de5574f26 100644 --- a/ext/wasm/wasmfs.make +++ b/ext/wasm/wasmfs.make @@ -6,19 +6,20 @@ # GNUMakefile. ######################################################################## MAKEFILE.wasmfs := $(lastword $(MAKEFILE_LIST)) -$(warning The WASMFS build is currently incomplete.) +# ensure that the following message starts on line 10 or higher for proper +# $(warning) alignment! +ifneq (1,$(MAKING_CLEAN)) + $(warning !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!) + $(warning !! The WASMFS build is not well-supported. WASMFS is a proverbial) + $(warning !! moving target, sometimes changing in incompatible ways between) + $(warning !! Emscripten versions. This build is provided for adventurous folks) + $(warning !! and is not a supported deliverable of the SQLite project.) + $(warning !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!) +endif -#dir.wasmfs := $(dir.wasm) -dir.wasmfs := $(dir.dout) sqlite3-wasmfs.js := $(dir.wasmfs)/sqlite3-wasmfs.js -sqlite3-wasmfs.mjs := $(dir.wasmfs)/sqlite3-wasmfs.mjs sqlite3-wasmfs.wasm := $(dir.wasmfs)/sqlite3-wasmfs.wasm -CLEAN_FILES += $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.wasm) \ - $(subst .js,.worker.js,$(sqlite3-wasmfs.js)) \ - $(sqlite3-wasmfs.mjs) \ - $(subst .mjs,.worker.mjs,$(sqlite3-wasmfs.mjs)) - ######################################################################## # emcc flags for .c/.o. cflags.sqlite3-wasmfs := @@ -30,12 +31,15 @@ cflags.sqlite3-wasmfs += -DSQLITE_ENABLE_WASMFS # emcc flags specific to building the final .js/.wasm file... emcc.flags.sqlite3-wasmfs := emcc.flags.sqlite3-wasmfs += \ - -sEXPORTED_RUNTIME_METHODS=wasmMemory,allocateUTF8OnStack,stringToUTF8OnStack + -sEXPORTED_RUNTIME_METHODS=wasmMemory # wasmMemory ==> for -sIMPORTED_MEMORY - # *OnStack ==> wasmfs internals (leaky abstraction) +# Some version of emcc between 3.1.60-ish(?) and 3.1.62 deprecated the +# use of (allocateUTF8OnStack,stringToUTF8OnStack). Earlier emcc +# versions will fail to build without those in the +# EXPORTED_RUNTIME_METHODS list. emcc.flags.sqlite3-wasmfs += -sUSE_CLOSURE_COMPILER=0 emcc.flags.sqlite3-wasmfs += -Wno-limited-postlink-optimizations -# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. +# ^^^^^ emcc likes to warn when we have "limited optimizations" via the -g3 flag. emcc.flags.sqlite3-wasmfs += -sMEMORY64=0 emcc.flags.sqlite3-wasmfs += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.128) # ^^^^ 64MB is not enough for WASMFS/OPFS test runs using batch-runner.js @@ -51,12 +55,7 @@ emcc.flags.sqlite3-wasmfs += -sALLOW_MEMORY_GROWTH=0 # And, indeed, it runs slowly if memory is permitted to grow. #emcc.flags.sqlite3-wasmfs.vanilla := #emcc.flags.sqlite3-wasmfs.esm := -sEXPORT_ES6 -sUSE_ES6_IMPORT_META -sqlite3-api.mjs.wasmfs := $(dir.tmp)/sqlite3-api-wasmfs.mjs -$(eval $(call SETUP_LIB_BUILD_MODE,sqlite3-wasmfs,esm,1,\ - $(sqlite3-api.mjs.wasmfs), $(sqlite3-wasmfs.mjs),\ - $(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs,\ - -sEXPORT_ES6 -sUSE_ES6_IMPORT_META\ -)) +all: $(sqlite3-wasmfs.mjs) $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs): $(MAKEFILE.wasmfs) ######################################################################## # Build quirk: we cannot build BOTH .js and .mjs with our current @@ -67,8 +66,8 @@ $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs): $(MAKEFILE.wasmfs) # build both modes they would need to have distinct base names or # output directories. "The problem" with giving them distinct base # names is that it means that the corresponding .wasm file is also -# built/saved multiple times. -# +# built/saved multiple times. It is likely that anyone wanting to use +# WASMFS will want an ES6 module, so that's what we build here. wasmfs.build.ext := mjs $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs): $(SOAP.js.bld) ifeq (js,$(wasmfs.build.ext)) @@ -78,7 +77,6 @@ else $(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.mjs) wasmfs: $(sqlite3-wasmfs.mjs) endif -#all: wasmfs ######################################################################## # speedtest1 for wasmfs. @@ -103,13 +101,11 @@ $(speedtest1-wasmfs.mjs): $(speedtest1.cfiles) $(sqlite3-wasmfs.js) \ $(emcc.flags.sqlite3-wasmfs) \ $(emcc.flags.speedtest1-wasmfs) \ -o $@ $(speedtest1.cfiles) -lm - @$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,1) + @$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,1,1) $(maybe-wasm-strip) $(speedtest1-wasmfs.wasm) chmod -x $(speedtest1-wasmfs.wasm) ls -la $@ $(speedtest1-wasmfs.wasm) wasmfs: $(speedtest1-wasmfs.mjs) -CLEAN_FILES += $(speedtest1-wasmfs.mjs) $(speedtest1-wasmfs.wasm) \ - $(subst .js,.worker.js,$(speedtest1-wasmfs.mjs)) # end speedtest1.js ######################################################################## diff --git a/main.mk b/main.mk index 98391b60f8..f81fd24128 100644 --- a/main.mk +++ b/main.mk @@ -562,14 +562,27 @@ sqlite3$(EXE): sqlite3.h libsqlite3.a shell.c $(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \ shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) -sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h - $(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \ +sqldiff$(EXE): $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h sqlite3.c sqlite3.h + $(TCCX) -I$(TOP)/ext/misc -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB) dbhash$(EXE): $(TOP)/tool/dbhash.c sqlite3.c sqlite3.h $(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB) +RSYNC_SRC = \ + $(TOP)/tool/sqlite3-rsync.c \ + sqlite3.c + +RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3-rsync$(EXE): $(RSYNC_SRC) + $(TCC) -o $@ $(RSYNC_OPT) $(RSYNC_SRC) $(TLIBS) + scrub$(EXE): $(TOP)/ext/misc/scrub.c sqlite3.o $(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB) @@ -752,8 +765,6 @@ keywordhash.h: $(TOP)/tool/mkkeywordhash.c # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)/src/shell.c.in \ - $(TOP)/ext/consio/console_io.c \ - $(TOP)/ext/consio/console_io.h \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/intck/sqlite3intck.c \ @@ -770,9 +781,13 @@ SHELL_DEP = \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/sha1.c \ $(TOP)/ext/misc/shathree.c \ $(TOP)/ext/misc/sqlar.c \ + $(TOP)/ext/misc/sqlite3_stdio.c \ + $(TOP)/ext/misc/sqlite3_stdio.h \ $(TOP)/ext/misc/uint.c \ + $(TOP)/ext/misc/vfstrace.c \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/sqlite3recover.c \ @@ -1150,7 +1165,7 @@ clean: rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) rm -f wordcount$(TEXE) changeset$(TEXE) version-info$(TEXE) rm -f *.dll *.lib *.exp *.def *.pc *.vsix - rm -f sqlite3_analyzer$(TEXE) + rm -f sqlite3_analyzer$(TEXE) sqlite3-rsync$(TEXE) rm -f mptester$(TEXE) rbu$(TEXE) srcck1$(TEXE) rm -f fuzzershell$(TEXE) fuzzcheck$(TEXE) sqldiff$(TEXE) dbhash$(TEXE) rm -f threadtest5$(TEXE) diff --git a/manifest b/manifest index c672a5fb61..b1ce724878 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Update\sthe\sbedrock\sbranch\sto\sinclude\sall\sthe\slatest\senhancements\sfrom\strunk. -D 2024-09-06T15:45:23.707 +C Merge\sthe\slatest\strunk\senhancements,\svia\sthe\swal2\sbranch,\sinto\sthe\sbedrock\sbranch. +D 2024-09-30T17:57:58.613 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 9cd7f11b35e8d1372f327831835273011877d2fafee62a4245c07aec5f3853ad +F Makefile.in b51056df3cb0d609daf2c54263104e748ddf79f65e46b77d347bedca6eecb9ce F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc d784087f454f007dc0b8f037fb16bfbdba2194356fe26b4add4f5e0e5ef8ce57 +F Makefile.msc 20f594d77f988ecacfe713442201e019066caf70274006b4bd8f7e1dae2177e9 F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -59,8 +59,8 @@ F ext/consio/console_io.c d2b74afae8d301de2e8447b1045fcd33eb59df13bf581d906d99c7 F ext/consio/console_io.h b5ebe34aa15b357621ebbea3d3f2e2b24750d4280b5802516409e23947fd9ee5 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 -F ext/expert/expert1.test 661f873fd451127edf822ef0d520088faa319135f6a15bd10be6801ac284ac9b -F ext/expert/sqlite3expert.c 8b09aeb2b95a9fca8b6628b522bf4d69aa746ff64c38eb1e99a9b5fad8cf03b9 +F ext/expert/expert1.test b10f9e20f64102a015c0fcf54cb7b7680266b397e91d93cdad45f57857cdfba6 +F ext/expert/sqlite3expert.c df417a6d91873a74d35daa9259171647c23c6601415e938e8a71702703f3d677 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c b767b2039a0df707eb3147e86bcf68b252d8455d9a41774b1a836cd052ceca70 F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c7cc3bf59ee @@ -95,15 +95,15 @@ F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7e F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 F ext/fts5/fts5.h efaaac0df3d3bc740383044c144b582f47921aafa21d7b10eb98f42c24c740b0 -F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d805655360 +F ext/fts5/fts5Int.h 93aba03ca417f403b07b2ab6f50aa0e0c1b8b031917a9026b81520e7047a168e F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 -F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81cbefc109ea +F ext/fts5/fts5_config.c da21548ddbc1a457cb42545f527065221ede8ada6a734891b8c34317a7a9506b F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 571483823193f09439356741669aa8c81da838ae6f5e1bfa7517f7ee2fb3addd -F ext/fts5/fts5_main.c 43990841e0221e0da1bf7ec687196683692b9e8783c372b8d58dea3e9866a5e4 -F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612 +F ext/fts5/fts5_index.c 368a968570ce12ba40223e284a588d9f93ee23a0133727f0df1fcd64086b1fb6 +F ext/fts5/fts5_main.c 2f7804238145ec847fde563872879080aba525bfb675235817332b82da84f3ac +F ext/fts5/fts5_storage.c 3332497823c3d171cf56379f2bd8c971ce15a19aadacff961106462022c92470 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -134,7 +134,7 @@ F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06c F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36 -F ext/fts5/test/fts5blob.test caa33369e93e99ff494cd1103506ae34c5afbc0bcc369ed5e58e135144e33689 +F ext/fts5/test/fts5blob.test 9644a5f917306690e08c5f89a470a3f2489376eaa52026eeca3209d149d6af74 F ext/fts5/test/fts5cat.test bf67dd335f964482ee658287521b81e2b88697b45eb7f73933e15f198ed447cb F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62 F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897 @@ -148,7 +148,7 @@ F ext/fts5/test/fts5contentless.test 606f063b29ba0f46d4b79aa36cdd1ef4dab5de53eae F ext/fts5/test/fts5contentless2.test 70ffe6c611d8f278240da56734df8a77948f04e2739b358439e9bdcf56ced35f F ext/fts5/test/fts5contentless3.test 75eaae5ad6b284ee447788943974d323228f27cc35a1681da997135cff95bc6a F ext/fts5/test/fts5contentless4.test ec34dc69ef474ca9997dae6d91e072906e0e9a5a4b05ea89964c863833b6eff8 -F ext/fts5/test/fts5contentless5.test 40cdcb4fe751672450829c5a96bd32c25fc2f6076279dd2ce5c58ac9a390132a +F ext/fts5/test/fts5contentless5.test 38cd0392c730dc7090c550321ce3c24ba4c392bc97308b51a4180e9959dca7b5 F ext/fts5/test/fts5corrupt.test 6485f721b88ba355ca5d701e7ee87a4efa3ea578d8e6adb26f51ef956c8328bd F ext/fts5/test/fts5corrupt2.test 335911e3f68b9625d850325f9e29a128db3f4276a8c9d4e32134580da8f924c4 F ext/fts5/test/fts5corrupt3.test 4fc3bf129f1616bea00884a23fd9d7b0e46d01791d2b57fe8d68ac36e8d3ff7c @@ -181,7 +181,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d -F ext/fts5/test/fts5faultI.test ae4b83ac953200bd7b66d53038f7d6a4fc29cd64831b8e1795538babcea7c638 +F ext/fts5/test/fts5faultI.test 0706b307b208638554c9e65b4091e1c0dd8c92941535089a301df454ff2c56f4 F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -192,7 +192,7 @@ F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba104 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 797cf6f5e017462ab11313ce884b9f1df8ff063811e74ef42190cd19ed6b600b +F ext/fts5/test/fts5locale.test 83ba7ee12628b540d3098f39c39c1de0c0440eddff8f7512c8c698d0c4a3ae3c F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -209,7 +209,7 @@ F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b3 F ext/fts5/test/fts5origintext3.test 45c33cf0c91a9ca0e36d298462db3edc7c8fe45fd185649a9dbfd66bb670058b F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8 F ext/fts5/test/fts5origintext5.test ee12b440ec335e5b422d1668aca0051b52ff28b6ee67073e8bbc29f509fd562b -F ext/fts5/test/fts5phrase.test 6260b8be40d51ec287992cd983a5e58a9be92e5dc6e01d48cbce1ad0e95a99d8 +F ext/fts5/test/fts5phrase.test bb2554bb61d15f859678c96dc89a7de415cd5fc3b7b54c29b82a0d0ad138091c F ext/fts5/test/fts5plan.test f8b0d752a818059a934cdc96c0f77de058a67a0a57bb3a8181d28307ab5b1626 F ext/fts5/test/fts5porter.test 15b514fac8690b58e99c330efe5bf5615bc43f2fae4a3cca3f923dbaff55a0c0 F ext/fts5/test/fts5porter2.test 94f0e4351e2c99b4e74f1fae05a4ddf1cb5b926620a8c14554160d075ddc7a59 @@ -403,7 +403,7 @@ F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f82 F ext/misc/decimal.c 172cf81a8634e6a0f0bedaf71a8372fee63348cf5a3c4e1b78bb233c35889fdc F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 F ext/misc/explain.c 606100185fb90d6a1eade1ed0414d53503c86820d8956a06e3b0a56291894f2b -F ext/misc/fileio.c 916638042f318701460485032e33981056747d0f92e6757aa9499f2363ea7047 +F ext/misc/fileio.c e6b34db4df4b55b96265086c0010264e257b6eab1644e665697a6da587659403 F ext/misc/fossildelta.c 8c026e086e406e2b69947f1856fa3b848fff5379962276430d10085b8756b05a F ext/misc/fuzzer.c 8b28acf1a7e95d50e332bdd47e792ff27054ad99d3f9bc2e91273814d4b31a5a F ext/misc/ieee754.c 62a90978204d2c956d5036eb89e548e736ca5fac0e965912867ddd7bb833256d @@ -424,11 +424,13 @@ F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6 F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c a6089b5e8e3002bd1e5d9877cee6aead0b9a6426e406c09a399817db9e9ae823 -F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d +F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 F ext/misc/shathree.c 1821d90a0040c9accdbe3e3527d378d30569475d758aa70f6848924c0b430e8c F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c c0aa7b80d6df45f7da59d912b38752bcac1af53a5766966160e6c5cdd397dbea +F ext/misc/spellfix.c bcc42ef3fd29429bc01a83e751332b8d4690e65d45008449bdffe7656371487f F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634 +F ext/misc/sqlite3_stdio.c f110e6f2dc97c67e89f941f82af7dbd221193fa44d1e3ef38a691454a2cbccda +F ext/misc/sqlite3_stdio.h f05eaf5e0258f0573910324a789a9586fc360a57678c57a6d63cfaa2245b6176 F ext/misc/stmt.c b090086cd6bd6281c21271d38d576eeffe662f0e6b67536352ce32bbaa438321 F ext/misc/stmtrand.c 59cffa5d8e158943ff1ce078956d8e208e8c04e67307e8f249dece2436dcb7fc F ext/misc/templatevtab.c 10f15b165b95423ddef593bc5dcb915ec4eb5e0f1066d585e5435a368b8bc22b @@ -439,10 +441,11 @@ F ext/misc/urifuncs.c f71360d14fa9e7626b563f1f781c6148109462741c5235ac63ae0f8917 F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 F ext/misc/vfsstat.c a85df08654743922a19410d7b1e3111de41bb7cd07d20dd16eda4e2b808d269d +F ext/misc/vfstrace.c ac76a4ac4d907774fd423cc2b61410c756f9d0782e27cf6032e058594accaaca w src/test_vfstrace.c F ext/misc/vtablog.c 1100250ce8782db37c833e3a9a5c9a3ecf1af5e15b8325572b82e6e0a138ffb5 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 -F ext/misc/zipfile.c 5a3bf1b9cccb8e0da2389fe9e39e9c7f2b1351474b7e5090635f817d495eee3f +F ext/misc/zipfile.c b62147ac4985eaac4e368d529b1f4f43ad6bc9ac13d6805d907fff3afdac64d3 F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64 F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255 @@ -580,7 +583,7 @@ F ext/session/sessionG.test 3efe388282d641b65485b5462e67851002cd91a282dc95b685d0 F ext/session/sessionH.test 29a5441c3dc0a63fa596d745e64bc6c636e062ae04cd89bc84e32c7d98b1fa9b F ext/session/session_common.tcl e5598096425486b363718e2cda48ee85d660c96b4f8ea9d9d7a4c3ef514769da F ext/session/session_speed_test.c dcf0ef58d76b70c8fbd9eab3be77cf9deb8bc1638fed8be518b62d6cbdef88b3 -F ext/session/sessionalter.test 460bdac2832a550519f6bc32e5db2c0cee94f335870aaf25a3a403a81ab20e17 +F ext/session/sessionalter.test e852acb3d2357aac7d0b920a2109da758c4331bfdf85b41d39aa3a8c18914f65 F ext/session/sessionat.test 00c8badb35e43a2f12a716d2734a44d614ff62361979b6b85419035bc04b45ee F ext/session/sessionbig.test 47c381e7acfabeef17d98519a3080d69151723354d220afa2053852182ca7adf F ext/session/sessionchange.test 77c4702050f24270b58070e2cf01c95c3d232a3ef164b70f31974b386ce69903 @@ -588,10 +591,10 @@ F ext/session/sessionconflict.test 8b8cbd98548e2e636ddc17d0986276f60e833fb865617 F ext/session/sessiondiff.test e89f7aedcdd89e5ebac3a455224eb553a171e9586fc3e1e6a7b3388d2648ba8d F ext/session/sessionfault.test c2b43d01213b389a3f518e90775fca2120812ba51e50444c4066962263e45c11 F ext/session/sessionfault2.test b0d6a7c1d7398a7e800d84657404909c7d385965ea8576dc79ed344c46fbf41c -F ext/session/sessionfault3.test 7c7547202775de268f3fe6f074c4d0d165151829710b4e64f90d4a01645ba9e7 +F ext/session/sessionfault3.test ce0b5d182133935c224d72507dbf1c5be1a1febf7e85d0b0fbd6d2f724b32b96 F ext/session/sessioninvert.test 04075517a9497a80d39c495ba6b44f3982c7371129b89e2c52219819bc105a25 F ext/session/sessionmem.test f2a735db84a3e9e19f571033b725b0b2daf847f3f28b1da55a0c1a4e74f1de09 -F ext/session/sessionnoact.test 506526a5fe29421ecc50d371774ef1bb04cbd9d906a8a468f0556cdbde184c22 +F ext/session/sessionnoact.test 2563dff62a2a80dc7c88002241b2fd1578c3e5438735e180fb7e941ebbc66214 F ext/session/sessionnoop.test a9366a36a95ef85f8a3687856ebef46983df399541174cb1ede2ee53b8011bc7 F ext/session/sessionnoop2.test de4672dce88464396ec9f30ed08c6c01643a69c53ae540fadbbf6d30642d64e8 F ext/session/sessionrebase.test 702378bdcb5062f1106e74457beca8797d09c113a81768734a58b197b5b334e2 @@ -601,14 +604,14 @@ F ext/session/sessionstat1.test 5e718d5888c0c49bbb33a7a4f816366db85f59f6a4f97544 F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc F ext/session/sqlite3changebatch.c d488b42d8fd49fb013a1e9c4535232680dabeb28ae8f9421b65ea0ccc3b430f7 F ext/session/sqlite3changebatch.h e72016998c9a22d439ddfd547b69e1ebac810c24 -F ext/session/sqlite3session.c 2190541e6870ceb6609e6cee35d2554c1843e5c556a93657e879c08deba6d124 +F ext/session/sqlite3session.c 830f5530a3240dd67740eaf81dc6e921a529d84cb476e2046ef6fcc8e0e08ec4 F ext/session/sqlite3session.h 3376dbf372cb00cc0f4e960ca0a0125418638da8c55aad749c9fe7a58a770506 F ext/session/test_session.c 6b2e04ebf69a7a5d81f425e5f8194fb03e41da0747f1d5c5555f1b9cfc1ce7e3 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt ca7e9ee82ca4e1c1744295f8184dd70edfae1992865d26c64303f539eb6c084c F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 -F ext/wasm/GNUmakefile d4f6586d9a36ee2869a8c7f77227a8b7f42b6c4623f3be594beafb1554ab20d9 +F ext/wasm/GNUmakefile 6b94c3125c40932e71bbe46e582d24ed4c829b9fa2ff34dd842b0ed8a948ac6a F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -617,20 +620,17 @@ F ext/wasm/SQLTester/SQLTester.run.mjs c72b7fe2072d05992f7a3d8c6a1d34e95712513ce F ext/wasm/SQLTester/index.html 3f8a016df0776be76605abf20e815ecaafbe055abac0e1fe5ea080e7846b760d F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth 7ac80cc3b6a6d52e041bb295e85555ce797be78c15ef2008a64ae58815014080 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate d1d62a2212099f2c0782d730beb8cb84a7a52d99c15ead2cb9b1411fff5fd6b1 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras fe40d6d758646e38f8b15f709044951e10884214f5453d35502100179c388c13 w ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session 213b6c04267cb9bd760172db011eb1650732805fb3d01f9395478a8ceec18eb0 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-vtab fd57af1f4502a052be27d8402df74be1dc60fcb6a687d372972abd90e424120a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 34fe11466f9c1d81b10a0469e1114e5f1c5a6365c73d80a1a6ca639a1a358b73 F ext/wasm/api/extern-post-js.c-pp.js c4154a7f90c2d7e51fd6738273908152036c3457fdc0b6523f1be3ef51105aac F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 04dc12c3edd666b64a1b4ef3b6690c88dcc653f26451fd4734472d8e29c1c122 -F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 +F ext/wasm/api/pre-js.c-pp.js a614a2c82b12c4d96d8e3ba77330329efc53c4d56a8a7e60ade900f341866cfb F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e -F ext/wasm/api/sqlite3-api-glue.c-pp.js 54b32b5321105a72d6f3d3e8b77f28f162d0367b08c63184263d3f85f3d7dbed +F ext/wasm/api/sqlite3-api-glue.c-pp.js fb6dbfe692cc23000a65a4cd95a1a47ed5eb592dc9d8b55363b3c2952a787244 F ext/wasm/api/sqlite3-api-oo1.c-pp.js f3a8e2004c6625d17946c11f2fb32008be78bc5207bf746fc77d59848813225f F ext/wasm/api/sqlite3-api-prologue.js 6f1257e04885632ed9f44d43aba200b86e0bc16709ffdba29abbbeb1bc8e8b76 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 5cc22a3c0d52828cb32aad8691488719f47d27567e63e8bc8b832d74371c352d @@ -640,14 +640,14 @@ F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js e529a99b7d5a088284821e2902b20d3404b561126969876997d5a73a656c9199 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e99e3d99f736937914527070f00ab13e9391d3f1cef884ab99a64cbcbee8d675 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js e809739d71e8b35dfe1b55d24d91f02d04239e6aef7ca1ea92a15a29e704f616 -F ext/wasm/api/sqlite3-wasm.c 09a938fc570f282e602acd111147c7b74b5332da72540c512a79b916ab57882a +F ext/wasm/api/sqlite3-wasm.c 83f5e9f998e9fa4261eb84e9f092210e3ffe03895119f5ded0429eb34ab9d2be F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js f10c3ecd9df06f6320073c2ce230a7ed7c56034d8b88c1e57095f2a97faf423a F ext/wasm/api/sqlite3-worker1.c-pp.js 5e8706c2c4af2a57fbcdc02f4e7ef79869971bc21bb8ede777687786ce1c92d5 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 F ext/wasm/batch-runner.js 05ec254f5dbfe605146d9640b3db17d6ef8c3fbef6aa8396051ca72bb5884e3f -F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4 +F ext/wasm/c-pp.c 6d131069644964223305582a80973477fa8b06b57306781690d7874ebd3a4f84 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f @@ -663,19 +663,20 @@ F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2 F ext/wasm/demo-worker1.js 836bece8615b17b1b572584f7b15912236a5947fe8c68b98d2737d7e287447ef F ext/wasm/dist.make 653e212c1e84aa3be168d62a10616ccea45ee9585b0192745d2706707a5248ce F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle.make 2406b02473878a99fb6a2eaff0923277017adc45eb041b2afb2d7707bf7b375c +F ext/wasm/fiddle.make ec2353f0eddade864f67b993376a0949e27b72465c24b1970940e48b70bc2df1 F ext/wasm/fiddle/fiddle-worker.js 850e66fce39b89d59e161d1abac43a181a4caa89ddeea162765d660277cd84ce F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/wasm/fiddle/fiddle.js b444a5646a9aac9f3fc06c53d78af5e1912eb235d69a8e6010723e4eb0e9d4a1 F ext/wasm/index-dist.html 564b5ec5669676482c5a25dea9e721d8eafed426ecb155f93d29aeff8507511f -F ext/wasm/index.html 4337f495416756802669f69f9f9f3df9f87ee4c1918e6718719b4b5718e4713a +F ext/wasm/index.html e4bbffdb3d40eff12b3f9c7abedef91787e2935620b7f8d40f2c774b80ad8fa9 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54 F ext/wasm/jaccwabyt/jaccwabyt.md 59a20df389abcc3606eb4eaea7fb7ba14504beb3e345dbea9b99a0618ba3bec8 +F ext/wasm/mkwasmbuilds.c e3580b26bc393e4e4beb25f6349b999878782f3319b740469f64c2e772632e03 F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d -F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe +F ext/wasm/speedtest1-wasmfs.mjs c77c7231338ed5c0e1ce16aa29106df8e5b5cf11a48319c49433490a8d3ded30 F ext/wasm/speedtest1-worker.html 864b65ed78ce24847a348c180e7f267621a02ca027068a1863ec1c90187c1852 F ext/wasm/speedtest1-worker.js 95e549e13a4d35863a9a7fc66122b5f546c0130d3be7b06dfcc556eb66d24bde F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da @@ -686,15 +687,15 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js a88b9c669715adc1c5e76750ca8c0994ae33d04572e3bf295b6f4f5870f3bdf3 +F ext/wasm/tester1.c-pp.js bb8c41a56ca0eabb945ca2e8f06324a7b63ad91630959d714b071bee88322019 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 -F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966f1bc16a +F ext/wasm/wasmfs.make bc8bb227f35d5bd3863a7bd2233437c37472a0d81585979f058f9b9b503bef35 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk b656a84515801f1bda32776e4e8ff75bdba5021b9fec5dd0fee9f84be3f5df7a +F main.mk 2edabaa7f5ae1de099ab5d4bfef787a6a4ae6e2d839dbaa2603d792100f434a0 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -721,9 +722,9 @@ F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d49 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 193f6f9a75204274b7e7f45ac6d6517c12c70b55a5dfb39312dfc3a52e2a8138 F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a -F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 +F src/dbpage.c 12e49515d67d4a59625d71f9aa42499556cfdc2e4f1ea49086e674a7f47f46e5 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c -F src/delete.c 444c4d1eaac40103461e3b6f0881846dd3aafc1cec1dd169d3482fa331667da7 +F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f @@ -737,7 +738,7 @@ F src/insert.c f8d1a0f8ee258411009c6b7f2d93170e351bd19f5ad89d57e1180644297cbe70 F src/json.c 68a98c020c22127f2d65f08855f7fc7460ff352a6ce0b543d8931dde83319c22 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 -F src/main.c def50c462ce61f722196f7615d79374e2e5b9e856cce2991192faea3b9dbd33c +F src/main.c c21e930f90e7099d0f3eb92ce999c38d51fd828bee9412b8cb72521fac4c84b6 F src/malloc.c 410e570b30c26cc36e3372577df50f7a96ee3eed5b2b161c6b6b48773c650c5e F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 @@ -747,7 +748,7 @@ F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff F src/memdb.c 16679def118b5fd75292a253166d3feba3ec9c6189205bf209643ecdb2174ecc F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9 -F src/mutex.c 1b4c7e5e3621b510e0c18397210be27cd54c8084141144fbbafd003fde948e88 +F src/mutex.c 06bcd9c3dbf2d9b21fcd182606c00fafb9bfe0287983c8e17acd13d2c81a2fa9 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c f7ee5a2061a4c11815a2bf4fc0e2bfa6fb8d9dc89390eb613ca0cec32fc9a3d1 @@ -758,7 +759,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c e33663c5a819e2c8532eedcefb1ea1ad7c1f0c9ac42e2149f632a642c2f9e621 +F src/os_unix.c 74de71b4979d2a3409224869b8738d64fab55dcc855162b20d1526544ae0efa1 F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c bfcfab4b210c602f77b601b07b6c4627e994e97218cb905ff73ab0231efd078f @@ -769,17 +770,17 @@ F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 430b86e10b8d6ae65832a19a0016f8f65f74207687bc9501c0b15a5908bcb6ad F src/pragma.c d206a9a757faceb812183faa2ed328145795c8067a44f981b0478736f68d4144 F src/pragma.h 6ebbdee90ed56a892d2c728e27fd9c1ce48c8a28841888d0c6c147946b38cb25 -F src/prepare.c d99931f45416652895e502328ca49fe782cfc4e1ebdcda13b3736d991ebf42ce +F src/prepare.c 3ba0ad907b7773ed642f66cea8a2c9c8edc18841aa1050b6218dbb3479e86225 F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2 F src/random.c 9bd018738ec450bf35d28050b4b33fa9a6eebf3aaefb1a1cff42dc14a7725673 -F src/resolve.c 2c127880c0634962837f16f2f48a295e514357af959330cc038de73015d5b5e8 +F src/resolve.c 9750a281f7ba073b4e6da2be1a6c4071f5d841a7746c5fb3f70d6d793b6675ea F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 108baa344f34a17a0723067d28b5b4b7da937d02ddc2d7c1a39a4a4815628ca4 -F src/shell.c.in 40de636c1d90fb8a9ca7f49dc8f50d930f1b60736e73aca5eb37c4c7d0e47f9d -F src/sqlite.h.in b7ff496637807ae88b2557039fc940518db328bf5d5621e2f7c048dfba32a52b +F src/shell.c.in 345173187067363374187176a8bbe779e359849e635d8288202000ee87f4405a +F src/sqlite.h.in cb20702355ffaeb6442b9bd3e7621471ffcd56d8c81d558eafdbd52c42ff7d56 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 913fb7b1b99ca00708eef5d99503f8f9d71b302e057572a9d7f56071ea7b79a6 +F src/sqliteInt.h e07d0efa86f45a6ad72a44e37bf5d3d806cd82a956cb0a38620c6eda2ba3280f F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -805,7 +806,7 @@ F src/test_demovfs.c 3efa2adf4f21e10d95521721687d5ca047aea91fa62dd8cc22ac9e5a9c9 F src/test_devsym.c 649434ed34d0b03fbd5a6b42df80f0f9a7e53f94dd1710aad5dd8831e91c4e86 F src/test_fs.c c411c40baba679536fc34e2679349f59d8225570aed3488b5b3ef1908525a3d5 F src/test_func.c 8c0e89192f70fac307822d1ac2911ee51751288780b3db0c5ab5ca75fa0fe851 -F src/test_hexio.c 35704e7db83fe1a0e1a014bacb17de97d7d1e89af3356a9da04e37cdaa087c09 +F src/test_hexio.c 7449504e4bde876ba91b202617a9228c7c8c2e7bd8b957302f3803ac0e9e353c F src/test_init.c 17313332d58e90defc527129d5eda4a08bd6b6e8de7207a231523c8d98fb445e F src/test_intarray.c e4216aadee9df2de7d1aee7e70f6b22c80ee79ece72a63d57105db74217639e5 F src/test_intarray.h 6c3534641108cd1bea517a8e117dcba237081310a29a4c35bd2190caa8972293 @@ -831,7 +832,6 @@ F src/test_tclvar.c ae873248a0188459b1c16ca7cc431265dacce524399e8b46725c2b3b7e04 F src/test_thread.c d7a8bcea7445f37cc2a1f7f81dd6059634f45e0c61bfe80182b02872fb0328bb F src/test_vdbecov.c 5c426d9cd2b351f5f9ceb30cabf8c64a63bfcad644c507e0bd9ce2f6ae1a3bf3 F src/test_vfs.c f298475e468c7e14945b20af885917181090c265aa3c4ade897849c9fbd396f2 -F src/test_vfstrace.c a2ea82df2ed8927e9eba49bdba1aa1aeb1dcb13dbf6558cff036da813031de9a F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1 F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb775ebc50 F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72 @@ -842,25 +842,25 @@ F src/treeview.c 88aa39b754f5ef7214385c1bbbdd2f3dc20efafeed0cf590e8d1199b9c6e44a F src/trigger.c 0bb986a5b96047fd597c6aac28588853df56064e576e6b81ba777ef2ccaac461 F src/update.c 2dd1b745acc9253df1b210ac69137c7a6b290e561d3f42da24418c4e807e889b F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 -F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e +F src/utf.c 7bc550af6f3ddd5f5dc82d092c41f728acb760c92e0b47f391963b01ae52569b F src/util.c 5d1a0134cf4240648d1c6bb5cc8efaca0ea2b5d5c840985aec7e947271f04375 F src/vacuum.c 25e407a6dc8b288fa4295b3d92fa9ce9318503e84df53cdf403a50fccbc1ba31 F src/vdbe.c 0f7c0fd2e821ef63a6cfaace9b46b053f99c96d9f03f1b497fe596d2ac7516e6 F src/vdbe.h c2549a215898a390de6669cfa32adba56f0d7e17ba5a7f7b14506d6fd5f0c36a -F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c -F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df -F src/vdbeaux.c 994c81eaed3b8305613ba707170ab55a9026ca0cfb8191989edc11ece6373519 +F src/vdbeInt.h af7d7e8291edd0b19f2cd698e60e4d4031078f9a2f2328ac8f0b7efb134f8a1d +F src/vdbeapi.c 53c7e26a2c0821a892b20eee2cde4656e31998212f3d515576c780dfaa45fd17 +F src/vdbeaux.c 0a6842a4c70cfa29946cfc73565ddf89f85980a54d07e01a36c886bc059c2906 F src/vdbeblob.c 255be187436da38b01f276c02e6a08103489bbe2a7c6c21537b7aecbe0e1f797 -F src/vdbemem.c 831a244831eaa45335f9ae276b50a7a82ee10d8c46c2c72492d4eb8c98d94d89 +F src/vdbemem.c df568ef0187e4be2788c35174f6d9b8566ab9475f9aff2d73907ed05aa5684b2 F src/vdbesort.c d0a3c7056c081703c8b6d91ad60f17da5e062a5c64bf568ed0fa1b5f4cae311f F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 -F src/vtab.c 5fb499d20494b7eecaadb7584634af9afcb374cb0524912b475fcb1712458a1b +F src/vtab.c 316cd48e9320660db3047cd306cd056e4361180cebb4d0f10a39244e10c11422 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c eb62325a874ff07aceb83fea2db85d36a49d472073b0df5a25782dafbfb7492c +F src/wal.c cc28bbf6c5229b6ff6e479d3db4182fffe53fb162131a1970481989800ca8d15 F src/wal.h 8d02ab8c2a93a941f5898eb3345bf711c1d3f8f86f4be8d5428fb6c074962d8a F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 7fb55836eb7fd07f0a0d8400c50619fc02cda1f46a617cfb003c2990f040193d +F src/where.c 461d41017d900d4248a268df96d2d30506c4dcc2257f4167c4f46072003ce2cf F src/whereInt.h a5d079c346a658b7a6e9e47bb943d021e02fa1e6aed3b964ca112112a4892192 F src/wherecode.c 5172d647798134e7c92536ddffe7e530c393d79b5dedd648b88faf2646c65baf F src/whereexpr.c 44f41ae554c7572e1de1485b3169b233ee04d464b2ee5881687ede3bf07cacfa @@ -1205,7 +1205,7 @@ F test/fts3conf.test c9cd45433b6787d48a43e84949aa2eb8b3b3d242bac7276731c1476290d F test/fts3corrupt.test 6732477c5ace050c5758a40a8b5706c8c0cccd416b9c558e0e15224805a40e57 F test/fts3corrupt2.test e318f0676e5e78d5a4b702637e2bb25265954c08a1b1e4aaf93c7880bb0c67d0 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f -F test/fts3corrupt4.test ab870772191b9e1d4f145eac340bc724e107d419bcfee0b197dc2026a2120c17 +F test/fts3corrupt4.test 28ce1af43ccfbb8960d755ad6a6473574f5294f004c0639cf4f5c080b9908c3c F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5 F test/fts3corrupt6.test f417c910254f32c0bc9ead7affa991a1d5aec35b3b32a183ffb05eea78289525 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf @@ -1288,7 +1288,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 3b8b39e3c0c88422c51ef0a93481d3d528fb370668344bf0ae4c87629c18b021 +F test/fuzzcheck.c 89b71d92b150a532e945e489d6e0721a4b15353c9255e079c198ed2a1958018b F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1300,14 +1300,14 @@ F test/fuzzdata8.db 4a53b6d077c6a5c23b609d8d3ac66996fa55ba3f8d02f9b6efdd0214a767 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c 3de49c7b33f5641b67edc2496328a49af029738e92c8499fafbf8618ad42f68d +F test/fuzzinvariants.c 057e910241d85aa4aaf75cef1a7adc45c632b173288d07d9dbbef4e6bda83d5a F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d F test/gencol1.test e169bdfa11c7ed5e9f322a98a7db3afe9e66235750b68c923efee8e1876b46ec F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/having.test a89236dd8d55aa50c4805f82ac9daf64d477a44d712d8209c118978d0ca21ec9 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test 18cae9140fa7f9a6f346e892a3fe3e31b2ca0be1494cd01b918adb74281016a6 +F test/hook.test 3481a68009fe143e3363fca922f6fc7a1e1f3776c51e42777f1a01b26ad2a9c8 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 8da7d52cd9722c82f33b0466ed915460cb03c23a38f18a9a2d3ff97da9a4a8c0 F test/ieee754.test b0945d12be7d255f3dfa18e2511b17ca37e0edd2b803231c52d05b86c04ab26e @@ -1646,7 +1646,7 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test 490bf9d0c7c9564fea318c46d49369f4690b825b584c9a544dbdccf61bc0babc +F test/shell1.test b02d628494fa284cdb2b7b2fecdadea96913796afd623f340a79d68f055dcb7e F test/shell2.test 01a01f76ed98088ce598794fbf5b359e148271541a8ddbf79d21cc353cc67a24 F test/shell3.test db1953a8e59d08e9240b7cc5948878e184f7eb2623591587f8fd1f1a5bd536d8 F test/shell4.test 522fdc628c55eff697b061504fb0a9e4e6dfc5d9087a633ab0f3dd11bcc4f807 @@ -2017,7 +2017,7 @@ F test/wal2recover2.test a7eece9892b125ef92343bba9e06edd5f6ad20c548dcbf79e3f2ab7 F test/wal2recover3.test 4a91689e165a38bc401736e6518188c2b0ff4fa1566d1810b8867536db128177 F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c F test/wal2rollback.test 23adc4a099b23f6aaea8b04fdca1c35861d887dd80f8be7da2d5273eb777e428 -F test/wal2savepoint.test 3793a0ae97011fca358f79775f5d7d9f85da75c8e67686e2e19713da0cb0d99c +F test/wal2savepoint.test c2410acac73bfa63827f9346d060205080b747f0414ff27273d417dadfea28d0 F test/wal2simple.test 320a08927f307d0ead26fa3fcef4e3f64279f49be9504f918cb33294f75aeec8 F test/wal2snapshot.test f6c3945bea572fd8745774e95b8dca1e5832c696e2251bb0db33391ee567fe60 F test/wal3.test 5de023bb862fd1eb9d2ad26fa8d9c43abb5370582e5b08b2ae0d6f93661bc310 @@ -2137,7 +2137,7 @@ F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e04534bbf F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 F tool/build-all-msvc.bat c817b716e0edeecaf265a6775b63e5f45c34a6544f1d4114a222701ed5ac79ab x -F tool/build-shell.sh f193b5e3eb4afcb4abbf96bf1475be6cfb74763ee2e50c82bc7ca105e8a136c5 +F tool/build-shell.sh 369c4b171cc877ad974fef691e4da782b4c1e99fe8f4361316c735f64d49280f F tool/buildtclext.tcl b64d250517b148e644d26fcbc097851867a0df52cd4bafe9bcd94b8421e1428a F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 @@ -2187,19 +2187,19 @@ F tool/offsets.c 8ed2b344d33f06e71366a9b93ccedaa38c096cc1dbd4c3c26ad08c611528584 F tool/omittest-msvc.tcl d6b8f501ac1d7798c4126065030f89812379012cad98a1735d6d7221492abc08 F tool/omittest.tcl 5ca5e4e01716d5f35b48b00fd351d929f01fbb98169a5a3cd00baf3d2e2019a9 F tool/opcodesum.tcl 740ed206ba8c5040018988129abbf3089a0ccf4a -F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b +F tool/pagesig.c f98909b4168d9cac11a2de7f031adea0e2f3131faa7515a72807c03ec58eafeb F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82d9a F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 -F tool/showdb.c 0f74b54cc67076c76cba9b2b7f54d3e05b78d130c70ffc394eb84c5b41bab017 +F tool/showdb.c 81b04bfaa9a63665f75945947323aa68b820570aa156b1574f440fc8276092c6 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9cc5e66d4ebbf2d194f39db2527ece92077e86ae627ddd233ee48e16e8142564 F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809 F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c2a1 F tool/showwal.c 11eca547980a066b081f512636151233350ac679f29ecf4ebfce7f4530230b3d F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe -F tool/spaceanal.tcl 3568b2b15b83dcaf789e787a4db0980da81eb6fa0e1e00783e4a927fdf584550 +F tool/spaceanal.tcl 1f83962090a6b60e1d7bf92495d643e622bef9fe82ea3f2d22350dcbce9a12d0 F tool/speed-check.sh e8d20cc2eb9c85ec1ba562226de144435456dcdff4ee618de49603c6958f6116 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e @@ -2208,8 +2208,9 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 -F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f +F tool/sqldiff.c 2a0987d183027c795ced13d6749061c1d2f38e24eddb428f56fa64c3a8f51e4b +F tool/sqlite3-rsync.c 187b262035c1159b047dbfa1959c168b87b5a153b63465e8c8bd1b54fabf4460 +F tool/sqlite3_analyzer.c.in 348ba349bbdc93c9866439f9f935d7284866a2a4e6898bc906ae1204ade56918 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c 41c586dee84d0b190ad13e0282ed83d4a65ec9fefde9adf4943efdf6558eea7f @@ -2249,8 +2250,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cf8f155280100c1a5b79b118875a93686a554d8be55114277f4f7b20cfd7835a f3e3ad6945647c88f41748b5e8ce71c06d6e77ac67c607898af63ea14a91910b -R 330a9c8704cb143ebf90334a590b8710 -U drh -Z 7d17e0ab995cc46d1cd21869a2b767f1 +P 4a3386ccb80e974538a96b830f0bb3c7802f5ef5a4f2f26d01fa0732fccae9e0 59fba0288d8fb68f9b6e4936c09ae9169bc002bfe6548354504563a674000c20 +R fa192ced6f57fe693d9e42d297c534f9 +U dan +Z 83fa2b380d84dcc6771d1124ae49ce40 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 58cb2cb963..3d7215319a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a3386ccb80e974538a96b830f0bb3c7802f5ef5a4f2f26d01fa0732fccae9e0 +fd6ff55141be82ba36513ba6273a60c6d8b3c496ed7890ab47cccb05bec7f7c2 diff --git a/src/dbpage.c b/src/dbpage.c index 73c31f0dab..42b24f9b8d 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -28,7 +28,13 @@ ** ** The data field of sqlite_dbpage table can be updated. The new ** value must be a BLOB which is the correct page size, otherwise the -** update fails. Rows may not be deleted or inserted. +** update fails. INSERT operations also work, and operate as if they +** where REPLACE. The size of the database can be extended by INSERT-ing +** new pages on the end. +** +** Rows may not be deleted. However, doing an INSERT to page number N +** with NULL page data causes the N-th page and all subsequent pages to be +** deleted and the database to be truncated. */ #include "sqliteInt.h" /* Requires access to internal data structures */ @@ -51,6 +57,8 @@ struct DbpageCursor { struct DbpageTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database */ + int nTrunc; /* Entries in aTrunc[] */ + Pgno *aTrunc; /* Truncation size for each database */ }; /* Columns */ @@ -59,7 +67,6 @@ struct DbpageTable { #define DBPAGE_COLUMN_SCHEMA 2 - /* ** Connect to or create a dbpagevfs virtual table. */ @@ -100,6 +107,8 @@ static int dbpageConnect( ** Disconnect from or destroy a dbpagevfs virtual table. */ static int dbpageDisconnect(sqlite3_vtab *pVtab){ + DbpageTable *pTab = (DbpageTable *)pVtab; + sqlite3_free(pTab->aTrunc); sqlite3_free(pVtab); return SQLITE_OK; } @@ -321,11 +330,11 @@ static int dbpageUpdate( DbPage *pDbPage = 0; int rc = SQLITE_OK; char *zErr = 0; - const char *zSchema; int iDb; Btree *pBt; Pager *pPager; int szPage; + int isInsert; (void)pRowid; if( pTab->db->flags & SQLITE_Defensive ){ @@ -336,21 +345,29 @@ static int dbpageUpdate( zErr = "cannot delete"; goto update_fail; } - pgno = sqlite3_value_int(argv[0]); - if( sqlite3_value_type(argv[0])==SQLITE_NULL - || (Pgno)sqlite3_value_int(argv[1])!=pgno - ){ - zErr = "cannot insert"; - goto update_fail; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + pgno = (Pgno)sqlite3_value_int(argv[2]); + isInsert = 1; + }else{ + pgno = sqlite3_value_int(argv[0]); + if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ + zErr = "cannot insert"; + goto update_fail; + } + isInsert = 0; } - zSchema = (const char*)sqlite3_value_text(argv[4]); - iDb = ALWAYS(zSchema) ? sqlite3FindDbName(pTab->db, zSchema) : -1; - if( NEVER(iDb<0) ){ - zErr = "no such schema"; - goto update_fail; + if( sqlite3_value_type(argv[4])==SQLITE_NULL ){ + iDb = 0; + }else{ + const char *zSchema = (const char*)sqlite3_value_text(argv[4]); + iDb = sqlite3FindDbName(pTab->db, zSchema); + if( iDb<0 ){ + zErr = "no such schema"; + goto update_fail; + } } pBt = pTab->db->aDb[iDb].pBt; - if( NEVER(pgno<1) || NEVER(pBt==0) || NEVER(pgno>sqlite3BtreeLastPage(pBt)) ){ + if( pgno<1 || NEVER(pBt==0) ){ zErr = "bad page number"; goto update_fail; } @@ -358,18 +375,31 @@ static int dbpageUpdate( if( sqlite3_value_type(argv[3])!=SQLITE_BLOB || sqlite3_value_bytes(argv[3])!=szPage ){ - zErr = "bad page value"; - goto update_fail; + if( sqlite3_value_type(argv[3])==SQLITE_NULL && isInsert ){ + if( iDb>=pTab->nTrunc ){ + testcase( pTab->aTrunc!=0 ); + pTab->aTrunc = sqlite3_realloc(pTab->aTrunc, (iDb+1)*sizeof(Pgno)); + if( pTab->aTrunc ){ + int j; + for(j=pTab->nTrunc; jaTrunc[j] = 0; + pTab->nTrunc = iDb+1; + }else{ + return SQLITE_NOMEM; + } + } + pTab->aTrunc[iDb] = pgno; + }else{ + zErr = "bad page value"; + goto update_fail; + } } pPager = sqlite3BtreePager(pBt); rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ const void *pData = sqlite3_value_blob(argv[3]); - assert( pData!=0 || pTab->db->mallocFailed ); - if( pData - && (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK - ){ - memcpy(sqlite3PagerGetData(pDbPage), pData, szPage); + if( (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK && pData ){ + unsigned char *aPage = sqlite3PagerGetData(pDbPage); + memcpy(aPage, pData, szPage); } } sqlite3PagerUnref(pDbPage); @@ -393,6 +423,26 @@ static int dbpageBegin(sqlite3_vtab *pVtab){ Btree *pBt = db->aDb[i].pBt; if( pBt ) (void)sqlite3BtreeBeginTrans(pBt, 1, 0); } + if( pTab->nTrunc>0 ){ + memset(pTab->aTrunc, 0, sizeof(pTab->aTrunc[0])*pTab->nTrunc); + } + return SQLITE_OK; +} + +/* Invoke sqlite3PagerTruncate() as necessary, just prior to COMMIT +*/ +static int dbpageSync(sqlite3_vtab *pVtab){ + int iDb; + DbpageTable *pTab = (DbpageTable *)pVtab; + + for(iDb=0; iDbnTrunc; iDb++){ + if( pTab->aTrunc[iDb]>0 ){ + Btree *pBt = pTab->db->aDb[iDb].pBt; + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerTruncateImage(pPager, pTab->aTrunc[iDb]); + pTab->aTrunc[iDb] = 0; + } + } return SQLITE_OK; } @@ -417,7 +467,7 @@ int sqlite3DbpageRegister(sqlite3 *db){ dbpageRowid, /* xRowid - read data */ dbpageUpdate, /* xUpdate */ dbpageBegin, /* xBegin */ - 0, /* xSync */ + dbpageSync, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindMethod */ diff --git a/src/delete.c b/src/delete.c index 4cdb3946e3..8fac7c2f32 100644 --- a/src/delete.c +++ b/src/delete.c @@ -75,6 +75,7 @@ void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char *zColName){ ** is for a top-level SQL statement. */ static int vtabIsReadOnly(Parse *pParse, Table *pTab){ + assert( IsVirtual(pTab) ); if( sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ){ return 1; } diff --git a/src/main.c b/src/main.c index bd68fa74d7..1c112be729 100644 --- a/src/main.c +++ b/src/main.c @@ -3482,6 +3482,7 @@ static int openDatabase( if( ((1<<(flags&7)) & 0x46)==0 ){ rc = SQLITE_MISUSE_BKPT; /* IMP: R-18321-05872 */ }else{ + if( zFilename==0 ) zFilename = ":memory:"; rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); } if( rc!=SQLITE_OK ){ diff --git a/src/mutex.c b/src/mutex.c index 381ffbdfd5..62e09cb4fa 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -347,15 +347,28 @@ void sqlite3_mutex_leave(sqlite3_mutex *p){ /* ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are ** intended for use inside assert() statements. +** +** Because these routines raise false-positive alerts in TSAN, disable +** them (make them always return 1) when compiling with TSAN. */ int sqlite3_mutex_held(sqlite3_mutex *p){ +# if defined(__has_feature) +# if __has_feature(thread_sanitizer) + p = 0; +# endif +# endif assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } int sqlite3_mutex_notheld(sqlite3_mutex *p){ +# if defined(__has_feature) +# if __has_feature(thread_sanitizer) + p = 0; +# endif +# endif assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } -#endif +#endif /* NDEBUG */ #endif /* !defined(SQLITE_MUTEX_OMIT) */ diff --git a/src/os_unix.c b/src/os_unix.c index 581905d863..ff7c5c83c0 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4191,7 +4191,7 @@ static void setDeviceCharacteristics(unixFile *pFd){ static void setDeviceCharacteristics(unixFile *pFile){ if( pFile->sectorSize == 0 ){ struct statvfs fsInfo; - + /* Set defaults for non-supported filesystems */ pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; pFile->deviceCharacteristics = 0; @@ -4231,7 +4231,7 @@ static void setDeviceCharacteristics(unixFile *pFile){ pFile->sectorSize = fsInfo.f_bsize; pFile->deviceCharacteristics = /* full bitset of atomics from max sector size and smaller */ - ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + (((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2) | SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind ** so it is ordered */ 0; @@ -4239,7 +4239,7 @@ static void setDeviceCharacteristics(unixFile *pFile){ pFile->sectorSize = fsInfo.f_bsize; pFile->deviceCharacteristics = /* full bitset of atomics from max sector size and smaller */ - ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + (((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2) | SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind ** so it is ordered */ 0; diff --git a/src/prepare.c b/src/prepare.c index df9c98f743..7aa1e1a022 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -1007,12 +1007,24 @@ static int sqlite3Prepare16( if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } + + /* Make sure nBytes is non-negative and correct. It should be the + ** number of bytes until the end of the input buffer or until the first + ** U+0000 character. If the input nBytes is odd, convert it into + ** an even number. If the input nBytes is negative, then the input + ** must be terminated by at least one U+0000 character */ if( nBytes>=0 ){ int sz; const char *z = (const char*)zSql; for(sz=0; szmutex); zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); if( zSql8 ){ @@ -1026,7 +1038,7 @@ static int sqlite3Prepare16( ** the same number of characters into the UTF-16 string. */ int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8)); - *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed); + *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, nBytes, chars_parsed); } sqlite3DbFree(db, zSql8); rc = sqlite3ApiExit(db, rc); diff --git a/src/resolve.c b/src/resolve.c index b755cc8646..8e8da66910 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -486,7 +486,7 @@ static int lookupName( */ if( cntTab==0 || (cntTab==1 - && ALWAYS(pMatch!=0) + && pMatch!=0 && ALWAYS(pMatch->pSTab!=0) && (pMatch->pSTab->tabFlags & TF_Ephemeral)!=0 && (pTab->tabFlags & TF_Ephemeral)==0) @@ -1119,8 +1119,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ /* Resolve function names */ case TK_FUNCTION: { - ExprList *pList = pExpr->x.pList; /* The argument list */ - int n = pList ? pList->nExpr : 0; /* Number of arguments */ + ExprList *pList; /* The argument list */ + int n; /* Number of arguments */ int no_such_func = 0; /* True if no such function exists */ int wrong_num_args = 0; /* True if wrong number of arguments */ int is_agg = 0; /* True if is an aggregate function */ @@ -1133,6 +1133,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif assert( !ExprHasProperty(pExpr, EP_xIsSelect|EP_IntValue) ); assert( pExpr->pLeft==0 || pExpr->pLeft->op==TK_ORDER ); + pList = pExpr->x.pList; + n = pList ? pList->nExpr : 0; zId = pExpr->u.zToken; pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); if( pDef==0 ){ diff --git a/src/shell.c.in b/src/shell.c.in index b8f9327948..d1fc6aa748 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -192,8 +192,6 @@ typedef unsigned char u8; # ifndef strdup # define strdup _strdup # endif -# undef popen -# define popen _popen # undef pclose # define pclose _pclose # endif @@ -237,6 +235,9 @@ extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); #endif +INCLUDE ../ext/misc/sqlite3_stdio.h +INCLUDE ../ext/misc/sqlite3_stdio.c + /* Use console I/O package as a direct INCLUDE. */ #define SQLITE_INTERNAL_LINKAGE static @@ -248,57 +249,9 @@ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); # define SQLITE_CIO_NO_SETMODE # define SQLITE_CIO_NO_FLUSH #endif -INCLUDE ../ext/consio/console_io.h -INCLUDE ../ext/consio/console_io.c -#ifndef SQLITE_SHELL_FIDDLE - -/* From here onward, fgets() is redirected to the console_io library. */ -# define fgets(b,n,f) fGetsUtf8(b,n,f) -/* - * Define macros for emitting output text in various ways: - * sputz(s, z) => emit 0-terminated string z to given stream s - * sputf(s, f, ...) => emit varargs per format f to given stream s - * oputz(z) => emit 0-terminated string z to default stream - * oputf(f, ...) => emit varargs per format f to default stream - * eputz(z) => emit 0-terminated string z to error stream - * eputf(f, ...) => emit varargs per format f to error stream - * oputb(b, n) => emit char buffer b[0..n-1] to default stream - * - * Note that the default stream is whatever has been last set via: - * setOutputStream(FILE *pf) - * This is normally the stream that CLI normal output goes to. - * For the stand-alone CLI, it is stdout with no .output redirect. - * - * The ?putz(z) forms are required for the Fiddle builds for string literal - * output, in aid of enforcing format string to argument correspondence. - */ -# define sputz(s,z) fPutsUtf8(z,s) -# define sputf fPrintfUtf8 -# define oputz(z) oPutsUtf8(z) -# define oputf oPrintfUtf8 -# define eputz(z) ePutsUtf8(z) -# define eputf ePrintfUtf8 -# define oputb(buf,na) oPutbUtf8(buf,na) -# define fflush(s) fFlushBuffer(s); - -#else -/* For Fiddle, all console handling and emit redirection is omitted. */ -/* These next 3 macros are for emitting formatted output. When complaints - * from the WASM build are issued for non-formatted output, when a mere - * string literal is to be emitted, the ?putz(z) forms should be used. - * (This permits compile-time checking of format string / argument mismatch.) - */ -# define oputf(fmt, ...) printf(fmt,__VA_ARGS__) -# define eputf(fmt, ...) fprintf(stderr,fmt,__VA_ARGS__) -# define sputf(fp,fmt, ...) fprintf(fp,fmt,__VA_ARGS__) -/* These next 3 macros are for emitting simple string literals. */ -# define oputz(z) fputs(z,stdout) -# define eputz(z) fputs(z,stderr) -# define sputz(fp,z) fputs(z,fp) -# define oputb(buf,na) fwrite(buf,1,na,stdout) -# undef fflush -#endif +#define eputz(z) sqlite3_fputs(z,stderr) +#define sputz(fp,z) sqlite3_fputs(z,fp) /* True if the timer is enabled */ static int enableTimer = 0; @@ -344,6 +297,7 @@ struct rusage { #define getrusage(A,B) memset(B,0,sizeof(*B)) #endif + /* Saved resource information for the beginning of an operation */ static struct rusage sBegin; /* CPU time at start */ static sqlite3_int64 iBegin; /* Wall-clock time at start */ @@ -367,12 +321,12 @@ static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ /* ** Print the timing results. */ -static void endTimer(void){ +static void endTimer(FILE *out){ if( enableTimer ){ sqlite3_int64 iEnd = timeOfDay(); struct rusage sEnd; getrusage(RUSAGE_SELF, &sEnd); - sputf(stdout, "Run Time: real %.3f user %f sys %f\n", + sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", (iEnd - iBegin)*0.001, timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); @@ -380,7 +334,7 @@ static void endTimer(void){ } #define BEGIN_TIMER beginTimer() -#define END_TIMER endTimer() +#define END_TIMER(X) endTimer(X) #define HAS_TIMER 1 #elif (defined(_WIN32) || defined(WIN32)) @@ -446,12 +400,12 @@ static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ /* ** Print the timing results. */ -static void endTimer(void){ +static void endTimer(FILE *out){ if( enableTimer && getProcessTimesAddr){ FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; sqlite3_int64 ftWallEnd = timeOfDay(); getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); - sputf(stdout, "Run Time: real %.3f user %f sys %f\n", + sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", (ftWallEnd - ftWallBegin)*0.001, timeDiff(&ftUserBegin, &ftUserEnd), timeDiff(&ftKernelBegin, &ftKernelEnd)); @@ -459,12 +413,12 @@ static void endTimer(void){ } #define BEGIN_TIMER beginTimer() -#define END_TIMER endTimer() +#define END_TIMER(X) endTimer(X) #define HAS_TIMER hasTimer() #else #define BEGIN_TIMER -#define END_TIMER +#define END_TIMER(X) /*no-op*/ #define HAS_TIMER 0 #endif @@ -665,37 +619,215 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); - sputf(iotrace, "%s", z); + sqlite3_fprintf(iotrace, "%s", z); sqlite3_free(z); } #endif +/* Lookup table to estimate the number of columns consumed by a Unicode +** character. +*/ +static const struct { + unsigned char w; /* Width of the character in columns */ + int iFirst; /* First character in a span having this width */ +} aUWidth[] = { + /* {0, 0x00000}, {1, 0x00020}, {0, 0x0007f}, {1, 0x000a0}, */ + {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, + {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, + {0, 0x005c1}, {1, 0x005c3}, {0, 0x005c4}, {1, 0x005c6}, {0, 0x005c7}, + {1, 0x005c8}, {0, 0x00600}, {1, 0x00604}, {0, 0x00610}, {1, 0x00616}, + {0, 0x0064b}, {1, 0x0065f}, {0, 0x00670}, {1, 0x00671}, {0, 0x006d6}, + {1, 0x006e5}, {0, 0x006e7}, {1, 0x006e9}, {0, 0x006ea}, {1, 0x006ee}, + {0, 0x0070f}, {1, 0x00710}, {0, 0x00711}, {1, 0x00712}, {0, 0x00730}, + {1, 0x0074b}, {0, 0x007a6}, {1, 0x007b1}, {0, 0x007eb}, {1, 0x007f4}, + {0, 0x00901}, {1, 0x00903}, {0, 0x0093c}, {1, 0x0093d}, {0, 0x00941}, + {1, 0x00949}, {0, 0x0094d}, {1, 0x0094e}, {0, 0x00951}, {1, 0x00955}, + {0, 0x00962}, {1, 0x00964}, {0, 0x00981}, {1, 0x00982}, {0, 0x009bc}, + {1, 0x009bd}, {0, 0x009c1}, {1, 0x009c5}, {0, 0x009cd}, {1, 0x009ce}, + {0, 0x009e2}, {1, 0x009e4}, {0, 0x00a01}, {1, 0x00a03}, {0, 0x00a3c}, + {1, 0x00a3d}, {0, 0x00a41}, {1, 0x00a43}, {0, 0x00a47}, {1, 0x00a49}, + {0, 0x00a4b}, {1, 0x00a4e}, {0, 0x00a70}, {1, 0x00a72}, {0, 0x00a81}, + {1, 0x00a83}, {0, 0x00abc}, {1, 0x00abd}, {0, 0x00ac1}, {1, 0x00ac6}, + {0, 0x00ac7}, {1, 0x00ac9}, {0, 0x00acd}, {1, 0x00ace}, {0, 0x00ae2}, + {1, 0x00ae4}, {0, 0x00b01}, {1, 0x00b02}, {0, 0x00b3c}, {1, 0x00b3d}, + {0, 0x00b3f}, {1, 0x00b40}, {0, 0x00b41}, {1, 0x00b44}, {0, 0x00b4d}, + {1, 0x00b4e}, {0, 0x00b56}, {1, 0x00b57}, {0, 0x00b82}, {1, 0x00b83}, + {0, 0x00bc0}, {1, 0x00bc1}, {0, 0x00bcd}, {1, 0x00bce}, {0, 0x00c3e}, + {1, 0x00c41}, {0, 0x00c46}, {1, 0x00c49}, {0, 0x00c4a}, {1, 0x00c4e}, + {0, 0x00c55}, {1, 0x00c57}, {0, 0x00cbc}, {1, 0x00cbd}, {0, 0x00cbf}, + {1, 0x00cc0}, {0, 0x00cc6}, {1, 0x00cc7}, {0, 0x00ccc}, {1, 0x00cce}, + {0, 0x00ce2}, {1, 0x00ce4}, {0, 0x00d41}, {1, 0x00d44}, {0, 0x00d4d}, + {1, 0x00d4e}, {0, 0x00dca}, {1, 0x00dcb}, {0, 0x00dd2}, {1, 0x00dd5}, + {0, 0x00dd6}, {1, 0x00dd7}, {0, 0x00e31}, {1, 0x00e32}, {0, 0x00e34}, + {1, 0x00e3b}, {0, 0x00e47}, {1, 0x00e4f}, {0, 0x00eb1}, {1, 0x00eb2}, + {0, 0x00eb4}, {1, 0x00eba}, {0, 0x00ebb}, {1, 0x00ebd}, {0, 0x00ec8}, + {1, 0x00ece}, {0, 0x00f18}, {1, 0x00f1a}, {0, 0x00f35}, {1, 0x00f36}, + {0, 0x00f37}, {1, 0x00f38}, {0, 0x00f39}, {1, 0x00f3a}, {0, 0x00f71}, + {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, + {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, + {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, + {0, 0x01036}, {1, 0x01038}, {0, 0x01039}, {1, 0x0103a}, {0, 0x01058}, + {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, + {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, + {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, + {1, 0x017b6}, {0, 0x017b7}, {1, 0x017be}, {0, 0x017c6}, {1, 0x017c7}, + {0, 0x017c9}, {1, 0x017d4}, {0, 0x017dd}, {1, 0x017de}, {0, 0x0180b}, + {1, 0x0180e}, {0, 0x018a9}, {1, 0x018aa}, {0, 0x01920}, {1, 0x01923}, + {0, 0x01927}, {1, 0x01929}, {0, 0x01932}, {1, 0x01933}, {0, 0x01939}, + {1, 0x0193c}, {0, 0x01a17}, {1, 0x01a19}, {0, 0x01b00}, {1, 0x01b04}, + {0, 0x01b34}, {1, 0x01b35}, {0, 0x01b36}, {1, 0x01b3b}, {0, 0x01b3c}, + {1, 0x01b3d}, {0, 0x01b42}, {1, 0x01b43}, {0, 0x01b6b}, {1, 0x01b74}, + {0, 0x01dc0}, {1, 0x01dcb}, {0, 0x01dfe}, {1, 0x01e00}, {0, 0x0200b}, + {1, 0x02010}, {0, 0x0202a}, {1, 0x0202f}, {0, 0x02060}, {1, 0x02064}, + {0, 0x0206a}, {1, 0x02070}, {0, 0x020d0}, {1, 0x020f0}, {2, 0x02329}, + {1, 0x0232b}, {2, 0x02e80}, {0, 0x0302a}, {2, 0x03030}, {1, 0x0303f}, + {2, 0x03040}, {0, 0x03099}, {2, 0x0309b}, {1, 0x0a4d0}, {0, 0x0a806}, + {1, 0x0a807}, {0, 0x0a80b}, {1, 0x0a80c}, {0, 0x0a825}, {1, 0x0a827}, + {2, 0x0ac00}, {1, 0x0d7a4}, {2, 0x0f900}, {1, 0x0fb00}, {0, 0x0fb1e}, + {1, 0x0fb1f}, {0, 0x0fe00}, {2, 0x0fe10}, {1, 0x0fe1a}, {0, 0x0fe20}, + {1, 0x0fe24}, {2, 0x0fe30}, {1, 0x0fe70}, {0, 0x0feff}, {2, 0x0ff00}, + {1, 0x0ff61}, {2, 0x0ffe0}, {1, 0x0ffe7}, {0, 0x0fff9}, {1, 0x0fffc}, + {0, 0x10a01}, {1, 0x10a04}, {0, 0x10a05}, {1, 0x10a07}, {0, 0x10a0c}, + {1, 0x10a10}, {0, 0x10a38}, {1, 0x10a3b}, {0, 0x10a3f}, {1, 0x10a40}, + {0, 0x1d167}, {1, 0x1d16a}, {0, 0x1d173}, {1, 0x1d183}, {0, 0x1d185}, + {1, 0x1d18c}, {0, 0x1d1aa}, {1, 0x1d1ae}, {0, 0x1d242}, {1, 0x1d245}, + {2, 0x20000}, {1, 0x2fffe}, {2, 0x30000}, {1, 0x3fffe}, {0, 0xe0001}, + {1, 0xe0002}, {0, 0xe0020}, {1, 0xe0080}, {0, 0xe0100}, {1, 0xe01f0} +}; + /* -** Output string zUtf to Out stream as w characters. If w is negative, +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different display devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. +*/ +int cli_wcwidth(int c){ + int iFirst, iLast; + + /* Fast path for common characters */ + if( c<0x20 ) return 0; + if( c<0x7f ) return 1; + if( c<0xa0 ) return 0; + if( c<=0x300 ) return 1; + + /* The general case */ + iFirst = 0; + iLast = sizeof(aUWidth)/sizeof(aUWidth[0]) - 1; + while( iFirst c ){ + iLast = iMid - 1; + }else{ + return aUWidth[iMid].w; + } + } + if( aUWidth[iLast].iFirst > c ) return aUWidth[iFirst].w; + return aUWidth[iLast].w; +} + +/* +** Compute the value and length of a multi-byte UTF-8 character that +** begins at z[0]. Return the length. Write the Unicode value into *pU. +** +** This routine only works for *multi-byte* UTF-8 characters. +*/ +static int decodeUtf8(const unsigned char *z, int *pU){ + if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); + return 2; + } + if( (z[0] & 0xf0)==0xe0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x0f)<<12) | ((z[1] & 0x3f)<<6) | (z[2] & 0x3f); + return 3; + } + if( (z[0] & 0xf8)==0xf0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 + && (z[3] & 0xc0)==0x80 + ){ + *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 + | (z[4] & 0x3f); + return 4; + } + *pU = 0; + return 1; +} + + +#if 0 /* NOT USED */ +/* +** Return the width, in display columns, of a UTF-8 string. +** +** Each normal character counts as 1. Zero-width characters count +** as zero, and double-width characters count as 2. +*/ +int cli_wcswidth(const char *z){ + const unsigned char *a = (const unsigned char*)z; + int n = 0; + int i = 0; + unsigned char c; + while( (c = a[i])!=0 ){ + if( c>=0xc0 ){ + int u; + int len = decodeUtf8(&a[i], &u); + i += len; + n += cli_wcwidth(u); + }else if( c>=' ' ){ + n++; + i++; + }else{ + i++; + } + } + return n; +} +#endif + +/* +** Output string zUtf to stdout as w characters. If w is negative, ** then right-justify the text. W is the width in UTF-8 characters, not ** in bytes. This is different from the %*.*s specification in printf ** since with %*.*s the width is measured in bytes, not characters. +** +** Take into account zero-width and double-width Unicode characters. +** In other words, a zero-width character does not count toward the +** the w limit. A double-width character counts as two. */ -static void utf8_width_print(int w, const char *zUtf){ - int i; - int n; +static void utf8_width_print(FILE *out, int w, const char *zUtf){ + const unsigned char *a = (const unsigned char*)zUtf; + unsigned char c; + int i = 0; + int n = 0; int aw = w<0 ? -w : w; if( zUtf==0 ) zUtf = ""; - for(i=n=0; zUtf[i]; i++){ - if( (zUtf[i]&0xc0)!=0x80 ){ - n++; - if( n==aw ){ - do{ i++; }while( (zUtf[i]&0xc0)==0x80 ); + while( (c = a[i])!=0 ){ + if( (c&0xc0)==0xc0 ){ + int u; + int len = decodeUtf8(a+i, &u); + int x = cli_wcwidth(u); + if( x+n>aw ){ break; } + i += len; + n += x; + }else if( n>=aw ){ + break; + }else{ + n++; + i++; } } if( n>=aw ){ - oputf("%.*s", i, zUtf); + sqlite3_fprintf(out, "%.*s", i, zUtf); }else if( w<0 ){ - oputf("%*s%s", aw-n, "", zUtf); + sqlite3_fprintf(out, "%*s%s", aw-n, "", zUtf); }else{ - oputf("%s%*s", zUtf, aw-n, ""); + sqlite3_fprintf(out, "%s%*s", zUtf, aw-n, ""); } } @@ -761,7 +893,7 @@ static FILE * openChrSource(const char *zFile){ /* On Windows, open first, then check the stream nature. This order ** is necessary because _stat() and sibs, when checking a named pipe, ** effectively break the pipe as its supplier sees it. */ - FILE *rv = fopen(zFile, "rb"); + FILE *rv = sqlite3_fopen(zFile, "rb"); if( rv==0 ) return 0; if( _fstat64(_fileno(rv), &x) != 0 || !STAT_CHR_SRC(x.st_mode)){ @@ -775,7 +907,7 @@ static FILE * openChrSource(const char *zFile){ # define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) if( rc!=0 ) return 0; if( STAT_CHR_SRC(x.st_mode) ){ - return fopen(zFile, "rb"); + return sqlite3_fopen(zFile, "rb"); }else{ return 0; } @@ -802,7 +934,7 @@ static char *local_getline(char *zLine, FILE *in){ zLine = realloc(zLine, nLine); shell_check_oom(zLine); } - if( fgets(&zLine[n], nLine - n, in)==0 ){ + if( sqlite3_fgets(&zLine[n], nLine - n, in)==0 ){ if( n==0 ){ free(zLine); return 0; @@ -1209,6 +1341,7 @@ INCLUDE test_windirent.c INCLUDE ../ext/misc/memtrace.c INCLUDE ../ext/misc/pcachetrace.c INCLUDE ../ext/misc/shathree.c +INCLUDE ../ext/misc/sha1.c INCLUDE ../ext/misc/uint.c INCLUDE ../ext/misc/decimal.c INCLUDE ../ext/misc/percentile.c @@ -1236,6 +1369,7 @@ INCLUDE ../ext/expert/sqlite3expert.c INCLUDE ../ext/intck/sqlite3intck.h INCLUDE ../ext/intck/sqlite3intck.c INCLUDE ../ext/misc/stmtrand.c +INCLUDE ../ext/misc/vfstrace.c #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) #define SQLITE_SHELL_HAVE_RECOVER 1 @@ -1462,6 +1596,7 @@ static ShellState shellState; #define MODE_Count 17 /* Output only a count of the rows of output */ #define MODE_Off 18 /* No query output shown */ #define MODE_ScanExp 19 /* Like MODE_Explain, but for ".scanstats vm" */ +#define MODE_Www 20 /* Full web-page output */ static const char *modeDescr[] = { "line", @@ -1482,7 +1617,9 @@ static const char *modeDescr[] = { "table", "box", "count", - "off" + "off", + "scanexp", + "www", }; /* @@ -1494,7 +1631,13 @@ static const char *modeDescr[] = { #define SEP_Tab "\t" #define SEP_Space " " #define SEP_Comma "," -#define SEP_CrLf "\r\n" +#ifdef SQLITE_U8TEXT_ONLY + /* With the SQLITE_U8TEXT_ONLY option, the output will always be in + ** text mode. The \r will be inserted automatically. */ +# define SEP_CrLf "\n" +#else +# define SEP_CrLf "\r\n" +#endif #define SEP_Unit "\x1F" #define SEP_Record "\x1E" @@ -1510,7 +1653,7 @@ static const char *modeDescr[] = { static void shellLog(void *pArg, int iErrCode, const char *zMsg){ ShellState *p = (ShellState*)pArg; if( p->pLog==0 ) return; - sputf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + sqlite3_fprintf(p->pLog, "(%d) %s\n", iErrCode, zMsg); fflush(p->pLog); } @@ -1525,9 +1668,9 @@ static void shellPutsFunc( int nVal, sqlite3_value **apVal ){ - /* Unused: (ShellState*)sqlite3_user_data(pCtx); */ + ShellState *p = (ShellState*)sqlite3_user_data(pCtx); (void)nVal; - oputf("%s\n", sqlite3_value_text(apVal[0])); + sqlite3_fprintf(p->out, "%s\n", sqlite3_value_text(apVal[0])); sqlite3_result_value(pCtx, apVal[0]); } @@ -1546,7 +1689,7 @@ static void failIfSafeMode( va_start(ap, zErrMsg); zMsg = sqlite3_vmprintf(zErrMsg, ap); va_end(ap); - eputf("line %d: %s\n", p->lineno, zMsg); + sqlite3_fprintf(stderr, "line %d: %s\n", p->lineno, zMsg); exit(1); } } @@ -1613,7 +1756,7 @@ static void editFunc( bBin = sqlite3_value_type(argv[0])==SQLITE_BLOB; /* When writing the file to be edited, do \n to \r\n conversions on systems ** that want \r\n line endings */ - f = fopen(zTempFile, bBin ? "wb" : "w"); + f = sqlite3_fopen(zTempFile, bBin ? "wb" : "w"); if( f==0 ){ sqlite3_result_error(context, "edit() cannot open temp file", -1); goto edit_func_end; @@ -1644,7 +1787,7 @@ static void editFunc( sqlite3_result_error(context, "EDITOR returned non-zero", -1); goto edit_func_end; } - f = fopen(zTempFile, "rb"); + f = sqlite3_fopen(zTempFile, "rb"); if( f==0 ){ sqlite3_result_error(context, "edit() cannot reopen temp file after edit", -1); @@ -1714,7 +1857,7 @@ static void outputModePop(ShellState *p){ /* ** Output the given string as a hex-encoded blob (eg. X'1234' ) */ -static void output_hex_blob(const void *pBlob, int nBlob){ +static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ int i; unsigned char *aBlob = (unsigned char*)pBlob; @@ -1731,7 +1874,7 @@ static void output_hex_blob(const void *pBlob, int nBlob){ } zStr[i*2] = '\0'; - oputf("X'%s'", zStr); + sqlite3_fprintf(out, "X'%s'", zStr); sqlite3_free(zStr); } @@ -1761,28 +1904,25 @@ static const char *unused_string( ** ** See also: output_quoted_escaped_string() */ -static void output_quoted_string(const char *z){ +static void output_quoted_string(FILE *out, const char *z){ int i; char c; -#ifndef SQLITE_SHELL_FIDDLE - FILE *pfO = setOutputStream(invalidFileStream); - setBinaryMode(pfO, 1); -#endif + sqlite3_fsetmode(out, _O_BINARY); if( z==0 ) return; for(i=0; (c = z[i])!=0 && c!='\''; i++){} if( c==0 ){ - oputf("'%s'",z); + sqlite3_fprintf(out, "'%s'",z); }else{ - oputz("'"); + sqlite3_fputs("'", out); while( *z ){ for(i=0; (c = z[i])!=0 && c!='\''; i++){} if( c=='\'' ) i++; if( i ){ - oputf("%.*s", i, z); + sqlite3_fprintf(out, "%.*s", i, z); z += i; } if( c=='\'' ){ - oputz("'"); + sqlite3_fputs("'", out); continue; } if( c==0 ){ @@ -1790,13 +1930,9 @@ static void output_quoted_string(const char *z){ } z++; } - oputz("'"); + sqlite3_fputs("'", out); } -#ifndef SQLITE_SHELL_FIDDLE - setTextMode(pfO, 1); -#else - setTextMode(stdout, 1); -#endif + sqlite3_fsetmode(out, _O_TEXT); } /* @@ -1808,16 +1944,13 @@ static void output_quoted_string(const char *z){ ** This is like output_quoted_string() but with the addition of the \r\n ** escape mechanism. */ -static void output_quoted_escaped_string(const char *z){ +static void output_quoted_escaped_string(FILE *out, const char *z){ int i; char c; -#ifndef SQLITE_SHELL_FIDDLE - FILE *pfO = setOutputStream(invalidFileStream); - setBinaryMode(pfO, 1); -#endif + sqlite3_fsetmode(out, _O_BINARY); for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} if( c==0 ){ - oputf("'%s'",z); + sqlite3_fprintf(out, "'%s'",z); }else{ const char *zNL = 0; const char *zCR = 0; @@ -1829,23 +1962,23 @@ static void output_quoted_escaped_string(const char *z){ if( z[i]=='\r' ) nCR++; } if( nNL ){ - oputz("replace("); + sqlite3_fputs("replace(", out); zNL = unused_string(z, "\\n", "\\012", zBuf1); } if( nCR ){ - oputz("replace("); + sqlite3_fputs("replace(", out); zCR = unused_string(z, "\\r", "\\015", zBuf2); } - oputz("'"); + sqlite3_fputs("'", out); while( *z ){ for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} if( c=='\'' ) i++; if( i ){ - oputf("%.*s", i, z); + sqlite3_fprintf(out, "%.*s", i, z); z += i; } if( c=='\'' ){ - oputz("'"); + sqlite3_fputs("'", out); continue; } if( c==0 ){ @@ -1853,24 +1986,20 @@ static void output_quoted_escaped_string(const char *z){ } z++; if( c=='\n' ){ - oputz(zNL); + sqlite3_fputs(zNL, out); continue; } - oputz(zCR); + sqlite3_fputs(zCR, out); } - oputz("'"); + sqlite3_fputs("'", out); if( nCR ){ - oputf(",'%s',char(13))", zCR); + sqlite3_fprintf(out, ",'%s',char(13))", zCR); } if( nNL ){ - oputf(",'%s',char(10))", zNL); + sqlite3_fprintf(out, ",'%s',char(10))", zNL); } } -#ifndef SQLITE_SHELL_FIDDLE - setTextMode(pfO, 1); -#else - setTextMode(stdout, 1); -#endif + sqlite3_fsetmode(stdout, _O_TEXT); } /* @@ -1890,22 +2019,60 @@ static const char *anyOfInStr(const char *s, const char *zAny, size_t ns){ } return pcFirst; } + +/* Skip over as much z[] input char sequence as is valid UTF-8, +** limited per nAccept char's or whole characters and containing +** no char cn such that ((1<=0 => char count, nAccept<0 => character + */ +const char *zSkipValidUtf8(const char *z, int nAccept, long ccm){ + int ng = (nAccept<0)? -nAccept : 0; + const char *pcLimit = (nAccept>=0)? z+nAccept : 0; + assert(z!=0); + while( (pcLimit)? (z= pcLimit ) return z; + else{ + char ct = *zt++; + if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){ + /* Trailing bytes are too few, too many, or invalid. */ + return z; + } + } + } while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */ + z = zt; + } + } + return z; +} + + /* ** Output the given string as a quoted according to C or TCL quoting rules. */ -static void output_c_string(const char *z){ +static void output_c_string(FILE *out, const char *z){ char c; static const char *zq = "\""; static long ctrlMask = ~0L; static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ char ace[3] = "\\?"; char cbsSay; - oputz(zq); + sqlite3_fputs(zq, out); while( *z!=0 ){ const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; - if( pcEnd > z ) oputb(z, (int)(pcEnd-z)); + if( pcEnd > z ){ + sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); + } if( (c = *pcEnd)==0 ) break; ++pcEnd; switch( c ){ @@ -1920,22 +2087,22 @@ static void output_c_string(const char *z){ } if( cbsSay ){ ace[1] = cbsSay; - oputz(ace); + sqlite3_fputs(ace, out); }else if( !isprint(c&0xff) ){ - oputf("\\%03o", c&0xff); + sqlite3_fprintf(out, "\\%03o", c&0xff); }else{ ace[1] = (char)c; - oputz(ace+1); + sqlite3_fputs(ace+1, out); } z = pcEnd; } - oputz(zq); + sqlite3_fputs(zq, out); } /* ** Output the given string as a quoted according to JSON quoting rules. */ -static void output_json_string(const char *z, i64 n){ +static void output_json_string(FILE *out, const char *z, i64 n){ char c; static const char *zq = "\""; static long ctrlMask = ~0L; @@ -1946,13 +2113,13 @@ static void output_json_string(const char *z, i64 n){ if( z==0 ) z = ""; pcLimit = z + ((n<0)? strlen(z) : (size_t)n); - oputz(zq); + sqlite3_fputs(zq, out); while( z < pcLimit ){ const char *pcDQBS = anyOfInStr(z, zDQBS, pcLimit-z); const char *pcPast = zSkipValidUtf8(z, (int)(pcLimit-z), ctrlMask); const char *pcEnd = (pcDQBS && pcDQBS < pcPast)? pcDQBS : pcPast; if( pcEnd > z ){ - oputb(z, (int)(pcEnd-z)); + sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); z = pcEnd; } if( z >= pcLimit ) break; @@ -1970,22 +2137,22 @@ static void output_json_string(const char *z, i64 n){ } if( cbsSay ){ ace[1] = cbsSay; - oputz(ace); + sqlite3_fputs(ace, out); }else if( c<=0x1f ){ - oputf("u%04x", c); + sqlite3_fprintf(out, "u%04x", c); }else{ ace[1] = (char)c; - oputz(ace+1); + sqlite3_fputs(ace+1, out); } } - oputz(zq); + sqlite3_fputs(zq, out); } /* ** Output the given string with characters that are special to ** HTML escaped. */ -static void output_html_string(const char *z){ +static void output_html_string(FILE *out, const char *z){ int i; if( z==0 ) z = ""; while( *z ){ @@ -1997,18 +2164,18 @@ static void output_html_string(const char *z){ && z[i]!='\''; i++){} if( i>0 ){ - oputf("%.*s",i,z); + sqlite3_fprintf(out, "%.*s",i,z); } if( z[i]=='<' ){ - oputz("<"); + sqlite3_fputs("<", out); }else if( z[i]=='&' ){ - oputz("&"); + sqlite3_fputs("&", out); }else if( z[i]=='>' ){ - oputz(">"); + sqlite3_fputs(">", out); }else if( z[i]=='\"' ){ - oputz("""); + sqlite3_fputs(""", out); }else if( z[i]=='\'' ){ - oputz("'"); + sqlite3_fputs("'", out); }else{ break; } @@ -2047,7 +2214,7 @@ static const char needCsvQuote[] = { */ static void output_csv(ShellState *p, const char *z, int bSep){ if( z==0 ){ - oputf("%s",p->nullValue); + sqlite3_fprintf(p->out, "%s",p->nullValue); }else{ unsigned i; for(i=0; z[i]; i++){ @@ -2059,14 +2226,14 @@ static void output_csv(ShellState *p, const char *z, int bSep){ if( i==0 || strstr(z, p->colSeparator)!=0 ){ char *zQuoted = sqlite3_mprintf("\"%w\"", z); shell_check_oom(zQuoted); - oputz(zQuoted); + sqlite3_fputs(zQuoted, p->out); sqlite3_free(zQuoted); }else{ - oputz(z); + sqlite3_fputs(z, p->out); } } if( bSep ){ - oputz(p->colSeparator); + sqlite3_fputs(p->colSeparator, p->out); } } @@ -2174,16 +2341,16 @@ static int shellAuth( az[1] = zA2; az[2] = zA3; az[3] = zA4; - oputf("authorizer: %s", azAction[op]); + sqlite3_fprintf(p->out, "authorizer: %s", azAction[op]); for(i=0; i<4; i++){ - oputz(" "); + sqlite3_fputs(" ", p->out); if( az[i] ){ - output_c_string(az[i]); + output_c_string(p->out, az[i]); }else{ - oputz("NULL"); + sqlite3_fputs("NULL", p->out); } } - oputz("\n"); + sqlite3_fputs("\n", p->out); if( p->bSafeMode ) (void)safeModeAuth(pClientData, op, zA1, zA2, zA3, zA4); return SQLITE_OK; } @@ -2199,7 +2366,7 @@ static int shellAuth( ** sqlite3_complete() returns false, try to terminate the comment before ** printing the result. https://sqlite.org/forum/forumpost/d7be961c5c */ -static void printSchemaLine(const char *z, const char *zTail){ +static void printSchemaLine(FILE *out, const char *z, const char *zTail){ char *zToFree = 0; if( z==0 ) return; if( zTail==0 ) return; @@ -2221,16 +2388,16 @@ static void printSchemaLine(const char *z, const char *zTail){ } } if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){ - oputf("CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); + sqlite3_fprintf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); }else{ - oputf("%s%s", z, zTail); + sqlite3_fprintf(out, "%s%s", z, zTail); } sqlite3_free(zToFree); } -static void printSchemaLineN(char *z, int n, const char *zTail){ +static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){ char c = z[n]; z[n] = 0; - printSchemaLine(z, zTail); + printSchemaLine(out, z, zTail); z[n] = c; } @@ -2258,7 +2425,7 @@ static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){ if( zText==0 ) return; nText = strlen(zText); if( p->autoEQPtest ){ - oputf("%d,%d,%s\n", iEqpId, p2, zText); + sqlite3_fprintf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); } pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); shell_check_oom(pNew); @@ -2306,7 +2473,8 @@ static void eqp_render_level(ShellState *p, int iEqpId){ for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ pNext = eqp_next_row(p, iEqpId, pRow); z = pRow->zText; - oputf("%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z); + sqlite3_fprintf(p->out, "%s%s%s\n", p->sGraph.zPrefix, + pNext ? "|--" : "`--", z); if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); eqp_render_level(p, pRow->iEqpId); @@ -2326,13 +2494,13 @@ static void eqp_render(ShellState *p, i64 nCycle){ eqp_reset(p); return; } - oputf("%s\n", pRow->zText+3); + sqlite3_fprintf(p->out, "%s\n", pRow->zText+3); p->sGraph.pRow = pRow->pNext; sqlite3_free(pRow); }else if( nCycle>0 ){ - oputf("QUERY PLAN (cycles=%lld [100%%])\n", nCycle); + sqlite3_fprintf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); }else{ - oputz("QUERY PLAN\n"); + sqlite3_fputs("QUERY PLAN\n", p->out); } p->sGraph.zPrefix[0] = 0; eqp_render_level(p, 0); @@ -2348,13 +2516,13 @@ static int progress_handler(void *pClientData) { ShellState *p = (ShellState*)pClientData; p->nProgress++; if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ - oputf("Progress limit reached (%u)\n", p->nProgress); + sqlite3_fprintf(p->out, "Progress limit reached (%u)\n", p->nProgress); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; return 1; } if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ - oputf("Progress %u\n", p->nProgress); + sqlite3_fprintf(p->out, "Progress %u\n", p->nProgress); } return 0; } @@ -2363,14 +2531,14 @@ static int progress_handler(void *pClientData) { /* ** Print N dashes */ -static void print_dashes(int N){ +static void print_dashes(FILE *out, int N){ const char zDash[] = "--------------------------------------------------"; const int nDash = sizeof(zDash) - 1; while( N>nDash ){ - oputz(zDash); + sqlite3_fputs(zDash, out); N -= nDash; } - oputf("%.*s", N, zDash); + sqlite3_fprintf(out, "%.*s", N, zDash); } /* @@ -2383,15 +2551,15 @@ static void print_row_separator( ){ int i; if( nArg>0 ){ - oputz(zSep); - print_dashes(p->actualWidth[0]+2); + sqlite3_fputs(zSep, p->out); + print_dashes(p->out, p->actualWidth[0]+2); for(i=1; iactualWidth[i]+2); + sqlite3_fputs(zSep, p->out); + print_dashes(p->out, p->actualWidth[i]+2); } - oputz(zSep); + sqlite3_fputs(zSep, p->out); } - oputz("\n"); + sqlite3_fputs("\n", p->out); } /* @@ -2421,9 +2589,9 @@ static int shell_callback( int len = strlen30(azCol[i] ? azCol[i] : ""); if( len>w ) w = len; } - if( p->cnt++>0 ) oputz(p->rowSeparator); + if( p->cnt++>0 ) sqlite3_fputs(p->rowSeparator, p->out); for(i=0; iout, "%*s = %s%s", w, azCol[i], azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator); } break; @@ -2451,12 +2619,12 @@ static int shell_callback( /* If this is the first row seen, print out the headers */ if( p->cnt++==0 ){ for(i=0; iout, aWidth[i], azCol[ aMap[i] ]); + sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); } for(i=0; iout, aWidth[i]); + sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); } } @@ -2474,17 +2642,17 @@ static int shell_callback( } if( i==iIndent && p->aiIndent && p->pStmt ){ if( p->iIndentnIndent ){ - oputf("%*.s", p->aiIndent[p->iIndent], ""); + sqlite3_fprintf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); } p->iIndent++; } - utf8_width_print(w, zVal ? zVal : p->nullValue); - oputz(i==nArg-1 ? "\n" : zSep); + utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); + sqlite3_fputs(i==nArg-1 ? "\n" : zSep, p->out); } break; } case MODE_Semi: { /* .schema and .fullschema output */ - printSchemaLine(azArg[0], ";\n"); + printSchemaLine(p->out, azArg[0], ";\n"); break; } case MODE_Pretty: { /* .schema and .fullschema with --indent */ @@ -2499,7 +2667,7 @@ static int shell_callback( if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 ){ - oputf("%s;\n", azArg[0]); + sqlite3_fprintf(p->out, "%s;\n", azArg[0]); break; } z = sqlite3_mprintf("%s", azArg[0]); @@ -2532,7 +2700,7 @@ static int shell_callback( }else if( c==')' ){ nParen--; if( nLine>0 && nParen==0 && j>0 ){ - printSchemaLineN(z, j, "\n"); + printSchemaLineN(p->out, z, j, "\n"); j = 0; } } @@ -2541,7 +2709,7 @@ static int shell_callback( && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) ){ if( c=='\n' ) j--; - printSchemaLineN(z, j, "\n "); + printSchemaLineN(p->out, z, j, "\n "); j = 0; nLine++; while( IsSpace(z[i+1]) ){ i++; } @@ -2549,118 +2717,128 @@ static int shell_callback( } z[j] = 0; } - printSchemaLine(z, ";\n"); + printSchemaLine(p->out, z, ";\n"); sqlite3_free(z); break; } case MODE_List: { if( p->cnt++==0 && p->showHeader ){ for(i=0; irowSeparator : p->colSeparator); + sqlite3_fprintf(p->out, "%s%s", azCol[i], + i==nArg-1 ? p->rowSeparator : p->colSeparator); } } if( azArg==0 ) break; for(i=0; inullValue; - oputz(z); - oputz((icolSeparator : p->rowSeparator); + sqlite3_fputs(z, p->out); + sqlite3_fputs((icolSeparator : p->rowSeparator, p->out); } break; } + case MODE_Www: case MODE_Html: { - if( p->cnt++==0 && p->showHeader ){ - oputz(""); + if( p->cnt==0 && p->cMode==MODE_Www ){ + sqlite3_fputs( + "\n" + "\n" + ,p->out + ); + } + if( p->cnt==0 && (p->showHeader || p->cMode==MODE_Www) ){ + sqlite3_fputs("", p->out); for(i=0; i"); - output_html_string(azCol[i]); - oputz("\n"); + sqlite3_fputs("\n", p->out); } - oputz("\n"); + sqlite3_fputs("\n", p->out); } + p->cnt++; if( azArg==0 ) break; - oputz(""); + sqlite3_fputs("", p->out); for(i=0; i"); - output_html_string(azArg[i] ? azArg[i] : p->nullValue); - oputz("\n"); + sqlite3_fputs("\n", p->out); } - oputz("\n"); + sqlite3_fputs("\n", p->out); break; } case MODE_Tcl: { if( p->cnt++==0 && p->showHeader ){ for(i=0; icolSeparator); + output_c_string(p->out, azCol[i] ? azCol[i] : ""); + if(icolSeparator, p->out); } - oputz(p->rowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); } if( azArg==0 ) break; for(i=0; inullValue); - if(icolSeparator); + output_c_string(p->out, azArg[i] ? azArg[i] : p->nullValue); + if(icolSeparator, p->out); } - oputz(p->rowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); break; } case MODE_Csv: { - setBinaryMode(p->out, 1); + sqlite3_fsetmode(p->out, _O_BINARY); if( p->cnt++==0 && p->showHeader ){ for(i=0; irowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); } if( nArg>0 ){ for(i=0; irowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); } - setTextMode(p->out, 1); + sqlite3_fsetmode(p->out, _O_TEXT); break; } case MODE_Insert: { if( azArg==0 ) break; - oputf("INSERT INTO %s",p->zDestTable); + sqlite3_fprintf(p->out, "INSERT INTO %s",p->zDestTable); if( p->showHeader ){ - oputz("("); + sqlite3_fputs("(", p->out); for(i=0; i0 ) oputz(","); + if( i>0 ) sqlite3_fputs(",", p->out); if( quoteChar(azCol[i]) ){ char *z = sqlite3_mprintf("\"%w\"", azCol[i]); shell_check_oom(z); - oputz(z); + sqlite3_fputs(z, p->out); sqlite3_free(z); }else{ - oputf("%s", azCol[i]); + sqlite3_fprintf(p->out, "%s", azCol[i]); } } - oputz(")"); + sqlite3_fputs(")", p->out); } p->cnt++; for(i=0; i0 ? "," : " VALUES("); + sqlite3_fputs(i>0 ? "," : " VALUES(", p->out); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - oputz("NULL"); + sqlite3_fputs("NULL", p->out); }else if( aiType && aiType[i]==SQLITE_TEXT ){ if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(azArg[i]); + output_quoted_string(p->out, azArg[i]); }else{ - output_quoted_escaped_string(azArg[i]); + output_quoted_escaped_string(p->out, azArg[i]); } }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - oputz(azArg[i]); + sqlite3_fputs(azArg[i], p->out); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - oputz("9.0e+999"); + sqlite3_fputs("9.0e+999", p->out); }else if( ur==0xfff0000000000000LL ){ - oputz("-9.0e+999"); + sqlite3_fputs("-9.0e+999", p->out); }else{ sqlite3_int64 ir = (sqlite3_int64)r; if( r==(double)ir ){ @@ -2668,115 +2846,115 @@ static int shell_callback( }else{ sqlite3_snprintf(50,z,"%!.20g", r); } - oputz(z); + sqlite3_fputs(z, p->out); } }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(pBlob, nBlob); + output_hex_blob(p->out, pBlob, nBlob); }else if( isNumber(azArg[i], 0) ){ - oputz(azArg[i]); + sqlite3_fputs(azArg[i], p->out); }else if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(azArg[i]); + output_quoted_string(p->out, azArg[i]); }else{ - output_quoted_escaped_string(azArg[i]); + output_quoted_escaped_string(p->out, azArg[i]); } } - oputz(");\n"); + sqlite3_fputs(");\n", p->out); break; } case MODE_Json: { if( azArg==0 ) break; if( p->cnt==0 ){ - fputs("[{", p->out); + sqlite3_fputs("[{", p->out); }else{ - fputs(",\n{", p->out); + sqlite3_fputs(",\n{", p->out); } p->cnt++; for(i=0; iout, azCol[i], -1); + sqlite3_fputs(":", p->out); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - oputz("null"); + sqlite3_fputs("null", p->out); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - oputz("9.0e+999"); + sqlite3_fputs("9.0e+999", p->out); }else if( ur==0xfff0000000000000LL ){ - oputz("-9.0e+999"); + sqlite3_fputs("-9.0e+999", p->out); }else{ sqlite3_snprintf(50,z,"%!.20g", r); - oputz(z); + sqlite3_fputs(z, p->out); } }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_json_string(pBlob, nBlob); + output_json_string(p->out, pBlob, nBlob); }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_json_string(azArg[i], -1); + output_json_string(p->out, azArg[i], -1); }else{ - oputz(azArg[i]); + sqlite3_fputs(azArg[i], p->out); } if( iout); } } - oputz("}"); + sqlite3_fputs("}", p->out); break; } case MODE_Quote: { if( azArg==0 ) break; if( p->cnt==0 && p->showHeader ){ for(i=0; i0 ) fputs(p->colSeparator, p->out); - output_quoted_string(azCol[i]); + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); + output_quoted_string(p->out, azCol[i]); } - fputs(p->rowSeparator, p->out); + sqlite3_fputs(p->rowSeparator, p->out); } p->cnt++; for(i=0; i0 ) fputs(p->colSeparator, p->out); + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - oputz("NULL"); + sqlite3_fputs("NULL", p->out); }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_quoted_string(azArg[i]); + output_quoted_string(p->out, azArg[i]); }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - oputz(azArg[i]); + sqlite3_fputs(azArg[i], p->out); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_snprintf(50,z,"%!.20g", r); - oputz(z); + sqlite3_fputs(z, p->out); }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(pBlob, nBlob); + output_hex_blob(p->out, pBlob, nBlob); }else if( isNumber(azArg[i], 0) ){ - oputz(azArg[i]); + sqlite3_fputs(azArg[i], p->out); }else{ - output_quoted_string(azArg[i]); + output_quoted_string(p->out, azArg[i]); } } - fputs(p->rowSeparator, p->out); + sqlite3_fputs(p->rowSeparator, p->out); break; } case MODE_Ascii: { if( p->cnt++==0 && p->showHeader ){ for(i=0; i0 ) oputz(p->colSeparator); - oputz(azCol[i] ? azCol[i] : ""); + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); + sqlite3_fputs(azCol[i] ? azCol[i] : "", p->out); } - oputz(p->rowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); } if( azArg==0 ) break; for(i=0; i0 ) oputz(p->colSeparator); - oputz(azArg[i] ? azArg[i] : p->nullValue); + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); + sqlite3_fputs(azArg[i] ? azArg[i] : p->nullValue, p->out); } - oputz(p->rowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); break; } case MODE_EQP: { @@ -2855,7 +3033,7 @@ static void createSelftestTable(ShellState *p){ "DROP TABLE [_shell$self];" ,0,0,&zErrMsg); if( zErrMsg ){ - eputf("SELFTEST initialization failure: %s\n", zErrMsg); + sqlite3_fprintf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); sqlite3_free(zErrMsg); } sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); @@ -2958,7 +3136,7 @@ static int run_table_dump_query( rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE_OK || !pSelect ){ char *zContext = shell_error_context(zSelect, p->db); - oputf("/**** ERROR: (%d) %s *****/\n%s", + sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n%s", rc, sqlite3_errmsg(p->db), zContext); sqlite3_free(zContext); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; @@ -2968,22 +3146,23 @@ static int run_table_dump_query( nResult = sqlite3_column_count(pSelect); while( rc==SQLITE_ROW ){ z = (const char*)sqlite3_column_text(pSelect, 0); - oputf("%s", z); + sqlite3_fprintf(p->out, "%s", z); for(i=1; iout, ",%s", sqlite3_column_text(pSelect, i)); } if( z==0 ) z = ""; while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; if( z[0] ){ - oputz("\n;\n"); + sqlite3_fputs("\n;\n", p->out); }else{ - oputz(";\n"); + sqlite3_fputs(";\n", p->out); } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); if( rc!=SQLITE_OK ){ - oputf("/**** ERROR: (%d) %s *****/\n", rc, sqlite3_errmsg(p->db)); + sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", + rc, sqlite3_errmsg(p->db)); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; } return rc; @@ -3019,13 +3198,13 @@ static char *save_err_msg( /* ** Attempt to display I/O stats on Linux using /proc/PID/io */ -static void displayLinuxIoStats(void){ +static void displayLinuxIoStats(FILE *out){ FILE *in; char z[200]; sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); - in = fopen(z, "rb"); + in = sqlite3_fopen(z, "rb"); if( in==0 ) return; - while( fgets(z, sizeof(z), in)!=0 ){ + while( sqlite3_fgets(z, sizeof(z), in)!=0 ){ static const struct { const char *zPattern; const char *zDesc; @@ -3042,7 +3221,7 @@ static void displayLinuxIoStats(void){ for(i=0; iout==0 ) return 0; + out = pArg->out; if( pArg->pStmt && pArg->statsOn==2 ){ int nCol, i, x; sqlite3_stmt *pStmt = pArg->pStmt; char z[100]; nCol = sqlite3_column_count(pStmt); - oputf("%-36s %d\n", "Number of output columns:", nCol); + sqlite3_fprintf(out, "%-36s %d\n", "Number of output columns:", nCol); for(i=0; istatsOn==3 ){ if( pArg->pStmt ){ iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); - oputf("VM-steps: %d\n", iCur); + sqlite3_fprintf(out, "VM-steps: %d\n", iCur); } return 0; } - displayStatLine("Memory Used:", + displayStatLine(out, "Memory Used:", "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); - displayStatLine("Number of Outstanding Allocations:", + displayStatLine(out, "Number of Outstanding Allocations:", "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); if( pArg->shellFlgs & SHFLG_Pagecache ){ - displayStatLine("Number of Pcache Pages Used:", + displayStatLine(out, "Number of Pcache Pages Used:", "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); } - displayStatLine("Number of Pcache Overflow Bytes:", + displayStatLine(out, "Number of Pcache Overflow Bytes:", "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); - displayStatLine("Largest Allocation:", + displayStatLine(out, "Largest Allocation:", "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); - displayStatLine("Largest Pcache Allocation:", + displayStatLine(out, "Largest Pcache Allocation:", "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH - displayStatLine("Deepest Parser Stack:", + displayStatLine(out, "Deepest Parser Stack:", "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif @@ -3144,68 +3327,87 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - oputf("Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); + sqlite3_fprintf(out, + "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); - oputf("Successful lookaside attempts: %d\n", iHiwtr); + sqlite3_fprintf(out, + "Successful lookaside attempts: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); - oputf("Lookaside failures due to size: %d\n", iHiwtr); + sqlite3_fprintf(out, + "Lookaside failures due to size: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); - oputf("Lookaside failures due to OOM: %d\n", iHiwtr); + sqlite3_fprintf(out, + "Lookaside failures due to OOM: %d\n", iHiwtr); } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - oputf("Pager Heap Usage: %d bytes\n", iCur); + sqlite3_fprintf(out, + "Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); - oputf("Page cache hits: %d\n", iCur); + sqlite3_fprintf(out, + "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - oputf("Page cache misses: %d\n", iCur); + sqlite3_fprintf(out, + "Page cache misses: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - oputf("Page cache writes: %d\n", iCur); + sqlite3_fprintf(out, + "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); - oputf("Page cache spills: %d\n", iCur); + sqlite3_fprintf(out, + "Page cache spills: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - oputf("Schema Heap Usage: %d bytes\n", iCur); + sqlite3_fprintf(out, + "Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - oputf("Statement Heap/Lookaside Usage: %d bytes\n", iCur); + sqlite3_fprintf(out, + "Statement Heap/Lookaside Usage: %d bytes\n", iCur); } if( pArg->pStmt ){ int iHit, iMiss; iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); - oputf("Fullscan Steps: %d\n", iCur); + sqlite3_fprintf(out, + "Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); - oputf("Sort Operations: %d\n", iCur); + sqlite3_fprintf(out, + "Sort Operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); - oputf("Autoindex Inserts: %d\n", iCur); + sqlite3_fprintf(out, + "Autoindex Inserts: %d\n", iCur); iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, bReset); iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, bReset); if( iHit || iMiss ){ - oputf("Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); + sqlite3_fprintf(out, + "Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); } iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); - oputf("Virtual Machine Steps: %d\n", iCur); + sqlite3_fprintf(out, + "Virtual Machine Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); - oputf("Reprepare operations: %d\n", iCur); + sqlite3_fprintf(out, + "Reprepare operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); - oputf("Number of times run: %d\n", iCur); + sqlite3_fprintf(out, + "Number of times run: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); - oputf("Memory used by prepared stmt: %d\n", iCur); + sqlite3_fprintf(out, + "Memory used by prepared stmt: %d\n", iCur); } #ifdef __linux__ - displayLinuxIoStats(); + displayLinuxIoStats(pArg->out); #endif /* Do not remove this machine readable comment: extra-stats-output-here */ @@ -3601,17 +3803,17 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ /* Draw horizontal line N characters long using unicode box ** characters */ -static void print_box_line(int N){ +static void print_box_line(FILE *out, int N){ const char zDash[] = BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; const int nDash = sizeof(zDash) - 1; N *= 3; while( N>nDash ){ - oputz(zDash); + sqlite3_fputs(zDash, out); N -= nDash; } - oputf("%.*s", N, zDash); + sqlite3_fprintf(out, "%.*s", N, zDash); } /* @@ -3626,15 +3828,15 @@ static void print_box_row_separator( ){ int i; if( nArg>0 ){ - oputz(zSep1); - print_box_line(p->actualWidth[0]+2); + sqlite3_fputs(zSep1, p->out); + print_box_line(p->out, p->actualWidth[0]+2); for(i=1; iactualWidth[i]+2); + sqlite3_fputs(zSep2, p->out); + print_box_line(p->out, p->actualWidth[i]+2); } - oputz(zSep3); + sqlite3_fputs(zSep3, p->out); } - oputz("\n"); + sqlite3_fputs("\n", p->out); } /* @@ -3668,12 +3870,22 @@ static char *translateForDisplayAndDup( if( mxWidth==0 ) mxWidth = 1000000; i = j = n = 0; while( n=' ' ){ - n++; - do{ i++; j++; }while( (z[i]&0xc0)==0x80 ); + unsigned char c = z[i]; + if( c>=0xc0 ){ + int u; + int len = decodeUtf8(&z[i], &u); + i += len; + j += len; + n += cli_wcwidth(u); continue; } - if( z[i]=='\t' ){ + if( c>=' ' ){ + n++; + i++; + j++; + continue; + } + if( c=='\t' ){ do{ n++; j++; @@ -3715,9 +3927,17 @@ static char *translateForDisplayAndDup( shell_check_oom(zOut); i = j = n = 0; while( i=' ' ){ + unsigned char c = z[i]; + if( c>=0xc0 ){ + int u; + int len = decodeUtf8(&z[i], &u); + do{ zOut[j++] = z[i++]; }while( (--len)>0 ); + n += cli_wcwidth(u); + continue; + } + if( c>=' ' ){ n++; - do{ zOut[j++] = z[i++]; }while( (z[i]&0xc0)==0x80 ); + zOut[j++] = z[i++]; continue; } if( z[i]=='\t' ){ @@ -3897,12 +4117,12 @@ static void exec_prepared_stmt_columnar( for(i=0; iactualWidth[i]; if( p->colWidth[i]<0 ) w = -w; - utf8_width_print(w, azData[i]); - fputs(i==nColumn-1?"\n":" ", p->out); + utf8_width_print(p->out, w, azData[i]); + sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); } for(i=0; iactualWidth[i]); - fputs(i==nColumn-1?"\n":" ", p->out); + print_dashes(p->out, p->actualWidth[i]); + sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); } } break; @@ -3911,12 +4131,13 @@ static void exec_prepared_stmt_columnar( colSep = " | "; rowSep = " |\n"; print_row_separator(p, nColumn, "+"); - fputs("| ", p->out); + sqlite3_fputs("| ", p->out); for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - oputf("%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); - oputz(i==nColumn-1?" |\n":" | "); + sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", + azData[i], (w-n+1)/2, ""); + sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); } print_row_separator(p, nColumn, "+"); break; @@ -3924,12 +4145,13 @@ static void exec_prepared_stmt_columnar( case MODE_Markdown: { colSep = " | "; rowSep = " |\n"; - fputs("| ", p->out); + sqlite3_fputs("| ", p->out); for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - oputf("%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); - oputz(i==nColumn-1?" |\n":" | "); + sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", + azData[i], (w-n+1)/2, ""); + sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); } print_row_separator(p, nColumn, "|"); break; @@ -3938,11 +4160,11 @@ static void exec_prepared_stmt_columnar( colSep = " " BOX_13 " "; rowSep = " " BOX_13 "\n"; print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); - oputz(BOX_13 " "); + sqlite3_fputs(BOX_13 " ", p->out); for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - oputf("%*s%s%*s%s", + sqlite3_fprintf(p->out, "%*s%s%*s%s", (w-n)/2, "", azData[i], (w-n+1)/2, "", i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); } @@ -3952,28 +4174,28 @@ static void exec_prepared_stmt_columnar( } for(i=nColumn, j=0; icMode!=MODE_Column ){ - oputz(p->cMode==MODE_Box?BOX_13" ":"| "); + sqlite3_fputs(p->cMode==MODE_Box?BOX_13" ":"| ", p->out); } z = azData[i]; if( z==0 ) z = p->nullValue; w = p->actualWidth[j]; if( p->colWidth[j]<0 ) w = -w; - utf8_width_print(w, z); + utf8_width_print(p->out, w, z); if( j==nColumn-1 ){ - oputz(rowSep); + sqlite3_fputs(rowSep, p->out); if( bMultiLineRowExists && abRowDiv[i/nColumn-1] && i+1cMode==MODE_Table ){ print_row_separator(p, nColumn, "+"); }else if( p->cMode==MODE_Box ){ print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); }else if( p->cMode==MODE_Column ){ - oputz("\n"); + sqlite3_fputs("\n", p->out); } } j = -1; if( seenInterrupt ) goto columnar_end; }else{ - oputz(colSep); + sqlite3_fputs(colSep, p->out); } } if( p->cMode==MODE_Table ){ @@ -3983,7 +4205,7 @@ static void exec_prepared_stmt_columnar( } columnar_end: if( seenInterrupt ){ - oputz("Interrupt\n"); + sqlite3_fputs("Interrupt\n", p->out); } nData = (nRow+1)*nColumn; for(i=0; icMode==MODE_Json ){ - fputs("]\n", pArg->out); + sqlite3_fputs("]\n", pArg->out); + }else if( pArg->cMode==MODE_Www ){ + sqlite3_fputs("
    ", p->out); + output_html_string(p->out, azCol[i]); + sqlite3_fputs("
    ", p->out); + output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); + sqlite3_fputs("
    \n
    \n", pArg->out);
       }else if( pArg->cMode==MODE_Count ){
         char zBuf[200];
         sqlite3_snprintf(sizeof(zBuf), zBuf, "%llu row%s\n",
@@ -4119,6 +4343,7 @@ static int expertFinish(
 ){
   int rc = SQLITE_OK;
   sqlite3expert *p = pState->expert.pExpert;
+  FILE *out = pState->out;
   assert( p );
   assert( bCancel || pzErr==0 || *pzErr==0 );
   if( bCancel==0 ){
@@ -4131,8 +4356,8 @@ static int expertFinish(
 
       if( bVerbose ){
         const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
-        oputz("-- Candidates -----------------------------\n");
-        oputf("%s\n", zCand);
+        sqlite3_fputs("-- Candidates -----------------------------\n", out);
+        sqlite3_fprintf(out, "%s\n", zCand);
       }
       for(i=0; i=2 && 0==cli_strncmp(z, "-sample", n) ){
       if( i==(nArg-1) ){
-        eputf("option requires an argument: %s\n", z);
+        sqlite3_fprintf(stderr, "option requires an argument: %s\n", z);
         rc = SQLITE_ERROR;
       }else{
         iSample = (int)integerValue(azArg[++i]);
         if( iSample<0 || iSample>100 ){
-          eputf("value out of range: %s\n", azArg[i]);
+          sqlite3_fprintf(stderr,"value out of range: %s\n", azArg[i]);
           rc = SQLITE_ERROR;
         }
       }
     }
     else{
-      eputf("unknown option: %s\n", z);
+      sqlite3_fprintf(stderr,"unknown option: %s\n", z);
       rc = SQLITE_ERROR;
     }
   }
@@ -4198,7 +4424,8 @@ static int expertDotCommand(
   if( rc==SQLITE_OK ){
     pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr);
     if( pState->expert.pExpert==0 ){
-      eputf("sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory");
+      sqlite3_fprintf(stderr,
+          "sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory");
       rc = SQLITE_ERROR;
     }else{
       sqlite3_expert_config(
@@ -4527,9 +4754,9 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
   noSys    = (p->shellFlgs & SHFLG_DumpNoSys)!=0;
 
   if( cli_strcmp(zTable, "sqlite_sequence")==0 && !noSys ){
-    if( !dataOnly ) oputz("DELETE FROM sqlite_sequence;\n");
+    if( !dataOnly ) sqlite3_fputs("DELETE FROM sqlite_sequence;\n", p->out);
   }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){
-    if( !dataOnly ) oputz("ANALYZE sqlite_schema;\n");
+    if( !dataOnly ) sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out);
   }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){
     return 0;
   }else if( dataOnly ){
@@ -4537,7 +4764,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
   }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){
     char *zIns;
     if( !p->writableSchema ){
-      oputz("PRAGMA writable_schema=ON;\n");
+      sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out);
       p->writableSchema = 1;
     }
     zIns = sqlite3_mprintf(
@@ -4545,11 +4772,11 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
        "VALUES('table','%q','%q',0,'%q');",
        zTable, zTable, zSql);
     shell_check_oom(zIns);
-    oputf("%s\n", zIns);
+    sqlite3_fprintf(p->out, "%s\n", zIns);
     sqlite3_free(zIns);
     return 0;
   }else{
-    printSchemaLine(zSql, ";\n");
+    printSchemaLine(p->out, zSql, ";\n");
   }
 
   if( cli_strcmp(zType, "table")==0 ){
@@ -4607,7 +4834,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
     p->mode = p->cMode = MODE_Insert;
     rc = shell_exec(p, sSelect.z, 0);
     if( (rc&0xff)==SQLITE_CORRUPT ){
-      oputz("/****** CORRUPTION ERROR *******/\n");
+      sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out);
       toggleSelectOrder(p->db);
       shell_exec(p, sSelect.z, 0);
       toggleSelectOrder(p->db);
@@ -4638,9 +4865,9 @@ static int run_schema_dump_query(
   if( rc==SQLITE_CORRUPT ){
     char *zQ2;
     int len = strlen30(zQuery);
-    oputz("/****** CORRUPTION ERROR *******/\n");
+    sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out);
     if( zErr ){
-      oputf("/****** %s ******/\n", zErr);
+      sqlite3_fprintf(p->out, "/****** %s ******/\n", zErr);
       sqlite3_free(zErr);
       zErr = 0;
     }
@@ -4649,7 +4876,7 @@ static int run_schema_dump_query(
     sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery);
     rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr);
     if( rc ){
-      oputf("/****** ERROR: %s ******/\n", zErr);
+      sqlite3_fprintf(p->out, "/****** ERROR: %s ******/\n", zErr);
     }else{
       rc = SQLITE_CORRUPT;
     }
@@ -4712,9 +4939,10 @@ static const char *(azHelp[]) = {
   ".clone NEWDB             Clone data into NEWDB from the existing database",
 #endif
   ".connection [close] [#]  Open or close an auxiliary database connection",
-#if defined(_WIN32) || defined(WIN32)
+#if defined(_WIN32) && !defined(SQLITE_U8TEXT_ONLY) \
+                    && !defined(SQLITE_U8TEXT_STDIO)
   ".crnl on|off             Translate \\n to \\r\\n.  Default ON",
-#endif
+#endif /* _WIN32 && U8TEXT_ONLY && U8TEXT_STDIO */
   ".databases               List names and files of attached databases",
   ".dbconfig ?op? ?val?     List or change sqlite3_db_config() options",
 #if SQLITE_SHELL_HAVE_RECOVER
@@ -4820,9 +5048,11 @@ static const char *(azHelp[]) = {
 #ifndef SQLITE_SHELL_FIDDLE
   ".once ?OPTIONS? ?FILE?   Output for the next SQL command only to FILE",
   "     If FILE begins with '|' then open as a pipe",
-  "       --bom  Put a UTF8 byte-order mark at the beginning",
-  "       -e     Send output to the system text editor",
-  "       -x     Send output as CSV to a spreadsheet (same as \".excel\")",
+  "       --bom    Put a UTF8 byte-order mark at the beginning",
+  "       -e       Send output to the system text editor",
+  "       --plain  Use text/plain output instead of HTML for -w option",
+  "       -w       Send output as HTML to a web browser (same as \".www\")",
+  "       -x       Send output as CSV to a spreadsheet (same as \".excel\")",
   /* Note that .open is (partially) available in WASM builds but is
   ** currently only intended to be used by the fiddle tool, not
   ** end users, so is "undocumented." */
@@ -4845,6 +5075,8 @@ static const char *(azHelp[]) = {
   "   Options:",
   "     --bom                 Prefix output with a UTF8 byte-order mark",
   "     -e                    Send output to the system text editor",
+  "     --plain               Use text/plain for -w option",
+  "     -w                    Send output to a web browser",
   "     -x                    Send output as CSV to a spreadsheet",
 #endif
   ".parameter CMD ...       Manage SQL parameter bindings",
@@ -4958,6 +5190,10 @@ static const char *(azHelp[]) = {
   ".vfsname ?AUX?           Print the name of the VFS stack",
   ".width NUM1 NUM2 ...     Set minimum column widths for columnar output",
   "     Negative values right-justify",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".www                     Display output of the next command in web browser",
+  "    --plain                 Show results as text/plain, not as HTML",
+#endif
 };
 
 /*
@@ -5006,10 +5242,10 @@ static int showHelp(FILE *out, const char *zPattern){
       }
       if( ((hw^hh)&HH_Undoc)==0 ){
         if( (hh&HH_Summary)!=0 ){
-          sputf(out, ".%s\n", azHelp[i]+1);
+          sqlite3_fprintf(out, ".%s\n", azHelp[i]+1);
           ++n;
         }else if( (hw&HW_SummaryOnly)==0 ){
-          sputf(out, "%s\n", azHelp[i]);
+          sqlite3_fprintf(out, "%s\n", azHelp[i]);
         }
       }
     }
@@ -5019,7 +5255,7 @@ static int showHelp(FILE *out, const char *zPattern){
     shell_check_oom(zPat);
     for(i=0; i65536 || (pgsz & (pgsz-1))!=0 ){
-    eputz("invalid pagesize\n");
+    sqlite3_fputs("invalid pagesize\n", stderr);
     goto readHexDb_error;
   }
-  for(nLine++; fgets(zLine, sizeof(zLine), in)!=0; nLine++){
+  for(nLine++; sqlite3_fgets(zLine, sizeof(zLine), in)!=0; nLine++){
     rc = sscanf(zLine, "| page %d offset %d", &j, &k);
     if( rc==2 ){
       iOffset = k;
@@ -5281,14 +5517,14 @@ readHexDb_error:
   if( in!=p->in ){
     fclose(in);
   }else{
-    while( fgets(zLine, sizeof(zLine), p->in)!=0 ){
+    while( sqlite3_fgets(zLine, sizeof(zLine), p->in)!=0 ){
       nLine++;
       if(cli_strncmp(zLine, "| end ", 6)==0 ) break;
     }
     p->lineno = nLine;
   }
   sqlite3_free(a);
-  eputf("Error on line %d of --hexdb input\n", nLine);
+  sqlite3_fprintf(stderr,"Error on line %d of --hexdb input\n", nLine);
   return 0;
 }
 #endif /* SQLITE_OMIT_DESERIALIZE */
@@ -5363,7 +5599,7 @@ static void open_db(ShellState *p, int openFlags){
       }
     }
     if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
-      eputf("Error: unable to open database \"%s\": %s\n",
+      sqlite3_fprintf(stderr,"Error: unable to open database \"%s\": %s\n",
             zDbFilename, sqlite3_errmsg(p->db));
       if( (openFlags & OPEN_DB_KEEPALIVE)==0 ){
         exit(1);
@@ -5371,10 +5607,12 @@ static void open_db(ShellState *p, int openFlags){
       sqlite3_close(p->db);
       sqlite3_open(":memory:", &p->db);
       if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
-        eputz("Also: unable to open substitute in-memory database.\n");
+        sqlite3_fputs("Also: unable to open substitute in-memory database.\n",
+                      stderr);
         exit(1);
       }else{
-        eputf("Notice: using substitute in-memory database instead of \"%s\"\n",
+        sqlite3_fprintf(stderr,
+              "Notice: using substitute in-memory database instead of \"%s\"\n",
               zDbFilename);
       }
     }
@@ -5391,6 +5629,7 @@ static void open_db(ShellState *p, int openFlags){
 #ifndef SQLITE_OMIT_LOAD_EXTENSION
     sqlite3_enable_load_extension(p->db, 1);
 #endif
+    sqlite3_sha_init(p->db, 0, 0);
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_uint_init(p->db, 0, 0);
     sqlite3_stmtrand_init(p->db, 0, 0);
@@ -5485,7 +5724,7 @@ static void open_db(ShellState *p, int openFlags){
                    SQLITE_DESERIALIZE_RESIZEABLE |
                    SQLITE_DESERIALIZE_FREEONCLOSE);
       if( rc ){
-        eputf("Error: sqlite3_deserialize() returns %d\n", rc);
+        sqlite3_fprintf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc);
       }
       if( p->szMax>0 ){
         sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax);
@@ -5509,7 +5748,8 @@ static void open_db(ShellState *p, int openFlags){
 void close_db(sqlite3 *db){
   int rc = sqlite3_close(db);
   if( rc ){
-    eputf("Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db));
+    sqlite3_fprintf(stderr,
+        "Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db));
   }
 }
 
@@ -5673,7 +5913,8 @@ static int booleanValue(const char *zArg){
   if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){
     return 0;
   }
-  eputf("ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", zArg);
+  sqlite3_fprintf(stderr,
+       "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", zArg);
   return 0;
 }
 
@@ -5709,9 +5950,9 @@ static FILE *output_file_open(const char *zFile, int bTextMode){
   }else if( cli_strcmp(zFile, "off")==0 ){
     f = 0;
   }else{
-    f = fopen(zFile, bTextMode ? "w" : "wb");
+    f = sqlite3_fopen(zFile, bTextMode ? "w" : "wb");
     if( f==0 ){
-      eputf("Error: cannot open \"%s\"\n", zFile);
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile);
     }
   }
   return f;
@@ -5764,12 +6005,13 @@ static int sql_trace_callback(
   switch( mType ){
     case SQLITE_TRACE_ROW:
     case SQLITE_TRACE_STMT: {
-      sputf(p->traceOut, "%.*s;\n", (int)nSql, zSql);
+      sqlite3_fprintf(p->traceOut, "%.*s;\n", (int)nSql, zSql);
       break;
     }
     case SQLITE_TRACE_PROFILE: {
       sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0;
-      sputf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec);
+      sqlite3_fprintf(p->traceOut,
+                      "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec);
       break;
     }
   }
@@ -5876,10 +6118,11 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
         break;
       }
       if( pc==cQuote && c!='\r' ){
-        eputf("%s:%d: unescaped %c character\n", p->zFile, p->nLine, cQuote);
+        sqlite3_fprintf(stderr,"%s:%d: unescaped %c character\n", 
+                        p->zFile, p->nLine, cQuote);
       }
       if( c==EOF ){
-        eputf("%s:%d: unterminated %c-quoted field\n",
+        sqlite3_fprintf(stderr,"%s:%d: unterminated %c-quoted field\n",
               p->zFile, startLine, cQuote);
         p->cTerm = c;
         break;
@@ -5978,7 +6221,7 @@ static void tryToCloneData(
   shell_check_oom(zQuery);
   rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
   if( rc ){
-    eputf("Error %d: %s on [%s]\n",
+    sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n",
           sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery);
     goto end_data_xfer;
   }
@@ -5995,7 +6238,7 @@ static void tryToCloneData(
   memcpy(zInsert+i, ");", 3);
   rc = sqlite3_prepare_v2(newDb, zInsert, -1, &pInsert, 0);
   if( rc ){
-    eputf("Error %d: %s on [%s]\n",
+    sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n",
           sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), zInsert);
     goto end_data_xfer;
   }
@@ -6031,7 +6274,7 @@ static void tryToCloneData(
       } /* End for */
       rc = sqlite3_step(pInsert);
       if( rc!=SQLITE_OK && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){
-        eputf("Error %d: %s\n",
+        sqlite3_fprintf(stderr,"Error %d: %s\n",
               sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb));
       }
       sqlite3_reset(pInsert);
@@ -6049,7 +6292,7 @@ static void tryToCloneData(
     shell_check_oom(zQuery);
     rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
     if( rc ){
-      eputf("Warning: cannot step \"%s\" backwards", zTable);
+      sqlite3_fprintf(stderr,"Warning: cannot step \"%s\" backwards", zTable);
       break;
     }
   } /* End for(k=0...) */
@@ -6086,7 +6329,8 @@ static void tryToCloneSchema(
   shell_check_oom(zQuery);
   rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
   if( rc ){
-    eputf("Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db),
+    sqlite3_fprintf(stderr,
+          "Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db),
           sqlite3_errmsg(p->db), zQuery);
     goto end_schema_xfer;
   }
@@ -6095,10 +6339,10 @@ static void tryToCloneSchema(
     zSql = sqlite3_column_text(pQuery, 1);
     if( zName==0 || zSql==0 ) continue;
     if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ){
-      sputf(stdout, "%s... ", zName); fflush(stdout);
+      sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout);
       sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
       if( zErrMsg ){
-        eputf("Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
+        sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
         sqlite3_free(zErrMsg);
         zErrMsg = 0;
       }
@@ -6116,7 +6360,7 @@ static void tryToCloneSchema(
     shell_check_oom(zQuery);
     rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
     if( rc ){
-      eputf("Error: (%d) %s on [%s]\n",
+      sqlite3_fprintf(stderr,"Error: (%d) %s on [%s]\n",
             sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery);
       goto end_schema_xfer;
     }
@@ -6125,10 +6369,10 @@ static void tryToCloneSchema(
       zSql = sqlite3_column_text(pQuery, 1);
       if( zName==0 || zSql==0 ) continue;
       if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue;
-      sputf(stdout, "%s... ", zName); fflush(stdout);
+      sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout);
       sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
       if( zErrMsg ){
-        eputf("Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
+        sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
         sqlite3_free(zErrMsg);
         zErrMsg = 0;
       }
@@ -6152,12 +6396,13 @@ static void tryToClone(ShellState *p, const char *zNewDb){
   int rc;
   sqlite3 *newDb = 0;
   if( access(zNewDb,0)==0 ){
-    eputf("File \"%s\" already exists.\n", zNewDb);
+    sqlite3_fprintf(stderr,"File \"%s\" already exists.\n", zNewDb);
     return;
   }
   rc = sqlite3_open(zNewDb, &newDb);
   if( rc ){
-    eputf("Cannot create output database: %s\n", sqlite3_errmsg(newDb));
+    sqlite3_fprintf(stderr,
+        "Cannot create output database: %s\n", sqlite3_errmsg(newDb));
   }else{
     sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0);
     sqlite3_exec(newDb, "BEGIN EXCLUSIVE;", 0, 0, 0);
@@ -6174,10 +6419,17 @@ static void tryToClone(ShellState *p, const char *zNewDb){
 ** Change the output stream (file or pipe or console) to something else.
 */
 static void output_redir(ShellState *p, FILE *pfNew){
-  if( p->out != stdout ) eputz("Output already redirected.\n");
-  else{
+  if( p->out != stdout ){
+    sqlite3_fputs("Output already redirected.\n", stderr);
+  }else{
     p->out = pfNew;
-    setOutputStream(pfNew);
+    if( p->mode==MODE_Www ){
+      sqlite3_fputs(
+        "\n"
+        "
    \n",
+        p->out
+      );
+    }
   }
 }
 
@@ -6194,6 +6446,9 @@ static void output_reset(ShellState *p){
     pclose(p->out);
 #endif
   }else{
+    if( p->mode==MODE_Www ){
+      sqlite3_fputs("
    \n", p->out);
+    }
     output_file_close(p->out);
 #ifndef SQLITE_NOHAVE_SYSTEM
     if( p->doXdgOpen ){
@@ -6208,7 +6463,7 @@ static void output_reset(ShellState *p){
       char *zCmd;
       zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile);
       if( system(zCmd) ){
-        eputf("Failed: [%s]\n", zCmd);
+        sqlite3_fprintf(stderr,"Failed: [%s]\n", zCmd);
       }else{
         /* Give the start/open/xdg-open command some time to get
         ** going before we continue, and potential delete the
@@ -6223,7 +6478,6 @@ static void output_reset(ShellState *p){
   }
   p->outfile[0] = 0;
   p->out = stdout;
-  setOutputStream(stdout);
 }
 #else
 # define output_redir(SS,pfO)
@@ -6299,7 +6553,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
              "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1",
              -1, &pStmt, 0);
   if( rc ){
-    eputf("error: %s\n", sqlite3_errmsg(p->db));
+    sqlite3_fprintf(stderr,"error: %s\n", sqlite3_errmsg(p->db));
     sqlite3_finalize(pStmt);
     return 1;
   }
@@ -6312,28 +6566,28 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
     memcpy(aHdr, pb, 100);
     sqlite3_finalize(pStmt);
   }else{
-    eputz("unable to read database header\n");
+    sqlite3_fputs("unable to read database header\n", stderr);
     sqlite3_finalize(pStmt);
     return 1;
   }
   i = get2byteInt(aHdr+16);
   if( i==1 ) i = 65536;
-  oputf("%-20s %d\n", "database page size:", i);
-  oputf("%-20s %d\n", "write format:", aHdr[18]);
-  oputf("%-20s %d\n", "read format:", aHdr[19]);
-  oputf("%-20s %d\n", "reserved bytes:", aHdr[20]);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "database page size:", i);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "write format:", aHdr[18]);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "read format:", aHdr[19]);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]);
   for(i=0; iout, "%-20s %u", aField[i].zName, val);
     switch( ofst ){
       case 56: {
-        if( val==1 ) oputz(" (utf8)");
-        if( val==2 ) oputz(" (utf16le)");
-        if( val==3 ) oputz(" (utf16be)");
+        if( val==1 ) sqlite3_fputs(" (utf8)", p->out);
+        if( val==2 ) sqlite3_fputs(" (utf16le)", p->out);
+        if( val==3 ) sqlite3_fputs(" (utf16be)", p->out);
       }
     }
-    oputz("\n");
+    sqlite3_fputs("\n", p->out);
   }
   if( zDb==0 ){
     zSchemaTab = sqlite3_mprintf("main.sqlite_schema");
@@ -6346,11 +6600,11 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
     char *zSql = sqlite3_mprintf(aQuery[i].zSql, zSchemaTab);
     int val = db_int(p->db, zSql);
     sqlite3_free(zSql);
-    oputf("%-20s %d\n", aQuery[i].zName, val);
+    sqlite3_fprintf(p->out, "%-20s %d\n", aQuery[i].zName, val);
   }
   sqlite3_free(zSchemaTab);
   sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion);
-  oputf("%-20s %u\n", "data version", iDataVersion);
+  sqlite3_fprintf(p->out, "%-20s %u\n", "data version", iDataVersion);
   return 0;
 }
 #endif /* SQLITE_SHELL_HAVE_RECOVER */
@@ -6359,7 +6613,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
 ** Print the given string as an error message.
 */
 static void shellEmitError(const char *zErr){
-  eputf("Error: %s\n", zErr);
+  sqlite3_fprintf(stderr,"Error: %s\n", zErr);
 }
 /*
 ** Print the current sqlite3_errmsg() value to stderr and return 1.
@@ -6606,6 +6860,7 @@ static int lintFkeyIndexes(
   const char *zIndent = "";       /* How much to indent CREATE INDEX by */
   int rc;                         /* Return code */
   sqlite3_stmt *pSql = 0;         /* Compiled version of SQL statement below */
+  FILE *out = pState->out;        /* Send output here */
 
   /*
   ** This SELECT statement returns one row for each foreign key constraint
@@ -6681,7 +6936,8 @@ static int lintFkeyIndexes(
       zIndent = "    ";
     }
     else{
-      eputf("Usage: %s %s ?-verbose? ?-groupbyparent?\n", azArg[0], azArg[1]);
+      sqlite3_fprintf(stderr,
+           "Usage: %s %s ?-verbose? ?-groupbyparent?\n", azArg[0], azArg[1]);
       return SQLITE_ERROR;
     }
   }
@@ -6725,22 +6981,23 @@ static int lintFkeyIndexes(
       if( rc!=SQLITE_OK ) break;
 
       if( res<0 ){
-        eputz("Error: internal error");
+        sqlite3_fputs("Error: internal error", stderr);
         break;
       }else{
         if( bGroupByParent
         && (bVerbose || res==0)
         && (zPrev==0 || sqlite3_stricmp(zParent, zPrev))
         ){
-          oputf("-- Parent table %s\n", zParent);
+          sqlite3_fprintf(out, "-- Parent table %s\n", zParent);
           sqlite3_free(zPrev);
           zPrev = sqlite3_mprintf("%s", zParent);
         }
 
         if( res==0 ){
-          oputf("%s%s --> %s\n", zIndent, zCI, zTarget);
+          sqlite3_fprintf(out, "%s%s --> %s\n", zIndent, zCI, zTarget);
         }else if( bVerbose ){
-          oputf("%s/* no extra indexes required for %s -> %s */\n",
+          sqlite3_fprintf(out,
+                "%s/* no extra indexes required for %s -> %s */\n",
                 zIndent, zFrom, zTarget
           );
         }
@@ -6749,16 +7006,16 @@ static int lintFkeyIndexes(
     sqlite3_free(zPrev);
 
     if( rc!=SQLITE_OK ){
-      eputf("%s\n", sqlite3_errmsg(db));
+      sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db));
     }
 
     rc2 = sqlite3_finalize(pSql);
     if( rc==SQLITE_OK && rc2!=SQLITE_OK ){
       rc = rc2;
-      eputf("%s\n", sqlite3_errmsg(db));
+      sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db));
     }
   }else{
-    eputf("%s\n", sqlite3_errmsg(db));
+    sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db));
   }
 
   return rc;
@@ -6778,9 +7035,9 @@ static int lintDotCommand(
   return lintFkeyIndexes(pState, azArg, nArg);
 
  usage:
-  eputf("Usage %s sub-command ?switches...?\n", azArg[0]);
-  eputz("Where sub-commands are:\n");
-  eputz("    fkey-indexes\n");
+  sqlite3_fprintf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]);
+  sqlite3_fprintf(stderr, "Where sub-commands are:\n");
+  sqlite3_fprintf(stderr, "    fkey-indexes\n");
   return SQLITE_ERROR;
 }
 
@@ -6794,7 +7051,8 @@ static void shellPrepare(
   if( *pRc==SQLITE_OK ){
     int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
     if( rc!=SQLITE_OK ){
-      eputf("sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db));
+      sqlite3_fprintf(stderr,
+         "sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db));
       *pRc = rc;
     }
   }
@@ -6838,7 +7096,7 @@ static void shellFinalize(
     int rc = sqlite3_finalize(pStmt);
     if( *pRc==SQLITE_OK ){
       if( rc!=SQLITE_OK ){
-        eputf("SQL error: %s\n", sqlite3_errmsg(db));
+        sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db));
       }
       *pRc = rc;
     }
@@ -6860,7 +7118,7 @@ void shellReset(
   if( *pRc==SQLITE_OK ){
     if( rc!=SQLITE_OK ){
       sqlite3 *db = sqlite3_db_handle(pStmt);
-      eputf("SQL error: %s\n", sqlite3_errmsg(db));
+      sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db));
     }
     *pRc = rc;
   }
@@ -6889,6 +7147,7 @@ struct ArCommand {
   const char *zDir;               /* --directory argument, or NULL */
   char **azArg;                   /* Array of command arguments */
   ShellState *p;                  /* Shell state */
+  FILE *out;                      /* Output to this stream */
   sqlite3 *db;                    /* Database containing the archive */
 };
 
@@ -6912,9 +7171,9 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){
   va_end(ap);
   shellEmitError(z);
   if( pAr->fromCmdLine ){
-    eputz("Use \"-A\" for more help\n");
+    sqlite3_fputs("Use \"-A\" for more help\n", stderr);
   }else{
-    eputz("Use \".archive --help\" for more help\n");
+    sqlite3_fputs("Use \".archive --help\" for more help\n", stderr);
   }
   sqlite3_free(z);
   return SQLITE_ERROR;
@@ -7014,7 +7273,7 @@ static int arParseCommand(
   struct ArSwitch *pEnd = &aSwitch[nSwitch];
 
   if( nArg<=1 ){
-    eputz("Wrong number of arguments.  Usage:\n");
+    sqlite3_fprintf(stderr, "Wrong number of arguments.  Usage:\n");
     return arUsage(stderr);
   }else{
     char *z = azArg[1];
@@ -7120,7 +7379,7 @@ static int arParseCommand(
     }
   }
   if( pAr->eCmd==0 ){
-    eputz("Required argument missing.  Usage:\n");
+    sqlite3_fprintf(stderr, "Required argument missing.  Usage:\n");
     return arUsage(stderr);
   }
   return SQLITE_OK;
@@ -7163,7 +7422,7 @@ static int arCheckEntries(ArCommand *pAr){
       }
       shellReset(&rc, pTest);
       if( rc==SQLITE_OK && bOk==0 ){
-        eputf("not found in archive: %s\n", z);
+        sqlite3_fprintf(stderr,"not found in archive: %s\n", z);
         rc = SQLITE_ERROR;
       }
     }
@@ -7230,15 +7489,15 @@ static int arListCommand(ArCommand *pAr){
   shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose],
                      pAr->zSrcTable, zWhere);
   if( pAr->bDryRun ){
-    oputf("%s\n", sqlite3_sql(pSql));
+    sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql));
   }else{
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
       if( pAr->bVerbose ){
-        oputf("%s % 10d  %s  %s\n",
+        sqlite3_fprintf(pAr->out, "%s % 10d  %s  %s\n",
               sqlite3_column_text(pSql, 0), sqlite3_column_int(pSql, 1),
               sqlite3_column_text(pSql, 2),sqlite3_column_text(pSql, 3));
       }else{
-        oputf("%s\n", sqlite3_column_text(pSql, 0));
+        sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0));
       }
     }
   }
@@ -7265,7 +7524,7 @@ static int arRemoveCommand(ArCommand *pAr){
     zSql = sqlite3_mprintf("DELETE FROM %s WHERE %s;",
                            pAr->zSrcTable, zWhere);
     if( pAr->bDryRun ){
-      oputf("%s\n", zSql);
+      sqlite3_fprintf(pAr->out, "%s\n", zSql);
     }else{
       char *zErr = 0;
       rc = sqlite3_exec(pAr->db, "SAVEPOINT ar;", 0, 0, 0);
@@ -7278,7 +7537,7 @@ static int arRemoveCommand(ArCommand *pAr){
         }
       }
       if( zErr ){
-        sputf(stdout, "ERROR: %s\n", zErr); /* stdout? */
+        sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); /* stdout? */
         sqlite3_free(zErr);
       }
     }
@@ -7342,11 +7601,11 @@ static int arExtractCommand(ArCommand *pAr){
       j = sqlite3_bind_parameter_index(pSql, "$dirOnly");
       sqlite3_bind_int(pSql, j, i);
       if( pAr->bDryRun ){
-        oputf("%s\n", sqlite3_sql(pSql));
+        sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql));
       }else{
         while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
           if( i==0 && pAr->bVerbose ){
-            oputf("%s\n", sqlite3_column_text(pSql, 0));
+            sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0));
           }
         }
       }
@@ -7366,13 +7625,13 @@ static int arExtractCommand(ArCommand *pAr){
 static int arExecSql(ArCommand *pAr, const char *zSql){
   int rc;
   if( pAr->bDryRun ){
-    oputf("%s\n", zSql);
+    sqlite3_fprintf(pAr->out, "%s\n", zSql);
     rc = SQLITE_OK;
   }else{
     char *zErr = 0;
     rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr);
     if( zErr ){
-      sputf(stdout, "ERROR: %s\n", zErr);
+      sqlite3_fprintf(stdout, "ERROR: %s\n", zErr);
       sqlite3_free(zErr);
     }
   }
@@ -7521,6 +7780,7 @@ static int arDotCommand(
   if( rc==SQLITE_OK ){
     int eDbType = SHELL_OPEN_UNSPEC;
     cmd.p = pState;
+    cmd.out = pState->out;
     cmd.db = pState->db;
     if( cmd.zFile ){
       eDbType = deduceDatabaseType(cmd.zFile, 1);
@@ -7547,13 +7807,14 @@ static int arDotCommand(
       }
       cmd.db = 0;
       if( cmd.bDryRun ){
-        oputf("-- open database '%s'%s\n", cmd.zFile,
+        sqlite3_fprintf(cmd.out, "-- open database '%s'%s\n", cmd.zFile,
               eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : "");
       }
       rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags,
              eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0);
       if( rc!=SQLITE_OK ){
-        eputf("cannot open file: %s (%s)\n", cmd.zFile, sqlite3_errmsg(cmd.db));
+        sqlite3_fprintf(stderr, "cannot open file: %s (%s)\n",
+                        cmd.zFile, sqlite3_errmsg(cmd.db));
         goto end_ar_command;
       }
       sqlite3_fileio_init(cmd.db, 0, 0);
@@ -7566,7 +7827,7 @@ static int arDotCommand(
       if( cmd.eCmd!=AR_CMD_CREATE
        && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0)
       ){
-        eputz("database does not contain an 'sqlar' table\n");
+        sqlite3_fprintf(stderr, "database does not contain an 'sqlar' table\n");
         rc = SQLITE_ERROR;
         goto end_ar_command;
       }
@@ -7624,7 +7885,7 @@ end_ar_command:
 */
 static int recoverSqlCb(void *pCtx, const char *zSql){
   ShellState *pState = (ShellState*)pCtx;
-  sputf(pState->out, "%s;\n", zSql);
+  sqlite3_fprintf(pState->out, "%s;\n", zSql);
   return SQLITE_OK;
 }
 
@@ -7667,7 +7928,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){
       bRowids = 0;
     }
     else{
-      eputf("unexpected option: %s\n", azArg[i]);
+      sqlite3_fprintf(stderr,"unexpected option: %s\n", azArg[i]);
       showHelp(pState->out, azArg[0]);
       return 1;
     }
@@ -7686,7 +7947,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){
   if( sqlite3_recover_errcode(p)!=SQLITE_OK ){
     const char *zErr = sqlite3_recover_errmsg(p);
     int errCode = sqlite3_recover_errcode(p);
-    eputf("sql error: %s (%d)\n", zErr, errCode);
+    sqlite3_fprintf(stderr,"sql error: %s (%d)\n", zErr, errCode);
   }
   rc = sqlite3_recover_finish(p);
   return rc;
@@ -7708,7 +7969,7 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){
     while( SQLITE_OK==sqlite3_intck_step(p) ){
       const char *zMsg = sqlite3_intck_message(p);
       if( zMsg ){
-        oputf("%s\n", zMsg);
+        sqlite3_fprintf(pState->out, "%s\n", zMsg);
         nError++;
       }
       nStep++;
@@ -7718,11 +7979,11 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){
     }
     rc = sqlite3_intck_error(p, &zErr);
     if( zErr ){
-      eputf("%s\n", zErr);
+      sqlite3_fprintf(stderr,"%s\n", zErr);
     }
     sqlite3_intck_close(p);
 
-    oputf("%lld steps, %lld errors\n", nStep, nError);
+    sqlite3_fprintf(pState->out, "%lld steps, %lld errors\n", nStep, nError);
   }
 
   return rc;
@@ -7745,7 +8006,7 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){
 #define rc_err_oom_die(rc) \
   if( rc==SQLITE_NOMEM ) shell_check_oom(0); \
   else if(!(rc==SQLITE_OK||rc==SQLITE_DONE)) \
-    eputf("E:%d\n",rc), assert(0)
+    sqlite3_fprintf(stderr,"E:%d\n",rc), assert(0)
 #else
 static void rc_err_oom_die(int rc){
   if( rc==SQLITE_NOMEM ) shell_check_oom(0);
@@ -7962,8 +8223,9 @@ static int outputDumpWarning(ShellState *p, const char *zLike){
     "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true"
   );
   if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
-    oputz("/* WARNING: "
-          "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n"
+    sqlite3_fputs("/* WARNING: "
+          "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n",
+          p->out
     );
   }
   shellFinalize(&rc, pStmt);
@@ -7994,12 +8256,14 @@ static int faultsim_callback(int iArg){
   if( faultsim_state.iCnt ){
     if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--;
     if( faultsim_state.eVerbose>=2 ){
-      oputf("FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt);
+      sqlite3_fprintf(stdout, 
+         "FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt);
     }
     return SQLITE_OK;
   }
   if( faultsim_state.eVerbose>=1 ){
-    oputf("FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr);
+    sqlite3_fprintf(stdout, 
+         "FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr);
   }
   faultsim_state.iCnt = faultsim_state.iInterval;
   faultsim_state.nHit++;
@@ -8062,7 +8326,7 @@ static int do_meta_command(char *zLine, ShellState *p){
 #ifndef SQLITE_OMIT_AUTHORIZATION
   if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){
     if( nArg!=2 ){
-      eputz("Usage: .auth ON|OFF\n");
+      sqlite3_fprintf(stderr, "Usage: .auth ON|OFF\n");
       rc = 1;
       goto meta_command_exit;
     }
@@ -8109,7 +8373,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           bAsync = 1;
         }else
         {
-          eputf("unknown option: %s\n", azArg[j]);
+          sqlite3_fprintf(stderr,"unknown option: %s\n", azArg[j]);
           return 1;
         }
       }else if( zDestFile==0 ){
@@ -8118,19 +8382,19 @@ static int do_meta_command(char *zLine, ShellState *p){
         zDb = zDestFile;
         zDestFile = azArg[j];
       }else{
-        eputz("Usage: .backup ?DB? ?OPTIONS? FILENAME\n");
+        sqlite3_fprintf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n");
         return 1;
       }
     }
     if( zDestFile==0 ){
-      eputz("missing FILENAME argument on .backup\n");
+      sqlite3_fprintf(stderr, "missing FILENAME argument on .backup\n");
       return 1;
     }
     if( zDb==0 ) zDb = "main";
     rc = sqlite3_open_v2(zDestFile, &pDest,
                   SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs);
     if( rc!=SQLITE_OK ){
-      eputf("Error: cannot open \"%s\"\n", zDestFile);
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zDestFile);
       close_db(pDest);
       return 1;
     }
@@ -8168,17 +8432,8 @@ static int do_meta_command(char *zLine, ShellState *p){
 
   /* Undocumented.  Legacy only.  See "crnl" below */
   if( c=='b' && n>=3 && cli_strncmp(azArg[0], "binary", n)==0 ){
-    if( nArg==2 ){
-      if( booleanValue(azArg[1]) ){
-        setBinaryMode(p->out, 1);
-      }else{
-        setTextMode(p->out, 1);
-      }
-    }else{
-      eputz("The \".binary\" command is deprecated. Use \".crnl\" instead.\n"
-            "Usage: .binary on|off\n");
-      rc = 1;
-    }
+    eputz("The \".binary\" command is deprecated. Use \".crnl\" instead.\n");
+    rc = 1;
   }else
 
   /* The undocumented ".breakpoint" command causes a call to the no-op
@@ -8200,7 +8455,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       rc = chdir(azArg[1]);
 #endif
       if( rc ){
-        eputf("Cannot change to directory \"%s\"\n", azArg[1]);
+        sqlite3_fprintf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]);
         rc = 1;
       }
     }else{
@@ -8233,11 +8488,12 @@ static int do_meta_command(char *zLine, ShellState *p){
     }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){
       rc = 2;
     }else if( testcase_glob(azArg[1],zRes)==0 ){
-      eputf("testcase-%s FAILED\n Expected: [%s]\n      Got: [%s]\n",
+      sqlite3_fprintf(stderr,
+            "testcase-%s FAILED\n Expected: [%s]\n      Got: [%s]\n",
             p->zTestcase, azArg[1], zRes);
       rc = 1;
     }else{
-      oputf("testcase-%s ok\n", p->zTestcase);
+      sqlite3_fprintf(p->out, "testcase-%s ok\n", p->zTestcase);
       p->nCheck++;
     }
     sqlite3_free(zRes);
@@ -8270,9 +8526,9 @@ static int do_meta_command(char *zLine, ShellState *p){
           zFile = "(temporary-file)";
         }
         if( p->pAuxDb == &p->aAuxDb[i] ){
-          sputf(stdout, "ACTIVE %d: %s\n", i, zFile);
+          sqlite3_fprintf(stdout, "ACTIVE %d: %s\n", i, zFile);
         }else if( p->aAuxDb[i].db!=0 ){
-          sputf(stdout, "       %d: %s\n", i, zFile);
+          sqlite3_fprintf(stdout, "       %d: %s\n", i, zFile);
         }
       }
     }else if( nArg==2 && IsDigit(azArg[1][0]) && azArg[1][1]==0 ){
@@ -8303,19 +8559,21 @@ static int do_meta_command(char *zLine, ShellState *p){
   }else
 
   if( c=='c' && n==4 && cli_strncmp(azArg[0], "crnl", n)==0 ){
+#if !defined(_WIN32) || defined(SQLITE_U8TEXT_ONLY) \
+                     || defined(SQLITE_U8TEXT_STDIO)
+    sqlite3_fputs("The \".crnl\" command is disable in this build.\n", p->out);
+#else
     if( nArg==2 ){
       if( booleanValue(azArg[1]) ){
-        setTextMode(p->out, 1);
+        sqlite3_fsetmode(p->out, _O_TEXT);
       }else{
-        setBinaryMode(p->out, 1);
+        sqlite3_fsetmode(p->out, _O_BINARY);
       }
     }else{
-#if !defined(_WIN32) && !defined(WIN32)
-      eputz("The \".crnl\" is a no-op on non-Windows machines.\n");
-#endif
       eputz("Usage: .crnl on|off\n");
       rc = 1;
     }
+#endif
   }else
 
   if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){
@@ -8345,7 +8603,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       int eTxn = sqlite3_txn_state(p->db, azName[i*2]);
       int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]);
       const char *z = azName[i*2+1];
-      oputf("%s: %s %s%s\n",
+      sqlite3_fprintf(p->out, "%s: %s %s%s\n",
             azName[i*2], z && z[0] ? z : "\"\"", bRdonly ? "r/o" : "r/w",
             eTxn==SQLITE_TXN_NONE ? "" :
             eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn");
@@ -8387,11 +8645,12 @@ static int do_meta_command(char *zLine, ShellState *p){
         sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0);
       }
       sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v);
-      oputf("%19s %s\n", aDbConfig[ii].zName, v ? "on" : "off");
+      sqlite3_fprintf(p->out, "%19s %s\n",
+                      aDbConfig[ii].zName, v ? "on" : "off");
       if( nArg>1 ) break;
     }
     if( nArg>1 && ii==ArraySize(aDbConfig) ){
-      eputf("Error: unknown dbconfig \"%s\"\n", azArg[1]);
+      sqlite3_fprintf(stderr,"Error: unknown dbconfig \"%s\"\n", azArg[1]);
       eputz("Enter \".dbconfig\" with no arguments for a list\n");
     }
   }else
@@ -8441,7 +8700,8 @@ static int do_meta_command(char *zLine, ShellState *p){
           ShellSetFlag(p, SHFLG_DumpNoSys);
         }else
         {
-          eputf("Unknown option \"%s\" on \".dump\"\n", azArg[i]);
+          sqlite3_fprintf(stderr,
+               "Unknown option \"%s\" on \".dump\"\n", azArg[i]);
           rc = 1;
           sqlite3_free(zLike);
           goto meta_command_exit;
@@ -8476,8 +8736,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       /* When playing back a "dump", the content might appear in an order
       ** which causes immediate foreign key constraints to be violated.
       ** So disable foreign-key constraint enforcement to prevent problems. */
-      oputz("PRAGMA foreign_keys=OFF;\n");
-      oputz("BEGIN TRANSACTION;\n");
+      sqlite3_fputs("PRAGMA foreign_keys=OFF;\n", p->out);
+      sqlite3_fputs("BEGIN TRANSACTION;\n", p->out);
     }
     p->writableSchema = 0;
     p->showHeader = 0;
@@ -8509,13 +8769,13 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
     sqlite3_free(zLike);
     if( p->writableSchema ){
-      oputz("PRAGMA writable_schema=OFF;\n");
+      sqlite3_fputs("PRAGMA writable_schema=OFF;\n", p->out);
       p->writableSchema = 0;
     }
     sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
     sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0);
     if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
-      oputz(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n");
+      sqlite3_fputs(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out);
     }
     p->showHeader = savedShowHeader;
     p->shellFlgs = savedShellFlags;
@@ -8595,7 +8855,8 @@ static int do_meta_command(char *zLine, ShellState *p){
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){
     if( p->bSafeMode ){
-      eputf("Cannot run experimental commands such as \"%s\" in safe mode\n",
+      sqlite3_fprintf(stderr,
+            "Cannot run experimental commands such as \"%s\" in safe mode\n",
             azArg[0]);
       rc = 1;
     }else{
@@ -8652,9 +8913,10 @@ static int do_meta_command(char *zLine, ShellState *p){
 
     /* --help lists all file-controls */
     if( cli_strcmp(zCmd,"help")==0 ){
-      oputz("Available file-controls:\n");
+      sqlite3_fputs("Available file-controls:\n", p->out);
       for(i=0; iout,
+               "  .filectrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage);
       }
       rc = 1;
       goto meta_command_exit;
@@ -8669,7 +8931,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           filectrl = aCtrl[i].ctrlCode;
           iCtrl = i;
         }else{
-          eputf("Error: ambiguous file-control: \"%s\"\n"
+          sqlite3_fprintf(stderr,"Error: ambiguous file-control: \"%s\"\n"
                 "Use \".filectrl --help\" for help\n", zCmd);
           rc = 1;
           goto meta_command_exit;
@@ -8677,7 +8939,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
     }
     if( filectrl<0 ){
-      eputf("Error: unknown file-control: %s\n"
+      sqlite3_fprintf(stderr,"Error: unknown file-control: %s\n"
             "Use \".filectrl --help\" for help\n", zCmd);
     }else{
       switch(filectrl){
@@ -8721,7 +8983,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           if( nArg!=2 ) break;
           sqlite3_file_control(p->db, zSchema, filectrl, &z);
           if( z ){
-            oputf("%s\n", z);
+            sqlite3_fprintf(p->out, "%s\n", z);
             sqlite3_free(z);
           }
           isOk = 2;
@@ -8735,19 +8997,20 @@ static int do_meta_command(char *zLine, ShellState *p){
           }
           x = -1;
           sqlite3_file_control(p->db, zSchema, filectrl, &x);
-          oputf("%d\n", x);
+          sqlite3_fprintf(p->out, "%d\n", x);
           isOk = 2;
           break;
         }
       }
     }
     if( isOk==0 && iCtrl>=0 ){
-      oputf("Usage: .filectrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage);
+      sqlite3_fprintf(p->out, "Usage: .filectrl %s %s\n",
+                      zCmd, aCtrl[iCtrl].zUsage);
       rc = 1;
     }else if( isOk==1 ){
       char zBuf[100];
       sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes);
-      oputf("%s\n", zBuf);
+      sqlite3_fprintf(p->out, "%s\n", zBuf);
     }
   }else
 
@@ -8788,15 +9051,15 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
     }
     if( doStats==0 ){
-      oputz("/* No STAT tables available */\n");
+      sqlite3_fputs("/* No STAT tables available */\n", p->out);
     }else{
-      oputz("ANALYZE sqlite_schema;\n");
+      sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out);
       data.cMode = data.mode = MODE_Insert;
       data.zDestTable = "sqlite_stat1";
       shell_exec(&data, "SELECT * FROM sqlite_stat1", 0);
       data.zDestTable = "sqlite_stat4";
       shell_exec(&data, "SELECT * FROM sqlite_stat4", 0);
-      oputz("ANALYZE sqlite_schema;\n");
+      sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out);
     }
   }else
 
@@ -8814,7 +9077,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( nArg>=2 ){
       n = showHelp(p->out, azArg[1]);
       if( n==0 ){
-        oputf("Nothing matches '%s'\n", azArg[1]);
+        sqlite3_fprintf(p->out, "Nothing matches '%s'\n", azArg[1]);
       }
     }else{
       showHelp(p->out, 0);
@@ -8857,7 +9120,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         }else if( zTable==0 ){
           zTable = z;
         }else{
-          oputf("ERROR: extra argument: \"%s\".  Usage:\n", z);
+          sqlite3_fprintf(p->out, "ERROR: extra argument: \"%s\". Usage:\n",z);
           showHelp(p->out, "import");
           goto meta_command_exit;
         }
@@ -8878,13 +9141,13 @@ static int do_meta_command(char *zLine, ShellState *p){
         xRead = csv_read_one_field;
         useOutputMode = 0;
       }else{
-        oputf("ERROR: unknown option: \"%s\".  Usage:\n", z);
+        sqlite3_fprintf(p->out, "ERROR: unknown option: \"%s\".  Usage:\n", z);
         showHelp(p->out, "import");
         goto meta_command_exit;
       }
     }
     if( zTable==0 ){
-      oputf("ERROR: missing %s argument. Usage:\n",
+      sqlite3_fprintf(p->out, "ERROR: missing %s argument. Usage:\n",
             zFile==0 ? "FILE" : "TABLE");
       showHelp(p->out, "import");
       goto meta_command_exit;
@@ -8934,28 +9197,28 @@ static int do_meta_command(char *zLine, ShellState *p){
       eputz("Error: pipes are not supported in this OS\n");
       goto meta_command_exit;
 #else
-      sCtx.in = popen(sCtx.zFile+1, "r");
+      sCtx.in = sqlite3_popen(sCtx.zFile+1, "r");
       sCtx.zFile = "";
       sCtx.xCloser = pclose;
 #endif
     }else{
-      sCtx.in = fopen(sCtx.zFile, "rb");
+      sCtx.in = sqlite3_fopen(sCtx.zFile, "rb");
       sCtx.xCloser = fclose;
     }
     if( sCtx.in==0 ){
-      eputf("Error: cannot open \"%s\"\n", zFile);
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile);
       goto meta_command_exit;
     }
     if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){
       char zSep[2];
       zSep[1] = 0;
       zSep[0] = sCtx.cColSep;
-      oputz("Column separator ");
-      output_c_string(zSep);
-      oputz(", row separator ");
+      sqlite3_fputs("Column separator ", p->out);
+      output_c_string(p->out, zSep);
+      sqlite3_fputs(", row separator ", p->out);
       zSep[0] = sCtx.cRowSep;
-      output_c_string(zSep);
-      oputz("\n");
+      output_c_string(p->out, zSep);
+      sqlite3_fputs("\n", p->out);
     }
     sCtx.z = sqlite3_malloc64(120);
     if( sCtx.z==0 ){
@@ -8980,14 +9243,14 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
       zColDefs = zAutoColumn(0, &dbCols, &zRenames);
       if( zRenames!=0 ){
-        sputf((stdin_is_interactive && p->in==stdin)? p->out : stderr,
+        sqlite3_fprintf((stdin_is_interactive && p->in==stdin)? p->out : stderr,
               "Columns renamed during .import %s due to duplicates:\n"
               "%s\n", sCtx.zFile, zRenames);
         sqlite3_free(zRenames);
       }
       assert(dbCols==0);
       if( zColDefs==0 ){
-        eputf("%s: empty file\n", sCtx.zFile);
+        sqlite3_fprintf(stderr,"%s: empty file\n", sCtx.zFile);
         import_cleanup(&sCtx);
         rc = 1;
         sqlite3_free(zCreate);
@@ -8999,13 +9262,16 @@ static int do_meta_command(char *zLine, ShellState *p){
         shell_out_of_memory();
       }
       if( eVerbose>=1 ){
-        oputf("%s\n", zCreate);
+        sqlite3_fprintf(p->out, "%s\n", zCreate);
       }
       rc = sqlite3_exec(p->db, zCreate, 0, 0, 0);
+      if( rc ){
+        sqlite3_fprintf(stderr,
+             "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db));
+      }
       sqlite3_free(zCreate);
       zCreate = 0;
       if( rc ){
-        eputf("%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db));
         import_cleanup(&sCtx);
         rc = 1;
         goto meta_command_exit;
@@ -9060,7 +9326,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     zSql[j] = 0;
     assert( j=2 ){
-      oputf("Insert using: %s\n", zSql);
+      sqlite3_fprintf(p->out, "Insert using: %s\n", zSql);
     }
     rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
     sqlite3_free(zSql);
@@ -9099,7 +9365,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         }
         sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT);
         if( i=nCol ){
         sqlite3_step(pStmt);
         rc = sqlite3_reset(pStmt);
         if( rc!=SQLITE_OK ){
-          eputf("%s:%d: INSERT failed: %s\n",
+          sqlite3_fprintf(stderr,"%s:%d: INSERT failed: %s\n",
                 sCtx.zFile, startLine, sqlite3_errmsg(p->db));
           sCtx.nErr++;
         }else{
@@ -9131,7 +9398,8 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_finalize(pStmt);
     if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0);
     if( eVerbose>0 ){
-      oputf("Added %d rows with %d errors using %d lines of input\n",
+      sqlite3_fprintf(p->out,
+            "Added %d rows with %d errors using %d lines of input\n",
             sCtx.nRow, sCtx.nErr, sCtx.nLine-1);
     }
   }else
@@ -9147,7 +9415,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */
     int i;
     if( !ShellHasFlag(p,SHFLG_TestingMode) ){
-      eputf(".%s unavailable without --unsafe-testing\n",
+      sqlite3_fprintf(stderr,".%s unavailable without --unsafe-testing\n",
             "imposter");
       rc = 1;
       goto meta_command_exit;
@@ -9213,7 +9481,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
     sqlite3_finalize(pStmt);
     if( i==0 || tnum==0 ){
-      eputf("no such index: \"%s\"\n", azArg[1]);
+      sqlite3_fprintf(stderr,"no such index: \"%s\"\n", azArg[1]);
       rc = 1;
       sqlite3_free(zCollist);
       goto meta_command_exit;
@@ -9228,14 +9496,16 @@ static int do_meta_command(char *zLine, ShellState *p){
       rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
       sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0);
       if( rc ){
-        eputf("Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db));
+        sqlite3_fprintf(stderr,
+              "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db));
       }else{
-        sputf(stdout, "%s;\n", zSql);
-        sputf(stdout, "WARNING: writing to an imposter table will corrupt"
+        sqlite3_fprintf(stdout, "%s;\n", zSql);
+        sqlite3_fprintf(stdout,
+              "WARNING: writing to an imposter table will corrupt"
               " the \"%s\" %s!\n", azArg[1], isWO ? "table" : "index");
       }
     }else{
-      eputf("SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc);
+      sqlite3_fprintf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc);
       rc = 1;
     }
     sqlite3_free(zSql);
@@ -9249,7 +9519,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( iArg==0 ) iArg = -1;
     }
     if( (nArg!=1 && nArg!=2) || iArg<0 ){
-      eputf("%s","Usage: .intck STEPS_PER_UNLOCK\n");
+      sqlite3_fprintf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n");
       rc = 1;
       goto meta_command_exit;
     }
@@ -9268,9 +9538,9 @@ static int do_meta_command(char *zLine, ShellState *p){
       sqlite3IoTrace = iotracePrintf;
       iotrace = stdout;
     }else{
-      iotrace = fopen(azArg[1], "w");
+      iotrace = sqlite3_fopen(azArg[1], "w");
       if( iotrace==0 ){
-        eputf("Error: cannot open \"%s\"\n", azArg[1]);
+        sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
         sqlite3IoTrace = 0;
         rc = 1;
       }else{
@@ -9302,7 +9572,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     open_db(p, 0);
     if( nArg==1 ){
       for(i=0; idb, aLimit[i].limitCode, -1));
       }
     }else if( nArg>3 ){
@@ -9317,14 +9587,14 @@ static int do_meta_command(char *zLine, ShellState *p){
           if( iLimit<0 ){
             iLimit = i;
           }else{
-            eputf("ambiguous limit: \"%s\"\n", azArg[1]);
+            sqlite3_fprintf(stderr,"ambiguous limit: \"%s\"\n", azArg[1]);
             rc = 1;
             goto meta_command_exit;
           }
         }
       }
       if( iLimit<0 ){
-        eputf("unknown limit: \"%s\"\n"
+        sqlite3_fprintf(stderr,"unknown limit: \"%s\"\n"
               "enter \".limits\" with no arguments for a list.\n",
               azArg[1]);
         rc = 1;
@@ -9334,7 +9604,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         sqlite3_limit(p->db, aLimit[iLimit].limitCode,
                       (int)integerValue(azArg[2]));
       }
-      sputf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName,
+      sqlite3_fprintf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName,
             sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1));
     }
   }else
@@ -9417,7 +9687,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }else if( zTabname==0 ){
         zTabname = z;
       }else if( z[0]=='-' ){
-        eputf("unknown option: %s\n", z);
+        sqlite3_fprintf(stderr,"unknown option: %s\n", z);
         eputz("options:\n"
               "  --noquote\n"
               "  --quote\n"
@@ -9427,7 +9697,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         rc = 1;
         goto meta_command_exit;
       }else{
-        eputf("extra argument: \"%s\"\n", z);
+        sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z);
         rc = 1;
         goto meta_command_exit;
       }
@@ -9436,12 +9706,14 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( p->mode==MODE_Column
        || (p->mode>=MODE_Markdown && p->mode<=MODE_Box)
       ){
-        oputf("current output mode: %s --wrap %d --wordwrap %s --%squote\n",
+        sqlite3_fprintf(p->out,
+              "current output mode: %s --wrap %d --wordwrap %s --%squote\n",
               modeDescr[p->mode], p->cmOpts.iWrap,
               p->cmOpts.bWordWrap ? "on" : "off",
               p->cmOpts.bQuote ? "" : "no");
       }else{
-        oputf("current output mode: %s\n", modeDescr[p->mode]);
+        sqlite3_fprintf(p->out,
+              "current output mode: %s\n", modeDescr[p->mode]);
       }
       zMode = modeDescr[p->mode];
     }
@@ -9514,7 +9786,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       eputz("Usage: .nonce NONCE\n");
       rc = 1;
     }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){
-      eputf("line %d: incorrect nonce: \"%s\"\n",
+      sqlite3_fprintf(stderr,"line %d: incorrect nonce: \"%s\"\n",
             p->lineno, azArg[1]);
       exit(1);
     }else{
@@ -9569,11 +9841,11 @@ static int do_meta_command(char *zLine, ShellState *p){
       }else
 #endif /* !SQLITE_SHELL_FIDDLE */
       if( z[0]=='-' ){
-        eputf("unknown option: %s\n", z);
+        sqlite3_fprintf(stderr,"unknown option: %s\n", z);
         rc = 1;
         goto meta_command_exit;
       }else if( zFN ){
-        eputf("extra argument: \"%s\"\n", z);
+        sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z);
         rc = 1;
         goto meta_command_exit;
       }else{
@@ -9615,7 +9887,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       p->pAuxDb->zDbFilename = zNewFilename;
       open_db(p, OPEN_DB_KEEPALIVE);
       if( p->db==0 ){
-        eputf("Error: cannot open '%s'\n", zNewFilename);
+        sqlite3_fprintf(stderr,"Error: cannot open '%s'\n", zNewFilename);
         sqlite3_free(zNewFilename);
       }else{
         p->pAuxDb->zFreeOnClose = zNewFilename;
@@ -9633,19 +9905,24 @@ static int do_meta_command(char *zLine, ShellState *p){
         && (cli_strncmp(azArg[0], "output", n)==0
             || cli_strncmp(azArg[0], "once", n)==0))
    || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0)
+   || (c=='w' && n==3 && cli_strcmp(azArg[0],"www")==0)
   ){
     char *zFile = 0;
     int bTxtMode = 0;
     int i;
     int eMode = 0;
-    int bOnce = 0;            /* 0: .output, 1: .once, 2: .excel */
-    static const char *zBomUtf8 = "\xef\xbb\xbf";
+    int bOnce = 0;            /* 0: .output, 1: .once, 2: .excel/.www */
+    int bPlain = 0;           /* --plain option */
+    static const char *zBomUtf8 = "\357\273\277";
     const char *zBom = 0;
 
     failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]);
     if( c=='e' ){
       eMode = 'x';
       bOnce = 2;
+    }else if( c=='w' ){
+      eMode = 'w';
+      bOnce = 2;
     }else if( cli_strncmp(azArg[0],"once",n)==0 ){
       bOnce = 1;
     }
@@ -9655,24 +9932,30 @@ static int do_meta_command(char *zLine, ShellState *p){
         if( z[1]=='-' ) z++;
         if( cli_strcmp(z,"-bom")==0 ){
           zBom = zBomUtf8;
-        }else if( c!='e' && cli_strcmp(z,"-x")==0 ){
+        }else if( cli_strcmp(z,"-plain")==0 ){
+          bPlain = 1;
+        }else if( c=='o' && cli_strcmp(z,"-x")==0 ){
           eMode = 'x';  /* spreadsheet */
-        }else if( c!='e' && cli_strcmp(z,"-e")==0 ){
+        }else if( c=='o' && cli_strcmp(z,"-e")==0 ){
           eMode = 'e';  /* text editor */
+        }else if( c=='o' && cli_strcmp(z,"-w")==0 ){
+          eMode = 'w';  /* Web browser */
         }else{
-          oputf("ERROR: unknown option: \"%s\".  Usage:\n", azArg[i]);
+          sqlite3_fprintf(p->out, 
+                          "ERROR: unknown option: \"%s\". Usage:\n", azArg[i]);
           showHelp(p->out, azArg[0]);
           rc = 1;
           goto meta_command_exit;
         }
-      }else if( zFile==0 && eMode!='e' && eMode!='x' ){
+      }else if( zFile==0 && eMode==0 ){
         zFile = sqlite3_mprintf("%s", z);
         if( zFile && zFile[0]=='|' ){
           while( i+1out,
+            "ERROR: extra parameter: \"%s\".  Usage:\n", azArg[i]);
         showHelp(p->out, azArg[0]);
         rc = 1;
         sqlite3_free(zFile);
@@ -9689,7 +9972,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
     output_reset(p);
 #ifndef SQLITE_NOHAVE_SYSTEM
-    if( eMode=='e' || eMode=='x' ){
+    if( eMode=='e' || eMode=='x' || eMode=='w' ){
       p->doXdgOpen = 1;
       outputModePush(p);
       if( eMode=='x' ){
@@ -9699,6 +9982,15 @@ static int do_meta_command(char *zLine, ShellState *p){
         p->mode = MODE_Csv;
         sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);
         sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf);
+#ifdef _WIN32
+        zBom = zBomUtf8;  /* Always include the BOM on Windows, as Excel does
+                          ** not work without it. */
+#endif
+      }else if( eMode=='w' ){
+        /* web-browser mode. */
+        newTempFile(p, "html");
+        if( !bPlain ) p->mode = MODE_Www;
+        bTxtMode = 1;
       }else{
         /* text editor mode */
         newTempFile(p, "txt");
@@ -9715,13 +10007,13 @@ static int do_meta_command(char *zLine, ShellState *p){
       rc = 1;
       output_redir(p, stdout);
 #else
-      FILE *pfPipe = popen(zFile + 1, "w");
+      FILE *pfPipe = sqlite3_popen(zFile + 1, "w");
       if( pfPipe==0 ){
-        eputf("Error: cannot open pipe \"%s\"\n", zFile + 1);
+        sqlite3_fprintf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1);
         rc = 1;
       }else{
         output_redir(p, pfPipe);
-        if( zBom ) oputz(zBom);
+        if( zBom ) sqlite3_fputs(zBom, pfPipe);
         sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile);
       }
 #endif
@@ -9729,12 +10021,18 @@ static int do_meta_command(char *zLine, ShellState *p){
       FILE *pfFile = output_file_open(zFile, bTxtMode);
       if( pfFile==0 ){
         if( cli_strcmp(zFile,"off")!=0 ){
-          eputf("Error: cannot write to \"%s\"\n", zFile);
+          sqlite3_fprintf(stderr,"Error: cannot write to \"%s\"\n", zFile);
         }
         rc = 1;
       } else {
         output_redir(p, pfFile);
-        if( zBom ) oputz(zBom);
+        if( bPlain && eMode=='w' ){
+          sqlite3_fputs(
+            "\n\n\n",
+            pfFile
+          );
+        }
+        if( zBom ) sqlite3_fputs(zBom, pfFile);
         sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile);
       }
     }
@@ -9775,7 +10073,8 @@ static int do_meta_command(char *zLine, ShellState *p){
              "SELECT key, quote(value) "
              "FROM temp.sqlite_parameters;", -1, &pStmt, 0);
         while( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
-          oputf("%-*s %s\n", len, sqlite3_column_text(pStmt,0),
+          sqlite3_fprintf(p->out,
+                "%-*s %s\n", len, sqlite3_column_text(pStmt,0),
                 sqlite3_column_text(pStmt,1));
         }
         sqlite3_finalize(pStmt);
@@ -9820,7 +10119,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
         sqlite3_free(zSql);
         if( rx!=SQLITE_OK ){
-          oputf("Error: %s\n", sqlite3_errmsg(p->db));
+          sqlite3_fprintf(p->out, "Error: %s\n", sqlite3_errmsg(p->db));
           sqlite3_finalize(pStmt);
           pStmt = 0;
           rc = 1;
@@ -9849,10 +10148,10 @@ static int do_meta_command(char *zLine, ShellState *p){
   if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){
     int i;
     for(i=1; i<nArg; i++){
-      if( i>1 ) oputz(" ");
-      oputz(azArg[i]);
+      if( i>1 ) sqlite3_fputs(" ", p->out);
+      sqlite3_fputs(azArg[i], p->out);
     }
-    oputz("\n");
+    sqlite3_fputs("\n", p->out);
   }else
 
 #ifndef SQLITE_OMIT_PROGRESS_CALLBACK
@@ -9889,7 +10188,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           }
           continue;
         }
-        eputf("Error: unknown option: \"%s\"\n", azArg[i]);
+        sqlite3_fprintf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]);
         rc = 1;
         goto meta_command_exit;
       }else{
@@ -9932,9 +10231,9 @@ static int do_meta_command(char *zLine, ShellState *p){
       rc = 1;
       p->out = stdout;
 #else
-      p->in = popen(azArg[1]+1, "r");
+      p->in = sqlite3_popen(azArg[1]+1, "r");
       if( p->in==0 ){
-        eputf("Error: cannot open \"%s\"\n", azArg[1]);
+        sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
         rc = 1;
       }else{
         rc = process_input(p);
@@ -9942,7 +10241,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
 #endif
     }else if( (p->in = openChrSource(azArg[1]))==0 ){
-      eputf("Error: cannot open \"%s\"\n", azArg[1]);
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
       rc = 1;
     }else{
       rc = process_input(p);
@@ -9975,7 +10274,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
     rc = sqlite3_open(zSrcFile, &pSrc);
     if( rc!=SQLITE_OK ){
-      eputf("Error: cannot open \"%s\"\n", zSrcFile);
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zSrcFile);
       close_db(pSrc);
       return 1;
     }
@@ -10058,7 +10357,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }else if( optionMatch(azArg[ii],"nosys") ){
         bNoSystemTabs = 1;
       }else if( azArg[ii][0]=='-' ){
-        eputf("Unknown option: \"%s\"\n", azArg[ii]);
+        sqlite3_fprintf(stderr,"Unknown option: \"%s\"\n", azArg[ii]);
         rc = 1;
         goto meta_command_exit;
       }else if( zName==0 ){
@@ -10159,7 +10458,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       appendText(&sSelect, "sql IS NOT NULL"
                            " ORDER BY snum, rowid", 0);
       if( bDebug ){
-        oputf("SQL: %s;\n", sSelect.z);
+        sqlite3_fprintf(p->out, "SQL: %s;\n", sSelect.z);
       }else{
         rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg);
       }
@@ -10220,7 +10519,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       }else{
         rc = sqlite3session_attach(pSession->p, azCmd[1]);
         if( rc ){
-          eputf("ERROR: sqlite3session_attach() returns %d\n",rc);
+          sqlite3_fprintf(stderr,
+               "ERROR: sqlite3session_attach() returns %d\n",rc);
           rc = 0;
         }
       }
@@ -10237,9 +10537,9 @@ static int do_meta_command(char *zLine, ShellState *p){
       failIfSafeMode(p, "cannot run \".session %s\" in safe mode", azCmd[0]);
       if( nCmd!=2 ) goto session_syntax_error;
       if( pSession->p==0 ) goto session_not_open;
-      out = fopen(azCmd[1], "wb");
+      out = sqlite3_fopen(azCmd[1], "wb");
       if( out==0 ){
-        eputf("ERROR: cannot open \"%s\" for writing\n",
+        sqlite3_fprintf(stderr,"ERROR: cannot open \"%s\" for writing\n",
               azCmd[1]);
       }else{
         int szChng;
@@ -10250,12 +10550,13 @@ static int do_meta_command(char *zLine, ShellState *p){
           rc = sqlite3session_patchset(pSession->p, &szChng, &pChng);
         }
         if( rc ){
-          sputf(stdout, "Error: error code %d\n", rc);
+          sqlite3_fprintf(stdout, "Error: error code %d\n", rc);
           rc = 0;
         }
         if( pChng
           && fwrite(pChng, szChng, 1, out)!=1 ){
-          eputf("ERROR: Failed to write entire %d-byte output\n", szChng);
+          sqlite3_fprintf(stderr,
+              "ERROR: Failed to write entire %d-byte output\n", szChng);
         }
         sqlite3_free(pChng);
         fclose(out);
@@ -10282,7 +10583,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
       if( pAuxDb->nSession ){
         ii = sqlite3session_enable(pSession->p, ii);
-        oputf("session %s enable flag = %d\n", pSession->zName, ii);
+        sqlite3_fprintf(p->out,
+            "session %s enable flag = %d\n", pSession->zName, ii);
       }
     }else
 
@@ -10317,7 +10619,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
       if( pAuxDb->nSession ){
         ii = sqlite3session_indirect(pSession->p, ii);
-        oputf("session %s indirect flag = %d\n", pSession->zName, ii);
+        sqlite3_fprintf(p->out,
+            "session %s indirect flag = %d\n", pSession->zName, ii);
       }
     }else
 
@@ -10329,7 +10632,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( nCmd!=1 ) goto session_syntax_error;
       if( pAuxDb->nSession ){
         ii = sqlite3session_isempty(pSession->p);
-        oputf("session %s isempty flag = %d\n", pSession->zName, ii);
+        sqlite3_fprintf(p->out,
+             "session %s isempty flag = %d\n", pSession->zName, ii);
       }
     }else
 
@@ -10338,7 +10642,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     */
     if( cli_strcmp(azCmd[0],"list")==0 ){
       for(i=0; i<pAuxDb->nSession; i++){
-        oputf("%d %s\n", i, pAuxDb->aSession[i].zName);
+        sqlite3_fprintf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName);
       }
     }else
 
@@ -10353,18 +10657,19 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( zName[0]==0 ) goto session_syntax_error;
       for(i=0; i<pAuxDb->nSession; i++){
         if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){
-          eputf("Session \"%s\" already exists\n", zName);
+          sqlite3_fprintf(stderr,"Session \"%s\" already exists\n", zName);
           goto meta_command_exit;
         }
       }
       if( pAuxDb->nSession>=ArraySize(pAuxDb->aSession) ){
-        eputf("Maximum of %d sessions\n", ArraySize(pAuxDb->aSession));
+        sqlite3_fprintf(stderr,
+           "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession));
         goto meta_command_exit;
       }
       pSession = &pAuxDb->aSession[pAuxDb->nSession];
       rc = sqlite3session_create(p->db, azCmd[1], &pSession->p);
       if( rc ){
-        eputf("Cannot open session: error code=%d\n", rc);
+        sqlite3_fprintf(stderr,"Cannot open session: error code=%d\n", rc);
         rc = 0;
         goto meta_command_exit;
       }
@@ -10388,7 +10693,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       int i, v;
       for(i=1; i<nArg; i++){
         v = booleanValue(azArg[i]);
-        oputf("%s: %d 0x%x\n", azArg[i], v, v);
+        sqlite3_fprintf(p->out, "%s: %d 0x%x\n", azArg[i], v, v);
       }
     }
     if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){
@@ -10397,7 +10702,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         char zBuf[200];
         v = integerValue(azArg[i]);
         sqlite3_snprintf(sizeof(zBuf),zBuf,"%s: %lld 0x%llx\n", azArg[i],v,v);
-        oputz(zBuf);
+        sqlite3_fputs(zBuf, p->out);
       }
     }
   }else
@@ -10424,8 +10729,9 @@ static int do_meta_command(char *zLine, ShellState *p){
         bVerbose++;
       }else
       {
-        eputf("Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]);
-        eputz("Should be one of: --init -v\n");
+        sqlite3_fprintf(stderr,
+              "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]);
+        sqlite3_fputs("Should be one of: --init -v\n", stderr);
         rc = 1;
         goto meta_command_exit;
       }
@@ -10470,10 +10776,10 @@ static int do_meta_command(char *zLine, ShellState *p){
         if( zAns==0 ) continue;
         k = 0;
         if( bVerbose>0 ){
-          sputf(stdout, "%d: %s %s\n", tno, zOp, zSql);
+          sqlite3_fprintf(stdout, "%d: %s %s\n", tno, zOp, zSql);
         }
         if( cli_strcmp(zOp,"memo")==0 ){
-          oputf("%s\n", zSql);
+          sqlite3_fprintf(p->out, "%s\n", zSql);
         }else
         if( cli_strcmp(zOp,"run")==0 ){
           char *zErrMsg = 0;
@@ -10482,22 +10788,23 @@ static int do_meta_command(char *zLine, ShellState *p){
           rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg);
           nTest++;
           if( bVerbose ){
-            oputf("Result: %s\n", str.z);
+            sqlite3_fprintf(p->out, "Result: %s\n", str.z);
           }
           if( rc || zErrMsg ){
             nErr++;
             rc = 1;
-            oputf("%d: error-code-%d: %s\n", tno, rc, zErrMsg);
+            sqlite3_fprintf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg);
             sqlite3_free(zErrMsg);
           }else if( cli_strcmp(zAns,str.z)!=0 ){
             nErr++;
             rc = 1;
-            oputf("%d: Expected: [%s]\n", tno, zAns);
-            oputf("%d:      Got: [%s]\n", tno, str.z);
+            sqlite3_fprintf(p->out, "%d: Expected: [%s]\n", tno, zAns);
+            sqlite3_fprintf(p->out, "%d:      Got: [%s]\n", tno, str.z);
           }
         }
         else{
-          eputf("Unknown operation \"%s\" on selftest line %d\n", zOp, tno);
+          sqlite3_fprintf(stderr,
+                "Unknown operation \"%s\" on selftest line %d\n", zOp, tno);
           rc = 1;
           break;
         }
@@ -10505,7 +10812,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       sqlite3_finalize(pStmt);
     } /* End loop over k */
     freeText(&str);
-    oputf("%d errors out of %d tests\n", nErr, nTest);
+    sqlite3_fprintf(p->out, "%d errors out of %d tests\n", nErr, nTest);
   }else
 
   if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){
@@ -10553,7 +10860,8 @@ static int do_meta_command(char *zLine, ShellState *p){
           bDebug = 1;
         }else
         {
-          eputf("Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]);
+          sqlite3_fprintf(stderr,
+                  "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]);
           showHelp(p->out, azArg[0]);
           rc = 1;
           goto meta_command_exit;
@@ -10631,7 +10939,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     freeText(&sQuery);
     freeText(&sSql);
     if( bDebug ){
-      oputf("%s\n", zSql);
+      sqlite3_fprintf(p->out, "%s\n", zSql);
     }else{
       shell_exec(p, zSql, 0);
     }
@@ -10661,7 +10969,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           "' OR ') as query, tname from tabcols group by tname)"
           , zRevText);
       shell_check_oom(zRevText);
-      if( bDebug ) oputf("%s\n", zRevText);
+      if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zRevText);
       lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0);
       if( lrc!=SQLITE_OK ){
         /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the
@@ -10674,7 +10982,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0);
           sqlite3_stmt *pCheckStmt;
           lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0);
-          if( bDebug ) oputf("%s\n", zGenQuery);
+          if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zGenQuery);
           if( lrc!=SQLITE_OK ){
             rc = 1;
           }else{
@@ -10682,7 +10990,8 @@ static int do_meta_command(char *zLine, ShellState *p){
               double countIrreversible = sqlite3_column_double(pCheckStmt, 0);
               if( countIrreversible>0 ){
                 int sz = (int)(countIrreversible + 0.5);
-                eputf("Digest includes %d invalidly encoded text field%s.\n",
+                sqlite3_fprintf(stderr,
+                      "Digest includes %d invalidly encoded text field%s.\n",
                       sz, (sz>1)? "s": "");
               }
             }
@@ -10716,11 +11025,11 @@ static int do_meta_command(char *zLine, ShellState *p){
       zCmd = sqlite3_mprintf(strchr(azArg[i],' ')==0?"%z %s":"%z \"%s\"",
                              zCmd, azArg[i]);
     }
-    consoleRestore();
+    /*consoleRestore();*/
     x = zCmd!=0 ? system(zCmd) : 1;
-    consoleRenewSetup();
+    /*consoleRenewSetup();*/
     sqlite3_free(zCmd);
-    if( x ) eputf("System command returns %d\n", x);
+    if( x ) sqlite3_fprintf(stderr,"System command returns %d\n", x);
   }else
 #endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */
 
@@ -10733,46 +11042,48 @@ static int do_meta_command(char *zLine, ShellState *p){
       rc = 1;
       goto meta_command_exit;
     }
-    oputf("%12.12s: %s\n","echo",
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","echo",
           azBool[ShellHasFlag(p, SHFLG_Echo)]);
-    oputf("%12.12s: %s\n","eqp", azBool[p->autoEQP&3]);
-    oputf("%12.12s: %s\n","explain",
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","explain",
           p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off");
-    oputf("%12.12s: %s\n","headers", azBool[p->showHeader!=0]);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","headers",
+          azBool[p->showHeader!=0]);
     if( p->mode==MODE_Column
      || (p->mode>=MODE_Markdown && p->mode<=MODE_Box)
     ){
-      oputf("%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode",
+      sqlite3_fprintf(p->out,
+            "%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode",
             modeDescr[p->mode], p->cmOpts.iWrap,
             p->cmOpts.bWordWrap ? "on" : "off",
             p->cmOpts.bQuote ? "" : "no");
     }else{
-      oputf("%12.12s: %s\n","mode", modeDescr[p->mode]);
+      sqlite3_fprintf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]);
     }
-    oputf("%12.12s: ", "nullvalue");
-    output_c_string(p->nullValue);
-    oputz("\n");
-    oputf("%12.12s: %s\n","output",
+    sqlite3_fprintf(p->out, "%12.12s: ", "nullvalue");
+    output_c_string(p->out, p->nullValue);
+    sqlite3_fputs("\n", p->out);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","output",
           strlen30(p->outfile) ? p->outfile : "stdout");
-    oputf("%12.12s: ", "colseparator");
-     output_c_string(p->colSeparator);
-     oputz("\n");
-    oputf("%12.12s: ", "rowseparator");
-     output_c_string(p->rowSeparator);
-     oputz("\n");
+    sqlite3_fprintf(p->out, "%12.12s: ", "colseparator");
+    output_c_string(p->out, p->colSeparator);
+    sqlite3_fputs("\n", p->out);
+    sqlite3_fprintf(p->out, "%12.12s: ", "rowseparator");
+    output_c_string(p->out, p->rowSeparator);
+    sqlite3_fputs("\n", p->out);
     switch( p->statsOn ){
       case 0:  zOut = "off";     break;
       default: zOut = "on";      break;
       case 2:  zOut = "stmt";    break;
       case 3:  zOut = "vmstep";  break;
     }
-    oputf("%12.12s: %s\n","stats", zOut);
-    oputf("%12.12s: ", "width");
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","stats", zOut);
+    sqlite3_fprintf(p->out, "%12.12s: ", "width");
     for (i=0;i<p->nWidth;i++) {
-      oputf("%d ", p->colWidth[i]);
+      sqlite3_fprintf(p->out, "%d ", p->colWidth[i]);
     }
-    oputz("\n");
-    oputf("%12.12s: %s\n", "filename",
+    sqlite3_fputs("\n", p->out);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n", "filename",
           p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : "");
   }else
 
@@ -10890,9 +11201,10 @@ static int do_meta_command(char *zLine, ShellState *p){
       for(i=0; i<nPrintRow; i++){
         for(j=i; j<nRow; j+=nPrintRow){
           char *zSp = j<nPrintRow ? "" : "  ";
-          oputf("%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:"");
+          sqlite3_fprintf(p->out,
+               "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:"");
         }
-        oputz("\n");
+        sqlite3_fputs("\n", p->out);
       }
     }
 
@@ -10968,10 +11280,10 @@ static int do_meta_command(char *zLine, ShellState *p){
 
     /* --help lists all test-controls */
     if( cli_strcmp(zCmd,"help")==0 ){
-      oputz("Available test-controls:\n");
+      sqlite3_fputs("Available test-controls:\n", p->out);
       for(i=0; i<ArraySize(aCtrl); i++){
         if( aCtrl[i].unSafe && !ShellHasFlag(p,SHFLG_TestingMode) ) continue;
-        oputf("  .testctrl %s %s\n",
+        sqlite3_fprintf(p->out, "  .testctrl %s %s\n",
               aCtrl[i].zCtrlName, aCtrl[i].zUsage);
       }
       rc = 1;
@@ -10988,7 +11300,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           testctrl = aCtrl[i].ctrlCode;
           iCtrl = i;
         }else{
-          eputf("Error: ambiguous test-control: \"%s\"\n"
+          sqlite3_fprintf(stderr,"Error: ambiguous test-control: \"%s\"\n"
                 "Use \".testctrl --help\" for help\n", zCmd);
           rc = 1;
           goto meta_command_exit;
@@ -10996,7 +11308,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
     }
     if( testctrl<0 ){
-      eputf("Error: unknown test-control: %s\n"
+      sqlite3_fprintf(stderr,"Error: unknown test-control: %s\n"
             "Use \".testctrl --help\" for help\n", zCmd);
     }else{
       switch(testctrl){
@@ -11072,12 +11384,13 @@ static int do_meta_command(char *zLine, ShellState *p){
                 if( sqlite3_stricmp(zLabel, aLabel[jj].zLabel)==0 ) break;
               }
               if( jj>=ArraySize(aLabel) ){
-                eputf("Error: no such optimization: \"%s\"\n", zLabel);
-                eputz("Should be one of:");
+                sqlite3_fprintf(stderr,
+                    "Error: no such optimization: \"%s\"\n", zLabel);
+                sqlite3_fputs("Should be one of:", stderr);
                 for(jj=0; jj<ArraySize(aLabel); jj++){
-                  eputf(" %s", aLabel[jj].zLabel);
+                  sqlite3_fprintf(stderr," %s", aLabel[jj].zLabel);
                 }
-                eputz("\n");
+                sqlite3_fputs("\n", stderr);
                 rc = 1;
                 goto meta_command_exit;
               }
@@ -11094,16 +11407,16 @@ static int do_meta_command(char *zLine, ShellState *p){
             curOpt = ~newOpt;
           }
           if( newOpt==0 ){
-            oputz("+All\n");
+            sqlite3_fputs("+All\n", p->out);
           }else if( newOpt==0xffffffff ){
-            oputz("-All\n");
+            sqlite3_fputs("-All\n", p->out);
           }else{
             int jj;
             for(jj=0; jj<ArraySize(aLabel); jj++){
               unsigned int m = aLabel[jj].mask;
               if( !aLabel[jj].bDsply  ) continue;
               if( (curOpt&m)!=(newOpt&m) ){
-                oputf("%c%s\n", (newOpt & m)==0 ? '+' : '-',
+                sqlite3_fprintf(p->out, "%c%s\n", (newOpt & m)==0 ? '+' : '-',
                       aLabel[jj].zLabel);
               }
             }
@@ -11147,7 +11460,7 @@ static int do_meta_command(char *zLine, ShellState *p){
             sqlite3 *db;
             if( ii==0 && cli_strcmp(azArg[2],"random")==0 ){
               sqlite3_randomness(sizeof(ii),&ii);
-              sputf(stdout, "-- random seed: %d\n", ii);
+              sqlite3_fprintf(stdout, "-- random seed: %d\n", ii);
             }
             if( nArg==3 ){
               db = 0;
@@ -11215,7 +11528,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         case SQLITE_TESTCTRL_SEEK_COUNT: {
           u64 x = 0;
           rc2 = sqlite3_test_control(testctrl, p->db, &x);
-          oputf("%llu\n", x);
+          sqlite3_fprintf(p->out, "%llu\n", x);
           isOk = 3;
           break;
         }
@@ -11246,11 +11559,11 @@ static int do_meta_command(char *zLine, ShellState *p){
               int val = 0;
               rc2 = sqlite3_test_control(testctrl, -id, &val);
               if( rc2!=SQLITE_OK ) break;
-              if( id>1 ) oputz("  ");
-              oputf("%d: %d", id, val);
+              if( id>1 ) sqlite3_fputs("  ", p->out);
+              sqlite3_fprintf(p->out, "%d: %d", id, val);
               id++;
             }
-            if( id>1 ) oputz("\n");
+            if( id>1 ) sqlite3_fputs("\n", p->out);
             isOk = 3;
           }
           break;
@@ -11292,14 +11605,22 @@ static int do_meta_command(char *zLine, ShellState *p){
               faultsim_state.nHit = 0;
               sqlite3_test_control(testctrl, faultsim_callback);
             }else if( cli_strcmp(z,"status")==0 ){
-              oputf("faultsim.iId:       %d\n", faultsim_state.iId);
-              oputf("faultsim.iErr:      %d\n", faultsim_state.iErr);
-              oputf("faultsim.iCnt:      %d\n", faultsim_state.iCnt);
-              oputf("faultsim.nHit:      %d\n", faultsim_state.nHit);
-              oputf("faultsim.iInterval: %d\n", faultsim_state.iInterval);
-              oputf("faultsim.eVerbose:  %d\n", faultsim_state.eVerbose);
-              oputf("faultsim.nRepeat:   %d\n", faultsim_state.nRepeat);
-              oputf("faultsim.nSkip:     %d\n", faultsim_state.nSkip);
+              sqlite3_fprintf(p->out, "faultsim.iId:       %d\n",
+                              faultsim_state.iId);
+              sqlite3_fprintf(p->out, "faultsim.iErr:      %d\n",
+                              faultsim_state.iErr);
+              sqlite3_fprintf(p->out, "faultsim.iCnt:      %d\n",
+                              faultsim_state.iCnt);
+              sqlite3_fprintf(p->out, "faultsim.nHit:      %d\n",
+                              faultsim_state.nHit);
+              sqlite3_fprintf(p->out, "faultsim.iInterval: %d\n",
+                              faultsim_state.iInterval);
+              sqlite3_fprintf(p->out, "faultsim.eVerbose:  %d\n",
+                              faultsim_state.eVerbose);
+              sqlite3_fprintf(p->out, "faultsim.nRepeat:   %d\n",
+                              faultsim_state.nRepeat);
+              sqlite3_fprintf(p->out, "faultsim.nSkip:     %d\n",
+                              faultsim_state.nSkip);
             }else if( cli_strcmp(z,"-v")==0 ){
               if( faultsim_state.eVerbose<2 ) faultsim_state.eVerbose++;
             }else if( cli_strcmp(z,"-q")==0 ){
@@ -11317,7 +11638,8 @@ static int do_meta_command(char *zLine, ShellState *p){
             }else if( cli_strcmp(z,"-?")==0 || sqlite3_strglob("*help*",z)==0){
               bShowHelp = 1;
             }else{
-              eputf("Unrecognized fault_install argument: \"%s\"\n",
+              sqlite3_fprintf(stderr,
+                  "Unrecognized fault_install argument: \"%s\"\n",
                   azArg[kk]);
               rc = 1;
               bShowHelp = 1;
@@ -11325,7 +11647,7 @@ static int do_meta_command(char *zLine, ShellState *p){
             }
           }
           if( bShowHelp ){
-            oputz(
+            sqlite3_fputs(
                "Usage: .testctrl fault_install ARGS\n"
                "Possible arguments:\n"
                "   off               Disable faultsim\n"
@@ -11339,6 +11661,7 @@ static int do_meta_command(char *zLine, ShellState *p){
                "   --interval N      Trigger only after every N-th call\n"
                "   --repeat N        Turn off after N hits.  0 means never\n"
                "   --skip N          Skip the first N encounters\n"
+               ,p->out
             );
           }
           break;
@@ -11346,12 +11669,13 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
     }
     if( isOk==0 && iCtrl>=0 ){
-      oputf("Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage);
+      sqlite3_fprintf(p->out,
+          "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage);
       rc = 1;
     }else if( isOk==1 ){
-      oputf("%d\n", rc2);
+      sqlite3_fprintf(p->out, "%d\n", rc2);
     }else if( isOk==2 ){
-      oputf("0x%08x\n", rc2);
+      sqlite3_fprintf(p->out, "0x%08x\n", rc2);
     }
   }else
 #endif /* !defined(SQLITE_UNTESTABLE) */
@@ -11406,7 +11730,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           mType |= SQLITE_TRACE_CLOSE;
         }
         else {
-          eputf("Unknown option \"%s\" on \".trace\"\n", z);
+          sqlite3_fprintf(stderr,"Unknown option \"%s\" on \".trace\"\n", z);
           rc = 1;
           goto meta_command_exit;
         }
@@ -11466,7 +11790,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3],
                                      strlen30(azArg[3]));
       if( rc ){
-        eputf("Authentication failed for user %s\n", azArg[2]);
+        sqlite3_fprintf(stderr,"Authentication failed for user %s\n", azArg[2]);
         rc = 1;
       }
     }else if( cli_strcmp(azArg[1],"add")==0 ){
@@ -11478,7 +11802,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       rc = sqlite3_user_add(p->db, azArg[2], azArg[3], strlen30(azArg[3]),
                             booleanValue(azArg[4]));
       if( rc ){
-        eputf("User-Add failed: %d\n", rc);
+        sqlite3_fprintf(stderr,"User-Add failed: %d\n", rc);
         rc = 1;
       }
     }else if( cli_strcmp(azArg[1],"edit")==0 ){
@@ -11490,7 +11814,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       rc = sqlite3_user_change(p->db, azArg[2], azArg[3], strlen30(azArg[3]),
                               booleanValue(azArg[4]));
       if( rc ){
-        eputf("User-Edit failed: %d\n", rc);
+        sqlite3_fprintf(stderr,"User-Edit failed: %d\n", rc);
         rc = 1;
       }
     }else if( cli_strcmp(azArg[1],"delete")==0 ){
@@ -11501,7 +11825,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
       rc = sqlite3_user_delete(p->db, azArg[2]);
       if( rc ){
-        eputf("User-Delete failed: %d\n", rc);
+        sqlite3_fprintf(stderr,"User-Delete failed: %d\n", rc);
         rc = 1;
       }
     }else{
@@ -11514,21 +11838,21 @@ static int do_meta_command(char *zLine, ShellState *p){
 
   if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){
     char *zPtrSz = sizeof(void*)==8 ? "64-bit" : "32-bit";
-    oputf("SQLite %s %s\n" /*extra-version-info*/,
+    sqlite3_fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/,
           sqlite3_libversion(), sqlite3_sourceid());
 #if SQLITE_HAVE_ZLIB
-    oputf("zlib version %s\n", zlibVersion());
+    sqlite3_fprintf(p->out, "zlib version %s\n", zlibVersion());
 #endif
 #define CTIMEOPT_VAL_(opt) #opt
 #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt)
 #if defined(__clang__) && defined(__clang_major__)
-    oputf("clang-" CTIMEOPT_VAL(__clang_major__) "."
+    sqlite3_fprintf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "."
           CTIMEOPT_VAL(__clang_minor__) "."
           CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz);
 #elif defined(_MSC_VER)
-    oputf("msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz);
+    sqlite3_fprintf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz);
 #elif defined(__GNUC__) && defined(__VERSION__)
-    oputf("gcc-" __VERSION__ " (%s)\n", zPtrSz);
+    sqlite3_fprintf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz);
 #endif
   }else
 
@@ -11538,10 +11862,10 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( p->db ){
       sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs);
       if( pVfs ){
-        oputf("vfs.zName      = \"%s\"\n", pVfs->zName);
-        oputf("vfs.iVersion   = %d\n", pVfs->iVersion);
-        oputf("vfs.szOsFile   = %d\n", pVfs->szOsFile);
-        oputf("vfs.mxPathname = %d\n", pVfs->mxPathname);
+        sqlite3_fprintf(p->out, "vfs.zName      = \"%s\"\n", pVfs->zName);
+        sqlite3_fprintf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
+        sqlite3_fprintf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
+        sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
       }
     }
   }else
@@ -11553,13 +11877,13 @@ static int do_meta_command(char *zLine, ShellState *p){
       sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent);
     }
     for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){
-      oputf("vfs.zName      = \"%s\"%s\n", pVfs->zName,
+      sqlite3_fprintf(p->out, "vfs.zName      = \"%s\"%s\n", pVfs->zName,
             pVfs==pCurrent ? "  <--- CURRENT" : "");
-      oputf("vfs.iVersion   = %d\n", pVfs->iVersion);
-      oputf("vfs.szOsFile   = %d\n", pVfs->szOsFile);
-      oputf("vfs.mxPathname = %d\n", pVfs->mxPathname);
+      sqlite3_fprintf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
+      sqlite3_fprintf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
+      sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
       if( pVfs->pNext ){
-        oputz("-----------------------------------\n");
+        sqlite3_fputs("-----------------------------------\n", p->out);
       }
     }
   }else
@@ -11570,7 +11894,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( p->db ){
       sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName);
       if( zVfsName ){
-        oputf("%s\n", zVfsName);
+        sqlite3_fprintf(p->out, "%s\n", zVfsName);
         sqlite3_free(zVfsName);
       }
     }
@@ -11594,7 +11918,7 @@ static int do_meta_command(char *zLine, ShellState *p){
   }else
 
   {
-    eputf("Error: unknown command or invalid arguments: "
+    sqlite3_fprintf(stderr,"Error: unknown command or invalid arguments: "
           " \"%s\". Enter \".help\" for help\n", azArg[0]);
     rc = 1;
   }
@@ -11846,7 +12170,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
   if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0;
   BEGIN_TIMER;
   rc = shell_exec(p, zSql, &zErrMsg);
-  END_TIMER;
+  END_TIMER(p->out);
   if( rc || zErrMsg ){
     char zPrefix[100];
     const char *zErrorTail;
@@ -11870,7 +12194,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
     }else{
       sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:", zErrorType);
     }
-    eputf("%s %s\n", zPrefix, zErrorTail);
+    sqlite3_fprintf(stderr,"%s %s\n", zPrefix, zErrorTail);
     sqlite3_free(zErrMsg);
     zErrMsg = 0;
     return 1;
@@ -11879,7 +12203,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
     sqlite3_snprintf(sizeof(zLineBuf), zLineBuf,
             "changes: %lld   total_changes: %lld",
             sqlite3_changes64(p->db), sqlite3_total_changes64(p->db));
-    oputf("%s\n", zLineBuf);
+    sqlite3_fprintf(p->out, "%s\n", zLineBuf);
   }
 
   if( doAutoDetectRestore(p, zSql) ) return 1;
@@ -11887,7 +12211,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
 }
 
 static void echo_group_input(ShellState *p, const char *zDo){
-  if( ShellHasFlag(p, SHFLG_Echo) ) oputf("%s\n", zDo);
+  if( ShellHasFlag(p, SHFLG_Echo) ) sqlite3_fprintf(p->out, "%s\n", zDo);
 }
 
 #ifdef SQLITE_SHELL_FIDDLE
@@ -11945,7 +12269,7 @@ static int process_input(ShellState *p){
 
   if( p->inputNesting==MAX_INPUT_NESTING ){
     /* This will be more informative in a later version. */
-    eputf("Input nesting limit (%d) reached at line %d."
+    sqlite3_fprintf(stderr,"Input nesting limit (%d) reached at line %d."
           " Check recursion.\n", MAX_INPUT_NESTING, p->lineno);
     return 1;
   }
@@ -11957,7 +12281,7 @@ static int process_input(ShellState *p){
     zLine = one_input_line(p->in, zLine, nSql>0);
     if( zLine==0 ){
       /* End of input */
-      if( p->in==0 && stdin_is_interactive ) oputz("\n");
+      if( p->in==0 && stdin_is_interactive ) sqlite3_fputs("\n", p->out);
       break;
     }
     if( seenInterrupt ){
@@ -12177,15 +12501,15 @@ static void process_sqliterc(
     shell_check_oom(zBuf);
     sqliterc = zBuf;
   }
-  p->in = fopen(sqliterc,"rb");
+  p->in = sqlite3_fopen(sqliterc,"rb");
   if( p->in ){
     if( stdin_is_interactive ){
-      eputf("-- Loading resources from %s\n", sqliterc);
+      sqlite3_fprintf(stderr,"-- Loading resources from %s\n", sqliterc);
     }
     if( process_input(p) && bail_on_error ) exit(1);
     fclose(p->in);
   }else if( sqliterc_override!=0 ){
-    eputf("cannot open: \"%s\"\n", sqliterc);
+    sqlite3_fprintf(stderr,"cannot open: \"%s\"\n", sqliterc);
     if( bail_on_error ) exit(1);
   }
   p->in = inSaved;
@@ -12254,19 +12578,17 @@ static const char zOptions[] =
   "   -unsafe-testing      allow unsafe commands and modes for testing\n"
   "   -version             show SQLite version\n"
   "   -vfs NAME            use NAME as the default VFS\n"
-#ifdef SQLITE_ENABLE_VFSTRACE
   "   -vfstrace            enable tracing of all VFS calls\n"
-#endif
 #ifdef SQLITE_HAVE_ZLIB
   "   -zip                 open the file as a ZIP Archive\n"
 #endif
 ;
 static void usage(int showDetail){
-  eputf("Usage: %s [OPTIONS] [FILENAME [SQL]]\n"
+  sqlite3_fprintf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL]]\n"
        "FILENAME is the name of an SQLite database. A new database is created\n"
        "if the file does not previously exist. Defaults to :memory:.\n", Argv0);
   if( showDetail ){
-    eputf("OPTIONS include:\n%s", zOptions);
+    sqlite3_fprintf(stderr,"OPTIONS include:\n%s", zOptions);
   }else{
     eputz("Use the -help option for additional information\n");
   }
@@ -12326,7 +12648,7 @@ static void printBold(const char *zText){
 }
 #else
 static void printBold(const char *zText){
-  sputf(stdout, "\033[1m%s\033[0m", zText);
+  sqlite3_fprintf(stdout, "\033[1m%s\033[0m", zText);
 }
 #endif
 
@@ -12336,7 +12658,8 @@ static void printBold(const char *zText){
 */
 static char *cmdline_option_value(int argc, char **argv, int i){
   if( i==argc ){
-    eputf("%s: Error: missing argument to %s\n", argv[0], argv[argc-1]);
+    sqlite3_fprintf(stderr,
+            "%s: Error: missing argument to %s\n", argv[0], argv[argc-1]);
     exit(1);
   }
   return argv[i];
@@ -12373,7 +12696,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #  define data shellState
 #else
   ShellState data;
-  StreamsAreConsole consStreams = SAC_NoConsole;
 #endif
   const char *zInitFile = 0;
   int i;
@@ -12382,6 +12704,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   int readStdin = 1;
   int nCmd = 0;
   int nOptsEnd = argc;
+  int bEnableVfstrace = 0;
   char **azCmd = 0;
   const char *zVfs = 0;           /* Value of -vfs command-line option */
 #if !SQLITE_SHELL_IS_UTF8
@@ -12395,10 +12718,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   stdout_is_console = 1;
   data.wasm.zDefaultDbName = "/fiddle.sqlite3";
 #else
-  consStreams = consoleClassifySetup(stdin, stdout, stderr);
-  stdin_is_interactive = (consStreams & SAC_InConsole)!=0;
-  stdout_is_console = (consStreams & SAC_OutConsole)!=0;
-  atexit(consoleRestore);
+  stdin_is_interactive = isatty(0);
+  stdout_is_console = isatty(1);
 #endif
   atexit(sayAbnormalExit);
 #ifdef SQLITE_DEBUG
@@ -12407,9 +12728,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #if !defined(_WIN32_WCE)
   if( getenv("SQLITE_DEBUG_BREAK") ){
     if( isatty(0) && isatty(2) ){
-      eputf("attach debugger to process %d and press any key to continue.\n",
+      char zLine[100];
+      sqlite3_fprintf(stderr,
+            "attach debugger to process %d and press ENTER to continue...",
             GETPID());
-      fgetc(stdin);
+      if( sqlite3_fgets(zLine, sizeof(zLine), stdin)!=0
+       && cli_strcmp(zLine,"stop")==0
+      ){
+        exit(1);
+      }
     }else{
 #if defined(_WIN32) || defined(WIN32)
 #if SQLITE_OS_WINRT
@@ -12434,7 +12761,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 
 #if USE_SYSTEM_SQLITE+0!=1
   if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){
-    eputf("SQLite header and source version mismatch\n%s\n%s\n",
+    sqlite3_fprintf(stderr,
+          "SQLite header and source version mismatch\n%s\n%s\n",
           sqlite3_sourceid(), SQLITE_SOURCE_ID);
     exit(1);
   }
@@ -12576,17 +12904,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
          case 2:  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);   break;
          default: sqlite3_config(SQLITE_CONFIG_SERIALIZED);    break;
       }
-#ifdef SQLITE_ENABLE_VFSTRACE
     }else if( cli_strcmp(z,"-vfstrace")==0 ){
-      extern int vfstrace_register(
-         const char *zTraceName,
-         const char *zOldVfsName,
-         int (*xOut)(const char*,void*),
-         void *pOutArg,
-         int makeDefault
-      );
-      vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1);
-#endif
+      vfstrace_register("trace",0,(int(*)(const char*,void*))sqlite3_fputs,
+                        stderr,1);
+      bEnableVfstrace = 1;
 #ifdef SQLITE_ENABLE_MULTIPLEX
     }else if( cli_strcmp(z,"-multiplex")==0 ){
       extern int sqlite3_multiplex_initialize(const char*,int);
@@ -12642,7 +12963,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }
   }
 #ifndef SQLITE_SHELL_FIDDLE
-  verify_uninitialized();
+  if( !bEnableVfstrace ) verify_uninitialized();
 #endif
 
 
@@ -12666,7 +12987,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     if( pVfs ){
       sqlite3_vfs_register(pVfs, 1);
     }else{
-      eputf("no such VFS: \"%s\"\n", zVfs);
+      sqlite3_fprintf(stderr,"no such VFS: \"%s\"\n", zVfs);
       exit(1);
     }
   }
@@ -12676,7 +12997,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     data.pAuxDb->zDbFilename = ":memory:";
     warnInmemoryDb = argc==1;
 #else
-    eputf("%s: Error: no database filename specified\n", Argv0);
+    sqlite3_fprintf(stderr,
+                    "%s: Error: no database filename specified\n", Argv0);
     return 1;
 #endif
   }
@@ -12793,7 +13115,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( cli_strcmp(z,"-bail")==0 ){
       /* No-op.  The bail_on_error flag should already be set. */
     }else if( cli_strcmp(z,"-version")==0 ){
-      sputf(stdout, "%s %s (%d-bit)\n",
+      sqlite3_fprintf(stdout, "%s %s (%d-bit)\n",
             sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*));
       return 0;
     }else if( cli_strcmp(z,"-interactive")==0 ){
@@ -12831,10 +13153,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #endif
     }else if( cli_strcmp(z,"-vfs")==0 ){
       i++;
-#ifdef SQLITE_ENABLE_VFSTRACE
     }else if( cli_strcmp(z,"-vfstrace")==0 ){
       i++;
-#endif
 #ifdef SQLITE_ENABLE_MULTIPLEX
     }else if( cli_strcmp(z,"-multiplex")==0 ){
       i++;
@@ -12858,14 +13178,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
           shellEmitError(zErrMsg);
           if( bail_on_error ) return rc!=0 ? rc : 1;
         }else if( rc!=0 ){
-          eputf("Error: unable to process SQL \"%s\"\n", z);
+          sqlite3_fprintf(stderr,"Error: unable to process SQL \"%s\"\n", z);
           if( bail_on_error ) return rc;
         }
       }
 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
     }else if( cli_strncmp(z, "-A", 2)==0 ){
       if( nCmd>0 ){
-        eputf("Error: cannot mix regular SQL or dot-commands"
+        sqlite3_fprintf(stderr,"Error: cannot mix regular SQL or dot-commands"
               " with \"%s\"\n", z);
         return 1;
       }
@@ -12884,7 +13204,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( cli_strcmp(z,"-unsafe-testing")==0 ){
       /* Acted upon in first pass. */
     }else{
-      eputf("%s: Error: unknown option: %s\n", Argv0, z);
+      sqlite3_fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
       eputz("Use -help for a list of options.\n");
       return 1;
     }
@@ -12911,7 +13231,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
           if( zErrMsg!=0 ){
             shellEmitError(zErrMsg);
           }else{
-            eputf("Error: unable to process SQL: %s\n", azCmd[i]);
+            sqlite3_fprintf(stderr,
+                            "Error: unable to process SQL: %s\n", azCmd[i]);
           }
           sqlite3_free(zErrMsg);
           if( rc==0 ) rc = 1;
@@ -12931,7 +13252,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #else
 # define SHELL_CIO_CHAR_SET ""
 #endif
-      sputf(stdout, "SQLite version %s %.19s%s\n" /*extra-version-info*/
+      sqlite3_fprintf(stdout,
+            "SQLite version %s %.19s%s\n" /*extra-version-info*/
             "Enter \".help\" for usage hints.\n",
             sqlite3_libversion(), sqlite3_sourceid(), SHELL_CIO_CHAR_SET);
       if( warnInmemoryDb ){
@@ -13002,9 +13324,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   /* Clear the global data structure so that valgrind will detect memory
   ** leaks */
   memset(&data, 0, sizeof(data));
+  if( bEnableVfstrace ){
+    vfstrace_unregister("trace");
+  }
 #ifdef SQLITE_DEBUG
   if( sqlite3_memory_used()>mem_main_enter ){
-    eputf("Memory leaked: %u bytes\n",
+    sqlite3_fprintf(stderr,"Memory leaked: %u bytes\n",
           (unsigned int)(sqlite3_memory_used()-mem_main_enter));
   }
 #endif
@@ -13044,7 +13369,7 @@ sqlite3_vfs * fiddle_db_vfs(const char *zDbName){
 
 /* Only for emcc experimentation purposes. */
 sqlite3 * fiddle_db_arg(sqlite3 *arg){
-    oputf("fiddle_db_arg(%p)\n", (const void*)arg);
+    sqlite3_fprintf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg);
     return arg;
 }
 
@@ -13081,7 +13406,7 @@ void fiddle_reset_db(void){
       ** Resolve problem reported in
       ** https://sqlite.org/forum/forumpost/0b41a25d65
       */
-      oputz("Rolling back in-progress transaction.\n");
+      sqlite3_fputs("Rolling back in-progress transaction.\n", stdout);
       sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0);
     }
     rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0);
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index fed9295587..0b24b41fa5 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -4222,13 +4222,17 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** and sqlite3_prepare16_v3() use UTF-16.
 **
 ** ^If the nByte argument is negative, then zSql is read up to the
-** first zero terminator. ^If nByte is positive, then it is the
-** number of bytes read from zSql.  ^If nByte is zero, then no prepared
+** first zero terminator. ^If nByte is positive, then it is the maximum
+** number of bytes read from zSql.  When nByte is positive, zSql is read
+** up to the first zero terminator or until the nByte bytes have been read,
+** whichever comes first.  ^If nByte is zero, then no prepared
 ** statement is generated.
 ** If the caller knows that the supplied string is nul-terminated, then
 ** there is a small performance advantage to passing an nByte parameter that
 ** is the number of bytes in the input string <i>including</i>
 ** the nul-terminator.
+** Note that nByte measure the length of the input in bytes, not
+** characters, even for the UTF-16 inferfaces.
 **
 ** ^If pzTail is not NULL then *pzTail is made to point to the first byte
 ** past the end of the first SQL statement in zSql.  These routines only
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index b9d5ff8c04..b802aeb686 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -5294,7 +5294,7 @@ int sqlite3GetInt32(const char *, int*);
 int sqlite3GetUInt32(const char*, u32*);
 int sqlite3Atoi(const char*);
 #ifndef SQLITE_OMIT_UTF16
-int sqlite3Utf16ByteLen(const void *pData, int nChar);
+int sqlite3Utf16ByteLen(const void *pData, int nByte, int nChar);
 #endif
 int sqlite3Utf8CharLen(const char *pData, int nByte);
 u32 sqlite3Utf8Read(const u8**);
diff --git a/src/test_hexio.c b/src/test_hexio.c
index 47c9d8fd2b..1a21e89aa0 100644
--- a/src/test_hexio.c
+++ b/src/test_hexio.c
@@ -232,9 +232,9 @@ static int SQLITE_TCLAPI hexio_get_int(
   }
   sqlite3_free(aOut);
   if( bLittle ){
-    val = (aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0];
+    val = (int)((u32)aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0];
   }else{
-    val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+    val = (int)((u32)aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
   }
   Tcl_SetObjResult(interp, Tcl_NewIntObj(val));
   return TCL_OK;
diff --git a/src/utf.c b/src/utf.c
index 216864f5c7..083ada7882 100644
--- a/src/utf.c
+++ b/src/utf.c
@@ -514,20 +514,22 @@ char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 enc){
 }
 
 /*
-** zIn is a UTF-16 encoded unicode string at least nChar characters long.
+** zIn is a UTF-16 encoded unicode string at least nByte bytes long.
 ** Return the number of bytes in the first nChar unicode characters
-** in pZ.  nChar must be non-negative.
+** in pZ.  nChar must be non-negative.  Surrogate pairs count as a single
+** character.
 */
-int sqlite3Utf16ByteLen(const void *zIn, int nChar){
+int sqlite3Utf16ByteLen(const void *zIn, int nByte, int nChar){
   int c;
   unsigned char const *z = zIn;
+  unsigned char const *zEnd = &z[nByte-1];
   int n = 0;
   
   if( SQLITE_UTF16NATIVE==SQLITE_UTF16LE ) z++;
-  while( n<nChar ){
+  while( n<nChar && ALWAYS(z<=zEnd) ){
     c = z[0];
     z += 2;
-    if( c>=0xd8 && c<0xdc && z[0]>=0xdc && z[0]<0xe0 ) z += 2;
+    if( c>=0xd8 && c<0xdc && z<=zEnd && z[0]>=0xdc && z[0]<0xe0 ) z += 2;
     n++;
   }
   return (int)(z-(unsigned char const *)zIn) 
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 2a23c3f285..b26860f3a6 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -543,6 +543,7 @@ struct PreUpdate {
   Mem *aNew;                      /* Array of new.* values */
   Table *pTab;                    /* Schema object being updated */
   Index *pPk;                     /* PK index if pTab is WITHOUT ROWID */
+  sqlite3_value **apDflt;         /* Array of default values, if required */
 };
 
 /*
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index 3182e4070f..b6ad5f3d95 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -1621,6 +1621,17 @@ const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){
 **
 ** The error code stored in database p->db is overwritten with the return
 ** value in any case.
+**
+** (tag-20240917-01) If  vdbeUnbind(p,(u32)(i-1))  returns SQLITE_OK,
+** that means all of the the following will be true:
+**
+**     p!=0
+**     p->pVar!=0
+**     i>0
+**     i<=p->nVar
+**
+** An assert() is normally added after vdbeUnbind() to help static analyzers
+** realize this.
 */
 static int vdbeUnbind(Vdbe *p, unsigned int i){
   Mem *pVar;
@@ -1678,6 +1689,7 @@ static int bindText(
 
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
     if( zData!=0 ){
       pVar = &p->aVar[i-1];
       rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel);
@@ -1727,6 +1739,7 @@ int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){
   Vdbe *p = (Vdbe *)pStmt;
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
     sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue);
     sqlite3_mutex_leave(p->db->mutex);
   }
@@ -1740,6 +1753,7 @@ int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){
   Vdbe *p = (Vdbe *)pStmt;
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
     sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue);
     sqlite3_mutex_leave(p->db->mutex);
   }
@@ -1750,6 +1764,7 @@ int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){
   Vdbe *p = (Vdbe*)pStmt;
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
     sqlite3_mutex_leave(p->db->mutex);
   }
   return rc;
@@ -1765,6 +1780,7 @@ int sqlite3_bind_pointer(
   Vdbe *p = (Vdbe*)pStmt;
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
     sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor);
     sqlite3_mutex_leave(p->db->mutex);
   }else if( xDestructor ){
@@ -1846,6 +1862,7 @@ int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){
   Vdbe *p = (Vdbe *)pStmt;
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
 #ifndef SQLITE_OMIT_INCRBLOB
     sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n);
 #else
@@ -2205,7 +2222,30 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
   if( iIdx==p->pTab->iPKey ){
     sqlite3VdbeMemSetInt64(pMem, p->iKey1);
   }else if( iIdx>=p->pUnpacked->nField ){
-    *ppValue = (sqlite3_value *)columnNullValue();
+    /* This occurs when the table has been extended using ALTER TABLE
+    ** ADD COLUMN. The value to return is the default value of the column. */
+    Column *pCol = &p->pTab->aCol[iIdx];
+    if( pCol->iDflt>0 ){
+      if( p->apDflt==0 ){
+        int nByte = sizeof(sqlite3_value*)*p->pTab->nCol;
+        p->apDflt = (sqlite3_value**)sqlite3DbMallocZero(db, nByte);
+        if( p->apDflt==0 ) goto preupdate_old_out;
+      }
+      if( p->apDflt[iIdx]==0 ){
+        sqlite3_value *pVal = 0;
+        Expr *pDflt;
+        assert( p->pTab!=0 && IsOrdinaryTable(p->pTab) );
+        pDflt = p->pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr;
+        rc = sqlite3ValueFromExpr(db, pDflt, ENC(db), pCol->affinity, &pVal);
+        if( rc==SQLITE_OK && pVal==0 ){
+          rc = SQLITE_CORRUPT_BKPT;
+        }
+        p->apDflt[iIdx] = pVal;
+      }
+      *ppValue = p->apDflt[iIdx];
+    }else{
+      *ppValue = (sqlite3_value *)columnNullValue();
+    }
   }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){
     if( pMem->flags & (MEM_Int|MEM_IntReal) ){
       testcase( pMem->flags & MEM_Int );
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 4caa6debb9..08296b55a6 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -5564,5 +5564,12 @@ void sqlite3VdbePreUpdateHook(
     }
     sqlite3DbNNFreeNN(db, preupdate.aNew);
   }
+  if( preupdate.apDflt ){
+    int i;
+    for(i=0; i<pTab->nCol; i++){
+      sqlite3ValueFree(preupdate.apDflt[i]);
+    }
+    sqlite3DbFree(db, preupdate.apDflt);
+  }
 }
 #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 8e2aa4a6c9..0fc6b68f5e 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1534,7 +1534,8 @@ static int valueFromFunction(
       goto value_from_function_out;
     }
     for(i=0; i<nVal; i++){
-      rc = sqlite3ValueFromExpr(db, pList->a[i].pExpr, enc, aff, &apVal[i]);
+      rc = sqlite3Stat4ValueFromExpr(pCtx->pParse, pList->a[i].pExpr, aff,
+                                     &apVal[i]);
       if( apVal[i]==0 || rc!=SQLITE_OK ) goto value_from_function_out;
     }
   }
diff --git a/src/vtab.c b/src/vtab.c
index 1036eed445..76ad3613e8 100644
--- a/src/vtab.c
+++ b/src/vtab.c
@@ -867,6 +867,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
       Table *pNew = sParse.pNewTable;
       Index *pIdx;
       pTab->aCol = pNew->aCol;
+      assert( IsOrdinaryTable(pNew) );
       sqlite3ExprListDelete(db, pNew->u.tab.pDfltList);
       pTab->nNVCol = pTab->nCol = pNew->nCol;
       pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid);
diff --git a/src/wal.c b/src/wal.c
index cf1b895cac..7a05d7d70f 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -44,7 +44,7 @@
 **    28: Checksum-2 (second part of checksum for first 24 bytes of header).
 **
 ** Immediately following the wal-header are zero or more frames. Each
-** frame consists of a 24-byte frame-header followed by a <page-size> bytes
+** frame consists of a 24-byte frame-header followed by <page-size> bytes
 ** of page data. The frame-header is six big-endian 32-bit unsigned
 ** integer values, as follows:
 **
diff --git a/src/where.c b/src/where.c
index 35da4caa6e..9aaa082cd3 100644
--- a/src/where.c
+++ b/src/where.c
@@ -1636,9 +1636,11 @@ static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){
 ** that this is required.
 */
 static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
-  sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
   int rc;
+  sqlite3_vtab *pVtab;
 
+  assert( IsVirtual(pTab) );
+  pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
   whereTraceIndexInfoInputs(p, pTab);
   pParse->db->nSchemaLock++;
   rc = pVtab->pModule->xBestIndex(pVtab, p);
diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test
index ee607f5c61..e97fc4b0a4 100644
--- a/test/fts3corrupt4.test
+++ b/test/fts3corrupt4.test
@@ -4379,6 +4379,8 @@ do_test 25.0 {
 proc rndblob {n} { return [sqlite3_randomness $n] }
 db func randomblob rndblob
 
+reset_prng_state
+
 do_catchsql_test 25.1 {
   PRAGMA writable_schema = 1;
   WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x%1 FROM c WHERE x<599237)
diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c
index f81e0982d9..9f339096bc 100644
--- a/test/fuzzcheck.c
+++ b/test/fuzzcheck.c
@@ -1038,11 +1038,11 @@ static int recoverDatabase(sqlite3 *db){
 static void bindDebugParameters(sqlite3_stmt *pStmt){
   int nVar = sqlite3_bind_parameter_count(pStmt);
   int i;
-  for(i=0; i<nVar; i++){
-    const char *zVar = sqlite3_bind_parameter_name(pStmt, i+1);
+  for(i=1; i<=nVar; i++){
+    const char *zVar = sqlite3_bind_parameter_name(pStmt, i);
     if( zVar==0 ) continue;
     if( strncmp(zVar, "$int_", 5)==0 ){
-      sqlite3_bind_int(pStmt, i+1, atoi(&zVar[5]));
+      sqlite3_bind_int(pStmt, i, atoi(&zVar[5]));
     }else
     if( strncmp(zVar, "$text_", 6)==0 ){
       size_t szVar = strlen(zVar);
diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c
index 2f09c43ab6..c8aae6e3dd 100644
--- a/test/fuzzinvariants.c
+++ b/test/fuzzinvariants.c
@@ -47,11 +47,11 @@ static void reportInvariantFailed(
 static void bindDebugParameters(sqlite3_stmt *pStmt){
   int nVar = sqlite3_bind_parameter_count(pStmt);
   int i;
-  for(i=0; i<nVar; i++){
-    const char *zVar = sqlite3_bind_parameter_name(pStmt, i+1);
+  for(i=1; i<=nVar; i++){
+    const char *zVar = sqlite3_bind_parameter_name(pStmt, i);
     if( zVar==0 ) continue;
     if( strncmp(zVar, "$int_", 5)==0 ){
-      sqlite3_bind_int(pStmt, i+1, atoi(&zVar[5]));
+      sqlite3_bind_int(pStmt, i, atoi(&zVar[5]));
     }else
     if( strncmp(zVar, "$text_", 6)==0 ){
       size_t szVar = strlen(zVar);
diff --git a/test/hook.test b/test/hook.test
index 3d735875df..8638d3a6ba 100644
--- a/test/hook.test
+++ b/test/hook.test
@@ -706,11 +706,13 @@ ifcapable altertable {
   }
 }
 
-if 0 {
+if 1 {
   # At time of writing, these two are broken. They demonstrate that the
   # sqlite3_preupdate_old() method does not handle the case where ALTER TABLE
   # has been used to add a column with a default value other than NULL.
   #
+  # 2024-09-18: These are now fixed.
+  #
   do_preupdate_test 7.5.2.1 {
     DELETE FROM t8 WHERE a = 'one'
   } {
@@ -1022,4 +1024,37 @@ do_catchsql_test 12.6 {
   INSERT INTO t4 VALUES('def', 3);
 } {1 {UNIQUE constraint failed: t4.a}}
 
+#-------------------------------------------------------------------------
+# Test adding non-NULL default values using ALTER TABLE.
+#
+reset_db
+db preupdate hook preupdate_hook
+do_execsql_test 13.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY);
+  INSERT INTO t1 VALUES(100), (200), (300), (400);
+}
+
+do_execsql_test 13.1 {
+  ALTER TABLE t1 ADD COLUMN b DEFAULT 1234;
+  ALTER TABLE t1 ADD COLUMN c DEFAULT 'abcdef';
+  ALTER TABLE t1 ADD COLUMN d DEFAULT NULL;
+}
+
+do_preupdate_test 13.2 {
+  DELETE FROM t1 WHERE a=300
+} {DELETE main t1 300 300 0 300 1234 abcdef {}}
+
+do_preupdate_test 13.3 {
+  UPDATE t1 SET d='hello world' WHERE a=200
+} {
+  UPDATE main t1 200 200 0 200 1234 abcdef {} 
+                           200 1234 abcdef {hello world}
+}
+
+do_preupdate_test 13.4 {
+  INSERT INTO t1 DEFAULT VALUES;
+} {
+  INSERT main t1 401 401 0 401 1234 abcdef {}
+}
+
 finish_test
diff --git a/test/shell1.test b/test/shell1.test
index f355989c31..b4883e3521 100644
--- a/test/shell1.test
+++ b/test/shell1.test
@@ -516,6 +516,8 @@ do_test shell1-3.15.3 {
    Options:
      --bom                 Prefix output with a UTF8 byte-order mark
      -e                    Send output to the system text editor
+     --plain               Use text/plain for -w option
+     -w                    Send output to a web browser
      -x                    Send output as CSV to a spreadsheet
 child process exited abnormally}}
 
@@ -532,6 +534,8 @@ do_test shell1-3.16.2 {
    Options:
      --bom                 Prefix output with a UTF8 byte-order mark
      -e                    Send output to the system text editor
+     --plain               Use text/plain for -w option
+     -w                    Send output to a web browser
      -x                    Send output as CSV to a spreadsheet
 child process exited abnormally}}
 
diff --git a/test/wal2savepoint.test b/test/wal2savepoint.test
index c35f86ee0b..2b09164394 100644
--- a/test/wal2savepoint.test
+++ b/test/wal2savepoint.test
@@ -22,6 +22,7 @@ source $testdir/wal_common.tcl
 set testprefix wal2savepoint
 ifcapable !wal {finish_test ; return }
 
+reset_prng_state
 do_execsql_test 1.0 {
   CREATE TABLE t1(a, b, c);
   CREATE INDEX t1a ON t1(a);
@@ -71,4 +72,3 @@ do_execsql_test 1.4 {
 
 
 finish_test
-
diff --git a/tool/build-shell.sh b/tool/build-shell.sh
index a57f753039..7899080ebe 100644
--- a/tool/build-shell.sh
+++ b/tool/build-shell.sh
@@ -17,5 +17,5 @@ gcc -o sqlite3 -g -Os -I. \
    -DSQLITE_ENABLE_RTREE \
    -DHAVE_READLINE \
    ../sqlite/src/shell.c \
-   ../sqlite/src/test_vfstrace.c \
+   ../sqlite/ext/misc/vfstrace.c \
    sqlite3.c -ldl -lreadline -lncurses
diff --git a/tool/pagesig.c b/tool/pagesig.c
index 540c9d7226..37cc804a53 100644
--- a/tool/pagesig.c
+++ b/tool/pagesig.c
@@ -26,7 +26,7 @@
 ** the entire block.
 **
 ** For blocks of more than 16 bytes, the signature is a hex dump of the
-** first 8 bytes followed by a 64-bit has of the entire block.
+** first 8 bytes followed by a 64-bit hash of the entire block.
 */
 static void vlogSignature(unsigned char *p, int n, char *zCksum){
   unsigned int s0 = 0, s1 = 0;
diff --git a/tool/showdb.c b/tool/showdb.c
index 1b80c7f170..12c2e271b7 100644
--- a/tool/showdb.c
+++ b/tool/showdb.c
@@ -1084,17 +1084,28 @@ static void ptrmap_coverage_report(const char *zDbName){
     printf("%5llu: PTRMAP page covering %llu..%llu\n", pgno,
            pgno+1, pgno+perPage);
     a = fileRead((pgno-1)*g.pagesize, usable);
-    for(i=0; i+5<=usable && pgno+1+i/5<=g.mxPage; i+=5){
-      const char *zType = "???";
+    for(i=0; i+5<=usable; i+=5){
+      const char *zType;
       u32 iFrom = decodeInt32(&a[i+1]);
+      const char *zExtra = pgno+1+i/5>g.mxPage ? " (off end of DB)" : "";
       switch( a[i] ){
         case 1:  zType = "b-tree root page";        break;
         case 2:  zType = "freelist page";           break;
         case 3:  zType = "first page of overflow";  break;
         case 4:  zType = "later page of overflow";  break;
         case 5:  zType = "b-tree non-root page";    break;
+        default: {
+          if( zExtra[0]==0 ){
+            printf("%5llu: invalid (0x%02x), parent=%u\n", 
+                   pgno+1+i/5, a[i], iFrom);
+          }
+          zType = 0;
+          break;
+        }
+      }
+      if( zType ){
+        printf("%5llu: %s, parent=%u%s\n", pgno+1+i/5, zType, iFrom, zExtra);
       }
-      printf("%5llu: %s, parent=%u\n", pgno+1+i/5, zType, iFrom);
     }
     sqlite3_free(a);
   }
diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl
index abc1e7e98b..e50415900a 100644
--- a/tool/spaceanal.tcl
+++ b/tool/spaceanal.tcl
@@ -74,6 +74,16 @@ Options:
 }
   exit 1
 }
+
+# Exit with given code, but first close db if open.
+#
+proc exit_clean {exit_code} {
+  if {0 < [llength [info commands db]]} {
+    db close
+  }
+  exit $exit_code
+}
+
 set file_to_analyze {}
 set flags(-pageinfo) 0
 set flags(-stats) 0
@@ -157,7 +167,7 @@ if {![db exists {SELECT 1 FROM pragma_compile_options
         lacks required capabilities. Recompile using the\
         -DSQLITE_ENABLE_DBSTAT_VTAB compile-time option to fix\
         this problem."
-  exit 1
+  exit_clean 1
 }
 
 db eval {SELECT count(*) FROM sqlite_schema}
@@ -168,7 +178,7 @@ if {$flags(-pageinfo)} {
   db eval {SELECT name, path, pageno FROM temp.stat ORDER BY pageno} {
     puts "$pageno $name $path"
   }
-  exit 0
+  exit_clean 0
 }
 if {$flags(-stats)} {
   db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat}
@@ -198,7 +208,7 @@ if {$flags(-stats)} {
     puts "INSERT INTO stats VALUES($x);"
   }
   puts "COMMIT;"
-  exit 0
+  exit_clean 0
 }
 
 
@@ -901,5 +911,7 @@ puts "COMMIT;"
 } err]} {
   puts "ERROR: $err"
   puts $errorInfo
-  exit 1
+  exit_clean 1
 }
+
+exit_clean 0
diff --git a/tool/sqldiff.c b/tool/sqldiff.c
index 8b2293cafd..bb26daf139 100644
--- a/tool/sqldiff.c
+++ b/tool/sqldiff.c
@@ -14,7 +14,7 @@
 ** between two SQLite databases.
 **
 ** To compile, simply link against SQLite.  (Windows builds must also link
-** against ext/consio/console_io.c.)
+** against ext/misc/sqlite3_stdio.c.)
 **
 ** See the showHelp() routine below for a brief description of how to
 ** run the utility.
@@ -26,19 +26,7 @@
 #include <string.h>
 #include <assert.h>
 #include "sqlite3.h"
-
-/* Output function substitutions that cause UTF8 characters to be rendered
-** correctly on Windows:
-**
-**     fprintf()  ->  Wfprintf()
-**
-*/
-#if defined(_WIN32)
-# include "console_io.h"
-# define Wfprintf fPrintfUtf8
-#else
-# define Wfprintf fprintf
-#endif
+#include "sqlite3_stdio.h"
 
 /*
 ** All global variables are gathered into the "g" singleton.
@@ -76,9 +64,9 @@ static void cmdlineError(const char *zFormat, ...){
   va_start(ap, zFormat);
   sqlite3_str_vappendf(pOut, zFormat, ap);
   va_end(ap);
-  Wfprintf(stderr, "%s: %s\n", g.zArgv0, sqlite3_str_value(pOut));
+  sqlite3_fprintf(stderr, "%s: %s\n", g.zArgv0, sqlite3_str_value(pOut));
   strFree(pOut);
-  Wfprintf(stderr, "\"%s --help\" for more help\n", g.zArgv0);
+  sqlite3_fprintf(stderr, "\"%s --help\" for more help\n", g.zArgv0);
   exit(1);
 }
 
@@ -92,7 +80,7 @@ static void runtimeError(const char *zFormat, ...){
   va_start(ap, zFormat);
   sqlite3_str_vappendf(pOut, zFormat, ap);
   va_end(ap);
-  Wfprintf(stderr, "%s: %s\n", g.zArgv0, sqlite3_str_value(pOut));
+  sqlite3_fprintf(stderr, "%s: %s\n", g.zArgv0, sqlite3_str_value(pOut));
   strFree(pOut);
   exit(1);
 }
@@ -349,11 +337,11 @@ static void printQuoted(FILE *out, sqlite3_value *X){
       char zBuf[50];
       r1 = sqlite3_value_double(X);
       sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1);
-      fprintf(out, "%s", zBuf);
+      sqlite3_fprintf(out, "%s", zBuf);
       break;
     }
     case SQLITE_INTEGER: {
-      fprintf(out, "%lld", sqlite3_value_int64(X));
+      sqlite3_fprintf(out, "%lld", sqlite3_value_int64(X));
       break;
     }
     case SQLITE_BLOB: {
@@ -361,14 +349,14 @@ static void printQuoted(FILE *out, sqlite3_value *X){
       int nBlob = sqlite3_value_bytes(X);
       if( zBlob ){
         int i;
-        fprintf(out, "x'");
+        sqlite3_fprintf(out, "x'");
         for(i=0; i<nBlob; i++){
-          fprintf(out, "%02x", zBlob[i]);
+          sqlite3_fprintf(out, "%02x", zBlob[i]);
         }
-        fprintf(out, "'");
+        sqlite3_fprintf(out, "'");
       }else{
         /* Could be an OOM, could be a zero-byte blob */
-        fprintf(out, "X''");
+        sqlite3_fprintf(out, "X''");
       }
       break;
     }
@@ -376,38 +364,38 @@ static void printQuoted(FILE *out, sqlite3_value *X){
       const unsigned char *zArg = sqlite3_value_text(X);
 
       if( zArg==0 ){
-        fprintf(out, "NULL");
+        sqlite3_fprintf(out, "NULL");
       }else{
         int inctl = 0;
         int i, j;
-        fprintf(out, "'");
+        sqlite3_fprintf(out, "'");
         for(i=j=0; zArg[i]; i++){
           char c = zArg[i];
           int ctl = iscntrl((unsigned char)c);
           if( ctl>inctl ){
             inctl = ctl;
-            fprintf(out, "%.*s'||X'%02x", i-j, &zArg[j], c);
+            sqlite3_fprintf(out, "%.*s'||X'%02x", i-j, &zArg[j], c);
             j = i+1;
           }else if( ctl ){
-            fprintf(out, "%02x", c);
+            sqlite3_fprintf(out, "%02x", c);
             j = i+1;
           }else{
             if( inctl ){
               inctl = 0;
-              fprintf(out, "'\n||'");
+              sqlite3_fprintf(out, "'\n||'");
             }
             if( c=='\'' ){
-              fprintf(out, "%.*s'", i-j+1, &zArg[j]);
+              sqlite3_fprintf(out, "%.*s'", i-j+1, &zArg[j]);
               j = i+1;
             }
           }
         }
-        fprintf(out, "%s'", &zArg[j]);
+        sqlite3_fprintf(out, "%s'", &zArg[j]);
       }
       break;
     }
     case SQLITE_NULL: {
-      fprintf(out, "NULL");
+      sqlite3_fprintf(out, "NULL");
       break;
     }
   }
@@ -428,7 +416,7 @@ static void dump_table(const char *zTab, FILE *out){
 
   pStmt = db_prepare("SELECT sql FROM aux.sqlite_schema WHERE name=%Q", zTab);
   if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
+    sqlite3_fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
   }
   sqlite3_finalize(pStmt);
   if( !g.bSchemaOnly ){
@@ -463,14 +451,14 @@ static void dump_table(const char *zTab, FILE *out){
     }
     nCol = sqlite3_column_count(pStmt);
     while( SQLITE_ROW==sqlite3_step(pStmt) ){
-      Wfprintf(out, "%s",sqlite3_str_value(pIns));
+      sqlite3_fprintf(out, "%s",sqlite3_str_value(pIns));
       zSep = "(";
       for(i=0; i<nCol; i++){
-        Wfprintf(out, "%s",zSep);
+        sqlite3_fprintf(out, "%s",zSep);
         printQuoted(out, sqlite3_column_value(pStmt,i));
         zSep = ",";
       }
-      Wfprintf(out, ");\n");
+      sqlite3_fprintf(out, ");\n");
     }
     sqlite3_finalize(pStmt);
     strFree(pIns);
@@ -479,7 +467,7 @@ static void dump_table(const char *zTab, FILE *out){
                      " WHERE type='index' AND tbl_name=%Q AND sql IS NOT NULL",
                      zTab);
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
-    Wfprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
+    sqlite3_fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
   }
   sqlite3_finalize(pStmt);
   sqlite3_free(zId);
@@ -514,14 +502,14 @@ static void diff_one_table(const char *zTab, FILE *out){
     */
     az = columnNames("aux",zTab, &nPk, 0);
     if( az==0 ){
-      Wfprintf(stdout, "Rowid not accessible for %s\n", zId);
+      sqlite3_fprintf(stdout, "Rowid not accessible for %s\n", zId);
     }else{
-      Wfprintf(stdout, "%s:", zId);
+      sqlite3_fprintf(stdout, "%s:", zId);
       for(i=0; az[i]; i++){
-        Wfprintf(stdout, " %s", az[i]);
-        if( i+1==nPk ) Wfprintf(stdout, " *");
+        sqlite3_fprintf(stdout, " %s", az[i]);
+        if( i+1==nPk ) sqlite3_fprintf(stdout, " *");
       }
-      Wfprintf(stdout, "\n");
+      sqlite3_fprintf(stdout, "\n");
     }
     goto end_diff_one_table;
   }
@@ -530,9 +518,9 @@ static void diff_one_table(const char *zTab, FILE *out){
     if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
       /* Table missing from second database. */
       if( g.bSchemaCompare )
-        Wfprintf(out, "-- 2nd DB has no %s table\n", zTab);
+        sqlite3_fprintf(out, "-- 2nd DB has no %s table\n", zTab);
       else
-        Wfprintf(out, "DROP TABLE %s;\n", zId);
+        sqlite3_fprintf(out, "DROP TABLE %s;\n", zId);
     }
     goto end_diff_one_table;
   }
@@ -540,7 +528,7 @@ static void diff_one_table(const char *zTab, FILE *out){
   if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
     /* Table missing from source */
     if( g.bSchemaCompare ){
-      Wfprintf(out, "-- 1st DB has no %s table\n", zTab);
+      sqlite3_fprintf(out, "-- 1st DB has no %s table\n", zTab);
     }else{
       dump_table(zTab, out);
     }
@@ -560,7 +548,7 @@ static void diff_one_table(const char *zTab, FILE *out){
    || az[n]
   ){
     /* Schema mismatch */
-    Wfprintf(out, "%sDROP TABLE %s; -- due to schema mismatch\n", zLead, zId);
+    sqlite3_fprintf(out, "%sDROP TABLE %s; -- due to schema mismatch\n", zLead, zId);
     dump_table(zTab, out);
     goto end_diff_one_table;
   }
@@ -568,7 +556,7 @@ static void diff_one_table(const char *zTab, FILE *out){
   /* Build the comparison query */
   for(n2=n; az2[n2]; n2++){
     char *zNTab = safeId(az2[n2]);
-    Wfprintf(out, "ALTER TABLE %s ADD COLUMN %s;\n", zId, zNTab);
+    sqlite3_fprintf(out, "ALTER TABLE %s ADD COLUMN %s;\n", zId, zNTab);
     sqlite3_free(zNTab);
   }
   nQ = nPk2+1+2*(n2-nPk2);
@@ -669,7 +657,7 @@ static void diff_one_table(const char *zTab, FILE *out){
     zTab, zTab);
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
     char *z = safeId((const char*)sqlite3_column_text(pStmt,0));
-    fprintf(out, "DROP INDEX %s;\n", z);
+    sqlite3_fprintf(out, "DROP INDEX %s;\n", z);
     sqlite3_free(z);
   }
   sqlite3_finalize(pStmt);
@@ -681,39 +669,39 @@ static void diff_one_table(const char *zTab, FILE *out){
       int iType = sqlite3_column_int(pStmt, nPk);
       if( iType==1 || iType==2 ){
         if( iType==1 ){       /* Change the content of a row */
-          fprintf(out, "%sUPDATE %s", zLead, zId);
+          sqlite3_fprintf(out, "%sUPDATE %s", zLead, zId);
           zSep = " SET";
           for(i=nPk+1; i<nQ; i+=2){
             if( sqlite3_column_int(pStmt,i)==0 ) continue;
-            fprintf(out, "%s %s=", zSep, az2[(i+nPk-1)/2]);
+            sqlite3_fprintf(out, "%s %s=", zSep, az2[(i+nPk-1)/2]);
             zSep = ",";
             printQuoted(out, sqlite3_column_value(pStmt,i+1));
           }
         }else{                /* Delete a row */
-          fprintf(out, "%sDELETE FROM %s", zLead, zId);
+          sqlite3_fprintf(out, "%sDELETE FROM %s", zLead, zId);
         }
         zSep = " WHERE";
         for(i=0; i<nPk; i++){
-          fprintf(out, "%s %s=", zSep, az2[i]);
+          sqlite3_fprintf(out, "%s %s=", zSep, az2[i]);
           printQuoted(out, sqlite3_column_value(pStmt,i));
           zSep = " AND";
         }
-        fprintf(out, ";\n");
+        sqlite3_fprintf(out, ";\n");
       }else{                  /* Insert a row */
-        fprintf(out, "%sINSERT INTO %s(%s", zLead, zId, az2[0]);
-        for(i=1; az2[i]; i++) fprintf(out, ",%s", az2[i]);
-        fprintf(out, ") VALUES");
+        sqlite3_fprintf(out, "%sINSERT INTO %s(%s", zLead, zId, az2[0]);
+        for(i=1; az2[i]; i++) sqlite3_fprintf(out, ",%s", az2[i]);
+        sqlite3_fprintf(out, ") VALUES");
         zSep = "(";
         for(i=0; i<nPk2; i++){
-          fprintf(out, "%s", zSep);
+          sqlite3_fprintf(out, "%s", zSep);
           zSep = ",";
           printQuoted(out, sqlite3_column_value(pStmt,i));
         }
         for(i=nPk2+2; i<nQ; i+=2){
-          fprintf(out, ",");
+          sqlite3_fprintf(out, ",");
           printQuoted(out, sqlite3_column_value(pStmt,i));
         }
-        fprintf(out, ");\n");
+        sqlite3_fprintf(out, ");\n");
       }
     }
     sqlite3_finalize(pStmt);
@@ -729,7 +717,7 @@ static void diff_one_table(const char *zTab, FILE *out){
     "                      AND sql IS NOT NULL)",
     zTab, zTab);
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
-    fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
+    sqlite3_fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
   }
   sqlite3_finalize(pStmt);
 
@@ -1283,17 +1271,17 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
     ** statement first. And reset pCt so that it will not be
     ** printed again.  */
     if( sqlite3_str_length(pCt) ){
-      fprintf(out, "%s\n", sqlite3_str_value(pCt));
+      sqlite3_fprintf(out, "%s\n", sqlite3_str_value(pCt));
       sqlite3_str_reset(pCt);
     }
 
     /* Output the first part of the INSERT statement */
-    fprintf(out, "%s", sqlite3_str_value(pInsert));
+    sqlite3_fprintf(out, "%s", sqlite3_str_value(pInsert));
     nRow++;
 
     if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){
       for(i=0; i<=nCol; i++){
-        if( i>0 ) fprintf(out, ", ");
+        if( i>0 ) sqlite3_fprintf(out, ", ");
         printQuoted(out, sqlite3_column_value(pStmt, i));
       }
     }else{
@@ -1320,9 +1308,9 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
           nDelta = rbuDeltaCreate(aSrc, nSrc, aFinal, nFinal, aDelta);
           if( nDelta<nFinal ){
             int j;
-            fprintf(out, "x'");
-            for(j=0; j<nDelta; j++) fprintf(out, "%02x", (u8)aDelta[j]);
-            fprintf(out, "'");
+            sqlite3_fprintf(out, "x'");
+            for(j=0; j<nDelta; j++) sqlite3_fprintf(out, "%02x", (u8)aDelta[j]);
+            sqlite3_fprintf(out, "'");
             zOtaControl[i-bOtaRowid] = 'f';
             bDone = 1;
           }
@@ -1332,14 +1320,14 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
         if( bDone==0 ){
           printQuoted(out, sqlite3_column_value(pStmt, i));
         }
-        fprintf(out, ", ");
+        sqlite3_fprintf(out, ", ");
       }
-      fprintf(out, "'%s'", zOtaControl);
+      sqlite3_fprintf(out, "'%s'", zOtaControl);
       sqlite3_free(zOtaControl);
     }
 
     /* And the closing bracket of the insert statement */
-    fprintf(out, ");\n");
+    sqlite3_fprintf(out, ");\n");
   }
 
   sqlite3_finalize(pStmt);
@@ -1347,7 +1335,7 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
     sqlite3_str *pCnt = sqlite3_str_new(0);
     sqlite3_str_appendf(pCnt,
          "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow);
-    fprintf(out, "%s\n", sqlite3_str_value(pCnt));
+    sqlite3_fprintf(out, "%s\n", sqlite3_str_value(pCnt));
     strFree(pCnt);
   }
 
@@ -1386,14 +1374,14 @@ static void summarize_one_table(const char *zTab, FILE *out){
   if( sqlite3_table_column_metadata(g.db,"aux",zTab,0,0,0,0,0,0) ){
     if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
       /* Table missing from second database. */
-      Wfprintf(out, "%s: missing from second database\n", zTab);
+      sqlite3_fprintf(out, "%s: missing from second database\n", zTab);
     }
     goto end_summarize_one_table;
   }
 
   if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
     /* Table missing from source */
-    Wfprintf(out, "%s: missing from first database\n", zTab);
+    sqlite3_fprintf(out, "%s: missing from first database\n", zTab);
     goto end_summarize_one_table;
   }
 
@@ -1410,7 +1398,7 @@ static void summarize_one_table(const char *zTab, FILE *out){
    || az[n]
   ){
     /* Schema mismatch */
-    Wfprintf(out, "%s: incompatible schema\n", zTab);
+    sqlite3_fprintf(out, "%s: incompatible schema\n", zTab);
     goto end_summarize_one_table;
   }
 
@@ -1455,7 +1443,7 @@ static void summarize_one_table(const char *zTab, FILE *out){
   sqlite3_str_appendf(pSql, ")\n ORDER BY 1;\n");
 
   if( (g.fDebug & DEBUG_DIFF_SQL)!=0 ){ 
-    Wfprintf(stdout, "SQL for %s:\n%s\n", zId, sqlite3_str_value(pSql));
+    sqlite3_fprintf(stdout, "SQL for %s:\n%s\n", zId, sqlite3_str_value(pSql));
     goto end_summarize_one_table;
   }
 
@@ -1480,7 +1468,7 @@ static void summarize_one_table(const char *zTab, FILE *out){
     }
   }
   sqlite3_finalize(pStmt);
-  Wfprintf(out,
+  sqlite3_fprintf(out,
           "%s: %lld changes, %lld inserts, %lld deletes, %lld unchanged\n",
           zTab, nUpdate, nInsert, nDelete, nUnchanged);
 
@@ -1661,7 +1649,7 @@ static void changeset_one_table(const char *zTab, FILE *out){
   sqlite3_str_appendf(pSql, ";\n");
 
   if( g.fDebug & DEBUG_DIFF_SQL ){ 
-    Wfprintf(stdout, "SQL for %s:\n%s\n", zId, sqlite3_str_value(pSql));
+    sqlite3_fprintf(stdout, "SQL for %s:\n%s\n", zId, sqlite3_str_value(pSql));
     goto end_changeset_one_table;
   }
 
@@ -1891,8 +1879,8 @@ const char *all_tables_sql(){
 ** Print sketchy documentation for this utility program
 */
 static void showHelp(void){
-  Wfprintf(stdout, "Usage: %s [options] DB1 DB2\n", g.zArgv0);
-  Wfprintf(stdout,
+  sqlite3_fprintf(stdout, "Usage: %s [options] DB1 DB2\n", g.zArgv0);
+  sqlite3_fprintf(stdout,
 "Output SQL text that would transform DB1 into DB2.\n"
 "Options:\n"
 "  --changeset FILE      Write a CHANGESET into FILE\n"
@@ -1935,7 +1923,7 @@ int main(int argc, char **argv){
       if( z[0]=='-' ) z++;
       if( strcmp(z,"changeset")==0 ){
         if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
-        out = fopen(argv[++i], "wb");
+        out = sqlite3_fopen(argv[++i], "wb");
         if( out==0 ) cmdlineError("cannot open: %s", argv[i]);
         xDiff = changeset_one_table;
         neverUseTransaction = 1;
@@ -2036,9 +2024,9 @@ int main(int argc, char **argv){
   }
 
   if( neverUseTransaction ) useTransaction = 0;
-  if( useTransaction ) Wfprintf(out, "BEGIN TRANSACTION;\n");
+  if( useTransaction ) sqlite3_fprintf(out, "BEGIN TRANSACTION;\n");
   if( xDiff==rbudiff_one_table ){
-    Wfprintf(out, "CREATE TABLE IF NOT EXISTS rbu_count"
+    sqlite3_fprintf(out, "CREATE TABLE IF NOT EXISTS rbu_count"
            "(tbl TEXT PRIMARY KEY COLLATE NOCASE, cnt INTEGER) "
            "WITHOUT ROWID;\n"
     );
@@ -2053,7 +2041,7 @@ int main(int argc, char **argv){
     }
     sqlite3_finalize(pStmt);
   }
-  if( useTransaction ) Wfprintf(stdout,"COMMIT;\n");
+  if( useTransaction ) sqlite3_fprintf(stdout,"COMMIT;\n");
 
   /* TBD: Handle trigger differences */
   /* TBD: Handle view differences */
diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c
new file mode 100644
index 0000000000..1be206a758
--- /dev/null
+++ b/tool/sqlite3-rsync.c
@@ -0,0 +1,1862 @@
+/*
+** 2024-09-10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This is a utility program that makes a copy of a live SQLite database
+** using a bandwidth-efficient protocol, similar to "rsync".
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdarg.h>
+#include "sqlite3.h"
+
+static const char zUsage[] = 
+  "sqlite3-rsync ORIGIN REPLICA ?OPTIONS?\n"
+  "\n"
+  "One of ORIGIN or REPLICA is a pathname to a database on the local\n"
+  "machine and the other is of the form \"USER@HOST:PATH\" describing\n"
+  "a database on a remote machine.  This utility makes REPLICA into a\n"
+  "copy of ORIGIN\n"
+  "\n"
+  "OPTIONS:\n"
+  "\n"
+  "   --exe PATH    Name of the sqlite3-rsync program on the remote side\n"
+  "   --help        Show this help screen\n"
+  "   --ssh PATH    Name of the SSH program used to reach the remote side\n"
+  "   -v            Verbose.  Multiple v's for increasing output\n"
+  "   --version     Show detailed version information\n"
+;
+
+typedef unsigned char u8;
+
+/* Context for the run */
+typedef struct SQLiteRsync SQLiteRsync;
+struct SQLiteRsync {
+  const char *zOrigin;     /* Name of the origin */
+  const char *zReplica;    /* Name of the replica */
+  const char *zErrFile;    /* Append error messages to this file */
+  FILE *pOut;              /* Transmit to the other side */
+  FILE *pIn;               /* Receive from the other side */
+  FILE *pLog;              /* Duplicate output here if not NULL */
+  sqlite3 *db;             /* Database connection */
+  int nErr;                /* Number of errors encountered */
+  int nWrErr;              /* Number of failed attempts to write on the pipe */
+  u8 eVerbose;             /* Bigger for more output.  0 means none. */
+  u8 bCommCheck;           /* True to debug the communication protocol */
+  u8 isRemote;             /* On the remote side of a connection */
+  u8 isReplica;            /* True if running on the replica side */
+  u8 iProtocol;            /* Protocol version number */
+  sqlite3_uint64 nOut;     /* Bytes transmitted */
+  sqlite3_uint64 nIn;      /* Bytes received */
+  unsigned int nPage;      /* Total number of pages in the database */
+  unsigned int szPage;     /* Database page size */
+  unsigned int nHashSent;  /* Hashes sent (replica to origin) */
+  unsigned int nPageSent;  /* Page contents sent (origin to replica) */
+};
+
+/* The version number of the protocol.  Sent in the *_BEGIN message
+** to verify that both sides speak the same dialect.
+*/
+#define PROTOCOL_VERSION  1
+
+
+/* Magic numbers to identify particular messages sent over the wire.
+*/
+#define ORIGIN_BEGIN    0x41     /* Initial message */
+#define ORIGIN_END      0x42     /* Time to quit */
+#define ORIGIN_ERROR    0x43     /* Error message from the remote */
+#define ORIGIN_PAGE     0x44     /* New page data */
+#define ORIGIN_TXN      0x45     /* Transaction commit */
+#define ORIGIN_MSG      0x46     /* Informational message */
+
+#define REPLICA_BEGIN   0x61     /* Welcome message */
+#define REPLICA_ERROR   0x62     /* Error.  Report and quit. */
+#define REPLICA_END     0x63     /* Replica wants to stop */
+#define REPLICA_HASH    0x64     /* One or more pages hashes to report */
+#define REPLICA_READY   0x65     /* Read to receive page content */
+#define REPLICA_MSG     0x66     /* Informational message */
+
+
+/****************************************************************************
+** Beginning of the popen2() implementation copied from Fossil  *************
+****************************************************************************/
+#ifdef _WIN32
+#include <windows.h>
+#include <fcntl.h>
+/*
+** Print a fatal error and quit.
+*/
+static void win32_fatal_error(const char *zMsg){
+  fprintf(stderr, "%s", zMsg);
+  exit(1);
+}
+extern int _open_osfhandle(intptr_t,int);
+#else
+#include <unistd.h>
+#include <signal.h>
+#include <sys/wait.h>
+#endif
+
+/*
+** The following macros are used to cast pointers to integers and
+** integers to pointers.  The way you do this varies from one compiler
+** to the next, so we have developed the following set of #if statements
+** to generate appropriate macros for a wide range of compilers.
+**
+** The correct "ANSI" way to do this is to use the intptr_t type.
+** Unfortunately, that typedef is not available on all compilers, or
+** if it is available, it requires an #include of specific headers
+** that vary from one machine to the next.
+**
+** This code is copied out of SQLite.
+*/
+#if defined(__PTRDIFF_TYPE__)  /* This case should work for GCC */
+# define INT_TO_PTR(X)  ((void*)(__PTRDIFF_TYPE__)(X))
+# define PTR_TO_INT(X)  ((int)(__PTRDIFF_TYPE__)(X))
+#elif !defined(__GNUC__)       /* Works for compilers other than LLVM */
+# define INT_TO_PTR(X)  ((void*)&((char*)0)[X])
+# define PTR_TO_INT(X)  ((int)(((char*)X)-(char*)0))
+#elif defined(HAVE_STDINT_H)   /* Use this case if we have ANSI headers */
+# define INT_TO_PTR(X)  ((void*)(intptr_t)(X))
+# define PTR_TO_INT(X)  ((int)(intptr_t)(X))
+#else                          /* Generates a warning - but it always works */
+# define INT_TO_PTR(X)  ((void*)(X))
+# define PTR_TO_INT(X)  ((int)(X))
+#endif
+
+/* Register SQL functions provided by ext/misc/sha1.c */
+extern int sqlite3_sha_init(
+  sqlite3 *db,
+  char **pzErrMsg,
+  const sqlite3_api_routines *pApi
+);
+
+#ifdef _WIN32
+/*
+** On windows, create a child process and specify the stdin, stdout,
+** and stderr channels for that process to use.
+**
+** Return the number of errors.
+*/
+static int win32_create_child_process(
+  wchar_t *zCmd,       /* The command that the child process will run */
+  HANDLE hIn,          /* Standard input */
+  HANDLE hOut,         /* Standard output */
+  HANDLE hErr,         /* Standard error */
+  DWORD *pChildPid     /* OUT: Child process handle */
+){
+  STARTUPINFOW si;
+  PROCESS_INFORMATION pi;
+  BOOL rc;
+
+  memset(&si, 0, sizeof(si));
+  si.cb = sizeof(si);
+  si.dwFlags = STARTF_USESTDHANDLES;
+  SetHandleInformation(hIn, HANDLE_FLAG_INHERIT, TRUE);
+  si.hStdInput  = hIn;
+  SetHandleInformation(hOut, HANDLE_FLAG_INHERIT, TRUE);
+  si.hStdOutput = hOut;
+  SetHandleInformation(hErr, HANDLE_FLAG_INHERIT, TRUE);
+  si.hStdError  = hErr;
+  rc = CreateProcessW(
+     NULL,  /* Application Name */
+     zCmd,  /* Command-line */
+     NULL,  /* Process attributes */
+     NULL,  /* Thread attributes */
+     TRUE,  /* Inherit Handles */
+     0,     /* Create flags  */
+     NULL,  /* Environment */
+     NULL,  /* Current directory */
+     &si,   /* Startup Info */
+     &pi    /* Process Info */
+  );
+  if( rc ){
+    CloseHandle( pi.hProcess );
+    CloseHandle( pi.hThread );
+    *pChildPid = pi.dwProcessId;
+  }else{
+    win32_fatal_error("cannot create child process");
+  }
+  return rc!=0;
+}
+void *win32_utf8_to_unicode(const char *zUtf8){
+  int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0);
+  wchar_t *zUnicode = malloc( nByte*2 );
+  MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte);
+  return zUnicode;
+}
+#endif
+
+/*
+** Create a child process running shell command "zCmd".  *ppOut is
+** a FILE that becomes the standard input of the child process.
+** (The caller writes to *ppOut in order to send text to the child.)
+** *ppIn is stdout from the child process.  (The caller
+** reads from *ppIn in order to receive input from the child.)
+** Note that *ppIn is an unbuffered file descriptor, not a FILE.
+** The process ID of the child is written into *pChildPid.
+**
+** Return the number of errors.
+*/
+static int popen2(
+  const char *zCmd,      /* Command to run in the child process */
+  FILE **ppIn,           /* Read from child using this file descriptor */
+  FILE **ppOut,          /* Write to child using this file descriptor */
+  int *pChildPid,        /* PID of the child process */
+  int bDirect            /* 0: run zCmd as a shell cmd.  1: run directly */
+){
+#ifdef _WIN32
+  HANDLE hStdinRd, hStdinWr, hStdoutRd, hStdoutWr, hStderr;
+  SECURITY_ATTRIBUTES saAttr;
+  DWORD childPid = 0;
+  int fd;
+
+  saAttr.nLength = sizeof(saAttr);
+  saAttr.bInheritHandle = TRUE;
+  saAttr.lpSecurityDescriptor = NULL;
+  hStderr = GetStdHandle(STD_ERROR_HANDLE);
+  if( !CreatePipe(&hStdoutRd, &hStdoutWr, &saAttr, 4096) ){
+    win32_fatal_error("cannot create pipe for stdout");
+  }
+  SetHandleInformation( hStdoutRd, HANDLE_FLAG_INHERIT, FALSE);
+
+  if( !CreatePipe(&hStdinRd, &hStdinWr, &saAttr, 4096) ){
+    win32_fatal_error("cannot create pipe for stdin");
+  }
+  SetHandleInformation( hStdinWr, HANDLE_FLAG_INHERIT, FALSE);
+
+  win32_create_child_process(win32_utf8_to_unicode(zCmd),
+                             hStdinRd, hStdoutWr, hStderr,&childPid);
+  *pChildPid = childPid;
+  fd = _open_osfhandle(PTR_TO_INT(hStdoutRd), 0);
+  *ppIn = fdopen(fd, "r");
+  fd = _open_osfhandle(PTR_TO_INT(hStdinWr), 0);
+  *ppOut = _fdopen(fd, "w");
+  CloseHandle(hStdinRd);
+  CloseHandle(hStdoutWr);
+  return 0;
+#else
+  int pin[2], pout[2];
+  *ppIn = 0;
+  *ppOut = 0;
+  *pChildPid = 0;
+
+  if( pipe(pin)<0 ){
+    return 1;
+  }
+  if( pipe(pout)<0 ){
+    close(pin[0]);
+    close(pin[1]);
+    return 1;
+  }
+  *pChildPid = fork();
+  if( *pChildPid<0 ){
+    close(pin[0]);
+    close(pin[1]);
+    close(pout[0]);
+    close(pout[1]);
+    *pChildPid = 0;
+    return 1;
+  }
+  signal(SIGPIPE,SIG_IGN);
+  if( *pChildPid==0 ){
+    int fd;
+    /* This is the child process */
+    close(0);
+    fd = dup(pout[0]);
+    if( fd!=0 ) {
+      fprintf(stderr,"popen2() failed to open file descriptor 0");
+      exit(1);
+    }
+    close(pout[0]);
+    close(pout[1]);
+    close(1);
+    fd = dup(pin[1]);
+    if( fd!=1 ){
+      fprintf(stderr,"popen() failed to open file descriptor 1");
+      exit(1);
+    }
+    close(pin[0]);
+    close(pin[1]);
+    if( bDirect ){
+      execl(zCmd, zCmd, (char*)0);
+    }else{
+      execl("/bin/sh", "/bin/sh", "-c", zCmd, (char*)0);
+    }
+    return 1;
+  }else{
+    /* This is the parent process */
+    close(pin[1]);
+    *ppIn = fdopen(pin[0], "r");
+    close(pout[0]);
+    *ppOut = fdopen(pout[1], "w");
+    return 0;
+  }
+#endif
+}
+
+/*
+** Close the connection to a child process previously created using
+** popen2().
+*/
+static void pclose2(FILE *pIn, FILE *pOut, int childPid){
+#ifdef _WIN32
+  /* Not implemented, yet */
+  fclose(pIn);
+  fclose(pOut);
+#else
+  fclose(pIn);
+  fclose(pOut);
+  while( waitpid(0, 0, WNOHANG)>0 ) {}
+#endif
+}
+/*****************************************************************************
+** End of the popen2() implementation copied from Fossil *********************
+*****************************************************************************/
+
+/*****************************************************************************
+** Beginning of the append_escaped_arg() routine, adapted from the Fossil   **
+** subroutine nameed blob_append_escaped_arg()                              **
+*****************************************************************************/
+/*
+** ASCII (for reference):
+**    x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf
+** 0x ^`  ^a  ^b  ^c  ^d  ^e  ^f  ^g  \b  \t  \n  ()  \f  \r  ^n  ^o
+** 1x ^p  ^q  ^r  ^s  ^t  ^u  ^v  ^w  ^x  ^y  ^z  ^{  ^|  ^}  ^~  ^
+** 2x ()  !   "   #   $   %   &   '   (   )   *   +   ,   -   .   /
+** 3x 0   1   2   3   4   5   6   7   8   9   :   ;   <   =   >   ?
+** 4x @   A   B   C   D   E   F   G   H   I   J   K   L   M   N   O
+** 5x P   Q   R   S   T   U   V   W   X   Y   Z   [   \   ]   ^   _
+** 6x `   a   b   c   d   e   f   g   h   i   j   k   l   m   n   o
+** 7x p   q   r   s   t   u   v   w   x   y   z   {   |   }   ~   ^_
+*/
+
+/*
+** Meanings for bytes in a filename:
+**
+**    0      Ordinary character.  No encoding required
+**    1      Needs to be escaped
+**    2      Illegal character.  Do not allow in a filename
+**    3      First byte of a 2-byte UTF-8
+**    4      First byte of a 3-byte UTF-8
+**    5      First byte of a 4-byte UTF-8
+*/
+static const char aSafeChar[256] = {
+#ifdef _WIN32
+/* Windows
+** Prohibit:  all control characters, including tab, \r and \n.
+** Escape:    (space) " # $ % & ' ( ) * ; < > ? [ ] ^ ` { | }
+*/
+/*  x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf  */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 0x */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 1x */
+     1,  0,  1,  1,  1,  1,  1,  1,  1,  1,  1,  0,  0,  0,  0,  0, /* 2x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  0,  1,  1, /* 3x */
+     1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, /* 4x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  0,  1,  1,  0, /* 5x */
+     1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, /* 6x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  1,  0,  1, /* 7x */
+#else
+/* Unix
+** Prohibit:  all control characters, including tab, \r and \n
+** Escape:    (space) ! " # $ % & ' ( ) * ; < > ? [ \ ] ^ ` { | }
+*/
+/*  x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf  */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 0x */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 1x */
+     1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  0,  0,  0,  0,  0, /* 2x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  0,  1,  1, /* 3x */
+     1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, /* 4x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  1,  1,  0, /* 5x */
+     1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, /* 6x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  1,  0,  1, /* 7x */
+#endif
+    /* all bytes 0x80 through 0xbf are unescaped, being secondary
+    ** bytes to UTF8 characters.  Bytes 0xc0 through 0xff are the
+    ** first byte of a UTF8 character and do get escaped */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 8x */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 9x */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* ax */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* bx */
+     3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3, /* cx */
+     3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3, /* dx */
+     4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4, /* ex */
+     5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5  /* fx */
+};
+
+/*
+** pStr is a shell command under construction.  This routine safely
+** appends filename argument zIn.  It returns 0 on success or non-zero
+** on any error.
+**
+** The argument is escaped if it contains white space or other characters
+** that need to be escaped for the shell.  If zIn contains characters
+** that cannot be safely escaped, then throw a fatal error.
+**
+** If the isFilename argument is true, then the argument is expected
+** to be a filename.  As shell commands commonly have command-line
+** options that begin with "-" and since we do not want an attacker
+** to be able to invoke these switches using filenames that begin
+** with "-", if zIn begins with "-", prepend an additional "./"
+** (or ".\\" on Windows).
+*/
+int append_escaped_arg(sqlite3_str *pStr, const char *zIn, int isFilename){
+  int i;
+  unsigned char c;
+  int needEscape = 0;
+  int n = sqlite3_str_length(pStr);
+  char *z = sqlite3_str_value(pStr);
+
+  /* Look for illegal byte-sequences and byte-sequences that require
+  ** escaping.  No control-characters are allowed.  All spaces and
+  ** non-ASCII unicode characters and some punctuation characters require
+  ** escaping. */
+  for(i=0; (c = (unsigned char)zIn[i])!=0; i++){
+    if( aSafeChar[c] ){
+      unsigned char x = aSafeChar[c];
+      needEscape = 1;
+      if( x==2 ){
+        /* Bad ASCII character */
+        return 1;
+      }else if( x>2 ){
+        if( (zIn[i+1]&0xc0)!=0x80
+         || (x>=4 && (zIn[i+2]&0xc0)!=0x80)
+         || (x==5 && (zIn[i+3]&0xc0)!=0x80)
+        ){
+          /* Bad UTF8 character */
+          return 1;
+        }
+        i += x-2;
+      }
+    }
+  }
+
+  /* Separate from the previous argument by a space */
+  if( n>0 && !isspace(z[n-1]) ){
+    sqlite3_str_appendchar(pStr, 1, ' ');
+  }
+
+  /* Check for characters that need quoting */
+  if( !needEscape ){
+    if( isFilename && zIn[0]=='-' ){
+      sqlite3_str_appendchar(pStr, 1, '.');
+#if defined(_WIN32)
+      sqlite3_str_appendchar(pStr, 1, '\\');
+#else
+      sqlite3_str_appendchar(pStr, 1, '/');
+#endif
+    }
+    sqlite3_str_appendall(pStr, zIn);
+  }else{
+#if defined(_WIN32)
+    /* Quoting strategy for windows:
+    ** Put the entire name inside of "...".  Any " characters within
+    ** the name get doubled.
+    */
+    sqlite3_str_appendchar(pStr, 1, '"');
+    if( isFilename && zIn[0]=='-' ){
+      sqlite3_str_appendchar(pStr, 1, '.');
+      sqlite3_str_appendchar(pStr, 1, '\\');
+    }else if( zIn[0]=='/' ){
+      sqlite3_str_appendchar(pStr, 1, '.');
+    }
+    for(i=0; (c = (unsigned char)zIn[i])!=0; i++){
+      sqlite3_str_appendchar(pStr, 1, (char)c);
+      if( c=='"' ) sqlite3_str_appendchar(pStr, 1, '"');
+      if( c=='\\' ) sqlite3_str_appendchar(pStr, 1, '\\');
+      if( c=='%' && isFilename ) sqlite3_str_append(pStr, "%cd:~,%", 7);
+    }
+    sqlite3_str_appendchar(pStr, 1, '"');
+#else
+    /* Quoting strategy for unix:
+    ** If the name does not contain ', then surround the whole thing
+    ** with '...'.   If there is one or more ' characters within the
+    ** name, then put \ before each special character.
+    */
+    if( strchr(zIn,'\'') ){
+      if( isFilename && zIn[0]=='-' ){
+        sqlite3_str_appendchar(pStr, 1, '.');
+        sqlite3_str_appendchar(pStr, 1, '/');
+      }
+      for(i=0; (c = (unsigned char)zIn[i])!=0; i++){
+        if( aSafeChar[c] && aSafeChar[c]!=2 ){
+          sqlite3_str_appendchar(pStr, 1, '\\');
+        }
+        sqlite3_str_appendchar(pStr, 1, (char)c);
+      }
+    }else{
+      sqlite3_str_appendchar(pStr, 1, '\'');
+      if( isFilename && zIn[0]=='-' ){
+        sqlite3_str_appendchar(pStr, 1, '.');
+        sqlite3_str_appendchar(pStr, 1, '/');
+      }
+      sqlite3_str_appendall(pStr, zIn);
+      sqlite3_str_appendchar(pStr, 1, '\'');
+    }
+#endif
+  }
+  return 0;
+}
+/*****************************************************************************
+** End of the append_escaped_arg() routine, adapted from the Fossil         **
+*****************************************************************************/
+
+/*****************************************************************************
+** The Hash Engine
+**
+** This is basically SHA3, though with a 160-bit hash, and reducing the
+** number of rounds in the KeccakF1600 step function from 24 to 6.
+*/
+/*
+** Macros to determine whether the machine is big or little endian,
+** and whether or not that determination is run-time or compile-time.
+**
+** For best performance, an attempt is made to guess at the byte-order
+** using C-preprocessor macros.  If that is unsuccessful, or if
+** -DHash_BYTEORDER=0 is set, then byte-order is determined
+** at run-time.
+*/
+#ifndef Hash_BYTEORDER
+# if defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
+     defined(__x86_64) || defined(__x86_64__) || defined(_M_X64)  ||    \
+     defined(_M_AMD64) || defined(_M_ARM)     || defined(__x86)   ||    \
+     defined(__arm__)
+#   define Hash_BYTEORDER    1234
+# elif defined(sparc)    || defined(__ppc__)
+#   define Hash_BYTEORDER    4321
+# else
+#   define Hash_BYTEORDER 0
+# endif
+#endif
+
+typedef sqlite3_uint64 u64;
+
+/*
+** State structure for a Hash hash in progress
+*/
+typedef struct HashContext HashContext;
+struct HashContext {
+  union {
+    u64 s[25];                /* Keccak state. 5x5 lines of 64 bits each */
+    unsigned char x[1600];    /* ... or 1600 bytes */
+  } u;
+  unsigned nRate;        /* Bytes of input accepted per Keccak iteration */
+  unsigned nLoaded;      /* Input bytes loaded into u.x[] so far this cycle */
+  unsigned ixMask;       /* Insert next input into u.x[nLoaded^ixMask]. */
+  unsigned iSize;        /* 224, 256, 358, or 512 */
+};
+
+/*
+** A single step of the Keccak mixing function for a 1600-bit state
+*/
+static void KeccakF1600Step(HashContext *p){
+  int i;
+  u64 b0, b1, b2, b3, b4;
+  u64 c0, c1, c2, c3, c4;
+  u64 d0, d1, d2, d3, d4;
+  static const u64 RC[] = {
+    0x0000000000000001ULL,  0x0000000000008082ULL,
+    0x800000000000808aULL,  0x8000000080008000ULL,
+    0x000000000000808bULL,  0x0000000080000001ULL,
+    0x8000000080008081ULL,  0x8000000000008009ULL,
+    0x000000000000008aULL,  0x0000000000000088ULL,
+    0x0000000080008009ULL,  0x000000008000000aULL,
+    0x000000008000808bULL,  0x800000000000008bULL,
+    0x8000000000008089ULL,  0x8000000000008003ULL,
+    0x8000000000008002ULL,  0x8000000000000080ULL,
+    0x000000000000800aULL,  0x800000008000000aULL,
+    0x8000000080008081ULL,  0x8000000000008080ULL,
+    0x0000000080000001ULL,  0x8000000080008008ULL
+  };
+# define a00 (p->u.s[0])
+# define a01 (p->u.s[1])
+# define a02 (p->u.s[2])
+# define a03 (p->u.s[3])
+# define a04 (p->u.s[4])
+# define a10 (p->u.s[5])
+# define a11 (p->u.s[6])
+# define a12 (p->u.s[7])
+# define a13 (p->u.s[8])
+# define a14 (p->u.s[9])
+# define a20 (p->u.s[10])
+# define a21 (p->u.s[11])
+# define a22 (p->u.s[12])
+# define a23 (p->u.s[13])
+# define a24 (p->u.s[14])
+# define a30 (p->u.s[15])
+# define a31 (p->u.s[16])
+# define a32 (p->u.s[17])
+# define a33 (p->u.s[18])
+# define a34 (p->u.s[19])
+# define a40 (p->u.s[20])
+# define a41 (p->u.s[21])
+# define a42 (p->u.s[22])
+# define a43 (p->u.s[23])
+# define a44 (p->u.s[24])
+# define ROL64(a,x) ((a<<x)|(a>>(64-x)))
+
+  /*         v---- Number of rounds.  SHA3 has 24 here. */
+  for(i=0; i<6; i++){
+    c0 = a00^a10^a20^a30^a40;
+    c1 = a01^a11^a21^a31^a41;
+    c2 = a02^a12^a22^a32^a42;
+    c3 = a03^a13^a23^a33^a43;
+    c4 = a04^a14^a24^a34^a44;
+    d0 = c4^ROL64(c1, 1);
+    d1 = c0^ROL64(c2, 1);
+    d2 = c1^ROL64(c3, 1);
+    d3 = c2^ROL64(c4, 1);
+    d4 = c3^ROL64(c0, 1);
+
+    b0 = (a00^d0);
+    b1 = ROL64((a11^d1), 44);
+    b2 = ROL64((a22^d2), 43);
+    b3 = ROL64((a33^d3), 21);
+    b4 = ROL64((a44^d4), 14);
+    a00 =   b0 ^((~b1)&  b2 );
+    a00 ^= RC[i];
+    a11 =   b1 ^((~b2)&  b3 );
+    a22 =   b2 ^((~b3)&  b4 );
+    a33 =   b3 ^((~b4)&  b0 );
+    a44 =   b4 ^((~b0)&  b1 );
+
+    b2 = ROL64((a20^d0), 3);
+    b3 = ROL64((a31^d1), 45);
+    b4 = ROL64((a42^d2), 61);
+    b0 = ROL64((a03^d3), 28);
+    b1 = ROL64((a14^d4), 20);
+    a20 =   b0 ^((~b1)&  b2 );
+    a31 =   b1 ^((~b2)&  b3 );
+    a42 =   b2 ^((~b3)&  b4 );
+    a03 =   b3 ^((~b4)&  b0 );
+    a14 =   b4 ^((~b0)&  b1 );
+
+    b4 = ROL64((a40^d0), 18);
+    b0 = ROL64((a01^d1), 1);
+    b1 = ROL64((a12^d2), 6);
+    b2 = ROL64((a23^d3), 25);
+    b3 = ROL64((a34^d4), 8);
+    a40 =   b0 ^((~b1)&  b2 );
+    a01 =   b1 ^((~b2)&  b3 );
+    a12 =   b2 ^((~b3)&  b4 );
+    a23 =   b3 ^((~b4)&  b0 );
+    a34 =   b4 ^((~b0)&  b1 );
+
+    b1 = ROL64((a10^d0), 36);
+    b2 = ROL64((a21^d1), 10);
+    b3 = ROL64((a32^d2), 15);
+    b4 = ROL64((a43^d3), 56);
+    b0 = ROL64((a04^d4), 27);
+    a10 =   b0 ^((~b1)&  b2 );
+    a21 =   b1 ^((~b2)&  b3 );
+    a32 =   b2 ^((~b3)&  b4 );
+    a43 =   b3 ^((~b4)&  b0 );
+    a04 =   b4 ^((~b0)&  b1 );
+
+    b3 = ROL64((a30^d0), 41);
+    b4 = ROL64((a41^d1), 2);
+    b0 = ROL64((a02^d2), 62);
+    b1 = ROL64((a13^d3), 55);
+    b2 = ROL64((a24^d4), 39);
+    a30 =   b0 ^((~b1)&  b2 );
+    a41 =   b1 ^((~b2)&  b3 );
+    a02 =   b2 ^((~b3)&  b4 );
+    a13 =   b3 ^((~b4)&  b0 );
+    a24 =   b4 ^((~b0)&  b1 );
+  }
+}
+
+/*
+** Initialize a new hash.  iSize determines the size of the hash
+** in bits and should be one of 224, 256, 384, or 512.  Or iSize
+** can be zero to use the default hash size of 256 bits.
+*/
+static void HashInit(HashContext *p, int iSize){
+  memset(p, 0, sizeof(*p));
+  p->iSize = iSize;
+  if( iSize>=128 && iSize<=512 ){
+    p->nRate = (1600 - ((iSize + 31)&~31)*2)/8;
+  }else{
+    p->nRate = (1600 - 2*256)/8;
+  }
+#if Hash_BYTEORDER==1234
+  /* Known to be little-endian at compile-time. No-op */
+#elif Hash_BYTEORDER==4321
+  p->ixMask = 7;  /* Big-endian */
+#else
+  {
+    static unsigned int one = 1;
+    if( 1==*(unsigned char*)&one ){
+      /* Little endian.  No byte swapping. */
+      p->ixMask = 0;
+    }else{
+      /* Big endian.  Byte swap. */
+      p->ixMask = 7;
+    }
+  }
+#endif
+}
+
+/*
+** Make consecutive calls to the HashUpdate function to add new content
+** to the hash
+*/
+static void HashUpdate(
+  HashContext *p,
+  const unsigned char *aData,
+  unsigned int nData
+){
+  unsigned int i = 0;
+  if( aData==0 ) return;
+#if Hash_BYTEORDER==1234
+  if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){
+    for(; i+7<nData; i+=8){
+      p->u.s[p->nLoaded/8] ^= *(u64*)&aData[i];
+      p->nLoaded += 8;
+      if( p->nLoaded>=p->nRate ){
+        KeccakF1600Step(p);
+        p->nLoaded = 0;
+      }
+    }
+  }
+#endif
+  for(; i<nData; i++){
+#if Hash_BYTEORDER==1234
+    p->u.x[p->nLoaded] ^= aData[i];
+#elif Hash_BYTEORDER==4321
+    p->u.x[p->nLoaded^0x07] ^= aData[i];
+#else
+    p->u.x[p->nLoaded^p->ixMask] ^= aData[i];
+#endif
+    p->nLoaded++;
+    if( p->nLoaded==p->nRate ){
+      KeccakF1600Step(p);
+      p->nLoaded = 0;
+    }
+  }
+}
+
+/*
+** After all content has been added, invoke HashFinal() to compute
+** the final hash.  The function returns a pointer to the binary
+** hash value.
+*/
+static unsigned char *HashFinal(HashContext *p){
+  unsigned int i;
+  if( p->nLoaded==p->nRate-1 ){
+    const unsigned char c1 = 0x86;
+    HashUpdate(p, &c1, 1);
+  }else{
+    const unsigned char c2 = 0x06;
+    const unsigned char c3 = 0x80;
+    HashUpdate(p, &c2, 1);
+    p->nLoaded = p->nRate - 1;
+    HashUpdate(p, &c3, 1);
+  }
+  for(i=0; i<p->nRate; i++){
+    p->u.x[i+p->nRate] = p->u.x[i^p->ixMask];
+  }
+  return &p->u.x[p->nRate];
+}
+
+/*
+** Implementation of the hash(X) function.
+**
+** Return a 160-bit BLOB which is the hash of X.
+*/
+static void hashFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  HashContext cx;
+  int eType = sqlite3_value_type(argv[0]);
+  int nByte = sqlite3_value_bytes(argv[0]);
+  if( eType==SQLITE_NULL ) return;
+  HashInit(&cx, 160);
+  if( eType==SQLITE_BLOB ){
+    HashUpdate(&cx, sqlite3_value_blob(argv[0]), nByte);
+  }else{
+    HashUpdate(&cx, sqlite3_value_text(argv[0]), nByte);
+  }
+  sqlite3_result_blob(context, HashFinal(&cx), 160/8, SQLITE_TRANSIENT);
+}
+
+/* Register the hash function */
+static int hashRegister(sqlite3 *db){
+   return sqlite3_create_function(db, "hash", 1,
+                SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC,
+                0, hashFunc, 0, 0);
+}
+
+/* End of the hashing logic
+*****************************************************************************/
+
+/*
+** Return the tail of a file pathname.  The tail is the last component
+** of the path.  For example, the tail of "/a/b/c.d" is "c.d".
+*/
+const char *file_tail(const char *z){
+  const char *zTail = z;
+  if( !zTail ) return 0;
+  while( z[0] ){
+    if( z[0]=='/' ) zTail = &z[1];
+    z++;
+  }
+  return zTail;
+}
+
+/*
+** Append error message text to the error file, if an error file is
+** specified.  In any case, increment the error count.
+*/
+static void logError(SQLiteRsync *p, const char *zFormat, ...){
+  if( p->zErrFile ){
+    FILE *pErr = fopen(p->zErrFile, "a");
+    if( pErr ){
+      va_list ap;
+      va_start(ap, zFormat);
+      vfprintf(pErr, zFormat, ap);
+      va_end(ap);
+      fclose(pErr);
+    }
+  }
+  p->nErr++;
+}
+
+
+/* Read a single big-endian 32-bit unsigned integer from the input
+** stream.  Return 0 on success and 1 if there are any errors.
+*/
+static int readUint32(SQLiteRsync *p, unsigned int *pU){
+  unsigned char buf[4];
+  if( fread(buf, sizeof(buf), 1, p->pIn)==1 ){
+    *pU = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3];
+    p->nIn += 4;
+    return 0;
+  }else{
+    logError(p, "failed to read a 32-bit integer\n");
+    return 1;
+  }
+}
+
+/* Write a single big-endian 32-bit unsigned integer to the output stream.
+** Return 0 on success and 1 if there are any errors.
+*/
+static int writeUint32(SQLiteRsync *p, unsigned int x){
+  unsigned char buf[4];
+  buf[3] = x & 0xff;
+  x >>= 8;
+  buf[2] = x & 0xff;
+  x >>= 8;
+  buf[1] = x & 0xff;
+  x >>= 8;
+  buf[0] = x;
+  if( p->pLog ) fwrite(buf, sizeof(buf), 1, p->pLog);
+  if( fwrite(buf, sizeof(buf), 1, p->pOut)!=1 ){
+    logError(p, "failed to write 32-bit integer 0x%x\n", x);
+    p->nWrErr++;
+    return 1;
+  }
+  p->nOut += 4;
+  return 0;
+}
+
+/* Read a single byte from the wire.
+*/
+int readByte(SQLiteRsync *p){
+  int c = fgetc(p->pIn);
+  if( c!=EOF ) p->nIn++;
+  return c;
+}
+
+/* Write a single byte into the wire.
+*/
+void writeByte(SQLiteRsync *p, int c){
+  if( p->pLog ) fputc(c, p->pLog);
+  fputc(c, p->pOut);
+  p->nOut++;
+}
+
+/* Read a power of two encoded as a single byte.
+*/
+int readPow2(SQLiteRsync *p){
+  int x = readByte(p);
+  if( x<0 || x>=32 ){
+    logError(p, "read invalid page size %d\n", x);
+    return 0;
+  }
+  return 1<<x;
+}
+
+/* Write a power-of-two value onto the wire as a single byte.
+*/
+void writePow2(SQLiteRsync *p, int c){
+  int n;
+  if( c<0 || (c&(c-1))!=0 ){
+    logError(p, "trying to read invalid page size %d\n", c);
+  }
+  for(n=0; c>1; n++){ c /= 2; }
+  writeByte(p, n);
+}
+
+/* Read an array of bytes from the wire.
+*/
+void readBytes(SQLiteRsync *p, int nByte, void *pData){
+  if( fread(pData, 1, nByte, p->pIn)==nByte ){
+    p->nIn += nByte;
+  }else{
+    logError(p, "failed to read %d bytes\n", nByte);
+  }
+}
+
+/* Write an array of bytes onto the wire.
+*/
+void writeBytes(SQLiteRsync *p, int nByte, const void *pData){
+  if( p->pLog ) fwrite(pData, 1, nByte, p->pLog);
+  if( fwrite(pData, 1, nByte, p->pOut)==nByte ){
+    p->nOut += nByte;
+  }else{
+    logError(p, "failed to write %d bytes\n", nByte);
+    p->nWrErr++;
+  }
+}
+
+/* Report an error.
+**
+** If this happens on the remote side, we send back a *_ERROR
+** message.  On the local side, the error message goes to stderr.
+*/
+static void reportError(SQLiteRsync *p, const char *zFormat, ...){
+  va_list ap;
+  char *zMsg;
+  unsigned int nMsg;
+  va_start(ap, zFormat);
+  zMsg = sqlite3_vmprintf(zFormat, ap);
+  va_end(ap);
+  nMsg = zMsg ? (unsigned int)strlen(zMsg) : 0;
+  if( p->isRemote ){
+    if( p->isReplica ){
+      putc(REPLICA_ERROR, p->pOut);
+    }else{
+      putc(ORIGIN_ERROR, p->pOut);
+    }
+    writeUint32(p, nMsg);
+    writeBytes(p, nMsg, zMsg);
+    fflush(p->pOut);
+  }else{
+    fprintf(stderr, "%s\n", zMsg);
+  }
+  logError(p, "%s\n", zMsg);
+  sqlite3_free(zMsg);
+}
+
+/* Send an informational message.
+**
+** If this happens on the remote side, we send back a *_MSG 
+** message.  On the local side, the message goes to stdout.
+*/
+static void infoMsg(SQLiteRsync *p, const char *zFormat, ...){
+  va_list ap;
+  char *zMsg;
+  unsigned int nMsg;
+  va_start(ap, zFormat);
+  zMsg = sqlite3_vmprintf(zFormat, ap);
+  va_end(ap);
+  nMsg = zMsg ? (unsigned int)strlen(zMsg) : 0;
+  if( p->isRemote ){
+    if( p->isReplica ){
+      putc(REPLICA_MSG, p->pOut);
+    }else{
+      putc(ORIGIN_MSG, p->pOut);
+    }
+    writeUint32(p, nMsg);
+    writeBytes(p, nMsg, zMsg);
+    fflush(p->pOut);
+  }else{
+    printf("%s\n", zMsg);
+  }
+  sqlite3_free(zMsg);
+}
+
+/* Receive and report an error message coming from the other side.
+*/
+static void readAndDisplayMessage(SQLiteRsync *p, int c){
+  unsigned int n = 0;
+  char *zMsg;
+  const char *zPrefix;
+  if( c==ORIGIN_ERROR || c==REPLICA_ERROR ){
+    zPrefix = "ERROR: ";
+  }else{
+    zPrefix = "";
+  }
+  readUint32(p, &n);
+  if( n==0 ){
+    fprintf(stderr,"ERROR: unknown (possibly out-of-memory)\n");
+  }else{
+    zMsg = sqlite3_malloc64( n+1 );
+    if( zMsg==0 ){
+      fprintf(stderr, "ERROR: out-of-memory\n");
+      return;
+    }
+    memset(zMsg, 0, n+1);
+    readBytes(p, n, zMsg);
+    fprintf(stderr,"%s%s\n", zPrefix, zMsg);
+    if( zPrefix[0] ) logError(p, "%s%s\n", zPrefix, zMsg);
+    sqlite3_free(zMsg);
+  }
+}
+
+/* Construct a new prepared statement.  Report an error and return NULL
+** if anything goes wrong.
+*/
+static sqlite3_stmt *prepareStmtVA(
+  SQLiteRsync *p,
+  char *zFormat,
+  va_list ap
+){
+  sqlite3_stmt *pStmt = 0;
+  char *zSql;
+  char *zToFree = 0;
+  int rc;
+
+  if( strchr(zFormat,'%') ){
+    zSql = sqlite3_vmprintf(zFormat, ap);
+    if( zSql==0 ){
+      reportError(p, "out-of-memory");
+      return 0;
+    }else{
+      zToFree = zSql;
+    }
+  }else{
+    zSql = zFormat;
+  }
+  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+  if( rc || pStmt==0 ){
+    reportError(p, "unable to prepare SQL [%s]: %s", zSql,
+                sqlite3_errmsg(p->db));
+    sqlite3_finalize(pStmt);
+    pStmt = 0;
+  }
+  if( zToFree ) sqlite3_free(zToFree);
+  return pStmt;
+}
+static sqlite3_stmt *prepareStmt(
+  SQLiteRsync *p,
+  char *zFormat,
+  ...
+){
+  sqlite3_stmt *pStmt;
+  va_list ap;
+  va_start(ap, zFormat);
+  pStmt = prepareStmtVA(p, zFormat, ap);
+  va_end(ap);
+  return pStmt;
+}
+
+/* Run a single SQL statement
+*/
+static void runSql(SQLiteRsync *p, char *zSql, ...){
+  sqlite3_stmt *pStmt;
+  va_list ap;
+
+  va_start(ap, zSql);
+  pStmt = prepareStmtVA(p, zSql, ap);
+  va_end(ap);
+  if( pStmt ){
+    int rc = sqlite3_step(pStmt);
+    if( rc==SQLITE_ROW ) rc = sqlite3_step(pStmt);
+    if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){
+      reportError(p, "SQL statement [%s] failed: %s", zSql,
+                  sqlite3_errmsg(p->db));
+    }
+    sqlite3_finalize(pStmt);
+  }
+}
+
+/* Run an SQL statement that returns a single unsigned 32-bit integer result
+*/
+static int runSqlReturnUInt(
+  SQLiteRsync *p,
+  unsigned int *pRes,
+  char *zSql,
+  ...
+){
+  sqlite3_stmt *pStmt;
+  int res = 0;
+  va_list ap;
+
+  va_start(ap, zSql);
+  pStmt = prepareStmtVA(p, zSql, ap);
+  va_end(ap);
+  if( pStmt==0 ){
+    res = 1;
+  }else{
+    int rc = sqlite3_step(pStmt);
+    if( rc==SQLITE_ROW ){
+      *pRes = (unsigned int)(sqlite3_column_int64(pStmt, 0)&0xffffffff);
+    }else{
+      reportError(p, "SQL statement [%s] failed: %s", zSql,
+                  sqlite3_errmsg(p->db));
+      res = 1;
+    }
+    sqlite3_finalize(pStmt);
+  }
+  return res;
+}
+
+/* Run an SQL statement that returns a single TEXT value that is no more
+** than 99 bytes in length.
+*/
+static int runSqlReturnText(
+  SQLiteRsync *p,
+  char *pRes,
+  char *zSql,
+  ...
+){
+  sqlite3_stmt *pStmt;
+  int res = 0;
+  va_list ap;
+
+  va_start(ap, zSql);
+  pStmt = prepareStmtVA(p, zSql, ap);
+  va_end(ap);
+  pRes[0] = 0;
+  if( pStmt==0 ){
+    res = 1;
+  }else{
+    int rc = sqlite3_step(pStmt);
+    if( rc==SQLITE_ROW ){
+      const unsigned char *a = sqlite3_column_text(pStmt, 0);
+      int n;
+      if( a==0 ){
+        pRes[0] = 0;
+      }else{
+        n = sqlite3_column_bytes(pStmt, 0);
+        if( n>99 ) n = 99;
+        memcpy(pRes, a, n);
+        pRes[n] = 0;
+      }
+    }else{
+      reportError(p, "SQL statement [%s] failed: %s", zSql,
+                  sqlite3_errmsg(p->db));
+      res = 1;
+    }
+    sqlite3_finalize(pStmt);
+  }
+  return res;
+}
+
+/* Close the database connection associated with p
+*/
+static void closeDb(SQLiteRsync *p){
+  if( p->db ){
+    sqlite3_close(p->db);
+    p->db = 0;
+  }
+}
+
+/*
+** Run the origin-side protocol.
+**
+** Begin by sending the ORIGIN_BEGIN message with two arguments,
+** nPage, and szPage.  Then enter a loop responding to message from
+** the replica:
+**
+**    REPLICA_ERROR  size  text
+**
+**         Report an error from the replica and quit
+**
+**    REPLICA_END
+**
+**         The replica is terminating.  Stop processing now.
+**
+**    REPLICA_HASH  hash
+**
+**         The argument is the 20-byte SHA1 hash for the next page
+**         page hashes appear in sequential order with no gaps.
+**
+**    REPLICA_READY
+**
+**         The replica has sent all the hashes that it intends to send.
+**         This side (the origin) can now start responding with page
+**         content for pages that do not have a matching hash.
+*/
+static void originSide(SQLiteRsync *p){
+  int rc = 0;
+  int c = 0;
+  unsigned int nPage = 0;
+  unsigned int iPage = 0;
+  unsigned int szPg = 0;
+  sqlite3_stmt *pCkHash = 0;
+  char buf[200];
+
+  p->isReplica = 0;
+  if( p->bCommCheck ){
+    infoMsg(p, "origin  zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d",
+               p->zOrigin, p->zReplica, p->isRemote, PROTOCOL_VERSION);
+    writeByte(p, ORIGIN_END);
+    fflush(p->pOut);
+  }else{
+    /* Open the ORIGIN database. */
+    rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READWRITE, 0);
+    if( rc ){
+      reportError(p, "cannot open origin \"%s\": %s",
+                  p->zOrigin, sqlite3_errmsg(p->db));
+      closeDb(p);
+      return;
+    }
+    hashRegister(p->db);
+    runSql(p, "BEGIN");
+    runSqlReturnText(p, buf, "PRAGMA journal_mode");
+    if( sqlite3_stricmp(buf,"wal")!=0 ){
+      reportError(p, "Origin database is not in WAL mode");
+    }
+    runSqlReturnUInt(p, &nPage, "PRAGMA page_count");
+    runSqlReturnUInt(p, &szPg, "PRAGMA page_size");
+  
+    if( p->nErr==0 ){
+      /* Send the ORIGIN_BEGIN message */
+      writeByte(p, ORIGIN_BEGIN);
+      writeByte(p, PROTOCOL_VERSION);
+      writePow2(p, szPg);
+      writeUint32(p, nPage);
+      fflush(p->pOut);
+      p->nPage = nPage;
+      p->szPage = szPg;
+      p->iProtocol = PROTOCOL_VERSION;
+    }
+  }
+  
+  /* Respond to message from the replica */
+  while( p->nErr<=p->nWrErr && (c = readByte(p))!=EOF && c!=REPLICA_END ){
+    switch( c ){
+      case REPLICA_BEGIN: {
+        /* This message is only sent if the replica received an origin-protocol
+        ** that is larger than what it knows about.  The replica sends back
+        ** a counter-proposal of an earlier protocol which the origin can
+        ** accept by resending a new ORIGIN_BEGIN. */
+        p->iProtocol = readByte(p);
+        writeByte(p, ORIGIN_BEGIN);
+        writeByte(p, p->iProtocol);
+        writePow2(p, p->szPage);
+        writeUint32(p, p->nPage);
+        break;
+      }
+      case REPLICA_MSG:
+      case REPLICA_ERROR: {
+        readAndDisplayMessage(p, c);
+        break;
+      }
+      case REPLICA_HASH: {
+        if( pCkHash==0 ){
+          runSql(p, "CREATE TEMP TABLE badHash(pgno INTEGER PRIMARY KEY)");
+          pCkHash = prepareStmt(p,
+            "INSERT INTO badHash SELECT pgno FROM sqlite_dbpage('main')"
+            " WHERE pgno=?1 AND hash(data)!=?2"
+          );
+          if( pCkHash==0 ) break;
+        }
+        p->nHashSent++;
+        iPage++;
+        sqlite3_bind_int64(pCkHash, 1, iPage);
+        readBytes(p, 20, buf);
+        sqlite3_bind_blob(pCkHash, 2, buf, 20, SQLITE_STATIC);
+        rc = sqlite3_step(pCkHash);
+        if( rc!=SQLITE_DONE ){
+          reportError(p, "SQL statement [%s] failed: %s",
+                      sqlite3_sql(pCkHash), sqlite3_errmsg(p->db));
+        }
+        sqlite3_reset(pCkHash);
+        break;
+      }
+      case REPLICA_READY: {
+        sqlite3_stmt *pStmt;
+        sqlite3_finalize(pCkHash);
+        pCkHash = 0;
+        if( iPage+1<p->nPage ){
+          runSql(p, "WITH RECURSIVE c(n) AS"
+                    " (VALUES(%d) UNION ALL SELECT n+1 FROM c WHERE n<%d)"
+                    " INSERT INTO badHash SELECT n FROM c",
+                    iPage+1, p->nPage);
+        }
+        pStmt = prepareStmt(p,
+               "SELECT pgno, data"
+               "  FROM badHash JOIN sqlite_dbpage('main') USING(pgno)");
+        if( pStmt==0 ) break;
+        while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 && p->nWrErr==0 ){
+          unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0);
+          const void *pContent = sqlite3_column_blob(pStmt, 1);
+          writeByte(p, ORIGIN_PAGE);
+          writeUint32(p, pgno);
+          writeBytes(p, szPg, pContent);
+          p->nPageSent++;
+        }
+        sqlite3_finalize(pStmt);
+        writeByte(p, ORIGIN_TXN);
+        writeUint32(p, nPage);
+        writeByte(p, ORIGIN_END);
+        fflush(p->pOut);
+        break;
+      }
+      default: {
+        reportError(p, "Unknown message 0x%02x %lld bytes into conversation",
+                       c, p->nIn);
+        break;
+      }
+    }
+  }
+
+  if( pCkHash ) sqlite3_finalize(pCkHash);
+  closeDb(p);
+}
+
+/*
+** Run the replica-side protocol.  The protocol is passive in the sense
+** that it only response to message from the origin side.
+**
+**    ORIGIN_BEGIN  idProtocol szPage nPage
+**
+**         The origin is reporting the protocol version number, the size of
+**         each page in the origin database (sent as a single-byte power-of-2),
+**         and the number of pages in the origin database.
+**         This procedure checks compatibility, and if everything is ok,
+**         it starts sending hashes of pages already present back to the origin.
+**
+**    ORIGIN_ERROR  size text
+**
+**         Report the received error and quit.
+**
+**    ORIGIN_PAGE  pgno content
+**
+**         Update the content of the given page.
+**
+**    ORIGIN_TXN   pgno
+**
+**         Close the update transaction.  The total database size is pgno
+**         pages.
+**
+**    ORIGIN_END
+**
+**         Expect no more transmissions from the origin.
+*/
+static void replicaSide(SQLiteRsync *p){
+  int c;
+  sqlite3_stmt *pIns = 0;
+  unsigned int szOPage = 0;
+  char buf[65536];
+
+  p->isReplica = 1;
+  if( p->bCommCheck ){
+    infoMsg(p, "replica zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d",
+               p->zOrigin, p->zReplica, p->isRemote, PROTOCOL_VERSION);
+    writeByte(p, REPLICA_END);
+    fflush(p->pOut);
+  }
+
+  /* Respond to message from the origin.  The origin will initiate the
+  ** the conversation with an ORIGIN_BEGIN message.
+  */
+  while( p->nErr<=p->nWrErr && (c = readByte(p))!=EOF && c!=ORIGIN_END ){
+    switch( c ){
+      case ORIGIN_MSG:
+      case ORIGIN_ERROR: {
+        readAndDisplayMessage(p, c);
+        break;
+      }
+      case ORIGIN_BEGIN: {
+        unsigned int nOPage = 0;
+        unsigned int nRPage = 0, szRPage = 0;
+        int rc = 0;
+        sqlite3_stmt *pStmt = 0;
+
+        closeDb(p);
+        p->iProtocol = readByte(p);
+        szOPage = readPow2(p);
+        readUint32(p, &nOPage);
+        if( p->nErr ) break;
+        if( p->iProtocol>PROTOCOL_VERSION ){
+          /* If the protocol version on the origin side is larger, send back
+          ** a REPLICA_BEGIN message with the protocol version number of the
+          ** replica side.  This gives the origin an opportunity to resend
+          ** a new ORIGIN_BEGIN with a reduced protocol version. */
+          writeByte(p, REPLICA_BEGIN);
+          writeByte(p, PROTOCOL_VERSION);
+          break;
+        }
+        p->nPage = nOPage;
+        p->szPage = szOPage;
+        rc = sqlite3_open(":memory:", &p->db);
+        if( rc ){
+          reportError(p, "cannot open in-memory database: %s",
+                      sqlite3_errmsg(p->db));
+          closeDb(p);
+          break;
+        }
+        runSql(p, "ATTACH %Q AS 'replica'", p->zReplica);
+        if( p->nErr ){
+          closeDb(p);
+          break;
+        }
+        hashRegister(p->db);
+        if( runSqlReturnUInt(p, &nRPage, "PRAGMA replica.page_count") ){
+          break;
+        }
+        if( nRPage==0 ){
+          runSql(p, "PRAGMA replica.page_size=%u", szOPage);
+          runSql(p, "PRAGMA replica.journal_mode=WAL");
+          runSql(p, "SELECT * FROM replica.sqlite_schema");
+        }
+        runSql(p, "BEGIN IMMEDIATE");
+        runSqlReturnText(p, buf, "PRAGMA replica.journal_mode");
+        if( strcmp(buf, "wal")!=0 ){
+          reportError(p, "replica is not in WAL mode");
+          break;
+        }
+        runSqlReturnUInt(p, &nRPage, "PRAGMA replica.page_count");
+        runSqlReturnUInt(p, &szRPage, "PRAGMA replica.page_size");
+        if( szRPage!=szOPage ){
+          reportError(p, "page size mismatch; origin is %d bytes and "
+                         "replica is %d bytes", szOPage, szRPage);
+          break;
+        }
+
+        pStmt = prepareStmt(p,
+                   "SELECT hash(data) FROM sqlite_dbpage('replica')"
+                   " WHERE pgno<=min(%d,%d)"
+                   " ORDER BY pgno", nRPage, nOPage);
+        while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 && p->nWrErr==0 ){
+          const unsigned char *a = sqlite3_column_blob(pStmt, 0);
+          writeByte(p, REPLICA_HASH);
+          writeBytes(p, 20, a);
+          p->nHashSent++;
+        }
+        sqlite3_finalize(pStmt);
+        writeByte(p, REPLICA_READY);
+        fflush(p->pOut);
+        runSql(p, "PRAGMA writable_schema=ON");
+        break;
+      }
+      case ORIGIN_TXN: {
+        unsigned int nOPage = 0;
+        readUint32(p, &nOPage);
+        if( pIns==0 ){
+          /* Nothing has changed */
+          runSql(p, "COMMIT");
+        }else if( p->nErr ){
+          runSql(p, "ROLLBACK");
+        }else{
+          int rc;
+          sqlite3_bind_int64(pIns, 1, nOPage);
+          sqlite3_bind_null(pIns, 2);
+          rc = sqlite3_step(pIns);
+          if( rc!=SQLITE_DONE ){
+            reportError(p, "SQL statement [%s] failed (pgno=%u, data=NULL): %s",
+                   sqlite3_sql(pIns), nOPage, sqlite3_errmsg(p->db));
+          }
+          sqlite3_reset(pIns);
+          p->nPage = nOPage;
+          runSql(p, "COMMIT");
+        }
+        break;
+      }
+      case ORIGIN_PAGE: {
+        unsigned int pgno = 0;
+        int rc;
+        readUint32(p, &pgno);
+        if( p->nErr ) break;
+        if( pIns==0 ){
+          pIns = prepareStmt(p,
+            "INSERT INTO sqlite_dbpage(pgno,data,schema)VALUES(?1,?2,'replica')"
+          );
+          if( pIns==0 ) break;
+        }
+        readBytes(p, szOPage, buf);
+        if( p->nErr ) break;
+        p->nPageSent++;
+        sqlite3_bind_int64(pIns, 1, pgno);
+        sqlite3_bind_blob(pIns, 2, buf, szOPage, SQLITE_STATIC);
+        rc = sqlite3_step(pIns);
+        if( rc!=SQLITE_DONE ){
+          reportError(p, "SQL statement [%s] failed (pgno=%u): %s",
+                 sqlite3_sql(pIns), pgno, sqlite3_errmsg(p->db));
+        }
+        sqlite3_reset(pIns);
+        break;
+      }
+      default: {
+        reportError(p, "Unknown message 0x%02x %lld bytes into conversation",
+                       c, p->nIn);
+        break;
+      }
+    }
+  }
+
+  if( pIns ) sqlite3_finalize(pIns);
+  closeDb(p);
+}
+
+/*
+** The argument might be -vvv...vv with any number of "v"s.  Return
+** the number of "v"s.  Return 0 if the argument is not a -vvv...v.
+*/
+static int numVs(const char *z){
+  int n = 0;
+  if( z[0]!='-' ) return 0;
+  z++;
+  if( z[0]=='-' ) z++;
+  while( z[0]=='v' ){ n++; z++; }
+  if( z[0]==0 ) return n;
+  return 0;
+}
+
+/*
+** Get the argument to an --option.  Throw an error and die if no argument
+** is available.
+*/
+static const char *cmdline_option_value(int argc, const char * const*argv,
+                                        int i){
+  if( i==argc ){
+    fprintf(stderr,"%s: Error: missing argument to %s\n",
+            argv[0], argv[argc-1]);
+    exit(1);
+  }
+  return argv[i];
+}
+
+/*
+** Return the current time in milliseconds since the Julian epoch.
+*/
+sqlite3_int64 currentTime(void){
+  sqlite3_int64 now = 0;
+  sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
+  if( pVfs && pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64!=0 ){
+    pVfs->xCurrentTimeInt64(pVfs, &now);
+  }
+  return now;
+}
+
+/*
+** Input string zIn might be in any of these formats:
+**
+**    (1) PATH
+**    (2) HOST:PATH
+**    (3) USER@HOST:PATH
+**
+** For format 1, return NULL.  For formats 2 and 3, return
+** a pointer to the ':' character that separates the hostname
+** from the path.
+*/
+static char *hostSeparator(const char *zIn){
+  char *zPath = strchr(zIn, ':');
+  if( zPath==0 ) return 0;
+#ifdef _WIN32
+  if( isalpha(zIn[0]) && zIn[1]==':' && (zIn[2]=='/' || zIn[2]=='\\') ){
+    return 0;
+  }
+#endif
+  while( zIn<zPath ){
+    if( zIn[0]=='/' ) return 0;
+    if( zIn[0]=='\\' ) return 0;
+    zIn++;
+  }
+  return zPath;
+
+}
+
+/*
+** Parse command-line arguments.  Dispatch subroutines to do the
+** requested work.
+**
+** Input formats:
+**
+**  (1)    sqlite3-rsync  FILENAME1  USER@HOST:FILENAME2
+**
+**  (2)    sqlite3-rsync  USER@HOST:FILENAME1  FILENAME2
+**
+**  (3)    sqlite3-rsync --origin FILENAME1
+**
+**  (4)    sqlite3-rsync --replica FILENAME2
+**
+** The user types (1) or (2).  SSH launches (3) or (4).
+**
+** If (1) is seen then popen2 is used launch (4) on the remote and
+** originSide() is called locally.
+**
+** If (2) is seen, then popen2() is used to launch (3) on the remote
+** and replicaSide() is run locally.
+**
+** If (3) is seen, call originSide() on stdin and stdout.
+**
+q** If (4) is seen, call replicaSide() on stdin and stdout.
+*/
+int main(int argc, char const * const *argv){
+  int isOrigin = 0;
+  int isReplica = 0;
+  int i;
+  SQLiteRsync ctx;
+  char *zDiv;
+  FILE *pIn = 0;
+  FILE *pOut = 0;
+  int childPid = 0;
+  const char *zSsh = "ssh";
+  const char *zExe = "sqlite3-rsync";
+  char *zCmd = 0;
+  sqlite3_int64 tmStart;
+  sqlite3_int64 tmEnd;
+  sqlite3_int64 tmElapse;
+  const char *zRemoteErrFile = 0;
+
+#define cli_opt_val cmdline_option_value(argc, argv, ++i)
+  memset(&ctx, 0, sizeof(ctx));
+  for(i=1; i<argc; i++){
+    const char *z = argv[i];
+    if( strcmp(z,"--origin")==0 ){
+      isOrigin = 1;
+      continue;
+    }
+    if( strcmp(z,"--replica")==0 ){
+      isReplica = 1;
+      continue;
+    }
+    if( numVs(z) ){
+      ctx.eVerbose += numVs(z);
+      continue;
+    }
+    if( strcmp(z, "--ssh")==0 ){
+      zSsh = cli_opt_val;
+      continue;
+    }
+    if( strcmp(z, "--exe")==0 ){
+      zExe = cli_opt_val;
+      continue;
+    }
+    if( strcmp(z, "--logfile")==0 ){
+      /* DEBUG OPTION:  --logfile FILENAME
+      ** Cause all local output traffic to be duplicated in FILENAME */
+      const char *zLog = cli_opt_val;
+      if( ctx.pLog ) fclose(ctx.pLog);
+      ctx.pLog = fopen(zLog, "wb");
+      if( ctx.pLog==0 ){
+        fprintf(stderr, "cannot open \"%s\" for writing\n", argv[i]);
+        return 1;
+      }
+      continue;
+    }
+    if( strcmp(z, "--errorfile")==0 ){
+      /* DEBUG OPTION:  --errorfile FILENAME
+      ** Error messages on the local side are written into FILENAME */
+      ctx.zErrFile = cli_opt_val;
+      continue;
+    }
+    if( strcmp(z, "--remote-errorfile")==0 ){
+      /* DEBUG OPTION:  --remote-errorfile FILENAME
+      ** Error messages on the remote side are written into FILENAME on
+      ** the remote side. */
+      zRemoteErrFile = cli_opt_val;
+      continue;
+    }
+    if( strcmp(z, "-help")==0 || strcmp(z, "--help")==0
+     || strcmp(z, "-?")==0
+    ){
+      printf("%s", zUsage);
+      return 0;
+    }
+    if( strcmp(z, "--version")==0 ){
+      printf("%s\n", sqlite3_sourceid());
+      return 0;
+    }
+    if( z[0]=='-' ){
+      if( strcmp(z,"--commcheck")==0 ){  /* DEBUG ONLY */
+        /* Run a communication check with the remote side.  Do not attempt
+        ** to exchange any database connection */
+        ctx.bCommCheck = 1;
+        continue;
+      }
+      if( strcmp(z,"--arg-escape-check")==0 ){  /* DEBUG ONLY */
+        /* Test the append_escaped_arg() routine by using it to render a
+        ** copy of the input command-line, assuming all arguments except
+        ** this one are filenames. */
+        sqlite3_str *pStr = sqlite3_str_new(0);
+        int k;
+        for(k=0; k<argc; k++){
+          append_escaped_arg(pStr, argv[k], i!=k);
+        }
+        printf("%s\n", sqlite3_str_value(pStr));
+        return 0;
+      }
+      fprintf(stderr,
+         "unknown option: \"%s\". Use --help for more detail.\n", z);
+      return 1;
+    }
+    if( ctx.zOrigin==0 ){
+      ctx.zOrigin = z;
+    }else if( ctx.zReplica==0 ){
+      ctx.zReplica = z;
+    }else{
+      fprintf(stderr, "Unknown argument: \"%s\"\n", z);
+      return 1;
+    }
+  }
+  if( ctx.zOrigin==0 ){
+    fprintf(stderr, "missing ORIGIN database filename\n");
+    return 1;
+  }
+  if( ctx.zReplica==0 ){
+    fprintf(stderr, "missing REPLICA database filename\n");
+    return 1;
+  }
+  if( isOrigin && isReplica ){
+    fprintf(stderr, "bad option combination\n");
+    return 1;
+  }
+  if( isOrigin ){
+    ctx.pIn = stdin;
+    ctx.pOut = stdout;
+    ctx.isRemote = 1;
+    originSide(&ctx);
+    return 0;
+  }
+  if( isReplica ){
+    ctx.pIn = stdin;
+    ctx.pOut = stdout;
+    ctx.isRemote = 1;
+    replicaSide(&ctx);
+    return 0;
+  }
+  if( ctx.zReplica==0 ){
+    fprintf(stderr, "missing REPLICA database filename\n");
+    return 1;
+  }
+  tmStart = currentTime();
+  zDiv = hostSeparator(ctx.zOrigin);
+  if( zDiv ){
+    if( hostSeparator(ctx.zReplica)!=0 ){
+      fprintf(stderr,
+         "At least one of ORIGIN and REPLICA must be a local database\n"
+         "You provided two remote databases.\n");
+      return 1;
+    }
+    /* Remote ORIGIN and local REPLICA */
+    sqlite3_str *pStr = sqlite3_str_new(0);
+    append_escaped_arg(pStr, zSsh, 1);
+    sqlite3_str_appendf(pStr, " -e none");
+    *(zDiv++) = 0;
+    append_escaped_arg(pStr, ctx.zOrigin, 0);
+    append_escaped_arg(pStr, zExe, 1);
+    append_escaped_arg(pStr, "--origin", 0);
+    if( ctx.bCommCheck ){
+      append_escaped_arg(pStr, "--commcheck", 0);
+      if( ctx.eVerbose==0 ) ctx.eVerbose = 1;
+    }
+    if( zRemoteErrFile ){
+      append_escaped_arg(pStr, "--errorfile", 0);
+      append_escaped_arg(pStr, zRemoteErrFile, 1);
+    }
+    append_escaped_arg(pStr, zDiv, 1);
+    append_escaped_arg(pStr, file_tail(ctx.zReplica), 1);
+    zCmd = sqlite3_str_finish(pStr);
+    if( ctx.eVerbose>=2 ) printf("%s\n", zCmd);
+    if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){
+      fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd);
+      return 1;
+    }
+    replicaSide(&ctx);
+  }else if( (zDiv = hostSeparator(ctx.zReplica))!=0 ){
+    /* Local ORIGIN and remote REPLICA */
+    sqlite3_str *pStr = sqlite3_str_new(0);
+    append_escaped_arg(pStr, zSsh, 1);
+    sqlite3_str_appendf(pStr, " -e none");
+    *(zDiv++) = 0;
+    append_escaped_arg(pStr, ctx.zReplica, 0);
+    append_escaped_arg(pStr, zExe, 1);
+    append_escaped_arg(pStr, "--replica", 0);
+    if( ctx.bCommCheck ){
+      append_escaped_arg(pStr, "--commcheck", 0);
+      if( ctx.eVerbose==0 ) ctx.eVerbose = 1;
+    }
+    if( zRemoteErrFile ){
+      append_escaped_arg(pStr, "--errorfile", 0);
+      append_escaped_arg(pStr, zRemoteErrFile, 1);
+    }
+    append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1);
+    append_escaped_arg(pStr, zDiv, 1);
+    zCmd = sqlite3_str_finish(pStr);
+    if( ctx.eVerbose>=2 ) printf("%s\n", zCmd);
+    if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){
+      fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd);
+      return 1;
+    }
+    originSide(&ctx);
+  }else{
+    /* Local ORIGIN and REPLICA */
+    sqlite3_str *pStr = sqlite3_str_new(0);
+    append_escaped_arg(pStr, argv[0], 1);
+    append_escaped_arg(pStr, "--replica", 0);
+    if( ctx.bCommCheck ){
+      append_escaped_arg(pStr, "--commcheck", 0);
+    }
+    if( zRemoteErrFile ){
+      append_escaped_arg(pStr, "--errorfile", 0);
+      append_escaped_arg(pStr, zRemoteErrFile, 1);
+    }
+    append_escaped_arg(pStr, ctx.zOrigin, 1);
+    append_escaped_arg(pStr, ctx.zReplica, 1);
+    zCmd = sqlite3_str_finish(pStr);
+    if( ctx.eVerbose>=2 ) printf("%s\n", zCmd);
+    if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){
+      fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd);
+      return 1;
+    }
+    originSide(&ctx);
+  }
+  pclose2(ctx.pIn, ctx.pOut, childPid);
+  if( ctx.pLog ) fclose(ctx.pLog);
+  tmEnd = currentTime();
+  tmElapse = tmEnd - tmStart;  /* Elapse time in milliseconds */
+  if( ctx.nErr ){
+    printf("Databases were not synced due to errors\n");
+  }
+  if( ctx.eVerbose>=1 ){
+    char *zMsg;
+    sqlite3_int64 szTotal = (sqlite3_int64)ctx.nPage*(sqlite3_int64)ctx.szPage;
+    sqlite3_int64 nIO = ctx.nOut +ctx.nIn;
+    zMsg = sqlite3_mprintf("sent %,lld bytes, received %,lld bytes",
+                           ctx.nOut, ctx.nIn);
+    printf("%s", zMsg);
+    sqlite3_free(zMsg);
+    if( tmElapse>0 ){
+      zMsg = sqlite3_mprintf(", %,.2f bytes/sec",
+                             1000.0*(double)nIO/(double)tmElapse);
+      printf("%s\n", zMsg);
+      sqlite3_free(zMsg);
+    }else{
+      printf("\n");
+    }
+    if( ctx.nErr==0 ){
+      if( nIO<=szTotal && nIO>0 ){
+        zMsg = sqlite3_mprintf("total size %,lld  speedup is %.2f",
+           szTotal, (double)szTotal/(double)nIO);
+      }else{
+        zMsg = sqlite3_mprintf("total size %,lld", szTotal);
+      }
+      printf("%s\n", zMsg);
+      sqlite3_free(zMsg);
+    }
+  }
+  sqlite3_free(zCmd);
+  if( pIn!=0 && pOut!=0 ){
+    pclose2(pIn, pOut, childPid);
+  }
+  return ctx.nErr;
+}
diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in
index 2d799ed250..1c9fc836a1 100644
--- a/tool/sqlite3_analyzer.c.in
+++ b/tool/sqlite3_analyzer.c.in
@@ -20,8 +20,8 @@ INCLUDE sqlite3.c
 INCLUDE $ROOT/src/tclsqlite.c
 
 #if defined(_WIN32)
-INCLUDE $ROOT/ext/consio/console_io.h
-INCLUDE $ROOT/ext/consio/console_io.c
+INCLUDE $ROOT/ext/misc/sqlite3_stdio.h
+INCLUDE $ROOT/ext/misc/sqlite3_stdio.c
 
 /* Substitute "puts" command.  Only these forms recognized:
 **
@@ -56,8 +56,8 @@ static int subst_puts(
       return TCL_ERROR;
     }
   }
-  fPutsUtf8(zOut, pOut);
-  if( addNewLine ) fPutsUtf8("\n", pOut);
+  sqlite3_fputs(zOut, pOut);
+  if( addNewLine ) sqlite3_fputs("\n", pOut);
   return TCL_OK;
 }
 #endif /* defined(_WIN32) */