From a43c8c8a6065b3eed4b1a7ec03a7535f3b3b537f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Oct 2017 13:48:11 +0000 Subject: [PATCH 01/22] Initial implementation of the "sqlite_dbpage" virtual table. Currently it is read-only and has a place-holder xBestIndex. FossilOrigin-Name: c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49 --- Makefile.in | 8 +- Makefile.msc | 9 +- main.mk | 5 +- manifest | 26 +++-- manifest.uuid | 2 +- src/dbpage.c | 241 ++++++++++++++++++++++++++++++++++++++++++++ src/main.c | 6 ++ src/sqliteInt.h | 3 + tool/mksqlite3c.tcl | 1 + 9 files changed, 285 insertions(+), 16 deletions(-) create mode 100644 src/dbpage.c diff --git a/Makefile.in b/Makefile.in index 58c2c2919b..7ea97e6195 100644 --- a/Makefile.in +++ b/Makefile.in @@ -166,7 +166,8 @@ USE_AMALGAMATION = @USE_AMALGAMATION@ # LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ - callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \ + callback.lo complete.lo ctime.lo \ + date.lo dbpage.lo dbstat.lo delete.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ @@ -215,6 +216,7 @@ SRC = \ $(TOP)/src/complete.c \ $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ @@ -449,6 +451,7 @@ TESTSRC2 = \ $(TOP)/src/build.c \ $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ $(TOP)/src/expr.c \ $(TOP)/src/func.c \ @@ -752,6 +755,9 @@ ctime.lo: $(TOP)/src/ctime.c $(HDR) date.lo: $(TOP)/src/date.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c +dbpage.lo: $(TOP)/src/dbpage.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbpage.c + dbstat.lo: $(TOP)/src/dbstat.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c diff --git a/Makefile.msc b/Makefile.msc index 10585e4a58..d59997388d 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1091,7 +1091,8 @@ LTLIBS = $(LTLIBS) $(LIBICU) # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ - callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \ + callback.lo complete.lo ctime.lo \ + date.lo dbpage.lo dbstat.lo delete.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ @@ -1154,6 +1155,7 @@ SRC00 = \ $(TOP)\src\complete.c \ $(TOP)\src\ctime.c \ $(TOP)\src\date.c \ + $(TOP)\src\dbpage.c \ $(TOP)\src\dbstat.c \ $(TOP)\src\delete.c \ $(TOP)\src\expr.c \ @@ -1747,7 +1749,10 @@ ctime.lo: $(TOP)\src\ctime.c $(HDR) date.lo: $(TOP)\src\date.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\date.c -dbstat.lo: $(TOP)\src\date.c $(HDR) +dbpage.lo: $(TOP)\src\dbpage.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbpage.c + +dbstat.lo: $(TOP)\src\dbstat.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbstat.c delete.lo: $(TOP)\src\delete.c $(HDR) diff --git a/main.mk b/main.mk index 7da6db15f1..3f83197d4d 100644 --- a/main.mk +++ b/main.mk @@ -55,7 +55,8 @@ THREADLIB += $(LIBS) LIBOBJ+= vdbe.o parse.o \ alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ - callback.o complete.o ctime.o date.o dbstat.o delete.o expr.o \ + callback.o complete.o ctime.o \ + date.o dbpage.o dbstat.o delete.o expr.o \ fault.o fkey.o \ fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ @@ -96,6 +97,7 @@ SRC = \ $(TOP)/src/complete.c \ $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ @@ -359,6 +361,7 @@ TESTSRC2 = \ $(TOP)/src/btree.c \ $(TOP)/src/build.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ $(TOP)/src/expr.c \ $(TOP)/src/func.c \ diff --git a/manifest b/manifest index b8e6ca811e..dbf72466e4 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C In\sthe\sspeed-check.sh\stest\sscript,\sallow\san\sadditional\stest-name\sargument\nwhich\sbecomes\sthe\scomparison\sbaseline,\sin\splace\sof\s"trunk". -D 2017-10-11T12:20:36.082 -F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff +C Initial\simplementation\sof\sthe\s"sqlite_dbpage"\svirtual\stable.\s\sCurrently\nit\sis\sread-only\sand\shas\sa\splace-holder\sxBestIndex. +D 2017-10-11T13:48:11.877 +F Makefile.in f7cba589198b8663d8a43f25ad001cf44fdac4fcd6216325f05775924a7af2f9 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f +F Makefile.msc 1224c8773fc7f89c57e83c7b452291d239aa2cff4af50a204c84129e295cc37d F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -382,7 +382,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk d0145f02deb67d65c4822225847cba112c237cdb62f4905eeb4b648e82bfc222 +F main.mk 656d9f31fdc01290094460f00255a45fdf4bedd08839467857919fea628582cf F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -409,6 +409,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 +F src/dbpage.c f274a9b7bb680cc2952ee78883d67e133be0ef6065317813586a5f723af35ad5 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 F src/expr.c 4d2d0aafd945424f638ee03e11330f03288ccf616e025498f3c8602d01609a0a @@ -423,7 +424,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 1f33ef4ca0553b60fff03aa171370f8709a3e945acfcc68ccafc92752d872f40 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2 -F src/main.c a4bdadaaa827e7380cba4de878ed7947dab5aeb84f617118ba6a0422cd745b4b +F src/main.c 54637b9e7f91de6d281e577cd1a997762a4613f51a0509790027ca9865185d7c F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -464,7 +465,7 @@ F src/shell.c.in e03f7d473e10b65c25836a058a3e7a1665ffb1fe712949dcd6e38c790e4eafd F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 -F src/sqliteInt.h c07bc88eca1f59ce73e1f486187d0df4effe67c4579e112dfdd91c159e5c0569 +F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1597,7 +1598,7 @@ F tool/mkshellc.tcl 950c36f45941c12140e346a907fb66198bc2770ff7a17c749201e78d34bb F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb -F tool/mksqlite3c.tcl b258d679829a9305f5cf107b7d97b9bf23adb3773df42947fed5ef7b180dfbd9 +F tool/mksqlite3c.tcl 1fb69d39166f52d802a70ec37d99bca51d011c8ab30be27bc495be493196ae41 F tool/mksqlite3h.tcl f92f994d9709aeb9e2b6e6f9fc8b069d2f55202c8e23f453edc44390a25982dc F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 @@ -1656,7 +1657,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32 -R 9a03aa36b167c86705816bfaa31cdc19 +P 0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49 +R 9d91f38d6de2e0aabd371a2c4dbcd832 +T *branch * dbpage +T *sym-dbpage * +T -sym-trunk * U drh -Z 8f5e2f1e221681e7d4bf20eb32b86754 +Z 95e75b385006fb35e51e104eb16c88ba diff --git a/manifest.uuid b/manifest.uuid index 7305d2515a..b0152fffa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49 \ No newline at end of file +c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49 \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c new file mode 100644 index 0000000000..89dcf04227 --- /dev/null +++ b/src/dbpage.c @@ -0,0 +1,241 @@ +/* +** 2017-10-11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the "sqlite_dbpage" virtual table. +** +** The sqlite_dbpage virtual table is used to read or write whole raw +** pages of the database file. The pager interface is used so that +** uncommitted changes and changes recorded in the WAL file are correctly +** retrieved. +*/ + +#include "sqliteInt.h" /* Requires access to internal data structures */ +#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ + && !defined(SQLITE_OMIT_VIRTUALTABLE) + +typedef struct DbpageTable DbpageTable; +typedef struct DbpageCursor DbpageCursor; + +struct DbpageCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + int pgno; /* Current page number */ +}; + +struct DbpageTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database */ + Pager *pPager; /* Pager being read/written */ + int iDb; /* Index of database to analyze */ + int szPage; /* Size of each page in bytes */ + int nPage; /* Number of pages in the file */ +}; + +/* +** Connect to or create a dbpagevfs virtual table. +*/ +static int dbpageConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + DbpageTable *pTab = 0; + int rc = SQLITE_OK; + int iDb; + + if( argc>=4 ){ + Token nm; + sqlite3TokenInit(&nm, (char*)argv[3]); + iDb = sqlite3FindDb(db, &nm); + if( iDb<0 ){ + *pzErr = sqlite3_mprintf("no such schema: %s", argv[3]); + return SQLITE_ERROR; + } + }else{ + iDb = 0; + } + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB)"); + if( rc==SQLITE_OK ){ + pTab = (DbpageTable *)sqlite3_malloc64(sizeof(DbpageTable)); + if( pTab==0 ) rc = SQLITE_NOMEM_BKPT; + } + + assert( rc==SQLITE_OK || pTab==0 ); + if( rc==SQLITE_OK ){ + Btree *pBt = db->aDb[iDb].pBt; + memset(pTab, 0, sizeof(DbpageTable)); + pTab->db = db; + pTab->iDb = iDb; + pTab->pPager = pBt ? sqlite3BtreePager(pBt) : 0; + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a dbpagevfs virtual table. +*/ +static int dbpageDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** idxNum: +** +** 0 full table scan +** 1 pgno=?1 +*/ +static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + pIdxInfo->estimatedCost = 1.0e6; /* Initial cost estimate */ + return SQLITE_OK; +} + +/* +** Open a new dbpagevfs cursor. +*/ +static int dbpageOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + DbpageCursor *pCsr; + + pCsr = (DbpageCursor *)sqlite3_malloc64(sizeof(DbpageCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + memset(pCsr, 0, sizeof(DbpageCursor)); + pCsr->base.pVtab = pVTab; + pCsr->pgno = -1; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a dbpagevfs cursor. +*/ +static int dbpageClose(sqlite3_vtab_cursor *pCursor){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Move a dbpagevfs cursor to the next entry in the file. +*/ +static int dbpageNext(sqlite3_vtab_cursor *pCursor){ + int rc = SQLITE_OK; + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + pCsr->pgno++; + return rc; +} + +static int dbpageEof(sqlite3_vtab_cursor *pCursor){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; + return pCsr->pgno >= pTab->nPage; +} + +static int dbpageFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; + int rc = SQLITE_OK; + Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; + + if( idxNum==1 ){ + pCsr->pgno = sqlite3_value_int(argv[0]); + }else{ + pCsr->pgno = 0; + } + pTab->szPage = sqlite3BtreeGetPageSize(pBt); + pTab->nPage = sqlite3BtreeLastPage(pBt); + return rc; +} + +static int dbpageColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; + int rc = SQLITE_OK; + switch( i ){ + case 0: { /* pgno */ + sqlite3_result_int(ctx, pCsr->pgno); + break; + } + case 1: { /* data */ + DbPage *pDbPage = 0; + rc = sqlite3PagerGet(pTab->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pTab->szPage, + SQLITE_TRANSIENT); + } + sqlite3PagerUnref(pDbPage); + break; + } + default: { /* schema */ + sqlite3 *db = sqlite3_context_db_handle(ctx); + sqlite3_result_text(ctx, db->aDb[pTab->iDb].zDbSName, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + *pRowid = pCsr->pgno; + return SQLITE_OK; +} + +/* +** Invoke this routine to register the "dbpage" virtual table module +*/ +int sqlite3DbpageRegister(sqlite3 *db){ + static sqlite3_module dbpage_module = { + 0, /* iVersion */ + dbpageConnect, /* xCreate */ + dbpageConnect, /* xConnect */ + dbpageBestIndex, /* xBestIndex */ + dbpageDisconnect, /* xDisconnect */ + dbpageDisconnect, /* xDestroy */ + dbpageOpen, /* xOpen - open a cursor */ + dbpageClose, /* xClose - close a cursor */ + dbpageFilter, /* xFilter - configure scan constraints */ + dbpageNext, /* xNext - advance a cursor */ + dbpageEof, /* xEof - check for end of scan */ + dbpageColumn, /* xColumn - read data */ + dbpageRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + }; + return sqlite3_create_module(db, "sqlite_dbpage", &dbpage_module, 0); +} +#elif defined(SQLITE_ENABLE_DBPAGE_VTAB) +int sqlite3DbpageRegister(sqlite3 *db){ return SQLITE_OK; } +#endif /* SQLITE_ENABLE_DBSTAT_VTAB */ diff --git a/src/main.c b/src/main.c index 3d7609ce5b..49613f6c74 100644 --- a/src/main.c +++ b/src/main.c @@ -3054,6 +3054,12 @@ static int openDatabase( } #endif +#ifdef SQLITE_ENABLE_DBPAGE_VTAB + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3DbpageRegister(db); + } +#endif + #ifdef SQLITE_ENABLE_DBSTAT_VTAB if( !db->mallocFailed && rc==SQLITE_OK){ rc = sqlite3DbstatRegister(db); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a8f1bed512..0cc435d7b7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4400,6 +4400,9 @@ int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); int sqlite3ThreadJoin(SQLiteThread*, void**); #endif +#if defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST) +int sqlite3DbpageRegister(sqlite3*); +#endif #if defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST) int sqlite3DbstatRegister(sqlite3*); #endif diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 933819d12b..8ea3e81c91 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -394,6 +394,7 @@ foreach file { fts3_icu.c sqlite3rbu.c dbstat.c + dbpage.c sqlite3session.c json1.c fts5.c From 34d0b1ac56e7cdf587dfbb06a136d560032deedf Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Oct 2017 15:02:53 +0000 Subject: [PATCH 02/22] Get writes working on the sqlite_dbpage virtual table. Add a few test cases. FossilOrigin-Name: a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f --- manifest | 16 ++++--- manifest.uuid | 2 +- src/dbpage.c | 106 +++++++++++++++++++++++++++++++++++++++++++---- test/dbpage.test | 69 ++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 19 deletions(-) create mode 100644 test/dbpage.test diff --git a/manifest b/manifest index dbf72466e4..3e606d8851 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\simplementation\sof\sthe\s"sqlite_dbpage"\svirtual\stable.\s\sCurrently\nit\sis\sread-only\sand\shas\sa\splace-holder\sxBestIndex. -D 2017-10-11T13:48:11.877 +C Get\swrites\sworking\son\sthe\ssqlite_dbpage\svirtual\stable.\s\sAdd\sa\sfew\stest\scases. +D 2017-10-11T15:02:53.047 F Makefile.in f7cba589198b8663d8a43f25ad001cf44fdac4fcd6216325f05775924a7af2f9 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1224c8773fc7f89c57e83c7b452291d239aa2cff4af50a204c84129e295cc37d @@ -409,7 +409,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 -F src/dbpage.c f274a9b7bb680cc2952ee78883d67e133be0ef6065317813586a5f723af35ad5 +F src/dbpage.c c625a0bd605d4cea9a3258b8db49a5474a04976e95a9fe380cdaf74e8eb6736d F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 F src/expr.c 4d2d0aafd945424f638ee03e11330f03288ccf616e025498f3c8602d01609a0a @@ -709,6 +709,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e +F test/dbpage.test 9cf4dc92a4de67c81e5c32b24e3fbb8c4757e4b642694a219b3090a4f9277a4d F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -1657,10 +1658,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49 -R 9d91f38d6de2e0aabd371a2c4dbcd832 -T *branch * dbpage -T *sym-dbpage * -T -sym-trunk * +P c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49 +R 92490bca397034b7f3fba4e951f0781d U drh -Z 95e75b385006fb35e51e104eb16c88ba +Z ddf133cf759c24442aeeb91134927fef diff --git a/manifest.uuid b/manifest.uuid index b0152fffa9..7a388fce6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49 \ No newline at end of file +a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index 89dcf04227..d21c5b6df1 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -16,6 +16,19 @@ ** pages of the database file. The pager interface is used so that ** uncommitted changes and changes recorded in the WAL file are correctly ** retrieved. +** +** Usage example: +** +** SELECT data FROM sqlite_dbpage('aux1') WHERE pgno=123; +** +** This is an eponymous virtual table so it does not need to be created before +** use. The optional argument to the sqlite_dbpage() table name is the +** schema for the database file that is to be read. The default schema is +** "main". +** +** The data field of sqlite_dbpage table can be updated. The new +** value must be a BLOB which is the correct page size, otherwise the +** update fails. Rows may not be deleted or inserted. */ #include "sqliteInt.h" /* Requires access to internal data structures */ @@ -28,6 +41,7 @@ typedef struct DbpageCursor DbpageCursor; struct DbpageCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ int pgno; /* Current page number */ + int mxPgno; /* Last page to visit on this scan */ }; struct DbpageTable { @@ -65,7 +79,7 @@ static int dbpageConnect( iDb = 0; } rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB)"); + "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)"); if( rc==SQLITE_OK ){ pTab = (DbpageTable *)sqlite3_malloc64(sizeof(DbpageTable)); if( pTab==0 ) rc = SQLITE_NOMEM_BKPT; @@ -99,7 +113,26 @@ static int dbpageDisconnect(sqlite3_vtab *pVtab){ ** 1 pgno=?1 */ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; pIdxInfo->estimatedCost = 1.0e6; /* Initial cost estimate */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->usable && p->iColumn<=0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + pIdxInfo->estimatedCost = 1.0; + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + } + if( pIdxInfo->nOrderBy>=1 + && pIdxInfo->aOrderBy[0].iColumn<=0 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } return SQLITE_OK; } @@ -143,8 +176,7 @@ static int dbpageNext(sqlite3_vtab_cursor *pCursor){ static int dbpageEof(sqlite3_vtab_cursor *pCursor){ DbpageCursor *pCsr = (DbpageCursor *)pCursor; - DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; - return pCsr->pgno >= pTab->nPage; + return pCsr->pgno > pCsr->mxPgno; } static int dbpageFilter( @@ -157,13 +189,20 @@ static int dbpageFilter( int rc = SQLITE_OK; Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; - if( idxNum==1 ){ - pCsr->pgno = sqlite3_value_int(argv[0]); - }else{ - pCsr->pgno = 0; - } pTab->szPage = sqlite3BtreeGetPageSize(pBt); pTab->nPage = sqlite3BtreeLastPage(pBt); + if( idxNum==1 ){ + pCsr->pgno = sqlite3_value_int(argv[0]); + if( pCsr->pgno<1 || pCsr->pgno>pTab->nPage ){ + pCsr->pgno = 1; + pCsr->mxPgno = 0; + }else{ + pCsr->mxPgno = pCsr->pgno; + } + }else{ + pCsr->pgno = 1; + pCsr->mxPgno = pTab->nPage; + } return rc; } @@ -205,6 +244,55 @@ static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ return SQLITE_OK; } +static int dbpageUpdate( + sqlite3_vtab *pVtab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + DbpageTable *pTab = (DbpageTable *)pVtab; + int pgno; + DbPage *pDbPage = 0; + int rc = SQLITE_OK; + char *zErr = 0; + + if( argc==1 ){ + zErr = "cannot delete"; + goto update_fail; + } + pgno = sqlite3_value_int(argv[0]); + if( pgno<1 || pgno>pTab->nPage ){ + zErr = "bad page number"; + goto update_fail; + } + if( sqlite3_value_int(argv[1])!=pgno ){ + zErr = "cannot insert"; + goto update_fail; + } + if( sqlite3_value_type(argv[3])!=SQLITE_BLOB + || sqlite3_value_bytes(argv[3])!=pTab->szPage + ){ + zErr = "bad page value"; + goto update_fail; + } + rc = sqlite3PagerGet(pTab->pPager, pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pDbPage); + if( rc==SQLITE_OK ){ + memcpy(sqlite3PagerGetData(pDbPage), + sqlite3_value_blob(argv[3]), + pTab->szPage); + } + } + sqlite3PagerUnref(pDbPage); + return rc; + +update_fail: + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = sqlite3_mprintf("%s", zErr); + return SQLITE_ERROR; +} + /* ** Invoke this routine to register the "dbpage" virtual table module */ @@ -223,7 +311,7 @@ int sqlite3DbpageRegister(sqlite3 *db){ dbpageEof, /* xEof - check for end of scan */ dbpageColumn, /* xColumn - read data */ dbpageRowid, /* xRowid - read data */ - 0, /* xUpdate */ + dbpageUpdate, /* xUpdate */ 0, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ diff --git a/test/dbpage.test b/test/dbpage.test new file mode 100644 index 0000000000..e29d4b33a9 --- /dev/null +++ b/test/dbpage.test @@ -0,0 +1,69 @@ +# 2017-10-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sqlite_dbpage virtual table. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix dbpage + +ifcapable !vtab||!compound { + finish_test + return +} + +do_execsql_test 100 { + PRAGMA page_size=4096; + PRAGMA journal_mode=WAL; + CREATE TABLE t1(a,b); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b) SELECT x, printf('%d-x%.*c',x,x,'x') FROM c; + PRAGMA integrity_check; +} {wal ok} +do_execsql_test 110 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('main') ORDER BY pgno; +} {1 X'53514C6974' 2 X'0500000001' 3 X'0D0000004E' 4 X'0D00000016'} +do_execsql_test 120 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=2; +} {2 X'0500000001'} +do_execsql_test 130 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=4; +} {4 X'0D00000016'} +do_execsql_test 140 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=5; +} {} +do_execsql_test 150 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=0; +} {} + +do_execsql_test 200 { + CREATE TEMP TABLE saved_content(x); + INSERT INTO saved_content(x) SELECT data FROM sqlite_dbpage WHERE pgno=4; + UPDATE sqlite_dbpage SET data=zeroblob(4096) WHERE pgno=4; +} {} +do_catchsql_test 210 { + PRAGMA integrity_check; +} {1 {database disk image is malformed}} +do_execsql_test 220 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('main') ORDER BY pgno; +} {1 X'53514C6974' 2 X'0500000001' 3 X'0D0000004E' 4 X'0000000000'} +do_execsql_test 230 { + UPDATE sqlite_dbpage SET data=(SELECT x FROM saved_content) WHERE pgno=4; +} {} +do_catchsql_test 230 { + PRAGMA integrity_check; +} {0 ok} + + + + +finish_test From ca5cf1229be7c3e60c6a2daf2c01feba6d9aeb2e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Oct 2017 17:13:29 +0000 Subject: [PATCH 03/22] Enable sqlite_dbpage and dbstat virtual tables in the shell, by default. FossilOrigin-Name: 01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db --- Makefile.in | 2 ++ Makefile.msc | 3 ++- main.mk | 2 ++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index 7ea97e6195..5e3f14279e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -572,6 +572,8 @@ SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB +SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 diff --git a/Makefile.msc b/Makefile.msc index d59997388d..febf496ee3 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1510,7 +1510,8 @@ FUZZDATA = \ # when the shell is not being dynamically linked. # !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB !ENDIF # <> diff --git a/main.mk b/main.mk index 3f83197d4d..26a0bd586d 100644 --- a/main.mk +++ b/main.mk @@ -483,6 +483,8 @@ SHELL_OPT += -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB +SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 diff --git a/manifest b/manifest index 3e606d8851..e250415fef 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Get\swrites\sworking\son\sthe\ssqlite_dbpage\svirtual\stable.\s\sAdd\sa\sfew\stest\scases. -D 2017-10-11T15:02:53.047 -F Makefile.in f7cba589198b8663d8a43f25ad001cf44fdac4fcd6216325f05775924a7af2f9 +C Enable\ssqlite_dbpage\sand\sdbstat\svirtual\stables\sin\sthe\sshell,\sby\sdefault. +D 2017-10-11T17:13:29.048 +F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 1224c8773fc7f89c57e83c7b452291d239aa2cff4af50a204c84129e295cc37d +F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -382,7 +382,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 656d9f31fdc01290094460f00255a45fdf4bedd08839467857919fea628582cf +F main.mk dc4d9dfe050e6d65671cb940974bd6b76451287db334c47332552ede28325714 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1658,7 +1658,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49 -R 92490bca397034b7f3fba4e951f0781d +P a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f +R 26ea83977ebe8e13c118f8a4befc3e5d U drh -Z ddf133cf759c24442aeeb91134927fef +Z d68d7c86064c38dfe4b5e452d55c3294 diff --git a/manifest.uuid b/manifest.uuid index 7a388fce6d..4b74c0454f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f \ No newline at end of file +01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db \ No newline at end of file From 512e6c3c5118cba59ed3fa66a586e87100b91299 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Oct 2017 17:51:08 +0000 Subject: [PATCH 04/22] Convert the implementation of the ".dbstat" dot-command of the command-line shell to use the sqlite_dbpage table. FossilOrigin-Name: 497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 18 +++++++++++------- src/shell.c.in | 18 +++++++++++------- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index e250415fef..92b51678a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\ssqlite_dbpage\sand\sdbstat\svirtual\stables\sin\sthe\sshell,\sby\sdefault. -D 2017-10-11T17:13:29.048 +C Convert\sthe\simplementation\sof\sthe\s".dbstat"\sdot-command\sof\sthe\scommand-line\nshell\sto\suse\sthe\ssqlite_dbpage\stable. +D 2017-10-11T17:51:08.392 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 @@ -460,8 +460,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 -F src/shell.c cbf450e75665a185c546adc702ec5fd091306ae7a08bc88b1508ac9c11acc7fe -F src/shell.c.in e03f7d473e10b65c25836a058a3e7a1665ffb1fe712949dcd6e38c790e4eafd0 +F src/shell.c b1c14539ae8f756a96a5604952e24fb8f2a65745290037f4f43dddfabac76e6e +F src/shell.c.in 73d8000bb066cd7ceb9655ffdb0e19a80779e3c64506f5a1ecfa9838511bee18 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 @@ -1658,7 +1658,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f -R 26ea83977ebe8e13c118f8a4befc3e5d +P 01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db +R 32cd47bb05391f34b12cc1216f3e5e53 U drh -Z d68d7c86064c38dfe4b5e452d55c3294 +Z ef5ad8ed0ad3bdbe43b1de17eb74fe58 diff --git a/manifest.uuid b/manifest.uuid index 4b74c0454f..12bf79e9d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db \ No newline at end of file +497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index f7be941570..2b77d482e6 100644 --- a/src/shell.c +++ b/src/shell.c @@ -4970,20 +4970,24 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ { "schema size:", "SELECT total(length(sql)) FROM %s" }, }; - sqlite3_file *pFile = 0; int i; char *zSchemaTab; char *zDb = nArg>=2 ? azArg[1] : "main"; + sqlite3_stmt *pStmt = 0; unsigned char aHdr[100]; open_db(p, 0); if( p->db==0 ) return 1; - sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFile); - if( pFile==0 || pFile->pMethods==0 || pFile->pMethods->xRead==0 ){ - return 1; - } - i = pFile->pMethods->xRead(pFile, aHdr, 100, 0); - if( i!=SQLITE_OK ){ + sqlite3_prepare_v2(p->db,"SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", + -1, &pStmt, 0); + sqlite3_bind_text(pStmt, 1, zDb, -1, SQLITE_STATIC); + if( sqlite3_step(pStmt)==SQLITE_ROW + && sqlite3_column_bytes(pStmt,0)>100 + ){ + memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100); + sqlite3_finalize(pStmt); + }else{ raw_printf(stderr, "unable to read database header\n"); + sqlite3_finalize(pStmt); return 1; } i = get2byteInt(aHdr+16); diff --git a/src/shell.c.in b/src/shell.c.in index db4f2e1128..54a61b9456 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3610,20 +3610,24 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ { "schema size:", "SELECT total(length(sql)) FROM %s" }, }; - sqlite3_file *pFile = 0; int i; char *zSchemaTab; char *zDb = nArg>=2 ? azArg[1] : "main"; + sqlite3_stmt *pStmt = 0; unsigned char aHdr[100]; open_db(p, 0); if( p->db==0 ) return 1; - sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFile); - if( pFile==0 || pFile->pMethods==0 || pFile->pMethods->xRead==0 ){ - return 1; - } - i = pFile->pMethods->xRead(pFile, aHdr, 100, 0); - if( i!=SQLITE_OK ){ + sqlite3_prepare_v2(p->db,"SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", + -1, &pStmt, 0); + sqlite3_bind_text(pStmt, 1, zDb, -1, SQLITE_STATIC); + if( sqlite3_step(pStmt)==SQLITE_ROW + && sqlite3_column_bytes(pStmt,0)>100 + ){ + memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100); + sqlite3_finalize(pStmt); + }else{ raw_printf(stderr, "unable to read database header\n"); + sqlite3_finalize(pStmt); return 1; } i = get2byteInt(aHdr+16); From 36187fe8c2b90068f3262f962654ba5632b530e7 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 11 Oct 2017 18:00:34 +0000 Subject: [PATCH 05/22] Add new extension "checkfreelist", which uses sqlite_dbpage to check that there are no invalid entries on the database free-list. FossilOrigin-Name: 21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 --- ext/misc/checkfreelist.c | 291 +++++++++++++++++++++++++++++++++++++++ manifest | 14 +- manifest.uuid | 2 +- test/checkfreelist.test | 114 +++++++++++++++ 4 files changed, 414 insertions(+), 7 deletions(-) create mode 100644 ext/misc/checkfreelist.c create mode 100644 test/checkfreelist.test diff --git a/ext/misc/checkfreelist.c b/ext/misc/checkfreelist.c new file mode 100644 index 0000000000..cec03b7999 --- /dev/null +++ b/ext/misc/checkfreelist.c @@ -0,0 +1,291 @@ +/* +** 2017 October 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module exports a single C function: +** +** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); +** +** This function checks the free-list in database zDb (one of "main", +** "temp", etc.) and reports any errors by invoking the sqlite3_log() +** function. It returns SQLITE_OK if successful, or an SQLite error +** code otherwise. It is not an error if the free-list is corrupted but +** no IO or OOM errors occur. +** +** If this file is compiled and loaded as an SQLite loadable extension, +** it adds an SQL function "checkfreelist" to the database handle, to +** be invoked as follows: +** +** SELECT checkfreelist(); +** +** This function performs the same checks as sqlite3_check_freelist(), +** except that it returns all error messages as a single text value, +** separated by newline characters. If the freelist is not corrupted +** in any way, an empty string is returned. +** +** To compile this module for use as an SQLite loadable extension: +** +** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so +*/ + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#ifndef SQLITE_AMALGAMATION +# include +# include +# include +# include +# define ALWAYS(X) 1 +# define NEVER(X) 0 + typedef unsigned char u8; + typedef unsigned short u16; + typedef unsigned int u32; +#define get4byte(x) ( \ + ((u32)((x)[0])<<24) + \ + ((u32)((x)[1])<<16) + \ + ((u32)((x)[2])<<8) + \ + ((u32)((x)[3])) \ +) +#endif + +/* +** Execute a single PRAGMA statement and return the integer value returned +** via output parameter (*pnOut). +** +** The SQL statement passed as the third argument should be a printf-style +** format string containing a single "%s" which will be replace by the +** value passed as the second argument. e.g. +** +** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) +** +** executes "PRAGMA main.page_count" and stores the results in (*pnOut). +*/ +static int sqlGetInteger( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + const char *zFmt, /* SQL statement format */ + u32 *pnOut /* OUT: Integer value */ +){ + int rc, rc2; + char *zSql; + sqlite3_stmt *pStmt = 0; + int bOk = 0; + + zSql = sqlite3_mprintf(zFmt, zDb); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnOut = (u32)sqlite3_column_int(pStmt, 0); + bOk = 1; + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; + return rc; +} + +/* +** Argument zFmt must be a printf-style format string and must be +** followed by its required arguments. If argument pzOut is NULL, +** then the results of printf()ing the format string are passed to +** sqlite3_log(). Otherwise, they are appended to the string +** at (*pzOut). +*/ +static int checkFreelistError(char **pzOut, const char *zFmt, ...){ + int rc = SQLITE_OK; + char *zErr = 0; + va_list ap; + + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + if( zErr==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pzOut ){ + *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); + if( *pzOut==0 ) rc = SQLITE_NOMEM; + }else{ + sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); + } + sqlite3_free(zErr); + } + va_end(ap); + return rc; +} + +static int checkFreelist( + sqlite3 *db, + const char *zDb, + char **pzOut +){ + /* This query returns one row for each page on the free list. Each row has + ** two columns - the page number and page content. */ + const char *zTrunk = + "WITH freelist_trunk(i, d, n) AS (" + "SELECT 1, NULL, sqlite_readint32(data, 32) " + "FROM sqlite_dbpage(:1) WHERE pgno=1 " + "UNION ALL " + "SELECT n, data, sqlite_readint32(data) " + "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " + ")" + "SELECT i, d FROM freelist_trunk WHERE i!=1;"; + + int rc, rc2; /* Return code */ + sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ + u32 nPage = 0; /* Number of pages in db */ + u32 nExpected = 0; /* Expected number of free pages */ + u32 nFree = 0; /* Number of pages on free list */ + + if( zDb==0 ) zDb = "main"; + + if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) + || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) + ){ + return rc; + } + + rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); + while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ + u32 i; + u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); + const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); + int nData = sqlite3_column_bytes(pTrunk, 1); + u32 iNext = get4byte(&aData[0]); + u32 nLeaf = get4byte(&aData[4]); + + nFree += 1+nLeaf; + if( iNext>nPage ){ + rc = checkFreelistError(pzOut, + "trunk page %d is out of range", (int)iNext + ); + } + + for(i=0; rc==SQLITE_OK && inPage ){ + rc = checkFreelistError(pzOut, + "leaf page %d is out of range (child %d of trunk page %d)", + (int)iLeaf, (int)i, (int)iTrunk + ); + } + } + } + + if( rc==SQLITE_OK && nFree!=nExpected ){ + rc = checkFreelistError(pzOut, + "free-list count mismatch: actual=%d header=%d", + (int)nFree, (int)nExpected + ); + } + + rc2 = sqlite3_finalize(pTrunk); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + +int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ + return checkFreelist(db, zDb, 0); +} + +static void checkfreelist_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const char *zDb; + int rc; + char *zOut = 0; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + assert( nArg==1 ); + zDb = sqlite3_value_text(apArg[0]); + rc = checkFreelist(db, zDb, &zOut); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + + sqlite3_free(zOut); +} + +/* +** An SQL function invoked as follows: +** +** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob +*/ +static void readint_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const u8 *zBlob; + int nBlob; + int iOff = 0; + u32 iRet = 0; + + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error( + pCtx, "wrong number of arguments to function sqlite_readint32()", -1 + ); + return; + } + if( nArg==2 ){ + iOff = sqlite3_value_int(apArg[1]); + } + + zBlob = sqlite3_value_blob(apArg[0]); + nBlob = sqlite3_value_bytes(apArg[0]); + + if( nBlob>=(iOff+4) ){ + iRet = get4byte(&zBlob[iOff]); + } + + sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); +} + +/* +** Register the SQL functions. +*/ +static int cflRegister(sqlite3 *db){ + int rc = sqlite3_create_function( + db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 + ); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3_create_function( + db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 + ); + return rc; +} + +/* +** Extension load function. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_checkfreelist_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return cflRegister(db); +} diff --git a/manifest b/manifest index 92b51678a2..d4e28e2d73 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sthe\simplementation\sof\sthe\s".dbstat"\sdot-command\sof\sthe\scommand-line\nshell\sto\suse\sthe\ssqlite_dbpage\stable. -D 2017-10-11T17:51:08.392 +C Add\snew\sextension\s"checkfreelist",\swhich\suses\ssqlite_dbpage\sto\scheck\sthat\nthere\sare\sno\sinvalid\sentries\son\sthe\sdatabase\sfree-list. +D 2017-10-11T18:00:34.689 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 @@ -259,6 +259,7 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 +F ext/misc/checkfreelist.c 043fdcc710f4147ff1deaf1bd6ea0a1c3eccb665ddd30d5623823a8eb4817eea w ext/misc/freelistchecker.c F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 @@ -647,6 +648,7 @@ F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a +F test/checkfreelist.test 6324b0a279eb101d698b31c12a65767b25f9b5c66d0d424943ae002e01f0de2f F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1658,7 +1660,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db -R 32cd47bb05391f34b12cc1216f3e5e53 -U drh -Z ef5ad8ed0ad3bdbe43b1de17eb74fe58 +P 497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 +R 7024a507e1ac7d6985bebba168f4b31f +U dan +Z 54a66c878cf2157bd3590abc0ee2a612 diff --git a/manifest.uuid b/manifest.uuid index 12bf79e9d8..e6ec495019 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 \ No newline at end of file +21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 \ No newline at end of file diff --git a/test/checkfreelist.test b/test/checkfreelist.test new file mode 100644 index 0000000000..f16d8b57bb --- /dev/null +++ b/test/checkfreelist.test @@ -0,0 +1,114 @@ +# 2017-10-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the checkfreelist extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix checkfreelist + +ifcapable !vtab||!compound { + finish_test + return +} + +if {[file exists ../checkfreelist.so]==0} { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); +} + +db enable_load_extension 1 +do_execsql_test 1.1 { + SELECT load_extension('../checkfreelist.so'); +} {{}} + +do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok} +do_execsql_test 1.3 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000 + ) + INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s; + DELETE FROM t1 WHERE rowid%3; + PRAGMA freelist_count; +} {6726} + +do_execsql_test 1.4 { SELECT checkfreelist('main') } {ok} +do_execsql_test 1.5 { + WITH freelist_trunk(i, d, n) AS ( + SELECT 1, NULL, sqlite_readint32(data, 32) FROM sqlite_dbpage WHERE pgno=1 + UNION ALL + SELECT n, data, sqlite_readint32(data) + FROM freelist_trunk, sqlite_dbpage WHERE pgno=n + ) + SELECT i FROM freelist_trunk WHERE i!=1; +} { + 10010 9716 9344 8970 8596 8223 7848 7475 7103 6728 6355 5983 5609 5235 + 4861 4488 4113 3741 3368 2993 2620 2248 1873 1500 1126 753 378 5 +} + +do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok} + +proc set_int {blob idx newval} { + binary scan $blob I* ints + lset ints $idx $newval + binary format I* $ints +} +db func set_int set_int + +proc get_int {blob idx} { + binary scan $blob I* ints + lindex $ints $idx +} +db func get_int get_int + +do_execsql_test 1.7 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 1, get_int(data, 1)-1) + WHERE pgno=4861; + SELECT checkfreelist('main'); + ROLLBACK; +} {{free-list count mismatch: actual=6725 header=6726}} + +do_execsql_test 1.8 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1) + WHERE pgno=4861; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 10093 is out of range (child 3 of trunk page 4861)}} + +do_execsql_test 1.9 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 5, 0) + WHERE pgno=4861; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 0 is out of range (child 3 of trunk page 4861)}} + +do_execsql_test 1.10 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, get_int(data, 1)+1, 0) + WHERE pgno=5; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 0 is out of range (child 247 of trunk page 5)}} + +finish_test + From 7d157f91f4a29e8f237f1f1d0705271a3fe45f5f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 11 Oct 2017 18:21:44 +0000 Subject: [PATCH 06/22] Check that the leaf count on each freelist trunk page is in range as part of checkfreelist processing. FossilOrigin-Name: 4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec --- ext/misc/checkfreelist.c | 8 ++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- test/checkfreelist.test | 9 +++++++++ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/ext/misc/checkfreelist.c b/ext/misc/checkfreelist.c index cec03b7999..345f26c223 100644 --- a/ext/misc/checkfreelist.c +++ b/ext/misc/checkfreelist.c @@ -170,6 +170,14 @@ static int checkFreelist( u32 iNext = get4byte(&aData[0]); u32 nLeaf = get4byte(&aData[4]); + if( nLeaf>((nData/4)-2-6) ){ + rc = checkFreelistError(pzOut, + "leaf count out of range (%d) on trunk page %d", + (int)nLeaf, (int)iTrunk + ); + nLeaf = (nData/4) - 2 - 6; + } + nFree += 1+nLeaf; if( iNext>nPage ){ rc = checkFreelistError(pzOut, diff --git a/manifest b/manifest index d4e28e2d73..7a6b18c230 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sextension\s"checkfreelist",\swhich\suses\ssqlite_dbpage\sto\scheck\sthat\nthere\sare\sno\sinvalid\sentries\son\sthe\sdatabase\sfree-list. -D 2017-10-11T18:00:34.689 +C Check\sthat\sthe\sleaf\scount\son\seach\sfreelist\strunk\spage\sis\sin\srange\sas\spart\sof\ncheckfreelist\sprocessing. +D 2017-10-11T18:21:44.429 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 @@ -259,7 +259,7 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 -F ext/misc/checkfreelist.c 043fdcc710f4147ff1deaf1bd6ea0a1c3eccb665ddd30d5623823a8eb4817eea w ext/misc/freelistchecker.c +F ext/misc/checkfreelist.c fc46557e73a6233bd698815d3963acc44bf4dba0ca9c91c90be361cca49d6b3e F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 @@ -648,7 +648,7 @@ F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a -F test/checkfreelist.test 6324b0a279eb101d698b31c12a65767b25f9b5c66d0d424943ae002e01f0de2f +F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1660,7 +1660,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 -R 7024a507e1ac7d6985bebba168f4b31f +P 21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 +R f25a07255c10ac9143e97a5d9d8e737e U dan -Z 54a66c878cf2157bd3590abc0ee2a612 +Z fe2b858df1a95da0fbc85fd6ca1ee6ea diff --git a/manifest.uuid b/manifest.uuid index e6ec495019..a9df98f06f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 \ No newline at end of file +4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec \ No newline at end of file diff --git a/test/checkfreelist.test b/test/checkfreelist.test index f16d8b57bb..93e4ecc234 100644 --- a/test/checkfreelist.test +++ b/test/checkfreelist.test @@ -110,5 +110,14 @@ do_execsql_test 1.10 { ROLLBACK; } {{leaf page 0 is out of range (child 247 of trunk page 5)}} +do_execsql_test 1.11 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 1, 249) + WHERE pgno=5; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf count out of range (249) on trunk page 5}} + finish_test From f294ce648b121fc5ed1c8e6456ed6b03fac34b13 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Oct 2017 18:26:26 +0000 Subject: [PATCH 07/22] Add the checkfreelist extension to the command-line shell. FossilOrigin-Name: 48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52 --- ext/misc/checkfreelist.c | 2 +- manifest | 18 +-- manifest.uuid | 2 +- src/shell.c | 302 +++++++++++++++++++++++++++++++++++++++ src/shell.c.in | 2 + 5 files changed, 315 insertions(+), 11 deletions(-) diff --git a/ext/misc/checkfreelist.c b/ext/misc/checkfreelist.c index 345f26c223..cd2801e040 100644 --- a/ext/misc/checkfreelist.c +++ b/ext/misc/checkfreelist.c @@ -223,7 +223,7 @@ static void checkfreelist_function( sqlite3 *db = sqlite3_context_db_handle(pCtx); assert( nArg==1 ); - zDb = sqlite3_value_text(apArg[0]); + zDb = (const char*)sqlite3_value_text(apArg[0]); rc = checkFreelist(db, zDb, &zOut); if( rc==SQLITE_OK ){ sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); diff --git a/manifest b/manifest index 7a6b18c230..c2dd3cb1eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Check\sthat\sthe\sleaf\scount\son\seach\sfreelist\strunk\spage\sis\sin\srange\sas\spart\sof\ncheckfreelist\sprocessing. -D 2017-10-11T18:21:44.429 +C Add\sthe\scheckfreelist\sextension\sto\sthe\scommand-line\sshell. +D 2017-10-11T18:26:26.636 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 @@ -259,7 +259,7 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 -F ext/misc/checkfreelist.c fc46557e73a6233bd698815d3963acc44bf4dba0ca9c91c90be361cca49d6b3e +F ext/misc/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 @@ -461,8 +461,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 -F src/shell.c b1c14539ae8f756a96a5604952e24fb8f2a65745290037f4f43dddfabac76e6e -F src/shell.c.in 73d8000bb066cd7ceb9655ffdb0e19a80779e3c64506f5a1ecfa9838511bee18 +F src/shell.c ffb06532d6667bf1bb64080a316120c67249636a12f008c2f9716d6778565d57 +F src/shell.c.in 7842db264d5512520c61d0353196eeefeb65b710dd0d97d4ad9844c56e313be5 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 @@ -1660,7 +1660,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 -R f25a07255c10ac9143e97a5d9d8e737e -U dan -Z fe2b858df1a95da0fbc85fd6ca1ee6ea +P 4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec +R 0bf65201c138febf2eab9baca075a19d +U drh +Z f892c9c71f9811efc31d97f9328db29a diff --git a/manifest.uuid b/manifest.uuid index a9df98f06f..eb5db27581 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec \ No newline at end of file +48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 2b77d482e6..6c14db9fdd 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2156,6 +2156,307 @@ int sqlite3_completion_init( } /************************* End ../ext/misc/completion.c ********************/ +/************************* Begin ../ext/misc/checkfreelist.c ******************/ +/* +** 2017 October 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module exports a single C function: +** +** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); +** +** This function checks the free-list in database zDb (one of "main", +** "temp", etc.) and reports any errors by invoking the sqlite3_log() +** function. It returns SQLITE_OK if successful, or an SQLite error +** code otherwise. It is not an error if the free-list is corrupted but +** no IO or OOM errors occur. +** +** If this file is compiled and loaded as an SQLite loadable extension, +** it adds an SQL function "checkfreelist" to the database handle, to +** be invoked as follows: +** +** SELECT checkfreelist(); +** +** This function performs the same checks as sqlite3_check_freelist(), +** except that it returns all error messages as a single text value, +** separated by newline characters. If the freelist is not corrupted +** in any way, an empty string is returned. +** +** To compile this module for use as an SQLite loadable extension: +** +** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so +*/ + +SQLITE_EXTENSION_INIT1 + +#ifndef SQLITE_AMALGAMATION +# include +# include +# include +# include +# define ALWAYS(X) 1 +# define NEVER(X) 0 + typedef unsigned char u8; + typedef unsigned short u16; + typedef unsigned int u32; +#define get4byte(x) ( \ + ((u32)((x)[0])<<24) + \ + ((u32)((x)[1])<<16) + \ + ((u32)((x)[2])<<8) + \ + ((u32)((x)[3])) \ +) +#endif + +/* +** Execute a single PRAGMA statement and return the integer value returned +** via output parameter (*pnOut). +** +** The SQL statement passed as the third argument should be a printf-style +** format string containing a single "%s" which will be replace by the +** value passed as the second argument. e.g. +** +** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) +** +** executes "PRAGMA main.page_count" and stores the results in (*pnOut). +*/ +static int sqlGetInteger( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + const char *zFmt, /* SQL statement format */ + u32 *pnOut /* OUT: Integer value */ +){ + int rc, rc2; + char *zSql; + sqlite3_stmt *pStmt = 0; + int bOk = 0; + + zSql = sqlite3_mprintf(zFmt, zDb); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnOut = (u32)sqlite3_column_int(pStmt, 0); + bOk = 1; + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; + return rc; +} + +/* +** Argument zFmt must be a printf-style format string and must be +** followed by its required arguments. If argument pzOut is NULL, +** then the results of printf()ing the format string are passed to +** sqlite3_log(). Otherwise, they are appended to the string +** at (*pzOut). +*/ +static int checkFreelistError(char **pzOut, const char *zFmt, ...){ + int rc = SQLITE_OK; + char *zErr = 0; + va_list ap; + + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + if( zErr==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pzOut ){ + *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); + if( *pzOut==0 ) rc = SQLITE_NOMEM; + }else{ + sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); + } + sqlite3_free(zErr); + } + va_end(ap); + return rc; +} + +static int checkFreelist( + sqlite3 *db, + const char *zDb, + char **pzOut +){ + /* This query returns one row for each page on the free list. Each row has + ** two columns - the page number and page content. */ + const char *zTrunk = + "WITH freelist_trunk(i, d, n) AS (" + "SELECT 1, NULL, sqlite_readint32(data, 32) " + "FROM sqlite_dbpage(:1) WHERE pgno=1 " + "UNION ALL " + "SELECT n, data, sqlite_readint32(data) " + "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " + ")" + "SELECT i, d FROM freelist_trunk WHERE i!=1;"; + + int rc, rc2; /* Return code */ + sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ + u32 nPage = 0; /* Number of pages in db */ + u32 nExpected = 0; /* Expected number of free pages */ + u32 nFree = 0; /* Number of pages on free list */ + + if( zDb==0 ) zDb = "main"; + + if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) + || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) + ){ + return rc; + } + + rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); + while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ + u32 i; + u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); + const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); + int nData = sqlite3_column_bytes(pTrunk, 1); + u32 iNext = get4byte(&aData[0]); + u32 nLeaf = get4byte(&aData[4]); + + if( nLeaf>((nData/4)-2-6) ){ + rc = checkFreelistError(pzOut, + "leaf count out of range (%d) on trunk page %d", + (int)nLeaf, (int)iTrunk + ); + nLeaf = (nData/4) - 2 - 6; + } + + nFree += 1+nLeaf; + if( iNext>nPage ){ + rc = checkFreelistError(pzOut, + "trunk page %d is out of range", (int)iNext + ); + } + + for(i=0; rc==SQLITE_OK && inPage ){ + rc = checkFreelistError(pzOut, + "leaf page %d is out of range (child %d of trunk page %d)", + (int)iLeaf, (int)i, (int)iTrunk + ); + } + } + } + + if( rc==SQLITE_OK && nFree!=nExpected ){ + rc = checkFreelistError(pzOut, + "free-list count mismatch: actual=%d header=%d", + (int)nFree, (int)nExpected + ); + } + + rc2 = sqlite3_finalize(pTrunk); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + +int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ + return checkFreelist(db, zDb, 0); +} + +static void checkfreelist_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const char *zDb; + int rc; + char *zOut = 0; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + assert( nArg==1 ); + zDb = (const char*)sqlite3_value_text(apArg[0]); + rc = checkFreelist(db, zDb, &zOut); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + + sqlite3_free(zOut); +} + +/* +** An SQL function invoked as follows: +** +** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob +*/ +static void readint_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const u8 *zBlob; + int nBlob; + int iOff = 0; + u32 iRet = 0; + + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error( + pCtx, "wrong number of arguments to function sqlite_readint32()", -1 + ); + return; + } + if( nArg==2 ){ + iOff = sqlite3_value_int(apArg[1]); + } + + zBlob = sqlite3_value_blob(apArg[0]); + nBlob = sqlite3_value_bytes(apArg[0]); + + if( nBlob>=(iOff+4) ){ + iRet = get4byte(&zBlob[iOff]); + } + + sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); +} + +/* +** Register the SQL functions. +*/ +static int cflRegister(sqlite3 *db){ + int rc = sqlite3_create_function( + db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 + ); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3_create_function( + db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 + ); + return rc; +} + +/* +** Extension load function. +*/ +#ifdef _WIN32 + +#endif +int sqlite3_checkfreelist_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return cflRegister(db); +} + +/************************* End ../ext/misc/checkfreelist.c ********************/ #if defined(SQLITE_ENABLE_SESSION) /* @@ -4246,6 +4547,7 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); + sqlite3_checkfreelist_init(p->db, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); } diff --git a/src/shell.c.in b/src/shell.c.in index 54a61b9456..de170887f7 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -796,6 +796,7 @@ static void shellAddSchemaName( INCLUDE ../ext/misc/shathree.c INCLUDE ../ext/misc/fileio.c INCLUDE ../ext/misc/completion.c +INCLUDE ../ext/misc/checkfreelist.c #if defined(SQLITE_ENABLE_SESSION) /* @@ -2886,6 +2887,7 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); + sqlite3_checkfreelist_init(p->db, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); } From 277ee81babd94ce08cda6c57596759dec457f35b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 12 Oct 2017 19:50:28 +0000 Subject: [PATCH 08/22] Create the new ext/repair folder and move checkfreelist.c there. Remove checkfreelist.c from the command-line shell (undoing check-in [48418f2e]). FossilOrigin-Name: dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab --- ext/repair/README.md | 16 ++ ext/{misc => repair}/checkfreelist.c | 0 manifest | 18 +- manifest.uuid | 2 +- src/shell.c | 302 --------------------------- src/shell.c.in | 2 - 6 files changed, 27 insertions(+), 313 deletions(-) create mode 100644 ext/repair/README.md rename ext/{misc => repair}/checkfreelist.c (100%) diff --git a/ext/repair/README.md b/ext/repair/README.md new file mode 100644 index 0000000000..927ceb7c44 --- /dev/null +++ b/ext/repair/README.md @@ -0,0 +1,16 @@ +This folder contains extensions and utility programs intended to analyze +live database files, detect problems, and possibly fix them. + +As SQLite is being used on larger and larger databases, database sizes +are growing into the terabyte range. At that size, hardware malfunctions +and/or cosmic rays will occasionally corrupt a database file. Detecting +problems and fixing errors a terabyte-sized databases can take hours or days, +and it is undesirable to take applications that depend on the databases +off-line for such a long time. +The utilities in the folder are intended to provide mechanisms for +detecting and fixing problems in large databases while those databases +are in active use. + +The utilities and extensions in this folder are experimental and under +active development at the time of this writing (2017-10-12). If and when +they stabilize, this README will be updated to reflect that fact. diff --git a/ext/misc/checkfreelist.c b/ext/repair/checkfreelist.c similarity index 100% rename from ext/misc/checkfreelist.c rename to ext/repair/checkfreelist.c diff --git a/manifest b/manifest index e5589b7672..5c8309403b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfixes\sfrom\sthe\s3.21\sbranch. -D 2017-10-12T01:24:36.582 +C Create\sthe\snew\sext/repair\sfolder\sand\smove\scheckfreelist.c\sthere.\s\sRemove\ncheckfreelist.c\sfrom\sthe\scommand-line\sshell\s(undoing\scheck-in\s[48418f2e]). +D 2017-10-12T19:50:28.969 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 523fd246fd801689c766945f6450615d3784fda2dc04993bcafc796ac6afd8df @@ -259,7 +259,6 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 -F ext/misc/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 @@ -326,6 +325,8 @@ F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa F ext/rbu/sqlite3rbu.c 64bd08c1011456f90564ed167abce3a9c2af421a924b21eb57231e078da04feb F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a +F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 +F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 w ext/misc/checkfreelist.c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c f2fd34db37ea053798f8e66b44a473449b21301d2b92505ee576823789e909fb F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -461,8 +462,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 -F src/shell.c ffb06532d6667bf1bb64080a316120c67249636a12f008c2f9716d6778565d57 -F src/shell.c.in 7842db264d5512520c61d0353196eeefeb65b710dd0d97d4ad9844c56e313be5 +F src/shell.c b1c14539ae8f756a96a5604952e24fb8f2a65745290037f4f43dddfabac76e6e +F src/shell.c.in 73d8000bb066cd7ceb9655ffdb0e19a80779e3c64506f5a1ecfa9838511bee18 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 @@ -1660,7 +1661,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52 f0a2724f0a255cd5a262f31e4ee1f99ae713c25a9ecc56dc794c95f223453b9b -R 9b46eb5073fd5db6110bab2092230089 +P 18d4654fd161900f98ff435ea9e0a3c44b9972f84ee9f43096f9998f844ff857 +Q -48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52 +R 4a0d06893cd053abce709ab7740cd918 U drh -Z b5c9a13e106677edbcab94b5f19597c5 +Z 46634faecfbbf65605db6fcff2dbe040 diff --git a/manifest.uuid b/manifest.uuid index f8e3904fb9..58e6b4a3f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18d4654fd161900f98ff435ea9e0a3c44b9972f84ee9f43096f9998f844ff857 \ No newline at end of file +dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 6c14db9fdd..2b77d482e6 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2156,307 +2156,6 @@ int sqlite3_completion_init( } /************************* End ../ext/misc/completion.c ********************/ -/************************* Begin ../ext/misc/checkfreelist.c ******************/ -/* -** 2017 October 11 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This module exports a single C function: -** -** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); -** -** This function checks the free-list in database zDb (one of "main", -** "temp", etc.) and reports any errors by invoking the sqlite3_log() -** function. It returns SQLITE_OK if successful, or an SQLite error -** code otherwise. It is not an error if the free-list is corrupted but -** no IO or OOM errors occur. -** -** If this file is compiled and loaded as an SQLite loadable extension, -** it adds an SQL function "checkfreelist" to the database handle, to -** be invoked as follows: -** -** SELECT checkfreelist(); -** -** This function performs the same checks as sqlite3_check_freelist(), -** except that it returns all error messages as a single text value, -** separated by newline characters. If the freelist is not corrupted -** in any way, an empty string is returned. -** -** To compile this module for use as an SQLite loadable extension: -** -** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so -*/ - -SQLITE_EXTENSION_INIT1 - -#ifndef SQLITE_AMALGAMATION -# include -# include -# include -# include -# define ALWAYS(X) 1 -# define NEVER(X) 0 - typedef unsigned char u8; - typedef unsigned short u16; - typedef unsigned int u32; -#define get4byte(x) ( \ - ((u32)((x)[0])<<24) + \ - ((u32)((x)[1])<<16) + \ - ((u32)((x)[2])<<8) + \ - ((u32)((x)[3])) \ -) -#endif - -/* -** Execute a single PRAGMA statement and return the integer value returned -** via output parameter (*pnOut). -** -** The SQL statement passed as the third argument should be a printf-style -** format string containing a single "%s" which will be replace by the -** value passed as the second argument. e.g. -** -** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) -** -** executes "PRAGMA main.page_count" and stores the results in (*pnOut). -*/ -static int sqlGetInteger( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Database name ("main", "temp" etc.) */ - const char *zFmt, /* SQL statement format */ - u32 *pnOut /* OUT: Integer value */ -){ - int rc, rc2; - char *zSql; - sqlite3_stmt *pStmt = 0; - int bOk = 0; - - zSql = sqlite3_mprintf(zFmt, zDb); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - } - - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnOut = (u32)sqlite3_column_int(pStmt, 0); - bOk = 1; - } - - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; - return rc; -} - -/* -** Argument zFmt must be a printf-style format string and must be -** followed by its required arguments. If argument pzOut is NULL, -** then the results of printf()ing the format string are passed to -** sqlite3_log(). Otherwise, they are appended to the string -** at (*pzOut). -*/ -static int checkFreelistError(char **pzOut, const char *zFmt, ...){ - int rc = SQLITE_OK; - char *zErr = 0; - va_list ap; - - va_start(ap, zFmt); - zErr = sqlite3_vmprintf(zFmt, ap); - if( zErr==0 ){ - rc = SQLITE_NOMEM; - }else{ - if( pzOut ){ - *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); - if( *pzOut==0 ) rc = SQLITE_NOMEM; - }else{ - sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); - } - sqlite3_free(zErr); - } - va_end(ap); - return rc; -} - -static int checkFreelist( - sqlite3 *db, - const char *zDb, - char **pzOut -){ - /* This query returns one row for each page on the free list. Each row has - ** two columns - the page number and page content. */ - const char *zTrunk = - "WITH freelist_trunk(i, d, n) AS (" - "SELECT 1, NULL, sqlite_readint32(data, 32) " - "FROM sqlite_dbpage(:1) WHERE pgno=1 " - "UNION ALL " - "SELECT n, data, sqlite_readint32(data) " - "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " - ")" - "SELECT i, d FROM freelist_trunk WHERE i!=1;"; - - int rc, rc2; /* Return code */ - sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ - u32 nPage = 0; /* Number of pages in db */ - u32 nExpected = 0; /* Expected number of free pages */ - u32 nFree = 0; /* Number of pages on free list */ - - if( zDb==0 ) zDb = "main"; - - if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) - || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) - ){ - return rc; - } - - rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); - while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ - u32 i; - u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); - const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); - int nData = sqlite3_column_bytes(pTrunk, 1); - u32 iNext = get4byte(&aData[0]); - u32 nLeaf = get4byte(&aData[4]); - - if( nLeaf>((nData/4)-2-6) ){ - rc = checkFreelistError(pzOut, - "leaf count out of range (%d) on trunk page %d", - (int)nLeaf, (int)iTrunk - ); - nLeaf = (nData/4) - 2 - 6; - } - - nFree += 1+nLeaf; - if( iNext>nPage ){ - rc = checkFreelistError(pzOut, - "trunk page %d is out of range", (int)iNext - ); - } - - for(i=0; rc==SQLITE_OK && inPage ){ - rc = checkFreelistError(pzOut, - "leaf page %d is out of range (child %d of trunk page %d)", - (int)iLeaf, (int)i, (int)iTrunk - ); - } - } - } - - if( rc==SQLITE_OK && nFree!=nExpected ){ - rc = checkFreelistError(pzOut, - "free-list count mismatch: actual=%d header=%d", - (int)nFree, (int)nExpected - ); - } - - rc2 = sqlite3_finalize(pTrunk); - if( rc==SQLITE_OK ) rc = rc2; - return rc; -} - -int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ - return checkFreelist(db, zDb, 0); -} - -static void checkfreelist_function( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - const char *zDb; - int rc; - char *zOut = 0; - sqlite3 *db = sqlite3_context_db_handle(pCtx); - - assert( nArg==1 ); - zDb = (const char*)sqlite3_value_text(apArg[0]); - rc = checkFreelist(db, zDb, &zOut); - if( rc==SQLITE_OK ){ - sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_code(pCtx, rc); - } - - sqlite3_free(zOut); -} - -/* -** An SQL function invoked as follows: -** -** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob -*/ -static void readint_function( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - const u8 *zBlob; - int nBlob; - int iOff = 0; - u32 iRet = 0; - - if( nArg!=1 && nArg!=2 ){ - sqlite3_result_error( - pCtx, "wrong number of arguments to function sqlite_readint32()", -1 - ); - return; - } - if( nArg==2 ){ - iOff = sqlite3_value_int(apArg[1]); - } - - zBlob = sqlite3_value_blob(apArg[0]); - nBlob = sqlite3_value_bytes(apArg[0]); - - if( nBlob>=(iOff+4) ){ - iRet = get4byte(&zBlob[iOff]); - } - - sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); -} - -/* -** Register the SQL functions. -*/ -static int cflRegister(sqlite3 *db){ - int rc = sqlite3_create_function( - db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 - ); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3_create_function( - db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 - ); - return rc; -} - -/* -** Extension load function. -*/ -#ifdef _WIN32 - -#endif -int sqlite3_checkfreelist_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - return cflRegister(db); -} - -/************************* End ../ext/misc/checkfreelist.c ********************/ #if defined(SQLITE_ENABLE_SESSION) /* @@ -4547,7 +4246,6 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); - sqlite3_checkfreelist_init(p->db, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); } diff --git a/src/shell.c.in b/src/shell.c.in index de170887f7..54a61b9456 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -796,7 +796,6 @@ static void shellAddSchemaName( INCLUDE ../ext/misc/shathree.c INCLUDE ../ext/misc/fileio.c INCLUDE ../ext/misc/completion.c -INCLUDE ../ext/misc/checkfreelist.c #if defined(SQLITE_ENABLE_SESSION) /* @@ -2887,7 +2886,6 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); - sqlite3_checkfreelist_init(p->db, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); } From c318f730a44618db5a84bba635be5b53ab876dcc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Oct 2017 15:06:06 +0000 Subject: [PATCH 09/22] Move some test logic out of tclsqlite.c and into auxiliary test_*.c files. This is a work in progress. FossilOrigin-Name: 95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf --- Makefile.in | 2 + Makefile.msc | 2 + main.mk | 2 + manifest | 24 +- manifest.uuid | 2 +- src/tclsqlite.c | 636 +++-------------------------------------------- src/test_md5.c | 450 +++++++++++++++++++++++++++++++++ src/test_tclsh.c | 196 +++++++++++++++ 8 files changed, 697 insertions(+), 617 deletions(-) create mode 100644 src/test_md5.c create mode 100644 src/test_tclsh.c diff --git a/Makefile.in b/Makefile.in index 5295c2b032..8c306629e0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -396,6 +396,7 @@ TESTSRC = \ $(TOP)/src/test_intarray.c \ $(TOP)/src/test_journal.c \ $(TOP)/src/test_malloc.c \ + $(TOP)/src/test_md5.c \ $(TOP)/src/test_multiplex.c \ $(TOP)/src/test_mutex.c \ $(TOP)/src/test_onefile.c \ @@ -407,6 +408,7 @@ TESTSRC = \ $(TOP)/src/test_server.c \ $(TOP)/src/test_superlock.c \ $(TOP)/src/test_syscall.c \ + $(TOP)/src/test_tclsh.c \ $(TOP)/src/test_tclvar.c \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vfs.c \ diff --git a/Makefile.msc b/Makefile.msc index da71df89f0..e4a72e872b 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1374,6 +1374,7 @@ TESTSRC = \ $(TOP)\src\test_intarray.c \ $(TOP)\src\test_journal.c \ $(TOP)\src\test_malloc.c \ + $(TOP)\src\test_md5.c \ $(TOP)\src\test_multiplex.c \ $(TOP)\src\test_mutex.c \ $(TOP)\src\test_onefile.c \ @@ -1385,6 +1386,7 @@ TESTSRC = \ $(TOP)\src\test_server.c \ $(TOP)\src\test_superlock.c \ $(TOP)\src\test_syscall.c \ + $(TOP)\src\test_tclsh.c \ $(TOP)\src\test_tclvar.c \ $(TOP)\src\test_thread.c \ $(TOP)\src\test_vfs.c \ diff --git a/main.mk b/main.mk index d03ee8d1d3..84d1a3eb76 100644 --- a/main.mk +++ b/main.mk @@ -308,6 +308,7 @@ TESTSRC = \ $(TOP)/src/test_intarray.c \ $(TOP)/src/test_journal.c \ $(TOP)/src/test_malloc.c \ + $(TOP)/src/test_md5.c \ $(TOP)/src/test_multiplex.c \ $(TOP)/src/test_mutex.c \ $(TOP)/src/test_onefile.c \ @@ -320,6 +321,7 @@ TESTSRC = \ $(TOP)/src/test_sqllog.c \ $(TOP)/src/test_superlock.c \ $(TOP)/src/test_syscall.c \ + $(TOP)/src/test_tclsh.c \ $(TOP)/src/test_tclvar.c \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vfs.c \ diff --git a/manifest b/manifest index 3360e30a3a..250e0c8001 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Add\sthe\ssqlite_dbpage\svirtual\stable\s(enabled\susing\sSQLITE_ENABLE_DBPAGE_VTAB).\nMake\sthat\svirtual\stable\sand\sdbstat\savailable\sto\sthe\scommand-line\sshell. -D 2017-10-12T20:37:20.278 -F Makefile.in cb88ca5a6d8e116a50bd19bf477384498df475c5463cbbf53f36624ce308ed62 +C Move\ssome\stest\slogic\sout\sof\stclsqlite.c\sand\sinto\sauxiliary\stest_*.c\sfiles.\nThis\sis\sa\swork\sin\sprogress. +D 2017-10-13T15:06:06.783 +F Makefile.in 8658943abe79d10adfed7b504936199ebcd5c1d372d176936d5edd075492b13b F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 918f07fee01bf858d20df6b37802161423f216da1ece14794afee12de4992e35 +F Makefile.msc 22fc2be4ac777553096de774b9b601568fa21904d1d64f595916f78957025558 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 8be17ffd37df86bfbd696072045cd360600499876f7144313090f96ce3a37a55 +F main.mk 03a3ae65678b238a787f0aa9162e0e2bd998561bc8d66b9b073f54d9025dee61 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -470,7 +470,7 @@ F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53b F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 -F src/tclsqlite.c 487951d81f9704800fd9f0ffdaa2f935a83ccb6be3575c2c4ef83e4789b4c828 +F src/tclsqlite.c 93b2267ade2bfaad9d4f0135e5928ee28fdd77a355a563ad444ead13e1c488b2 F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b @@ -499,6 +499,7 @@ F src/test_intarray.h f3b7672f5d1056eac563c0d6ea8480a660b1475c F src/test_journal.c 619f2aa10e0d7a5f87c0f06825bc61dfce1c6b9c7f3ad990fb13de6c3b8874a3 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd F src/test_malloc.c 4f06a805de86be5216a127b3777ca2d5a1ff99d1a9238374ce136a47411be36c +F src/test_md5.c 7268e1e8c399d4a5e181b64ac20e1e6f3bc4dd9fc87abac02db145a3d951fa8c F src/test_multiplex.c e054459f7633f3ff8ce1245da724f9a8be189e4e F src/test_multiplex.h 5436d03f2d0501d04f3ed50a75819e190495b635 F src/test_mutex.c 7f4337ba23ee6b1d2ec81c189653608cb069926a @@ -513,6 +514,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 +F src/test_tclsh.c 2a1ff3eac1ac3ee070a80209384d6cb353d4ad4a549400e3cd29445c672b2b87 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e @@ -1661,8 +1663,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 36acc0a97fdcc6f54f29c68c4e131702f69c3e59e58237ff4e5c647928699956 dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab -R a8e3c658e72c296e51cc10d88be952b0 -T +closed dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab +P eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a +R 225e4cd689e0c61d406db3daa52debdc +T *branch * tclsqlite-cleanup +T *sym-tclsqlite-cleanup * +T -sym-trunk * U drh -Z 03ff5b730193797e00df33319c2ae82d +Z 604c544fbf5b9e259c249448beb617c7 diff --git a/manifest.uuid b/manifest.uuid index 50f9e1660f..705f05a85e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a \ No newline at end of file +95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 1b9f91405e..a66fe6832e 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3546,443 +3546,16 @@ int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } #endif +/* +** If the TCLSH macro is defined to be either 1 or 2, then a main() +** routine is inserted that starts up a Tcl interpreter. When TCLSH==1 +** the interpreter works like an ordinary tclsh. When TCLSH==2 then the +** startup script is supplied by an routine named "tclsh_main_loop()" +** that must be linked separately. The TCLSH==2 technique is used to +** generate stand-alone executables based on TCL, such as +** sqlite3_analyzer.exe. +*/ #ifdef TCLSH -/***************************************************************************** -** All of the code that follows is used to build standalone TCL interpreters -** that are statically linked with SQLite. Enable these by compiling -** with -DTCLSH=n where n can be 1 or 2. An n of 1 generates a standard -** tclsh but with SQLite built in. An n of 2 generates the SQLite space -** analysis program. -*/ - -#if defined(SQLITE_TEST) || defined(SQLITE_TCLMD5) -/* - * This code implements the MD5 message-digest algorithm. - * The algorithm is due to Ron Rivest. This code was - * written by Colin Plumb in 1993, no copyright is claimed. - * This code is in the public domain; do with it what you wish. - * - * Equivalent code is available from RSA Data Security, Inc. - * This code has been tested against that, and is equivalent, - * except that you don't need to include two pages of legalese - * with every copy. - * - * To compute the message digest of a chunk of bytes, declare an - * MD5Context structure, pass it to MD5Init, call MD5Update as - * needed on buffers full of bytes, and then call MD5Final, which - * will fill a supplied 16-byte array with the digest. - */ - -/* - * If compiled on a machine that doesn't have a 32-bit integer, - * you just set "uint32" to the appropriate datatype for an - * unsigned 32-bit integer. For example: - * - * cc -Duint32='unsigned long' md5.c - * - */ -#ifndef uint32 -# define uint32 unsigned int -#endif - -struct MD5Context { - int isInit; - uint32 buf[4]; - uint32 bits[2]; - unsigned char in[64]; -}; -typedef struct MD5Context MD5Context; - -/* - * Note: this code is harmless on little-endian machines. - */ -static void byteReverse (unsigned char *buf, unsigned longs){ - uint32 t; - do { - t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 | - ((unsigned)buf[1]<<8 | buf[0]); - *(uint32 *)buf = t; - buf += 4; - } while (--longs); -} -/* The four core functions - F1 is optimized somewhat */ - -/* #define F1(x, y, z) (x & y | ~x & z) */ -#define F1(x, y, z) (z ^ (x & (y ^ z))) -#define F2(x, y, z) F1(z, x, y) -#define F3(x, y, z) (x ^ y ^ z) -#define F4(x, y, z) (y ^ (x | ~z)) - -/* This is the central step in the MD5 algorithm. */ -#define MD5STEP(f, w, x, y, z, data, s) \ - ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) - -/* - * The core of the MD5 algorithm, this alters an existing MD5 hash to - * reflect the addition of 16 longwords of new data. MD5Update blocks - * the data and converts bytes into longwords for this routine. - */ -static void MD5Transform(uint32 buf[4], const uint32 in[16]){ - register uint32 a, b, c, d; - - a = buf[0]; - b = buf[1]; - c = buf[2]; - d = buf[3]; - - MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7); - MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12); - MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17); - MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22); - MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7); - MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12); - MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17); - MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22); - MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7); - MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12); - MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17); - MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22); - MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7); - MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12); - MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17); - MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22); - - MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5); - MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9); - MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14); - MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20); - MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5); - MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9); - MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14); - MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20); - MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5); - MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9); - MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14); - MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20); - MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5); - MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9); - MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14); - MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20); - - MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4); - MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11); - MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16); - MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23); - MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4); - MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11); - MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16); - MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23); - MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4); - MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11); - MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16); - MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23); - MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4); - MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11); - MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16); - MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23); - - MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6); - MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10); - MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15); - MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21); - MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6); - MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10); - MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15); - MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21); - MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6); - MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10); - MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15); - MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21); - MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6); - MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10); - MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15); - MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21); - - buf[0] += a; - buf[1] += b; - buf[2] += c; - buf[3] += d; -} - -/* - * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious - * initialization constants. - */ -static void MD5Init(MD5Context *ctx){ - ctx->isInit = 1; - ctx->buf[0] = 0x67452301; - ctx->buf[1] = 0xefcdab89; - ctx->buf[2] = 0x98badcfe; - ctx->buf[3] = 0x10325476; - ctx->bits[0] = 0; - ctx->bits[1] = 0; -} - -/* - * Update context to reflect the concatenation of another buffer full - * of bytes. - */ -static -void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ - uint32 t; - - /* Update bitcount */ - - t = ctx->bits[0]; - if ((ctx->bits[0] = t + ((uint32)len << 3)) < t) - ctx->bits[1]++; /* Carry from low to high */ - ctx->bits[1] += len >> 29; - - t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ - - /* Handle any leading odd-sized chunks */ - - if ( t ) { - unsigned char *p = (unsigned char *)ctx->in + t; - - t = 64-t; - if (len < t) { - memcpy(p, buf, len); - return; - } - memcpy(p, buf, t); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); - buf += t; - len -= t; - } - - /* Process data in 64-byte chunks */ - - while (len >= 64) { - memcpy(ctx->in, buf, 64); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); - buf += 64; - len -= 64; - } - - /* Handle any remaining bytes of data. */ - - memcpy(ctx->in, buf, len); -} - -/* - * Final wrapup - pad to 64-byte boundary with the bit pattern - * 1 0* (64-bit count of bits processed, MSB-first) - */ -static void MD5Final(unsigned char digest[16], MD5Context *ctx){ - unsigned count; - unsigned char *p; - - /* Compute number of bytes mod 64 */ - count = (ctx->bits[0] >> 3) & 0x3F; - - /* Set the first char of padding to 0x80. This is safe since there is - always at least one byte free */ - p = ctx->in + count; - *p++ = 0x80; - - /* Bytes of padding needed to make 64 bytes */ - count = 64 - 1 - count; - - /* Pad out to 56 mod 64 */ - if (count < 8) { - /* Two lots of padding: Pad the first block to 64 bytes */ - memset(p, 0, count); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); - - /* Now fill the next block with 56 bytes */ - memset(ctx->in, 0, 56); - } else { - /* Pad block to 56 bytes */ - memset(p, 0, count-8); - } - byteReverse(ctx->in, 14); - - /* Append length in bits and transform */ - memcpy(ctx->in + 14*4, ctx->bits, 8); - - MD5Transform(ctx->buf, (uint32 *)ctx->in); - byteReverse((unsigned char *)ctx->buf, 4); - memcpy(digest, ctx->buf, 16); -} - -/* -** Convert a 128-bit MD5 digest into a 32-digit base-16 number. -*/ -static void MD5DigestToBase16(unsigned char *digest, char *zBuf){ - static char const zEncode[] = "0123456789abcdef"; - int i, j; - - for(j=i=0; i<16; i++){ - int a = digest[i]; - zBuf[j++] = zEncode[(a>>4)&0xf]; - zBuf[j++] = zEncode[a & 0xf]; - } - zBuf[j] = 0; -} - - -/* -** Convert a 128-bit MD5 digest into sequency of eight 5-digit integers -** each representing 16 bits of the digest and separated from each -** other by a "-" character. -*/ -static void MD5DigestToBase10x8(unsigned char digest[16], char zDigest[50]){ - int i, j; - unsigned int x; - for(i=j=0; i<16; i+=2){ - x = digest[i]*256 + digest[i+1]; - if( i>0 ) zDigest[j++] = '-'; - sqlite3_snprintf(50-j, &zDigest[j], "%05u", x); - j += 5; - } - zDigest[j] = 0; -} - -/* -** A TCL command for md5. The argument is the text to be hashed. The -** Result is the hash in base64. -*/ -static int SQLITE_TCLAPI md5_cmd( - void*cd, - Tcl_Interp *interp, - int argc, - const char **argv -){ - MD5Context ctx; - unsigned char digest[16]; - char zBuf[50]; - void (*converter)(unsigned char*, char*); - - if( argc!=2 ){ - Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], - " TEXT\"", (char*)0); - return TCL_ERROR; - } - MD5Init(&ctx); - MD5Update(&ctx, (unsigned char*)argv[1], (unsigned)strlen(argv[1])); - MD5Final(digest, &ctx); - converter = (void(*)(unsigned char*,char*))cd; - converter(digest, zBuf); - Tcl_AppendResult(interp, zBuf, (char*)0); - return TCL_OK; -} - -/* -** A TCL command to take the md5 hash of a file. The argument is the -** name of the file. -*/ -static int SQLITE_TCLAPI md5file_cmd( - void*cd, - Tcl_Interp *interp, - int argc, - const char **argv -){ - FILE *in; - int ofst; - int amt; - MD5Context ctx; - void (*converter)(unsigned char*, char*); - unsigned char digest[16]; - char zBuf[10240]; - - if( argc!=2 && argc!=4 ){ - Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], - " FILENAME [OFFSET AMT]\"", (char*)0); - return TCL_ERROR; - } - if( argc==4 ){ - ofst = atoi(argv[2]); - amt = atoi(argv[3]); - }else{ - ofst = 0; - amt = 2147483647; - } - in = fopen(argv[1],"rb"); - if( in==0 ){ - Tcl_AppendResult(interp,"unable to open file \"", argv[1], - "\" for reading", (char*)0); - return TCL_ERROR; - } - fseek(in, ofst, SEEK_SET); - MD5Init(&ctx); - while( amt>0 ){ - int n; - n = (int)fread(zBuf, 1, sizeof(zBuf)<=amt ? sizeof(zBuf) : amt, in); - if( n<=0 ) break; - MD5Update(&ctx, (unsigned char*)zBuf, (unsigned)n); - amt -= n; - } - fclose(in); - MD5Final(digest, &ctx); - converter = (void(*)(unsigned char*,char*))cd; - converter(digest, zBuf); - Tcl_AppendResult(interp, zBuf, (char*)0); - return TCL_OK; -} - -/* -** Register the four new TCL commands for generating MD5 checksums -** with the TCL interpreter. -*/ -int Md5_Init(Tcl_Interp *interp){ - Tcl_CreateCommand(interp, "md5", (Tcl_CmdProc*)md5_cmd, - MD5DigestToBase16, 0); - Tcl_CreateCommand(interp, "md5-10x8", (Tcl_CmdProc*)md5_cmd, - MD5DigestToBase10x8, 0); - Tcl_CreateCommand(interp, "md5file", (Tcl_CmdProc*)md5file_cmd, - MD5DigestToBase16, 0); - Tcl_CreateCommand(interp, "md5file-10x8", (Tcl_CmdProc*)md5file_cmd, - MD5DigestToBase10x8, 0); - return TCL_OK; -} -#endif /* defined(SQLITE_TEST) || defined(SQLITE_TCLMD5) */ - -#if defined(SQLITE_TEST) -/* -** During testing, the special md5sum() aggregate function is available. -** inside SQLite. The following routines implement that function. -*/ -static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){ - MD5Context *p; - int i; - if( argc<1 ) return; - p = sqlite3_aggregate_context(context, sizeof(*p)); - if( p==0 ) return; - if( !p->isInit ){ - MD5Init(p); - } - for(i=0; i -#endif - #define TCLSH_MAIN main /* Needed to fake out mktclapp */ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ Tcl_Interp *interp; @@ -4271,17 +3690,6 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ } #endif - /* Since the primary use case for this binary is testing of SQLite, - ** be sure to generate core files if we crash */ -#if defined(SQLITE_TEST) && defined(unix) - { struct rlimit x; - getrlimit(RLIMIT_CORE, &x); - x.rlim_cur = x.rlim_max; - setrlimit(RLIMIT_CORE, &x); - } -#endif /* SQLITE_TEST && unix */ - - /* Call sqlite3_shutdown() once before doing anything else. This is to ** test that sqlite3_shutdown() can be safely called by a process before ** sqlite3_initialize() is. */ @@ -4295,7 +3703,23 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); #endif - init_all(interp); + /* Add extensions */ +#if !defined(SQLITE_TEST) + /* Normally we only initialize the TCL extension */ + Sqlite3_Init(interp); +#else + /* For testing, do lots of extra initialization */ + { + extern void sqlite3InitTclTestLogic(Tcl_Interp*); + sqlite3InitTclTestLogic(interp); + Tcl_CreateObjCommand( + interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0 + ); + Tcl_CreateObjCommand( + interp, "db_last_stmt_ptr", db_last_stmt_ptr, 0, 0 + ); + } +#endif /* SQLITE_TEST */ if( argc>=2 ){ int i; char zArgc[32]; diff --git a/src/test_md5.c b/src/test_md5.c new file mode 100644 index 0000000000..b670026861 --- /dev/null +++ b/src/test_md5.c @@ -0,0 +1,450 @@ +/* +** 2017-10-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code to implement an MD5 extension to TCL. +*/ +#include "sqlite3.h" +#include +#include +#include "sqlite3.h" +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif + +/* + * This code implements the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + */ + +/* + * If compiled on a machine that doesn't have a 32-bit integer, + * you just set "uint32" to the appropriate datatype for an + * unsigned 32-bit integer. For example: + * + * cc -Duint32='unsigned long' md5.c + * + */ +#ifndef uint32 +# define uint32 unsigned int +#endif + +struct MD5Context { + int isInit; + uint32 buf[4]; + uint32 bits[2]; + unsigned char in[64]; +}; +typedef struct MD5Context MD5Context; + +/* + * Note: this code is harmless on little-endian machines. + */ +static void byteReverse (unsigned char *buf, unsigned longs){ + uint32 t; + do { + t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 | + ((unsigned)buf[1]<<8 | buf[0]); + *(uint32 *)buf = t; + buf += 4; + } while (--longs); +} +/* The four core functions - F1 is optimized somewhat */ + +/* #define F1(x, y, z) (x & y | ~x & z) */ +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +/* This is the central step in the MD5 algorithm. */ +#define MD5STEP(f, w, x, y, z, data, s) \ + ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) + +/* + * The core of the MD5 algorithm, this alters an existing MD5 hash to + * reflect the addition of 16 longwords of new data. MD5Update blocks + * the data and converts bytes into longwords for this routine. + */ +static void MD5Transform(uint32 buf[4], const uint32 in[16]){ + register uint32 a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +/* + * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious + * initialization constants. + */ +static void MD5Init(MD5Context *ctx){ + ctx->isInit = 1; + ctx->buf[0] = 0x67452301; + ctx->buf[1] = 0xefcdab89; + ctx->buf[2] = 0x98badcfe; + ctx->buf[3] = 0x10325476; + ctx->bits[0] = 0; + ctx->bits[1] = 0; +} + +/* + * Update context to reflect the concatenation of another buffer full + * of bytes. + */ +static +void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ + uint32 t; + + /* Update bitcount */ + + t = ctx->bits[0]; + if ((ctx->bits[0] = t + ((uint32)len << 3)) < t) + ctx->bits[1]++; /* Carry from low to high */ + ctx->bits[1] += len >> 29; + + t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ + + /* Handle any leading odd-sized chunks */ + + if ( t ) { + unsigned char *p = (unsigned char *)ctx->in + t; + + t = 64-t; + if (len < t) { + memcpy(p, buf, len); + return; + } + memcpy(p, buf, t); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->in); + buf += t; + len -= t; + } + + /* Process data in 64-byte chunks */ + + while (len >= 64) { + memcpy(ctx->in, buf, 64); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + + memcpy(ctx->in, buf, len); +} + +/* + * Final wrapup - pad to 64-byte boundary with the bit pattern + * 1 0* (64-bit count of bits processed, MSB-first) + */ +static void MD5Final(unsigned char digest[16], MD5Context *ctx){ + unsigned count; + unsigned char *p; + + /* Compute number of bytes mod 64 */ + count = (ctx->bits[0] >> 3) & 0x3F; + + /* Set the first char of padding to 0x80. This is safe since there is + always at least one byte free */ + p = ctx->in + count; + *p++ = 0x80; + + /* Bytes of padding needed to make 64 bytes */ + count = 64 - 1 - count; + + /* Pad out to 56 mod 64 */ + if (count < 8) { + /* Two lots of padding: Pad the first block to 64 bytes */ + memset(p, 0, count); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->in); + + /* Now fill the next block with 56 bytes */ + memset(ctx->in, 0, 56); + } else { + /* Pad block to 56 bytes */ + memset(p, 0, count-8); + } + byteReverse(ctx->in, 14); + + /* Append length in bits and transform */ + memcpy(ctx->in + 14*4, ctx->bits, 8); + + MD5Transform(ctx->buf, (uint32 *)ctx->in); + byteReverse((unsigned char *)ctx->buf, 4); + memcpy(digest, ctx->buf, 16); +} + +/* +** Convert a 128-bit MD5 digest into a 32-digit base-16 number. +*/ +static void MD5DigestToBase16(unsigned char *digest, char *zBuf){ + static char const zEncode[] = "0123456789abcdef"; + int i, j; + + for(j=i=0; i<16; i++){ + int a = digest[i]; + zBuf[j++] = zEncode[(a>>4)&0xf]; + zBuf[j++] = zEncode[a & 0xf]; + } + zBuf[j] = 0; +} + + +/* +** Convert a 128-bit MD5 digest into sequency of eight 5-digit integers +** each representing 16 bits of the digest and separated from each +** other by a "-" character. +*/ +static void MD5DigestToBase10x8(unsigned char digest[16], char zDigest[50]){ + int i, j; + unsigned int x; + for(i=j=0; i<16; i+=2){ + x = digest[i]*256 + digest[i+1]; + if( i>0 ) zDigest[j++] = '-'; + sqlite3_snprintf(50-j, &zDigest[j], "%05u", x); + j += 5; + } + zDigest[j] = 0; +} + +/* +** A TCL command for md5. The argument is the text to be hashed. The +** Result is the hash in base64. +*/ +static int SQLITE_TCLAPI md5_cmd( + void*cd, + Tcl_Interp *interp, + int argc, + const char **argv +){ + MD5Context ctx; + unsigned char digest[16]; + char zBuf[50]; + void (*converter)(unsigned char*, char*); + + if( argc!=2 ){ + Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], + " TEXT\"", (char*)0); + return TCL_ERROR; + } + MD5Init(&ctx); + MD5Update(&ctx, (unsigned char*)argv[1], (unsigned)strlen(argv[1])); + MD5Final(digest, &ctx); + converter = (void(*)(unsigned char*,char*))cd; + converter(digest, zBuf); + Tcl_AppendResult(interp, zBuf, (char*)0); + return TCL_OK; +} + +/* +** A TCL command to take the md5 hash of a file. The argument is the +** name of the file. +*/ +static int SQLITE_TCLAPI md5file_cmd( + void*cd, + Tcl_Interp *interp, + int argc, + const char **argv +){ + FILE *in; + int ofst; + int amt; + MD5Context ctx; + void (*converter)(unsigned char*, char*); + unsigned char digest[16]; + char zBuf[10240]; + + if( argc!=2 && argc!=4 ){ + Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], + " FILENAME [OFFSET AMT]\"", (char*)0); + return TCL_ERROR; + } + if( argc==4 ){ + ofst = atoi(argv[2]); + amt = atoi(argv[3]); + }else{ + ofst = 0; + amt = 2147483647; + } + in = fopen(argv[1],"rb"); + if( in==0 ){ + Tcl_AppendResult(interp,"unable to open file \"", argv[1], + "\" for reading", (char*)0); + return TCL_ERROR; + } + fseek(in, ofst, SEEK_SET); + MD5Init(&ctx); + while( amt>0 ){ + int n; + n = (int)fread(zBuf, 1, sizeof(zBuf)<=amt ? sizeof(zBuf) : amt, in); + if( n<=0 ) break; + MD5Update(&ctx, (unsigned char*)zBuf, (unsigned)n); + amt -= n; + } + fclose(in); + MD5Final(digest, &ctx); + converter = (void(*)(unsigned char*,char*))cd; + converter(digest, zBuf); + Tcl_AppendResult(interp, zBuf, (char*)0); + return TCL_OK; +} + +/* +** Register the four new TCL commands for generating MD5 checksums +** with the TCL interpreter. +*/ +int Md5_Init(Tcl_Interp *interp){ + Tcl_CreateCommand(interp, "md5", (Tcl_CmdProc*)md5_cmd, + MD5DigestToBase16, 0); + Tcl_CreateCommand(interp, "md5-10x8", (Tcl_CmdProc*)md5_cmd, + MD5DigestToBase10x8, 0); + Tcl_CreateCommand(interp, "md5file", (Tcl_CmdProc*)md5file_cmd, + MD5DigestToBase16, 0); + Tcl_CreateCommand(interp, "md5file-10x8", (Tcl_CmdProc*)md5file_cmd, + MD5DigestToBase10x8, 0); + return TCL_OK; +} + +/* +** During testing, the special md5sum() aggregate function is available. +** inside SQLite. The following routines implement that function. +*/ +static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){ + MD5Context *p; + int i; + if( argc<1 ) return; + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + MD5Init(p); + } + for(i=0; i +#endif + +/* Forward declaration */ +static int SQLITE_TCLAPI load_testfixture_extensions( + ClientData cd, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +); + +/* +** This routine is the primary export of this file. +** +** Configure the interpreter passed as the first argument to have access +** to the commands and linked variables that make up: +** +** * the [sqlite3] extension itself, +** +** * If SQLITE_TCLMD5 or SQLITE_TEST is defined, the Md5 commands, and +** +** * If SQLITE_TEST is set, the various test interfaces used by the Tcl +** test suite. +*/ +void sqlite3InitTclTestLogic(Tcl_Interp *interp){ + /* Since the primary use case for this binary is testing of SQLite, + ** be sure to generate core files if we crash */ +#if defined(unix) + { struct rlimit x; + getrlimit(RLIMIT_CORE, &x); + x.rlim_cur = x.rlim_max; + setrlimit(RLIMIT_CORE, &x); + } +#endif /* unix */ + + { + extern int Sqlite3_Init(Tcl_Interp*); + extern int Sqliteconfig_Init(Tcl_Interp*); + extern int Sqlitetest1_Init(Tcl_Interp*); + extern int Sqlitetest2_Init(Tcl_Interp*); + extern int Sqlitetest3_Init(Tcl_Interp*); + extern int Sqlitetest4_Init(Tcl_Interp*); + extern int Sqlitetest5_Init(Tcl_Interp*); + extern int Sqlitetest6_Init(Tcl_Interp*); + extern int Sqlitetest7_Init(Tcl_Interp*); + extern int Sqlitetest8_Init(Tcl_Interp*); + extern int Sqlitetest9_Init(Tcl_Interp*); + extern int Sqlitetestasync_Init(Tcl_Interp*); + extern int Sqlitetest_autoext_Init(Tcl_Interp*); + extern int Sqlitetest_blob_Init(Tcl_Interp*); + extern int Sqlitetest_demovfs_Init(Tcl_Interp *); + extern int Sqlitetest_func_Init(Tcl_Interp*); + extern int Sqlitetest_hexio_Init(Tcl_Interp*); + extern int Sqlitetest_init_Init(Tcl_Interp*); + extern int Sqlitetest_malloc_Init(Tcl_Interp*); + extern int Sqlitetest_mutex_Init(Tcl_Interp*); + extern int Sqlitetestschema_Init(Tcl_Interp*); + extern int Sqlitetestsse_Init(Tcl_Interp*); + extern int Sqlitetesttclvar_Init(Tcl_Interp*); + extern int Sqlitetestfs_Init(Tcl_Interp*); + extern int SqlitetestThread_Init(Tcl_Interp*); + extern int SqlitetestOnefile_Init(); + extern int SqlitetestOsinst_Init(Tcl_Interp*); + extern int Sqlitetestbackup_Init(Tcl_Interp*); + extern int Sqlitetestintarray_Init(Tcl_Interp*); + extern int Sqlitetestvfs_Init(Tcl_Interp *); + extern int Sqlitetestrtree_Init(Tcl_Interp*); + extern int Sqlitequota_Init(Tcl_Interp*); + extern int Sqlitemultiplex_Init(Tcl_Interp*); + extern int SqliteSuperlock_Init(Tcl_Interp*); + extern int SqlitetestSyscall_Init(Tcl_Interp*); +#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK) + extern int TestSession_Init(Tcl_Interp*); +#endif + extern int Md5_Init(Tcl_Interp*); + extern int Fts5tcl_Init(Tcl_Interp *); + extern int SqliteRbu_Init(Tcl_Interp*); + extern int Sqlitetesttcl_Init(Tcl_Interp*); +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + extern int Sqlitetestfts3_Init(Tcl_Interp *interp); +#endif +#ifdef SQLITE_ENABLE_ZIPVFS + extern int Zipvfs_Init(Tcl_Interp*); +#endif + + Sqlite3_Init(interp); +#ifdef SQLITE_ENABLE_ZIPVFS + Zipvfs_Init(interp); +#endif + Md5_Init(interp); + Sqliteconfig_Init(interp); + Sqlitetest1_Init(interp); + Sqlitetest2_Init(interp); + Sqlitetest3_Init(interp); + Sqlitetest4_Init(interp); + Sqlitetest5_Init(interp); + Sqlitetest6_Init(interp); + Sqlitetest7_Init(interp); + Sqlitetest8_Init(interp); + Sqlitetest9_Init(interp); + Sqlitetestasync_Init(interp); + Sqlitetest_autoext_Init(interp); + Sqlitetest_blob_Init(interp); + Sqlitetest_demovfs_Init(interp); + Sqlitetest_func_Init(interp); + Sqlitetest_hexio_Init(interp); + Sqlitetest_init_Init(interp); + Sqlitetest_malloc_Init(interp); + Sqlitetest_mutex_Init(interp); + Sqlitetestschema_Init(interp); + Sqlitetesttclvar_Init(interp); + Sqlitetestfs_Init(interp); + SqlitetestThread_Init(interp); + SqlitetestOnefile_Init(); + SqlitetestOsinst_Init(interp); + Sqlitetestbackup_Init(interp); + Sqlitetestintarray_Init(interp); + Sqlitetestvfs_Init(interp); + Sqlitetestrtree_Init(interp); + Sqlitequota_Init(interp); + Sqlitemultiplex_Init(interp); + SqliteSuperlock_Init(interp); + SqlitetestSyscall_Init(interp); +#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK) + TestSession_Init(interp); +#endif + Fts5tcl_Init(interp); + SqliteRbu_Init(interp); + Sqlitetesttcl_Init(interp); + +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + Sqlitetestfts3_Init(interp); +#endif + + Tcl_CreateObjCommand( + interp, "load_testfixture_extensions", load_testfixture_extensions,0,0 + ); + } +} + +/* tclcmd: load_testfixture_extensions +*/ +static int SQLITE_TCLAPI load_testfixture_extensions( + ClientData cd, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + + Tcl_Interp *slave; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SLAVE"); + return TCL_ERROR; + } + + slave = Tcl_GetSlave(interp, Tcl_GetString(objv[1])); + if( !slave ){ + return TCL_ERROR; + } + + sqlite3InitTclTestLogic(slave); + return TCL_OK; +} From 1df6470d359e9e9cc4eacbd1932dce5354b8b18a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Oct 2017 15:56:26 +0000 Subject: [PATCH 10/22] When compiling with SQLITE_TEST, extra options are available on the "DB version" command for controlling the internal state of the SqliteDb object, for testing purposes. FossilOrigin-Name: bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896 --- Makefile.in | 1 + Makefile.msc | 1 + main.mk | 1 + manifest | 23 ++++---- manifest.uuid | 2 +- src/tclsqlite.c | 126 +++++++++++++------------------------------ test/scanstatus.test | 4 +- 7 files changed, 52 insertions(+), 106 deletions(-) diff --git a/Makefile.in b/Makefile.in index 8c306629e0..d474222fb8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1119,6 +1119,7 @@ TESTFIXTURE_FLAGS += -DBUILD_sqlite TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la TESTFIXTURE_SRC1 = sqlite3.c diff --git a/Makefile.msc b/Makefile.msc index e4a72e872b..0611d8a8b3 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2111,6 +2111,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN) TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) diff --git a/main.mk b/main.mk index 84d1a3eb76..bdbb56e229 100644 --- a/main.mk +++ b/main.mk @@ -803,6 +803,7 @@ TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ diff --git a/manifest b/manifest index 250e0c8001..74a206741c 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Move\ssome\stest\slogic\sout\sof\stclsqlite.c\sand\sinto\sauxiliary\stest_*.c\sfiles.\nThis\sis\sa\swork\sin\sprogress. -D 2017-10-13T15:06:06.783 -F Makefile.in 8658943abe79d10adfed7b504936199ebcd5c1d372d176936d5edd075492b13b +C When\scompiling\swith\sSQLITE_TEST,\sextra\soptions\sare\savailable\son\sthe\n"DB\sversion"\scommand\sfor\scontrolling\sthe\sinternal\sstate\sof\sthe\sSqliteDb\nobject,\sfor\stesting\spurposes. +D 2017-10-13T15:56:26.514 +F Makefile.in d9acf9cc5252f406c2a417c12475ef14fe49096d121bd025a230fe3b58e0f294 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 22fc2be4ac777553096de774b9b601568fa21904d1d64f595916f78957025558 +F Makefile.msc c20c8f2945955931d0f790104b335b37ba511eeb0d75eded23b7f878077ee1f5 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 03a3ae65678b238a787f0aa9162e0e2bd998561bc8d66b9b073f54d9025dee61 +F main.mk 9311f1d83efd6818b7fbeb1256dbc5377159359528be4e7d925345e4d80446bf F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -470,7 +470,7 @@ F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53b F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 -F src/tclsqlite.c 93b2267ade2bfaad9d4f0135e5928ee28fdd77a355a563ad444ead13e1c488b2 +F src/tclsqlite.c 7ffde8e2cd556ed74b5d40ae0ee924aa974ee3b3dacd5114c5c98feb9f651ac4 F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b @@ -1157,7 +1157,7 @@ F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 -F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285 +F test/scanstatus.test 1ba0e2ee25dcd1d55ec770803b19832cffaecbf0b15d376807759ebeed3669b0 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1663,10 +1663,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a -R 225e4cd689e0c61d406db3daa52debdc -T *branch * tclsqlite-cleanup -T *sym-tclsqlite-cleanup * -T -sym-trunk * +P 95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf +R beaa5ad001aa19fb6dcaa6d5d2766f78 U drh -Z 604c544fbf5b9e259c249448beb617c7 +Z 978089b1abbc5f3c189d1cdd601af59d diff --git a/manifest.uuid b/manifest.uuid index 705f05a85e..ddda81051f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf \ No newline at end of file +bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index a66fe6832e..0009eab695 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3286,7 +3286,42 @@ static int SQLITE_TCLAPI DbObjCmd( ** Return the version string for this database. */ case DB_VERSION: { - Tcl_SetResult(interp, (char *)sqlite3_libversion(), TCL_STATIC); + int i; + for(i=2; ibLegacyPrepare) ){ + return TCL_ERROR; + } + }else + + /* $db version -last-stmt-ptr + ** + ** Return a string which is a hex encoding of the pointer to the + ** most recent sqlite3_stmt in the statement cache. + */ + if( strcmp(zArg, "-last-stmt-ptr")==0 ){ + char zBuf[100]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "%p", + pDb->stmtList ? pDb->stmtList->pStmt: 0); + Tcl_SetResult(interp, zBuf, TCL_VOLATILE); + }else +#endif /* SQLITE_TEST */ + { + Tcl_AppendResult(interp, "unknown argument: ", zArg, (char*)0); + return TCL_ERROR; + } + } + if( i==2 ){ + Tcl_SetResult(interp, (char *)sqlite3_libversion(), TCL_STATIC); + } break; } @@ -3594,89 +3629,6 @@ static const char *tclsh_main_loop(void){ static const char *tclsh_main_loop(void); #endif -/* The following to TCL commands used for testing must appear in this -** file (they cannot be factored out into one of the test_*.c files where -** they belong) because they require access to the SqliteDb object. -*/ -#ifdef SQLITE_TEST -/* -** Tclcmd: db_use_legacy_prepare DB BOOLEAN -** -** The first argument to this command must be a database command created by -** [sqlite3]. If the second argument is true, then the handle is configured -** to use the sqlite3_prepare_v2() function to prepare statements. If it -** is false, sqlite3_prepare(). -*/ -static int SQLITE_TCLAPI db_use_legacy_prepare_cmd( - ClientData cd, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - Tcl_CmdInfo cmdInfo; - SqliteDb *pDb; - int bPrepare; - - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB BOOLEAN"); - return TCL_ERROR; - } - - if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ - Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0); - return TCL_ERROR; - } - pDb = (SqliteDb*)cmdInfo.objClientData; - if( Tcl_GetBooleanFromObj(interp, objv[2], &bPrepare) ){ - return TCL_ERROR; - } - - pDb->bLegacyPrepare = bPrepare; - - Tcl_ResetResult(interp); - return TCL_OK; -} - -/* -** Tclcmd: db_last_stmt_ptr DB -** -** If the statement cache associated with database DB is not empty, -** return the text representation of the most recently used statement -** handle. -*/ -static int SQLITE_TCLAPI db_last_stmt_ptr( - ClientData cd, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); - Tcl_CmdInfo cmdInfo; - SqliteDb *pDb; - sqlite3_stmt *pStmt = 0; - char zBuf[100]; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - - if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ - Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0); - return TCL_ERROR; - } - pDb = (SqliteDb*)cmdInfo.objClientData; - - if( pDb->stmtList ) pStmt = pDb->stmtList->pStmt; - if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ){ - return TCL_ERROR; - } - Tcl_SetResult(interp, zBuf, TCL_VOLATILE); - - return TCL_OK; -} -#endif /* SQLITE_TEST */ - #define TCLSH_MAIN main /* Needed to fake out mktclapp */ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ Tcl_Interp *interp; @@ -3712,12 +3664,6 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ { extern void sqlite3InitTclTestLogic(Tcl_Interp*); sqlite3InitTclTestLogic(interp); - Tcl_CreateObjCommand( - interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0 - ); - Tcl_CreateObjCommand( - interp, "db_last_stmt_ptr", db_last_stmt_ptr, 0, 0 - ); } #endif /* SQLITE_TEST */ if( argc>=2 ){ diff --git a/test/scanstatus.test b/test/scanstatus.test index ed24d97437..fdd6476ae9 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -30,7 +30,7 @@ do_execsql_test 1.0 { } proc do_scanstatus_test {tn res} { - set stmt [db_last_stmt_ptr db] + set stmt [db version -last-stmt-ptr] set idx 0 set ret [list] while {1} { @@ -79,7 +79,7 @@ do_scanstatus_test 1.9 { } do_test 1.9 { - sqlite3_stmt_scanstatus_reset [db_last_stmt_ptr db] + sqlite3_stmt_scanstatus_reset [db version -last-stmt-ptr] } {} do_scanstatus_test 1.10 { From 8544d43e4ae349ab04aea9fe6c075627abaafeeb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Oct 2017 16:01:36 +0000 Subject: [PATCH 11/22] The db_use_legacy_prepare command no longer exists. Use the undocumented -use-legacy-prepare option to DB version instead. FossilOrigin-Name: ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/permutations.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 74a206741c..cc4d9453b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\scompiling\swith\sSQLITE_TEST,\sextra\soptions\sare\savailable\son\sthe\n"DB\sversion"\scommand\sfor\scontrolling\sthe\sinternal\sstate\sof\sthe\sSqliteDb\nobject,\sfor\stesting\spurposes. -D 2017-10-13T15:56:26.514 +C The\sdb_use_legacy_prepare\scommand\sno\slonger\sexists.\s\sUse\sthe\sundocumented\n-use-legacy-prepare\soption\sto\sDB\sversion\sinstead. +D 2017-10-13T16:01:36.071 F Makefile.in d9acf9cc5252f406c2a417c12475ef14fe49096d121bd025a230fe3b58e0f294 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc c20c8f2945955931d0f790104b335b37ba511eeb0d75eded23b7f878077ee1f5 @@ -1105,7 +1105,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test d911c9ba49088d22054a05dc73743f677872a92ac89288bcdeafa0ebf3f9c531 +F test/permutations.test 53fb87f0a345259f884fa8b5fc62ee327b20be378ebe5149833b1b28c58a42f6 F test/pragma.test c31b5e98998c160a4c85b1e04f590655c67f2daa7f73854640cd120610e3ac15 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1663,7 +1663,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf -R beaa5ad001aa19fb6dcaa6d5d2766f78 +P bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896 +R ef5c1b585866b0d225763ce3d0edba7c U drh -Z 978089b1abbc5f3c189d1cdd601af59d +Z e1aad8bb9151be888a26c10aa66d2e50 diff --git a/manifest.uuid b/manifest.uuid index ddda81051f..5b87ac07a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896 \ No newline at end of file +ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 5afc51cb7d..1e04265668 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -1065,7 +1065,7 @@ test_suite "no_optimization" -description { test_suite "prepare" -description { Run tests with the db connection using sqlite3_prepare() instead of _v2(). } -dbconfig { - db_use_legacy_prepare $::dbhandle 1 + $::dbhandle version -use-legacy-prepare 1 #$::dbhandle cache size 0 } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ From 903b23022d6d39fbc18d619dfdead39277bed72d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Oct 2017 18:58:55 +0000 Subject: [PATCH 12/22] Add the tcl/mkccode.tcl script used to construct a single C-language source fiel for programs that combine C-code, SQLite, and TCL. Use this script to construct the sqlite3_analyzer program. FossilOrigin-Name: 298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82 --- Makefile.in | 10 +- Makefile.msc | 10 +- main.mk | 10 +- manifest | 20 +- manifest.uuid | 2 +- tool/mkccode.tcl | 93 ++++ tool/sqlite3_analyzer.c.in | 903 +++++++++++++++++++++++++++++++++++++ tool/tostr.tcl | 12 - 8 files changed, 1013 insertions(+), 47 deletions(-) create mode 100755 tool/mkccode.tcl create mode 100644 tool/sqlite3_analyzer.c.in delete mode 100644 tool/tostr.tcl diff --git a/Makefile.in b/Makefile.in index d474222fb8..e2934d80b1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1180,14 +1180,8 @@ valgrindtest: $(TESTPROGS) valgrindfuzz smoketest: $(TESTPROGS) fuzzcheck$(TEXE) ./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS) -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl - echo "#define TCLSH 2" > $@ - echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ - cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@ - echo "static const char *tclsh_main_loop(void){" >> $@ - echo "static const char *zMainloop = " >> $@ - $(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@ - echo "; return zMainloop; }" >> $@ +sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in + $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index 0611d8a8b3..bfd368c558 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2191,14 +2191,8 @@ smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP) - echo #define TCLSH 2 > $@ - echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ - copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ - echo static const char *tclsh_main_loop(void){ >> $@ - echo static const char *zMainloop = >> $@ - $(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@ - echo ; return zMainloop; } >> $@ +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP) + $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ diff --git a/main.mk b/main.mk index bdbb56e229..18d7cd3459 100644 --- a/main.mk +++ b/main.mk @@ -780,14 +780,8 @@ tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a $(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \ $(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB) -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl - echo "#define TCLSH 2" > $@ - echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ - cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@ - echo "static const char *tclsh_main_loop(void){" >> $@ - echo "static const char *zMainloop = " >> $@ - tclsh $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@ - echo "; return zMainloop; }" >> $@ +sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/sqlite3_analyzer.c.in $(TOP)/tool/mkccode.tcl + tclsh $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c sqlite3_analyzer$(EXE): sqlite3_analyzer.c $(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) diff --git a/manifest b/manifest index c45a7ce09e..669c1b284a 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Move\sa\sbunch\sof\sunrelated\stest\scode\sout\sof\stclsqlite.c\sand\sinto\ntest_*.c\sfiles.\s\sThere\sis\sstill\ssome\stest\scode\sin\stclsqlite.c,\sbut\sthe\samount\nis\sgreatly\sreduced. -D 2017-10-13T16:19:48.503 -F Makefile.in d9acf9cc5252f406c2a417c12475ef14fe49096d121bd025a230fe3b58e0f294 +C Add\sthe\stcl/mkccode.tcl\sscript\sused\sto\sconstruct\sa\ssingle\sC-language\ssource\nfiel\sfor\sprograms\sthat\scombine\sC-code,\sSQLite,\sand\sTCL.\s\sUse\sthis\sscript\sto\nconstruct\sthe\ssqlite3_analyzer\sprogram. +D 2017-10-13T18:58:55.897 +F Makefile.in a99044d778e2a83ba6cf78b88937d3d667cee425dca5915d73eeba1d56b98082 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc c20c8f2945955931d0f790104b335b37ba511eeb0d75eded23b7f878077ee1f5 +F Makefile.msc b864c4f906031a37e425466d8214df2da2f01703956dad5290156f03548d7277 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 9311f1d83efd6818b7fbeb1256dbc5377159359528be4e7d925345e4d80446bf +F main.mk a1948e0509f143fe4395a449cf61039bb6c7db3a93ae6af65092a72556718026 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1593,6 +1593,7 @@ F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7 +F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3 F tool/mkkeywordhash.c 2e852ac0dfdc5af18886dc1ce7e9676d11714ae3df0a282dc7d90b3a0fe2033c F tool/mkmsvcmin.tcl cbd93f1cfa3a0a9ae56fc958510aa3fc3ac65e29cb111716199e3d0e66eefaa4 @@ -1631,11 +1632,11 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb +F tool/sqlite3_analyzer.c.in 3d53f06e04619f43b7aebc01cc318ef215bfd09ea1947fe9e485bed6aa6cb4b9 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh c5a617b8c61a0926747a56c65f5671ef8ac0e148 -F tool/tostr.tcl 96022f35ada2194f6f8ccf6fd95809e90ed277c4 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f @@ -1663,8 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b -R ef5c1b585866b0d225763ce3d0edba7c -T +closed ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b +P a9c4bc88fcf985a0bea14ed5381239cfb697886287998da04a10230b6858ab5d +R 78f34b33921be1c7a7c8587f43b33b81 U drh -Z 0c47718eb3eab9783f8c2bca1833cb14 +Z 180ea09ee34fefc9e97e7461c13719ef diff --git a/manifest.uuid b/manifest.uuid index 05172a0f2b..fabaeec3c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9c4bc88fcf985a0bea14ed5381239cfb697886287998da04a10230b6858ab5d \ No newline at end of file +298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82 \ No newline at end of file diff --git a/tool/mkccode.tcl b/tool/mkccode.tcl new file mode 100755 index 0000000000..41b09f1e81 --- /dev/null +++ b/tool/mkccode.tcl @@ -0,0 +1,93 @@ +#!/usr/bin/tclsh +# +# Use this script to build C-language source code for a program that uses +# tclsqlite.c together with custom TCL scripts and/or C extensions for +# either SQLite or TCL. +# +# Usage example: +# +# tclsh mktclsqliteprog.tcl demoapp.c.in >demoapp.c +# +# The demoapp.c.in file contains a mixture of C code, TCL script, and +# processing directives used by mktclsqliteprog.tcl to build the final C-code +# output file. Most lines of demoapp.c.in are copied straight through into +# the output. The following control directives are recognized: +# +# BEGIN_STRING +# +# This marks the beginning of large string literal - usually a TCL +# script of some kind. Subsequent lines of text through the first +# line that begins with END_STRING are converted into a C-language +# string literal. +# +# INCLUDE path +# +# The path argument is the name of a file to be inserted in place of +# the INCLUDE line. The path can begin with $ROOT to signify the +# root of the SQLite source tree, or $HOME to signify the directory +# that contains the demoapp.c.in input script itself. If the path does +# not begin with either $ROOT or $HOME, then it is interpreted relative +# to the current working directory. +# +# If the INCLUDE occurs in the middle of BEGIN_STRING...END_STRING +# then all of the text in the input file is converted into C-language +# string literals. +# +# None of the control directives described above will nest. Only the +# top-level input file ("demoapp.c.in" in the example) is interpreted. +# referenced files are copied verbatim. +# +if {[llength $argv]!=1} { + puts stderr "Usage: $argv0 TEMPLATE >OUTPUT" + exit 1 +} +set infile [lindex $argv 0] +set ROOT [file normalize [file dir $argv0]/..] +set HOME [file normalize [file dir $infile]] +set in [open $infile rb] +puts [subst {/* DO NOT EDIT +** +** This file was generated by \"$argv0 $infile\". +** To make changes, edit $infile then rerun the generator +** command. +*/}] +set instr 0 +while {1} { + set line [gets $in] + if {[eof $in]} break + if {[regexp {^INCLUDE (.*)} $line all path]} { + regsub {^\$ROOT\y} $path $ROOT path + regsub {^\$HOME\y} $path $HOME path + set in2 [open $path rb] + puts "/* INCLUDE $path */" + if {$instr} { + while {1} { + set line [gets $in2] + if {[eof $in2]} break + set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line] + puts "\"$x\\n\"" + } + } else { + puts [read $in2] + } + puts "/* END $path */" + close $in2 + continue + } + if {[regexp {^BEGIN_STRING} $line]} { + set instr 1 + puts "/* BEGIN_STRING */" + continue + } + if {[regexp {^END_STRING} $line]} { + set instr 0 + puts "/* END_STRING */" + continue + } + if {$instr} { + set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line] + puts "\"$x\\n\"" + } else { + puts $line + } +} diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in new file mode 100644 index 0000000000..54f323a427 --- /dev/null +++ b/tool/sqlite3_analyzer.c.in @@ -0,0 +1,903 @@ +/* +** Read an SQLite database file and analyze its space utilization. Generate +** text on standard output. +*/ +#define TCLSH 2 +#define SQLITE_ENABLE_DBSTAT_VTAB 1 +#undef SQLITE_THREADSAFE +#define SQLITE_THREADSAFE 0 +#undef SQLITE_ENABLE_COLUMN_METADATA +#define SQLITE_OMIT_DECLTYPE 1 +#define SQLITE_OMIT_DEPRECATED 1 +#define SQLITE_OMIT_PROGRESS_CALLBACK 1 +#define SQLITE_OMIT_SHARED_CACHE 1 +#define SQLITE_DEFAULT_MEMSTATUS 0 +#define SQLITE_MAX_EXPR_DEPTH 0 +#define SQLITE_OMIT_LOAD_EXTENSION 1 +INCLUDE sqlite3.c +INCLUDE $ROOT/src/tclsqlite.c + +static const char *tclsh_main_loop(void){ +return +BEGIN_STRING +# Run this TCL script using "testfixture" in order get a report that shows +# how much disk space is used by a particular data to actually store data +# versus how much space is unused. +# + +if {[catch { + +# Argument $tname is the name of a table within the database opened by +# database handle [db]. Return true if it is a WITHOUT ROWID table, or +# false otherwise. +# +proc is_without_rowid {tname} { + set t [string map {' ''} $tname] + db eval "PRAGMA index_list = '$t'" o { + if {$o(origin) == "pk"} { + set n $o(name) + if {0==[db one { SELECT count(*) FROM sqlite_master WHERE name=$n }]} { + return 1 + } + } + } + return 0 +} + +# Read and run TCL commands from standard input. Used to implement +# the --tclsh option. +# +proc tclsh {} { + set line {} + while {![eof stdin]} { + if {$line!=""} { + puts -nonewline "> " + } else { + puts -nonewline "% " + } + flush stdout + append line [gets stdin] + if {[info complete $line]} { + if {[catch {uplevel #0 $line} result]} { + puts stderr "Error: $result" + } elseif {$result!=""} { + puts $result + } + set line {} + } else { + append line \n + } + } +} + + +# Get the name of the database to analyze +# +proc usage {} { + set argv0 [file rootname [file tail [info nameofexecutable]]] + puts stderr "Usage: $argv0 ?--pageinfo? ?--stats? database-filename" + puts stderr { +Analyze the SQLite3 database file specified by the "database-filename" +argument and output a report detailing size and storage efficiency +information for the database and its constituent tables and indexes. + +Options: + + --pageinfo Show how each page of the database-file is used + + --stats Output SQL text that creates a new database containing + statistics about the database that was analyzed + + --tclsh Run the built-in TCL interpreter interactively (for debugging) + + --version Show the version number of SQLite +} + exit 1 +} +set file_to_analyze {} +set flags(-pageinfo) 0 +set flags(-stats) 0 +set flags(-debug) 0 +append argv {} +foreach arg $argv { + if {[regexp {^-+pageinfo$} $arg]} { + set flags(-pageinfo) 1 + } elseif {[regexp {^-+stats$} $arg]} { + set flags(-stats) 1 + } elseif {[regexp {^-+debug$} $arg]} { + set flags(-debug) 1 + } elseif {[regexp {^-+tclsh$} $arg]} { + tclsh + exit 0 + } elseif {[regexp {^-+version$} $arg]} { + sqlite3 mem :memory: + puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] + mem close + exit 0 + } elseif {[regexp {^-} $arg]} { + puts stderr "Unknown option: $arg" + usage + } elseif {$file_to_analyze!=""} { + usage + } else { + set file_to_analyze $arg + } +} +if {$file_to_analyze==""} usage +set root_filename $file_to_analyze +regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename +if {![file exists $root_filename]} { + puts stderr "No such file: $root_filename" + exit 1 +} +if {![file readable $root_filename]} { + puts stderr "File is not readable: $root_filename" + exit 1 +} +set true_file_size [file size $root_filename] +if {$true_file_size<512} { + puts stderr "Empty or malformed database: $root_filename" + exit 1 +} + +# Compute the total file size assuming test_multiplexor is being used. +# Assume that SQLITE_ENABLE_8_3_NAMES might be enabled +# +set extension [file extension $root_filename] +set pattern $root_filename +append pattern {[0-3][0-9][0-9]} +foreach f [glob -nocomplain $pattern] { + incr true_file_size [file size $f] + set extension {} +} +if {[string length $extension]>=2 && [string length $extension]<=4} { + set pattern [file rootname $root_filename] + append pattern {.[0-3][0-9][0-9]} + foreach f [glob -nocomplain $pattern] { + incr true_file_size [file size $f] + } +} + +# Open the database +# +if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} { + puts stderr "error trying to open $file_to_analyze: $msg" + exit 1 +} +if {$flags(-debug)} { + proc dbtrace {txt} {puts $txt; flush stdout;} + db trace ::dbtrace +} + +db eval {SELECT count(*) FROM sqlite_master} +set pageSize [expr {wide([db one {PRAGMA page_size}])}] + +if {$flags(-pageinfo)} { + db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} + db eval {SELECT name, path, pageno FROM temp.stat ORDER BY pageno} { + puts "$pageno $name $path" + } + exit 0 +} +if {$flags(-stats)} { + db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} + puts "BEGIN;" + puts "CREATE TABLE stats(" + puts " name STRING, /* Name of table or index */" + puts " path INTEGER, /* Path to page from root */" + puts " pageno INTEGER, /* Page number */" + puts " pagetype STRING, /* 'internal', 'leaf' or 'overflow' */" + puts " ncell INTEGER, /* Cells on page (0 for overflow) */" + puts " payload INTEGER, /* Bytes of payload on this page */" + puts " unused INTEGER, /* Bytes of unused space on this page */" + puts " mx_payload INTEGER, /* Largest payload size of all cells */" + puts " pgoffset INTEGER, /* Offset of page in file */" + puts " pgsize INTEGER /* Size of the page */" + puts ");" + db eval {SELECT quote(name) || ',' || + quote(path) || ',' || + quote(pageno) || ',' || + quote(pagetype) || ',' || + quote(ncell) || ',' || + quote(payload) || ',' || + quote(unused) || ',' || + quote(mx_payload) || ',' || + quote(pgoffset) || ',' || + quote(pgsize) AS x FROM stat} { + puts "INSERT INTO stats VALUES($x);" + } + puts "COMMIT;" + exit 0 +} + + +# In-memory database for collecting statistics. This script loops through +# the tables and indices in the database being analyzed, adding a row for each +# to an in-memory database (for which the schema is shown below). It then +# queries the in-memory db to produce the space-analysis report. +# +sqlite3 mem :memory: +if {$flags(-debug)} { + proc dbtrace {txt} {puts $txt; flush stdout;} + mem trace ::dbtrace +} +set tabledef {CREATE TABLE space_used( + name clob, -- Name of a table or index in the database file + tblname clob, -- Name of associated table + is_index boolean, -- TRUE if it is an index, false for a table + is_without_rowid boolean, -- TRUE if WITHOUT ROWID table + nentry int, -- Number of entries in the BTree + leaf_entries int, -- Number of leaf entries + depth int, -- Depth of the b-tree + payload int, -- Total amount of data stored in this table or index + ovfl_payload int, -- Total amount of data stored on overflow pages + ovfl_cnt int, -- Number of entries that use overflow + mx_payload int, -- Maximum payload size + int_pages int, -- Number of interior pages used + leaf_pages int, -- Number of leaf pages used + ovfl_pages int, -- Number of overflow pages used + int_unused int, -- Number of unused bytes on interior pages + leaf_unused int, -- Number of unused bytes on primary pages + ovfl_unused int, -- Number of unused bytes on overflow pages + gap_cnt int, -- Number of gaps in the page layout + compressed_size int -- Total bytes stored on disk +);} +mem eval $tabledef + +# Create a temporary "dbstat" virtual table. +# +db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} +db eval {CREATE TEMP TABLE dbstat AS SELECT * FROM temp.stat + ORDER BY name, path} +db eval {DROP TABLE temp.stat} + +set isCompressed 0 +set compressOverhead 0 +set depth 0 +set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 } +foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] { + + set is_index [expr {$name!=$tblname}] + set is_without_rowid [is_without_rowid $name] + db eval { + SELECT + sum(ncell) AS nentry, + sum((pagetype=='leaf')*ncell) AS leaf_entries, + sum(payload) AS payload, + sum((pagetype=='overflow') * payload) AS ovfl_payload, + sum(path LIKE '%+000000') AS ovfl_cnt, + max(mx_payload) AS mx_payload, + sum(pagetype=='internal') AS int_pages, + sum(pagetype=='leaf') AS leaf_pages, + sum(pagetype=='overflow') AS ovfl_pages, + sum((pagetype=='internal') * unused) AS int_unused, + sum((pagetype=='leaf') * unused) AS leaf_unused, + sum((pagetype=='overflow') * unused) AS ovfl_unused, + sum(pgsize) AS compressed_size, + max((length(CASE WHEN path LIKE '%+%' THEN '' ELSE path END)+3)/4) + AS depth + FROM temp.dbstat WHERE name = $name + } break + + set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}] + set storage [expr {$total_pages*$pageSize}] + if {!$isCompressed && $storage>$compressed_size} { + set isCompressed 1 + set compressOverhead 14 + } + + # Column 'gap_cnt' is set to the number of non-contiguous entries in the + # list of pages visited if the b-tree structure is traversed in a top-down + # fashion (each node visited before its child-tree is passed). Any overflow + # chains present are traversed from start to finish before any child-tree + # is. + # + set gap_cnt 0 + set prev 0 + db eval { + SELECT pageno, pagetype FROM temp.dbstat + WHERE name=$name + ORDER BY pageno + } { + if {$prev>0 && $pagetype=="leaf" && $pageno!=$prev+1} { + incr gap_cnt + } + set prev $pageno + } + mem eval { + INSERT INTO space_used VALUES( + $name, + $tblname, + $is_index, + $is_without_rowid, + $nentry, + $leaf_entries, + $depth, + $payload, + $ovfl_payload, + $ovfl_cnt, + $mx_payload, + $int_pages, + $leaf_pages, + $ovfl_pages, + $int_unused, + $leaf_unused, + $ovfl_unused, + $gap_cnt, + $compressed_size + ); + } +} + +proc integerify {real} { + if {[string is double -strict $real]} { + return [expr {wide($real)}] + } else { + return 0 + } +} +mem function int integerify + +# Quote a string for use in an SQL query. Examples: +# +# [quote {hello world}] == {'hello world'} +# [quote {hello world's}] == {'hello world''s'} +# +proc quote {txt} { + return [string map {' ''} $txt] +} + +# Output a title line +# +proc titleline {title} { + if {$title==""} { + puts [string repeat * 79] + } else { + set len [string length $title] + set stars [string repeat * [expr 79-$len-5]] + puts "*** $title $stars" + } +} + +# Generate a single line of output in the statistics section of the +# report. +# +proc statline {title value {extra {}}} { + set len [string length $title] + set dots [string repeat . [expr 50-$len]] + set len [string length $value] + set sp2 [string range { } $len end] + if {$extra ne ""} { + set extra " $extra" + } + puts "$title$dots $value$sp2$extra" +} + +# Generate a formatted percentage value for $num/$denom +# +proc percent {num denom {of {}}} { + if {$denom==0.0} {return ""} + set v [expr {$num*100.0/$denom}] + set of {} + if {$v==100.0 || $v<0.001 || ($v>1.0 && $v<99.0)} { + return [format {%5.1f%% %s} $v $of] + } elseif {$v<0.1 || $v>99.9} { + return [format {%7.3f%% %s} $v $of] + } else { + return [format {%6.2f%% %s} $v $of] + } +} + +proc divide {num denom} { + if {$denom==0} {return 0.0} + return [format %.2f [expr double($num)/double($denom)]] +} + +# Generate a subreport that covers some subset of the database. +# the $where clause determines which subset to analyze. +# +proc subreport {title where showFrag} { + global pageSize file_pgcnt compressOverhead + + # Query the in-memory database for the sum of various statistics + # for the subset of tables/indices identified by the WHERE clause in + # $where. Note that even if the WHERE clause matches no rows, the + # following query returns exactly one row (because it is an aggregate). + # + # The results of the query are stored directly by SQLite into local + # variables (i.e. $nentry, $payload etc.). + # + mem eval " + SELECT + int(sum( + CASE WHEN (is_without_rowid OR is_index) THEN nentry + ELSE leaf_entries + END + )) AS nentry, + int(sum(payload)) AS payload, + int(sum(ovfl_payload)) AS ovfl_payload, + max(mx_payload) AS mx_payload, + int(sum(ovfl_cnt)) as ovfl_cnt, + int(sum(leaf_pages)) AS leaf_pages, + int(sum(int_pages)) AS int_pages, + int(sum(ovfl_pages)) AS ovfl_pages, + int(sum(leaf_unused)) AS leaf_unused, + int(sum(int_unused)) AS int_unused, + int(sum(ovfl_unused)) AS ovfl_unused, + int(sum(gap_cnt)) AS gap_cnt, + int(sum(compressed_size)) AS compressed_size, + int(max(depth)) AS depth, + count(*) AS cnt + FROM space_used WHERE $where" {} {} + + # Output the sub-report title, nicely decorated with * characters. + # + puts "" + titleline $title + puts "" + + # Calculate statistics and store the results in TCL variables, as follows: + # + # total_pages: Database pages consumed. + # total_pages_percent: Pages consumed as a percentage of the file. + # storage: Bytes consumed. + # payload_percent: Payload bytes used as a percentage of $storage. + # total_unused: Unused bytes on pages. + # avg_payload: Average payload per btree entry. + # avg_fanout: Average fanout for internal pages. + # avg_unused: Average unused bytes per btree entry. + # avg_meta: Average metadata overhead per entry. + # ovfl_cnt_percent: Percentage of btree entries that use overflow pages. + # + set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}] + set total_pages_percent [percent $total_pages $file_pgcnt] + set storage [expr {$total_pages*$pageSize}] + set payload_percent [percent $payload $storage {of storage consumed}] + set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}] + set avg_payload [divide $payload $nentry] + set avg_unused [divide $total_unused $nentry] + set total_meta [expr {$storage - $payload - $total_unused}] + set total_meta [expr {$total_meta + 4*($ovfl_pages - $ovfl_cnt)}] + set meta_percent [percent $total_meta $storage {of metadata}] + set avg_meta [divide $total_meta $nentry] + if {$int_pages>0} { + # TODO: Is this formula correct? + set nTab [mem eval " + SELECT count(*) FROM ( + SELECT DISTINCT tblname FROM space_used WHERE $where AND is_index=0 + ) + "] + set avg_fanout [mem eval " + SELECT (sum(leaf_pages+int_pages)-$nTab)/sum(int_pages) FROM space_used + WHERE $where + "] + set avg_fanout [format %.2f $avg_fanout] + } + set ovfl_cnt_percent [percent $ovfl_cnt $nentry {of all entries}] + + # Print out the sub-report statistics. + # + statline {Percentage of total database} $total_pages_percent + statline {Number of entries} $nentry + statline {Bytes of storage consumed} $storage + if {$compressed_size!=$storage} { + set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}] + set pct [expr {$compressed_size*100.0/$storage}] + set pct [format {%5.1f%%} $pct] + statline {Bytes used after compression} $compressed_size $pct + } + statline {Bytes of payload} $payload $payload_percent + statline {Bytes of metadata} $total_meta $meta_percent + if {$cnt==1} {statline {B-tree depth} $depth} + statline {Average payload per entry} $avg_payload + statline {Average unused bytes per entry} $avg_unused + statline {Average metadata per entry} $avg_meta + if {[info exists avg_fanout]} { + statline {Average fanout} $avg_fanout + } + if {$showFrag && $total_pages>1} { + set fragmentation [percent $gap_cnt [expr {$total_pages-1}]] + statline {Non-sequential pages} $gap_cnt $fragmentation + } + statline {Maximum payload per entry} $mx_payload + statline {Entries that use overflow} $ovfl_cnt $ovfl_cnt_percent + if {$int_pages>0} { + statline {Index pages used} $int_pages + } + statline {Primary pages used} $leaf_pages + statline {Overflow pages used} $ovfl_pages + statline {Total pages used} $total_pages + if {$int_unused>0} { + set int_unused_percent [ + percent $int_unused [expr {$int_pages*$pageSize}] {of index space}] + statline "Unused bytes on index pages" $int_unused $int_unused_percent + } + statline "Unused bytes on primary pages" $leaf_unused [ + percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}] + statline "Unused bytes on overflow pages" $ovfl_unused [ + percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}] + statline "Unused bytes on all pages" $total_unused [ + percent $total_unused $storage {of all space}] + return 1 +} + +# Calculate the overhead in pages caused by auto-vacuum. +# +# This procedure calculates and returns the number of pages used by the +# auto-vacuum 'pointer-map'. If the database does not support auto-vacuum, +# then 0 is returned. The two arguments are the size of the database file in +# pages and the page size used by the database (in bytes). +proc autovacuum_overhead {filePages pageSize} { + + # Set $autovacuum to non-zero for databases that support auto-vacuum. + set autovacuum [db one {PRAGMA auto_vacuum}] + + # If the database is not an auto-vacuum database or the file consists + # of one page only then there is no overhead for auto-vacuum. Return zero. + if {0==$autovacuum || $filePages==1} { + return 0 + } + + # The number of entries on each pointer map page. The layout of the + # database file is one pointer-map page, followed by $ptrsPerPage other + # pages, followed by a pointer-map page etc. The first pointer-map page + # is the second page of the file overall. + set ptrsPerPage [expr double($pageSize/5)] + + # Return the number of pointer map pages in the database. + return [expr wide(ceil( ($filePages-1.0)/($ptrsPerPage+1.0) ))] +} + + +# Calculate the summary statistics for the database and store the results +# in TCL variables. They are output below. Variables are as follows: +# +# pageSize: Size of each page in bytes. +# file_bytes: File size in bytes. +# file_pgcnt: Number of pages in the file. +# file_pgcnt2: Number of pages in the file (calculated). +# av_pgcnt: Pages consumed by the auto-vacuum pointer-map. +# av_percent: Percentage of the file consumed by auto-vacuum pointer-map. +# inuse_pgcnt: Data pages in the file. +# inuse_percent: Percentage of pages used to store data. +# free_pgcnt: Free pages calculated as ( - ) +# free_pgcnt2: Free pages in the file according to the file header. +# free_percent: Percentage of file consumed by free pages (calculated). +# free_percent2: Percentage of file consumed by free pages (header). +# ntable: Number of tables in the db. +# nindex: Number of indices in the db. +# nautoindex: Number of indices created automatically. +# nmanindex: Number of indices created manually. +# user_payload: Number of bytes of payload in table btrees +# (not including sqlite_master) +# user_percent: $user_payload as a percentage of total file size. + +### The following, setting $file_bytes based on the actual size of the file +### on disk, causes this tool to choke on zipvfs databases. So set it based +### on the return of [PRAGMA page_count] instead. +if 0 { + set file_bytes [file size $file_to_analyze] + set file_pgcnt [expr {$file_bytes/$pageSize}] +} +set file_pgcnt [db one {PRAGMA page_count}] +set file_bytes [expr {$file_pgcnt * $pageSize}] + +set av_pgcnt [autovacuum_overhead $file_pgcnt $pageSize] +set av_percent [percent $av_pgcnt $file_pgcnt] + +set sql {SELECT sum(leaf_pages+int_pages+ovfl_pages) FROM space_used} +set inuse_pgcnt [expr wide([mem eval $sql])] +set inuse_percent [percent $inuse_pgcnt $file_pgcnt] + +set free_pgcnt [expr {$file_pgcnt-$inuse_pgcnt-$av_pgcnt}] +set free_percent [percent $free_pgcnt $file_pgcnt] +set free_pgcnt2 [db one {PRAGMA freelist_count}] +set free_percent2 [percent $free_pgcnt2 $file_pgcnt] + +set file_pgcnt2 [expr {$inuse_pgcnt+$free_pgcnt2+$av_pgcnt}] + +set ntable [db eval {SELECT count(*)+1 FROM sqlite_master WHERE type='table'}] +set nindex [db eval {SELECT count(*) FROM sqlite_master WHERE type='index'}] +set sql {SELECT count(*) FROM sqlite_master WHERE name LIKE 'sqlite_autoindex%'} +set nautoindex [db eval $sql] +set nmanindex [expr {$nindex-$nautoindex}] + +# set total_payload [mem eval "SELECT sum(payload) FROM space_used"] +set user_payload [mem one {SELECT int(sum(payload)) FROM space_used + WHERE NOT is_index AND name NOT LIKE 'sqlite_master'}] +set user_percent [percent $user_payload $file_bytes] + +# Output the summary statistics calculated above. +# +puts "/** Disk-Space Utilization Report For $root_filename" +puts "" +statline {Page size in bytes} $pageSize +statline {Pages in the whole file (measured)} $file_pgcnt +statline {Pages in the whole file (calculated)} $file_pgcnt2 +statline {Pages that store data} $inuse_pgcnt $inuse_percent +statline {Pages on the freelist (per header)} $free_pgcnt2 $free_percent2 +statline {Pages on the freelist (calculated)} $free_pgcnt $free_percent +statline {Pages of auto-vacuum overhead} $av_pgcnt $av_percent +statline {Number of tables in the database} $ntable +statline {Number of indices} $nindex +statline {Number of defined indices} $nmanindex +statline {Number of implied indices} $nautoindex +if {$isCompressed} { + statline {Size of uncompressed content in bytes} $file_bytes + set efficiency [percent $true_file_size $file_bytes] + statline {Size of compressed file on disk} $true_file_size $efficiency +} else { + statline {Size of the file in bytes} $file_bytes +} +statline {Bytes of user payload stored} $user_payload $user_percent + +# Output table rankings +# +puts "" +titleline "Page counts for all tables with their indices" +puts "" +mem eval {SELECT tblname, count(*) AS cnt, + int(sum(int_pages+leaf_pages+ovfl_pages)) AS size + FROM space_used GROUP BY tblname ORDER BY size+0 DESC, tblname} {} { + statline [string toupper $tblname] $size [percent $size $file_pgcnt] +} +puts "" +titleline "Page counts for all tables and indices separately" +puts "" +mem eval { + SELECT + upper(name) AS nm, + int(int_pages+leaf_pages+ovfl_pages) AS size + FROM space_used + ORDER BY size+0 DESC, name} {} { + statline $nm $size [percent $size $file_pgcnt] +} +if {$isCompressed} { + puts "" + titleline "Bytes of disk space used after compression" + puts "" + set csum 0 + mem eval {SELECT tblname, + int(sum(compressed_size)) + + $compressOverhead*sum(int_pages+leaf_pages+ovfl_pages) + AS csize + FROM space_used GROUP BY tblname ORDER BY csize+0 DESC, tblname} {} { + incr csum $csize + statline [string toupper $tblname] $csize [percent $csize $true_file_size] + } + set overhead [expr {$true_file_size - $csum}] + if {$overhead>0} { + statline {Header and free space} $overhead [percent $overhead $true_file_size] + } +} + +# Output subreports +# +if {$nindex>0} { + subreport {All tables and indices} 1 0 +} +subreport {All tables} {NOT is_index} 0 +if {$nindex>0} { + subreport {All indices} {is_index} 0 +} +foreach tbl [mem eval {SELECT DISTINCT tblname name FROM space_used + ORDER BY name}] { + set qn [quote $tbl] + set name [string toupper $tbl] + set n [mem eval {SELECT count(*) FROM space_used WHERE tblname=$tbl}] + if {$n>1} { + set idxlist [mem eval "SELECT name FROM space_used + WHERE tblname='$qn' AND is_index + ORDER BY 1"] + subreport "Table $name and all its indices" "tblname='$qn'" 0 + subreport "Table $name w/o any indices" "name='$qn'" 1 + if {[llength $idxlist]>1} { + subreport "Indices of table $name" "tblname='$qn' AND is_index" 0 + } + foreach idx $idxlist { + set qidx [quote $idx] + subreport "Index [string toupper $idx] of table $name" "name='$qidx'" 1 + } + } else { + subreport "Table $name" "name='$qn'" 1 + } +} + +# Output instructions on what the numbers above mean. +# +puts "" +titleline Definitions +puts { +Page size in bytes + + The number of bytes in a single page of the database file. + Usually 1024. + +Number of pages in the whole file +} +puts " The number of $pageSize-byte pages that go into forming the complete + database" +puts { +Pages that store data + + The number of pages that store data, either as primary B*Tree pages or + as overflow pages. The number at the right is the data pages divided by + the total number of pages in the file. + +Pages on the freelist + + The number of pages that are not currently in use but are reserved for + future use. The percentage at the right is the number of freelist pages + divided by the total number of pages in the file. + +Pages of auto-vacuum overhead + + The number of pages that store data used by the database to facilitate + auto-vacuum. This is zero for databases that do not support auto-vacuum. + +Number of tables in the database + + The number of tables in the database, including the SQLITE_MASTER table + used to store schema information. + +Number of indices + + The total number of indices in the database. + +Number of defined indices + + The number of indices created using an explicit CREATE INDEX statement. + +Number of implied indices + + The number of indices used to implement PRIMARY KEY or UNIQUE constraints + on tables. + +Size of the file in bytes + + The total amount of disk space used by the entire database files. + +Bytes of user payload stored + + The total number of bytes of user payload stored in the database. The + schema information in the SQLITE_MASTER table is not counted when + computing this number. The percentage at the right shows the payload + divided by the total file size. + +Percentage of total database + + The amount of the complete database file that is devoted to storing + information described by this category. + +Number of entries + + The total number of B-Tree key/value pairs stored under this category. + +Bytes of storage consumed + + The total amount of disk space required to store all B-Tree entries + under this category. The is the total number of pages used times + the pages size. + +Bytes of payload + + The amount of payload stored under this category. Payload is the data + part of table entries and the key part of index entries. The percentage + at the right is the bytes of payload divided by the bytes of storage + consumed. + +Bytes of metadata + + The amount of formatting and structural information stored in the + table or index. Metadata includes the btree page header, the cell pointer + array, the size field for each cell, the left child pointer or non-leaf + cells, the overflow pointers for overflow cells, and the rowid value for + rowid table cells. In other words, metadata is everything that is neither + unused space nor content. The record header in the payload is counted as + content, not metadata. + +Average payload per entry + + The average amount of payload on each entry. This is just the bytes of + payload divided by the number of entries. + +Average unused bytes per entry + + The average amount of free space remaining on all pages under this + category on a per-entry basis. This is the number of unused bytes on + all pages divided by the number of entries. + +Non-sequential pages + + The number of pages in the table or index that are out of sequence. + Many filesystems are optimized for sequential file access so a small + number of non-sequential pages might result in faster queries, + especially for larger database files that do not fit in the disk cache. + Note that after running VACUUM, the root page of each table or index is + at the beginning of the database file and all other pages are in a + separate part of the database file, resulting in a single non- + sequential page. + +Maximum payload per entry + + The largest payload size of any entry. + +Entries that use overflow + + The number of entries that user one or more overflow pages. + +Total pages used + + This is the number of pages used to hold all information in the current + category. This is the sum of index, primary, and overflow pages. + +Index pages used + + This is the number of pages in a table B-tree that hold only key (rowid) + information and no data. + +Primary pages used + + This is the number of B-tree pages that hold both key and data. + +Overflow pages used + + The total number of overflow pages used for this category. + +Unused bytes on index pages + + The total number of bytes of unused space on all index pages. The + percentage at the right is the number of unused bytes divided by the + total number of bytes on index pages. + +Unused bytes on primary pages + + The total number of bytes of unused space on all primary pages. The + percentage at the right is the number of unused bytes divided by the + total number of bytes on primary pages. + +Unused bytes on overflow pages + + The total number of bytes of unused space on all overflow pages. The + percentage at the right is the number of unused bytes divided by the + total number of bytes on overflow pages. + +Unused bytes on all pages + + The total number of bytes of unused space on all primary and overflow + pages. The percentage at the right is the number of unused bytes + divided by the total number of bytes. +} + +# Output a dump of the in-memory database. This can be used for more +# complex offline analysis. +# +titleline {} +puts "The entire text of this report can be sourced into any SQL database" +puts "engine for further analysis. All of the text above is an SQL comment." +puts "The data used to generate this report follows:" +puts "*/" +puts "BEGIN;" +puts $tabledef +unset -nocomplain x +mem eval {SELECT * FROM space_used} x { + puts -nonewline "INSERT INTO space_used VALUES" + set sep ( + foreach col $x(*) { + set v $x($col) + if {$v=="" || ![string is double $v]} {set v '[quote $v]'} + puts -nonewline $sep$v + set sep , + } + puts ");" +} +puts "COMMIT;" + +} err]} { + puts "ERROR: $err" + puts $errorInfo + exit 1 +} +END_STRING +; +} diff --git a/tool/tostr.tcl b/tool/tostr.tcl deleted file mode 100644 index cb06ee947f..0000000000 --- a/tool/tostr.tcl +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/tcl -# -# Convert input text into a C string -# -set in [open [lindex $argv 0] rb] -while {![eof $in]} { - set line [gets $in] - if {[eof $in]} break; - set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line] - puts "\"$x\\n\"" -} -close $in From 96a206fa1055e1f6fa3d0bf8a221bdf18bc7195d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Oct 2017 20:14:06 +0000 Subject: [PATCH 13/22] Improved ability to generate stand-alone program using TCL and SQLite by compiling with -DTCLSH_INIT_PROC=name to cause the TCL interpreter to be initialized using procedure name(). Both sqlite3_analyzer and testfixture are now built this way. FossilOrigin-Name: d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4 --- Makefile.in | 5 +- Makefile.msc | 4 +- main.mk | 9 +- manifest | 22 +- manifest.uuid | 2 +- src/tclsqlite.c | 138 +++--- src/test_tclsh.c | 194 ++++---- tool/sqlite3_analyzer.c.in | 886 +------------------------------------ 8 files changed, 187 insertions(+), 1073 deletions(-) diff --git a/Makefile.in b/Makefile.in index e2934d80b1..5f6be57c37 100644 --- a/Makefile.in +++ b/Makefile.in @@ -947,7 +947,7 @@ tclsqlite.lo: $(TOP)/src/tclsqlite.c $(HDR) $(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c tclsqlite-shell.lo: $(TOP)/src/tclsqlite.c $(HDR) - $(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c + $(LTCOMPILE) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c tclsqlite-stubs.lo: $(TOP)/src/tclsqlite.c $(HDR) $(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c @@ -1113,7 +1113,8 @@ sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). # -TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 +TESTFIXTURE_FLAGS = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 +TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE TESTFIXTURE_FLAGS += -DBUILD_sqlite TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 diff --git a/Makefile.msc b/Makefile.msc index bfd368c558..ce867ba3f3 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1934,7 +1934,7 @@ tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) - $(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c + $(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS) @@ -2105,7 +2105,7 @@ sqlite3rbu.lo: $(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR) # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). # -TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 +TESTFIXTURE_FLAGS = -DTCLSH_INIT_PROC=sqlite3TestInit -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN) TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 diff --git a/main.mk b/main.mk index 18d7cd3459..c7711fb6e9 100644 --- a/main.mk +++ b/main.mk @@ -777,7 +777,7 @@ sqlite3rbu.o: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) # Rules for building test programs and for running tests # tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a - $(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \ + $(TCCX) $(TCL_FLAGS) -DTCLSH -o tclsqlite3 \ $(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB) sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/sqlite3_analyzer.c.in $(TOP)/tool/mkccode.tcl @@ -798,21 +798,22 @@ TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB +TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c - $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ + $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ $(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \ -o testfixture$(EXE) $(LIBTCL) libsqlite3.a $(THREADLIB) amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c \ $(TOP)/ext/session/test_session.c - $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ + $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \ $(TOP)/ext/session/test_session.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c - $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ + $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ -DSQLITE_ENABLE_FTS3=1 \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) diff --git a/manifest b/manifest index 669c1b284a..2599e89765 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Add\sthe\stcl/mkccode.tcl\sscript\sused\sto\sconstruct\sa\ssingle\sC-language\ssource\nfiel\sfor\sprograms\sthat\scombine\sC-code,\sSQLite,\sand\sTCL.\s\sUse\sthis\sscript\sto\nconstruct\sthe\ssqlite3_analyzer\sprogram. -D 2017-10-13T18:58:55.897 -F Makefile.in a99044d778e2a83ba6cf78b88937d3d667cee425dca5915d73eeba1d56b98082 +C Improved\sability\sto\sgenerate\sstand-alone\sprogram\susing\sTCL\sand\sSQLite\sby\ncompiling\swith\s-DTCLSH_INIT_PROC=name\sto\scause\sthe\sTCL\sinterpreter\sto\sbe\ninitialized\susing\sprocedure\sname().\s\sBoth\ssqlite3_analyzer\sand\stestfixture\nare\snow\sbuilt\sthis\sway. +D 2017-10-13T20:14:06.865 +F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc b864c4f906031a37e425466d8214df2da2f01703956dad5290156f03548d7277 +F Makefile.msc a341cc7d737f596a1074e47fc9eb17bde689413d891e3ef2f26e98126da950a4 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a1948e0509f143fe4395a449cf61039bb6c7db3a93ae6af65092a72556718026 +F main.mk a39528d993afc1f0c0aebde2e3623ab4171d3bba484eea1e5241615c706c9ce8 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -470,7 +470,7 @@ F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53b F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 -F src/tclsqlite.c 7ffde8e2cd556ed74b5d40ae0ee924aa974ee3b3dacd5114c5c98feb9f651ac4 +F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696 F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b @@ -514,7 +514,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 -F src/test_tclsh.c 2a1ff3eac1ac3ee070a80209384d6cb353d4ad4a549400e3cd29445c672b2b87 +F src/test_tclsh.c 74fcfb7f3b0ff1f871e62263dd84ffba46a8e9d477439115e0fb2035e4bf69e1 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e @@ -1632,7 +1632,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb -F tool/sqlite3_analyzer.c.in 3d53f06e04619f43b7aebc01cc318ef215bfd09ea1947fe9e485bed6aa6cb4b9 +F tool/sqlite3_analyzer.c.in 771d15fb9c67645fd8ef932a438f98959da4b7c7da3cb87ae1850b27c969edf3 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a9c4bc88fcf985a0bea14ed5381239cfb697886287998da04a10230b6858ab5d -R 78f34b33921be1c7a7c8587f43b33b81 +P 298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82 +R df005e0c0c347d89af378f70d82f057b U drh -Z 180ea09ee34fefc9e97e7461c13719ef +Z 8b50e8b5594af446543ab2fc55fee9a2 diff --git a/manifest.uuid b/manifest.uuid index fabaeec3c0..d905d370b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82 \ No newline at end of file +d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 0009eab695..eed86eee34 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -14,17 +14,19 @@ ** ** Compile-time options: ** -** -DTCLSH=1 Add a "main()" routine that works as a tclsh. +** -DTCLSH Add a "main()" routine that works as a tclsh. ** -** -DSQLITE_TCLMD5 When used in conjuction with -DTCLSH=1, add -** four new commands to the TCL interpreter for -** generating MD5 checksums: md5, md5file, -** md5-10x8, and md5file-10x8. +** -DTCLSH_INIT_PROC=name ** -** -DSQLITE_TEST When used in conjuction with -DTCLSH=1, add -** hundreds of new commands used for testing -** SQLite. This option implies -DSQLITE_TCLMD5. +** Invoke name(interp) to initialize the Tcl interpreter. +** If name(interp) returns a non-NULL string, then run +** that string as a Tcl script to launch the application. +** If name(interp) returns NULL, then run the regular +** tclsh-emulator code. */ +#ifdef TCLSH_INIT_PROC +# define TCLSH 1 +#endif /* ** If requested, include the SQLite compiler options file for MSVC. @@ -3582,56 +3584,55 @@ int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } #endif /* -** If the TCLSH macro is defined to be either 1 or 2, then a main() -** routine is inserted that starts up a Tcl interpreter. When TCLSH==1 -** the interpreter works like an ordinary tclsh. When TCLSH==2 then the -** startup script is supplied by an routine named "tclsh_main_loop()" -** that must be linked separately. The TCLSH==2 technique is used to -** generate stand-alone executables based on TCL, such as -** sqlite3_analyzer.exe. +** If the TCLSH macro is defined, add code to make a stand-alone program. */ -#ifdef TCLSH +#if defined(TCLSH) -/* -** If the macro TCLSH is one, then put in code this for the -** "main" routine that will initialize Tcl and take input from -** standard input, or if a file is named on the command line -** the TCL interpreter reads and evaluates that file. +/* This is the main routine for an ordinary TCL shell. If there are +** are arguments, run the first argument as a script. Otherwise, +** read TCL commands from standard input */ -#if TCLSH==1 static const char *tclsh_main_loop(void){ static const char zMainloop[] = - "set line {}\n" - "while {![eof stdin]} {\n" - "if {$line!=\"\"} {\n" - "puts -nonewline \"> \"\n" - "} else {\n" - "puts -nonewline \"% \"\n" - "}\n" - "flush stdout\n" - "append line [gets stdin]\n" - "if {[info complete $line]} {\n" - "if {[catch {uplevel #0 $line} result]} {\n" - "puts stderr \"Error: $result\"\n" - "} elseif {$result!=\"\"} {\n" - "puts $result\n" + "if {[llength $argv]>=1} {\n" + "set argv0 [lindex $argv 0]\n" + "set argv [lrange $argv 1 end]\n" + "source $argv0\n" + "} else {\n" + "set line {}\n" + "while {![eof stdin]} {\n" + "if {$line!=\"\"} {\n" + "puts -nonewline \"> \"\n" + "} else {\n" + "puts -nonewline \"% \"\n" + "}\n" + "flush stdout\n" + "append line [gets stdin]\n" + "if {[info complete $line]} {\n" + "if {[catch {uplevel #0 $line} result]} {\n" + "puts stderr \"Error: $result\"\n" + "} elseif {$result!=\"\"} {\n" + "puts $result\n" + "}\n" + "set line {}\n" + "} else {\n" + "append line \\n\n" "}\n" - "set line {}\n" - "} else {\n" - "append line \\n\n" "}\n" "}\n" ; return zMainloop; } -#endif -#if TCLSH==2 -static const char *tclsh_main_loop(void); -#endif #define TCLSH_MAIN main /* Needed to fake out mktclapp */ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ Tcl_Interp *interp; + int i; + const char *zScript = 0; + char zArgc[32]; +#if defined(TCLSH_INIT_PROC) + extern const char *TCLSH_INIT_PROC(Tcl_Interp*); +#endif #if !defined(_WIN32_WCE) if( getenv("BREAK") ){ @@ -3650,42 +3651,27 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ Tcl_FindExecutable(argv[0]); Tcl_SetSystemEncoding(NULL, "utf-8"); interp = Tcl_CreateInterp(); - -#if TCLSH==2 - sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); -#endif - - /* Add extensions */ -#if !defined(SQLITE_TEST) - /* Normally we only initialize the TCL extension */ Sqlite3_Init(interp); -#else - /* For testing, do lots of extra initialization */ - { - extern void sqlite3InitTclTestLogic(Tcl_Interp*); - sqlite3InitTclTestLogic(interp); + + sqlite3_snprintf(sizeof(zArgc), zArgc, "%d", argc-1); + Tcl_SetVar(interp,"argc", zArgc, TCL_GLOBAL_ONLY); + Tcl_SetVar(interp,"argv0",argv[0],TCL_GLOBAL_ONLY); + Tcl_SetVar(interp,"argv", "", TCL_GLOBAL_ONLY); + for(i=1; i=2 ){ - int i; - char zArgc[32]; - sqlite3_snprintf(sizeof(zArgc), zArgc, "%d", argc-(3-TCLSH)); - Tcl_SetVar(interp,"argc", zArgc, TCL_GLOBAL_ONLY); - Tcl_SetVar(interp,"argv0",argv[1],TCL_GLOBAL_ONLY); - Tcl_SetVar(interp,"argv", "", TCL_GLOBAL_ONLY); - for(i=3-TCLSH; i " - } else { - puts -nonewline "% " - } - flush stdout - append line [gets stdin] - if {[info complete $line]} { - if {[catch {uplevel #0 $line} result]} { - puts stderr "Error: $result" - } elseif {$result!=""} { - puts $result - } - set line {} - } else { - append line \n - } - } -} - - -# Get the name of the database to analyze -# -proc usage {} { - set argv0 [file rootname [file tail [info nameofexecutable]]] - puts stderr "Usage: $argv0 ?--pageinfo? ?--stats? database-filename" - puts stderr { -Analyze the SQLite3 database file specified by the "database-filename" -argument and output a report detailing size and storage efficiency -information for the database and its constituent tables and indexes. - -Options: - - --pageinfo Show how each page of the database-file is used - - --stats Output SQL text that creates a new database containing - statistics about the database that was analyzed - - --tclsh Run the built-in TCL interpreter interactively (for debugging) - - --version Show the version number of SQLite -} - exit 1 -} -set file_to_analyze {} -set flags(-pageinfo) 0 -set flags(-stats) 0 -set flags(-debug) 0 -append argv {} -foreach arg $argv { - if {[regexp {^-+pageinfo$} $arg]} { - set flags(-pageinfo) 1 - } elseif {[regexp {^-+stats$} $arg]} { - set flags(-stats) 1 - } elseif {[regexp {^-+debug$} $arg]} { - set flags(-debug) 1 - } elseif {[regexp {^-+tclsh$} $arg]} { - tclsh - exit 0 - } elseif {[regexp {^-+version$} $arg]} { - sqlite3 mem :memory: - puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] - mem close - exit 0 - } elseif {[regexp {^-} $arg]} { - puts stderr "Unknown option: $arg" - usage - } elseif {$file_to_analyze!=""} { - usage - } else { - set file_to_analyze $arg - } -} -if {$file_to_analyze==""} usage -set root_filename $file_to_analyze -regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename -if {![file exists $root_filename]} { - puts stderr "No such file: $root_filename" - exit 1 -} -if {![file readable $root_filename]} { - puts stderr "File is not readable: $root_filename" - exit 1 -} -set true_file_size [file size $root_filename] -if {$true_file_size<512} { - puts stderr "Empty or malformed database: $root_filename" - exit 1 -} - -# Compute the total file size assuming test_multiplexor is being used. -# Assume that SQLITE_ENABLE_8_3_NAMES might be enabled -# -set extension [file extension $root_filename] -set pattern $root_filename -append pattern {[0-3][0-9][0-9]} -foreach f [glob -nocomplain $pattern] { - incr true_file_size [file size $f] - set extension {} -} -if {[string length $extension]>=2 && [string length $extension]<=4} { - set pattern [file rootname $root_filename] - append pattern {.[0-3][0-9][0-9]} - foreach f [glob -nocomplain $pattern] { - incr true_file_size [file size $f] - } -} - -# Open the database -# -if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} { - puts stderr "error trying to open $file_to_analyze: $msg" - exit 1 -} -if {$flags(-debug)} { - proc dbtrace {txt} {puts $txt; flush stdout;} - db trace ::dbtrace -} - -db eval {SELECT count(*) FROM sqlite_master} -set pageSize [expr {wide([db one {PRAGMA page_size}])}] - -if {$flags(-pageinfo)} { - db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} - db eval {SELECT name, path, pageno FROM temp.stat ORDER BY pageno} { - puts "$pageno $name $path" - } - exit 0 -} -if {$flags(-stats)} { - db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} - puts "BEGIN;" - puts "CREATE TABLE stats(" - puts " name STRING, /* Name of table or index */" - puts " path INTEGER, /* Path to page from root */" - puts " pageno INTEGER, /* Page number */" - puts " pagetype STRING, /* 'internal', 'leaf' or 'overflow' */" - puts " ncell INTEGER, /* Cells on page (0 for overflow) */" - puts " payload INTEGER, /* Bytes of payload on this page */" - puts " unused INTEGER, /* Bytes of unused space on this page */" - puts " mx_payload INTEGER, /* Largest payload size of all cells */" - puts " pgoffset INTEGER, /* Offset of page in file */" - puts " pgsize INTEGER /* Size of the page */" - puts ");" - db eval {SELECT quote(name) || ',' || - quote(path) || ',' || - quote(pageno) || ',' || - quote(pagetype) || ',' || - quote(ncell) || ',' || - quote(payload) || ',' || - quote(unused) || ',' || - quote(mx_payload) || ',' || - quote(pgoffset) || ',' || - quote(pgsize) AS x FROM stat} { - puts "INSERT INTO stats VALUES($x);" - } - puts "COMMIT;" - exit 0 -} - - -# In-memory database for collecting statistics. This script loops through -# the tables and indices in the database being analyzed, adding a row for each -# to an in-memory database (for which the schema is shown below). It then -# queries the in-memory db to produce the space-analysis report. -# -sqlite3 mem :memory: -if {$flags(-debug)} { - proc dbtrace {txt} {puts $txt; flush stdout;} - mem trace ::dbtrace -} -set tabledef {CREATE TABLE space_used( - name clob, -- Name of a table or index in the database file - tblname clob, -- Name of associated table - is_index boolean, -- TRUE if it is an index, false for a table - is_without_rowid boolean, -- TRUE if WITHOUT ROWID table - nentry int, -- Number of entries in the BTree - leaf_entries int, -- Number of leaf entries - depth int, -- Depth of the b-tree - payload int, -- Total amount of data stored in this table or index - ovfl_payload int, -- Total amount of data stored on overflow pages - ovfl_cnt int, -- Number of entries that use overflow - mx_payload int, -- Maximum payload size - int_pages int, -- Number of interior pages used - leaf_pages int, -- Number of leaf pages used - ovfl_pages int, -- Number of overflow pages used - int_unused int, -- Number of unused bytes on interior pages - leaf_unused int, -- Number of unused bytes on primary pages - ovfl_unused int, -- Number of unused bytes on overflow pages - gap_cnt int, -- Number of gaps in the page layout - compressed_size int -- Total bytes stored on disk -);} -mem eval $tabledef - -# Create a temporary "dbstat" virtual table. -# -db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} -db eval {CREATE TEMP TABLE dbstat AS SELECT * FROM temp.stat - ORDER BY name, path} -db eval {DROP TABLE temp.stat} - -set isCompressed 0 -set compressOverhead 0 -set depth 0 -set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 } -foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] { - - set is_index [expr {$name!=$tblname}] - set is_without_rowid [is_without_rowid $name] - db eval { - SELECT - sum(ncell) AS nentry, - sum((pagetype=='leaf')*ncell) AS leaf_entries, - sum(payload) AS payload, - sum((pagetype=='overflow') * payload) AS ovfl_payload, - sum(path LIKE '%+000000') AS ovfl_cnt, - max(mx_payload) AS mx_payload, - sum(pagetype=='internal') AS int_pages, - sum(pagetype=='leaf') AS leaf_pages, - sum(pagetype=='overflow') AS ovfl_pages, - sum((pagetype=='internal') * unused) AS int_unused, - sum((pagetype=='leaf') * unused) AS leaf_unused, - sum((pagetype=='overflow') * unused) AS ovfl_unused, - sum(pgsize) AS compressed_size, - max((length(CASE WHEN path LIKE '%+%' THEN '' ELSE path END)+3)/4) - AS depth - FROM temp.dbstat WHERE name = $name - } break - - set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}] - set storage [expr {$total_pages*$pageSize}] - if {!$isCompressed && $storage>$compressed_size} { - set isCompressed 1 - set compressOverhead 14 - } - - # Column 'gap_cnt' is set to the number of non-contiguous entries in the - # list of pages visited if the b-tree structure is traversed in a top-down - # fashion (each node visited before its child-tree is passed). Any overflow - # chains present are traversed from start to finish before any child-tree - # is. - # - set gap_cnt 0 - set prev 0 - db eval { - SELECT pageno, pagetype FROM temp.dbstat - WHERE name=$name - ORDER BY pageno - } { - if {$prev>0 && $pagetype=="leaf" && $pageno!=$prev+1} { - incr gap_cnt - } - set prev $pageno - } - mem eval { - INSERT INTO space_used VALUES( - $name, - $tblname, - $is_index, - $is_without_rowid, - $nentry, - $leaf_entries, - $depth, - $payload, - $ovfl_payload, - $ovfl_cnt, - $mx_payload, - $int_pages, - $leaf_pages, - $ovfl_pages, - $int_unused, - $leaf_unused, - $ovfl_unused, - $gap_cnt, - $compressed_size - ); - } -} - -proc integerify {real} { - if {[string is double -strict $real]} { - return [expr {wide($real)}] - } else { - return 0 - } -} -mem function int integerify - -# Quote a string for use in an SQL query. Examples: -# -# [quote {hello world}] == {'hello world'} -# [quote {hello world's}] == {'hello world''s'} -# -proc quote {txt} { - return [string map {' ''} $txt] -} - -# Output a title line -# -proc titleline {title} { - if {$title==""} { - puts [string repeat * 79] - } else { - set len [string length $title] - set stars [string repeat * [expr 79-$len-5]] - puts "*** $title $stars" - } -} - -# Generate a single line of output in the statistics section of the -# report. -# -proc statline {title value {extra {}}} { - set len [string length $title] - set dots [string repeat . [expr 50-$len]] - set len [string length $value] - set sp2 [string range { } $len end] - if {$extra ne ""} { - set extra " $extra" - } - puts "$title$dots $value$sp2$extra" -} - -# Generate a formatted percentage value for $num/$denom -# -proc percent {num denom {of {}}} { - if {$denom==0.0} {return ""} - set v [expr {$num*100.0/$denom}] - set of {} - if {$v==100.0 || $v<0.001 || ($v>1.0 && $v<99.0)} { - return [format {%5.1f%% %s} $v $of] - } elseif {$v<0.1 || $v>99.9} { - return [format {%7.3f%% %s} $v $of] - } else { - return [format {%6.2f%% %s} $v $of] - } -} - -proc divide {num denom} { - if {$denom==0} {return 0.0} - return [format %.2f [expr double($num)/double($denom)]] -} - -# Generate a subreport that covers some subset of the database. -# the $where clause determines which subset to analyze. -# -proc subreport {title where showFrag} { - global pageSize file_pgcnt compressOverhead - - # Query the in-memory database for the sum of various statistics - # for the subset of tables/indices identified by the WHERE clause in - # $where. Note that even if the WHERE clause matches no rows, the - # following query returns exactly one row (because it is an aggregate). - # - # The results of the query are stored directly by SQLite into local - # variables (i.e. $nentry, $payload etc.). - # - mem eval " - SELECT - int(sum( - CASE WHEN (is_without_rowid OR is_index) THEN nentry - ELSE leaf_entries - END - )) AS nentry, - int(sum(payload)) AS payload, - int(sum(ovfl_payload)) AS ovfl_payload, - max(mx_payload) AS mx_payload, - int(sum(ovfl_cnt)) as ovfl_cnt, - int(sum(leaf_pages)) AS leaf_pages, - int(sum(int_pages)) AS int_pages, - int(sum(ovfl_pages)) AS ovfl_pages, - int(sum(leaf_unused)) AS leaf_unused, - int(sum(int_unused)) AS int_unused, - int(sum(ovfl_unused)) AS ovfl_unused, - int(sum(gap_cnt)) AS gap_cnt, - int(sum(compressed_size)) AS compressed_size, - int(max(depth)) AS depth, - count(*) AS cnt - FROM space_used WHERE $where" {} {} - - # Output the sub-report title, nicely decorated with * characters. - # - puts "" - titleline $title - puts "" - - # Calculate statistics and store the results in TCL variables, as follows: - # - # total_pages: Database pages consumed. - # total_pages_percent: Pages consumed as a percentage of the file. - # storage: Bytes consumed. - # payload_percent: Payload bytes used as a percentage of $storage. - # total_unused: Unused bytes on pages. - # avg_payload: Average payload per btree entry. - # avg_fanout: Average fanout for internal pages. - # avg_unused: Average unused bytes per btree entry. - # avg_meta: Average metadata overhead per entry. - # ovfl_cnt_percent: Percentage of btree entries that use overflow pages. - # - set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}] - set total_pages_percent [percent $total_pages $file_pgcnt] - set storage [expr {$total_pages*$pageSize}] - set payload_percent [percent $payload $storage {of storage consumed}] - set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}] - set avg_payload [divide $payload $nentry] - set avg_unused [divide $total_unused $nentry] - set total_meta [expr {$storage - $payload - $total_unused}] - set total_meta [expr {$total_meta + 4*($ovfl_pages - $ovfl_cnt)}] - set meta_percent [percent $total_meta $storage {of metadata}] - set avg_meta [divide $total_meta $nentry] - if {$int_pages>0} { - # TODO: Is this formula correct? - set nTab [mem eval " - SELECT count(*) FROM ( - SELECT DISTINCT tblname FROM space_used WHERE $where AND is_index=0 - ) - "] - set avg_fanout [mem eval " - SELECT (sum(leaf_pages+int_pages)-$nTab)/sum(int_pages) FROM space_used - WHERE $where - "] - set avg_fanout [format %.2f $avg_fanout] - } - set ovfl_cnt_percent [percent $ovfl_cnt $nentry {of all entries}] - - # Print out the sub-report statistics. - # - statline {Percentage of total database} $total_pages_percent - statline {Number of entries} $nentry - statline {Bytes of storage consumed} $storage - if {$compressed_size!=$storage} { - set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}] - set pct [expr {$compressed_size*100.0/$storage}] - set pct [format {%5.1f%%} $pct] - statline {Bytes used after compression} $compressed_size $pct - } - statline {Bytes of payload} $payload $payload_percent - statline {Bytes of metadata} $total_meta $meta_percent - if {$cnt==1} {statline {B-tree depth} $depth} - statline {Average payload per entry} $avg_payload - statline {Average unused bytes per entry} $avg_unused - statline {Average metadata per entry} $avg_meta - if {[info exists avg_fanout]} { - statline {Average fanout} $avg_fanout - } - if {$showFrag && $total_pages>1} { - set fragmentation [percent $gap_cnt [expr {$total_pages-1}]] - statline {Non-sequential pages} $gap_cnt $fragmentation - } - statline {Maximum payload per entry} $mx_payload - statline {Entries that use overflow} $ovfl_cnt $ovfl_cnt_percent - if {$int_pages>0} { - statline {Index pages used} $int_pages - } - statline {Primary pages used} $leaf_pages - statline {Overflow pages used} $ovfl_pages - statline {Total pages used} $total_pages - if {$int_unused>0} { - set int_unused_percent [ - percent $int_unused [expr {$int_pages*$pageSize}] {of index space}] - statline "Unused bytes on index pages" $int_unused $int_unused_percent - } - statline "Unused bytes on primary pages" $leaf_unused [ - percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}] - statline "Unused bytes on overflow pages" $ovfl_unused [ - percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}] - statline "Unused bytes on all pages" $total_unused [ - percent $total_unused $storage {of all space}] - return 1 -} - -# Calculate the overhead in pages caused by auto-vacuum. -# -# This procedure calculates and returns the number of pages used by the -# auto-vacuum 'pointer-map'. If the database does not support auto-vacuum, -# then 0 is returned. The two arguments are the size of the database file in -# pages and the page size used by the database (in bytes). -proc autovacuum_overhead {filePages pageSize} { - - # Set $autovacuum to non-zero for databases that support auto-vacuum. - set autovacuum [db one {PRAGMA auto_vacuum}] - - # If the database is not an auto-vacuum database or the file consists - # of one page only then there is no overhead for auto-vacuum. Return zero. - if {0==$autovacuum || $filePages==1} { - return 0 - } - - # The number of entries on each pointer map page. The layout of the - # database file is one pointer-map page, followed by $ptrsPerPage other - # pages, followed by a pointer-map page etc. The first pointer-map page - # is the second page of the file overall. - set ptrsPerPage [expr double($pageSize/5)] - - # Return the number of pointer map pages in the database. - return [expr wide(ceil( ($filePages-1.0)/($ptrsPerPage+1.0) ))] -} - - -# Calculate the summary statistics for the database and store the results -# in TCL variables. They are output below. Variables are as follows: -# -# pageSize: Size of each page in bytes. -# file_bytes: File size in bytes. -# file_pgcnt: Number of pages in the file. -# file_pgcnt2: Number of pages in the file (calculated). -# av_pgcnt: Pages consumed by the auto-vacuum pointer-map. -# av_percent: Percentage of the file consumed by auto-vacuum pointer-map. -# inuse_pgcnt: Data pages in the file. -# inuse_percent: Percentage of pages used to store data. -# free_pgcnt: Free pages calculated as ( - ) -# free_pgcnt2: Free pages in the file according to the file header. -# free_percent: Percentage of file consumed by free pages (calculated). -# free_percent2: Percentage of file consumed by free pages (header). -# ntable: Number of tables in the db. -# nindex: Number of indices in the db. -# nautoindex: Number of indices created automatically. -# nmanindex: Number of indices created manually. -# user_payload: Number of bytes of payload in table btrees -# (not including sqlite_master) -# user_percent: $user_payload as a percentage of total file size. - -### The following, setting $file_bytes based on the actual size of the file -### on disk, causes this tool to choke on zipvfs databases. So set it based -### on the return of [PRAGMA page_count] instead. -if 0 { - set file_bytes [file size $file_to_analyze] - set file_pgcnt [expr {$file_bytes/$pageSize}] -} -set file_pgcnt [db one {PRAGMA page_count}] -set file_bytes [expr {$file_pgcnt * $pageSize}] - -set av_pgcnt [autovacuum_overhead $file_pgcnt $pageSize] -set av_percent [percent $av_pgcnt $file_pgcnt] - -set sql {SELECT sum(leaf_pages+int_pages+ovfl_pages) FROM space_used} -set inuse_pgcnt [expr wide([mem eval $sql])] -set inuse_percent [percent $inuse_pgcnt $file_pgcnt] - -set free_pgcnt [expr {$file_pgcnt-$inuse_pgcnt-$av_pgcnt}] -set free_percent [percent $free_pgcnt $file_pgcnt] -set free_pgcnt2 [db one {PRAGMA freelist_count}] -set free_percent2 [percent $free_pgcnt2 $file_pgcnt] - -set file_pgcnt2 [expr {$inuse_pgcnt+$free_pgcnt2+$av_pgcnt}] - -set ntable [db eval {SELECT count(*)+1 FROM sqlite_master WHERE type='table'}] -set nindex [db eval {SELECT count(*) FROM sqlite_master WHERE type='index'}] -set sql {SELECT count(*) FROM sqlite_master WHERE name LIKE 'sqlite_autoindex%'} -set nautoindex [db eval $sql] -set nmanindex [expr {$nindex-$nautoindex}] - -# set total_payload [mem eval "SELECT sum(payload) FROM space_used"] -set user_payload [mem one {SELECT int(sum(payload)) FROM space_used - WHERE NOT is_index AND name NOT LIKE 'sqlite_master'}] -set user_percent [percent $user_payload $file_bytes] - -# Output the summary statistics calculated above. -# -puts "/** Disk-Space Utilization Report For $root_filename" -puts "" -statline {Page size in bytes} $pageSize -statline {Pages in the whole file (measured)} $file_pgcnt -statline {Pages in the whole file (calculated)} $file_pgcnt2 -statline {Pages that store data} $inuse_pgcnt $inuse_percent -statline {Pages on the freelist (per header)} $free_pgcnt2 $free_percent2 -statline {Pages on the freelist (calculated)} $free_pgcnt $free_percent -statline {Pages of auto-vacuum overhead} $av_pgcnt $av_percent -statline {Number of tables in the database} $ntable -statline {Number of indices} $nindex -statline {Number of defined indices} $nmanindex -statline {Number of implied indices} $nautoindex -if {$isCompressed} { - statline {Size of uncompressed content in bytes} $file_bytes - set efficiency [percent $true_file_size $file_bytes] - statline {Size of compressed file on disk} $true_file_size $efficiency -} else { - statline {Size of the file in bytes} $file_bytes -} -statline {Bytes of user payload stored} $user_payload $user_percent - -# Output table rankings -# -puts "" -titleline "Page counts for all tables with their indices" -puts "" -mem eval {SELECT tblname, count(*) AS cnt, - int(sum(int_pages+leaf_pages+ovfl_pages)) AS size - FROM space_used GROUP BY tblname ORDER BY size+0 DESC, tblname} {} { - statline [string toupper $tblname] $size [percent $size $file_pgcnt] -} -puts "" -titleline "Page counts for all tables and indices separately" -puts "" -mem eval { - SELECT - upper(name) AS nm, - int(int_pages+leaf_pages+ovfl_pages) AS size - FROM space_used - ORDER BY size+0 DESC, name} {} { - statline $nm $size [percent $size $file_pgcnt] -} -if {$isCompressed} { - puts "" - titleline "Bytes of disk space used after compression" - puts "" - set csum 0 - mem eval {SELECT tblname, - int(sum(compressed_size)) + - $compressOverhead*sum(int_pages+leaf_pages+ovfl_pages) - AS csize - FROM space_used GROUP BY tblname ORDER BY csize+0 DESC, tblname} {} { - incr csum $csize - statline [string toupper $tblname] $csize [percent $csize $true_file_size] - } - set overhead [expr {$true_file_size - $csum}] - if {$overhead>0} { - statline {Header and free space} $overhead [percent $overhead $true_file_size] - } -} - -# Output subreports -# -if {$nindex>0} { - subreport {All tables and indices} 1 0 -} -subreport {All tables} {NOT is_index} 0 -if {$nindex>0} { - subreport {All indices} {is_index} 0 -} -foreach tbl [mem eval {SELECT DISTINCT tblname name FROM space_used - ORDER BY name}] { - set qn [quote $tbl] - set name [string toupper $tbl] - set n [mem eval {SELECT count(*) FROM space_used WHERE tblname=$tbl}] - if {$n>1} { - set idxlist [mem eval "SELECT name FROM space_used - WHERE tblname='$qn' AND is_index - ORDER BY 1"] - subreport "Table $name and all its indices" "tblname='$qn'" 0 - subreport "Table $name w/o any indices" "name='$qn'" 1 - if {[llength $idxlist]>1} { - subreport "Indices of table $name" "tblname='$qn' AND is_index" 0 - } - foreach idx $idxlist { - set qidx [quote $idx] - subreport "Index [string toupper $idx] of table $name" "name='$qidx'" 1 - } - } else { - subreport "Table $name" "name='$qn'" 1 - } -} - -# Output instructions on what the numbers above mean. -# -puts "" -titleline Definitions -puts { -Page size in bytes - - The number of bytes in a single page of the database file. - Usually 1024. - -Number of pages in the whole file -} -puts " The number of $pageSize-byte pages that go into forming the complete - database" -puts { -Pages that store data - - The number of pages that store data, either as primary B*Tree pages or - as overflow pages. The number at the right is the data pages divided by - the total number of pages in the file. - -Pages on the freelist - - The number of pages that are not currently in use but are reserved for - future use. The percentage at the right is the number of freelist pages - divided by the total number of pages in the file. - -Pages of auto-vacuum overhead - - The number of pages that store data used by the database to facilitate - auto-vacuum. This is zero for databases that do not support auto-vacuum. - -Number of tables in the database - - The number of tables in the database, including the SQLITE_MASTER table - used to store schema information. - -Number of indices - - The total number of indices in the database. - -Number of defined indices - - The number of indices created using an explicit CREATE INDEX statement. - -Number of implied indices - - The number of indices used to implement PRIMARY KEY or UNIQUE constraints - on tables. - -Size of the file in bytes - - The total amount of disk space used by the entire database files. - -Bytes of user payload stored - - The total number of bytes of user payload stored in the database. The - schema information in the SQLITE_MASTER table is not counted when - computing this number. The percentage at the right shows the payload - divided by the total file size. - -Percentage of total database - - The amount of the complete database file that is devoted to storing - information described by this category. - -Number of entries - - The total number of B-Tree key/value pairs stored under this category. - -Bytes of storage consumed - - The total amount of disk space required to store all B-Tree entries - under this category. The is the total number of pages used times - the pages size. - -Bytes of payload - - The amount of payload stored under this category. Payload is the data - part of table entries and the key part of index entries. The percentage - at the right is the bytes of payload divided by the bytes of storage - consumed. - -Bytes of metadata - - The amount of formatting and structural information stored in the - table or index. Metadata includes the btree page header, the cell pointer - array, the size field for each cell, the left child pointer or non-leaf - cells, the overflow pointers for overflow cells, and the rowid value for - rowid table cells. In other words, metadata is everything that is neither - unused space nor content. The record header in the payload is counted as - content, not metadata. - -Average payload per entry - - The average amount of payload on each entry. This is just the bytes of - payload divided by the number of entries. - -Average unused bytes per entry - - The average amount of free space remaining on all pages under this - category on a per-entry basis. This is the number of unused bytes on - all pages divided by the number of entries. - -Non-sequential pages - - The number of pages in the table or index that are out of sequence. - Many filesystems are optimized for sequential file access so a small - number of non-sequential pages might result in faster queries, - especially for larger database files that do not fit in the disk cache. - Note that after running VACUUM, the root page of each table or index is - at the beginning of the database file and all other pages are in a - separate part of the database file, resulting in a single non- - sequential page. - -Maximum payload per entry - - The largest payload size of any entry. - -Entries that use overflow - - The number of entries that user one or more overflow pages. - -Total pages used - - This is the number of pages used to hold all information in the current - category. This is the sum of index, primary, and overflow pages. - -Index pages used - - This is the number of pages in a table B-tree that hold only key (rowid) - information and no data. - -Primary pages used - - This is the number of B-tree pages that hold both key and data. - -Overflow pages used - - The total number of overflow pages used for this category. - -Unused bytes on index pages - - The total number of bytes of unused space on all index pages. The - percentage at the right is the number of unused bytes divided by the - total number of bytes on index pages. - -Unused bytes on primary pages - - The total number of bytes of unused space on all primary pages. The - percentage at the right is the number of unused bytes divided by the - total number of bytes on primary pages. - -Unused bytes on overflow pages - - The total number of bytes of unused space on all overflow pages. The - percentage at the right is the number of unused bytes divided by the - total number of bytes on overflow pages. - -Unused bytes on all pages - - The total number of bytes of unused space on all primary and overflow - pages. The percentage at the right is the number of unused bytes - divided by the total number of bytes. -} - -# Output a dump of the in-memory database. This can be used for more -# complex offline analysis. -# -titleline {} -puts "The entire text of this report can be sourced into any SQL database" -puts "engine for further analysis. All of the text above is an SQL comment." -puts "The data used to generate this report follows:" -puts "*/" -puts "BEGIN;" -puts $tabledef -unset -nocomplain x -mem eval {SELECT * FROM space_used} x { - puts -nonewline "INSERT INTO space_used VALUES" - set sep ( - foreach col $x(*) { - set v $x($col) - if {$v=="" || ![string is double $v]} {set v '[quote $v]'} - puts -nonewline $sep$v - set sep , - } - puts ");" -} -puts "COMMIT;" - -} err]} { - puts "ERROR: $err" - puts $errorInfo - exit 1 -} +INCLUDE $ROOT/tool/spaceanal.tcl END_STRING ; } From 835cd436e1e0a964d84034a88303a1b4c0e8083c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 14 Oct 2017 19:54:06 +0000 Subject: [PATCH 14/22] Updates to the Makefiles for MSVC. FossilOrigin-Name: ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 --- Makefile.msc | 4 ++-- autoconf/Makefile.msc | 5 +++-- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index ce867ba3f3..2804c25ab4 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1984,8 +1984,8 @@ SHELL_SRC = \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\completion.c -shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl - $(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl > shell.c +shell.c: $(SHELL_SRC) $(TOP)\tool\mkshellc.tcl + $(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c # Rules to build the extension objects. diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index f535d1450a..e0f7ad8615 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -928,6 +928,7 @@ LIBRESOBJS = # !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB !ENDIF @@ -957,8 +958,8 @@ sqlite3.def: Replace.exe $(LIBOBJ) | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ | sort >> sqlite3.def -$(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) - $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \ +$(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) + $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \ /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) diff --git a/manifest b/manifest index 2599e89765..1ce91e738e 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Improved\sability\sto\sgenerate\sstand-alone\sprogram\susing\sTCL\sand\sSQLite\sby\ncompiling\swith\s-DTCLSH_INIT_PROC=name\sto\scause\sthe\sTCL\sinterpreter\sto\sbe\ninitialized\susing\sprocedure\sname().\s\sBoth\ssqlite3_analyzer\sand\stestfixture\nare\snow\sbuilt\sthis\sway. -D 2017-10-13T20:14:06.865 +C Updates\sto\sthe\sMakefiles\sfor\sMSVC. +D 2017-10-14T19:54:06.773 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc a341cc7d737f596a1074e47fc9eb17bde689413d891e3ef2f26e98126da950a4 +F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 -F autoconf/Makefile.msc 55328895d224f1b81298224929a16b8601baa75f9d7badd493224f68a22700dd +F autoconf/Makefile.msc 645b8a9774281515dc4a8de65d8a914f7b418ba8bd1c48b53ccbf43d3b339715 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82 -R df005e0c0c347d89af378f70d82f057b -U drh -Z 8b50e8b5594af446543ab2fc55fee9a2 +P d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4 +R 49fa7e54acb9ad8a392e6c25fe8116d5 +U mistachkin +Z 3270bf7f35234fd560a264ffe40cbf35 diff --git a/manifest.uuid b/manifest.uuid index d905d370b6..63b3b0cf2b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4 \ No newline at end of file +ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 \ No newline at end of file From 47b1d68f25f2c921497c6a642a111d2d8e7d00cc Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 15 Oct 2017 22:16:25 +0000 Subject: [PATCH 15/22] In the query planner, do not consider index X to be a proper subset of index Y if X is a covering index but Y is not. FossilOrigin-Name: ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 17 ++++++++++------- test/analyze9.test | 5 ++++- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 1ce91e738e..b77b1484ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\sMakefiles\sfor\sMSVC. -D 2017-10-14T19:54:06.773 +C In\sthe\squery\splanner,\sdo\snot\sconsider\sindex\sX\sto\sbe\sa\sproper\ssubset\sof\s\nindex\sY\sif\sX\sis\sa\scovering\sindex\sbut\sY\sis\snot. +D 2017-10-15T22:16:25.229 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -544,7 +544,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 -F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5 +F src/where.c 248f0dc6085b37b4b2ccad2de06b90f1a48b1932403ab0ad9100a1f2a6990763 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1 F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4 @@ -568,7 +568,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test c05a461d0a6b05991106467d0c47480f2e709c82 -F test/analyze9.test b817b8e798315fc65b820a5463f73ad5f48ed8dd +F test/analyze9.test dac0bdc7eab965b9ad639ca83564d98717aaf13ce5a776f23d9a3680238cecd8 F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a F test/analyzeB.test a4c1c3048f6d9e090eb76e83eecb18bcf6d31a70 F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4 -R 49fa7e54acb9ad8a392e6c25fe8116d5 -U mistachkin -Z 3270bf7f35234fd560a264ffe40cbf35 +P ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 +R 88b26813bcaf999746e4cf55503b4797 +U drh +Z fe6d8478e3f31cb2dfbec6ce858d4ad8 diff --git a/manifest.uuid b/manifest.uuid index 63b3b0cf2b..12179a84a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 \ No newline at end of file +ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 5a3d44f9ee..95bbdb157e 100644 --- a/src/where.c +++ b/src/where.c @@ -1885,16 +1885,15 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** Return TRUE if all of the following are true: ** ** (1) X has the same or lower cost that Y -** (2) X is a proper subset of Y -** (3) X skips at least as many columns as Y -** -** By "proper subset" we mean that X uses fewer WHERE clause terms -** than Y and that every WHERE clause term used by X is also used -** by Y. +** (2) X users fewer WHERE clause terms than Y +** (3) Every WHERE clause term used by X is also used by Y +** (4) X skips at least as many columns as Y +** (5) If X is a covering index, than Y is too ** +** Conditions (2) and (3) mean that X is a "proper subset" of Y. ** If X is a proper subset of Y then Y is a better choice and ought ** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. The third rule +** relationship is inverted and needs to be adjusted. Constraint (4) ** was added because if X uses skip-scan less than Y it still might ** deserve a lower cost even if it is a proper subset of Y. */ @@ -1918,6 +1917,10 @@ static int whereLoopCheaperProperSubset( } if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ } + if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 + && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){ + return 0; /* Constraint (5) */ + } return 1; /* All conditions meet */ } diff --git a/test/analyze9.test b/test/analyze9.test index 918577bd19..7c69806f89 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -1052,8 +1052,11 @@ do_execsql_test 23.0 { do_eqp_test 23.1 { SELECT * FROM t4 WHERE (e=1 AND b='xyz' AND c='zyx' AND a<'AEA') AND f<300 + -- Formerly used index i41. But i41 is not a covering index whereas + -- the PRIMARY KEY is a covering index, and so as of 2017-10-15, the + -- PRIMARY KEY is preferred. } { - 0 0 0 {SEARCH TABLE t4 USING INDEX i41 (e=? AND c=? AND b=? AND a Date: Mon, 16 Oct 2017 11:50:12 +0000 Subject: [PATCH 16/22] Improved comments, including a typo fix, on the whereLoopCheaperProperSubset() function. FossilOrigin-Name: 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b77b1484ce..ab44e57d39 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\squery\splanner,\sdo\snot\sconsider\sindex\sX\sto\sbe\sa\sproper\ssubset\sof\s\nindex\sY\sif\sX\sis\sa\scovering\sindex\sbut\sY\sis\snot. -D 2017-10-15T22:16:25.229 +C Improved\scomments,\sincluding\sa\stypo\sfix,\son\sthe\swhereLoopCheaperProperSubset()\nfunction. +D 2017-10-16T11:50:12.951 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -544,7 +544,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 -F src/where.c 248f0dc6085b37b4b2ccad2de06b90f1a48b1932403ab0ad9100a1f2a6990763 +F src/where.c d8c6d690c4b11f30211de073011fe19352364a6303ae053f45cb66f9576ba8a9 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1 F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 -R 88b26813bcaf999746e4cf55503b4797 +P ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d +R 593b5394fda7f5c3a7e7e5312c0a0c37 U drh -Z fe6d8478e3f31cb2dfbec6ce858d4ad8 +Z d6213a58e95c0fa7e25d95c2efa3d8ac diff --git a/manifest.uuid b/manifest.uuid index 12179a84a0..3281f881f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d \ No newline at end of file +700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f \ No newline at end of file diff --git a/src/where.c b/src/where.c index 95bbdb157e..6021edf47f 100644 --- a/src/where.c +++ b/src/where.c @@ -1885,7 +1885,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** Return TRUE if all of the following are true: ** ** (1) X has the same or lower cost that Y -** (2) X users fewer WHERE clause terms than Y +** (2) X uses fewer WHERE clause terms than Y ** (3) Every WHERE clause term used by X is also used by Y ** (4) X skips at least as many columns as Y ** (5) If X is a covering index, than Y is too @@ -1895,7 +1895,9 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** to have a lower cost. This routine returns TRUE when that cost ** relationship is inverted and needs to be adjusted. Constraint (4) ** was added because if X uses skip-scan less than Y it still might -** deserve a lower cost even if it is a proper subset of Y. +** deserve a lower cost even if it is a proper subset of Y. Constraint (5) +** was added because a covering index probably deserves to have a lower cost +** than a non-covering index even if it is a proper subset. */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ From f30d345241ac0d260c5a8034b1366c5bee596ee3 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 17 Oct 2017 13:44:46 +0000 Subject: [PATCH 17/22] Fix the ".check" command of the command-line shell to return 1 (error) rather than 2 (exit) on a test failure. FossilOrigin-Name: e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ab44e57d39..de1b01a07e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments,\sincluding\sa\stypo\sfix,\son\sthe\swhereLoopCheaperProperSubset()\nfunction. -D 2017-10-16T11:50:12.951 +C Fix\sthe\s".check"\scommand\sof\sthe\scommand-line\sshell\sto\sreturn\s1\s(error)\nrather\sthan\s2\s(exit)\son\sa\stest\sfailure. +D 2017-10-17T13:44:46.075 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -462,7 +462,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 -F src/shell.c.in 5446de0a90c15d713bbdb5827cf57ec30d1c3497097f39ec2c2e874dcca34ca3 +F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d -R 593b5394fda7f5c3a7e7e5312c0a0c37 +P 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f +R 7b11fa4b75f3d72b968740b4ad577de5 U drh -Z d6213a58e95c0fa7e25d95c2efa3d8ac +Z eb279a84da6335af377cd536eb14b148 diff --git a/manifest.uuid b/manifest.uuid index 3281f881f9..cd80d78996 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f \ No newline at end of file +e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index b62c055076..8f5ed59e93 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4248,7 +4248,7 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(stderr, "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", p->zTestcase, azArg[1], zRes); - rc = 2; + rc = 1; }else{ utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase); p->nCheck++; From f7fc4c22f45adcf20fa88febad0a0e046a04f9ae Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 17 Oct 2017 18:33:22 +0000 Subject: [PATCH 18/22] The source code files generated for the 'sqlite3-all.c' target should not have Windows line-endings. FossilOrigin-Name: 6a08c43431be18a08bdcbf33d327513f72fff72dac5d02103dab8399d8c3d668 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/split-sqlite3c.tcl | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index de1b01a07e..d6e49a102d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s".check"\scommand\sof\sthe\scommand-line\sshell\sto\sreturn\s1\s(error)\nrather\sthan\s2\s(exit)\son\sa\stest\sfailure. -D 2017-10-17T13:44:46.075 +C The\ssource\scode\sfiles\sgenerated\sfor\sthe\s'sqlite3-all.c'\starget\sshould\snot\shave\sWindows\sline-endings. +D 2017-10-17T18:33:22.646 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -1630,7 +1630,7 @@ F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd -F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c +F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb F tool/sqlite3_analyzer.c.in 771d15fb9c67645fd8ef932a438f98959da4b7c7da3cb87ae1850b27c969edf3 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f -R 7b11fa4b75f3d72b968740b4ad577de5 -U drh -Z eb279a84da6335af377cd536eb14b148 +P e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d +R d1f05de03c82774074759a8848af421f +U mistachkin +Z cf3e5ebd3225c433ae20a29b56e94ef7 diff --git a/manifest.uuid b/manifest.uuid index cd80d78996..4c5b7a09c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d \ No newline at end of file +6a08c43431be18a08bdcbf33d327513f72fff72dac5d02103dab8399d8c3d668 \ No newline at end of file diff --git a/tool/split-sqlite3c.tcl b/tool/split-sqlite3c.tcl index 287b752828..230e3f2549 100644 --- a/tool/split-sqlite3c.tcl +++ b/tool/split-sqlite3c.tcl @@ -15,6 +15,7 @@ set END {^/\*+ End of %s \*+/} set in [open sqlite3.c] set out1 [open sqlite3-all.c w] +fconfigure $out1 -translation lf # Copy the header from sqlite3.c into sqlite3-all.c # @@ -48,6 +49,7 @@ proc write_one_file {content} { global filecnt incr filecnt set out [open sqlite3-$filecnt.c w] + fconfigure $out -translation lf puts -nonewline $out $content close $out puts $::out1 "#include \"sqlite3-$filecnt.c\"" From af124faf3e6e6284161b84e68f7e13235bd6afb4 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 18 Oct 2017 14:29:36 +0000 Subject: [PATCH 19/22] Avoid using snprintf as a member variable of a struct in sqlite3ext.h. This causes problems on OSX. FossilOrigin-Name: b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/sqlite3ext.h | 4 ++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index de1b01a07e..365c4e7bb8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s".check"\scommand\sof\sthe\scommand-line\sshell\sto\sreturn\s1\s(error)\nrather\sthan\s2\s(exit)\son\sa\stest\sfailure. -D 2017-10-17T13:44:46.075 +C Avoid\susing\ssnprintf\sas\sa\smember\svariable\sof\sa\sstruct\sin\ssqlite3ext.h.\sThis\ncauses\sproblems\son\sOSX. +D 2017-10-18T14:29:36.881 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -465,7 +465,7 @@ F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 +F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 @@ -1664,7 +1664,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f -R 7b11fa4b75f3d72b968740b4ad577de5 -U drh -Z eb279a84da6335af377cd536eb14b148 +P e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d +R af758af6d9338e0da5d96870c08db747 +T *branch * sqlite3ext-fix +T *sym-sqlite3ext-fix * +T -sym-trunk * +U dan +Z d17bbc38b957a75c44d151990964eb3a diff --git a/manifest.uuid b/manifest.uuid index cd80d78996..73db2c4c9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d \ No newline at end of file +b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c \ No newline at end of file diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 4aade278bd..d1d2c574ae 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -134,7 +134,7 @@ struct sqlite3_api_routines { int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, const char*,const char*),void*); void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); - char * (*snprintf)(int,char*,const char*,...); + char * (*xsnprintf)(int,char*,const char*,...); int (*step)(sqlite3_stmt*); int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, char const**,char const**,int*,int*,int*); @@ -418,7 +418,7 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_rollback_hook sqlite3_api->rollback_hook #define sqlite3_set_authorizer sqlite3_api->set_authorizer #define sqlite3_set_auxdata sqlite3_api->set_auxdata -#define sqlite3_snprintf sqlite3_api->snprintf +#define sqlite3_snprintf sqlite3_api->xsnprintf #define sqlite3_step sqlite3_api->step #define sqlite3_table_column_metadata sqlite3_api->table_column_metadata #define sqlite3_thread_cleanup sqlite3_api->thread_cleanup From f2f72a0f9fe7125285f233011bb10257fa5a6a88 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 19 Oct 2017 15:17:38 +0000 Subject: [PATCH 20/22] Take extra care to avoid an OOB read caused by a corrupt b-tree page. FossilOrigin-Name: 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b --- manifest | 18 +++---- manifest.uuid | 2 +- src/btree.c | 13 +++-- test/corruptK.test | 109 +++++++++++++++++++++++++++++++++++++++++ test/permutations.test | 2 +- 5 files changed, 129 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 75ecef5321..5c8ae4d13c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\susing\ssnprintf\sas\sa\smember\svariable\sof\sa\sstruct\sin\ssqlite3ext.h.\sThis\ncauses\sproblems\son\sOSX.\sSimilar\sto\sfix\s[a1fd3aa8]. -D 2017-10-18T15:02:01.361 +C Take\sextra\scare\sto\savoid\san\sOOB\sread\scaused\sby\sa\scorrupt\sb-tree\spage. +D 2017-10-19T15:17:38.752 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -403,7 +403,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 8565b061a6a6fad850230c73d6a7a8ffb88f3370e3352a8689a9a672160c5cc5 +F src/btree.c 75229a5a47985997f861b428552acd14fe42b657f755cba5e0b1a007bd77b2ea F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc F src/build.c 6ffe76970aeee4bc94e60cf0138269e67109061a853e13098c38a904dd66e673 @@ -690,7 +690,7 @@ F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 -F test/corruptK.test 814a59ec699d8546b4e29005fba3d16e933ef2fe +F test/corruptK.test 251ef631d095e882d455d2183961fa9ba879b4156e18e96c5d2b84aa7ef5f7a9 F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c @@ -1105,7 +1105,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 53fb87f0a345259f884fa8b5fc62ee327b20be378ebe5149833b1b28c58a42f6 +F test/permutations.test 490e3333b9b1aefb7ebc6e9ab2ae0e382b7dd8713ccc4a2786b0f75467c2ab6b F test/pragma.test c31b5e98998c160a4c85b1e04f590655c67f2daa7f73854640cd120610e3ac15 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1664,8 +1664,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6a08c43431be18a08bdcbf33d327513f72fff72dac5d02103dab8399d8c3d668 b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c -R bae812ccbd328c244f2fa888e3ba625e -T +closed b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c +P cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46 +R 14641e8822b464d39bf472f899e19240 +T +closed 9dd591ef24b302a5fe2af0619d0cda6733348bacc541b3c0a134ac25981d4b2a U dan -Z 4879d15d8231001302e10b2e0a3c2891 +Z 576e55698c218b4128bbad5de39dc1b0 diff --git a/manifest.uuid b/manifest.uuid index 2f3fcd4b40..26a2375f60 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46 \ No newline at end of file +04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 89222197bc..ddcb6cfd35 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4824,7 +4824,7 @@ static const void *fetchPayload( BtCursor *pCur, /* Cursor pointing to entry to read from */ u32 *pAmt /* Write the number of available bytes here */ ){ - u32 amt; + int amt; assert( pCur!=0 && pCur->iPage>=0 && pCur->pPage); assert( pCur->eState==CURSOR_VALID ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); @@ -4833,9 +4833,14 @@ static const void *fetchPayload( assert( pCur->info.nSize>0 ); assert( pCur->info.pPayload>pCur->pPage->aData || CORRUPT_DB ); assert( pCur->info.pPayloadpPage->aDataEnd ||CORRUPT_DB); - amt = (int)(pCur->pPage->aDataEnd - pCur->info.pPayload); - if( pCur->info.nLocalinfo.nLocal; - *pAmt = amt; + amt = pCur->info.nLocal; + if( amt>(int)(pCur->pPage->aDataEnd - pCur->info.pPayload) ){ + /* There is too little space on the page for the expected amount + ** of local content. Database must be corrupt. */ + assert( CORRUPT_DB ); + amt = MAX(0, (int)(pCur->pPage->aDataEnd - pCur->info.pPayload)); + } + *pAmt = (u32)amt; return (void*)pCur->info.pPayload; } diff --git a/test/corruptK.test b/test/corruptK.test index b20c2d8bf0..b921176974 100644 --- a/test/corruptK.test +++ b/test/corruptK.test @@ -107,6 +107,115 @@ do_catchsql_test 2.3 { INSERT INTO t1 VALUES(randomblob(900)); } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- + +proc hex2blob {hex} { + # Split on newlines: + set bytes [list] + foreach l [split $hex "\n"] { + if {[string is space $l]} continue + set L [list] + foreach b [split $l] { + if {[string is xdigit $b] && [string length $b]==2} { + lappend L [expr "0x$b"] + } + } + if {[llength $L]!=16} { + error "Badly formed hex (1)" + } + set bytes [concat $bytes $L] + } + + binary format c* $bytes +} + +reset_db +db func hex2blob hex2blob + +do_execsql_test 3.1 { + PRAGMA page_size=1024; + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(a, b, c); + CREATE TABLE t3(a, b, c); + CREATE TABLE t4(a, b, c); + CREATE TABLE t5(a, b, c); +} + +do_execsql_test 3.2 { + UPDATE sqlite_dbpage SET data = hex2blob(' + 000: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. + 010: 04 00 01 01 20 40 20 20 00 00 3e d9 00 00 00 06 .... @ ..>..... + 020: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ + 030: 0f 00 00 00 00 00 00 00 00 00 00 01 00 00 83 00 ................ + 040: 00 00 00 00 00 00 00 00 00 00 00 00 00 38 00 00 .............8.. + 050: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3e d9 ..............>. + 060: 00 2d e6 07 0d 00 00 00 01 03 a0 00 03 e0 00 00 .-.............. + 070: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 090: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0d0: 00 00 00 00 00 c1 00 00 00 00 00 00 00 00 00 00 ................ + 0e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 100: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 120: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 130: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 140: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 150: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 160: 00 83 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 170: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 180: 00 00 00 00 00 00 00 00 00 00 07 00 30 00 00 00 ............0... + 190: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1c0: 02 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 ................ + 1d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1f0: 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 200: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 210: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 220: 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 230: 0c 00 00 00 00 00 00 60 00 00 00 06 00 00 c3 00 .......`........ + 240: 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 250: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 260: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 270: 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 280: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 290: 04 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2b0: 00 00 00 00 83 00 8c 00 00 00 00 00 00 00 00 00 ................ + 2c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 310: 00 78 00 00 00 00 00 00 00 00 00 00 00 00 70 00 .x............p. + 320: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 330: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 340: 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 350: 00 00 00 00 00 68 00 00 00 00 00 00 00 00 00 00 .....h.......... + 360: 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 ................ + 370: 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 ................ + 380: 00 00 00 00 70 00 00 00 00 00 00 00 00 00 00 00 ....p........... + 390: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 3a0: 5e 01 07 17 1b 1b 01 81 13 74 61 62 6c 65 73 65 ^........tablese + 3b0: 6e 73 6f 32 73 73 65 6e 73 6f 72 73 02 43 52 45 nso2ssensors.CRE + 3c0: 41 54 45 20 54 41 42 4c 45 20 73 65 6e 73 6f 72 ATE TABLE sensor + 3d0: 73 20 0a 20 20 24 20 20 20 20 20 20 20 20 20 20 s . $ + 3e0: b8 6e 61 6d 65 21 74 65 78 74 2c 20 79 61 6c 20 .name!text, yal + 3f0: 72 65 61 6c 2c 20 74 69 6d 65 20 74 65 78 74 29 real, time text) + ') WHERE pgno=1 +} + +db close +sqlite3 db test.db + +do_catchsql_test 3.3 { + PRAGMA integrity_check; +} {1 {database disk image is malformed}} diff --git a/test/permutations.test b/test/permutations.test index 1e04265668..76fb72ace5 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -194,7 +194,7 @@ test_suite "valgrind" -prefix "" -description { } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* *_err* wal.test \ shell*.test crash8.test atof1.test selectG.test \ - tkt-fc62af4523.test numindex1.test + tkt-fc62af4523.test numindex1.test corruptK.test ] -initialize { set ::G(valgrind) 1 } -shutdown { From 316ab93e2c92526d0fd2b42f53b75be32e6a0428 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 21 Oct 2017 13:29:26 +0000 Subject: [PATCH 21/22] Remove unnecessary "#if SQLITE_MAX_COLUMN". SQLITE_MAX_COLUMN is always defined. FossilOrigin-Name: 6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a --- manifest | 19 +++++++++---------- manifest.uuid | 2 +- src/build.c | 2 -- src/resolve.c | 4 ---- src/select.c | 2 -- 5 files changed, 10 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 5c8ae4d13c..cd878b65fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\sextra\scare\sto\savoid\san\sOOB\sread\scaused\sby\sa\scorrupt\sb-tree\spage. -D 2017-10-19T15:17:38.752 +C Remove\sunnecessary\s"#if\sSQLITE_MAX_COLUMN".\s\sSQLITE_MAX_COLUMN\sis\salways\ndefined. +D 2017-10-21T13:29:26.479 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -406,7 +406,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 75229a5a47985997f861b428552acd14fe42b657f755cba5e0b1a007bd77b2ea F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc -F src/build.c 6ffe76970aeee4bc94e60cf0138269e67109061a853e13098c38a904dd66e673 +F src/build.c e24b61144f9c9b15c4aa05954514190361061da721e56dcd1af6e0e945ee5909 F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 @@ -459,9 +459,9 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 +F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 +F src/select.c 78b81b0d0f8ba2445e4de5ca3c97a9fd317240a9c5e4994887d4ae8a1d5a3367 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1664,8 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46 -R 14641e8822b464d39bf472f899e19240 -T +closed 9dd591ef24b302a5fe2af0619d0cda6733348bacc541b3c0a134ac25981d4b2a -U dan -Z 576e55698c218b4128bbad5de39dc1b0 +P 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b +R 3fe93e66991cd1a89b7cde18365d4335 +U drh +Z 82fbf688aff072892ed2c0d525c25911 diff --git a/manifest.uuid b/manifest.uuid index 26a2375f60..14db0bd9b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b \ No newline at end of file +6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a \ No newline at end of file diff --git a/src/build.c b/src/build.c index cb8c374d1a..a9428104a3 100644 --- a/src/build.c +++ b/src/build.c @@ -1063,12 +1063,10 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ Column *pCol; sqlite3 *db = pParse->db; if( (p = pParse->pNewTable)==0 ) return; -#if SQLITE_MAX_COLUMN if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName); return; } -#endif z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2); if( z==0 ) return; memcpy(z, pName->z, pName->n); diff --git a/src/resolve.c b/src/resolve.c index 78f37512a2..4a1e8284c8 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -959,12 +959,10 @@ static int resolveCompoundOrderBy( pOrderBy = pSelect->pOrderBy; if( pOrderBy==0 ) return 0; db = pParse->db; -#if SQLITE_MAX_COLUMN if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); return 1; } -#endif for(i=0; inExpr; i++){ pOrderBy->a[i].done = 0; } @@ -1056,12 +1054,10 @@ int sqlite3ResolveOrderGroupBy( struct ExprList_item *pItem; if( pOrderBy==0 || pParse->db->mallocFailed ) return 0; -#if SQLITE_MAX_COLUMN if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); return 1; } -#endif pEList = pSelect->pEList; assert( pEList!=0 ); /* sqlite3SelectNew() guarantees this */ for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ diff --git a/src/select.c b/src/select.c index 6f524fa2e1..9639cfde1a 100644 --- a/src/select.c +++ b/src/select.c @@ -4596,12 +4596,10 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3ExprListDelete(db, pEList); p->pEList = pNew; } -#if SQLITE_MAX_COLUMN if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many columns in result set"); return WRC_Abort; } -#endif return WRC_Continue; } From 6fe3733ba97193227a395c7796bb9ca0d6f82134 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 21 Oct 2017 14:17:31 +0000 Subject: [PATCH 22/22] Catch and avoid a 16-bit integer overflow on the number of columns in a common table expression. This fixes a problem found by OSS-Fuzz. The test case is in TH3. FossilOrigin-Name: 6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index cd878b65fe..da985ae44f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunnecessary\s"#if\sSQLITE_MAX_COLUMN".\s\sSQLITE_MAX_COLUMN\sis\salways\ndefined. -D 2017-10-21T13:29:26.479 +C Catch\sand\savoid\sa\s16-bit\sinteger\soverflow\son\sthe\snumber\sof\scolumns\sin\sa\ncommon\stable\sexpression.\s\sThis\sfixes\sa\sproblem\sfound\sby\sOSS-Fuzz.\s\sThe\ntest\scase\sis\sin\sTH3. +D 2017-10-21T14:17:31.555 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -461,7 +461,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 78b81b0d0f8ba2445e4de5ca3c97a9fd317240a9c5e4994887d4ae8a1d5a3367 +F src/select.c e6a068d9ea54417d625578086d3d482284af8d5a449bb3593d40c257080806a8 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b -R 3fe93e66991cd1a89b7cde18365d4335 +P 6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a +R 201775ba1e14aa2d5f8d0e57defde230 U drh -Z 82fbf688aff072892ed2c0d525c25911 +Z bad3386d6737a5472f5b033811b6e996 diff --git a/manifest.uuid b/manifest.uuid index 14db0bd9b2..60279d9206 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a \ No newline at end of file +6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 9639cfde1a..0e2328120b 100644 --- a/src/select.c +++ b/src/select.c @@ -1689,6 +1689,7 @@ int sqlite3ColumnsFromExprList( nCol = pEList->nExpr; aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); testcase( aCol==0 ); + if( nCol>32767 ) nCol = 32767; }else{ nCol = 0; aCol = 0;