From 41113b6429967b290386c05a4cbf6b9efc632b7a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 5 Apr 2016 21:07:58 +0000 Subject: [PATCH 01/63] Defer opening the file used for the temp database (where CREATE TEMP TABLE tables are stored) until the database is too large to reside entirely within the cache. There are likely still problems on this branch. FossilOrigin-Name: be5a549eba6cf8e29cb6b9824fd6d0db9d03ca7f --- manifest | 20 +++--- manifest.uuid | 2 +- src/pager.c | 24 +++++-- src/pcache.c | 15 ++++- test/temptable2.test | 157 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 200 insertions(+), 18 deletions(-) create mode 100644 test/temptable2.test diff --git a/manifest b/manifest index 04532b4880..261df5d6bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Carry\stable\scolumn\stypes\sthrough\sinto\sVIEW\sdefinitions,\swhere\spossible. -D 2016-04-05T20:59:12.069 +C Defer\sopening\sthe\sfile\sused\sfor\sthe\stemp\sdatabase\s(where\sCREATE\sTEMP\sTABLE\stables\sare\sstored)\suntil\sthe\sdatabase\sis\stoo\slarge\sto\sreside\sentirely\swithin\sthe\scache.\sThere\sare\slikely\sstill\sproblems\son\sthis\sbranch. +D 2016-04-05T21:07:58.179 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -362,10 +362,10 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c b1ccb273771f41dbdbe0ba7c1ad63c38ad5972ec F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 38718a019ca762ba4f6795425d5a54db70d1790d +F src/pager.c 123dbae47bf27915a4b567915ff8dcc27cfcd369 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821 -F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df +F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84 @@ -1110,6 +1110,7 @@ F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 +F test/temptable2.test f3b198e386f6494082c87accc8780cdf84b36e6d F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1482,7 +1483,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0bf9926c7a7865694edd48535777248e73d86bbf -R c7f5a7683cc4a0d14362e20e6891847a -U drh -Z ec8e2cff7764d808576889c190bbf60c +P fb555c3c2af7f5e62ff839658f4fba7b645d3a68 +R f511e5baaeaad693345498b91b98e90f +T *branch * tempfiles-lazy-open +T *sym-tempfiles-lazy-open * +T -sym-trunk * +U dan +Z 505ba6148fb8f2c79237e86b07655c1c diff --git a/manifest.uuid b/manifest.uuid index d21138d9ad..0e8a9ef130 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb555c3c2af7f5e62ff839658f4fba7b645d3a68 \ No newline at end of file +be5a549eba6cf8e29cb6b9824fd6d0db9d03ca7f \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index c18b3a32f7..45f54ef30d 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2006,7 +2006,11 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - sqlite3PcacheCleanAll(pPager->pPCache); + if( pPager->tempFile==0 || MEMDB ){ + sqlite3PcacheCleanAll(pPager->pPCache); + }else{ + sqlite3PcacheClearWritable(pPager->pPCache); + } sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( pagerUseWal(pPager) ){ @@ -2374,9 +2378,13 @@ static int pager_playback_one_page( ** be written out into the database file before its journal file ** segment is synced. If a crash occurs during or following this, ** database corruption may ensue. + ** + ** Update: Another exception is for temp files that are not + ** in-memory databases. In this case the page may have been dirty + ** at the start of the transaction. */ assert( !pagerUseWal(pPager) ); - sqlite3PcacheMakeClean(pPg); + if( pPager->tempFile==0 ) sqlite3PcacheMakeClean(pPg); } pager_set_pagehash(pPg); @@ -4258,8 +4266,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); assert( pPager->eLock==EXCLUSIVE_LOCK ); + assert( pPager->tempFile==0 || pList->pDirty==0 ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -5941,6 +5950,7 @@ int sqlite3PagerWrite(PgHdr *pPg){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ + assert( pPager->tempFile==0 ); return pagerWriteLargeSector(pPg); }else{ return pager_write(pPg); @@ -6179,11 +6189,11 @@ int sqlite3PagerCommitPhaseOne( /* If no database changes have been made, return early. */ if( pPager->eStatetempFile ); + if( pPager->tempFile ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any - ** backup in progress needs to be restarted. - */ + ** backup in progress needs to be restarted. */ sqlite3BackupRestart(pPager->pBackup); }else{ if( pagerUseWal(pPager) ){ @@ -6845,7 +6855,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ - if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit && pPager->tempFile==0 ){ needSyncPgno = pPg->pgno; assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || pageInJournal(pPager, pPg) || pPg->pgno>pPager->dbOrigSize ); diff --git a/src/pcache.c b/src/pcache.c index 61e7587422..52661c28a9 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -254,7 +254,7 @@ sqlite3_pcache_page *sqlite3PcacheFetch( /* ** If the sqlite3PcacheFetch() routine is unable to allocate a new -** page because new clean pages are available for reuse and the cache +** page because no clean pages are available for reuse and the cache ** size limit has been reached, then this routine can be invoked to ** try harder to allocate a page. This routine might invoke the stress ** callback to spill dirty pages to the journal. It will then try to @@ -439,6 +439,17 @@ void sqlite3PcacheCleanAll(PCache *pCache){ } } +/* +** Clear the PGHDR_NEED_SYNC and PGHDR_WRITEABLE flag from all dirty pages. +*/ +void sqlite3PcacheClearWritable(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + } + pCache->pSynced = pCache->pDirtyTail; +} + /* ** Clear the PGHDR_NEED_SYNC flag from all dirty pages. */ @@ -484,7 +495,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ ** it must be that pgno==0. */ assert( p->pgno>0 ); - if( ALWAYS(p->pgno>pgno) ){ + if( p->pgno>pgno ){ assert( p->flags&PGHDR_DIRTY ); sqlite3PcacheMakeClean(p); } diff --git a/test/temptable2.test b/test/temptable2.test new file mode 100644 index 0000000000..daeb206a29 --- /dev/null +++ b/test/temptable2.test @@ -0,0 +1,157 @@ +# 2016 March 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix temptable2 + +do_execsql_test 1.1 { + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); +} + +do_execsql_test 1.2 { + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100000 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM X; +} {} + +do_execsql_test 1.3 { + PRAGMA temp.integrity_check; +} {ok} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.1 { + CREATE TEMP TABLE t2(a, b); + INSERT INTO t2 VALUES(1, 2); +} {} + +do_execsql_test 2.2 { + BEGIN; + INSERT INTO t2 VALUES(3, 4); + SELECT * FROM t2; +} {1 2 3 4} + +do_execsql_test 2.3 { + ROLLBACK; + SELECT * FROM t2; +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.1.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + + SELECT count(*) FROM t1; +} {1000} +do_execsql_test 3.1.2 { + BEGIN; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0; + ROLLBACK; +} +do_execsql_test 3.1.3 { + SELECT count(*) FROM t1; +} {1000} +do_execsql_test 3.1.4 { PRAGMA temp.integrity_check } {ok} + +do_execsql_test 3.2.1 { + BEGIN; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0; + SAVEPOINT abc; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==1; + ROLLBACK TO abc; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==2; + COMMIT; +} +do_execsql_test 3.2.2 { PRAGMA temp.integrity_check } {ok} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.1.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<10 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + + SELECT count(*) FROM t1; + PRAGMA temp.page_count; +} {10 9} + +do_execsql_test 4.1.2 { + BEGIN; + UPDATE t1 SET b=randomblob(100); + ROLLBACK; +} + +do_execsql_test 4.1.3 { + CREATE TEMP TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; + + SELECT count(*) FROM t2; + SELECT count(*) FROM t1; + PRAGMA temp.page_count; +} {500 10 292} + +do_execsql_test 4.1.4 { PRAGMA temp.integrity_check } {ok} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.1.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; + + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES(1, 2); + + PRAGMA temp.page_count; +} {286} + +do_execsql_test 5.1.2 { + BEGIN; + UPDATE t1 SET a=2; + UPDATE t2 SET a=randomblob(100); + SELECT count(*) FROM t1; + ROLLBACK; +} {1} + +do_execsql_test 5.1.3 { + UPDATE t2 SET a=randomblob(100); + + SELECT * FROM t1; +} {1 2} + +do_execsql_test 5.1.4 { PRAGMA temp.integrity_check } {ok} + +finish_test + From d87efd7221c3a4edefe09bd2c18bf29b042ad542 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Apr 2016 15:39:03 +0000 Subject: [PATCH 02/63] Add tests to this branch. Fix a problem with temporary databases in auto-vacuum mode. FossilOrigin-Name: afe9bd9b4b5dc98dccf1bced80567515ab5c0117 --- manifest | 17 ++++++------- manifest.uuid | 2 +- src/pager.c | 8 +++--- test/temptable2.test | 58 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 261df5d6bf..0084c64a0f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Defer\sopening\sthe\sfile\sused\sfor\sthe\stemp\sdatabase\s(where\sCREATE\sTEMP\sTABLE\stables\sare\sstored)\suntil\sthe\sdatabase\sis\stoo\slarge\sto\sreside\sentirely\swithin\sthe\scache.\sThere\sare\slikely\sstill\sproblems\son\sthis\sbranch. -D 2016-04-05T21:07:58.179 +C Add\stests\sto\sthis\sbranch.\sFix\sa\sproblem\swith\stemporary\sdatabases\sin\sauto-vacuum\smode. +D 2016-04-06T15:39:03.038 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c b1ccb273771f41dbdbe0ba7c1ad63c38ad5972ec F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 123dbae47bf27915a4b567915ff8dcc27cfcd369 +F src/pager.c 685f3cc08d045f64d20133144fb9a437914ea4c7 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821 F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 @@ -1110,7 +1110,7 @@ F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test f3b198e386f6494082c87accc8780cdf84b36e6d +F test/temptable2.test c16f96e996bf6f587a4df4199c62cf0ac9a1c0ea F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1483,10 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb555c3c2af7f5e62ff839658f4fba7b645d3a68 -R f511e5baaeaad693345498b91b98e90f -T *branch * tempfiles-lazy-open -T *sym-tempfiles-lazy-open * -T -sym-trunk * +P be5a549eba6cf8e29cb6b9824fd6d0db9d03ca7f +R 71d9b48ce9c812093890a1f227fcf740 U dan -Z 505ba6148fb8f2c79237e86b07655c1c +Z 749bb77cf880284a6ac38e76d6c94ae0 diff --git a/manifest.uuid b/manifest.uuid index 0e8a9ef130..f61492cdc3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be5a549eba6cf8e29cb6b9824fd6d0db9d03ca7f \ No newline at end of file +afe9bd9b4b5dc98dccf1bced80567515ab5c0117 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 45f54ef30d..3d8df3cdbb 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6815,7 +6815,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. */ - if( MEMDB ){ + if( pPager->tempFile ){ rc = sqlite3PagerWrite(pPg); if( rc ) return rc; } @@ -6855,7 +6855,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ - if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit && pPager->tempFile==0 ){ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit /* && pPager->tempFile==0 */ ){ needSyncPgno = pPg->pgno; assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || pageInJournal(pPager, pPg) || pPg->pgno>pPager->dbOrigSize ); @@ -6872,7 +6872,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); - if( MEMDB ){ + if( pPager->tempFile ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ sqlite3PcacheMove(pPgOld, pPager->dbSize+1); @@ -6889,7 +6889,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** to exist, in case the transaction needs to roll back. Use pPgOld ** as the original page since it has already been allocated. */ - if( MEMDB ){ + if( pPager->tempFile ){ assert( pPgOld ); sqlite3PcacheMove(pPgOld, origPgno); sqlite3PagerUnrefNotNull(pPgOld); diff --git a/test/temptable2.test b/test/temptable2.test index daeb206a29..e702410f68 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -153,5 +153,63 @@ do_execsql_test 5.1.3 { do_execsql_test 5.1.4 { PRAGMA temp.integrity_check } {ok} +#------------------------------------------------------------------------- +# Test this: +# +# 1. Page is DIRTY at the start of a transaction. +# 2. Page is written out as part of the transaction. +# 3. Page is then read back in. +# 4. Transaction is rolled back. Is the page now clean or dirty? +# +# This actually does work. Step 4 marks the page as clean. But it also +# writes to the database file itself. So marking it clean is correct - +# the page does match the contents of the db file. +# +reset_db + +do_execsql_test 6.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t1(x); + INSERT INTO t1 VALUES('one'); + + CREATE TEMP TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; +} + +do_execsql_test 6.2 { + UPDATE t1 SET x='two'; -- step 1 + BEGIN; + UPDATE t2 SET a=randomblob(100); -- step 2 + SELECT * FROM t1; -- step 3 + ROLLBACK; -- step 4 + + SELECT count(*) FROM t2; + SELECT * FROM t1; +} {two 500 two} + +#------------------------------------------------------------------------- + +reset_db +sqlite3 db "" +do_execsql_test 7.1 { + PRAGMA auto_vacuum=INCREMENTAL; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(zeroblob(900)); + INSERT INTO t1 VALUES(zeroblob(900)); + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1 SELECT x FROM t1; + BEGIN; + DELETE FROM t1 WHERE rowid%2; + PRAGMA incremental_vacuum(4); + ROLLBACK; + PRAGMA integrity_check; +} {ok} + finish_test From 7f8def285efbf12c337050ac81788e38345909c6 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Apr 2016 16:27:50 +0000 Subject: [PATCH 03/63] Fix a problem in test script lock.test. Fix minor issues in pager.c. FossilOrigin-Name: 84de8813c3b4007e3c7c3a286ce13020f2065c7b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 5 +++-- test/lock.test | 5 +++-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0084c64a0f..fefd0eeca8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\sthis\sbranch.\sFix\sa\sproblem\swith\stemporary\sdatabases\sin\sauto-vacuum\smode. -D 2016-04-06T15:39:03.038 +C Fix\sa\sproblem\sin\stest\sscript\slock.test.\sFix\sminor\sissues\sin\spager.c. +D 2016-04-06T16:27:50.054 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c b1ccb273771f41dbdbe0ba7c1ad63c38ad5972ec F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 685f3cc08d045f64d20133144fb9a437914ea4c7 +F src/pager.c 2047e837a5df26c16271e77e6b30021d0cdc98b7 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821 F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 @@ -877,7 +877,7 @@ F test/like3.test 3608a2042b6f922f900fbfd5d3ce4e7eca57f7c4 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e F test/loadext.test 648cb95f324d1775c54a55c12271b2d1156b633b F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 -F test/lock.test b984ab9034e7389be0d863fe4e64cbbc4d2028f5 +F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P be5a549eba6cf8e29cb6b9824fd6d0db9d03ca7f -R 71d9b48ce9c812093890a1f227fcf740 +P afe9bd9b4b5dc98dccf1bced80567515ab5c0117 +R 35c99f2c80a6bf5e586b897ba6d7e956 U dan -Z 749bb77cf880284a6ac38e76d6c94ae0 +Z 67c50ed57a780aa24194c52733684bdf diff --git a/manifest.uuid b/manifest.uuid index f61492cdc3..832abb0052 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afe9bd9b4b5dc98dccf1bced80567515ab5c0117 \ No newline at end of file +84de8813c3b4007e3c7c3a286ce13020f2065c7b \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 3d8df3cdbb..c1c1134047 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6855,7 +6855,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ - if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit /* && pPager->tempFile==0 */ ){ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ needSyncPgno = pPg->pgno; assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || pageInJournal(pPager, pPg) || pPg->pgno>pPager->dbOrigSize ); @@ -7142,7 +7142,8 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ ** Unless this is an in-memory or temporary database, clear the pager cache. */ void sqlite3PagerClearCache(Pager *pPager){ - if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); + assert( MEMDB==0 || pPager->tempFile ); + if( pPager->tempFile==0 ) pager_reset(pPager); } #endif diff --git a/test/lock.test b/test/lock.test index f785c4bb23..534aa3b9a4 100644 --- a/test/lock.test +++ b/test/lock.test @@ -423,8 +423,9 @@ do_test lock-6.5 { # * there exists one or more active read-only statements, and # * a transaction that modified zero database pages is committed. # -set temp_status unlocked -if {$TEMP_STORE>=2} {set temp_status unknown} +#set temp_status unlocked +#if {$TEMP_STORE>=2} {set temp_status unknown} +set temp_status unknown do_test lock-7.1 { set STMT [sqlite3_prepare $DB "SELECT * FROM sqlite_master" -1 TAIL] sqlite3_step $STMT From 9131ab93865d961f6408800f969e3996d78db34e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Apr 2016 18:20:51 +0000 Subject: [PATCH 04/63] For a pager backed by a temp file, store the main journal in memory until it is at least sqlite3_config.nStmtSpill bytes in size. Prevent the backup API from being used to change the page-size of a temp file. FossilOrigin-Name: 84c557010c211595d2ec80b62c63af1c7f4714bd --- manifest | 18 +++++------ manifest.uuid | 2 +- src/pager.c | 30 +++++++++--------- src/test3.c | 6 ++-- test/pragma.test | 15 ++++++++- test/temptable2.test | 74 +++++++++++++++++++++++++++++++++++++++----- 6 files changed, 110 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index fefd0eeca8..103a4b00bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\stest\sscript\slock.test.\sFix\sminor\sissues\sin\spager.c. -D 2016-04-06T16:27:50.054 +C For\sa\spager\sbacked\sby\sa\stemp\sfile,\sstore\sthe\smain\sjournal\sin\smemory\suntil\sit\sis\sat\sleast\ssqlite3_config.nStmtSpill\sbytes\sin\ssize.\sPrevent\sthe\sbackup\sAPI\sfrom\sbeing\sused\sto\schange\sthe\spage-size\sof\sa\stemp\sfile. +D 2016-04-06T18:20:51.565 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c b1ccb273771f41dbdbe0ba7c1ad63c38ad5972ec F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 2047e837a5df26c16271e77e6b30021d0cdc98b7 +F src/pager.c 91d7462aa2429e30d64c705f75ba36cace90c241 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821 F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 @@ -387,7 +387,7 @@ F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 F src/tclsqlite.c 56569acc73d36e836b64aefecbbb709a92ba0077 F src/test1.c 7187b7e924bfc97780e6fd2a40dad94a32bddca0 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b -F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f +F src/test3.c 0df6f8dbb4cbaa7106397c70a271fa6a43659042 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 F src/test6.c 41cacf3b0dd180823919bf9e1fbab287c9266723 @@ -976,7 +976,7 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff F test/permutations.test cd1fa041074ed08eeaa563e4d1bacb0c69337ec1 -F test/pragma.test afbf028be1c35b68f57db8eb015c4a3c59d8f28e +F test/pragma.test 36daccdbfbdd958a5f65c034141ab2bfd959808e F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 3f1984a04657331f838df5c519b443c2088df922 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc @@ -1110,7 +1110,7 @@ F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test c16f96e996bf6f587a4df4199c62cf0ac9a1c0ea +F test/temptable2.test 707c01c00b4a8fd050e0db5dcef448e998b9ddb5 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P afe9bd9b4b5dc98dccf1bced80567515ab5c0117 -R 35c99f2c80a6bf5e586b897ba6d7e956 +P 84de8813c3b4007e3c7c3a286ce13020f2065c7b +R 54ab030014fa87b230c7ab4f5fdc139b U dan -Z 67c50ed57a780aa24194c52733684bdf +Z dbab9fc4f54135fb156cdc6ccf0d8253 diff --git a/manifest.uuid b/manifest.uuid index 832abb0052..515a3c2874 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84de8813c3b4007e3c7c3a286ce13020f2065c7b \ No newline at end of file +84c557010c211595d2ec80b62c63af1c7f4714bd \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index c1c1134047..79dbaec5c2 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1170,6 +1170,8 @@ static int jrnlBufferSize(Pager *pPager){ return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); } +#else +# define jrnlBufferSize(x) 0 #endif /* @@ -5562,24 +5564,24 @@ static int pager_open_journal(Pager *pPager){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3MemJournalOpen(pPager->jfd); }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) - ); + int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + int nSpill; + if( pPager->tempFile ){ + flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + nSpill = sqlite3Config.nStmtSpill; + }else{ + flags |= SQLITE_OPEN_MAIN_JOURNAL; + nSpill = jrnlBufferSize(pPager); + } + /* Verify that the database still has the same name as it did when ** it was originally opened. */ rc = databaseIsUnmoved(pPager); if( rc==SQLITE_OK ){ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + rc = sqlite3JournalOpen ( + pVfs, pPager->zJournal, pPager->jfd, flags, nSpill ); -#else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); -#endif } } assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); @@ -6532,10 +6534,10 @@ void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ } /* -** Return true if this is an in-memory pager. +** Return true if this is an in-memory or temp-file backed pager. */ int sqlite3PagerIsMemdb(Pager *pPager){ - return MEMDB; + return pPager->tempFile; } /* diff --git a/src/test3.c b/src/test3.c index 2a41068e5f..817e8a88fd 100644 --- a/src/test3.c +++ b/src/test3.c @@ -547,7 +547,7 @@ static int btree_from_db( /* ** Usage: btree_ismemdb ID ** -** Return true if the B-Tree is in-memory. +** Return true if the B-Tree is currently stored entirely in memory. */ static int btree_ismemdb( void *NotUsed, @@ -557,6 +557,7 @@ static int btree_ismemdb( ){ Btree *pBt; int res; + sqlite3_file *pFile; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], @@ -566,7 +567,8 @@ static int btree_ismemdb( pBt = sqlite3TestTextToPtr(argv[1]); sqlite3_mutex_enter(pBt->db->mutex); sqlite3BtreeEnter(pBt); - res = sqlite3PagerIsMemdb(sqlite3BtreePager(pBt)); + pFile = sqlite3PagerFile(sqlite3BtreePager(pBt)); + res = (pFile->pMethods==0); sqlite3BtreeLeave(pBt); sqlite3_mutex_leave(pBt->db->mutex); Tcl_SetObjResult(interp, Tcl_NewBooleanObj(res)); diff --git a/test/pragma.test b/test/pragma.test index acbe74599e..2c73090551 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1083,7 +1083,20 @@ do_test pragma-8.2.15 { # "memory" or "disk" as appropriate. # proc check_temp_store {} { - db eval {CREATE TEMP TABLE IF NOT EXISTS a(b)} + db eval { + PRAGMA temp.cache_size = 1; + CREATE TEMP TABLE IF NOT EXISTS a(b); + DELETE FROM a; + INSERT INTO a VALUES(randomblob(1000)); + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + } db eval {PRAGMA database_list} { if {$name=="temp"} { set bt [btree_from_db db 1] diff --git a/test/temptable2.test b/test/temptable2.test index e702410f68..e87ffa89d9 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -113,8 +113,12 @@ do_execsql_test 4.1.3 { SELECT count(*) FROM t2; SELECT count(*) FROM t1; - PRAGMA temp.page_count; -} {500 10 292} +} {500 10} + +do_test 4.1.4 { + set n [db one { PRAGMA temp.page_count }] + expr ($n >280 && $n < 300) +} 1 do_execsql_test 4.1.4 { PRAGMA temp.integrity_check } {ok} @@ -133,11 +137,14 @@ do_execsql_test 5.1.1 { CREATE TEMP TABLE t1(a, b); CREATE INDEX i1 ON t1(a, b); INSERT INTO t1 VALUES(1, 2); +} - PRAGMA temp.page_count; -} {286} +do_test 5.1.2 { + set n [db one { PRAGMA temp.page_count }] + expr ($n > 280 && $n < 290) +} {1} -do_execsql_test 5.1.2 { +do_execsql_test 5.1.3 { BEGIN; UPDATE t1 SET a=2; UPDATE t2 SET a=randomblob(100); @@ -145,13 +152,13 @@ do_execsql_test 5.1.2 { ROLLBACK; } {1} -do_execsql_test 5.1.3 { +do_execsql_test 5.1.4 { UPDATE t2 SET a=randomblob(100); SELECT * FROM t1; } {1 2} -do_execsql_test 5.1.4 { PRAGMA temp.integrity_check } {ok} +do_execsql_test 5.1.5 { PRAGMA temp.integrity_check } {ok} #------------------------------------------------------------------------- # Test this: @@ -192,7 +199,7 @@ do_execsql_test 6.2 { } {two 500 two} #------------------------------------------------------------------------- - +# reset_db sqlite3 db "" do_execsql_test 7.1 { @@ -211,5 +218,56 @@ do_execsql_test 7.1 { PRAGMA integrity_check; } {ok} +#------------------------------------------------------------------------- +# Try changing the page size using a backup operation when pages are +# stored in main-memory only. +# +reset_db +do_execsql_test 8.1 { + CREATE TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<20 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; + PRAGMA page_count; +} {13} + +do_test 8.2 { + sqlite3 tmp "" + execsql { + PRAGMA page_size = 8192; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + PRAGMA page_count; + } tmp +} {10} + +do_test 8.3 { + sqlite3_backup B tmp main db main + B step 5 + B finish +} {SQLITE_READONLY} + +do_test 8.4 { + execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + PRAGMA page_size; + } tmp +} {100 ok 8192} + +do_test 8.5 { + tmp eval { UPDATE t1 SET a=randomblob(100) } +} {} + +do_test 8.6 { + sqlite3_backup B tmp main db main + B step 1000 + B finish +} {SQLITE_READONLY} + +tmp close + finish_test From 67330a1224271e0186445c8f68754b214991df3a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Apr 2016 18:07:47 +0000 Subject: [PATCH 05/63] Fixes for OOM and IO error handling with temp file databases. FossilOrigin-Name: 4eb06e843af60d5e533793618c6e0e9b7ef7a1a6 --- manifest | 15 +++--- manifest.uuid | 2 +- src/pager.c | 15 +++--- test/cffault.test | 2 +- test/tempfault.test | 114 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 15 deletions(-) create mode 100644 test/tempfault.test diff --git a/manifest b/manifest index 2cc5625474..1c7823a2ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthis\sbranch\swith\sthe\slatest\schanges\sfrom\sthe\strunk. -D 2016-04-11T09:39:25.021 +C Fixes\sfor\sOOM\sand\sIO\serror\shandling\swith\stemp\sfile\sdatabases. +D 2016-04-11T18:07:47.205 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bde4844f0849cab5924c6a81178f8500774ce76b F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 91d7462aa2429e30d64c705f75ba36cace90c241 +F src/pager.c 4cc8a5b3979db65725eaa03650a6d54219122b8d F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 52cdeb4f37634d0ccd2998aab099b7bbb690b0d3 F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 @@ -551,7 +551,7 @@ F test/capi3c.test 0b9edb0c2156a964b9271cd5ea7ae56736cc2fcb F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 -F test/cffault.test aadc1f61f8811cb600e3e069acbf8796f472a096 +F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 85a84bfc4be0e83f668747211c7cd45a6721d485 F test/close.test 83947daf3b700631f90f4850ddaab455be4af73d F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 @@ -1109,6 +1109,7 @@ F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 +F test/tempfault.test 0c80f23c651cf6452c27cacf09f4c2ee9527a912 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 707c01c00b4a8fd050e0db5dcef448e998b9ddb5 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 @@ -1483,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 84c557010c211595d2ec80b62c63af1c7f4714bd ef1966c2469a0f5dbdb31a0287bd37badb2b8f28 -R 95ac2720b81ae4f698b1e227fe8fc71b +P 982b753d0b6a3ed9fba33ed41523b2cd42280276 +R 46e56336ea4184d65ce3043234f11f03 U dan -Z 4e2c4b80dd3173e2cb79a80f72f7a7a8 +Z 58afa17a4e51610cdca345488b854923 diff --git a/manifest.uuid b/manifest.uuid index 9b0cbf0915..9e8a4101e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -982b753d0b6a3ed9fba33ed41523b2cd42280276 \ No newline at end of file +4eb06e843af60d5e533793618c6e0e9b7ef7a1a6 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 79dbaec5c2..9baddc2986 100644 --- a/src/pager.c +++ b/src/pager.c @@ -958,7 +958,7 @@ static int assert_pager_state(Pager *p){ ** back to OPEN state. */ assert( pPager->errCode!=SQLITE_OK ); - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 || pPager->tempFile ); break; } @@ -1819,11 +1819,14 @@ static void pager_unlock(Pager *pPager){ ** trusted. Now that there are no outstanding references to the pager, ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. - */ - if( pPager->errCode ){ - assert( !MEMDB ); + ** + ** Exception: There is no way out of the error state for temp files. + ** This is because it is not possible to call pager_reset() on a temp + ** file pager (as this may discard the only copy of some data). */ + assert( pPager->errCode==SQLITE_OK || !MEMDB ); + if( pPager->tempFile==0 && pPager->errCode ){ pager_reset(pPager); - pPager->changeCountDone = pPager->tempFile; + pPager->changeCountDone = 0; pPager->eState = PAGER_OPEN; pPager->errCode = SQLITE_OK; if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); @@ -5034,8 +5037,8 @@ int sqlite3PagerSharedLock(Pager *pPager){ */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); + if( pPager->tempFile && pPager->errCode ) { return pPager->errCode; } assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); - if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ diff --git a/test/cffault.test b/test/cffault.test index 79cefd24c4..0d029ece37 100644 --- a/test/cffault.test +++ b/test/cffault.test @@ -15,7 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix cacheflush +set testprefix cffault source $testdir/malloc_common.tcl # Run the supplied SQL on a copy of the database currently stored on diff --git a/test/tempfault.test b/test/tempfault.test new file mode 100644 index 0000000000..da1ae12375 --- /dev/null +++ b/test/tempfault.test @@ -0,0 +1,114 @@ +# 2016 April 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for fault-injection when SQLite is used with +# a temp file database. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix tempfault + +sqlite3_memdebug_vfs_oom_test 0 + +do_faultsim_test 1 -faults oom* -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } +} -body { + execsql { INSERT INTO t1 VALUES(5, 6) } +} -test { + faultsim_test_result {0 {}} + set rc [catch { execsql { SELECT * FROM t1 } } msg] + if {$rc==0 && $msg != "1 2 3 4 5 6" && $msg != "1 2 3 4"} { + error "data mismatch 1: $msg" + } + if {$testrc==0 && $msg != "1 2 3 4 5 6"} { + error "data mismatch 2: $msg" + } + faultsim_integrity_check +} + +do_faultsim_test 2 -faults oom* -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } +} -body { + execsql { UPDATE t1 SET a = randomblob(99) } +} -test { + faultsim_test_result {0 {}} + faultsim_integrity_check db +} + +do_faultsim_test 3 -faults oom* -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } +} -body { + execsql { + BEGIN; + UPDATE t1 SET a = randomblob(99); + SAVEPOINT abc; + UPDATE t1 SET a = randomblob(98) WHERE (rowid%10)==0; + ROLLBACK TO abc; + UPDATE t1 SET a = randomblob(97) WHERE (rowid%5)==0; + ROLLBACK TO abc; + COMMIT; + } +} -test { + faultsim_test_result {0 {}} + faultsim_integrity_check db +} + +do_faultsim_test 4 -faults * -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } +} -body { + execsql { + BEGIN; + UPDATE t1 SET a = randomblob(99); + SAVEPOINT abc; + UPDATE t1 SET a = randomblob(98) WHERE (rowid%10)==0; + ROLLBACK TO abc; + UPDATE t1 SET a = randomblob(97) WHERE (rowid%5)==0; + ROLLBACK TO abc; + COMMIT; + } +} -test { + faultsim_test_result {0 {}} +} + +sqlite3_memdebug_vfs_oom_test 1 +finish_test From b9f11f932dd6b924d5707222c43a72ea2291464a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Apr 2016 18:49:37 +0000 Subject: [PATCH 06/63] Add extra tests to temptable2.test. FossilOrigin-Name: 7dd9d4c15bb6b6ef9127dc7c80a189c55e714dc9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/temptable2.test | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 1c7823a2ff..58157eb4f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sOOM\sand\sIO\serror\shandling\swith\stemp\sfile\sdatabases. -D 2016-04-11T18:07:47.205 +C Add\sextra\stests\sto\stemptable2.test. +D 2016-04-11T18:49:37.527 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -1111,7 +1111,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c80f23c651cf6452c27cacf09f4c2ee9527a912 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test 707c01c00b4a8fd050e0db5dcef448e998b9ddb5 +F test/temptable2.test f04c411560dadebcc44aba631c602c908dfe1dd8 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 982b753d0b6a3ed9fba33ed41523b2cd42280276 -R 46e56336ea4184d65ce3043234f11f03 +P 4eb06e843af60d5e533793618c6e0e9b7ef7a1a6 +R d2dfbf56b62d35c4cf496af1896eae37 U dan -Z 58afa17a4e51610cdca345488b854923 +Z db78b11117984d67b6a3aa7fe90754f9 diff --git a/manifest.uuid b/manifest.uuid index 9e8a4101e2..66abf69ec2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4eb06e843af60d5e533793618c6e0e9b7ef7a1a6 \ No newline at end of file +7dd9d4c15bb6b6ef9127dc7c80a189c55e714dc9 \ No newline at end of file diff --git a/test/temptable2.test b/test/temptable2.test index e87ffa89d9..af566ab529 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -269,5 +269,42 @@ do_test 8.6 { tmp close +#------------------------------------------------------------------------- +# Try inserts and deletes with a large db in auto-vacuum mode. Check +# +reset_db +do_execsql_test 9.1 { + PRAGMA cache_size = 15; + CREATE TABLE tx(a, b); + CREATE INDEX i1 ON tx(a); + CREATE INDEX i2 ON tx(b); + + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; +} +for {set i 2} {$i<20} {incr i} { + + do_execsql_test 9.$i.1 { DELETE FROM tx WHERE (random()%3)==0 } + + do_execsql_test 9.$i.2 { PRAGMA integrity_check } ok + + do_execsql_test 9.$i.3 { + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<400 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; + } + + do_execsql_test 9.$i.4 { PRAGMA integrity_check } ok + + do_execsql_test 9.$i.5 { + BEGIN; + DELETE FROM tx WHERE (random()%3)==0; + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; + COMMIT; + } + + do_execsql_test 9.$i.6 { PRAGMA integrity_check } ok +} + finish_test From b5a2592a9e1625b84b975d2bfbd2b8587bf58185 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Apr 2016 19:23:36 +0000 Subject: [PATCH 07/63] Add tests for wal mode to temptable2.test. FossilOrigin-Name: c6d0d441a1ab4873caedf2f23543444cea5d7f5a --- manifest | 12 +++++----- manifest.uuid | 2 +- test/temptable2.test | 55 ++++++++++++++++++++++++++------------------ 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index d5aafb59c4..017d301d54 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2016-04-11T18:50:25.500 +C Add\stests\sfor\swal\smode\sto\stemptable2.test. +D 2016-04-11T19:23:36.940 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -1111,7 +1111,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c80f23c651cf6452c27cacf09f4c2ee9527a912 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test f04c411560dadebcc44aba631c602c908dfe1dd8 +F test/temptable2.test 79a30edb79c4d5a4916f208f97935a70efd4b3ba F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7dd9d4c15bb6b6ef9127dc7c80a189c55e714dc9 2764aeaa11f38cf2ff4d6191e6d5466ddb203022 -R 7b4c1f71edaf782a86682a72496a355b +P ffc62af1d503c6e7fc0059d9c2ee57a3bef9dd32 +R b5aaac24011d4d477c13603957856121 U dan -Z 4c9d4d3f9ec993332df2230efd24eb4f +Z 91cb540a3f9a28d737c21f7feeaff7cc diff --git a/manifest.uuid b/manifest.uuid index 0dd130d179..fa8dceac08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffc62af1d503c6e7fc0059d9c2ee57a3bef9dd32 \ No newline at end of file +c6d0d441a1ab4873caedf2f23543444cea5d7f5a \ No newline at end of file diff --git a/test/temptable2.test b/test/temptable2.test index af566ab529..44a148ddfb 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -272,38 +272,47 @@ tmp close #------------------------------------------------------------------------- # Try inserts and deletes with a large db in auto-vacuum mode. Check # -reset_db -do_execsql_test 9.1 { - PRAGMA cache_size = 15; - CREATE TABLE tx(a, b); - CREATE INDEX i1 ON tx(a); - CREATE INDEX i2 ON tx(b); +foreach {tn mode} { + 1 delete + 2 wal +} { + reset_db + do_execsql_test 9.$tn.1.1 { + PRAGMA cache_size = 15; + PRAGMA auto_vacuum = 1; + } + do_execsql_test 9.$tn.1.2 "PRAGMA journal_mode = $mode" $mode - WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) - INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; -} -for {set i 2} {$i<20} {incr i} { - - do_execsql_test 9.$i.1 { DELETE FROM tx WHERE (random()%3)==0 } - - do_execsql_test 9.$i.2 { PRAGMA integrity_check } ok - - do_execsql_test 9.$i.3 { - WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<400 ) + do_execsql_test 9.$tn.1.3 { + CREATE TABLE tx(a, b); + CREATE INDEX i1 ON tx(a); + CREATE INDEX i2 ON tx(b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; } - do_execsql_test 9.$i.4 { PRAGMA integrity_check } ok + for {set i 2} {$i<20} {incr i} { + do_execsql_test 9.$tn.$i.1 { DELETE FROM tx WHERE (random()%3)==0 } - do_execsql_test 9.$i.5 { - BEGIN; + do_execsql_test 9.$tn.$i.2 { PRAGMA integrity_check } ok + + do_execsql_test 9.$tn.$i.3 { + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<400 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; + } + + do_execsql_test 9.$tn.$i.4 { PRAGMA integrity_check } ok + + do_execsql_test 9.$tn.$i.5 { + BEGIN; DELETE FROM tx WHERE (random()%3)==0; WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; - COMMIT; - } + COMMIT; + } - do_execsql_test 9.$i.6 { PRAGMA integrity_check } ok + do_execsql_test 9.$tn.$i.6 { PRAGMA integrity_check } ok + } } finish_test From 0e55da2ef8999a44fe094b4f385423b1b60b304d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Apr 2016 19:24:56 +0000 Subject: [PATCH 08/63] Fix a typo in temptable2.test. FossilOrigin-Name: 04b1890fbc19eb3fa935083c1664fbbfb67dad93 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/temptable2.test | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 017d301d54..6d6c99e5a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\swal\smode\sto\stemptable2.test. -D 2016-04-11T19:23:36.940 +C Fix\sa\stypo\sin\stemptable2.test. +D 2016-04-11T19:24:56.197 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -1111,7 +1111,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c80f23c651cf6452c27cacf09f4c2ee9527a912 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test 79a30edb79c4d5a4916f208f97935a70efd4b3ba +F test/temptable2.test e62783549be26283e1b3725f0de3309411be6c84 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ffc62af1d503c6e7fc0059d9c2ee57a3bef9dd32 -R b5aaac24011d4d477c13603957856121 +P c6d0d441a1ab4873caedf2f23543444cea5d7f5a +R 24eeba37ecde224a7caa9038421064fb U dan -Z 91cb540a3f9a28d737c21f7feeaff7cc +Z 162e6a9a240e1cae98b0ba71f957c9d5 diff --git a/manifest.uuid b/manifest.uuid index fa8dceac08..3e7e5faf24 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6d0d441a1ab4873caedf2f23543444cea5d7f5a \ No newline at end of file +04b1890fbc19eb3fa935083c1664fbbfb67dad93 \ No newline at end of file diff --git a/test/temptable2.test b/test/temptable2.test index 44a148ddfb..c5e53de635 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -277,11 +277,12 @@ foreach {tn mode} { 2 wal } { reset_db + sqlite3 db "" do_execsql_test 9.$tn.1.1 { PRAGMA cache_size = 15; PRAGMA auto_vacuum = 1; } - do_execsql_test 9.$tn.1.2 "PRAGMA journal_mode = $mode" $mode + do_execsql_test 9.$tn.1.2 "PRAGMA journal_mode = $mode" delete do_execsql_test 9.$tn.1.3 { CREATE TABLE tx(a, b); From 199f56b98455731ccfd8297bf6f3906866c8309c Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Apr 2016 19:09:29 +0000 Subject: [PATCH 09/63] Once a temporary database file has been opened, flush all dirty pages to disk when comitting a transaction. FossilOrigin-Name: bbac71aa2aa2380d393cda3be64b0208b464b27e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 6c3dd2f0cc..1f71471219 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sfrom\strunk. -D 2016-04-12T16:10:10.308 +C Once\sa\stemporary\sdatabase\sfile\shas\sbeen\sopened,\sflush\sall\sdirty\spages\sto\sdisk\swhen\scomitting\sa\stransaction. +D 2016-04-12T19:09:29.339 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bde4844f0849cab5924c6a81178f8500774ce76b F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 4cc8a5b3979db65725eaa03650a6d54219122b8d +F src/pager.c 525f3698f9a6b47a6e0f5496675529a183c2462b F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04b1890fbc19eb3fa935083c1664fbbfb67dad93 7e7289655185e7643ead6d685922528bc4d9e0ae -R 5d35ce920fcfc7013551c54ce7d1cde0 -U drh -Z 97342c74358cb773c29965a93119c3f8 +P 9682c0433c04713c28bd9105a7e20af7372f873e +R ade8e2f9d92ad43fcc4c6c4e5d1634ec +U dan +Z 4a91ff9d1af5229ff41acb5284c79d70 diff --git a/manifest.uuid b/manifest.uuid index 01a6eda77e..90747752ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9682c0433c04713c28bd9105a7e20af7372f873e \ No newline at end of file +bbac71aa2aa2380d393cda3be64b0208b464b27e \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 9baddc2986..e8c612e5c8 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2011,7 +2011,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - if( pPager->tempFile==0 || MEMDB ){ + if( isOpen(pPager->fd) || MEMDB ){ sqlite3PcacheCleanAll(pPager->pPCache); }else{ sqlite3PcacheClearWritable(pPager->pPCache); @@ -4273,7 +4273,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ assert( !pagerUseWal(pPager) ); assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); assert( pPager->eLock==EXCLUSIVE_LOCK ); - assert( pPager->tempFile==0 || pList->pDirty==0 ); + assert( isOpen(pPager->fd) || pList->pDirty==0 ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -6195,7 +6195,8 @@ int sqlite3PagerCommitPhaseOne( if( pPager->eStatetempFile ); - if( pPager->tempFile ){ + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( !isOpen(pPager->fd) ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any ** backup in progress needs to be restarted. */ From 0f52455a357769627eaaab4575dd8f5760b729a0 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 13 Apr 2016 16:52:11 +0000 Subject: [PATCH 10/63] On commit, flush dirty temp-file pages only if the file is already open and 25% or more of the cache is dirty. FossilOrigin-Name: f6babf2920340f25815c0a3c58de1e902c2f5542 --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/pager.c | 21 ++++++++++++++++++++- src/pcache.c | 11 +++++++++++ src/pcache.h | 3 +++ 5 files changed, 46 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1f71471219..0105dfb0f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Once\sa\stemporary\sdatabase\sfile\shas\sbeen\sopened,\sflush\sall\sdirty\spages\sto\sdisk\swhen\scomitting\sa\stransaction. -D 2016-04-12T19:09:29.339 +C On\scommit,\sflush\sdirty\stemp-file\spages\sonly\sif\sthe\sfile\sis\salready\sopen\sand\s25%\sor\smore\sof\sthe\scache\sis\sdirty. +D 2016-04-13T16:52:11.775 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -362,11 +362,11 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bde4844f0849cab5924c6a81178f8500774ce76b F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 525f3698f9a6b47a6e0f5496675529a183c2462b +F src/pager.c 4d849ad718980d698157cd136a40dc91cbeff4d3 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e -F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 -F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 +F src/pcache.c d63b34cce0a8aba1fa552428b2790e13877db553 +F src/pcache.h 60bc9893bfc0e16f8178fb5d8b6fcb8fab1d93c0 F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c @@ -1484,7 +1484,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9682c0433c04713c28bd9105a7e20af7372f873e -R ade8e2f9d92ad43fcc4c6c4e5d1634ec +P bbac71aa2aa2380d393cda3be64b0208b464b27e +R d455655e720096e5ec7c439c248aa874 +T *branch * tempfiles-25 +T *sym-tempfiles-25 * +T -sym-tempfiles-lazy-open * U dan -Z 4a91ff9d1af5229ff41acb5284c79d70 +Z 1d78549afb34b97c4ff4b3e2f6fd7c59 diff --git a/manifest.uuid b/manifest.uuid index 90747752ed..5f347a440e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbac71aa2aa2380d393cda3be64b0208b464b27e \ No newline at end of file +f6babf2920340f25815c0a3c58de1e902c2f5542 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e8c612e5c8..2530a2836e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1873,6 +1873,25 @@ static int pager_error(Pager *pPager, int rc){ static int pager_truncate(Pager *pPager, Pgno nPage); +/* +** The write transaction open on the pager passed as the only argument is +** being committed. This function returns true if all dirty pages should +** be flushed to disk, or false otherwise. Pages should be flushed to disk +** unless one of the following is true: +** +** * The db is an in-memory database. +** +** * The db is a temporary database and the db file has not been opened. +** +** * The db is a temporary database and the cache contains less than +** C/4 dirty pages, where C is the configured cache-size. +*/ +static int pagerFlushOnCommit(Pager *pPager){ + if( pPager->tempFile==0 ) return 1; + if( !isOpen(pPager->fd) ) return 0; + return (sqlite3PCachePercentDirty(pPager->pPCache)>=25); +} + /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -2011,7 +2030,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - if( isOpen(pPager->fd) || MEMDB ){ + if( pagerFlushOnCommit(pPager) ){ sqlite3PcacheCleanAll(pPager->pPCache); }else{ sqlite3PcacheClearWritable(pPager->pPCache); diff --git a/src/pcache.c b/src/pcache.c index 52661c28a9..561672b9ce 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -686,6 +686,17 @@ void sqlite3PcacheShrink(PCache *pCache){ */ int sqlite3HeaderSizePcache(void){ return ROUND8(sizeof(PgHdr)); } +/* +** Return the number of dirty pages currently in the cache, as a percentage +** of the configured cache size. +*/ +int sqlite3PCachePercentDirty(PCache *pCache){ + PgHdr *pDirty; + int nDirty = 0; + int nCache = numberOfCachePages(pCache); + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; + return (int)(((i64)nDirty * 100) / nCache); +} #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* diff --git a/src/pcache.h b/src/pcache.h index 475c04c061..0caf605ff8 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -173,4 +173,7 @@ void sqlite3PCacheSetDefault(void); int sqlite3HeaderSizePcache(void); int sqlite3HeaderSizePcache1(void); +/* Number of dirty pages as a percentage of the configured cache size */ +int sqlite3PCachePercentDirty(PCache*); + #endif /* _PCACHE_H_ */ From f89157653a8a4586b2498cc6b8f64cd8896e1e9f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Apr 2016 20:46:41 +0000 Subject: [PATCH 11/63] Add the experimental sqlite3rbu_vacuum() API function. For opening an RBU handle that rebuilds a database from scratch. FossilOrigin-Name: 0216b48f28042ad86711e00802c2da8ce9be3044 --- ext/rbu/rbuvacuum.test | 181 +++++++++++++++ ext/rbu/sqlite3rbu.c | 501 +++++++++++++++++++++++++++++------------ ext/rbu/sqlite3rbu.h | 8 + ext/rbu/test_rbu.c | 29 +++ manifest | 22 +- manifest.uuid | 2 +- 6 files changed, 595 insertions(+), 148 deletions(-) create mode 100644 ext/rbu/rbuvacuum.test diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test new file mode 100644 index 0000000000..045b2d1bb7 --- /dev/null +++ b/ext/rbu/rbuvacuum.test @@ -0,0 +1,181 @@ +# 2016 April 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that the sqlite3rbu_vacuum() API works as +# expected. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuvacuum + +proc do_rbu_vacuum_test {tn} { + uplevel [list do_test $tn.1 { + forcedelete state.db + if {$::step==0} { sqlite3rbu_vacuum rbu test.db state.db } + while 1 { + if {$::step==1} { sqlite3rbu_vacuum rbu test.db state.db } + set rc [rbu step] + if {$rc!="SQLITE_OK"} break + if {$::step==1} { rbu close } + } + rbu close + } {SQLITE_DONE}] + + uplevel [list do_execsql_test $tn.2 { + PRAGMA integrity_check + } ok] +} + +foreach step {0 1} { + + set ::testprefix rbuvacuum-step=$step + reset_db + + # Simplest possible vacuum. + do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + PRAGMA integrity_check; + } {ok} + do_rbu_vacuum_test 1.1 + + # A vacuum that actually reclaims space. + do_execsql_test 1.2.1 { + INSERT INTO t1 VALUES(8, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(9, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(10, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(11, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(12, randomblob(900), randomblob(900)); + PRAGMA page_count; + } {12} + do_execsql_test 1.2.2 { + DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11; + PRAGMA page_count; + } {12} + do_rbu_vacuum_test 1.2.3 + do_execsql_test 1.2.4 { + PRAGMA page_count; + } {3} + + # Add an index to the table. + do_execsql_test 1.3.1 { + CREATE INDEX t1b ON t1(b); + INSERT INTO t1 VALUES(13, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(14, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(15, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(16, randomblob(900), randomblob(900)); + PRAGMA page_count; + } {18} + do_execsql_test 1.3.2 { + DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15; + PRAGMA page_count; + } {18} + do_rbu_vacuum_test 1.3.3 + do_execsql_test 1.3.4 { + PRAGMA page_count; + } {5} + + # WITHOUT ROWID table. + do_execsql_test 1.4.1 { + CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; + + INSERT INTO t2 VALUES(randomblob(900), 1, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 2, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 3, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 4, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 6, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 7, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 8, randomblob(900)); + + DELETE FROM t2 WHERE b BETWEEN 2 AND 7; + PRAGMA page_count; + } {20} + do_rbu_vacuum_test 1.4.2 + do_execsql_test 1.4.3 { + PRAGMA page_count; + } {10} + + # WITHOUT ROWID table with an index. + do_execsql_test 1.4.1 { + CREATE INDEX t2c ON t2(c); + + INSERT INTO t2 VALUES(randomblob(900), 9, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 10, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 11, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 12, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 13, randomblob(900)); + + DELETE FROM t2 WHERE b BETWEEN 8 AND 12; + PRAGMA page_count; + } {35} + do_rbu_vacuum_test 1.4.2 + do_execsql_test 1.4.3 { + PRAGMA page_count; + } {15} + do_execsql_test 1.4.4 { + VACUUM; + PRAGMA page_count; + } {15} + + do_execsql_test 1.5.1 { + CREATE TABLE t3(a, b, c); + INSERT INTO t3 VALUES('a', 'b', 'c'); + INSERT INTO t3 VALUES('d', 'e', 'f'); + INSERT INTO t3 VALUES('g', 'h', 'i'); + } + do_rbu_vacuum_test 1.5.2 + do_execsql_test 1.5.3 { + SELECT * FROM t3 + } {a b c d e f g h i} + do_execsql_test 1.5.4 { + CREATE INDEX t3a ON t3(a); + CREATE INDEX t3b ON t3(b); + CREATE INDEX t3c ON t3(c); + INSERT INTO t3 VALUES('j', 'k', 'l'); + DELETE FROM t3 WHERE a = 'g'; + } + do_rbu_vacuum_test 1.5.5 + do_execsql_test 1.5.6 { + SELECT rowid, * FROM t3 ORDER BY b + } {1 a b c 2 d e f 4 j k l} + + do_execsql_test 1.6.1 { + CREATE TABLE t4(a PRIMARY KEY, b, c); + INSERT INTO t4 VALUES('a', 'b', 'c'); + INSERT INTO t4 VALUES('d', 'e', 'f'); + INSERT INTO t4 VALUES('g', 'h', 'i'); + } + do_rbu_vacuum_test 1.6.2 + do_execsql_test 1.6.3 { + SELECT * FROM t4 + } {a b c d e f g h i} + do_execsql_test 1.6.4 { + CREATE INDEX t4a ON t4(a); + CREATE INDEX t4b ON t4(b); + CREATE INDEX t4c ON t4(c); + + INSERT INTO t4 VALUES('j', 'k', 'l'); + DELETE FROM t4 WHERE a='g'; + } + do_rbu_vacuum_test 1.6.5 + do_execsql_test 1.6.6 { + SELECT * FROM t4 ORDER BY b + } {a b c d e f j k l} + +} + +catch { db close } +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index f553ebdac5..e71690769b 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -176,6 +176,7 @@ typedef struct RbuUpdateStmt RbuUpdateStmt; #if !defined(SQLITE_AMALGAMATION) typedef unsigned int u32; +typedef unsigned short u16; typedef unsigned char u8; typedef sqlite3_int64 i64; #endif @@ -402,6 +403,11 @@ struct rbu_file { rbu_file *pMainNext; /* Next MAIN_DB file */ }; +/* +** True for an RBU vacuum handle, or false otherwise. +*/ +#define rbuIsVacuum(p) ((p)->zTarget==0) + /************************************************************************* ** The following three functions, found below: @@ -850,8 +856,11 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ /* ** The implementation of the rbu_target_name() SQL function. This function -** accepts one argument - the name of a table in the RBU database. If the -** table name matches the pattern: +** accepts one or two arguments. The first argument is the name of a table - +** the name of a table in the RBU database. The second, if it is present, is 1 +** for a view or 0 for a table. +** +** For a non-vacuum RBU handle, if the table name matches the pattern: ** ** data[0-9]_ ** @@ -862,21 +871,33 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ ** "data_t1" -> "t1" ** "data0123_t2" -> "t2" ** "dataAB_t3" -> NULL +** +** For an rbu vacuum handle, a copy of the first argument is returned if +** the second argument is either missing or 0 (not a view). */ static void rbuTargetNameFunc( - sqlite3_context *context, + sqlite3_context *pCtx, int argc, sqlite3_value **argv ){ + sqlite3rbu *p = sqlite3_user_data(pCtx); const char *zIn; - assert( argc==1 ); + assert( argc==1 || argc==2 ); zIn = (const char*)sqlite3_value_text(argv[0]); - if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ - int i; - for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); - if( zIn[i]=='_' && zIn[i+1] ){ - sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC); + if( zIn ){ + if( rbuIsVacuum(p) ){ + if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ + sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); + } + }else{ + if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); + } + } } } } @@ -894,7 +915,8 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ memset(pIter, 0, sizeof(RbuObjIter)); rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, - "SELECT rbu_target_name(name) AS target, name FROM sqlite_master " + "SELECT rbu_target_name(name, type='view') AS target, name " + "FROM sqlite_master " "WHERE type IN ('table', 'view') AND target IS NOT NULL " "ORDER BY name" ); @@ -1270,6 +1292,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ pStmt = 0; if( p->rc==SQLITE_OK + && rbuIsVacuum(p)==0 && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) ){ p->rc = SQLITE_ERROR; @@ -1409,6 +1432,8 @@ static char *rbuObjIterGetIndexCols( for(i=0; pIter->abTblPk[i]==0; i++); assert( inTblCol ); zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; }else{ zCol = "rbu_rowid"; } @@ -1949,7 +1974,7 @@ static int rbuObjIterPrepareAll( } /* And to delete index entries */ - if( p->rc==SQLITE_OK ){ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError( p->dbMain, &pIter->pDelete, &p->zErrmsg, sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) @@ -1959,6 +1984,15 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys in sorted order */ if( p->rc==SQLITE_OK ){ char *zSql; + if( rbuIsVacuum(p) ){ + zSql = sqlite3_mprintf( + "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", + zCollist, + pIter->zDataTbl, + zCollist, zLimit + ); + }else + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", @@ -1985,7 +2019,9 @@ static int rbuObjIterPrepareAll( sqlite3_free(zWhere); sqlite3_free(zBind); }else{ - int bRbuRowid = (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE); + int bRbuRowid = (pIter->eType==RBU_PK_VTAB) + ||(pIter->eType==RBU_PK_NONE) + ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); const char *zTbl = pIter->zTbl; /* Table this step applies to */ const char *zWrite; /* Imposter table name */ @@ -2012,8 +2048,10 @@ static int rbuObjIterPrepareAll( ); } - /* Create the DELETE statement to write to the target PK b-tree */ - if( p->rc==SQLITE_OK ){ + /* Create the DELETE statement to write to the target PK b-tree. + ** Because it only performs INSERT operations, this is not required for + ** an rbu vacuum handle. */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, sqlite3_mprintf( "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere @@ -2021,7 +2059,7 @@ static int rbuObjIterPrepareAll( ); } - if( pIter->abIndexed ){ + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ const char *zRbuRowid = ""; if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zRbuRowid = ", rbu_rowid"; @@ -2071,10 +2109,16 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ + const char *zRbuRowid = ""; + if( bRbuRowid ){ + zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; + } p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, sqlite3_mprintf( - "SELECT %s, rbu_control%s FROM '%q'%s", - zCollist, (bRbuRowid ? ", rbu_rowid" : ""), + "SELECT %s,%s rbu_control%s FROM '%q'%s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, pIter->zDataTbl, zLimit ) ); @@ -2183,6 +2227,95 @@ static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ return db; } +/* +** Free an RbuState object allocated by rbuLoadState(). +*/ +static void rbuFreeState(RbuState *p){ + if( p ){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Allocate an RbuState object and load the contents of the rbu_state +** table into it. Return a pointer to the new object. It is the +** responsibility of the caller to eventually free the object using +** sqlite3_free(). +** +** If an error occurs, leave an error code and message in the rbu handle +** and return NULL. +*/ +static RbuState *rbuLoadState(sqlite3rbu *p){ + RbuState *pRet = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int rc2; + + pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); + if( pRet==0 ) return 0; + + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt, 0) ){ + case RBU_STATE_STAGE: + pRet->eStage = sqlite3_column_int(pStmt, 1); + if( pRet->eStage!=RBU_STAGE_OAL + && pRet->eStage!=RBU_STAGE_MOVE + && pRet->eStage!=RBU_STAGE_CKPT + ){ + p->rc = SQLITE_CORRUPT; + } + break; + + case RBU_STATE_TBL: + pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_IDX: + pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_ROW: + pRet->nRow = sqlite3_column_int(pStmt, 1); + break; + + case RBU_STATE_PROGRESS: + pRet->nProgress = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_CKPT: + pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_COOKIE: + pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_OALSZ: + pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + + default: + rc = SQLITE_CORRUPT; + break; + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + + p->rc = rc; + return pRet; +} + + /* ** Open the database handle and attach the RBU database as "rbu". If an ** error occurs, leave an error code and message in the RBU handle. @@ -2191,8 +2324,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ assert( p->rc==SQLITE_OK ); assert( p->dbMain==0 && p->dbRbu==0 ); - p->eStage = 0; - p->dbMain = rbuOpenDbhandle(p, p->zTarget); + /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu); /* If using separate RBU and state databases, attach the state database to @@ -2204,6 +2336,38 @@ static void rbuOpenDatabase(sqlite3rbu *p){ memcpy(p->zStateDb, "main", 4); } + /* If it has not already been created, create the rbu_state table */ + rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); + + if( rbuIsVacuum(p) ){ + int bOpen = 0; + if( p->eStage>=RBU_STAGE_MOVE ){ + bOpen = 1; + }else{ + RbuState *pState = rbuLoadState(p); + if( pState ){ + bOpen = (pState->eStage>RBU_STAGE_MOVE); + rbuFreeState(pState); + } + } + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu); + } + + p->eStage = 0; + if( p->dbMain==0 ){ + if( p->zTarget ){ + p->dbMain = rbuOpenDbhandle(p, p->zTarget); + }else{ + char *zTarget = sqlite3_mprintf("%s-vacuum", p->zRbu); + if( zTarget==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + p->dbMain = rbuOpenDbhandle(p, zTarget); + sqlite3_free(zTarget); + } + } + if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbMain, "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 @@ -2218,7 +2382,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbRbu, - "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 ); } @@ -2477,9 +2641,14 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ */ static void rbuMoveOalFile(sqlite3rbu *p){ const char *zBase = sqlite3_db_filename(p->dbMain, "main"); - - char *zWal = sqlite3_mprintf("%s-wal", zBase); char *zOal = sqlite3_mprintf("%s-oal", zBase); + char *zWal; + + if( rbuIsVacuum(p) ){ + zWal = sqlite3_mprintf("%s-wal", sqlite3_db_filename(p->dbRbu, "main")); + }else{ + zWal = sqlite3_mprintf("%s-wal", zBase); + } assert( p->eStage==RBU_STAGE_MOVE ); assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); @@ -2500,8 +2669,8 @@ static void rbuMoveOalFile(sqlite3rbu *p){ /* Re-open the databases. */ rbuObjIterFinalize(&p->objiter); - sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); p->dbMain = 0; p->dbRbu = 0; @@ -2663,19 +2832,24 @@ static void rbuStepOneOp(sqlite3rbu *p, int eType){ p->rc = sqlite3_bind_value(pWriter, i+1, pVal); if( p->rc ) return; } - if( pIter->zIdx==0 - && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) - ){ - /* For a virtual table, or a table with no primary key, the - ** SELECT statement is: - ** - ** SELECT , rbu_control, rbu_rowid FROM .... - ** - ** Hence column_value(pIter->nCol+1). - */ - assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid"); - pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); - p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + if( pIter->zIdx==0 ){ + if( pIter->eType==RBU_PK_VTAB + || pIter->eType==RBU_PK_NONE + || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)) + ){ + /* For a virtual table, or a table with no primary key, the + ** SELECT statement is: + ** + ** SELECT , rbu_control, rbu_rowid FROM .... + ** + ** Hence column_value(pIter->nCol+1). + */ + assertColumnName(pIter->pSelect, pIter->nCol+1, + rbuIsVacuum(p) ? "rowid" : "rbu_rowid" + ); + pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); + p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + } } if( p->rc==SQLITE_OK ){ sqlite3_step(pWriter); @@ -2840,7 +3014,7 @@ int sqlite3rbu_step(sqlite3rbu *p){ /* Clean up the rbu_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ - if( pIter->abIndexed ){ + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ rbuMPrintfExec(p, p->dbRbu, "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl ); @@ -2927,94 +3101,6 @@ int sqlite3rbu_step(sqlite3rbu *p){ } } -/* -** Free an RbuState object allocated by rbuLoadState(). -*/ -static void rbuFreeState(RbuState *p){ - if( p ){ - sqlite3_free(p->zTbl); - sqlite3_free(p->zIdx); - sqlite3_free(p); - } -} - -/* -** Allocate an RbuState object and load the contents of the rbu_state -** table into it. Return a pointer to the new object. It is the -** responsibility of the caller to eventually free the object using -** sqlite3_free(). -** -** If an error occurs, leave an error code and message in the rbu handle -** and return NULL. -*/ -static RbuState *rbuLoadState(sqlite3rbu *p){ - RbuState *pRet = 0; - sqlite3_stmt *pStmt = 0; - int rc; - int rc2; - - pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); - if( pRet==0 ) return 0; - - rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - switch( sqlite3_column_int(pStmt, 0) ){ - case RBU_STATE_STAGE: - pRet->eStage = sqlite3_column_int(pStmt, 1); - if( pRet->eStage!=RBU_STAGE_OAL - && pRet->eStage!=RBU_STAGE_MOVE - && pRet->eStage!=RBU_STAGE_CKPT - ){ - p->rc = SQLITE_CORRUPT; - } - break; - - case RBU_STATE_TBL: - pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_IDX: - pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_ROW: - pRet->nRow = sqlite3_column_int(pStmt, 1); - break; - - case RBU_STATE_PROGRESS: - pRet->nProgress = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_CKPT: - pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_COOKIE: - pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_OALSZ: - pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_PHASEONESTEP: - pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); - break; - - default: - rc = SQLITE_CORRUPT; - break; - } - } - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - - p->rc = rc; - return pRet; -} - /* ** Compare strings z1 and z2, returning 0 if they are identical, or non-zero ** otherwise. Either or both argument may be NULL. Two NULL values are @@ -3205,15 +3291,63 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ } /* -** Open and return a new RBU handle. +** The RBU handle passed as the only argument has just been opened and +** the state database is empty. If this RBU handle was opened for an +** RBU vacuum operation, create the schema in the target db. */ -sqlite3rbu *sqlite3rbu_open( +static void rbuCreateTargetSchema(sqlite3rbu *p){ + sqlite3_stmt *pSql = 0; + sqlite3_stmt *pInsert = 0; + int rc2; + + assert( rbuIsVacuum(p) ); + + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" + " ORDER BY type DESC" + ); + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + const char *zSql = sqlite3_column_text(pSql, 0); + p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); + } + rbuFinalize(p, pSql); + if( p->rc!=SQLITE_OK ) return; + + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg, + "INSERT INTO sqlite_master VALUES(?,?,?,?,?)" + ); + } + + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + int i; + for(i=0; i<5; i++){ + sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i)); + } + sqlite3_step(pInsert); + p->rc = sqlite3_reset(pInsert); + } + + rbuFinalize(p, pSql); + rbuFinalize(p, pInsert); +} + + +static sqlite3rbu *openRbuHandle( const char *zTarget, const char *zRbu, const char *zState ){ sqlite3rbu *p; - size_t nTarget = strlen(zTarget); + size_t nTarget = zTarget ? strlen(zTarget) : 0; size_t nRbu = strlen(zRbu); size_t nState = zState ? strlen(zState) : 0; size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; @@ -3226,22 +3360,24 @@ sqlite3rbu *sqlite3rbu_open( memset(p, 0, sizeof(sqlite3rbu)); rbuCreateVfs(p); - /* Open the target database */ + /* Open the target, RBU and state databases */ if( p->rc==SQLITE_OK ){ - p->zTarget = (char*)&p[1]; - memcpy(p->zTarget, zTarget, nTarget+1); - p->zRbu = &p->zTarget[nTarget+1]; + char *pCsr = (char*)&p[1]; + if( zTarget ){ + p->zTarget = pCsr; + memcpy(p->zTarget, zTarget, nTarget+1); + pCsr += nTarget+1; + } + p->zRbu = pCsr; memcpy(p->zRbu, zRbu, nRbu+1); + pCsr += nRbu+1; if( zState ){ - p->zState = &p->zRbu[nRbu+1]; + p->zState = pCsr; memcpy(p->zState, zState, nState+1); } rbuOpenDatabase(p); } - /* If it has not already been created, create the rbu_state table */ - rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); - if( p->rc==SQLITE_OK ){ pState = rbuLoadState(p); assert( pState || p->rc!=SQLITE_OK ); @@ -3304,6 +3440,12 @@ sqlite3rbu *sqlite3rbu_open( } } + /* If this is an RBU vacuum operation and the state table was empty + ** when this handle was opened, create the target database schema. */ + if( pState->eStage==0 && rbuIsVacuum(p) ){ + rbuCreateTargetSchema(p); + } + /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ p->rc = rbuObjIterFirst(p, &p->objiter); @@ -3336,6 +3478,28 @@ sqlite3rbu *sqlite3rbu_open( return p; } +/* +** Open and return a new RBU handle. +*/ +sqlite3rbu *sqlite3rbu_open( + const char *zTarget, + const char *zRbu, + const char *zState +){ + /* TODO: Check that zTarget and zRbu are non-NULL */ + return openRbuHandle(zTarget, zRbu, zState); +} + +/* +** Open a handle to begin or resume an RBU VACUUM operation. +*/ +sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +){ + /* TODO: Check that both arguments are non-NULL */ + return openRbuHandle(0, zTarget, zState); +} /* ** Return the database handle used by pRbu. @@ -3391,8 +3555,8 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ rbuObjIterFinalize(&p->objiter); /* Close the open database handle and VFS object. */ - sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); rbuDeleteVfs(p); sqlite3_free(p->aBuf); sqlite3_free(p->aFrame); @@ -3594,6 +3758,26 @@ static u32 rbuGetU32(u8 *aBuf){ + ((u32)aBuf[3]); } +/* +** Write an unsigned 32-bit value in big-endian format to the supplied +** buffer. +*/ +static void rbuPutU32(u8 *aBuf, u32 iVal){ + aBuf[0] = (iVal >> 24) & 0xFF; + aBuf[1] = (iVal >> 16) & 0xFF; + aBuf[2] = (iVal >> 8) & 0xFF; + aBuf[3] = (iVal >> 0) & 0xFF; +} + +/* +** Write an unsigned 16-bit value in big-endian format to the supplied +** buffer. +*/ +static void rbuPutU16(u8 *aBuf, u16 iVal){ + aBuf[0] = (iVal >> 8) & 0xFF; + aBuf[1] = (iVal >> 0) & 0xFF; +} + /* ** Read data from an rbuVfs-file. */ @@ -3619,6 +3803,34 @@ static int rbuVfsRead( memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); + /* If this is being called to read the first page of the target + ** database as part of an rbu vacuum operation, synthesize the + ** contents of the first page if it does not yet exist. Otherwise, + ** SQLite will not check for a *-wal file. */ + if( p->pRbu && rbuIsVacuum(p->pRbu) + && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + sqlite3_file *pFd = 0; + rc = sqlite3_file_control( + p->pRbu->dbRbu, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFd + ); + if( rc==SQLITE_OK ){ + rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); + } + if( rc==SQLITE_OK ){ + rbuPutU32(&zBuf[52], 0); /* largest root page number */ + rbuPutU32(&zBuf[36], 0); /* number of free pages */ + rbuPutU32(&zBuf[32], 0); /* first page on free list trunk */ + rbuPutU32(&zBuf[28], 1); /* size of db file in pages */ + + if( iAmt>100 ){ + assert( iAmt>=101 ); + memset(&zBuf[101], 0, iAmt-101); + rbuPutU16(&zBuf[105], iAmt & 0xFFFF); + } + } + } } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part @@ -3693,7 +3905,20 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ */ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ rbu_file *p = (rbu_file *)pFile; - return p->pReal->pMethods->xFileSize(p->pReal, pSize); + int rc; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + + /* If this is an RBU vacuum operation and this is the target database, + ** pretend that it has at least one page. Otherwise, SQLite will not + ** check for the existance of a *-wal file. rbuVfsRead() contains + ** similar logic. */ + if( rc==SQLITE_OK && *pSize==0 + && p->pRbu && rbuIsVacuum(p->pRbu) + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + *pSize = 1024; + } + return rc; } /* diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index f379bb5b41..b910866293 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -314,6 +314,14 @@ sqlite3rbu *sqlite3rbu_open( const char *zState ); +/* +** Open an RBU handle to perform an RBU vacuum database file zTarget. +*/ +sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +); + /* ** Internally, each RBU connection uses a separate SQLite database ** connection to access the target and rbu update databases. This diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index 5e8640a9f8..629a33cd0e 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -187,6 +187,34 @@ static int test_sqlite3rbu( return TCL_OK; } +/* +** Tclcmd: sqlite3rbu_vacuum CMD +*/ +static int test_sqlite3rbu_vacuum( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3rbu *pRbu = 0; + const char *zCmd; + const char *zTarget; + const char *zStateDb = 0; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB STATE-DB"); + return TCL_ERROR; + } + zCmd = Tcl_GetString(objv[1]); + zTarget = Tcl_GetString(objv[2]); + zStateDb = Tcl_GetString(objv[3]); + + pRbu = sqlite3rbu_vacuum(zTarget, zStateDb); + Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + /* ** Tclcmd: sqlite3rbu_create_vfs ?-default? NAME PARENT */ @@ -274,6 +302,7 @@ int SqliteRbu_Init(Tcl_Interp *interp){ Tcl_ObjCmdProc *xProc; } aObjCmd[] = { { "sqlite3rbu", test_sqlite3rbu }, + { "sqlite3rbu_vacuum", test_sqlite3rbu_vacuum }, { "sqlite3rbu_create_vfs", test_sqlite3rbu_create_vfs }, { "sqlite3rbu_destroy_vfs", test_sqlite3rbu_destroy_vfs }, { "sqlite3rbu_internal_test", test_sqlite3rbu_internal_test }, diff --git a/manifest b/manifest index c039180bbf..91d7041221 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C CLI\senhancement:\s\sAdd\sthe\s".eqp\sfull"\soption,\sthat\sshows\sboth\sthe\sEXPLAIN\nQUERY\sPLAN\sand\sthe\sEXPLAIN\soutput\sfor\seach\scommand\srun.\s\sAlso\sdisable\nany\s".wheretrace"\sand\s".selecttrace"\swhen\sshowing\sEQP\soutput. -D 2016-04-15T15:03:27.144 +C Add\sthe\sexperimental\ssqlite3rbu_vacuum()\sAPI\sfunction.\sFor\sopening\san\sRBU\shandle\sthat\srebuilds\sa\sdatabase\sfrom\sscratch. +D 2016-04-15T20:46:41.467 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -246,9 +246,10 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c 9097f1d95666dbef72ca61d5b6a13a84660735ac -F ext/rbu/sqlite3rbu.h d7cc99350c10134f358fe1a8997d9225b3f712b2 -F ext/rbu/test_rbu.c 3505641a78b723589b8780d5f9b2faeeb73e037d +F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 +F ext/rbu/sqlite3rbu.c 79b8be4a0c8276b2b2b24c88edf3944216ccd35b +F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b +F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1482,7 +1483,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d23e581351fb8eea28e7b13b3dcadfc817c3a05f -R e880e1f26ac9fef78137656188a6e0aa -U drh -Z dc2294cde78bca18f07e3e2fb59a4dd5 +P 3e217d6265ecd16db783bed7ce1d9d0f9c4828bb +R d2c575fe1215bc1b298a105940b3f837 +T *branch * rbu-vacuum +T *sym-rbu-vacuum * +T -sym-trunk * +U dan +Z af92587ff7c9268b9a60aaab80a1a468 diff --git a/manifest.uuid b/manifest.uuid index 453393af12..66b85c22cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e217d6265ecd16db783bed7ce1d9d0f9c4828bb \ No newline at end of file +0216b48f28042ad86711e00802c2da8ce9be3044 \ No newline at end of file From ee65eea4af7d4eec9f0c17d39236faffcb10678c Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 16 Apr 2016 15:03:20 +0000 Subject: [PATCH 12/63] Fix a couple of assert() statements that were failing with OOM error tests. FossilOrigin-Name: 8eb3d7d8360530f364bbbebac53e1f0e6753d924 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/expr.c | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 91d7041221..eeb8cf07ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sexperimental\ssqlite3rbu_vacuum()\sAPI\sfunction.\sFor\sopening\san\sRBU\shandle\sthat\srebuilds\sa\sdatabase\sfrom\sscratch. -D 2016-04-15T20:46:41.467 +C Fix\sa\scouple\sof\sassert()\sstatements\sthat\swere\sfailing\swith\sOOM\serror\stests. +D 2016-04-16T15:03:20.537 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -330,7 +330,7 @@ F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198 F src/date.c 1cc9fb516ec9932c6fd4d2a0d2f8bc4480145c39 F src/dbstat.c c845548d4346e606e2f2b7d2e714ace2b8a7dd1b F src/delete.c 78eb999114ec04fcf1b7d123ccedb4b5b734930e -F src/expr.c 17d4e745ef6a3fd2e4ef863f5f9a4912f1ba1198 +F src/expr.c 8796c0739b2ad091e6779253f62aad6e767e2be1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 4c0bd09e602b8ae8d36d81e31e4872d0b53c87bb F src/func.c 552d300265aed09eea21f68ac742a440550c0062 @@ -1483,10 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3e217d6265ecd16db783bed7ce1d9d0f9c4828bb -R d2c575fe1215bc1b298a105940b3f837 -T *branch * rbu-vacuum -T *sym-rbu-vacuum * -T -sym-trunk * +P 0216b48f28042ad86711e00802c2da8ce9be3044 +R 0603f47373db6aeb2d2c22dfe403e0e2 U dan -Z af92587ff7c9268b9a60aaab80a1a468 +Z 6e5ab10ab6efe323abcf341183f13408 diff --git a/manifest.uuid b/manifest.uuid index 66b85c22cc..773e6bf755 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0216b48f28042ad86711e00802c2da8ce9be3044 \ No newline at end of file +8eb3d7d8360530f364bbbebac53e1f0e6753d924 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 094a454ab1..9e3b664e8d 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2360,7 +2360,7 @@ static void cacheEntryClear(Parse *pParse, struct yColCache *p){ } p->iReg = 0; pParse->nColCache--; - assert( cacheIsValid(pParse) ); + assert( pParse->db->mallocFailed || cacheIsValid(pParse) ); } @@ -2405,7 +2405,7 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ p->tempReg = 0; p->lru = pParse->iCacheCnt++; pParse->nColCache++; - assert( cacheIsValid(pParse) ); + assert( pParse->db->mallocFailed || cacheIsValid(pParse) ); return; } } From 977cbe78b3945be39c0e353c77e25d6e1d161d35 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 16 Apr 2016 17:53:14 +0000 Subject: [PATCH 13/63] Avoid creating any extra files ("-vacuum") when running an RBU vacuum. Ensure that the OAL file created is "-oal", not "-vacuum-oal". FossilOrigin-Name: dc19aacc7e99213edca9bb57b5c11a8a1ac99113 --- ext/rbu/sqlite3rbu.c | 147 ++++++++++++++++++++++--------------------- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 84 insertions(+), 77 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index e71690769b..ff01f30725 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2323,6 +2323,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ static void rbuOpenDatabase(sqlite3rbu *p){ assert( p->rc==SQLITE_OK ); assert( p->dbMain==0 && p->dbRbu==0 ); + assert( rbuIsVacuum(p) || p->zTarget!=0 ); /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu); @@ -2355,16 +2356,21 @@ static void rbuOpenDatabase(sqlite3rbu *p){ p->eStage = 0; if( p->dbMain==0 ){ - if( p->zTarget ){ + if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget); }else{ - char *zTarget = sqlite3_mprintf("%s-vacuum", p->zRbu); + char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); if( zTarget==0 ){ p->rc = SQLITE_NOMEM; return; } p->dbMain = rbuOpenDbhandle(p, zTarget); sqlite3_free(zTarget); + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, + "PRAGMA journal_mode=off; BEGIN EXCLUSIVE; COMMIT;", 0, 0, 0 + ); + } } } @@ -2641,14 +2647,15 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ */ static void rbuMoveOalFile(sqlite3rbu *p){ const char *zBase = sqlite3_db_filename(p->dbMain, "main"); - char *zOal = sqlite3_mprintf("%s-oal", zBase); + const char *zMove = zBase; + char *zOal; char *zWal; if( rbuIsVacuum(p) ){ - zWal = sqlite3_mprintf("%s-wal", sqlite3_db_filename(p->dbRbu, "main")); - }else{ - zWal = sqlite3_mprintf("%s-wal", zBase); + zMove = sqlite3_db_filename(p->dbRbu, "main"); } + zOal = sqlite3_mprintf("%s-oal", zMove); + zWal = sqlite3_mprintf("%s-wal", zMove); assert( p->eStage==RBU_STAGE_MOVE ); assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); @@ -2928,13 +2935,18 @@ static int rbuStep(sqlite3rbu *p){ /* ** Increment the schema cookie of the main database opened by p->dbMain. +** +** Or, if this is an RBU vacuum, set the schema cookie of the main db +** opened by p->dbMain to one more than the schema cookie of the main +** db opened by p->dbRbu. */ static void rbuIncrSchemaCookie(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ + sqlite3 *dbread = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain); int iCookie = 1000000; sqlite3_stmt *pStmt; - p->rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg, "PRAGMA schema_version" ); if( p->rc==SQLITE_OK ){ @@ -3408,6 +3420,7 @@ static sqlite3rbu *openRbuHandle( } if( p->rc==SQLITE_OK + && !rbuIsVacuum(p) && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE) && pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie ){ @@ -3803,34 +3816,6 @@ static int rbuVfsRead( memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); - /* If this is being called to read the first page of the target - ** database as part of an rbu vacuum operation, synthesize the - ** contents of the first page if it does not yet exist. Otherwise, - ** SQLite will not check for a *-wal file. */ - if( p->pRbu && rbuIsVacuum(p->pRbu) - && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 - && (p->openFlags & SQLITE_OPEN_MAIN_DB) - ){ - sqlite3_file *pFd = 0; - rc = sqlite3_file_control( - p->pRbu->dbRbu, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFd - ); - if( rc==SQLITE_OK ){ - rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); - } - if( rc==SQLITE_OK ){ - rbuPutU32(&zBuf[52], 0); /* largest root page number */ - rbuPutU32(&zBuf[36], 0); /* number of free pages */ - rbuPutU32(&zBuf[32], 0); /* first page on free list trunk */ - rbuPutU32(&zBuf[28], 1); /* size of db file in pages */ - - if( iAmt>100 ){ - assert( iAmt>=101 ); - memset(&zBuf[101], 0, iAmt-101); - rbuPutU16(&zBuf[105], iAmt & 0xFFFF); - } - } - } } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part @@ -3838,6 +3823,13 @@ static int rbuVfsRead( u8 *pBuf = (u8*)zBuf; p->iCookie = rbuGetU32(&pBuf[24]); p->iWriteVer = pBuf[19]; + if( pRbu && rbuIsVacuum(p->pRbu) ){ + rbu_file *pRbuFd = 0; + sqlite3_file_control(pRbu->dbRbu, "main", + SQLITE_FCNTL_FILE_POINTER, (void*)&pRbuFd + ); + rbuPutU32(&pBuf[24], pRbuFd->iCookie+1); + } } } return rc; @@ -3905,20 +3897,7 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ */ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ rbu_file *p = (rbu_file *)pFile; - int rc; - rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); - - /* If this is an RBU vacuum operation and this is the target database, - ** pretend that it has at least one page. Otherwise, SQLite will not - ** check for the existance of a *-wal file. rbuVfsRead() contains - ** similar logic. */ - if( rc==SQLITE_OK && *pSize==0 - && p->pRbu && rbuIsVacuum(p->pRbu) - && (p->openFlags & SQLITE_OPEN_MAIN_DB) - ){ - *pSize = 1024; - } - return rc; + return p->pReal->pMethods->xFileSize(p->pReal, pSize); } /* @@ -4156,6 +4135,33 @@ static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){ return pDb; } +/* +** A main database named zName has just been opened. The following +** function returns a pointer to a buffer owned by SQLite that contains +** the name of the *-wal file this db connection will use. SQLite +** happens to pass a pointer to this buffer when using xAccess() +** or xOpen() to operate on the *-wal file. +*/ +static const char *rbuMainToWal(const char *zName, int flags){ + int n = (int)strlen(zName); + const char *z = &zName[n]; + if( flags & SQLITE_OPEN_URI ){ + int odd = 0; + while( 1 ){ + if( z[0]==0 ){ + odd = 1 - odd; + if( odd && z[1]==0 ) break; + } + z++; + } + z += 2; + }else{ + while( *z==0 ) z++; + } + z += (n + 8 + 1); + return z; +} + /* ** Open an rbu file handle. */ @@ -4191,6 +4197,7 @@ static int rbuVfsOpen( rbu_file *pFd = (rbu_file *)pFile; int rc = SQLITE_OK; const char *zOpen = zName; + int oflags = flags; memset(pFd, 0, sizeof(rbu_file)); pFd->pReal = (sqlite3_file*)&pFd[1]; @@ -4203,23 +4210,7 @@ static int rbuVfsOpen( ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ - int n = (int)strlen(zName); - const char *z = &zName[n]; - if( flags & SQLITE_OPEN_URI ){ - int odd = 0; - while( 1 ){ - if( z[0]==0 ){ - odd = 1 - odd; - if( odd && z[1]==0 ) break; - } - z++; - } - z += 2; - }else{ - while( *z==0 ) z++; - } - z += (n + 8 + 1); - pFd->zWal = z; + pFd->zWal = rbuMainToWal(zName, flags); } else if( flags & SQLITE_OPEN_WAL ){ rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName); @@ -4229,10 +4220,17 @@ static int rbuVfsOpen( ** code ensures that the string passed to xOpen() is terminated by a ** pair of '\0' bytes in case the VFS attempts to extract a URI ** parameter from it. */ - size_t nCopy = strlen(zName); - char *zCopy = sqlite3_malloc64(nCopy+2); + const char *zBase = zName; + size_t nCopy; + char *zCopy; + if( rbuIsVacuum(pDb->pRbu) ){ + zBase = sqlite3_db_filename(pDb->pRbu->dbRbu, "main"); + zBase = rbuMainToWal(zBase, SQLITE_OPEN_URI); + } + nCopy = strlen(zBase); + zCopy = sqlite3_malloc64(nCopy+2); if( zCopy ){ - memcpy(zCopy, zName, nCopy); + memcpy(zCopy, zBase, nCopy); zCopy[nCopy-3] = 'o'; zCopy[nCopy] = '\0'; zCopy[nCopy+1] = '\0'; @@ -4247,8 +4245,17 @@ static int rbuVfsOpen( } } + if( oflags & SQLITE_OPEN_MAIN_DB + && sqlite3_uri_boolean(zName, "rbu_memory", 0) + ){ + assert( oflags & SQLITE_OPEN_MAIN_DB ); + oflags = SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; + zOpen = 0; + } + if( rc==SQLITE_OK ){ - rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, flags, pOutFlags); + rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags); } if( pFd->pReal->pMethods ){ /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods diff --git a/manifest b/manifest index eeb8cf07ef..0e404f7022 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\sassert()\sstatements\sthat\swere\sfailing\swith\sOOM\serror\stests. -D 2016-04-16T15:03:20.537 +C Avoid\screating\sany\sextra\sfiles\s("-vacuum")\swhen\srunning\san\sRBU\svacuum.\sEnsure\sthat\sthe\sOAL\sfile\screated\sis\s"-oal",\snot\s"-vacuum-oal". +D 2016-04-16T17:53:14.824 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c 79b8be4a0c8276b2b2b24c88edf3944216ccd35b +F ext/rbu/sqlite3rbu.c 471b4055618473612e9ae7d7e4f1922559b59aaf F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0216b48f28042ad86711e00802c2da8ce9be3044 -R 0603f47373db6aeb2d2c22dfe403e0e2 +P 8eb3d7d8360530f364bbbebac53e1f0e6753d924 +R 4d03a7f5bef5f70852d283bb46ff80e2 U dan -Z 6e5ab10ab6efe323abcf341183f13408 +Z 0a7d529e6f2d77bae387935d2996f1da diff --git a/manifest.uuid b/manifest.uuid index 773e6bf755..56172c7c76 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8eb3d7d8360530f364bbbebac53e1f0e6753d924 \ No newline at end of file +dc19aacc7e99213edca9bb57b5c11a8a1ac99113 \ No newline at end of file From 2bb5f4478ea2c4fc2e5889a1586153e394f37ffd Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 16 Apr 2016 19:02:53 +0000 Subject: [PATCH 14/63] Changes so that rbu vacuum works with zipvfs databases. FossilOrigin-Name: bae7f875f476f6e01112751cb404fef42ba0a01c --- ext/rbu/sqlite3rbu.c | 32 ++++++++++++++------------------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index ff01f30725..280b64bb2a 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2213,11 +2213,15 @@ static int rbuGetUpdateStmt( return p->rc; } -static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ +static sqlite3 *rbuOpenDbhandle( + sqlite3rbu *p, + const char *zName, + int bUseVfs +){ sqlite3 *db = 0; if( p->rc==SQLITE_OK ){ const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; - p->rc = sqlite3_open_v2(zName, &db, flags, p->zVfsName); + p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); sqlite3_close(db); @@ -2326,7 +2330,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ assert( rbuIsVacuum(p) || p->zTarget!=0 ); /* Open the RBU database */ - p->dbRbu = rbuOpenDbhandle(p, p->zRbu); + p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); /* If using separate RBU and state databases, attach the state database to ** the RBU db handle now. */ @@ -2351,24 +2355,26 @@ static void rbuOpenDatabase(sqlite3rbu *p){ rbuFreeState(pState); } } - if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu); + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, 1); } p->eStage = 0; if( p->dbMain==0 ){ if( !rbuIsVacuum(p) ){ - p->dbMain = rbuOpenDbhandle(p, p->zTarget); + p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); }else{ + int frc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_ZIPVFS, 0); char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); if( zTarget==0 ){ p->rc = SQLITE_NOMEM; return; } - p->dbMain = rbuOpenDbhandle(p, zTarget); + p->dbMain = rbuOpenDbhandle(p, zTarget, frc!=SQLITE_OK); sqlite3_free(zTarget); if( p->rc==SQLITE_OK ){ p->rc = sqlite3_exec(p->dbMain, - "PRAGMA journal_mode=off; BEGIN EXCLUSIVE; COMMIT;", 0, 0, 0 + "PRAGMA journal_mode=off; PRAGMA zipvfs_journal_mode = off;" + "BEGIN EXCLUSIVE; COMMIT;", 0, 0, 0 ); } } @@ -3310,7 +3316,6 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ static void rbuCreateTargetSchema(sqlite3rbu *p){ sqlite3_stmt *pSql = 0; sqlite3_stmt *pInsert = 0; - int rc2; assert( rbuIsVacuum(p) ); @@ -3319,7 +3324,7 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ " ORDER BY type DESC" ); while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ - const char *zSql = sqlite3_column_text(pSql, 0); + const char *zSql = (const char*)sqlite3_column_text(pSql, 0); p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); } rbuFinalize(p, pSql); @@ -3782,15 +3787,6 @@ static void rbuPutU32(u8 *aBuf, u32 iVal){ aBuf[3] = (iVal >> 0) & 0xFF; } -/* -** Write an unsigned 16-bit value in big-endian format to the supplied -** buffer. -*/ -static void rbuPutU16(u8 *aBuf, u16 iVal){ - aBuf[0] = (iVal >> 8) & 0xFF; - aBuf[1] = (iVal >> 0) & 0xFF; -} - /* ** Read data from an rbuVfs-file. */ diff --git a/manifest b/manifest index 0e404f7022..bcd6ca601c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\screating\sany\sextra\sfiles\s("-vacuum")\swhen\srunning\san\sRBU\svacuum.\sEnsure\sthat\sthe\sOAL\sfile\screated\sis\s"-oal",\snot\s"-vacuum-oal". -D 2016-04-16T17:53:14.824 +C Changes\sso\sthat\srbu\svacuum\sworks\swith\szipvfs\sdatabases. +D 2016-04-16T19:02:53.248 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c 471b4055618473612e9ae7d7e4f1922559b59aaf +F ext/rbu/sqlite3rbu.c 721c6c116018b5d02f6318b6bbb7834098bc6a07 F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8eb3d7d8360530f364bbbebac53e1f0e6753d924 -R 4d03a7f5bef5f70852d283bb46ff80e2 +P dc19aacc7e99213edca9bb57b5c11a8a1ac99113 +R 51e4311e92cd324079b6a63abfa03656 U dan -Z 0a7d529e6f2d77bae387935d2996f1da +Z c0a4c5c182b536f7f5ddad73d82153a8 diff --git a/manifest.uuid b/manifest.uuid index 56172c7c76..3a2c90ce4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc19aacc7e99213edca9bb57b5c11a8a1ac99113 \ No newline at end of file +bae7f875f476f6e01112751cb404fef42ba0a01c \ No newline at end of file From 303a094466c7c5bb76e80cac61c79123b4b95c41 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 18 Apr 2016 09:17:05 +0000 Subject: [PATCH 15/63] Add the -vacuum switch to the "rbu" demonstration program. FossilOrigin-Name: 9a0078a538c7e73a009960347b8953c5af99fefd --- ext/rbu/rbu.c | 56 ++++++++++++++++++++++++++++++++++++--------------- manifest | 12 +++++------ manifest.uuid | 2 +- 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/ext/rbu/rbu.c b/ext/rbu/rbu.c index 018296b8dd..a3efad0d44 100644 --- a/ext/rbu/rbu.c +++ b/ext/rbu/rbu.c @@ -24,12 +24,22 @@ */ void usage(const char *zArgv0){ fprintf(stderr, -"Usage: %s [-step NSTEP] TARGET-DB RBU-DB\n" +"Usage: %s ?OPTIONS? TARGET-DB RBU-DB\n" "\n" -" Argument RBU-DB must be an RBU database containing an update suitable for\n" -" target database TARGET-DB. If NSTEP is set to less than or equal to zero\n" -" (the default value), this program attempts to apply the entire update to\n" -" the target database.\n" +"Where options are:\n" +"\n" +" -step NSTEP\n" +" -vacuum\n" +"\n" +" If the -vacuum switch is not present, argument RBU-DB must be an RBU\n" +" database containing an update suitable for target database TARGET-DB.\n" +" Or, if -vacuum is specified, then TARGET-DB is a database to vacuum using\n" +" RBU, and RBU-DB is used as the state database for the vacuum (refer to\n" +" API documentation for details).\n" +"\n" +" If NSTEP is set to less than or equal to zero (the default value), this \n" +" program attempts to perform the entire update or vacuum operation before\n" +" exiting\n" "\n" " If NSTEP is greater than zero, then a maximum of NSTEP calls are made\n" " to sqlite3rbu_step(). If the RBU update has not been completely applied\n" @@ -69,29 +79,43 @@ int main(int argc, char **argv){ char *zErrmsg; /* Error message, if any */ sqlite3rbu *pRbu; /* RBU handle */ int nStep = 0; /* Maximum number of step() calls */ + int bVacuum = 0; int rc; sqlite3_int64 nProgress = 0; + int nArg = argc-2; - /* Process command line arguments. Following this block local variables - ** zTarget, zRbu and nStep are all set. */ - if( argc==5 ){ - size_t nArg1 = strlen(argv[1]); - if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]); - nStep = atoi(argv[2]); - }else if( argc!=3 ){ - usage(argv[0]); + if( argc<3 ) usage(argv[0]); + for(i=1; i1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){ + bVacuum = 1; + }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i Date: Mon, 18 Apr 2016 13:30:50 +0000 Subject: [PATCH 16/63] sqlite3PCachePerecentDirty() should return 0 if the pcache is empty. FossilOrigin-Name: 85f3d80eb7319754792eef76996b2c740a2b0073 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pcache.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index acc14071dc..4b67df4189 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sfrom\strunk\svia\stempfiles-lazy-open. -D 2016-04-13T19:20:23.336 +C sqlite3PCachePerecentDirty()\sshould\sreturn\s0\sif\sthe\spcache\sis\sempty. +D 2016-04-18T13:30:50.144 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -365,7 +365,7 @@ F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 4d849ad718980d698157cd136a40dc91cbeff4d3 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e -F src/pcache.c d63b34cce0a8aba1fa552428b2790e13877db553 +F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 F src/pcache.h 60bc9893bfc0e16f8178fb5d8b6fcb8fab1d93c0 F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f6babf2920340f25815c0a3c58de1e902c2f5542 e2edd34e79af906e7069e3e837730b12de0b73fe -R ae2dffba3d0e430c251b8372421692c3 +P ae16310c4e9a9b7e94874bd912fc6a4324cfaa2b +R 0620b8f840563243b73e6076ebdcd81f U drh -Z 2aed413a070df399f71581c0517ce593 +Z 04d9a6e624a4591b3aa3f8ec5734f1ff diff --git a/manifest.uuid b/manifest.uuid index de9f4ee7ee..9ac43a726e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae16310c4e9a9b7e94874bd912fc6a4324cfaa2b \ No newline at end of file +85f3d80eb7319754792eef76996b2c740a2b0073 \ No newline at end of file diff --git a/src/pcache.c b/src/pcache.c index 561672b9ce..f700c2ff66 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -695,7 +695,7 @@ int sqlite3PCachePercentDirty(PCache *pCache){ int nDirty = 0; int nCache = numberOfCachePages(pCache); for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; - return (int)(((i64)nDirty * 100) / nCache); + return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; } #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) From beccf1d16f3cf00c58349dd1a6114b13a920ee4b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 18 Apr 2016 18:18:18 +0000 Subject: [PATCH 17/63] Fix some zipvfs related problems in RBU vacuum. FossilOrigin-Name: d76f4aaa4caab713460421bd27365a82ac986c20 --- ext/rbu/sqlite3rbu.c | 81 +++++++++++++++++++++++++++++++++++++------- manifest | 12 +++---- manifest.uuid | 2 +- 3 files changed, 76 insertions(+), 19 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 280b64bb2a..f98a836852 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -190,6 +190,8 @@ typedef sqlite3_int64 i64; #define WAL_LOCK_CKPT 1 #define WAL_LOCK_READ0 3 +#define SQLITE_FCNTL_RBUCNT 5149216 + /* ** A structure to store values read from the rbu_state table in memory. */ @@ -393,6 +395,7 @@ struct rbu_file { int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ u8 iWriteVer; /* "write-version" value for main db files */ + u8 bNolock; int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ @@ -2325,6 +2328,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ ** error occurs, leave an error code and message in the RBU handle. */ static void rbuOpenDatabase(sqlite3rbu *p){ + int nRbu = 0; assert( p->rc==SQLITE_OK ); assert( p->dbMain==0 && p->dbRbu==0 ); assert( rbuIsVacuum(p) || p->zTarget!=0 ); @@ -2332,6 +2336,10 @@ static void rbuOpenDatabase(sqlite3rbu *p){ /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + if( rbuIsVacuum(p) ){ + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, &nRbu); + } + /* If using separate RBU and state databases, attach the state database to ** the RBU db handle now. */ if( p->zState ){ @@ -2355,7 +2363,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ rbuFreeState(pState); } } - if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, 1); + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, nRbu<=1); } p->eStage = 0; @@ -2363,20 +2371,13 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); }else{ - int frc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_ZIPVFS, 0); char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); if( zTarget==0 ){ p->rc = SQLITE_NOMEM; return; } - p->dbMain = rbuOpenDbhandle(p, zTarget, frc!=SQLITE_OK); + p->dbMain = rbuOpenDbhandle(p, zTarget, nRbu<=1); sqlite3_free(zTarget); - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbMain, - "PRAGMA journal_mode=off; PRAGMA zipvfs_journal_mode = off;" - "BEGIN EXCLUSIVE; COMMIT;", 0, 0, 0 - ); - } } } @@ -3445,7 +3446,7 @@ static sqlite3rbu *openRbuHandle( ** created at this point. */ p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); } /* Check if the main database is a zipvfs db. If it is, set the upper @@ -3787,6 +3788,11 @@ static void rbuPutU32(u8 *aBuf, u32 iVal){ aBuf[3] = (iVal >> 0) & 0xFF; } +static void rbuPutU16(u8 *aBuf, u16 iVal){ + aBuf[0] = (iVal >> 8) & 0xFF; + aBuf[1] = (iVal >> 0) & 0xFF; +} + /* ** Read data from an rbuVfs-file. */ @@ -3812,6 +3818,37 @@ static int rbuVfsRead( memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); +#if 1 + /* If this is being called to read the first page of the target + ** database as part of an rbu vacuum operation, synthesize the + ** contents of the first page if it does not yet exist. Otherwise, + ** SQLite will not check for a *-wal file. */ + if( p->pRbu && rbuIsVacuum(p->pRbu) + && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + sqlite3_file *pFd = 0; + rc = sqlite3_file_control( + p->pRbu->dbRbu, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFd + ); + if( rc==SQLITE_OK ){ + rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); + } + if( rc==SQLITE_OK ){ + u8 *aBuf = (u8*)zBuf; + rbuPutU32(&aBuf[52], 0); /* largest root page number */ + rbuPutU32(&aBuf[36], 0); /* number of free pages */ + rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ + rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ + + if( iAmt>100 ){ + assert( iAmt>=101 ); + memset(&aBuf[101], 0, iAmt-101); + rbuPutU16(&aBuf[105], iAmt & 0xFFFF); + } + } + } +#endif } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part @@ -3893,7 +3930,20 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ */ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ rbu_file *p = (rbu_file *)pFile; - return p->pReal->pMethods->xFileSize(p->pReal, pSize); + int rc; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + + /* If this is an RBU vacuum operation and this is the target database, + ** pretend that it has at least one page. Otherwise, SQLite will not + ** check for the existance of a *-wal file. rbuVfsRead() contains + ** similar logic. */ + if( rc==SQLITE_OK && *pSize==0 + && p->pRbu && rbuIsVacuum(p->pRbu) + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + *pSize = 1024; + } + return rc; } /* @@ -3905,7 +3955,9 @@ static int rbuVfsLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); - if( pRbu && eLock==SQLITE_LOCK_EXCLUSIVE && pRbu->eStage!=RBU_STAGE_DONE ){ + if( eLock==SQLITE_LOCK_EXCLUSIVE + && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE)) + ){ /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this ** prevents it from checkpointing the database from sqlite3_close(). */ rc = SQLITE_BUSY; @@ -3968,6 +4020,11 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ } return rc; } + else if( op==SQLITE_FCNTL_RBUCNT ){ + int *pnRbu = (int*)pArg; + (*pnRbu)++; + p->bNolock = 1; + } rc = xControl(p->pReal, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ diff --git a/manifest b/manifest index 6840069cfa..d23cec1ffd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s-vacuum\sswitch\sto\sthe\s"rbu"\sdemonstration\sprogram. -D 2016-04-18T09:17:05.073 +C Fix\ssome\szipvfs\srelated\sproblems\sin\sRBU\svacuum. +D 2016-04-18T18:18:18.881 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c 721c6c116018b5d02f6318b6bbb7834098bc6a07 +F ext/rbu/sqlite3rbu.c 372ed3aaa396e3edfacdb8976d49aafdbe330cfa F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bae7f875f476f6e01112751cb404fef42ba0a01c -R 446c70a62698b2fec86a1f84d13dd5af +P 9a0078a538c7e73a009960347b8953c5af99fefd +R dc7cff23cf20a157412c6c371d56e10b U dan -Z be7942fa7aee76e8d89cc36d886361a8 +Z 72aacc0891d9c46e37fe5c65eb96b678 diff --git a/manifest.uuid b/manifest.uuid index c5a45627c9..e3fa7de1a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a0078a538c7e73a009960347b8953c5af99fefd \ No newline at end of file +d76f4aaa4caab713460421bd27365a82ac986c20 \ No newline at end of file From 58e3812988e4eea3140ae365d0325254555c71bc Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 18 Apr 2016 21:00:01 +0000 Subject: [PATCH 18/63] Another fix to rbu vacuum for a zipvfs case. FossilOrigin-Name: 29407d70e44ad9ea5ddaf1011d0e212b602a3ddf --- ext/rbu/sqlite3rbu.c | 30 ++++++++++++++++-------------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index f98a836852..60773b79f8 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -370,6 +370,10 @@ struct sqlite3rbu { int pgsz; u8 *aBuf; i64 iWalCksum; + + /* Used in RBU vacuum mode only */ + int nRbu; /* Number of RBU VFS in the stack */ + rbu_file *pRbuFd; /* Fd for main db of dbRbu */ }; /* @@ -2328,7 +2332,6 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ ** error occurs, leave an error code and message in the RBU handle. */ static void rbuOpenDatabase(sqlite3rbu *p){ - int nRbu = 0; assert( p->rc==SQLITE_OK ); assert( p->dbMain==0 && p->dbRbu==0 ); assert( rbuIsVacuum(p) || p->zTarget!=0 ); @@ -2337,7 +2340,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); if( rbuIsVacuum(p) ){ - sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, &nRbu); + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); } /* If using separate RBU and state databases, attach the state database to @@ -2354,6 +2357,9 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( rbuIsVacuum(p) ){ int bOpen = 0; + p->nRbu = 0; + p->pRbuFd = 0; + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); if( p->eStage>=RBU_STAGE_MOVE ){ bOpen = 1; }else{ @@ -2363,7 +2369,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ rbuFreeState(pState); } } - if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, nRbu<=1); + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); } p->eStage = 0; @@ -2376,7 +2382,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ p->rc = SQLITE_NOMEM; return; } - p->dbMain = rbuOpenDbhandle(p, zTarget, nRbu<=1); + p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); sqlite3_free(zTarget); } } @@ -3823,17 +3829,12 @@ static int rbuVfsRead( ** database as part of an rbu vacuum operation, synthesize the ** contents of the first page if it does not yet exist. Otherwise, ** SQLite will not check for a *-wal file. */ - if( p->pRbu && rbuIsVacuum(p->pRbu) + if( pRbu && rbuIsVacuum(pRbu) && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ - sqlite3_file *pFd = 0; - rc = sqlite3_file_control( - p->pRbu->dbRbu, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFd - ); - if( rc==SQLITE_OK ){ - rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); - } + sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd; + rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); if( rc==SQLITE_OK ){ u8 *aBuf = (u8*)zBuf; rbuPutU32(&aBuf[52], 0); /* largest root page number */ @@ -4021,8 +4022,9 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ return rc; } else if( op==SQLITE_FCNTL_RBUCNT ){ - int *pnRbu = (int*)pArg; - (*pnRbu)++; + sqlite3rbu *pRbu = (sqlite3rbu*)pArg; + pRbu->nRbu++; + pRbu->pRbuFd = p; p->bNolock = 1; } diff --git a/manifest b/manifest index d23cec1ffd..068554e465 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\szipvfs\srelated\sproblems\sin\sRBU\svacuum. -D 2016-04-18T18:18:18.881 +C Another\sfix\sto\srbu\svacuum\sfor\sa\szipvfs\scase. +D 2016-04-18T21:00:01.258 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c 372ed3aaa396e3edfacdb8976d49aafdbe330cfa +F ext/rbu/sqlite3rbu.c ac4621fc62e0bc3e9e92fceb483c0207af25addf F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9a0078a538c7e73a009960347b8953c5af99fefd -R dc7cff23cf20a157412c6c371d56e10b +P d76f4aaa4caab713460421bd27365a82ac986c20 +R 02609be0d1cf35faad7663d479babac3 U dan -Z 72aacc0891d9c46e37fe5c65eb96b678 +Z ecb53ed56b3181cf4b46470b55ef0f36 diff --git a/manifest.uuid b/manifest.uuid index e3fa7de1a3..74ea0959b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d76f4aaa4caab713460421bd27365a82ac986c20 \ No newline at end of file +29407d70e44ad9ea5ddaf1011d0e212b602a3ddf \ No newline at end of file From bff4b6367c4a216c71f3e6f7ef6c613da82f6341 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 Apr 2016 16:20:24 +0000 Subject: [PATCH 19/63] Detect attempts to use rbu vacuum on a wal mode database (not allowed). And attempts to write to a database in the middle of an rbu vacuum (which prevents the vacuum from resuming). FossilOrigin-Name: 00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0 --- ext/rbu/rbuvacuum.test | 82 +++++++++++++++++++++++++++++++++++------- ext/rbu/sqlite3rbu.c | 42 +++++++++++----------- manifest | 14 ++++---- manifest.uuid | 2 +- 4 files changed, 99 insertions(+), 41 deletions(-) diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test index 045b2d1bb7..4380b78fbf 100644 --- a/ext/rbu/rbuvacuum.test +++ b/ext/rbu/rbuvacuum.test @@ -17,15 +17,15 @@ source [file join [file dirname [info script]] rbu_common.tcl] set ::testprefix rbuvacuum -proc do_rbu_vacuum_test {tn} { +proc do_rbu_vacuum_test {tn step} { uplevel [list do_test $tn.1 { forcedelete state.db - if {$::step==0} { sqlite3rbu_vacuum rbu test.db state.db } + if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db } while 1 { - if {$::step==1} { sqlite3rbu_vacuum rbu test.db state.db } + if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db } set rc [rbu step] if {$rc!="SQLITE_OK"} break - if {$::step==1} { rbu close } + if {$step==1} { rbu close } } rbu close } {SQLITE_DONE}] @@ -49,7 +49,7 @@ foreach step {0 1} { INSERT INTO t1 VALUES(7, 8, 9); PRAGMA integrity_check; } {ok} - do_rbu_vacuum_test 1.1 + do_rbu_vacuum_test 1.1 $step # A vacuum that actually reclaims space. do_execsql_test 1.2.1 { @@ -64,7 +64,7 @@ foreach step {0 1} { DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11; PRAGMA page_count; } {12} - do_rbu_vacuum_test 1.2.3 + do_rbu_vacuum_test 1.2.3 $step do_execsql_test 1.2.4 { PRAGMA page_count; } {3} @@ -82,7 +82,7 @@ foreach step {0 1} { DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15; PRAGMA page_count; } {18} - do_rbu_vacuum_test 1.3.3 + do_rbu_vacuum_test 1.3.3 $step do_execsql_test 1.3.4 { PRAGMA page_count; } {5} @@ -102,7 +102,7 @@ foreach step {0 1} { DELETE FROM t2 WHERE b BETWEEN 2 AND 7; PRAGMA page_count; } {20} - do_rbu_vacuum_test 1.4.2 + do_rbu_vacuum_test 1.4.2 $step do_execsql_test 1.4.3 { PRAGMA page_count; } {10} @@ -120,7 +120,7 @@ foreach step {0 1} { DELETE FROM t2 WHERE b BETWEEN 8 AND 12; PRAGMA page_count; } {35} - do_rbu_vacuum_test 1.4.2 + do_rbu_vacuum_test 1.4.2 $step do_execsql_test 1.4.3 { PRAGMA page_count; } {15} @@ -135,7 +135,7 @@ foreach step {0 1} { INSERT INTO t3 VALUES('d', 'e', 'f'); INSERT INTO t3 VALUES('g', 'h', 'i'); } - do_rbu_vacuum_test 1.5.2 + do_rbu_vacuum_test 1.5.2 $step do_execsql_test 1.5.3 { SELECT * FROM t3 } {a b c d e f g h i} @@ -146,7 +146,7 @@ foreach step {0 1} { INSERT INTO t3 VALUES('j', 'k', 'l'); DELETE FROM t3 WHERE a = 'g'; } - do_rbu_vacuum_test 1.5.5 + do_rbu_vacuum_test 1.5.5 $step do_execsql_test 1.5.6 { SELECT rowid, * FROM t3 ORDER BY b } {1 a b c 2 d e f 4 j k l} @@ -157,7 +157,7 @@ foreach step {0 1} { INSERT INTO t4 VALUES('d', 'e', 'f'); INSERT INTO t4 VALUES('g', 'h', 'i'); } - do_rbu_vacuum_test 1.6.2 + do_rbu_vacuum_test 1.6.2 $step do_execsql_test 1.6.3 { SELECT * FROM t4 } {a b c d e f g h i} @@ -169,13 +169,69 @@ foreach step {0 1} { INSERT INTO t4 VALUES('j', 'k', 'l'); DELETE FROM t4 WHERE a='g'; } - do_rbu_vacuum_test 1.6.5 + do_rbu_vacuum_test 1.6.5 $step do_execsql_test 1.6.6 { SELECT * FROM t4 ORDER BY b } {a b c d e f j k l} } +#------------------------------------------------------------------------- +# Test some error cases: +# +# 2.1.* the db being vacuumed being in wal mode already. +# 2.2.* database modified mid vacuum. +# +reset_db +do_execsql_test 2.1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(9, 10); +} wal +do_test 2.1.1 { + forcedelete state.db + sqlite3rbu_vacuum rbu test.db state.db + rbu step +} {SQLITE_ERROR} +do_test 2.1.2 { + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - cannot vacuum wal mode database}} + +reset_db +do_execsql_test 2.2.0 { + CREATE TABLE tx(a PRIMARY KEY, b BLOB); + INSERT INTO tx VALUES(1, randomblob(900)); + INSERT INTO tx SELECT a+1, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+2, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+4, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+8, randomblob(900) FROM tx; +} +db_save_and_close +for {set i 1} 1 {incr i} { + forcedelete state.db + db_restore_and_reopen + + sqlite3rbu_vacuum rbu test.db state.db + for {set step 0} {$step<$i} {incr step} { rbu step } + rbu close + if {[file exists test.db-wal]} break + + execsql { INSERT INTO tx VALUES(20, 20) } + + do_test 2.2.$i.1 { + sqlite3rbu_vacuum rbu test.db state.db + rbu step + } {SQLITE_BUSY} + do_test 2.2.$i.2 { + list [catch { rbu close } msg] $msg + } {1 {SQLITE_BUSY - database modified during rbu vacuum}} + +} + catch { db close } finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 60773b79f8..6ccf54826c 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -399,7 +399,7 @@ struct rbu_file { int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ u8 iWriteVer; /* "write-version" value for main db files */ - u8 bNolock; + u8 bNolock; /* True to fail EXCLUSIVE locks */ int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ @@ -2373,9 +2373,12 @@ static void rbuOpenDatabase(sqlite3rbu *p){ } p->eStage = 0; - if( p->dbMain==0 ){ + if( p->rc==SQLITE_OK && p->dbMain==0 ){ if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + }else if( p->pRbuFd->pWalFd ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); }else{ char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); if( zTarget==0 ){ @@ -2987,6 +2990,7 @@ static void rbuIncrSchemaCookie(sqlite3rbu *p){ static void rbuSaveState(sqlite3rbu *p, int eStage){ if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ sqlite3_stmt *pInsert = 0; + rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); int rc; assert( p->zErrmsg==0 ); @@ -3009,7 +3013,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ RBU_STATE_ROW, p->nStep, RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, - RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie, + RBU_STATE_COOKIE, (i64)pFd->iCookie, RBU_STATE_OALSZ, p->iOalSz, RBU_STATE_PHASEONESTEP, p->nPhaseOneStep ) @@ -3431,17 +3435,21 @@ static sqlite3rbu *openRbuHandle( } } - if( p->rc==SQLITE_OK - && !rbuIsVacuum(p) + if( p->rc==SQLITE_OK && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE) - && pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie - ){ - /* At this point (pTargetFd->iCookie) contains the value of the - ** change-counter cookie (the thing that gets incremented when a - ** transaction is committed in rollback mode) currently stored on - ** page 1 of the database file. */ - p->rc = SQLITE_BUSY; - p->zErrmsg = sqlite3_mprintf("database modified during rbu update"); + && pState->eStage!=0 + ){ + rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); + if( pFd->iCookie!=pState->iCookie ){ + /* At this point (pTargetFd->iCookie) contains the value of the + ** change-counter cookie (the thing that gets incremented when a + ** transaction is committed in rollback mode) currently stored on + ** page 1 of the database file. */ + p->rc = SQLITE_BUSY; + p->zErrmsg = sqlite3_mprintf("database modified during rbu %s", + (rbuIsVacuum(p) ? "vacuum" : "update") + ); + } } if( p->rc==SQLITE_OK ){ @@ -3841,6 +3849,7 @@ static int rbuVfsRead( rbuPutU32(&aBuf[36], 0); /* number of free pages */ rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ + rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */ if( iAmt>100 ){ assert( iAmt>=101 ); @@ -3857,13 +3866,6 @@ static int rbuVfsRead( u8 *pBuf = (u8*)zBuf; p->iCookie = rbuGetU32(&pBuf[24]); p->iWriteVer = pBuf[19]; - if( pRbu && rbuIsVacuum(p->pRbu) ){ - rbu_file *pRbuFd = 0; - sqlite3_file_control(pRbu->dbRbu, "main", - SQLITE_FCNTL_FILE_POINTER, (void*)&pRbuFd - ); - rbuPutU32(&pBuf[24], pRbuFd->iCookie+1); - } } } return rc; diff --git a/manifest b/manifest index 068554e465..fe489077ea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\sfix\sto\srbu\svacuum\sfor\sa\szipvfs\scase. -D 2016-04-18T21:00:01.258 +C Detect\sattempts\sto\suse\srbu\svacuum\son\sa\swal\smode\sdatabase\s(not\sallowed).\sAnd\sattempts\sto\swrite\sto\sa\sdatabase\sin\sthe\smiddle\sof\san\srbu\svacuum\s(which\sprevents\sthe\svacuum\sfrom\sresuming). +D 2016-04-19T16:20:24.513 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -246,8 +246,8 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c ac4621fc62e0bc3e9e92fceb483c0207af25addf +F ext/rbu/rbuvacuum.test b7194617c22f69e2015082a5414585aae2827a29 +F ext/rbu/sqlite3rbu.c d269d60bddb770fc9bbe747c47522f946562878d F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d76f4aaa4caab713460421bd27365a82ac986c20 -R 02609be0d1cf35faad7663d479babac3 +P 29407d70e44ad9ea5ddaf1011d0e212b602a3ddf +R da49b342c9cd61c56db951e40b34afc9 U dan -Z ecb53ed56b3181cf4b46470b55ef0f36 +Z 3a78d0d54b42b6cec75432abbb63bed7 diff --git a/manifest.uuid b/manifest.uuid index 74ea0959b8..4483a9f107 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29407d70e44ad9ea5ddaf1011d0e212b602a3ddf \ No newline at end of file +00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0 \ No newline at end of file From 46f0035b4829af743b83b6b9efd431031a716d35 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 Apr 2016 17:11:05 +0000 Subject: [PATCH 20/63] When an RBU vacuum is started on a db identified using a URI filename, pass the same URI parameters when creating the new version of the db. This ensures that RBU vacuum works with password protected databases. FossilOrigin-Name: ca021ba88106500f347ed65199a4832bc9eb5ef8 --- ext/rbu/sqlite3rbu.c | 16 +++++++++++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 6ccf54826c..0670b9b8b8 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2380,7 +2380,21 @@ static void rbuOpenDatabase(sqlite3rbu *p){ p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); }else{ - char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); + char *zTarget; + char *zExtra = 0; + if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ + zExtra = &p->zRbu[5]; + while( *zExtra ){ + if( *zExtra++=='?' ) break; + } + if( *zExtra=='\0' ) zExtra = 0; + } + + zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", + sqlite3_db_filename(p->dbRbu, "main"), + (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) + ); + if( zTarget==0 ){ p->rc = SQLITE_NOMEM; return; diff --git a/manifest b/manifest index fe489077ea..2edcbd1077 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Detect\sattempts\sto\suse\srbu\svacuum\son\sa\swal\smode\sdatabase\s(not\sallowed).\sAnd\sattempts\sto\swrite\sto\sa\sdatabase\sin\sthe\smiddle\sof\san\srbu\svacuum\s(which\sprevents\sthe\svacuum\sfrom\sresuming). -D 2016-04-19T16:20:24.513 +C When\san\sRBU\svacuum\sis\sstarted\son\sa\sdb\sidentified\susing\sa\sURI\sfilename,\spass\sthe\ssame\sURI\sparameters\swhen\screating\sthe\snew\sversion\sof\sthe\sdb.\sThis\sensures\sthat\sRBU\svacuum\sworks\swith\spassword\sprotected\sdatabases. +D 2016-04-19T17:11:05.418 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test b7194617c22f69e2015082a5414585aae2827a29 -F ext/rbu/sqlite3rbu.c d269d60bddb770fc9bbe747c47522f946562878d +F ext/rbu/sqlite3rbu.c 0447a8a306411e585d8595465abbed7709f1feb7 F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 29407d70e44ad9ea5ddaf1011d0e212b602a3ddf -R da49b342c9cd61c56db951e40b34afc9 +P 00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0 +R e7445097e6cb895f2eea40503c71f11e U dan -Z 3a78d0d54b42b6cec75432abbb63bed7 +Z 90c42b90d6761cf98f1c67d89391cb68 diff --git a/manifest.uuid b/manifest.uuid index 4483a9f107..f1fc314990 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0 \ No newline at end of file +ca021ba88106500f347ed65199a4832bc9eb5ef8 \ No newline at end of file From 1aea63032a02a2f38b673243b63a71a80bb4aadb Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 Apr 2016 19:27:09 +0000 Subject: [PATCH 21/63] Updates to ensure the values of PRAGMA settings like "page_size", "auto_vacuum", "user_version" and "application_id" are not lost when a database is RBU vacuumed. FossilOrigin-Name: 74ffea76b8c9ddd7699ce6ca77dce005bd7922ac --- ext/rbu/rbuvacuum.test | 68 +++++++++++++++++++++++++++++++++++++++++- ext/rbu/sqlite3rbu.c | 57 ++++++++++++++++++++++++++++++----- manifest | 14 ++++----- manifest.uuid | 2 +- 4 files changed, 125 insertions(+), 16 deletions(-) diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test index 4380b78fbf..85ecfc4c13 100644 --- a/ext/rbu/rbuvacuum.test +++ b/ext/rbu/rbuvacuum.test @@ -174,6 +174,73 @@ foreach step {0 1} { SELECT * FROM t4 ORDER BY b } {a b c d e f j k l} + reset_db + do_execsql_test 1.7.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + DELETE FROM t1 WHERE a=2; + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + DELETE FROM t1 WHERE a=4; + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + DELETE FROM t1 WHERE a=6; + SELECT * FROM t1; + } {1 one 3 three 5 five} + do_rbu_vacuum_test 1.7.1 $step + do_execsql_test 1.7.2 { + INSERT INTO t1 VALUES(NULL, 'seven'); + SELECT * FROM t1; + } {1 one 3 three 5 five 7 seven} + + reset_db + do_execsql_test 1.8.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + ANALYZE; + SELECT * FROM sqlite_stat1; + } {t1 i1 {6 1}} + do_rbu_vacuum_test 1.8.1 $step + do_execsql_test 1.7.2 { + SELECT * FROM sqlite_stat1; + } {t1 i1 {6 1}} + + reset_db + do_execsql_test 1.9.0 { + PRAGMA page_size = 8192; + PRAGMA auto_vacuum = 2; + PRAGMA user_version = 412; + PRAGMA application_id = 413; + + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + + PRAGMA main.page_size; + PRAGMA main.auto_vacuum; + PRAGMA main.user_version; + PRAGMA main.application_id; + } {8192 2 412 413} + + do_rbu_vacuum_test 1.9.1 $step + do_execsql_test 1.9.2 { + PRAGMA main.page_size; + PRAGMA main.auto_vacuum; + PRAGMA main.user_version; + PRAGMA main.application_id; + } {8192 2 412 413} } #------------------------------------------------------------------------- @@ -229,7 +296,6 @@ for {set i 1} 1 {incr i} { do_test 2.2.$i.2 { list [catch { rbu close } msg] $msg } {1 {SQLITE_BUSY - database modified during rbu vacuum}} - } catch { db close } diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 0670b9b8b8..8c7682abea 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3333,6 +3333,38 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ } } +/* +** The second argument passed to this function is the name of a PRAGMA +** setting - "page_size", "auto_vacuum", "user_version" or "application_id". +** This function executes the following on sqlite3rbu.dbRbu: +** +** "PRAGMA main.$zPragma" +** +** where $zPragma is the string passed as the second argument, then +** on sqlite3rbu.dbMain: +** +** "PRAGMA main.$zPragma = $val" +** +** where $val is the value returned by the first PRAGMA invocation. +** +** In short, it copies the value of the specified PRAGMA setting from +** dbRbu to dbMain. +*/ +static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pPragma = 0; + p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.%s", zPragma) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){ + p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d", + zPragma, sqlite3_column_int(pPragma, 0) + ); + } + rbuFinalize(p, pPragma); + } +} + /* ** The RBU handle passed as the only argument has just been opened and ** the state database is empty. If this RBU handle was opened for an @@ -3343,11 +3375,15 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ sqlite3_stmt *pInsert = 0; assert( rbuIsVacuum(p) ); + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" + " AND name!='sqlite_sequence' " + " ORDER BY type DESC" + ); + } - p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, - "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" - " ORDER BY type DESC" - ); while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ const char *zSql = (const char*)sqlite3_column_text(pSql, 0); p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); @@ -3355,8 +3391,6 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ rbuFinalize(p, pSql); if( p->rc!=SQLITE_OK ) return; - p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); - if( p->rc==SQLITE_OK ){ p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" @@ -3470,6 +3504,11 @@ static sqlite3rbu *openRbuHandle( if( p->eStage==RBU_STAGE_OAL ){ sqlite3 *db = p->dbMain; + if( pState->eStage==0 && rbuIsVacuum(p) ){ + rbuCopyPragma(p, "page_size"); + rbuCopyPragma(p, "auto_vacuum"); + } + /* Open transactions both databases. The *-oal file is opened or ** created at this point. */ p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); @@ -3491,6 +3530,8 @@ static sqlite3rbu *openRbuHandle( ** when this handle was opened, create the target database schema. */ if( pState->eStage==0 && rbuIsVacuum(p) ){ rbuCreateTargetSchema(p); + rbuCopyPragma(p, "user_version"); + rbuCopyPragma(p, "application_id"); } /* Point the object iterator at the first object */ @@ -3854,12 +3895,14 @@ static int rbuVfsRead( if( pRbu && rbuIsVacuum(pRbu) && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) + && pRbu->pRbuFd->base.pMethods ){ sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd; rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); if( rc==SQLITE_OK ){ u8 *aBuf = (u8*)zBuf; - rbuPutU32(&aBuf[52], 0); /* largest root page number */ + u32 iRoot = rbuGetU32(&aBuf[52]) ? 1 : 0; + rbuPutU32(&aBuf[52], iRoot); /* largest root page number */ rbuPutU32(&aBuf[36], 0); /* number of free pages */ rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ diff --git a/manifest b/manifest index 2edcbd1077..d233334b3c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\san\sRBU\svacuum\sis\sstarted\son\sa\sdb\sidentified\susing\sa\sURI\sfilename,\spass\sthe\ssame\sURI\sparameters\swhen\screating\sthe\snew\sversion\sof\sthe\sdb.\sThis\sensures\sthat\sRBU\svacuum\sworks\swith\spassword\sprotected\sdatabases. -D 2016-04-19T17:11:05.418 +C Updates\sto\sensure\sthe\svalues\sof\sPRAGMA\ssettings\slike\s"page_size",\s"auto_vacuum",\s"user_version"\sand\s"application_id"\sare\snot\slost\swhen\sa\sdatabase\sis\sRBU\svacuumed. +D 2016-04-19T19:27:09.289 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -246,8 +246,8 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/rbuvacuum.test b7194617c22f69e2015082a5414585aae2827a29 -F ext/rbu/sqlite3rbu.c 0447a8a306411e585d8595465abbed7709f1feb7 +F ext/rbu/rbuvacuum.test ecd7ce1cd8528d6230b56f4840bbb7188cee6946 +F ext/rbu/sqlite3rbu.c a6c02a67834f9e6c2adb18a9191aa0ae65057cfb F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0 -R e7445097e6cb895f2eea40503c71f11e +P ca021ba88106500f347ed65199a4832bc9eb5ef8 +R 8122ed3aabc11f4475527c60a5badfa5 U dan -Z 90c42b90d6761cf98f1c67d89391cb68 +Z 8cbed13b6f72dec5a4cc65417abbee63 diff --git a/manifest.uuid b/manifest.uuid index f1fc314990..590abb9da7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca021ba88106500f347ed65199a4832bc9eb5ef8 \ No newline at end of file +74ffea76b8c9ddd7699ce6ca77dce005bd7922ac \ No newline at end of file From 1a55dedf7ba312cc357fdfa669b14323693b6b90 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Apr 2016 00:30:05 +0000 Subject: [PATCH 22/63] API Change: Modify sqlite3_enable_load_extension() so that it only enables/disables the load_extension() SQL function, and leaves the C-APIs enabled at all times. In this way, applications can enable extension loading for the C interface without having to expose that capability to the SQL. FossilOrigin-Name: edb454e45ae008e051e2f48d704a855b0c3e4be9 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/func.c | 8 ++++++++ src/loadext.c | 15 --------------- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index eb27e8449c..154ad7c55d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\squotes\sfrom\sidentifiers\sused\sas\sdatatype\snames\sin\sa\sCREATE\sTABLE\nstatement.\s\sFix\sfor\sticket\s[7d7525cb01b68] -D 2016-04-18T15:46:14.499 +C API\sChange:\sModify\ssqlite3_enable_load_extension()\sso\sthat\sit\sonly\nenables/disables\sthe\sload_extension()\sSQL\sfunction,\sand\sleaves\sthe\sC-APIs\nenabled\sat\sall\stimes.\s\sIn\sthis\sway,\sapplications\scan\senable\sextension\sloading\nfor\sthe\sC\sinterface\swithout\shaving\sto\sexpose\sthat\scapability\sto\sthe\sSQL. +D 2016-04-20T00:30:05.107 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -332,14 +332,14 @@ F src/delete.c 78eb999114ec04fcf1b7d123ccedb4b5b734930e F src/expr.c 17d4e745ef6a3fd2e4ef863f5f9a4912f1ba1198 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 4c0bd09e602b8ae8d36d81e31e4872d0b53c87bb -F src/func.c 552d300265aed09eea21f68ac742a440550c0062 +F src/func.c 2105701329de3fc2bf47c4153181d412f9f1531c F src/global.c c45ea22aff29334f6a9ec549235ac3357c970015 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 8f4e9fcbd8e95e85f15647ba8b413b18d556ec2b F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e -F src/loadext.c e70f8f9e97624a232870ea5486e682c813ac3002 +F src/loadext.c 3f74ec102096acc2e33379c8379fced14b66858a F src/main.c 5ac9dccc03faadd6f867f67b9018ff41eeeadb46 F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -1482,7 +1482,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a7d72986fabe9434ff5bd02c93169314f072b23 -R 761e3cafabae5f2f003bf21bdc2befe8 +P eba27d4d17a76884292667d570d542e580ee3e77 +R 9873f5bcb700ee71f0df5e53b0b9f6e3 +T *branch * load-ext-security +T *sym-load-ext-security * +T -sym-trunk * U drh -Z 8a9e64fdc933327d2408de0f01f55efc +Z 9467a7a3e92ef67650d6a6e01e05bbf1 diff --git a/manifest.uuid b/manifest.uuid index 540eda5836..6aaf8e7765 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eba27d4d17a76884292667d570d542e580ee3e77 \ No newline at end of file +edb454e45ae008e051e2f48d704a855b0c3e4be9 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 662a08f504..17ca7ab9a8 100644 --- a/src/func.c +++ b/src/func.c @@ -1386,6 +1386,14 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3 *db = sqlite3_context_db_handle(context); char *zErrMsg = 0; + /* Disallow the load_extension function unless the SQLITE_LoadExtension + ** flag is set. See the sqlite3_enable_load_extension() API. + */ + if( (db->flags & SQLITE_LoadExtension)==0 ){ + sqlite3_result_error(context, "not authorized", -1); + return; + } + if( argc==2 ){ zProc = (const char *)sqlite3_value_text(argv[1]); }else{ diff --git a/src/loadext.c b/src/loadext.c index 495001e55a..7be43cb86d 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -460,22 +460,7 @@ static int sqlite3LoadExtension( if( pzErrMsg ) *pzErrMsg = 0; - - /* Ticket #1863. To avoid a creating security problems for older - ** applications that relink against newer versions of SQLite, the - ** ability to run load_extension is turned off by default. One - ** must call sqlite3_enable_load_extension() to turn on extension - ** loading. Otherwise you get the following error. - */ - if( (db->flags & SQLITE_LoadExtension)==0 ){ - if( pzErrMsg ){ - *pzErrMsg = sqlite3_mprintf("not authorized"); - } - return SQLITE_ERROR; - } - zEntry = zProc ? zProc : "sqlite3_extension_init"; - handle = sqlite3OsDlOpen(pVfs, zFile); #if SQLITE_OS_UNIX || SQLITE_OS_WIN for(ii=0; ii Date: Wed, 20 Apr 2016 17:47:52 +0000 Subject: [PATCH 23/63] Add fault injection tests for rbu vacuum. Fix some problems revealed by the same. FossilOrigin-Name: f042fdd1ea7febec7228e51efc2b0281805e196a --- ext/rbu/rbufault3.test | 98 ++++++++++++++++++++++++++++++++++++++++++ ext/rbu/sqlite3rbu.c | 19 +++++--- manifest | 13 +++--- manifest.uuid | 2 +- 4 files changed, 119 insertions(+), 13 deletions(-) create mode 100644 ext/rbu/rbufault3.test diff --git a/ext/rbu/rbufault3.test b/ext/rbu/rbufault3.test new file mode 100644 index 0000000000..e72eb14f7d --- /dev/null +++ b/ext/rbu/rbufault3.test @@ -0,0 +1,98 @@ +# 2016 April 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault injection tests for RBU vacuum operations. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +source $testdir/malloc_common.tcl +set ::testprefix rbufault3 + +foreach {fault errlist} { + oom-* { + {1 SQLITE_NOMEM} + {1 SQLITE_IOERR_NOMEM} + {1 {SQLITE_NOMEM - out of memory}} + } + + ioerr-* { + {1 {SQLITE_IOERR - disk I/O error}} + {1 SQLITE_IOERR} + {1 SQLITE_IOERR_WRITE} + {1 SQLITE_IOERR_FSYNC} + {1 SQLITE_IOERR_READ} + {1 {SQLITE_IOERR - unable to open database: test.db2}} + {1 {SQLITE_ERROR - unable to open database: test.db2}} + {1 {SQLITE_ERROR - SQL logic error or missing database}} + } + + cantopen* { + {1 {SQLITE_CANTOPEN - unable to open database: test.db2}} + {1 {SQLITE_CANTOPEN - unable to open database: test.db2}} + {1 {SQLITE_CANTOPEN - unable to open database file}} + {1 SQLITE_CANTOPEN} + } + +} { + + reset_db + do_execsql_test 0 { + CREATE TABLE target(x UNIQUE, y, z, PRIMARY KEY(y)); + INSERT INTO target VALUES(1, 2, 3); + INSERT INTO target VALUES(4, 5, 6); + INSERT INTO target VALUES(7, 8, 9); + CREATE INDEX i1 ON target(z); + } + faultsim_save_and_close + + do_faultsim_test 1 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_DONE} {*}$::errlist] + } + + do_faultsim_test 2 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist] + } + + forcedelete test.db2 + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + faultsim_save_and_close + + do_faultsim_test 3 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist] + } + +} + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 8c7682abea..8a1b06dd62 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2339,7 +2339,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); - if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); } @@ -2355,11 +2355,13 @@ static void rbuOpenDatabase(sqlite3rbu *p){ /* If it has not already been created, create the rbu_state table */ rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); - if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ int bOpen = 0; + int rc; p->nRbu = 0; p->pRbuFd = 0; - sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( rc!=SQLITE_NOTFOUND ) p->rc = rc; if( p->eStage>=RBU_STAGE_MOVE ){ bOpen = 1; }else{ @@ -3411,6 +3413,9 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ sqlite3_step(pInsert); p->rc = sqlite3_reset(pInsert); } + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg); + } rbuFinalize(p, pSql); rbuFinalize(p, pInsert); @@ -3511,7 +3516,9 @@ static sqlite3rbu *openRbuHandle( /* Open transactions both databases. The *-oal file is opened or ** created at this point. */ - p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + } if( p->rc==SQLITE_OK ){ p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); } @@ -3528,7 +3535,7 @@ static sqlite3rbu *openRbuHandle( /* If this is an RBU vacuum operation and the state table was empty ** when this handle was opened, create the target database schema. */ - if( pState->eStage==0 && rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){ rbuCreateTargetSchema(p); rbuCopyPragma(p, "user_version"); rbuCopyPragma(p, "application_id"); @@ -3895,7 +3902,7 @@ static int rbuVfsRead( if( pRbu && rbuIsVacuum(pRbu) && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) - && pRbu->pRbuFd->base.pMethods + && pRbu->rc==SQLITE_OK ){ sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd; rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); diff --git a/manifest b/manifest index d233334b3c..e640d710d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sensure\sthe\svalues\sof\sPRAGMA\ssettings\slike\s"page_size",\s"auto_vacuum",\s"user_version"\sand\s"application_id"\sare\snot\slost\swhen\sa\sdatabase\sis\sRBU\svacuumed. -D 2016-04-19T19:27:09.289 +C Add\sfault\sinjection\stests\sfor\srbu\svacuum.\sFix\ssome\sproblems\srevealed\sby\sthe\ssame. +D 2016-04-20T17:47:52.967 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -243,11 +243,12 @@ F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 F ext/rbu/rbudiff.test 2df0a8a7d998ecf81764c21eeda3cde5611c5091 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 +F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test ecd7ce1cd8528d6230b56f4840bbb7188cee6946 -F ext/rbu/sqlite3rbu.c a6c02a67834f9e6c2adb18a9191aa0ae65057cfb +F ext/rbu/sqlite3rbu.c cc44505ef06053077f35b08c4ba11dc8bcf310dc F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ca021ba88106500f347ed65199a4832bc9eb5ef8 -R 8122ed3aabc11f4475527c60a5badfa5 +P 74ffea76b8c9ddd7699ce6ca77dce005bd7922ac +R f86f4403b5c8adf842292353673eca81 U dan -Z 8cbed13b6f72dec5a4cc65417abbee63 +Z 6e60b344cf662b4bf49e9c442360b79f diff --git a/manifest.uuid b/manifest.uuid index 590abb9da7..ce2dab7023 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -74ffea76b8c9ddd7699ce6ca77dce005bd7922ac \ No newline at end of file +f042fdd1ea7febec7228e51efc2b0281805e196a \ No newline at end of file From 333f80562b6bd9bb976f9d276e9586bc82b673cc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Apr 2016 18:31:27 +0000 Subject: [PATCH 24/63] Fix a locking race condition in Windows 10 that can occur when two or more processes attempt to recover the same hot journal at the same time. FossilOrigin-Name: 38a4e9d92887898b779493c71e4500f777a4e2e7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index eb27e8449c..0271d96ed6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\squotes\sfrom\sidentifiers\sused\sas\sdatatype\snames\sin\sa\sCREATE\sTABLE\nstatement.\s\sFix\sfor\sticket\s[7d7525cb01b68] -D 2016-04-18T15:46:14.499 +C Fix\sa\slocking\srace\scondition\sin\sWindows\s10\sthat\scan\soccur\swhen\stwo\sor\smore\nprocesses\sattempt\sto\srecover\sthe\ssame\shot\sjournal\sat\sthe\ssame\stime. +D 2016-04-20T18:31:27.150 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -360,7 +360,7 @@ F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c d0b41a47eb5f0dc00e423a1723aadeab0e78c85f -F src/os_win.c b169437dff859e308b3726594094a2f8ca922941 +F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 38718a019ca762ba4f6795425d5a54db70d1790d F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 @@ -1482,7 +1482,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a7d72986fabe9434ff5bd02c93169314f072b23 -R 761e3cafabae5f2f003bf21bdc2befe8 +P eba27d4d17a76884292667d570d542e580ee3e77 +R 91e3493cbc66272eab1ca1f4455591b7 U drh -Z 8a9e64fdc933327d2408de0f01f55efc +Z f74379d7acb8dab3d54fff08dd9ef3d2 diff --git a/manifest.uuid b/manifest.uuid index 540eda5836..b7ae2958b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eba27d4d17a76884292667d570d542e580ee3e77 \ No newline at end of file +38a4e9d92887898b779493c71e4500f777a4e2e7 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 929ad346cc..1f26463453 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3169,9 +3169,8 @@ static int winLock(sqlite3_file *id, int locktype){ ** the PENDING_LOCK byte is temporary. */ newLocktype = pFile->locktype; - if( (pFile->locktype==NO_LOCK) - || ( (locktype==EXCLUSIVE_LOCK) - && (pFile->locktype==RESERVED_LOCK)) + if( pFile->locktype==NO_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) ){ int cnt = 3; while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, From cdce61e133e58112db4377123380de20e37cfc83 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Apr 2016 19:30:47 +0000 Subject: [PATCH 25/63] When an error occurs while transitioning out of WAL mode, make sure the locking state is not left at EXCLUSIVE. FossilOrigin-Name: 3340f086510b08ce5b42a8781f1df51bf7c27701 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 0271d96ed6..3b102ff170 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\slocking\srace\scondition\sin\sWindows\s10\sthat\scan\soccur\swhen\stwo\sor\smore\nprocesses\sattempt\sto\srecover\sthe\ssame\shot\sjournal\sat\sthe\ssame\stime. -D 2016-04-20T18:31:27.150 +C When\san\serror\soccurs\swhile\stransitioning\sout\sof\sWAL\smode,\smake\ssure\sthe\nlocking\sstate\sis\snot\sleft\sat\sEXCLUSIVE. +D 2016-04-20T19:30:47.222 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c d0b41a47eb5f0dc00e423a1723aadeab0e78c85f F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 38718a019ca762ba4f6795425d5a54db70d1790d +F src/pager.c d20fa46d5beda38095be6070dd4c59e502817c72 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df @@ -1482,7 +1482,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eba27d4d17a76884292667d570d542e580ee3e77 -R 91e3493cbc66272eab1ca1f4455591b7 +P 38a4e9d92887898b779493c71e4500f777a4e2e7 +R d900f17deba1f548d837d30ffbaf8602 U drh -Z f74379d7acb8dab3d54fff08dd9ef3d2 +Z 44c6d07231ede184477737327be6aaec diff --git a/manifest.uuid b/manifest.uuid index b7ae2958b9..14e0ef795f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38a4e9d92887898b779493c71e4500f777a4e2e7 \ No newline at end of file +3340f086510b08ce5b42a8781f1df51bf7c27701 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index c18b3a32f7..2015808089 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7311,6 +7311,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); + if( rc && !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } } return rc; From 94d49697b506e9c612352809d0ef100d6c5d4fc8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Apr 2016 20:08:58 +0000 Subject: [PATCH 26/63] Add a documentation comment for sqlite3rbu_vacuum() to sqlite3rbu.h. FossilOrigin-Name: da5c753ddac3cbfdf03710a82f5fd9fa2e29e819 --- ext/rbu/sqlite3rbu.c | 36 ++++++++++++++++++++++++++++++++++++ ext/rbu/sqlite3rbu.h | 22 +++++++++++++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 8a1b06dd62..022e682bde 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2352,9 +2352,45 @@ static void rbuOpenDatabase(sqlite3rbu *p){ memcpy(p->zStateDb, "main", 4); } +#if 0 + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); + } +#endif + /* If it has not already been created, create the rbu_state table */ rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); +#if 0 + if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK ){ + int rc2; + int bOk = 0; + sqlite3_stmt *pCnt = 0; + p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, + "SELECT count(*) FROM stat.sqlite_master" + ); + if( p->rc==SQLITE_OK + && sqlite3_step(pCnt)==SQLITE_ROW + && 1==sqlite3_column_int(pCnt, 0) + ){ + bOk = 1; + } + rc2 = sqlite3_finalize(pCnt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + if( p->rc==SQLITE_OK && bOk==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid state database"); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + } + } + } +#endif + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ int bOpen = 0; int rc; diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index b910866293..c0c95e51fd 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -315,7 +315,27 @@ sqlite3rbu *sqlite3rbu_open( ); /* -** Open an RBU handle to perform an RBU vacuum database file zTarget. +** Open an RBU handle to perform an RBU vacuum on database file zTarget. +** An RBU vacuum is similar to SQLite's built-in VACUUM command, except +** that it can be suspended and resumed like an RBU update. +** +** The second argument to this function, which may not be NULL, identifies +** a database in which to store the state of the RBU vacuum operation if +** it is suspended. The first time sqlite3rbu_vacuum() is called, to start +** an RBU vacuum operation, the state database should either not exist or +** be empty (contain no tables). If an RBU vacuum is suspended by calling +** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has +** returned SQLITE_DONE, the vacuum state is stored in the state database. +** The vacuum can be resumed by calling this function to open a new RBU +** handle specifying the same target and state databases. +** +** This function does not delete the state database after an RBU vacuum +** is completed, even if it created it. +** +** As with sqlite3rbu_open(), Zipvfs users should rever to the comment +** describing the sqlite3rbu_create_vfs() API function below for +** a description of the complications associated with using RBU with +** zipvfs databases. */ sqlite3rbu *sqlite3rbu_vacuum( const char *zTarget, diff --git a/manifest b/manifest index e640d710d0..1ec956e979 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfault\sinjection\stests\sfor\srbu\svacuum.\sFix\ssome\sproblems\srevealed\sby\sthe\ssame. -D 2016-04-20T17:47:52.967 +C Add\sa\sdocumentation\scomment\sfor\ssqlite3rbu_vacuum()\sto\ssqlite3rbu.h. +D 2016-04-20T20:08:58.990 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -248,8 +248,8 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test ecd7ce1cd8528d6230b56f4840bbb7188cee6946 -F ext/rbu/sqlite3rbu.c cc44505ef06053077f35b08c4ba11dc8bcf310dc -F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b +F ext/rbu/sqlite3rbu.c 80ab68658a1dc1921a3eee7c666c948ea640087c +F ext/rbu/sqlite3rbu.h efcafd0e36861943a88deb181753236f84dbc244 F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 74ffea76b8c9ddd7699ce6ca77dce005bd7922ac -R f86f4403b5c8adf842292353673eca81 +P f042fdd1ea7febec7228e51efc2b0281805e196a +R 20401012c91c212b91bec1accca1e5a8 U dan -Z 6e60b344cf662b4bf49e9c442360b79f +Z ebff3ad2f969914e02c51eda888da39a diff --git a/manifest.uuid b/manifest.uuid index ce2dab7023..ffe63d9829 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f042fdd1ea7febec7228e51efc2b0281805e196a \ No newline at end of file +da5c753ddac3cbfdf03710a82f5fd9fa2e29e819 \ No newline at end of file From 191dd06195817f9bf0b4250a28dddd97e9622762 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 21 Apr 2016 01:30:09 +0000 Subject: [PATCH 27/63] Revert sqlite3_enable_load_extension() to its original long-standing behavior. Add SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION which will enable only the C-API and leave the SQL function disabled. FossilOrigin-Name: b2ae5bfa32e608625bd177907596df3dbc2212e1 --- manifest | 25 +++++++++++-------------- manifest.uuid | 2 +- src/func.c | 7 ++----- src/loadext.c | 20 ++++++++++++++++++-- src/main.c | 1 + src/sqlite.h.in | 40 +++++++++++++++++++++++++++++++++++++++- src/sqliteInt.h | 15 ++++++++------- src/test1.c | 1 + 8 files changed, 81 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 154ad7c55d..a06de3ef6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C API\sChange:\sModify\ssqlite3_enable_load_extension()\sso\sthat\sit\sonly\nenables/disables\sthe\sload_extension()\sSQL\sfunction,\sand\sleaves\sthe\sC-APIs\nenabled\sat\sall\stimes.\s\sIn\sthis\sway,\sapplications\scan\senable\sextension\sloading\nfor\sthe\sC\sinterface\swithout\shaving\sto\sexpose\sthat\scapability\sto\sthe\sSQL. -D 2016-04-20T00:30:05.107 +C Revert\ssqlite3_enable_load_extension()\sto\sits\soriginal\slong-standing\sbehavior.\nAdd\sSQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION\swhich\swill\senable\sonly\sthe\sC-API\sand\nleave\sthe\sSQL\sfunction\sdisabled. +D 2016-04-21T01:30:09.828 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -332,15 +332,15 @@ F src/delete.c 78eb999114ec04fcf1b7d123ccedb4b5b734930e F src/expr.c 17d4e745ef6a3fd2e4ef863f5f9a4912f1ba1198 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 4c0bd09e602b8ae8d36d81e31e4872d0b53c87bb -F src/func.c 2105701329de3fc2bf47c4153181d412f9f1531c +F src/func.c b61726e7a1b8f41464d8391d0f2d8a6be1a45281 F src/global.c c45ea22aff29334f6a9ec549235ac3357c970015 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 8f4e9fcbd8e95e85f15647ba8b413b18d556ec2b F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e -F src/loadext.c 3f74ec102096acc2e33379c8379fced14b66858a -F src/main.c 5ac9dccc03faadd6f867f67b9018ff41eeeadb46 +F src/loadext.c 8b3a73f0624c5f7cadbd5cb89940783bee1d39a6 +F src/main.c 405d13e3a4f7c5add9fb27702ae70ed0a6e32cca F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b @@ -377,15 +377,15 @@ F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e F src/select.c 30217121bdf6b587462150b8ee9e1467f7a6036b F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6 -F src/sqlite.h.in 64eb70a3b309751bebf73a5552a51244f68f0ea5 +F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 98f72cbfe00169c39089115427d06ea05fe4b4a2 -F src/sqliteInt.h 49cd2b5cd07cca7c462608540cb6dfa8ab03ba89 +F src/sqliteInt.h ec538389481a3d093f07fb344c5a9dc988042304 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 F src/tclsqlite.c 9c4c4589d078de37813ded708d8838b338ffb060 -F src/test1.c 457c601302b8a0f5960dffd17b6a2877603841dd +F src/test1.c abc10e3e81258835aeb59616685d7369ba99ad1e F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e @@ -1482,10 +1482,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eba27d4d17a76884292667d570d542e580ee3e77 -R 9873f5bcb700ee71f0df5e53b0b9f6e3 -T *branch * load-ext-security -T *sym-load-ext-security * -T -sym-trunk * +P edb454e45ae008e051e2f48d704a855b0c3e4be9 +R 1ffd5268513054a5c042ef7eabb85336 U drh -Z 9467a7a3e92ef67650d6a6e01e05bbf1 +Z 6cdb5487d5064dce1cac61ac30e9bbbf diff --git a/manifest.uuid b/manifest.uuid index 6aaf8e7765..5c2ea41ae7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -edb454e45ae008e051e2f48d704a855b0c3e4be9 \ No newline at end of file +b2ae5bfa32e608625bd177907596df3dbc2212e1 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 17ca7ab9a8..651591d8c8 100644 --- a/src/func.c +++ b/src/func.c @@ -1386,13 +1386,10 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3 *db = sqlite3_context_db_handle(context); char *zErrMsg = 0; - /* Disallow the load_extension function unless the SQLITE_LoadExtension + /* Disallow the load_extension() SQL function unless the SQLITE_LoadExtFunc ** flag is set. See the sqlite3_enable_load_extension() API. */ - if( (db->flags & SQLITE_LoadExtension)==0 ){ - sqlite3_result_error(context, "not authorized", -1); - return; - } + if( (db->flags & SQLITE_LoadExtFunc)==0 ) return; if( argc==2 ){ zProc = (const char *)sqlite3_value_text(argv[1]); diff --git a/src/loadext.c b/src/loadext.c index 7be43cb86d..f881d99710 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -460,7 +460,23 @@ static int sqlite3LoadExtension( if( pzErrMsg ) *pzErrMsg = 0; + + /* Ticket #1863. To avoid a creating security problems for older + ** applications that relink against newer versions of SQLite, the + ** ability to run load_extension is turned off by default. One + ** must call either sqlite3_enable_load_extension(db) or + ** sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, 1, 0) + ** to turn on extension loading. + */ + if( (db->flags & SQLITE_LoadExtension)==0 ){ + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("not authorized"); + } + return SQLITE_ERROR; + } + zEntry = zProc ? zProc : "sqlite3_extension_init"; + handle = sqlite3OsDlOpen(pVfs, zFile); #if SQLITE_OS_UNIX || SQLITE_OS_WIN for(ii=0; iimutex); if( onoff ){ - db->flags |= SQLITE_LoadExtension; + db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; }else{ - db->flags &= ~SQLITE_LoadExtension; + db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc); } sqlite3_mutex_leave(db->mutex); return SQLITE_OK; diff --git a/src/main.c b/src/main.c index 9f773667af..30370f8cab 100644 --- a/src/main.c +++ b/src/main.c @@ -804,6 +804,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, + { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d50d826b41..795236f587 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1932,12 +1932,30 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the new setting is not reported back. ** +**
SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
+**
^This option is used to enable or disable the [sqlite3_load_extension()] +** interface independently of the [load_extension()] SQL function. +** The [sqlite3_enable_load_extension()] API enables or disables both the +** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. +** There should be two additional arguments. +** When the first argument to this interface is 1, then only the C-API is +** enabled and the SQL function remains disabled. If the first argment to +** this interface is 0, then both the C-API and the SQL function are disabled. +** If the first argument is -1, then no changes are made to state of either the +** C-API or the SQL function. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface +** is disabled or enabled following this call. The second parameter may +** be a NULL pointer, in which case the new setting is not reported back. +**
+** ** */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ #define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */ /* @@ -5474,9 +5492,18 @@ int sqlite3_table_column_metadata( ** should free this memory by calling [sqlite3_free()]. ** ** ^Extension loading must be enabled using -** [sqlite3_enable_load_extension()] prior to calling this API, +** [sqlite3_enable_load_extension()] or +** [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],1,NULL) +** prior to calling this API, ** otherwise an error will be returned. ** +** Security warning: It is recommended that the +** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this +** interface. The use of the [sqlite3_enable_load_extension()] interface +** should be avoided. This will keep the SQL function [load_extension()] +** disabled and prevent SQL injections from giving attackers +** access to extension loading capabilities. +** ** See also the [load_extension() SQL function]. */ int sqlite3_load_extension( @@ -5499,6 +5526,17 @@ int sqlite3_load_extension( ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. +** +** ^This interface enables or disables both the C-API +** [sqlite3_load_extension()] and the SQL function [load_extension()]. +** Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) +** to enable or disable only the C-API. +** +** Security warning: It is recommended that extension loading +** be disabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method +** rather than this interface, so the [load_extension()] SQL function +** remains disabled. This will prevent SQL injections from giving attackers +** access to extension loading capabilities. */ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index fe7203b8e1..7bb15d0bf5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1382,13 +1382,14 @@ struct sqlite3 { #define SQLITE_AutoIndex 0x00100000 /* Enable automatic indexes */ #define SQLITE_PreferBuiltin 0x00200000 /* Preference to built-in funcs */ #define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ -#define SQLITE_EnableTrigger 0x00800000 /* True to enable triggers */ -#define SQLITE_DeferFKs 0x01000000 /* Defer all FK constraints */ -#define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ -#define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ -#define SQLITE_Vacuum 0x08000000 /* Currently in a VACUUM */ -#define SQLITE_CellSizeCk 0x10000000 /* Check btree cell sizes on load */ -#define SQLITE_Fts3Tokenizer 0x20000000 /* Enable fts3_tokenizer(2) */ +#define SQLITE_LoadExtFunc 0x00800000 /* Enable load_extension() SQL func */ +#define SQLITE_EnableTrigger 0x01000000 /* True to enable triggers */ +#define SQLITE_DeferFKs 0x02000000 /* Defer all FK constraints */ +#define SQLITE_QueryOnly 0x04000000 /* Disable database changes */ +#define SQLITE_VdbeEQP 0x08000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_Vacuum 0x10000000 /* Currently in a VACUUM */ +#define SQLITE_CellSizeCk 0x20000000 /* Check btree cell sizes on load */ +#define SQLITE_Fts3Tokenizer 0x40000000 /* Enable fts3_tokenizer(2) */ /* diff --git a/src/test1.c b/src/test1.c index 0c5af822c9..5478a72549 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6988,6 +6988,7 @@ static int test_sqlite3_db_config( { "FKEY", SQLITE_DBCONFIG_ENABLE_FKEY }, { "TRIGGER", SQLITE_DBCONFIG_ENABLE_TRIGGER }, { "FTS3_TOKENIZER", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, + { "LOAD_EXTENSION", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, }; int i; int v; From f602a1612338718d8b20bfc7d5467fb46b4cb228 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 21 Apr 2016 01:58:21 +0000 Subject: [PATCH 28/63] Test cases for SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION. FossilOrigin-Name: debafa5efd37ac60e030d0963ce8e7c4f51a0f10 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/func.c | 5 ++++- test/loadext.test | 12 +++++++++++- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a06de3ef6e..05c486ec05 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Revert\ssqlite3_enable_load_extension()\sto\sits\soriginal\slong-standing\sbehavior.\nAdd\sSQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION\swhich\swill\senable\sonly\sthe\sC-API\sand\nleave\sthe\sSQL\sfunction\sdisabled. -D 2016-04-21T01:30:09.828 +C Test\scases\sfor\sSQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION. +D 2016-04-21T01:58:21.573 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -332,7 +332,7 @@ F src/delete.c 78eb999114ec04fcf1b7d123ccedb4b5b734930e F src/expr.c 17d4e745ef6a3fd2e4ef863f5f9a4912f1ba1198 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 4c0bd09e602b8ae8d36d81e31e4872d0b53c87bb -F src/func.c b61726e7a1b8f41464d8391d0f2d8a6be1a45281 +F src/func.c ef4c18c8a66143413ce41a58d582d2c14ddf78e1 F src/global.c c45ea22aff29334f6a9ec549235ac3357c970015 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -875,7 +875,7 @@ F test/like.test 81632c437a947bf1f7130b19537da6a1a844806a F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/like3.test 3608a2042b6f922f900fbfd5d3ce4e7eca57f7c4 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e -F test/loadext.test 648cb95f324d1775c54a55c12271b2d1156b633b +F test/loadext.test 42a3b8166dfcadcb0e0c8710dc520d97c31a8b98 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test b984ab9034e7389be0d863fe4e64cbbc4d2028f5 F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff @@ -1482,7 +1482,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P edb454e45ae008e051e2f48d704a855b0c3e4be9 -R 1ffd5268513054a5c042ef7eabb85336 +P b2ae5bfa32e608625bd177907596df3dbc2212e1 +R 142ad50676c76bf1cd4debbac256d360 U drh -Z 6cdb5487d5064dce1cac61ac30e9bbbf +Z d6546b0e1c5ecba4524ffa3ce100a1b6 diff --git a/manifest.uuid b/manifest.uuid index 5c2ea41ae7..7305430aec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2ae5bfa32e608625bd177907596df3dbc2212e1 \ No newline at end of file +debafa5efd37ac60e030d0963ce8e7c4f51a0f10 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 651591d8c8..4feedc7440 100644 --- a/src/func.c +++ b/src/func.c @@ -1389,7 +1389,10 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ /* Disallow the load_extension() SQL function unless the SQLITE_LoadExtFunc ** flag is set. See the sqlite3_enable_load_extension() API. */ - if( (db->flags & SQLITE_LoadExtFunc)==0 ) return; + if( (db->flags & SQLITE_LoadExtFunc)==0 ){ + sqlite3_result_error(context, "not authorized", -1); + return; + } if( argc==2 ){ zProc = (const char *)sqlite3_value_text(argv[1]); diff --git a/test/loadext.test b/test/loadext.test index 7ba4c0cf77..e6ba21e187 100644 --- a/test/loadext.test +++ b/test/loadext.test @@ -111,7 +111,7 @@ do_test loadext-1.2 { # do_test loadext-1.3 { sqlite3 db2 test.db - sqlite3_enable_load_extension db2 1 + sqlite3_db_config db2 SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1 catchsql { SELECT half(1.0); } db2 @@ -256,6 +256,7 @@ do_test loadext-4.2 { } } {0 {{}}} +# disable all extension loading do_test loadext-4.3 { sqlite3_enable_load_extension db 0 catchsql { @@ -263,6 +264,15 @@ do_test loadext-4.3 { } } {1 {not authorized}} +# enable C-api extension loading only. Show that the SQL function +# still does not work. +do_test loadext-4.4 { + sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1 + catchsql { + SELECT load_extension($::testextension,'testloadext_init') + } +} {1 {not authorized}} + source $testdir/malloc_common.tcl From 6bcfe8b61f6b9390622670de373359159fe06d44 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 21 Apr 2016 15:24:46 +0000 Subject: [PATCH 29/63] Add a function prototype in order to fix a compiler warning. FossilOrigin-Name: 49aec9718d61c9f12aed96f530128666c3b01c81 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pcache.h | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 5048173d7c..8a5fd27015 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\supdates\sfrom\strunk. -D 2016-04-21T15:03:37.535 +C Add\sa\sfunction\sprototype\sin\sorder\sto\sfix\sa\scompiler\swarning. +D 2016-04-21T15:24:46.372 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -366,7 +366,7 @@ F src/pager.c bb8c7ff706aa88e7c2ec9a6175a128c01bc3ae5f F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 -F src/pcache.h 60bc9893bfc0e16f8178fb5d8b6fcb8fab1d93c0 +F src/pcache.h 6b865be765d1ebd06145219550b10921c7da7cc9 F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9768de502e34da42f3ad955947c23da50f57bce c4f165c460c4244ed434107feac005efa3c386cf -R 46a947ce1c80cd48530d3e6dff00dfca +P cc28106e5c196c0a9678fbbd06f6afd49271a8e8 +R c97bc8da233adc2bbb054b217c885bd5 U drh -Z 049109eccf475c334bc4eb599a0e0693 +Z fa8f1e96dc1996c819bff77f7b75b34d diff --git a/manifest.uuid b/manifest.uuid index 8cdc841c3b..31bd76e737 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc28106e5c196c0a9678fbbd06f6afd49271a8e8 \ No newline at end of file +49aec9718d61c9f12aed96f530128666c3b01c81 \ No newline at end of file diff --git a/src/pcache.h b/src/pcache.h index 0caf605ff8..04bce289d0 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -99,6 +99,7 @@ void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ +void sqlite3PcacheClearWritable(PCache*); /* Change a page number. Used by incr-vacuum. */ void sqlite3PcacheMove(PgHdr*, Pgno); From 2e8bfb67d4a6de7029f62573bdc44b6ce9c47d78 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Apr 2016 15:26:26 +0000 Subject: [PATCH 30/63] Fix a problem in rbu vacuum on tables with a large sqlite_master table. FossilOrigin-Name: 23eac52e987996afe8db847921aef434cebd25b5 --- ext/rbu/rbuvacuum.test | 15 +++++++++++++++ ext/rbu/sqlite3rbu.c | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test index 85ecfc4c13..c67162c567 100644 --- a/ext/rbu/rbuvacuum.test +++ b/ext/rbu/rbuvacuum.test @@ -241,6 +241,21 @@ foreach step {0 1} { PRAGMA main.user_version; PRAGMA main.application_id; } {8192 2 412 413} + + # Vacuum a database with a large sqlite_master table. + # + reset_db + do_test 1.10.1 { + for {set i 1} {$i < 50} {incr i} { + execsql "PRAGMA page_size = 1024" + execsql "CREATE TABLE t$i (a, b, c, PRIMARY KEY(a, b));" + execsql " + INSERT INTO t$i VALUES(1, 2, 3); + INSERT INTO t$i VALUES(4, 5, 6); + " + } + } {} + do_rbu_vacuum_test 1.10.2 $step } #------------------------------------------------------------------------- diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 022e682bde..04dc9944d7 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3952,9 +3952,9 @@ static int rbuVfsRead( rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */ if( iAmt>100 ){ - assert( iAmt>=101 ); - memset(&aBuf[101], 0, iAmt-101); + memset(&aBuf[100], 0, iAmt-100); rbuPutU16(&aBuf[105], iAmt & 0xFFFF); + aBuf[100] = 0x0D; } } } diff --git a/manifest b/manifest index 1ec956e979..dbe7f79adc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sdocumentation\scomment\sfor\ssqlite3rbu_vacuum()\sto\ssqlite3rbu.h. -D 2016-04-20T20:08:58.990 +C Fix\sa\sproblem\sin\srbu\svacuum\son\stables\swith\sa\slarge\ssqlite_master\stable. +D 2016-04-21T15:26:26.501 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,8 +247,8 @@ F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/rbuvacuum.test ecd7ce1cd8528d6230b56f4840bbb7188cee6946 -F ext/rbu/sqlite3rbu.c 80ab68658a1dc1921a3eee7c666c948ea640087c +F ext/rbu/rbuvacuum.test aa405cff38b15a89f6c97f91f1c6a67b2140b358 +F ext/rbu/sqlite3rbu.c 8310553b029e749329a89b3880a21f80876755c3 F ext/rbu/sqlite3rbu.h efcafd0e36861943a88deb181753236f84dbc244 F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f042fdd1ea7febec7228e51efc2b0281805e196a -R 20401012c91c212b91bec1accca1e5a8 +P da5c753ddac3cbfdf03710a82f5fd9fa2e29e819 +R e2636288f7318e2301180a849b5ff72b U dan -Z ebff3ad2f969914e02c51eda888da39a +Z d9d3e7dacedd64ed84574ca4c0316d8e diff --git a/manifest.uuid b/manifest.uuid index ffe63d9829..8f13cab8e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da5c753ddac3cbfdf03710a82f5fd9fa2e29e819 \ No newline at end of file +23eac52e987996afe8db847921aef434cebd25b5 \ No newline at end of file From b7a208686ae0a1cbe8aac5919e9502a023ec8409 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Apr 2016 16:44:38 +0000 Subject: [PATCH 31/63] If a call to sqlite3rbu_close() on an rbu handle opened by sqlite3rbu_vacuum() returns other than SQLITE_OK, delete the contents of the rbu_state table. This ensures that if an RBU vacuum operation either completes successfully or encounters an error, the next call to sqlite3rbu_vacuum() with the same parameters starts a new RBU vacuum operation. FossilOrigin-Name: 6e4ac704f51bdd1e0b16814f6a45ab29f26400ee --- ext/rbu/rbuvacuum.test | 3 --- ext/rbu/sqlite3rbu.c | 10 ++++++++++ ext/rbu/sqlite3rbu.h | 6 +++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test index c67162c567..96a37d1f71 100644 --- a/ext/rbu/rbuvacuum.test +++ b/ext/rbu/rbuvacuum.test @@ -19,7 +19,6 @@ set ::testprefix rbuvacuum proc do_rbu_vacuum_test {tn step} { uplevel [list do_test $tn.1 { - forcedelete state.db if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db } while 1 { if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db } @@ -275,7 +274,6 @@ do_execsql_test 2.1.0 { INSERT INTO t1 VALUES(9, 10); } wal do_test 2.1.1 { - forcedelete state.db sqlite3rbu_vacuum rbu test.db state.db rbu step } {SQLITE_ERROR} @@ -294,7 +292,6 @@ do_execsql_test 2.2.0 { } db_save_and_close for {set i 1} 1 {incr i} { - forcedelete state.db db_restore_and_reopen sqlite3rbu_vacuum rbu test.db state.db diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 04dc9944d7..e7e4ddef58 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3685,6 +3685,16 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ /* Close any open statement handles. */ rbuObjIterFinalize(&p->objiter); + /* If this is an RBU vacuum handle and the vacuum has either finished + ** successfully or encountered an error, delete the contents of the + ** state table. This causes the next call to sqlite3rbu_vacuum() + ** specifying the current target and state databases to start a new + ** vacuum from scratch. */ + if( rbuIsVacuum(p) && p->rc!=SQLITE_OK && p->dbRbu ){ + int rc2 = sqlite3_exec(p->dbRbu, "DELETE FROM stat.rbu_state", 0, 0, 0); + if( p->rc==SQLITE_DONE && rc2!=SQLITE_OK ) p->rc = rc2; + } + /* Close the open database handle and VFS object. */ sqlite3_close(p->dbRbu); sqlite3_close(p->dbMain); diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index c0c95e51fd..fce9e493db 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -330,7 +330,11 @@ sqlite3rbu *sqlite3rbu_open( ** handle specifying the same target and state databases. ** ** This function does not delete the state database after an RBU vacuum -** is completed, even if it created it. +** is completed, even if it created it. However, if the call to +** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents +** of the state tables within the state database are zeroed. This way, +** the next call to sqlite3rbu_vacuum() opens a handle that starts a +** new RBU vacuum operation. ** ** As with sqlite3rbu_open(), Zipvfs users should rever to the comment ** describing the sqlite3rbu_create_vfs() API function below for diff --git a/manifest b/manifest index dbe7f79adc..a1b6be3a3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\srbu\svacuum\son\stables\swith\sa\slarge\ssqlite_master\stable. -D 2016-04-21T15:26:26.501 +C If\sa\scall\sto\ssqlite3rbu_close()\son\san\srbu\shandle\sopened\sby\ssqlite3rbu_vacuum()\sreturns\sother\sthan\sSQLITE_OK,\sdelete\sthe\scontents\sof\sthe\srbu_state\stable.\sThis\sensures\sthat\sif\san\sRBU\svacuum\soperation\seither\scompletes\ssuccessfully\sor\sencounters\san\serror,\sthe\snext\scall\sto\ssqlite3rbu_vacuum()\swith\sthe\ssame\sparameters\sstarts\sa\snew\sRBU\svacuum\soperation. +D 2016-04-21T16:44:38.130 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,9 +247,9 @@ F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/rbuvacuum.test aa405cff38b15a89f6c97f91f1c6a67b2140b358 -F ext/rbu/sqlite3rbu.c 8310553b029e749329a89b3880a21f80876755c3 -F ext/rbu/sqlite3rbu.h efcafd0e36861943a88deb181753236f84dbc244 +F ext/rbu/rbuvacuum.test bcbc1dcd8e2a46a811e46477692ae1c0e8917a85 +F ext/rbu/sqlite3rbu.c 20922328dcebe89589638923bb46840df8bc7733 +F ext/rbu/sqlite3rbu.h 2acd0a6344a6079de15c8dc9d84d3df83a665930 F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P da5c753ddac3cbfdf03710a82f5fd9fa2e29e819 -R e2636288f7318e2301180a849b5ff72b +P 23eac52e987996afe8db847921aef434cebd25b5 +R 9f2af869b142a7169421543b9ca51ae8 U dan -Z d9d3e7dacedd64ed84574ca4c0316d8e +Z 717b7e491d25b95d9f80b6c2df00ade7 diff --git a/manifest.uuid b/manifest.uuid index 8f13cab8e3..ef1798e1f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23eac52e987996afe8db847921aef434cebd25b5 \ No newline at end of file +6e4ac704f51bdd1e0b16814f6a45ab29f26400ee \ No newline at end of file From 7694e06461203a466865fc8ed3068e0990a7ca50 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 21 Apr 2016 23:37:24 +0000 Subject: [PATCH 32/63] Change the temporary directory search algorithm on unix so that directories with only -wx permission are allowed. And do not allow "." to be returned if it lacks -wx permission. FossilOrigin-Name: 67985761aa93fb613b87d340e75371fa55b0f778 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/os_unix.c | 8 +++++--- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 084953dacb..df9a5ec95f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3rbu_vacuum()\sAPI,\sto\screate\san\sRBU\shandle\sthat\scan\sbe\sused\sto\srebuild\sa\sdatabase\sin\sthe\ssame\sway\sas\sthe\sVACUUM\scommand. -D 2016-04-21T18:13:37.597 +C Change\sthe\stemporary\sdirectory\ssearch\salgorithm\son\sunix\sso\sthat\sdirectories\nwith\sonly\s-wx\spermission\sare\sallowed.\s\sAnd\sdo\snot\sallow\s"."\sto\sbe\sreturned\sif\nit\slacks\s-wx\spermission. +D 2016-04-21T23:37:24.378 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -361,7 +361,7 @@ F src/os.c 4d83917f072ad958fba9235136fa2ed43df47905 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c d0b41a47eb5f0dc00e423a1723aadeab0e78c85f +F src/os_unix.c c96826e21e897bf9c6ae896be9de8d55d63a4ed5 F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c d20fa46d5beda38095be6070dd4c59e502817c72 @@ -1484,8 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c4f165c460c4244ed434107feac005efa3c386cf 6e4ac704f51bdd1e0b16814f6a45ab29f26400ee -R 4089c7bed4eb395c7a148d5eb16b1be4 -T +closed 6e4ac704f51bdd1e0b16814f6a45ab29f26400ee -U dan -Z 0e5a7e5d2cf2990714a283851c232d22 +P bd4a6f8313eb71399a7141ebf7f2617d7b1464c3 +R 90c3ca98d0d4d69b4119d1fe5513a916 +U drh +Z f4304caf089d5c38523c57ca13360c36 diff --git a/manifest.uuid b/manifest.uuid index a983c0514a..65f89ba88b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bd4a6f8313eb71399a7141ebf7f2617d7b1464c3 \ No newline at end of file +67985761aa93fb613b87d340e75371fa55b0f778 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index aa86f00d70..9a97515fdc 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5420,10 +5420,10 @@ static const char *unixTempFileDir(void){ if( zDir==0 ) continue; if( osStat(zDir, &buf) ) continue; if( !S_ISDIR(buf.st_mode) ) continue; - if( osAccess(zDir, 07) ) continue; - break; + if( osAccess(zDir, 03) ) continue; + return zDir; } - return zDir; + return 0; } /* @@ -5439,9 +5439,11 @@ static int unixGetTempname(int nBuf, char *zBuf){ ** using the io-error infrastructure to test that SQLite handles this ** function failing. */ + zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); zDir = unixTempFileDir(); + if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; do{ u64 r; sqlite3_randomness(sizeof(r), &r); From 6572c16ae1a1f554475c4993eb0c753a2c5cf8d9 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Apr 2016 14:55:28 +0000 Subject: [PATCH 33/63] Fix some problems to do with temp-file databases and recovering from IO and SQLITE_FULL errors. FossilOrigin-Name: 3d61da4a76af8c9c2a293df085f3ed5a7bb447df --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 26 +++++++++++++++----------- test/tempfault.test | 30 +++++++++++++++++++++++++----- 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 8a5fd27015..ec3e51504b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sfunction\sprototype\sin\sorder\sto\sfix\sa\scompiler\swarning. -D 2016-04-21T15:24:46.372 +C Fix\ssome\sproblems\sto\sdo\swith\stemp-file\sdatabases\sand\srecovering\sfrom\sIO\sand\sSQLITE_FULL\serrors. +D 2016-04-23T14:55:28.020 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c d0b41a47eb5f0dc00e423a1723aadeab0e78c85f F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c bb8c7ff706aa88e7c2ec9a6175a128c01bc3ae5f +F src/pager.c 4981dc6154ce111361a7cd35eaa26aadf8914ea3 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 @@ -1109,7 +1109,7 @@ F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 -F test/tempfault.test 0c80f23c651cf6452c27cacf09f4c2ee9527a912 +F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test e62783549be26283e1b3725f0de3309411be6c84 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cc28106e5c196c0a9678fbbd06f6afd49271a8e8 -R c97bc8da233adc2bbb054b217c885bd5 -U drh -Z fa8f1e96dc1996c819bff77f7b75b34d +P 49aec9718d61c9f12aed96f530128666c3b01c81 +R 98471ae1c4ed8f399ba91e3e119d0ef8 +U dan +Z 4115a15d29a4b7f5f76b2491e48cc4f1 diff --git a/manifest.uuid b/manifest.uuid index 31bd76e737..7b1996a745 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49aec9718d61c9f12aed96f530128666c3b01c81 \ No newline at end of file +3d61da4a76af8c9c2a293df085f3ed5a7bb447df \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 41d3297e29..a749b8a6b3 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1824,12 +1824,16 @@ static void pager_unlock(Pager *pPager){ ** This is because it is not possible to call pager_reset() on a temp ** file pager (as this may discard the only copy of some data). */ assert( pPager->errCode==SQLITE_OK || !MEMDB ); - if( pPager->tempFile==0 && pPager->errCode ){ - pager_reset(pPager); - pPager->changeCountDone = 0; - pPager->eState = PAGER_OPEN; - pPager->errCode = SQLITE_OK; + if( pPager->errCode ){ + if( pPager->tempFile==0 ){ + pager_reset(pPager); + pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + }else{ + pPager->eState = (isOpen(pPager->jfd) ? PAGER_OPEN : PAGER_READER); + } if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + pPager->errCode = SQLITE_OK; } pPager->journalOff = 0; @@ -2319,7 +2323,7 @@ static int pager_playback_one_page( pPg = sqlite3PagerLookup(pPager, pgno); } assert( pPg || !MEMDB ); - assert( pPager->eState!=PAGER_OPEN || pPg==0 ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 || pPager->tempFile ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") @@ -5052,17 +5056,17 @@ int sqlite3PagerSharedLock(Pager *pPager){ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either ** be OPEN or READER. READER is only possible if the pager is or was in - ** exclusive access mode. - */ + ** exclusive access mode. */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); - if( pPager->tempFile && pPager->errCode ) { return pPager->errCode; } assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); + assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ assert( !MEMDB ); + assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ @@ -5148,7 +5152,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ - rc = pager_playback(pPager, 1); + rc = pager_playback(pPager, !pPager->tempFile); pPager->eState = PAGER_OPEN; } }else if( !pPager->exclusiveMode ){ @@ -5244,7 +5248,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerBeginReadTransaction(pPager); } - if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ rc = pagerPagecount(pPager, &pPager->dbSize); } diff --git a/test/tempfault.test b/test/tempfault.test index da1ae12375..4eae116a72 100644 --- a/test/tempfault.test +++ b/test/tempfault.test @@ -18,9 +18,9 @@ source $testdir/tester.tcl source $testdir/malloc_common.tcl set testprefix tempfault -sqlite3_memdebug_vfs_oom_test 0 +# sqlite3_memdebug_vfs_oom_test 0 -do_faultsim_test 1 -faults oom* -prep { +do_faultsim_test 1 -faults * -prep { sqlite3 db "" db eval { PRAGMA page_size = 1024; @@ -42,14 +42,14 @@ do_faultsim_test 1 -faults oom* -prep { faultsim_integrity_check } -do_faultsim_test 2 -faults oom* -prep { +do_faultsim_test 2 -faults * -prep { sqlite3 db "" db eval { PRAGMA page_size = 1024; PRAGMA cache_size = 10; CREATE TABLE t1(a, b); CREATE INDEX i1 ON t1(b, a); - WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50) + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100) INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; } } -body { @@ -59,7 +59,27 @@ do_faultsim_test 2 -faults oom* -prep { faultsim_integrity_check db } -do_faultsim_test 3 -faults oom* -prep { +catch { db close } +do_faultsim_test 2.1 -faults * -prep { + if {[info commands db]==""} { + sqlite3 db "" + execsql { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } + } +} -body { + execsql { UPDATE t1 SET a = randomblob(99) } +} -test { + faultsim_test_result {0 {}} + faultsim_integrity_check db +} + +do_faultsim_test 3 -faults * -prep { sqlite3 db "" db eval { PRAGMA page_size = 1024; From 2d36f065e2843175057dba4e4e34fa3df6541357 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Apr 2016 17:24:16 +0000 Subject: [PATCH 34/63] Fix a problem with mixing temp-files and mmap-mode. FossilOrigin-Name: c80c5c62b2e2c5e47e0839f8e2d5b6341ca4a249 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 6 +++--- test/temptable2.test | 29 +++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index ec3e51504b..c7f8ab468f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\sto\sdo\swith\stemp-file\sdatabases\sand\srecovering\sfrom\sIO\sand\sSQLITE_FULL\serrors. -D 2016-04-23T14:55:28.020 +C Fix\sa\sproblem\swith\smixing\stemp-files\sand\smmap-mode. +D 2016-04-23T17:24:16.091 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c d0b41a47eb5f0dc00e423a1723aadeab0e78c85f F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 4981dc6154ce111361a7cd35eaa26aadf8914ea3 +F src/pager.c cbc8996b773c191107b771424b529307ffdf19ba F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 @@ -1111,7 +1111,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test e62783549be26283e1b3725f0de3309411be6c84 +F test/temptable2.test 31485911fb33e72c7737087ba5a2b35acafba55a F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 49aec9718d61c9f12aed96f530128666c3b01c81 -R 98471ae1c4ed8f399ba91e3e119d0ef8 +P 3d61da4a76af8c9c2a293df085f3ed5a7bb447df +R 85df4f851dc9e804fc35d28f4dbcf65b U dan -Z 4115a15d29a4b7f5f76b2491e48cc4f1 +Z 50e48f2ab5de65ac7aff6f1c82162e94 diff --git a/manifest.uuid b/manifest.uuid index 7b1996a745..865e4fbe13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3d61da4a76af8c9c2a293df085f3ed5a7bb447df \ No newline at end of file +c80c5c62b2e2c5e47e0839f8e2d5b6341ca4a249 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index a749b8a6b3..3f2a131877 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2034,7 +2034,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - if( pagerFlushOnCommit(pPager) ){ + if( MEMDB || pagerFlushOnCommit(pPager) ){ sqlite3PcacheCleanAll(pPager->pPCache); }else{ sqlite3PcacheClearWritable(pPager->pPCache); @@ -5381,7 +5381,7 @@ int sqlite3PagerGet( ); if( rc==SQLITE_OK && pData ){ - if( pPager->eState>PAGER_READER ){ + if( pPager->eState>PAGER_READER || pPager->tempFile ){ pPg = sqlite3PagerLookup(pPager, pgno); } if( pPg==0 ){ @@ -6219,7 +6219,7 @@ int sqlite3PagerCommitPhaseOne( assert( MEMDB==0 || pPager->tempFile ); assert( isOpen(pPager->fd) || pPager->tempFile ); - if( !isOpen(pPager->fd) ){ + if( 0==pagerFlushOnCommit(pPager) ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any ** backup in progress needs to be restarted. */ diff --git a/test/temptable2.test b/test/temptable2.test index c5e53de635..d71c168ebf 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -316,5 +316,34 @@ foreach {tn mode} { } } +#------------------------------------------------------------------------- +# When using mmap mode with a temp file, SQLite must search the cache +# before using a mapped page even when there is no write transaction +# open. For a temp file, the on-disk version may not be up to date. +# +sqlite3 db "" +do_execsql_test 10.0 { + PRAGMA cache_size = 50; + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b, PRIMARY KEY(a)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(a); + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES(1, 2); +} + +do_execsql_test 10.1 { + BEGIN; + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + COMMIT; + INSERT INTO t2 VALUES(3, 4); +} + +do_execsql_test 10.2 { + PRAGMA mmap_size = 512000; + SELECT * FROM t2; + PRAGMA integrity_check; +} {512000 1 2 3 4 ok} + finish_test From 23ee89375284ba23b42d81c4ce3a39d656b0996d Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Apr 2016 20:59:23 +0000 Subject: [PATCH 35/63] Fix the fix to the temporary directory search algorithm so that it continues to return "." as a fallback if that directory has the correct permissions. FossilOrigin-Name: b38fe522cfc971b37ca04e7b63a92bbb6e0b01e1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index df9a5ec95f..366814ad84 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\stemporary\sdirectory\ssearch\salgorithm\son\sunix\sso\sthat\sdirectories\nwith\sonly\s-wx\spermission\sare\sallowed.\s\sAnd\sdo\snot\sallow\s"."\sto\sbe\sreturned\sif\nit\slacks\s-wx\spermission. -D 2016-04-21T23:37:24.378 +C Fix\sthe\sfix\sto\sthe\stemporary\sdirectory\ssearch\salgorithm\sso\sthat\sit\scontinues\nto\sreturn\s"."\sas\sa\sfallback\sif\sthat\sdirectory\shas\sthe\scorrect\spermissions. +D 2016-04-23T20:59:23.071 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -361,7 +361,7 @@ F src/os.c 4d83917f072ad958fba9235136fa2ed43df47905 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c c96826e21e897bf9c6ae896be9de8d55d63a4ed5 +F src/os_unix.c 2488a2b6456709ad6398df2302d427a980e2695a F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c d20fa46d5beda38095be6070dd4c59e502817c72 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bd4a6f8313eb71399a7141ebf7f2617d7b1464c3 -R 90c3ca98d0d4d69b4119d1fe5513a916 +P 67985761aa93fb613b87d340e75371fa55b0f778 +R 0a3904801876ad35cfb663a63395f63c U drh -Z f4304caf089d5c38523c57ca13360c36 +Z 2d6519bfb47a62ce49b104a4f37d1acb diff --git a/manifest.uuid b/manifest.uuid index 65f89ba88b..f26bce27a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67985761aa93fb613b87d340e75371fa55b0f778 \ No newline at end of file +b38fe522cfc971b37ca04e7b63a92bbb6e0b01e1 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 9a97515fdc..aadb414afe 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5416,7 +5416,7 @@ static const char *unixTempFileDir(void){ if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); - for(i=0; i Date: Mon, 25 Apr 2016 01:43:24 +0000 Subject: [PATCH 36/63] Fix a typo in the Makefile. FossilOrigin-Name: d0a579b35105810821bbfec6b50ecfebac7a17ee --- Makefile.in | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile.in b/Makefile.in index 2606b5a654..8200cf2357 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1032,7 +1032,7 @@ FTS5_SRC = \ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h - ./lemon $(OPTS) fts5parse.y + ./lemon$(BEXE) $(OPTS) fts5parse.y fts5parse.h: fts5parse.c diff --git a/manifest b/manifest index 366814ad84..aba5d698fd 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Fix\sthe\sfix\sto\sthe\stemporary\sdirectory\ssearch\salgorithm\sso\sthat\sit\scontinues\nto\sreturn\s"."\sas\sa\sfallback\sif\sthat\sdirectory\shas\sthe\scorrect\spermissions. -D 2016-04-23T20:59:23.071 -F Makefile.in eba680121821b8a60940a81454316f47a341487a +C Fix\sa\stypo\sin\sthe\sMakefile. +D 2016-04-25T01:43:24.306 +F Makefile.in a905f3180accdafbd5a534bf26126ee5306d5056 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 67985761aa93fb613b87d340e75371fa55b0f778 -R 0a3904801876ad35cfb663a63395f63c -U drh -Z 2d6519bfb47a62ce49b104a4f37d1acb +P b38fe522cfc971b37ca04e7b63a92bbb6e0b01e1 +R 5aa002889bfcaf382a8424adff459dd7 +U mistachkin +Z 272f168dd75d48a808379cca7049b3ce diff --git a/manifest.uuid b/manifest.uuid index f26bce27a6..41e73dfc0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b38fe522cfc971b37ca04e7b63a92bbb6e0b01e1 \ No newline at end of file +d0a579b35105810821bbfec6b50ecfebac7a17ee \ No newline at end of file From b1ec87afdbed6096a2f968becb619710eb12a122 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Apr 2016 02:20:10 +0000 Subject: [PATCH 37/63] When checking for the WHERE-clause push-down optimization, verify that all terms of the compound inner SELECT are non-aggregate, not just the last term. Fix for ticket [f7f8c97e97597]. FossilOrigin-Name: ec215f94ac9748c0acd82af0cc9e7a92249462f9 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 14 +++++++++----- test/select4.test | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index aba5d698fd..10b7a29fe4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\sMakefile. -D 2016-04-25T01:43:24.306 +C When\schecking\sfor\sthe\sWHERE-clause\spush-down\soptimization,\sverify\sthat\nall\sterms\sof\sthe\scompound\sinner\sSELECT\sare\snon-aggregate,\snot\sjust\sthe\nlast\sterm.\s\sFix\sfor\sticket\s[f7f8c97e97597]. +D 2016-04-25T02:20:10.236 F Makefile.in a905f3180accdafbd5a534bf26126ee5306d5056 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -377,7 +377,7 @@ F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e -F src/select.c 30217121bdf6b587462150b8ee9e1467f7a6036b +F src/select.c fd4a7ce2937497181063cfedb92058ac89491a5d F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6 F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1026,7 +1026,7 @@ F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054 -F test/select4.test d926792a5e4d88fef0ddcddeb45d27ce75f7296c +F test/select4.test 5389d9895968d1196c457d59b3ee6515d771d328 F test/select5.test e758b8ef94f69b111df4cb819008856655dcd535 F test/select6.test 39eac4a5c03650b2b473c532882273283ee8b7a0 F test/select7.test 95e370c42d47c3c52377d05e9ffc01ccff7c1f61 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b38fe522cfc971b37ca04e7b63a92bbb6e0b01e1 -R 5aa002889bfcaf382a8424adff459dd7 -U mistachkin -Z 272f168dd75d48a808379cca7049b3ce +P d0a579b35105810821bbfec6b50ecfebac7a17ee +R 858c551be0612332d0982a3a42946fb6 +U drh +Z c1158477a3c6966fdb5709eb9b3c8fe5 diff --git a/manifest.uuid b/manifest.uuid index 41e73dfc0b..580a058689 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0a579b35105810821bbfec6b50ecfebac7a17ee \ No newline at end of file +ec215f94ac9748c0acd82af0cc9e7a92249462f9 \ No newline at end of file diff --git a/src/select.c b/src/select.c index b86e040f0f..eb37536b6f 100644 --- a/src/select.c +++ b/src/select.c @@ -3785,14 +3785,18 @@ static int pushDownWhereTerms( ){ Expr *pNew; int nChng = 0; + Select *pX; /* For looping over compound SELECTs in pSubq */ if( pWhere==0 ) return 0; - if( (pSubq->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ - testcase( pSubq->selFlags & SF_Aggregate ); - testcase( pSubq->selFlags & SF_Recursive ); - return 0; /* restrictions (1) and (2) */ + for(pX=pSubq; pX; pX=pX->pPrior){ + if( (pX->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ + testcase( pX->selFlags & SF_Aggregate ); + testcase( pX->selFlags & SF_Recursive ); + testcase( pX!=pSubq ); + return 0; /* restrictions (1) and (2) */ + } } if( pSubq->pLimit!=0 ){ - return 0; /* restriction (3) */ + return 0; /* restriction (3) */ } while( pWhere->op==TK_AND ){ nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); diff --git a/test/select4.test b/test/select4.test index 1f29f29073..51a1b1c4c5 100644 --- a/test/select4.test +++ b/test/select4.test @@ -968,6 +968,42 @@ do_execsql_test select4-16.3 { ORDER BY t3.a; } {95 96 97 98 99} +# Ticket https://www.sqlite.org/src/tktview/f7f8c97e975978d45 on 2016-04-25 +# +# The where push-down optimization from 2015-06-02 is suppose to disable +# on aggregate subqueries. But if the subquery is a compound where the +# last SELECT is non-aggregate but some other SELECT is an aggregate, the +# test is incomplete and the optimization is not properly disabled. +# +# The following test cases verify that the fix works. +# +do_execsql_test select4-17.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a int, b int); + INSERT INTO t1 VALUES(1,2),(1,18),(2,19); + SELECT x, y FROM ( + SELECT 98 AS x, 99 AS y + UNION + SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a + ) AS w WHERE y>=20 + ORDER BY +x; +} {1 20 98 99} +do_execsql_test select4-17.2 { + SELECT x, y FROM ( + SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a + UNION + SELECT 98 AS x, 99 AS y + ) AS w WHERE y>=20 + ORDER BY +x; +} {1 20 98 99} +do_catchsql_test select4-17.3 { + SELECT x, y FROM ( + SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a LIMIT 3 + UNION + SELECT 98 AS x, 99 AS y + ) AS w WHERE y>=20 + ORDER BY +x; +} {1 {LIMIT clause should come after UNION not before}} From 835f22dedade668d99034711a3758ca32752e716 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Apr 2016 19:20:56 +0000 Subject: [PATCH 38/63] Fix some unreachable branches in the pager. FossilOrigin-Name: 3ae44770fdecc40c8097f1de0b504f36585a2232 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 18 ++++++++++-------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 713cf4421c..acd08f3f4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sbug\sfix\sfrom\strunk. -D 2016-04-25T15:03:49.467 +C Fix\ssome\sunreachable\sbranches\sin\sthe\spager. +D 2016-04-25T19:20:56.371 F Makefile.in a905f3180accdafbd5a534bf26126ee5306d5056 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -364,7 +364,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 2488a2b6456709ad6398df2302d427a980e2695a F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c cbc8996b773c191107b771424b529307ffdf19ba +F src/pager.c a31af55dd6139c6ece7642667a035028a266d344 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9b8fec60d8e576cd09e1d075a59bfad1c6169d7a ec215f94ac9748c0acd82af0cc9e7a92249462f9 -R e941df834a8d6e2084dd43227adc9b5d +P a905d5e08de3f3c60b667d840b5995911372647d +R 1f1507f9b43194c78dbe59034ff5cd5c U drh -Z 530f6cc77507869285e9eb18ff03544a +Z 8869ed19649f951f40d0b8fe31c4bd6b diff --git a/manifest.uuid b/manifest.uuid index 02d571129d..010c2a556f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a905d5e08de3f3c60b667d840b5995911372647d \ No newline at end of file +3ae44770fdecc40c8097f1de0b504f36585a2232 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 3f2a131877..0a185c3e37 100644 --- a/src/pager.c +++ b/src/pager.c @@ -872,6 +872,7 @@ static int assert_pager_state(Pager *p){ ** state. */ if( MEMDB ){ + assert( !isOpen(p->fd) ); assert( p->noSync ); assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_MEMORY @@ -3204,6 +3205,8 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ */ assert( pPager->eState==PAGER_OPEN ); assert( pPager->eLock>=SHARED_LOCK ); + assert( isOpen(pPager->fd) ); + assert( pPager->tempFile==0 ); nPage = sqlite3WalDbsize(pPager->pWal); /* If the number of pages in the database is not available from the @@ -3211,14 +3214,11 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ ** the database file. If the size of the database file is not an ** integer multiple of the page-size, round up the result. */ - if( nPage==0 ){ + if( nPage==0 && ALWAYS(isOpen(pPager->fd)) ){ i64 n = 0; /* Size of db file in bytes */ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - int rc = sqlite3OsFileSize(pPager->fd, &n); - if( rc!=SQLITE_OK ){ - return rc; - } + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; } nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); } @@ -4964,6 +4964,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK && !locked ){ Pgno nPage; /* Number of pages in database file */ + assert( pPager->tempFile==0 ); rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ /* If the database is zero pages in size, that means that either (1) the @@ -5448,7 +5449,8 @@ int sqlite3PagerGet( goto pager_acquire_err; } - if( MEMDB || pPager->dbSizefd) ){ + assert( !isOpen(pPager->fd) || !MEMDB ); + if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; From ae20690e2ce48e9c519baddce5fc48d2f97deb10 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Apr 2016 19:25:12 +0000 Subject: [PATCH 39/63] Update the RBU vacuum code so that databases that use custom collation sequences can be vacuumed. FossilOrigin-Name: 7dd48c10790a7b9c4165214399c261a0aa701297 --- ext/rbu/rbuvacuum.test | 95 +++++++++++++++++++++ ext/rbu/sqlite3rbu.c | 189 +++++++++++++++++++++-------------------- ext/rbu/test_rbu.c | 22 ++++- manifest | 18 ++-- manifest.uuid | 2 +- 5 files changed, 220 insertions(+), 106 deletions(-) diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test index 96a37d1f71..7d82e380d6 100644 --- a/ext/rbu/rbuvacuum.test +++ b/ext/rbu/rbuvacuum.test @@ -255,7 +255,41 @@ foreach step {0 1} { } } {} do_rbu_vacuum_test 1.10.2 $step + + # Database with empty tables. + # + reset_db + do_execsql_test 1.11.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + INSERT INTO t4 VALUES(1, 2); + } + do_rbu_vacuum_test 1.11.2 $step + do_execsql_test 1.11.3 { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + SELECT * FROM t4; + } {1 2} + reset_db + do_execsql_test 1.12.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + } + do_rbu_vacuum_test 1.12.2 $step + do_execsql_test 1.12.3 { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + SELECT * FROM t4; + } {1 2} } +set ::testprefix rbuvacuum #------------------------------------------------------------------------- # Test some error cases: @@ -310,6 +344,67 @@ for {set i 1} 1 {incr i} { } {1 {SQLITE_BUSY - database modified during rbu vacuum}} } +#------------------------------------------------------------------------- +# Test that a database that uses custom collation sequences can be RBU +# vacuumed. +# +reset_db +forcedelete state.db +proc noop {args} {} +proc length_cmp {x y} { + set n1 [string length $x] + set n2 [string length $y] + return [expr $n1 - $n2] +} +sqlite3_create_collation_v2 db length length_cmp noop + +do_execsql_test 3.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'i'); + INSERT INTO t1 VALUES(2, 'iiii'); + INSERT INTO t1 VALUES(3, 'ii'); + INSERT INTO t1 VALUES(4, 'iii'); + SELECT a FROM t1 ORDER BY b COLLATE length; +} {1 3 4 2} +do_execsql_test 3.1 { + CREATE INDEX i1 ON t1(b COLLATE length); +} + +do_test 3.2 { + sqlite3rbu_vacuum rbu test.db state.db + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.3 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 0] + sqlite3_create_collation_v2 $db1 length length_cmp noop + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.4 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 1] + sqlite3_create_collation_v2 $db1 length length_cmp noop + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.5 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 0] + set db2 [rbu db 1] + + sqlite3_create_collation_v2 $db1 length length_cmp noop + sqlite3_create_collation_v2 $db2 length length_cmp noop + + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {0 SQLITE_DONE} + + catch { db close } finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index e7e4ddef58..dc80935ecf 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3081,6 +3081,92 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ } +/* +** The second argument passed to this function is the name of a PRAGMA +** setting - "page_size", "auto_vacuum", "user_version" or "application_id". +** This function executes the following on sqlite3rbu.dbRbu: +** +** "PRAGMA main.$zPragma" +** +** where $zPragma is the string passed as the second argument, then +** on sqlite3rbu.dbMain: +** +** "PRAGMA main.$zPragma = $val" +** +** where $val is the value returned by the first PRAGMA invocation. +** +** In short, it copies the value of the specified PRAGMA setting from +** dbRbu to dbMain. +*/ +static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pPragma = 0; + p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.%s", zPragma) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){ + p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d", + zPragma, sqlite3_column_int(pPragma, 0) + ); + } + rbuFinalize(p, pPragma); + } +} + +/* +** The RBU handle passed as the only argument has just been opened and +** the state database is empty. If this RBU handle was opened for an +** RBU vacuum operation, create the schema in the target db. +*/ +static void rbuCreateTargetSchema(sqlite3rbu *p){ + sqlite3_stmt *pSql = 0; + sqlite3_stmt *pInsert = 0; + + assert( rbuIsVacuum(p) ); + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" + " AND name!='sqlite_sequence' " + " ORDER BY type DESC" + ); + } + + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pSql, 0); + p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); + } + rbuFinalize(p, pSql); + if( p->rc!=SQLITE_OK ) return; + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg, + "INSERT INTO sqlite_master VALUES(?,?,?,?,?)" + ); + } + + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + int i; + for(i=0; i<5; i++){ + sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i)); + } + sqlite3_step(pInsert); + p->rc = sqlite3_reset(pInsert); + } + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg); + } + + rbuFinalize(p, pSql); + rbuFinalize(p, pInsert); +} + /* ** Step the RBU object. */ @@ -3089,6 +3175,15 @@ int sqlite3rbu_step(sqlite3rbu *p){ switch( p->eStage ){ case RBU_STAGE_OAL: { RbuObjIter *pIter = &p->objiter; + + /* If this is an RBU vacuum operation and the state table was empty + ** when this handle was opened, create the target database schema. */ + if( rbuIsVacuum(p) && p->nProgress==0 && p->rc==SQLITE_OK ){ + rbuCreateTargetSchema(p); + rbuCopyPragma(p, "user_version"); + rbuCopyPragma(p, "application_id"); + } + while( p->rc==SQLITE_OK && pIter->zTbl ){ if( pIter->bCleanup ){ @@ -3371,92 +3466,6 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ } } -/* -** The second argument passed to this function is the name of a PRAGMA -** setting - "page_size", "auto_vacuum", "user_version" or "application_id". -** This function executes the following on sqlite3rbu.dbRbu: -** -** "PRAGMA main.$zPragma" -** -** where $zPragma is the string passed as the second argument, then -** on sqlite3rbu.dbMain: -** -** "PRAGMA main.$zPragma = $val" -** -** where $val is the value returned by the first PRAGMA invocation. -** -** In short, it copies the value of the specified PRAGMA setting from -** dbRbu to dbMain. -*/ -static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){ - if( p->rc==SQLITE_OK ){ - sqlite3_stmt *pPragma = 0; - p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.%s", zPragma) - ); - if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){ - p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d", - zPragma, sqlite3_column_int(pPragma, 0) - ); - } - rbuFinalize(p, pPragma); - } -} - -/* -** The RBU handle passed as the only argument has just been opened and -** the state database is empty. If this RBU handle was opened for an -** RBU vacuum operation, create the schema in the target db. -*/ -static void rbuCreateTargetSchema(sqlite3rbu *p){ - sqlite3_stmt *pSql = 0; - sqlite3_stmt *pInsert = 0; - - assert( rbuIsVacuum(p) ); - p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); - if( p->rc==SQLITE_OK ){ - p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, - "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" - " AND name!='sqlite_sequence' " - " ORDER BY type DESC" - ); - } - - while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ - const char *zSql = (const char*)sqlite3_column_text(pSql, 0); - p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); - } - rbuFinalize(p, pSql); - if( p->rc!=SQLITE_OK ) return; - - if( p->rc==SQLITE_OK ){ - p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, - "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" - ); - } - - if( p->rc==SQLITE_OK ){ - p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg, - "INSERT INTO sqlite_master VALUES(?,?,?,?,?)" - ); - } - - while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ - int i; - for(i=0; i<5; i++){ - sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i)); - } - sqlite3_step(pInsert); - p->rc = sqlite3_reset(pInsert); - } - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg); - } - - rbuFinalize(p, pSql); - rbuFinalize(p, pInsert); -} - static sqlite3rbu *openRbuHandle( const char *zTarget, @@ -3569,14 +3578,6 @@ static sqlite3rbu *openRbuHandle( } } - /* If this is an RBU vacuum operation and the state table was empty - ** when this handle was opened, create the target database schema. */ - if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){ - rbuCreateTargetSchema(p); - rbuCopyPragma(p, "user_version"); - rbuCopyPragma(p, "application_id"); - } - /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ p->rc = rbuObjIterFirst(p, &p->objiter); diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index 629a33cd0e..2d52262c71 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -20,8 +20,9 @@ #include #include -/* From main.c (apparently...) */ +/* From main.c */ extern const char *sqlite3ErrName(int); +extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); void test_rbu_delta(sqlite3_context *pCtx, int nArg, sqlite3_value **apVal){ Tcl_Interp *interp = (Tcl_Interp*)sqlite3_user_data(pCtx); @@ -66,7 +67,8 @@ static int test_sqlite3rbu_cmd( {"create_rbu_delta", 2, ""}, /* 2 */ {"savestate", 2, ""}, /* 3 */ {"dbMain_eval", 3, "SQL"}, /* 4 */ - {"bp_progress", 2, ""}, /* 5 */ + {"bp_progress", 2, ""}, /* 5 */ + {"db", 3, "RBU"}, /* 6 */ {0,0,0} }; int iCmd; @@ -149,6 +151,22 @@ static int test_sqlite3rbu_cmd( break; } + case 6: /* db */ { + int bArg; + if( Tcl_GetBooleanFromObj(interp, objv[2], &bArg) ){ + ret = TCL_ERROR; + }else{ + char zBuf[50]; + sqlite3 *db = sqlite3rbu_db(pRbu, bArg); + if( sqlite3TestMakePointerStr(interp, zBuf, (void*)db) ){ + ret = TCL_ERROR; + }else{ + Tcl_SetResult(interp, zBuf, TCL_VOLATILE); + } + } + break; + } + default: /* seems unlikely */ assert( !"cannot happen" ); break; diff --git a/manifest b/manifest index 10b7a29fe4..d07dc12346 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\schecking\sfor\sthe\sWHERE-clause\spush-down\soptimization,\sverify\sthat\nall\sterms\sof\sthe\scompound\sinner\sSELECT\sare\snon-aggregate,\snot\sjust\sthe\nlast\sterm.\s\sFix\sfor\sticket\s[f7f8c97e97597]. -D 2016-04-25T02:20:10.236 +C Update\sthe\sRBU\svacuum\scode\sso\sthat\sdatabases\sthat\suse\scustom\scollation\ssequences\scan\sbe\svacuumed. +D 2016-04-25T19:25:12.645 F Makefile.in a905f3180accdafbd5a534bf26126ee5306d5056 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,10 +247,10 @@ F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/rbuvacuum.test bcbc1dcd8e2a46a811e46477692ae1c0e8917a85 -F ext/rbu/sqlite3rbu.c 20922328dcebe89589638923bb46840df8bc7733 +F ext/rbu/rbuvacuum.test 66e02cf299836770e718e95c36686be0b26dbda3 +F ext/rbu/sqlite3rbu.c bf36625990c6865ecf08bd844d8097ed2d0a6958 F ext/rbu/sqlite3rbu.h 2acd0a6344a6079de15c8dc9d84d3df83a665930 -F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 +F ext/rbu/test_rbu.c 9bbdf6bd8efd58fbc4f192698df50569598fbb9e F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d0a579b35105810821bbfec6b50ecfebac7a17ee -R 858c551be0612332d0982a3a42946fb6 -U drh -Z c1158477a3c6966fdb5709eb9b3c8fe5 +P ec215f94ac9748c0acd82af0cc9e7a92249462f9 +R c0a43d6aa5e9f6d906be746461438b7f +U dan +Z 33a592aa08c77e06ce6e19e096f78faa diff --git a/manifest.uuid b/manifest.uuid index 580a058689..2b1a4a4c78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec215f94ac9748c0acd82af0cc9e7a92249462f9 \ No newline at end of file +7dd48c10790a7b9c4165214399c261a0aa701297 \ No newline at end of file From 6c96358601f1da67c58661706c6066dce51214a8 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Apr 2016 19:28:34 +0000 Subject: [PATCH 40/63] Remove an obsolete comment from the pager. No code changes. FossilOrigin-Name: 36f97ca8874a03ac8699f44fe0da95b0be507cc7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 5 +---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index acd08f3f4f..ad1b6577cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sunreachable\sbranches\sin\sthe\spager. -D 2016-04-25T19:20:56.371 +C Remove\san\sobsolete\scomment\sfrom\sthe\spager.\s\sNo\scode\schanges. +D 2016-04-25T19:28:34.469 F Makefile.in a905f3180accdafbd5a534bf26126ee5306d5056 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -364,7 +364,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 2488a2b6456709ad6398df2302d427a980e2695a F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c a31af55dd6139c6ece7642667a035028a266d344 +F src/pager.c 456ea27c2a467c161b1466b8f4f5307f12314d31 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a905d5e08de3f3c60b667d840b5995911372647d -R 1f1507f9b43194c78dbe59034ff5cd5c +P 3ae44770fdecc40c8097f1de0b504f36585a2232 +R db63125d858bb9f2f1e3791a791b0b06 U drh -Z 8869ed19649f951f40d0b8fe31c4bd6b +Z 7c3cc56899db1d72bed3171f2d6f3ea7 diff --git a/manifest.uuid b/manifest.uuid index 010c2a556f..5ba52e8d51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ae44770fdecc40c8097f1de0b504f36585a2232 \ No newline at end of file +36f97ca8874a03ac8699f44fe0da95b0be507cc7 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 0a185c3e37..6e464ba82f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1820,10 +1820,7 @@ static void pager_unlock(Pager *pPager){ ** trusted. Now that there are no outstanding references to the pager, ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. - ** - ** Exception: There is no way out of the error state for temp files. - ** This is because it is not possible to call pager_reset() on a temp - ** file pager (as this may discard the only copy of some data). */ + */ assert( pPager->errCode==SQLITE_OK || !MEMDB ); if( pPager->errCode ){ if( pPager->tempFile==0 ){ From ead01fd27250ce29a44734a050954ac48434607a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Apr 2016 22:39:42 +0000 Subject: [PATCH 41/63] Add an sqlite3FaultSim() all to make it easier to simulate IO errors in a VACUUM commit. FossilOrigin-Name: 8bfde416c51f4e087275aebf652a73985515e810 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index ad1b6577cc..07bba5bb6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sobsolete\scomment\sfrom\sthe\spager.\s\sNo\scode\schanges. -D 2016-04-25T19:28:34.469 +C Add\san\ssqlite3FaultSim()\sall\sto\smake\sit\seasier\sto\ssimulate\sIO\serrors\sin\na\sVACUUM\scommit. +D 2016-04-25T22:39:42.651 F Makefile.in a905f3180accdafbd5a534bf26126ee5306d5056 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -364,7 +364,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 2488a2b6456709ad6398df2302d427a980e2695a F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 456ea27c2a467c161b1466b8f4f5307f12314d31 +F src/pager.c a8d30c49c231e9a20d05257613db922532588963 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3ae44770fdecc40c8097f1de0b504f36585a2232 -R db63125d858bb9f2f1e3791a791b0b06 +P 36f97ca8874a03ac8699f44fe0da95b0be507cc7 +R 54e681c99d31e5095466803c5f1b64d8 U drh -Z 7c3cc56899db1d72bed3171f2d6f3ea7 +Z 6d6cde3ceb3c27e62273721031ebad91 diff --git a/manifest.uuid b/manifest.uuid index 5ba52e8d51..bbdd0e7139 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -36f97ca8874a03ac8699f44fe0da95b0be507cc7 \ No newline at end of file +8bfde416c51f4e087275aebf652a73985515e810 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 6e464ba82f..d24337cd50 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6210,6 +6210,9 @@ int sqlite3PagerCommitPhaseOne( /* If a prior error occurred, report that error again. */ if( NEVER(pPager->errCode) ) return pPager->errCode; + /* Provide the ability to easily simulate an I/O error during testing */ + if( (rc = sqlite3FaultSim(400))!=SQLITE_OK ) return rc; + PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); From b283a0cde90b25fd01054228e4c5a680233d4742 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Apr 2016 13:13:53 +0000 Subject: [PATCH 42/63] Fix the "checksymbols" target in Makefile.in to be able to deal with the sqlite3changegroup family of interfaces. FossilOrigin-Name: d819bfbd46dc7f065d2a3ab0ef9f8322d8394e11 --- Makefile.in | 3 ++- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Makefile.in b/Makefile.in index 8200cf2357..e4687626f0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1164,8 +1164,9 @@ loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. # +VALIDIDS=' sqlite3(changeset|changegroup|session)?_' checksymbols: sqlite3.lo - nm -g --defined-only sqlite3.o | egrep -v ' sqlite3(changeset|session)?_' ; test $$? -ne 0 + nm -g --defined-only sqlite3.o | egrep -v $(VALIDIDS); test $$? -ne 0 echo '0 errors out of 1 tests' # Build the amalgamation-autoconf package. The amalamgation-tarball target builds diff --git a/manifest b/manifest index d07dc12346..01396063e2 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Update\sthe\sRBU\svacuum\scode\sso\sthat\sdatabases\sthat\suse\scustom\scollation\ssequences\scan\sbe\svacuumed. -D 2016-04-25T19:25:12.645 -F Makefile.in a905f3180accdafbd5a534bf26126ee5306d5056 +C Fix\sthe\s"checksymbols"\starget\sin\sMakefile.in\sto\sbe\sable\sto\sdeal\swith\sthe\nsqlite3changegroup\sfamily\sof\sinterfaces. +D 2016-04-26T13:13:53.622 +F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec215f94ac9748c0acd82af0cc9e7a92249462f9 -R c0a43d6aa5e9f6d906be746461438b7f -U dan -Z 33a592aa08c77e06ce6e19e096f78faa +P 7dd48c10790a7b9c4165214399c261a0aa701297 +R 5836a38779892d93c5f443eeb9cd4151 +U drh +Z a0cff4b059a2c6bf807105b3b6e856e6 diff --git a/manifest.uuid b/manifest.uuid index 2b1a4a4c78..cff4e97ad7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7dd48c10790a7b9c4165214399c261a0aa701297 \ No newline at end of file +d819bfbd46dc7f065d2a3ab0ef9f8322d8394e11 \ No newline at end of file From 45164826b76087ae15c714b87d35336aee056637 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Apr 2016 17:10:03 +0000 Subject: [PATCH 43/63] Fix an issue in temptable2.test preventing it from working with SQLITE_DEFAULT_AUTOVACUUM=1 builds. FossilOrigin-Name: e790aac02e4b427b4891b514a050699d159b03b1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/temptable2.test | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3224381eed..52a995d5ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\schecksymbols\sfix\sfrom\strunk\sinto\sthis\sbranch. -D 2016-04-26T16:03:58.388 +C Fix\san\sissue\sin\stemptable2.test\spreventing\sit\sfrom\sworking\swith\sSQLITE_DEFAULT_AUTOVACUUM=1\sbuilds. +D 2016-04-26T17:10:03.745 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -1113,7 +1113,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test 31485911fb33e72c7737087ba5a2b35acafba55a +F test/temptable2.test d4e967c355b154e8bc387de961430e94c20f2dca F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8bfde416c51f4e087275aebf652a73985515e810 d819bfbd46dc7f065d2a3ab0ef9f8322d8394e11 -R 0f2ec21167579c0f8c46594d0d9ecc27 +P 04911cee0cd5467ee99744b774d692219e5e1425 +R 849d538a2e2ee898d96d5ba70eff854f U dan -Z 444c73c0b2d802c9a660647c7b166bf1 +Z 496f6fcde8513321efb142fd9a4f11c8 diff --git a/manifest.uuid b/manifest.uuid index 6798dc8b47..021372e495 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04911cee0cd5467ee99744b774d692219e5e1425 \ No newline at end of file +e790aac02e4b427b4891b514a050699d159b03b1 \ No newline at end of file diff --git a/test/temptable2.test b/test/temptable2.test index d71c168ebf..21e06a3a4f 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -224,6 +224,7 @@ do_execsql_test 7.1 { # reset_db do_execsql_test 8.1 { + PRAGMA auto_vacuum = OFF; CREATE TABLE t2(a, b); CREATE INDEX i2 ON t2(a, b); WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<20 ) @@ -234,6 +235,7 @@ do_execsql_test 8.1 { do_test 8.2 { sqlite3 tmp "" execsql { + PRAGMA auto_vacuum = OFF; PRAGMA page_size = 8192; CREATE TABLE t1(a, b); CREATE INDEX i1 ON t1(a, b); From 97a7e5e68cdfe31e9e020ed59d8f93787e18cd25 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Apr 2016 18:58:54 +0000 Subject: [PATCH 44/63] The pcache and the built-in VFSes should not use mutexes when SQLITE_CONFIG_SINGLETHREAD is set. FossilOrigin-Name: 12418b100196abbfbfb85e0ab4bb6b1cbf335df7 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 10 ++++++---- src/os_win.c | 14 ++++++++------ src/pcache1.c | 4 ++-- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 01396063e2..a96ddd6939 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s"checksymbols"\starget\sin\sMakefile.in\sto\sbe\sable\sto\sdeal\swith\sthe\nsqlite3changegroup\sfamily\sof\sinterfaces. -D 2016-04-26T13:13:53.622 +C The\spcache\sand\sthe\sbuilt-in\sVFSes\sshould\snot\suse\smutexes\swhen\nSQLITE_CONFIG_SINGLETHREAD\sis\sset. +D 2016-04-26T18:58:54.655 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -361,15 +361,15 @@ F src/os.c 4d83917f072ad958fba9235136fa2ed43df47905 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 2488a2b6456709ad6398df2302d427a980e2695a -F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 +F src/os_unix.c 8422fba2eb592fbbb2d4006b6f2a67cad8951495 +F src/os_win.c 852fc2ff6084296348ed3739c548b2cf32df394e F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c d20fa46d5beda38095be6070dd4c59e502817c72 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 -F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 +F src/pcache1.c 7f51d2b541aab57596adf62db2c4bb025d34f04d F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7dd48c10790a7b9c4165214399c261a0aa701297 -R 5836a38779892d93c5f443eeb9cd4151 +P d819bfbd46dc7f065d2a3ab0ef9f8322d8394e11 +R 2246a8395360136a36ef2a0e4a464571 U drh -Z a0cff4b059a2c6bf807105b3b6e856e6 +Z fce32975c383f8c036923c0ce041e29e diff --git a/manifest.uuid b/manifest.uuid index cff4e97ad7..d1ff167696 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d819bfbd46dc7f065d2a3ab0ef9f8322d8394e11 \ No newline at end of file +12418b100196abbfbfb85e0ab4bb6b1cbf335df7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index aadb414afe..01de00e0dd 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4288,10 +4288,12 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_NOMEM_BKPT; - goto shm_open_err; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } } if( pInode->bProcessLock==0 ){ diff --git a/src/os_win.c b/src/os_win.c index 1f26463453..9a34d9a106 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1260,8 +1260,8 @@ int sqlite3_win32_reset_heap(){ int rc; MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ MUTEX_LOGIC( sqlite3_mutex *pMem; ) /* The memsys static mutex */ - MUTEX_LOGIC( pMaster = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); ) - MUTEX_LOGIC( pMem = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM); ) + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + MUTEX_LOGIC( pMem = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); ) sqlite3_mutex_enter(pMaster); sqlite3_mutex_enter(pMem); winMemAssertMagic(); @@ -3764,10 +3764,12 @@ static int winOpenSharedMemory(winFile *pDbFd){ pShmNode->pNext = winShmNodeList; winShmNodeList = pShmNode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_IOERR_NOMEM_BKPT; - goto shm_open_err; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shm_open_err; + } } rc = winOpen(pDbFd->pVfs, diff --git a/src/pcache1.c b/src/pcache1.c index d168e7fbcb..5fe963ad05 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -690,8 +690,8 @@ static int pcache1Init(void *NotUsed){ #if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ - pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); - pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); + pcache1.grp.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU); + pcache1.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PMEM); } #endif if( pcache1.separateCache From e5f06fdc8718d60ae90b5791026b58fd812c3963 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Apr 2016 21:02:53 +0000 Subject: [PATCH 45/63] Remove an unnecessary line from the sqlite3AtoF() routine. FossilOrigin-Name: a9d007494cfe30a26dca5352655dc7702351476e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/util.c | 3 --- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a96ddd6939..bd6eed15fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\spcache\sand\sthe\sbuilt-in\sVFSes\sshould\snot\suse\smutexes\swhen\nSQLITE_CONFIG_SINGLETHREAD\sis\sset. -D 2016-04-26T18:58:54.655 +C Remove\san\sunnecessary\sline\sfrom\sthe\ssqlite3AtoF()\sroutine. +D 2016-04-26T21:02:53.535 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -441,7 +441,7 @@ F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 F src/update.c 3e67ab3c0814635f355fb1f8ab010a2b9e016e7d F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c 187a0a2aaa3c5d2ccd2ab0143b2fd9e86d6bc816 +F src/util.c 8ae5e894687fb90cb22a870284eb4a8d077f4f93 F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52 F src/vdbe.c d3843a66d74a7696477ee5141e5eb9a7e5e2401c F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d819bfbd46dc7f065d2a3ab0ef9f8322d8394e11 -R 2246a8395360136a36ef2a0e4a464571 +P 12418b100196abbfbfb85e0ab4bb6b1cbf335df7 +R 2a12fe749cf4b5592aed6dd4a1552e1f U drh -Z fce32975c383f8c036923c0ce041e29e +Z ab5ecd1a857f2f93572fff644c4c92bc diff --git a/manifest.uuid b/manifest.uuid index d1ff167696..128c730aca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -12418b100196abbfbfb85e0ab4bb6b1cbf335df7 \ No newline at end of file +a9d007494cfe30a26dca5352655dc7702351476e \ No newline at end of file diff --git a/src/util.c b/src/util.c index 2f77a6033f..e01928815b 100644 --- a/src/util.c +++ b/src/util.c @@ -384,9 +384,6 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ z+=incr; } - /* skip leading zeroes */ - while( z Date: Tue, 26 Apr 2016 23:14:45 +0000 Subject: [PATCH 46/63] Further simplifications to sqlite3AtoF() to remove unneeded branches. FossilOrigin-Name: dd69e53cb077873171af5312c633ca185595bf31 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/util.c | 11 ++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index bd6eed15fa..2dd1865c29 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sline\sfrom\sthe\ssqlite3AtoF()\sroutine. -D 2016-04-26T21:02:53.535 +C Further\ssimplifications\sto\ssqlite3AtoF()\sto\sremove\sunneeded\sbranches. +D 2016-04-26T23:14:45.113 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -441,7 +441,7 @@ F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 F src/update.c 3e67ab3c0814635f355fb1f8ab010a2b9e016e7d F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c 8ae5e894687fb90cb22a870284eb4a8d077f4f93 +F src/util.c 7ea35eef53451786b4729d2a64ba9a8574a11e31 F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52 F src/vdbe.c d3843a66d74a7696477ee5141e5eb9a7e5e2401c F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 12418b100196abbfbfb85e0ab4bb6b1cbf335df7 -R 2a12fe749cf4b5592aed6dd4a1552e1f +P a9d007494cfe30a26dca5352655dc7702351476e +R bdb81af968b0eff03040af96472e5367 U drh -Z ab5ecd1a857f2f93572fff644c4c92bc +Z 7162b1b4b927db763c29ab3a71a81028 diff --git a/manifest.uuid b/manifest.uuid index 128c730aca..48cc5d7fb4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9d007494cfe30a26dca5352655dc7702351476e \ No newline at end of file +dd69e53cb077873171af5312c633ca185595bf31 \ No newline at end of file diff --git a/src/util.c b/src/util.c index e01928815b..771213be13 100644 --- a/src/util.c +++ b/src/util.c @@ -400,12 +400,13 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ z+=incr; /* copy digits from after decimal to significand ** (decrease exponent by d to shift decimal right) */ - while( z=zEnd ) goto do_atof_calc; From c6daa01c058378590d0a219d857dfde1acb12866 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Apr 2016 02:35:03 +0000 Subject: [PATCH 47/63] Remove still more unnecessary branches from sqlite3AtoF(). FossilOrigin-Name: 3adfe9f3e6ce7cc09fcb570d9b65e830a96cac15 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/util.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 2dd1865c29..06c6814c38 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\ssimplifications\sto\ssqlite3AtoF()\sto\sremove\sunneeded\sbranches. -D 2016-04-26T23:14:45.113 +C Remove\sstill\smore\sunnecessary\sbranches\sfrom\ssqlite3AtoF(). +D 2016-04-27T02:35:03.572 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -441,7 +441,7 @@ F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 F src/update.c 3e67ab3c0814635f355fb1f8ab010a2b9e016e7d F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c 7ea35eef53451786b4729d2a64ba9a8574a11e31 +F src/util.c 562f7a85d933b7173a29e331deb28d85d6208f7c F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52 F src/vdbe.c d3843a66d74a7696477ee5141e5eb9a7e5e2401c F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a9d007494cfe30a26dca5352655dc7702351476e -R bdb81af968b0eff03040af96472e5367 +P dd69e53cb077873171af5312c633ca185595bf31 +R 4f22b05e3bc28723a991a2f87c87962e U drh -Z 7162b1b4b927db763c29ab3a71a81028 +Z 439e39a464407086f87a7be758f8e1c8 diff --git a/manifest.uuid b/manifest.uuid index 48cc5d7fb4..1f512c3c71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dd69e53cb077873171af5312c633ca185595bf31 \ No newline at end of file +3adfe9f3e6ce7cc09fcb570d9b65e830a96cac15 \ No newline at end of file diff --git a/src/util.c b/src/util.c index 771213be13..c85ae6a199 100644 --- a/src/util.c +++ b/src/util.c @@ -431,9 +431,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ } /* skip trailing spaces */ - if( nDigits && eValid ){ - while( z0 ){ From 9bf0136384c90f4f9e7aacb705575617b459792c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 27 Apr 2016 11:24:42 +0000 Subject: [PATCH 48/63] Fix a couple of test script problems on this branch. Both related to the fact that temp database page sizes can no longer be changed by VACUUM or the backup API after the temp db is populated. FossilOrigin-Name: 6bb5aa2be2a6c5ca66a98a9387b2845507a6bdd2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/backup.test | 21 ++++++++++----------- test/pagerfault.test | 30 ++++++++++++++++-------------- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 52a995d5ff..45d2492933 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\sin\stemptable2.test\spreventing\sit\sfrom\sworking\swith\sSQLITE_DEFAULT_AUTOVACUUM=1\sbuilds. -D 2016-04-26T17:10:03.745 +C Fix\sa\scouple\sof\stest\sscript\sproblems\son\sthis\sbranch.\sBoth\srelated\sto\sthe\sfact\sthat\stemp\sdatabase\spage\ssizes\scan\sno\slonger\sbe\schanged\sby\sVACUUM\sor\sthe\sbackup\sAPI\safter\sthe\stemp\sdb\sis\spopulated. +D 2016-04-27T11:24:42.387 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -512,7 +512,7 @@ F test/autovacuum.test 92c24eedbdb68e49f3fb71f26f9ce6d8988cac15 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d -F test/backup.test b79299a536a4c6d919094786595b95be56d02014 +F test/backup.test dd4a5ff756e3df3931dacb1791db0584d4bad989 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32 F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 @@ -968,7 +968,7 @@ F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pager4.test a122e9e6925d5b23b31e3dfef8c6a44bbf19590e -F test/pagerfault.test ae9ee0db5a30aecda9db8290ce3dd12e5f7bbaa1 +F test/pagerfault.test 42ff797b1e6426c141cc7ee8b7417c9f27427950 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04911cee0cd5467ee99744b774d692219e5e1425 -R 849d538a2e2ee898d96d5ba70eff854f +P e790aac02e4b427b4891b514a050699d159b03b1 +R 167ee4a4f32b4f8204918ef34f7c21ee U dan -Z 496f6fcde8513321efb142fd9a4f11c8 +Z bbfd58a46d92c2867a1001ebb15e237d diff --git a/manifest.uuid b/manifest.uuid index 021372e495..eadf0ef5a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e790aac02e4b427b4891b514a050699d159b03b1 \ No newline at end of file +6bb5aa2be2a6c5ca66a98a9387b2845507a6bdd2 \ No newline at end of file diff --git a/test/backup.test b/test/backup.test index 3b1e1db9e5..829b32452d 100644 --- a/test/backup.test +++ b/test/backup.test @@ -164,7 +164,7 @@ foreach zOpenScript [list { set file_dest temp }] { foreach rows_dest {0 3 10} { -foreach pgsz_dest {512 1024 2048} { +foreach pgsz_dest {512 1024 2048 4096} { foreach nPagePerStep {1 200} { # Open the databases. @@ -176,17 +176,16 @@ foreach nPagePerStep {1 200} { # in-memory destination is only possible if the initial destination # page size is the same as the source page size (in this case 1024 bytes). # - set isMemDest [expr { - $zDestFile eq ":memory:" || $file_dest eq "temp" && $TEMP_STORE>=2 - }] + set isMemDest [expr { $zDestFile eq ":memory:" || $file_dest eq "temp" }] - if { $isMemDest==0 || $pgsz_dest == 1024 } { - if 0 { - puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile" - puts -nonewline " (as $db_dest.$file_dest)" - puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest" - puts "" - } + if 0 { + puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile" + puts -nonewline " (as $db_dest.$file_dest)" + puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest" + puts "" + } + + if { $isMemDest==0 || $pgsz_dest==1024 || $rows_dest==0 } { # Set up the content of the source database. execsql { diff --git a/test/pagerfault.test b/test/pagerfault.test index c0f5de69ac..2e70171b20 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -684,22 +684,24 @@ do_faultsim_test pagerfault-14a -prep { # is not possible to change the page-size of an in-memory database. Even # using the backup API. # -if {$TEMP_STORE<2} { - do_faultsim_test pagerfault-14b -prep { - catch { db2 close } - faultsim_restore_and_reopen +# Update: It is no longer possible to change the page size of any temp +# database after it has been created. +# +do_faultsim_test pagerfault-14b -prep { + catch { db2 close } + faultsim_restore_and_reopen sqlite3 db2 "" db2 eval { PRAGMA page_size = 4096; CREATE TABLE xx(a) } - } -body { - sqlite3_backup B db2 main db main - B step 200 - set rc [B finish] - if {[string match SQLITE_IOERR_* $rc]} {set rc SQLITE_IOERR} - if {$rc != "SQLITE_OK"} { error [sqlite3_test_errstr $rc] } - set {} {} - } -test { - faultsim_test_result {0 {}} {1 {sqlite3_backup_init() failed}} - } +} -body { + sqlite3_backup B db2 main db main + B step 200 + set rc [B finish] + if {[string match SQLITE_IOERR_* $rc]} {set rc SQLITE_IOERR} + if {$rc != "SQLITE_OK"} { error [sqlite3_test_errstr $rc] } + set {} {} +} -test { + faultsim_test_result {1 {attempt to write a readonly database}} \ + {1 {sqlite3_backup_init() failed}} } do_faultsim_test pagerfault-14c -prep { From ad975d539e1357f62b4e6501c2252490dbb9f43b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Apr 2016 15:24:13 +0000 Subject: [PATCH 49/63] More simplification of the sqlite3AtoF() routine. Add special comments to indicate branches that are for optimization purposes only and that give the correct answer even if always or never taken. FossilOrigin-Name: 0065fe97cb8e5076acae1bf1560fd2f69dab9014 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/util.c | 77 ++++++++++++++++++++++++++++++--------------------- 3 files changed, 52 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index 06c6814c38..7fa7ebedf8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sstill\smore\sunnecessary\sbranches\sfrom\ssqlite3AtoF(). -D 2016-04-27T02:35:03.572 +C More\ssimplification\sof\sthe\ssqlite3AtoF()\sroutine.\s\sAdd\sspecial\scomments\nto\sindicate\sbranches\sthat\sare\sfor\soptimization\spurposes\sonly\sand\sthat\sgive\nthe\scorrect\sanswer\seven\sif\salways\sor\snever\staken. +D 2016-04-27T15:24:13.573 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -441,7 +441,7 @@ F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 F src/update.c 3e67ab3c0814635f355fb1f8ab010a2b9e016e7d F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c 562f7a85d933b7173a29e331deb28d85d6208f7c +F src/util.c d0899604c30b4a9d493980aa7742eeda383fff6d F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52 F src/vdbe.c d3843a66d74a7696477ee5141e5eb9a7e5e2401c F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dd69e53cb077873171af5312c633ca185595bf31 -R 4f22b05e3bc28723a991a2f87c87962e +P 3adfe9f3e6ce7cc09fcb570d9b65e830a96cac15 +R 676347b15924e075b34cabb43dd7e840 U drh -Z 439e39a464407086f87a7be758f8e1c8 +Z 680dc6a7c6e0d675cd4dc0c16abcab9d diff --git a/manifest.uuid b/manifest.uuid index 1f512c3c71..a6a5e1cabb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3adfe9f3e6ce7cc09fcb570d9b65e830a96cac15 \ No newline at end of file +0065fe97cb8e5076acae1bf1560fd2f69dab9014 \ No newline at end of file diff --git a/src/util.c b/src/util.c index c85ae6a199..1f59a9f731 100644 --- a/src/util.c +++ b/src/util.c @@ -355,7 +355,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ int eValid = 1; /* True exponent is either not used or is well-formed */ double result; int nDigits = 0; - int nonNum = 0; + int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ @@ -368,7 +368,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); for(i=3-enc; i=zEnd ) goto do_atof_calc; + + /* This branch is needed to avoid a (harmless) buffer overread. The + ** special comment alerts the mutation tester that the correct answer + ** is obtained even if the branch is omitted */ + if( z>=zEnd ) goto do_atof_calc; /*PREVENTS-HARMLESS-OVERREAD*/ + /* get sign of exponent */ if( *z=='-' ){ esign = -1; @@ -443,41 +448,51 @@ do_atof_calc: esign = 1; } - /* if 0 significand */ - if( !s ) { - /* In the IEEE 754 standard, zero is signed. - ** Add the sign if we've seen at least one digit */ + if( s==0 ) { + /* In the IEEE 754 standard, zero is signed. */ result = sign<0 ? -(double)0 : (double)0; } else { - /* attempt to reduce exponent */ - if( esign>0 ){ - while( s<(LARGEST_INT64/10) && e>0 ) e--,s*=10; - }else{ - while( !(s%10) && e>0 ) e--,s/=10; + /* Attempt to reduce exponent. + ** + ** Branches that are not required for the correct answer but which only + ** help to obtain the correct answer faster are marked with special + ** comments, as a hint to the mutation tester. + */ + while( e>0 ){ /*OPTIMIZATION-IF-TRUE*/ + if( esign>0 ){ + if( s>=(LARGEST_INT64/10) ) break; /*OPTIMIZATION-IF-FALSE*/ + s *= 10; + }else{ + if( s%10!=0 ) break; /*OPTIMIZATION-IF-FALSE*/ + s /= 10; + } + e--; } /* adjust the sign of significand */ s = sign<0 ? -s : s; - /* if exponent, scale significand as appropriate - ** and store in result. */ - if( e ){ + if( e==0 ){ /*OPTIMIZATION-IF-TRUE*/ + result = (double)s; + }else{ LONGDOUBLE_TYPE scale = 1.0; /* attempt to handle extremely small/large numbers better */ - if( e>307 && e<342 ){ - while( e%308 ) { scale *= 1.0e+1; e -= 1; } - if( esign<0 ){ - result = s / scale; - result /= 1.0e+308; - }else{ - result = s * scale; - result *= 1.0e+308; - } - }else if( e>=342 ){ - if( esign<0 ){ - result = 0.0*s; - }else{ - result = 1e308*1e308*s; /* Infinity */ + if( e>307 ){ /*OPTIMIZATION-IF-TRUE*/ + if( e<342 ){ /*OPTIMIZATION-IF-TRUE*/ + while( e%308 ) { scale *= 1.0e+1; e -= 1; } + if( esign<0 ){ + result = s / scale; + result /= 1.0e+308; + }else{ + result = s * scale; + result *= 1.0e+308; + } + }else{ assert( e>=342 ); + if( esign<0 ){ + result = 0.0*s; + }else{ + result = 1e308*1e308*s; /* Infinity */ + } } }else{ /* 1.0e+22 is the largest power of 10 than can be @@ -490,8 +505,6 @@ do_atof_calc: result = s * scale; } } - } else { - result = (double)s; } } @@ -499,7 +512,7 @@ do_atof_calc: *pResult = result; /* return true if number and no extra non-whitespace chracters after */ - return z>=zEnd && nDigits>0 && eValid && nonNum==0; + return z==zEnd && nDigits>0 && eValid && nonNum==0; #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ From 05accd2216c25d31d98a19c6d8d14a260751452e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 27 Apr 2016 18:54:49 +0000 Subject: [PATCH 50/63] Modify the permutations.test script so as to set any permutation specific configuration values before running each individual test script. FossilOrigin-Name: 2662d8fef791f7b8b3b14f9c27dfedec84620dee --- ext/session/sessionfault.test | 1 + ext/session/sessionfault2.test | 1 + manifest | 61 ++++++++++++++++++---------------- manifest.uuid | 2 +- src/test1.c | 8 +++-- src/test6.c | 28 +++++++++++++++- src/test_devsym.c | 7 ++++ src/test_journal.c | 27 +++++++++------ src/test_syscall.c | 12 +++++-- test/dbstatus2.test | 2 +- test/e_vacuum.test | 2 +- test/exists.test | 1 + test/hook.test | 2 +- test/incrvacuum2.test | 2 +- test/journal2.test | 2 +- test/permutations.test | 10 +++--- test/pragma3.test | 2 +- test/stat.test | 2 +- test/sync.test | 2 ++ test/tester.tcl | 6 ++++ test/tkt-2d1a5c67d.test | 3 +- test/tkt-313723c356.test | 2 +- test/tkt-5d863f876e.test | 2 +- test/tkt-9d68c883.test | 2 ++ test/zerodamage.test | 2 +- 25 files changed, 127 insertions(+), 64 deletions(-) diff --git a/ext/session/sessionfault.test b/ext/session/sessionfault.test index 6560ea8755..be6c4568ce 100644 --- a/ext/session/sessionfault.test +++ b/ext/session/sessionfault.test @@ -17,6 +17,7 @@ if {![info exists testdir]} { } source [file join [file dirname [info script]] session_common.tcl] source $testdir/tester.tcl +ifcapable !session {finish_test; return} set testprefix sessionfault diff --git a/ext/session/sessionfault2.test b/ext/session/sessionfault2.test index 6b257cf41b..ffdc57b9bc 100644 --- a/ext/session/sessionfault2.test +++ b/ext/session/sessionfault2.test @@ -17,6 +17,7 @@ if {![info exists testdir]} { } source [file join [file dirname [info script]] session_common.tcl] source $testdir/tester.tcl +ifcapable !session {finish_test; return} set testprefix sessionfault2 do_execsql_test 1.0.0 { diff --git a/manifest b/manifest index 7fa7ebedf8..e587ac6205 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\ssimplification\sof\sthe\ssqlite3AtoF()\sroutine.\s\sAdd\sspecial\scomments\nto\sindicate\sbranches\sthat\sare\sfor\soptimization\spurposes\sonly\sand\sthat\sgive\nthe\scorrect\sanswer\seven\sif\salways\sor\snever\staken. -D 2016-04-27T15:24:13.573 +C Modify\sthe\spermutations.test\sscript\sso\sas\sto\sset\sany\spermutation\sspecific\sconfiguration\svalues\sbefore\srunning\seach\sindividual\stest\sscript. +D 2016-04-27T18:54:49.294 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -291,8 +291,8 @@ F ext/session/sessionE.test e60a238c47f0feb3bb707e7f35e22be09c7e8f26 F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596 -F ext/session/sessionfault.test d52cbb7bee48cc8ee80335e07eb72fcb6b15eb40 -F ext/session/sessionfault2.test ac1dfd77a0fb0ea310aee40a16645ef1b66d3f88 +F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 +F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 F ext/session/sqlite3session.c beb300cd1b5c5054062c8e6e807b10475e363410 F ext/session/sqlite3session.h 64e9e7f185725ef43b97f4a9a0c0df0669844f1d F ext/session/test_session.c 464f2c8bf502795d95969387eb8e93f68c513c15 @@ -387,12 +387,12 @@ F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 F src/tclsqlite.c 9c4c4589d078de37813ded708d8838b338ffb060 -F src/test1.c abc10e3e81258835aeb59616685d7369ba99ad1e +F src/test1.c 0b0b9483649d48846655ea30345bb3fe917555d6 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 -F src/test6.c 41cacf3b0dd180823919bf9e1fbab287c9266723 +F src/test6.c 2c014d4977efd6107ec9eef3dfdec56ac516f824 F src/test7.c 9c89a4f1ed6bb13af0ed805b8d782bd83fcd57e3 F src/test8.c fa262391d3edea6490a71bfaa8fed477ccbbac75 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 @@ -404,14 +404,14 @@ F src/test_blob.c b2551a9b5573232db5f66f292307c37067937239 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f F src/test_config.c 7003f6f35134de6f19c6588f44783e43390ea277 F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 -F src/test_devsym.c 7e73009d5297b603c11c66d7c7d6723d4b5c73e1 +F src/test_devsym.c 4e58dec2602d8e139ca08659f62a62450587cb58 F src/test_fs.c f10f840ca4f8c72e4837908bd8347ac4bcab074b F src/test_func.c 37453d346cfcf118774efd5bf6187f7e6a7e3254 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 F src/test_intarray.c 870124b95ec4c645d4eb84f15efb7133528fb1a5 F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 -F src/test_journal.c da540964c675fde39487da2bc664096af97190d8 +F src/test_journal.c d3b83f2bcb7792c709e57abddc456a2b1818643a F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd F src/test_malloc.c 94c18e88d6df6d471a0d04ddb809f833d1739950 F src/test_multiplex.c eafc567ebe162e36f17b5062285dfe90461cf8e9 @@ -427,7 +427,7 @@ F src/test_schema.c 2bdba21b82f601da69793e1f1d11bf481a79b091 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 0d138a8180a312bf996b37fa66da5c5799d4d57b F src/test_superlock.c 06797157176eb7085027d9dd278c0d7a105e3ec9 -F src/test_syscall.c 4889d374f5a5856b7951f7a67c6401f7b938c6f5 +F src/test_syscall.c 268c072541162564a882c57f54a6fee12ef4a4d2 F src/test_tclvar.c d86412527da65468ee6fa1b8607c65d0af736bc4 F src/test_thread.c af391ec03d23486dffbcc250b7e58e073f172af9 F src/test_vfs.c 4d02f38bfb8f7f273da7ba84bfe000f5babf206c @@ -612,7 +612,7 @@ F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 F test/date.test 984ac1e3e5e031386866f034006148d3972b4a65 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204 -F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 +F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d F test/delete.test e1bcdf8926234e27aac24b346ad83d3329ec8b6f F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -644,7 +644,7 @@ F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 F test/e_uri.test 25385396082b67fd02ae0038b95a3b3575fe0519 -F test/e_vacuum.test 4d5b391384bb7d56bb9337d956f08035332421fc +F test/e_vacuum.test 120f29ea56bdce4d43279527ece894ab5d1729d3 F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 @@ -659,7 +659,7 @@ F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9 F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 -F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 +F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac F test/expr.test 79c3e7502d9e571553b85f0ecc8ff2ac7d0e4931 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 @@ -808,7 +808,7 @@ F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/hexlit.test d7b0a5f41123df1e43985b91b8b2e70f95282d21 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test 40523db3aa76d62bda71c26f824fa0eabc420f0e +F test/hook.test 3b7b99d0eece6d279812c2aef6fa08bdfabc633e F test/icu.test 73956798bace8982909c00476b216714a6d0559a F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 @@ -825,7 +825,7 @@ F test/incrblob_err.test 69f9247fed50278d48ea710d1a8f9cdb09e4c0b8 F test/incrblobfault.test 280474078f6da9e732cd2a215d3d854969014b6e F test/incrcorrupt.test 6c567fbf870aa9e91866fe52ce6f200cd548939a F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 -F test/incrvacuum2.test 676c41428765d58f1da7dbe659ef27726d3d30ac +F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 F test/index.test fe3c7a1aad82af92623747e9c3f3aa94ccd51238 @@ -862,7 +862,7 @@ F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 8a5c0be6f0c260a5c7177c3b8f07c7856141038a F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 69abc726c51b4a0409189f9a85191205297c0577 -F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 +F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d @@ -977,10 +977,10 @@ F test/parser1.test 222b5cbf3e2e659fec1bf7d723488c8b9c94f1d0 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test cd1fa041074ed08eeaa563e4d1bacb0c69337ec1 +F test/permutations.test 44b5d257ff1e2bf102ffea50bd31f793de0c14f7 F test/pragma.test dd5313eee9c6d9d4726593a68ede8768d3b50ccc F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f -F test/pragma3.test 3f1984a04657331f838df5c519b443c2088df922 +F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc F test/printf2.test 0b61566dd1c0f0b802f59dffa228c5dc5aa6b054 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb @@ -1092,7 +1092,7 @@ F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a -F test/stat.test b65bad7120c52583b8f0054d99eff80718119a77 +F test/stat.test 66e95f97b9f724f9ab921d054ee0db3c2689f1ee F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1 F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f @@ -1102,7 +1102,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 -F test/sync.test 2f607e1821aa3af3c5c53b58835c05e511c95899 +F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 F test/tabfunc01.test f977868fa8bb7beb4b2072883190411653473906 @@ -1113,7 +1113,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 +F test/tester.tcl 30c7a9be8601d1c1c9c93d013545ebcb28d64254 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1129,10 +1129,10 @@ F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 -F test/tkt-2d1a5c67d.test d371279946622698ab393ff88cad9f5f6d82960b +F test/tkt-2d1a5c67d.test be1326f3061caec85085f4c9ee4490561ca037c0 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 6fb8807851964da0d24e942f2e19c7c705b9fb58 -F test/tkt-313723c356.test c47f8a9330523e6f35698bf4489bcb29609b53ac +F test/tkt-313723c356.test 4b306ad45c736cedf2f5221f6155b92143244b6d F test/tkt-385a5b56b9.test c0a06ada41d7f06b1686da0e718553f853771d1e F test/tkt-38cb5df375.test f3cc8671f1eb604d4ae9cf886ed4366bec656678 F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7 @@ -1143,7 +1143,7 @@ F test/tkt-4c86b126f2.test cbcc611becd0396890169ab23102dd70048bbc9a F test/tkt-4dd95f6943.test 3d0ce415d2ee15d3d564121960016b9c7be79407 F test/tkt-4ef7e3cfca.test 3965ae11cc9cf6e334f9d7d3c1e20bf8d56254b1 F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 -F test/tkt-5d863f876e.test c9f36ca503fa154a3655f92a69d2c30da1747bfa +F test/tkt-5d863f876e.test 726e76d725f6fe0eb2fc8a522b721b79807380ee F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f F test/tkt-6bfb98dfc0.test 24780633627b5cfc0635a5500c2389ebfb563336 @@ -1159,7 +1159,7 @@ F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 -F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 +F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 F test/tkt-a8a0d2996a.test eb597379dbcefa24765763d7f682c00cb5924fa9 @@ -1398,7 +1398,7 @@ F test/without_rowid5.test 89b1c587bd92a0590e440da33e7666bf4891572a F test/without_rowid6.test 1f99644e6508447fb050f73697350c7ceca3392e F test/wordcount.c 2a0a6c0d0e8e8bbbac1f06d72a6791828c37c0cf F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa -F test/zerodamage.test 2d725c214b883e25ae6bb85ef228ecdfa03c6a7b +F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 629d87562e0487c386db630033931d12d62e6372 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1484,7 +1484,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3adfe9f3e6ce7cc09fcb570d9b65e830a96cac15 -R 676347b15924e075b34cabb43dd7e840 -U drh -Z 680dc6a7c6e0d675cd4dc0c16abcab9d +P 0065fe97cb8e5076acae1bf1560fd2f69dab9014 +R 7df43ef0b7e3be4f42f2974bf9ebb1f9 +T *branch * permutations-fix +T *sym-permutations-fix * +T -sym-trunk * +U dan +Z 8c7c480375d878fe85148ba2747a0470 diff --git a/manifest.uuid b/manifest.uuid index a6a5e1cabb..a2dbc6c3ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0065fe97cb8e5076acae1bf1560fd2f69dab9014 \ No newline at end of file +2662d8fef791f7b8b3b14f9c27dfedec84620dee \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 5478a72549..2540f3e804 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5213,7 +5213,9 @@ static int vfs_unregister_all( /* ** tclcmd: vfs_reregister_all ** -** Restore all VFSes that were removed using vfs_unregister_all +** Restore all VFSes that were removed using vfs_unregister_all. Taking +** care to put the linked list back together in the same order as it was +** in before vfs_unregister_all was invoked. */ static int vfs_reregister_all( ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ @@ -5222,8 +5224,8 @@ static int vfs_reregister_all( Tcl_Obj *CONST objv[] /* Command arguments */ ){ int i; - for(i=0; i=0; i--){ + sqlite3_vfs_register(apVfs[i], 1); } return TCL_OK; } diff --git a/src/test6.c b/src/test6.c index 306482dcd3..2a09122c6c 100644 --- a/src/test6.c +++ b/src/test6.c @@ -701,6 +701,10 @@ static int cfCurrentTime(sqlite3_vfs *pCfVfs, double *pTimeOut){ sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData; return pVfs->xCurrentTime(pVfs, pTimeOut); } +static int cfGetLastError(sqlite3_vfs *pCfVfs, int n, char *z){ + sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData; + return pVfs->xGetLastError(pVfs, n, z); +} static int processDevSymArgs( Tcl_Interp *interp, @@ -827,7 +831,7 @@ static int crashEnableCmd( cfRandomness, /* xRandomness */ cfSleep, /* xSleep */ cfCurrentTime, /* xCurrentTime */ - 0, /* xGetlastError */ + cfGetLastError, /* xGetLastError */ 0, /* xCurrentTimeInt64 */ }; @@ -940,6 +944,27 @@ static int devSymObjCmd( devsym_register(iDc, iSectorSize); return TCL_OK; + +} + +/* +** tclcmd: unregister_devsim +*/ +static int dsUnregisterObjCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + void devsym_unregister(void); + + if( objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + + devsym_unregister(); + return TCL_OK; } /* @@ -1010,6 +1035,7 @@ int Sqlitetest6_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "sqlite3_crash_enable", crashEnableCmd, 0, 0); Tcl_CreateObjCommand(interp, "sqlite3_crashparams", crashParamsObjCmd, 0, 0); Tcl_CreateObjCommand(interp, "sqlite3_simulate_device", devSymObjCmd, 0, 0); + Tcl_CreateObjCommand(interp, "unregister_devsim", dsUnregisterObjCmd, 0, 0); Tcl_CreateObjCommand(interp, "register_jt_vfs", jtObjCmd, 0, 0); Tcl_CreateObjCommand(interp, "unregister_jt_vfs", jtUnregisterObjCmd, 0, 0); #endif diff --git a/src/test_devsym.c b/src/test_devsym.c index 5fd0935846..9a1ba09d69 100644 --- a/src/test_devsym.c +++ b/src/test_devsym.c @@ -396,4 +396,11 @@ void devsym_register(int iDeviceChar, int iSectorSize){ } } +void devsym_unregister(){ + sqlite3_vfs_unregister(&devsym_vfs); + g.pVfs = 0; + g.iDeviceChar = 0; + g.iSectorSize = 0; +} + #endif diff --git a/src/test_journal.c b/src/test_journal.c index 84c80546a6..4e63bccf76 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -160,6 +160,7 @@ static int jtRandomness(sqlite3_vfs*, int nByte, char *zOut); static int jtSleep(sqlite3_vfs*, int microseconds); static int jtCurrentTime(sqlite3_vfs*, double*); static int jtCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int jtGetLastError(sqlite3_vfs*, int, char*); static sqlite3_vfs jt_vfs = { 2, /* iVersion */ @@ -179,7 +180,7 @@ static sqlite3_vfs jt_vfs = { jtRandomness, /* xRandomness */ jtSleep, /* xSleep */ jtCurrentTime, /* xCurrentTime */ - 0, /* xGetLastError */ + jtGetLastError, /* xGetLastError */ jtCurrentTimeInt64 /* xCurrentTimeInt64 */ }; @@ -285,9 +286,10 @@ static int jtRead( ** b) The file-name specified when the file was opened matches ** all but the final 8 characters of the journal file name. ** -** c) There is currently a reserved lock on the file. +** c) There is currently a reserved lock on the file. This +** condition is waived if the noLock argument is non-zero. **/ -static jt_file *locateDatabaseHandle(const char *zJournal){ +static jt_file *locateDatabaseHandle(const char *zJournal, int noLock){ jt_file *pMain = 0; enterJtMutex(); for(pMain=g.pList; pMain; pMain=pMain->pNext){ @@ -295,7 +297,7 @@ static jt_file *locateDatabaseHandle(const char *zJournal){ if( (pMain->flags&SQLITE_OPEN_MAIN_DB) && ((int)strlen(pMain->zName)==nName) && 0==memcmp(pMain->zName, zJournal, nName) - && (pMain->eLock>=SQLITE_LOCK_RESERVED) + && ((pMain->eLock>=SQLITE_LOCK_RESERVED) || noLock) ){ break; } @@ -517,7 +519,7 @@ static int jtWrite( jt_file *p = (jt_file *)pFile; if( p->flags&SQLITE_OPEN_MAIN_JOURNAL ){ if( iOfst==0 ){ - jt_file *pMain = locateDatabaseHandle(p->zName); + jt_file *pMain = locateDatabaseHandle(p->zName, 0); assert( pMain ); if( iAmt==28 ){ @@ -562,7 +564,7 @@ static int jtWrite( rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); if( (p->flags&SQLITE_OPEN_MAIN_JOURNAL) && iAmt==12 ){ - jt_file *pMain = locateDatabaseHandle(p->zName); + jt_file *pMain = locateDatabaseHandle(p->zName, 0); int rc2 = readJournalFile(p, pMain); if( rc==SQLITE_OK ) rc = rc2; } @@ -576,7 +578,7 @@ static int jtTruncate(sqlite3_file *pFile, sqlite_int64 size){ jt_file *p = (jt_file *)pFile; if( p->flags&SQLITE_OPEN_MAIN_JOURNAL && size==0 ){ /* Truncating a journal file. This is the end of a transaction. */ - jt_file *pMain = locateDatabaseHandle(p->zName); + jt_file *pMain = locateDatabaseHandle(p->zName, 0); closeTransaction(pMain); } if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){ @@ -604,11 +606,10 @@ static int jtSync(sqlite3_file *pFile, int flags){ ** jt_file.pWritable bitvec of the main database file associated with ** this journal file. */ - pMain = locateDatabaseHandle(p->zName); - assert(pMain); + pMain = locateDatabaseHandle(p->zName, 0); /* Set the bitvec values */ - if( pMain->pWritable ){ + if( pMain && pMain->pWritable ){ pMain->nSync++; rc = readJournalFile(p, pMain); if( rc!=SQLITE_OK ){ @@ -730,7 +731,7 @@ static int jtDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ int nPath = (int)strlen(zPath); if( nPath>8 && 0==strcmp("-journal", &zPath[nPath-8]) ){ /* Deleting a journal file. The end of a transaction. */ - jt_file *pMain = locateDatabaseHandle(zPath); + jt_file *pMain = locateDatabaseHandle(zPath, 0); if( pMain ){ closeTransaction(pMain); } @@ -825,6 +826,10 @@ static int jtCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ return g.pVfs->xCurrentTimeInt64(g.pVfs, pTimeOut); } +static int jtGetLastError(sqlite3_vfs *pVfs, int n, char *z){ + return g.pVfs->xGetLastError(g.pVfs, n, z); +} + /************************************************************************** ** Start of public API. */ diff --git a/src/test_syscall.c b/src/test_syscall.c index f9abc1e46d..f1d5c61bc3 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -722,14 +722,20 @@ static int test_syscall( }; int iCmd; int rc; + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); if( objc<2 ){ Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ..."); return TCL_ERROR; } - rc = Tcl_GetIndexFromObjStruct(interp, - objv[1], aCmd, sizeof(aCmd[0]), "sub-command", 0, &iCmd - ); + if( pVfs->iVersion<3 || pVfs->xSetSystemCall==0 ){ + Tcl_AppendResult(interp, "VFS does not support xSetSystemCall", 0); + rc = TCL_ERROR; + }else{ + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aCmd, sizeof(aCmd[0]), "sub-command", 0, &iCmd + ); + } if( rc!=TCL_OK ) return rc; return aCmd[iCmd].xCmd(clientData, interp, objc, objv); } diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 2541a1a823..eff4b0207f 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -86,7 +86,7 @@ do_test 2.3 { db_write db 1 } {0 4 0} do_test 2.4 { db_write db 0 } {0 0 0} do_test 2.5 { db_write db 1 } {0 0 0} -ifcapable wal { +if {[wal_is_capable]} { do_test 2.6 { execsql { PRAGMA journal_mode = WAL } db_write db 1 diff --git a/test/e_vacuum.test b/test/e_vacuum.test index 1113a1fb6e..c3e517a892 100644 --- a/test/e_vacuum.test +++ b/test/e_vacuum.test @@ -176,7 +176,7 @@ if {![nonzero_reserved_bytes]} { # EVIDENCE-OF: R-48521-51450 When in write-ahead log mode, only the # auto_vacuum support property can be changed using VACUUM. # - ifcapable wal { + if {[wal_is_capable]} { do_test e_vacuum-1.3.3.1 { execsql { PRAGMA journal_mode = wal } execsql { PRAGMA page_size ; PRAGMA auto_vacuum } diff --git a/test/exists.test b/test/exists.test index fb73797d29..5f9e36eb00 100644 --- a/test/exists.test +++ b/test/exists.test @@ -19,6 +19,7 @@ source $testdir/lock_common.tcl foreach jm {rollback wal} { + if {![wal_is_capable] && $jm=="wal"} continue set testprefix exists-$jm diff --git a/test/hook.test b/test/hook.test index 45e2959591..12e5bc0656 100644 --- a/test/hook.test +++ b/test/hook.test @@ -704,7 +704,7 @@ do_execsql_test 7.5.2.0 { ALTER TABLE t8 ADD COLUMN c DEFAULT 'xxx'; } -ifcapable !session { +if 0 { # At time of writing, these two are broken. They demonstrate that the # sqlite3_preupdate_old() method does not handle the case where ALTER TABLE # has been used to add a column with a default value other than NULL. diff --git a/test/incrvacuum2.test b/test/incrvacuum2.test index 2219d54195..6ec66c9deb 100644 --- a/test/incrvacuum2.test +++ b/test/incrvacuum2.test @@ -134,7 +134,7 @@ do_test incrvacuum2-3.2 { integrity_check incrvacuum2-3.3 -ifcapable wal { +if {[wal_is_capable]} { # At one point, when a specific page was being extracted from the b-tree # free-list (e.g. during an incremental-vacuum), all trunk pages that # occurred before the specific page in the free-list trunk were being diff --git a/test/journal2.test b/test/journal2.test index 8f9b4d0b71..7cbe0ef2c5 100644 --- a/test/journal2.test +++ b/test/journal2.test @@ -204,7 +204,7 @@ db close # delete the journal file when committing the transaction that switches # the system to WAL mode. # -ifcapable wal { +if {[wal_is_capable]} { do_test journal2-2.1 { faultsim_delete_and_reopen set ::oplog [list] diff --git a/test/permutations.test b/test/permutations.test index 308d8bde77..15576e4f9f 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -942,7 +942,9 @@ test_suite "journaltest" -description { unregister_jt_vfs } -files [test_set $::allquicktests -exclude { wal* incrvacuum.test ioerr.test corrupt4.test io.test crash8.test - async4.test bigfile.test backcompat.test + async4.test bigfile.test backcompat.test e_wal* fstat.test mmap2.test + pager1.test syscall.test tkt3457.test *malloc* mmap* multiplex* nolock* + pager2.test *fault* rowal* snapshot* superlock* symlink.test }] if {[info commands register_demovfs] != ""} { @@ -1050,15 +1052,13 @@ proc run_tests {name args} { set ::G(isquick) 1 set ::G(perm:dbconfig) $options(-dbconfig) - uplevel $options(-initialize) - foreach file [lsort $options(-files)] { + uplevel $options(-initialize) if {[file tail $file] == $file} { set file [file join $::testdir $file] } slave_test_file $file + uplevel $options(-shutdown) } - uplevel $options(-shutdown) - unset ::G(perm:name) unset ::G(perm:prefix) unset ::G(perm:presql) diff --git a/test/pragma3.test b/test/pragma3.test index 1a67d72350..8a4f4f9d84 100644 --- a/test/pragma3.test +++ b/test/pragma3.test @@ -221,7 +221,7 @@ ifcapable shared_cache { # This will not work with the in-memory journal permutation, as opening # [db2] switches the journal mode back to "memory" # -ifcapable wal { +if {[wal_is_capable]} { if {[permutation]!="inmemory_journal"} { sqlite3 db test.db diff --git a/test/stat.test b/test/stat.test index 95586e94bc..d106894d71 100644 --- a/test/stat.test +++ b/test/stat.test @@ -37,7 +37,7 @@ do_execsql_test stat-0.0 { SELECT * FROM stat; } {} -ifcapable wal { +if {[wal_is_capable]} { do_execsql_test stat-0.1 { PRAGMA journal_mode = WAL; PRAGMA journal_mode = delete; diff --git a/test/sync.test b/test/sync.test index 2ee636a266..210039acb5 100644 --- a/test/sync.test +++ b/test/sync.test @@ -81,6 +81,7 @@ do_test sync-1.3 { set sqlite_sync_count } 11 ifcapable pager_pragmas { +if {[permutation]!="journaltest"} { do_test sync-1.4 { set sqlite_sync_count 0 execsql { @@ -94,6 +95,7 @@ ifcapable pager_pragmas { set sqlite_sync_count } 0 } +} finish_test diff --git a/test/tester.tcl b/test/tester.tcl index 2fcae7b5a2..23810d09ee 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1944,6 +1944,12 @@ proc wal_check_journal_mode {testname {db db}} { } } +proc wal_is_capable {} { + ifcapable !wal { return 0 } + if {[permutation]=="journaltest"} { return 0 } + return 1 +} + proc permutation {} { set perm "" catch {set perm $::G(perm:name)} diff --git a/test/tkt-2d1a5c67d.test b/test/tkt-2d1a5c67d.test index 3fef187ecd..0d12a6ec43 100644 --- a/test/tkt-2d1a5c67d.test +++ b/test/tkt-2d1a5c67d.test @@ -19,7 +19,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix tkt-2d1a5c67d -ifcapable {!wal || !vtab} {finish_test; return} +ifcapable {!vtab} {finish_test; return} +if {[wal_is_capable]==0} {finish_test; return} for {set ii 1} {$ii<=10} {incr ii} { do_test tkt-2d1a5c67d.1.$ii { diff --git a/test/tkt-313723c356.test b/test/tkt-313723c356.test index 8c08c34976..5325fd2292 100644 --- a/test/tkt-313723c356.test +++ b/test/tkt-313723c356.test @@ -18,7 +18,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl -ifcapable !wal { finish_test ; return } +if {![wal_is_capable]} { finish_test ; return } do_execsql_test tkt-313723c356.1 { PRAGMA page_size = 1024; diff --git a/test/tkt-5d863f876e.test b/test/tkt-5d863f876e.test index 86024e300b..9a2fa3f470 100644 --- a/test/tkt-5d863f876e.test +++ b/test/tkt-5d863f876e.test @@ -18,7 +18,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl set ::testprefix tkt-5d863f876e -ifcapable !wal {finish_test ; return } +if {![wal_is_capable]} {finish_test ; return } do_multiclient_test tn { do_test $tn.1 { diff --git a/test/tkt-9d68c883.test b/test/tkt-9d68c883.test index 18dc6ccd67..ba91b39e39 100644 --- a/test/tkt-9d68c883.test +++ b/test/tkt-9d68c883.test @@ -50,4 +50,6 @@ for {set i 0} {$i < 100} {incr i} { } {ok} } +catch { db close } +unregister_devsim finish_test diff --git a/test/zerodamage.test b/test/zerodamage.test index d781ab89a3..a87e50b7b5 100644 --- a/test/zerodamage.test +++ b/test/zerodamage.test @@ -89,7 +89,7 @@ do_test zerodamage-2.1 { concat [file_control_powersafe_overwrite db -1] [set ::max_journal_size] } {0 0 24704} -ifcapable wal { +if {[wal_is_capable]} { # Run a WAL-mode transaction with POWERSAFE_OVERWRITE on to verify that the # WAL file does not get too big. # From 609d5846baf14c8886a690ead6fb3c0172f6ad4f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 28 Apr 2016 00:32:16 +0000 Subject: [PATCH 51/63] Change the sqlite3Atoi64() routine so that it returns failure if not all of the input characters are consumed, even if it consumed all characters up to the first 0x00. This has no impact on external APIs as far as I can tell. FossilOrigin-Name: 46d4ffff3bd33d7e901e76cfac1cbde38d4f61d0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test1.c | 2 +- src/util.c | 11 +++++++---- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 7fa7ebedf8..a6ca358f63 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\ssimplification\sof\sthe\ssqlite3AtoF()\sroutine.\s\sAdd\sspecial\scomments\nto\sindicate\sbranches\sthat\sare\sfor\soptimization\spurposes\sonly\sand\sthat\sgive\nthe\scorrect\sanswer\seven\sif\salways\sor\snever\staken. -D 2016-04-27T15:24:13.573 +C Change\sthe\ssqlite3Atoi64()\sroutine\sso\sthat\sit\sreturns\sfailure\sif\snot\sall\sof\nthe\sinput\scharacters\sare\sconsumed,\seven\sif\sit\sconsumed\sall\scharacters\sup\sto\nthe\sfirst\s0x00.\s\sThis\shas\sno\simpact\son\sexternal\sAPIs\sas\sfar\sas\sI\scan\stell. +D 2016-04-28T00:32:16.814 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -387,7 +387,7 @@ F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 F src/tclsqlite.c 9c4c4589d078de37813ded708d8838b338ffb060 -F src/test1.c abc10e3e81258835aeb59616685d7369ba99ad1e +F src/test1.c 922fd57b211e14761ef846cebfd09a1922771960 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e @@ -441,7 +441,7 @@ F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 F src/update.c 3e67ab3c0814635f355fb1f8ab010a2b9e016e7d F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c d0899604c30b4a9d493980aa7742eeda383fff6d +F src/util.c 1d2f9cc9ea43c5bc9a3b7e29b953001b0e85f5c7 F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52 F src/vdbe.c d3843a66d74a7696477ee5141e5eb9a7e5e2401c F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3adfe9f3e6ce7cc09fcb570d9b65e830a96cac15 -R 676347b15924e075b34cabb43dd7e840 +P 0065fe97cb8e5076acae1bf1560fd2f69dab9014 +R 3ff6e84daa3c00c689a29794b4bf7803 U drh -Z 680dc6a7c6e0d675cd4dc0c16abcab9d +Z 1cade5a69af13a9b74de489bf6c2518b diff --git a/manifest.uuid b/manifest.uuid index a6a5e1cabb..fa2b210a37 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0065fe97cb8e5076acae1bf1560fd2f69dab9014 \ No newline at end of file +46d4ffff3bd33d7e901e76cfac1cbde38d4f61d0 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 5478a72549..98afb03e12 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1271,7 +1271,7 @@ static int sqlite3_mprintf_int64( return TCL_ERROR; } for(i=2; i<5; i++){ - if( sqlite3Atoi64(argv[i], &a[i-2], 1000000, SQLITE_UTF8) ){ + if( sqlite3Atoi64(argv[i], &a[i-2], sqlite3Strlen30(argv[i]), SQLITE_UTF8) ){ Tcl_AppendResult(interp, "argument is not a valid 64-bit integer", 0); return TCL_ERROR; } diff --git a/src/util.c b/src/util.c index 1f59a9f731..e883b1b9eb 100644 --- a/src/util.c +++ b/src/util.c @@ -574,7 +574,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ int neg = 0; /* assume positive */ int i; int c = 0; - int nonNum = 0; + int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ const char *zStart; const char *zEnd = zNum + length; assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); @@ -585,7 +585,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); for(i=3-enc; i19*incr || nonNum ){ + if( &zNum[i]19*incr /* Too many digits */ + || nonNum /* UTF16 with high-order bytes non-zero */ + ){ /* zNum is empty or contains non-numeric text or is longer ** than 19 digits (thus guaranteeing that it is too large) */ return 1; From 10c0a2a7c9e40ac583d5fb02701c19510110c78f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 28 Apr 2016 03:52:18 +0000 Subject: [PATCH 52/63] Remove an unnecessary conditional from the sqlite3DecOrHexToI64() routine. FossilOrigin-Name: fcf85bfe50b254e825ee63a4cd0aa0b333b06eed --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/util.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a6ca358f63..0e64cb6d65 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\ssqlite3Atoi64()\sroutine\sso\sthat\sit\sreturns\sfailure\sif\snot\sall\sof\nthe\sinput\scharacters\sare\sconsumed,\seven\sif\sit\sconsumed\sall\scharacters\sup\sto\nthe\sfirst\s0x00.\s\sThis\shas\sno\simpact\son\sexternal\sAPIs\sas\sfar\sas\sI\scan\stell. -D 2016-04-28T00:32:16.814 +C Remove\san\sunnecessary\sconditional\sfrom\sthe\ssqlite3DecOrHexToI64()\sroutine. +D 2016-04-28T03:52:18.237 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -441,7 +441,7 @@ F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 F src/update.c 3e67ab3c0814635f355fb1f8ab010a2b9e016e7d F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c 1d2f9cc9ea43c5bc9a3b7e29b953001b0e85f5c7 +F src/util.c 41cac9db948cd1f746636cc6a0c60d4234d4053f F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52 F src/vdbe.c d3843a66d74a7696477ee5141e5eb9a7e5e2401c F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0065fe97cb8e5076acae1bf1560fd2f69dab9014 -R 3ff6e84daa3c00c689a29794b4bf7803 +P 46d4ffff3bd33d7e901e76cfac1cbde38d4f61d0 +R 229317c8b5ac201d05335df0607fe343 U drh -Z 1cade5a69af13a9b74de489bf6c2518b +Z 98856d3d087818ace0a186a1d76b304b diff --git a/manifest.uuid b/manifest.uuid index fa2b210a37..61f1101a9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46d4ffff3bd33d7e901e76cfac1cbde38d4f61d0 \ No newline at end of file +fcf85bfe50b254e825ee63a4cd0aa0b333b06eed \ No newline at end of file diff --git a/src/util.c b/src/util.c index e883b1b9eb..748f8e802a 100644 --- a/src/util.c +++ b/src/util.c @@ -658,7 +658,6 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') - && sqlite3Isxdigit(z[2]) ){ u64 u = 0; int i, k; From 3104c4d6f432335fade4e254580f7019c1d71038 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 28 Apr 2016 09:53:14 +0000 Subject: [PATCH 53/63] Ensure that quota.test deletes the directory named "test.db" that it creates. FossilOrigin-Name: 045edd4b17c6b76079ac90773c035d8c1f8d525d --- manifest | 15 ++++++--------- manifest.uuid | 2 +- test/quota.test | 2 ++ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index e587ac6205..4851ef79ae 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\spermutations.test\sscript\sso\sas\sto\sset\sany\spermutation\sspecific\sconfiguration\svalues\sbefore\srunning\seach\sindividual\stest\sscript. -D 2016-04-27T18:54:49.294 +C Ensure\sthat\squota.test\sdeletes\sthe\sdirectory\snamed\s"test.db"\sthat\sit\screates. +D 2016-04-28T09:53:14.110 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -988,7 +988,7 @@ F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 -F test/quota.test 36cd78b178c4eb0401d4f25754ef410fbd9df2a7 +F test/quota.test bfb269ce81ea52f593f9648316cd5013d766dd2a F test/quota2.test 7dc12e08b11cbc4c16c9ba2aa2e040ea8d8ab4b8 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 @@ -1484,10 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0065fe97cb8e5076acae1bf1560fd2f69dab9014 -R 7df43ef0b7e3be4f42f2974bf9ebb1f9 -T *branch * permutations-fix -T *sym-permutations-fix * -T -sym-trunk * +P 2662d8fef791f7b8b3b14f9c27dfedec84620dee +R c0f9313eefa2a94509287d7302d3c22e U dan -Z 8c7c480375d878fe85148ba2747a0470 +Z e1c90065dd206cf7c15216b8288594e5 diff --git a/manifest.uuid b/manifest.uuid index a2dbc6c3ba..be4490f793 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2662d8fef791f7b8b3b14f9c27dfedec84620dee \ No newline at end of file +045edd4b17c6b76079ac90773c035d8c1f8d525d \ No newline at end of file diff --git a/test/quota.test b/test/quota.test index 7af55a8330..5d0bda3ad3 100644 --- a/test/quota.test +++ b/test/quota.test @@ -527,4 +527,6 @@ do_faultsim_test quota-5.6 -prep { } catch { sqlite3_quota_shutdown } +catch { db close } +forcedelete test.db finish_test From 75ab50ce8ff0821bae0989a86f037024fd6a8711 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 28 Apr 2016 14:15:12 +0000 Subject: [PATCH 54/63] Use comments to mark several branches as optimizations. No changes to code. FossilOrigin-Name: 33e627472780b872716c504f2d585cc057c390a5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/hash.c | 4 ++-- src/util.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 0e64cb6d65..4e86475b14 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sconditional\sfrom\sthe\ssqlite3DecOrHexToI64()\sroutine. -D 2016-04-28T03:52:18.237 +C Use\scomments\sto\smark\sseveral\sbranches\sas\soptimizations.\s\sNo\schanges\sto\scode. +D 2016-04-28T14:15:12.903 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -336,7 +336,7 @@ F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 4c0bd09e602b8ae8d36d81e31e4872d0b53c87bb F src/func.c ef4c18c8a66143413ce41a58d582d2c14ddf78e1 F src/global.c c45ea22aff29334f6a9ec549235ac3357c970015 -F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 +F src/hash.c 55b5fb474100cee0b901edaf203e26c970940f36 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 8f4e9fcbd8e95e85f15647ba8b413b18d556ec2b @@ -441,7 +441,7 @@ F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 F src/update.c 3e67ab3c0814635f355fb1f8ab010a2b9e016e7d F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c 41cac9db948cd1f746636cc6a0c60d4234d4053f +F src/util.c 810ec3f22e2d1b62e66c30fe3621ebdedd23584d F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52 F src/vdbe.c d3843a66d74a7696477ee5141e5eb9a7e5e2401c F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 46d4ffff3bd33d7e901e76cfac1cbde38d4f61d0 -R 229317c8b5ac201d05335df0607fe343 +P fcf85bfe50b254e825ee63a4cd0aa0b333b06eed +R 197c644a6cd9b0806f84f77482d9fe15 U drh -Z 98856d3d087818ace0a186a1d76b304b +Z 413f6650964e5d2ab342130728703294 diff --git a/manifest.uuid b/manifest.uuid index 61f1101a9b..abe023c752 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fcf85bfe50b254e825ee63a4cd0aa0b333b06eed \ No newline at end of file +33e627472780b872716c504f2d585cc057c390a5 \ No newline at end of file diff --git a/src/hash.c b/src/hash.c index b5886e0641..eea2dd1ac2 100644 --- a/src/hash.c +++ b/src/hash.c @@ -55,7 +55,7 @@ void sqlite3HashClear(Hash *pH){ static unsigned int strHash(const char *z){ unsigned int h = 0; unsigned char c; - while( (c = (unsigned char)*z++)!=0 ){ + while( (c = (unsigned char)*z++)!=0 ){ /*OPTIMIZATION-IF-TRUE*/ h = (h<<3) ^ h ^ sqlite3UpperToLower[c]; } return h; @@ -148,7 +148,7 @@ static HashElem *findElementWithHash( int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ - if( pH->ht ){ + if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ struct _ht *pEntry; h = strHash(pKey) % pH->htsize; pEntry = &pH->ht[h]; diff --git a/src/util.c b/src/util.c index 748f8e802a..db6163c3c7 100644 --- a/src/util.c +++ b/src/util.c @@ -1419,7 +1419,7 @@ LogEst sqlite3LogEst(u64 x){ if( x<2 ) return 0; while( x<8 ){ y -= 10; x <<= 1; } }else{ - while( x>255 ){ y += 40; x >>= 4; } + while( x>255 ){ y += 40; x >>= 4; } /*OPTIMIZATION-IF-TRUE*/ while( x>15 ){ y += 10; x >>= 1; } } return a[x&7] + y - 10; From 14cee38be8249d7d15e07b4a27a3d0e2b98dcac0 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 28 Apr 2016 14:54:54 +0000 Subject: [PATCH 55/63] Avoid running sessionfault.test and sessionfault2.test as part of the "inmemory_journal" permutation. FossilOrigin-Name: 5973d3bf2e5aa979ccc9a4a3a0eb08b67098b499 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/permutations.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4851ef79ae..24ba903e1f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\squota.test\sdeletes\sthe\sdirectory\snamed\s"test.db"\sthat\sit\screates. -D 2016-04-28T09:53:14.110 +C Avoid\srunning\ssessionfault.test\sand\ssessionfault2.test\sas\spart\sof\sthe\s"inmemory_journal"\spermutation. +D 2016-04-28T14:54:54.590 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -977,7 +977,7 @@ F test/parser1.test 222b5cbf3e2e659fec1bf7d723488c8b9c94f1d0 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 44b5d257ff1e2bf102ffea50bd31f793de0c14f7 +F test/permutations.test b6b3e165fdc1b8c82a820033646dbfc6a7a01746 F test/pragma.test dd5313eee9c6d9d4726593a68ede8768d3b50ccc F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2662d8fef791f7b8b3b14f9c27dfedec84620dee -R c0f9313eefa2a94509287d7302d3c22e +P 045edd4b17c6b76079ac90773c035d8c1f8d525d +R e73b09d1e2cd9d2d69ece961d0e8362f U dan -Z e1c90065dd206cf7c15216b8288594e5 +Z aff771281cb418c17327081bdcd3c618 diff --git a/manifest.uuid b/manifest.uuid index be4490f793..7a52ce8e55 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -045edd4b17c6b76079ac90773c035d8c1f8d525d \ No newline at end of file +5973d3bf2e5aa979ccc9a4a3a0eb08b67098b499 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 15576e4f9f..df3f500cda 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -728,7 +728,7 @@ test_suite "inmemory_journal" -description { ioerr.test ioerr2.test ioerr3.test ioerr4.test ioerr5.test vacuum3.test incrblob_err.test diskfull.test backup_ioerr.test e_fts3.test fts3cov.test fts3malloc.test fts3rnd.test - fts3snippet.test mmapfault.test + fts3snippet.test mmapfault.test sessionfault.test sessionfault2.test # Exclude test scripts that use tcl IO to access journal files or count # the number of fsync() calls. From cb6d66becc06dd0ad293779e02b8b9b25dfe3301 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 28 Apr 2016 18:53:08 +0000 Subject: [PATCH 56/63] Rearrange some code in the RowSet logic for clarity of presentation, while adding an /*OPTIMIZATION-IF-TRUE*/ comment. It should operate identically. FossilOrigin-Name: 5748e64376c1c2be5154a632d1527cfebbb9ec74 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/rowset.c | 23 +++++++++++++---------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 4e86475b14..0837962a87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\scomments\sto\smark\sseveral\sbranches\sas\soptimizations.\s\sNo\schanges\sto\scode. -D 2016-04-28T14:15:12.903 +C Rearrange\ssome\scode\sin\sthe\sRowSet\slogic\sfor\sclarity\sof\spresentation,\swhile\nadding\san\s/*OPTIMIZATION-IF-TRUE*/\scomment.\s\sIt\sshould\soperate\sidentically. +D 2016-04-28T18:53:08.454 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -376,7 +376,7 @@ F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20 -F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e +F src/rowset.c d68d95e167cb234f51d655c9546f46752c9f6d45 F src/select.c fd4a7ce2937497181063cfedb92058ac89491a5d F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6 F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fcf85bfe50b254e825ee63a4cd0aa0b333b06eed -R 197c644a6cd9b0806f84f77482d9fe15 +P 33e627472780b872716c504f2d585cc057c390a5 +R a33e3686da943dcb038124577baa15b4 U drh -Z 413f6650964e5d2ab342130728703294 +Z bac1d4beccda46e9aa1657c47d678425 diff --git a/manifest.uuid b/manifest.uuid index abe023c752..976aebefbd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -33e627472780b872716c504f2d585cc057c390a5 \ No newline at end of file +5748e64376c1c2be5154a632d1527cfebbb9ec74 \ No newline at end of file diff --git a/src/rowset.c b/src/rowset.c index c2e73ed72e..65fcdb2318 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -338,20 +338,23 @@ static struct RowSetEntry *rowSetNDeepTree( if( *ppList==0 ){ return 0; } - if( iDepth==1 ){ + if( iDepth>1 ){ /*OPTIMIZATION-IF-TRUE*/ + /* This branch cases a *balanced* tree to be generated. A valid tree + ** is still generated without this branch, but it is wildly unbalanced + ** and inefficient. */ + pLeft = rowSetNDeepTree(ppList, iDepth-1); + p = *ppList; + if( p==0 ){ + return pLeft; + } + p->pLeft = pLeft; + *ppList = p->pRight; + p->pRight = rowSetNDeepTree(ppList, iDepth-1); + }else{ p = *ppList; *ppList = p->pRight; p->pLeft = p->pRight = 0; - return p; } - pLeft = rowSetNDeepTree(ppList, iDepth-1); - p = *ppList; - if( p==0 ){ - return pLeft; - } - p->pLeft = pLeft; - *ppList = p->pRight; - p->pRight = rowSetNDeepTree(ppList, iDepth-1); return p; } From 2075fb08e66d6b21c471d9bbdd88773007342543 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 28 Apr 2016 19:23:10 +0000 Subject: [PATCH 57/63] Fix typo in comment. No changes to code. FossilOrigin-Name: 9f6f17b5c97c9576c340173d068485fdbfd53253 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/rowset.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0837962a87..c30368cd24 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rearrange\ssome\scode\sin\sthe\sRowSet\slogic\sfor\sclarity\sof\spresentation,\swhile\nadding\san\s/*OPTIMIZATION-IF-TRUE*/\scomment.\s\sIt\sshould\soperate\sidentically. -D 2016-04-28T18:53:08.454 +C Fix\stypo\sin\scomment.\s\sNo\schanges\sto\scode. +D 2016-04-28T19:23:10.406 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -376,7 +376,7 @@ F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20 -F src/rowset.c d68d95e167cb234f51d655c9546f46752c9f6d45 +F src/rowset.c 156cb68c126b50222f16e7bab4d31ef03866c7e3 F src/select.c fd4a7ce2937497181063cfedb92058ac89491a5d F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6 F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 33e627472780b872716c504f2d585cc057c390a5 -R a33e3686da943dcb038124577baa15b4 -U drh -Z bac1d4beccda46e9aa1657c47d678425 +P 5748e64376c1c2be5154a632d1527cfebbb9ec74 +R 14e25db2c4fe1efcd0c8b2319d192798 +U mistachkin +Z 0a6ff0ee2ee8b77d1f440385eb5ab0b6 diff --git a/manifest.uuid b/manifest.uuid index 976aebefbd..a3d1de6609 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5748e64376c1c2be5154a632d1527cfebbb9ec74 \ No newline at end of file +9f6f17b5c97c9576c340173d068485fdbfd53253 \ No newline at end of file diff --git a/src/rowset.c b/src/rowset.c index 65fcdb2318..bf4382ddd8 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -339,7 +339,7 @@ static struct RowSetEntry *rowSetNDeepTree( return 0; } if( iDepth>1 ){ /*OPTIMIZATION-IF-TRUE*/ - /* This branch cases a *balanced* tree to be generated. A valid tree + /* This branch causes a *balanced* tree to be generated. A valid tree ** is still generated without this branch, but it is wildly unbalanced ** and inefficient. */ pLeft = rowSetNDeepTree(ppList, iDepth-1); From 396794f0ee581d61d213e9bec23ef6b53e0516b5 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 28 Apr 2016 20:11:12 +0000 Subject: [PATCH 58/63] Comment changes only: Add several optimization marks in rowset.c. Add a header comment that explains what the various special comments mean. FossilOrigin-Name: 8cdbe89ac6c22d632f677eb293111b3dbae7d6c1 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/rowset.c | 21 ++++++++++++++------- src/sqliteInt.h | 27 +++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index c30368cd24..4e21e75b83 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\scomment.\s\sNo\schanges\sto\scode. -D 2016-04-28T19:23:10.406 +C Comment\schanges\sonly:\s\sAdd\sseveral\soptimization\smarks\sin\srowset.c.\s\sAdd\na\sheader\scomment\sthat\sexplains\swhat\sthe\svarious\sspecial\scomments\smean. +D 2016-04-28T20:11:12.680 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -376,13 +376,13 @@ F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20 -F src/rowset.c 156cb68c126b50222f16e7bab4d31ef03866c7e3 +F src/rowset.c fb413e336e765481e4d8368377c444904cecdf32 F src/select.c fd4a7ce2937497181063cfedb92058ac89491a5d F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6 F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 98f72cbfe00169c39089115427d06ea05fe4b4a2 -F src/sqliteInt.h ec538389481a3d093f07fb344c5a9dc988042304 +F src/sqliteInt.h f4a53f3547dab80dc7db975fa1192d9bad1f38e8 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5748e64376c1c2be5154a632d1527cfebbb9ec74 -R 14e25db2c4fe1efcd0c8b2319d192798 -U mistachkin -Z 0a6ff0ee2ee8b77d1f440385eb5ab0b6 +P 9f6f17b5c97c9576c340173d068485fdbfd53253 +R 868dfe1770715db047dd242154ff598b +U drh +Z b6de1d1adc478f4213336a399ac69fc9 diff --git a/manifest.uuid b/manifest.uuid index a3d1de6609..7cd7864f7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f6f17b5c97c9576c340173d068485fdbfd53253 \ No newline at end of file +8cdbe89ac6c22d632f677eb293111b3dbae7d6c1 \ No newline at end of file diff --git a/src/rowset.c b/src/rowset.c index bf4382ddd8..5420ab991c 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -179,7 +179,9 @@ void sqlite3RowSetClear(RowSet *p){ */ static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ assert( p!=0 ); - if( p->nFresh==0 ){ + if( p->nFresh==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* We could allocate a fresh RowSetEntry each time one is needed, but it + ** is more efficient to pull a preallocated entry from the pool */ struct RowSetChunk *pNew; pNew = sqlite3DbMallocRawNN(p->db, sizeof(*pNew)); if( pNew==0 ){ @@ -213,7 +215,9 @@ void sqlite3RowSetInsert(RowSet *p, i64 rowid){ pEntry->pRight = 0; pLast = p->pLast; if( pLast ){ - if( (p->rsFlags & ROWSET_SORTED)!=0 && rowid<=pLast->v ){ + if( rowid<=pLast->v ){ /*OPTIMIZATION-IF-FALSE*/ + /* Avoid unnecessary sorts by preserving the ROWSET_SORTED flags + ** where possible */ p->rsFlags &= ~ROWSET_SORTED; } pLast->pRight = pEntry; @@ -335,16 +339,19 @@ static struct RowSetEntry *rowSetNDeepTree( ){ struct RowSetEntry *p; /* Root of the new tree */ struct RowSetEntry *pLeft; /* Left subtree */ - if( *ppList==0 ){ - return 0; + if( *ppList==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Prevent unnecessary deep recursion when we run out of entries */ + return 0; } if( iDepth>1 ){ /*OPTIMIZATION-IF-TRUE*/ /* This branch causes a *balanced* tree to be generated. A valid tree - ** is still generated without this branch, but it is wildly unbalanced - ** and inefficient. */ + ** is still generated without this branch, but the tree is wildly + ** unbalanced and inefficient. */ pLeft = rowSetNDeepTree(ppList, iDepth-1); p = *ppList; - if( p==0 ){ + if( p==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* It is safe to always return here, but the resulting tree + ** would be unbalanced */ return pLeft; } p->pLeft = pLeft; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7bb15d0bf5..76bf809624 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -15,6 +15,33 @@ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ +/* Special Comments: +** +** Some comments have special meaning to the tools that measure test +** coverage: +** +** NO_TEST - The branches on this line are not +** measured by branch coverage. This is +** used on lines of code that actually +** implement parts of coverage testing. +** +** OPTIMIZATION-IF-TRUE - This branch is allowed to alway be false +** and the correct answer is still obtained, +** though perhaps more slowly. +** +** OPTIMIZATION-IF-FALSE - This branch is allowed to alway be true +** and the correct answer is still obtained, +** though perhaps more slowly. +** +** PREVENTS-HARMLESS-OVERREAD - This branch prevents a buffer overread +** that would be harmless and undetectable +** if it did occur. +** +** In all cases, the special comment must be enclosed in the usual +** slash-asterisk...asterisk-slash comment marks, with no spaces between the +** asterisks and the comment text. +*/ + /* ** Make sure that rand_s() is available on Windows systems with MSVC 2005 ** or higher. From aa50271a2432fb6a85c1f42b39317500c7cda950 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 28 Apr 2016 22:29:31 +0000 Subject: [PATCH 59/63] Further simplifications and improved commentting on the rowset.c module, including several optimization comments. FossilOrigin-Name: 9f15a520deb9f1d4ecaa3bfff82bd57ef122aadb --- manifest | 12 +++++----- manifest.uuid | 2 +- src/rowset.c | 65 ++++++++++++++++++--------------------------------- 3 files changed, 30 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index 4e21e75b83..4d312042a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Comment\schanges\sonly:\s\sAdd\sseveral\soptimization\smarks\sin\srowset.c.\s\sAdd\na\sheader\scomment\sthat\sexplains\swhat\sthe\svarious\sspecial\scomments\smean. -D 2016-04-28T20:11:12.680 +C Further\ssimplifications\sand\simproved\scommentting\son\sthe\srowset.c\smodule,\nincluding\sseveral\soptimization\scomments. +D 2016-04-28T22:29:31.056 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -376,7 +376,7 @@ F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20 -F src/rowset.c fb413e336e765481e4d8368377c444904cecdf32 +F src/rowset.c 49eb91c588a2bab36647368e031dc5b66928149d F src/select.c fd4a7ce2937497181063cfedb92058ac89491a5d F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6 F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9f6f17b5c97c9576c340173d068485fdbfd53253 -R 868dfe1770715db047dd242154ff598b +P 8cdbe89ac6c22d632f677eb293111b3dbae7d6c1 +R 9629e5ecfb28b4deca7492eeb801c3d3 U drh -Z b6de1d1adc478f4213336a399ac69fc9 +Z f98c64d26c6e12bfaaf00c7f14039634 diff --git a/manifest.uuid b/manifest.uuid index 7cd7864f7b..fd604e04b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8cdbe89ac6c22d632f677eb293111b3dbae7d6c1 \ No newline at end of file +9f15a520deb9f1d4ecaa3bfff82bd57ef122aadb \ No newline at end of file diff --git a/src/rowset.c b/src/rowset.c index 5420ab991c..a70264ed1b 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -57,8 +57,9 @@ ** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST ** primitives are constant time. The cost of DESTROY is O(N). ** -** There is an added cost of O(N) when switching between TEST and -** SMALLEST primitives. +** TEST and SMALLEST may not be used by the same RowSet. This used to +** be possible, but the feature was not used, so it was removed in order +** to simplify the code. */ #include "sqliteInt.h" @@ -388,59 +389,37 @@ static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ return p; } -/* -** Take all the entries on p->pEntry and on the trees in p->pForest and -** sort them all together into one big ordered list on p->pEntry. -** -** This routine should only be called once in the life of a RowSet. -*/ -static void rowSetToList(RowSet *p){ - - /* This routine is called only once */ - assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); - - if( (p->rsFlags & ROWSET_SORTED)==0 ){ - p->pEntry = rowSetEntrySort(p->pEntry); - } - - /* While this module could theoretically support it, sqlite3RowSetNext() - ** is never called after sqlite3RowSetText() for the same RowSet. So - ** there is never a forest to deal with. Should this change, simply - ** remove the assert() and the #if 0. */ - assert( p->pForest==0 ); -#if 0 - while( p->pForest ){ - struct RowSetEntry *pTree = p->pForest->pLeft; - if( pTree ){ - struct RowSetEntry *pHead, *pTail; - rowSetTreeToList(pTree, &pHead, &pTail); - p->pEntry = rowSetEntryMerge(p->pEntry, pHead); - } - p->pForest = p->pForest->pRight; - } -#endif - p->rsFlags |= ROWSET_NEXT; /* Verify this routine is never called again */ -} - /* ** Extract the smallest element from the RowSet. ** Write the element into *pRowid. Return 1 on success. Return ** 0 if the RowSet is already empty. ** ** After this routine has been called, the sqlite3RowSetInsert() -** routine may not be called again. +** routine may not be called again. +** +** This routine may not be called after sqlite3RowSetTest() has +** been used. Older versions of RowSet allowed that, but as the +** capability was not used by the code generator, it was removed +** for code economy. */ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ assert( p!=0 ); + assert( p->pForest==0 ); /* Cannot be used with sqlite3RowSetText() */ /* Merge the forest into a single sorted list on first call */ - if( (p->rsFlags & ROWSET_NEXT)==0 ) rowSetToList(p); + if( (p->rsFlags & ROWSET_NEXT)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + if( (p->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + p->pEntry = rowSetEntrySort(p->pEntry); + } + p->rsFlags |= ROWSET_SORTED|ROWSET_NEXT; + } /* Return the next entry on the list */ if( p->pEntry ){ *pRowid = p->pEntry->v; p->pEntry = p->pEntry->pRight; - if( p->pEntry==0 ){ + if( p->pEntry==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Free memory immediately, rather than waiting on sqlite3_finalize() */ sqlite3RowSetClear(p); } return 1; @@ -463,13 +442,15 @@ int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ /* This routine is never called after sqlite3RowSetNext() */ assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); - /* Sort entries into the forest on the first test of a new batch + /* Sort entries into the forest on the first test of a new batch. + ** To save unnecessary work, only do this when the batch number changes. */ - if( iBatch!=pRowSet->iBatch ){ + if( iBatch!=pRowSet->iBatch ){ /*OPTIMIZATION-IF-FALSE*/ p = pRowSet->pEntry; if( p ){ struct RowSetEntry **ppPrevTree = &pRowSet->pForest; - if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* Only sort the current set of entiries if they need it */ p = rowSetEntrySort(p); } for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ From a3fa140fb46e74cb86cd1b44905842efd03c9c5e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Apr 2016 02:55:05 +0000 Subject: [PATCH 60/63] Some optimization comments added to vdbe.c. No functional changes to code. FossilOrigin-Name: e7c22e3bffce63f3b47fa3683be8c00c42b2a7d3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 17 ++++++++++------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 4d312042a1..da4cac8196 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\ssimplifications\sand\simproved\scommentting\son\sthe\srowset.c\smodule,\nincluding\sseveral\soptimization\scomments. -D 2016-04-28T22:29:31.056 +C Some\soptimization\scomments\sadded\sto\svdbe.c.\s\sNo\sfunctional\schanges\sto\scode. +D 2016-04-29T02:55:05.603 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -443,7 +443,7 @@ F src/update.c 3e67ab3c0814635f355fb1f8ab010a2b9e016e7d F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c 810ec3f22e2d1b62e66c30fe3621ebdedd23584d F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52 -F src/vdbe.c d3843a66d74a7696477ee5141e5eb9a7e5e2401c +F src/vdbe.c 08fbea00a7f7f723973093c5f5bf7c40c025e2b3 F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170 F src/vdbeInt.h ddb157974436d87652de7dc641f7191496d9a8cd F src/vdbeapi.c ba85b78fe08dc4a9ce747e62c89a2b4a4547e74c @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8cdbe89ac6c22d632f677eb293111b3dbae7d6c1 -R 9629e5ecfb28b4deca7492eeb801c3d3 +P 9f15a520deb9f1d4ecaa3bfff82bd57ef122aadb +R 9e8b60c02f680bebef2b9615706e75d6 U drh -Z f98c64d26c6e12bfaaf00c7f14039634 +Z f25847a4d8f7c8c164def037ab0b4e49 diff --git a/manifest.uuid b/manifest.uuid index fd604e04b3..74550e35d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f15a520deb9f1d4ecaa3bfff82bd57ef122aadb \ No newline at end of file +e7c22e3bffce63f3b47fa3683be8c00c42b2a7d3 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 244a4ad063..5fba14a5c3 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -215,7 +215,7 @@ static VdbeCursor *allocateCursor( (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); assert( iCur>=0 && iCurnCursor ); - if( p->apCsr[iCur] ){ + if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } @@ -292,7 +292,7 @@ static void applyAffinity( if( affinity>=SQLITE_AFF_NUMERIC ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL || affinity==SQLITE_AFF_NUMERIC ); - if( (pRec->flags & MEM_Int)==0 ){ + if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/ if( (pRec->flags & MEM_Real)==0 ){ if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); }else{ @@ -302,10 +302,13 @@ static void applyAffinity( }else if( affinity==SQLITE_AFF_TEXT ){ /* Only attempt the conversion to TEXT if there is an integer or real ** representation (blob and NULL do not get converted) but no string - ** representation. - */ - if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ - sqlite3VdbeMemStringify(pRec, enc, 1); + ** representation. It would be harmless to repeat the conversion if + ** there is already a string rep, but it is pointless to waste those + ** CPU cycles. */ + if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/ + if( (pRec->flags&(MEM_Real|MEM_Int)) ){ + sqlite3VdbeMemStringify(pRec, enc, 1); + } } pRec->flags &= ~(MEM_Real|MEM_Int); } @@ -542,7 +545,7 @@ static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); pOut = &p->aMem[pOp->p2]; memAboutToChange(p, pOut); - if( VdbeMemDynamic(pOut) ){ + if( VdbeMemDynamic(pOut) ){ /*OPTIMIZATION-IF-FALSE*/ return out2PrereleaseWithClear(pOut); }else{ pOut->flags = MEM_Int; From 14faa061fd4815f6f8f63bc4d00889b676c85c20 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 29 Apr 2016 10:13:22 +0000 Subject: [PATCH 61/63] Fix an almost entirely harmless buffer overread in the sessions module. FossilOrigin-Name: 89b0ee3351381f7bc666cb206f77a56f2e0d4a0e --- ext/session/sqlite3session.c | 17 +++++++++++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index b058d357e2..e333dffcc7 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -592,14 +592,19 @@ static int sessionChangeEqual( int iCol; /* Used to iterate through table columns */ for(iCol=0; iColnCol; iCol++){ - int n1 = sessionSerialLen(a1); - int n2 = sessionSerialLen(a2); + if( pTab->abPK[iCol] ){ + int n1 = sessionSerialLen(a1); + int n2 = sessionSerialLen(a2); - if( pTab->abPK[iCol] && (n1!=n2 || memcmp(a1, a2, n1)) ){ - return 0; + if( pTab->abPK[iCol] && (n1!=n2 || memcmp(a1, a2, n1)) ){ + return 0; + } + a1 += n1; + a2 += n2; + }else{ + if( bLeftPkOnly==0 ) a1 += sessionSerialLen(a1); + if( bRightPkOnly==0 ) a2 += sessionSerialLen(a2); } - if( pTab->abPK[iCol] || bLeftPkOnly==0 ) a1 += n1; - if( pTab->abPK[iCol] || bRightPkOnly==0 ) a2 += n2; } return 1; diff --git a/manifest b/manifest index 61077e7077..77d5df7f09 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\schanges\sfrom\strunk\swith\sthis\sbranch. -D 2016-04-28T14:59:05.489 +C Fix\san\salmost\sentirely\sharmless\sbuffer\soverread\sin\sthe\ssessions\smodule. +D 2016-04-29T10:13:22.999 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -293,7 +293,7 @@ F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 -F ext/session/sqlite3session.c beb300cd1b5c5054062c8e6e807b10475e363410 +F ext/session/sqlite3session.c beb43b6b888801bb006320bc236baa95f4cc32ae F ext/session/sqlite3session.h 64e9e7f185725ef43b97f4a9a0c0df0669844f1d F ext/session/test_session.c 464f2c8bf502795d95969387eb8e93f68c513c15 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5973d3bf2e5aa979ccc9a4a3a0eb08b67098b499 33e627472780b872716c504f2d585cc057c390a5 -R 66281bc4e8738b7ab348496eadeeb2b1 +P 45467ee49872f495d5276cf830f10c3cf4ac8e3c +R 50a6a2b7911c16c6704703bf8b17ce71 U dan -Z 4d2c4cbb1442cfb28a5ae33052756c7a +Z da972510c8d8b75cf481b4f80618fa01 diff --git a/manifest.uuid b/manifest.uuid index b1984ea171..f4d85f9254 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -45467ee49872f495d5276cf830f10c3cf4ac8e3c \ No newline at end of file +89b0ee3351381f7bc666cb206f77a56f2e0d4a0e \ No newline at end of file From afb8cd977d09759a29b6c89d3882e431fb86a480 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Apr 2016 11:28:35 +0000 Subject: [PATCH 62/63] Lemon bug fix: Do not merge action routines unless their destructors are also identical. Problem and suggested fix reported on the mailing list by Kelvin Sherlock. FossilOrigin-Name: 4792d6dbba6857f74d27332dcc1468e39c767c71 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lemon.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index da4cac8196..df4dd75c55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Some\soptimization\scomments\sadded\sto\svdbe.c.\s\sNo\sfunctional\schanges\sto\scode. -D 2016-04-29T02:55:05.603 +C Lemon\sbug\sfix:\s\sDo\snot\smerge\saction\sroutines\sunless\stheir\sdestructors\sare\nalso\sidentical.\s\sProblem\sand\ssuggested\sfix\sreported\son\sthe\smailing\slist\nby\sKelvin\sSherlock. +D 2016-04-29T11:28:35.306 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -1414,7 +1414,7 @@ F tool/fuzzershell.c 94019b185caceffc9f7c7b678a6489e42bc2aefa F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c cfbfe061a4b2766512f6b484882eee2c86a14506 +F tool/lemon.c 83318dff3911e47f2b85e136e56aa1c4674a2d2b F tool/lempar.c 404ea3dc27dbeed343f0e61b1d36e97b9f5f0fb6 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9f15a520deb9f1d4ecaa3bfff82bd57ef122aadb -R 9e8b60c02f680bebef2b9615706e75d6 +P e7c22e3bffce63f3b47fa3683be8c00c42b2a7d3 +R 2a18b1b7e68842336a01e91672c4d16e U drh -Z f25847a4d8f7c8c164def037ab0b4e49 +Z 9e05af624d486885b70c45e730afddce diff --git a/manifest.uuid b/manifest.uuid index 74550e35d8..d7daf8acb8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e7c22e3bffce63f3b47fa3683be8c00c42b2a7d3 \ No newline at end of file +4792d6dbba6857f74d27332dcc1468e39c767c71 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 903fe97cfb..2f76465d89 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -4402,7 +4402,8 @@ void ReportTable( writeRuleText(out, rp); fprintf(out, " */\n"); lineno++; for(rp2=rp->next; rp2; rp2=rp2->next){ - if( rp2->code==rp->code ){ + if( rp2->code==rp->code && rp2->codePrefix==rp->codePrefix + && rp2->codeSuffix==rp->codeSuffix ){ fprintf(out," case %d: /* ", rp2->iRule); writeRuleText(out, rp2); fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++; From 4a030c643070a939ef2b3f86b56b64a38693868d Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 29 Apr 2016 14:12:48 +0000 Subject: [PATCH 63/63] Fix test script temptable2.test so that it works with the "inmemory_journal" and "journaltest" permutations. FossilOrigin-Name: b7bec7f2d3d08d0c14f77654be2c5cedf1619e58 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/temptable2.test | 21 +++++++++++++-------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c04f4869ed..1a928a99b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges,\sincluding\stest\scase\sfixes,\swith\sthis\sbranch. -D 2016-04-29T11:35:28.485 +C Fix\stest\sscript\stemptable2.test\sso\sthat\sit\sworks\swith\sthe\s"inmemory_journal"\sand\s"journaltest"\spermutations. +D 2016-04-29T14:12:48.901 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -1113,7 +1113,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test d4e967c355b154e8bc387de961430e94c20f2dca +F test/temptable2.test c3d8c138f493207612960bbd6a8c50e84975e2ee F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 30c7a9be8601d1c1c9c93d013545ebcb28d64254 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6bb5aa2be2a6c5ca66a98a9387b2845507a6bdd2 4cbd50245440e714935399a5c285cf32d8bfc424 -R 7ae3c477f36afa854d04f85029639199 +P 99794aca7b6cb40f08ce4db9889a989fc597eac9 +R f9ef4e1af8653e902d9192ffb04cabc2 U dan -Z 1ae9b89d9de363b516966d4d50de5d49 +Z 5d059cb204727917e4a7564cbd5ed599 diff --git a/manifest.uuid b/manifest.uuid index e7c3b7b953..db83d58f1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -99794aca7b6cb40f08ce4db9889a989fc597eac9 \ No newline at end of file +b7bec7f2d3d08d0c14f77654be2c5cedf1619e58 \ No newline at end of file diff --git a/test/temptable2.test b/test/temptable2.test index 21e06a3a4f..890b3b996d 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -139,9 +139,11 @@ do_execsql_test 5.1.1 { INSERT INTO t1 VALUES(1, 2); } +# Test that the temp database is now much bigger than the configured +# cache size (10 pages). do_test 5.1.2 { set n [db one { PRAGMA temp.page_count }] - expr ($n > 280 && $n < 290) + expr ($n > 270 && $n < 290) } {1} do_execsql_test 5.1.3 { @@ -284,9 +286,9 @@ foreach {tn mode} { PRAGMA cache_size = 15; PRAGMA auto_vacuum = 1; } - do_execsql_test 9.$tn.1.2 "PRAGMA journal_mode = $mode" delete + execsql "PRAGMA journal_mode = $mode" - do_execsql_test 9.$tn.1.3 { + do_execsql_test 9.$tn.1.2 { CREATE TABLE tx(a, b); CREATE INDEX i1 ON tx(a); CREATE INDEX i2 ON tx(b); @@ -341,11 +343,14 @@ do_execsql_test 10.1 { INSERT INTO t2 VALUES(3, 4); } -do_execsql_test 10.2 { - PRAGMA mmap_size = 512000; - SELECT * FROM t2; - PRAGMA integrity_check; -} {512000 1 2 3 4 ok} +if {[permutation]!="journaltest"} { + # The journaltest permutation does not support mmap, so this part of + # the test is omitted. + do_execsql_test 10.2 { PRAGMA mmap_size = 512000 } 512000 +} + +do_execsql_test 10.3 { SELECT * FROM t2 } {1 2 3 4} +do_execsql_test 10.4 { PRAGMA integrity_check } ok finish_test