From 0410302e585d278cde32238360c269281b5e6717 Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Tue, 3 Feb 2009 16:51:24 +0000 Subject: [PATCH] Commit first version of the 'backup' feature. (CVS 6241) FossilOrigin-Name: 663479b417fc06ba1790a544f28694f8797cee57 --- Makefile.in | 3 + main.mk | 7 +- manifest | 55 +-- manifest.uuid | 2 +- src/attach.c | 4 +- src/backup.c | 559 +++++++++++++++++++++++++++++ src/btree.c | 227 +----------- src/btree.h | 3 +- src/btreeInt.h | 3 +- src/build.c | 38 +- src/main.c | 12 +- src/pager.c | 32 +- src/pager.h | 5 +- src/sqlite.h.in | 174 ++++++++- src/sqliteInt.h | 6 +- src/tclsqlite.c | 4 +- src/test1.c | 4 +- src/test_backup.c | 148 ++++++++ src/vacuum.c | 3 +- src/vdbeaux.c | 4 +- test/backup.test | 767 ++++++++++++++++++++++++++++++++++++++++ test/backup_ioerr.test | 287 +++++++++++++++ test/backup_malloc.test | 86 +++++ test/quick.test | 3 +- test/tester.tcl | 8 +- tool/mksqlite3c.tcl | 1 + 26 files changed, 2157 insertions(+), 288 deletions(-) create mode 100644 src/backup.c create mode 100644 src/test_backup.c create mode 100644 test/backup.test create mode 100644 test/backup_ioerr.test create mode 100644 test/backup_malloc.test diff --git a/Makefile.in b/Makefile.in index 89c2867e35..5461633eac 100644 --- a/Makefile.in +++ b/Makefile.in @@ -195,6 +195,7 @@ SRC = \ $(TOP)/src/analyze.c \ $(TOP)/src/attach.c \ $(TOP)/src/auth.c \ + $(TOP)/src/backup.c \ $(TOP)/src/bitvec.c \ $(TOP)/src/btmutex.c \ $(TOP)/src/btree.c \ @@ -328,6 +329,7 @@ SRC += \ # TESTSRC2 = \ $(TOP)/src/attach.c \ + $(TOP)/src/backup.c \ $(TOP)/src/bitvec.c \ $(TOP)/src/btree.c \ $(TOP)/src/build.c \ @@ -372,6 +374,7 @@ TESTSRC = \ $(TOP)/src/test9.c \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ + $(TOP)/src/test_backup.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_devsym.c \ diff --git a/main.mk b/main.mk index c9f654cac0..f5b6231a9e 100644 --- a/main.mk +++ b/main.mk @@ -50,7 +50,7 @@ TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 # Object files for the SQLite library. # LIBOBJ+= alter.o analyze.o attach.o auth.o \ - bitvec.o btmutex.o btree.o build.o \ + backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o date.o delete.o expr.o fault.o \ fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_tokenizer.o fts3_tokenizer1.o \ @@ -76,6 +76,7 @@ SRC = \ $(TOP)/src/analyze.c \ $(TOP)/src/attach.c \ $(TOP)/src/auth.c \ + $(TOP)/src/backup.c \ $(TOP)/src/bitvec.c \ $(TOP)/src/btmutex.c \ $(TOP)/src/btree.c \ @@ -220,6 +221,7 @@ TESTSRC = \ $(TOP)/src/test9.c \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ + $(TOP)/src/test_backup.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_devsym.c \ @@ -242,7 +244,8 @@ TESTSRC = \ #TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c TESTSRC2 = \ - $(TOP)/src/attach.c $(TOP)/src/btree.c $(TOP)/src/build.c $(TOP)/src/date.c \ + $(TOP)/src/attach.c $(TOP)/src/backup.c $(TOP)/src/btree.c \ + $(TOP)/src/build.c $(TOP)/src/date.c \ $(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/os.c \ $(TOP)/src/os_os2.c $(TOP)/src/os_unix.c $(TOP)/src/os_win.c \ $(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c \ diff --git a/manifest b/manifest index 4fc8c8f0c4..5d41cc6fe5 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Fix\sthe\ssqlite3_mprintf_long\stest\scommand\s(added\sby\scheck-in\s(6224)\sin\sorder\nto\saddress\sticket\s#3621)\sso\sthat\sit\sworks\son\ssystems\swith\ssizeof(int)==4\sand\nsizeof(long)==8.\s(CVS\s6240) -D 2009-02-03T16:25:48 +C Commit\sfirst\sversion\sof\sthe\s'backup'\sfeature.\s(CVS\s6241) +D 2009-02-03T16:51:25 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 -F Makefile.in 3871d308188cefcb7c5ab20da4c7b6aad023bc52 +F Makefile.in c7a5a30fb6852bd7839b1024e1661da8549878ee F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 F Makefile.vxwSH4 d53b4be86491060d498b22148951b6d765884cab F README b974cdc3f9f12b87e851b04e75996d720ebf81ac @@ -83,7 +83,7 @@ F ext/rtree/tkt3363.test 6662237ea75bb431cd5d262dfc9535e1023315fc F ext/rtree/viewrtree.tcl 09526398dae87a5a87c5aac2b3854dbaf8376869 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 189d17c22bc35a9223f2de0eb9ac6e818439cef7 +F main.mk 2193e5939dbf91449f9b72178d543d31b2315360 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -99,14 +99,15 @@ F sqlite3.def a1be7b9a4b8b51ac41c6ff6e8e44a14ef66b338b F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 0ec29744c36c6e976596ce38c16289ebc5dc94db F src/analyze.c c86fd6a1425b22b3a46ce72ad403e4280026364f -F src/attach.c 1c35f95da3c62d19de75b44cfefd12c81c1791b3 +F src/attach.c 81d37d1948f409146a7b22b96998fd90649d1fd3 F src/auth.c c8b2ab5c8bad4bd90ed7c294694f48269162c627 +F src/backup.c a7605687863424d5d5a7ff8271f0bbcfd4fc0b57 F src/bitvec.c 44f7059ac1f874d364b34af31b9617e52223ba75 F src/btmutex.c 63c5cc4ad5715690767ffcb741e185d7bc35ec1a -F src/btree.c dfbbfc396fdd8cbc29754864a97c4df484b78870 -F src/btree.h 07359623fa24748709dd61212a32364a6adc6b56 -F src/btreeInt.h 44bcbfe387ba99a3a9f2527bd12fa1bb8bc574b3 -F src/build.c 1d755e4920d94b7b470f47d1915b131b92fe6f6b +F src/btree.c 800a065686c49a0cdefc933779a750a7c3c0509f +F src/btree.h 4eab72af6adf95f0b08b61a72ef9781bdb0bf63f +F src/btreeInt.h 0a4884e6152d7cae9c741e91b830064c19fd2c05 +F src/build.c ed7a59fa45823464b7d5fdca7712a5fb3433f757 F src/callback.c 5f10bca853e59a2c272bbfd5b720303f8b69e520 F src/complete.c cb14e06dbe79dee031031f0d9e686ff306afe07c F src/date.c 870770dde3fb56772ab247dfb6a6eda44d16cfbc @@ -122,7 +123,7 @@ F src/insert.c f6db1e6f43aae337e64a755208abb6ff124edc19 F src/journal.c e00df0c0da8413ab6e1bb7d7cab5665d4a9000d0 F src/legacy.c 8b3b95d48d202614946d7ce7256e7ba898905c3b F src/loadext.c 3f96631089fc4f3871a67f02f2e4fc7ea4d51edc -F src/main.c a7d7fd7df4e9f8fa3418258619436c969234fd82 +F src/main.c da51988dd4d75de4ccc66d2c99dd1b5b3b266e6c F src/malloc.c bc408056b126db37b6fba00e170d578cc67be6b3 F src/mem0.c f2f84062d1f35814d6535c9f9e33de3bfb3b132c F src/mem1.c 3bfb39e4f60b0179713a7c087b2d4f0dc205735f @@ -142,8 +143,8 @@ F src/os_common.h 24525d8b7bce66c374dfc1810a6c9043f3359b60 F src/os_os2.c bed77dc26e3a95ce4a204936b9a1ca6fe612fcc5 F src/os_unix.c f0fce3042011d462b8ae633564a5668260bd3636 F src/os_win.c ec133f2a3c0da786995ea09ba67056af8f18cc2e -F src/pager.c 72f4e7b3076584889ce6286cd15ff2d985325026 -F src/pager.h eccf5cdeebd79006ba7f9577dd30d8179b1430da +F src/pager.c 9e7c6db1635be2caf31ff3d407ecb2e145f89a8a +F src/pager.h 0c9f3520c00d8a3b8e792ca56c9a11b6b02b4b0f F src/parse.y 4f4d16aee0d11f69fec2adb77dac88878043ed8d F src/pcache.c fcf7738c83c4d3e9d45836b2334c8a368cc41274 F src/pcache.h 9b927ccc5a538e31b4c3bc7eec4f976db42a1324 @@ -156,14 +157,14 @@ F src/resolve.c 18dc9f0df1d60048e012ce6632251063e0dd356a F src/rowset.c ba9375f37053d422dd76965a9c370a13b6e1aac4 F src/select.c ae72b604e47092521c4d9ae54e1b1cbeb872a747 F src/shell.c 8965cf0cd7a7dac39d586a43c97adb00930e025d -F src/sqlite.h.in 8821a61dceff26993ed6689239b6fbcd8d8f6e50 +F src/sqlite.h.in e0d54b3a93489154151f49007a2f1219171945fa F src/sqlite3ext.h 1db7d63ab5de4b3e6b83dd03d1a4e64fef6d2a17 -F src/sqliteInt.h 3ee870a4d5886992cd09af62f0d13dc7a6033f9f +F src/sqliteInt.h 73c1d4f9716fe21f202f9d05c4fd9e6281f2636f F src/sqliteLimit.h ffe93f5a0c4e7bd13e70cd7bf84cfb5c3465f45d F src/status.c 237b193efae0cf6ac3f0817a208de6c6c6ef6d76 F src/table.c 332ab0ea691e63862e2a8bdfe2c0617ee61062a3 -F src/tclsqlite.c 7d77c3899d0244804d2773c9157e783788627762 -F src/test1.c 461b793df7db8a8d48bf261a813b2a7ef2417e6d +F src/tclsqlite.c 7b3e7fc4856e8280939c9ca0c3a6e49bd2c4bb46 +F src/test1.c f88b447699786d58a0136a3a48b12990abc72c8a F src/test2.c 9689e7d3b7791da8c03f9acd1ea801802cb83c17 F src/test3.c 88a246b56b824275300e6c899634fbac1dc94b14 F src/test4.c f79ab52d27ff49b784b631a42e2ccd52cfd5c84c @@ -174,6 +175,7 @@ F src/test8.c 3637439424d0d21ff2dcf9b015c30fcc1e7bcb24 F src/test9.c 904ebe0ed1472d6bad17a81e2ecbfc20017dc237 F src/test_async.c 45024094ed7cf780c5d5dccda645145f95cf78ef F src/test_autoext.c f53b0cdf7bf5f08100009572a5d65cdb540bd0ad +F src/test_backup.c 5b41518c5499dafe65177b0813b71ac356ee9df1 F src/test_btree.c d7b8716544611c323860370ee364e897c861f1b0 F src/test_config.c 9dd62f4bb725ad87d28b187b07377cb4f4a43197 F src/test_devsym.c 9f4bc2551e267ce7aeda195f3897d0f30c5228f4 @@ -197,12 +199,12 @@ F src/trigger.c ca6d78f7c1314053800386ca64361e487774fda3 F src/update.c 8c4925f9ca664effc8a1faaad67449d2074567b1 F src/utf.c 1da9c832dba0fa8f865b5b902d93f420a1ee4245 F src/util.c f1ac1bcd3ec5e3300982031504659b6f9435de33 -F src/vacuum.c b78c2bfdefc1b1d9aa5d82d57c333c5fde7be5a6 +F src/vacuum.c 4929a585ef0fb1dfaf46302f8a9c4aa30c2d9cf5 F src/vdbe.c 81120d5a5ba2d93eb7d7f66e814bbc811305daa2 F src/vdbe.h 03516f28bf5aca00a53c4dccd6c313f96adb94f6 F src/vdbeInt.h 13cb4868ea579b5a8f6b6b5098caa99cd5a14078 F src/vdbeapi.c 85c33cfbfa56249cbe627831610afafba754477d -F src/vdbeaux.c 30c1bbc1d2876c5bbe84d52dab9980ed032bca98 +F src/vdbeaux.c 75c3ac2a3c37747ae66ea0935f8f48bb1879234a F src/vdbeblob.c b0dcebfafedcf9c0addc7901ad98f6f986c08935 F src/vdbemem.c c6127c335f802ba159c6fec4e3284ba82a070602 F src/vtab.c e39e011d7443a8d574b1b9cde207a35522e6df43 @@ -231,6 +233,9 @@ F test/autoinc.test ab549b48b389cabd92967b86c379ec8b31fa6c16 F test/autovacuum.test 61260e25744189ff766f61ca3df23c1eeec0060e F test/autovacuum_ioerr2.test 598b0663074d3673a9c1bc9a16e80971313bafe6 F test/avtrans.test 1e901d8102706b63534dbd2bdd4d8f16c4082650 +F test/backup.test bd478dd20a092a99d98943dee9d92d69823a6820 +F test/backup_ioerr.test 2edd5e347e263733cae8c08f41bf3dbd7277b33d +F test/backup_malloc.test 471fb098dae228ca840d4d51e41481901ac03578 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/between.test 16b1776c6323faadb097a52d673e8e3d8be7d070 F test/bigfile.test 6adfef13d24bbe0c504b4547f292b9a170184f25 @@ -487,7 +492,7 @@ F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 47e9e5bbec8509023479d54ceb71c9d05a95308a F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 x F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/quick.test 9ab91798b047684f0dd26ee698920dbb69a30a10 +F test/quick.test 4a09b89a44be46b3ee5a5dbe25b72cb1e5c3ead8 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a @@ -546,7 +551,7 @@ F test/tableapi.test 505031f15b18a750184d967d2c896cf88fcc969c F test/tclsqlite.test 30636c3151ccc2d553aa09020b885054141a1963 F test/tempdb.test b88ac8a19823cf771d742bf61eef93ef337c06b1 F test/temptable.test 19b851b9e3e64d91e9867619b2a3f5fffee6e125 -F test/tester.tcl 57b8ad3e60bd14e93c88c9b8f1106221e677d17e +F test/tester.tcl 3d11a8c1d05535400880ac4f8c5402b8dee14b7f F test/thread001.test 71dca5edec5e44b56a9043da1ce7651c12216fe1 F test/thread002.test 84c03a9fc4f7a5f92eefe551266afa840c2eb6ae F test/thread003.test e17754799649c2b732c295620dca041c32f01e16 @@ -681,7 +686,7 @@ F tool/lempar.c aeba88b8566ff66f8a67c96b3eb2dd95e7d8908d F tool/mkkeywordhash.c 8e57fbe8c4fe2f1800f9190fd361231cb8558407 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c.tcl d4668afb9b48533eed969c98787fea9a3d07b565 +F tool/mksqlite3c.tcl b3dcc7a9610baf36545dab6acf19605505016409 F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87 F tool/omittest.tcl 27d6f6e3b1e95aeb26a1c140e6eb57771c6d794a F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -695,7 +700,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e -P 85e9196d79ef8500300abb215a31e0519b2e8d02 -R 8b6fc22195ba110d87608f7198abea10 -U drh -Z b41cee41042712f485446405aaaaa49c +P 2e45c2a85183f7430225aa8dd89ee05028afecf2 +R 4e142df07079d9b7b2e171c0f7313ba2 +U danielk1977 +Z aac63fa067e376f14e29eb95cbe2ffd4 diff --git a/manifest.uuid b/manifest.uuid index 1e36e3bb10..2cae1aff8a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e45c2a85183f7430225aa8dd89ee05028afecf2 \ No newline at end of file +663479b417fc06ba1790a544f28694f8797cee57 \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index 2a7525ae93..bf2683fb30 100644 --- a/src/attach.c +++ b/src/attach.c @@ -11,7 +11,7 @@ ************************************************************************* ** This file contains code used to implement the ATTACH and DETACH commands. ** -** $Id: attach.c,v 1.81 2008/12/10 16:45:51 drh Exp $ +** $Id: attach.c,v 1.82 2009/02/03 16:51:25 danielk1977 Exp $ */ #include "sqliteInt.h" @@ -265,7 +265,7 @@ static void detachFunc( "cannot DETACH database within transaction"); goto detach_error; } - if( sqlite3BtreeIsInReadTrans(pDb->pBt) ){ + if( sqlite3BtreeIsInReadTrans(pDb->pBt) || sqlite3BtreeIsInBackup(pDb->pBt) ){ sqlite3_snprintf(sizeof(zErr),zErr, "database %s is locked", zName); goto detach_error; } diff --git a/src/backup.c b/src/backup.c new file mode 100644 index 0000000000..a03ac2ad54 --- /dev/null +++ b/src/backup.c @@ -0,0 +1,559 @@ +/* +** 2009 January 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the implementation of the sqlite3_backup_XXX() +** API functions and the related features. +** +** $Id: backup.c,v 1.1 2009/02/03 16:51:25 danielk1977 Exp $ +*/ +#include "sqliteInt.h" +#include "btreeInt.h" + +/* Macro to find the minimum of two numeric values. +*/ +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif + +/* +** Structure allocated for each backup operation. +*/ +struct sqlite3_backup { + sqlite3* pDestDb; /* Destination database handle */ + Btree *pDest; /* Destination b-tree file */ + u32 iDestSchema; /* Original schema cookie in destination */ + int bDestLocked; /* True once a write-transaction is open on pDest */ + + Pgno iNext; /* Page number of the next source page to copy */ + sqlite3* pSrcDb; /* Source database handle */ + Btree *pSrc; /* Source b-tree file */ + + int rc; /* Backup process error code */ + + /* These two variables are set by every call to backup_step(). They are + ** read by calls to backup_remaining() and backup_pagecount(). + */ + Pgno nRemaining; /* Number of pages left to copy */ + Pgno nPagecount; /* Total number of pages to copy */ + + sqlite3_backup *pNext; /* Next backup associated with source pager */ +}; + +/* +** THREAD SAFETY NOTES: +** +** Once it has been created using backup_init(), a single sqlite3_backup +** structure may be accessed via two groups of thread-safe entry points: +** +** * Via the sqlite3_backup_XXX() API function backup_step() and +** backup_finish(). Both these functions obtain the source database +** handle mutex and the mutex associated with the source BtShared +** structure, in that order. +** +** * Via the BackupUpdate() and BackupRestart() functions, which are +** invoked by the pager layer to report various state changes in +** the page cache associated with the source database. The mutex +** associated with the source database BtShared structure will always +** be held when either of these functions are invoked. +** +** The other sqlite3_backup_XXX() API functions, backup_remaining() and +** backup_pagecount() are not thread-safe functions. If they are called +** while some other thread is calling backup_step() or backup_finish(), +** the values returned may be invalid. There is no way for a call to +** BackupUpdate() or BackupRestart() to interfere with backup_remaining() +** or backup_pagecount(). +** +** Depending on the SQLite configuration, the database handles and/or +** the Btree objects may have their own mutexes that require locking. +** Non-sharable Btrees (in-memory databases for example), do not have +** associated mutexes. +*/ + +/* +** Return a pointer corresponding to database zDb (i.e. "main", "temp") +** in connection handle pDb. If such a database cannot be found, return +** a NULL pointer and write an error message to pErrorDb. +** +** If the "temp" database is requested, it may need to be opened by this +** function. If an error occurs while doing so, return 0 and write an +** error message to pErrorDb. +*/ +static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ + int i = sqlite3FindDbName(pDb, zDb); + + if( i==1 ){ + Parse sParse; + memset(&sParse, 0, sizeof(sParse)); + sParse.db = pDb; + if( sqlite3OpenTempDatabase(&sParse) ){ + sqlite3ErrorClear(&sParse); + sqlite3Error(pErrorDb, sParse.rc, "%s", sParse.zErrMsg); + return 0; + } + assert( sParse.zErrMsg==0 ); + } + + if( i<0 ){ + sqlite3Error(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); + return 0; + } + + return pDb->aDb[i].pBt; +} + +/* +** Create an sqlite3_backup process to copy the contents of zSrcDb from +** connection handle pSrcDb to zDestDb in pDestDb. If successful, return +** a pointer to the new sqlite3_backup object. +** +** If an error occurs, NULL is returned and an error code and error message +** stored in database handle pDestDb. +*/ +sqlite3_backup *sqlite3_backup_init( + sqlite3* pDestDb, /* Database to write to */ + const char *zDestDb, /* Name of database within pDestDb */ + sqlite3* pSrcDb, /* Database connection to read from */ + const char *zSrcDb /* Name of database within pSrcDb */ +){ + sqlite3_backup *p; /* Value to return */ + + /* Lock the source database handle. The destination database + ** handle is not locked. The user is required to ensure that no + ** other thread accesses the destination handle for the duration + ** of the backup operation. + */ + sqlite3_mutex_enter(pSrcDb->mutex); + + if( pSrcDb==pDestDb ){ + sqlite3Error( + pDestDb, SQLITE_ERROR, "Source and destination handles must be distinct" + ); + p = 0; + }else { + /* Allocate space for a new sqlite3_backup object */ + p = (sqlite3_backup *)sqlite3_malloc(sizeof(sqlite3_backup)); + if( !p ){ + sqlite3Error(pDestDb, SQLITE_NOMEM, 0); + } + } + + /* If the allocation succeeded, populate the new object. */ + if( p ){ + memset(p, 0, sizeof(sqlite3_backup)); + p->pSrc = findBtree(pDestDb, pSrcDb, zSrcDb); + p->pDest = findBtree(pDestDb, pDestDb, zDestDb); + p->pDestDb = pDestDb; + p->pSrcDb = pSrcDb; + p->iNext = 1; + + if( 0==p->pSrc || 0==p->pDest ){ + /* One (or both) of the named databases did not exist. An error has + ** already been written into the pDestDb handle. All that is left + ** to do here is free the sqlite3_backup structure. + */ + sqlite3_free(p); + p = 0; + } + } + + /* If everything has gone as planned, attach the backup object to the + ** source pager. The source pager calls BackupUpdate() and BackupRestart() + ** to notify this module if the source file is modified mid-backup. + */ + if( p ){ + sqlite3_backup **pp; /* Pointer to head of pagers backup list */ + sqlite3BtreeEnter(p->pSrc); + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + p->pNext = *pp; + *pp = p; + sqlite3BtreeLeave(p->pSrc); + p->pSrc->nBackup++; + } + + sqlite3_mutex_leave(pSrcDb->mutex); + return p; +} + +/* +** Parameter zSrcData points to a buffer containing the data for +** page iSrcPg from the source database. Copy this data into the +** destination database. +*/ +static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ + Pager * const pDestPager = sqlite3BtreePager(p->pDest); + const int nSrcPgsz = sqlite3BtreeGetPageSize(p->pSrc); + int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); + const int nCopy = MIN(nSrcPgsz, nDestPgsz); + const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; + + int rc = SQLITE_OK; + i64 iOff; + + assert( p->bDestLocked ); + assert( p->rc==SQLITE_OK ); + assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); + assert( zSrcData ); + + /* Catch the case where the destination is an in-memory database and the + ** page sizes of the source and destination differ. + */ + if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(sqlite3BtreePager(p->pDest)) ){ + rc = SQLITE_READONLY; + } + + /* This loop runs once for each destination page spanned by the source + ** page. For each iteration, variable iOff is set to the byte offset + ** of the destination page. + */ + for(iOff=iEnd-(i64)nSrcPgsz; rc==SQLITE_OK && iOffpDest->pBt) ) continue; + if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg)) + && SQLITE_OK==(rc = sqlite3PagerWrite(pDestPg)) + ){ + const u8 *zIn = &zSrcData[iOff%nSrcPgsz]; + u8 *zDestData = sqlite3PagerGetData(pDestPg); + u8 *zOut = &zDestData[iOff%nDestPgsz]; + + /* Copy the data from the source page into the destination page. + ** Then clear the Btree layer MemPage.isInit flag. Both this module + ** and the pager code use this trick (clearing the first byte + ** of the page 'extra' space to invalidate the Btree layers + ** cached parse of the page). MemPage.isInit is marked + ** "MUST BE FIRST" for this purpose. + */ + memcpy(zOut, zIn, nCopy); + ((u8 *)sqlite3PagerGetExtra(pDestPg))[0] = 0; + } + sqlite3PagerUnref(pDestPg); + } + + return rc; +} + +/* +** Copy nPage pages from the source b-tree to the destination. +*/ +int sqlite3_backup_step(sqlite3_backup *p, int nPage){ + int rc; + + sqlite3_mutex_enter(p->pSrcDb->mutex); + sqlite3BtreeEnter(p->pSrc); + + rc = p->rc; + if( rc==SQLITE_OK ){ + Pager * const pSrcPager = sqlite3BtreePager(p->pSrc); /* Source pager */ + Pager * const pDestPager = sqlite3BtreePager(p->pDest); /* Dest pager */ + int ii; /* Iterator variable */ + int nSrcPage; /* Size of source db in pages */ + int bCloseTrans = 0; /* True if src db requires unlocking */ + + /* If the source pager is currently in a write-transaction, return + ** SQLITE_LOCKED immediately. + */ + if( p->pDestDb && p->pSrc->pBt->inTransaction==TRANS_WRITE ){ + rc = SQLITE_LOCKED; + } + + /* Lock the destination database, if it is not locked already. */ + if( SQLITE_OK==rc && p->bDestLocked==0 + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) + ){ + p->bDestLocked = 1; + rc = sqlite3BtreeGetMeta(p->pDest, 1, &p->iDestSchema); + } + + /* If there is no open read-transaction on the source database, open + ** one now. If a transaction is opened here, then it will be closed + ** before this function exits. + */ + if( rc==SQLITE_OK && 0==sqlite3BtreeIsInReadTrans(p->pSrc) ){ + rc = sqlite3BtreeBeginTrans(p->pSrc, 0); + bCloseTrans = 1; + } + + /* Now that there is a read-lock on the source database, query the + ** source pager for the number of pages in the database. + */ + if( rc==SQLITE_OK ){ + rc = sqlite3PagerPagecount(pSrcPager, &nSrcPage); + } + for(ii=0; iiiNext<=nSrcPage && rc==SQLITE_OK; ii++){ + const Pgno iSrcPg = p->iNext; /* Source page number */ + if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ + DbPage *pSrcPg; /* Source page object */ + rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + if( rc==SQLITE_OK ){ + rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg)); + sqlite3PagerUnref(pSrcPg); + } + } + p->iNext++; + } + if( rc==SQLITE_OK ){ + p->nPagecount = nSrcPage; + p->nRemaining = nSrcPage+1-p->iNext; + if( p->iNext>nSrcPage ){ + rc = SQLITE_DONE; + } + } + + if( rc==SQLITE_DONE ){ + const int nSrcPagesize = sqlite3BtreeGetPageSize(p->pSrc); + const int nDestPagesize = sqlite3BtreeGetPageSize(p->pDest); + int nDestTruncate; + sqlite3_file *pFile = 0; + i64 iSize; + + /* Update the schema version field in the destination database. This + ** is to make sure that the schema-version really does change in + ** the case where the source and destination databases have the + ** same schema version. + */ + sqlite3BtreeUpdateMeta(p->pDest, 1, p->iDestSchema+1); + + /* Set nDestTruncate to the final number of pages in the destination + ** database. The complication here is that the destination page + ** size may be different to the source page size. + ** + ** If the source page size is smaller than the destination page size, + ** round up. In this case the call to sqlite3OsTruncate() below will + ** fix the size of the file. However it is important to call + ** sqlite3PagerTruncateImage() here so that any pages in the + ** destination file that lie beyond the nDestTruncate page mark are + ** journalled by PagerCommitPhaseOne() before they are destroyed + ** by the file truncation. + */ + if( nSrcPagesizepDest)) + ){ + rc = SQLITE_DONE; + } + } + + /* If bCloseTrans is true, then this function opened a read transaction + ** on the source database. Close the read transaction here. There is + ** no need to check the return values of the btree methods here, as + ** "committing" a read-only transaction cannot fail. + */ + if( bCloseTrans ){ + TESTONLY( int rc2 ); + TESTONLY( rc2 = ) sqlite3BtreeCommitPhaseOne(p->pSrc, 0); + TESTONLY( rc2 |= ) sqlite3BtreeCommitPhaseTwo(p->pSrc); + assert( rc2==SQLITE_OK ); + } + + if( rc!=SQLITE_LOCKED && rc!=SQLITE_BUSY ){ + p->rc = rc; + } + } + sqlite3BtreeLeave(p->pSrc); + sqlite3_mutex_leave(p->pSrcDb->mutex); + return rc; +} + +/* +** Release all resources associated with an sqlite3_backup* handle. +*/ +int sqlite3_backup_finish(sqlite3_backup *p){ + sqlite3_backup **pp; /* Ptr to head of pagers backup list */ + sqlite3_mutex *mutex; /* Mutex to protect source database */ + int rc; /* Value to return */ + + /* Enter the mutexes */ + sqlite3_mutex_enter(p->pSrcDb->mutex); + sqlite3BtreeEnter(p->pSrc); + mutex = p->pSrcDb->mutex; + + /* Detach this backup from the source pager. */ + if( p->pDestDb ){ + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + while( *pp!=p ){ + pp = &(*pp)->pNext; + } + *pp = p->pNext; + p->pSrc->nBackup--; + } + + /* If a transaction is still open on the Btree, roll it back. */ + sqlite3BtreeRollback(p->pDest); + + /* Set the error code of the destination database handle. */ + rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; + sqlite3Error(p->pDestDb, rc, 0); + + /* Exit the mutexes and free the backup context structure. */ + sqlite3BtreeLeave(p->pSrc); + if( p->pDestDb ){ + sqlite3_free(p); + } + sqlite3_mutex_leave(mutex); + return rc; +} + +/* +** Return the number of pages still to be backed up as of the most recent +** call to sqlite3_backup_step(). +*/ +int sqlite3_backup_remaining(sqlite3_backup *p){ + return p->nRemaining; +} + +/* +** Return the total number of pages in the source database as of the most +** recent call to sqlite3_backup_step(). +*/ +int sqlite3_backup_pagecount(sqlite3_backup *p){ + return p->nPagecount; +} + +/* +** This function is called after the contents of page iPage of the +** source database have been modified. If page iPage has already been +** copied into the destination database, then the data written to the +** destination is now invalidated. The destination copy of iPage needs +** to be updated with the new data before the backup operation is +** complete. +** +** It is assumed that the mutex associated with the BtShared object +** corresponding to the source database is held when this function is +** called. +*/ +void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ + sqlite3_backup *p; /* Iterator variable */ + for(p=pBackup; p; p=p->pNext){ + assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); + if( p->rc==SQLITE_OK && iPageiNext ){ + /* The backup process p has already copied page iPage. But now it + ** has been modified by a transaction on the source pager. Copy + ** the new data into the backup. + */ + int rc = backupOnePage(p, iPage, aData); + if( rc!=SQLITE_OK ){ + p->rc = rc; + } + } + } +} + +/* +** Restart the backup process. This is called when the pager layer +** detects that the database has been modified by an external database +** connection. In this case there is no way of knowing which of the +** pages that have been copied into the destination database are still +** valid and which are not, so the entire process needs to be restarted. +** +** It is assumed that the mutex associated with the BtShared object +** corresponding to the source database is held when this function is +** called. +*/ +void sqlite3BackupRestart(sqlite3_backup *pBackup){ + sqlite3_backup *p; /* Iterator variable */ + for(p=pBackup; p; p=p->pNext){ + assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); + p->iNext = 1; + } +} + +#ifndef SQLITE_OMIT_VACUUM +/* +** Copy the complete content of pBtFrom into pBtTo. A transaction +** must be active for both files. +** +** The size of file pTo may be reduced by this operation. If anything +** goes wrong, the transaction on pTo is rolled back. If successful, the +** transaction is committed before returning. +*/ +int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ + int rc; + sqlite3_backup b; + sqlite3BtreeEnter(pTo); + sqlite3BtreeEnter(pFrom); + + /* Set up an sqlite3_backup object. sqlite3_backup.pDestDb must be set + ** to 0. This is used by the implementations of sqlite3_backup_step() + ** and sqlite3_backup_finish() to detect that they are being called + ** from this function, not directly by the user. + */ + memset(&b, 0, sizeof(b)); + b.pSrcDb = pFrom->db; + b.pSrc = pFrom; + b.pDest = pTo; + b.iNext = 1; + + /* 0x7FFFFFFF is the hard limit for the number of pages in a database + ** file. By passing this as the number of pages to copy to + ** sqlite3_backup_step(), we can guarantee that the copy finishes + ** within a single call (unless an error occurs). The assert() statement + ** checks this assumption - (p->rc) should be set to either SQLITE_DONE + ** or an error code. + */ + sqlite3_backup_step(&b, 0x7FFFFFFF); + assert( b.rc!=SQLITE_OK ); + rc = sqlite3_backup_finish(&b); + if( rc==SQLITE_OK ){ + pTo->pBt->pageSizeFixed = 0; + } + + sqlite3BtreeLeave(pFrom); + sqlite3BtreeLeave(pTo); + return rc; +} +#endif /* SQLITE_OMIT_VACUUM */ + diff --git a/src/btree.c b/src/btree.c index d48f20880d..b4a70bd971 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.563 2009/01/31 14:54:07 danielk1977 Exp $ +** $Id: btree.c,v 1.564 2009/02/03 16:51:25 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** See the header comment on "btreeInt.h" for additional information. @@ -7239,225 +7239,6 @@ const char *sqlite3BtreeGetJournalname(Btree *p){ return sqlite3PagerJournalname(p->pBt->pPager); } -#ifndef SQLITE_OMIT_VACUUM -/* -** Copy the complete content of pBtFrom into pBtTo. A transaction -** must be active for both files. -** -** The size of file pTo may be reduced by this operation. -** If anything goes wrong, the transaction on pTo is rolled back. -** -** If successful, CommitPhaseOne() may be called on pTo before returning. -** The caller should finish committing the transaction on pTo by calling -** sqlite3BtreeCommit(). -*/ -static int btreeCopyFile(Btree *pTo, Btree *pFrom){ - int rc = SQLITE_OK; - Pgno i; - - Pgno nFromPage; /* Number of pages in pFrom */ - Pgno nToPage; /* Number of pages in pTo */ - Pgno nNewPage; /* Number of pages in pTo after the copy */ - - Pgno iSkip; /* Pending byte page in pTo */ - int nToPageSize; /* Page size of pTo in bytes */ - int nFromPageSize; /* Page size of pFrom in bytes */ - - BtShared *pBtTo = pTo->pBt; - BtShared *pBtFrom = pFrom->pBt; - pBtTo->db = pTo->db; - pBtFrom->db = pFrom->db; - - nToPageSize = pBtTo->pageSize; - nFromPageSize = pBtFrom->pageSize; - - assert( pTo->inTrans==TRANS_WRITE ); - assert( pFrom->inTrans==TRANS_WRITE ); - if( NEVER(pBtTo->pCursor) ){ - return SQLITE_BUSY; - } - - nToPage = pagerPagecount(pBtTo); - nFromPage = pagerPagecount(pBtFrom); - iSkip = PENDING_BYTE_PAGE(pBtTo); - - /* Variable nNewPage is the number of pages required to store the - ** contents of pFrom using the current page-size of pTo. - */ - nNewPage = (Pgno) - (((i64)nFromPage*(i64)nFromPageSize+(i64)nToPageSize-1)/(i64)nToPageSize); - - for(i=1; rc==SQLITE_OK && (i<=nToPage || i<=nNewPage); i++){ - - /* Journal the original page. - ** - ** iSkip is the page number of the locking page (PENDING_BYTE_PAGE) - ** in database *pTo (before the copy). This page is never written - ** into the journal file. Unless i==iSkip or the page was not - ** present in pTo before the copy operation, journal page i from pTo. - */ - if( i!=iSkip && i<=nToPage ){ - DbPage *pDbPage = 0; - rc = sqlite3PagerGet(pBtTo->pPager, i, &pDbPage); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pDbPage); - if( rc==SQLITE_OK && i>nFromPage ){ - /* Yeah. It seems wierd to call DontWrite() right after Write(). But - ** that is because the names of those procedures do not exactly - ** represent what they do. Write() really means "put this page in the - ** rollback journal and mark it as dirty so that it will be written - ** to the database file later." DontWrite() undoes the second part of - ** that and prevents the page from being written to the database. The - ** page is still on the rollback journal, though. And that is the - ** whole point of this block: to put pages on the rollback journal. - */ - sqlite3PagerDontWrite(pDbPage); - } - sqlite3PagerUnref(pDbPage); - } - } - - /* Overwrite the data in page i of the target database */ - if( rc==SQLITE_OK && i!=iSkip && i<=nNewPage ){ - - DbPage *pToPage = 0; - sqlite3_int64 iOff; - - rc = sqlite3PagerGet(pBtTo->pPager, i, &pToPage); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pToPage); - } - - for( - iOff=(i-1)*nToPageSize; - rc==SQLITE_OK && iOffpPager, iFrom, &pFromPage); - if( rc==SQLITE_OK ){ - char *zTo = sqlite3PagerGetData(pToPage); - char *zFrom = sqlite3PagerGetData(pFromPage); - int nCopy; - - if( nFromPageSize>=nToPageSize ){ - zFrom += ((i-1)*nToPageSize - ((iFrom-1)*nFromPageSize)); - nCopy = nToPageSize; - }else{ - zTo += (((iFrom-1)*nFromPageSize) - (i-1)*nToPageSize); - nCopy = nFromPageSize; - } - - memcpy(zTo, zFrom, nCopy); - sqlite3PagerUnref(pFromPage); - } - } - - if( pToPage ){ - MemPage *p = (MemPage *)sqlite3PagerGetExtra(pToPage); - p->isInit = 0; - sqlite3PagerUnref(pToPage); - } - } - } - - /* If things have worked so far, the database file may need to be - ** truncated. The complex part is that it may need to be truncated to - ** a size that is not an integer multiple of nToPageSize - the current - ** page size used by the pager associated with B-Tree pTo. - ** - ** For example, say the page-size of pTo is 2048 bytes and the original - ** number of pages is 5 (10 KB file). If pFrom has a page size of 1024 - ** bytes and 9 pages, then the file needs to be truncated to 9KB. - */ - if( rc==SQLITE_OK ){ - sqlite3_file *pFile = sqlite3PagerFile(pBtTo->pPager); - i64 iSize = (i64)nFromPageSize * (i64)nFromPage; - i64 iNow = (i64)((nToPage>nNewPage)?nToPage:nNewPage) * (i64)nToPageSize; - i64 iPending = ((i64)PENDING_BYTE_PAGE(pBtTo)-1) *(i64)nToPageSize; - - assert( iSize<=iNow ); - - /* Commit phase one syncs the journal file associated with pTo - ** containing the original data. It does not sync the database file - ** itself. After doing this it is safe to use OsTruncate() and other - ** file APIs on the database file directly. - */ - pBtTo->db = pTo->db; - if( nFromPageSize==nToPageSize ){ - sqlite3PagerTruncateImage(pBtTo->pPager, nFromPage); - iNow = iSize; - } - rc = sqlite3PagerCommitPhaseOne(pBtTo->pPager, 0, 1); - if( iSizeiPending){ - i64 iOff; - for( - iOff=iPending; - rc==SQLITE_OK && iOff<(iPending+nToPageSize); - iOff += nFromPageSize - ){ - DbPage *pFromPage = 0; - Pgno iFrom = (Pgno)(iOff/nFromPageSize)+1; - - if( iFrom==PENDING_BYTE_PAGE(pBtFrom) || iFrom>nFromPage ){ - continue; - } - - rc = sqlite3PagerGet(pBtFrom->pPager, iFrom, &pFromPage); - if( rc==SQLITE_OK ){ - char *zFrom = sqlite3PagerGetData(pFromPage); - rc = sqlite3OsWrite(pFile, zFrom, nFromPageSize, iOff); - sqlite3PagerUnref(pFromPage); - } - } - } - } - - /* Sync the database file */ - if( rc==SQLITE_OK ){ - rc = sqlite3PagerSync(pBtTo->pPager); - } - if( rc==SQLITE_OK ){ - pBtTo->pageSizeFixed = 0; - }else{ - sqlite3BtreeRollback(pTo); - } - - return rc; -} -int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ - int rc; - sqlite3BtreeEnter(pTo); - sqlite3BtreeEnter(pFrom); - rc = btreeCopyFile(pTo, pFrom); - sqlite3BtreeLeave(pFrom); - sqlite3BtreeLeave(pTo); - return rc; -} - -#endif /* SQLITE_OMIT_VACUUM */ - /* ** Return non-zero if a transaction is active. */ @@ -7483,6 +7264,12 @@ int sqlite3BtreeIsInReadTrans(Btree *p){ return p->inTrans!=TRANS_NONE; } +int sqlite3BtreeIsInBackup(Btree *p){ + assert( p ); + assert( sqlite3_mutex_held(p->db->mutex) ); + return p->nBackup!=0; +} + /* ** This function returns a pointer to a blob of memory associated with ** a single shared-btree. The memory is used by client code for its own diff --git a/src/btree.h b/src/btree.h index 49744583b6..dff9fe1be4 100644 --- a/src/btree.h +++ b/src/btree.h @@ -13,7 +13,7 @@ ** subsystem. See comments in the source code for a detailed description ** of what each interface routine does. ** -** @(#) $Id: btree.h,v 1.107 2009/01/24 11:30:43 drh Exp $ +** @(#) $Id: btree.h,v 1.108 2009/02/03 16:51:25 danielk1977 Exp $ */ #ifndef _BTREE_H_ #define _BTREE_H_ @@ -98,6 +98,7 @@ int sqlite3BtreeCreateTable(Btree*, int*, int flags); int sqlite3BtreeIsInTrans(Btree*); int sqlite3BtreeIsInStmt(Btree*); int sqlite3BtreeIsInReadTrans(Btree*); +int sqlite3BtreeIsInBackup(Btree*); void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); int sqlite3BtreeSchemaLocked(Btree *); int sqlite3BtreeLockTable(Btree *, int, u8); diff --git a/src/btreeInt.h b/src/btreeInt.h index 2a97f9ee41..32aec96151 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btreeInt.h,v 1.41 2009/01/20 17:06:27 danielk1977 Exp $ +** $Id: btreeInt.h,v 1.42 2009/02/03 16:51:25 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -324,6 +324,7 @@ struct Btree { u8 sharable; /* True if we can share pBt with another db */ u8 locked; /* True if db currently has pBt locked */ int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ + int nBackup; /* Number of backup operations reading this btree */ Btree *pNext; /* List of other sharable Btrees from the same db */ Btree *pPrev; /* Back pointer of the same list */ }; diff --git a/src/build.c b/src/build.c index 300546f0ef..b54bfe06ce 100644 --- a/src/build.c +++ b/src/build.c @@ -22,7 +22,7 @@ ** COMMIT ** ROLLBACK ** -** $Id: build.c,v 1.514 2009/02/03 15:50:34 drh Exp $ +** $Id: build.c,v 1.515 2009/02/03 16:51:25 danielk1977 Exp $ */ #include "sqliteInt.h" @@ -621,31 +621,41 @@ void sqlite3OpenMasterTable(Parse *p, int iDb){ } /* -** The token *pName contains the name of a database (either "main" or -** "temp" or the name of an attached db). This routine returns the -** index of the named database in db->aDb[], or -1 if the named db -** does not exist. +** Parameter zName points to a nul-terminated buffer containing the name +** of a database ("main", "temp" or the name of an attached db). This +** function returns the index of the named database in db->aDb[], or +** -1 if the named db cannot be found. */ -int sqlite3FindDb(sqlite3 *db, Token *pName){ - int i = -1; /* Database number */ - int n; /* Number of characters in the name */ - Db *pDb; /* A database whose name space is being searched */ - char *zName; /* Name we are searching for */ - - zName = sqlite3NameFromToken(db, pName); +int sqlite3FindDbName(sqlite3 *db, const char *zName){ + int i = -1; /* Database number */ if( zName ){ - n = sqlite3Strlen30(zName); + Db *pDb; + int n = sqlite3Strlen30(zName); for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite3Strlen30(pDb->zName) && 0==sqlite3StrICmp(pDb->zName, zName) ){ break; } } - sqlite3DbFree(db, zName); } return i; } +/* +** The token *pName contains the name of a database (either "main" or +** "temp" or the name of an attached db). This routine returns the +** index of the named database in db->aDb[], or -1 if the named db +** does not exist. +*/ +int sqlite3FindDb(sqlite3 *db, Token *pName){ + int i; /* Database number */ + char *zName; /* Name we are searching for */ + zName = sqlite3NameFromToken(db, pName); + i = sqlite3FindDbName(db, zName); + sqlite3DbFree(db, zName); + return i; +} + /* The table or view or trigger name is passed to this routine via tokens ** pName1 and pName2. If the table name was fully qualified, for example: ** diff --git a/src/main.c b/src/main.c index f104e41ec1..58f5443ca3 100644 --- a/src/main.c +++ b/src/main.c @@ -14,7 +14,7 @@ ** other files are for internal use by SQLite and should not be ** accessed by users of the library. ** -** $Id: main.c,v 1.524 2009/02/03 15:50:34 drh Exp $ +** $Id: main.c,v 1.525 2009/02/03 16:51:25 danielk1977 Exp $ */ #include "sqliteInt.h" @@ -603,6 +603,16 @@ int sqlite3_close(sqlite3 *db){ } assert( sqlite3SafetyCheckSickOrOk(db) ); + for(j=0; jnDb; j++){ + Btree *pBt = db->aDb[j].pBt; + if( pBt && sqlite3BtreeIsInBackup(pBt) ){ + sqlite3Error(db, SQLITE_BUSY, + "Unable to close due to unfinished backup operation"); + sqlite3_mutex_leave(db->mutex); + return SQLITE_BUSY; + } + } + /* Free any outstanding Savepoint structures. */ sqlite3CloseSavepoints(db); diff --git a/src/pager.c b/src/pager.c index 820e8bcabb..10c09e7499 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.561 2009/01/31 14:54:07 danielk1977 Exp $ +** @(#) $Id: pager.c,v 1.562 2009/02/03 16:51:25 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" @@ -306,6 +306,7 @@ struct Pager { char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ i64 journalSizeLimit; /* Size limit for persistent journal files */ PCache *pPCache; /* Pointer to page cache object */ + sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ }; /* @@ -1039,9 +1040,12 @@ static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ /* ** Unless the pager is in error-state, discard all in-memory pages. If ** the pager is in error-state, then this call is a no-op. +** +** TODO: Why can we not reset the pager while in error state? */ static void pager_reset(Pager *pPager){ if( SQLITE_OK==pPager->errCode ){ + sqlite3BackupRestart(pPager->pBackup); sqlite3PcacheClear(pPager->pPCache); } } @@ -1513,6 +1517,7 @@ static int pager_playback_one_page( if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } + sqlite3BackupUpdate(pPager->pBackup, pgno, aData); }else if( !isMainJrnl && pPg==0 ){ /* If this is a rollback of a savepoint and data was not written to ** the database and the page is not in-memory, there is a potential @@ -2875,6 +2880,9 @@ static int pager_write_pagelist(PgHdr *pList){ pPager->dbFileSize = pgno; } + /* Update any backup objects copying the contents of this pager. */ + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8 *)pData); + PAGERTRACE(("STORE %d page %d hash(%08x)\n", PAGERID(pPager), pgno, pager_pagehash(pList))); IOTRACE(("PGOUT %p %d\n", pPager, pgno)); @@ -3549,7 +3557,7 @@ static int pagerSharedLock(Pager *pPager){ ** playing back the hot-journal so that we don't end up with ** an inconsistent cache. */ - sqlite3PcacheClear(pPager->pPCache); + pager_reset(pPager); rc = pager_playback(pPager, 1); if( rc!=SQLITE_OK ){ rc = pager_error(pPager, rc); @@ -4436,7 +4444,9 @@ int sqlite3PagerCommitPhaseOne( /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is a no-op. */ - if( pPager->state!=PAGER_SYNCED && !MEMDB && pPager->dbModified ){ + if( MEMDB && pPager->dbModified ){ + sqlite3BackupRestart(pPager->pBackup); + }else if( pPager->state!=PAGER_SYNCED && pPager->dbModified ){ /* The following block updates the change-counter. Exactly how it ** does this depends on whether or not the atomic-update optimization @@ -4747,10 +4757,14 @@ int *sqlite3PagerStats(Pager *pPager){ a[10] = pPager->nWrite; return a; } +#endif + +/* +** Return true if this is an in-memory pager. +*/ int sqlite3PagerIsMemdb(Pager *pPager){ return MEMDB; } -#endif /* ** Check that there are at least nSavepoint savepoints open. If there are @@ -5152,4 +5166,14 @@ i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ return pPager->journalSizeLimit; } +/* +** Return a pointer to the pPager->pBackup variable. The backup module +** in backup.c maintains the content of this variable. This module +** uses it opaquely as an argument to sqlite3BackupRestart() and +** sqlite3BackupUpdate() only. +*/ +sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ + return &pPager->pBackup; +} + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index 3d6e4bb85e..011f91c403 100644 --- a/src/pager.h +++ b/src/pager.h @@ -13,7 +13,7 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.99 2009/01/31 14:54:07 danielk1977 Exp $ +** @(#) $Id: pager.h,v 1.100 2009/02/03 16:51:25 danielk1977 Exp $ */ #ifndef _PAGER_H_ @@ -100,6 +100,7 @@ void sqlite3PagerSetSafetyLevel(Pager*,int,int); int sqlite3PagerLockingMode(Pager *, int); int sqlite3PagerJournalMode(Pager *, int); i64 sqlite3PagerJournalSizeLimit(Pager *, i64); +sqlite3_backup **sqlite3PagerBackupPtr(Pager*); /* Functions used to obtain and release page references. */ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); @@ -135,6 +136,7 @@ sqlite3_file *sqlite3PagerFile(Pager*); const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); +int sqlite3PagerIsMemdb(Pager*); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); @@ -152,7 +154,6 @@ void sqlite3PagerTruncateImage(Pager*,Pgno); #ifdef SQLITE_TEST int *sqlite3PagerStats(Pager*); void sqlite3PagerRefdump(Pager*); - int sqlite3PagerIsMemdb(Pager*); void disable_simulated_io_errors(void); void enable_simulated_io_errors(void); #else diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 950b09f8de..0d23bf213a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -30,7 +30,7 @@ ** the version number) and changes its name to "sqlite3.h" as ** part of the build process. ** -** @(#) $Id: sqlite.h.in,v 1.422 2009/01/23 16:45:01 danielk1977 Exp $ +** @(#) $Id: sqlite.h.in,v 1.423 2009/02/03 16:51:25 danielk1977 Exp $ */ #ifndef _SQLITE3_H_ #define _SQLITE3_H_ @@ -6722,6 +6722,178 @@ struct sqlite3_pcache_methods { void (*xDestroy)(sqlite3_pcache*); }; +/* +** CAPI3REF: Online Backup API. +** EXPERIMENTAL +** +** This API is used to overwrite the contents of one database with that +** of another. It is useful either for creating backups of databases or +** for copying in-memory databases to or from persistent files. +** +** Exclusive access is required to the destination database for the +** duration of the operation. However the source database is only +** read-locked while it is actually being read, it is not locked +** continuously for the entire operation. Thus, the backup may be +** performed on a live database without preventing other users from +** writing to the database for an extended period of time. +** +** To perform a backup operation: +**
    +**
  1. [sqlite3_backup_init()] is called once to initialize the backup, +**
  2. [sqlite3_backup_step()] is called one or more times to transfer +** the data between the two databases, and finally +**
  3. [sqlite3_backup_finish()] is called to release all resources +** associated with the backup operation. +**
+** There should be exactly one call to sqlite3_backup_finish() for each +** successful call to sqlite3_backup_init(). +** +** sqlite3_backup_init() +** +** The first two arguments passed to [sqlite3_backup_init()] are the database +** handle associated with the destination database and the database name +** used to attach the destination database to the handle. The database name +** is "main" for the main database, "temp" for the temporary database, or +** the name specified as part of the ATTACH statement if the destination is +** an attached database. The third and fourth arguments passed to +** sqlite3_backup_init() identify the database handle and database name used +** to access the source database. The values passed for the source and +** destination database handle parameters must not be the same. +** +** If an error occurs within sqlite3_backup_init(), then NULL is returned +** and an error code and error message written into the database handle +** passed as the first argument. They may be retrieved using the +** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16() functions. +** Otherwise, if successful, an opaque handle of type sqlite3_backup* is +** returned. This handle may be used with the sqlite3_backup_step() and +** sqlite3_backup_finish() functions to perform the specified backup +** operation. +** +** sqlite3_backup_step() +** +** Function [sqlite3_backup_step()] is used to copy up to nPage pages between +** the source and destination databases, where nPage is the value of the +** second parameter passed to sqlite3_backup_step(). If nPage pages are +** succesfully copied, but there are still more pages to copy before the +** backup is complete, it returns SQLITE_OK. If no error occured and there +** are no more pages to copy, then SQLITE_DONE is returned. If an error +** occurs, then an SQLite error code is returned. As well as SQLITE_OK and +** SQLITE_DONE, a call to sqlite3_backup_step() may return SQLITE_READONLY, +** SQLITE_NOMEM, SQLITE_BUSY, SQLITE_LOCKED or an SQLITE_IOERR_XXX error code. +** +** As well as the case where the destination database file was opened for +** read-only access, sqlite3_backup_step() may return SQLITE_READONLY if +** the destination is an in-memory database with a different page size +** from the source database. +** +** If sqlite3_backup_step() cannot obtain a required file-system lock, then +** the busy-handler function is invoked (if one is specified). If the +** busy-handler returns non-zero before the lock is available, then +** SQLITE_BUSY is returned to the caller. In this case the call to +** sqlite3_backup_step() can be retried later. If the source database handle +** is being used to write to the source database when sqlite3_backup_step() +** is called, then SQLITE_LOCKED is returned immediately. Again, in this +** case the call to sqlite3_backup_step() can be retried later on. If +** SQLITE_IOERR_XXX, SQLITE_NOMEM or SQLITE_READONLY is returned, then +** there is no point in retrying the call to sqlite3_backup_step(). These +** errors are considered fatal. At this point the application must accept +** that the backup operation has failed and pass the backup operation handle +** to the sqlite3_backup_finish() to release associated resources. +** +** Following the first call to sqlite3_backup_step(), an exclusive lock is +** obtained on the destination file. It is not released until either +** sqlite3_backup_finish() is called or the backup operation is complete +** and sqlite3_backup_step() returns SQLITE_DONE. Additionally, each time +** a call to sqlite3_backup_step() is made a shared lock is obtained on +** the source database file. This lock is released before the +** sqlite3_backup_step() call returns. Because the source database is not +** locked between calls to sqlite3_backup_step(), it may be modified mid-way +** through the backup procedure. If the source database is modified by an +** external process or via a database connection other than the one being +** used by the backup operation, then the backup will be transparently +** restarted by the next call to sqlite3_backup_step(). If the source +** database is modified by the using the same database connection as is used +** by the backup operation, then the backup database is transparently +** updated at the same time. +** +** sqlite3_backup_finish() +** +** Once sqlite3_backup_step() has returned SQLITE_DONE, or when the +** application wishes to abandon the backup operation, the sqlite3_backup* +** handle should be passed to sqlite3_backup_finish(). This releases all +** resources associated with the backup operation. If sqlite3_backup_step() +** has not yet returned SQLITE_DONE, then any active write-transaction on the +** destination database is rolled back. The sqlite3_backup* handle is invalid +** and may not be used following a call to sqlite3_backup_finish(). +** +** The value returned by sqlite3_backup_finish is SQLITE_OK if no error +** occured, regardless or whether or not sqlite3_backup_step() was called +** a sufficient number of times to complete the backup operation. Or, if +** an out-of-memory condition or IO error occured during a call to +** sqlite3_backup_step() then SQLITE_NOMEM or an SQLITE_IOERR_XXX error code +** is returned. In this case the error code and an error message are +** written to the destination database handle. +** +** A return of SQLITE_BUSY or SQLITE_LOCKED from sqlite3_backup_step() is +** not considered an error and does not affect the return value of +** sqlite3_backup_finish(). +** +** sqlite3_backup_remaining(), sqlite3_backup_pagecount() +** +** Each call to sqlite3_backup_step() sets two values stored internally +** by an sqlite3_backup* handle. The number of pages still to be backed +** up, which may be queried by sqlite3_backup_remaining(), and the total +** number of pages in the source database file, which may be queried by +** sqlite3_backup_pagecount(). +** +** The values returned by these functions are only updated by +** sqlite3_backup_step(). If the source database is modified during a backup +** operation, then the values are not updated to account for any extra +** pages that need to be updated or the size of the source database file +** changing. +** +** Concurrent Usage of Database Handles +** +** The source database handle may be used by the application for other +** purposes while a backup operation is underway or being initialized. +** If SQLite is compiled and configured to support threadsafe database +** connections, then the source database connection may be used concurrently +** from within other threads. +** +** However, the application must guarantee that the destination database +** connection handle is not passed to any other API (by any thread) after +** sqlite3_backup_init() is called and before the corresponding call to +** sqlite3_backup_finish(). Unfortunately SQLite does not currently check +** for this, if the application does use the destination database handle +** for some other purpose during a backup operation, things may appear to +** work correctly but in fact be subtly malfunctioning. +** +** Furthermore, if running in shared cache mode, the application must +** guarantee that the shared cache used by the destination database +** is not accessed while the backup is running. In practice this means +** that the application must guarantee that the file-system file being +** backed up to is not accessed by any connection within the process, +** not just the specific connection that was passed to sqlite3_backup_init(). +** +** The sqlite3_backup handle itself is partially threadsafe. Multiple +** threads may safely make multiple concurrent calls to sqlite3_backup_step(). +** However, the sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** APIs are not strictly speaking threadsafe. If they are invoked at the +** same time as another thread is invoking sqlite3_backup_step() it is +** possible that they return invalid values. +*/ +typedef struct sqlite3_backup sqlite3_backup; +sqlite3_backup *sqlite3_backup_init( + sqlite3 *pDest, /* Destination database handle */ + const char *zDestName, /* Destination database name */ + sqlite3 *pSource, /* Source database handle */ + const char *zSourceName /* Source database name */ +); +int sqlite3_backup_step(sqlite3_backup *p, int nPage); +int sqlite3_backup_finish(sqlite3_backup *p); +int sqlite3_backup_remaining(sqlite3_backup *p); +int sqlite3_backup_pagecount(sqlite3_backup *p); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 287d2c95f1..8db2e69e32 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.830 2009/01/24 11:30:43 drh Exp $ +** @(#) $Id: sqliteInt.h,v 1.831 2009/02/03 16:51:25 danielk1977 Exp $ */ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -2552,6 +2552,7 @@ char sqlite3AffinityType(const Token*); void sqlite3Analyze(Parse*, Token*, Token*); int sqlite3InvokeBusyHandler(BusyHandler*); int sqlite3FindDb(sqlite3*, Token*); +int sqlite3FindDbName(sqlite3 *, const char *); int sqlite3AnalysisLoad(sqlite3*,int iDB); void sqlite3DefaultRowEst(Index*); void sqlite3RegisterLikeFunctions(sqlite3*, int); @@ -2573,6 +2574,9 @@ char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumReset(StrAccum*); void sqlite3SelectDestInit(SelectDest*,int,int); +void sqlite3BackupRestart(sqlite3_backup *); +void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); + /* ** The interface to the LEMON-generated parser */ diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 0e35f41e41..1516728d55 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -12,7 +12,7 @@ ** A TCL Interface to SQLite. Append this file to sqlite3.c and ** compile the whole thing to build a TCL-enabled version of SQLite. ** -** $Id: tclsqlite.c,v 1.234 2009/01/14 23:38:03 drh Exp $ +** $Id: tclsqlite.c,v 1.235 2009/02/03 16:51:25 danielk1977 Exp $ */ #include "tcl.h" #include @@ -2663,6 +2663,7 @@ int TCLSH_MAIN(int argc, char **argv){ extern int SqlitetestThread_Init(Tcl_Interp*); extern int SqlitetestOnefile_Init(); extern int SqlitetestOsinst_Init(Tcl_Interp*); + extern int Sqlitetestbackup_Init(Tcl_Interp*); Md5_Init(interp); Sqliteconfig_Init(interp); @@ -2686,6 +2687,7 @@ int TCLSH_MAIN(int argc, char **argv){ SqlitetestThread_Init(interp); SqlitetestOnefile_Init(interp); SqlitetestOsinst_Init(interp); + Sqlitetestbackup_Init(interp); #ifdef SQLITE_SSE Sqlitetestsse_Init(interp); diff --git a/src/test1.c b/src/test1.c index eff89a7de6..326c67daf3 100644 --- a/src/test1.c +++ b/src/test1.c @@ -13,7 +13,7 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** -** $Id: test1.c,v 1.346 2009/02/03 16:25:48 drh Exp $ +** $Id: test1.c,v 1.347 2009/02/03 16:51:25 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -3217,7 +3217,7 @@ static int test_errcode( } /* -** Usage: test_errmsg DB +** Usage: sqlite3_errmsg DB ** ** Returns the UTF-8 representation of the error message string for the ** most recent sqlite3_* API call. diff --git a/src/test_backup.c b/src/test_backup.c new file mode 100644 index 0000000000..1a1418d1c2 --- /dev/null +++ b/src/test_backup.c @@ -0,0 +1,148 @@ +/* +** 2009 January 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** $Id: test_backup.c,v 1.1 2009/02/03 16:51:25 danielk1977 Exp $ +*/ + +#include "tcl.h" +#include +#include + +/* These functions are implemented in test1.c. */ +int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); +const char *sqlite3TestErrorName(int); + +static int backupTestCmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *const*objv +){ + enum BackupSubCommandEnum { + BACKUP_STEP, BACKUP_FINISH, BACKUP_REMAINING, BACKUP_PAGECOUNT + }; + struct BackupSubCommand { + const char *zCmd; + enum BackupSubCommandEnum eCmd; + int nArg; + const char *zArg; + } aSub[] = { + {"step", BACKUP_STEP , 1, "npage" }, + {"finish", BACKUP_FINISH , 0, "" }, + {"remaining", BACKUP_REMAINING , 0, "" }, + {"pagecount", BACKUP_PAGECOUNT , 0, "" }, + {0, 0, 0, 0} + }; + + sqlite3_backup *p = (sqlite3_backup *)clientData; + int iCmd; + int rc; + + rc = Tcl_GetIndexFromObjStruct( + interp, objv[1], aSub, sizeof(aSub[0]), "option", 0, &iCmd + ); + if( rc!=TCL_OK ){ + return rc; + } + if( objc!=(2 + aSub[iCmd].nArg) ){ + Tcl_WrongNumArgs(interp, 2, objv, aSub[iCmd].zArg); + return TCL_ERROR; + } + + switch( aSub[iCmd].eCmd ){ + + case BACKUP_FINISH: { + Tcl_CmdInfo cmdInfo; + Tcl_Command cmd = Tcl_GetCommandFromObj(interp, objv[0]); + Tcl_GetCommandInfoFromToken(cmd, &cmdInfo); + cmdInfo.deleteProc = 0; + Tcl_SetCommandInfoFromToken(cmd, &cmdInfo); + Tcl_DeleteCommandFromToken(interp, cmd); + + rc = sqlite3_backup_finish(p); + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + break; + } + + case BACKUP_STEP: { + int nPage; + if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &nPage) ){ + return TCL_ERROR; + } + rc = sqlite3_backup_step(p, nPage); + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + break; + } + + case BACKUP_REMAINING: + Tcl_SetObjResult(interp, Tcl_NewIntObj(sqlite3_backup_remaining(p))); + break; + + case BACKUP_PAGECOUNT: + Tcl_SetObjResult(interp, Tcl_NewIntObj(sqlite3_backup_pagecount(p))); + break; + } + + return TCL_OK; +} + +static void backupTestFinish(ClientData clientData){ + sqlite3_backup *pBackup = (sqlite3_backup *)clientData; + sqlite3_backup_finish(pBackup); +} + +/* +** sqlite3_backup CMDNAME DESTHANDLE DESTNAME SRCHANDLE SRCNAME +** +*/ +static int backupTestInit( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *const*objv +){ + sqlite3_backup *pBackup; + sqlite3 *pDestDb; + sqlite3 *pSrcDb; + const char *zDestName; + const char *zSrcName; + const char *zCmd; + + if( objc!=6 ){ + Tcl_WrongNumArgs( + interp, 1, objv, "CMDNAME DESTHANDLE DESTNAME SRCHANDLE SRCNAME" + ); + return TCL_ERROR; + } + + zCmd = Tcl_GetString(objv[1]); + getDbPointer(interp, Tcl_GetString(objv[2]), &pDestDb); + zDestName = Tcl_GetString(objv[3]); + getDbPointer(interp, Tcl_GetString(objv[4]), &pSrcDb); + zSrcName = Tcl_GetString(objv[5]); + + pBackup = sqlite3_backup_init(pDestDb, zDestName, pSrcDb, zSrcName); + if( !pBackup ){ + Tcl_AppendResult(interp, "sqlite3_backup_init() failed", 0); + return TCL_ERROR; + } + + Tcl_CreateObjCommand(interp, zCmd, backupTestCmd, pBackup, backupTestFinish); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + +int Sqlitetestbackup_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "sqlite3_backup", backupTestInit, 0, 0); + return TCL_OK; +} + diff --git a/src/vacuum.c b/src/vacuum.c index 2a16df43eb..c6227303d2 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -14,7 +14,7 @@ ** Most of the code in this file may be omitted by defining the ** SQLITE_OMIT_VACUUM macro. ** -** $Id: vacuum.c,v 1.85 2009/01/22 23:04:46 drh Exp $ +** $Id: vacuum.c,v 1.86 2009/02/03 16:51:25 danielk1977 Exp $ */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -265,7 +265,6 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ #ifndef SQLITE_OMIT_AUTOVACUUM sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp)); #endif - rc = sqlite3BtreeCommit(pMain); } if( rc==SQLITE_OK ){ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 54528ea38f..9bc5beff47 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -14,7 +14,7 @@ ** to version 2.8.7, all this code was combined into the vdbe.c source file. ** But that file was getting too big so this subroutines were split out. ** -** $Id: vdbeaux.c,v 1.434 2009/01/20 17:06:27 danielk1977 Exp $ +** $Id: vdbeaux.c,v 1.435 2009/02/03 16:51:25 danielk1977 Exp $ */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -1298,7 +1298,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt)) || nTrans<=1 ){ - for(i=0; rc==SQLITE_OK && inDb; i++){ + for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ rc = sqlite3BtreeCommitPhaseOne(pBt, 0); diff --git a/test/backup.test b/test/backup.test new file mode 100644 index 0000000000..92eadf3f18 --- /dev/null +++ b/test/backup.test @@ -0,0 +1,767 @@ +# 2008 January 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sqlite3_backup_XXX API. +# +# $Id: backup.test,v 1.1 2009/02/03 16:51:25 danielk1977 Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +#--------------------------------------------------------------------- +# Test organization: +# +# backup-1.*: Warm-body tests. +# +# backup-2.*: Test backup under various conditions. To and from in-memory +# databases. To and from empty/populated databases. etc. +# +# backup-3.*: Verify that the locking-page (pending byte page) is handled. +# +# backup-4.*: Test various error conditions. +# +# backup-5.*: Test the source database being modified during a backup. +# +# backup-6.*: Test the backup_remaining() and backup_pagecount() APIs. +# +# backup-7.*: Test SQLITE_BUSY and SQLITE_LOCKED errors. +# +# backup-8.*: Test multiple simultaneous backup operations. +# + +proc data_checksum {db file} { $db one "SELECT md5sum(a, b) FROM ${file}.t1" } +proc test_contents {name db1 file1 db2 file2} { + $db2 eval {select * from sqlite_master} + $db1 eval {select * from sqlite_master} + set checksum [data_checksum $db2 $file2] + uplevel [list do_test $name [list data_checksum $db1 $file1] $checksum] +} + +do_test backup-1.1 { + execsql { + BEGIN; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES(1, randstr(1000,1000)); + INSERT INTO t1 VALUES(2, randstr(1000,1000)); + INSERT INTO t1 VALUES(3, randstr(1000,1000)); + INSERT INTO t1 VALUES(4, randstr(1000,1000)); + INSERT INTO t1 VALUES(5, randstr(1000,1000)); + COMMIT; + } +} {} + +# Sanity check to verify that the [test_contents] proc works. +# +test_contents backup-1.2 db main db main + +# Check that it is possible to create and finish backup operations. +# +do_test backup-1.3.1 { + file delete test2.db + sqlite3 db2 test2.db + sqlite3_backup B db2 main db main +} {B} +do_test backup-1.3.2 { + B finish +} {SQLITE_OK} +do_test backup-1.3.3 { + info commands B +} {} + +# Simplest backup operation. Backup test.db to test2.db. test2.db is +# initially empty. test.db uses the default page size. +# +do_test backup-1.4.1 { + sqlite3_backup B db2 main db main +} {B} +do_test backup-1.4.2 { + B step 200 +} {SQLITE_DONE} +do_test backup-1.4.3 { + B finish +} {SQLITE_OK} +do_test backup-1.4.4 { + info commands B +} {} +test_contents backup-1.4.5 db2 main db main +db close +db2 close +# +# End of backup-1.* tests. +#--------------------------------------------------------------------- + + +#--------------------------------------------------------------------- +# The following tests, backup-2.*, are based on the following procedure: +# +# 1) Populate the source database. +# 2) Populate the destination database. +# 3) Run the backup to completion. (backup-2.*.1) +# 4) Integrity check the destination db. (backup-2.*.2) +# 5) Check that the contents of the destination db is the same as that +# of the source db. (backup-2.*.3) +# +# The test is run with all possible combinations of the following +# input parameters, except that if the destination is an in-memory +# database, the only page size tested is 1024 bytes (the same as the +# source page-size). +# +# * Source database is an in-memory database, OR +# * Source database is a file-backed database. +# +# * Target database is an in-memory database, OR +# * Target database is a file-backed database. +# +# * Destination database is a main file, OR +# * Destination database is an attached file, OR +# * Destination database is a temp database. +# +# * Target database is empty (zero bytes), OR +# * Target database is larger than the source, OR +# * Target database is smaller than the source. +# +# * Target database page-size is the same as the source, OR +# * Target database page-size is larger than the source, OR +# * Target database page-size is smaller than the source. +# +# * Each call to step copies a single page, OR +# * A single call to step copies the entire source database. +# +set iTest 1 +foreach zSrcFile {test.db :memory:} { +foreach zDestFile {test2.db :memory:} { +foreach zOpenScript [list { + sqlite3 db $zSrcFile + sqlite3 db2 $zSrcFile + db2 eval "ATTACH '$zDestFile' AS bak" + set db_dest db2 + set file_dest bak +} { + sqlite3 db $zSrcFile + sqlite3 db2 $zDestFile + set db_dest db2 + set file_dest main +} { + sqlite3 db $zSrcFile + sqlite3 db2 $zDestFile + set db_dest db2 + set file_dest temp +}] { +foreach rows_dest {0 3 10} { +foreach pgsz_dest {512 1024 2048} { +foreach nPagePerStep {1 200} { + + # Open the databases. + catch { file delete test.db } + catch { file delete test2.db } + eval $zOpenScript + + if {$zDestFile ne ":memory:" || $pgsz_dest == 1024 } { + + if 0 { + puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile" + puts -nonewline " (as $db_dest.$file_dest)" + puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest" + puts "" + } + + # Set up the content of the source database. + execsql { + BEGIN; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES(1, randstr(1000,1000)); + INSERT INTO t1 VALUES(2, randstr(1000,1000)); + INSERT INTO t1 VALUES(3, randstr(1000,1000)); + INSERT INTO t1 VALUES(4, randstr(1000,1000)); + INSERT INTO t1 VALUES(5, randstr(1000,1000)); + COMMIT; + } + + + + # Set up the content of the target database. + execsql "PRAGMA ${file_dest}.page_size = ${pgsz_dest}" $db_dest + if {$rows_dest != 0} { + execsql " + BEGIN; + CREATE TABLE ${file_dest}.t1(a, b); + CREATE INDEX ${file_dest}.i1 ON t1(a, b); + " $db_dest + for {set ii 0} {$ii < $rows_dest} {incr ii} { + execsql " + INSERT INTO ${file_dest}.t1 VALUES(1, randstr(1000,1000)) + " $db_dest + } + } + + # Backup the source database. + do_test backup-2.$iTest.1 { + sqlite3_backup B $db_dest $file_dest db main + while {[B step $nPagePerStep]=="SQLITE_OK"} {} + B finish + } {SQLITE_OK} + + # Run integrity check on the backup. + do_test backup-2.$iTest.2 { + execsql "PRAGMA ${file_dest}.integrity_check" $db_dest + } {ok} + + test_contents backup-2.$iTest.3 db main $db_dest $file_dest + + } + + db close + catch {db2 close} + incr iTest + +} } } } } } +# +# End of backup-2.* tests. +#--------------------------------------------------------------------- + +#--------------------------------------------------------------------- +# These tests, backup-3.*, ensure that nothing goes wrong if either +# the source or destination database are large enough to include the +# the locking-page (the page that contains the range of bytes that +# the locks are applied to). These tests assume that the pending +# byte is at offset 0x00010000 (64KB offset), as set by tester.tcl, +# not at the 1GB offset as it usually is. +# +# The test procedure is as follows (same procedure as used for +# the backup-2.* tests): +# +# 1) Populate the source database. +# 2) Populate the destination database. +# 3) Run the backup to completion. (backup-3.*.1) +# 4) Integrity check the destination db. (backup-3.*.2) +# 5) Check that the contents of the destination db is the same as that +# of the source db. (backup-3.*.3) +# +# The test procedure is run with the following parameters varied: +# +# * Source database includes pending-byte page. +# * Source database does not include pending-byte page. +# +# * Target database includes pending-byte page. +# * Target database does not include pending-byte page. +# +# * Target database page-size is the same as the source, OR +# * Target database page-size is larger than the source, OR +# * Target database page-size is smaller than the source. +# +set iTest 1 +foreach nSrcRow {10 100} { +foreach nDestRow {10 100} { +foreach nDestPgsz {512 1024 2048 4096} { + + catch { file delete test.db } + catch { file delete test2.db } + sqlite3 db test.db + sqlite3 db2 test2.db + + # Set up the content of the two databases. + # + foreach {db nRow} [list db $nSrcRow db2 $nDestRow] { + execsql { + BEGIN; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + } $db + for {set ii 0} {$ii < $nSrcRow} {incr ii} { + execsql { INSERT INTO t1 VALUES($ii, randstr(1000,1000)) } $db + } + execsql COMMIT $db + } + + # Backup the source database. + do_test backup-3.$iTest.1 { + sqlite3_backup B db main db2 main + while {[B step 10]=="SQLITE_OK"} {} + B finish + } {SQLITE_OK} + + # Run integrity check on the backup. + do_test backup-3.$iTest.2 { + execsql "PRAGMA integrity_check" db2 + } {ok} + + test_contents backup-3.$iTest.3 db main db2 main + + db close + db2 close + incr iTest +} +} +} +# +# End of backup-3.* tests. +#--------------------------------------------------------------------- + + +#--------------------------------------------------------------------- +# The following tests, backup-4.*, test various error conditions: +# +# backup-4.1.*: Test invalid database names. +# +# backup-4.2.*: Test that the source database cannot be detached while +# a backup is in progress. +# +# backup-4.3.*: Test that the source database handle cannot be closed +# while a backup is in progress. +# +# backup-4.4.*: Test an attempt to specify the same handle for the +# source and destination databases. +# +# backup-4.5.*: Test that an in-memory destination with a different +# page-size to the source database is an error. +# +sqlite3 db test.db +sqlite3 db2 test2.db + +do_test backup-4.1.1 { + catch { sqlite3_backup B db aux db2 main } +} {1} +do_test backup-4.1.2 { + sqlite3_errmsg db +} {unknown database aux} +do_test backup-4.1.3 { + catch { sqlite3_backup B db main db2 aux } +} {1} +do_test backup-4.1.4 { + sqlite3_errmsg db +} {unknown database aux} + +do_test backup-4.2.1 { + execsql { ATTACH 'test3.db' AS aux1 } + execsql { ATTACH 'test4.db' AS aux2 } db2 + sqlite3_backup B db aux1 db2 aux2 +} {B} +do_test backup-4.2.2 { + catchsql { DETACH aux2 } db2 +} {1 {database aux2 is locked}} +do_test backup-4.2.3 { + B step 50 +} {SQLITE_DONE} +do_test backup-4.2.4 { + B finish +} {SQLITE_OK} + +do_test backup-4.3.1 { + sqlite3_backup B db aux1 db2 aux2 +} {B} +do_test backup-4.3.2 { + db2 cache flush + sqlite3_close db2 +} {SQLITE_BUSY} +do_test backup-4.3.3 { + sqlite3_errmsg db2 +} {Unable to close due to unfinished backup operation} +do_test backup-4.3.4 { + B step 50 +} {SQLITE_DONE} +do_test backup-4.3.5 { + B finish +} {SQLITE_OK} + +do_test backup-4.4.1 { + set rc [catch {sqlite3_backup B db main db aux1}] + list $rc [sqlite3_errcode db] [sqlite3_errmsg db] +} {1 SQLITE_ERROR {Source and destination handles must be distinct}} +db close +db2 close + +do_test backup-4.5.1 { + catch { file delete -force test.db } + sqlite3 db test.db + sqlite3 db2 :memory: + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } + execsql { + PRAGMA page_size = 4096; + CREATE TABLE t2(a, b); + INSERT INTO t2 VALUES(3, 4); + } db2 + sqlite3_backup B db2 main db main +} {B} +do_test backup-4.5.2 { + B step 5000 +} {SQLITE_READONLY} +do_test backup-4.5.3 { + B finish +} {SQLITE_READONLY} + +db close +db2 close +# +# End of backup-5.* tests. +#--------------------------------------------------------------------- + +#--------------------------------------------------------------------- +# The following tests, backup-5.*, test that the backup works properly +# when the source database is modified during the backup. Test cases +# are organized as follows: +# +# backup-5.x.1.*: Nothing special. Modify the database mid-backup. +# +# backup-5.x.2.*: Modify the database mid-backup so that one or more +# pages are written out due to cache stress. Then +# rollback the transaction. +# +# backup-5.x.3.*: Database is vacuumed. +# +# backup-5.x.4.*: Database is vacuumed and the page-size modified. +# +# backup-5.x.5.*: Database is shrunk via incr-vacuum. +# +# Each test is run three times, in the following configurations: +# +# 1) Backing up file-to-file. The writer writes via an external pager. +# 2) Backing up file-to-file. The writer writes via the same pager as +# is used by the backup operation. +# 3) Backing up memory-to-file. +# +set iTest 0 +foreach {writer file} {db test.db db3 test.db db :memory:} { + incr iTest + catch { file delete bak.db } + sqlite3 db2 bak.db + catch { file delete $file } + sqlite3 db $file + sqlite3 db3 $file + + do_test backup-5.$iTest.1.1 { + execsql { + BEGIN; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES(1, randstr(1000,1000)); + INSERT INTO t1 VALUES(2, randstr(1000,1000)); + INSERT INTO t1 VALUES(3, randstr(1000,1000)); + INSERT INTO t1 VALUES(4, randstr(1000,1000)); + INSERT INTO t1 VALUES(5, randstr(1000,1000)); + COMMIT; + } + expr {[execsql {PRAGMA page_count}] > 10} + } {1} + do_test backup-5.$iTest.1.2 { + sqlite3_backup B db2 main db main + B step 5 + } {SQLITE_OK} + do_test backup-5.$iTest.1.3 { + execsql { UPDATE t1 SET a = a + 1 } $writer + B step 50 + } {SQLITE_DONE} + do_test backup-5.$iTest.1.4 { + B finish + } {SQLITE_OK} + integrity_check backup-5.$iTest.1.5 db2 + test_contents backup-5.$iTest.1.6 db main db2 main + + do_test backup-5.$iTest.2.1 { + execsql { + PRAGMA cache_size = 10; + BEGIN; + INSERT INTO t1 SELECT '', randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT '', randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT '', randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT '', randstr(1000,1000) FROM t1; + COMMIT; + } + } {} + do_test backup-5.$iTest.2.2 { + sqlite3_backup B db2 main db main + B step 50 + } {SQLITE_OK} + do_test backup-5.$iTest.2.3 { + execsql { + BEGIN; + UPDATE t1 SET a = a + 1; + ROLLBACK; + } $writer + B step 5000 + } {SQLITE_DONE} + do_test backup-5.$iTest.2.4 { + B finish + } {SQLITE_OK} + integrity_check backup-5.$iTest.2.5 db2 + test_contents backup-5.$iTest.2.6 db main db2 main + + do_test backup-5.$iTest.3.1 { + execsql { UPDATE t1 SET b = randstr(1000,1000) } + } {} + do_test backup-5.$iTest.3.2 { + sqlite3_backup B db2 main db main + B step 50 + } {SQLITE_OK} + do_test backup-5.$iTest.3.3 { + execsql { VACUUM } $writer + B step 5000 + } {SQLITE_DONE} + do_test backup-5.$iTest.3.4 { + B finish + } {SQLITE_OK} + integrity_check backup-5.$iTest.3.5 db2 + test_contents backup-5.$iTest.3.6 db main db2 main + + do_test backup-5.$iTest.4.1 { + execsql { UPDATE t1 SET b = randstr(1000,1000) } + } {} + do_test backup-5.$iTest.4.2 { + sqlite3_backup B db2 main db main + B step 50 + } {SQLITE_OK} + do_test backup-5.$iTest.4.3 { + execsql { + PRAGMA page_size = 2048; + VACUUM; + } $writer + B step 5000 + } {SQLITE_DONE} + do_test backup-5.$iTest.4.4 { + B finish + } {SQLITE_OK} + integrity_check backup-5.$iTest.4.5 db2 + test_contents backup-5.$iTest.4.6 db main db2 main + + catch { file delete bak.db } + sqlite3 db2 bak.db + catch { file delete $file } + sqlite3 db $file + sqlite3 db3 $file + do_test backup-5.$iTest.5.1 { + execsql { + PRAGMA auto_vacuum = incremental; + BEGIN; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES(1, randstr(1000,1000)); + INSERT INTO t1 VALUES(2, randstr(1000,1000)); + INSERT INTO t1 VALUES(3, randstr(1000,1000)); + INSERT INTO t1 VALUES(4, randstr(1000,1000)); + INSERT INTO t1 VALUES(5, randstr(1000,1000)); + COMMIT; + } + } {} + do_test backup-5.$iTest.5.2 { + sqlite3_backup B db2 main db main + B step 8 + } {SQLITE_OK} + do_test backup-5.$iTest.5.3 { + execsql { + DELETE FROM t1; + PRAGMA incremental_vacuum; + } $writer + B step 50 + } {SQLITE_DONE} + do_test backup-5.$iTest.5.4 { + B finish + } {SQLITE_OK} + integrity_check backup-5.$iTest.5.5 db2 + test_contents backup-5.$iTest.5.6 db main db2 main +} +catch {db close} +catch {db2 close} +catch {db3 close} +# +# End of backup-5.* tests. +#--------------------------------------------------------------------- + +#--------------------------------------------------------------------- +# Test the sqlite3_backup_remaining() and backup_pagecount() APIs. +# +do_test backup-6.1 { + catch { file delete -force test.db } + catch { file delete -force test2.db } + sqlite3 db test.db + sqlite3 db2 test2.db + execsql { + BEGIN; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES(1, randstr(1000,1000)); + INSERT INTO t1 VALUES(2, randstr(1000,1000)); + INSERT INTO t1 VALUES(3, randstr(1000,1000)); + INSERT INTO t1 VALUES(4, randstr(1000,1000)); + INSERT INTO t1 VALUES(5, randstr(1000,1000)); + COMMIT; + } +} {} +do_test backup-6.2 { + set nTotal [expr {[file size test.db]/1024}] + sqlite3_backup B db2 main db main + B step 1 +} {SQLITE_OK} +do_test backup-6.3 { + B pagecount +} $nTotal +do_test backup-6.4 { + B remaining +} [expr $nTotal-1] +do_test backup-6.5 { + B step 5 + list [B remaining] [B pagecount] +} [list [expr $nTotal-6] $nTotal] +do_test backup-6.6 { + execsql { CREATE TABLE t2(a PRIMARY KEY, b) } + B step 1 + list [B remaining] [B pagecount] +} [list [expr $nTotal-5] [expr $nTotal+2]] + +do_test backup-6.X { + B finish +} {SQLITE_OK} + + + +#--------------------------------------------------------------------- +# Test cases backup-7.* test that SQLITE_BUSY and SQLITE_LOCKED errors +# are returned correctly: +# +# backup-7.1.*: Source database is externally locked (return SQLITE_BUSY). +# +# backup-7.2.*: Attempt to step the backup process while a +# write-transaction is underway on the source pager (return +# SQLITE_LOCKED). +# +# backup-7.3.*: Destination database is externally locked (return SQLITE_BUSY). +# +do_test backup-7.0 { + catch { file delete -force test.db } + catch { file delete -force test2.db } + sqlite3 db2 test2.db + sqlite3 db test.db + execsql { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES(1, randstr(1000,1000)); + INSERT INTO t1 SELECT a+ 1, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+ 2, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+ 4, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+ 8, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+16, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+32, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+64, randstr(1000,1000) FROM t1; + } +} {} + +do_test backup-7.1.1 { + sqlite3_backup B db2 main db main + B step 5 +} {SQLITE_OK} +do_test backup-7.1.2 { + sqlite3 db3 test.db + execsql { BEGIN EXCLUSIVE } db3 + B step 5 +} {SQLITE_BUSY} +do_test backup-7.1.3 { + execsql { ROLLBACK } db3 + B step 5 +} {SQLITE_OK} +do_test backup-7.2.1 { + execsql { + BEGIN; + INSERT INTO t1 VALUES(1, 4); + } +} {} +do_test backup-7.2.2 { + B step 5000 +} {SQLITE_LOCKED} +do_test backup-7.2.3 { + execsql { ROLLBACK } + B step 5000 +} {SQLITE_DONE} +do_test backup-7.2.4 { + B finish +} {SQLITE_OK} +test_contents backup-7.2.5 db main db2 main +integrity_check backup-7.3.6 db2 + +do_test backup-7.3.1 { + db2 close + db3 close + file delete -force test2.db + sqlite3 db2 test2.db + sqlite3 db3 test2.db + + sqlite3_backup B db2 main db main + execsql { BEGIN ; CREATE TABLE t2(a, b); } db3 + + B step 5 +} {SQLITE_BUSY} +do_test backup-7.3.2 { + execsql { COMMIT } db3 + B step 5000 +} {SQLITE_DONE} +do_test backup-7.3.3 { + B finish +} {SQLITE_OK} +test_contents backup-7.3.4 db main db2 main +integrity_check backup-7.3.5 db2 +catch { db2 close } +catch { db3 close } + +#----------------------------------------------------------------------- +# The following tests, backup-8.*, test attaching multiple backup +# processes to the same source database. Also, reading from the source +# database while a read transaction is active. +# +# These tests reuse the database "test.db" left over from backup-7.*. +# +do_test backup-8.1 { + catch { file delete -force test2.db } + catch { file delete -force test3.db } + sqlite3 db2 test2.db + sqlite3 db3 test3.db + + sqlite3_backup B2 db2 main db main + sqlite3_backup B3 db3 main db main + list [B2 finish] [B3 finish] +} {SQLITE_OK SQLITE_OK} +do_test backup-8.2 { + sqlite3_backup B3 db3 main db main + sqlite3_backup B2 db2 main db main + list [B2 finish] [B3 finish] +} {SQLITE_OK SQLITE_OK} +do_test backup-8.3 { + sqlite3_backup B2 db2 main db main + sqlite3_backup B3 db3 main db main + B2 step 5 +} {SQLITE_OK} +do_test backup-8.4 { + execsql { + BEGIN; + SELECT * FROM sqlite_master; + } + B3 step 5 +} {SQLITE_OK} +do_test backup-8.5 { + list [B3 step 5000] [B3 finish] +} {SQLITE_DONE SQLITE_OK} +do_test backup-8.6 { + list [B2 step 5000] [B2 finish] +} {SQLITE_DONE SQLITE_OK} +test_contents backup-8.7 db main db2 main +test_contents backup-8.8 db main db3 main +do_test backup-8.9 { + execsql { PRAGMA lock_status } +} {main shared temp closed} +do_test backup-8.10 { + execsql COMMIT +} {} +catch { db2 close } +catch { db3 close } + + +finish_test diff --git a/test/backup_ioerr.test b/test/backup_ioerr.test new file mode 100644 index 0000000000..eae88641ea --- /dev/null +++ b/test/backup_ioerr.test @@ -0,0 +1,287 @@ +# 2008 January 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the handling of IO errors by the +# sqlite3_backup_XXX APIs. +# +# $Id: backup_ioerr.test,v 1.1 2009/02/03 16:51:25 danielk1977 Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +proc data_checksum {db file} { + $db one "SELECT md5sum(a, b) FROM ${file}.t1" +} +proc test_contents {name db1 file1 db2 file2} { + $db2 eval {select * from sqlite_master} + $db1 eval {select * from sqlite_master} + set checksum [data_checksum $db2 $file2] + uplevel [list do_test $name [list data_checksum $db1 $file1] $checksum] +} + +#-------------------------------------------------------------------- +# This proc creates a database of approximately 290 pages. Depending +# on whether or not auto-vacuum is configured. Test cases backup_ioerr-1.* +# verify nothing more than this assumption. +# +proc populate_database {db {xtra_large 0}} { + execsql { + BEGIN; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, randstr(1000,1000)); + INSERT INTO t1 SELECT a+ 1, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+ 2, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+ 4, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+ 8, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+16, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+32, randstr(1000,1000) FROM t1; + CREATE INDEX i1 ON t1(b); + COMMIT; + } $db + if {$xtra_large} { + execsql { INSERT INTO t1 SELECT a+64, randstr(1000,1000) FROM t1 } $db + } +} +do_test backup_ioerr-1.1 { + populate_database db + set nPage [expr {[file size test.db] / 1024}] + expr {$nPage>140 && $nPage<150} +} {1} +do_test backup_ioerr-1.2 { + expr {[file size test.db] > $sqlite_pending_byte} +} {1} +do_test backup_ioerr-1.3 { + db close + file delete -force test.db +} {} + +# Turn off IO error simulation. +# +proc clear_ioerr_simulation {} { + set ::sqlite_io_error_hit 0 + set ::sqlite_io_error_hardhit 0 + set ::sqlite_io_error_pending 0 + set ::sqlite_io_error_persist 0 +} + +#-------------------------------------------------------------------- +# The following procedure runs with SQLite's IO error simulation +# enabled. +# +# 1) Start with a reasonably sized database. One that includes the +# pending-byte (locking) page. +# +# 2) Open a backup process. Set the cache-size for the destination +# database to 10 pages only. +# +# 3) Step the backup process N times to partially backup the database +# file. If an IO error is reported, then the backup process is +# concluded with a call to backup_finish(). +# +# If an IO error occurs, verify that: +# +# * the call to backup_step() returns an SQLITE_IOERR_XXX error code. +# +# * after the failed call to backup_step() but before the call to +# backup_finish() the destination database handle error code and +# error message remain unchanged. +# +# * the call to backup_finish() returns an SQLITE_IOERR_XXX error code. +# +# * following the call to backup_finish(), the destination database +# handle has been populated with an error code and error message. +# +# 4) Write to the database via the source database connection. Check +# that: +# +# * If an IO error occurs while writing the source database, the +# write operation should report an IO error. The backup should +# proceed as normal. +# +# * If an IO error occurs while updating the backup, the write +# operation should proceed normally. The error should be reported +# from the next call to backup_step() (in step 5 of this test +# procedure). +# +# 5) Step the backup process to finish the backup. If an IO error is +# reported, then the backup process is concluded with a call to +# backup_finish(). +# +# Test that if an IO error occurs, or if one occured while updating +# the backup database during step 4, then the conditions listed +# under step 3 are all true. +# +# 6) Finish the backup process. +# +# * If the backup succeeds (backup_finish() returns SQLITE_OK), then +# the contents of the backup database should match that of the +# source database. +# +# * If the backup fails (backup_finish() returns other than SQLITE_OK), +# then the contents of the backup database should be as they were +# before the operation was started. +# +# The following factors are varied: +# +# * Destination database is initially larger than the source database, OR +# * Destination database is initially smaller than the source database. +# +# * IO errors are transient, OR +# * IO errors are persistent. +# +# * Destination page-size is smaller than the source. +# * Destination page-size is the same as the source. +# * Destination page-size is larger than the source. +# + +set iTest 1 +foreach bPersist {0 1} { +foreach iDestPagesize {512 1024 4096} { +foreach zSetupBak [list "" {populate_database ddb 1}] { + + incr iTest + set bStop 0 +for {set iError 1} {$bStop == 0} {incr iError} { + # Disable IO error simulation. + clear_ioerr_simulation + + catch { ddb close } + catch { sdb close } + catch { file delete -force test.db } + catch { file delete -force bak.db } + + # Open the source and destination databases. + sqlite3 sdb test.db + sqlite3 ddb bak.db + + # Step 1: Populate the source and destination databases. + populate_database sdb + ddb eval "PRAGMA page_size = $iDestPagesize" + ddb eval "PRAGMA cache_size = 10" + eval $zSetupBak + + # Step 2: Open the backup process. + sqlite3_backup B ddb main sdb main + + # Enable IO error simulation. + set ::sqlite_io_error_pending $iError + set ::sqlite_io_error_persist $bPersist + + # Step 3: Partially backup the database. If an IO error occurs, check + # a few things then skip to the next iteration of the loop. + # + set rc [B step 100] + if {$::sqlite_io_error_hardhit} { + + do_test backup_ioerr-$iTest.$iError.1 { + string match SQLITE_IOERR* $rc + } {1} + do_test backup_ioerr-$iTest.$iError.2 { + list [sqlite3_errcode ddb] [sqlite3_errmsg ddb] + } {SQLITE_OK {not an error}} + + set rc [B finish] + do_test backup_ioerr-$iTest.$iError.3 { + string match SQLITE_IOERR* $rc + } {1} + + do_test backup_ioerr-$iTest.$iError.4 { + sqlite3_errmsg ddb + } {disk I/O error} + + clear_ioerr_simulation + sqlite3 ddb bak.db + integrity_check backup_ioerr-$iTest.$iError.5 ddb + + continue + } + + # No IO error was encountered during step 3. Check that backup_step() + # returned SQLITE_OK before proceding. + do_test backup_ioerr-$iTest.$iError.6 { + expr {$rc eq "SQLITE_OK"} + } {1} + + # Step 4: Write to the source database. + set rc [catchsql { UPDATE t1 SET b = randstr(1000,1000) WHERE a < 50 } sdb] + + if {[lindex $rc 0] && $::sqlite_io_error_persist==0} { + # The IO error occured while updating the source database. In this + # case the backup should be able to continue. + set rc [B step 5000] + if { $rc != "SQLITE_IOERR_UNLOCK" } { + do_test backup_ioerr-$iTest.$iError.7 { + list [B step 5000] [B finish] + } {SQLITE_DONE SQLITE_OK} + + clear_ioerr_simulation + test_contents backup_ioerr-$iTest.$iError.8 ddb main sdb main + integrity_check backup_ioerr-$iTest.$iError.9 ddb + } else { + do_test backup_ioerr-$iTest.$iError.10 { + B finish + } {SQLITE_IOERR_UNLOCK} + } + + clear_ioerr_simulation + sqlite3 ddb bak.db + integrity_check backup_ioerr-$iTest.$iError.11 ddb + + continue + } + + # Step 5: Finish the backup operation. If an IO error occurs, check that + # it is reported correctly and skip to the next iteration of the loop. + # + set rc [B step 5000] + if {$rc != "SQLITE_DONE"} { + do_test backup_ioerr-$iTest.$iError.12 { + string match SQLITE_IOERR* $rc + } {1} + do_test backup_ioerr-$iTest.$iError.13 { + list [sqlite3_errcode ddb] [sqlite3_errmsg ddb] + } {SQLITE_OK {not an error}} + + set rc [B finish] + do_test backup_ioerr-$iTest.$iError.14 { + string match SQLITE_IOERR* $rc + } {1} + do_test backup_ioerr-$iTest.$iError.15 { + sqlite3_errmsg ddb + } {disk I/O error} + + clear_ioerr_simulation + sqlite3 ddb bak.db + integrity_check backup_ioerr-$iTest.$iError.16 ddb + + continue + } + + # The backup was successfully completed. + # + do_test backup_ioerr-$iTest.$iError.17 { + list [set rc] [B finish] + } {SQLITE_DONE SQLITE_OK} + + clear_ioerr_simulation + sqlite3 sdb test.db + sqlite3 ddb bak.db + + test_contents backup_ioerr-$iTest.$iError.18 ddb main sdb main + integrity_check backup_ioerr-$iTest.$iError.19 ddb + + set bStop [expr $::sqlite_io_error_pending<=0] +}}}} + +catch { sdb close } +catch { ddb close } +finish_test + diff --git a/test/backup_malloc.test b/test/backup_malloc.test new file mode 100644 index 0000000000..53486cdbe0 --- /dev/null +++ b/test/backup_malloc.test @@ -0,0 +1,86 @@ +# 2008 January 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the handling of OOM errors by the +# sqlite3_backup_XXX APIs. +# +# $Id: backup_malloc.test,v 1.1 2009/02/03 16:51:25 danielk1977 Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +source $testdir/malloc_common.tcl + +do_malloc_test backup_malloc-1 -tclprep { + execsql { + PRAGMA cache_size = 10; + BEGIN; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, randstr(1000,1000)); + INSERT INTO t1 SELECT a+ 1, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+ 2, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+ 4, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+ 8, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+16, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+32, randstr(1000,1000) FROM t1; + INSERT INTO t1 SELECT a+64, randstr(1000,1000) FROM t1; + CREATE INDEX i1 ON t1(b); + COMMIT; + } + sqlite3 db2 test2.db + execsql { PRAGMA cache_size = 10 } db2 +} -tclbody { + + # Create a backup object. + # + set rc [catch {sqlite3_backup B db2 main db main}] + if {$rc && [sqlite3_errcode db2] == "SQLITE_NOMEM"} { + error "out of memory" + } + + # Run the backup process some. + # + set rc [B step 50] + if {$rc == "SQLITE_NOMEM" || $rc == "SQLITE_IOERR_NOMEM"} { + error "out of memory" + } + + # Update the database. + # + execsql { UPDATE t1 SET a = a + 1 } + + # Finish doing the backup. + # + set rc [B step 5000] + if {$rc == "SQLITE_NOMEM" || $rc == "SQLITE_IOERR_NOMEM"} { + error "out of memory" + } + + # Finalize the backup. + B finish +} -cleanup { + catch { B finish } +} + +do_malloc_test backup_malloc-1 -tclprep { + sqlite3 db2 test2.db +} -tclbody { + set rc [catch {sqlite3_backup B db2 temp db main}] + set errcode [sqlite3_errcode db2] + if {$rc && ($errcode == "SQLITE_NOMEM" || $errcode == "SQLITE_IOERR_NOMEM")} { + error "out of memory" + } +} -cleanup { + catch { B finish } + db2 close +} + +finish_test diff --git a/test/quick.test b/test/quick.test index 64d238c207..c27f1d23fd 100644 --- a/test/quick.test +++ b/test/quick.test @@ -6,7 +6,7 @@ #*********************************************************************** # This file runs all tests. # -# $Id: quick.test,v 1.91 2009/01/03 10:41:29 danielk1977 Exp $ +# $Id: quick.test,v 1.92 2009/02/03 16:51:25 danielk1977 Exp $ proc lshift {lvar} { upvar $lvar l @@ -46,6 +46,7 @@ set EXCLUDE { async.test async2.test async3.test + backup_ioerr.test corrupt.test corruptC.test crash.test diff --git a/test/tester.tcl b/test/tester.tcl index 2e222d5215..6b2d749a7d 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -11,7 +11,7 @@ # This file implements some common TCL routines used for regression # testing the SQLite library # -# $Id: tester.tcl,v 1.136 2009/01/09 10:49:14 danielk1977 Exp $ +# $Id: tester.tcl,v 1.137 2009/02/03 16:51:25 danielk1977 Exp $ # # What for user input before continuing. This gives an opportunity @@ -485,11 +485,9 @@ proc forcedelete {filename} { # Do an integrity check of the entire database # -proc integrity_check {name} { +proc integrity_check {name {db db}} { ifcapable integrityck { - do_test $name { - execsql {PRAGMA integrity_check} - } {ok} + do_test $name [list execsql {PRAGMA integrity_check} $db] {ok} } } diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 1be74093fb..e13c86c5c3 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -247,6 +247,7 @@ foreach file { btmutex.c btree.c + backup.c vdbemem.c vdbeaux.c