From 6e09d69c92ca81c7a8ffda7a26bf74e807ad3371 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Jul 2010 18:34:15 +0000 Subject: [PATCH 01/30] Add experimental unix-only file-control to grow and truncate the database file by a configurable chunk size. FossilOrigin-Name: 7cf0e851d4c5e826ea22ed08291b7c91d7b1abc7 --- manifest | 32 +++++++++-------------- manifest.uuid | 2 +- src/os_unix.c | 64 +++++++++++++++++++++++++++++++++++++++++---- src/sqlite.h.in | 1 + src/test1.c | 39 +++++++++++++++++++++++++++ test/fallocate.test | 56 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 169 insertions(+), 25 deletions(-) create mode 100644 test/fallocate.test diff --git a/manifest b/manifest index a21c43a058..167b5239a5 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Increase\sthe\sversion\snumber\sto\s3.7.1\sdue\sto\sthe\sAPI\senhancement\sof\sthe\sprevious\ncheck-in. -D 2010-07-26T16:40:04 +C Add\sexperimental\sunix-only\sfile-control\sto\sgrow\sand\struncate\sthe\sdatabase\sfile\sby\sa\sconfigurable\schunk\ssize. +D 2010-07-27T18:34:16 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -157,7 +154,7 @@ F src/os.c 60178f518c4d6c0dcb59f7292232281d7bea2dcf F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e -F src/os_unix.c 3109e0e5a0d5551bab2e8c7322b20a3b8b171248 +F src/os_unix.c 77e963fbbed6a2ca9b7c9115ae06f05181729490 F src/os_win.c 1f8b0a1a5bcf6289e7754d0d3c16cec16d4c93ab F src/pager.c 78ca1e1f3315c8227431c403c04d791dccf242fb F src/pager.h 879fdde5a102d2f21a3135d6f647530b21c2796c @@ -173,14 +170,14 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 74fef1334bec27e606ef0b19e5c41cd0a639e69c F src/shell.c fd4ccdb37c3b68de0623eb938a649e0990710714 -F src/sqlite.h.in e789728101d821fd4307208aa11e332e51eedbf9 +F src/sqlite.h.in 3b201a7aced2227339c2bba8dfbee47a823d43ad F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h a9be6badc6cd6a3c1ae54475a98661cf351ecad5 F src/sqliteLimit.h 196e2f83c3b444c4548fc1874f52f84fdbda40f3 F src/status.c 51f65ba9768846c071fa621624d55d6866fce28b F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c ae1e4fb653c91ddad7e2534d209711a12604ccc4 -F src/test1.c ff3b4533fc4d78d1bff2ef831a5791db55096ed3 +F src/test1.c 0bfcda72f9f8ab5c0e90fac05cc22c1c21131ddf F src/test2.c e3f564ab1e9fd0b47b0c9e23e7054e38bf0836cf F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 F src/test4.c 0528360b5025688002a5feb6be906ddce52eaaee @@ -354,6 +351,7 @@ F test/exclusive.test 5fe18e10a159342dd52ca14b1554e33f98734267 F test/exclusive2.test fcbb1c9ca9739292a0a22a3763243ad6d868086b F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/expr.test 9f521ae22f00e074959f72ce2e55d46b9ed23f68 +F test/fallocate.test 2aa524a237893aca5b2a7ad5d450ee4801b4abdd F test/filectrl.test 97003734290887566e01dded09dc9e99cb937e9e F test/filefmt.test 5d271bf467e6557fe7499dcc8203069c9dc5825e F test/fkey1.test 01c7de578e11747e720c2d9aeef27f239853c4da @@ -841,14 +839,10 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 008368b2bd1513e2df21c2070548c6fc94abebb1 -R e0e547721eadb5865947ac95d9a5637b -U drh -Z b2f076ca75f409c11b1173e904fc1e43 ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFMTbpnoxKgR168RlERAgRsAKCEDv9PrVdoUqb+GgrX9p308vOjrgCgi4/A -Wv1UV7AZmSjiGasHOtjRZXM= -=082N ------END PGP SIGNATURE----- +P 1d7571e4be6d96aec13cd8b3b7b10b8dc44d19a5 +R 06d24f984109c26ad9ccb04075848e30 +T *branch * experimental +T *sym-experimental * +T -sym-trunk * +U dan +Z 84d3b6ef35de3d3ded4f7f20ba0de5c7 diff --git a/manifest.uuid b/manifest.uuid index 2e36131874..e26c476d95 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d7571e4be6d96aec13cd8b3b7b10b8dc44d19a5 \ No newline at end of file +7cf0e851d4c5e826ea22ed08291b7c91d7b1abc7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 9457516cac..467409b198 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -210,6 +210,7 @@ struct unixFile { int fileFlags; /* Miscellanous flags */ const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ + int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ #if SQLITE_ENABLE_LOCKING_STYLE int openFlags; /* The flags specified at open() */ #endif @@ -2763,6 +2764,43 @@ static int unixWrite( } SimulateIOError(( wrote=(-1), amt=1 )); SimulateDiskfullError(( wrote=0, amt=1 )); + + /* If the user has configured a chunk-size for this file, it could be + ** that the file needs to be extended at this point. + */ + if( pFile->szChunk && amt==0 ){ + i64 nSize; /* Required file size */ + struct stat buf; /* Used to hold return values of fstat() */ + int rc = fstat(pFile->h, &buf); + if( rc!=0 ) return SQLITE_IOERR_FSTAT; + nSize = ((offset+amt+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; + if( nSize>(i64)buf.st_size ){ +#ifdef HAVE_POSIX_FALLOCATE + if( posix_fallocate(pFile->h, buf.st_size, nSize-buf.st_size) ){ + return SQLITE_IOERR_WRITE; + } +#else + /* If the OS does not have posix_fallocate(), fake it. First use + ** ftruncate() to set the file size, then write a single byte to + ** the last byte in each block within the extended region. + */ + int nBlk = buf.st_blksize; /* File-system block size */ + i64 iWrite; /* Next offset to write to */ + + if( ftruncate(pFile->h, nSize) ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_TRUNCATE; + } + iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; + do { + wrote = seekAndWrite(pFile, iWrite, "", 1); + iWrite += nBlk; + } while( wrote==1 && iWrite0 ){ if( wrote<0 ){ /* lastErrno set by seekAndWrite */ @@ -2772,6 +2810,7 @@ static int unixWrite( return SQLITE_FULL; } } + return SQLITE_OK; } @@ -2973,12 +3012,23 @@ static int unixSync(sqlite3_file *id, int flags){ ** Truncate an open file to a specified size */ static int unixTruncate(sqlite3_file *id, i64 nByte){ + unixFile *pFile = (unixFile *)id; int rc; - assert( id ); + assert( pFile ); SimulateIOError( return SQLITE_IOERR_TRUNCATE ); - rc = ftruncate(((unixFile*)id)->h, (off_t)nByte); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + rc = ftruncate(pFile->h, (off_t)nByte); if( rc ){ - ((unixFile*)id)->lastErrno = errno; + pFile->lastErrno = errno; return SQLITE_IOERR_TRUNCATE; }else{ #ifndef NDEBUG @@ -2989,8 +3039,8 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ ** when restoring a database using the backup API from a zero-length ** source. */ - if( ((unixFile*)id)->inNormalWrite && nByte==0 ){ - ((unixFile*)id)->transCntrChng = 1; + if( pFile->inNormalWrite && nByte==0 ){ + pFile->transCntrChng = 1; } #endif @@ -3047,6 +3097,10 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ *(int*)pArg = ((unixFile*)id)->lastErrno; return SQLITE_OK; } + case SQLITE_FCNTL_CHUNK_SIZE: { + ((unixFile*)id)->szChunk = *(int *)pArg; + return SQLITE_OK; + } case SQLITE_FCNTL_SIZE_HINT: { #if 0 /* No performance advantage seen on Linux */ sqlite3_int64 szFile = *(sqlite3_int64*)pArg; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 89bcf1c400..ca0ea5dd4b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -696,6 +696,7 @@ struct sqlite3_io_methods { #define SQLITE_SET_LOCKPROXYFILE 3 #define SQLITE_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 /* ** CAPI3REF: Mutex Handle diff --git a/src/test1.c b/src/test1.c index 33ac7a1bd4..acb812b372 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4613,6 +4613,44 @@ static int file_control_lasterrno_test( return TCL_OK; } +/* +** tclcmd: file_control_chunksize_test DB DBNAME SIZE +** +** This TCL command runs the sqlite3_file_control interface and +** verifies correct operation of the SQLITE_GET_LOCKPROXYFILE and +** SQLITE_SET_LOCKPROXYFILE verbs. +*/ +static int file_control_chunksize_test( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + int nSize; /* New chunk size */ + char *zDb; /* Db name ("main", "temp" etc.) */ + sqlite3 *db; /* Database handle */ + int rc; /* file_control() return code */ + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME SIZE"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) + || Tcl_GetIntFromObj(interp, objv[3], &nSize) + ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + if( zDb[0]=='\0' ) zDb = NULL; + + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_CHUNK_SIZE, (void *)&nSize); + if( rc ){ + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + return TCL_ERROR; + } + return TCL_OK; +} + /* ** tclcmd: file_control_lockproxy_test DB PWD ** @@ -5106,6 +5144,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "file_control_test", file_control_test, 0 }, { "file_control_lasterrno_test", file_control_lasterrno_test, 0 }, { "file_control_lockproxy_test", file_control_lockproxy_test, 0 }, + { "file_control_chunksize_test", file_control_chunksize_test, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, /* Functions from os.h */ diff --git a/test/fallocate.test b/test/fallocate.test new file mode 100644 index 0000000000..b9c4f460d4 --- /dev/null +++ b/test/fallocate.test @@ -0,0 +1,56 @@ +# 2010 July 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +if {$::tcl_platform(platform)!="unix"} { + finish_test + return +} + +file_control_chunksize_test db main [expr 1024*1024] + +do_test fallocate-1.1 { + execsql { + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 1; + CREATE TABLE t1(a, b); + } + file size test.db +} [expr 1*1024*1024] + +do_test fallocate-1.2 { + execsql { INSERT INTO t1 VALUES(1, zeroblob(1024*900)) } + file size test.db +} [expr 1*1024*1024] + +do_test fallocate-1.3 { + execsql { INSERT INTO t1 VALUES(2, zeroblob(1024*900)) } + file size test.db +} [expr 2*1024*1024] + +do_test fallocate-1.4 { + execsql { DELETE FROM t1 WHERE a = 1 } + file size test.db +} [expr 1*1024*1024] + +do_test fallocate-1.5 { + execsql { DELETE FROM t1 WHERE a = 2 } + file size test.db +} [expr 1*1024*1024] + +do_test fallocate-1.6 { + execsql { PRAGMA freelist_count } +} {0} + +finish_test From 502019c8bbf382e97720c711af03b24147107eff Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 28 Jul 2010 14:26:17 +0000 Subject: [PATCH 02/30] Support FCNTL_CHUNK_SIZE on windows too. FossilOrigin-Name: a038688c991435967b935946c2283707820bb5da --- manifest | 22 +++--- manifest.uuid | 2 +- src/os_unix.c | 93 +++++++++++++------------ src/os_win.c | 160 ++++++++++++++++++++++++++------------------ src/pager.c | 2 +- src/sqlite.h.in | 8 +++ src/wal.c | 15 ++++- test/fallocate.test | 89 ++++++++++++++++++++++-- 8 files changed, 264 insertions(+), 127 deletions(-) diff --git a/manifest b/manifest index 35fb977532..ab3b4c342d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sexperimental\sbranch. -D 2010-07-27T18:36:38 +C Support\sFCNTL_CHUNK_SIZE\son\swindows\stoo. +D 2010-07-28T14:26:18 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -154,9 +154,9 @@ F src/os.c 60178f518c4d6c0dcb59f7292232281d7bea2dcf F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e -F src/os_unix.c 77e963fbbed6a2ca9b7c9115ae06f05181729490 -F src/os_win.c 1f8b0a1a5bcf6289e7754d0d3c16cec16d4c93ab -F src/pager.c 78ca1e1f3315c8227431c403c04d791dccf242fb +F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 +F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 +F src/pager.c 27ace2e07c8bfe2e04428eba03b2ddb5b03abd7d F src/pager.h 879fdde5a102d2f21a3135d6f647530b21c2796c F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -170,7 +170,7 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 74fef1334bec27e606ef0b19e5c41cd0a639e69c F src/shell.c fd4ccdb37c3b68de0623eb938a649e0990710714 -F src/sqlite.h.in 2dfa3db44fd123ea5f0e20e87fa3d9db189feb98 +F src/sqlite.h.in 10bd4aed244fc0eadb10ec01cedb60bafc2d7bfe F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h a9be6badc6cd6a3c1ae54475a98661cf351ecad5 F src/sqliteLimit.h 196e2f83c3b444c4548fc1874f52f84fdbda40f3 @@ -227,7 +227,7 @@ F src/vdbeblob.c 258a6010ba7a82b72b327fb24c55790655689256 F src/vdbemem.c 5e579abf6532001dfbee0e640dc34eae897a9807 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c 82200af3881fa4e1c9cf07cf31d98c09d437e3ab -F src/wal.c 0925601f3299c2941a67c9cfff41ee710f70ca82 +F src/wal.c 72cb5df7f4c26f83cb661d5a607b9918da99f758 F src/wal.h 906c85760598b18584921fe08008435aa4eeeeb2 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 79202ca81e740eeb1f54512147e29b6c518d84ca @@ -351,7 +351,7 @@ F test/exclusive.test 5fe18e10a159342dd52ca14b1554e33f98734267 F test/exclusive2.test fcbb1c9ca9739292a0a22a3763243ad6d868086b F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/expr.test 9f521ae22f00e074959f72ce2e55d46b9ed23f68 -F test/fallocate.test 2aa524a237893aca5b2a7ad5d450ee4801b4abdd +F test/fallocate.test 0594314eb04268f7d0779d054fa850a36a5ae8bc F test/filectrl.test 97003734290887566e01dded09dc9e99cb937e9e F test/filefmt.test 5d271bf467e6557fe7499dcc8203069c9dc5825e F test/fkey1.test 01c7de578e11747e720c2d9aeef27f239853c4da @@ -839,7 +839,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 7cf0e851d4c5e826ea22ed08291b7c91d7b1abc7 8118de2af33557f75b4f2f1b1194a21d46ccf7fe -R db6f0268ed3b851f8967be0e4ae70758 +P 621824092d443425c420ba9010bbe1202fe99ea2 +R d7ede0d4642b161610b7d818427c7a8c U dan -Z d9d588012d8704dd90455874cd937598 +Z 95d00fbeff8589e69f2af3194811a965 diff --git a/manifest.uuid b/manifest.uuid index 9d45064bef..1ec6093ac1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -621824092d443425c420ba9010bbe1202fe99ea2 \ No newline at end of file +a038688c991435967b935946c2283707820bb5da \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 467409b198..5c648e3f9c 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -2765,42 +2765,6 @@ static int unixWrite( SimulateIOError(( wrote=(-1), amt=1 )); SimulateDiskfullError(( wrote=0, amt=1 )); - /* If the user has configured a chunk-size for this file, it could be - ** that the file needs to be extended at this point. - */ - if( pFile->szChunk && amt==0 ){ - i64 nSize; /* Required file size */ - struct stat buf; /* Used to hold return values of fstat() */ - int rc = fstat(pFile->h, &buf); - if( rc!=0 ) return SQLITE_IOERR_FSTAT; - nSize = ((offset+amt+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; - if( nSize>(i64)buf.st_size ){ -#ifdef HAVE_POSIX_FALLOCATE - if( posix_fallocate(pFile->h, buf.st_size, nSize-buf.st_size) ){ - return SQLITE_IOERR_WRITE; - } -#else - /* If the OS does not have posix_fallocate(), fake it. First use - ** ftruncate() to set the file size, then write a single byte to - ** the last byte in each block within the extended region. - */ - int nBlk = buf.st_blksize; /* File-system block size */ - i64 iWrite; /* Next offset to write to */ - - if( ftruncate(pFile->h, nSize) ){ - pFile->lastErrno = errno; - return SQLITE_IOERR_TRUNCATE; - } - iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; - do { - wrote = seekAndWrite(pFile, iWrite, "", 1); - iWrite += nBlk; - } while( wrote==1 && iWrite0 ){ if( wrote<0 ){ /* lastErrno set by seekAndWrite */ @@ -3083,6 +3047,54 @@ static int unixFileSize(sqlite3_file *id, i64 *pSize){ static int proxyFileControl(sqlite3_file*,int,void*); #endif +/* +** This function is called to handle the SQLITE_FCNTL_SIZE_HINT +** file-control operation. +** +** If the user has configured a chunk-size for this file, it could be +** that the file needs to be extended at this point. Otherwise, the +** SQLITE_FCNTL_SIZE_HINT operation is a no-op for Unix. +*/ +static int fcntlSizeHint(unixFile *pFile, i64 nByte){ + if( pFile->szChunk ){ + i64 nSize; /* Required file size */ + struct stat buf; /* Used to hold return values of fstat() */ + + if( fstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; + + nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; + if( nSize>(i64)buf.st_size ){ +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + if( posix_fallocate(pFile->h, buf.st_size, nSize-buf.st_size) ){ + return SQLITE_IOERR_WRITE; + } +#else + /* If the OS does not have posix_fallocate(), fake it. First use + ** ftruncate() to set the file size, then write a single byte to + ** the last byte in each block within the extended region. This + ** is the same technique used by glibc to implement posix_fallocate() + ** on systems that do not have a real fallocate() system call. + */ + int nBlk = buf.st_blksize; /* File-system block size */ + i64 iWrite; /* Next offset to write to */ + int nWrite; /* Return value from seekAndWrite() */ + + if( ftruncate(pFile->h, nSize) ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_TRUNCATE; + } + iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; + do { + nWrite = seekAndWrite(pFile, iWrite, "", 1); + iWrite += nBlk; + } while( nWrite==1 && iWriteszChunk = *(int *)pArg; - return SQLITE_OK; + return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { -#if 0 /* No performance advantage seen on Linux */ - sqlite3_int64 szFile = *(sqlite3_int64*)pArg; - unixFile *pFile = (unixFile*)id; - ftruncate(pFile->h, szFile); -#endif - return SQLITE_OK; + return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); } #ifndef NDEBUG /* The pager calls this method to signal that it has done diff --git a/src/os_win.c b/src/os_win.c index 0951312327..562282f20b 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -108,6 +108,7 @@ struct winFile { DWORD sectorSize; /* Sector size of the device file is on */ winShm *pShm; /* Instance of shared memory on this file */ const char *zPath; /* Full pathname of this file */ + int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ #if SQLITE_OS_WINCE WCHAR *zDeleteOnClose; /* Name of file to delete when closing */ HANDLE hMutex; /* Mutex used to control access to shared lock */ @@ -620,6 +621,42 @@ static BOOL winceLockFileEx( ** by the sqlite3_io_methods object. ******************************************************************************/ +/* +** Some microsoft compilers lack this definition. +*/ +#ifndef INVALID_SET_FILE_POINTER +# define INVALID_SET_FILE_POINTER ((DWORD)-1) +#endif + +/* +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. +*/ +static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ + LONG upperBits; /* Most sig. 32 bits of new offset */ + LONG lowerBits; /* Least sig. 32 bits of new offset */ + DWORD dwRet; /* Value returned by SetFilePointer() */ + + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); + lowerBits = (LONG)(iOffset & 0xffffffff); + + /* API oddity: If successful, SetFilePointer() returns a dword + ** containing the lower 32-bits of the new file-offset. Or, if it fails, + ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, + ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine + ** whether an error has actually occured, it is also necessary to call + ** GetLastError(). + */ + dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + if( (dwRet==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR) ){ + pFile->lastErrno = GetLastError(); + return 1; + } + + return 0; +} + /* ** Close a file. ** @@ -662,13 +699,6 @@ static int winClose(sqlite3_file *id){ return rc ? SQLITE_OK : SQLITE_IOERR; } -/* -** Some microsoft compilers lack this definition. -*/ -#ifndef INVALID_SET_FILE_POINTER -# define INVALID_SET_FILE_POINTER ((DWORD)-1) -#endif - /* ** Read data from a file into a buffer. Return SQLITE_OK if all ** bytes were read successfully and SQLITE_IOERR if anything goes @@ -680,32 +710,27 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ - LONG upperBits = (LONG)((offset>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(offset & 0xffffffff); - DWORD rc; - winFile *pFile = (winFile*)id; - DWORD error; - DWORD got; + winFile *pFile = (winFile*)id; /* file handle */ + DWORD nRead; /* Number of bytes actually read from file */ assert( id!=0 ); SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); - rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( rc==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; + + if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; } - if( !ReadFile(pFile->h, pBuf, amt, &got, 0) ){ + if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ pFile->lastErrno = GetLastError(); return SQLITE_IOERR_READ; } - if( got==(DWORD)amt ){ - return SQLITE_OK; - }else{ + if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ - memset(&((char*)pBuf)[got], 0, amt-got); + memset(&((char*)pBuf)[nRead], 0, amt-nRead); return SQLITE_IOERR_SHORT_READ; } + + return SQLITE_OK; } /* @@ -713,47 +738,42 @@ static int winRead( ** or some other error code on failure. */ static int winWrite( - sqlite3_file *id, /* File to write into */ - const void *pBuf, /* The bytes to be written */ - int amt, /* Number of bytes to write */ - sqlite3_int64 offset /* Offset into the file to begin writing at */ + sqlite3_file *id, /* File to write into */ + const void *pBuf, /* The bytes to be written */ + int amt, /* Number of bytes to write */ + sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - LONG upperBits = (LONG)((offset>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(offset & 0xffffffff); - DWORD rc; - winFile *pFile = (winFile*)id; - DWORD error; - DWORD wrote = 0; + int rc; /* True if error has occured, else false */ + winFile *pFile = (winFile*)id; /* File handle */ - assert( id!=0 ); + assert( amt>0 ); + assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); + OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); - rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( rc==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; - if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ - return SQLITE_FULL; - }else{ - return SQLITE_IOERR_WRITE; + + rc = seekWinFile(pFile, offset); + if( rc==0 ){ + u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ + int nRem = amt; /* Number of bytes yet to be written */ + DWORD nWrite; /* Bytes written by each WriteFile() call */ + + while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){ + aRem += nWrite; + nRem -= nWrite; + } + if( nRem>0 ){ + pFile->lastErrno = GetLastError(); + rc = 1; } } - assert( amt>0 ); - while( - amt>0 - && (rc = WriteFile(pFile->h, pBuf, amt, &wrote, 0))!=0 - && wrote>0 - ){ - amt -= wrote; - pBuf = &((char*)pBuf)[wrote]; - } - if( !rc || amt>(int)wrote ){ - pFile->lastErrno = GetLastError(); + + if( rc ){ if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ return SQLITE_FULL; - }else{ - return SQLITE_IOERR_WRITE; } + return SQLITE_IOERR_WRITE; } return SQLITE_OK; } @@ -762,26 +782,32 @@ static int winWrite( ** Truncate an open file to a specified size */ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ - LONG upperBits = (LONG)((nByte>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(nByte & 0xffffffff); - DWORD dwRet; - winFile *pFile = (winFile*)id; - DWORD error; - int rc = SQLITE_OK; + winFile *pFile = (winFile*)id; /* File handle object */ + int rc = SQLITE_OK; /* Return code for this function */ + + assert( pFile ); - assert( id!=0 ); OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); - dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( dwRet==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ + if( seekWinFile(pFile, nByte) ){ rc = SQLITE_IOERR_TRUNCATE; - /* SetEndOfFile will fail if nByte is negative */ - }else if( !SetEndOfFile(pFile->h) ){ + }else if( 0==SetEndOfFile(pFile->h) ){ pFile->lastErrno = GetLastError(); rc = SQLITE_IOERR_TRUNCATE; } - OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc==SQLITE_OK ? "ok" : "failed")); + + OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); return rc; } @@ -1146,6 +1172,10 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ *(int*)pArg = (int)((winFile*)id)->lastErrno; return SQLITE_OK; } + case SQLITE_FCNTL_CHUNK_SIZE: { + ((winFile*)id)->szChunk = *(int *)pArg; + return SQLITE_OK; + } case SQLITE_FCNTL_SIZE_HINT: { sqlite3_int64 sz = *(sqlite3_int64*)pArg; SimulateIOErrorBenign(1); diff --git a/src/pager.c b/src/pager.c index dfb29e3df7..603e7b5026 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3365,7 +3365,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>(pPager->dbOrigSize+1) ){ + if( rc==SQLITE_OK && pPager->dbSize>pPager->dbOrigSize ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index eab2b3a84e..209abd4823 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -690,6 +690,14 @@ struct sqlite3_io_methods { ** is often close. The underlying VFS might choose to preallocate database ** file space based on this hint in order to help writes to the database ** file run faster. +** +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 diff --git a/src/wal.c b/src/wal.c index 54274640ed..3ac8071548 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1521,6 +1521,7 @@ static int walCheckpoint( u32 iDbpage = 0; /* Next database page to write */ u32 iFrame = 0; /* Wal frame containing data for iDbpage */ u32 mxSafeFrame; /* Max frame that can be backfilled */ + u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ @@ -1545,6 +1546,7 @@ static int walCheckpoint( ** cannot be backfilled from the WAL. */ mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; pInfo = walCkptInfo(pWal); for(i=1; iaReadMark[i]; @@ -1565,6 +1567,8 @@ static int walCheckpoint( if( pInfo->nBackfillnBackfill; /* Sync the WAL to disk */ @@ -1572,11 +1576,20 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } + /* If the database file may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + nReq = ((i64)mxPage * szPage); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; assert( walFramePgno(pWal, iFrame)==iDbpage ); - if( iFrame<=nBackfill || iFrame>mxSafeFrame ) continue; + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); diff --git a/test/fallocate.test b/test/fallocate.test index b9c4f460d4..82ceb1d8fa 100644 --- a/test/fallocate.test +++ b/test/fallocate.test @@ -13,11 +13,6 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -if {$::tcl_platform(platform)!="unix"} { - finish_test - return -} - file_control_chunksize_test db main [expr 1024*1024] do_test fallocate-1.1 { @@ -53,4 +48,88 @@ do_test fallocate-1.6 { execsql { PRAGMA freelist_count } } {0} +# Start a write-transaction and read the "database file size" field from +# the journal file. This field should be set to the number of pages in +# the database file based on the size of the file on disk, not the actual +# logical size of the database within the file. +# +# We need to check this to verify that if in the unlikely event a rollback +# causes a database file to grow, the database grows to its previous size +# on disk, not to the minimum size required to hold the database image. +# +do_test fallocate-1.7 { + execsql { BEGIN; INSERT INTO t1 VALUES(1, 2); } + hexio_get_int [hexio_read test.db-journal 16 4] +} {1024} +do_test fallocate-1.8 { execsql { COMMIT } } {} + + +#------------------------------------------------------------------------- +# The following tests - fallocate-2.* - test that things work in WAL +# mode as well. +# +db close +file delete -force test.db +sqlite3 db test.db +file_control_chunksize_test db main [expr 32*1024] + +do_test fallocate-2.1 { + execsql { + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a, b); + } + file size test.db +} [expr 32*1024] + +do_test fallocate-2.2 { + execsql { INSERT INTO t1 VALUES(1, zeroblob(35*1024)) } + execsql { PRAGMA wal_checkpoint } + file size test.db +} [expr 64*1024] + +do_test fallocate-2.3 { + execsql { DELETE FROM t1 } + execsql { VACUUM } + file size test.db +} [expr 64*1024] + +do_test fallocate-2.4 { + execsql { PRAGMA wal_checkpoint } + file size test.db +} [expr 32*1024] + +do_test fallocate-2.5 { + execsql { + INSERT INTO t1 VALUES(2, randomblob(35*1024)); + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES(3, randomblob(128)); + DELETE FROM t1 WHERE a = 2; + VACUUM; + } + file size test.db +} [expr 64*1024] + +do_test fallocate-2.6 { + sqlite3 db2 test.db + execsql { BEGIN ; SELECT count(a) FROM t1 } db2 + execsql { + INSERT INTO t1 VALUES(4, randomblob(128)); + PRAGMA wal_checkpoint; + } + file size test.db +} [expr 64*1024] + +do_test fallocate-2.7 { + execsql { SELECT count(b) FROM t1 } db2 +} {1} + +do_test fallocate-2.8 { + execsql { COMMIT } db2 + execsql { PRAGMA wal_checkpoint } + file size test.db +} [expr 32*1024] + + finish_test + From bfcaec729c1f15653ed75e73525c101170e2405e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 28 Jul 2010 15:10:37 +0000 Subject: [PATCH 03/30] Fix errors in wal3.test caused by recent modifications. FossilOrigin-Name: 0714aeccd82d82686c074c1bb74ea794b7bfa3a3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal3.test | 30 +++++++++++++++++++++--------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index ab3b4c342d..56239ec085 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Support\sFCNTL_CHUNK_SIZE\son\swindows\stoo. -D 2010-07-28T14:26:18 +C Fix\serrors\sin\swal3.test\scaused\sby\srecent\smodifications. +D 2010-07-28T15:10:38 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -782,7 +782,7 @@ F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test 1891e6f72dd437a1c2a48091aa9182ba17a8f780 F test/wal2.test fa6dc4457b46988f46cf6c68ea51ebe341765f4a -F test/wal3.test d2ae7e66f973bd6b58ce49e546b2c00f44fe0485 +F test/wal3.test 695ea0f6c516423c611891df9a285aacd33344e3 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 F test/wal_common.tcl 895d76138043b86bdccf36494054bdabcf65837b F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 @@ -839,7 +839,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 621824092d443425c420ba9010bbe1202fe99ea2 -R d7ede0d4642b161610b7d818427c7a8c +P a038688c991435967b935946c2283707820bb5da +R 27913a854c524e2ed86aec3ca2a1108c U dan -Z 95d00fbeff8589e69f2af3194811a965 +Z bf8c1f74204eb606b6c5a25382962d26 diff --git a/manifest.uuid b/manifest.uuid index 1ec6093ac1..b65a0b6612 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a038688c991435967b935946c2283707820bb5da \ No newline at end of file +0714aeccd82d82686c074c1bb74ea794b7bfa3a3 \ No newline at end of file diff --git a/test/wal3.test b/test/wal3.test index b344c1423d..b53b844052 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -110,6 +110,11 @@ for {set i 1} {$i < 50} {incr i} { db2 close } +proc byte_is_zero {file offset} { + if {[file size test.db] <= $offset} { return 1 } + expr { [hexio_read $file $offset 1] == "00" } +} + do_multiclient_test i { set testname(1) multiproc @@ -158,22 +163,24 @@ do_multiclient_test i { COMMIT; PRAGMA wal_checkpoint; } - file size test.db - } [expr $AUTOVACUUM ? 4*1024 : 3*1024] + byte_is_zero test.db [expr $AUTOVACUUM ? 4*1024 : 3*1024] + } {1} do_test wal3-2.$tn.5 { sql2 { COMMIT; PRAGMA wal_checkpoint; } - file size test.db - } [expr $AUTOVACUUM ? 5*1024 : 4*1024] + list [byte_is_zero test.db [expr $AUTOVACUUM ? 4*1024 : 3*1024]] \ + [byte_is_zero test.db [expr $AUTOVACUUM ? 5*1024 : 4*1024]] + } {0 1} do_test wal3-2.$tn.6 { sql3 { COMMIT; PRAGMA wal_checkpoint; } - file size test.db - } [expr $AUTOVACUUM ? 5*1024 : 4*1024] + list [byte_is_zero test.db [expr $AUTOVACUUM ? 4*1024 : 3*1024]] \ + [byte_is_zero test.db [expr $AUTOVACUUM ? 5*1024 : 4*1024]] + } {0 1} } catch {db close} @@ -700,6 +707,7 @@ do_test wal3-9.0 { file delete -force test.db test.db-journal test.db wal sqlite3 db test.db execsql { + PRAGMA page_size = 1024; PRAGMA journal_mode = WAL; CREATE TABLE whoami(x); INSERT INTO whoami VALUES('nobody'); @@ -722,15 +730,19 @@ for {set i 0} {$i < 50} {incr i} { execsql { SELECT * FROM whoami } $c } $c } + +set sz [expr 1024 * (2+$AUTOVACUUM)] do_test wal3-9.3 { for {set i 0} {$i < 49} {incr i} { db$i close } execsql { PRAGMA wal_checkpoint } - set sz1 [file size test.db] + byte_is_zero test.db [expr $sz-1024] +} {1} +do_test wal3-9.4 { db49 close execsql { PRAGMA wal_checkpoint } set sz2 [file size test.db] - expr {$sz2 > $sz1} -} {1} + byte_is_zero test.db [expr $sz-1024] +} {0} db close From eb9444a4b33eac8abd881c0c8fb3f427dedb5d0a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Jul 2010 10:02:24 +0000 Subject: [PATCH 04/30] Make sure a connection has an exclusive lock on all database files involved in a multi-file transaction before writing the master-journal pointer into any journal files. Fix for [f3e5abed55]. FossilOrigin-Name: 50c0f2202d21bbf6b593d75fd20f13c0fac23eff --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 20 ++++++++++++++++++++ src/vdbeaux.c | 9 +++++---- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 8c0289ea3a..77245c477f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfurther\strunk\schanges\sinto\sexperimental\sbranch. -D 2010-07-30T07:26:51 +C Make\ssure\sa\sconnection\shas\san\sexclusive\slock\son\sall\sdatabase\sfiles\sinvolved\sin\sa\smulti-file\stransaction\sbefore\swriting\sthe\smaster-journal\spointer\sinto\sany\sjournal\sfiles.\sFix\sfor\s[f3e5abed55]. +D 2010-07-30T10:02:24 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 65efcf8cc70de5facf4375dbd78a99ceb3d19d6e +F src/pager.c 237bee4c1cec621821e05c7c808f631d4ee3aa32 F src/pager.h 879fdde5a102d2f21a3135d6f647530b21c2796c F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -222,7 +222,7 @@ F src/vdbe.c cefff41564b68a412e65e6a1013ec1b1c1ece6c4 F src/vdbe.h 4de0efb4b0fdaaa900cf419b35c458933ef1c6d2 F src/vdbeInt.h ffd68c4d4229227a5089bec53a1c635146177abc F src/vdbeapi.c d0f4407e465f261780ad725c1caece7d66a6aa35 -F src/vdbeaux.c 77442ab4233858cf603910429033fbbd997ecdef +F src/vdbeaux.c bb14feb1f210aec69889e145cc41b466319bfd2b F src/vdbeblob.c 258a6010ba7a82b72b327fb24c55790655689256 F src/vdbemem.c 5e579abf6532001dfbee0e640dc34eae897a9807 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 @@ -840,7 +840,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P aa81900153a2762cb2ad41e6710c1f1e7dc8b41e 451d965742cc219db709939b4ba1da2f2343dbce -R ed3ad896c2cd90b3c1af9b65becc7431 +P fb847d70407b0f0e548919b7554f62bc1dab8a6c +R ed4b8e20864f45ad745b5c55b793d01a U dan -Z 9e7bcf251287b29d791a1cd12b29e0e0 +Z a4643485bfc292d567c95a06fb090017 diff --git a/manifest.uuid b/manifest.uuid index 57955c6cf0..2866ee3183 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb847d70407b0f0e548919b7554f62bc1dab8a6c \ No newline at end of file +50c0f2202d21bbf6b593d75fd20f13c0fac23eff \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index de3547a897..2e484611b1 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5061,6 +5061,26 @@ int sqlite3PagerSync(Pager *pPager){ return rc; } +/* +** This function may only be called while a write-transaction is active in +** rollback. If the connection is in WAL mode, this call is a no-op. +** Otherwise, if the connection does not already have an EXCLUSIVE lock on +** the database file, an attempt is made to obtain one. +** +** If the EXCLUSIVE lock is already held or the attempt to obtain it is +** successful, or the connection is in WAL mode, SQLITE_OK is returned. +** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is +** returned. +*/ +int sqlite3PagerExclusiveLock(Pager *pPager){ + int rc = SQLITE_OK; + assert( pPager->state>=PAGER_RESERVED ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, PAGER_EXCLUSIVE); + } + return rc; +} + /* ** Sync the database file for the pager pPager. zMaster points to the name ** of a master journal file that should be written into the individual diff --git a/src/vdbeaux.c b/src/vdbeaux.c index c7cd3e4284..46748a9530 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1646,9 +1646,6 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ** to the transaction. */ rc = sqlite3VtabSync(db, &p->zErrMsg); - if( rc!=SQLITE_OK ){ - return rc; - } /* This loop determines (a) if the commit hook should be invoked and ** (b) how many database files have open write transactions, not @@ -1656,13 +1653,17 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ** one database file has an open write transaction, a master journal ** file is required for an atomic commit. */ - for(i=0; inDb; i++){ + for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ needXcommit = 1; if( i!=1 ) nTrans++; + rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); } } + if( rc!=SQLITE_OK ){ + return rc; + } /* If there are any write-transactions at all, invoke the commit hook */ if( needXcommit && db->xCommitCallback ){ From 2554ea7b0bac21683d265120bb2608ea3ea13d3d Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Jul 2010 10:09:12 +0000 Subject: [PATCH 05/30] Add the test cases for bug [f3e5abed55]. FossilOrigin-Name: cbbaf8e67a23dc332ce935bc7234246eacba60bf --- manifest | 13 ++--- manifest.uuid | 2 +- src/vdbeaux.c | 1 + test/tkt-f3e5abed55.test | 113 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 7 deletions(-) create mode 100644 test/tkt-f3e5abed55.test diff --git a/manifest b/manifest index 77245c477f..a82fdbb928 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sa\sconnection\shas\san\sexclusive\slock\son\sall\sdatabase\sfiles\sinvolved\sin\sa\smulti-file\stransaction\sbefore\swriting\sthe\smaster-journal\spointer\sinto\sany\sjournal\sfiles.\sFix\sfor\s[f3e5abed55]. -D 2010-07-30T10:02:24 +C Add\sthe\stest\scases\sfor\sbug\s[f3e5abed55]. +D 2010-07-30T10:09:12 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/vdbe.c cefff41564b68a412e65e6a1013ec1b1c1ece6c4 F src/vdbe.h 4de0efb4b0fdaaa900cf419b35c458933ef1c6d2 F src/vdbeInt.h ffd68c4d4229227a5089bec53a1c635146177abc F src/vdbeapi.c d0f4407e465f261780ad725c1caece7d66a6aa35 -F src/vdbeaux.c bb14feb1f210aec69889e145cc41b466319bfd2b +F src/vdbeaux.c 8a443e73760ca65ffdfda3e26df4c8c90eeefa11 F src/vdbeblob.c 258a6010ba7a82b72b327fb24c55790655689256 F src/vdbemem.c 5e579abf6532001dfbee0e640dc34eae897a9807 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 @@ -646,6 +646,7 @@ F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 F test/tkt-cbd054fa6b.test f14f97ea43662e6f70c9e63287081e8be5d9d589 F test/tkt-d11f09d36e.test fb44f7961aa6d4b632fb7b9768239832210b5fc7 F test/tkt-d82e3f3721.test 731359dfdcdb36fea0559cd33fec39dd0ceae8e6 +F test/tkt-f3e5abed55.test 91713833e266fbdc60f2030e05647ad4762073f6 F test/tkt-f777251dc7a.test 6f24c053bc5cdb7e1e19be9a72c8887cf41d5e87 F test/tkt-f973c7ac31.test 1da0ed15ec2c7749fb5ce2828cd69d07153ad9f4 F test/tkt-fc62af4523.test 72825d3febdedcd5593a27989fc05accdbfc2bb4 @@ -840,7 +841,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P fb847d70407b0f0e548919b7554f62bc1dab8a6c -R ed4b8e20864f45ad745b5c55b793d01a +P 50c0f2202d21bbf6b593d75fd20f13c0fac23eff +R 90c2b80ec44f603bca769c7fc17c6688 U dan -Z a4643485bfc292d567c95a06fb090017 +Z 1b5da3789844ce06cb648b709ee8c441 diff --git a/manifest.uuid b/manifest.uuid index 2866ee3183..b6eb3e72e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -50c0f2202d21bbf6b593d75fd20f13c0fac23eff \ No newline at end of file +cbbaf8e67a23dc332ce935bc7234246eacba60bf \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 46748a9530..295a8c3a8d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1802,6 +1802,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ } } sqlite3OsCloseFree(pMaster); + assert( rc!=SQLITE_BUSY ); if( rc!=SQLITE_OK ){ sqlite3DbFree(db, zMaster); return rc; diff --git a/test/tkt-f3e5abed55.test b/test/tkt-f3e5abed55.test new file mode 100644 index 0000000000..d574480b24 --- /dev/null +++ b/test/tkt-f3e5abed55.test @@ -0,0 +1,113 @@ +# 2010 July 29 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl + +foreach f [glob -nocomplain test.db*mj*] { file delete -force $f } +file delete -force test.db2 + +do_test tkt-f3e5abed55-1.1 { + execsql { + ATTACH 'test.db2' AS aux; + CREATE TABLE main.t1(a, b); + CREATE TABLE aux.t2(c, d); + } +} {} + +do_test tkt-f3e5abed55-1.2 { + glob -nocomplain test.db*mj* +} {} + +do_test tkt-f3e5abed55-1.3 { + sqlite3 db2 test.db + execsql { BEGIN; SELECT * FROM t1 } db2 +} {} + +do_test tkt-f3e5abed55-1.4 { + execsql { + BEGIN; + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t2 VALUES(1, 2); + } + catchsql COMMIT +} {1 {database is locked}} + +do_test tkt-f3e5abed55-1.5 { + execsql COMMIT db2 + execsql COMMIT +} {} + +do_test tkt-f3e5abed55-1.6 { + glob -nocomplain test.db*mj* +} {} +foreach f [glob -nocomplain test.db*mj*] { file delete -force $f } +db close +db2 close + + + +# Set up a testvfs so that the next time SQLite tries to delete the +# file "test.db-journal", a snapshot of the current file-system contents +# is taken. +# +testvfs tvfs -default 1 +tvfs script xDelete +tvfs filter xDelete +proc xDelete {method file args} { + if {[file tail $file] == "test.db-journal"} { + faultsim_save + tvfs filter {} + } + return "SQLITE_OK" +} + +sqlite3 db test.db +sqlite3 db2 test.db +do_test tkt-f3e5abed55-2.1 { + execsql { + ATTACH 'test.db2' AS aux; + BEGIN; + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t2 VALUES(3, 4); + } +} {} +do_test tkt-f3e5abed55-2.2 { + execsql { BEGIN; SELECT * FROM t1 } db2 +} {1 2} +do_test tkt-f3e5abed55-2.3 { + catchsql COMMIT +} {1 {database is locked}} + +do_test tkt-f3e5abed55-2.4 { + execsql COMMIT db2 + execsql { + COMMIT; + SELECT * FROM t1; + SELECT * FROM t2; + } +} {1 2 3 4 1 2 3 4} +do_test tkt-f3e5abed55-2.5 { + db close + db2 close + faultsim_restore_and_reopen + execsql { + ATTACH 'test.db2' AS aux; + SELECT * FROM t1; + SELECT * FROM t2; + } +} {1 2 3 4 1 2 3 4} + + +finish_test + From 51133eaefa1b9aeb676d581c6c1f08c8a2412c71 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Jul 2010 15:43:13 +0000 Subject: [PATCH 06/30] Remove variable Pager.needSync, which was almost completely unused. FossilOrigin-Name: 347f22a5b777af92873590a5b9af5a6498bef918 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/pager.c | 65 ++++++++++++++++++++------------------------------- 3 files changed, 32 insertions(+), 47 deletions(-) diff --git a/manifest b/manifest index c592de600e..fd0f62e526 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sexperimental\sagain. -D 2010-07-30T14:39:13 +C Remove\svariable\sPager.needSync,\swhich\swas\salmost\scompletely\sunused. +D 2010-07-30T15:43:13 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c d493b8164eee3339730983121cf1c6215af157aa +F src/pager.c 3e152b44733459c0a1f39d9a60646ab3aa4a7845 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -841,7 +841,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P cbbaf8e67a23dc332ce935bc7234246eacba60bf ad78ccacb00e2f8a5ca93e2c9006266d9477fe48 -R acbdc50f69dbd02f5f58643fea54f04c +P 87e0f4e184284bf775c2fc7c4e9a334f4f237c59 +R 3e31f33b6e6cf5ef4a3978e597fbc7d4 U dan -Z 94891c66bfdc0aff59f3a65e9a0b9b92 +Z 9b7608ba93c0b902202a5a1f630e015f diff --git a/manifest.uuid b/manifest.uuid index c68a24af75..d9b2afbfd1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -87e0f4e184284bf775c2fc7c4e9a334f4f237c59 \ No newline at end of file +347f22a5b777af92873590a5b9af5a6498bef918 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 6ebe39482a..e3b590d5ca 100644 --- a/src/pager.c +++ b/src/pager.c @@ -325,12 +325,6 @@ struct PagerSavepoint { ** doNotSpill value set to prevent pagerStress() from trying to use ** the journal during a rollback. ** -** needSync -** -** TODO: It might be easier to set this variable in writeJournalHdr() -** and writeMasterJournal() only. Change its meaning to "unsynced data -** has been written to the journal". -** ** subjInMemory ** ** This is a boolean variable. If true, then any required sub-journal @@ -363,7 +357,6 @@ struct Pager { */ u8 state; /* PAGER_UNLOCK, _SHARED, _RESERVED, etc. */ u8 dbModified; /* True if there are any changes to the Db */ - u8 needSync; /* True if an fsync() is needed on the journal */ u8 journalStarted; /* True if header of journal is synced */ u8 changeCountDone; /* Set after incrementing the change-counter */ u8 setMaster; /* True if a m-j name has been written to jrnl */ @@ -1119,7 +1112,6 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ return rc; } pPager->journalOff += (nMaster+20); - pPager->needSync = !pPager->noSync; /* If the pager is in peristent-journal mode, then the physical ** journal-file may extend past the end of the master-journal name @@ -1485,7 +1477,6 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ pPager->state = PAGER_EXCLUSIVE; } pPager->setMaster = 0; - pPager->needSync = 0; pPager->dbModified = 0; /* TODO: Is this optimal? Why is the db size invalidated here @@ -2681,7 +2672,6 @@ void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int bFullFsync){ pPager->noSync = (level==1 || pPager->tempFile) ?1:0; pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; pPager->sync_flags = (bFullFsync?SQLITE_SYNC_FULL:SQLITE_SYNC_NORMAL); - if( pPager->noSync ) pPager->needSync = 0; } #endif @@ -3181,9 +3171,9 @@ void sqlite3PagerRef(DbPage *pPg){ ** been written to the journal have actually reached the surface of the ** disk and can be restored in the event of a hot-journal rollback. ** -** If the Pager.needSync flag is not set, then this function is a -** no-op. Otherwise, the actions required depend on the journal-mode -** and the device characteristics of the the file-system, as follows: +** If the Pager.noSync flag is set, then this function is a no-op. +** Otherwise, the actions required depend on the journal-mode and the +** device characteristics of the the file-system, as follows: ** ** * If the journal file is an in-memory journal file, no action need ** be taken. @@ -3207,17 +3197,16 @@ void sqlite3PagerRef(DbPage *pPg){ ** if( NOT SEQUENTIAL ) xSync(); ** } ** -** The Pager.needSync flag is never be set for temporary files, or any -** file operating in no-sync mode (Pager.noSync set to non-zero). -** ** If successful, this routine clears the PGHDR_NEED_SYNC flag of every ** page currently held in memory before returning SQLITE_OK. If an IO ** error is encountered, then the IO error code is returned to the caller. */ static int syncJournal(Pager *pPager){ - if( pPager->needSync ){ + if( !pPager->noSync ){ assert( !pPager->tempFile ); - if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ + if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY + && pPager->journalMode!=PAGER_JOURNALMODE_OFF + ){ int rc; /* Return code */ const int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); assert( isOpen(pPager->jfd) ); @@ -3295,10 +3284,9 @@ static int syncJournal(Pager *pPager){ } } - /* The journal file was just successfully synced. Set Pager.needSync - ** to zero and clear the PGHDR_NEED_SYNC flag on all pagess. + /* The journal file was just successfully synced. Clear the + ** PGHDR_NEED_SYNC flag on all pagess. */ - pPager->needSync = 0; pPager->journalStarted = 1; pPager->journalHdr = pPager->journalOff; sqlite3PcacheClearSyncFlags(pPager->pPCache); @@ -3395,6 +3383,8 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ char *pData; /* Data to write */ + assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); + /* Encode the database */ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); @@ -3604,6 +3594,7 @@ static int pagerStress(void *p, PgHdr *pPg){ /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ + assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); rc = pager_write_pagelist(pPager, pPg); } } @@ -3885,7 +3876,6 @@ int sqlite3PagerOpen( pPager->changeCountDone = pPager->tempFile; pPager->memDb = (u8)memDb; pPager->readOnly = (u8)readOnly; - /* pPager->needSync = 0; */ assert( useJournal || pPager->tempFile ); pPager->noSync = pPager->tempFile; pPager->fullSync = pPager->noSync ?0:1; @@ -4550,7 +4540,6 @@ static int pager_open_journal(Pager *pPager){ /* TODO: Check if all of these are really required. */ pPager->dbOrigSize = pPager->dbSize; pPager->journalStarted = 0; - pPager->needSync = 0; pPager->nRec = 0; pPager->journalOff = 0; pPager->setMaster = 0; @@ -4770,7 +4759,6 @@ static int pager_write(PgHdr *pPg){ */ if( !pPager->noSync ){ pPg->flags |= PGHDR_NEED_SYNC; - pPager->needSync = 1; } /* An error has occurred writing to the journal file. The @@ -4793,7 +4781,6 @@ static int pager_write(PgHdr *pPg){ }else{ if( !pPager->journalStarted && !pPager->noSync ){ pPg->flags |= PGHDR_NEED_SYNC; - pPager->needSync = 1; } PAGERTRACE(("APPEND %d page %d needSync=%d\n", PAGERID(pPager), pPg->pgno, @@ -4886,7 +4873,6 @@ int sqlite3PagerWrite(DbPage *pDbPage){ rc = pager_write(pPage); if( pPage->flags&PGHDR_NEED_SYNC ){ needSync = 1; - assert(pPager->needSync); } sqlite3PagerUnref(pPage); } @@ -4914,7 +4900,6 @@ int sqlite3PagerWrite(DbPage *pDbPage){ sqlite3PagerUnref(pPage); } } - assert(pPager->needSync); } assert( pPager->doNotSyncSpill==1 ); @@ -5245,14 +5230,20 @@ int sqlite3PagerCommitPhaseOne( rc = writeMasterJournal(pPager, zMaster); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* Sync the journal file. If the atomic-update optimization is being - ** used, this call will not create the journal file or perform any - ** real IO. + /* Sync the journal file and write all dirty pages to the database. + ** If the atomic-update optimization is being used, this sync will not + ** create the journal file or perform any real IO. + ** + ** Because the change-counter page was just modified, unless the + ** atomic-update optimization is used it is almost certain that the + ** journal requires a sync here. However, in locking_mode=exclusive + ** on a system under memory pressure it is just possible that this is + ** not the case. In this case it is likely enough that the redundant + ** xSync() call will be changed to a no-op by the OS anyhow. */ rc = syncJournal(pPager); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - - /* Write all dirty pages to the database file. */ + rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); @@ -5766,11 +5757,10 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ needSyncPgno = pPg->pgno; assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); assert( pPg->flags&PGHDR_DIRTY ); - assert( pPager->needSync ); } /* If the cache contains a page with page-number pgno, remove it - ** from its hash chain. Also, if the PgHdr.needSync was set for + ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for ** page pgno before the 'move' operation, it needs to be retained ** for the page moved there. */ @@ -5799,7 +5789,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** sync()ed before any data is written to database file page needSyncPgno. ** Currently, no such page exists in the page-cache and the ** "is journaled" bitvec flag has been set. This needs to be remedied by - ** loading the page into the pager-cache and setting the PgHdr.needSync + ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC ** flag. ** ** If the attempt to load the page into the page-cache fails, (due @@ -5808,12 +5798,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** this transaction, it may be written to the database file before ** it is synced into the journal file. This way, it may end up in ** the journal file twice, but that is not a problem. - ** - ** The sqlite3PagerGet() call may cause the journal to sync. So make - ** sure the Pager.needSync flag is set too. */ PgHdr *pPgHdr; - assert( pPager->needSync ); rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); if( rc!=SQLITE_OK ){ if( needSyncPgno<=pPager->dbOrigSize ){ @@ -5822,7 +5808,6 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ } return rc; } - pPager->needSync = 1; assert( pPager->noSync==0 && !MEMDB ); pPgHdr->flags |= PGHDR_NEED_SYNC; sqlite3PcacheMakeDirty(pPgHdr); From d08640871c85800f5953c5e32a1dcfc6b2af06b6 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 2 Aug 2010 14:32:52 +0000 Subject: [PATCH 07/30] Experimental refactoring of the Pager object state. FossilOrigin-Name: 03a240514aa07a22db787d221641253f23933e88 --- manifest | 18 +- manifest.uuid | 2 +- src/pager.c | 806 ++++++++++++++++++++++++++++---------------- test/analyze3.test | 1 - test/exclusive.test | 6 + test/wal2.test | 18 +- 6 files changed, 546 insertions(+), 305 deletions(-) diff --git a/manifest b/manifest index fd0f62e526..d8a6e2a31d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\svariable\sPager.needSync,\swhich\swas\salmost\scompletely\sunused. -D 2010-07-30T15:43:13 +C Experimental\srefactoring\sof\sthe\sPager\sobject\sstate. +D 2010-08-02T14:32:52 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 3e152b44733459c0a1f39d9a60646ab3aa4a7845 +F src/pager.c 94f00bedf9afb83c47f9b5d8bf8608f0eff088dd F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -241,7 +241,7 @@ F test/alter4.test 9386ffd1e9c7245f43eca412b2058d747509cc1f F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/analyze.test bf692e7db414f268a136bade16c03a1bdbb9240c F test/analyze2.test 59dac6c399c0c5d1a90a11ee7cc606743fb6db93 -F test/analyze3.test 535bf0762f49fa96885efe8568738276c2204a2a +F test/analyze3.test 6d4f4b0929545a9d1af803a0608a0c51b92a3537 F test/async.test ad4ba51b77cd118911a3fe1356b0809da9c108c3 F test/async2.test bf5e2ca2c96763b4cba3d016249ad7259a5603b6 F test/async3.test 93edaa9122f498e56ea98c36c72abc407f4fb11e @@ -347,7 +347,7 @@ F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 6d91a5286f59add0cfcbb2d0da913b76f2242398 F test/enc3.test 5c550d59ff31dccdba5d1a02ae11c7047d77c041 F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 -F test/exclusive.test 5fe18e10a159342dd52ca14b1554e33f98734267 +F test/exclusive.test b1f9012cabc124af947165d15ffa62ad20f63db8 F test/exclusive2.test fcbb1c9ca9739292a0a22a3763243ad6d868086b F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/expr.test 9f521ae22f00e074959f72ce2e55d46b9ed23f68 @@ -782,7 +782,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test 1891e6f72dd437a1c2a48091aa9182ba17a8f780 -F test/wal2.test fa6dc4457b46988f46cf6c68ea51ebe341765f4a +F test/wal2.test 8581b133ef58d48f24c56f645a20d6527723557c F test/wal3.test 695ea0f6c516423c611891df9a285aacd33344e3 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 F test/wal_common.tcl 895d76138043b86bdccf36494054bdabcf65837b @@ -841,7 +841,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 87e0f4e184284bf775c2fc7c4e9a334f4f237c59 -R 3e31f33b6e6cf5ef4a3978e597fbc7d4 +P 347f22a5b777af92873590a5b9af5a6498bef918 +R fb51b6835fc754e61c37a4fea5d6298b U dan -Z 9b7608ba93c0b902202a5a1f630e015f +Z 550ac78d14ba82c7a2ec704cc3fbe702 diff --git a/manifest.uuid b/manifest.uuid index d9b2afbfd1..0099265206 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -347f22a5b777af92873590a5b9af5a6498bef918 \ No newline at end of file +03a240514aa07a22db787d221641253f23933e88 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e3b590d5ca..947194927f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -125,6 +125,72 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ #define PAGERID(p) ((int)(p->fd)) #define FILEHANDLEID(fd) ((int)fd) +/* +** The Pager.eState variable stores the current 'state' of a pager. A +** pager may be in any one of the following six states: +** +** NONE: +** * No read or write transaction is active. +** * Any lock, or no lock at all, may be held on the database file. +** +** READER: +** * A read transaction is active. +** * A SHARED or greater lock is held on the database file. +** * The dbSize variable is valid. +** +** WRITER_INITIAL: +** * A write transaction is active. +** * A RESERVED or greater lock is held on the database file. +** * The dbSize, dbOrigSize and dbFileSize variables are all valid. +** * The contents of the pager cache have not been modified. +** +** WRITER_CACHEMOD: +** * A write transaction is active. +** * A RESERVED or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** to it, but the header has not been synced to disk. +** * The contents of the page cache have been modified. +** +** WRITER_DBMOD: +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** and synced to disk. +** * The contents of the page cache have been modified (and possibly +** written to disk). +** +** WRITER_FINISHED: +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * All writing and syncing of journal and database data has finished. +** If no error occured, all that remains is to finalize the journal to +** commit the transaction. If an error did occur, the caller will need +** to rollback the transaction. +** +** +** Allowable transitions and the [function] that performe each: +** +** NONE -> READER [PagerSharedLock] +** READER -> WRITER_INITIAL [PagerBegin] +** WRITER_INITIAL -> WRITER_CACHEMOD [pager_open_journal] +** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] +** WRITER_DBMOD -> WRITER_FINISHED [PagerCommitPhaseOne] +** +** WRITER_INITIAL -> READER [pager_end_transaction] +** WRITER_CACHEMOD -> READER [pager_end_transaction] +** WRITER_DBMOD -> READER [pager_end_transaction] +** WRITER_FINISHED -> READER [pager_end_transaction] +** +** READER -> NONE [pager_unlock] +*/ +#define PAGER_NONE 0 +#define PAGER_READER 1 +#define PAGER_WRITER_INITIAL 2 +#define PAGER_WRITER_CACHEMOD 3 +#define PAGER_WRITER_DBMOD 4 +#define PAGER_WRITER_FINISHED 5 + + /* ** The page cache as a whole is always in one of the following ** states: @@ -355,14 +421,19 @@ struct Pager { ** other variables in this block are described in the comment directly ** above this class definition. */ +#if 0 u8 state; /* PAGER_UNLOCK, _SHARED, _RESERVED, etc. */ u8 dbModified; /* True if there are any changes to the Db */ u8 journalStarted; /* True if header of journal is synced */ +#endif + u8 dbSizeValid; /* Set when dbSize is correct */ + u8 eState; /* Pager state (NONE, READER, WRITER_INITIAL..) */ + u8 eLock; /* Current lock held on database file */ + u8 changeCountDone; /* Set after incrementing the change-counter */ u8 setMaster; /* True if a m-j name has been written to jrnl */ u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 doNotSyncSpill; /* Do not do a spill that requires jrnl sync */ - u8 dbSizeValid; /* Set when dbSize is correct */ u8 subjInMemory; /* True to use in-memory sub-journals */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ @@ -484,22 +555,151 @@ static const unsigned char aJournalMagic[] = { */ #define PAGER_MAX_PGNO 2147483647 +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. +** +** This is so that expressions can be written as: +** +** if( isOpen(pPager->jfd) ){ ... +** +** instead of +** +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods) + +/* +** Return true if this pager uses a write-ahead log instead of the usual +** rollback journal. Otherwise false. +*/ +#ifndef SQLITE_OMIT_WAL +static int pagerUseWal(Pager *pPager){ + return (pPager->pWal!=0); +} +#else +# define pagerUseWal(x) 0 +# define pagerRollbackWal(x) 0 +# define pagerWalFrames(v,w,x,y,z) 0 +# define pagerOpenWalIfPresent(z) SQLITE_OK +# define pagerBeginReadTransaction(z) SQLITE_OK +#endif + #ifndef NDEBUG /* ** Usage: ** ** assert( assert_pager_state(pPager) ); */ -static int assert_pager_state(Pager *pPager){ +static int assert_pager_state(Pager *p){ + Pager *pPager = p; - /* A temp-file is always in PAGER_EXCLUSIVE or PAGER_SYNCED state. */ - assert( pPager->tempFile==0 || pPager->state>=PAGER_EXCLUSIVE ); + /* State must be valid. */ + assert( p->eState==PAGER_NONE + || p->eState==PAGER_READER + || p->eState==PAGER_WRITER_INITIAL + || p->eState==PAGER_WRITER_CACHEMOD + || p->eState==PAGER_WRITER_DBMOD + || p->eState==PAGER_WRITER_FINISHED + ); - /* The changeCountDone flag is always set for temp-files */ - assert( pPager->tempFile==0 || pPager->changeCountDone ); + /* Regardless of the current state, a temp-file connection always behaves + ** as if it has an exclusive lock on the database file. It never updates + ** the change-counter field, so the changeCountDone flag is always set. + */ + assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); + assert( p->tempFile==0 || pPager->changeCountDone ); + + /* If the useJournal flag is clear, the journal-mode must be "OFF". + ** And if the journal-mode is "OFF", the journal file must not be open. + */ + assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); + assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); + + /* Check that MEMDB implies noSync. */ + assert( !MEMDB || p->noSync ); + + switch( p->eState ){ + case PAGER_NONE: + assert( !MEMDB ); + assert( !p->tempFile ); + assert( !pagerUseWal(pPager) ); + break; + + case PAGER_READER: + assert( p->eLock>=SHARED_LOCK || p->noReadlock ); + break; + + case PAGER_WRITER_INITIAL: + if( !pagerUseWal(pPager) ){ + assert( p->eLock>=RESERVED_LOCK ); + } + break; + + case PAGER_WRITER_CACHEMOD: + if( !pagerUseWal(pPager) ){ + /* It is possible that if journal_mode=wal here that neither the + ** journal file nor the WAL file are open. This happens during + ** a rollback transaction that switches from journal_mode=off + ** to journal_mode=wal. + */ + assert( p->eLock>=RESERVED_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + } + break; + + case PAGER_WRITER_DBMOD: + assert( !pagerUseWal(pPager) ); + assert( p->eLock>=EXCLUSIVE_LOCK || pagerUseWal(pPager) ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + break; + + case PAGER_WRITER_FINISHED: + assert( !pagerUseWal(pPager) ); + assert( p->eLock>=EXCLUSIVE_LOCK || pagerUseWal(pPager) ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + break; + } return 1; } + +/* +** (gdb) printf "%s", print_pager_state(pPager) +*/ +static char *print_pager_state(Pager *p){ + static char zRet[1024]; + + sqlite3_snprintf(1024, zRet, + "State: %s\n" + "Lock: %s\n" + "Locking mode: locking_mode=%s\n" + "Backing store: tempFile=%d memDb=%d\n" + , p->eState==PAGER_NONE ? "NONE" : + p->eState==PAGER_READER ? "READER" : + p->eState==PAGER_WRITER_INITIAL ? "WRITER_INITIAL" : + p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : + p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : + p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : "?error?" + , p->eLock==NO_LOCK ? "NONE" : + p->eLock==RESERVED_LOCK ? "RESERVED" : + p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : + p->eLock==SHARED_LOCK ? "SHARED" : "?error?" + , p->exclusiveMode ? "exclusive" : "normal" + , (int)p->tempFile, (int)p->memDb + ); + + return zRet; +} #endif /* @@ -552,6 +752,7 @@ static int read32bits(sqlite3_file *fd, i64 offset, u32 *pRes){ */ #define put32bits(A,B) sqlite3Put4byte((u8*)A,B) + /* ** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK ** on success or an error code is something goes wrong. @@ -562,28 +763,17 @@ static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ return sqlite3OsWrite(fd, ac, 4, offset); } -/* -** The argument to this macro is a file descriptor (type sqlite3_file*). -** Return 0 if it is not open, or non-zero (but not 1) if it is. -** -** This is so that expressions can be written as: -** -** if( isOpen(pPager->jfd) ){ ... -** -** instead of -** -** if( pPager->jfd->pMethods ){ ... -*/ -#define isOpen(pFd) ((pFd)->pMethods) - /* ** If file pFd is open, call sqlite3OsUnlock() on it. */ -static int osUnlock(sqlite3_file *pFd, int eLock){ - if( !isOpen(pFd) ){ +static int osUnlock(Pager *pPager, int eLock){ + if( !isOpen(pPager->fd) ){ return SQLITE_OK; } - return sqlite3OsUnlock(pFd, eLock); + assert( pPager->eLock>=eLock ); + assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); + pPager->eLock = eLock; + return sqlite3OsUnlock(pPager->fd, eLock); } /* @@ -875,7 +1065,7 @@ static int writeJournalHdr(Pager *pPager){ ** that garbage data is never appended to the journal file. */ assert( isOpen(pPager->fd) || pPager->noSync ); - if( (pPager->noSync) || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) + if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) ){ memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); @@ -1075,6 +1265,8 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ u32 cksum = 0; /* Checksum of string zMaster */ assert( pPager->setMaster==0 ); + assert( !zMaster || pPager->journalMode!=PAGER_JOURNALMODE_WAL ); + assert( !pagerUseWal(pPager) ); if( !zMaster || pPager->journalMode==PAGER_JOURNALMODE_MEMORY @@ -1199,22 +1391,6 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ return rc; } -/* -** Return true if this pager uses a write-ahead log instead of the usual -** rollback journal. Otherwise false. -*/ -#ifndef SQLITE_OMIT_WAL -static int pagerUseWal(Pager *pPager){ - return (pPager->pWal!=0); -} -#else -# define pagerUseWal(x) 0 -# define pagerRollbackWal(x) 0 -# define pagerWalFrames(v,w,x,y,z) 0 -# define pagerOpenWalIfPresent(z) SQLITE_OK -# define pagerBeginReadTransaction(z) SQLITE_OK -#endif - /* ** Unlock the database file. This function is a no-op if the pager ** is in exclusive mode. @@ -1241,6 +1417,7 @@ static void pager_unlock(Pager *pPager){ assert( (PAGER_JOURNALMODE_DELETE & 5)!=1 ); assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); + if( 0==(iDc & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN) || 1!=(pPager->journalMode & 5) ){ @@ -1262,7 +1439,7 @@ static void pager_unlock(Pager *pPager){ if( pagerUseWal(pPager) ){ sqlite3WalEndReadTransaction(pPager->pWal); }else{ - rc = osUnlock(pPager->fd, NO_LOCK); + rc = osUnlock(pPager, NO_LOCK); } if( rc ){ pPager->errCode = rc; @@ -1281,8 +1458,7 @@ static void pager_unlock(Pager *pPager){ } pPager->changeCountDone = 0; - pPager->state = PAGER_UNLOCK; - pPager->dbModified = 0; + pPager->eState = PAGER_NONE; } } @@ -1319,32 +1495,6 @@ static int pager_error(Pager *pPager, int rc){ return rc; } -/* -** Execute a rollback if a transaction is active and unlock the -** database file. -** -** If the pager has already entered the error state, do not attempt -** the rollback at this time. Instead, pager_unlock() is called. The -** call to pager_unlock() will discard all in-memory pages, unlock -** the database file and clear the error state. If this means that -** there is a hot-journal left in the file-system, the next connection -** to obtain a shared lock on the pager (which may be this one) will -** roll it back. -** -** If the pager has not already entered the error state, but an IO or -** malloc error occurs during a rollback, then this will itself cause -** the pager to enter the error state. Which will be cleared by the -** call to pager_unlock(), as described above. -*/ -static void pagerUnlockAndRollback(Pager *pPager){ - if( pPager->errCode==SQLITE_OK && pPager->state>=PAGER_RESERVED ){ - sqlite3BeginBenignMalloc(); - sqlite3PagerRollback(pPager); - sqlite3EndBenignMalloc(); - } - pager_unlock(pPager); -} - /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -1405,11 +1555,13 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ int rc = SQLITE_OK; /* Error code from journal finalization operation */ int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ - if( pPager->stateeStateeLockjfd) || pPager->pInJournal==0 ); if( isOpen(pPager->jfd) ){ assert( !pagerUseWal(pPager) ); @@ -1425,14 +1577,12 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ rc = sqlite3OsTruncate(pPager->jfd, 0); } pPager->journalOff = 0; - pPager->journalStarted = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) ){ rc = zeroJournalHdr(pPager, hasMaster); pager_error(pPager, rc); pPager->journalOff = 0; - pPager->journalStarted = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal @@ -1457,39 +1607,58 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ pPager->pInJournal = 0; pPager->nRec = 0; sqlite3PcacheCleanAll(pPager->pPCache); + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( pagerUseWal(pPager) ){ + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE + ** lock held on the database file. + */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); assert( rc2==SQLITE_OK ); - pPager->state = PAGER_SHARED; - - /* If the connection was in locking_mode=exclusive mode but is no longer, - ** drop the EXCLUSIVE lock held on the database file. - */ if( !pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, 0) ){ - rc2 = osUnlock(pPager->fd, SHARED_LOCK); + rc2 = osUnlock(pPager, SHARED_LOCK); } }else if( !pPager->exclusiveMode ){ - rc2 = osUnlock(pPager->fd, SHARED_LOCK); - pPager->state = PAGER_SHARED; + rc2 = osUnlock(pPager, SHARED_LOCK); pPager->changeCountDone = 0; - }else if( pPager->state==PAGER_SYNCED ){ - pPager->state = PAGER_EXCLUSIVE; } + pPager->eState = PAGER_READER; pPager->setMaster = 0; - pPager->dbModified = 0; - - /* TODO: Is this optimal? Why is the db size invalidated here - ** when the database file is not unlocked? */ - pPager->dbOrigSize = 0; - sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); - if( !MEMDB ){ - pPager->dbSizeValid = 0; - } return (rc==SQLITE_OK?rc2:rc); } +/* +** Execute a rollback if a transaction is active and unlock the +** database file. +** +** If the pager has already entered the error state, do not attempt +** the rollback at this time. Instead, pager_unlock() is called. The +** call to pager_unlock() will discard all in-memory pages, unlock +** the database file and clear the error state. If this means that +** there is a hot-journal left in the file-system, the next connection +** to obtain a shared lock on the pager (which may be this one) will +** roll it back. +** +** If the pager has not already entered the error state, but an IO or +** malloc error occurs during a rollback, then this will itself cause +** the pager to enter the error state. Which will be cleared by the +** call to pager_unlock(), as described above. +*/ +static void pagerUnlockAndRollback(Pager *pPager){ + if( pPager->errCode==SQLITE_OK ){ + if( pPager->eState>=PAGER_WRITER_INITIAL ){ + sqlite3BeginBenignMalloc(); + sqlite3PagerRollback(pPager); + sqlite3EndBenignMalloc(); + }else if( pPager->eLock>=RESERVED_LOCK && !pPager->exclusiveMode ){ + pager_end_transaction(pPager, 0); + } + } + pager_unlock(pPager); +} + /* ** Parameter aData must point to a buffer of pPager->pageSize bytes ** of data. Compute and return a checksum based ont the contents of the @@ -1632,7 +1801,7 @@ static int pager_playback_one_page( if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ return rc; } - assert( pPager->state==PAGER_RESERVED || pPager->state>=PAGER_EXCLUSIVE ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); /* When playing back page 1, restore the nReserve setting */ @@ -1688,7 +1857,7 @@ static int pager_playback_one_page( }else{ isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); } - if( (pPager->state>=PAGER_EXCLUSIVE) + if( (pPager->eState>=PAGER_WRITER_DBMOD) && isOpen(pPager->fd) && isSynced ){ @@ -1936,7 +2105,7 @@ delmaster_out: */ static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; - if( pPager->state>=PAGER_EXCLUSIVE && isOpen(pPager->fd) ){ + if( pPager->eState>=PAGER_WRITER_DBMOD && isOpen(pPager->fd) ){ i64 currentSize, newSize; /* TODO: Is it safe to use Pager.dbFileSize here? */ rc = sqlite3OsFileSize(pPager->fd, ¤tSize); @@ -2056,10 +2225,13 @@ static int pager_playback(Pager *pPager, int isHot){ char *zMaster = 0; /* Name of master journal file if any */ int needPagerReset; /* True to reset page prior to first page rollback */ + if( !isOpen(pPager->jfd) ){ + return SQLITE_OK; + } + /* Figure out how many records are in the journal. Abort early if ** the journal is empty. */ - assert( isOpen(pPager->jfd) ); rc = sqlite3OsFileSize(pPager->jfd, &szJ); if( rc!=SQLITE_OK || szJ==0 ){ goto end_playback; @@ -2209,10 +2381,10 @@ end_playback: rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); testcase( rc!=SQLITE_OK ); } - if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ + if( rc==SQLITE_OK && !pPager->noSync && pPager->eState>=PAGER_WRITER_DBMOD ){ rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); } - if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ + if( rc==SQLITE_OK && !pPager->noSync && pPager->eState>=PAGER_WRITER_DBMOD ){ rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); } if( rc==SQLITE_OK ){ @@ -2254,7 +2426,7 @@ static int readDbPage(PgHdr *pPg){ int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ - assert( pPager->state>=PAGER_SHARED && !MEMDB ); + assert( pPager->eState>=PAGER_READER && !MEMDB ); assert( isOpen(pPager->fd) ); if( NEVER(!isOpen(pPager->fd)) ){ @@ -2434,7 +2606,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ } rc = sqlite3PagerPagecount(pPager, &dummy); } - pPager->state = PAGER_SHARED; + pPager->eState = PAGER_READER; return rc; } @@ -2464,7 +2636,8 @@ static int pagerOpenWalIfPresent(Pager *pPager){ if( !pPager->tempFile ){ int isWal; /* True if WAL file exists */ int nPage; /* Size of the database file */ - assert( pPager->state>=SHARED_LOCK ); + + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); rc = sqlite3PagerPagecount(pPager, &nPage); if( rc ) return rc; if( nPage==0 ){ @@ -2533,7 +2706,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ int rc = SQLITE_OK; /* Return code */ Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ - assert( pPager->state>=PAGER_SHARED ); + assert( pPager->eState>=PAGER_WRITER_INITIAL ); /* Allocate a bitvec to use to store the set of pages rolled back */ if( pSavepoint ){ @@ -2831,7 +3004,7 @@ int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ if( mxPage>0 ){ pPager->mxPgno = mxPage; } - if( pPager->state!=PAGER_UNLOCK ){ + if( pPager->eState!=PAGER_NONE ){ sqlite3PagerPagecount(pPager, &nPage); assert( (int)pPager->mxPgno>=nPage ); } @@ -2922,7 +3095,7 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ int rc; /* Error returned by OsFileSize() */ i64 n = 0; /* File size in bytes returned by OsFileSize() */ - if( pagerUseWal(pPager) && pPager->state!=PAGER_UNLOCK ){ + if( pagerUseWal(pPager) && pPager->eState!=PAGER_NONE ){ sqlite3WalDbsize(pPager->pWal, &nPage); } @@ -2940,7 +3113,7 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ nPage = (Pgno)(n / pPager->pageSize); } } - if( pPager->state!=PAGER_UNLOCK ){ + if( pPager->eState!=PAGER_NONE ){ pPager->dbSize = nPage; pPager->dbFileSize = nPage; pPager->dbSizeValid = 1; @@ -2961,6 +3134,20 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ } +static int pagerLock(Pager *pPager, int eLock){ + int rc; + assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); + if( pPager->eLock>=eLock ){ + rc = SQLITE_OK; + }else{ + rc = sqlite3OsLock(pPager->fd, eLock); + if( rc==SQLITE_OK ){ + pPager->eLock = eLock; + } + } + return rc; +} + /* ** Try to obtain a lock of type locktype on the database file. If ** a similar or greater lock is already held, this function is a no-op @@ -2983,30 +3170,24 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ assert( PAGER_RESERVED==RESERVED_LOCK ); assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK ); - /* If the file is currently unlocked then the size must be unknown. It - ** must not have been modified at this point. - */ - assert( pPager->state>=PAGER_SHARED || pPager->dbSizeValid==0 ); - assert( pPager->state>=PAGER_SHARED || pPager->dbModified==0 ); - /* Check that this is either a no-op (because the requested lock is ** already held, or one of the transistions that the busy-handler ** may be invoked during, according to the comment above ** sqlite3PagerSetBusyhandler(). */ - assert( (pPager->state>=locktype) - || (pPager->state==PAGER_UNLOCK && locktype==PAGER_SHARED) - || (pPager->state==PAGER_RESERVED && locktype==PAGER_EXCLUSIVE) + assert( (pPager->eLock>=locktype) + || (pPager->eLock==NO_LOCK && locktype==SHARED_LOCK) + || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) ); - if( pPager->state>=locktype ){ + if( pPager->eLock>=locktype ){ rc = SQLITE_OK; }else{ do { rc = sqlite3OsLock(pPager->fd, locktype); }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); if( rc==SQLITE_OK ){ - pPager->state = (u8)locktype; + pPager->eLock = (u8)locktype; IOTRACE(("LOCK %p %d\n", pPager, locktype)) } } @@ -3056,7 +3237,7 @@ static void assertTruncateConstraint(Pager *pPager){ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ assert( pPager->dbSizeValid ); assert( pPager->dbSize>=nPage ); - assert( pPager->state>=PAGER_RESERVED ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); pPager->dbSize = nPage; assertTruncateConstraint(pPager); } @@ -3202,12 +3383,19 @@ void sqlite3PagerRef(DbPage *pPg){ ** error is encountered, then the IO error code is returned to the caller. */ static int syncJournal(Pager *pPager){ + int rc; /* Return code */ + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + + rc = sqlite3PagerExclusiveLock(pPager); + if( rc!=SQLITE_OK ) return rc; + if( !pPager->noSync ){ assert( !pPager->tempFile ); - if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY - && pPager->journalMode!=PAGER_JOURNALMODE_OFF - ){ - int rc; /* Return code */ + if( isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ const int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); assert( isOpen(pPager->jfd) ); @@ -3284,14 +3472,16 @@ static int syncJournal(Pager *pPager){ } } - /* The journal file was just successfully synced. Clear the - ** PGHDR_NEED_SYNC flag on all pagess. - */ - pPager->journalStarted = 1; pPager->journalHdr = pPager->journalOff; - sqlite3PcacheClearSyncFlags(pPager->pPCache); } + /* Unless the pager is in noSync mode, the journal file was just + ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on + ** all pages. + */ + sqlite3PcacheClearSyncFlags(pPager->pPCache); + pPager->eState = PAGER_WRITER_DBMOD; + assert( assert_pager_state(pPager) ); return SQLITE_OK; } @@ -3346,8 +3536,21 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** EXCLUSIVE, it means the database file has been changed and any rollback ** will require a journal playback. */ + + /* Normally, this function is called in WRITER_DBMOD state. + ** + ** However it may be called in WRITER_CACHEMOD state if the page being + ** written (and all other pages that reside on the same disk sector) was + ** a free-list leaf page at the start of the transaction. In that case + ** the database file is not really being modified, so it is Ok to write + ** to it in CACHEMOD state. + */ assert( !pagerUseWal(pPager) ); - assert( pPager->state>=PAGER_RESERVED ); + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( pPager->eState==PAGER_WRITER_DBMOD || pList->pDirty==0 ); + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); /* If the file is a temp-file has not yet been opened, open it now. It @@ -3548,9 +3751,9 @@ static int pagerStress(void *p, PgHdr *pPg){ /* Sync the journal file if required. */ if( pPg->flags&PGHDR_NEED_SYNC ){ - assert( !pPager->noSync ); rc = syncJournal(pPager); if( rc==SQLITE_OK && + !pPager->noSync && !(pPager->journalMode==PAGER_JOURNALMODE_MEMORY) && !(sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) ){ @@ -3823,7 +4026,8 @@ int sqlite3PagerOpen( ** disk and uses an in-memory rollback journal. */ tempFile = 1; - pPager->state = PAGER_EXCLUSIVE; + pPager->eState = PAGER_READER; + pPager->eLock = EXCLUSIVE_LOCK; readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } @@ -3866,7 +4070,9 @@ int sqlite3PagerOpen( /* pPager->nPage = 0; */ pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; /* pPager->state = PAGER_UNLOCK; */ +#if 0 assert( pPager->state == (tempFile ? PAGER_EXCLUSIVE : PAGER_UNLOCK) ); +#endif /* pPager->errMask = 0; */ pPager->tempFile = (u8)tempFile; assert( tempFile==PAGER_LOCKINGMODE_NORMAL @@ -3943,7 +4149,8 @@ static int hasHotJournal(Pager *pPager, int *pExists){ assert( pPager!=0 ); assert( pPager->useJournal ); assert( isOpen(pPager->fd) ); - assert( pPager->state <= PAGER_SHARED ); + assert( pPager->eState<=PAGER_SHARED ); + assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN )); @@ -4079,28 +4286,27 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( pagerUseWal(pPager) ){ rc = pagerBeginReadTransaction(pPager); - }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ + }else if( pPager->eState==PAGER_NONE || isErrorReset ){ sqlite3_vfs * const pVfs = pPager->pVfs; int isHotJournal = 0; - assert( !MEMDB ); + + assert( !MEMDB && !pPager->tempFile ); assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); - if( pPager->noReadlock ){ - assert( pPager->readOnly ); - pPager->state = PAGER_SHARED; - }else{ + assert( pPager->noReadlock==0 || pPager->readOnly ); + + if( pPager->noReadlock==0 ){ rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ - assert( pPager->state==PAGER_UNLOCK ); + assert( pPager->eLock==PAGER_UNLOCK ); return pager_error(pPager, rc); } } - assert( pPager->state>=SHARED_LOCK ); + pPager->eState = PAGER_READER; /* If a journal file exists, and there is no RESERVED lock on the ** database file, then it either needs to be played back or deleted. */ if( !isErrorReset ){ - assert( pPager->state <= PAGER_SHARED ); rc = hasHotJournal(pPager, &isHotJournal); if( rc!=SQLITE_OK ){ goto failed; @@ -4118,21 +4324,28 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** other process attempting to access the database file will get to ** this point in the code and fail to obtain its own EXCLUSIVE lock ** on the database file. + ** + ** Unless the pager is in locking_mode=exclusive mode, the lock is + ** downgraded to SHARED_LOCK before this function returns. */ - if( pPager->statefd, EXCLUSIVE_LOCK); - if( rc!=SQLITE_OK ){ - rc = pager_error(pPager, rc); - goto failed; - } - pPager->state = PAGER_EXCLUSIVE; + rc = pagerLock(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + rc = pager_error(pPager, rc); + goto failed; } - /* Open the journal for read/write access. This is because in - ** exclusive-access mode the file descriptor will be kept open and - ** possibly used for a transaction later on. On some systems, the - ** OsTruncate() call used in exclusive-access mode also requires - ** a read/write file handle. + /* If it is not already open and the file exists on disk, open the + ** journal for read/write access. Write access is required because + ** in exclusive-access mode the file descriptor will be kept open + ** and possibly used for a transaction later on. Also, write-access + ** is usually required to finalize the journal in journal_mode=persist + ** mode (and also for journal_mode=truncate on some systems). + ** + ** If the journal does not exist, it usually means that some + ** other connection managed to get in and roll it back before + ** this connection obtained the exclusive lock above. Or, it + ** may mean that the pager was in the error-state when this + ** function was called and the journal file does not exist. */ if( !isOpen(pPager->jfd) ){ int res; @@ -4148,13 +4361,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = SQLITE_CANTOPEN_BKPT; sqlite3OsClose(pPager->jfd); } - }else{ - /* If the journal does not exist, it usually means that some - ** other connection managed to get in and roll it back before - ** this connection obtained the exclusive lock above. Or, it - ** may mean that the pager was in the error-state when this - ** function was called and the journal file does not exist. */ - rc = pager_end_transaction(pPager, 0); } } } @@ -4164,7 +4370,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ /* Reset the journal status fields to indicates that we have no ** rollback journal at this time. */ - pPager->journalStarted = 0; pPager->journalOff = 0; pPager->setMaster = 0; pPager->journalHdr = 0; @@ -4182,15 +4387,21 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( isOpen(pPager->jfd) ){ rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ + pPager->eState = PAGER_WRITER_FINISHED; rc = pager_playback(pPager, 1); + pPager->eState = PAGER_READER; } if( rc!=SQLITE_OK ){ rc = pager_error(pPager, rc); goto failed; } + }else{ + osUnlock(pPager, SHARED_LOCK); } - assert( (pPager->state==PAGER_SHARED) - || (pPager->exclusiveMode && pPager->state>PAGER_SHARED) + + assert( pPager->eState==PAGER_READER ); + assert( (pPager->eLock==SHARED_LOCK) + || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) ); } @@ -4234,7 +4445,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ pager_reset(pPager); } } - assert( pPager->exclusiveMode || pPager->state==PAGER_SHARED ); + assert( pPager->eState==PAGER_READER ); /* If there is a WAL file in the file-system, open this database in WAL ** mode. Otherwise, the following function call is a no-op. @@ -4325,8 +4536,8 @@ int sqlite3PagerAcquire( int rc; PgHdr *pPg; + assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); - assert( pPager->state>PAGER_UNLOCK ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -4444,7 +4655,7 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pPager!=0 ); assert( pgno!=0 ); assert( pPager->pPCache!=0 ); - assert( pPager->state > PAGER_UNLOCK ); + assert( pPager->eState>=PAGER_READER ); sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); return pPg; } @@ -4492,9 +4703,8 @@ static int pager_open_journal(Pager *pPager){ int nPage; /* Size of database file */ sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ - assert( pPager->state>=PAGER_RESERVED ); - assert( pPager->useJournal ); - assert( pPager->journalMode!=PAGER_JOURNALMODE_OFF ); + assert( pPager->eState==PAGER_WRITER_INITIAL ); + assert( assert_pager_state(pPager) ); assert( pPager->pInJournal==0 ); /* If already in the error state, this function is a no-op. But on @@ -4502,55 +4712,59 @@ static int pager_open_journal(Pager *pPager){ ** an error state. */ if( NEVER(pPager->errCode) ) return pPager->errCode; - testcase( pPager->dbSizeValid==0 ); - rc = sqlite3PagerPagecount(pPager, &nPage); - if( rc ) return rc; - pPager->pInJournal = sqlite3BitvecCreate(nPage); - if( pPager->pInJournal==0 ){ - return SQLITE_NOMEM; - } - - /* Open the journal file if it is not already open. */ - if( !isOpen(pPager->jfd) ){ - if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ - sqlite3MemJournalOpen(pPager->jfd); - }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) - ); -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) - ); -#else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); -#endif + if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + testcase( pPager->dbSizeValid==0 ); + rc = sqlite3PagerPagecount(pPager, &nPage); + if( rc ) return rc; + pPager->pInJournal = sqlite3BitvecCreate(nPage); + if( pPager->pInJournal==0 ){ + return SQLITE_NOMEM; + } + + /* Open the journal file if it is not already open. */ + if( !isOpen(pPager->jfd) ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + sqlite3MemJournalOpen(pPager->jfd); + }else{ + const int flags = /* VFS flags to open journal file */ + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| + (pPager->tempFile ? + (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): + (SQLITE_OPEN_MAIN_JOURNAL) + ); + #ifdef SQLITE_ENABLE_ATOMIC_WRITE + rc = sqlite3JournalOpen( + pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + ); + #else + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); + #endif + } + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + } + + + /* Write the first journal header to the journal file and open + ** the sub-journal if necessary. + */ + if( rc==SQLITE_OK ){ + /* TODO: Check if all of these are really required. */ + pPager->dbOrigSize = pPager->dbSize; + pPager->nRec = 0; + pPager->journalOff = 0; + pPager->setMaster = 0; + pPager->journalHdr = 0; + rc = writeJournalHdr(pPager); } - assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); - } - - - /* Write the first journal header to the journal file and open - ** the sub-journal if necessary. - */ - if( rc==SQLITE_OK ){ - /* TODO: Check if all of these are really required. */ - pPager->dbOrigSize = pPager->dbSize; - pPager->journalStarted = 0; - pPager->nRec = 0; - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - rc = writeJournalHdr(pPager); } if( rc!=SQLITE_OK ){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; + }else{ + pPager->eState = PAGER_WRITER_CACHEMOD; } + return rc; } @@ -4581,20 +4795,18 @@ static int pager_open_journal(Pager *pPager){ */ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ int rc = SQLITE_OK; - assert( pPager->state!=PAGER_UNLOCK ); + assert( pPager->eState>=PAGER_READER ); pPager->subjInMemory = (u8)subjInMemory; - if( pPager->state==PAGER_SHARED ){ + if( pPager->eState==PAGER_READER ){ assert( pPager->pInJournal==0 ); - assert( !MEMDB && !pPager->tempFile ); if( pagerUseWal(pPager) ){ /* If the pager is configured to use locking_mode=exclusive, and an ** exclusive lock on the database is not already held, obtain it now. */ if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ - rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK); - pPager->state = PAGER_SHARED; + rc = pagerLock(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ return rc; } @@ -4605,45 +4817,35 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** PAGER_RESERVED state. Otherwise, return an error code to the caller. ** The busy-handler is not invoked if another connection already ** holds the write-lock. If possible, the upper layer will call it. - ** - ** WAL mode sets Pager.state to PAGER_RESERVED when it has an open - ** transaction, but never to PAGER_EXCLUSIVE. This is because in - ** PAGER_EXCLUSIVE state the code to roll back savepoint transactions - ** may copy data from the sub-journal into the database file as well - ** as into the page cache. Which would be incorrect in WAL mode. */ rc = sqlite3WalBeginWriteTransaction(pPager->pWal); - if( rc==SQLITE_OK ){ - pPager->dbOrigSize = pPager->dbSize; - pPager->state = PAGER_RESERVED; - pPager->journalOff = 0; - } - - assert( rc!=SQLITE_OK || pPager->state==PAGER_RESERVED ); - assert( rc==SQLITE_OK || pPager->state==PAGER_SHARED ); }else{ /* Obtain a RESERVED lock on the database file. If the exFlag parameter ** is true, then immediately upgrade this to an EXCLUSIVE lock. The ** busy-handler callback can be used when upgrading to the EXCLUSIVE ** lock, but not when obtaining the RESERVED lock. */ - rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); - if( rc==SQLITE_OK ){ - pPager->state = PAGER_RESERVED; - if( exFlag ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - } + rc = pagerLock(pPager, RESERVED_LOCK); + if( rc==SQLITE_OK && exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } } - /* No need to open the journal file at this time. It will be - ** opened before it is written to. If we defer opening the journal, - ** we might save the work of creating a file if the transaction - ** ends up being a no-op. - */ - if( rc!=SQLITE_OK ){ - assert( !pPager->dbModified ); + if( rc==SQLITE_OK ){ + /* Change to WRITER_INITIAL state. + ** + ** WAL mode sets Pager.eState to PAGER_WRITER_INITIAL or CACHEMOD + ** when it has an open transaction, but never to DBMOD or FINISHED. + ** This is because in those states the code to roll back savepoint + ** transactions may copy data from the sub-journal into the database + ** file as well as into the page cache. Which would be incorrect in + ** WAL mode. + */ + pPager->eState = PAGER_WRITER_INITIAL; + pPager->dbOrigSize = pPager->dbSize; + pPager->journalOff = 0; + }else{ /* Ignore any IO error that occurs within pager_end_transaction(). The ** purpose of this call is to reset the internal state of the pager ** sub-system. It doesn't matter if the journal-file is not properly @@ -4651,6 +4853,10 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ */ pager_end_transaction(pPager, 0); } + + assert( rc==SQLITE_OK || pPager->eState==PAGER_READER ); + assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_INITIAL ); + assert( assert_pager_state(pPager) ); } PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager))); @@ -4669,10 +4875,12 @@ static int pager_write(PgHdr *pPg){ Pager *pPager = pPg->pPager; int rc = SQLITE_OK; - /* This routine is not called unless a transaction has already been - ** started. + /* This routine is not called unless a write-transaction has already + ** been started. The journal file may or may not be open at this point. */ - assert( pPager->state>=PAGER_RESERVED ); + assert( pPager->eState>=PAGER_WRITER_INITIAL ); + assert( pPager->eState!=PAGER_WRITER_FINISHED ); + assert( assert_pager_state(pPager) ); /* If an error has been previously detected, report the same error ** again. @@ -4683,8 +4891,6 @@ static int pager_write(PgHdr *pPg){ ** writable. But check anyway, just for robustness. */ if( NEVER(pPager->readOnly) ) return SQLITE_PERM; - assert( !pPager->setMaster ); - CHECK_PAGE(pPg); /* Mark the page as dirty. If the page has already been written @@ -4693,27 +4899,22 @@ static int pager_write(PgHdr *pPg){ sqlite3PcacheMakeDirty(pPg); if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ assert( !pagerUseWal(pPager) ); - pPager->dbModified = 1; + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); }else{ /* If we get this far, it means that the page needs to be - ** written to the transaction journal or the ckeckpoint journal + ** written to the transaction journal or the checkpoint journal ** or both. ** - ** Higher level routines should have already started a transaction, - ** which means they have acquired the necessary locks but the rollback - ** journal might not yet be open. + ** Higher level routines have already obtained the necessary locks + ** to begin the write-transaction, but the rollback journal might not + ** yet be open. Open it now if this is the case. */ - assert( pPager->state>=RESERVED_LOCK ); - if( pPager->pInJournal==0 - && pPager->journalMode!=PAGER_JOURNALMODE_OFF - && !pagerUseWal(pPager) - ){ - assert( pPager->useJournal ); + if( pPager->eState==PAGER_WRITER_INITIAL ){ rc = pager_open_journal(pPager); if( rc!=SQLITE_OK ) return rc; } - pPager->dbModified = 1; + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); /* The transaction journal now exists and we have a RESERVED or an ** EXCLUSIVE lock on the main database file. Write the current page to @@ -4757,7 +4958,7 @@ static int pager_write(PgHdr *pPg){ ** in the database file. And if an IO error occurs while doing so, ** then corruption may follow. */ - if( !pPager->noSync ){ + if( 1 || !pPager->noSync ){ pPg->flags |= PGHDR_NEED_SYNC; } @@ -4779,7 +4980,7 @@ static int pager_write(PgHdr *pPg){ return rc; } }else{ - if( !pPager->journalStarted && !pPager->noSync ){ + if( pPager->eState!=PAGER_WRITER_DBMOD && (!pPager->noSync || 1)){ pPg->flags |= PGHDR_NEED_SYNC; } PAGERTRACE(("APPEND %d page %d needSync=%d\n", @@ -4800,7 +5001,6 @@ static int pager_write(PgHdr *pPg){ /* Update the database size and return. */ - assert( pPager->state>=PAGER_SHARED ); if( pPager->dbSizepgno ){ pPager->dbSize = pPg->pgno; } @@ -4966,6 +5166,11 @@ void sqlite3PagerDontWrite(PgHdr *pPg){ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ int rc = SQLITE_OK; + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + /* Declare and initialize constant integer 'isDirect'. If the ** atomic-write optimization is enabled in this build, then isDirect ** is initialized to the value passed as the isDirectMode parameter @@ -4984,7 +5189,6 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ # define DIRECT_MODE isDirectMode #endif - assert( pPager->state>=PAGER_RESERVED ); if( !pPager->changeCountDone && pPager->dbSize>0 ){ PgHdr *pPgHdr; /* Reference to page 1 */ u32 change_counter; /* Initial value of change-counter field */ @@ -5069,7 +5273,11 @@ int sqlite3PagerSync(Pager *pPager){ */ int sqlite3PagerExclusiveLock(Pager *pPager){ int rc = SQLITE_OK; - assert( pPager->state>=PAGER_RESERVED ); + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_INITIAL + ); + assert( assert_pager_state(pPager) ); if( 0==pagerUseWal(pPager) ){ rc = pager_wait_on_lock(pPager, PAGER_EXCLUSIVE); } @@ -5109,8 +5317,16 @@ int sqlite3PagerCommitPhaseOne( ){ int rc = SQLITE_OK; /* Return code */ + assert( pPager->eState==PAGER_WRITER_INITIAL + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + +#if 0 /* The dbOrigSize is never set if journal_mode=OFF */ assert( pPager->journalMode!=PAGER_JOURNALMODE_OFF || pPager->dbOrigSize==0 ); +#endif /* If a prior error occurred, report that error again. */ if( pPager->errCode ) return pPager->errCode; @@ -5118,13 +5334,16 @@ int sqlite3PagerCommitPhaseOne( PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); - if( MEMDB && pPager->dbModified ){ + /* If no database changes have been made, return early. */ + if( pPager->eStatepBackup); - }else if( pPager->state!=PAGER_SYNCED && pPager->dbModified ){ + }else{ if( pagerUseWal(pPager) ){ PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList ){ @@ -5203,7 +5422,7 @@ int sqlite3PagerCommitPhaseOne( */ #ifndef SQLITE_OMIT_AUTOVACUUM if( pPager->dbSizedbOrigSize - && ALWAYS(pPager->journalMode!=PAGER_JOURNALMODE_OFF) + && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ Pgno i; /* Iterator variable */ const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ @@ -5256,7 +5475,7 @@ int sqlite3PagerCommitPhaseOne( */ if( pPager->dbSize!=pPager->dbFileSize ){ Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); - assert( pPager->state>=PAGER_EXCLUSIVE ); + assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; } @@ -5267,11 +5486,12 @@ int sqlite3PagerCommitPhaseOne( } IOTRACE(("DBSYNC %p\n", pPager)) } - - pPager->state = PAGER_SYNCED; } commit_phase_one_exit: + if( rc==SQLITE_OK && !pagerUseWal(pPager) ){ + pPager->eState = PAGER_WRITER_FINISHED; + } return rc; } @@ -5304,7 +5524,15 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** fails - make it so that we never reach this point if we do not hold ** all necessary locks. */ + assert( pPager->eState==PAGER_WRITER_INITIAL + || pPager->eState==PAGER_WRITER_FINISHED + || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) + ); + assert( assert_pager_state(pPager) ); + +#if 0 if( NEVER(pPager->statedbModified==0 && pPager->exclusiveMode + if( pPager->eState==PAGER_WRITER_INITIAL + && pPager->exclusiveMode && pPager->journalMode==PAGER_JOURNALMODE_PERSIST ){ assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); + pPager->eState = PAGER_READER; return SQLITE_OK; } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - assert( pPager->state==PAGER_SYNCED || MEMDB || !pPager->dbModified ); rc = pager_end_transaction(pPager, pPager->setMaster); return pager_error(pPager, rc); } @@ -5376,6 +5605,11 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ int sqlite3PagerRollback(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); + + /* PagerRollback() is a no-op if called in READER or NONE state. */ + assert( assert_pager_state(pPager) ); + if( pPager->eState<=PAGER_READER ) return SQLITE_OK; + if( pagerUseWal(pPager) ){ int rc2; @@ -5383,15 +5617,15 @@ int sqlite3PagerRollback(Pager *pPager){ rc2 = pager_end_transaction(pPager, pPager->setMaster); if( rc==SQLITE_OK ) rc = rc2; rc = pager_error(pPager, rc); - }else if( !pPager->dbModified || !isOpen(pPager->jfd) ){ + }else if( pPager->eState==PAGER_WRITER_INITIAL ){ rc = pager_end_transaction(pPager, pPager->setMaster); }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ - if( pPager->state>=PAGER_EXCLUSIVE ){ + if( pPager->eState>=PAGER_WRITER_DBMOD ){ pager_playback(pPager, 0); } rc = pPager->errCode; }else{ - if( pPager->state==PAGER_RESERVED ){ + if( pPager->eState==PAGER_WRITER_CACHEMOD ){ int rc2; rc = pager_playback(pPager, 0); rc2 = pager_end_transaction(pPager, pPager->setMaster); @@ -5459,7 +5693,7 @@ int *sqlite3PagerStats(Pager *pPager){ a[1] = sqlite3PcachePagecount(pPager->pPCache); a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); a[3] = pPager->dbSizeValid ? (int) pPager->dbSize : -1; - a[4] = pPager->state; + a[4] = pPager->eState; a[5] = pPager->errCode; a[6] = pPager->nHit; a[7] = pPager->nMiss; @@ -5708,6 +5942,10 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ Pgno origPgno; /* The original page number */ assert( pPg->nRef>0 ); + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. @@ -5782,7 +6020,6 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ origPgno = pPg->pgno; sqlite3PcacheMove(pPg, pgno); sqlite3PcacheMakeDirty(pPg); - pPager->dbModified = 1; if( needSyncPgno ){ /* If needSyncPgno is non-zero, then the journal file needs to be @@ -5808,7 +6045,6 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ } return rc; } - assert( pPager->noSync==0 && !MEMDB ); pPgHdr->flags |= PGHDR_NEED_SYNC; sqlite3PcacheMakeDirty(pPgHdr); sqlite3PagerUnref(pPgHdr); @@ -5949,23 +6185,23 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ ** while it is in use by some other client. */ int rc = SQLITE_OK; - int state = pPager->state; - if( stateeState; + if( state==PAGER_NONE ){ rc = sqlite3PagerSharedLock(pPager); } - if( pPager->state==PAGER_SHARED ){ + if( pPager->eState==PAGER_READER ){ assert( rc==SQLITE_OK ); rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); } if( rc==SQLITE_OK ){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } - if( rc==SQLITE_OK && state==PAGER_SHARED ){ + if( rc==SQLITE_OK && state==PAGER_READER ){ sqlite3OsUnlock(pPager->fd, SHARED_LOCK); - }else if( state==PAGER_UNLOCK ){ + }else if( state==PAGER_NONE ){ pager_unlock(pPager); } - assert( state==pPager->state ); + assert( state==pPager->eState ); } } @@ -5986,7 +6222,7 @@ int sqlite3PagerGetJournalMode(Pager *pPager){ ** is unmodified. */ int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ - if( pPager->dbModified ) return 0; + if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; return 1; } @@ -6064,7 +6300,7 @@ int sqlite3PagerOpenWal( ){ int rc = SQLITE_OK; /* Return code */ - assert( pPager->state>=PAGER_SHARED ); + assert( pPager->eState>=PAGER_READER ); assert( (pisOpen==0 && !pPager->tempFile && !pPager->pWal) || *pisOpen==0 ); if( !pPager->tempFile && !pPager->pWal ){ diff --git a/test/analyze3.test b/test/analyze3.test index 6fc70d99c5..50a2aecfb2 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -403,7 +403,6 @@ do_test analyze3-3.6.5 { } {SQLITE_OK} do_test analyze3-3.7.1 { -breakpoint set S [sqlite3_prepare_v2 db { SELECT * FROM t1 WHERE a IN ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?33, diff --git a/test/exclusive.test b/test/exclusive.test index 4f20a6b9fc..a4ac9eacd6 100644 --- a/test/exclusive.test +++ b/test/exclusive.test @@ -253,6 +253,11 @@ db2 close # from being inspected externally. # if {$tcl_platform(platform) != "windows"} { + + # Return a list of two booleans (either 0 or 1). The first is true + # if the named file exists. The second is true only if the file + # exists and the first 28 bytes contain at least one non-zero byte. + # proc filestate {fname} { set exists 0 set content 0 @@ -263,6 +268,7 @@ if {$tcl_platform(platform) != "windows"} { } list $exists $content } + do_test exclusive-3.0 { filestate test.db-journal } {0 0} diff --git a/test/wal2.test b/test/wal2.test index c4d7fa7bc5..0b6f24a3e4 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -456,7 +456,7 @@ do_test wal2-6.1.5 { SELECT * FROM t1; PRAGMA lock_status; } -} {1 2 main exclusive temp closed} +} {1 2 main shared temp closed} do_test wal2-6.1.6 { execsql { INSERT INTO t1 VALUES(3, 4); @@ -522,7 +522,7 @@ do_test wal2-6.2.8 { SELECT * FROM t1; pragma lock_status; } -} {1 2 3 4 main exclusive temp closed} +} {1 2 3 4 main shared temp closed} do_test wal2-6.2.9 { execsql { INSERT INTO t1 VALUES(5, 6); @@ -608,6 +608,11 @@ set READMARK1_SET { set READMARK1_READ { {4 1 lock shared} {4 1 unlock shared} } +set READMARK1_WRITE { + {4 1 lock shared} + {0 1 lock exclusive} {0 1 unlock exclusive} + {4 1 unlock shared} +} foreach {tn sql res expected_locks} { 2 { @@ -664,13 +669,9 @@ foreach {tn sql res expected_locks} { 9 { SELECT * FROM t1 ORDER BY x - } {Arthur {Julius Henry} Karl Leonard} { } + } {Arthur {Julius Henry} Karl Leonard} $READMARK1_READ - 10 { - DELETE FROM t1 - } {} { - $READMARK1_READ - } + 10 { DELETE FROM t1 } {} $READMARK1_WRITE 11 { SELECT * FROM t1 @@ -942,7 +943,6 @@ do_test wal2-10.2.2 { set hdr [set_tvfs_hdr $::filename] lindex $hdr 0 } {3007000} -breakpoint do_test wal2-10.2.3 { lset hdr 0 3007001 wal_fix_walindex_cksum hdr From 763afe62e1defca58ce0f865af2bfe0f51bca804 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 3 Aug 2010 06:42:39 +0000 Subject: [PATCH 08/30] Incremental checkin on pager state refactoring. FossilOrigin-Name: 0a636798bdb6961a47327091715b254f79add823 --- manifest | 20 ++--- manifest.uuid | 2 +- src/pager.c | 183 ++++++++++++++++++++++++++--------------- src/test_stat.c | 14 ++-- src/wal.c | 10 ++- src/wal.h | 7 +- test/permutations.test | 2 +- 7 files changed, 144 insertions(+), 94 deletions(-) diff --git a/manifest b/manifest index d8a6e2a31d..625d681ee0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\srefactoring\sof\sthe\sPager\sobject\sstate. -D 2010-08-02T14:32:52 +C Incremental\scheckin\son\spager\sstate\srefactoring. +D 2010-08-03T06:42:40 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 94f00bedf9afb83c47f9b5d8bf8608f0eff088dd +F src/pager.c a22869257641c379545e482d843904702d027db7 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -207,7 +207,7 @@ F src/test_osinst.c f408c6a181f2fb04c56273afd5c3e1e82f60392c F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6 -F src/test_stat.c 6ebaf2a86d01ccda24e49c148f1d33e8badda06e +F src/test_stat.c dcabb7d39da058cbfc9ddc427cd92fbf14822b28 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c bedd05cad673dba53326f3aa468cc803038896c0 F src/test_vfs.c 7e291f85256516ebde6633bc381ff7eedfa30234 @@ -227,8 +227,8 @@ F src/vdbeblob.c 258a6010ba7a82b72b327fb24c55790655689256 F src/vdbemem.c 5e579abf6532001dfbee0e640dc34eae897a9807 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c 82200af3881fa4e1c9cf07cf31d98c09d437e3ab -F src/wal.c 72cb5df7f4c26f83cb661d5a607b9918da99f758 -F src/wal.h 906c85760598b18584921fe08008435aa4eeeeb2 +F src/wal.c 6e04bccccd75acf86725cc8cb4b107cd245e018c +F src/wal.h 96669b645e27cd5a111ba59f0cae7743a207bc3c F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 79202ca81e740eeb1f54512147e29b6c518d84ca F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -547,7 +547,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test 4118a183908ecaed343a06fcef3ba82e87e0129d F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 3fe47c21c32b294b2354e702a25bfbff65747bb1 +F test/permutations.test 17498d1219f922d5a6da893a94c4dc7766fb2426 F test/pragma.test ed78d200f65c6998df51196cb8c39d5300570f24 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -841,7 +841,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 347f22a5b777af92873590a5b9af5a6498bef918 -R fb51b6835fc754e61c37a4fea5d6298b +P 03a240514aa07a22db787d221641253f23933e88 +R 0befc920b01bc332d217d8425323f579 U dan -Z 550ac78d14ba82c7a2ec704cc3fbe702 +Z 0621f056ce54ddd71f6be378b7d6f549 diff --git a/manifest.uuid b/manifest.uuid index 0099265206..2daa895065 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03a240514aa07a22db787d221641253f23933e88 \ No newline at end of file +0a636798bdb6961a47327091715b254f79add823 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 947194927f..c9f2d2ebf5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -130,13 +130,24 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** pager may be in any one of the following six states: ** ** NONE: +** The pager starts up in this state. Nothing is guaranteed in this +** state - the file may or may not be locked and the database size is +** unknown. The database may not be read or written. +** ** * No read or write transaction is active. ** * Any lock, or no lock at all, may be held on the database file. +** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted. ** ** READER: -** * A read transaction is active. +** In this state all the requirements for reading the database in +** rollback (non-WAL) mode are met. Unless the pager is (or recently +** was) in exclusive-locking mode, a user-level read transaction is +** open. The database size is known in this state. +** +** * A read transaction may be active. ** * A SHARED or greater lock is held on the database file. -** * The dbSize variable is valid. +** * The dbSize variable may be trusted (even if a user-level read +** transaction is not active). ** ** WRITER_INITIAL: ** * A write transaction is active. @@ -167,8 +178,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** commit the transaction. If an error did occur, the caller will need ** to rollback the transaction. ** -** -** Allowable transitions and the [function] that performe each: +** Allowable transitions and the [function] that performs each: ** ** NONE -> READER [PagerSharedLock] ** READER -> WRITER_INITIAL [PagerBegin] @@ -182,6 +192,19 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** WRITER_FINISHED -> READER [pager_end_transaction] ** ** READER -> NONE [pager_unlock] +** +** Notes: +** +** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the +** connection is open in WAL mode. A WAL connection is always in one +** of the first four states. +** +** * Normally, a connection open in exclusive mode is never in PAGER_NONE +** state. There are two exceptions: immediately after exclusive-mode has +** been turned on (and before any read or write transactions are +** executed), and when the pager is leaving the "error state". +** +** * See also: assert_pager_state(). */ #define PAGER_NONE 0 #define PAGER_READER 1 @@ -425,8 +448,8 @@ struct Pager { u8 state; /* PAGER_UNLOCK, _SHARED, _RESERVED, etc. */ u8 dbModified; /* True if there are any changes to the Db */ u8 journalStarted; /* True if header of journal is synced */ -#endif u8 dbSizeValid; /* Set when dbSize is correct */ +#endif u8 eState; /* Pager state (NONE, READER, WRITER_INITIAL..) */ u8 eLock; /* Current lock held on database file */ @@ -623,7 +646,6 @@ static int assert_pager_state(Pager *p){ case PAGER_NONE: assert( !MEMDB ); assert( !p->tempFile ); - assert( !pagerUseWal(pPager) ); break; case PAGER_READER: @@ -1348,7 +1370,6 @@ static void pager_reset(Pager *pPager){ if( SQLITE_OK==pPager->errCode ){ sqlite3BackupRestart(pPager->pBackup); sqlite3PcacheClear(pPager->pPCache); - pPager->dbSizeValid = 0; } } @@ -1428,14 +1449,6 @@ static void pager_unlock(Pager *pPager){ pPager->pInJournal = 0; releaseAllSavepoints(pPager); - /* If the file is unlocked, somebody else might change it. The - ** values stored in Pager.dbSize etc. might become invalid if - ** this happens. One can argue that this doesn't need to be cleared - ** until the change-counter check fails in PagerSharedLock(). - ** Clearing the page size cache here is being conservative. - */ - pPager->dbSizeValid = 0; - if( pagerUseWal(pPager) ){ sqlite3WalEndReadTransaction(pPager->pWal); }else{ @@ -2589,6 +2602,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ int changed = 0; /* True if cache must be reset */ assert( pagerUseWal(pPager) ); + assert( pPager->eState==PAGER_NONE || pPager->eState==PAGER_SHARED ); /* sqlite3WalEndReadTransaction() was not called for the previous ** transaction in locking_mode=EXCLUSIVE. So call it now. If we @@ -2598,19 +2612,41 @@ static int pagerBeginReadTransaction(Pager *pPager){ sqlite3WalEndReadTransaction(pPager->pWal); rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); - if( rc==SQLITE_OK ){ - int dummy; - if( changed ){ - pager_reset(pPager); - assert( pPager->errCode || pPager->dbSizeValid==0 ); - } - rc = sqlite3PagerPagecount(pPager, &dummy); + if( rc==SQLITE_OK && changed ){ + pager_reset(pPager); } - pPager->eState = PAGER_READER; return rc; } + +/* +** TODO: Description here. +*/ +static int pagerPagecount(Pager *pPager, Pgno *pnPage){ + Pgno nPage; /* Value to return via *pnPage */ + + assert( pPager->eState==PAGER_NONE ); + nPage = sqlite3WalDbsize(pPager->pWal); + if( nPage==0 ){ + i64 n = 0; /* Size of db file in bytes */ + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( isOpen(pPager->fd) ){ + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; + } + } + nPage = (Pgno)(n / pPager->pageSize); + if( nPage==0 && n>0 ){ + nPage = 1; + } + } + *pnPage = nPage; + return SQLITE_OK; +} + + /* ** Check if the *-wal file that corresponds to the database opened by pPager ** exists if the database is not empy, or verify that the *-wal file does @@ -2635,10 +2671,11 @@ static int pagerOpenWalIfPresent(Pager *pPager){ int rc = SQLITE_OK; if( !pPager->tempFile ){ int isWal; /* True if WAL file exists */ - int nPage; /* Size of the database file */ + Pgno nPage; /* Size of the database file */ + assert( pPager->eState==PAGER_NONE ); assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); - rc = sqlite3PagerPagecount(pPager, &nPage); + rc = pagerPagecount(pPager, &nPage); if( rc ) return rc; if( nPage==0 ){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); @@ -2650,7 +2687,7 @@ static int pagerOpenWalIfPresent(Pager *pPager){ } if( rc==SQLITE_OK ){ if( isWal ){ - pager_reset(pPager); + assert( sqlite3PcachePagecount(pPager->pPCache)==0 ); rc = sqlite3PagerOpenWal(pPager, 0); if( rc==SQLITE_OK ){ rc = pagerBeginReadTransaction(pPager); @@ -2960,11 +2997,19 @@ int sqlite3PagerSetPagesize(Pager *pPager, u16 *pPageSize, int nReserve){ && sqlite3PcacheRefCount(pPager->pPCache)==0 && pageSize && pageSize!=pPager->pageSize ){ - char *pNew = (char *)sqlite3PageMalloc(pageSize); + char *pNew; /* New temp space */ + i64 nByte = 0; + if( pPager->eState>PAGER_NONE && isOpen(pPager->fd) ){ + rc = sqlite3OsFileSize(pPager->fd, &nByte); + if( rc!=SQLITE_OK ) return rc; + } + pNew = (char *)sqlite3PageMalloc(pageSize); if( !pNew ){ rc = SQLITE_NOMEM; }else{ + assert( pPager->eState==PAGER_NONE || pPager->eState==PAGER_READER ); pager_reset(pPager); + pPager->dbSize = nByte/pageSize; pPager->pageSize = pageSize; sqlite3PageFree(pPager->pTmpSpace); pPager->pTmpSpace = pNew; @@ -3088,15 +3133,18 @@ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ Pgno nPage = 0; /* Value to return via *pnPage */ + assert( pPager->eState>=PAGER_SHARED ); + assert( pPager->eState!=PAGER_WRITER_FINISHED ); + /* Determine the number of pages in the file. Store this in nPage. */ - if( pPager->dbSizeValid ){ + if( pPager->eState>PAGER_NONE ){ nPage = pPager->dbSize; }else{ int rc; /* Error returned by OsFileSize() */ i64 n = 0; /* File size in bytes returned by OsFileSize() */ if( pagerUseWal(pPager) && pPager->eState!=PAGER_NONE ){ - sqlite3WalDbsize(pPager->pWal, &nPage); + nPage = sqlite3WalDbsize(pPager->pWal); } if( nPage==0 ){ @@ -3116,7 +3164,6 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ if( pPager->eState!=PAGER_NONE ){ pPager->dbSize = nPage; pPager->dbFileSize = nPage; - pPager->dbSizeValid = 1; } } @@ -3235,7 +3282,6 @@ static void assertTruncateConstraint(Pager *pPager){ ** truncation will be done when the current transaction is committed. */ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ - assert( pPager->dbSizeValid ); assert( pPager->dbSize>=nPage ); assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); pPager->dbSize = nPage; @@ -4064,7 +4110,6 @@ int sqlite3PagerOpen( /* pPager->stmtOpen = 0; */ /* pPager->stmtInUse = 0; */ /* pPager->nRef = 0; */ - pPager->dbSizeValid = (u8)memDb; /* pPager->stmtSize = 0; */ /* pPager->stmtJSize = 0; */ /* pPager->nPage = 0; */ @@ -4146,10 +4191,9 @@ static int hasHotJournal(Pager *pPager, int *pExists){ int exists = 1; /* True if a journal file is present */ int jrnlOpen = !!isOpen(pPager->jfd); - assert( pPager!=0 ); assert( pPager->useJournal ); assert( isOpen(pPager->fd) ); - assert( pPager->eState<=PAGER_SHARED ); + assert( pPager->eState==PAGER_NONE ); assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN @@ -4172,7 +4216,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ */ rc = sqlite3OsCheckReservedLock(pPager->fd, &locked); if( rc==SQLITE_OK && !locked ){ - int nPage; + Pgno nPage; /* Number of pages in database file */ /* Check the size of the database file. If it consists of 0 pages, ** then delete the journal file. See the header comment above for @@ -4180,7 +4224,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** a RESERVED lock to avoid race conditions and to avoid violating ** [H33020]. */ - rc = sqlite3PagerPagecount(pPager, &nPage); + rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ if( nPage==0 ){ sqlite3BeginBenignMalloc(); @@ -4268,8 +4312,13 @@ int sqlite3PagerSharedLock(Pager *pPager){ int isErrorReset = 0; /* True if recovering from error state */ /* This routine is only called from b-tree and only when there are no - ** outstanding pages */ + ** outstanding pages. This implies that the pager state should either + ** be NONE or READER. READER is only possible if the pager is or was in + ** exclusive access mode. + */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + assert( pPager->eState==PAGER_NONE || pPager->eState==PAGER_READER ); + assert( assert_pager_state(pPager) ); if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } /* If this database is in an error-state, now is a chance to clear @@ -4281,14 +4330,12 @@ int sqlite3PagerSharedLock(Pager *pPager){ isErrorReset = 1; } pPager->errCode = SQLITE_OK; + pPager->eState = PAGER_NONE; pager_reset(pPager); } - if( pagerUseWal(pPager) ){ - rc = pagerBeginReadTransaction(pPager); - }else if( pPager->eState==PAGER_NONE || isErrorReset ){ + if( !pagerUseWal(pPager) && pPager->eState==PAGER_NONE ){ sqlite3_vfs * const pVfs = pPager->pVfs; - int isHotJournal = 0; assert( !MEMDB && !pPager->tempFile ); assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); @@ -4301,18 +4348,17 @@ int sqlite3PagerSharedLock(Pager *pPager){ return pager_error(pPager, rc); } } - pPager->eState = PAGER_READER; /* If a journal file exists, and there is no RESERVED lock on the ** database file, then it either needs to be played back or deleted. */ if( !isErrorReset ){ - rc = hasHotJournal(pPager, &isHotJournal); + rc = hasHotJournal(pPager, &isErrorReset); if( rc!=SQLITE_OK ){ goto failed; } } - if( isErrorReset || isHotJournal ){ + if( isErrorReset ){ /* Get an EXCLUSIVE lock on the database file. At this point it is ** important that a RESERVED lock is not obtained on the way to the ** EXCLUSIVE lock. If it were, another process might open the @@ -4389,7 +4435,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( rc==SQLITE_OK ){ pPager->eState = PAGER_WRITER_FINISHED; rc = pager_playback(pPager, 1); - pPager->eState = PAGER_READER; + pPager->eState = PAGER_NONE; } if( rc!=SQLITE_OK ){ rc = pager_error(pPager, rc); @@ -4399,7 +4445,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ osUnlock(pPager, SHARED_LOCK); } - assert( pPager->eState==PAGER_READER ); + assert( pPager->eState==PAGER_NONE ); assert( (pPager->eLock==SHARED_LOCK) || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) ); @@ -4422,14 +4468,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** detected. The chance of an undetected change is so small that ** it can be neglected. */ - int nPage = 0; + Pgno nPage = 0; char dbFileVers[sizeof(pPager->dbFileVers)]; - sqlite3PagerPagecount(pPager, &nPage); - if( pPager->errCode ){ - rc = pPager->errCode; - goto failed; - } + rc = pagerPagecount(pPager, &nPage); + if( rc ) goto failed; if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); @@ -4445,7 +4488,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ pager_reset(pPager); } } - assert( pPager->eState==PAGER_READER ); /* If there is a WAL file in the file-system, open this database in WAL ** mode. Otherwise, the following function call is a no-op. @@ -4453,10 +4495,21 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerOpenWalIfPresent(pPager); } + if( pagerUseWal(pPager) && rc==SQLITE_OK ){ + rc = pagerBeginReadTransaction(pPager); + } + + if( pPager->eState==PAGER_NONE && rc==SQLITE_OK ){ + pPager->eState = PAGER_NONE; + rc = pagerPagecount(pPager, &pPager->dbSize); + } + failed: if( rc!=SQLITE_OK ){ /* pager_unlock() is a no-op for exclusive mode and in-memory databases. */ pager_unlock(pPager); + }else{ + pPager->eState = PAGER_READER; } return rc; } @@ -4713,7 +4766,6 @@ static int pager_open_journal(Pager *pPager){ if( NEVER(pPager->errCode) ) return pPager->errCode; if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ - testcase( pPager->dbSizeValid==0 ); rc = sqlite3PagerPagecount(pPager, &nPage); if( rc ) return rc; pPager->pInJournal = sqlite3BitvecCreate(nPage); @@ -5636,10 +5688,6 @@ int sqlite3PagerRollback(Pager *pPager){ rc = pager_playback(pPager, 0); } - if( !MEMDB ){ - pPager->dbSizeValid = 0; - } - /* If an error occurs during a ROLLBACK, we can no longer trust the pager ** cache. So call pager_error() on the way out to make any error ** persistent. @@ -5692,7 +5740,7 @@ int *sqlite3PagerStats(Pager *pPager){ a[0] = sqlite3PcacheRefCount(pPager->pPCache); a[1] = sqlite3PcachePagecount(pPager->pPCache); a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); - a[3] = pPager->dbSizeValid ? (int) pPager->dbSize : -1; + a[3] = pPager->eState==PAGER_NONE ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; a[6] = pPager->nHit; @@ -6010,7 +6058,6 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ if( MEMDB ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ - assert( pPager->dbSizeValid ); sqlite3PcacheMove(pPgOld, pPager->dbSize+1); }else{ sqlite3PcacheDrop(pPgOld); @@ -6287,21 +6334,24 @@ int sqlite3PagerWalSupported(Pager *pPager){ ** file (not a temp file or an in-memory database), and the WAL file ** is not already open, make an attempt to open it now. If successful, ** return SQLITE_OK. If an error occurs or the VFS used by the pager does -** not support the xShmXXX() methods, return an error code. *pisOpen is +** not support the xShmXXX() methods, return an error code. *pbOpen is ** not modified in either case. ** ** If the pager is open on a temp-file (or in-memory database), or if -** the WAL file is already open, set *pisOpen to 1 and return SQLITE_OK +** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK ** without doing anything. */ int sqlite3PagerOpenWal( Pager *pPager, /* Pager object */ - int *pisOpen /* OUT: Set to true if call is a no-op */ + int *pbOpen /* OUT: Set to true if call is a no-op */ ){ int rc = SQLITE_OK; /* Return code */ - assert( pPager->eState>=PAGER_READER ); - assert( (pisOpen==0 && !pPager->tempFile && !pPager->pWal) || *pisOpen==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_NONE || pbOpen ); + assert( pPager->eState==PAGER_READER || !pbOpen ); + assert( pbOpen==0 || *pbOpen==0 ); + assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; @@ -6313,9 +6363,10 @@ int sqlite3PagerOpenWal( rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, &pPager->pWal); if( rc==SQLITE_OK ){ pPager->journalMode = PAGER_JOURNALMODE_WAL; + pPager->eState = PAGER_NONE; } }else{ - *pisOpen = 1; + *pbOpen = 1; } return rc; diff --git a/src/test_stat.c b/src/test_stat.c index 894be80ecb..1534621114 100644 --- a/src/test_stat.c +++ b/src/test_stat.c @@ -376,7 +376,13 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->aPage[0].pPg==0 ){ rc = sqlite3_step(pCsr->pStmt); if( rc==SQLITE_ROW ){ + int nPage; u32 iRoot = sqlite3_column_int64(pCsr->pStmt, 1); + sqlite3PagerPagecount(pPager, &nPage); + if( nPage==0 ){ + pCsr->isEof = 1; + return sqlite3_reset(pCsr->pStmt); + } rc = sqlite3PagerGet(pPager, iRoot, &pCsr->aPage[0].pPg); pCsr->aPage[0].iPgno = iRoot; pCsr->aPage[0].iCell = 0; @@ -486,17 +492,9 @@ static int statFilter( int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ - sqlite3 *db = ((StatTable *)(pCursor->pVtab))->db; StatCursor *pCsr = (StatCursor *)pCursor; - int nPage = 0; statResetCsr((StatCursor *)pCursor); - sqlite3PagerPagecount(sqlite3BtreePager(db->aDb[0].pBt), &nPage); - if( nPage==0 ){ - pCsr->isEof = 1; - return SQLITE_OK; - } - return statNext(pCursor); } diff --git a/src/wal.c b/src/wal.c index 3ac8071548..be8c466428 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2166,11 +2166,13 @@ int sqlite3WalRead( /* -** Set *pPgno to the size of the database file (or zero, if unknown). +** Return the size of the database in pages (or zero, if unknown). */ -void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno){ - assert( pWal->readLock>=0 || pWal->lockError ); - *pPgno = pWal->hdr.nPage; +Pgno sqlite3WalDbsize(Wal *pWal){ + if( pWal && pWal->readLock>=0 ){ + return pWal->hdr.nPage; + } + return 0; } diff --git a/src/wal.h b/src/wal.h index f29d5838f5..f20dfa8e17 100644 --- a/src/wal.h +++ b/src/wal.h @@ -25,7 +25,7 @@ # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) # define sqlite3WalRead(v,w,x,y,z) 0 -# define sqlite3WalDbsize(y,z) +# define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 # define sqlite3WalUndo(x,y,z) 0 @@ -61,9 +61,8 @@ void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); -/* Return the size of the database as it existed at the beginning -** of the snapshot */ -void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno); +/* If the WAL is not empty, return the size of the database. */ +Pgno sqlite3WalDbsize(Wal *pWal); /* Obtain or release the WRITER lock. */ int sqlite3WalBeginWriteTransaction(Wal *pWal); diff --git a/test/permutations.test b/test/permutations.test index f1fb74fe61..086897c468 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -129,7 +129,7 @@ test_suite "veryquick" -prefix "" -description { This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* + test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] test_suite "quick" -prefix "" -description { From 937ac9da3802ef3e855ee64b268ccb1930e7503a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 3 Aug 2010 12:48:45 +0000 Subject: [PATCH 09/30] Add state diagram to comments in experimental version of pager.c. FossilOrigin-Name: 16dcf5a6d3ebff9a734705693b5ac740f4ff2cc1 --- manifest | 12 +-- manifest.uuid | 2 +- src/pager.c | 228 ++++++++++++++++++++++---------------------------- 3 files changed, 107 insertions(+), 135 deletions(-) diff --git a/manifest b/manifest index 625d681ee0..b67585edd7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Incremental\scheckin\son\spager\sstate\srefactoring. -D 2010-08-03T06:42:40 +C Add\sstate\sdiagram\sto\scomments\sin\sexperimental\sversion\sof\spager.c. +D 2010-08-03T12:48:46 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c a22869257641c379545e482d843904702d027db7 +F src/pager.c 24f978063ff3f18f1632b05b695717cc9818538a F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -841,7 +841,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 03a240514aa07a22db787d221641253f23933e88 -R 0befc920b01bc332d217d8425323f579 +P 0a636798bdb6961a47327091715b254f79add823 +R b916884b782901df5d25760d516d1243 U dan -Z 0621f056ce54ddd71f6be378b7d6f549 +Z a547af01cff05efb8820775070ecaafe diff --git a/manifest.uuid b/manifest.uuid index 2daa895065..34229d28a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a636798bdb6961a47327091715b254f79add823 \ No newline at end of file +16dcf5a6d3ebff9a734705693b5ac740f4ff2cc1 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index c9f2d2ebf5..100f4e90c9 100644 --- a/src/pager.c +++ b/src/pager.c @@ -130,6 +130,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** pager may be in any one of the following six states: ** ** NONE: +** ** The pager starts up in this state. Nothing is guaranteed in this ** state - the file may or may not be locked and the database size is ** unknown. The database may not be read or written. @@ -147,7 +148,8 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** * A read transaction may be active. ** * A SHARED or greater lock is held on the database file. ** * The dbSize variable may be trusted (even if a user-level read -** transaction is not active). +** transaction is not active). The dbOrigSize and dbFileSize variables +** may not be trusted at this point. ** ** WRITER_INITIAL: ** * A write transaction is active. @@ -178,7 +180,27 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** commit the transaction. If an error did occur, the caller will need ** to rollback the transaction. ** -** Allowable transitions and the [function] that performs each: +** State diagram: +** +** NONE <-------------+ +** | | +** V | +** +---------> READER------------->| +** | | | +** | V | +** |<-------WRITER_INITIAL-----> ERROR +** | | ^ +** | V | +** |<------WRITER_CACHEMOD-------->| +** | | | +** | V | +** |<-------WRITER_DBMOD---------->| +** | | | +** | V | +** +<------WRITER_FINISHED-------->+ +** +** +** State transitions and the [function] that performs each: ** ** NONE -> READER [PagerSharedLock] ** READER -> WRITER_INITIAL [PagerBegin] @@ -328,7 +350,7 @@ struct PagerSavepoint { ** SQLITE_FULL does not affect the sqlite3PagerGet() and sqlite3PagerLookup() ** APIs, they may still be used successfully. ** -** dbSizeValid, dbSize, dbOrigSize, dbFileSize +** dbSize, dbOrigSize, dbFileSize ** ** Managing the size of the database file in pages is a little complicated. ** The variable Pager.dbSize contains the number of pages that the database @@ -444,15 +466,8 @@ struct Pager { ** other variables in this block are described in the comment directly ** above this class definition. */ -#if 0 - u8 state; /* PAGER_UNLOCK, _SHARED, _RESERVED, etc. */ - u8 dbModified; /* True if there are any changes to the Db */ - u8 journalStarted; /* True if header of journal is synced */ - u8 dbSizeValid; /* Set when dbSize is correct */ -#endif u8 eState; /* Pager state (NONE, READER, WRITER_INITIAL..) */ u8 eLock; /* Current lock held on database file */ - u8 changeCountDone; /* Set after incrementing the change-counter */ u8 setMaster; /* True if a m-j name has been written to jrnl */ u8 doNotSpill; /* Do not spill the cache when non-zero */ @@ -656,6 +671,8 @@ static int assert_pager_state(Pager *p){ if( !pagerUseWal(pPager) ){ assert( p->eLock>=RESERVED_LOCK ); } + assert( pPager->dbSize==pPager->dbOrigSize ); + assert( pPager->dbOrigSize==pPager->dbFileSize ); break; case PAGER_WRITER_CACHEMOD: @@ -671,6 +688,7 @@ static int assert_pager_state(Pager *p){ || p->journalMode==PAGER_JOURNALMODE_WAL ); } + assert( pPager->dbOrigSize==pPager->dbFileSize ); break; case PAGER_WRITER_DBMOD: @@ -702,10 +720,11 @@ static char *print_pager_state(Pager *p){ static char zRet[1024]; sqlite3_snprintf(1024, zRet, - "State: %s\n" - "Lock: %s\n" + "State: %s\n" + "Lock: %s\n" "Locking mode: locking_mode=%s\n" - "Backing store: tempFile=%d memDb=%d\n" + "Journal mode: journal_mode=%s\n" + "Backing store: tempFile=%d memDb=%d useJournal=%d\n" , p->eState==PAGER_NONE ? "NONE" : p->eState==PAGER_READER ? "READER" : p->eState==PAGER_WRITER_INITIAL ? "WRITER_INITIAL" : @@ -717,7 +736,13 @@ static char *print_pager_state(Pager *p){ p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : p->eLock==SHARED_LOCK ? "SHARED" : "?error?" , p->exclusiveMode ? "exclusive" : "normal" - , (int)p->tempFile, (int)p->memDb + , p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" : + p->journalMode==PAGER_JOURNALMODE_OFF ? "off" : + p->journalMode==PAGER_JOURNALMODE_DELETE ? "delete" : + p->journalMode==PAGER_JOURNALMODE_PERSIST ? "persist" : + p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" : + p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" + , (int)p->tempFile, (int)p->memDb, (int)p->useJournal ); return zRet; @@ -2642,6 +2667,15 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ nPage = 1; } } + + /* If the current number of pages in the file is greater than the + ** configured maximum pager number, increase the allowed limit so + ** that the file can be read. + */ + if( nPage>pPager->mxPgno ){ + pPager->mxPgno = (Pgno)nPage; + } + *pnPage = nPage; return SQLITE_OK; } @@ -3045,13 +3079,11 @@ void *sqlite3PagerTempSpace(Pager *pPager){ ** Regardless of mxPage, return the current maximum page count. */ int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ - int nPage; if( mxPage>0 ){ pPager->mxPgno = mxPage; } - if( pPager->eState!=PAGER_NONE ){ - sqlite3PagerPagecount(pPager, &nPage); - assert( (int)pPager->mxPgno>=nPage ); + if( pPager->eState!=PAGER_NONE && pPager->mxPgnodbSize ){ + pPager->mxPgno = pPager->dbSize; } return pPager->mxPgno; } @@ -3116,67 +3148,16 @@ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ } /* -** Return the total number of pages in the database file associated -** with pPager. Normally, this is calculated as (/). +** This function may only be called when a read-transaction is open on +** the pager. It returns the total number of pages in the database. +** ** However, if the file is between 1 and bytes in size, then ** this is considered a 1 page file. -** -** If the pager is in error state when this function is called, then the -** error state error code is returned and *pnPage left unchanged. Or, -** if the file system has to be queried for the size of the file and -** the query attempt returns an IO error, the IO error code is returned -** and *pnPage is left unchanged. -** -** Otherwise, if everything is successful, then SQLITE_OK is returned -** and *pnPage is set to the number of pages in the database. */ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ - Pgno nPage = 0; /* Value to return via *pnPage */ - assert( pPager->eState>=PAGER_SHARED ); assert( pPager->eState!=PAGER_WRITER_FINISHED ); - - /* Determine the number of pages in the file. Store this in nPage. */ - if( pPager->eState>PAGER_NONE ){ - nPage = pPager->dbSize; - }else{ - int rc; /* Error returned by OsFileSize() */ - i64 n = 0; /* File size in bytes returned by OsFileSize() */ - - if( pagerUseWal(pPager) && pPager->eState!=PAGER_NONE ){ - nPage = sqlite3WalDbsize(pPager->pWal); - } - - if( nPage==0 ){ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - if( SQLITE_OK!=(rc = sqlite3OsFileSize(pPager->fd, &n)) ){ - pager_error(pPager, rc); - return rc; - } - } - if( n>0 && npageSize ){ - nPage = 1; - }else{ - nPage = (Pgno)(n / pPager->pageSize); - } - } - if( pPager->eState!=PAGER_NONE ){ - pPager->dbSize = nPage; - pPager->dbFileSize = nPage; - } - } - - /* If the current number of pages in the file is greater than the - ** configured maximum pager number, increase the allowed limit so - ** that the file can be read. - */ - if( nPage>pPager->mxPgno ){ - pPager->mxPgno = (Pgno)nPage; - } - - /* Set the output variable and return SQLITE_OK */ - *pnPage = nPage; + *pnPage = (int)pPager->dbSize; return SQLITE_OK; } @@ -3398,7 +3379,7 @@ void sqlite3PagerRef(DbPage *pPg){ ** been written to the journal have actually reached the surface of the ** disk and can be restored in the event of a hot-journal rollback. ** -** If the Pager.noSync flag is set, then this function is a no-op. +** If the Pager.noSync flag is set, then this function is a no-op. ** Otherwise, the actions required depend on the journal-mode and the ** device characteristics of the the file-system, as follows: ** @@ -3428,13 +3409,14 @@ void sqlite3PagerRef(DbPage *pPg){ ** page currently held in memory before returning SQLITE_OK. If an IO ** error is encountered, then the IO error code is returned to the caller. */ -static int syncJournal(Pager *pPager){ +static int syncJournal(Pager *pPager, int newHdr){ int rc; /* Return code */ assert( pPager->eState==PAGER_WRITER_CACHEMOD || pPager->eState==PAGER_WRITER_DBMOD ); assert( assert_pager_state(pPager) ); + assert( !pagerUseWal(pPager) ); rc = sqlite3PagerExclusiveLock(pPager); if( rc!=SQLITE_OK ) return rc; @@ -3516,9 +3498,15 @@ static int syncJournal(Pager *pPager){ ); if( rc!=SQLITE_OK ) return rc; } - } - pPager->journalHdr = pPager->journalOff; + pPager->journalHdr = pPager->journalOff; + if( newHdr && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + } + }else{ + pPager->journalHdr = pPager->journalOff; + } } /* Unless the pager is in noSync mode, the journal file was just @@ -3592,10 +3580,12 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** to it in CACHEMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD + assert( pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_CACHEMOD + ); + assert( pPager->eState==PAGER_WRITER_DBMOD + || (pList->pDirty==0 && pList->pgno<=pPager->dbFileSize) ); - assert( pPager->eState==PAGER_WRITER_DBMOD || pList->pDirty==0 ); rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); @@ -3797,15 +3787,7 @@ static int pagerStress(void *p, PgHdr *pPg){ /* Sync the journal file if required. */ if( pPg->flags&PGHDR_NEED_SYNC ){ - rc = syncJournal(pPager); - if( rc==SQLITE_OK && - !pPager->noSync && - !(pPager->journalMode==PAGER_JOURNALMODE_MEMORY) && - !(sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) - ){ - pPager->nRec = 0; - rc = writeJournalHdr(pPager); - } + rc = syncJournal(pPager, 1); } /* If the page number of this page is larger than the current size of @@ -4624,7 +4606,6 @@ int sqlite3PagerAcquire( }else{ /* The pager cache has created a new page. Its content needs to ** be initialized. */ - int nMax; PAGER_INCR(pPager->nMiss); pPg = *ppPage; @@ -4637,12 +4618,7 @@ int sqlite3PagerAcquire( goto pager_acquire_err; } - rc = sqlite3PagerPagecount(pPager, &nMax); - if( rc!=SQLITE_OK ){ - goto pager_acquire_err; - } - - if( MEMDB || nMax<(int)pgno || noContent || !isOpen(pPager->fd) ){ + if( MEMDB || pPager->dbSize<(int)pgno || noContent || !isOpen(pPager->fd) ){ if( pgno>pPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; @@ -4753,7 +4729,6 @@ void sqlite3PagerUnref(DbPage *pPg){ */ static int pager_open_journal(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ - int nPage; /* Size of database file */ sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ assert( pPager->eState==PAGER_WRITER_INITIAL ); @@ -4766,9 +4741,7 @@ static int pager_open_journal(Pager *pPager){ if( NEVER(pPager->errCode) ) return pPager->errCode; if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ - rc = sqlite3PagerPagecount(pPager, &nPage); - if( rc ) return rc; - pPager->pInJournal = sqlite3BitvecCreate(nPage); + pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); if( pPager->pInJournal==0 ){ return SQLITE_NOMEM; } @@ -4801,7 +4774,6 @@ static int pager_open_journal(Pager *pPager){ */ if( rc==SQLITE_OK ){ /* TODO: Check if all of these are really required. */ - pPager->dbOrigSize = pPager->dbSize; pPager->nRec = 0; pPager->journalOff = 0; pPager->setMaster = 0; @@ -4814,6 +4786,7 @@ static int pager_open_journal(Pager *pPager){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; }else{ + assert( pPager->eState==PAGER_WRITER_INITIAL ); pPager->eState = PAGER_WRITER_CACHEMOD; } @@ -4895,7 +4868,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** WAL mode. */ pPager->eState = PAGER_WRITER_INITIAL; - pPager->dbOrigSize = pPager->dbSize; + pPager->dbFileSize = pPager->dbOrigSize = pPager->dbSize; pPager->journalOff = 0; }else{ /* Ignore any IO error that occurs within pager_end_transaction(). The @@ -4967,14 +4940,15 @@ static int pager_write(PgHdr *pPg){ if( rc!=SQLITE_OK ) return rc; } assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + assert( assert_pager_state(pPager) ); /* The transaction journal now exists and we have a RESERVED or an ** EXCLUSIVE lock on the main database file. Write the current page to ** the transaction journal if it is not there already. */ - if( !pageInJournal(pPg) && isOpen(pPager->jfd) ){ - assert( !pagerUseWal(pPager) ); - if( pPg->pgno<=pPager->dbOrigSize ){ + if( !pageInJournal(pPg) && !pagerUseWal(pPager) ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ u32 cksum; char *pData2; @@ -5010,9 +4984,7 @@ static int pager_write(PgHdr *pPg){ ** in the database file. And if an IO error occurs while doing so, ** then corruption may follow. */ - if( 1 || !pPager->noSync ){ - pPg->flags |= PGHDR_NEED_SYNC; - } + pPg->flags |= PGHDR_NEED_SYNC; /* An error has occurred writing to the journal file. The ** transaction will be rolled back by the layer above. @@ -5032,7 +5004,7 @@ static int pager_write(PgHdr *pPg){ return rc; } }else{ - if( pPager->eState!=PAGER_WRITER_DBMOD && (!pPager->noSync || 1)){ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ pPg->flags |= PGHDR_NEED_SYNC; } PAGERTRACE(("APPEND %d page %d needSync=%d\n", @@ -5080,6 +5052,9 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( pPager->eState>=PAGER_WRITER_INITIAL ); + assert( assert_pager_state(pPager) ); + if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ @@ -5101,19 +5076,17 @@ int sqlite3PagerWrite(DbPage *pDbPage){ */ pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; - rc = sqlite3PagerPagecount(pPager, (int *)&nPageCount); - if( rc==SQLITE_OK ){ - if( pPg->pgno>nPageCount ){ - nPage = (pPg->pgno - pg1)+1; - }else if( (pg1+nPagePerSector-1)>nPageCount ){ - nPage = nPageCount+1-pg1; - }else{ - nPage = nPagePerSector; - } - assert(nPage>0); - assert(pg1<=pPg->pgno); - assert((pg1+nPage)>pPg->pgno); + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; + }else{ + nPage = nPagePerSector; } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); for(ii=0; iipPCache)); @@ -5773,13 +5746,12 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ int rc = SQLITE_OK; /* Return code */ int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ + assert( pPager->eState>=PAGER_WRITER_INITIAL ); + assert( assert_pager_state(pPager) ); + if( nSavepoint>nCurrent && pPager->useJournal ){ int ii; /* Iterator variable */ PagerSavepoint *aNew; /* New Pager.aSavepoint array */ - int nPage; /* Size of database file */ - - rc = sqlite3PagerPagecount(pPager, &nPage); - if( rc ) return rc; /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM ** if the allocation fails. Otherwise, zero the new portion in case a @@ -5796,14 +5768,14 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ /* Populate the PagerSavepoint structures just allocated. */ for(ii=nCurrent; iidbSize; if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ aNew[ii].iOffset = pPager->journalOff; }else{ aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); } aNew[ii].iSubRec = pPager->nSubRec; - aNew[ii].pInSavepoint = sqlite3BitvecCreate(nPage); + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); if( !aNew[ii].pInSavepoint ){ return SQLITE_NOMEM; } From b22aa4a63de7874a9eb898f3497f8a47d9455774 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 3 Aug 2010 15:57:09 +0000 Subject: [PATCH 10/30] Add comments to clarify the purpose of the pager "error state". FossilOrigin-Name: fde4c59782e98366792e1552fd122e7d80a277ff --- manifest | 12 +++--- manifest.uuid | 2 +- src/pager.c | 103 ++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 82 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index b67585edd7..736464b9c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sstate\sdiagram\sto\scomments\sin\sexperimental\sversion\sof\spager.c. -D 2010-08-03T12:48:46 +C Add\scomments\sto\sclarify\sthe\spurpose\sof\sthe\spager\s"error\sstate". +D 2010-08-03T15:57:10 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 24f978063ff3f18f1632b05b695717cc9818538a +F src/pager.c 809f4dfbca43fd64ef27855e193d8544a5054668 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -841,7 +841,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 0a636798bdb6961a47327091715b254f79add823 -R b916884b782901df5d25760d516d1243 +P 16dcf5a6d3ebff9a734705693b5ac740f4ff2cc1 +R f5f08ae2bf16feac4ddffb2e7526923b U dan -Z a547af01cff05efb8820775070ecaafe +Z 158c2b787243a4910abe77703b2eb315 diff --git a/manifest.uuid b/manifest.uuid index 34229d28a1..c2bab4d837 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16dcf5a6d3ebff9a734705693b5ac740f4ff2cc1 \ No newline at end of file +fde4c59782e98366792e1552fd122e7d80a277ff \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 100f4e90c9..1129021e91 100644 --- a/src/pager.c +++ b/src/pager.c @@ -140,6 +140,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted. ** ** READER: +** ** In this state all the requirements for reading the database in ** rollback (non-WAL) mode are met. Unless the pager is (or recently ** was) in exclusive-locking mode, a user-level read transaction is @@ -152,12 +153,16 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** may not be trusted at this point. ** ** WRITER_INITIAL: +** ** * A write transaction is active. ** * A RESERVED or greater lock is held on the database file. ** * The dbSize, dbOrigSize and dbFileSize variables are all valid. ** * The contents of the pager cache have not been modified. +** * The journal file may or may not be open. +** * Nothing (not even the first header) has been written to the journal. ** ** WRITER_CACHEMOD: +** ** * A write transaction is active. ** * A RESERVED or greater lock is held on the database file. ** * The journal file is open and the first header has been written @@ -165,6 +170,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** * The contents of the page cache have been modified. ** ** WRITER_DBMOD: +** ** * A write transaction is active. ** * An EXCLUSIVE or greater lock is held on the database file. ** * The journal file is open and the first header has been written @@ -173,6 +179,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** written to disk). ** ** WRITER_FINISHED: +** ** * A write transaction is active. ** * An EXCLUSIVE or greater lock is held on the database file. ** * All writing and syncing of journal and database data has finished. @@ -180,12 +187,62 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** commit the transaction. If an error did occur, the caller will need ** to rollback the transaction. ** +** ERROR: +** +** The ERROR state is entered when an IO, OOM or disk-full error +** occurs at a point in the code that makes it difficult to be sure +** that the in-memory pager state (cache contents, db size etc.) are +** consistent with the contents of the file-system. +** +** For example, if an IO error occurs while performing a rollback, +** the contents of the page-cache may be left in an inconsistent state. +** At this point it would be dangerous to change back to READER state +** (as usually happens after a rollback). Any subsequent readers might +** report database corruption (due to the inconsistent cache), and if +** they upgrade to writers, they may inadvertently corrupt the database +** file. To avoid this hazard, the pager switches into the ERROR state +** instead of READER following such an error. +** +** Once it has entered the ERROR state, any attempt to use the pager +** to read or write data returns an error. Eventually, once all +** outstanding transactions have been abandoned, the pager is able to +** transition back to NONE state, discarding the contents of the +** page-cache and any other in-memory state at the same time. Everything +** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** when a read-transaction is next opened on the pager (transitioning +** the pager into READER state). At that point the system has recovered +** from the error. +** +** Specifically, the pager jumps into the ERROR state if: +** +** 1. An error occurs while attempting a rollback. This happens in +** function sqlite3PagerRollback(). +** +** 2. An error occurs while attempting to finalize a journal file +** following a commit in function sqlite3PagerCommitPhaseTwo(). +** +** 3. An error occurs while attempting to write to the journal or +** database file in function pagerStress() in order to free up +** memory. +** +** In other cases, the error is returned to the b-tree layer. The b-tree +** layer then attempts a rollback operation. If the error condition +** persists, the pager enters the ERROR state via condition (1) above. +** +** Condition (3) is necessary because it can be triggered by a read-only +** statement executed within a transaction. In this case, if the error +** code were simply returned to the user, the b-tree layer would not +** automatically attempt a rollback, as it assumes that an error in a +** read-only statement cannot leave the pager in an internally inconsistent +** state. +** +** ** State diagram: ** -** NONE <-------------+ -** | | -** V | -** +---------> READER------------->| +** NONE <------+------+ +** | | | +** V | | +** +---------> READER-------+ | ** | | | ** | V | ** |<-------WRITER_INITIAL-----> ERROR @@ -1619,7 +1676,6 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) ){ rc = zeroJournalHdr(pPager, hasMaster); - pager_error(pPager, rc); pPager->journalOff = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if @@ -4327,7 +4383,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ assert( pPager->eLock==PAGER_UNLOCK ); - return pager_error(pPager, rc); + goto failed; } } @@ -4358,7 +4414,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ */ rc = pagerLock(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ - rc = pager_error(pPager, rc); goto failed; } @@ -4411,6 +4466,10 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** it back since the process that crashed and left the hot journal ** probably did not sync it and we are required to always sync ** the journal before playing it back. + ** + ** Even if an error occurs while syncing or rolling back the + ** hot-journal, there is no need to enter the ERROR state here, as + ** the pager never left state NONE anyhow. */ if( isOpen(pPager->jfd) ){ rc = pagerSyncHotJournal(pPager); @@ -4420,7 +4479,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ pPager->eState = PAGER_NONE; } if( rc!=SQLITE_OK ){ - rc = pager_error(pPager, rc); goto failed; } }else{ @@ -4488,8 +4546,10 @@ int sqlite3PagerSharedLock(Pager *pPager){ failed: if( rc!=SQLITE_OK ){ - /* pager_unlock() is a no-op for exclusive mode and in-memory databases. */ pager_unlock(pPager); + assert( (pPager->eState==PAGER_NONE) + || (pPager->exclusiveMode && pagerUseWal(pPager)) + ); }else{ pPager->eState = PAGER_READER; } @@ -5544,21 +5604,12 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** called, just return the same error code without doing anything. */ if( NEVER(pPager->errCode) ) return pPager->errCode; - /* This function should not be called if the pager is not in at least - ** PAGER_RESERVED state. **FIXME**: Make it so that this test always - ** fails - make it so that we never reach this point if we do not hold - ** all necessary locks. - */ assert( pPager->eState==PAGER_WRITER_INITIAL || pPager->eState==PAGER_WRITER_FINISHED || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) ); assert( assert_pager_state(pPager) ); -#if 0 - if( NEVER(pPager->statesetMaster); if( rc==SQLITE_OK ) rc = rc2; - rc = pager_error(pPager, rc); }else if( pPager->eState==PAGER_WRITER_INITIAL ){ - rc = pager_end_transaction(pPager, pPager->setMaster); + rc = pager_end_transaction(pPager, 0); }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ if( pPager->eState>=PAGER_WRITER_DBMOD ){ pager_playback(pPager, 0); @@ -5660,14 +5709,12 @@ int sqlite3PagerRollback(Pager *pPager){ }else{ rc = pager_playback(pPager, 0); } - - /* If an error occurs during a ROLLBACK, we can no longer trust the pager - ** cache. So call pager_error() on the way out to make any error - ** persistent. - */ - rc = pager_error(pPager, rc); } - return rc; + + /* If an error occurs during a ROLLBACK, we can no longer trust the pager + ** cache. So call pager_error() on the way out to make any error persistent. + */ + return pager_error(pPager, rc); } /* From a42c66bdfb009404579af14a7ded35c03f5dab73 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 3 Aug 2010 18:18:41 +0000 Subject: [PATCH 11/30] Set the Pager.eState variable to PAGER_ERROR whenever the pager enters the error state. FossilOrigin-Name: 4d384761d27425b71555f4f3262778277a955835 --- manifest | 12 ++-- manifest.uuid | 2 +- src/pager.c | 182 ++++++++++++++++++++++++++------------------------ 3 files changed, 101 insertions(+), 95 deletions(-) diff --git a/manifest b/manifest index 736464b9c6..056d09ed02 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scomments\sto\sclarify\sthe\spurpose\sof\sthe\spager\s"error\sstate". -D 2010-08-03T15:57:10 +C Set\sthe\sPager.eState\svariable\sto\sPAGER_ERROR\swhenever\sthe\spager\senters\sthe\serror\sstate. +D 2010-08-03T18:18:42 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 809f4dfbca43fd64ef27855e193d8544a5054668 +F src/pager.c b48eb75d41fd1bce0e9911209785aef4381258ef F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -841,7 +841,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 16dcf5a6d3ebff9a734705693b5ac740f4ff2cc1 -R f5f08ae2bf16feac4ddffb2e7526923b +P fde4c59782e98366792e1552fd122e7d80a277ff +R dd009cdb2bebe6c9ea312f79af5244c4 U dan -Z 158c2b787243a4910abe77703b2eb315 +Z 18695678602885240568b79d8561ccf5 diff --git a/manifest.uuid b/manifest.uuid index c2bab4d837..9923f3bbac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fde4c59782e98366792e1552fd122e7d80a277ff \ No newline at end of file +4d384761d27425b71555f4f3262778277a955835 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 1129021e91..e3f73d953f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -291,6 +291,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ #define PAGER_WRITER_CACHEMOD 3 #define PAGER_WRITER_DBMOD 4 #define PAGER_WRITER_FINISHED 5 +#define PAGER_ERROR 6 /* @@ -696,6 +697,7 @@ static int assert_pager_state(Pager *p){ || p->eState==PAGER_WRITER_CACHEMOD || p->eState==PAGER_WRITER_DBMOD || p->eState==PAGER_WRITER_FINISHED + || p->eState==PAGER_ERROR ); /* Regardless of the current state, a temp-file connection always behaves @@ -718,21 +720,27 @@ static int assert_pager_state(Pager *p){ case PAGER_NONE: assert( !MEMDB ); assert( !p->tempFile ); + assert( pPager->errCode==SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); break; case PAGER_READER: + assert( pPager->errCode==SQLITE_OK ); assert( p->eLock>=SHARED_LOCK || p->noReadlock ); break; case PAGER_WRITER_INITIAL: + assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) ){ assert( p->eLock>=RESERVED_LOCK ); } assert( pPager->dbSize==pPager->dbOrigSize ); assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->setMaster==0 ); break; case PAGER_WRITER_CACHEMOD: + assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) ){ /* It is possible that if journal_mode=wal here that neither the ** journal file nor the WAL file are open. This happens during @@ -749,6 +757,7 @@ static int assert_pager_state(Pager *p){ break; case PAGER_WRITER_DBMOD: + assert( pPager->errCode==SQLITE_OK ); assert( !pagerUseWal(pPager) ); assert( p->eLock>=EXCLUSIVE_LOCK || pagerUseWal(pPager) ); assert( isOpen(p->jfd) @@ -758,6 +767,7 @@ static int assert_pager_state(Pager *p){ break; case PAGER_WRITER_FINISHED: + assert( pPager->errCode==SQLITE_OK ); assert( !pagerUseWal(pPager) ); assert( p->eLock>=EXCLUSIVE_LOCK || pagerUseWal(pPager) ); assert( isOpen(p->jfd) @@ -765,6 +775,15 @@ static int assert_pager_state(Pager *p){ || p->journalMode==PAGER_JOURNALMODE_WAL ); break; + + case PAGER_ERROR: + /* There must be at least one outstanding reference to the pager if + ** in ERROR state. Otherwise the pager should have already dropped + ** back to NONE state. + */ + assert( pPager->errCode!=SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + break; } return 1; @@ -787,7 +806,8 @@ static char *print_pager_state(Pager *p){ p->eState==PAGER_WRITER_INITIAL ? "WRITER_INITIAL" : p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : - p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : "?error?" + p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : + p->eState==PAGER_ERROR ? "ERROR" : "?error?" , p->eLock==NO_LOCK ? "NONE" : p->eLock==RESERVED_LOCK ? "RESERVED" : p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : @@ -1443,16 +1463,11 @@ static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ } /* -** Unless the pager is in error-state, discard all in-memory pages. If -** the pager is in error-state, then this call is a no-op. -** -** TODO: Why can we not reset the pager while in error state? +** Discard the entire contents of the in-memory page-cache. */ static void pager_reset(Pager *pPager){ - if( SQLITE_OK==pPager->errCode ){ - sqlite3BackupRestart(pPager->pBackup); - sqlite3PcacheClear(pPager->pPCache); - } + sqlite3BackupRestart(pPager->pBackup); + sqlite3PcacheClear(pPager->pPCache); } /* @@ -1505,8 +1520,16 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ ** treated as a hot-journal and rolled back. */ static void pager_unlock(Pager *pPager){ - if( !pPager->exclusiveMode ){ - int rc = SQLITE_OK; /* Return code */ + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + releaseAllSavepoints(pPager); + + if( pagerUseWal(pPager) ){ + assert( !isOpen(pPager->jfd) ); + sqlite3WalEndReadTransaction(pPager->pWal); + pPager->eState = PAGER_NONE; + }else if( !pPager->exclusiveMode ){ int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; /* If the operating system support deletion of open files, then @@ -1520,40 +1543,33 @@ static void pager_unlock(Pager *pPager){ assert( (PAGER_JOURNALMODE_DELETE & 5)!=1 ); assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); - if( 0==(iDc & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN) || 1!=(pPager->journalMode & 5) ){ sqlite3OsClose(pPager->jfd); } + osUnlock(pPager, NO_LOCK); - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; - releaseAllSavepoints(pPager); - - if( pagerUseWal(pPager) ){ - sqlite3WalEndReadTransaction(pPager->pWal); - }else{ - rc = osUnlock(pPager, NO_LOCK); - } - if( rc ){ - pPager->errCode = rc; - } - IOTRACE(("UNLOCK %p\n", pPager)) - - /* If Pager.errCode is set, the contents of the pager cache cannot be - ** trusted. Now that the pager file is unlocked, the contents of the - ** cache can be discarded and the error code safely cleared. + /* The pager state may be changed from PAGER_ERROR to PAGER_NONE here + ** without clearing the error code. This is intentional - the error + ** code is cleared and the cache reset in the block below. */ - if( pPager->errCode ){ - if( rc==SQLITE_OK ){ - pPager->errCode = SQLITE_OK; - } - pager_reset(pPager); - } - + assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); pPager->changeCountDone = 0; pPager->eState = PAGER_NONE; + IOTRACE(("UNLOCK %p\n", pPager)) + } + + /* If Pager.errCode is set, the contents of the pager cache cannot be + ** trusted. Now that there are no outstanding references to the pager, + ** it can safely move back to PAGER_NONE state. This happens in both + ** normal and exclusive-locking mode. + */ + if( pPager->errCode ){ + pager_reset(pPager); + pPager->changeCountDone = 0; + pPager->eState = PAGER_NONE; + pPager->errCode = SQLITE_OK; } } @@ -1586,6 +1602,7 @@ static int pager_error(Pager *pPager, int rc){ ); if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ pPager->errCode = rc; + pPager->eState = PAGER_ERROR; } return rc; } @@ -1652,6 +1669,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ /* Do nothing if the pager does not have an open write transaction. */ assert( assert_pager_state(pPager) ); + assert( pPager->eState!=PAGER_ERROR ); if( pPager->eStateeLockerrCode==SQLITE_OK ){ + if( pPager->eState!=PAGER_ERROR ){ + assert( assert_pager_state(pPager) ); if( pPager->eState>=PAGER_WRITER_INITIAL ){ sqlite3BeginBenignMalloc(); sqlite3PagerRollback(pPager); @@ -2199,6 +2218,7 @@ delmaster_out: */ static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; + assert( pPager->eState!=PAGER_ERROR ); if( pPager->eState>=PAGER_WRITER_DBMOD && isOpen(pPager->fd) ){ i64 currentSize, newSize; /* TODO: Is it safe to use Pager.dbFileSize here? */ @@ -2833,6 +2853,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ int rc = SQLITE_OK; /* Return code */ Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ + assert( pPager->eState!=PAGER_ERROR ); assert( pPager->eState>=PAGER_WRITER_INITIAL ); /* Allocate a bitvec to use to store the set of pages rolled back */ @@ -3080,6 +3101,12 @@ void sqlite3PagerSetBusyhandler( int sqlite3PagerSetPagesize(Pager *pPager, u16 *pPageSize, int nReserve){ int rc = pPager->errCode; + /* It is not possible to do a full assert_pager_state() here, as this + ** function may be called from within PagerOpen(), before the state + ** of the Pager object is internally consistent. + */ + assert( rc==SQLITE_OK || pPager->eState==PAGER_ERROR ); + if( rc==SQLITE_OK ){ u16 pageSize = *pPageSize; assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); @@ -3370,7 +3397,7 @@ int sqlite3PagerClose(Pager *pPager){ disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); - pPager->errCode = 0; + /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL sqlite3WalClose(pPager->pWal, @@ -3383,14 +3410,19 @@ int sqlite3PagerClose(Pager *pPager){ if( MEMDB ){ pager_unlock(pPager); }else{ - /* Set Pager.journalHdr to -1 for the benefit of the pager_playback() - ** call which may be made from within pagerUnlockAndRollback(). If it - ** is not -1, then the unsynced portion of an open journal file may - ** be played back into the database. If a power failure occurs while - ** this is happening, the database may become corrupt. + /* If it is open, sync the journal file before calling UnlockAndRollback. + ** If this is not done, then an unsynced portion of the open journal + ** file may be played back into the database. If a power failure occurs + ** while this is happening, the database could become corrupt. + ** + ** If an error occurs while trying to sync the journal, shift the pager + ** into the ERROR state. This causes UnlockAndRollback to unlock the + ** database and close the journal file without attempting to roll it + ** back or finalize it. The next database user will have to do hot-journal + ** rollback before accessing the database file. */ if( isOpen(pPager->jfd) ){ - pPager->errCode = pagerSyncHotJournal(pPager); + pager_error(pPager, pagerSyncHotJournal(pPager)); } pagerUnlockAndRollback(pPager); } @@ -4359,24 +4391,10 @@ int sqlite3PagerSharedLock(Pager *pPager){ assert( assert_pager_state(pPager) ); if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } - /* If this database is in an error-state, now is a chance to clear - ** the error. Discard the contents of the pager-cache and rollback - ** any hot journal in the file-system. - */ - if( pPager->errCode ){ - if( isOpen(pPager->jfd) || pPager->zJournal ){ - isErrorReset = 1; - } - pPager->errCode = SQLITE_OK; - pPager->eState = PAGER_NONE; - pager_reset(pPager); - } - if( !pagerUseWal(pPager) && pPager->eState==PAGER_NONE ){ sqlite3_vfs * const pVfs = pPager->pVfs; assert( !MEMDB && !pPager->tempFile ); - assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( pPager->noReadlock==0 || pPager->readOnly ); if( pPager->noReadlock==0 ){ @@ -4474,6 +4492,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( isOpen(pPager->jfd) ){ rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ + assert( pPager->eState==PAGER_NONE ); pPager->eState = PAGER_WRITER_FINISHED; rc = pager_playback(pPager, 1); pPager->eState = PAGER_NONE; @@ -4540,16 +4559,13 @@ int sqlite3PagerSharedLock(Pager *pPager){ } if( pPager->eState==PAGER_NONE && rc==SQLITE_OK ){ - pPager->eState = PAGER_NONE; rc = pagerPagecount(pPager, &pPager->dbSize); } failed: if( rc!=SQLITE_OK ){ pager_unlock(pPager); - assert( (pPager->eState==PAGER_NONE) - || (pPager->exclusiveMode && pagerUseWal(pPager)) - ); + assert( pPager->eState==PAGER_NONE ); }else{ pPager->eState = PAGER_READER; } @@ -4962,14 +4978,16 @@ static int pager_write(PgHdr *pPg){ /* This routine is not called unless a write-transaction has already ** been started. The journal file may or may not be open at this point. + ** It is never called in the ERROR state. */ - assert( pPager->eState>=PAGER_WRITER_INITIAL ); - assert( pPager->eState!=PAGER_WRITER_FINISHED ); + assert( pPager->eState==PAGER_WRITER_INITIAL + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); assert( assert_pager_state(pPager) ); /* If an error has been previously detected, report the same error - ** again. - */ + ** again. This should not happen, but the check provides robustness. */ if( NEVER(pPager->errCode) ) return pPager->errCode; /* Higher-level routines never call this function if database is not @@ -5113,6 +5131,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); assert( pPager->eState>=PAGER_WRITER_INITIAL ); + assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); if( nPagePerSector>1 ){ @@ -5408,11 +5427,6 @@ int sqlite3PagerCommitPhaseOne( ); assert( assert_pager_state(pPager) ); -#if 0 - /* The dbOrigSize is never set if journal_mode=OFF */ - assert( pPager->journalMode!=PAGER_JOURNALMODE_OFF || pPager->dbOrigSize==0 ); -#endif - /* If a prior error occurred, report that error again. */ if( pPager->errCode ) return pPager->errCode; @@ -5682,8 +5696,12 @@ int sqlite3PagerRollback(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); - /* PagerRollback() is a no-op if called in READER or NONE state. */ + /* PagerRollback() is a no-op if called in READER or NONE state. If + ** the pager is already in the ERROR state, the rollback is not + ** attempted here. Instead, the error code is returned to the caller. + */ assert( assert_pager_state(pPager) ); + if( pPager->eState==PAGER_ERROR ) return pPager->errCode; if( pPager->eState<=PAGER_READER ) return SQLITE_OK; if( pagerUseWal(pPager) ){ @@ -5691,25 +5709,13 @@ int sqlite3PagerRollback(Pager *pPager){ rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); rc2 = pager_end_transaction(pPager, pPager->setMaster); if( rc==SQLITE_OK ) rc = rc2; - }else if( pPager->eState==PAGER_WRITER_INITIAL ){ + }else if( !isOpen(pPager->jfd) ){ rc = pager_end_transaction(pPager, 0); - }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ - if( pPager->eState>=PAGER_WRITER_DBMOD ){ - pager_playback(pPager, 0); - } - rc = pPager->errCode; }else{ - if( pPager->eState==PAGER_WRITER_CACHEMOD ){ - int rc2; - rc = pager_playback(pPager, 0); - rc2 = pager_end_transaction(pPager, pPager->setMaster); - if( rc==SQLITE_OK ){ - rc = rc2; - } - }else{ - rc = pager_playback(pPager, 0); - } + rc = pager_playback(pPager, 0); } + assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); + assert( rc==SQLITE_OK || rc==SQLITE_FULL || (rc&0xFF)==SQLITE_IOERR ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager ** cache. So call pager_error() on the way out to make any error persistent. From 431b0b4291042f6250071088fa5590842eb3180e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 4 Aug 2010 19:14:22 +0000 Subject: [PATCH 12/30] Fix some problems with error recovery introduced while reworking pager state. FossilOrigin-Name: 77eaab6f77c53cc4f429b65dfcf287ad6084c2da --- manifest | 12 ++-- manifest.uuid | 2 +- src/pager.c | 179 +++++++++++++++++++++++++------------------------- 3 files changed, 96 insertions(+), 97 deletions(-) diff --git a/manifest b/manifest index b0af40bea5..ee1423fe59 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sexperimental\sbranch. -D 2010-08-03T18:29:05 +C Fix\ssome\sproblems\swith\serror\srecovery\sintroduced\swhile\sreworking\spager\sstate. +D 2010-08-04T19:14:22 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c a5f55fb219e53bd064a81ccf697da30285ba2cc6 +F src/pager.c d181fe863c886da94d7c19cf62385858e610384e F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -842,7 +842,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 4d384761d27425b71555f4f3262778277a955835 57c0960038b8ce97f9d6665f15e7f6ec310c681f -R a5e53c42c5ef67b21638e39c5b136ceb +P 15368a9f8523d5fb611cd576080daed2cf2f1500 +R 38aa93658d08e993fa60d40008fca82a U dan -Z c69428969280d297bd2cfd79a9c3f85e +Z 9ffe03344bd2caf8cb58c57ef391fd3e diff --git a/manifest.uuid b/manifest.uuid index bf9ed997d3..f37f20b9ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15368a9f8523d5fb611cd576080daed2cf2f1500 \ No newline at end of file +77eaab6f77c53cc4f429b65dfcf287ad6084c2da \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 52065f1f39..da70b6eb53 100644 --- a/src/pager.c +++ b/src/pager.c @@ -127,7 +127,25 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ /* ** The Pager.eState variable stores the current 'state' of a pager. A -** pager may be in any one of the following six states: +** pager may be in any one of the seven states shown in the following +** state diagram. +** +** NONE <------+------+ +** | | | +** V | | +** +---------> READER-------+ | +** | | | +** | V | +** |<-------WRITER_INITIAL-----> ERROR +** | | ^ +** | V | +** |<------WRITER_CACHEMOD-------->| +** | | | +** | V | +** |<-------WRITER_DBMOD---------->| +** | | | +** | V | +** +<------WRITER_FINISHED-------->+ ** ** NONE: ** @@ -237,26 +255,6 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** state. ** ** -** State diagram: -** -** NONE <------+------+ -** | | | -** V | | -** +---------> READER-------+ | -** | | | -** | V | -** |<-------WRITER_INITIAL-----> ERROR -** | | ^ -** | V | -** |<------WRITER_CACHEMOD-------->| -** | | | -** | V | -** |<-------WRITER_DBMOD---------->| -** | | | -** | V | -** +<------WRITER_FINISHED-------->+ -** -** ** State transitions and the [function] that performs each: ** ** NONE -> READER [PagerSharedLock] @@ -265,12 +263,11 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] ** WRITER_DBMOD -> WRITER_FINISHED [PagerCommitPhaseOne] ** -** WRITER_INITIAL -> READER [pager_end_transaction] -** WRITER_CACHEMOD -> READER [pager_end_transaction] -** WRITER_DBMOD -> READER [pager_end_transaction] -** WRITER_FINISHED -> READER [pager_end_transaction] +** WRITER_*** -> READER [pager_end_transaction] +** WRITER_*** -> ERROR [pager_error] ** ** READER -> NONE [pager_unlock] +** ERROR -> NONE [pager_unlock] ** ** Notes: ** @@ -716,6 +713,11 @@ static int assert_pager_state(Pager *p){ /* Check that MEMDB implies noSync. */ assert( !MEMDB || p->noSync ); + /* If changeCountDone is set, a RESERVED lock or greater must be held + ** on the file. + */ + assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); + switch( p->eState ){ case PAGER_NONE: assert( !MEMDB ); @@ -891,13 +893,31 @@ static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ ** If file pFd is open, call sqlite3OsUnlock() on it. */ static int osUnlock(Pager *pPager, int eLock){ - if( !isOpen(pPager->fd) ){ - return SQLITE_OK; + int rc = SQLITE_OK; + if( isOpen(pPager->fd) ){ + assert( pPager->eLock>=eLock ); + assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); + rc = sqlite3OsUnlock(pPager->fd, eLock); + if( rc==SQLITE_OK ){ + pPager->eLock = eLock; + } } - assert( pPager->eLock>=eLock ); - assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); - pPager->eLock = eLock; - return sqlite3OsUnlock(pPager->fd, eLock); + return rc; +} + +static int osLock(Pager *pPager, int eLock){ + int rc; + assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); + if( pPager->eLock>=eLock ){ + rc = SQLITE_OK; + }else{ + rc = sqlite3OsLock(pPager->fd, eLock); + if( rc==SQLITE_OK ){ + pPager->eLock = eLock; + IOTRACE(("LOCK %p %d\n", pPager, locktype)) + } + } + return rc; } /* @@ -1565,7 +1585,7 @@ static void pager_unlock(Pager *pPager){ ** it can safely move back to PAGER_NONE state. This happens in both ** normal and exclusive-locking mode. */ - if( pPager->errCode ){ + if( pPager->errCode && !pPager->tempFile ){ pager_reset(pPager); pPager->changeCountDone = 0; pPager->eState = PAGER_NONE; @@ -1728,10 +1748,10 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); assert( rc2==SQLITE_OK ); - if( !pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, 0) ){ - rc2 = osUnlock(pPager, SHARED_LOCK); - } - }else if( !pPager->exclusiveMode ){ + } + if( !pPager->exclusiveMode + && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) + ){ rc2 = osUnlock(pPager, SHARED_LOCK); pPager->changeCountDone = 0; } @@ -3245,20 +3265,6 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ } -static int pagerLock(Pager *pPager, int eLock){ - int rc; - assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); - if( pPager->eLock>=eLock ){ - rc = SQLITE_OK; - }else{ - rc = sqlite3OsLock(pPager->fd, eLock); - if( rc==SQLITE_OK ){ - pPager->eLock = eLock; - } - } - return rc; -} - /* ** Try to obtain a lock of type locktype on the database file. If ** a similar or greater lock is already held, this function is a no-op @@ -3295,12 +3301,8 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ rc = SQLITE_OK; }else{ do { - rc = sqlite3OsLock(pPager->fd, locktype); + rc = osLock(pPager, locktype); }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); - if( rc==SQLITE_OK ){ - pPager->eLock = (u8)locktype; - IOTRACE(("LOCK %p %d\n", pPager, locktype)) - } } return rc; } @@ -4274,7 +4276,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ rc = sqlite3OsAccess(pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &exists); } if( rc==SQLITE_OK && exists ){ - int locked; /* True if some process holds a RESERVED lock */ + int locked = 0; /* True if some process holds a RESERVED lock */ /* Race condition here: Another process might have been holding the ** the RESERVED lock and have a journal open at the sqlite3OsAccess() @@ -4298,9 +4300,9 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK ){ if( nPage==0 ){ sqlite3BeginBenignMalloc(); - if( sqlite3OsLock(pPager->fd, RESERVED_LOCK)==SQLITE_OK ){ + if( osLock(pPager, RESERVED_LOCK)==SQLITE_OK ){ sqlite3OsDelete(pVfs, pPager->zJournal, 0); - sqlite3OsUnlock(pPager->fd, SHARED_LOCK); + osUnlock(pPager, SHARED_LOCK); } sqlite3EndBenignMalloc(); }else{ @@ -4379,7 +4381,6 @@ static int hasHotJournal(Pager *pPager, int *pExists){ */ int sqlite3PagerSharedLock(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ - int isErrorReset = 0; /* True if recovering from error state */ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either @@ -4392,7 +4393,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } if( !pagerUseWal(pPager) && pPager->eState==PAGER_NONE ){ - sqlite3_vfs * const pVfs = pPager->pVfs; + int bHotJournal = 1; /* True if there exists a hot journal-file */ assert( !MEMDB && !pPager->tempFile ); assert( pPager->noReadlock==0 || pPager->readOnly ); @@ -4408,13 +4409,13 @@ int sqlite3PagerSharedLock(Pager *pPager){ /* If a journal file exists, and there is no RESERVED lock on the ** database file, then it either needs to be played back or deleted. */ - if( !isErrorReset ){ - rc = hasHotJournal(pPager, &isErrorReset); - if( rc!=SQLITE_OK ){ - goto failed; - } + if( pPager->eLock<=SHARED_LOCK ){ + rc = hasHotJournal(pPager, &bHotJournal); } - if( isErrorReset ){ + if( rc!=SQLITE_OK ){ + goto failed; + } + if( bHotJournal ){ /* Get an EXCLUSIVE lock on the database file. At this point it is ** important that a RESERVED lock is not obtained on the way to the ** EXCLUSIVE lock. If it were, another process might open the @@ -4430,7 +4431,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** Unless the pager is in locking_mode=exclusive mode, the lock is ** downgraded to SHARED_LOCK before this function returns. */ - rc = pagerLock(pPager, EXCLUSIVE_LOCK); + rc = osLock(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ goto failed; } @@ -4449,19 +4450,19 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** function was called and the journal file does not exist. */ if( !isOpen(pPager->jfd) ){ - int res; - rc = sqlite3OsAccess(pVfs,pPager->zJournal,SQLITE_ACCESS_EXISTS,&res); - if( rc==SQLITE_OK ){ - if( res ){ - int fout = 0; - int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; - assert( !pPager->tempFile ); - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); - assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); - if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ - rc = SQLITE_CANTOPEN_BKPT; - sqlite3OsClose(pPager->jfd); - } + sqlite3_vfs * const pVfs = pPager->pVfs; + int bExists; /* True if journal file exists */ + rc = sqlite3OsAccess( + pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &bExists); + if( rc==SQLITE_OK && bExists ){ + int fout = 0; + int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + assert( !pPager->tempFile ); + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ + rc = SQLITE_CANTOPEN_BKPT; + sqlite3OsClose(pPager->jfd); } } } @@ -4581,9 +4582,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) - && (!pPager->exclusiveMode || pPager->journalOff>0) - ){ + if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } @@ -4907,7 +4906,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** exclusive lock on the database is not already held, obtain it now. */ if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ - rc = pagerLock(pPager, EXCLUSIVE_LOCK); + rc = osLock(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ return rc; } @@ -4926,7 +4925,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** busy-handler callback can be used when upgrading to the EXCLUSIVE ** lock, but not when obtaining the RESERVED lock. */ - rc = pagerLock(pPager, RESERVED_LOCK); + rc = osLock(pPager, RESERVED_LOCK); if( rc==SQLITE_OK && exFlag ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } @@ -6263,13 +6262,13 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } if( pPager->eState==PAGER_READER ){ assert( rc==SQLITE_OK ); - rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); + rc = osLock(pPager, RESERVED_LOCK); } if( rc==SQLITE_OK ){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } if( rc==SQLITE_OK && state==PAGER_READER ){ - sqlite3OsUnlock(pPager->fd, SHARED_LOCK); + osUnlock(pPager, SHARED_LOCK); }else if( state==PAGER_NONE ){ pager_unlock(pPager); } @@ -6417,7 +6416,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ */ if( !pPager->pWal ){ int logexists = 0; - rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_SHARED); + rc = osLock(pPager, SHARED_LOCK); if( rc==SQLITE_OK ){ rc = sqlite3OsAccess( pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists @@ -6433,7 +6432,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ ** the database file, the log and log-summary files will be deleted. */ if( rc==SQLITE_OK && pPager->pWal ){ - rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE); + rc = osLock(pPager, EXCLUSIVE_LOCK); if( rc==SQLITE_OK ){ rc = sqlite3WalClose(pPager->pWal, (pPager->noSync ? 0 : pPager->sync_flags), @@ -6443,7 +6442,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ }else{ /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock ** that we did get back to SHARED. */ - sqlite3OsUnlock(pPager->fd, SQLITE_LOCK_SHARED); + osUnlock(pPager, SQLITE_LOCK_SHARED); } } return rc; From 4e004aa6b92a3fc01aaec1647bc291004d68eae3 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 5 Aug 2010 15:30:22 +0000 Subject: [PATCH 13/30] Fixes for error handling with temp databases. And for errors that occur within OS locking primitives. FossilOrigin-Name: f99a902f9b3d0bf6f607c4ba641b4096fbbef5d5 --- manifest | 12 ++-- manifest.uuid | 2 +- src/pager.c | 181 ++++++++++++++++++++++++++------------------------ 3 files changed, 102 insertions(+), 93 deletions(-) diff --git a/manifest b/manifest index ee1423fe59..406d746d4a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\swith\serror\srecovery\sintroduced\swhile\sreworking\spager\sstate. -D 2010-08-04T19:14:22 +C Fixes\sfor\serror\shandling\swith\stemp\sdatabases.\sAnd\sfor\serrors\sthat\soccur\swithin\sOS\slocking\sprimitives. +D 2010-08-05T15:30:22 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c d181fe863c886da94d7c19cf62385858e610384e +F src/pager.c c09ca0cf83f928f0299be6620322acd9c38fa3ce F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -842,7 +842,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 15368a9f8523d5fb611cd576080daed2cf2f1500 -R 38aa93658d08e993fa60d40008fca82a +P 77eaab6f77c53cc4f429b65dfcf287ad6084c2da +R 002859c356edc1f0f7f124dc9ba8ad0d U dan -Z 9ffe03344bd2caf8cb58c57ef391fd3e +Z f6fbebc4624605af3301d8eb4f042add diff --git a/manifest.uuid b/manifest.uuid index f37f20b9ad..aab1319941 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77eaab6f77c53cc4f429b65dfcf287ad6084c2da \ No newline at end of file +f99a902f9b3d0bf6f607c4ba641b4096fbbef5d5 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index da70b6eb53..ac172c6ae2 100644 --- a/src/pager.c +++ b/src/pager.c @@ -345,6 +345,8 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ #define PAGER_EXCLUSIVE 4 /* same as EXCLUSIVE_LOCK */ #define PAGER_SYNCED 5 +#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) + /* ** A macro used for invoking the codec if there is one */ @@ -721,17 +723,18 @@ static int assert_pager_state(Pager *p){ switch( p->eState ){ case PAGER_NONE: assert( !MEMDB ); - assert( !p->tempFile ); assert( pPager->errCode==SQLITE_OK ); - assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile ); break; case PAGER_READER: assert( pPager->errCode==SQLITE_OK ); + assert( p->eLock!=UNKNOWN_LOCK ); assert( p->eLock>=SHARED_LOCK || p->noReadlock ); break; case PAGER_WRITER_INITIAL: + assert( p->eLock!=UNKNOWN_LOCK ); assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) ){ assert( p->eLock>=RESERVED_LOCK ); @@ -742,6 +745,7 @@ static int assert_pager_state(Pager *p){ break; case PAGER_WRITER_CACHEMOD: + assert( p->eLock!=UNKNOWN_LOCK ); assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) ){ /* It is possible that if journal_mode=wal here that neither the @@ -759,9 +763,10 @@ static int assert_pager_state(Pager *p){ break; case PAGER_WRITER_DBMOD: + assert( p->eLock==EXCLUSIVE_LOCK ); assert( pPager->errCode==SQLITE_OK ); assert( !pagerUseWal(pPager) ); - assert( p->eLock>=EXCLUSIVE_LOCK || pagerUseWal(pPager) ); + assert( p->eLock>=EXCLUSIVE_LOCK ); assert( isOpen(p->jfd) || p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_WAL @@ -769,9 +774,9 @@ static int assert_pager_state(Pager *p){ break; case PAGER_WRITER_FINISHED: + assert( p->eLock==EXCLUSIVE_LOCK ); assert( pPager->errCode==SQLITE_OK ); assert( !pagerUseWal(pPager) ); - assert( p->eLock>=EXCLUSIVE_LOCK || pagerUseWal(pPager) ); assert( isOpen(p->jfd) || p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_WAL @@ -798,11 +803,12 @@ static char *print_pager_state(Pager *p){ static char zRet[1024]; sqlite3_snprintf(1024, zRet, - "State: %s\n" + "State: %s errCode=%d\n" "Lock: %s\n" "Locking mode: locking_mode=%s\n" "Journal mode: journal_mode=%s\n" "Backing store: tempFile=%d memDb=%d useJournal=%d\n" + "Journal: journalOff=%lld journalHdr=%lld\n" , p->eState==PAGER_NONE ? "NONE" : p->eState==PAGER_READER ? "READER" : p->eState==PAGER_WRITER_INITIAL ? "WRITER_INITIAL" : @@ -810,10 +816,12 @@ static char *print_pager_state(Pager *p){ p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : p->eState==PAGER_ERROR ? "ERROR" : "?error?" + , (int)p->errCode , p->eLock==NO_LOCK ? "NONE" : p->eLock==RESERVED_LOCK ? "RESERVED" : p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : - p->eLock==SHARED_LOCK ? "SHARED" : "?error?" + p->eLock==SHARED_LOCK ? "SHARED" : + p->eLock==UNKNOWN_LOCK ? "UNKNOWN" : "?error?" , p->exclusiveMode ? "exclusive" : "normal" , p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" : p->journalMode==PAGER_JOURNALMODE_OFF ? "off" : @@ -822,6 +830,7 @@ static char *print_pager_state(Pager *p){ p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" : p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" , (int)p->tempFile, (int)p->memDb, (int)p->useJournal + , p->journalOff, p->journalHdr ); return zRet; @@ -890,31 +899,35 @@ static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ } /* -** If file pFd is open, call sqlite3OsUnlock() on it. +** This function and pagerLockDb() are wrappers around sqlite3OsLock() and +** sqlite3OsUnlock() that set the Pager.eLock variable to reflect the +** current lock held on the database file. */ -static int osUnlock(Pager *pPager, int eLock){ +static int pagerUnlockDb(Pager *pPager, int eLock){ int rc = SQLITE_OK; + assert( !pPager->exclusiveMode ); if( isOpen(pPager->fd) ){ assert( pPager->eLock>=eLock ); assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); rc = sqlite3OsUnlock(pPager->fd, eLock); - if( rc==SQLITE_OK ){ + if( pPager->eLock!=UNKNOWN_LOCK ){ pPager->eLock = eLock; } + IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) } return rc; } -static int osLock(Pager *pPager, int eLock){ +static int pagerLockDb(Pager *pPager, int eLock){ int rc; assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); - if( pPager->eLock>=eLock ){ + if( pPager->eLock>=eLock && pPager->eLock!=UNKNOWN_LOCK ){ rc = SQLITE_OK; }else{ rc = sqlite3OsLock(pPager->fd, eLock); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ pPager->eLock = eLock; - IOTRACE(("LOCK %p %d\n", pPager, locktype)) + IOTRACE(("LOCK %p %d\n", pPager, eLock)) } } return rc; @@ -1550,6 +1563,7 @@ static void pager_unlock(Pager *pPager){ sqlite3WalEndReadTransaction(pPager->pWal); pPager->eState = PAGER_NONE; }else if( !pPager->exclusiveMode ){ + int rc; /* Error code returned by pagerUnlockDb() */ int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; /* If the operating system support deletion of open files, then @@ -1568,7 +1582,11 @@ static void pager_unlock(Pager *pPager){ ){ sqlite3OsClose(pPager->jfd); } - osUnlock(pPager, NO_LOCK); + + rc = pagerUnlockDb(pPager, NO_LOCK); + if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){ + pPager->eLock = UNKNOWN_LOCK; + } /* The pager state may be changed from PAGER_ERROR to PAGER_NONE here ** without clearing the error code. This is intentional - the error @@ -1577,7 +1595,6 @@ static void pager_unlock(Pager *pPager){ assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); pPager->changeCountDone = 0; pPager->eState = PAGER_NONE; - IOTRACE(("UNLOCK %p\n", pPager)) } /* If Pager.errCode is set, the contents of the pager cache cannot be @@ -1585,12 +1602,16 @@ static void pager_unlock(Pager *pPager){ ** it can safely move back to PAGER_NONE state. This happens in both ** normal and exclusive-locking mode. */ - if( pPager->errCode && !pPager->tempFile ){ + if( pPager->errCode && !MEMDB ){ pager_reset(pPager); - pPager->changeCountDone = 0; + pPager->changeCountDone = pPager->tempFile; pPager->eState = PAGER_NONE; pPager->errCode = SQLITE_OK; } + + pPager->journalOff = 0; + pPager->journalHdr = 0; + pPager->setMaster = 0; } /* @@ -1752,7 +1773,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ - rc2 = osUnlock(pPager, SHARED_LOCK); + rc2 = pagerUnlockDb(pPager, SHARED_LOCK); pPager->changeCountDone = 0; } pPager->eState = PAGER_READER; @@ -1934,7 +1955,9 @@ static int pager_playback_one_page( if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ return rc; } - assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD + || (pPager->eState==PAGER_NONE && isMainJrnl) + ); /* When playing back page 1, restore the nReserve setting */ @@ -1990,7 +2013,7 @@ static int pager_playback_one_page( }else{ isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); } - if( (pPager->eState>=PAGER_WRITER_DBMOD) + if( (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_NONE) && isOpen(pPager->fd) && isSynced ){ @@ -2239,7 +2262,11 @@ delmaster_out: static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; assert( pPager->eState!=PAGER_ERROR ); - if( pPager->eState>=PAGER_WRITER_DBMOD && isOpen(pPager->fd) ){ + assert( pPager->eState!=PAGER_READER ); + + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_NONE) + ){ i64 currentSize, newSize; /* TODO: Is it safe to use Pager.dbFileSize here? */ rc = sqlite3OsFileSize(pPager->fd, ¤tSize); @@ -2817,7 +2844,7 @@ static int pagerOpenWalIfPresent(Pager *pPager){ } if( rc==SQLITE_OK ){ if( isWal ){ - assert( sqlite3PcachePagecount(pPager->pPCache)==0 ); + testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); rc = sqlite3PagerOpenWal(pPager, 0); if( rc==SQLITE_OK ){ rc = pagerBeginReadTransaction(pPager); @@ -3144,7 +3171,6 @@ int sqlite3PagerSetPagesize(Pager *pPager, u16 *pPageSize, int nReserve){ if( !pNew ){ rc = SQLITE_NOMEM; }else{ - assert( pPager->eState==PAGER_NONE || pPager->eState==PAGER_READER ); pager_reset(pPager); pPager->dbSize = nByte/pageSize; pPager->pageSize = pageSize; @@ -3297,13 +3323,9 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) ); - if( pPager->eLock>=locktype ){ - rc = SQLITE_OK; - }else{ - do { - rc = osLock(pPager, locktype); - }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); - } + do { + rc = pagerLockDb(pPager, locktype); + }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); return rc; } @@ -4300,9 +4322,9 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK ){ if( nPage==0 ){ sqlite3BeginBenignMalloc(); - if( osLock(pPager, RESERVED_LOCK)==SQLITE_OK ){ + if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ sqlite3OsDelete(pVfs, pPager->zJournal, 0); - osUnlock(pPager, SHARED_LOCK); + pagerUnlockDb(pPager, SHARED_LOCK); } sqlite3EndBenignMalloc(); }else{ @@ -4388,14 +4410,14 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** exclusive access mode. */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); - assert( pPager->eState==PAGER_NONE || pPager->eState==PAGER_READER ); assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_NONE || pPager->eState==PAGER_READER ); if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } if( !pagerUseWal(pPager) && pPager->eState==PAGER_NONE ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ - assert( !MEMDB && !pPager->tempFile ); + assert( !MEMDB ); assert( pPager->noReadlock==0 || pPager->readOnly ); if( pPager->noReadlock==0 ){ @@ -4431,7 +4453,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** Unless the pager is in locking_mode=exclusive mode, the lock is ** downgraded to SHARED_LOCK before this function returns. */ - rc = osLock(pPager, EXCLUSIVE_LOCK); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ goto failed; } @@ -4466,17 +4488,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ } } } - if( rc!=SQLITE_OK ){ - goto failed; - } - - /* Reset the journal status fields to indicates that we have no - ** rollback journal at this time. */ - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - - /* Make sure the journal file has been synced to disk. */ /* Playback and delete the journal. Drop the database write ** lock and reacquire the read lock. Purge the cache before @@ -4485,24 +4496,21 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** it back since the process that crashed and left the hot journal ** probably did not sync it and we are required to always sync ** the journal before playing it back. - ** - ** Even if an error occurs while syncing or rolling back the - ** hot-journal, there is no need to enter the ERROR state here, as - ** the pager never left state NONE anyhow. */ if( isOpen(pPager->jfd) ){ + assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ - assert( pPager->eState==PAGER_NONE ); - pPager->eState = PAGER_WRITER_FINISHED; rc = pager_playback(pPager, 1); pPager->eState = PAGER_NONE; } - if( rc!=SQLITE_OK ){ - goto failed; - } - }else{ - osUnlock(pPager, SHARED_LOCK); + }else if( !pPager->exclusiveMode ){ + pagerUnlockDb(pPager, SHARED_LOCK); + } + + if( rc!=SQLITE_OK ){ + pager_error(pPager, rc); + goto failed; } assert( pPager->eState==PAGER_NONE ); @@ -4511,7 +4519,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 ){ + if( !pPager->tempFile + && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) + ){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -4906,7 +4916,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** exclusive lock on the database is not already held, obtain it now. */ if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ - rc = osLock(pPager, EXCLUSIVE_LOCK); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ return rc; } @@ -4925,7 +4935,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** busy-handler callback can be used when upgrading to the EXCLUSIVE ** lock, but not when obtaining the RESERVED lock. */ - rc = osLock(pPager, RESERVED_LOCK); + rc = pagerLockDb(pPager, RESERVED_LOCK); if( rc==SQLITE_OK && exFlag ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } @@ -5874,12 +5884,12 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ ** savepoint. If no errors occur, SQLITE_OK is returned. */ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ - int rc = SQLITE_OK; + int rc = pPager->errCode; /* Return code */ assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); - if( iSavepointnSavepoint ){ + if( rc==SQLITE_OK && iSavepointnSavepoint ){ int ii; /* Iterator variable */ int nNew; /* Number of remaining savepoints after this op. */ @@ -5915,8 +5925,8 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } - } + return rc; } @@ -6092,6 +6102,16 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ sqlite3PcacheMove(pPg, pgno); sqlite3PcacheMakeDirty(pPg); + /* For an in-memory database, make sure the original page continues + ** to exist, in case the transaction needs to roll back. Use pPgOld + ** as the original page since it has already been allocated. + */ + if( MEMDB ){ + assert( pPgOld ); + sqlite3PcacheMove(pPgOld, origPgno); + sqlite3PagerUnref(pPgOld); + } + if( needSyncPgno ){ /* If needSyncPgno is non-zero, then the journal file needs to be ** sync()ed before any data is written to database file page needSyncPgno. @@ -6121,16 +6141,6 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ sqlite3PagerUnref(pPgHdr); } - /* - ** For an in-memory database, make sure the original page continues - ** to exist, in case the transaction needs to roll back. Use pPgOld - ** as the original page since it has already been allocated. - */ - if( MEMDB ){ - sqlite3PcacheMove(pPgOld, origPgno); - sqlite3PagerUnref(pPgOld); - } - return SQLITE_OK; } #endif @@ -6221,14 +6231,6 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } if( eMode!=eOld ){ - /* When changing between rollback modes, close the journal file prior - ** to the change. But when changing from a rollback mode to WAL, keep - ** the journal open since there is a rollback-style transaction in play - ** used to convert the version numbers in the btree header. - */ - if( isOpen(pPager->jfd) && eMode!=PAGER_JOURNALMODE_WAL ){ - sqlite3OsClose(pPager->jfd); - } /* Change the journal mode. */ pPager->journalMode = (u8)eMode; @@ -6247,6 +6249,8 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ assert( isOpen(pPager->fd) || pPager->exclusiveMode ); if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){ + sqlite3OsClose(pPager->jfd); + /* In this case we would like to delete the journal file. If it is ** not possible, then that is not a problem. Deleting the journal file ** here is an optimization only. @@ -6262,13 +6266,13 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } if( pPager->eState==PAGER_READER ){ assert( rc==SQLITE_OK ); - rc = osLock(pPager, RESERVED_LOCK); + rc = pagerLockDb(pPager, RESERVED_LOCK); } if( rc==SQLITE_OK ){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } if( rc==SQLITE_OK && state==PAGER_READER ){ - osUnlock(pPager, SHARED_LOCK); + pagerUnlockDb(pPager, SHARED_LOCK); }else if( state==PAGER_NONE ){ pager_unlock(pPager); } @@ -6293,6 +6297,7 @@ int sqlite3PagerGetJournalMode(Pager *pPager){ ** is unmodified. */ int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ + assert( assert_pager_state(pPager) ); if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; return 1; @@ -6380,6 +6385,10 @@ int sqlite3PagerOpenWal( if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; + if( isOpen(pPager->jfd) ){ + sqlite3OsClose(pPager->jfd); + } + /* Open the connection to the log file. If this operation fails, ** (e.g. due to malloc() failure), unlock the database file and ** return an error code. @@ -6416,7 +6425,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ */ if( !pPager->pWal ){ int logexists = 0; - rc = osLock(pPager, SHARED_LOCK); + rc = pagerLockDb(pPager, SHARED_LOCK); if( rc==SQLITE_OK ){ rc = sqlite3OsAccess( pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists @@ -6432,7 +6441,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ ** the database file, the log and log-summary files will be deleted. */ if( rc==SQLITE_OK && pPager->pWal ){ - rc = osLock(pPager, EXCLUSIVE_LOCK); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc==SQLITE_OK ){ rc = sqlite3WalClose(pPager->pWal, (pPager->noSync ? 0 : pPager->sync_flags), @@ -6442,7 +6451,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ }else{ /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock ** that we did get back to SHARED. */ - osUnlock(pPager, SQLITE_LOCK_SHARED); + pagerUnlockDb(pPager, SQLITE_LOCK_SHARED); } } return rc; From 5761dbe4a36a768310e402393113e20d36d46db4 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 5 Aug 2010 16:08:26 +0000 Subject: [PATCH 14/30] Catch an error code that was not being propagated back to the caller. FossilOrigin-Name: 800f496929fb0d45d16c573c5dca0197ac922e2b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 406d746d4a..24129bb6ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\serror\shandling\swith\stemp\sdatabases.\sAnd\sfor\serrors\sthat\soccur\swithin\sOS\slocking\sprimitives. -D 2010-08-05T15:30:22 +C Catch\san\serror\scode\sthat\swas\snot\sbeing\spropagated\sback\sto\sthe\scaller. +D 2010-08-05T16:08:26 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c c09ca0cf83f928f0299be6620322acd9c38fa3ce +F src/pager.c 7232578996c9ecd7b57bb9952a81549f2f133a4b F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -842,7 +842,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 77eaab6f77c53cc4f429b65dfcf287ad6084c2da -R 002859c356edc1f0f7f124dc9ba8ad0d +P f99a902f9b3d0bf6f607c4ba641b4096fbbef5d5 +R 619c44a049e165e5bdd234c179e111ef U dan -Z f6fbebc4624605af3301d8eb4f042add +Z 4f4c91f530cb0822ccee88049df426ee diff --git a/manifest.uuid b/manifest.uuid index aab1319941..21086831e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f99a902f9b3d0bf6f607c4ba641b4096fbbef5d5 \ No newline at end of file +800f496929fb0d45d16c573c5dca0197ac922e2b \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index ac172c6ae2..cbbce3ff0a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3615,6 +3615,7 @@ static int syncJournal(Pager *pPager, int newHdr){ if( newHdr && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ pPager->nRec = 0; rc = writeJournalHdr(pPager); + if( rc!=SQLITE_OK ) return rc; } }else{ pPager->journalHdr = pPager->journalOff; From 54919f82386d6fe6e38bfcb71520ec950b2cab0e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 5 Aug 2010 18:53:26 +0000 Subject: [PATCH 15/30] Add comments describing UNKNOWN_LOCK to pager.c. Improve some other comments i the same file. FossilOrigin-Name: 54eff6de9d8d87f33192c192ca91907c4c090988 --- manifest | 12 ++-- manifest.uuid | 2 +- src/pager.c | 168 +++++++++++++++++++++++++++----------------------- 3 files changed, 98 insertions(+), 84 deletions(-) diff --git a/manifest b/manifest index 4b915c1909..510ddb5501 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\swith\sexperimental\sbranch. -D 2010-08-05T16:22:50 +C Add\scomments\sdescribing\sUNKNOWN_LOCK\sto\spager.c.\sImprove\ssome\sother\scomments\si\sthe\ssame\sfile. +D 2010-08-05T18:53:27 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 7232578996c9ecd7b57bb9952a81549f2f133a4b +F src/pager.c 07bda904e5b3e6fe6c9425a205fcaf1dca5a444c F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -842,7 +842,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 800f496929fb0d45d16c573c5dca0197ac922e2b ca479f3de2927ccc05dc76d10e40c00b8e0c88d1 -R 5a6390c995b69a7f924deee65f1b7dc5 +P acd26b8b746980c344db017a0e96dbd92c89acdf +R c079a70d95550574b0e39982e236e749 U dan -Z 434bd8bfb5691e86b993c0ccfbfc4d4c +Z 83b26f553c42e8e1edd2dad0b7de50dd diff --git a/manifest.uuid b/manifest.uuid index c5356f714b..8f9281da3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -acd26b8b746980c344db017a0e96dbd92c89acdf \ No newline at end of file +54eff6de9d8d87f33192c192ca91907c4c090988 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index cbbce3ff0a..f5a2c4c4b3 100644 --- a/src/pager.c +++ b/src/pager.c @@ -163,12 +163,23 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** rollback (non-WAL) mode are met. Unless the pager is (or recently ** was) in exclusive-locking mode, a user-level read transaction is ** open. The database size is known in this state. +** +** A connection running with locking_mode=normal enters this state when +** it opens a read-transaction on the database and returns to state +** NONE after the read-transaction is completed. However a connection +** running in locking_mode=exclusive (including temp databases) remains in +** this state even after the read-transaction is closed. The only way +** a locking_mode=exclusive connection can transition from READER to NONE +** is via the ERROR state (see below). ** -** * A read transaction may be active. +** * A read transaction may be active (but a write-transaction cannot). ** * A SHARED or greater lock is held on the database file. ** * The dbSize variable may be trusted (even if a user-level read ** transaction is not active). The dbOrigSize and dbFileSize variables ** may not be trusted at this point. +** * If the database is a WAL database, then the WAL connection is open. +** * Even if a read-transaction is not open, it is guaranteed that +** there is no hot-journal in the file-system. ** ** WRITER_INITIAL: ** @@ -290,62 +301,55 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ #define PAGER_WRITER_FINISHED 5 #define PAGER_ERROR 6 - /* -** The page cache as a whole is always in one of the following -** states: +** The Pager.eLock variable is almost always set to one of the +** following locking-states, according to the lock currently held on +** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** This variable is kept up to date as locks are taken and released by +** the pagerLockDb() and pagerUnlockDb() wrappers. ** -** PAGER_UNLOCK The page cache is not currently reading or -** writing the database file. There is no -** data held in memory. This is the initial -** state. +** If the VFS xLock() or xUnlock() returns an error other than SQLITE_BUSY +** (i.e. one of the SQLITE_IOERR subtypes), it is not clear whether or not +** the operation was successful. In these circumstances pagerLockDb() and +** pagerUnlockDb() take a conservative approach - eLock is always updated +** when unlocking the file, and only updated when locking the file if the +** VFS call is successful. This way, the Pager.eLock variable may be set +** to a less exclusive (lower) value than the lock that is actually held +** at the system level, but it is never set to a more exclusive value. ** -** PAGER_SHARED The page cache is reading the database. -** Writing is not permitted. There can be -** multiple readers accessing the same database -** file at the same time. +** This is usually safe. If an xUnlock fails or appears to fail, there may +** be a few redundant xLock() calls or a lock may be held for longer than +** required, but nothing really goes wrong. ** -** PAGER_RESERVED This process has reserved the database for writing -** but has not yet made any changes. Only one process -** at a time can reserve the database. The original -** database file has not been modified so other -** processes may still be reading the on-disk -** database file. +** The exception is when the database file is unlocked as the pager moves +** from ERROR to NONE state. At this point there may be a hot-journal file +** in the file-system that needs to be rolled back (as part of a NONE->SHARED +** transition, by the same pager or any other). If the call to xUnlock() +** fails at this point and the pager is left holding an EXCLUSIVE lock, this +** can confuse the call to xCheckReservedLock() call made later as part +** of hot-journal detection. ** -** PAGER_EXCLUSIVE The page cache is writing the database. -** Access is exclusive. No other processes or -** threads can be reading or writing while one -** process is writing. +** xCheckReservedLock() is defined as returning true "if there is a RESERVED +** lock held by this process or any others". So xCheckReservedLock may +** return true because the caller itself is holding an EXCLUSIVE lock (but +** doesn't know it because of a previous error in xUnlock). If this happens +** a hot-journal may be mistaken for a journal being created by an active +** transaction in another process, causing SQLite to read from the database +** without rolling it back. ** -** PAGER_SYNCED The pager moves to this state from PAGER_EXCLUSIVE -** after all dirty pages have been written to the -** database file and the file has been synced to -** disk. All that remains to do is to remove or -** truncate the journal file and the transaction -** will be committed. +** To work around this, if a call to xUnlock() fails when unlocking the +** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It +** is only changed back to a real locking state after a successful call +** to xLock(EXCLUSIVE). Also, the code to do the NONE->SHARED state transition +** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK +** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE +** lock on the database file before attempting to roll it back. See function +** PagerSharedLock() for more detail. ** -** The page cache comes up in PAGER_UNLOCK. The first time a -** sqlite3PagerGet() occurs, the state transitions to PAGER_SHARED. -** After all pages have been released using sqlite_page_unref(), -** the state transitions back to PAGER_UNLOCK. The first time -** that sqlite3PagerWrite() is called, the state transitions to -** PAGER_RESERVED. (Note that sqlite3PagerWrite() can only be -** called on an outstanding page which means that the pager must -** be in PAGER_SHARED before it transitions to PAGER_RESERVED.) -** PAGER_RESERVED means that there is an open rollback journal. -** The transition to PAGER_EXCLUSIVE occurs before any changes -** are made to the database file, though writes to the rollback -** journal occurs with just PAGER_RESERVED. After an sqlite3PagerRollback() -** or sqlite3PagerCommitPhaseTwo(), the state can go back to PAGER_SHARED, -** or it can stay at PAGER_EXCLUSIVE if we are in exclusive access mode. +** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in +** PAGER_NONE state. */ -#define PAGER_UNLOCK 0 -#define PAGER_SHARED 1 /* same as SHARED_LOCK */ -#define PAGER_RESERVED 2 /* same as RESERVED_LOCK */ -#define PAGER_EXCLUSIVE 4 /* same as EXCLUSIVE_LOCK */ -#define PAGER_SYNCED 5 - -#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) +#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) /* ** A macro used for invoking the codec if there is one @@ -398,14 +402,11 @@ struct PagerSavepoint { ** A open page cache is an instance of the following structure. ** ** errCode +** The Pager.errCode variable is only ever non-zero when the condition +** (Pager.eState==PAGER_ERROR) is true. ** -** Pager.errCode may be set to SQLITE_IOERR, SQLITE_CORRUPT, or -** or SQLITE_FULL. Once one of the first three errors occurs, it persists -** and is returned as the result of every major pager API call. The -** SQLITE_FULL return code is slightly different. It persists only until the -** next successful rollback is performed on the pager cache. Also, -** SQLITE_FULL does not affect the sqlite3PagerGet() and sqlite3PagerLookup() -** APIs, they may still be used successfully. +** Pager.errCode may be set to SQLITE_FULL, SQLITE_IOERR or one of the +** SQLITE_IOERR_XXX sub-codes. ** ** dbSize, dbOrigSize, dbFileSize ** @@ -418,10 +419,7 @@ struct PagerSavepoint { ** out from the cache to the actual file on disk. Or if the image has been ** truncated by an incremental-vacuum operation. The Pager.dbOrigSize variable ** contains the number of pages in the database image when the current -** transaction was opened. The contents of all three of these variables is -** only guaranteed to be correct if the boolean Pager.dbSizeValid is true. -** -** TODO: Under what conditions is dbSizeValid set? Cleared? +** transaction was opened. ** ** changeCountDone ** @@ -719,6 +717,7 @@ static int assert_pager_state(Pager *p){ ** on the file. */ assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); + assert( p->eLock!=PENDING_LOCK ); switch( p->eState ){ case PAGER_NONE: @@ -899,16 +898,22 @@ static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ } /* -** This function and pagerLockDb() are wrappers around sqlite3OsLock() and -** sqlite3OsUnlock() that set the Pager.eLock variable to reflect the -** current lock held on the database file. +** Unlock the database file to level eLock, which must be either NO_LOCK +** or SHARED_LOCK. Regardless of whether or not the call to xUnlock() +** succeeds, set the Pager.eLock variable to match the (attempted) new lock. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it. See the comment above the #define of +** UNKNOWN_LOCK for an explanation of this. */ static int pagerUnlockDb(Pager *pPager, int eLock){ int rc = SQLITE_OK; + assert( !pPager->exclusiveMode ); + assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); + assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); if( isOpen(pPager->fd) ){ assert( pPager->eLock>=eLock ); - assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); rc = sqlite3OsUnlock(pPager->fd, eLock); if( pPager->eLock!=UNKNOWN_LOCK ){ pPager->eLock = eLock; @@ -918,12 +923,21 @@ static int pagerUnlockDb(Pager *pPager, int eLock){ return rc; } +/* +** Lock the database file to level eLock, which must be either SHARED_LOCK, +** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the +** Pager.eLock variable to the new locking state. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. +** See the comment above the #define of UNKNOWN_LOCK for an explanation +** of this. +*/ static int pagerLockDb(Pager *pPager, int eLock){ - int rc; + int rc = SQLITE_OK; + assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); - if( pPager->eLock>=eLock && pPager->eLock!=UNKNOWN_LOCK ){ - rc = SQLITE_OK; - }else{ + if( pPager->eLockeLock==UNKNOWN_LOCK ){ rc = sqlite3OsLock(pPager->fd, eLock); if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ pPager->eLock = eLock; @@ -1583,6 +1597,11 @@ static void pager_unlock(Pager *pPager){ sqlite3OsClose(pPager->jfd); } + /* If the pager is in the ERROR state and the call to unlock the database + ** file fails, set the current lock to UNKNOWN_LOCK. See the comment + ** above the #define for UNKNOWN_LOCK for an explanation of why this + ** is necessary. + */ rc = pagerUnlockDb(pPager, NO_LOCK); if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){ pPager->eLock = UNKNOWN_LOCK; @@ -2750,7 +2769,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ int changed = 0; /* True if cache must be reset */ assert( pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_NONE || pPager->eState==PAGER_SHARED ); + assert( pPager->eState==PAGER_NONE || pPager->eState==PAGER_READER ); /* sqlite3WalEndReadTransaction() was not called for the previous ** transaction in locking_mode=EXCLUSIVE. So call it now. If we @@ -3284,7 +3303,7 @@ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ ** this is considered a 1 page file. */ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ - assert( pPager->eState>=PAGER_SHARED ); + assert( pPager->eState>=PAGER_READER ); assert( pPager->eState!=PAGER_WRITER_FINISHED ); *pnPage = (int)pPager->dbSize; return SQLITE_OK; @@ -3308,11 +3327,6 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; /* Return code */ - /* The OS lock values must be the same as the Pager lock values */ - assert( PAGER_SHARED==SHARED_LOCK ); - assert( PAGER_RESERVED==RESERVED_LOCK ); - assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK ); - /* Check that this is either a no-op (because the requested lock is ** already held, or one of the transistions that the busy-handler ** may be invoked during, according to the comment above @@ -4424,7 +4438,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( pPager->noReadlock==0 ){ rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ - assert( pPager->eLock==PAGER_UNLOCK ); + assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); goto failed; } } @@ -5393,7 +5407,7 @@ int sqlite3PagerExclusiveLock(Pager *pPager){ ); assert( assert_pager_state(pPager) ); if( 0==pagerUseWal(pPager) ){ - rc = pager_wait_on_lock(pPager, PAGER_EXCLUSIVE); + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } return rc; } From 11f47a9b4e425b3029888df5ebbcd600bc2d6dde Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 6 Aug 2010 06:54:47 +0000 Subject: [PATCH 16/30] Fix a bug to do with deleting the journal file when exiting exclusive-locking mode. FossilOrigin-Name: 6217b607f0cd60383c6cb4ab0fe9da008f611244 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/pager.c | 48 ++++++++++++++++++++++++++++++++---------------- test/pager3.test | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 23 deletions(-) create mode 100644 test/pager3.test diff --git a/manifest b/manifest index 510ddb5501..d9b39071df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scomments\sdescribing\sUNKNOWN_LOCK\sto\spager.c.\sImprove\ssome\sother\scomments\si\sthe\ssame\sfile. -D 2010-08-05T18:53:27 +C Fix\sa\sbug\sto\sdo\swith\sdeleting\sthe\sjournal\sfile\swhen\sexiting\sexclusive-locking\smode. +D 2010-08-06T06:54:48 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 07bda904e5b3e6fe6c9425a205fcaf1dca5a444c +F src/pager.c f8fdab85e2dd0a3a55fc2bf212035f50ffcada13 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -541,6 +541,7 @@ F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test af02ed0a9cbc0d2a61b8f35171d4d117e588e4ec F test/pager1.test d8e4b2bc8164c920e6ea0572c9e13576d6e4f3fa F test/pager2.test f5c757c271ce642d36a393ecbfb3aef1c240dcef +F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test c1d176326ce244db157ce9c3ba128be2a9b172d6 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 @@ -842,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P acd26b8b746980c344db017a0e96dbd92c89acdf -R c079a70d95550574b0e39982e236e749 +P 54eff6de9d8d87f33192c192ca91907c4c090988 +R d766f7dfb5a440ae684ea90a650f2fcb U dan -Z 83b26f553c42e8e1edd2dad0b7de50dd +Z d043fca41f68ab4a258746c2916bd634 diff --git a/manifest.uuid b/manifest.uuid index 8f9281da3f..6fc4825e63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54eff6de9d8d87f33192c192ca91907c4c090988 \ No newline at end of file +6217b607f0cd60383c6cb4ab0fe9da008f611244 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index f5a2c4c4b3..c4833f1521 100644 --- a/src/pager.c +++ b/src/pager.c @@ -147,6 +147,22 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** | V | ** +<------WRITER_FINISHED-------->+ ** +** +** List of state transitions and the C [function] that performs each: +** +** NONE -> READER [sqlite3PagerSharedLock] +** READER -> NONE [pager_unlock] +** +** READER -> WRITER_INITIAL [sqlite3PagerBegin] +** WRITER_INITIAL -> WRITER_CACHEMOD [pager_open_journal] +** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] +** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne] +** WRITER_*** -> READER [pager_end_transaction] +** +** WRITER_*** -> ERROR [pager_error] +** ERROR -> NONE [pager_unlock] +** +** ** NONE: ** ** The pager starts up in this state. Nothing is guaranteed in this @@ -171,6 +187,10 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** this state even after the read-transaction is closed. The only way ** a locking_mode=exclusive connection can transition from READER to NONE ** is via the ERROR state (see below). +** +** TODO: Maybe WAL connections should behave like locking_mode=exclusive +** connections and remain in READER state even when there is no +** active read transaction. ** ** * A read transaction may be active (but a write-transaction cannot). ** * A SHARED or greater lock is held on the database file. @@ -183,8 +203,15 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** WRITER_INITIAL: ** +** The pager moves to this state from READER when a write-transaction +** is first opened on the database. +** ** * A write transaction is active. -** * A RESERVED or greater lock is held on the database file. +** * If the connection is open in rollback-mode, a RESERVED or greater +** lock is held on the database file. +** * If the connection is open in WAL-mode, a WAL write transaction +** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully +** called). ** * The dbSize, dbOrigSize and dbFileSize variables are all valid. ** * The contents of the pager cache have not been modified. ** * The journal file may or may not be open. @@ -266,20 +293,6 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** state. ** ** -** State transitions and the [function] that performs each: -** -** NONE -> READER [PagerSharedLock] -** READER -> WRITER_INITIAL [PagerBegin] -** WRITER_INITIAL -> WRITER_CACHEMOD [pager_open_journal] -** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] -** WRITER_DBMOD -> WRITER_FINISHED [PagerCommitPhaseOne] -** -** WRITER_*** -> READER [pager_end_transaction] -** WRITER_*** -> ERROR [pager_error] -** -** READER -> NONE [pager_unlock] -** ERROR -> NONE [pager_unlock] -** ** Notes: ** ** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the @@ -802,12 +815,14 @@ static char *print_pager_state(Pager *p){ static char zRet[1024]; sqlite3_snprintf(1024, zRet, + "Filename: %s\n" "State: %s errCode=%d\n" "Lock: %s\n" "Locking mode: locking_mode=%s\n" "Journal mode: journal_mode=%s\n" "Backing store: tempFile=%d memDb=%d useJournal=%d\n" "Journal: journalOff=%lld journalHdr=%lld\n" + , p->zFilename , p->eState==PAGER_NONE ? "NONE" : p->eState==PAGER_READER ? "READER" : p->eState==PAGER_WRITER_INITIAL ? "WRITER_INITIAL" : @@ -1819,13 +1834,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ ** call to pager_unlock(), as described above. */ static void pagerUnlockAndRollback(Pager *pPager){ - if( pPager->eState!=PAGER_ERROR ){ + if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_NONE ){ assert( assert_pager_state(pPager) ); if( pPager->eState>=PAGER_WRITER_INITIAL ){ sqlite3BeginBenignMalloc(); sqlite3PagerRollback(pPager); sqlite3EndBenignMalloc(); }else if( pPager->eLock>=RESERVED_LOCK && !pPager->exclusiveMode ){ + assert( pPager->eState==PAGER_READER ); pager_end_transaction(pPager, 0); } } diff --git a/test/pager3.test b/test/pager3.test new file mode 100644 index 0000000000..23435a79b7 --- /dev/null +++ b/test/pager3.test @@ -0,0 +1,33 @@ +# 2010 June 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + + +foreach {tn sql res j} { + 1 "PRAGMA journal_mode = DELETE" delete 0 + 2 "CREATE TABLE t1(a, b)" {} 0 + 3 "PRAGMA locking_mode=EXCLUSIVE" {exclusive} 0 + 4 "INSERT INTO t1 VALUES(1, 2)" {} 1 + 5 "PRAGMA locking_mode=NORMAL" {normal} 1 + 6 "SELECT * FROM t1" {1 2} 0 +} { + do_execsql_test pager3-1.$tn.1 $sql $res + do_test pager3-1.$tn.2 { file exists test.db-journal } $j +} + + +finish_test From dc1106148aee75dc069e820fef0b4894d652582b Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 6 Aug 2010 09:43:03 +0000 Subject: [PATCH 17/30] Modify test_journal.c to work with pre-allocated databases. FossilOrigin-Name: 4894a5d21044b169f412fef095e689cb8a8575dc --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_journal.c | 10 +++++++++- test/fallocate.test | 6 ++++++ 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d9b39071df..3770cf9225 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sto\sdo\swith\sdeleting\sthe\sjournal\sfile\swhen\sexiting\sexclusive-locking\smode. -D 2010-08-06T06:54:48 +C Modify\stest_journal.c\sto\swork\swith\spre-allocated\sdatabases. +D 2010-08-06T09:43:04 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -198,7 +198,7 @@ F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 -F src/test_journal.c 424a334cdfdc8a6f975abe3641440147bded3185 +F src/test_journal.c 5ac89727cadbcc71954067b319eb40aff71dead2 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 058e41dc853b1fa8ccd1083e9571018a74aeb3a7 F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3 @@ -351,7 +351,7 @@ F test/exclusive.test b1f9012cabc124af947165d15ffa62ad20f63db8 F test/exclusive2.test fcbb1c9ca9739292a0a22a3763243ad6d868086b F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/expr.test 9f521ae22f00e074959f72ce2e55d46b9ed23f68 -F test/fallocate.test 0594314eb04268f7d0779d054fa850a36a5ae8bc +F test/fallocate.test 56242d30325fa451247d13e103cf1c357751a066 F test/filectrl.test 97003734290887566e01dded09dc9e99cb937e9e F test/filefmt.test f77c92141960b7933bc6691631d2ad62257ef40a F test/fkey1.test 01c7de578e11747e720c2d9aeef27f239853c4da @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 54eff6de9d8d87f33192c192ca91907c4c090988 -R d766f7dfb5a440ae684ea90a650f2fcb +P 6217b607f0cd60383c6cb4ab0fe9da008f611244 +R 14b2ef30676c4e420e673009cd2ed209 U dan -Z d043fca41f68ab4a258746c2916bd634 +Z f16fc7fc34cf746cc2538d8ca729728e diff --git a/manifest.uuid b/manifest.uuid index 6fc4825e63..41a2b36786 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6217b607f0cd60383c6cb4ab0fe9da008f611244 \ No newline at end of file +4894a5d21044b169f412fef095e689cb8a8575dc \ No newline at end of file diff --git a/src/test_journal.c b/src/test_journal.c index 97e0e1b70a..0300fb8716 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -379,6 +379,15 @@ static int openTransaction(jt_file *pMain, jt_file *pJournal){ ** leaf to the jt_file.pWritable bitvec. */ rc = sqlite3OsRead(p, aData, pMain->nPagesize, 0); + if( rc==SQLITE_OK ){ + u32 nDbsize = decodeUint32(&aData[28]); + if( nDbsize>0 && memcmp(&aData[24], &aData[92], 4)==0 ){ + u32 iPg; + for(iPg=nDbsize+1; iPg<=pMain->nPage; iPg++){ + sqlite3BitvecSet(pMain->pWritable, iPg); + } + } + } iTrunk = decodeUint32(&aData[32]); while( rc==SQLITE_OK && iTrunk>0 ){ u32 nLeaf; @@ -492,7 +501,6 @@ finish_rjf: return rc; } - /* ** Write data to an jt-file. */ diff --git a/test/fallocate.test b/test/fallocate.test index 82ceb1d8fa..ab41fe0a23 100644 --- a/test/fallocate.test +++ b/test/fallocate.test @@ -68,6 +68,10 @@ do_test fallocate-1.8 { execsql { COMMIT } } {} # The following tests - fallocate-2.* - test that things work in WAL # mode as well. # +set skipwaltests [expr {[permutation]=="journaltest"}] +ifcapable !wal { set skipwaltests 1 } + +if {!$skipwaltests} { db close file delete -force test.db sqlite3 db test.db @@ -130,6 +134,8 @@ do_test fallocate-2.8 { file size test.db } [expr 32*1024] +} + finish_test From de1ae34eedfdc0851609323a060f0f6e71b2ec0e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 6 Aug 2010 12:00:27 +0000 Subject: [PATCH 18/30] Further updates to comments in pager.c. FossilOrigin-Name: 5f4c17a33fd442d66c1bf75af9b21954f6906897 --- manifest | 12 +- manifest.uuid | 2 +- src/pager.c | 354 ++++++++++++++++++++++++++++++-------------------- 3 files changed, 219 insertions(+), 149 deletions(-) diff --git a/manifest b/manifest index 3770cf9225..64c799c7db 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\stest_journal.c\sto\swork\swith\spre-allocated\sdatabases. -D 2010-08-06T09:43:04 +C Further\supdates\sto\scomments\sin\spager.c. +D 2010-08-06T12:00:27 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c f8fdab85e2dd0a3a55fc2bf212035f50ffcada13 +F src/pager.c 3ac13a537a5a1c3fe8cbb39a52e255a52b8b3c99 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 6217b607f0cd60383c6cb4ab0fe9da008f611244 -R 14b2ef30676c4e420e673009cd2ed209 +P 4894a5d21044b169f412fef095e689cb8a8575dc +R c8de9a7d2c35078aa3e4ea331272cd57 U dan -Z f16fc7fc34cf746cc2538d8ca729728e +Z e0fe6da6eb2070e7d54dde6c557a0593 diff --git a/manifest.uuid b/manifest.uuid index 41a2b36786..0e41de40a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4894a5d21044b169f412fef095e689cb8a8575dc \ No newline at end of file +5f4c17a33fd442d66c1bf75af9b21954f6906897 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index c4833f1521..a1d6b513ed 100644 --- a/src/pager.c +++ b/src/pager.c @@ -130,13 +130,13 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** pager may be in any one of the seven states shown in the following ** state diagram. ** -** NONE <------+------+ +** OPEN <------+------+ ** | | | ** V | | ** +---------> READER-------+ | ** | | | ** | V | -** |<-------WRITER_INITIAL-----> ERROR +** |<-------WRITER_LOCKED------> ERROR ** | | ^ ** | V | ** |<------WRITER_CACHEMOD-------->| @@ -150,20 +150,20 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** List of state transitions and the C [function] that performs each: ** -** NONE -> READER [sqlite3PagerSharedLock] -** READER -> NONE [pager_unlock] +** OPEN -> READER [sqlite3PagerSharedLock] +** READER -> OPEN [pager_unlock] ** -** READER -> WRITER_INITIAL [sqlite3PagerBegin] -** WRITER_INITIAL -> WRITER_CACHEMOD [pager_open_journal] +** READER -> WRITER_LOCKED [sqlite3PagerBegin] +** WRITER_LOCKED -> WRITER_CACHEMOD [pager_open_journal] ** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] ** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne] ** WRITER_*** -> READER [pager_end_transaction] ** ** WRITER_*** -> ERROR [pager_error] -** ERROR -> NONE [pager_unlock] +** ERROR -> OPEN [pager_unlock] ** ** -** NONE: +** OPEN: ** ** The pager starts up in this state. Nothing is guaranteed in this ** state - the file may or may not be locked and the database size is @@ -182,15 +182,11 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** A connection running with locking_mode=normal enters this state when ** it opens a read-transaction on the database and returns to state -** NONE after the read-transaction is completed. However a connection +** OPEN after the read-transaction is completed. However a connection ** running in locking_mode=exclusive (including temp databases) remains in ** this state even after the read-transaction is closed. The only way -** a locking_mode=exclusive connection can transition from READER to NONE +** a locking_mode=exclusive connection can transition from READER to OPEN ** is via the ERROR state (see below). -** -** TODO: Maybe WAL connections should behave like locking_mode=exclusive -** connections and remain in READER state even when there is no -** active read transaction. ** ** * A read transaction may be active (but a write-transaction cannot). ** * A SHARED or greater lock is held on the database file. @@ -201,10 +197,23 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** * Even if a read-transaction is not open, it is guaranteed that ** there is no hot-journal in the file-system. ** -** WRITER_INITIAL: +** WRITER_LOCKED: ** ** The pager moves to this state from READER when a write-transaction -** is first opened on the database. +** is first opened on the database. In WRITER_LOCKED state, all locks +** required to start a write-transaction are held, but no actual +** modifications to the cache or database have taken place. +** +** In rollback mode, a RESERVED or (if the transaction was opened with +** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when +** moving to this state, but the journal file is not written to or opened +** to in this state. If the transaction is committed or rolled back while +** in WRITER_LOCKED state, all that is required is to unlock the database +** file. +** +** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. +** If the connection is running with locking_mode=exclusive, an attempt +** is made to obtain an EXCLUSIVE lock on the database file. ** ** * A write transaction is active. ** * If the connection is open in rollback-mode, a RESERVED or greater @@ -219,6 +228,11 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** WRITER_CACHEMOD: ** +** A pager moves from WRITER_LOCKED state to this state when a page is +** first modified by the upper layer. In rollback mode the journal file +** is opened (if it is not already open) and a header written to the +** start of it. The database file on disk has not been modified. +** ** * A write transaction is active. ** * A RESERVED or greater lock is held on the database file. ** * The journal file is open and the first header has been written @@ -262,7 +276,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** Once it has entered the ERROR state, any attempt to use the pager ** to read or write data returns an error. Eventually, once all ** outstanding transactions have been abandoned, the pager is able to -** transition back to NONE state, discarding the contents of the +** transition back to OPEN state, discarding the contents of the ** page-cache and any other in-memory state at the same time. Everything ** is reloaded from disk (and, if necessary, hot-journal rollback peformed) ** when a read-transaction is next opened on the pager (transitioning @@ -291,6 +305,10 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** automatically attempt a rollback, as it assumes that an error in a ** read-only statement cannot leave the pager in an internally inconsistent ** state. +** +** * The Pager.errCode variable is set to something other than SQLITE_OK. +** * There are one or more outstanding references to pages (after the +** last reference is dropped the pager should move back to OPEN state). ** ** ** Notes: @@ -299,16 +317,16 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** connection is open in WAL mode. A WAL connection is always in one ** of the first four states. ** -** * Normally, a connection open in exclusive mode is never in PAGER_NONE +** * Normally, a connection open in exclusive mode is never in PAGER_OPEN ** state. There are two exceptions: immediately after exclusive-mode has ** been turned on (and before any read or write transactions are ** executed), and when the pager is leaving the "error state". ** ** * See also: assert_pager_state(). */ -#define PAGER_NONE 0 +#define PAGER_OPEN 0 #define PAGER_READER 1 -#define PAGER_WRITER_INITIAL 2 +#define PAGER_WRITER_LOCKED 2 #define PAGER_WRITER_CACHEMOD 3 #define PAGER_WRITER_DBMOD 4 #define PAGER_WRITER_FINISHED 5 @@ -335,8 +353,8 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** required, but nothing really goes wrong. ** ** The exception is when the database file is unlocked as the pager moves -** from ERROR to NONE state. At this point there may be a hot-journal file -** in the file-system that needs to be rolled back (as part of a NONE->SHARED +** from ERROR to OPEN state. At this point there may be a hot-journal file +** in the file-system that needs to be rolled back (as part of a OPEN->SHARED ** transition, by the same pager or any other). If the call to xUnlock() ** fails at this point and the pager is left holding an EXCLUSIVE lock, this ** can confuse the call to xCheckReservedLock() call made later as part @@ -353,14 +371,14 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** To work around this, if a call to xUnlock() fails when unlocking the ** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It ** is only changed back to a real locking state after a successful call -** to xLock(EXCLUSIVE). Also, the code to do the NONE->SHARED state transition +** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition ** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK ** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE ** lock on the database file before attempting to roll it back. See function ** PagerSharedLock() for more detail. ** ** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in -** PAGER_NONE state. +** PAGER_OPEN state. */ #define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) @@ -412,27 +430,28 @@ struct PagerSavepoint { }; /* -** A open page cache is an instance of the following structure. +** A open page cache is an instance of struct Pager. A description of +** some of the more important member variables follows: ** -** errCode -** The Pager.errCode variable is only ever non-zero when the condition -** (Pager.eState==PAGER_ERROR) is true. +** eState ** -** Pager.errCode may be set to SQLITE_FULL, SQLITE_IOERR or one of the -** SQLITE_IOERR_XXX sub-codes. +** The current 'state' of the pager object. See the comment and state +** diagram above for a description of the pager state. ** -** dbSize, dbOrigSize, dbFileSize +** eLock ** -** Managing the size of the database file in pages is a little complicated. -** The variable Pager.dbSize contains the number of pages that the database -** image currently contains. As the database image grows or shrinks this -** variable is updated. The variable Pager.dbFileSize contains the number -** of pages in the database file. This may be different from Pager.dbSize -** if some pages have been appended to the database image but not yet written -** out from the cache to the actual file on disk. Or if the image has been -** truncated by an incremental-vacuum operation. The Pager.dbOrigSize variable -** contains the number of pages in the database image when the current -** transaction was opened. +** For a real on-disk database, the current lock held on the database file - +** NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** +** For a temporary or in-memory database (neither of which require any +** locks), this variable is always set to EXCLUSIVE_LOCK. Since such +** databases always have Pager.exclusiveMode==1, this tricks the pager +** logic into thinking that it already has all the locks it will ever +** need (and no reason to release them). +** +** In some (obscure) circumstances, this variable may also be set to +** UNKNOWN_LOCK. See the comment above the #define of UNKNOWN_LOCK for +** details. ** ** changeCountDone ** @@ -451,24 +470,6 @@ struct PagerSavepoint { ** need only update the change-counter once, for the first transaction ** committed. ** -** dbModified -** -** The dbModified flag is set whenever a database page is dirtied. -** It is cleared at the end of each transaction. -** -** It is used when committing or otherwise ending a transaction. If -** the dbModified flag is clear then less work has to be done. -** -** journalStarted -** -** This flag is set during a write-transaction after the first -** journal-header is written and synced to disk. -** -** After this has happened, new pages appended to the database -** do not need the PGHDR_NEED_SYNC flag set, as they do not need -** to wait for a journal sync before they can be written out to -** the database file (see function pager_write()). -** ** setMaster ** ** When PagerCommitPhaseOne() is called to commit a transaction, it may @@ -479,10 +480,10 @@ struct PagerSavepoint { ** the way in which the journal file is finalized after the transaction is ** committed or rolled back when running in "journal_mode=PERSIST" mode. ** If a journal file does not contain a master-journal pointer, it is -** finalized by overwriting the first journal header with zeroes. If, -** on the other hand, it does contain a master-journal pointer, the -** journal file is finalized by truncating it to zero bytes, just as if -** the connection were running in "journal_mode=truncate" mode. +** finalized by overwriting the first journal header with zeroes. If +** it does contain a master-journal pointer the journal file is finalized +** by truncating it to zero bytes, just as if the connection were +** running in "journal_mode=truncate" mode. ** ** Journal files that contain master journal pointers cannot be finalized ** simply by overwriting the first journal-header with zeroes, as the @@ -492,7 +493,7 @@ struct PagerSavepoint { ** The flag is cleared as soon as the journal file is finalized (either ** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the ** journal file from being successfully finalized, the setMaster flag -** is cleared anyway. +** is cleared anyway (and the pager will move to ERROR state). ** ** doNotSpill, doNotSyncSpill ** @@ -509,6 +510,58 @@ struct PagerSavepoint { ** This is a boolean variable. If true, then any required sub-journal ** is opened as an in-memory journal file. If false, then in-memory ** sub-journals are only used for in-memory pager files. +** +** This variable is updated by the upper layer each time a new +** write-transaction is opened. +** +** dbSize, dbOrigSize, dbFileSize +** +** Variable dbSize is set to the number of pages in the database file. +** It is valid in PAGER_READER and higher states (all states except for +** OPEN and ERROR). +** +** dbSize is set based on the size of the database file, which may be +** larger than the size of the database (the value stored at offset +** 28 of the database header by the btree). If the size of the file +** is not an integer multiple of the page-size, the value stored in +** dbSize is rounded down (i.e. a 5KB file with 2K page-size has dbSize==2). +** Except, any file that is greater than 0 bytes in size is considered +** to have at least one page. (i.e. a 1KB file with 2K page-size leads +** to dbSize==1). +** +** During a write-transaction, if pages with page-numbers greater than +** dbSize are modified in the cache, dbSize is updated accordingly. +** Similarly, if the database is truncated using PagerTruncateImage(), +** dbSize is updated. +** +** Variables dbOrigSize and dbFileSize are valid in states +** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize +** variable at the start of the transaction. It is used during rollback, +** and to determine whether or not pages need to be journalled before +** being modified. +** +** Throughout a write-transaction, dbFileSize contains the size of +** the file on disk in pages. It is set to a copy of dbSize when the +** write-transaction is first opened, and updated when VFS calls are made +** to write or truncate the database file on disk. +** +** errCode +** +** The Pager.errCode variable is only ever used in PAGER_ERROR state. It +** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode +** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX +** sub-codes. +** +** If Pager.errCode is set to SQLITE_IOERR or one of its subcodes, then +** this value is immediately returned when ever any sqlite3PagerXXX() method +** that returns an error code is called. If it is set to SQLITE_FULL, +** then it is returned whenever any such sqlite3PagerXXX() method except +** for PagerAcquire() or PagerLookup() is called. +** +** TODO: Review the SQLITE_FULL/PagerAcquire() exception. Is it a good idea? +** If so, are there bugs whereby shared-cache clients can see +** uncommitted data when the pager is in the ERROR state? +** */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ @@ -534,7 +587,7 @@ struct Pager { ** other variables in this block are described in the comment directly ** above this class definition. */ - u8 eState; /* Pager state (NONE, READER, WRITER_INITIAL..) */ + u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */ u8 eLock; /* Current lock held on database file */ u8 changeCountDone; /* Set after incrementing the change-counter */ u8 setMaster; /* True if a m-j name has been written to jrnl */ @@ -696,14 +749,17 @@ static int pagerUseWal(Pager *pPager){ ** Usage: ** ** assert( assert_pager_state(pPager) ); +** +** This function runs many asserts to try to find inconsistencies in +** the internal state of the Pager object. */ static int assert_pager_state(Pager *p){ Pager *pPager = p; /* State must be valid. */ - assert( p->eState==PAGER_NONE + assert( p->eState==PAGER_OPEN || p->eState==PAGER_READER - || p->eState==PAGER_WRITER_INITIAL + || p->eState==PAGER_WRITER_LOCKED || p->eState==PAGER_WRITER_CACHEMOD || p->eState==PAGER_WRITER_DBMOD || p->eState==PAGER_WRITER_FINISHED @@ -733,7 +789,7 @@ static int assert_pager_state(Pager *p){ assert( p->eLock!=PENDING_LOCK ); switch( p->eState ){ - case PAGER_NONE: + case PAGER_OPEN: assert( !MEMDB ); assert( pPager->errCode==SQLITE_OK ); assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile ); @@ -745,7 +801,7 @@ static int assert_pager_state(Pager *p){ assert( p->eLock>=SHARED_LOCK || p->noReadlock ); break; - case PAGER_WRITER_INITIAL: + case PAGER_WRITER_LOCKED: assert( p->eLock!=UNKNOWN_LOCK ); assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) ){ @@ -798,7 +854,7 @@ static int assert_pager_state(Pager *p){ case PAGER_ERROR: /* There must be at least one outstanding reference to the pager if ** in ERROR state. Otherwise the pager should have already dropped - ** back to NONE state. + ** back to OPEN state. */ assert( pPager->errCode!=SQLITE_OK ); assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); @@ -809,6 +865,11 @@ static int assert_pager_state(Pager *p){ } /* +** Return a pointer to a human readable string in a static buffer +** containing the state of the Pager object passed as an argument. This +** is intended to be used within debuggers. For example, as an alternative +** to "print *pPager" in gdb: +** ** (gdb) printf "%s", print_pager_state(pPager) */ static char *print_pager_state(Pager *p){ @@ -823,15 +884,15 @@ static char *print_pager_state(Pager *p){ "Backing store: tempFile=%d memDb=%d useJournal=%d\n" "Journal: journalOff=%lld journalHdr=%lld\n" , p->zFilename - , p->eState==PAGER_NONE ? "NONE" : + , p->eState==PAGER_OPEN ? "OPEN" : p->eState==PAGER_READER ? "READER" : - p->eState==PAGER_WRITER_INITIAL ? "WRITER_INITIAL" : + p->eState==PAGER_WRITER_LOCKED ? "WRITER_LOCKED" : p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : p->eState==PAGER_ERROR ? "ERROR" : "?error?" , (int)p->errCode - , p->eLock==NO_LOCK ? "NONE" : + , p->eLock==NO_LOCK ? "OPEN" : p->eLock==RESERVED_LOCK ? "RESERVED" : p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : p->eLock==SHARED_LOCK ? "SHARED" : @@ -1590,7 +1651,7 @@ static void pager_unlock(Pager *pPager){ if( pagerUseWal(pPager) ){ assert( !isOpen(pPager->jfd) ); sqlite3WalEndReadTransaction(pPager->pWal); - pPager->eState = PAGER_NONE; + pPager->eState = PAGER_OPEN; }else if( !pPager->exclusiveMode ){ int rc; /* Error code returned by pagerUnlockDb() */ int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; @@ -1622,24 +1683,24 @@ static void pager_unlock(Pager *pPager){ pPager->eLock = UNKNOWN_LOCK; } - /* The pager state may be changed from PAGER_ERROR to PAGER_NONE here + /* The pager state may be changed from PAGER_ERROR to PAGER_OPEN here ** without clearing the error code. This is intentional - the error ** code is cleared and the cache reset in the block below. */ assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); pPager->changeCountDone = 0; - pPager->eState = PAGER_NONE; + pPager->eState = PAGER_OPEN; } /* If Pager.errCode is set, the contents of the pager cache cannot be ** trusted. Now that there are no outstanding references to the pager, - ** it can safely move back to PAGER_NONE state. This happens in both + ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. */ if( pPager->errCode && !MEMDB ){ pager_reset(pPager); pPager->changeCountDone = pPager->tempFile; - pPager->eState = PAGER_NONE; + pPager->eState = PAGER_OPEN; pPager->errCode = SQLITE_OK; } @@ -1745,7 +1806,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ /* Do nothing if the pager does not have an open write transaction. */ assert( assert_pager_state(pPager) ); assert( pPager->eState!=PAGER_ERROR ); - if( pPager->eStateeLockeStateeLockeState!=PAGER_ERROR && pPager->eState!=PAGER_NONE ){ + if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_OPEN ){ assert( assert_pager_state(pPager) ); - if( pPager->eState>=PAGER_WRITER_INITIAL ){ + if( pPager->eState>=PAGER_WRITER_LOCKED ){ sqlite3BeginBenignMalloc(); sqlite3PagerRollback(pPager); sqlite3EndBenignMalloc(); @@ -1991,7 +2052,7 @@ static int pager_playback_one_page( return rc; } assert( pPager->eState>=PAGER_WRITER_CACHEMOD - || (pPager->eState==PAGER_NONE && isMainJrnl) + || (pPager->eState==PAGER_OPEN && isMainJrnl) ); /* When playing back page 1, restore the nReserve setting @@ -2048,7 +2109,7 @@ static int pager_playback_one_page( }else{ isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); } - if( (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_NONE) + if( (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) && isOpen(pPager->fd) && isSynced ){ @@ -2300,7 +2361,7 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ assert( pPager->eState!=PAGER_READER ); if( isOpen(pPager->fd) - && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_NONE) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) ){ i64 currentSize, newSize; /* TODO: Is it safe to use Pager.dbFileSize here? */ @@ -2785,7 +2846,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ int changed = 0; /* True if cache must be reset */ assert( pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_NONE || pPager->eState==PAGER_READER ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); /* sqlite3WalEndReadTransaction() was not called for the previous ** transaction in locking_mode=EXCLUSIVE. So call it now. If we @@ -2809,7 +2870,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ Pgno nPage; /* Value to return via *pnPage */ - assert( pPager->eState==PAGER_NONE ); + assert( pPager->eState==PAGER_OPEN ); nPage = sqlite3WalDbsize(pPager->pWal); if( nPage==0 ){ i64 n = 0; /* Size of db file in bytes */ @@ -2865,7 +2926,7 @@ static int pagerOpenWalIfPresent(Pager *pPager){ int isWal; /* True if WAL file exists */ Pgno nPage; /* Size of the database file */ - assert( pPager->eState==PAGER_NONE ); + assert( pPager->eState==PAGER_OPEN ); assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); rc = pagerPagecount(pPager, &nPage); if( rc ) return rc; @@ -2936,7 +2997,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ assert( pPager->eState!=PAGER_ERROR ); - assert( pPager->eState>=PAGER_WRITER_INITIAL ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); /* Allocate a bitvec to use to store the set of pages rolled back */ if( pSavepoint ){ @@ -3198,7 +3259,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u16 *pPageSize, int nReserve){ ){ char *pNew; /* New temp space */ i64 nByte = 0; - if( pPager->eState>PAGER_NONE && isOpen(pPager->fd) ){ + if( pPager->eState>PAGER_OPEN && isOpen(pPager->fd) ){ rc = sqlite3OsFileSize(pPager->fd, &nByte); if( rc!=SQLITE_OK ) return rc; } @@ -3246,7 +3307,7 @@ int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ if( mxPage>0 ){ pPager->mxPgno = mxPage; } - if( pPager->eState!=PAGER_NONE && pPager->mxPgnodbSize ){ + if( pPager->eState!=PAGER_OPEN && pPager->mxPgnodbSize ){ pPager->mxPgno = pPager->dbSize; } return pPager->mxPgno; @@ -3697,23 +3758,6 @@ static int syncJournal(Pager *pPager, int newHdr){ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ int rc; /* Return code */ - /* At this point there may be either a RESERVED or EXCLUSIVE lock on the - ** database file. If there is already an EXCLUSIVE lock, the following - ** call is a no-op. - ** - ** Moving the lock from RESERVED to EXCLUSIVE actually involves going - ** through an intermediate state PENDING. A PENDING lock prevents new - ** readers from attaching to the database but is unsufficient for us to - ** write. The idea of a PENDING lock is to prevent new readers from - ** coming in while we wait for existing readers to clear. - ** - ** While the pager is in the RESERVED state, the original database file - ** is unchanged and we can rollback without having to playback the - ** journal into the original database file. Once we transition to - ** EXCLUSIVE, it means the database file has been changed and any rollback - ** will require a journal playback. - */ - /* Normally, this function is called in WRITER_DBMOD state. ** ** However it may be called in WRITER_CACHEMOD state if the page being @@ -3730,6 +3774,22 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ || (pList->pDirty==0 && pList->pgno<=pPager->dbFileSize) ); + /* At this point there may be either a RESERVED or EXCLUSIVE lock on the + ** database file. If there is already an EXCLUSIVE lock, the following + ** call is a no-op. + ** + ** Moving the lock from RESERVED to EXCLUSIVE actually involves going + ** through an intermediate state PENDING. A PENDING lock prevents new + ** readers from attaching to the database but is unsufficient for us to + ** write. The idea of a PENDING lock is to prevent new readers from + ** coming in while we wait for existing readers to clear. + ** + ** While the pager is in the RESERVED state, the original database file + ** is unchanged and we can rollback without having to playback the + ** journal into the original database file. Once we transition to + ** EXCLUSIVE, it means the database file has been changed and any rollback + ** will require a journal playback. + */ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); /* If the file is a temp-file has not yet been opened, open it now. It @@ -4318,7 +4378,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ assert( pPager->useJournal ); assert( isOpen(pPager->fd) ); - assert( pPager->eState==PAGER_NONE ); + assert( pPager->eState==PAGER_OPEN ); assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN @@ -4437,15 +4497,15 @@ int sqlite3PagerSharedLock(Pager *pPager){ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either - ** be NONE or READER. READER is only possible if the pager is or was in + ** be OPEN or READER. READER is only possible if the pager is or was in ** exclusive access mode. */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); - assert( pPager->eState==PAGER_NONE || pPager->eState==PAGER_READER ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } - if( !pagerUseWal(pPager) && pPager->eState==PAGER_NONE ){ + if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ assert( !MEMDB ); @@ -4533,18 +4593,34 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ rc = pager_playback(pPager, 1); - pPager->eState = PAGER_NONE; + pPager->eState = PAGER_OPEN; } }else if( !pPager->exclusiveMode ){ pagerUnlockDb(pPager, SHARED_LOCK); } if( rc!=SQLITE_OK ){ + /* This branch is taken if an error occurs while trying to open + ** or roll back a hot-journal while holding an EXCLUSIVE lock. The + ** pager_unlock() routine will be called before returning to unlock + ** the file. If the unlock attempt fails, then Pager.eLock must be + ** set to UNKNOWN_LOCK (see the comment above the #define for + ** UNKNOWN_LOCK above for an explanation). + ** + ** In order to get pager_unlock() to do this, set Pager.eState to + ** PAGER_ERROR now. This is not actually counted as a transition + ** to ERROR state in the state diagram at the top of this file, + ** since we know that the same call to pager_unlock() will very + ** shortly transition the pager object to the OPEN state. Calling + ** assert_pager_state() would fail now, as it should not be possible + ** to be in ERROR state when there are zero outstanding page + ** references. + */ pager_error(pPager, rc); goto failed; } - assert( pPager->eState==PAGER_NONE ); + assert( pPager->eState==PAGER_OPEN ); assert( (pPager->eLock==SHARED_LOCK) || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) ); @@ -4600,14 +4676,14 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerBeginReadTransaction(pPager); } - if( pPager->eState==PAGER_NONE && rc==SQLITE_OK ){ + if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ rc = pagerPagecount(pPager, &pPager->dbSize); } failed: if( rc!=SQLITE_OK ){ pager_unlock(pPager); - assert( pPager->eState==PAGER_NONE ); + assert( pPager->eState==PAGER_OPEN ); }else{ pPager->eState = PAGER_READER; } @@ -4847,7 +4923,7 @@ static int pager_open_journal(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ - assert( pPager->eState==PAGER_WRITER_INITIAL ); + assert( pPager->eState==PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); assert( pPager->pInJournal==0 ); @@ -4902,7 +4978,7 @@ static int pager_open_journal(Pager *pPager){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; }else{ - assert( pPager->eState==PAGER_WRITER_INITIAL ); + assert( pPager->eState==PAGER_WRITER_LOCKED ); pPager->eState = PAGER_WRITER_CACHEMOD; } @@ -4974,29 +5050,22 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ if( rc==SQLITE_OK ){ - /* Change to WRITER_INITIAL state. + /* Change to WRITER_LOCKED state. ** - ** WAL mode sets Pager.eState to PAGER_WRITER_INITIAL or CACHEMOD + ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD ** when it has an open transaction, but never to DBMOD or FINISHED. ** This is because in those states the code to roll back savepoint ** transactions may copy data from the sub-journal into the database ** file as well as into the page cache. Which would be incorrect in ** WAL mode. */ - pPager->eState = PAGER_WRITER_INITIAL; + pPager->eState = PAGER_WRITER_LOCKED; pPager->dbFileSize = pPager->dbOrigSize = pPager->dbSize; pPager->journalOff = 0; - }else{ - /* Ignore any IO error that occurs within pager_end_transaction(). The - ** purpose of this call is to reset the internal state of the pager - ** sub-system. It doesn't matter if the journal-file is not properly - ** finalized at this point (since it is not a valid journal file anyway). - */ - pager_end_transaction(pPager, 0); } assert( rc==SQLITE_OK || pPager->eState==PAGER_READER ); - assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_INITIAL ); + assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); } @@ -5020,7 +5089,7 @@ static int pager_write(PgHdr *pPg){ ** been started. The journal file may or may not be open at this point. ** It is never called in the ERROR state. */ - assert( pPager->eState==PAGER_WRITER_INITIAL + assert( pPager->eState==PAGER_WRITER_LOCKED || pPager->eState==PAGER_WRITER_CACHEMOD || pPager->eState==PAGER_WRITER_DBMOD ); @@ -5053,7 +5122,7 @@ static int pager_write(PgHdr *pPg){ ** to begin the write-transaction, but the rollback journal might not ** yet be open. Open it now if this is the case. */ - if( pPager->eState==PAGER_WRITER_INITIAL ){ + if( pPager->eState==PAGER_WRITER_LOCKED ){ rc = pager_open_journal(pPager); if( rc!=SQLITE_OK ) return rc; } @@ -5170,7 +5239,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); - assert( pPager->eState>=PAGER_WRITER_INITIAL ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); @@ -5419,7 +5488,7 @@ int sqlite3PagerExclusiveLock(Pager *pPager){ int rc = SQLITE_OK; assert( pPager->eState==PAGER_WRITER_CACHEMOD || pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_WRITER_INITIAL + || pPager->eState==PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); if( 0==pagerUseWal(pPager) ){ @@ -5461,9 +5530,10 @@ int sqlite3PagerCommitPhaseOne( ){ int rc = SQLITE_OK; /* Return code */ - assert( pPager->eState==PAGER_WRITER_INITIAL + assert( pPager->eState==PAGER_WRITER_LOCKED || pPager->eState==PAGER_WRITER_CACHEMOD || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_ERROR ); assert( assert_pager_state(pPager) ); @@ -5658,7 +5728,7 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** called, just return the same error code without doing anything. */ if( NEVER(pPager->errCode) ) return pPager->errCode; - assert( pPager->eState==PAGER_WRITER_INITIAL + assert( pPager->eState==PAGER_WRITER_LOCKED || pPager->eState==PAGER_WRITER_FINISHED || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) ); @@ -5675,7 +5745,7 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** header. Since the pager is in exclusive mode, there is no need ** to drop any locks either. */ - if( pPager->eState==PAGER_WRITER_INITIAL + if( pPager->eState==PAGER_WRITER_LOCKED && pPager->exclusiveMode && pPager->journalMode==PAGER_JOURNALMODE_PERSIST ){ @@ -5736,7 +5806,7 @@ int sqlite3PagerRollback(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); - /* PagerRollback() is a no-op if called in READER or NONE state. If + /* PagerRollback() is a no-op if called in READER or OPEN state. If ** the pager is already in the ERROR state, the rollback is not ** attempted here. Instead, the error code is returned to the caller. */ @@ -5806,7 +5876,7 @@ int *sqlite3PagerStats(Pager *pPager){ a[0] = sqlite3PcacheRefCount(pPager->pPCache); a[1] = sqlite3PcachePagecount(pPager->pPCache); a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); - a[3] = pPager->eState==PAGER_NONE ? -1 : (int) pPager->dbSize; + a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; a[6] = pPager->nHit; @@ -5839,7 +5909,7 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ int rc = SQLITE_OK; /* Return code */ int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ - assert( pPager->eState>=PAGER_WRITER_INITIAL ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); if( nSavepoint>nCurrent && pPager->useJournal ){ @@ -6292,7 +6362,7 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ */ int rc = SQLITE_OK; int state = pPager->eState; - if( state==PAGER_NONE ){ + if( state==PAGER_OPEN ){ rc = sqlite3PagerSharedLock(pPager); } if( pPager->eState==PAGER_READER ){ @@ -6304,7 +6374,7 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } if( rc==SQLITE_OK && state==PAGER_READER ){ pagerUnlockDb(pPager, SHARED_LOCK); - }else if( state==PAGER_NONE ){ + }else if( state==PAGER_OPEN ){ pager_unlock(pPager); } assert( state==pPager->eState ); @@ -6408,7 +6478,7 @@ int sqlite3PagerOpenWal( int rc = SQLITE_OK; /* Return code */ assert( assert_pager_state(pPager) ); - assert( pPager->eState==PAGER_NONE || pbOpen ); + assert( pPager->eState==PAGER_OPEN || pbOpen ); assert( pPager->eState==PAGER_READER || !pbOpen ); assert( pbOpen==0 || *pbOpen==0 ); assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); @@ -6427,7 +6497,7 @@ int sqlite3PagerOpenWal( rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, &pPager->pWal); if( rc==SQLITE_OK ){ pPager->journalMode = PAGER_JOURNALMODE_WAL; - pPager->eState = PAGER_NONE; + pPager->eState = PAGER_OPEN; } }else{ *pbOpen = 1; From 4f7b8d62606b14f23dac84c68bffd6c52dc939b6 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 6 Aug 2010 14:37:13 +0000 Subject: [PATCH 19/30] Fix some problems with running test scripts with the inmemory_journal permutation. FossilOrigin-Name: 7bd8ba084e75bcd5c744e1d4a1812df3a4d91887 --- manifest | 14 ++--- manifest.uuid | 2 +- test/dbstatus.test | 7 +++ test/fallocate.test | 133 +++++++++++++++++++++++--------------------- 4 files changed, 84 insertions(+), 72 deletions(-) diff --git a/manifest b/manifest index dc83753762..f3b4bb44fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sexperimental\sbranch. -D 2010-08-06T13:53:10 +C Fix\ssome\sproblems\swith\srunning\stest\sscripts\swith\sthe\sinmemory_journal\spermutation. +D 2010-08-06T14:37:13 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -330,7 +330,7 @@ F test/createtab.test 199cf68f44e5d9e87a0b8afc7130fdeb4def3272 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test 6354b883f922c38046a8efbad187cc95df6da023 -F test/dbstatus.test f3c88a3f8d15716e6ae73567a78ee96420c294a3 +F test/dbstatus.test ba072efbd7fcbeb4da324bfbdf1e596d994c159e F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test f7629d9eb245dfca170169cc5c7a735dec34aeb4 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -351,7 +351,7 @@ F test/exclusive.test b1f9012cabc124af947165d15ffa62ad20f63db8 F test/exclusive2.test fcbb1c9ca9739292a0a22a3763243ad6d868086b F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/expr.test 9f521ae22f00e074959f72ce2e55d46b9ed23f68 -F test/fallocate.test 56242d30325fa451247d13e103cf1c357751a066 +F test/fallocate.test 43dc34b8c24be6baffadc3b4401ee15710ce83c6 F test/filectrl.test 97003734290887566e01dded09dc9e99cb937e9e F test/filefmt.test f77c92141960b7933bc6691631d2ad62257ef40a F test/fkey1.test 01c7de578e11747e720c2d9aeef27f239853c4da @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 5f4c17a33fd442d66c1bf75af9b21954f6906897 698fba826e40ce6414cf6c261441d68b174c1637 -R 608fe42d5e0f4b194d191a9617ba7a7e +P aef6698c732f3f9e46986f53e63ca2bdf5f7d208 +R d400d2e52912473cf6c4aca53a46678c U dan -Z cb0c5df5af28e7e96ef8d49f9e0760f1 +Z e0e135b259997ce301d4de7d751dee90 diff --git a/manifest.uuid b/manifest.uuid index ec439f5a86..9840d1e499 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aef6698c732f3f9e46986f53e63ca2bdf5f7d208 \ No newline at end of file +7bd8ba084e75bcd5c744e1d4a1812df3a4d91887 \ No newline at end of file diff --git a/test/dbstatus.test b/test/dbstatus.test index 46291b1ce5..53d3d66f0a 100644 --- a/test/dbstatus.test +++ b/test/dbstatus.test @@ -55,6 +55,12 @@ proc lookaside {db} { # foreach ::lookaside_buffer_size {0 64 120} { + # Do not run any of these tests if there is SQL configured to run + # as part of the [sqlite3] command. This prevents the script from + # configuring the size of the lookaside buffer after [sqlite3] has + # returned. + if {[presql] != ""} break + #------------------------------------------------------------------------- # Tests for SQLITE_DBSTATUS_SCHEMA_USED. # @@ -317,6 +323,7 @@ foreach ::lookaside_buffer_size {0 64 120} { # implementation using sqlite3_mprintf() is technically considered # external and so is not counted as "statement memory". # +puts "$nStmt1 $nFree" if {[string match *x $tn]} { do_test dbstatus-3.$tn.bx { expr $nStmt1<=$nFree } {1} } else { diff --git a/test/fallocate.test b/test/fallocate.test index ab41fe0a23..05aa2a81e5 100644 --- a/test/fallocate.test +++ b/test/fallocate.test @@ -59,7 +59,11 @@ do_test fallocate-1.6 { # do_test fallocate-1.7 { execsql { BEGIN; INSERT INTO t1 VALUES(1, 2); } - hexio_get_int [hexio_read test.db-journal 16 4] + if {[permutation] != "inmemory_journal"} { + hexio_get_int [hexio_read test.db-journal 16 4] + } else { + set {} 1024 + } } {1024} do_test fallocate-1.8 { execsql { COMMIT } } {} @@ -68,72 +72,73 @@ do_test fallocate-1.8 { execsql { COMMIT } } {} # The following tests - fallocate-2.* - test that things work in WAL # mode as well. # -set skipwaltests [expr {[permutation]=="journaltest"}] +set skipwaltests [expr { + [permutation]=="journaltest" || [permutation]=="inmemory_journal" +}] ifcapable !wal { set skipwaltests 1 } if {!$skipwaltests} { -db close -file delete -force test.db -sqlite3 db test.db -file_control_chunksize_test db main [expr 32*1024] - -do_test fallocate-2.1 { - execsql { - PRAGMA page_size = 1024; - PRAGMA journal_mode = WAL; - CREATE TABLE t1(a, b); - } - file size test.db -} [expr 32*1024] - -do_test fallocate-2.2 { - execsql { INSERT INTO t1 VALUES(1, zeroblob(35*1024)) } - execsql { PRAGMA wal_checkpoint } - file size test.db -} [expr 64*1024] - -do_test fallocate-2.3 { - execsql { DELETE FROM t1 } - execsql { VACUUM } - file size test.db -} [expr 64*1024] - -do_test fallocate-2.4 { - execsql { PRAGMA wal_checkpoint } - file size test.db -} [expr 32*1024] - -do_test fallocate-2.5 { - execsql { - INSERT INTO t1 VALUES(2, randomblob(35*1024)); - PRAGMA wal_checkpoint; - INSERT INTO t1 VALUES(3, randomblob(128)); - DELETE FROM t1 WHERE a = 2; - VACUUM; - } - file size test.db -} [expr 64*1024] - -do_test fallocate-2.6 { - sqlite3 db2 test.db - execsql { BEGIN ; SELECT count(a) FROM t1 } db2 - execsql { - INSERT INTO t1 VALUES(4, randomblob(128)); - PRAGMA wal_checkpoint; - } - file size test.db -} [expr 64*1024] - -do_test fallocate-2.7 { - execsql { SELECT count(b) FROM t1 } db2 -} {1} - -do_test fallocate-2.8 { - execsql { COMMIT } db2 - execsql { PRAGMA wal_checkpoint } - file size test.db -} [expr 32*1024] - + db close + file delete -force test.db + sqlite3 db test.db + file_control_chunksize_test db main [expr 32*1024] + + do_test fallocate-2.1 { + execsql { + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a, b); + } + file size test.db + } [expr 32*1024] + + do_test fallocate-2.2 { + execsql { INSERT INTO t1 VALUES(1, zeroblob(35*1024)) } + execsql { PRAGMA wal_checkpoint } + file size test.db + } [expr 64*1024] + + do_test fallocate-2.3 { + execsql { DELETE FROM t1 } + execsql { VACUUM } + file size test.db + } [expr 64*1024] + + do_test fallocate-2.4 { + execsql { PRAGMA wal_checkpoint } + file size test.db + } [expr 32*1024] + + do_test fallocate-2.5 { + execsql { + INSERT INTO t1 VALUES(2, randomblob(35*1024)); + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES(3, randomblob(128)); + DELETE FROM t1 WHERE a = 2; + VACUUM; + } + file size test.db + } [expr 64*1024] + + do_test fallocate-2.6 { + sqlite3 db2 test.db + execsql { BEGIN ; SELECT count(a) FROM t1 } db2 + execsql { + INSERT INTO t1 VALUES(4, randomblob(128)); + PRAGMA wal_checkpoint; + } + file size test.db + } [expr 64*1024] + + do_test fallocate-2.7 { + execsql { SELECT count(b) FROM t1 } db2 + } {1} + + do_test fallocate-2.8 { + execsql { COMMIT } db2 + execsql { PRAGMA wal_checkpoint } + file size test.db + } [expr 32*1024] } From 85d14ed238f9c60333148062f28390092e3de68a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 6 Aug 2010 17:18:00 +0000 Subject: [PATCH 20/30] Further enhancements to comments in pager.c. FossilOrigin-Name: 876162c7e036af1cb447409b685afc72c0061a32 --- manifest | 12 ++--- manifest.uuid | 2 +- src/pager.c | 130 +++++++++++++++++++++++++++++++------------------- 3 files changed, 87 insertions(+), 57 deletions(-) diff --git a/manifest b/manifest index f3b4bb44fc..e52daaa123 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\swith\srunning\stest\sscripts\swith\sthe\sinmemory_journal\spermutation. -D 2010-08-06T14:37:13 +C Further\senhancements\sto\scomments\sin\spager.c. +D 2010-08-06T17:18:01 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c c905bca55b0d507a2c5c9ee97895ba0aaff2a969 +F src/pager.c 2dffe4d468c7665cceaa43eb56e4b5c2f0d28396 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P aef6698c732f3f9e46986f53e63ca2bdf5f7d208 -R d400d2e52912473cf6c4aca53a46678c +P 7bd8ba084e75bcd5c744e1d4a1812df3a4d91887 +R 8a9a6e1ca5d584d1153bedf7c51336b6 U dan -Z e0e135b259997ce301d4de7d751dee90 +Z 9aa10a3fe3170bea3315ad90ad87b34f diff --git a/manifest.uuid b/manifest.uuid index 9840d1e499..4ff4fab4ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7bd8ba084e75bcd5c744e1d4a1812df3a4d91887 \ No newline at end of file +876162c7e036af1cb447409b685afc72c0061a32 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 6383beb8e5..767426fefa 100644 --- a/src/pager.c +++ b/src/pager.c @@ -497,13 +497,21 @@ struct PagerSavepoint { ** ** doNotSpill, doNotSyncSpill ** -** When enabled, cache spills are prohibited. The doNotSpill variable -** inhibits all cache spill and doNotSyncSpill inhibits those spills that -** would require a journal sync. The doNotSyncSpill is set and cleared -** by sqlite3PagerWrite() in order to prevent a journal sync from happening -** in between the journalling of two pages on the same sector. The -** doNotSpill value set to prevent pagerStress() from trying to use -** the journal during a rollback. +** These two boolean variables control the behaviour of cache-spills +** (calls made by the pcache module to the pagerStress() routine to +** write cached data to the file-system in order to free up memory). +** +** When doNotSpill is non-zero, writing to the database from pagerStress() +** is disabled altogether. This is done in a very obscure case that +** comes up during savepoint rollback that requires the pcache module +** to allocate a new page to prevent the journal file from being written +** while it is being traversed by code in pager_playback(). +** +** If doNotSyncSpill is non-zero, writing to the database from pagerStress() +** is permitted, but syncing the journal file is not. This flag is set +** by sqlite3PagerWrite() when the file-system sector-size is larger than +** the database page-size in order to prevent a journal sync from happening +** in between the journalling of two pages on the same sector. ** ** subjInMemory ** @@ -1750,8 +1758,9 @@ static int pager_error(Pager *pPager, int rc){ ** the journal file or writing the very first journal-header of a ** database transaction. ** -** If the pager is in PAGER_SHARED or PAGER_UNLOCK state when this -** routine is called, it is a no-op (returns SQLITE_OK). +** This routine is never called in PAGER_ERROR state. If it is called +** in PAGER_NONE or PAGER_SHARED state and the lock held is less +** exclusive than a RESERVED lock, it is a no-op. ** ** Otherwise, any active savepoints are released. ** @@ -1782,13 +1791,9 @@ static int pager_error(Pager *pPager, int rc){ ** DELETE and the pager is in exclusive mode, the method described under ** journalMode==PERSIST is used instead. ** -** After the journal is finalized, if running in non-exclusive mode, the -** pager moves to PAGER_SHARED state (and downgrades the lock on the -** database file accordingly). -** -** If the pager is running in exclusive mode and is in PAGER_SYNCED state, -** it moves to PAGER_EXCLUSIVE. No locks are downgraded when running in -** exclusive mode. +** After the journal is finalized, the pager moves to PAGER_READER state. +** If running in non-exclusive rollback mode, the lock on the file is +** downgraded to a SHARED_LOCK. ** ** SQLITE_OK is returned if no error occurs. If an error occurs during ** any of the IO operations to finalize the journal file or unlock the @@ -1803,7 +1808,19 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ int rc = SQLITE_OK; /* Error code from journal finalization operation */ int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ - /* Do nothing if the pager does not have an open write transaction. */ + /* Do nothing if the pager does not have an open write transaction + ** or at least a RESERVED lock. This function may be called when there + ** is no write-transaction active but a RESERVED or greater lock is + ** held under two circumstances: + ** + ** 1. After a successful hot-journal rollback, it is called with + ** eState==PAGER_NONE and eLock==EXCLUSIVE_LOCK. + ** + ** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE + ** lock switches back to locking_mode=normal and then executes a + ** read-transaction, this function is called with eState==PAGER_READER + ** and eLock==EXCLUSIVE_LOCK when the read-transaction is closed. + */ assert( assert_pager_state(pPager) ); assert( pPager->eState!=PAGER_ERROR ); if( pPager->eStateeLockeLock>=RESERVED_LOCK && !pPager->exclusiveMode ){ + }else if( !pPager->exclusiveMode ){ assert( pPager->eState==PAGER_READER ); pager_end_transaction(pPager, 0); } @@ -1959,9 +1976,8 @@ static void pagerReportSize(Pager *pPager){ ** The page begins at offset *pOffset into the file. The *pOffset ** value is increased to the start of the next page in the journal. ** -** The isMainJrnl flag is true if this is the main rollback journal and -** false for the statement journal. The main rollback journal uses -** checksums - the statement journal does not. +** The main rollback journal uses checksums - the statement journal does +** not. ** ** If the page number of the page record read from the (sub-)journal file ** is greater than the current value of Pager.dbSize, then playback is @@ -2015,6 +2031,17 @@ static int pager_playback_one_page( assert( aData ); /* Temp storage must have already been allocated */ assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); + /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction + ** or savepoint rollback done at the request of the caller) or this is + ** a hot-journal rollback. If it is a hot-journal rollback, the pager + ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback + ** only reads from the main journal, not the sub-journal. + */ + assert( pPager->eState>=PAGER_WRITER_CACHEMOD + || (pPager->eState==PAGER_OPEN && pPager->eLock==EXCLUSIVE_LOCK) + ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD || isMainJrnl ); + /* Read the page number and page data from the journal or sub-journal ** file. Return an error code to the caller if an IO error occurs. */ @@ -2051,9 +2078,6 @@ static int pager_playback_one_page( if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ return rc; } - assert( pPager->eState>=PAGER_WRITER_CACHEMOD - || (pPager->eState==PAGER_OPEN && isMainJrnl) - ); /* When playing back page 1, restore the nReserve setting */ @@ -2863,15 +2887,34 @@ static int pagerBeginReadTransaction(Pager *pPager){ return rc; } - /* -** TODO: Description here. +** This function is called as part of the transition from PAGER_OPEN +** to PAGER_READER state to determine the size of the database file +** in pages (assuming the page size currently stored in Pager.pageSize). +** +** If no error occurs, SQLITE_OK is returned and the size of the database +** in pages is stored in *pnPage. Otherwise, an error code (perhaps +** SQLITE_IOERR_FSTAT) is returned and *pnPage is left unmodified. */ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ Pgno nPage; /* Value to return via *pnPage */ + /* Query the WAL sub-system for the database size. The WalDbsize() + ** function returns zero if the WAL is not open (i.e. Pager.pWal==0), or + ** if the database size is not available. The database size is not + ** available from the WAL sub-system if the log file is empty or + ** contains no valid committed transactions. + */ assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); nPage = sqlite3WalDbsize(pPager->pWal); + + /* If the database size was not available from the WAL sub-system, + ** determine it based on the size of the database file. If the size + ** of the database file is not an integer multiple of the page-size, + ** round down to the nearest page. Except, any file larger than 0 + ** bytes in size is considered to contain at least one page. + */ if( nPage==0 ){ i64 n = 0; /* Size of db file in bytes */ assert( isOpen(pPager->fd) || pPager->tempFile ); @@ -2887,7 +2930,7 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ } } - /* If the current number of pages in the file is greater than the + /* If the current number of pages in the file is greater than the ** configured maximum pager number, increase the allowed limit so ** that the file can be read. */ @@ -2912,8 +2955,6 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ ** ** Return SQLITE_OK or an error code. ** -** If the WAL file is opened, also open a snapshot (read transaction). -** ** The caller must hold a SHARED lock on the database file to call this ** function. Because an EXCLUSIVE lock on the db file is required to delete ** a WAL on a none-empty database, this ensures there is no race condition @@ -2922,12 +2963,13 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ */ static int pagerOpenWalIfPresent(Pager *pPager){ int rc = SQLITE_OK; + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); + if( !pPager->tempFile ){ int isWal; /* True if WAL file exists */ Pgno nPage; /* Size of the database file */ - assert( pPager->eState==PAGER_OPEN ); - assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); rc = pagerPagecount(pPager, &nPage); if( rc ) return rc; if( nPage==0 ){ @@ -2942,9 +2984,6 @@ static int pagerOpenWalIfPresent(Pager *pPager){ if( isWal ){ testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); rc = sqlite3PagerOpenWal(pPager, 0); - if( rc==SQLITE_OK ){ - rc = pagerBeginReadTransaction(pPager); - } }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ pPager->journalMode = PAGER_JOURNALMODE_DELETE; } @@ -4994,14 +5033,6 @@ static int pager_open_journal(Pager *pPager){ ** an EXCLUSIVE lock. If such a lock is already held, no locking ** functions need be called. ** -** If this is not a temporary or in-memory file and, the journal file is -** opened if it has not been already. For a temporary file, the opening -** of the journal file is deferred until there is an actual need to -** write to the journal. TODO: Why handle temporary files differently? -** -** If the journal file is opened (or if it is already open), then a -** journal-header is written to the start of it. -** ** If the subjInMemory argument is non-zero, then any sub-journal opened ** within this transaction will be opened as an in-memory file. This ** has no effect if the sub-journal is already opened (as it may be when @@ -5048,7 +5079,6 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ } } - if( rc==SQLITE_OK ){ /* Change to WRITER_LOCKED state. ** From 16f7720ede96b0bda39705803c20ccd4900ac0a8 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 7 Aug 2010 05:15:22 +0000 Subject: [PATCH 21/30] Add test case 'cgt_pager_1', intended for use with callgrind to detect performance regression in the pager module, to threadtest3.c. FossilOrigin-Name: b5d46f1ea08db2b88d2205bc283b9262ad970b55 --- manifest | 12 +++++----- manifest.uuid | 2 +- test/threadtest3.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e52daaa123..ac5dfd9e0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\senhancements\sto\scomments\sin\spager.c. -D 2010-08-06T17:18:01 +C Add\stest\scase\s'cgt_pager_1',\sintended\sfor\suse\swith\scallgrind\sto\sdetect\sperformance\sregression\sin\sthe\spager\smodule,\sto\sthreadtest3.c. +D 2010-08-07T05:15:23 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -632,7 +632,7 @@ F test/thread2.test e08034b83fe9693ade77049732518e5b3d2d700d F test/thread_common.tcl 2aa6f2fdcd4d6e461169c3e5ca098eebf643b863 F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c f7e21a9bcddff1499d46a1fc68d1ba31e4063be4 +F test/threadtest3.c 58df1e3c060f534fd7fb0702331b0acc41c381d8 F test/tkt-02a8e81d44.test 58494de77be2cf249228ada3f313fa399821c6ab F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 7bd8ba084e75bcd5c744e1d4a1812df3a4d91887 -R 8a9a6e1ca5d584d1153bedf7c51336b6 +P 876162c7e036af1cb447409b685afc72c0061a32 +R 27f96e8eaa225660cd3eee27494507ba U dan -Z 9aa10a3fe3170bea3315ad90ad87b34f +Z 3bc5778d1d89a64090d5ae07cc0b0720 diff --git a/manifest.uuid b/manifest.uuid index 4ff4fab4ca..6c2974f53e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -876162c7e036af1cb447409b685afc72c0061a32 \ No newline at end of file +b5d46f1ea08db2b88d2205bc283b9262ad970b55 \ No newline at end of file diff --git a/test/threadtest3.c b/test/threadtest3.c index 57404faa67..6373926c13 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -517,6 +517,7 @@ static void opendb_x( pDb->db, "md5sum", -1, SQLITE_UTF8, 0, 0, md5step, md5finalize ); sqlite3_busy_handler(pDb->db, busyhandler, 0); + sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0); } } } @@ -1219,6 +1220,58 @@ static void walthread5(int nMs){ print_and_free_err(&err); } +/*------------------------------------------------------------------------ +** Test case "cgt_pager_1" +*/ +#define CALLGRINDTEST1_NROW 10000 +static void cgt_pager_1_populate(Error *pErr, Sqlite *pDb){ + const char *zInsert = "INSERT INTO t1 VALUES(:iRow, zeroblob(:iBlob))"; + i64 iRow; + sql_script(pErr, pDb, "BEGIN"); + for(iRow=1; iRow<=CALLGRINDTEST1_NROW; iRow++){ + i64 iBlob = 600 + (iRow%300); + execsql(pErr, pDb, zInsert, &iRow, &iBlob); + } + sql_script(pErr, pDb, "COMMIT"); +} +static void cgt_pager_1_update(Error *pErr, Sqlite *pDb){ + const char *zUpdate = "UPDATE t1 SET b = zeroblob(:iBlob) WHERE a = :iRow"; + i64 iRow; + sql_script(pErr, pDb, "BEGIN"); + for(iRow=1; iRow<=CALLGRINDTEST1_NROW; iRow++){ + i64 iBlob = 600 + ((iRow+100)%300); + execsql(pErr, pDb, zUpdate, &iBlob, &iRow); + } + sql_script(pErr, pDb, "COMMIT"); +} +static void cgt_pager_1_read(Error *pErr, Sqlite *pDb){ + i64 iRow; + sql_script(pErr, pDb, "BEGIN"); + for(iRow=1; iRow<=CALLGRINDTEST1_NROW; iRow++){ + execsql(pErr, pDb, "SELECT * FROM t1 WHERE a = :iRow", &iRow); + } + sql_script(pErr, pDb, "COMMIT"); +} +static void cgt_pager_1(int nMs){ + void (*xSub)(Error *, Sqlite *); + Error err = {0}; + Sqlite db = {0}; + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, + "PRAGMA cache_size = 2000;" + "PRAGMA page_size = 1024;" + "CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB);" + ); + + xSub = cgt_pager_1_populate; xSub(&err, &db); + xSub = cgt_pager_1_update; xSub(&err, &db); + xSub = cgt_pager_1_read; xSub(&err, &db); + + closedb(&err, &db); + print_and_free_err(&err); +} + int main(int argc, char **argv){ struct ThreadTest { void (*xTest)(int); @@ -1230,6 +1283,9 @@ int main(int argc, char **argv){ { walthread3, "walthread3", 20000 }, { walthread4, "walthread4", 20000 }, { walthread5, "walthread5", 1000 }, + { walthread5, "walthread5", 1000 }, + + { cgt_pager_1, "cgt_pager_1", 0 }, }; int i; From e5953ccdb20f8f3e6b9c29928fb64ef83a8744f5 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 7 Aug 2010 09:31:14 +0000 Subject: [PATCH 22/30] Fix a problem wherein changing the journal-mode immediately after leaving exclusive-locking mode could lead to the database being unlocked without clearing the changeCountDone flag. FossilOrigin-Name: 531abc808526d607768bf6f503268d4cc66ab169 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 42 +++++++++++++++++++++++------------------- test/jrnlmode.test | 31 +++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index ac5dfd9e0d..13283aff95 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scase\s'cgt_pager_1',\sintended\sfor\suse\swith\scallgrind\sto\sdetect\sperformance\sregression\sin\sthe\spager\smodule,\sto\sthreadtest3.c. -D 2010-08-07T05:15:23 +C Fix\sa\sproblem\swherein\schanging\sthe\sjournal-mode\simmediately\safter\sleaving\sexclusive-locking\smode\scould\slead\sto\sthe\sdatabase\sbeing\sunlocked\swithout\sclearing\sthe\schangeCountDone\sflag. +D 2010-08-07T09:31:14 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 2dffe4d468c7665cceaa43eb56e4b5c2f0d28396 +F src/pager.c 9621456e9fdbd134273a434efb2c873c4c4f2a9e F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -471,7 +471,7 @@ F test/join6.test bf82cf3f979e9eade83ad0d056a66c5ed71d1901 F test/journal1.test 36f2d1bb9bf03f790f43fbdb439e44c0657fab19 F test/journal2.test 50a3604768494d4a337f194f0a9480e7c57dcb72 F test/journal3.test ff175219be1b02d2f7e54297ad7e491b7533edb6 -F test/jrnlmode.test 2d5a8b6d68de8512f522532731d90ca96912f3b7 +F test/jrnlmode.test e3fe6c4a2c3213d285650dc8e33aab7eaaa5ce53 F test/jrnlmode2.test a19e28de1a6ec898067e46a122f1b71c9323bf00 F test/jrnlmode3.test cfcdb12b90e640a23b92785a002d96c0624c8710 F test/keyword1.test a2400977a2e4fde43bf33754c2929fda34dbca05 @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 876162c7e036af1cb447409b685afc72c0061a32 -R 27f96e8eaa225660cd3eee27494507ba +P b5d46f1ea08db2b88d2205bc283b9262ad970b55 +R 1c87633c0c4928fc4e767694680c3b36 U dan -Z 3bc5778d1d89a64090d5ae07cc0b0720 +Z d18018a31ac0c912f6e4858813e6f216 diff --git a/manifest.uuid b/manifest.uuid index 6c2974f53e..33f193000c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5d46f1ea08db2b88d2205bc283b9262ad970b55 \ No newline at end of file +531abc808526d607768bf6f503268d4cc66ab169 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 767426fefa..765a64c34e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6364,10 +6364,11 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ if( eMode!=eOld ){ /* Change the journal mode. */ + assert( pPager->eState!=PAGER_ERROR ); pPager->journalMode = (u8)eMode; /* When transistioning from TRUNCATE or PERSIST to any other journal - ** mode except WAL (and we are not in locking_mode=EXCLUSIVE) then + ** mode except WAL, unless the pager is in locking_mode=exclusive mode, ** delete the journal file. */ assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); @@ -6380,8 +6381,6 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ assert( isOpen(pPager->fd) || pPager->exclusiveMode ); if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){ - sqlite3OsClose(pPager->jfd); - /* In this case we would like to delete the journal file. If it is ** not possible, then that is not a problem. Deleting the journal file ** here is an optimization only. @@ -6390,24 +6389,29 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ ** database file. This ensures that the journal file is not deleted ** while it is in use by some other client. */ - int rc = SQLITE_OK; - int state = pPager->eState; - if( state==PAGER_OPEN ){ - rc = sqlite3PagerSharedLock(pPager); - } - if( pPager->eState==PAGER_READER ){ - assert( rc==SQLITE_OK ); - rc = pagerLockDb(pPager, RESERVED_LOCK); - } - if( rc==SQLITE_OK ){ + sqlite3OsClose(pPager->jfd); + if( pPager->eLock>=RESERVED_LOCK ){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + }else{ + int rc = SQLITE_OK; + int state = pPager->eState; + if( state==PAGER_OPEN ){ + rc = sqlite3PagerSharedLock(pPager); + } + if( pPager->eState==PAGER_READER ){ + assert( rc==SQLITE_OK ); + rc = pagerLockDb(pPager, RESERVED_LOCK); + } + if( rc==SQLITE_OK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + if( rc==SQLITE_OK && state==PAGER_READER ){ + pagerUnlockDb(pPager, SHARED_LOCK); + }else if( state==PAGER_OPEN ){ + pager_unlock(pPager); + } + assert( state==pPager->eState ); } - if( rc==SQLITE_OK && state==PAGER_READER ){ - pagerUnlockDb(pPager, SHARED_LOCK); - }else if( state==PAGER_OPEN ){ - pager_unlock(pPager); - } - assert( state==pPager->eState ); } } diff --git a/test/jrnlmode.test b/test/jrnlmode.test index f1e9c786bb..eab74d65ac 100644 --- a/test/jrnlmode.test +++ b/test/jrnlmode.test @@ -524,4 +524,35 @@ ifcapable pragma { do_test jrnlmode-7.2 { file size test.db } {1024} } +do_execsql_test jrnlmode-8.1 { PRAGMA locking_mode=EXCLUSIVE } {exclusive} +do_execsql_test jrnlmode-8.2 { CREATE TABLE t1(x) } {} +do_execsql_test jrnlmode-8.3 { INSERT INTO t1 VALUES(123) } {} +do_execsql_test jrnlmode-8.4 { SELECT * FROM t1 } {123} +do_execsql_test jrnlmode-8.5 { PRAGMA journal_mode=PERSIST } {persist} +do_execsql_test jrnlmode-8.6 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.7 { PRAGMA journal_mode=TRUNCATE } {truncate} +do_execsql_test jrnlmode-8.8 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.9 { CREATE TABLE t2(y) } {} +do_execsql_test jrnlmode-8.10 { INSERT INTO t2 VALUES(456) } {} +do_execsql_test jrnlmode-8.11 { SELECT * FROM t1, t2 } {123 456} +do_execsql_test jrnlmode-8.12 { PRAGMA locking_mode=NORMAL } {normal} +do_execsql_test jrnlmode-8.13 { PRAGMA journal_mode=PERSIST } {persist} +do_execsql_test jrnlmode-8.14 { PRAGMA journal_mode=TRUNCATE } {truncate} +do_execsql_test jrnlmode-8.15 { PRAGMA journal_mode=PERSIST } {persist} +do_execsql_test jrnlmode-8.16 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.17 { PRAGMA journal_mode=TRUNCATE } {truncate} +do_execsql_test jrnlmode-8.18 { PRAGMA locking_mode=EXCLUSIVE } {exclusive} +do_execsql_test jrnlmode-8.19 { CREATE TABLE t3(z) } {} +do_execsql_test jrnlmode-8.20 { BEGIN IMMEDIATE } {} +do_execsql_test jrnlmode-8.21 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.22 { COMMIT } {} +do_execsql_test jrnlmode-8.23 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.24 { PRAGMA journal_mode=TRUNCATE } {truncate} +do_execsql_test jrnlmode-8.25 { PRAGMA locking_mode=NORMAL } {normal} +do_execsql_test jrnlmode-8.26 { CREATE TABLE t4(w) } {} +do_execsql_test jrnlmode-8.27 { BEGIN IMMEDIATE } {} +do_execsql_test jrnlmode-8.28 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.29 { COMMIT } {} +do_execsql_test jrnlmode-8.30 { PRAGMA journal_mode=DELETE } {delete} + finish_test From 73d66fdb223a696b65c0a6c3ff12641fde1462af Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 7 Aug 2010 16:17:48 +0000 Subject: [PATCH 23/30] Fix minor problems and update comments in pager.c. FossilOrigin-Name: 92e456374b052aceff356d10317b5b94fcdeaa5c --- manifest | 16 ++++---- manifest.uuid | 2 +- src/pager.c | 96 ++++++++++++++++++----------------------------- src/wal.c | 1 + test/walmode.test | 7 ++++ 5 files changed, 54 insertions(+), 68 deletions(-) diff --git a/manifest b/manifest index 13283aff95..6455d6683b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swherein\schanging\sthe\sjournal-mode\simmediately\safter\sleaving\sexclusive-locking\smode\scould\slead\sto\sthe\sdatabase\sbeing\sunlocked\swithout\sclearing\sthe\schangeCountDone\sflag. -D 2010-08-07T09:31:14 +C Fix\sminor\sproblems\sand\supdate\scomments\sin\spager.c. +D 2010-08-07T16:17:49 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 9621456e9fdbd134273a434efb2c873c4c4f2a9e +F src/pager.c 3af4caaa32f26d537ff857ac30d3ac3ce7e46ea5 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -227,7 +227,7 @@ F src/vdbeblob.c 258a6010ba7a82b72b327fb24c55790655689256 F src/vdbemem.c e5673f81a2381b35c60e73ef0a8502be2ab1041e F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c 82200af3881fa4e1c9cf07cf31d98c09d437e3ab -F src/wal.c 6e04bccccd75acf86725cc8cb4b107cd245e018c +F src/wal.c ef5e37b8cf0b617b787d00eb9c63c803f1f790b1 F src/wal.h 96669b645e27cd5a111ba59f0cae7743a207bc3c F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 7db3e41c2a846f9deeb24f1bbb75461b4010b7b5 @@ -794,7 +794,7 @@ F test/walcrash.test e763841551d6b23677ccb419797c1589dcbdbaf5 F test/walcrash2.test 019d60b89d96c1937adb2b30b850ac7e86e5a142 F test/walfault.test 05c470688d742688e455dd56816bd6bcffa298f8 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 -F test/walmode.test 5dc3008ef71988ecdd949ea16e5750e325b92b54 +F test/walmode.test 4ecd37284f245247f7935896ab66f6943f0432a0 F test/walshared.test 985b4a3406b2b2dace1d52a42d26a11dd6900981 F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933 F test/walthread.test a25a393c068a2b42b44333fa3fdaae9072f1617c @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P b5d46f1ea08db2b88d2205bc283b9262ad970b55 -R 1c87633c0c4928fc4e767694680c3b36 +P 531abc808526d607768bf6f503268d4cc66ab169 +R 8bd570ff99007432bc59ff36d6f70bed U dan -Z d18018a31ac0c912f6e4858813e6f216 +Z ab88a6e6bc1538b57c390502ab433fb7 diff --git a/manifest.uuid b/manifest.uuid index 33f193000c..802595050a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -531abc808526d607768bf6f503268d4cc66ab169 \ No newline at end of file +92e456374b052aceff356d10317b5b94fcdeaa5c \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 765a64c34e..b68d3cbb8f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -891,6 +891,7 @@ static char *print_pager_state(Pager *p){ "Journal mode: journal_mode=%s\n" "Backing store: tempFile=%d memDb=%d useJournal=%d\n" "Journal: journalOff=%lld journalHdr=%lld\n" + "Size: dbsize=%d dbOrigSize=%d dbFileSize=%d\n" , p->zFilename , p->eState==PAGER_OPEN ? "OPEN" : p->eState==PAGER_READER ? "READER" : @@ -914,6 +915,7 @@ static char *print_pager_state(Pager *p){ p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" , (int)p->tempFile, (int)p->memDb, (int)p->useJournal , p->journalOff, p->journalHdr + , (int)p->dbSize, (int)p->dbOrigSize, (int)p->dbFileSize ); return zRet; @@ -1520,7 +1522,6 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ u32 cksum = 0; /* Checksum of string zMaster */ assert( pPager->setMaster==0 ); - assert( !zMaster || pPager->journalMode!=PAGER_JOURNALMODE_WAL ); assert( !pagerUseWal(pPager) ); if( !zMaster @@ -5168,33 +5169,18 @@ static int pager_write(PgHdr *pPg){ if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ u32 cksum; char *pData2; + i64 iOff = pPager->journalOff; /* We should never write to the journal file the page that ** contains the database locks. The following assert verifies ** that we do not. */ assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); - assert( pPager->journalHdr <= pPager->journalOff ); + assert( pPager->journalHdr<=pPager->journalOff ); CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); cksum = pager_cksum(pPager, (u8*)pData2); - rc = write32bits(pPager->jfd, pPager->journalOff, pPg->pgno); - if( rc==SQLITE_OK ){ - rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, - pPager->journalOff + 4); - pPager->journalOff += pPager->pageSize+4; - } - if( rc==SQLITE_OK ){ - rc = write32bits(pPager->jfd, pPager->journalOff, cksum); - pPager->journalOff += 4; - } - IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, - pPager->journalOff, pPager->pageSize)); - PAGER_INCR(sqlite3_pager_writej_count); - PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); - /* Even if an IO or diskfull error occurred while journalling the + /* Even if an IO or diskfull error occurs while journalling the ** page in the block above, set the need-sync flag for the page. ** Otherwise, when the transaction is rolled back, the logic in ** playback_one_page() will think that the page needs to be restored @@ -5203,13 +5189,21 @@ static int pager_write(PgHdr *pPg){ */ pPg->flags |= PGHDR_NEED_SYNC; - /* An error has occurred writing to the journal file. The - ** transaction will be rolled back by the layer above. - */ - if( rc!=SQLITE_OK ){ - return rc; - } + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + pPager->journalOff, pPager->pageSize)); + PAGER_INCR(sqlite3_pager_writej_count); + PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); + + pPager->journalOff += 8 + pPager->pageSize; pPager->nRec++; assert( pPager->pInJournal!=0 ); rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); @@ -5335,7 +5329,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){ ** before any of them can be written out to the database file. */ if( rc==SQLITE_OK && needSync ){ - assert( !MEMDB && pPager->noSync==0 ); + assert( !MEMDB ); for(ii=0; iisetMaster); if( rc==SQLITE_OK ) rc = rc2; - }else if( !isOpen(pPager->jfd) ){ + }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ rc = pager_end_transaction(pPager, 0); }else{ rc = pager_playback(pPager, 0); } + assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); assert( rc==SQLITE_OK || rc==SQLITE_FULL || (rc&0xFF)==SQLITE_IOERR ); diff --git a/src/wal.c b/src/wal.c index be8c466428..57d8e462df 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2044,6 +2044,7 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ ** read-lock. */ void sqlite3WalEndReadTransaction(Wal *pWal){ + sqlite3WalEndWriteTransaction(pWal); if( pWal->readLock>=0 ){ walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->readLock = -1; diff --git a/test/walmode.test b/test/walmode.test index 1a54277418..2506c90e1e 100644 --- a/test/walmode.test +++ b/test/walmode.test @@ -377,4 +377,11 @@ sqlite3 db2 test.db2 do_test walmode-8.19 { execsql { PRAGMA main.journal_mode } db2 } {wal} db2 close +do_execsql_test walmode-8.20 { PRAGMA journal_mode = DELETE } {delete} +do_execsql_test walmode-8.21 { PRAGMA main.journal_mode } {delete} +do_execsql_test walmode-8.22 { PRAGMA two.journal_mode } {delete} +do_execsql_test walmode-8.21 { PRAGMA journal_mode = WAL } {wal} +do_execsql_test walmode-8.21 { PRAGMA main.journal_mode } {wal} +do_execsql_test walmode-8.22 { PRAGMA two.journal_mode } {wal} + finish_test From 7a76197f40df894de9d0ff64324013b22233d6ee Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 9 Aug 2010 07:07:37 +0000 Subject: [PATCH 24/30] In WAL mode, ignore any error returned by an xFileControl(FCNTL_SIZE_HINT) call. This matches the behaviour in rollback mode. FossilOrigin-Name: 158a309737bd4cdc033cd504a6d0cc43c15b6e17 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6455d6683b..cf37d80174 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\sproblems\sand\supdate\scomments\sin\spager.c. -D 2010-08-07T16:17:49 +C In\sWAL\smode,\signore\sany\serror\sreturned\sby\san\sxFileControl(FCNTL_SIZE_HINT)\scall.\sThis\smatches\sthe\sbehaviour\sin\srollback\smode. +D 2010-08-09T07:07:38 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -227,7 +227,7 @@ F src/vdbeblob.c 258a6010ba7a82b72b327fb24c55790655689256 F src/vdbemem.c e5673f81a2381b35c60e73ef0a8502be2ab1041e F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c 82200af3881fa4e1c9cf07cf31d98c09d437e3ab -F src/wal.c ef5e37b8cf0b617b787d00eb9c63c803f1f790b1 +F src/wal.c 4ac3c3d757447b54f6911d11843a5a40e2a14ab8 F src/wal.h 96669b645e27cd5a111ba59f0cae7743a207bc3c F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 7db3e41c2a846f9deeb24f1bbb75461b4010b7b5 @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 531abc808526d607768bf6f503268d4cc66ab169 -R 8bd570ff99007432bc59ff36d6f70bed +P 92e456374b052aceff356d10317b5b94fcdeaa5c +R 8b8b84b2159488c8018e6a3bb6917d04 U dan -Z ab88a6e6bc1538b57c390502ab433fb7 +Z c0e146dd93d7d775035eb9a6bc3d7235 diff --git a/manifest.uuid b/manifest.uuid index 802595050a..93a3181d4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92e456374b052aceff356d10317b5b94fcdeaa5c \ No newline at end of file +158a309737bd4cdc033cd504a6d0cc43c15b6e17 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 57d8e462df..792a656623 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1582,7 +1582,7 @@ static int walCheckpoint( nReq = ((i64)mxPage * szPage); rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); } /* Iterate through the contents of the WAL, copying data to the db file. */ From 007820d65d1ed51da24b8edfb9fb1855deb9de3c Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 9 Aug 2010 07:51:40 +0000 Subject: [PATCH 25/30] Fix a problem causing the return code of an xSync call to be ignored in wal.c. FossilOrigin-Name: f1b2b5f9c3a5561ea6108283ae08404106c3f8bc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 11 ++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index cf37d80174..9a4fbe11aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sWAL\smode,\signore\sany\serror\sreturned\sby\san\sxFileControl(FCNTL_SIZE_HINT)\scall.\sThis\smatches\sthe\sbehaviour\sin\srollback\smode. -D 2010-08-09T07:07:38 +C Fix\sa\sproblem\scausing\sthe\sreturn\scode\sof\san\sxSync\scall\sto\sbe\signored\sin\swal.c. +D 2010-08-09T07:51:41 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -227,7 +227,7 @@ F src/vdbeblob.c 258a6010ba7a82b72b327fb24c55790655689256 F src/vdbemem.c e5673f81a2381b35c60e73ef0a8502be2ab1041e F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c 82200af3881fa4e1c9cf07cf31d98c09d437e3ab -F src/wal.c 4ac3c3d757447b54f6911d11843a5a40e2a14ab8 +F src/wal.c 582acbaa606c748675fafff516ec1b34a7bdae62 F src/wal.h 96669b645e27cd5a111ba59f0cae7743a207bc3c F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 7db3e41c2a846f9deeb24f1bbb75461b4010b7b5 @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 92e456374b052aceff356d10317b5b94fcdeaa5c -R 8b8b84b2159488c8018e6a3bb6917d04 +P 158a309737bd4cdc033cd504a6d0cc43c15b6e17 +R 79082b63de1369184eea76cab5557bfc U dan -Z c0e146dd93d7d775035eb9a6bc3d7235 +Z 5fdda4c9d9e63d074670dd4c1c360d96 diff --git a/manifest.uuid b/manifest.uuid index 93a3181d4d..45c1738165 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -158a309737bd4cdc033cd504a6d0cc43c15b6e17 \ No newline at end of file +f1b2b5f9c3a5561ea6108283ae08404106c3f8bc \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 792a656623..5924a63756 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1567,7 +1567,6 @@ static int walCheckpoint( if( pInfo->nBackfillnBackfill; @@ -1579,10 +1578,12 @@ static int walCheckpoint( /* If the database file may grow as a result of this checkpoint, hint ** about the eventual size of the db file to the VFS layer. */ - nReq = ((i64)mxPage * szPage); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } } /* Iterate through the contents of the WAL, copying data to the db file. */ From ec561a351a08fb842caa2e65f4a3b601f4446793 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 9 Aug 2010 14:47:50 +0000 Subject: [PATCH 26/30] Change to tcl test infrastructure so that --malloctrace=1 works when sizeof(int)!=sizeof(void*). FossilOrigin-Name: c3e771b3cf685f4237a2516ee7111849bf0ffb3f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_malloc.c | 19 +++++++++++-------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 9a4fbe11aa..d49f525643 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\sthe\sreturn\scode\sof\san\sxSync\scall\sto\sbe\signored\sin\swal.c. -D 2010-08-09T07:51:41 +C Change\sto\stcl\stest\sinfrastructure\sso\sthat\s--malloctrace=1\sworks\swhen\ssizeof(int)!=sizeof(void*). +D 2010-08-09T14:47:50 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -200,7 +200,7 @@ F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 5ac89727cadbcc71954067b319eb40aff71dead2 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e -F src/test_malloc.c 058e41dc853b1fa8ccd1083e9571018a74aeb3a7 +F src/test_malloc.c 09a88f0c111201dc4f8c20470aa1b5f611d59200 F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3 F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec F src/test_osinst.c f408c6a181f2fb04c56273afd5c3e1e82f60392c @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 158a309737bd4cdc033cd504a6d0cc43c15b6e17 -R 79082b63de1369184eea76cab5557bfc +P f1b2b5f9c3a5561ea6108283ae08404106c3f8bc +R 0c13ff5ef493cf88f58734e32a21c9d6 U dan -Z 5fdda4c9d9e63d074670dd4c1c360d96 +Z 48ba547a168a1ffe5803ea79aa90257c diff --git a/manifest.uuid b/manifest.uuid index 45c1738165..9547915834 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1b2b5f9c3a5561ea6108283ae08404106c3f8bc \ No newline at end of file +c3e771b3cf685f4237a2516ee7111849bf0ffb3f \ No newline at end of file diff --git a/src/test_malloc.c b/src/test_malloc.c index 3fa753b7e0..08642c9fec 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -728,7 +728,10 @@ static int test_memdebug_settitle( return TCL_OK; } -#define MALLOC_LOG_FRAMES 10 +#define MALLOC_LOG_FRAMES 10 +#define MALLOC_LOG_KEYINTS ( \ + 10 * ((sizeof(int)>=sizeof(void*)) ? 1 : sizeof(void*)/sizeof(int)) \ +) static Tcl_HashTable aMallocLog; static int mallocLogEnabled = 0; @@ -745,8 +748,8 @@ static void test_memdebug_callback(int nByte, int nFrame, void **aFrame){ Tcl_HashEntry *pEntry; int isNew; - int aKey[MALLOC_LOG_FRAMES]; - int nKey = sizeof(int)*MALLOC_LOG_FRAMES; + int aKey[MALLOC_LOG_KEYINTS]; + int nKey = sizeof(int)*MALLOC_LOG_KEYINTS; memset(aKey, 0, nKey); if( (sizeof(void*)*nFrame)=sizeof(void*)); for( pEntry=Tcl_FirstHashEntry(&aMallocLog, &search); @@ -836,13 +839,13 @@ static int test_memdebug_log( ){ Tcl_Obj *apElem[MALLOC_LOG_FRAMES+2]; MallocLog *pLog = (MallocLog *)Tcl_GetHashValue(pEntry); - int *aKey = (int *)Tcl_GetHashKey(&aMallocLog, pEntry); + Tcl_WideInt *aKey = (Tcl_WideInt *)Tcl_GetHashKey(&aMallocLog, pEntry); int ii; apElem[0] = Tcl_NewIntObj(pLog->nCall); apElem[1] = Tcl_NewIntObj(pLog->nByte); for(ii=0; ii Date: Mon, 9 Aug 2010 16:12:51 +0000 Subject: [PATCH 27/30] Fix a memory leak in test_journal.c causing tests to fail. FossilOrigin-Name: f229487cccc7514b2663e8e6e04798702c9cfb10 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 2 +- src/test_journal.c | 1 + 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d49f525643..8b71df0195 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sto\stcl\stest\sinfrastructure\sso\sthat\s--malloctrace=1\sworks\swhen\ssizeof(int)!=sizeof(void*). -D 2010-08-09T14:47:50 +C Fix\sa\smemory\sleak\sin\stest_journal.c\scausing\stests\sto\sfail. +D 2010-08-09T16:12:52 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 3af4caaa32f26d537ff857ac30d3ac3ce7e46ea5 +F src/pager.c ad850e0b0547ff3781676c311c276123b5e07902 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -198,7 +198,7 @@ F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 -F src/test_journal.c 5ac89727cadbcc71954067b319eb40aff71dead2 +F src/test_journal.c 785edd54f963aefb3c1628124170a56697c68c70 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 09a88f0c111201dc4f8c20470aa1b5f611d59200 F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3 @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P f1b2b5f9c3a5561ea6108283ae08404106c3f8bc -R 0c13ff5ef493cf88f58734e32a21c9d6 +P c3e771b3cf685f4237a2516ee7111849bf0ffb3f +R fafdcc6c6bb38040ee797c40ede9da75 U dan -Z 48ba547a168a1ffe5803ea79aa90257c +Z 19e4a407fe616523b7236ad07579dec7 diff --git a/manifest.uuid b/manifest.uuid index 9547915834..55a281afff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3e771b3cf685f4237a2516ee7111849bf0ffb3f \ No newline at end of file +f229487cccc7514b2663e8e6e04798702c9cfb10 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index b68d3cbb8f..4c5bd662cd 100644 --- a/src/pager.c +++ b/src/pager.c @@ -901,7 +901,7 @@ static char *print_pager_state(Pager *p){ p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : p->eState==PAGER_ERROR ? "ERROR" : "?error?" , (int)p->errCode - , p->eLock==NO_LOCK ? "OPEN" : + , p->eLock==NO_LOCK ? "NO_LOCK" : p->eLock==RESERVED_LOCK ? "RESERVED" : p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : p->eLock==SHARED_LOCK ? "SHARED" : diff --git a/src/test_journal.c b/src/test_journal.c index 0300fb8716..ca4c5c3884 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -361,6 +361,7 @@ static int openTransaction(jt_file *pMain, jt_file *pJournal){ sqlite3_file *p = pMain->pReal; int rc = SQLITE_OK; + closeTransaction(pMain); aData = sqlite3_malloc(pMain->nPagesize); pMain->pWritable = sqlite3BitvecCreate(pMain->nPage); pMain->aCksum = sqlite3_malloc(sizeof(u32) * (pMain->nPage + 1)); From de5fd22f3d302a4125213f6c35f441edcb9b8856 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 9 Aug 2010 19:17:29 +0000 Subject: [PATCH 28/30] Modify some comments in pager.c. FossilOrigin-Name: 5662da6d4f648e5d07d7cbea6bbd92fa684c02c9 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/pager.c | 81 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 64 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 32b6a4f7db..866a7705ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sexperimental\sbranch. -D 2010-08-09T16:52:12 +C Modify\ssome\scomments\sin\spager.c. +D 2010-08-09T19:17:29 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c ad850e0b0547ff3781676c311c276123b5e07902 +F src/pager.c df8e60594f095c3bd6b74c7c8a2585655b055536 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P f229487cccc7514b2663e8e6e04798702c9cfb10 9cebaf2dca6dc35e489537fe7c55474e1029a98e -R a8ec38097bd382b8337fdc3c678431ed +P aecbd890327dc676d6c2366b07f3d2e636a4983e +R f5e40f8996c812e7f2b0312f4a214200 U dan -Z 5adcbb92ea687904b54262b46cb56097 +Z 6addb6181e8f60b5f4d1be2660768d4f diff --git a/manifest.uuid b/manifest.uuid index f41751d6b9..2f79639b12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aecbd890327dc676d6c2366b07f3d2e636a4983e \ No newline at end of file +5662da6d4f648e5d07d7cbea6bbd92fa684c02c9 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 4c5bd662cd..3544032e3d 100644 --- a/src/pager.c +++ b/src/pager.c @@ -241,6 +241,11 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** WRITER_DBMOD: ** +** The pager transitions from WRITER_CACHEMOD into WRITER_DBMOD state +** when it modifies the contents of the database file. WAL connections +** never enter this state (since they do not modify the database file, +** just the log file). +** ** * A write transaction is active. ** * An EXCLUSIVE or greater lock is held on the database file. ** * The journal file is open and the first header has been written @@ -250,6 +255,15 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** WRITER_FINISHED: ** +** It is not possible for a WAL connection to enter this state. +** +** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD +** state after the entire transaction has been successfully written into the +** database file. In this state the transaction may be committed simply +** by finalizing the journal file. Once in WRITER_FINISHED state, it is +** not possible to modify the database further. At this point, the upper +** layer must either commit or rollback the transaction. +** ** * A write transaction is active. ** * An EXCLUSIVE or greater lock is held on the database file. ** * All writing and syncing of journal and database data has finished. @@ -1642,17 +1656,29 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ } /* -** Unlock the database file. This function is a no-op if the pager -** is in exclusive mode. +** This function is a no-op if the pager is in exclusive mode and not +** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN +** state. ** -** If the pager is currently in error state, discard the contents of -** the cache and reset the Pager structure internal state. If there is -** an open journal-file, then the next time a shared-lock is obtained -** on the pager file (by this or any other process), it will be -** treated as a hot-journal and rolled back. +** If the pager is not in exclusive-access mode, the database file is +** completely unlocked. If the file is unlocked and the file-system does +** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is +** closed (if it is open). +** +** If the pager is in ERROR state when this function is called, the +** contents of the pager cache are discarded before switching back to +** the OPEN state. Regardless of whether the pager is in exclusive-mode +** or not, any journal file left in the file-system will be treated +** as a hot-journal and rolled back the next time a read-transaction +** is opened (by this or by any other connection). */ static void pager_unlock(Pager *pPager){ + assert( pPager->eState==PAGER_READER + || pPager->eState==PAGER_OPEN + || pPager->eState==PAGER_ERROR + ); + sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; releaseAllSavepoints(pPager); @@ -1719,18 +1745,20 @@ static void pager_unlock(Pager *pPager){ } /* -** This function should be called when an IOERR, CORRUPT or FULL error -** may have occurred. The first argument is a pointer to the pager -** structure, the second the error-code about to be returned by a pager -** API function. The value returned is a copy of the second argument -** to this function. +** This function is called whenever an IOERR or FULL error that requires +** the pager to transition into the ERROR state may ahve occurred. +** The first argument is a pointer to the pager structure, the second +** the error-code about to be returned by a pager API function. The +** value returned is a copy of the second argument to this function. ** -** If the second argument is SQLITE_IOERR, SQLITE_CORRUPT, or SQLITE_FULL -** the error becomes persistent. Until the persistent error is cleared, -** subsequent API calls on this Pager will immediately return the same -** error code. +** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the +** IOERR sub-codes, the pager enters the ERROR state and the error code +** is stored in Pager.errCode. While the pager remains in the ERROR state, +** all major API calls on the Pager will immediately return Pager.errCode. +** Except, if the error-code is SQLITE_FULL, calls to PagerLookup() and +** PagerAcquire are handled as if the pager were in PAGER_READER state. ** -** A persistent error indicates that the contents of the pager-cache +** The ERROR state indicates that the contents of the pager-cache ** cannot be trusted. This state can be cleared by completely discarding ** the contents of the pager-cache. If a transaction was active when ** the persistent error occurred, then the rollback journal may need @@ -2087,7 +2115,7 @@ static int pager_playback_one_page( pagerReportSize(pPager); } - /* If the pager is in RESERVED state, then there must be a copy of this + /* If the pager is in CACHEMOD state, then there must be a copy of this ** page in the pager cache. In this case just update the pager cache, ** not the database file. The page is left marked dirty in this case. ** @@ -2098,8 +2126,11 @@ static int pager_playback_one_page( ** either. So the condition described in the above paragraph is not ** assert()able. ** - ** If in EXCLUSIVE state, then we update the pager cache if it exists - ** and the main file. The page is then marked not dirty. + ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the + ** pager cache if it exists and the main file. The page is then marked + ** not dirty. Since this code is only executed in PAGER_OPEN state for + ** a hot-journal rollback, it is guaranteed that the page-cache is empty + ** if the pager is in OPEN state. ** ** Ticket #1171: The statement journal might contain page content that is ** different from the page content at the start of the transaction. @@ -2125,6 +2156,7 @@ static int pager_playback_one_page( pPg = pager_lookup(pPager, pgno); } assert( pPg || !MEMDB ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") @@ -2365,10 +2397,10 @@ delmaster_out: ** file in the file-system. This only happens when committing a transaction, ** or rolling back a transaction (including rolling back a hot-journal). ** -** If the main database file is not open, or an exclusive lock is not -** held, this function is a no-op. Otherwise, the size of the file is -** changed to nPage pages (nPage*pPager->pageSize bytes). If the file -** on disk is currently larger than nPage pages, then use the VFS +** If the main database file is not open, or the pager is not in either +** DBMOD or OPEN state, this function is a no-op. Otherwise, the size +** of the file is changed to nPage pages (nPage*pPager->pageSize bytes). +** If the file on disk is currently larger than nPage pages, then use the VFS ** xTruncate() method to truncate it. ** ** Or, it might might be the case that the file on disk is smaller than @@ -2389,6 +2421,7 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) ){ i64 currentSize, newSize; + assert( pPager->eLock==EXCLUSIVE_LOCK ); /* TODO: Is it safe to use Pager.dbFileSize here? */ rc = sqlite3OsFileSize(pPager->fd, ¤tSize); newSize = pPager->pageSize*(i64)nPage; From c864912ad16cc8b86e3b371e35cbd45ff7067008 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 10 Aug 2010 09:58:57 +0000 Subject: [PATCH 29/30] Add variable pager.dbHintSize, used to limit the number of calls made to the xFileControl(FCNTL_SIZE_HINT) method. FossilOrigin-Name: eb3ac895bd92c880ac6acca58de8cf9643c2b5e4 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/pager.c | 77 ++++++++++++++++++++++++++------------------------- 3 files changed, 47 insertions(+), 44 deletions(-) diff --git a/manifest b/manifest index 866a7705ba..4ff62443c9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\ssome\scomments\sin\spager.c. -D 2010-08-09T19:17:29 +C Add\svariable\spager.dbHintSize,\sused\sto\slimit\sthe\snumber\sof\scalls\smade\sto\sthe\sxFileControl(FCNTL_SIZE_HINT)\smethod. +D 2010-08-10T09:58:57 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c df8e60594f095c3bd6b74c7c8a2585655b055536 +F src/pager.c 6e8316c94f4ce46480f7f628c3e819d922af2111 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P aecbd890327dc676d6c2366b07f3d2e636a4983e -R f5e40f8996c812e7f2b0312f4a214200 +P 5662da6d4f648e5d07d7cbea6bbd92fa684c02c9 +R 016b700a84062c14051dadc45f2eaff7 U dan -Z 6addb6181e8f60b5f4d1be2660768d4f +Z bdf1c4ed8f87e806444ebc6f32ecd642 diff --git a/manifest.uuid b/manifest.uuid index 2f79639b12..4d160d37cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5662da6d4f648e5d07d7cbea6bbd92fa684c02c9 \ No newline at end of file +eb3ac895bd92c880ac6acca58de8cf9643c2b5e4 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 3544032e3d..e2c2f715a3 100644 --- a/src/pager.c +++ b/src/pager.c @@ -565,7 +565,30 @@ struct PagerSavepoint { ** Throughout a write-transaction, dbFileSize contains the size of ** the file on disk in pages. It is set to a copy of dbSize when the ** write-transaction is first opened, and updated when VFS calls are made -** to write or truncate the database file on disk. +** to write or truncate the database file on disk. +** +** The only reason the dbFileSize variable is required is to suppress +** unnecessary calls to xTruncate() after committing a transaction. If, +** when a transaction is committed, the dbFileSize variable indicates +** that the database file is larger than the database image (Pager.dbSize), +** pager_truncate() is called. The pager_truncate() call uses xFilesize() +** to measure the database file on disk, and then truncates it if required. +** dbFileSize is not used when rolling back a transaction. In this case +** pager_truncate() is called unconditionally (which means there may be +** a call to xFilesize() that is not strictly required). In either case, +** pager_truncate() may cause the file to become smaller or larger. +** +** dbHintSize +** +** The dbHintSize variable is used to limit the number of calls made to +** the VFS xFileControl(FCNTL_SIZE_HINT) method. +** +** dbHintSize is set to a copy of the dbSize variable when a +** write-transaction is opened (at the same time as dbFileSize and +** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, +** dbHintSize is increased to the number of pages that correspond to the +** size-hint passed to the method call. See pager_write_pagelist() for +** details. ** ** errCode ** @@ -619,6 +642,7 @@ struct Pager { Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ + Pgno dbHintSize; /* Value passed to FCNTL_SIZE_HINT call */ int errCode; /* One of several kinds of errors */ int nRec; /* Pages journalled since last j-header written */ u32 cksumInit; /* Quasi-random value added to every checksum */ @@ -831,6 +855,7 @@ static int assert_pager_state(Pager *p){ } assert( pPager->dbSize==pPager->dbOrigSize ); assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); assert( pPager->setMaster==0 ); break; @@ -850,6 +875,7 @@ static int assert_pager_state(Pager *p){ ); } assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); break; case PAGER_WRITER_DBMOD: @@ -861,6 +887,7 @@ static int assert_pager_state(Pager *p){ || p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_WAL ); + assert( pPager->dbOrigSize<=pPager->dbHintSize ); break; case PAGER_WRITER_FINISHED: @@ -3829,41 +3856,12 @@ static int syncJournal(Pager *pPager, int newHdr){ ** be obtained, SQLITE_BUSY is returned. */ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ - /* Normally, this function is called in WRITER_DBMOD state. - ** - ** However it may be called in WRITER_CACHEMOD state if the page being - ** written (and all other pages that reside on the same disk sector) was - ** a free-list leaf page at the start of the transaction. In that case - ** the database file is not really being modified, so it is Ok to write - ** to it in CACHEMOD state. - */ + /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_WRITER_CACHEMOD - ); - assert( pPager->eState==PAGER_WRITER_DBMOD - || (pList->pDirty==0 && pList->pgno<=pPager->dbFileSize) - ); - - /* At this point there may be either a RESERVED or EXCLUSIVE lock on the - ** database file. If there is already an EXCLUSIVE lock, the following - ** call is a no-op. - ** - ** Moving the lock from RESERVED to EXCLUSIVE actually involves going - ** through an intermediate state PENDING. A PENDING lock prevents new - ** readers from attaching to the database but is unsufficient for us to - ** write. The idea of a PENDING lock is to prevent new readers from - ** coming in while we wait for existing readers to clear. - ** - ** While the pager is in the RESERVED state, the original database file - ** is unchanged and we can rollback without having to playback the - ** journal into the original database file. Once we transition to - ** EXCLUSIVE, it means the database file has been changed and any rollback - ** will require a journal playback. - */ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->eLock==EXCLUSIVE_LOCK ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -3878,9 +3876,10 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbOrigSize ){ + if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); + pPager->dbHintSize = pPager->dbSize; } while( rc==SQLITE_OK && pList ){ @@ -4062,7 +4061,9 @@ static int pagerStress(void *p, PgHdr *pPg){ }else{ /* Sync the journal file if required. */ - if( pPg->flags&PGHDR_NEED_SYNC ){ + if( pPg->flags&PGHDR_NEED_SYNC + || pPager->eState==PAGER_WRITER_CACHEMOD + ){ rc = syncJournal(pPager, 1); } @@ -5124,7 +5125,9 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** WAL mode. */ pPager->eState = PAGER_WRITER_LOCKED; - pPager->dbFileSize = pPager->dbOrigSize = pPager->dbSize; + pPager->dbHintSize = pPager->dbSize; + pPager->dbFileSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; pPager->journalOff = 0; } From 7e68423894fae26d25ef4e1d6b8513469020db07 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 10 Aug 2010 15:46:06 +0000 Subject: [PATCH 30/30] Fix a bug in pager.c causing it to omit the xSync() call required following a hot-journal rollback. FossilOrigin-Name: 42ba43ac690dc800fb8b969ae257a79942918342 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 7 +++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4ff62443c9..96901ca253 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\svariable\spager.dbHintSize,\sused\sto\slimit\sthe\snumber\sof\scalls\smade\sto\sthe\sxFileControl(FCNTL_SIZE_HINT)\smethod. -D 2010-08-10T09:58:57 +C Fix\sa\sbug\sin\spager.c\scausing\sit\sto\somit\sthe\sxSync()\scall\srequired\sfollowing\sa\shot-journal\srollback. +D 2010-08-10T15:46:07 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c ae5ca8a6031380708f3fec7be325233d49944914 F src/os_win.c 51cb62f76262d961ea4249489383d714501315a7 -F src/pager.c 6e8316c94f4ce46480f7f628c3e819d922af2111 +F src/pager.c b3db762350ee71f5f8bde04f21ca2ffcccded483 F src/pager.h 80726162dc3942f59ab27b738fb667b9ba0a89d5 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07 @@ -843,7 +843,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 5662da6d4f648e5d07d7cbea6bbd92fa684c02c9 -R 016b700a84062c14051dadc45f2eaff7 +P eb3ac895bd92c880ac6acca58de8cf9643c2b5e4 +R e4d9a3fc6619bf1654c442eda9a22d0e U dan -Z bdf1c4ed8f87e806444ebc6f32ecd642 +Z d2fd5616e29600e0ee8548b4ff5a7e14 diff --git a/manifest.uuid b/manifest.uuid index 4d160d37cb..219b4e1fad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb3ac895bd92c880ac6acca58de8cf9643c2b5e4 \ No newline at end of file +42ba43ac690dc800fb8b969ae257a79942918342 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e2c2f715a3..4d9be40eb8 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2723,10 +2723,9 @@ end_playback: rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); testcase( rc!=SQLITE_OK ); } - if( rc==SQLITE_OK && !pPager->noSync && pPager->eState>=PAGER_WRITER_DBMOD ){ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); - } - if( rc==SQLITE_OK && !pPager->noSync && pPager->eState>=PAGER_WRITER_DBMOD ){ + if( rc==SQLITE_OK && !pPager->noSync + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); } if( rc==SQLITE_OK ){