From 2be25bffcad4002b02a08a84eccb6bf72037721d Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 15 Dec 2011 02:22:16 +0000 Subject: [PATCH 01/10] In the multiplexor, do not try to delete overflow files that do not exist. And assume all but the last overflow file is the size of the chunk size. FossilOrigin-Name: a822a80d3cfe42b2fca6f8c9ff11762993114a27 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_multiplex.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e9459b2243..cd4f6d1738 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sVFSNAME\sfile-control\saddition\sinto\sthe\snx-devkit\sbranch. -D 2011-12-14T18:28:18.920 +C In\sthe\smultiplexor,\sdo\snot\stry\sto\sdelete\soverflow\sfiles\sthat\sdo\snot\sexist.\nAnd\sassume\sall\sbut\sthe\slast\soverflow\sfile\sis\sthe\ssize\sof\sthe\schunk\ssize. +D 2011-12-15T02:22:16.244 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 8d416f29ad8573f32601f6056c9d2b17472e9ad5 -F src/test_multiplex.c 80677a47c75ca6a035d1ac66fb59186ffd4cdd19 +F src/test_multiplex.c cee2474efe6ee92732e43184f62422bc17623d1f F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 08bbbd8e38697908ab8ae0f6b512d01c32f7e4da cdbfb553af02d10767a905489d9dfc57fc6be54e -R d213e3c5e90076919d50d6149e76976e +P 08c1dc517c1340737a55ad9012b7b06f72899c6f +R 05ea69e1c214c57dec4ba569476ada26 U drh -Z 8e788ffb68fb72b933914781f94a7874 +Z 5b0957f1b7fd94c5b03cf783382a1410 diff --git a/manifest.uuid b/manifest.uuid index d03e81ee9a..18315cb8d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -08c1dc517c1340737a55ad9012b7b06f72899c6f \ No newline at end of file +a822a80d3cfe42b2fca6f8c9ff11762993114a27 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index f6540d82dc..b9688e34df 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -365,7 +365,9 @@ static void multiplexSubClose( sqlite3_file *pSubOpen = pGroup->aReal[iChunk].p; if( pSubOpen ){ pSubOpen->pMethods->xClose(pSubOpen); - if( pOrigVfs ) pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0); + if( pOrigVfs && pGroup->aReal[iChunk].z ){ + pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0); + } sqlite3_free(pGroup->aReal[iChunk].p); } sqlite3_free(pGroup->aReal[iChunk].z); @@ -768,7 +770,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ if( rc==SQLITE_OK && sz>pGroup->szChunk ){ rc = SQLITE_IOERR_FSTAT; } - *pSize += sz; + *pSize = i*(sqlite3_int64)pGroup->szChunk + sz; } } } From 31b21295b0f291b2e4f37442b687672e1feffac3 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Dec 2011 11:45:19 +0000 Subject: [PATCH 02/10] Fix some problems with dropped error codes in multiplexOpen(). FossilOrigin-Name: 2d50f78188e3297e8cefdf73cff51fa0a3b36e65 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_multiplex.c | 18 +++++++++++------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index cd4f6d1738..33561aa1a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\smultiplexor,\sdo\snot\stry\sto\sdelete\soverflow\sfiles\sthat\sdo\snot\sexist.\nAnd\sassume\sall\sbut\sthe\slast\soverflow\sfile\sis\sthe\ssize\sof\sthe\schunk\ssize. -D 2011-12-15T02:22:16.244 +C Fix\ssome\sproblems\swith\sdropped\serror\scodes\sin\smultiplexOpen(). +D 2011-12-15T11:45:19.350 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 8d416f29ad8573f32601f6056c9d2b17472e9ad5 -F src/test_multiplex.c cee2474efe6ee92732e43184f62422bc17623d1f +F src/test_multiplex.c 7e8d8303b8ba74ff50fbb40502522e576760266e F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 08c1dc517c1340737a55ad9012b7b06f72899c6f -R 05ea69e1c214c57dec4ba569476ada26 -U drh -Z 5b0957f1b7fd94c5b03cf783382a1410 +P a822a80d3cfe42b2fca6f8c9ff11762993114a27 +R 4637baef8720406f37351464996699d4 +U dan +Z a1536baba57190ddd9d59e1c01fe9761 diff --git a/manifest.uuid b/manifest.uuid index 18315cb8d5..b07b3ebcac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a822a80d3cfe42b2fca6f8c9ff11762993114a27 \ No newline at end of file +2d50f78188e3297e8cefdf73cff51fa0a3b36e65 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index b9688e34df..f0d495f491 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -466,13 +466,15 @@ static int multiplexOpen( rc = multiplexSubFilename(pGroup, 1); if( rc==SQLITE_OK ){ pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags); + assert( pSubOpen || rc!=SQLITE_OK ); } - if( pSubOpen ){ - int exists, rc2, rc3; + if( rc==SQLITE_OK ){ sqlite3_int64 sz; - rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); - if( rc2==SQLITE_OK && zName ){ + rc = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); + if( rc==SQLITE_OK && zName ){ + int exists; + /* If the first overflow file exists and if the size of the main file ** is different from the chunk size, that means the chunk size is set ** set incorrectly. So fix it. @@ -482,16 +484,18 @@ static int multiplexOpen( ** But we have no way of determining the intended chunk size, so ** just disable the multiplexor all togethre. */ - rc3 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, + rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, SQLITE_ACCESS_EXISTS, &exists); - if( rc3==SQLITE_OK && exists && sz==(sz&0xffff0000) && sz>0 + if( rc==SQLITE_OK && exists && sz==(sz&0xffff0000) && sz>0 && sz!=pGroup->szChunk ){ pGroup->szChunk = sz; - }else if( rc3==SQLITE_OK && !exists && sz>pGroup->szChunk ){ + }else if( rc==SQLITE_OK && !exists && sz>pGroup->szChunk ){ pGroup->bEnabled = 0; } } + } + if( rc==SQLITE_OK ){ if( pSubOpen->pMethods->iVersion==1 ){ pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV1; }else{ From 78c0eafb3535fd70d74b9753b4e52f7a6225b5a2 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 15 Dec 2011 13:29:12 +0000 Subject: [PATCH 03/10] Make sure the chunksize in test_multiplex does not cause the pending byte to fall near the end of a chunk. Adjust the chunksize upward as necessary to prevent this. FossilOrigin-Name: e05f8a2998f4f4cbdb65702baa65893c538b3d38 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_multiplex.c | 10 ++++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 33561aa1a4..21cbee27f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\swith\sdropped\serror\scodes\sin\smultiplexOpen(). -D 2011-12-15T11:45:19.350 +C Make\ssure\sthe\schunksize\sin\stest_multiplex\sdoes\snot\scause\sthe\spending\sbyte\nto\sfall\snear\sthe\send\sof\sa\schunk.\s\sAdjust\sthe\schunksize\supward\sas\snecessary\nto\sprevent\sthis. +D 2011-12-15T13:29:12.575 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 8d416f29ad8573f32601f6056c9d2b17472e9ad5 -F src/test_multiplex.c 7e8d8303b8ba74ff50fbb40502522e576760266e +F src/test_multiplex.c 7b65eea8b390dd223bd6e0ba9d7879389bd075c0 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P a822a80d3cfe42b2fca6f8c9ff11762993114a27 -R 4637baef8720406f37351464996699d4 -U dan -Z a1536baba57190ddd9d59e1c01fe9761 +P 2d50f78188e3297e8cefdf73cff51fa0a3b36e65 +R db6d46332c6333123f8fa60ccb204a1d +U drh +Z f65d5d73aa828ce8ac2cca0eb8289aa5 diff --git a/manifest.uuid b/manifest.uuid index b07b3ebcac..ce1d054b8f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d50f78188e3297e8cefdf73cff51fa0a3b36e65 \ No newline at end of file +e05f8a2998f4f4cbdb65702baa65893c538b3d38 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index f0d495f491..12e420fb5d 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -462,6 +462,16 @@ static int multiplexOpen( memcpy(pGroup->zName, zName, nName+1); pGroup->nName = nName; } + if( pGroup->bEnabled ){ + /* Make sure that the chunksize is not such that the pending byte + ** falls at the end of a chunk. A region of up to 64K following + ** the pending byte is never written, so if the pending byte occurs + ** near the end of a chunk, that chunk will be too small. */ + extern int sqlite3PendingByte; + while( (sqlite3PendingByte % pGroup->szChunk)>=(pGroup->szChunk-65536) ){ + pGroup->szChunk += 65536; + } + } pGroup->flags = flags; rc = multiplexSubFilename(pGroup, 1); if( rc==SQLITE_OK ){ From e712b5823e8c984491a8ac34e43fb5e50a995058 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Dec 2011 17:00:10 +0000 Subject: [PATCH 04/10] When deleting a file with the multiplexor VFS, also delete any overflow files that exist. FossilOrigin-Name: 3af1feaa35d3fb2e7be550cd32a727001b874938 --- manifest | 14 ++--- manifest.uuid | 2 +- src/test_multiplex.c | 132 +++++++++++++++++++++++++++++++------------ 3 files changed, 105 insertions(+), 43 deletions(-) diff --git a/manifest b/manifest index 21cbee27f3..aec2b27521 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\schunksize\sin\stest_multiplex\sdoes\snot\scause\sthe\spending\sbyte\nto\sfall\snear\sthe\send\sof\sa\schunk.\s\sAdjust\sthe\schunksize\supward\sas\snecessary\nto\sprevent\sthis. -D 2011-12-15T13:29:12.575 +C When\sdeleting\sa\sfile\swith\sthe\smultiplexor\sVFS,\salso\sdelete\sany\soverflow\sfiles\sthat\sexist. +D 2011-12-15T17:00:10.854 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 8d416f29ad8573f32601f6056c9d2b17472e9ad5 -F src/test_multiplex.c 7b65eea8b390dd223bd6e0ba9d7879389bd075c0 +F src/test_multiplex.c c87d27283e2f578c17a8db7a8cac095f576f4323 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 2d50f78188e3297e8cefdf73cff51fa0a3b36e65 -R db6d46332c6333123f8fa60ccb204a1d -U drh -Z f65d5d73aa828ce8ac2cca0eb8289aa5 +P e05f8a2998f4f4cbdb65702baa65893c538b3d38 +R 45b2cf463e6285a0d96697332c4d9aa9 +U dan +Z 16b7798bc9bd7dfe8e0e630d52e6558b diff --git a/manifest.uuid b/manifest.uuid index ce1d054b8f..e4ed76872d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e05f8a2998f4f4cbdb65702baa65893c538b3d38 \ No newline at end of file +3af1feaa35d3fb2e7be550cd32a727001b874938 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 12e420fb5d..d844604ab5 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -216,6 +216,39 @@ static int multiplexStrlen30(const char *z){ return 0x3fffffff & (int)(z2 - z); } +/* +** Generate the file-name for chunk iChunk of the group with base name +** zBase. The file-name is written to buffer zOut before returning. Buffer +** zOut must be allocated by the caller so that it is at least (nBase+4) +** bytes in size, where nBase is the length of zBase, not including the +** nul-terminator. +*/ +static void multiplexFilename( + const char *zBase, /* Filename for chunk 0 */ + int nBase, /* Size of zBase in bytes (without \0) */ + int flags, /* Flags used to open file */ + int iChunk, /* Chunk to generate filename for */ + char *zOut /* Buffer to write generated name to */ +){ + memcpy(zOut, zBase, nBase+1); + if( iChunk!=0 && iChunk!=SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET ){ + int n = nBase; +#ifdef SQLITE_ENABLE_8_3_NAMES + int i; + for(i=n-1; i>0 && i>=n-4 && zOut[i]!='.'; i--){} + if( i>=n-4 ) n = i+1; + if( flags & SQLITE_OPEN_MAIN_JOURNAL ){ + /* The extensions on overflow files for main databases are 001, 002, + ** 003 and so forth. To avoid name collisions, add 400 to the + ** extensions of journal files so that they are 401, 402, 403, .... + */ + iChunk += SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET; + } +#endif + sqlite3_snprintf(4,&zOut[n],"%03d",iChunk); + } +} + /* Compute the filename for the iChunk-th chunk */ static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){ @@ -236,22 +269,7 @@ static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){ if( z==0 ){ return SQLITE_NOMEM; } - memcpy(z, pGroup->zName, n+1); - if( iChunk>0 ){ -#ifdef SQLITE_ENABLE_8_3_NAMES - int i; - for(i=n-1; i>0 && i>=n-4 && z[i]!='.'; i--){} - if( i>=n-4 ) n = i+1; - if( pGroup->flags & (SQLITE_OPEN_MAIN_JOURNAL|SQLITE_OPEN_TEMP_JOURNAL) ){ - /* The extensions on overflow files for main databases are 001, 002, - ** 003 and so forth. To avoid name collisions, add 400 to the - ** extensions of journal files so that they are 401, 402, 403, .... - */ - iChunk += SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET; - } -#endif - sqlite3_snprintf(4,&z[n],"%03d",iChunk); - } + multiplexFilename(pGroup->zName, pGroup->nName, pGroup->flags, iChunk, z); } return SQLITE_OK; } @@ -483,24 +501,43 @@ static int multiplexOpen( rc = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); if( rc==SQLITE_OK && zName ){ - int exists; - - /* If the first overflow file exists and if the size of the main file - ** is different from the chunk size, that means the chunk size is set - ** set incorrectly. So fix it. - ** - ** Or, if the first overflow file does not exist and the main file is - ** larger than the chunk size, that means the chunk size is too small. - ** But we have no way of determining the intended chunk size, so - ** just disable the multiplexor all togethre. - */ - rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, - SQLITE_ACCESS_EXISTS, &exists); - if( rc==SQLITE_OK && exists && sz==(sz&0xffff0000) && sz>0 - && sz!=pGroup->szChunk ){ - pGroup->szChunk = sz; - }else if( rc==SQLITE_OK && !exists && sz>pGroup->szChunk ){ - pGroup->bEnabled = 0; + int bExists; + if( sz==0 ){ + if( flags & SQLITE_OPEN_MAIN_JOURNAL ){ + /* If opening a main journal file and the first chunk is zero + ** bytes in size, delete any subsequent chunks from the + ** file-system. */ + int iChunk = 1; + do { + rc = pOrigVfs->xAccess(pOrigVfs, + pGroup->aReal[iChunk].z, SQLITE_ACCESS_EXISTS, &bExists + ); + if( rc==SQLITE_OK && bExists ){ + rc = pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0); + if( rc==SQLITE_OK ){ + rc = multiplexSubFilename(pGroup, ++iChunk); + } + } + }while( rc==SQLITE_OK && bExists ); + } + }else{ + /* If the first overflow file exists and if the size of the main file + ** is different from the chunk size, that means the chunk size is set + ** set incorrectly. So fix it. + ** + ** Or, if the first overflow file does not exist and the main file is + ** larger than the chunk size, that means the chunk size is too small. + ** But we have no way of determining the intended chunk size, so + ** just disable the multiplexor all togethre. + */ + rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, + SQLITE_ACCESS_EXISTS, &bExists); + if( rc==SQLITE_OK && bExists && sz==(sz&0xffff0000) && sz>0 + && sz!=pGroup->szChunk ){ + pGroup->szChunk = sz; + }else if( rc==SQLITE_OK && !bExists && sz>pGroup->szChunk ){ + pGroup->bEnabled = 0; + } } } } @@ -534,8 +571,33 @@ static int multiplexDelete( const char *zName, /* Name of file to delete */ int syncDir ){ + int rc; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - return pOrigVfs->xDelete(pOrigVfs, zName, syncDir); + rc = pOrigVfs->xDelete(pOrigVfs, zName, syncDir); + if( rc==SQLITE_OK ){ + /* If the main chunk was deleted successfully, also delete any subsequent + ** chunks - starting with the last (highest numbered). + */ + int nName = strlen(zName); + char *z; + z = sqlite3_malloc(nName + 4); + if( z==0 ){ + rc = SQLITE_IOERR_NOMEM; + }else{ + int iChunk = 0; + int bExists; + do{ + multiplexFilename(zName, nName, SQLITE_OPEN_MAIN_JOURNAL, ++iChunk, z); + rc = pOrigVfs->xAccess(pOrigVfs, z, SQLITE_ACCESS_EXISTS, &bExists); + }while( rc==SQLITE_OK && bExists ); + while( rc==SQLITE_OK && iChunk>1 ){ + multiplexFilename(zName, nName, SQLITE_OPEN_MAIN_JOURNAL, --iChunk, z); + rc = pOrigVfs->xDelete(pOrigVfs, z, syncDir); + } + } + sqlite3_free(z); + } + return rc; } static int multiplexAccess(sqlite3_vfs *a, const char *b, int c, int *d){ From f580860f5196fc51fd5d9b6f42fe89e5b0573adc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 Dec 2011 00:33:04 +0000 Subject: [PATCH 05/10] Make sure the antipenultimate character of master-journal filenames is a "9" in order to avoid collisions with other files in 8+3 filename mode. Also, limit the number of attempts at finding a unique master-journal filename. FossilOrigin-Name: 34a0483605d36e6cf03065ed0df33fb1f7c8a272 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/util.c | 1 + src/vdbeaux.c | 12 +++++++++++- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index aec2b27521..c6cfc989a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sdeleting\sa\sfile\swith\sthe\smultiplexor\sVFS,\salso\sdelete\sany\soverflow\sfiles\sthat\sexist. -D 2011-12-15T17:00:10.854 +C Make\ssure\sthe\santipenultimate\scharacter\sof\smaster-journal\sfilenames\sis\sa\s"9"\nin\sorder\sto\savoid\scollisions\swith\sother\sfiles\sin\s8+3\sfilename\smode.\s\sAlso,\nlimit\sthe\snumber\sof\sattempts\sat\sfinding\sa\sunique\smaster-journal\sfilename. +D 2011-12-16T00:33:04.352 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -237,13 +237,13 @@ F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 F src/trigger.c 1cfb80e2290ef66ea89cb4e821caae65a02c0d56 F src/update.c 25e046a8f69d5e557aabde2000487b8545509d8d F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 -F src/util.c 01238e2b0f24a14779181dbf991fe02620a80e31 +F src/util.c 343508d359df65685b62e63964a40e7af4cfbe05 F src/vacuum.c 0c0ba2242355c6048d65e2b333abe0f7c06348fa F src/vdbe.c a7ab9993ec5a4d9479dc99671faec061fbf9b889 F src/vdbe.h f0725ee997db869ecae5bb70a71612aabeca7755 F src/vdbeInt.h 9498fc98a2c9e349a4ef13455ff5a3e898f40176 F src/vdbeapi.c 4dbba7f94f127f6ea8d2d0505ee1f98e5ffbf546 -F src/vdbeaux.c 45713a5f8f4f36195f503b30153ddef292323f88 +F src/vdbeaux.c 896ce3c48bf1bf27a1330589a20daa2db0222381 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c 2fc78b3e0fabcc1eaa23cd79dd2e30e6dcfe1e56 F src/vdbesort.c 468d43c057063e54da4f1988b38b4f46d60e7790 @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P e05f8a2998f4f4cbdb65702baa65893c538b3d38 -R 45b2cf463e6285a0d96697332c4d9aa9 -U dan -Z 16b7798bc9bd7dfe8e0e630d52e6558b +P 3af1feaa35d3fb2e7be550cd32a727001b874938 +R 398b3e3805c532cf24ac8d99750e0da2 +U drh +Z ebbc2fb95389a75c4e570b61ca8c6d99 diff --git a/manifest.uuid b/manifest.uuid index e4ed76872d..b3c3bab442 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3af1feaa35d3fb2e7be550cd32a727001b874938 \ No newline at end of file +34a0483605d36e6cf03065ed0df33fb1f7c8a272 \ No newline at end of file diff --git a/src/util.c b/src/util.c index ffbf7daadf..d27f6fdf26 100644 --- a/src/util.c +++ b/src/util.c @@ -1169,6 +1169,7 @@ int sqlite3AbsInt32(int x){ ** test.db-journal => test.nal ** test.db-wal => test.wal ** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa */ void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ #if SQLITE_ENABLE_8_3_NAMES<2 diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 7c69d28cf6..2e01c982e5 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1824,16 +1824,26 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ sqlite3_file *pMaster = 0; i64 offset = 0; int res; + int retryCount = 0; /* Select a master journal file name */ do { u32 iRandom; + if( retryCount++>100 ){ + sqlite3_log(SQLITE_FULL, "cannot create a master journal filename"); + rc = SQLITE_FULL; + break; + } sqlite3DbFree(db, zMaster); sqlite3_randomness(sizeof(iRandom), &iRandom); - zMaster = sqlite3MPrintf(db, "%s-mj%08X", zMainFile, iRandom&0x7fffffff); + zMaster = sqlite3MPrintf(db, "%s-mj%06X9%02X", zMainFile, + (iRandom>>8)&0xffffff, iRandom&0xff); if( !zMaster ){ return SQLITE_NOMEM; } + /* The antipenultimate character of the master journal name must + ** be "9" to avoid name collisions when using 8+3 filenames. */ + assert( zMaster[strlen(zMaster)-3]=='9' ); sqlite3FileSuffix3(zMainFile, zMaster); rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); }while( rc==SQLITE_OK && res ); From 5c531a4aabce7f5a1f60ec87576f24092259a848 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 Dec 2011 01:21:31 +0000 Subject: [PATCH 06/10] A better solution to being unable to find a unique master-journal filename: just delete an existing master-journal and reuse it. FossilOrigin-Name: 2685c2b949061f18bf6a4940eac8c8148873abb6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 15 +++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c6cfc989a6..322d293528 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\santipenultimate\scharacter\sof\smaster-journal\sfilenames\sis\sa\s"9"\nin\sorder\sto\savoid\scollisions\swith\sother\sfiles\sin\s8+3\sfilename\smode.\s\sAlso,\nlimit\sthe\snumber\sof\sattempts\sat\sfinding\sa\sunique\smaster-journal\sfilename. -D 2011-12-16T00:33:04.352 +C A\sbetter\ssolution\sto\sbeing\sunable\sto\sfind\sa\sunique\smaster-journal\sfilename:\njust\sdelete\san\sexisting\smaster-journal\sand\sreuse\sit. +D 2011-12-16T01:21:31.882 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -243,7 +243,7 @@ F src/vdbe.c a7ab9993ec5a4d9479dc99671faec061fbf9b889 F src/vdbe.h f0725ee997db869ecae5bb70a71612aabeca7755 F src/vdbeInt.h 9498fc98a2c9e349a4ef13455ff5a3e898f40176 F src/vdbeapi.c 4dbba7f94f127f6ea8d2d0505ee1f98e5ffbf546 -F src/vdbeaux.c 896ce3c48bf1bf27a1330589a20daa2db0222381 +F src/vdbeaux.c 7bae4d94d7e41e8005824a6d1da54b73d47f52b1 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c 2fc78b3e0fabcc1eaa23cd79dd2e30e6dcfe1e56 F src/vdbesort.c 468d43c057063e54da4f1988b38b4f46d60e7790 @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 3af1feaa35d3fb2e7be550cd32a727001b874938 -R 398b3e3805c532cf24ac8d99750e0da2 +P 34a0483605d36e6cf03065ed0df33fb1f7c8a272 +R dac1ae186d64bf5089bb1df4a91a9793 U drh -Z ebbc2fb95389a75c4e570b61ca8c6d99 +Z cdecba57f424e5a33e7e2e3955b5bd95 diff --git a/manifest.uuid b/manifest.uuid index b3c3bab442..6fdf08c334 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34a0483605d36e6cf03065ed0df33fb1f7c8a272 \ No newline at end of file +2685c2b949061f18bf6a4940eac8c8148873abb6 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 2e01c982e5..6e183d0b8c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1825,25 +1825,24 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ i64 offset = 0; int res; int retryCount = 0; + int nMainFile; /* Select a master journal file name */ + nMainFile = sqlite3Strlen30(zMainFile); + zMaster = sqlite3MPrintf(db, "%s-mjXXXXXX9XX", zMainFile); + if( zMaster==0 ) return SQLITE_NOMEM; do { u32 iRandom; if( retryCount++>100 ){ - sqlite3_log(SQLITE_FULL, "cannot create a master journal filename"); - rc = SQLITE_FULL; + sqlite3OsDelete(pVfs, zMaster, 0); break; } - sqlite3DbFree(db, zMaster); sqlite3_randomness(sizeof(iRandom), &iRandom); - zMaster = sqlite3MPrintf(db, "%s-mj%06X9%02X", zMainFile, + sqlite3_snprintf(13, &zMaster[nMainFile], "-mj%06X9%02X", (iRandom>>8)&0xffffff, iRandom&0xff); - if( !zMaster ){ - return SQLITE_NOMEM; - } /* The antipenultimate character of the master journal name must ** be "9" to avoid name collisions when using 8+3 filenames. */ - assert( zMaster[strlen(zMaster)-3]=='9' ); + assert( zMaster[sqlite3Strlen30(zMaster)-3]=='9' ); sqlite3FileSuffix3(zMainFile, zMaster); rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); }while( rc==SQLITE_OK && res ); From 38deeb9763133eea8ffbf1d7c1d3843bb8f2fc9e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 Dec 2011 01:30:15 +0000 Subject: [PATCH 07/10] Call sqlite3_log() with an appropriate message if unable to find a unique master-journal filename. FossilOrigin-Name: e9177f7d4ec19f0841d0613990d2fb5725700ba1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 322d293528..0d3df0583b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\sbetter\ssolution\sto\sbeing\sunable\sto\sfind\sa\sunique\smaster-journal\sfilename:\njust\sdelete\san\sexisting\smaster-journal\sand\sreuse\sit. -D 2011-12-16T01:21:31.882 +C Call\ssqlite3_log()\swith\san\sappropriate\smessage\sif\sunable\sto\sfind\sa\sunique\nmaster-journal\sfilename. +D 2011-12-16T01:30:15.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -243,7 +243,7 @@ F src/vdbe.c a7ab9993ec5a4d9479dc99671faec061fbf9b889 F src/vdbe.h f0725ee997db869ecae5bb70a71612aabeca7755 F src/vdbeInt.h 9498fc98a2c9e349a4ef13455ff5a3e898f40176 F src/vdbeapi.c 4dbba7f94f127f6ea8d2d0505ee1f98e5ffbf546 -F src/vdbeaux.c 7bae4d94d7e41e8005824a6d1da54b73d47f52b1 +F src/vdbeaux.c 52ebf2a62d6b66ded536d0a2745f2236771097c7 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c 2fc78b3e0fabcc1eaa23cd79dd2e30e6dcfe1e56 F src/vdbesort.c 468d43c057063e54da4f1988b38b4f46d60e7790 @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 34a0483605d36e6cf03065ed0df33fb1f7c8a272 -R dac1ae186d64bf5089bb1df4a91a9793 +P 2685c2b949061f18bf6a4940eac8c8148873abb6 +R 45a748a7c5e87682cd34056f04633da7 U drh -Z cdecba57f424e5a33e7e2e3955b5bd95 +Z 8c7f37ab067ed94fde60690f325712e2 diff --git a/manifest.uuid b/manifest.uuid index 6fdf08c334..57de583d49 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2685c2b949061f18bf6a4940eac8c8148873abb6 \ No newline at end of file +e9177f7d4ec19f0841d0613990d2fb5725700ba1 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 6e183d0b8c..bb1844dfb4 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1834,6 +1834,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ do { u32 iRandom; if( retryCount++>100 ){ + sqlite3_log(SQLITE_FULL, "cannot find unique master-journal"); sqlite3OsDelete(pVfs, zMaster, 0); break; } From e7d9f13d99667f89606a617eb9ffc01eb0fc8649 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 Dec 2011 04:57:32 +0000 Subject: [PATCH 08/10] The xTruncate method for the multiplexor now changes surplus overflow files to zero-length by default. Or if the "truncate" query parameter is used, it actually deletes the surplus overflow files. This allows VACUUM to shrink the database again. FossilOrigin-Name: 5aefef04d6cc3b734c3f05e036b3869a52248f4d --- manifest | 12 ++-- manifest.uuid | 2 +- src/test_multiplex.c | 168 ++++++++++++++++++++++++------------------- 3 files changed, 103 insertions(+), 79 deletions(-) diff --git a/manifest b/manifest index 0d3df0583b..a12b10d33a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Call\ssqlite3_log()\swith\san\sappropriate\smessage\sif\sunable\sto\sfind\sa\sunique\nmaster-journal\sfilename. -D 2011-12-16T01:30:15.388 +C The\sxTruncate\smethod\sfor\sthe\smultiplexor\snow\schanges\ssurplus\soverflow\nfiles\sto\szero-length\sby\sdefault.\s\sOr\sif\sthe\s"truncate"\squery\sparameter\s\nis\sused,\sit\sactually\sdeletes\sthe\ssurplus\soverflow\sfiles.\s\sThis\sallows\nVACUUM\sto\sshrink\sthe\sdatabase\sagain. +D 2011-12-16T04:57:32.390 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 8d416f29ad8573f32601f6056c9d2b17472e9ad5 -F src/test_multiplex.c c87d27283e2f578c17a8db7a8cac095f576f4323 +F src/test_multiplex.c 993f930c9c8e1aa0f4fe7b5a74c9a60fbf3447e4 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 2685c2b949061f18bf6a4940eac8c8148873abb6 -R 45a748a7c5e87682cd34056f04633da7 +P e9177f7d4ec19f0841d0613990d2fb5725700ba1 +R c09d355f93cdab856d4274295151320d U drh -Z 8c7f37ab067ed94fde60690f325712e2 +Z 4efbc5088c9568c2b7090122e09b39c8 diff --git a/manifest.uuid b/manifest.uuid index 57de583d49..4852f04967 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9177f7d4ec19f0841d0613990d2fb5725700ba1 \ No newline at end of file +5aefef04d6cc3b734c3f05e036b3869a52248f4d \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index d844604ab5..f01a7f585b 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -133,7 +133,8 @@ struct multiplexGroup { int nName; /* Length of base filename */ int flags; /* Flags used for original opening */ unsigned int szChunk; /* Chunk size used for this group */ - int bEnabled; /* TRUE to use Multiplex VFS for this file */ + unsigned char bEnabled; /* TRUE to use Multiplex VFS for this file */ + unsigned char bTruncate; /* TRUE to enable truncation of databases */ multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ }; @@ -276,12 +277,17 @@ static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){ /* Translate an sqlite3_file* that is really a multiplexGroup* into ** the sqlite3_file* for the underlying original VFS. +** +** For chunk 0, the pGroup->flags determines whether or not a new file +** is created if it does not already exist. For chunks 1 and higher, the +** file is created only if createFlag is 1. */ static sqlite3_file *multiplexSubOpen( - multiplexGroup *pGroup, - int iChunk, - int *rc, - int *pOutFlags + multiplexGroup *pGroup, /* The multiplexor group */ + int iChunk, /* Which chunk to open. 0==original file */ + int *rc, /* Result code in and out */ + int *pOutFlags, /* Output flags */ + int createFlag /* True to create if iChunk>0 */ ){ sqlite3_file *pSubOpen = 0; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ @@ -299,15 +305,28 @@ static sqlite3_file *multiplexSubOpen( *rc = multiplexSubFilename(pGroup, iChunk); if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){ + int flags, rc2, bExists; + if( iChunk==0 ) createFlag = (pGroup->flags & SQLITE_OPEN_CREATE)!=0; + flags = pGroup->flags; + if( createFlag ){ + flags |= SQLITE_OPEN_CREATE; + }else if( pGroup->aReal[iChunk].z==0 ){ + return 0; + }else{ + *rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[iChunk].z, + SQLITE_ACCESS_EXISTS, &bExists); + if( *rc || !bExists ) return 0; + flags &= ~SQLITE_OPEN_CREATE; + } pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile ); if( pSubOpen==0 ){ *rc = SQLITE_NOMEM; return 0; } pGroup->aReal[iChunk].p = pSubOpen; - *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, - pGroup->flags, pOutFlags); - if( *rc!=SQLITE_OK ){ + rc2 = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, + flags, pOutFlags); + if( rc2!=SQLITE_OK ){ sqlite3_free(pSubOpen); pGroup->aReal[iChunk].p = 0; return 0; @@ -316,6 +335,25 @@ static sqlite3_file *multiplexSubOpen( return pSubOpen; } +/* +** Return the size, in bytes, of chunk number iChunk. If that chunk +** does not exist, then return 0. This function does not distingish between +** non-existant files and zero-length files. +*/ +static sqlite3_int64 multiplexSubSize( + multiplexGroup *pGroup, /* The multiplexor group */ + int iChunk, /* Which chunk to open. 0==original file */ + int *rc /* Result code in and out */ +){ + sqlite3_file *pSub; + sqlite3_int64 sz = 0; + + pSub = multiplexSubOpen(pGroup, iChunk, rc, NULL, 0); + if( pSub==0 ) return 0; + *rc = pSub->pMethods->xFileSize(pSub, &sz); + return sz; +} + /* ** This is the implementation of the multiplex_control() SQL function. */ @@ -455,6 +493,7 @@ static int multiplexOpen( memset(pGroup, 0, sz); pMultiplexOpen->pGroup = pGroup; pGroup->bEnabled = -1; + pGroup->bTruncate = (flags & SQLITE_OPEN_MAIN_DB)==0; pGroup->szChunk = SQLITE_MULTIPLEX_CHUNK_SIZE; if( zName ){ @@ -475,6 +514,7 @@ static int multiplexOpen( pGroup->bEnabled = 0; } } + if( sqlite3_uri_parameter(zName, "truncate") ) pGroup->bTruncate = 1; } pGroup->zName = p; memcpy(pGroup->zName, zName, nName+1); @@ -493,8 +533,8 @@ static int multiplexOpen( pGroup->flags = flags; rc = multiplexSubFilename(pGroup, 1); if( rc==SQLITE_OK ){ - pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags); - assert( pSubOpen || rc!=SQLITE_OK ); + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags, 0); + if( pSubOpen==0 && rc==SQLITE_OK ) rc = SQLITE_CANTOPEN; } if( rc==SQLITE_OK ){ sqlite3_int64 sz; @@ -522,17 +562,18 @@ static int multiplexOpen( } }else{ /* If the first overflow file exists and if the size of the main file - ** is different from the chunk size, that means the chunk size is set - ** set incorrectly. So fix it. - ** - ** Or, if the first overflow file does not exist and the main file is - ** larger than the chunk size, that means the chunk size is too small. - ** But we have no way of determining the intended chunk size, so - ** just disable the multiplexor all togethre. - */ + ** is different from the chunk size, that means the chunk size is set + ** set incorrectly. So fix it. + ** + ** Or, if the first overflow file does not exist and the main file is + ** larger than the chunk size, that means the chunk size is too small. + ** But we have no way of determining the intended chunk size, so + ** just disable the multiplexor all togethre. + */ rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, SQLITE_ACCESS_EXISTS, &bExists); - if( rc==SQLITE_OK && bExists && sz==(sz&0xffff0000) && sz>0 + bExists = multiplexSubSize(pGroup, 1, &rc)>0; + if( rc==SQLITE_OK && bExists && sz==(sz&0xffff0000) && sz>0 && sz!=pGroup->szChunk ){ pGroup->szChunk = sz; }else if( rc==SQLITE_OK && !bExists && sz>pGroup->szChunk ){ @@ -674,7 +715,7 @@ static int multiplexRead( int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_READ; }else{ @@ -683,7 +724,7 @@ static int multiplexRead( }else{ while( iAmt > 0 ){ int i = (int)(iOfst / pGroup->szChunk); - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL, 1); if( pSubOpen ){ int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk; if( extra<0 ) extra = 0; @@ -719,7 +760,7 @@ static int multiplexWrite( int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_WRITE; }else{ @@ -728,7 +769,7 @@ static int multiplexWrite( }else{ while( rc==SQLITE_OK && iAmt>0 ){ int i = (int)(iOfst / pGroup->szChunk); - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL, 1); if( pSubOpen ){ int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk; @@ -756,28 +797,35 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_TRUNCATE; }else{ rc = pSubOpen->pMethods->xTruncate(pSubOpen, size); } - }else if( (pGroup->flags & SQLITE_OPEN_MAIN_DB)==0 ){ - int rc2; + }else{ int i; + int iBaseGroup = (int)(size / pGroup->szChunk); sqlite3_file *pSubOpen; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ /* delete the chunks above the truncate limit */ - for(i=(int)(size / pGroup->szChunk)+1; inReal; i++){ - multiplexSubClose(pGroup, i, pOrigVfs); + for(i = pGroup->nReal-1; i>iBaseGroup && rc==SQLITE_OK; i--){ + if( pGroup->bTruncate ){ + multiplexSubClose(pGroup, i, pOrigVfs); + }else{ + pSubOpen = multiplexSubOpen(pGroup, i, &rc, 0, 0); + if( pSubOpen ){ + rc = pSubOpen->pMethods->xTruncate(pSubOpen, 0); + } + } } - pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->szChunk), &rc2,0); - if( pSubOpen ){ - rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk); - if( rc2!=SQLITE_OK ) rc = rc2; - }else{ - rc = SQLITE_IOERR_TRUNCATE; + if( rc==SQLITE_OK ){ + pSubOpen = multiplexSubOpen(pGroup, iBaseGroup, &rc, 0, 0); + if( pSubOpen ){ + rc = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk); + } } + if( rc ) rc = SQLITE_IOERR_TRUNCATE; } multiplexLeave(); return rc; @@ -812,42 +860,18 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ int i; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_FSTAT; }else{ rc = pSubOpen->pMethods->xFileSize(pSubOpen, pSize); } }else{ - sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; *pSize = 0; for(i=0; rc==SQLITE_OK; i++){ - sqlite3_file *pSubOpen = 0; - int exists = 0; - rc = multiplexSubFilename(pGroup, i); - if( rc!=SQLITE_OK ) break; - if( pGroup->nReal>i && pGroup->aReal[i].p!=0 ){ - exists = 1; - }else if( (pGroup->flags & SQLITE_OPEN_DELETEONCLOSE)==0 ){ - const char *zReal = pGroup->aReal[i].z; - rc = pOrigVfs->xAccess(pOrigVfs, zReal, SQLITE_ACCESS_EXISTS, &exists); - } - if( exists==0 ){ - /* stop at first "gap" or IO error. */ - break; - } - if( rc==SQLITE_OK ){ - pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); - } - assert( pSubOpen || rc!=SQLITE_OK ); - if( rc==SQLITE_OK ){ - sqlite3_int64 sz = 0; - rc = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); - if( rc==SQLITE_OK && sz>pGroup->szChunk ){ - rc = SQLITE_IOERR_FSTAT; - } - *pSize = i*(sqlite3_int64)pGroup->szChunk + sz; - } + sqlite3_int64 sz = multiplexSubSize(pGroup, i, &rc); + if( sz==0 ) break; + *pSize = i*(sqlite3_int64)pGroup->szChunk + sz; } } multiplexLeave(); @@ -859,7 +883,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ static int multiplexLock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xLock(pSubOpen, lock); } @@ -871,7 +895,7 @@ static int multiplexLock(sqlite3_file *pConn, int lock){ static int multiplexUnlock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xUnlock(pSubOpen, lock); } @@ -883,7 +907,7 @@ static int multiplexUnlock(sqlite3_file *pConn, int lock){ static int multiplexCheckReservedLock(sqlite3_file *pConn, int *pResOut){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xCheckReservedLock(pSubOpen, pResOut); } @@ -931,7 +955,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ rc = SQLITE_OK; break; default: - pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg); if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ @@ -948,7 +972,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ static int multiplexSectorSize(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen && pSubOpen->pMethods->xSectorSize ){ return pSubOpen->pMethods->xSectorSize(pSubOpen); } @@ -960,7 +984,7 @@ static int multiplexSectorSize(sqlite3_file *pConn){ static int multiplexDeviceCharacteristics(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xDeviceCharacteristics(pSubOpen); } @@ -978,7 +1002,7 @@ static int multiplexShmMap( ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend,pp); } @@ -995,7 +1019,7 @@ static int multiplexShmLock( ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xShmLock(pSubOpen, ofst, n, flags); } @@ -1007,7 +1031,7 @@ static int multiplexShmLock( static void multiplexShmBarrier(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ pSubOpen->pMethods->xShmBarrier(pSubOpen); } @@ -1018,7 +1042,7 @@ static void multiplexShmBarrier(sqlite3_file *pConn){ static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xShmUnmap(pSubOpen, deleteFlag); } From 06999667980faaf686d7232e7d97bf1cf4907849 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 Dec 2011 05:09:46 +0000 Subject: [PATCH 09/10] Fix a potential infinite loop (inserted by the previous check-in) on the multiplexWrite() method of the multiplexor. FossilOrigin-Name: 022bf427c2edfe494ec7c222f436953ff56574a6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_multiplex.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a12b10d33a..88df5c4fc4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sxTruncate\smethod\sfor\sthe\smultiplexor\snow\schanges\ssurplus\soverflow\nfiles\sto\szero-length\sby\sdefault.\s\sOr\sif\sthe\s"truncate"\squery\sparameter\s\nis\sused,\sit\sactually\sdeletes\sthe\ssurplus\soverflow\sfiles.\s\sThis\sallows\nVACUUM\sto\sshrink\sthe\sdatabase\sagain. -D 2011-12-16T04:57:32.390 +C Fix\sa\spotential\sinfinite\sloop\s(inserted\sby\sthe\sprevious\scheck-in)\non\sthe\smultiplexWrite()\smethod\sof\sthe\smultiplexor. +D 2011-12-16T05:09:46.415 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 8d416f29ad8573f32601f6056c9d2b17472e9ad5 -F src/test_multiplex.c 993f930c9c8e1aa0f4fe7b5a74c9a60fbf3447e4 +F src/test_multiplex.c ccc5f2feae52c4fbef1a88205667a6bcf2078844 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P e9177f7d4ec19f0841d0613990d2fb5725700ba1 -R c09d355f93cdab856d4274295151320d +P 5aefef04d6cc3b734c3f05e036b3869a52248f4d +R 8d7b158bf3a519193898c5ead6ba7790 U drh -Z 4efbc5088c9568c2b7090122e09b39c8 +Z d8c8d1df41588adf9be8799f58ddd721 diff --git a/manifest.uuid b/manifest.uuid index 4852f04967..8cbc3f560c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5aefef04d6cc3b734c3f05e036b3869a52248f4d \ No newline at end of file +022bf427c2edfe494ec7c222f436953ff56574a6 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index f01a7f585b..47d553cc27 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -305,7 +305,7 @@ static sqlite3_file *multiplexSubOpen( *rc = multiplexSubFilename(pGroup, iChunk); if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){ - int flags, rc2, bExists; + int flags, bExists; if( iChunk==0 ) createFlag = (pGroup->flags & SQLITE_OPEN_CREATE)!=0; flags = pGroup->flags; if( createFlag ){ @@ -324,9 +324,9 @@ static sqlite3_file *multiplexSubOpen( return 0; } pGroup->aReal[iChunk].p = pSubOpen; - rc2 = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, + *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, flags, pOutFlags); - if( rc2!=SQLITE_OK ){ + if( (*rc)!=SQLITE_OK ){ sqlite3_free(pSubOpen); pGroup->aReal[iChunk].p = 0; return 0; From 8c24a369a5e5efab956a7dac56083ecd0fdd4e6b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 Dec 2011 05:50:39 +0000 Subject: [PATCH 10/10] Tweaks to the way multiplexSubOpen() works, for backwards compatibility. FossilOrigin-Name: bb40338887c912be70cb6fe3b760d87c14bb88a5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_multiplex.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 88df5c4fc4..1257f0f4ca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sinfinite\sloop\s(inserted\sby\sthe\sprevious\scheck-in)\non\sthe\smultiplexWrite()\smethod\sof\sthe\smultiplexor. -D 2011-12-16T05:09:46.415 +C Tweaks\sto\sthe\sway\smultiplexSubOpen()\sworks,\sfor\sbackwards\scompatibility. +D 2011-12-16T05:50:39.331 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 8d416f29ad8573f32601f6056c9d2b17472e9ad5 -F src/test_multiplex.c ccc5f2feae52c4fbef1a88205667a6bcf2078844 +F src/test_multiplex.c 6e07b94e2fe430f7f4f0d7d67b5e58f504dea655 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 5aefef04d6cc3b734c3f05e036b3869a52248f4d -R 8d7b158bf3a519193898c5ead6ba7790 +P 022bf427c2edfe494ec7c222f436953ff56574a6 +R 29ea54dcd671c93952fcbf93e86a1060 U drh -Z d8c8d1df41588adf9be8799f58ddd721 +Z b9fbbc1fe20301c015fc0588c12a3621 diff --git a/manifest.uuid b/manifest.uuid index 8cbc3f560c..6edf6058c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -022bf427c2edfe494ec7c222f436953ff56574a6 \ No newline at end of file +bb40338887c912be70cb6fe3b760d87c14bb88a5 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 47d553cc27..25bfcdddd2 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -306,10 +306,12 @@ static sqlite3_file *multiplexSubOpen( *rc = multiplexSubFilename(pGroup, iChunk); if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){ int flags, bExists; - if( iChunk==0 ) createFlag = (pGroup->flags & SQLITE_OPEN_CREATE)!=0; + createFlag = (pGroup->flags & SQLITE_OPEN_CREATE)!=0; flags = pGroup->flags; if( createFlag ){ flags |= SQLITE_OPEN_CREATE; + }else if( iChunk==0 ){ + /* Fall through */ }else if( pGroup->aReal[iChunk].z==0 ){ return 0; }else{ @@ -320,7 +322,7 @@ static sqlite3_file *multiplexSubOpen( } pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile ); if( pSubOpen==0 ){ - *rc = SQLITE_NOMEM; + *rc = SQLITE_IOERR_NOMEM; return 0; } pGroup->aReal[iChunk].p = pSubOpen;