From 050d09abdca959e7afe9c810f1e51ab7d62a8cd5 Mon Sep 17 00:00:00 2001 From: shaneh Date: Mon, 8 Nov 2010 19:16:16 +0000 Subject: [PATCH] Changes to the multiplex VFS to optionally (compiler define) allow the "chunk extension" to overwrite the right-most chars of the filename instead of simply being appended. FossilOrigin-Name: 07da0a0beffda324d28fd2768c542ff69d4dbff2 --- manifest | 16 +++---- manifest.uuid | 2 +- src/test_config.c | 6 +++ src/test_multiplex.c | 110 +++++++++++++++++++++++++++++++++---------- test/multiplex.test | 48 ++++++++++++------- 5 files changed, 131 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index fdf2f98586..02f35da05d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sto\sxTruncate\sand\smore\sjournal\smode\stests\sfor\sthe\smultiplex\sVFS. -D 2010-11-05T20:50:44 +C Changes\sto\sthe\smultiplex\sVFS\sto\soptionally\s(compiler\sdefine)\sallow\s\nthe\s"chunk\sextension"\sto\soverwrite\sthe\sright-most\schars\sof\sthe\sfilename\s\ninstead\sof\ssimply\sbeing\sappended. +D 2010-11-08T19:16:16 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -196,7 +196,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 -F src/test_config.c 5a11c51af2156e2d07186930b36f2b8239a4393f +F src/test_config.c 55918873f0a2ebbd5b52417319987f9f99c401fa F src/test_demovfs.c 0aed671636735116fc872c5b03706fd5612488b5 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 13b582345fb1185a93e46c53310fae8547dcce20 @@ -207,7 +207,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 785edd54f963aefb3c1628124170a56697c68c70 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 09a88f0c111201dc4f8c20470aa1b5f611d59200 -F src/test_multiplex.c 479a51f7eec4b05b7e4b2ce43c4b4ba9b53a03ca +F src/test_multiplex.c 5c2b53d620613b46e4cfdde66f82612e896ac7b9 F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3 F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec F src/test_osinst.c f408c6a181f2fb04c56273afd5c3e1e82f60392c @@ -559,7 +559,7 @@ F test/misc5.test 45b2e3ed5f79af2b4f38ae362eaf4c49674575bd F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test c5f4e6a82e04e71820c0f9f64f6733f04c8ae0ae F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 -F test/multiplex.test bbd039c94e7be031ce890e302b7a266bcbe0f56f +F test/multiplex.test 6b19e95d572363f21d3e67bc984ec721a7a7158a F test/mutex1.test 5b71777fc127509cd257910c8db799de557a02de F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test a44e04df1486fcfb02d32468cbcd3c8e1e433723 @@ -885,7 +885,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 72ba3e368bec34532ec7b5e856a4daa7e1c8cccb -R 0042c6a8b282a3f9ef09bfa98caed3ba +P 65fa1164f035d270db48db6474da888aacfba3bd +R b100d3e162d12518a1c689502bc517e4 U shaneh -Z 7b8161cae1f9c054697e43cb6ccb5f60 +Z 5f41c44eac49aaf452bba5b19211e7d7 diff --git a/manifest.uuid b/manifest.uuid index cc3e0f4a7b..04db900ece 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65fa1164f035d270db48db6474da888aacfba3bd \ No newline at end of file +07da0a0beffda324d28fd2768c542ff69d4dbff2 \ No newline at end of file diff --git a/src/test_config.c b/src/test_config.c index 7ada13f4f6..12df78709a 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -523,6 +523,12 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", Tcl_SetVar2(interp, "sqlite_options", "secure_delete", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_MULTIPLEX_EXT_OVWR + Tcl_SetVar2(interp, "sqlite_options", "multiplex_ext_overwrite", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "multiplex_ext_overwrite", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef YYTRACKMAXSTACKDEPTH Tcl_SetVar2(interp, "sqlite_options", "yytrackmaxstackdepth", "1", TCL_GLOBAL_ONLY); #else diff --git a/src/test_multiplex.c b/src/test_multiplex.c index c7e87d46ce..442b39e0d8 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -26,9 +26,27 @@ /************************ Shim Definitions ******************************/ +/* This is the limit on the chunk size. It may be changed by calling +** the sqlite3_multiplex_set() interface. +*/ #define SQLITE_MULTIPLEX_CHUNK_SIZE 0x40000000 +/* Default limit on number of chunks. Care should be taken +** so that values for chunks numbers fit in the SQLITE_MULTIPLEX_EXT_FMT +** format specifier. It may be changed by calling +** the sqlite3_multiplex_set() interface. +*/ #define SQLITE_MULTIPLEX_MAX_CHUNKS 32 -#define SQLITE_MULTIPLEX_EXT_FMT "-%04d" + +/* If SQLITE_MULTIPLEX_EXT_OVWR is defined, the +** last SQLITE_MULTIPLEX_EXT_SZ characters of the +** filename will be overwritten, otherwise, the +** multiplex extension is simply appended to the filename. +** Ex. (undefined) test.db -> test.db01 +** (defined) test.db -> test.01 +** Chunk 0 does not have a modified extension. +*/ +#define SQLITE_MULTIPLEX_EXT_FMT "%02d" +#define SQLITE_MULTIPLEX_EXT_SZ 2 /************************ Object Definitions ******************************/ @@ -45,11 +63,11 @@ typedef struct multiplexConn multiplexConn; ** group. */ struct multiplexGroup { - sqlite3_file *pReal[SQLITE_MULTIPLEX_MAX_CHUNKS]; /* Handles to each chunk */ - char bOpen[SQLITE_MULTIPLEX_MAX_CHUNKS]; /* 0 if chunk not opened */ - char *zName; /* Base filename of this group */ - int nName; /* Length of base filename */ - int flags; /* Flags used for original opening */ + sqlite3_file **pReal; /* Handles to each chunk */ + char *bOpen; /* 0 if chunk not opened */ + char *zName; /* Base filename of this group */ + int nName; /* Length of base filename */ + int flags; /* Flags used for original opening */ multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ }; @@ -137,9 +155,15 @@ static sqlite3_file *multiplexSubOpen(multiplexConn *pConn, int iChunk, int *rc, if( iChunkpReal[iChunk]; /* Real file descriptor */ if( !pGroup->bOpen[iChunk] ){ - pGroup->zName[pGroup->nName] = '\0'; - if( iChunk ) sqlite3_snprintf(pGroup->nName+6, pGroup->zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, iChunk); - *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->zName, pSubOpen, pGroup->flags, pOutFlags); + memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); + if( iChunk ){ +#ifdef SQLITE_MULTIPLEX_EXT_OVWR + sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, iChunk); +#else + sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, iChunk); +#endif + } + *rc = pOrigVfs->xOpen(pOrigVfs, gMultiplex.zName, pSubOpen, pGroup->flags, pOutFlags); if( *rc==SQLITE_OK ){ pGroup->bOpen[iChunk] = -1; return pSubOpen; @@ -176,6 +200,7 @@ static int multiplexOpen( sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ int nName = sqlite3Strlen30(zName); int i; + int sz; UNUSED_PARAMETER(pVfs); @@ -184,20 +209,37 @@ static int multiplexOpen( */ multiplexEnter(); pMultiplexOpen = (multiplexConn*)pConn; - /* allocate space for group, file handles, - ** and file name (+ extra for "-0000\0") - */ - pGroup = sqlite3_malloc( sizeof(multiplexGroup) + (pOrigVfs->szOsFile*gMultiplex.nMaxChunks) + nName + 6 ); + /* allocate space for group */ + sz = sizeof(multiplexGroup) /* multiplexGroup */ + + (sizeof(sqlite3_file *)*gMultiplex.nMaxChunks) /* pReal[] */ + + (pOrigVfs->szOsFile*gMultiplex.nMaxChunks) /* *pReal */ + + gMultiplex.nMaxChunks /* bOpen[] */ + + nName + 1; /* zName */ +#ifndef SQLITE_MULTIPLEX_EXT_OVWR + sz += SQLITE_MULTIPLEX_EXT_SZ; + assert(nName+SQLITE_MULTIPLEX_EXT_SZ < pOrigVfs->mxPathname); +#else + assert(nName >= SQLITE_MULTIPLEX_EXT_SZ); + assert(nName < pOrigVfs->mxPathname); +#endif + pGroup = sqlite3_malloc( sz ); if( pGroup==0 ){ rc=SQLITE_NOMEM; }else{ + /* assign pointers to extra space allocated */ + char *p = (char *)&pGroup[1]; pMultiplexOpen->pGroup = pGroup; - memset(pGroup, 0, sizeof(multiplexGroup) + (pOrigVfs->szOsFile*gMultiplex.nMaxChunks) + nName + 6); - /* assign pointers to extra space for file handles */ + memset(pGroup, 0, sz); + pGroup->pReal = (sqlite3_file **)p; + p += (sizeof(sqlite3_file *)*gMultiplex.nMaxChunks); for(i=0; ipReal[i] = (sqlite3_file *)((char *)&pGroup[1] + (pOrigVfs->szOsFile*i)); + pGroup->pReal[i] = (sqlite3_file *)p; + p += pOrigVfs->szOsFile; } - pGroup->zName = (char *)&pGroup[1] + (pOrigVfs->szOsFile*gMultiplex.nMaxChunks); + pGroup->bOpen = p; + p += gMultiplex.nMaxChunks; + pGroup->zName = p; + /* save off base filename, name length, and original open flags */ memcpy(pGroup->zName, zName, nName+1); pGroup->nName = nName; pGroup->flags = flags; @@ -223,7 +265,7 @@ static int multiplexOpen( /* ** This is the xDelete method used for the "multiplex" VFS. ** It attempts to delete the filename specified, as well -** as addiitional files with the "-####" extension. +** as additional files with the SQLITE_MULTIPLEX_EXT_FMT extension. */ static int multiplexDelete( sqlite3_vfs *pVfs, /* The multiplex VFS */ @@ -242,7 +284,13 @@ static int multiplexDelete( for(i=0; ixAccess(pOrigVfs, gMultiplex.zName, SQLITE_ACCESS_EXISTS, &exists); if( rc2==SQLITE_OK && exists){ /* if it exists, delete it */ @@ -372,6 +420,7 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ sqlite3_file *pSubOpen; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ multiplexEnter(); + memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); /* delete the chunks above the truncate limit */ for(i=(int)(size/gMultiplex.nChunkSize)+1; ibOpen[i] = 0; } - sqlite3_snprintf(pGroup->nName+6, pGroup->zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, i); - rc2 = pOrigVfs->xDelete(pOrigVfs, pGroup->zName, 0); +#ifdef SQLITE_MULTIPLEX_EXT_OVWR + sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, i); +#else + sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, i); +#endif + rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, 0); if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; } pSubOpen = multiplexSubOpen(p, (int)(size/gMultiplex.nChunkSize), &rc2, NULL); @@ -435,9 +488,15 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ }else{ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ int exists = 0; - pGroup->zName[pGroup->nName] = '\0'; - if( i ) sqlite3_snprintf(pGroup->nName+6, pGroup->zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, i); - rc2 = pOrigVfs->xAccess(pOrigVfs, pGroup->zName, SQLITE_ACCESS_EXISTS, &exists); + memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); + if( i ){ +#ifdef SQLITE_MULTIPLEX_EXT_OVWR + sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, i); +#else + sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, i); +#endif + } + rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, SQLITE_ACCESS_EXISTS, &exists); if( rc2==SQLITE_OK && exists){ /* if it exists, open it */ pSubOpen = multiplexSubOpen(p, i, &rc, NULL); @@ -453,6 +512,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ rc = rc2; }else{ *pSize += sz; + assert(sz<=gMultiplex.nChunkSize); } }else{ break; @@ -687,7 +747,7 @@ int sqlite3_multiplex_set( if( gMultiplex.pGroups ) return SQLITE_MISUSE; if( nChunkSize<32 ) return SQLITE_MISUSE; if( nMaxChunks<1 ) return SQLITE_MISUSE; - if( nMaxChunks>SQLITE_MULTIPLEX_MAX_CHUNKS ) return SQLITE_MISUSE; + if( nMaxChunks>99 ) return SQLITE_MISUSE; multiplexEnter(); gMultiplex.nChunkSize = nChunkSize; gMultiplex.nMaxChunks = nMaxChunks; diff --git a/test/multiplex.test b/test/multiplex.test index dc245e8527..2adbfcb693 100644 --- a/test/multiplex.test +++ b/test/multiplex.test @@ -17,6 +17,22 @@ source $testdir/malloc_common.tcl set g_chunk_size 2147483648 set g_max_chunks 32 +# This handles appending the chunk number +# to the end of the filename. if +# SQLITE_MULTIPLEX_EXT_OVWR is defined, then +# it overwrites the last 2 bytes of the +# file name with the chunk number. +proc multiplex_name {name chunk} { + if {$chunk==0} { return $name } + set num [format "%02d" $chunk] + ifcapable {multiplex_ext_overwrite} { + set name [string range $name 0 [expr [string length $name]-2-1]] + } + return $name$num +} + +# This saves off the parameters and calls the +# underlying sqlite3_multiplex_set() API. proc multiplex_set {chunk_size max_chunks} { global g_chunk_size global g_max_chunks @@ -25,16 +41,14 @@ proc multiplex_set {chunk_size max_chunks} { sqlite3_multiplex_set $chunk_size $max_chunks } +# This attempts to delete the base file and +# and files with the chunk extension. proc multiplex_delete {name} { global g_max_chunks - forcedelete $name - forcedelete $name-journal - forcedelete $name-wal - for {set i 1} {$i<$g_max_chunks} {incr i} { - set num [format "%04d" $i] - forcedelete $name-$num - forcedelete $name-journal-$num - forcedelete $name-wal-$num + for {set i 0} {$i<$g_max_chunks} {incr i} { + forcedelete [multiplex_name $name $i] + forcedelete [multiplex_name $name-journal $i] + forcedelete [multiplex_name $name-wal $i] } } @@ -58,7 +72,7 @@ do_test multiplex-1.10.1 { multiplex_set 32768 16 } {SQLITE_OK} do_test multiplex-1.10.2 { multiplex_set 32768 -1 } {SQLITE_MISUSE} do_test multiplex-1.10.3 { multiplex_set -1 16 } {SQLITE_MISUSE} do_test multiplex-1.10.4 { multiplex_set 31 16 } {SQLITE_MISUSE} -do_test multiplex-1.10.5 { multiplex_set 32768 33 } {SQLITE_MISUSE} +do_test multiplex-1.10.5 { multiplex_set 32768 100 } {SQLITE_MISUSE} do_test multiplex-1.11 { sqlite3_multiplex_shutdown } {SQLITE_OK} @@ -98,7 +112,7 @@ do_test multiplex-2.1.2 { INSERT INTO t1 VALUES(2, randomblob(1100)); } } {} -do_test multiplex-2.1.3 { file size test.db } {4096} +do_test multiplex-2.1.3 { file size [multiplex_name test.db 0] } {4096} do_test multiplex-2.1.4 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} @@ -106,7 +120,7 @@ do_test multiplex-2.1.4 { do_test multiplex-2.2.1 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} -do_test multiplex-2.2.3 { file size test.db } {6144} +do_test multiplex-2.2.3 { file size [multiplex_name test.db 0] } {6144} do_test multiplex-2.3.1 { sqlite3 db2 bak.db @@ -119,7 +133,7 @@ do_test multiplex-2.4.1 { do_test multiplex-2.4.2 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} -do_test multiplex-2.4.4 { file size test.db } {7168} +do_test multiplex-2.4.4 { file size [multiplex_name test.db 0] } {7168} do_test multiplex-2.4.99 { db close sqlite3_multiplex_shutdown @@ -172,8 +186,8 @@ do_test multiplex-2.5.8 { db eval {SELECT a,length(b) FROM t1 WHERE a=4} } {4 4000} -do_test multiplex-2.5.9 { file size test.db } [list $g_chunk_size] -do_test multiplex-2.5.10 { file size test.db-0001 } [list $g_chunk_size] +do_test multiplex-2.5.9 { file size [multiplex_name test.db 0] } [list $g_chunk_size] +do_test multiplex-2.5.10 { file size [multiplex_name test.db 1] } [list $g_chunk_size] do_test multiplex-2.5.99 { db close @@ -216,7 +230,7 @@ foreach jmode $all_journal_modes { db eval {SELECT length(b) FROM t1 WHERE a=2} } [list $g_chunk_size] - do_test multiplex-2.6.6.$sz.$jmode { file size test.db } [list $g_chunk_size] + do_test multiplex-2.6.6.$sz.$jmode { file size [multiplex_name test.db 0] } [list $g_chunk_size] do_test multiplex-2.6.99.$sz.$jmode { db close @@ -249,7 +263,7 @@ do_test multiplex-3.1.2 { CREATE TABLE t1(a PRIMARY KEY, b); INSERT INTO t1 VALUES(1, 'one'); } - file size test.db + file size [multiplex_name test.db 0] } {3072} do_test multiplex-3.1.3 { sqlite3 db2 test.db @@ -283,7 +297,7 @@ do_test multiplex-3.2.1a { } $db } - list [file size test.db] [file size test2.db] + list [file size [multiplex_name test.db 0]] [file size [multiplex_name test2.db 0]] } {2048 2048} do_test multiplex-3.2.1b {