diff --git a/Makefile.in b/Makefile.in index e3dfb01baa..0b326ebd59 100644 --- a/Makefile.in +++ b/Makefile.in @@ -180,7 +180,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ func.lo global.lo hash.lo \ icu.lo insert.lo json1.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ - memjournal.lo \ + memdb.lo memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ @@ -240,6 +240,7 @@ SRC = \ $(TOP)/src/mem2.c \ $(TOP)/src/mem3.c \ $(TOP)/src/mem5.c \ + $(TOP)/src/memdb.c \ $(TOP)/src/memjournal.c \ $(TOP)/src/msvc.h \ $(TOP)/src/mutex.c \ @@ -824,6 +825,9 @@ mem3.lo: $(TOP)/src/mem3.c $(HDR) mem5.lo: $(TOP)/src/mem5.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem5.c +memdb.lo: $(TOP)/src/memdb.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memdb.c + memjournal.lo: $(TOP)/src/memjournal.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memjournal.c diff --git a/Makefile.msc b/Makefile.msc index 2f038ab1c1..35eb97261d 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1101,7 +1101,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ func.lo global.lo hash.lo \ icu.lo insert.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ - memjournal.lo \ + memdb.lo memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ @@ -1174,6 +1174,7 @@ SRC00 = \ $(TOP)\src\mem2.c \ $(TOP)\src\mem3.c \ $(TOP)\src\mem5.c \ + $(TOP)\src\memdb.c \ $(TOP)\src\memjournal.c \ $(TOP)\src\mutex.c \ $(TOP)\src\mutex_noop.c \ @@ -1807,6 +1808,9 @@ mem3.lo: $(TOP)\src\mem3.c $(HDR) mem5.lo: $(TOP)\src\mem5.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\mem5.c +memdb.lo: $(TOP)\src\memjournal.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\memdb.c + memjournal.lo: $(TOP)\src\memjournal.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\memjournal.c diff --git a/main.mk b/main.mk index 85588cd01a..4e13c3e608 100644 --- a/main.mk +++ b/main.mk @@ -65,7 +65,7 @@ LIBOBJ+= vdbe.o parse.o \ fts3_write.o fts5.o func.o global.o hash.o \ icu.o insert.o json1.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ - memjournal.o \ + memdb.o memjournal.o \ mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \ notify.o opcodes.o os.o os_unix.o os_win.o \ pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \ @@ -118,6 +118,7 @@ SRC = \ $(TOP)/src/mem2.c \ $(TOP)/src/mem3.c \ $(TOP)/src/mem5.c \ + $(TOP)/src/memdb.c \ $(TOP)/src/memjournal.c \ $(TOP)/src/msvc.h \ $(TOP)/src/mutex.c \ diff --git a/manifest b/manifest index 774549626b..f282ff78d7 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\scode\sindentation.\s\sNo\slogic\schanges. -D 2018-01-02T21:29:42.540 -F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1 +C Add\ssupport\sfor\sthe\s"memdb"\sVFS\sand\sthe\ssqlite3_memdb_ptr()\sand\nsqlite3_memdb_config()\sinterfaces,\sto\senable\san\sSQLite\sdatabase\sto\sbe\nmanipulated\sas\san\sin-memory\sobject. +D 2018-01-03T01:28:46.871 +F Makefile.in 892bf253c48f3d2d8d4e4e89b44b71aa548a0eba11b148c338690cfb99822859 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89 +F Makefile.msc bddc04c05f1ab348be051da17827b123fa004005c9ee47489eaf3a2759dcdc45 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -401,7 +401,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 59744c818d349d170ff56cdbdfb5af0e0a2029db18ce2824fcd1b0a3fa317d84 +F main.mk da996649117a4563f04b046d73ce82237ff544138b252245111703f6b916fcd4 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -443,13 +443,14 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 14686083cedc198540b15a79586cdd4be2acf6d5fa97627e355f817ab07e9fee F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 55bcc3c741059a1056859e8adaf133aa179e22be12215c0936b2f354ef71209b -F src/main.c 690c4134f944cbd5b71d59dd6e61ce4131f6a50ab774f38108e57d07d79cf876 +F src/main.c 37139a23caa1a9864b42b0e4bd6416b0d9387e8a7423ebd0ca70f32d03018f46 F src/malloc.c 6f684fd039f53bf9195193eb0cde731a2954970fabc6ef054ba379b6052dc296 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 +F src/memdb.c 46109bc6c890c21a802b6aad6e667c81aee53e5623e2068c6fa9ded1b9634558 F src/memjournal.c 6f3d36a0a8f72f48f6c3c722f04301ac64f2515435fa42924293e46fc7994661 F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81 F src/mutex.c b021263554c8a3995e9d53193b8194b96d1ed28e06c3b532dd7f7d29cf0c7d53 @@ -465,7 +466,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 7fc2735390a7809d5d893ed735d994ff12521224b89738226fff6f1a0aa1c932 F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 9b9cb4e06c03d43d62480a7a685a012d645fcf3a39e7767ccb505fb41ee083ec +F src/pager.c ede1e65f465f6c507c876b87c3f796a7809b52bd9f82da14bac2680f5df8ebe6 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a F src/parse.y 44cbbc3e132ea128258eff7be7f6d5c5dfa25522f89ec8b5501276966511bd50 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 @@ -480,14 +481,14 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74 F src/shell.c.in a418ddceef7a2789f18bdc2bcdd481b2562fe4a7754b8009c8dd33d5a67da332 -F src/sqlite.h.in c597ba5d11666bb2d0179a173cd25e44f0b0333f9e18ce60b07eac8bdc3de67f +F src/sqlite.h.in 8b6cd7fd8b286d567bcc1ee3a750686cbb1c0962dc453b2cfa034684fe5808db F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 -F src/sqliteInt.h fd8702c65994d7de3e2d8f7d85d958731da1ed29476571fdfa2290fd8ec0bf80 +F src/sqliteInt.h 2567341e37050ad542d57f3027b8717cb9b14ff363fdfeecf85421e0d973787f F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 -F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696 +F src/tclsqlite.c 45bcce558f40f245e25d1dd7c208f6845137fbee042cced2a4116f05349ffd9d F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b @@ -503,7 +504,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 -F src/test_config.c cc8a1d44648d9392a14f4ecfc841d027daaf61f952b9f70792edf11373aaa3dd +F src/test_config.c 6b749332d98924c52d5b08d1fd847bbe8ff049fc4446af738e64e2532a804348 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2 @@ -1055,6 +1056,7 @@ F test/malloc_common.tcl aac62499b76be719fac31e7a3e54a7fd53272e7f F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 +F test/memdb1.test 3d8ebffdc64ea1a7b4ccacd5502e61191233d979451747f4af88c9f2ee9b34c5 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 F test/memsubsys1.test 9e7555a22173b8f1c96c281ce289b338fcba2abe8b157f8798ca195bbf1d347e F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08 @@ -1628,7 +1630,7 @@ F tool/mkshellc.tcl 574307265b49d813301fba91ccd74e6a26d33f65f74b6891c320a0ffbee0 F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb -F tool/mksqlite3c.tcl 1fb69d39166f52d802a70ec37d99bca51d011c8ab30be27bc495be493196ae41 +F tool/mksqlite3c.tcl a03cee30de81a2e67b93e5c659f24113a003677c557daeb008205c8e6d4345d6 F tool/mksqlite3h.tcl f92f994d9709aeb9e2b6e6f9fc8b069d2f55202c8e23f453edc44390a25982dc F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 @@ -1688,7 +1690,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2846458af5d029a8e4fdcc8f50873a44e57897bbfe6aee8a23a01ffc34c5579f -R 9613ba300b7bd9cce413f2672960fb05 +P e115f2583499df0c7ee991d372bed4b89aa717b10a4e4b10977864390cb4fc11 +R dd0ff0dcdd2908eb59154bae6d4e7abd +T *branch * memdb +T *sym-memdb * +T -sym-trunk * U drh -Z 9fca118926b581510cc96e0ce3702687 +Z 4f847deac1204c1b3d7c3decd276393a diff --git a/manifest.uuid b/manifest.uuid index 4bf24d86c2..1ad14f7d1d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e115f2583499df0c7ee991d372bed4b89aa717b10a4e4b10977864390cb4fc11 \ No newline at end of file +fb2ac2d2fa6374084f3325b41b257c7a3ace43aade4b666ec4be93b6b70dc39a \ No newline at end of file diff --git a/src/main.c b/src/main.c index 3c8035c120..bcede6483f 100644 --- a/src/main.c +++ b/src/main.c @@ -239,6 +239,11 @@ int sqlite3_initialize(void){ sqlite3GlobalConfig.isPCacheInit = 1; rc = sqlite3OsInit(); } +#ifdef SQLITE_ENABLE_MEMDB + if( rc==SQLITE_OK ){ + rc = sqlite3MemdbInit(); + } +#endif if( rc==SQLITE_OK ){ sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); @@ -271,7 +276,7 @@ int sqlite3_initialize(void){ #ifndef NDEBUG #ifndef SQLITE_OMIT_FLOATING_POINT /* This section of code's only "output" is via assert() statements. */ - if ( rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ u64 x = (((u64)1)<<63)-1; double y; assert(sizeof(x)==8); diff --git a/src/memdb.c b/src/memdb.c new file mode 100644 index 0000000000..7392513126 --- /dev/null +++ b/src/memdb.c @@ -0,0 +1,497 @@ +/* +** 2016-09-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an in-memory VFS implementation. The application supplies +** a chunk of memory to hold the database file. +** +** USAGE: +** +** sqlite3_open_v2("whatever", &db, SQLITE_OPEN_READWRITE, "memdb"); +** void *sqlite3_memdb_ptr(db, "main", &sz); +** int sqlite3_memdb_config(db, "main", pMem, szData, szMem, mFlags); +** +** Flags: +** +** SQLITE_MEMDB_FREEONCLOSE Free pMem when closing the connection +** SQLITE_MEMDB_RESIZEABLE Use sqlite3_realloc64() to resize pMem +*/ +#ifdef SQLITE_ENABLE_MEMDB +#include "sqliteInt.h" + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs MemVfs; +typedef struct MemFile MemFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) + +/* An open file */ +struct MemFile { + sqlite3_file base; /* IO methods */ + sqlite3_int64 sz; /* Size of the file */ + sqlite3_int64 szMax; /* Space allocated to aData */ + unsigned char *aData; /* content of the file */ + int nMmap; /* Number of memory mapped pages */ + unsigned mFlags; /* Flags */ + int eLock; /* Most recent lock against this file */ +}; + +/* +** Methods for MemFile +*/ +static int memdbClose(sqlite3_file*); +static int memdbRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int memdbWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int memdbTruncate(sqlite3_file*, sqlite3_int64 size); +static int memdbSync(sqlite3_file*, int flags); +static int memdbFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int memdbLock(sqlite3_file*, int); +static int memdbCheckReservedLock(sqlite3_file*, int *pResOut); +static int memdbFileControl(sqlite3_file*, int op, void *pArg); +static int memdbSectorSize(sqlite3_file*); +static int memdbDeviceCharacteristics(sqlite3_file*); +static int memdbShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int memdbShmLock(sqlite3_file*, int offset, int n, int flags); +static void memdbShmBarrier(sqlite3_file*); +static int memdbShmUnmap(sqlite3_file*, int deleteFlag); +static int memdbFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int memdbUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for MemVfs +*/ +static int memdbOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int memdbDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int memdbAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int memdbFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *memdbDlOpen(sqlite3_vfs*, const char *zFilename); +static void memdbDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*memdbDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void memdbDlClose(sqlite3_vfs*, void*); +static int memdbRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int memdbSleep(sqlite3_vfs*, int microseconds); +static int memdbCurrentTime(sqlite3_vfs*, double*); +static int memdbGetLastError(sqlite3_vfs*, int, char *); +static int memdbCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs memdb_vfs = { + 2, /* iVersion */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "memdb", /* zName */ + 0, /* pAppData (set when registered) */ + memdbOpen, /* xOpen */ + memdbDelete, /* xDelete */ + memdbAccess, /* xAccess */ + memdbFullPathname, /* xFullPathname */ + memdbDlOpen, /* xDlOpen */ + memdbDlError, /* xDlError */ + memdbDlSym, /* xDlSym */ + memdbDlClose, /* xDlClose */ + memdbRandomness, /* xRandomness */ + memdbSleep, /* xSleep */ + memdbCurrentTime, /* xCurrentTime */ + memdbGetLastError, /* xGetLastError */ + memdbCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +static const sqlite3_io_methods memdb_io_methods = { + 3, /* iVersion */ + memdbClose, /* xClose */ + memdbRead, /* xRead */ + memdbWrite, /* xWrite */ + memdbTruncate, /* xTruncate */ + memdbSync, /* xSync */ + memdbFileSize, /* xFileSize */ + memdbLock, /* xLock */ + memdbLock, /* xUnlock - same as xLock in this case */ + memdbCheckReservedLock, /* xCheckReservedLock */ + memdbFileControl, /* xFileControl */ + memdbSectorSize, /* xSectorSize */ + memdbDeviceCharacteristics, /* xDeviceCharacteristics */ + memdbShmMap, /* xShmMap */ + memdbShmLock, /* xShmLock */ + memdbShmBarrier, /* xShmBarrier */ + memdbShmUnmap, /* xShmUnmap */ + memdbFetch, /* xFetch */ + memdbUnfetch /* xUnfetch */ +}; + + + +/* +** Close an memdb-file. +** +** The pData pointer is owned by the application, so there is nothing +** to free. +*/ +static int memdbClose(sqlite3_file *pFile){ + MemFile *p = (MemFile *)pFile; + if( p->mFlags & SQLITE_MEMDB_FREEONCLOSE ) sqlite3_free(p->aData); + return SQLITE_OK; +} + +/* +** Read data from an memdb-file. +*/ +static int memdbRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + MemFile *p = (MemFile *)pFile; + if( iOfst+iAmt>p->sz ){ + memset(zBuf, 0, iAmt); + return SQLITE_IOERR_SHORT_READ; + } + memcpy(zBuf, p->aData+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Try to enlarge the memory allocation to hold at least sz bytes +*/ +static int memdbEnlarge(MemFile *p, sqlite3_int64 newSz){ + unsigned char *pNew; + if( (p->mFlags & SQLITE_MEMDB_RESIZEABLE)==0 ) return SQLITE_FULL; + if( p->nMmap>0 ) return SQLITE_FULL; + pNew = sqlite3_realloc64(p->aData, newSz); + if( pNew==0 ) return SQLITE_FULL; + p->aData = pNew; + p->szMax = newSz; + return SQLITE_OK; +} + +/* +** Write data to an memdb-file. +*/ +static int memdbWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + MemFile *p = (MemFile *)pFile; + if( iOfst+iAmt>p->sz ){ + if( iOfst+iAmt>p->szMax && memdbEnlarge(p, (iOfst+iAmt)*2) ){ + return SQLITE_FULL; + } + if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz); + p->sz = iOfst+iAmt; + } + memcpy(p->aData+iOfst, z, iAmt); + return SQLITE_OK; +} + +/* +** Truncate an memdb-file. +*/ +static int memdbTruncate(sqlite3_file *pFile, sqlite_int64 size){ + MemFile *p = (MemFile *)pFile; + if( size>p->sz ){ + if( size>p->szMax && memdbEnlarge(p, size) ) return SQLITE_FULL; + memset(p->aData+p->sz, 0, size-p->sz); + } + p->sz = size; + return SQLITE_OK; +} + +/* +** Sync an memdb-file. +*/ +static int memdbSync(sqlite3_file *pFile, int flags){ + return SQLITE_OK; +} + +/* +** Return the current file-size of an memdb-file. +*/ +static int memdbFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + MemFile *p = (MemFile *)pFile; + *pSize = p->sz; + return SQLITE_OK; +} + +/* +** Lock an memdb-file. +*/ +static int memdbLock(sqlite3_file *pFile, int eLock){ + MemFile *p = (MemFile *)pFile; + p->eLock = eLock; + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an memdb-file. +*/ +static int memdbCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an memdb-file. +*/ +static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){ + MemFile *p = (MemFile *)pFile; + int rc = SQLITE_NOTFOUND; + if( op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz); + rc = SQLITE_OK; + } + return rc; +} + +/* +** Return the sector-size in bytes for an memdb-file. +*/ +static int memdbSectorSize(sqlite3_file *pFile){ + return 1024; +} + +/* +** Return the device characteristic flags supported by an memdb-file. +*/ +static int memdbDeviceCharacteristics(sqlite3_file *pFile){ + return SQLITE_IOCAP_ATOMIC | + SQLITE_IOCAP_POWERSAFE_OVERWRITE | + SQLITE_IOCAP_SAFE_APPEND | + SQLITE_IOCAP_SEQUENTIAL; +} + +/* Create a shared memory file mapping */ +static int memdbShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + return SQLITE_IOERR_SHMMAP; +} + +/* Perform locking on a shared-memory segment */ +static int memdbShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + return SQLITE_IOERR_SHMLOCK; +} + +/* Memory barrier operation on shared memory */ +static void memdbShmBarrier(sqlite3_file *pFile){ + return; +} + +/* Unmap a shared memory segment */ +static int memdbShmUnmap(sqlite3_file *pFile, int deleteFlag){ + return SQLITE_OK; +} + +/* Fetch a page of a memory-mapped file */ +static int memdbFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + MemFile *p = (MemFile *)pFile; + p->nMmap++; + *pp = (void*)(p->aData + iOfst); + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int memdbUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + MemFile *p = (MemFile *)pFile; + p->nMmap--; + return SQLITE_OK; +} + +/* +** Open an mem file handle. +*/ +static int memdbOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + MemFile *p = (MemFile*)pFile; + memset(p, 0, sizeof(*p)); + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN; + p->mFlags = SQLITE_MEMDB_RESIZEABLE | SQLITE_MEMDB_FREEONCLOSE; + *pOutFlags = flags | SQLITE_OPEN_MEMORY; + p->base.pMethods = &memdb_io_methods; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int memdbDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return SQLITE_IOERR_DELETE; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int memdbAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int memdbFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + sqlite3_snprintf(nOut, zOut, "%s", zPath); + return SQLITE_OK; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *memdbDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void memdbDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*memdbDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void memdbDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int memdbRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int memdbSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int memdbCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} + +static int memdbGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int memdbCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} + +/* +** Translate a database connection pointer and schema name into a +** MemFile pointer. +*/ +static MemFile *memdbFromDbSchema(sqlite3 *db, const char *zSchema){ + MemFile *p = 0; + int rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p); + if( rc ) return 0; + if( p->base.pMethods!=&memdb_io_methods ) return 0; + return p; +} + +/* +** Return a pointer to the memory used to hold the database. +** Return NULL if the arguments do not describe a memdb database. +*/ +void *sqlite3_memdb_ptr(sqlite3 *db, const char *zSchema, sqlite3_int64 *pSz){ + MemFile *p = memdbFromDbSchema(db, zSchema); + if( p==0 ){ + *pSz = 0; + return 0; + } + *pSz = p->sz; + return p->aData; +} + +/* +** Reconfigure a memdb database. +*/ +int sqlite3_memdb_config( + sqlite3 *db, + const char *zSchema, + void *aData, + sqlite3_int64 sz, + sqlite3_int64 szMax, + unsigned int mFlags +){ + MemFile *p = memdbFromDbSchema(db, zSchema); + if( p==0 ) return SQLITE_ERROR; + if( p->eLock!=SQLITE_LOCK_NONE || p->nMmap>0 ) return SQLITE_BUSY; + if( p->mFlags & SQLITE_MEMDB_FREEONCLOSE ) sqlite3_free(p->aData); + p->aData = aData; + p->sz = sz; + p->szMax = szMax; + p->mFlags = mFlags; + return SQLITE_OK; +} + +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3MemdbInit(void){ + memdb_vfs.pAppData = sqlite3_vfs_find(0); + memdb_vfs.szOsFile = sizeof(MemFile); + return sqlite3_vfs_register(&memdb_vfs, 0); +} +#endif /* SQLITE_ENABLE_MEMDB */ diff --git a/src/pager.c b/src/pager.c index 295cbe04c5..4f50963b22 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4694,6 +4694,7 @@ int sqlite3PagerOpen( int rc = SQLITE_OK; /* Return code */ int tempFile = 0; /* True for temp files (incl. in-memory files) */ int memDb = 0; /* True if this is an in-memory file */ + int memJM = 0; /* Memory journal mode */ int readOnly = 0; /* True if this is a read-only file */ int journalFileSize; /* Bytes to allocate for each journal fd */ char *zPathname = 0; /* Full path to database file */ @@ -4821,7 +4822,8 @@ int sqlite3PagerOpen( int fout = 0; /* VFS flags returned by xOpen() */ rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); assert( !memDb ); - readOnly = (fout&SQLITE_OPEN_READONLY); + memJM = (fout&SQLITE_OPEN_MEMORY)!=0; + readOnly = (fout&SQLITE_OPEN_READONLY)!=0; /* If the file was successfully opened for read/write access, ** choose a default page size in case we have to create the @@ -4952,7 +4954,7 @@ act_like_temp_file: setSectorSize(pPager); if( !useJournal ){ pPager->journalMode = PAGER_JOURNALMODE_OFF; - }else if( memDb ){ + }else if( memDb || memJM ){ pPager->journalMode = PAGER_JOURNALMODE_MEMORY; } /* pPager->xBusyHandler = 0; */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a6d328360a..1434485532 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8758,6 +8758,46 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( */ SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); +/* +** CAPI3REF: Retrieve the current MEMDB buffer +** EXPERIMENTAL +** +** This interface is only available when SQLite is compiled +** with SQLITE_ENABLE_MEMDB. +** +** The sqlite3_memdb_ptr(D,S,P) interface returns a pointer to the +** memory buffer that is the database file used for [database connection] D +** and schema S. If schema S of database connection D is not a MEMDB +** database, then this routine returns NULL. If P is not NULL, then it must +** be a pointer to a 64-bit signed integer into which the size of the +** database file is written. +*/ +SQLITE_EXPERIMENTAL void *sqlite3_memdb_ptr(sqlite3*,const char*,sqlite3_int64*); + +/* +** CAPI3REF: Set the current MEMDB buffer +** EXPERIMENTAL +** +** This interface is only available when SQLite is compiled +** with SQLITE_ENABLE_MEMDB. +** +** The sqlite3_memdb_config(D,S,P,N,M,F) interface initializes a MEMDB database. +** The database identified by D and S must not be in active use when this +** interface is called, or [SQLITE_BUSY] is returned. +*/ +SQLITE_EXPERIMENTAL int sqlite3_memdb_config(sqlite3*,const char*,void*,sqlite3_int64,sqlite3_int64,unsigned); + +/* +** CAPI3REF: Flags for configuring MEMDB databases +** EXPERIMENTAL +** +** The following are allowed values for the 6th argument (the "flags" +** argument) of the [sqlite3_memdb_config()] interface. +*/ +#define SQLITE_MEMDB_FREEONCLOSE 0x001 /* Free the memory buffer on close */ +#define SQLITE_MEMDB_RESIZEABLE 0x002 /* Resize using sqlite3_realloc64() */ + + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8e79133200..002fb4e916 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4019,6 +4019,10 @@ int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); const char *sqlite3ErrName(int); #endif +#ifdef SQLITE_ENABLE_MEMDB +int sqlite3MemdbInit(void); +#endif + const char *sqlite3ErrStr(int); int sqlite3ReadSchema(Parse *pParse); CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); diff --git a/src/tclsqlite.c b/src/tclsqlite.c index eed86eee34..0e86d727b3 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -1852,14 +1852,13 @@ static int SQLITE_TCLAPI DbObjCmd( "complete", "copy", "enable_load_extension", "errorcode", "eval", "exists", "function", "incrblob", "interrupt", - "last_insert_rowid", "nullvalue", "onecolumn", - "preupdate", "profile", "progress", - "rekey", "restore", "rollback_hook", - "status", "timeout", "total_changes", - "trace", "trace_v2", "transaction", - "unlock_notify", "update_hook", "version", - "wal_hook", - 0 + "last_insert_rowid", "memdb", "nullvalue", + "onecolumn", "preupdate", "profile", + "progress", "rekey", "restore", + "rollback_hook", "status", "timeout", + "total_changes", "trace", "trace_v", + "transaction", "unlock_notify", "update_hook", + "version", "wal_hook", 0 }; enum DB_enum { DB_AUTHORIZER, DB_BACKUP, DB_BUSY, @@ -1868,13 +1867,13 @@ static int SQLITE_TCLAPI DbObjCmd( DB_COMPLETE, DB_COPY, DB_ENABLE_LOAD_EXTENSION, DB_ERRORCODE, DB_EVAL, DB_EXISTS, DB_FUNCTION, DB_INCRBLOB, DB_INTERRUPT, - DB_LAST_INSERT_ROWID, DB_NULLVALUE, DB_ONECOLUMN, - DB_PREUPDATE, DB_PROFILE, DB_PROGRESS, - DB_REKEY, DB_RESTORE, DB_ROLLBACK_HOOK, - DB_STATUS, DB_TIMEOUT, DB_TOTAL_CHANGES, - DB_TRACE, DB_TRACE_V2, DB_TRANSACTION, - DB_UNLOCK_NOTIFY, DB_UPDATE_HOOK, DB_VERSION, - DB_WAL_HOOK, + DB_LAST_INSERT_ROWID, DB_MEMDB, DB_NULLVALUE, + DB_ONECOLUMN, DB_PREUPDATE, DB_PROFILE, + DB_PROGRESS, DB_REKEY, DB_RESTORE, + DB_ROLLBACK_HOOK, DB_STATUS, DB_TIMEOUT, + DB_TOTAL_CHANGES, DB_TRACE, DB_TRACE_V2, + DB_TRANSACTION, DB_UNLOCK_NOTIFY, DB_UPDATE_HOOK, + DB_VERSION, DB_WAL_HOOK }; /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */ @@ -2667,6 +2666,54 @@ static int SQLITE_TCLAPI DbObjCmd( break; } + /* + ** $db memdb DATABASE ?BLOB? + ** + ** Set or query the content of a MEMDB database. + ** + */ + case DB_MEMDB: { +#ifndef SQLITE_ENABLE_MEMDB + Tcl_AppendResult(interp, "MEMDB not available in this build", + (char*)0); + rc = TCL_ERROR; +#else + const char *zSchema = Tcl_GetString(objv[2]); + sqlite3_int64 sz = 0; + unsigned char *pData; + if( objc==3 ){ + pData = sqlite3_memdb_ptr(pDb->db, zSchema, &sz); + if( pData==0 ){ + Tcl_AppendResult(interp, "not a MEMDB database", (char*)0); + rc = TCL_ERROR; + }else{ + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pData,sz)); + } + }else if( objc==4 ){ + int len = 0, xrc; + unsigned char *pBA = Tcl_GetByteArrayFromObj(objv[3], &len); + pData = sqlite3_malloc64( len ); + if( pData==0 ){ + Tcl_AppendResult(interp, "out of memory", (char*)0); + rc = TCL_ERROR; + }else{ + memcpy(pData, pBA, len); + xrc = sqlite3_memdb_config(pDb->db, zSchema, pData, len, len, + SQLITE_MEMDB_FREEONCLOSE|SQLITE_MEMDB_RESIZEABLE); + if( xrc ){ + sqlite3_free(pData); + Tcl_AppendResult(interp, "unable to set MEMDB content", (char*)0); + rc = TCL_ERROR; + } + } + }else{ + Tcl_WrongNumArgs(interp, 2, objv, "?SCRIPT?"); + rc = TCL_ERROR; + } +#endif + break; + } + /* ** $db nullvalue ?STRING? ** diff --git a/src/test_config.c b/src/test_config.c index ad63016baf..aeb3393f0b 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -148,6 +148,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "hiddencolumns", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_ENABLE_MEMDB + Tcl_SetVar2(interp, "sqlite_options", "memdb", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "memdb", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_ENABLE_MEMSYS3 Tcl_SetVar2(interp, "sqlite_options", "mem3", "1", TCL_GLOBAL_ONLY); #else diff --git a/test/memdb1.test b/test/memdb1.test new file mode 100644 index 0000000000..2be9cf62c5 --- /dev/null +++ b/test/memdb1.test @@ -0,0 +1,57 @@ +# 2018-01-02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is the "memdb" VFS +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix memdb1 +do_not_use_codec + +ifcapable !memdb { + finish_test + return +} + +# Create a MEMDB and populate it with some dummy data. +# Then extract the database into the $::db1 variable. +# Verify that the size of $::db1 is the same as the size of +# the database. +# +db close +sqlite3 db dummy -vfs memdb +unset -nocomplain db1 +unset -nocomplain sz1 +unset -nocomplain pgsz +do_test 100 { + db eval { + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(1,2); + } + set ::pgsz [db one {PRAGMA page_size}] + set ::sz1 [expr {$::pgsz*[db one {PRAGMA page_count}]}] + set ::db1 [db memdb main] + expr {[string length $::db1]==$::sz1} +} 1 + +# Create a new MEMDB and initialize it to the content of $::db1 +# Verify that the content is the same. +# +db close +sqlite3 db dummy2 -vfs memdb +db memdb main $db1 +do_execsql_test 110 { + SELECT * FROM t1; +} {1 2} + + +finish_test diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 8ea3e81c91..f2d93aadd0 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -320,6 +320,7 @@ foreach file { os_unix.c os_win.c + memdb.c bitvec.c pcache.c