1
0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00

Add the ability to write to an appended database. This check-in compiles

but is otherwise untested.

FossilOrigin-Name: e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e
This commit is contained in:
drh
2017-12-14 16:28:27 +00:00
parent 32b5fdcaea
commit e483d349ab
3 changed files with 155 additions and 46 deletions

View File

@ -18,8 +18,31 @@
** best performance page 1 should be located at a disk page boundary, though ** best performance page 1 should be located at a disk page boundary, though
** that is not required. ** that is not required.
** **
** An appended database is considered immutable. It is read-only and no ** When opening a database using this VFS, the connection might treat
** locks are ever taken. ** the file as an ordinary SQLite database, or it might treat is as a
** database appended onto some other file. Here are the rules:
**
** (1) When opening a new empty file, that file is treated as an ordinary
** database.
**
** (2) When opening a file that begins with the standard SQLite prefix
** string "SQLite format 3", that file is treated as an ordinary
** database.
**
** (3) When opening a file that ends with the appendvfs trailer string
** "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
** database.
**
** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is
** set, then a new database is appended to the already existing file.
**
** (5) Otherwise, SQLITE_CANTOPEN is returned.
**
** To avoid unnecessary complications with the PENDING_BYTE, the size of
** the file containing the database is limited to 1GB. This VFS will refuse
** to read or write past the 1GB mark. This restriction might be lifted in
** future versions. For now, if you need a large database, then keep the
** database in a separate file.
** **
** If the file being opened is not an appended database, then this shim is ** If the file being opened is not an appended database, then this shim is
** a pass-through into the default underlying VFS. ** a pass-through into the default underlying VFS.
@ -41,6 +64,12 @@ SQLITE_EXTENSION_INIT1
#define APND_MARK_PREFIX_SZ 17 #define APND_MARK_PREFIX_SZ 17
#define APND_MARK_SIZE 25 #define APND_MARK_SIZE 25
/*
** Maximum size of the combined prefix + database + append-mark. This
** must be less than 0x40000000 to avoid locking issues on Windows.
*/
#define APND_MAX_SIZE (65536*15259)
/* /*
** Forward declaration of objects used by this utility ** Forward declaration of objects used by this utility
*/ */
@ -57,6 +86,7 @@ typedef struct ApndFile ApndFile;
struct ApndFile { struct ApndFile {
sqlite3_file base; /* IO methods */ sqlite3_file base; /* IO methods */
sqlite3_int64 iPgOne; /* File offset to page 1 */ sqlite3_int64 iPgOne; /* File offset to page 1 */
sqlite3_int64 iMark; /* Start of the append-mark */
}; };
/* /*
@ -172,37 +202,75 @@ static int apndRead(
return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne); return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
} }
/*
** Add the append-mark onto the end of the file.
*/
static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
int i;
unsigned char a[APND_MARK_SIZE];
memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
for(i=0; i<8; i++){
a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
}
return pFile->pMethods->xWrite(pFile, a, APND_MARK_PREFIX_SZ, p->iMark);
}
/* /*
** Write data to an apnd-file. ** Write data to an apnd-file.
*/ */
static int apndWrite( static int apndWrite(
sqlite3_file *pFile, sqlite3_file *pFile,
const void *z, const void *zBuf,
int iAmt, int iAmt,
sqlite_int64 iOfst sqlite_int64 iOfst
){ ){
return SQLITE_READONLY; int rc;
ApndFile *p = (ApndFile *)pFile;
pFile = ORIGFILE(pFile);
if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL;
rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
if( rc==SQLITE_OK && iOfst + iAmt + p->iPgOne > p->iMark ){
sqlite3_int64 sz = 0;
rc = pFile->pMethods->xFileSize(pFile, &sz);
if( rc==SQLITE_OK ){
p->iMark = sz - APND_MARK_SIZE;
if( iOfst + iAmt + p->iPgOne > p->iMark ){
p->iMark = p->iPgOne + iOfst + iAmt;
rc = apndWriteMark(p, pFile);
}
}
}
return rc;
} }
/* /*
** Truncate an apnd-file. ** Truncate an apnd-file.
*/ */
static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
return SQLITE_READONLY; int rc;
ApndFile *p = (ApndFile *)pFile;
pFile = ORIGFILE(pFile);
rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
if( rc==SQLITE_OK ){
p->iMark = p->iPgOne+size;
rc = apndWriteMark(p, pFile);
}
return rc;
} }
/* /*
** Sync an apnd-file. ** Sync an apnd-file.
*/ */
static int apndSync(sqlite3_file *pFile, int flags){ static int apndSync(sqlite3_file *pFile, int flags){
return SQLITE_READONLY; pFile = ORIGFILE(pFile);
return pFile->pMethods->xSync(pFile, flags);
} }
/* /*
** Return the current file-size of an apnd-file. ** Return the current file-size of an apnd-file.
*/ */
static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
ApndFile *p = (ApndFile*)pFile; ApndFile *p = (ApndFile *)pFile;
int rc; int rc;
pFile = ORIGFILE(p); pFile = ORIGFILE(p);
rc = pFile->pMethods->xFileSize(pFile, pSize); rc = pFile->pMethods->xFileSize(pFile, pSize);
@ -216,22 +284,24 @@ static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
** Lock an apnd-file. ** Lock an apnd-file.
*/ */
static int apndLock(sqlite3_file *pFile, int eLock){ static int apndLock(sqlite3_file *pFile, int eLock){
return SQLITE_READONLY; pFile = ORIGFILE(pFile);
return pFile->pMethods->xLock(pFile, eLock);
} }
/* /*
** Unlock an apnd-file. ** Unlock an apnd-file.
*/ */
static int apndUnlock(sqlite3_file *pFile, int eLock){ static int apndUnlock(sqlite3_file *pFile, int eLock){
return SQLITE_OK; pFile = ORIGFILE(pFile);
return pFile->pMethods->xUnlock(pFile, eLock);
} }
/* /*
** Check if another file-handle holds a RESERVED lock on an apnd-file. ** Check if another file-handle holds a RESERVED lock on an apnd-file.
*/ */
static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
*pResOut = 0; pFile = ORIGFILE(pFile);
return SQLITE_OK; return pFile->pMethods->xCheckReservedLock(pFile, pResOut);
} }
/* /*
@ -273,22 +343,26 @@ static int apndShmMap(
int bExtend, int bExtend,
void volatile **pp void volatile **pp
){ ){
return SQLITE_READONLY; pFile = ORIGFILE(pFile);
return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp);
} }
/* Perform locking on a shared-memory segment */ /* Perform locking on a shared-memory segment */
static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){ static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
return SQLITE_READONLY; pFile = ORIGFILE(pFile);
return pFile->pMethods->xShmLock(pFile,offset,n,flags);
} }
/* Memory barrier operation on shared memory */ /* Memory barrier operation on shared memory */
static void apndShmBarrier(sqlite3_file *pFile){ static void apndShmBarrier(sqlite3_file *pFile){
return; pFile = ORIGFILE(pFile);
pFile->pMethods->xShmBarrier(pFile);
} }
/* Unmap a shared memory segment */ /* Unmap a shared memory segment */
static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){ static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
return SQLITE_OK; pFile = ORIGFILE(pFile);
return pFile->pMethods->xShmUnmap(pFile,deleteFlag);
} }
/* Fetch a page of a memory-mapped file */ /* Fetch a page of a memory-mapped file */
@ -310,6 +384,40 @@ static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
} }
/*
** Check to see if the file is an ordinary SQLite database file.
*/
static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
int rc;
char zHdr[16];
static const char aSqliteHdr[] = "SQLite format 3";
if( sz<512 ) return 0;
rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0);
if( rc ) return 0;
return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0;
}
/*
** Try to read the append-mark off the end of a file. Return the
** start of the appended database if the append-mark is present. If
** there is no append-mark, return -1;
*/
static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){
int rc, i;
sqlite3_int64 iMark;
unsigned char a[APND_MARK_SIZE];
if( sz<=APND_MARK_SIZE ) return -1;
rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
if( rc ) return -1;
if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1;
iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56;
for(i=1; i<8; i++){
iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i);
}
return iMark;
}
/* /*
** Open an apnd file handle. ** Open an apnd file handle.
*/ */
@ -321,38 +429,39 @@ static int apndOpen(
int *pOutFlags int *pOutFlags
){ ){
ApndFile *p; ApndFile *p;
sqlite3_file *pSubFile;
sqlite3_vfs *pSubVfs;
int rc; int rc;
sqlite3_int64 sz; sqlite3_int64 sz;
pVfs = ORIGVFS(pVfs); pSubVfs = ORIGVFS(pVfs);
if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
return pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags); return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
} }
p = (ApndFile*)pFile; p = (ApndFile*)pFile;
memset(p, 0, sizeof(*p)); memset(p, 0, sizeof(*p));
p->base.pMethods = &apnd_io_methods; p->base.pMethods = &apnd_io_methods;
pFile = ORIGFILE(pFile); pSubFile = ORIGFILE(pFile);
flags &= ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE); rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
flags |= SQLITE_OPEN_READONLY;
rc = pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags);
if( rc ) return rc; if( rc ) return rc;
rc = pFile->pMethods->xFileSize(pFile, &sz); rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
if( rc==SQLITE_OK && sz>512 ){ if( rc ){
unsigned char a[APND_MARK_SIZE]; pSubFile->pMethods->xClose(pSubFile);
rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); return rc;
if( rc==SQLITE_OK
&& memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)==0
){
p->iPgOne =
((sqlite3_uint64)a[APND_MARK_PREFIX_SZ]<<56) +
((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+1]<<48) +
((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+2]<<40) +
((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+3]<<32) +
((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+4]<<24) +
((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+5]<<16) +
((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+6]<<8) +
((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+7]);
}
} }
if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
memmove(pFile, pSubFile, pSubVfs->szOsFile);
return SQLITE_OK;
}
p->iMark = 0;
p->iPgOne = apndReadMark(sz, pFile);
if( p->iPgOne>0 ){
return SQLITE_OK;
}
if( (flags & SQLITE_OPEN_CREATE)==0 ){
pSubFile->pMethods->xClose(pSubFile);
return SQLITE_CANTOPEN;
}
p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
return SQLITE_OK; return SQLITE_OK;
} }
@ -442,7 +551,7 @@ int sqlite3_appendvfs_init(
apnd_vfs.iVersion = pOrig->iVersion; apnd_vfs.iVersion = pOrig->iVersion;
apnd_vfs.pAppData = pOrig; apnd_vfs.pAppData = pOrig;
apnd_vfs.szOsFile = sizeof(ApndFile); apnd_vfs.szOsFile = sizeof(ApndFile);
rc = sqlite3_vfs_register(&apnd_vfs, 1); rc = sqlite3_vfs_register(&apnd_vfs, 0);
#ifdef APPENDVFS_TEST #ifdef APPENDVFS_TEST
if( rc==SQLITE_OK ){ if( rc==SQLITE_OK ){
rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);

View File

@ -1,5 +1,5 @@
C Bring\sin\sthe\slatest\strunk\schanges. C Add\sthe\sability\sto\swrite\sto\san\sappended\sdatabase.\s\sThis\scheck-in\scompiles\nbut\sis\sotherwise\suntested.
D 2017-12-14T14:50:49.020 D 2017-12-14T16:28:27.146
F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
@ -260,7 +260,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607
F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
F ext/misc/appendvfs.c 487a5db4427d118413346780c4c90d0256315e8de3fb528ee54daef3bb73276f F ext/misc/appendvfs.c c8b18caae0d22762080a9c48a4e46933f48625bac7fde2af9ef8ec2a23755ac9
F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676
F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 1a1a73b821eb1a22ca335f582a0aea31c71ca9f5b09d54f26409691c90e38c4a 3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3 P 75d8517703f7efa33437079108e2c4ef0de1a118bbe1f4a86afdc34da09d3008
R 29a07f33957881a5b837419ff7f21683 R 15eddfa7673da0a2579cc83183e243e0
U drh U drh
Z d18bd4436a6dfca865f7b12ca3ce5865 Z 4e88658420f57b7b81dd23d0961f71ee

View File

@ -1 +1 @@
75d8517703f7efa33437079108e2c4ef0de1a118bbe1f4a86afdc34da09d3008 e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e