database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00
Code Activity

Fix appendvfs bug exposed by docsapp build (in docsrc), and tighten/revise published behavior.

Browse Source 
FossilOrigin-Name: 7f0f2eacdb496f7f36865ba62164a48edb324a3c6baede736b4e7edf5e8b0fdb
This commit is contained in:
larrybr
2021-03-05 20:46:53 +00:00
parent b14d14696b
commit feecc9f547
3 changed files with 172 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

241
ext/misc/appendvfs.c
View File

@ -14,24 +14,25 @@
** appended onto the end of some other file, such as an executable.
**
** A special record must appear at the end of the file that identifies the
** file as an appended database and provides an offset to page 1. For
** best performance page 1 should be located at a disk page boundary, though
** that is not required.
** file as an appended database and provides the offset to the first page
** of the exposed content. (Or, it is the length of the content prefix.)
** For best performance page 1 should be located at a disk page boundary,
** though that is not required.
**
** When opening a database using this VFS, the connection might treat
** the file as an ordinary SQLite database, or it might treat is as a
** the file as an ordinary SQLite database, or it might treat it as a
** database appended onto some other file. Here are the rules:
**
** (1) When opening a new empty file, that file is treated as an ordinary
** database.
**
** (2) When opening a file that begins with the standard SQLite prefix
** string "SQLite format 3", that file is treated as an ordinary
** database.
**
** (3) When opening a file that ends with the appendvfs trailer string
** (2) When opening a file that ends with the appendvfs trailer string
** "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
** database.
** database, even if rule 3 otherwise applies.
**
** (3) When opening a file that begins with the standard SQLite prefix
** string "SQLite format 3", that file is treated as an ordinary
** database, unless rule 2 applies.
**
** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is
** set, then a new database is appended to the already existing file.
@ -39,13 +40,13 @@
** (5) Otherwise, SQLITE_CANTOPEN is returned.
**
** To avoid unnecessary complications with the PENDING_BYTE, the size of
** the file containing the database is limited to 1GB. This VFS will refuse
** to read or write past the 1GB mark. This restriction might be lifted in
** future versions. For now, if you need a large database, then keep the
** database in a separate file.
** the file containing the database is limited to 1GB. (1000013824 bytes)
** This VFS will not read or write past the 1GB mark. This restriction
** might be lifted in future versions. For now, if you need a larger
** database, then keep it in a separate file.
**
** If the file being opened is not an appended database, then this shim is
** a pass-through into the default underlying VFS.
** If the file being opened is a plain database (not an appended one), then
** this shim is a pass-through into the default underlying VFS. (rule 3)
**/
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
@ -58,11 +59,12 @@ SQLITE_EXTENSION_INIT1
** 123456789 123456789 12345
**
** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
** the offset to page 1.
** the offset to page 1, and also the length of the prefix content.
*/
#define APND_MARK_PREFIX "Start-Of-SQLite3-"
#define APND_MARK_PREFIX_SZ 17
#define APND_MARK_SIZE 25
#define APND_MARK_FOS_SZ 8
#define APND_MARK_SIZE (APND_MARK_PREFIX_SZ+APND_MARK_FOS_SZ)
/*
** Maximum size of the combined prefix + database + append-mark. This
@ -70,6 +72,11 @@ SQLITE_EXTENSION_INIT1
*/
#define APND_MAX_SIZE (65536*15259)
/*
** Size of storage page upon which to align appendvfs portion.
*/
#define APND_ROUNDUP_BITS 12
/*
** Forward declaration of objects used by this utility
*/
@ -82,11 +89,39 @@ typedef struct ApndFile ApndFile;
#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))
#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
/* An open file */
/* Invariants for an open appendvfs file:
* Once an appendvfs file is opened, it will be in one of three states:
* State 0: Never written. Underlying file (if any) is unaltered.
* State 1: Append mark is persisted, content write is in progress.
* State 2: Append mark is persisted, content writes are complete.
*
* State 0 is persistent in the sense that nothing will have been done
* to the underlying file, including any attempt to convert it to an
* appendvfs file.
*
* State 1 is normally transitory. However, if a write operation ends
* abnormally (disk full, power loss, process kill, etc.), then State 1
* may be persistent on disk with an incomplete content write-out. This
* is logically equivalent to an interrupted write to an ordinary file,
* where some unknown portion of to-be-written data is persisted while
* the remainder is not. Database integrity in such cases is maintained
* (or not) by the same measures available for ordinary file access.
*
* State 2 is persistent under normal circumstances (when there is no
* abnormal termination of a write operation such that data provided
* to the underlying VFS write method has not yet reached storage.)
*
* In order to maintain the state invariant, the append mark is written
* in advance of content writes where any part of such content would
* overwrite an existing (or yet to be written) append mark.
*/
struct ApndFile {
sqlite3_file base; /* IO methods */
sqlite3_int64 iPgOne; /* File offset to page 1 */
sqlite3_int64 iMark; /* Start of the append-mark */
/* IO methods of the underlying file */
sqlite3_file base;
/* File offset to beginning of appended content (unchanging) */
sqlite3_int64 iPgOne;
/* File offset of written append-mark, or -1 if unwritten */
sqlite3_int64 iMark;
};
/*
@ -178,8 +213,6 @@ static const sqlite3_io_methods apnd_io_methods = {
apndUnfetch /* xUnfetch */
};
/*
** Close an apnd-file.
*/
@ -203,16 +236,29 @@ static int apndRead(
}
/*
** Add the append-mark onto the end of the file.
** Add the append-mark onto what should become the end of the file.
* If and only if this succeeds, the ApndFile.iMark is updated.
* If it fails, there is little reason to proceed with content writes.
* Parameter imoNext is the appendvfs-relative offset of the new mark.
*/
static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
int i;
static int apndWriteMark(
ApndFile *p,
sqlite3_file *pFile,
sqlite_int64 imoNext
){
unsigned char a[APND_MARK_SIZE];
int ibs = (APND_MARK_FOS_SZ - 1) * 8;
int i, rc;
memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
for(i=0; i<8; i++){
a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
for(i=0; i<APND_MARK_FOS_SZ; ibs -= 8, i++){
a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> ibs) & 0xff;
}
return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark);
imoNext += p->iPgOne;
if( SQLITE_OK==(rc = pFile->pMethods->xWrite
(pFile, a, APND_MARK_SIZE, imoNext)) ){
p->iMark = imoNext;
}
return rc;
}
/*
@ -224,23 +270,16 @@ static int apndWrite(
int iAmt,
sqlite_int64 iOfst
){
int rc;
ApndFile *p = (ApndFile *)pFile;
sqlite_int64 imoNext = iOfst + iAmt;
if( imoNext>=APND_MAX_SIZE ) return SQLITE_FULL;
pFile = ORIGFILE(pFile);
if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL;
rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
if( rc==SQLITE_OK && iOfst + iAmt + p->iPgOne > p->iMark ){
sqlite3_int64 sz = 0;
rc = pFile->pMethods->xFileSize(pFile, &sz);
if( rc==SQLITE_OK ){
p->iMark = sz - APND_MARK_SIZE;
if( iOfst + iAmt + p->iPgOne > p->iMark ){
p->iMark = p->iPgOne + iOfst + iAmt;
rc = apndWriteMark(p, pFile);
}
}
}
if( p->iMark < 0 || imoNext + p->iPgOne > p->iMark ){
int rc = apndWriteMark(p, pFile, imoNext);
if( SQLITE_OK!=rc )
return rc;
}
return pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
}
/*
@ -249,11 +288,13 @@ static int apndWrite(
static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
int rc;
ApndFile *p = (ApndFile *)pFile;
sqlite_int64 iomNext = size+p->iPgOne;
pFile = ORIGFILE(pFile);
rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
rc = apndWriteMark(p, pFile, iomNext);
if( rc==SQLITE_OK ){
p->iMark = p->iPgOne+size;
rc = apndWriteMark(p, pFile);
rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
if( rc==SQLITE_OK )
p->iMark = iomNext;
}
return rc;
}
@ -268,16 +309,12 @@ static int apndSync(sqlite3_file *pFile, int flags){
/*
** Return the current file-size of an apnd-file.
** If the append mark is not yet there, the file-size is 0.
*/
static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
ApndFile *p = (ApndFile *)pFile;
int rc;
pFile = ORIGFILE(p);
rc = pFile->pMethods->xFileSize(pFile, pSize);
if( rc==SQLITE_OK && p->iPgOne ){
*pSize -= p->iPgOne + APND_MARK_SIZE;
}
return rc;
ApndFile *paf = (ApndFile *)pFile;
*pSize = ( paf->iMark >= 0 )? (paf->iMark - paf->iPgOne) : 0;
return SQLITE_OK;
}
/*
@ -372,6 +409,8 @@ static int apndFetch(
void **pp
){
ApndFile *p = (ApndFile *)pFile;
if( p->iMark < 0 || iOfst+iAmt > p->iMark)
return SQLITE_IOERR; /* Cannot read what is not yet there. */
pFile = ORIGFILE(pFile);
return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
}
@ -383,40 +422,69 @@ static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
}
/*
** Check to see if the file is an ordinary SQLite database file.
*/
static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
int rc;
char zHdr[16];
static const char aSqliteHdr[] = "SQLite format 3";
if( sz<512 ) return 0;
rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0);
if( rc ) return 0;
return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0;
}
/*
** Try to read the append-mark off the end of a file. Return the
** start of the appended database if the append-mark is present. If
** there is no append-mark, return -1;
** start of the appended database if the append-mark is present.
** If there is no valid append-mark, return -1;
*/
static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){
int rc, i;
sqlite3_int64 iMark;
int msbs = 8 * (APND_MARK_FOS_SZ-1);
unsigned char a[APND_MARK_SIZE];
if( sz<=APND_MARK_SIZE ) return -1;
if( APND_MARK_SIZE!=(sz & 0x1ff) ) return -1;
rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
if( rc ) return -1;
if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1;
iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56;
iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ] & 0x7f)) << msbs;
for(i=1; i<8; i++){
iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i);
msbs -= 8;
iMark |= (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<msbs;
}
return iMark;
}
static const char apvfsSqliteHdr[] = "SQLite format 3";
/*
** Check to see if the file is an appendvfs SQLite database file.
** Return true iff it is such. Parameter sz is the file's size.
*/
static int apndIsAppendvfsDatabase(sqlite3_int64 sz, sqlite3_file *pFile){
int rc;
char zHdr[16];
sqlite3_int64 iMark = apndReadMark(sz, pFile);
if( iMark>=0 ){
/* If file has right end-marker, the expected odd size, and the
* SQLite DB type marker where the end-marker puts it, then it
* is an appendvfs database (to be treated as such.)
*/
rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), iMark);
if( SQLITE_OK==rc && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0
&& (sz & 0x1ff)== APND_MARK_SIZE && sz>=512+APND_MARK_SIZE )
return 1; /* It's an appendvfs database */
}
return 0;
}
/*
** Check to see if the file is an ordinary SQLite database file.
** Return true iff so. Parameter sz is the file's size.
*/
static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
char zHdr[16];
if( apndIsAppendvfsDatabase(sz, pFile) /* rule 2 */
|| (sz & 0x1ff) != 0
|| SQLITE_OK!=pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0)
|| memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))!=0 )
return 0;
return 1;
}
/* Round-up used to get appendvfs portion to begin at a page boundary. */
#define APND_START_ROUNDUP(fsz, nPageBits) \
((fsz) + ((1<<nPageBits)-1) & ~(sqlite3_int64)((1<<nPageBits)-1))
/*
** Open an apnd file handle.
*/
@ -433,6 +501,10 @@ static int apndOpen(
int rc;
sqlite3_int64 sz;
pSubVfs = ORIGVFS(pVfs);
/* The appendvfs is not to be used for auxillary DB files.
* Attempting such will result in simply opening the named
* file however the underlying VFS does that.
*/
if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
}
@ -451,27 +523,42 @@ static int apndOpen(
memmove(pFile, pSubFile, pSubVfs->szOsFile);
return SQLITE_OK;
}
p->iMark = 0;
/* Record that append mark has not been written until seen otherwise. */
p->iMark = -1;
p->iPgOne = apndReadMark(sz, pFile);
if( p->iPgOne>0 ){
if( p->iPgOne>=0 ){
/* Append mark was found, infer its offset */
p->iMark = sz - p->iPgOne - APND_MARK_SIZE;
return SQLITE_OK;
}
if( (flags & SQLITE_OPEN_CREATE)==0 ){
pSubFile->pMethods->xClose(pSubFile);
rc = SQLITE_CANTOPEN;
}
p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
/* Round newly added appendvfs location to #define'd page boundary.
* Note that nothing has yet been written to the underlying file.
* The append mark will be written along with first content write.
* Until then, the p->iMark value indicates it is not yet written.
*/
p->iPgOne = APND_START_ROUNDUP(sz, APND_ROUNDUP_BITS);
apnd_open_done:
if( rc ) pFile->pMethods = 0;
return rc;
}
/*
** All other VFS methods are pass-thrus.
** Delete an apnd file.
** For an appendvfs, this could mean delete the appendvfs portion,
** leaving the appendee as it was before it gained an appendvfs.
** For now, this code deletes the underlying file too.
*/
static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync);
}
/*
** All other VFS methods are pass-thrus.
*/
static int apndAccess(
sqlite3_vfs *pVfs,
const char *zPath,

15
manifest
View File

@ -1,5 +1,5 @@
C Create\snew\sbranch\snamed\s"appendvfs_tighten"
D 2021-03-05T09:42:58.132
C Fix\sappendvfs\sbug\sexposed\sby\sdocsapp\sbuild\s(in\sdocsrc),\sand\stighten/revise\spublished\sbehavior.
D 2021-03-05T20:46:53.770
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -286,7 +286,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607
F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240
F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358
F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
F ext/misc/appendvfs.c 55121d311d408ba9c62c3cfa367408887638f02f9522dd9859891d0ee69a7eba
F ext/misc/appendvfs.c 82cd0dc37b5645b079365c2ebd5b361548896c401db9280da2f7f64d4f02f618
F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a
F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9
F ext/misc/carray.c b75a0f207391038bf1540d3372f482a95c3613511c7c474db51ede1196321c7c
@ -1909,10 +1909,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 23459071091248e94202e609bb8031c3e34722b8ae8ff5a66851dcc528a2d2c2
R 3f829008faf6bb2f8b2a8086ffa5bb99
T *branch * appendvfs_tighten
T *sym-appendvfs_tighten *
T -sym-trunk *
P 48c968bf4c8e1517864bec68d3072441994ea511a9cb446d2fdf031d5f43b077
R d9b9b7ebcf10ff9c889756135ed9e77d
U larrybr
Z 63ac0fc408a0a285700284c9e62510e0
Z c2502dcdc1270b481b2f881df0282e61

2
manifest.uuid
View File

@ -1 +1 @@
48c968bf4c8e1517864bec68d3072441994ea511a9cb446d2fdf031d5f43b077
7f0f2eacdb496f7f36865ba62164a48edb324a3c6baede736b4e7edf5e8b0fdb