database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-08 14:02:16 +03:00
Code Activity

Add an experimental extension for applying bulk updates to databases.

Browse Source 
FossilOrigin-Name: 2954ab501049968430011b63d046eb42ff37a56c
This commit is contained in:
dan
2014-09-02 19:59:40 +00:00
parent f8ede57a61
commit b0083756f2
22 changed files with 1878 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
ext/ota/ota1.test Normal file
View File

@@ -0,0 +1,181 @@
# 2014 August 30
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
set testdir [file join [file dirname $argv0] .. .. test]
source $testdir/tester.tcl
set ::testprefix ota1
# Create a simple OTA database. That expects to write to a table:
#
# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
#
proc create_ota1 {filename} {
forcedelete $filename
sqlite3 ota1 $filename
ota1 eval {
CREATE TABLE data_t1(a, b, c, ota_control);
INSERT INTO data_t1 VALUES(1, 2, 3, 0);
INSERT INTO data_t1 VALUES(2, 'two', 'three', 0);
INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0);
}
ota1 close
return $filename
}
# Create an empty target database suitable for the OTA created by
# [create_ota1].
#
# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
#
proc create_target1 {filename} {
forcedelete $filename
sqlite3 target1 $filename
target1 eval { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) }
target1 close
return $filename
}
# Run the OTA in file $ota on target database $target until completion.
#
proc run_ota {target ota} {
sqlite3ota ota $target $ota
while { [ota step]=="SQLITE_OK" } {}
ota close
}
proc step_ota {target ota} {
while 1 {
sqlite3ota ota $target $ota
set rc [ota step]
ota close
if {$rc != "SQLITE_OK"} break
}
set rc
}
foreach {tn2 cmd} {1 step_ota 2 run_ota} {
foreach {tn schema} {
1 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
}
2 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b);
}
3 {
CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID;
}
4 {
CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID;
CREATE INDEX i1 ON t1(b);
}
5 {
CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)) WITHOUT ROWID;
CREATE INDEX i1 ON t1(b);
}
6 {
CREATE TABLE t1(a, b, c, PRIMARY KEY(c)) WITHOUT ROWID;
CREATE INDEX i1 ON t1(b, a);
}
7 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b, c);
CREATE INDEX i2 ON t1(c, b);
CREATE INDEX i3 ON t1(a, b, c, a, b, c);
}
} {
reset_db
execsql $schema
do_test 1.$tn2.$tn.1 {
create_ota1 ota.db
$cmd test.db ota.db
} {SQLITE_DONE}
do_execsql_test 1.$tn2.$tn.2 {
SELECT * FROM t1 ORDER BY a ASC;
} {
1 2 3
2 two three
3 {} 8.2
}
do_execsql_test 1.$tn2.$tn.3 { PRAGMA integrity_check } ok
}
}
#-------------------------------------------------------------------------
# Check that an OTA cannot be applied to a table that has no PK.
#
reset_db
create_ota1 ota.db
do_execsql_test 2.1 { CREATE TABLE t1(a, b, c) }
do_test 2.2 {
sqlite3ota ota test.db ota.db
ota step
} {SQLITE_ERROR}
do_test 2.3 {
list [catch { ota close } msg] $msg
} {1 {SQLITE_ERROR - table t1 has no PRIMARY KEY}}
reset_db
do_execsql_test 2.4 { CREATE TABLE t1(a PRIMARY KEY, b, c) }
do_test 2.5 {
sqlite3ota ota test.db ota.db
ota step
} {SQLITE_ERROR}
do_test 2.6 {
list [catch { ota close } msg] $msg
} {1 {SQLITE_ERROR - table t1 has no PRIMARY KEY}}
#-------------------------------------------------------------------------
# Check that if a UNIQUE constraint is violated the current and all
# subsequent [ota step] calls return SQLITE_CONSTRAINT. And that the OTA
# transaction is rolled back by the [ota close] that deletes the ota
# handle.
#
foreach {tn errcode errmsg schema} {
1 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
INSERT INTO t1 VALUES(3, 2, 1);
}
2 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c UNIQUE);
INSERT INTO t1 VALUES(4, 2, 'three');
}
} {
reset_db
execsql $schema
set cksum [dbcksum db main]
do_test 3.$tn.1 {
create_ota1 ota.db
sqlite3ota ota test.db ota.db
while {[set res [ota step]]=="SQLITE_OK"} {}
set res
} $errcode
do_test 3.$tn.2 { ota step } $errcode
do_test 3.$tn.3 {
list [catch { ota close } msg] $msg
} [list 1 "$errcode - $errmsg"]
do_test 3.$tn.4 { dbcksum db main } $cksum
}
finish_test

62
ext/ota/ota2.test Normal file
View File

@@ -0,0 +1,62 @@
# 2014 August 30
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
set testdir [file join [file dirname $argv0] .. .. test]
source $testdir/tester.tcl
set ::testprefix ota2
do_execsql_test 1.0 {
PRAGMA ota_mode = 1;
PRAGMA journal_mode = wal;
CREATE TABLE t1(a, b);
BEGIN;
INSERT INTO t1 VALUES(1, 2);
} {wal}
do_test 1.1 {
set state [sqlite3_transaction_save db]
db close
file exists test.db-wal
} {1}
do_test 1.2 {
sqlite3 db test.db
db eval {SELECT * FROM t1}
} {}
do_test 1.3 {
execsql {BEGIN IMMEDIATE}
sqlite3_transaction_restore db $::state
db eval {SELECT * FROM t1}
} {1 2}
do_test 1.4 {
execsql {
INSERT INTO t1 VALUES(3, 4);
COMMIT;
SELECT * FROM t1;
}
} {1 2 3 4}
do_test 1.5 {
db close
file exists test.db-wal
} {0}
do_test 1.5 {
sqlite3 db test.db
db eval {SELECT * FROM t1}
} {1 2 3 4}
finish_test

811
ext/ota/sqlite3ota.c Normal file
View File

@@ -0,0 +1,811 @@
/*
** 2014 August 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
*/
#include <assert.h>
#include <string.h>
#include "sqlite3.h"
#include "sqlite3ota.h"
/*
** The ota_state table is used to save the state of a partially applied
** update so that it can be resumed later. The table contains at most a
** single row:
**
** "wal_state" -> Blob to use with sqlite3_transaction_restore().
**
** "tbl" -> Table currently being written (target database names).
**
** "idx" -> Index currently being written (target database names).
** Or, if the main table is being written, a NULL value.
**
** "row" -> Last rowid processed from ota database table (i.e. data_%).
**
** "progress" -> total number of key/value b-tree operations performed
** so far as part of this ota update.
*/
#define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota_state" \
"(wal_state, tbl, idx, row, progress)"
typedef struct OtaTblIter OtaTblIter;
typedef struct OtaIdxIter OtaIdxIter;
/*
** Iterator used to iterate through all data tables in the OTA. As follows:
**
** OtaTblIter iter;
** for(rc=tblIterFirst(db, &iter);
** rc==SQLITE_OK && iter.zTarget;
** rc=tblIterNext(&iter)
** ){
** }
*/
struct OtaTblIter {
sqlite3_stmt *pTblIter; /* Iterate through tables */
int iEntry; /* Index of current entry (from 1) */
/* Output varibles. zTarget==0 implies EOF. */
const char *zTarget; /* Name of target table */
const char *zSource; /* Name of source table */
/* Useful things populated by a call to tblIterPrepareAll() */
int nCol; /* Number of columns in this table */
char **azCol; /* Array of quoted column names */
sqlite3_stmt *pSelect; /* PK b-tree SELECT statement */
sqlite3_stmt *pInsert; /* PK b-tree INSERT statement */
};
/*
** API is:
**
** idxIterFirst()
** idxIterNext()
** idxIterFinalize()
** idxIterPrepareAll()
*/
struct OtaIdxIter {
sqlite3_stmt *pIdxIter; /* Iterate through indexes */
int iEntry; /* Index of current entry (from 1) */
/* Output varibles. zTarget==0 implies EOF. */
const char *zIndex; /* Name of index */
int nCol; /* Number of columns in index */
int *aiCol; /* Array of column indexes */
sqlite3_stmt *pWriter; /* Index writer */
sqlite3_stmt *pSelect; /* Select to read values in index order */
};
struct sqlite3ota {
sqlite3 *dbDest; /* Target db */
sqlite3 *dbOta; /* Ota db */
int rc; /* Value returned by last ota_step() call */
char *zErrmsg; /* Error message if rc!=SQLITE_OK */
OtaTblIter tbliter; /* Used to iterate through tables */
OtaIdxIter idxiter; /* Used to iterate through indexes */
};
static int prepareAndCollectError(
sqlite3 *db,
const char *zSql,
sqlite3_stmt **ppStmt,
char **pzErrmsg
){
int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
if( rc!=SQLITE_OK ){
*pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
*ppStmt = 0;
}
return rc;
}
/*
** Unless it is NULL, argument zSql points to a buffer allocated using
** sqlite3_malloc containing an SQL statement. This function prepares the SQL
** statement against database db and frees the buffer. If statement
** compilation is successful, *ppStmt is set to point to the new statement
** handle and SQLITE_OK is returned.
**
** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code
** returned. In this case, *pzErrmsg may also be set to point to an error
** message. It is the responsibility of the caller to free this error message
** buffer using sqlite3_free().
**
** If argument zSql is NULL, this function assumes that an OOM has occurred.
** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL.
*/
static int prepareFreeAndCollectError(
sqlite3 *db,
char *zSql,
sqlite3_stmt **ppStmt,
char **pzErrmsg
){
int rc;
assert( *pzErrmsg==0 );
if( zSql==0 ){
rc = SQLITE_NOMEM;
*ppStmt = 0;
}else{
rc = prepareAndCollectError(db, zSql, ppStmt, pzErrmsg);
sqlite3_free(zSql);
}
return rc;
}
static char *quoteSqlName(const char *zName){
int nName = strlen(zName);
char *zRet = sqlite3_malloc(nName * 2 + 2 + 1);
if( zRet ){
int i;
char *p = zRet;
*p++ = '"';
for(i=0; i<nName; i++){
if( zName[i]=='"' ) *p++ = '"';
*p++ = zName[i];
}
*p++ = '"';
*p++ = '\0';
}
return zRet;
}
static int tblIterPrepareAll(sqlite3ota *p){
OtaTblIter *pIter = &p->tbliter;
int rc = SQLITE_OK;
char *zCol = 0;
char *zBindings = 0;
char *zSql;
sqlite3_stmt *pPragma = 0;
int i;
int bSeenPk = 0; /* Set to true once PK column seen */
/* Allocate and populate the azCol[] array */
zSql = sqlite3_mprintf("PRAGMA main.table_info(%Q)", pIter->zTarget);
rc = prepareFreeAndCollectError(p->dbDest, zSql, &pPragma, &p->zErrmsg);
pIter->nCol = 0;
if( rc==SQLITE_OK ){
while( SQLITE_ROW==sqlite3_step(pPragma) ){
const char *zName = (const char*)sqlite3_column_text(pPragma, 1);
if( (pIter->nCol % 4)==0 ){
int nByte = sizeof(char*) * (pIter->nCol+4);
char **azNew = (char**)sqlite3_realloc(pIter->azCol, nByte);
if( azNew==0 ){
rc = SQLITE_NOMEM;
break;
}
pIter->azCol = azNew;
}
pIter->azCol[pIter->nCol] = quoteSqlName(zName);
if( pIter->azCol[pIter->nCol]==0 ){
rc = SQLITE_NOMEM;
break;
}
pIter->nCol++;
if( sqlite3_column_int(pPragma, 5) ) bSeenPk = 1;
}
if( rc==SQLITE_OK ){
rc = sqlite3_finalize(pPragma);
}else{
sqlite3_finalize(pPragma);
}
}
/* If the table has no PRIMARY KEY, throw an exception. */
if( bSeenPk==0 ){
p->zErrmsg = sqlite3_mprintf("table %s has no PRIMARY KEY", pIter->zTarget);
rc = SQLITE_ERROR;
}
/* Populate the zCol variable */
for(i=0; rc==SQLITE_OK && i<pIter->nCol; i++){
zCol = sqlite3_mprintf("%z%s%s", zCol, (i==0?"":", "), pIter->azCol[i]);
if( zCol==0 ){
rc = SQLITE_NOMEM;
}
}
/* Allocate and populate zBindings */
if( rc==SQLITE_OK ){
zBindings = (char*)sqlite3_malloc(pIter->nCol * 2);
if( zBindings==0 ){
rc = SQLITE_NOMEM;
}else{
int i;
for(i=0; i<pIter->nCol; i++){
zBindings[i*2] = '?';
zBindings[i*2+1] = ',';
}
zBindings[pIter->nCol*2-1] = '\0';
}
}
/* Create OtaTblIter.pSelect */
if( rc==SQLITE_OK ){
zSql = sqlite3_mprintf("SELECT rowid, %s FROM %Q", zCol, pIter->zSource);
rc = prepareFreeAndCollectError(p->dbOta,zSql,&pIter->pSelect, &p->zErrmsg);
}
/* Create OtaTblIter.pInsert */
if( rc==SQLITE_OK ){
zSql = sqlite3_mprintf("INSERT INTO %Q(%s) VALUES(%s)",
pIter->zTarget, zCol, zBindings
);
rc = prepareFreeAndCollectError(p->dbDest,zSql,&pIter->pInsert,&p->zErrmsg);
}
sqlite3_free(zCol);
sqlite3_free(zBindings);
return rc;
}
static void tblIterFreeAll(OtaTblIter *pIter){
int i;
sqlite3_finalize(pIter->pSelect);
sqlite3_finalize(pIter->pInsert);
for(i=0; i<pIter->nCol; i++) sqlite3_free(pIter->azCol[i]);
sqlite3_free(pIter->azCol);
pIter->azCol = 0;
pIter->pSelect = 0;
pIter->pInsert = 0;
pIter->nCol = 0;
}
static int tblIterNext(OtaTblIter *pIter){
int rc;
tblIterFreeAll(pIter);
assert( pIter->pTblIter );
rc = sqlite3_step(pIter->pTblIter);
if( rc==SQLITE_ROW ){
pIter->zSource = (const char*)sqlite3_column_text(pIter->pTblIter, 0);
pIter->zTarget = &pIter->zSource[5]; assert( 5==strlen("data_") );
pIter->iEntry++;
}else{
pIter->zSource = 0;
pIter->zTarget = 0;
}
if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK;
return rc;
}
static int tblIterFirst(sqlite3 *db, OtaTblIter *pIter){
int rc; /* return code */
memset(pIter, 0, sizeof(OtaTblIter));
rc = sqlite3_prepare_v2(db,
"SELECT name FROM sqlite_master "
"WHERE type='table' AND name LIKE 'data_%'", -1, &pIter->pTblIter, 0
);
if( rc==SQLITE_OK ){
rc = tblIterNext(pIter);
}
return rc;
}
static void tblIterFinalize(OtaTblIter *pIter){
tblIterFreeAll(pIter);
sqlite3_finalize(pIter->pTblIter);
memset(pIter, 0, sizeof(OtaTblIter));
}
static void idxIterFreeAll(OtaIdxIter *pIter){
sqlite3_finalize(pIter->pWriter);
sqlite3_finalize(pIter->pSelect);
pIter->pWriter = 0;
pIter->pSelect = 0;
pIter->aiCol = 0;
pIter->nCol = 0;
}
static int idxIterPrepareAll(sqlite3ota *p){
int rc;
int i; /* Iterator variable */
char *zSql = 0;
char *zCols = 0; /* Columns list */
OtaIdxIter *pIter = &p->idxiter;
/* Prepare the writer statement to write (insert) entries into the index. */
rc = sqlite3_index_writer(
p->dbDest, 0, pIter->zIndex, &pIter->pWriter, &pIter->aiCol, &pIter->nCol
);
/* Prepare a SELECT statement to read values from the source table in
** the same order as they are stored in the current index. The statement
** is:
**
** SELECT rowid, <cols> FROM data_<tbl> ORDER BY <cols>
*/
for(i=0; rc==SQLITE_OK && i<pIter->nCol; i++){
const char *zQuoted = p->tbliter.azCol[ pIter->aiCol[i] ];
zCols = sqlite3_mprintf("%z%s%s", zCols, zCols?", ":"", zQuoted);
if( !zCols ){
rc = SQLITE_NOMEM;
}
}
if( rc==SQLITE_OK ){
const char *zFmt = "SELECT rowid, %s FROM %Q ORDER BY %s";
zSql = sqlite3_mprintf(zFmt, zCols, p->tbliter.zSource, zCols);
if( zSql ){
sqlite3_stmt **pp = &p->idxiter.pSelect;
rc = prepareFreeAndCollectError(p->dbOta, zSql, pp, &p->zErrmsg);
}else{
rc = SQLITE_NOMEM;
}
}
sqlite3_free(zCols);
return rc;
}
static int idxIterNext(OtaIdxIter *pIter){
int rc;
idxIterFreeAll(pIter);
assert( pIter->pIdxIter );
rc = sqlite3_step(pIter->pIdxIter);
if( rc==SQLITE_ROW ){
pIter->zIndex = (const char*)sqlite3_column_text(pIter->pIdxIter, 0);
pIter->iEntry++;
}else{
pIter->zIndex = 0;
rc = sqlite3_finalize(pIter->pIdxIter);
pIter->pIdxIter = 0;
}
if( rc==SQLITE_ROW ) rc = SQLITE_OK;
return rc;
}
static int idxIterFirst(sqlite3 *db, const char *zTable, OtaIdxIter *pIter){
int rc; /* return code */
memset(pIter, 0, sizeof(OtaIdxIter));
rc = sqlite3_prepare_v2(db,
"SELECT name FROM sqlite_master "
"WHERE type='index' AND tbl_name = ?", -1, &pIter->pIdxIter, 0
);
if( rc==SQLITE_OK ){
rc = sqlite3_bind_text(pIter->pIdxIter, 1, zTable, -1, SQLITE_TRANSIENT);
}
if( rc==SQLITE_OK ){
rc = idxIterNext(pIter);
}
return rc;
}
static void idxIterFinalize(OtaIdxIter *pIter){
idxIterFreeAll(pIter);
sqlite3_finalize(pIter->pIdxIter);
memset(pIter, 0, sizeof(OtaIdxIter));
}
/*
** Call sqlite3_reset() on the SQL statement passed as the second argument.
** If it returns anything other than SQLITE_OK, store the error code and
** error message in the OTA handle.
*/
static void otaResetStatement(sqlite3ota *p, sqlite3_stmt *pStmt){
assert( p->rc==SQLITE_OK );
assert( p->zErrmsg==0 );
p->rc = sqlite3_reset(pStmt);
if( p->rc!=SQLITE_OK ){
sqlite3 *db = sqlite3_db_handle(pStmt);
p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
}
}
/*
** Check that all SQL statements required to process the current
** table and index have been prepared. If not, prepare them. If
** an error occurs, store the error code and message in the OTA
** handle before returning.
*/
static int otaPrepareAll(sqlite3ota *p){
assert( p->rc==SQLITE_OK );
assert( p->zErrmsg==0 );
assert( p->tbliter.zTarget );
if( p->tbliter.pSelect==0 ){
p->rc = tblIterPrepareAll(p);
}
if( p->rc==SQLITE_OK && p->idxiter.zIndex && 0==p->idxiter.pSelect ){
p->rc = idxIterPrepareAll(p);
}
return p->rc;
}
int sqlite3ota_step(sqlite3ota *p){
if( p ){
while( p && p->rc==SQLITE_OK && p->tbliter.zTarget ){
sqlite3_stmt *pSelect;
int i;
otaPrepareAll(p);
pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect);
/* Advance to the next input row. */
if( p->rc==SQLITE_OK ){
int rc = sqlite3_step(pSelect);
if( rc!=SQLITE_ROW ){
otaResetStatement(p, pSelect);
/* Go to the next index. */
if( p->rc==SQLITE_OK ){
if( p->idxiter.zIndex ){
p->rc = idxIterNext(&p->idxiter);
}else{
p->rc = idxIterFirst(p->dbDest, p->tbliter.zTarget, &p->idxiter);
}
}
/* If there is no next index, go to the next table. */
if( p->rc==SQLITE_OK && p->idxiter.zIndex==0 ){
p->rc = tblIterNext(&p->tbliter);
}
continue;
}
}
/* Update the target database PK table according to the row that
** tbliter.pSelect currently points to.
**
** todo: For now, we assume all rows are INSERT commands - this will
** change. */
if( p->rc==SQLITE_OK ){
sqlite3_stmt *pInsert;
int nCol;
if( p->idxiter.zIndex ){
pInsert = p->idxiter.pWriter;
nCol = p->idxiter.nCol;
}else{
pInsert = p->tbliter.pInsert;
nCol = p->tbliter.nCol;
}
for(i=0; i<nCol; i++){
sqlite3_value *pVal = sqlite3_column_value(pSelect, i+1);
sqlite3_bind_value(pInsert, i+1, pVal);
}
sqlite3_step(pInsert);
otaResetStatement(p, pInsert);
}
break;
}
if( p->rc==SQLITE_OK && p->tbliter.zTarget==0 ) p->rc = SQLITE_DONE;
}
return (p ? p->rc : SQLITE_NOMEM);
}
static void otaOpenDatabase(sqlite3ota *p, sqlite3 **pDb, const char *zFile){
if( p->rc==SQLITE_OK ){
p->rc = sqlite3_open(zFile, pDb);
if( p->rc ){
const char *zErr = sqlite3_errmsg(*pDb);
p->zErrmsg = sqlite3_mprintf("sqlite3_open(): %s", zErr);
}
}
}
static void otaSaveTransactionState(sqlite3ota *p){
sqlite3_stmt *pStmt = 0;
void *pWalState = 0;
int nWalState = 0;
int rc;
const char *zInsert =
"INSERT INTO ota_state(wal_state, tbl, idx, row, progress)"
"VALUES(:wal_state, :tbl, :idx, :row, :progress)";
rc = sqlite3_transaction_save(p->dbDest, &pWalState, &nWalState);
if( rc==SQLITE_OK ){
rc = sqlite3_exec(p->dbOta, "DELETE FROM ota_state", 0, 0, 0);
}
if( rc==SQLITE_OK ){
rc = prepareAndCollectError(p->dbOta, zInsert, &pStmt, &p->zErrmsg);
}
if( rc==SQLITE_OK ){
sqlite3_stmt *pSelect;
pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect);
sqlite3_bind_blob(pStmt, 1, pWalState, nWalState, SQLITE_STATIC);
sqlite3_bind_text(pStmt, 2, p->tbliter.zTarget, -1, SQLITE_STATIC);
if( p->idxiter.zIndex ){
sqlite3_bind_text(pStmt, 3, p->idxiter.zIndex, -1, SQLITE_STATIC);
}
sqlite3_bind_int64(pStmt, 4, sqlite3_column_int64(pSelect, 0));
sqlite3_step(pStmt);
rc = sqlite3_finalize(pStmt);
if( rc==SQLITE_OK ){
rc = sqlite3_exec(p->dbOta, "COMMIT", 0, 0, 0);
}
if( rc!=SQLITE_OK ){
p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->dbOta));
}
}
sqlite3_free(pWalState);
assert( p->rc==SQLITE_OK );
p->rc = rc;
}
static void otaLoadTransactionState(sqlite3ota *p){
sqlite3_stmt *pStmt = 0;
int rc;
const char *zSelect =
"SELECT wal_state, tbl, idx, row, progress FROM ota_state";
rc = prepareAndCollectError(p->dbOta, zSelect, &pStmt, &p->zErrmsg);
if( rc==SQLITE_OK ){
if( SQLITE_ROW==sqlite3_step(pStmt) ){
const void *pWalState = 0;
int nWalState = 0;
const char *zTbl;
const char *zIdx;
sqlite3_int64 iRowid;
pWalState = sqlite3_column_blob(pStmt, 0);
nWalState = sqlite3_column_bytes(pStmt, 0);
zTbl = (const char*)sqlite3_column_text(pStmt, 1);
zIdx = (const char*)sqlite3_column_text(pStmt, 2);
iRowid = sqlite3_column_int64(pStmt, 3);
rc = sqlite3_transaction_restore(p->dbDest, pWalState, nWalState);
while( rc==SQLITE_OK
&& p->tbliter.zTarget
&& sqlite3_stricmp(p->tbliter.zTarget, zTbl)
){
rc = tblIterNext(&p->tbliter);
}
if( rc==SQLITE_OK && !p->tbliter.zTarget ){
rc = SQLITE_ERROR;
p->zErrmsg = sqlite3_mprintf("ota_state mismatch error");
}
if( rc==SQLITE_OK && zIdx ){
rc = idxIterFirst(p->dbDest, p->tbliter.zTarget, &p->idxiter);
while( rc==SQLITE_OK
&& p->idxiter.zIndex
&& sqlite3_stricmp(p->idxiter.zIndex, zIdx)
){
rc = idxIterNext(&p->idxiter);
}
if( rc==SQLITE_OK && !p->idxiter.zIndex ){
rc = SQLITE_ERROR;
p->zErrmsg = sqlite3_mprintf("ota_state mismatch error");
}
}
if( rc==SQLITE_OK ){
rc = otaPrepareAll(p);
}
if( rc==SQLITE_OK ){
sqlite3_stmt *pSelect;
pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect);
while( sqlite3_column_int64(pSelect, 0)!=iRowid ){
rc = sqlite3_step(pSelect);
if( rc!=SQLITE_ROW ) break;
}
if( rc==SQLITE_ROW ){
rc = SQLITE_OK;
}else{
rc = SQLITE_ERROR;
p->zErrmsg = sqlite3_mprintf("ota_state mismatch error");
}
}
}
if( rc==SQLITE_OK ){
rc = sqlite3_finalize(pStmt);
}else{
sqlite3_finalize(pStmt);
}
}
p->rc = rc;
}
/*
** Open and return a new OTA handle.
*/
sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){
sqlite3ota *p;
p = (sqlite3ota*)sqlite3_malloc(sizeof(sqlite3ota));
if( p ){
/* Open the target database */
memset(p, 0, sizeof(sqlite3ota));
otaOpenDatabase(p, &p->dbDest, zTarget);
otaOpenDatabase(p, &p->dbOta, zOta);
/* If it has not already been created, create the ota_state table */
if( p->rc==SQLITE_OK ){
p->rc = sqlite3_exec(p->dbOta, OTA_CREATE_STATE, 0, 0, &p->zErrmsg);
}
if( p->rc==SQLITE_OK ){
const char *zScript =
"PRAGMA ota_mode=1;"
"PRAGMA journal_mode=wal;"
"BEGIN IMMEDIATE;"
;
p->rc = sqlite3_exec(p->dbDest, zScript, 0, 0, &p->zErrmsg);
}
if( p->rc==SQLITE_OK ){
const char *zScript = "BEGIN IMMEDIATE";
p->rc = sqlite3_exec(p->dbOta, zScript, 0, 0, &p->zErrmsg);
}
/* Point the table iterator at the first table */
if( p->rc==SQLITE_OK ){
p->rc = tblIterFirst(p->dbOta, &p->tbliter);
}
if( p->rc==SQLITE_OK ){
otaLoadTransactionState(p);
}
}
return p;
}
static void otaCloseHandle(sqlite3 *db){
int rc = sqlite3_close(db);
assert( rc==SQLITE_OK );
}
int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){
int rc;
if( p ){
/* If the update has not been fully applied, save the state in
** the ota db. If successful, this call also commits the open
** transaction on the ota db. */
assert( p->rc!=SQLITE_ROW );
if( p->rc==SQLITE_OK ){
assert( p->zErrmsg==0 );
otaSaveTransactionState(p);
}
/* Close all open statement handles. */
tblIterFinalize(&p->tbliter);
idxIterFinalize(&p->idxiter);
/* If the ota update has been fully applied, commit the transaction
** on the target database. */
if( p->rc==SQLITE_DONE ){
rc = sqlite3_exec(p->dbDest, "COMMIT", 0, 0, &p->zErrmsg);
if( rc!=SQLITE_OK ) p->rc = rc;
}
rc = p->rc;
*pzErrmsg = p->zErrmsg;
otaCloseHandle(p->dbDest);
otaCloseHandle(p->dbOta);
sqlite3_free(p);
}else{
rc = SQLITE_NOMEM;
*pzErrmsg = 0;
}
return rc;
}
/**************************************************************************/
#ifdef SQLITE_TEST
#include <tcl.h>
/* From main.c (apparently...) */
extern const char *sqlite3ErrName(int);
static int test_sqlite3ota_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
int ret = TCL_OK;
sqlite3ota *pOta = (sqlite3ota*)clientData;
const char *azMethod[] = { "step", "close", 0 };
int iMethod;
if( objc!=2 ){
Tcl_WrongNumArgs(interp, 1, objv, "METHOD");
return TCL_ERROR;
}
if( Tcl_GetIndexFromObj(interp, objv[1], azMethod, "method", 0, &iMethod) ){
return TCL_ERROR;
}
switch( iMethod ){
case 0: /* step */ {
int rc = sqlite3ota_step(pOta);
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
break;
}
case 1: /* close */ {
char *zErrmsg = 0;
int rc;
Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
rc = sqlite3ota_close(pOta, &zErrmsg);
if( rc==SQLITE_OK || rc==SQLITE_DONE ){
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
assert( zErrmsg==0 );
}else{
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
if( zErrmsg ){
Tcl_AppendResult(interp, " - ", zErrmsg, 0);
sqlite3_free(zErrmsg);
}
ret = TCL_ERROR;
}
break;
}
default: /* seems unlikely */
assert( !"cannot happen" );
break;
}
return ret;
}
/*
** Tclcmd: sqlite3ota CMD <target-db> <ota-db>
*/
static int test_sqlite3ota(
ClientData clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
sqlite3ota *pOta = 0;
const char *zCmd;
const char *zTarget;
const char *zOta;
if( objc!=4 ){
Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB OTA-DB");
return TCL_ERROR;
}
zCmd = Tcl_GetString(objv[1]);
zTarget = Tcl_GetString(objv[2]);
zOta = Tcl_GetString(objv[3]);
pOta = sqlite3ota_open(zTarget, zOta);
Tcl_CreateObjCommand(interp, zCmd, test_sqlite3ota_cmd, (ClientData)pOta, 0);
Tcl_SetObjResult(interp, objv[1]);
return TCL_OK;
}
int SqliteOta_Init(Tcl_Interp *interp){
Tcl_CreateObjCommand(interp, "sqlite3ota", test_sqlite3ota, 0, 0);
return TCL_OK;
}
#endif /* ifdef SQLITE_TEST */

212
ext/ota/sqlite3ota.h Normal file
View File

@@ -0,0 +1,212 @@
/*
** 2014 August 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
**
** This file contains the public interface for the OTA extension.
*/
/*
** SUMMARY
**
** Writing a transaction containing a large number of operations on
** b-tree indexes that are collectively larger than the available cache
** memory can be very inefficient.
**
** The problem is that in order to update a b-tree, the leaf page (at least)
** containing the entry being inserted or deleted must be modified. If the
** working set of leaves is larger than the available cache memory, then a
** single leaf that is modified more than once as part of the transaction
** may be loaded from or written to the persistent media more than once.
** Additionally, because the index updates are likely to be applied in
** random order, access to pages within the databse is also likely to be in
** random order, which is itself quite inefficient.
**
** One way to improve the situation is to sort the operations on each index
** by index key before applying them to the b-tree. This leads to an IO
** pattern that resembles a single linear scan through the index b-tree,
** and all but guarantees each modified leaf page is loaded and stored
** exactly once. SQLite uses this trick to improve the performance of
** CREATE INDEX commands. This extension allows it to be used to improve
** the performance of large transactions on existing databases.
**
** Additionally, this extension allows the work involved in writing the
** large transaction to be broken down into sub-transactions performed
** sequentially by separate processes. This is useful if the system cannot
** guarantee that a single update process may run for long enough to apply
** the entire update, for example because the update is running on a mobile
** device that is frequently rebooted. Even after the writer process has
** committed one or more sub-transactions, other database clients continue
** to read from the original database snapshot. In other words, partially
** applied transactions are not visible to other clients.
**
** "OTA" stands for "Over The Air" update. As in a large database update
** transmitted via a wireless network to a mobile device. A transaction
** applied using this extension is hence refered to as an "OTA update".
**
**
** LIMITATIONS
**
** An "OTA update" transaction is subject to the following limitations:
**
** * The transaction must consist of INSERT, UPDATE and DELETE operations
** only.
**
** * INSERT statements may not use any default values.
**
** * UPDATE and DELETE statements must identify their target rows by
** real PRIMARY KEY values - i.e. INTEGER PRIMARY KEY columns or
** by the PRIMARY KEY columns of WITHOUT ROWID tables.
**
** * UPDATE statements may not modify real PRIMARY KEY columns.
**
** * No triggers will be fired.
**
** * No foreign key violations are detected or reported.
**
** * No constraint handling mode except for "OR ROLLBACK" is supported.
**
**
** PREPARATION
**
** An "OTA update" is stored as a separate SQLite database. A database
** containing an OTA update is an "OTA database". For each table in the
** target database to be updated, the OTA database should contain a table
** named "data_<target name>" containing the same set of columns as the
** target table, and one more - "ota_control". The data_% table should
** have no PRIMARY KEY or UNIQUE constraints, but each column should have
** the same type as the corresponding column in the target database.
** The "ota_control" column should have no type at all. For example, if
** the target database contains:
**
** CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c UNIQUE);
**
** Then the OTA database should contain:
**
** CREATE TABLE data_t1(a INTEGER, b TEXT, c, ota_control);
**
** The order of the columns in the data_% table does not matter.
**
** For each row to INSERT into the target database as part of the OTA
** update, the corresponding data_% table should contain a single record
** with the "ota_control" column set to contain integer value 0. The
** other columns should be set to the values that make up the new record
** to insert.
**
** For each row to DELETE from the target database as part of the OTA
** update, the corresponding data_% table should contain a single record
** with the "ota_control" column set to contain integer value 1. The
** real primary key values of the row to delete should be stored in the
** corresponding columns of the data_% table. The values stored in the
** other columns are not used.
**
** For each row to DELETE from the target database as part of the OTA
** update, the corresponding data_% table should contain a single record
** with the "ota_control" column set to contain a value of type text.
** The real primary key values identifying the row to update should be
** stored in the corresponding columns of the data_% table row, as should
** the new values of all columns being update. The text value in the
** "ota_control" column must contain the same number of characters as
** there are column in the target database table, and must consist entirely
** of "x" and "." characters. For each column that is being updated,
** the corresponding character is set to "x". For those that remain as
** they are, the corresponding character of the ota_control value should
** be set to ".". For example, given the tables above, the update
** statement:
**
** UPDATE t1 SET c = 'usa' WHERE a = 4;
**
** is represented by the data_t1 row created by:
**
** INSERT INTO data_t1(a, b, c, ota_control) VALUES(4, NULL, 'usa', '..x');
**
**
** USAGE
**
** The API declared below allows an application to apply an OTA update
** stored on disk to an existing target database. Essentially, the
** application:
**
** 1) Opens an OTA handle using the sqlite3ota_open() function.
**
** 2) Calls the sqlite3ota_step() function one or more times on
** the new handle. Each call to sqlite3ota_step() performs a single
** b-tree operation, so thousands of calls may be required to apply
** a complete update.
**
** 3) Calls sqlite3ota_close() to close the OTA update handle. If
** sqlite3ota_step() has been called enough times to completely
** apply the update to the target database, then it is committed
** and made visible to other database clients at this point.
** Otherwise, the state of the OTA update application is saved
** in the OTA database for later resumption.
**
** See comments below for more detail on APIs.
**
** If an update is only partially applied to the target database by the
** time sqlite3ota_close() is called, various state information is saved
** within the OTA database. This allows subsequent processes to automatically
** resume the OTA update from where it left off.
**
** To remove all OTA extension state information, returning an OTA database
** to its original contents, it is sufficient to drop all tables that begin
** with the prefix "ota_"
*/
#ifndef _SQLITE3OTA_H
#define _SQLITE3OTA_H
typedef struct sqlite3ota sqlite3ota;
/*
** Open an OTA handle.
**
** Argument zTarget is the path to the target database. Argument zOta is
** the path to the OTA database. Each call to this function must be matched
** by a call to sqlite3ota_close().
*/
sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta);
/*
** Do some work towards applying the OTA update to the target db.
**
** Return SQLITE_DONE if the update has been completely applied, or
** SQLITE_OK if no error occurs but there remains work to do to apply
** the OTA update. If an error does occur, some other error code is
** returned.
**
** Once a call to sqlite3ota_step() has returned a value other than
** SQLITE_OK, all subsequent calls on the same OTA handle are no-ops
** that immediately return the same value.
*/
int sqlite3ota_step(sqlite3ota *pOta);
/*
** Close an OTA handle.
**
** If the OTA update has been completely applied, commit it to the target
** database. Otherwise, assuming no error has occurred, save the current
** state of the OTA update appliation to the OTA database.
**
** If an error has already occurred as part of an sqlite3ota_step()
** or sqlite3ota_open() call, or if one occurs within this function, an
** SQLite error code is returned. Additionally, *pzErrmsg may be set to
** point to a buffer containing a utf-8 formatted English language error
** message. It is the responsibility of the caller to eventually free any
** such buffer using sqlite3_free().
**
** Otherwise, if no error occurs, this function returns SQLITE_OK if the
** update has been partially applied, or SQLITE_DONE if it has been
** completely applied.
*/
int sqlite3ota_close(sqlite3ota *pOta, char **pzErrmsg);
#endif /* _SQLITE3OTA_H */

3
main.mk
View File

@@ -331,7 +331,8 @@ TESTSRC2 = \
$(TOP)/ext/fts3/fts3_expr.c \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/fts3/fts3_write.c \
$(TOP)/ext/async/sqlite3async.c
$(TOP)/ext/async/sqlite3async.c \
$(TOP)/ext/ota/sqlite3ota.c
# Header files used by all library source files.
#

51
manifest
View File

@@ -1,5 +1,5 @@
C Update\scomments\sin\sthe\sANALYZE\scommand\sthat\sdescribe\show\sthe\sStat4Accum\nobjecct\sis\spassed\saround\swithin\sthe\sVDBE.\s\sNo\schanges\sto\sfunctional\scode.
D 2014-09-01T23:06:44.401
C Add\san\sexperimental\sextension\sfor\sapplying\sbulk\supdates\sto\sdatabases.
D 2014-09-02T19:59:40.729
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -121,6 +121,10 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e
F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
F ext/ota/ota1.test ea2865997ce573fadaf12eb0a0f80ef22d9dd77f
F ext/ota/ota2.test 4f7abfe1dfb7c3709bf45e94f3e65f3839b4f115
F ext/ota/sqlite3ota.c ad55821883e4110367a30ffca282032d2bf36e45
F ext/ota/sqlite3ota.h d3187a98fe1e3445c58f7a27d96ac385b78486a1
F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b
F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -147,7 +151,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
F main.mk 9b5ccf1097050b1f16681f7d4beeea4f7f7ac2c3
F main.mk 566c36f247d19525264e584466b5f07c0a48302e
F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5
F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@@ -168,7 +172,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e
F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb
F src/btmutex.c ec9d3f1295dafeb278c3830211cc5584132468f4
F src/btree.c 2a483a8045118faa99867a8679da42754b532318
F src/btree.c c46043fbb09c18a19bdb96eadde6e724901d6fcf
F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8
F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3
F src/build.c c26b233dcdb1e2c8f468d49236c266f9f3de96d8
@@ -185,12 +189,12 @@ F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4
F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5
F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
F src/insert.c d1a104e67b33314d4cc5c1356147446086ab9fc8
F src/insert.c 62b0ceab1720dc74ed1fbcf953224132245704d8
F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a
F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b
F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab
F src/main.c d2ef03a45552e11813c68326d5edfda992e319d4
F src/main.c c9802dc99c019fbba516202300d56be2c478fa93
F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b
@@ -211,30 +215,30 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
F src/os_unix.c 8525ca79457c5b4673a5fda2774ee39fe155f40f
F src/os_win.c 2aa8aa7780d7cf03e912d2088ab2ec5c32f33dc5
F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21
F src/pager.c 3e732d2bbdd8d8d95fed0c5ae7e718d73153c4c5
F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428
F src/pager.c a3caa08db8227c5a32f388be67f33d8cb44d5e35
F src/pager.h 1acd367a0ffb63026b0461ea5eaeeb8046414a71
F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0
F src/pcache.c 3b3791297e8977002e56b4a9b8916f2039abad9b
F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a
F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c
F src/pragma.c 14bcdb504128a476cce5bbc086d5226c5e46c225
F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d
F src/pragma.c d252459fb3ce19448d1a2f41000c780fac4c0c26
F src/prepare.c 314961aa6650cc860394cb2f31931cf2de93baa8
F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74
F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece
F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697
F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be
F src/select.c 89e569b263535662f54b537eb9118b2c554ae7aa
F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50
F src/sqlite.h.in 43852c8b68b4c579948cb37182918078836c5c06
F src/sqlite.h.in 706b420dc3390532435a3bd196360a940805f02f
F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
F src/sqliteInt.h 6244ee9052752e26d1275ab20c9b774385aa57d2
F src/sqliteInt.h 7c090825333d91ca392c2479a9e835e7b6a5eb12
F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158
F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
F src/tclsqlite.c 7d100e2e7aad614bb3d7026a41a0e3827dbaaebc
F src/test1.c 363a5089230a92cf0aaa7a2945da7f2bf3b0a8d3
F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712
F src/tclsqlite.c 29357f2be7b0d00e8ea900eaf727e0c5ffeaa660
F src/test1.c e9a0e5804b078532e69e69ec14c8326bf2cfc318
F src/test2.c 84f6a786aa7ffa12fff83acb52660e337ffe642a
F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c
F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df
F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1
@@ -290,13 +294,13 @@ F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8
F src/vdbeInt.h cdc8e421f85beb1ac9b4669ec5beadab6faa15e0
F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441
F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36
F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4
F src/vdbeblob.c 0bc9d22578d87ad9ff1c16e20a36863326f34fd7
F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62
F src/vdbesort.c 02646a9f86421776ae5d7594f620f9ed669d3698
F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767
F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f
F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a
F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
F src/wal.c 93b4fcb56a98f435a2cb66024bb2b12d66d1ff53
F src/wal.h 237bc4484f7c289f094ecb6efb2b6c02005484e1
F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45
F src/where.c d9eae96b2cbbe4842eac3ee156ccd1b933d802c4
F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3
@@ -1193,7 +1197,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 4cae93f8ae8fb3fe38fd5dc7d3a5ea0d11552841
R fe37d23fb1f96547febf2ff7c0ba49b9
U drh
Z 29e2a2847ff584a00093c5b89ffca704
P 9779c7a9eb1e2bd36e9286331a9314f064014d80
R 5df4197c767caad19feca7391f16a42d
T *branch * experimental-bulk-update
T *sym-experimental-bulk-update *
T -sym-trunk *
U dan
Z 195de077202fec58ba90fc725d7305e7

2
manifest.uuid
View File

@@ -1 +1 @@
9779c7a9eb1e2bd36e9286331a9314f064014d80
2954ab501049968430011b63d046eb42ff37a56c

7
src/btree.c
View File

@@ -151,7 +151,8 @@ static int hasSharedCacheTableLock(
** and has the read-uncommitted flag set, then no lock is required.
** Return true immediately.
*/
if( (pBtree->sharable==0)
if( (pBtree->db->flags & SQLITE_OtaMode)
|| (pBtree->sharable==0)
|| (eLockType==READ_LOCK && (pBtree->db->flags & SQLITE_ReadUncommitted))
){
return 1;
@@ -2045,7 +2046,7 @@ int sqlite3BtreeOpen(
btree_open_out:
if( rc!=SQLITE_OK ){
if( pBt && pBt->pPager ){
sqlite3PagerClose(pBt->pPager);
sqlite3PagerClose(pBt->pPager, 0);
}
sqlite3_free(pBt);
sqlite3_free(p);
@@ -2174,7 +2175,7 @@ int sqlite3BtreeClose(Btree *p){
** Clean out and delete the BtShared object.
*/
assert( !pBt->pCursor );
sqlite3PagerClose(pBt->pPager);
sqlite3PagerClose(pBt->pPager, (p->db->flags & SQLITE_OtaMode)!=0);
if( pBt->xFreeSchema && pBt->pSchema ){
pBt->xFreeSchema(pBt->pSchema);
}

13
src/insert.c
View File

@@ -1365,6 +1365,10 @@ void sqlite3GenerateConstraintChecks(
int iThisCur; /* Cursor for this UNIQUE index */
int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */
/* If the "ota_mode" flag is set, ignore all indexes except the PK
** index of WITHOUT ROWID tables. */
if( (db->flags & SQLITE_OtaMode) && pIdx!=pPk) continue;
if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */
if( bAffinityDone==0 ){
sqlite3TableAffinity(v, pTab, regNewData+1);
@@ -1556,6 +1560,15 @@ void sqlite3CompleteInsertion(
assert( pTab->pSelect==0 ); /* This table is not a VIEW */
for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){
if( aRegIdx[i]==0 ) continue;
/* If the "ota_mode" flag is set, ignore all indexes except the PK
** index of WITHOUT ROWID tables. */
if( (pParse->db->flags & SQLITE_OtaMode)
&& (pTab->iPKey>=0 || pIdx->idxType!=SQLITE_IDXTYPE_PRIMARYKEY)
){
continue;
}
bAffinityDone = 1;
if( pIdx->pPartIdxWhere ){
sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2);

12
src/main.c
View File

@@ -3470,3 +3470,15 @@ int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){
Btree *pBt = sqlite3DbNameToBtree(db, zDbName);
return pBt ? sqlite3BtreeIsReadonly(pBt) : -1;
}
int sqlite3_transaction_save(sqlite3 *db, void **ppState, int *pnState){
Pager *pPager = sqlite3BtreePager(db->aDb[0].pBt);
return sqlite3PagerSaveState(pPager, ppState, pnState);
}
int sqlite3_transaction_restore(sqlite3 *db, const void *pState, int nState){
Pager *pPager = sqlite3BtreePager(db->aDb[0].pBt);
return sqlite3PagerRestoreState(pPager, pState, nState);
}

41
src/pager.c
View File

@@ -3947,7 +3947,7 @@ static void pagerFreeMapHdrs(Pager *pPager){
** a hot journal may be left in the filesystem but no error is returned
** to the caller.
*/
int sqlite3PagerClose(Pager *pPager){
int sqlite3PagerClose(Pager *pPager, int bOtaMode){
u8 *pTmp = (u8 *)pPager->pTmpSpace;
assert( assert_pager_state(pPager) );
@@ -3957,7 +3957,9 @@ int sqlite3PagerClose(Pager *pPager){
/* pPager->errCode = 0; */
pPager->exclusiveMode = 0;
#ifndef SQLITE_OMIT_WAL
sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp);
sqlite3WalClose(
pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (bOtaMode?0:pTmp)
);
pPager->pWal = 0;
#endif
pager_reset(pPager);
@@ -7217,6 +7219,41 @@ int sqlite3PagerCloseWal(Pager *pPager){
return rc;
}
int sqlite3PagerSaveState(Pager *pPager, void **ppState, int *pnState){
int rc = SQLITE_OK;
*ppState = 0;
*pnState = 0;
if( pPager->pWal==0 || pPager->eState<PAGER_WRITER_LOCKED ){
rc = SQLITE_ERROR;
}else{
/* Flush all dirty pages to the wal. */
PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache);
rc = sqlite3WalFrames(pPager->pWal,
pPager->pageSize, pList, 0, 0, pPager->walSyncFlags
);
if( rc==SQLITE_OK ){
rc = sqlite3WalSaveState(pPager->pWal, ppState, pnState);
}
}
return rc;
}
int sqlite3PagerRestoreState(Pager *pPager, const void *pState, int nState){
int rc = SQLITE_OK;
if( pPager->pWal==0
|| pPager->eState<PAGER_WRITER_LOCKED
|| sqlite3PcacheDirtyList(pPager->pPCache)
){
rc = SQLITE_ERROR;
}else{
sqlite3PcacheTruncate(pPager->pPCache, 1);
rc = sqlite3WalRestoreState(pPager->pWal, pState, nState);
pPager->eState = PAGER_WRITER_CACHEMOD;
}
return rc;
}
#endif /* !SQLITE_OMIT_WAL */
#ifdef SQLITE_ENABLE_ZIPVFS

5
src/pager.h
View File

@@ -112,7 +112,7 @@ int sqlite3PagerOpen(
int,
void(*)(DbPage*)
);
int sqlite3PagerClose(Pager *pPager);
int sqlite3PagerClose(Pager *pPager, int);
int sqlite3PagerReadFileheader(Pager*, int, unsigned char*);
/* Functions used to configure a Pager object. */
@@ -207,4 +207,7 @@ void *sqlite3PagerCodec(DbPage *);
# define enable_simulated_io_errors()
#endif
int sqlite3PagerSaveState(Pager *pPager, void **ppState, int *pnState);
int sqlite3PagerRestoreState(Pager *pPager, const void *pState, int nState);
#endif /* _PAGER_H_ */

12
src/pragma.c
View File

@@ -308,6 +308,12 @@ static const struct sPragmaNames {
/* ePragTyp: */ PragTyp_MMAP_SIZE,
/* ePragFlag: */ 0,
/* iArg: */ 0 },
#endif
{ /* zName: */ "ota_mode",
/* ePragTyp: */ PragTyp_FLAG,
/* ePragFlag: */ 0,
/* iArg: */ SQLITE_OtaMode },
#if !defined(SQLITE_OMIT_PAGER_PRAGMAS)
{ /* zName: */ "page_count",
/* ePragTyp: */ PragTyp_PAGE_COUNT,
/* ePragFlag: */ PragFlag_NeedSchema,
@@ -1467,7 +1473,11 @@ void sqlite3Pragma(
}else if( pPk==0 ){
k = 1;
}else{
for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){}
if( (db->flags & SQLITE_OtaMode) && HasRowid(pTab) ){
k = 0;
}else{
for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){}
}
}
sqlite3VdbeAddOp2(v, OP_Integer, k, 6);
sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6);

1
src/prepare.c
View File

@@ -796,7 +796,6 @@ int sqlite3_prepare_v2(
return rc;
}
#ifndef SQLITE_OMIT_UTF16
/*
** Compile the UTF-16 encoded SQL statement zSql into a statement handle.

72
src/sqlite.h.in
View File

@@ -7363,7 +7363,79 @@ int sqlite3_vtab_on_conflict(sqlite3 *);
/* #define SQLITE_ABORT 4 // Also an error code */
#define SQLITE_REPLACE 5
/*
** Allocate a statement handle that may be used to write directly to an
** index b-tree. This allows the user to create a corrupt database. Once
** the statement handle is allocated, it may be used with the same APIs
** as any statement handle created with sqlite3_prepare().
**
** The statement writes to the index specified by parameter zIndex, which
** must be in the "main" database. If argument bDelete is false, then each
** time the statement is sqlite3_step()ed, an entry is inserted into the
** b-tree index. If it is true, then an entry may be deleted (or may not, if
** the specified key is not found) each time the statement is
** sqlite3_step()ed.
**
** If statement compilation is successful, *ppStmt is set to point to the
** new statement handle and SQLITE_OK is returned. Otherwise, if an error
** occurs, *ppStmt is set to NULL and an error code returned. An error
** message may be left in the database handle in this case.
**
** If statement compilation succeeds, output variable *pnCol is set to the
** total number of columns in the index, including the primary key columns
** at the end. Variable *paiCol is set to point to an array *pnCol entries
** in size. Each entry is the table column index, numbered from zero from left
** to right, of the corresponding index column. For example, if:
**
** CREATE TABLE t1(a, b, c, d);
** CREATE INDEX i1 ON t1(b, c);
**
** then *pnCol is 3 and *paiCol points to an array containing {1, 2, -1}.
** If table t1 had an explicit INTEGER PRIMARY KEY, then the "-1" in the
** *paiCol array would be replaced by its column index. Or if:
**
** CREATE TABLE t2(a, b, c, d, PRIMARY KEY(d, c)) WITHOUT ROWID;
** CREATE INDEX i2 ON t2(a);
**
** then (*pnCol) is 3 and *paiCol points to an array containing {0, 3, 2}.
**
** The lifetime of the array is the same as that of the statement handle -
** it is automatically freed when the statement handle is passed to
** sqlite3_finalize().
**
** The statement has (*pnCol) SQL variables that values may be bound to.
** They correspond to the values used to create the index key that is
** inserted or deleted when the statement is stepped.
**
** If the index is a UNIQUE index, the usual checking and error codes apply
** to insert operations.
*/
int sqlite3_index_writer(
sqlite3 *db,
int bDelete, /* Zero for insert, non-zero for delete */
const char *zIndex, /* Index to write to */
sqlite3_stmt**, /* OUT: New statement handle */
int **paiCol, int *pnCol /* OUT: See above */
);
/*
** This function is used to save the state of an ongoing WAL mode write
** transaction on the "main" database of the supplied database handle.
**
** If successful, SQLITE_OK is returned and output variable (*ppState)
** is set to point to a buffer containing the transaction state data.
** (*pnState) is set to the size of that buffer in bytes. Otherwise, if
** an error occurs, an SQLite error code is returned and both output
** variables are zeroed.
**
** A transaction state may be saved if:
**
** * the transaction does not contain any schema modifications.
** * there are no open sub-transactions.
*/
int sqlite3_transaction_save(sqlite3 *db, void **ppState, int *pnState);
int sqlite3_transaction_restore(sqlite3 *db, const void *pState, int nState);
/*
** Undo the hack that converts floating point types to integer for

3
src/sqliteInt.h
View File

@@ -1141,6 +1141,8 @@ struct sqlite3 {
#define SQLITE_QueryOnly 0x02000000 /* Disable database changes */
#define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */
#define SQLITE_OtaMode 0x08000000 /* True in "ota mode" */
/*
** Bits of the sqlite3.dbOptFlags field that are used by the
@@ -3726,6 +3728,7 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...);
#define MEMTYPE_PCACHE 0x08 /* Page cache allocations */
#define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */
/*
** Threading interface
*/

3
src/tclsqlite.c
View File

@@ -3699,6 +3699,8 @@ static void init_all(Tcl_Interp *interp){
extern int SqliteSuperlock_Init(Tcl_Interp*);
extern int SqlitetestSyscall_Init(Tcl_Interp*);
extern int SqliteOta_Init(Tcl_Interp*);
#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
#endif
@@ -3740,6 +3742,7 @@ static void init_all(Tcl_Interp *interp){
Sqlitemultiplex_Init(interp);
SqliteSuperlock_Init(interp);
SqlitetestSyscall_Init(interp);
SqliteOta_Init(interp);
#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
Sqlitetestfts3_Init(interp);

66
src/test1.c
View File

@@ -6497,6 +6497,70 @@ static int sorter_test_sort4_helper(
}
/*
** tclcmd: sqlite3_transaction_save DB
*/
static int testTransactionSave(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
void *pState;
int nState;
sqlite3 *db;
int rc;
if( objc!=2 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB");
return TCL_ERROR;
}
if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
rc = sqlite3_transaction_save(db, &pState, &nState);
if( rc==SQLITE_OK ){
Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pState, nState));
}else{
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
return TCL_ERROR;
}
sqlite3_free(pState);
return TCL_OK;
}
/*
** tclcmd: sqlite3_transaction_restore DB BLOB
*/
static int testTransactionRestore(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
void *pState;
int nState;
sqlite3 *db;
int rc;
if( objc!=3 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB BLOB");
return TCL_ERROR;
}
if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
pState = (void*)Tcl_GetByteArrayFromObj(objv[2], &nState);
rc = sqlite3_transaction_restore(db, pState, nState);
if( rc==SQLITE_OK ){
Tcl_ResetResult(interp);
}else{
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
return TCL_ERROR;
}
return TCL_OK;
}
/*
** Register commands with the TCL interpreter.
*/
@@ -6734,6 +6798,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
{ "load_static_extension", tclLoadStaticExtensionCmd },
{ "sorter_test_fakeheap", sorter_test_fakeheap },
{ "sorter_test_sort4_helper", sorter_test_sort4_helper },
{ "sqlite3_transaction_save", testTransactionSave },
{ "sqlite3_transaction_restore", testTransactionRestore },
};
static int bitmask_size = sizeof(Bitmask)*8;
int i;

2
src/test2.c
View File

@@ -89,7 +89,7 @@ static int pager_close(
return TCL_ERROR;
}
pPager = sqlite3TestTextToPtr(argv[1]);
rc = sqlite3PagerClose(pPager);
rc = sqlite3PagerClose(pPager, 0);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
return TCL_ERROR;

112
src/vdbeblob.c
View File

@@ -463,4 +463,116 @@ int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){
return rc;
}
int sqlite3_index_writer(
sqlite3 *db,
int bDelete,
const char *zIndex,
sqlite3_stmt **ppStmt,
int **paiCol, int *pnCol
){
int rc = SQLITE_OK;
Parse *pParse = 0;
Index *pIdx = 0; /* The index to write to */
Table *pTab;
int i; /* Used to iterate through index columns */
Vdbe *v = 0;
int regRec; /* Register to assemble record in */
int *aiCol = 0;
sqlite3_mutex_enter(db->mutex);
sqlite3BtreeEnterAll(db);
/* Allocate the parse context */
pParse = sqlite3StackAllocRaw(db, sizeof(*pParse));
if( !pParse ) goto index_writer_out;
memset(pParse, 0, sizeof(Parse));
pParse->db = db;
/* Allocate the Vdbe */
v = sqlite3GetVdbe(pParse);
if( v==0 ) goto index_writer_out;
/* Find the index to write to */
pIdx = sqlite3FindIndex(db, zIndex, "main");
if( pIdx==0 ){
sqlite3ErrorMsg(pParse, "no such index: %s", zIndex);
goto index_writer_out;
}
pTab = pIdx->pTable;
/* Populate the two output variables, *pnCol and *pnAiCol. */
*pnCol = pIdx->nColumn;
*paiCol = aiCol = sqlite3DbMallocZero(db, sizeof(int) * pIdx->nColumn);
if( aiCol==0 ){
rc = SQLITE_NOMEM;
goto index_writer_out;
}
for(i=0; i<pIdx->nKeyCol; i++){
aiCol[i] = pIdx->aiColumn[i];
}
if( !HasRowid(pTab) ){
Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable);
assert( pIdx->nColumn==pIdx->nKeyCol+pPk->nKeyCol );
if( pPk==pIdx ){
rc = SQLITE_ERROR;
goto index_writer_out;
}
for(i=0; i<pPk->nKeyCol; i++){
aiCol[pIdx->nKeyCol+i] = pPk->aiColumn[i];
}
}else{
assert( pIdx->nColumn==pIdx->nKeyCol+1 );
aiCol[i] = pTab->iPKey;
}
/* Add an OP_Noop to the VDBE program. Then store a pointer to the
** output array *paiCol as its P4 value. This is so that the array
** is automatically deleted when the user finalizes the statement. The
** OP_Noop serves no other purpose. */
sqlite3VdbeAddOp0(v, OP_Noop);
sqlite3VdbeChangeP4(v, -1, (const char*)aiCol, P4_INTARRAY);
sqlite3BeginWriteOperation(pParse, 0, 0);
/* Open a write cursor on the index */
pParse->nTab = 1;
sqlite3VdbeAddOp3(v, OP_OpenWrite, 0, pIdx->tnum, 0);
sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
/* Create the record to insert into the index. Store it in register regRec. */
pParse->nVar = pIdx->nColumn;
pParse->nMem = pIdx->nColumn;
for(i=1; i<=pIdx->nColumn; i++){
sqlite3VdbeAddOp2(v, OP_Variable, i, i);
}
regRec = ++pParse->nMem;
sqlite3VdbeAddOp3(v, OP_MakeRecord, 1, pIdx->nColumn, regRec);
/* If this is a UNIQUE index, check the constraint. */
if( pIdx->onError ){
int addr = sqlite3VdbeAddOp4Int(v, OP_NoConflict, 0, 0, 1, pIdx->nKeyCol);
sqlite3UniqueConstraint(pParse, SQLITE_ABORT, pIdx);
sqlite3VdbeJumpHere(v, addr);
}
/* Code the IdxInsert to write to the b-tree index. */
sqlite3VdbeAddOp2(v, OP_IdxInsert, 0, regRec);
sqlite3FinishCoding(pParse);
index_writer_out:
if( rc==SQLITE_OK && db->mallocFailed==0 ){
*ppStmt = (sqlite3_stmt*)v;
}else{
*ppStmt = 0;
if( v ) sqlite3VdbeFinalize(v);
}
sqlite3ParserReset(pParse);
sqlite3StackFree(db, pParse);
sqlite3BtreeLeaveAll(db);
rc = sqlite3ApiExit(db, rc);
sqlite3_mutex_leave(db->mutex);
return rc;
}
#endif /* #ifndef SQLITE_OMIT_INCRBLOB */

306
src/wal.c
View File

@@ -1046,6 +1046,62 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){
return rc;
}
static int walFileReadHdr(Wal *pWal, int *pbValid){
u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */
int rc; /* Return code */
u32 magic; /* Magic value read from WAL header */
int szPage; /* Page size according to the log */
u32 version; /* Magic value read from WAL header */
*pbValid = 0;
/* Read in the WAL header. */
rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0);
if( rc!=SQLITE_OK ){
return rc;
}
/* If the database page size is not a power of two, or is greater than
** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid
** data. Similarly, if the 'magic' value is invalid, ignore the whole
** WAL file.
*/
magic = sqlite3Get4byte(&aBuf[0]);
szPage = sqlite3Get4byte(&aBuf[8]);
if( (magic&0xFFFFFFFE)!=WAL_MAGIC
|| szPage&(szPage-1)
|| szPage>SQLITE_MAX_PAGE_SIZE
|| szPage<512
){
return SQLITE_OK;
}
pWal->hdr.bigEndCksum = (u8)(magic&0x00000001);
pWal->szPage = szPage;
pWal->nCkpt = sqlite3Get4byte(&aBuf[12]);
memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);
/* Verify that the WAL header checksum is correct */
walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN,
aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum
);
if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24])
|| pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28])
){
return SQLITE_OK;
}
/* Verify that the version number on the WAL format is one that
** are able to understand */
version = sqlite3Get4byte(&aBuf[4]);
if( version!=WAL_MAX_VERSION ){
return SQLITE_CANTOPEN_BKPT;
}
*pbValid = 1;
return SQLITE_OK;
}
/*
** Recover the wal-index by reading the write-ahead log file.
@@ -1090,59 +1146,18 @@ static int walIndexRecover(Wal *pWal){
}
if( nSize>WAL_HDRSIZE ){
u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */
u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */
int szFrame; /* Number of bytes in buffer aFrame[] */
u8 *aData; /* Pointer to data part of aFrame buffer */
int iFrame; /* Index of last frame read */
i64 iOffset; /* Next offset to read from log file */
int szPage; /* Page size according to the log */
u32 magic; /* Magic value read from WAL header */
u32 version; /* Magic value read from WAL header */
int isValid; /* True if this frame is valid */
/* Read in the WAL header. */
rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0);
if( rc!=SQLITE_OK ){
goto recovery_error;
}
/* If the database page size is not a power of two, or is greater than
** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid
** data. Similarly, if the 'magic' value is invalid, ignore the whole
** WAL file.
*/
magic = sqlite3Get4byte(&aBuf[0]);
szPage = sqlite3Get4byte(&aBuf[8]);
if( (magic&0xFFFFFFFE)!=WAL_MAGIC
|| szPage&(szPage-1)
|| szPage>SQLITE_MAX_PAGE_SIZE
|| szPage<512
){
goto finished;
}
pWal->hdr.bigEndCksum = (u8)(magic&0x00000001);
pWal->szPage = szPage;
pWal->nCkpt = sqlite3Get4byte(&aBuf[12]);
memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);
/* Verify that the WAL header checksum is correct */
walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN,
aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum
);
if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24])
|| pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28])
){
goto finished;
}
/* Verify that the version number on the WAL format is one that
** are able to understand */
version = sqlite3Get4byte(&aBuf[4]);
if( version!=WAL_MAX_VERSION ){
rc = SQLITE_CANTOPEN_BKPT;
goto finished;
}
rc = walFileReadHdr(pWal, &isValid);
if( rc!=SQLITE_OK ) goto recovery_error;
if( isValid==0 ) goto finished;
szPage = pWal->szPage;
/* Malloc a buffer to read frames into. */
szFrame = szPage + WAL_FRAME_HDRSIZE;
@@ -1837,32 +1852,34 @@ int sqlite3WalClose(
**
** The EXCLUSIVE lock is not released before returning.
*/
rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE);
if( rc==SQLITE_OK ){
if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
}
rc = sqlite3WalCheckpoint(
pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
);
if( zBuf ){
rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE);
if( rc==SQLITE_OK ){
int bPersist = -1;
sqlite3OsFileControlHint(
pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist
if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
}
rc = sqlite3WalCheckpoint(
pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
);
if( bPersist!=1 ){
/* Try to delete the WAL file if the checkpoint completed and
** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal
** mode (!bPersist) */
isDelete = 1;
}else if( pWal->mxWalSize>=0 ){
/* Try to truncate the WAL file to zero bytes if the checkpoint
** completed and fsynced (rc==SQLITE_OK) and we are in persistent
** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a
** non-negative value (pWal->mxWalSize>=0). Note that we truncate
** to zero bytes as truncating to the journal_size_limit might
** leave a corrupt WAL file on disk. */
walLimitSize(pWal, 0);
if( rc==SQLITE_OK ){
int bPersist = -1;
sqlite3OsFileControlHint(
pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist
);
if( bPersist!=1 ){
/* Try to delete the WAL file if the checkpoint completed and
** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal
** mode (!bPersist) */
isDelete = 1;
}else if( pWal->mxWalSize>=0 ){
/* Try to truncate the WAL file to zero bytes if the checkpoint
** completed and fsynced (rc==SQLITE_OK) and we are in persistent
** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a
** non-negative value (pWal->mxWalSize>=0). Note that we truncate
** to zero bytes as truncating to the journal_size_limit might
** leave a corrupt WAL file on disk. */
walLimitSize(pWal, 0);
}
}
}
}
@@ -3079,6 +3096,157 @@ int sqlite3WalHeapMemory(Wal *pWal){
return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE );
}
/*
** Save current transaction state.
**
** The transaction state consists of a series of 32-bit big-endian integers:
**
** * initial number of frames in WAL file.
** * initial checksum values (2 integers).
** * current number of frames.
** * current checksum values (2 integers).
*/
int sqlite3WalSaveState(Wal *pWal, void **ppState, int *pnState){
int rc = SQLITE_OK;
*ppState = 0;
*pnState = 0;
if( pWal->writeLock==0 ){
/* Must be in a write transaction to call this function. */
rc = SQLITE_ERROR;
}else{
WalIndexHdr *pOrig = (WalIndexHdr*)walIndexHdr(pWal);
int nBuf = 6 * 4; /* Bytes of space to allocate */
u8 *aBuf;
aBuf = sqlite3_malloc(nBuf);
if( aBuf==0 ){
rc = SQLITE_NOMEM;
}else{
sqlite3Put4byte(&aBuf[0], pOrig->mxFrame);
sqlite3Put4byte(&aBuf[4], pOrig->aFrameCksum[0]);
sqlite3Put4byte(&aBuf[8], pOrig->aFrameCksum[1]);
sqlite3Put4byte(&aBuf[12], pWal->hdr.mxFrame);
sqlite3Put4byte(&aBuf[16], pWal->hdr.aFrameCksum[0]);
sqlite3Put4byte(&aBuf[20], pWal->hdr.aFrameCksum[1]);
*ppState = (void*)aBuf;
*pnState = nBuf;
}
}
return rc;
}
static int walUndoNoop(void *pUndoCtx, Pgno pgno){
UNUSED_PARAMETER(pUndoCtx);
UNUSED_PARAMETER(pgno);
return SQLITE_OK;
}
/*
** If possible, restore the state of the curent transaction to that
** described by the second and third arguments.
*/
int sqlite3WalRestoreState(Wal *pWal, const void *pState, int nState){
int rc = SQLITE_OK;
if( pWal->writeLock==0 ){
/* Must have opened a write transaction to call this */
rc = SQLITE_ERROR;
}else{
u8 *aBuf = (u8*)pState;
int szFrame; /* Size of each frame in WAL file */
u8 *aFrame = 0; /* Buffer to read data into */
u8 *aData; /* Data part of aFrame[] buffer */
u32 mxFrame; /* Maximum frame following restoration */
int i; /* Iterator variable */
WalIndexHdr *pOrig = (WalIndexHdr*)walIndexHdr(pWal);
/* Check that no dirty pages have been written to the WAL file since
** the current transaction was opened. */
if( pOrig->mxFrame!=pWal->hdr.mxFrame
|| pOrig->aFrameCksum[0]!=pWal->hdr.aFrameCksum[0]
|| pOrig->aFrameCksum[1]!=pWal->hdr.aFrameCksum[1]
){
rc = SQLITE_ERROR;
}
/* Check that the WAL file is in the same state that it was when the
** transaction was saved. If not, return SQLITE_MISMATCH - cannot
** resume this transaction */
if( rc==SQLITE_OK && (
pWal->hdr.mxFrame!=sqlite3Get4byte(&aBuf[0])
|| pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[4])
|| pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[8])
)){
rc = SQLITE_MISMATCH;
}
if( rc==SQLITE_OK && pWal->readLock==0 ){
int cnt = 0;
walUnlockShared(pWal, WAL_READ_LOCK(0));
pWal->readLock = -1;
do{
int notUsed;
rc = walTryBeginRead(pWal, &notUsed, 1, ++cnt);
}while( rc==WAL_RETRY );
if( rc==SQLITE_OK ){
int bValid;
rc = walFileReadHdr(pWal, &bValid);
if( rc==SQLITE_OK && bValid==0 ) rc = SQLITE_MISMATCH;
pWal->hdr.szPage = (u16)((pWal->szPage&0xff00) | (pWal->szPage>>16));
}
}
/* Malloc a buffer to read frames into. */
if( rc==SQLITE_OK ){
szFrame = pWal->szPage + WAL_FRAME_HDRSIZE;
aFrame = (u8*)sqlite3_malloc(szFrame);
if( !aFrame ){
rc = SQLITE_NOMEM;
}else{
aData = &aFrame[WAL_FRAME_HDRSIZE];
}
}
mxFrame = sqlite3Get4byte(&aBuf[12]);
for(i=pWal->hdr.mxFrame+1; rc==SQLITE_OK && i<=mxFrame; i++){
sqlite3_int64 iOff = walFrameOffset(i, pWal->szPage);
rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOff);
if( rc==SQLITE_OK ){
u32 iPg;
u32 dummy;
if( 0==walDecodeFrame(pWal, &iPg, &dummy, aData, aFrame) ){
rc = SQLITE_MISMATCH;
}else{
rc = walIndexAppend(pWal, i, iPg);
if( iPg>pWal->hdr.nPage ) pWal->hdr.nPage = iPg;
}
pWal->hdr.mxFrame = i;
}
}
sqlite3_free(aFrame);
if( rc==SQLITE_OK ){
assert( pWal->hdr.mxFrame==mxFrame );
if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[16])
|| pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[20])
){
rc = SQLITE_MISMATCH;
}
}
if( rc!=SQLITE_OK ){
sqlite3WalUndo(pWal, walUndoNoop, 0);
}
}
return rc;
}
#ifdef SQLITE_ENABLE_ZIPVFS
/*
** If the argument is not NULL, it points to a Wal object that holds a

3
src/wal.h
View File

@@ -126,6 +126,9 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op);
*/
int sqlite3WalHeapMemory(Wal *pWal);
int sqlite3WalSaveState(Wal *pWal, void **ppState, int *pnState);
int sqlite3WalRestoreState(Wal *pWal, const void *pState, int nState);
#ifdef SQLITE_ENABLE_ZIPVFS
/* If the WAL file is not empty, return the number of bytes of content
** stored in each frame (i.e. the db page-size when the WAL was created).