From b0083756f274674359f3d8b2e1aef3c3ef7530af Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Sep 2014 19:59:40 +0000 Subject: [PATCH 001/116] Add an experimental extension for applying bulk updates to databases. FossilOrigin-Name: 2954ab501049968430011b63d046eb42ff37a56c --- ext/ota/ota1.test | 181 ++++++++++ ext/ota/ota2.test | 62 ++++ ext/ota/sqlite3ota.c | 811 +++++++++++++++++++++++++++++++++++++++++++ ext/ota/sqlite3ota.h | 212 +++++++++++ main.mk | 3 +- manifest | 51 +-- manifest.uuid | 2 +- src/btree.c | 7 +- src/insert.c | 13 + src/main.c | 12 + src/pager.c | 41 ++- src/pager.h | 5 +- src/pragma.c | 12 +- src/prepare.c | 1 - src/sqlite.h.in | 72 ++++ src/sqliteInt.h | 3 + src/tclsqlite.c | 3 + src/test1.c | 66 ++++ src/test2.c | 2 +- src/vdbeblob.c | 112 ++++++ src/wal.c | 306 ++++++++++++---- src/wal.h | 3 + 22 files changed, 1878 insertions(+), 102 deletions(-) create mode 100644 ext/ota/ota1.test create mode 100644 ext/ota/ota2.test create mode 100644 ext/ota/sqlite3ota.c create mode 100644 ext/ota/sqlite3ota.h diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test new file mode 100644 index 0000000000..aa334b9581 --- /dev/null +++ b/ext/ota/ota1.test @@ -0,0 +1,181 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file join [file dirname $argv0] .. .. test] +source $testdir/tester.tcl +set ::testprefix ota1 + + +# Create a simple OTA database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_ota1 {filename} { + forcedelete $filename + sqlite3 ota1 $filename + ota1 eval { + CREATE TABLE data_t1(a, b, c, ota_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(2, 'two', 'three', 0); + INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0); + } + ota1 close + return $filename +} + +# Create an empty target database suitable for the OTA created by +# [create_ota1]. +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_target1 {filename} { + forcedelete $filename + sqlite3 target1 $filename + target1 eval { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + target1 close + return $filename +} + +# Run the OTA in file $ota on target database $target until completion. +# +proc run_ota {target ota} { + sqlite3ota ota $target $ota + while { [ota step]=="SQLITE_OK" } {} + ota close +} + +proc step_ota {target ota} { + while 1 { + sqlite3ota ota $target $ota + set rc [ota step] + ota close + if {$rc != "SQLITE_OK"} break + } + set rc +} + +foreach {tn2 cmd} {1 step_ota 2 run_ota} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + } + 3 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + } + 4 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + 5 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + 6 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b, a); + } + 7 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(a, b, c, a, b, c); + } + } { + reset_db + execsql $schema + + do_test 1.$tn2.$tn.1 { + create_ota1 ota.db + $cmd test.db ota.db + } {SQLITE_DONE} + + do_execsql_test 1.$tn2.$tn.2 { + SELECT * FROM t1 ORDER BY a ASC; + } { + 1 2 3 + 2 two three + 3 {} 8.2 + } + + do_execsql_test 1.$tn2.$tn.3 { PRAGMA integrity_check } ok + } +} + +#------------------------------------------------------------------------- +# Check that an OTA cannot be applied to a table that has no PK. +# +reset_db +create_ota1 ota.db +do_execsql_test 2.1 { CREATE TABLE t1(a, b, c) } +do_test 2.2 { + sqlite3ota ota test.db ota.db + ota step +} {SQLITE_ERROR} +do_test 2.3 { + list [catch { ota close } msg] $msg +} {1 {SQLITE_ERROR - table t1 has no PRIMARY KEY}} + +reset_db +do_execsql_test 2.4 { CREATE TABLE t1(a PRIMARY KEY, b, c) } +do_test 2.5 { + sqlite3ota ota test.db ota.db + ota step +} {SQLITE_ERROR} +do_test 2.6 { + list [catch { ota close } msg] $msg +} {1 {SQLITE_ERROR - table t1 has no PRIMARY KEY}} + +#------------------------------------------------------------------------- +# Check that if a UNIQUE constraint is violated the current and all +# subsequent [ota step] calls return SQLITE_CONSTRAINT. And that the OTA +# transaction is rolled back by the [ota close] that deletes the ota +# handle. +# +foreach {tn errcode errmsg schema} { + 1 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(3, 2, 1); + } + + 2 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c UNIQUE); + INSERT INTO t1 VALUES(4, 2, 'three'); + } + +} { + reset_db + execsql $schema + set cksum [dbcksum db main] + + do_test 3.$tn.1 { + create_ota1 ota.db + sqlite3ota ota test.db ota.db + while {[set res [ota step]]=="SQLITE_OK"} {} + set res + } $errcode + + do_test 3.$tn.2 { ota step } $errcode + + do_test 3.$tn.3 { + list [catch { ota close } msg] $msg + } [list 1 "$errcode - $errmsg"] + + do_test 3.$tn.4 { dbcksum db main } $cksum +} + + +finish_test + diff --git a/ext/ota/ota2.test b/ext/ota/ota2.test new file mode 100644 index 0000000000..dfa4a48e9d --- /dev/null +++ b/ext/ota/ota2.test @@ -0,0 +1,62 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file join [file dirname $argv0] .. .. test] +source $testdir/tester.tcl +set ::testprefix ota2 + + +do_execsql_test 1.0 { + PRAGMA ota_mode = 1; + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + BEGIN; + INSERT INTO t1 VALUES(1, 2); +} {wal} + +do_test 1.1 { + set state [sqlite3_transaction_save db] + db close + file exists test.db-wal +} {1} + +do_test 1.2 { + sqlite3 db test.db + db eval {SELECT * FROM t1} +} {} + +do_test 1.3 { + execsql {BEGIN IMMEDIATE} + sqlite3_transaction_restore db $::state + db eval {SELECT * FROM t1} +} {1 2} + +do_test 1.4 { + execsql { + INSERT INTO t1 VALUES(3, 4); + COMMIT; + SELECT * FROM t1; + } +} {1 2 3 4} + +do_test 1.5 { + db close + file exists test.db-wal +} {0} + +do_test 1.5 { + sqlite3 db test.db + db eval {SELECT * FROM t1} +} {1 2 3 4} + +finish_test + diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c new file mode 100644 index 0000000000..5483b9049f --- /dev/null +++ b/ext/ota/sqlite3ota.c @@ -0,0 +1,811 @@ +/* +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#include +#include + +#include "sqlite3.h" +#include "sqlite3ota.h" + + +/* +** The ota_state table is used to save the state of a partially applied +** update so that it can be resumed later. The table contains at most a +** single row: +** +** "wal_state" -> Blob to use with sqlite3_transaction_restore(). +** +** "tbl" -> Table currently being written (target database names). +** +** "idx" -> Index currently being written (target database names). +** Or, if the main table is being written, a NULL value. +** +** "row" -> Last rowid processed from ota database table (i.e. data_%). +** +** "progress" -> total number of key/value b-tree operations performed +** so far as part of this ota update. +*/ +#define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota_state" \ + "(wal_state, tbl, idx, row, progress)" + +typedef struct OtaTblIter OtaTblIter; +typedef struct OtaIdxIter OtaIdxIter; + +/* +** Iterator used to iterate through all data tables in the OTA. As follows: +** +** OtaTblIter iter; +** for(rc=tblIterFirst(db, &iter); +** rc==SQLITE_OK && iter.zTarget; +** rc=tblIterNext(&iter) +** ){ +** } +*/ +struct OtaTblIter { + sqlite3_stmt *pTblIter; /* Iterate through tables */ + int iEntry; /* Index of current entry (from 1) */ + + /* Output varibles. zTarget==0 implies EOF. */ + const char *zTarget; /* Name of target table */ + const char *zSource; /* Name of source table */ + + /* Useful things populated by a call to tblIterPrepareAll() */ + int nCol; /* Number of columns in this table */ + char **azCol; /* Array of quoted column names */ + sqlite3_stmt *pSelect; /* PK b-tree SELECT statement */ + sqlite3_stmt *pInsert; /* PK b-tree INSERT statement */ +}; + +/* +** API is: +** +** idxIterFirst() +** idxIterNext() +** idxIterFinalize() +** idxIterPrepareAll() +*/ +struct OtaIdxIter { + sqlite3_stmt *pIdxIter; /* Iterate through indexes */ + int iEntry; /* Index of current entry (from 1) */ + + /* Output varibles. zTarget==0 implies EOF. */ + const char *zIndex; /* Name of index */ + + int nCol; /* Number of columns in index */ + int *aiCol; /* Array of column indexes */ + sqlite3_stmt *pWriter; /* Index writer */ + sqlite3_stmt *pSelect; /* Select to read values in index order */ +}; + + +struct sqlite3ota { + sqlite3 *dbDest; /* Target db */ + sqlite3 *dbOta; /* Ota db */ + + int rc; /* Value returned by last ota_step() call */ + char *zErrmsg; /* Error message if rc!=SQLITE_OK */ + + OtaTblIter tbliter; /* Used to iterate through tables */ + OtaIdxIter idxiter; /* Used to iterate through indexes */ +}; + +static int prepareAndCollectError( + sqlite3 *db, + const char *zSql, + sqlite3_stmt **ppStmt, + char **pzErrmsg +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + *ppStmt = 0; + } + return rc; +} + +/* +** Unless it is NULL, argument zSql points to a buffer allocated using +** sqlite3_malloc containing an SQL statement. This function prepares the SQL +** statement against database db and frees the buffer. If statement +** compilation is successful, *ppStmt is set to point to the new statement +** handle and SQLITE_OK is returned. +** +** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code +** returned. In this case, *pzErrmsg may also be set to point to an error +** message. It is the responsibility of the caller to free this error message +** buffer using sqlite3_free(). +** +** If argument zSql is NULL, this function assumes that an OOM has occurred. +** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. +*/ +static int prepareFreeAndCollectError( + sqlite3 *db, + char *zSql, + sqlite3_stmt **ppStmt, + char **pzErrmsg +){ + int rc; + assert( *pzErrmsg==0 ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + *ppStmt = 0; + }else{ + rc = prepareAndCollectError(db, zSql, ppStmt, pzErrmsg); + sqlite3_free(zSql); + } + return rc; +} + +static char *quoteSqlName(const char *zName){ + int nName = strlen(zName); + char *zRet = sqlite3_malloc(nName * 2 + 2 + 1); + if( zRet ){ + int i; + char *p = zRet; + *p++ = '"'; + for(i=0; itbliter; + int rc = SQLITE_OK; + char *zCol = 0; + char *zBindings = 0; + char *zSql; + sqlite3_stmt *pPragma = 0; + int i; + int bSeenPk = 0; /* Set to true once PK column seen */ + + /* Allocate and populate the azCol[] array */ + zSql = sqlite3_mprintf("PRAGMA main.table_info(%Q)", pIter->zTarget); + rc = prepareFreeAndCollectError(p->dbDest, zSql, &pPragma, &p->zErrmsg); + pIter->nCol = 0; + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3_step(pPragma) ){ + const char *zName = (const char*)sqlite3_column_text(pPragma, 1); + if( (pIter->nCol % 4)==0 ){ + int nByte = sizeof(char*) * (pIter->nCol+4); + char **azNew = (char**)sqlite3_realloc(pIter->azCol, nByte); + if( azNew==0 ){ + rc = SQLITE_NOMEM; + break; + } + pIter->azCol = azNew; + } + pIter->azCol[pIter->nCol] = quoteSqlName(zName); + if( pIter->azCol[pIter->nCol]==0 ){ + rc = SQLITE_NOMEM; + break; + } + pIter->nCol++; + if( sqlite3_column_int(pPragma, 5) ) bSeenPk = 1; + } + if( rc==SQLITE_OK ){ + rc = sqlite3_finalize(pPragma); + }else{ + sqlite3_finalize(pPragma); + } + } + + /* If the table has no PRIMARY KEY, throw an exception. */ + if( bSeenPk==0 ){ + p->zErrmsg = sqlite3_mprintf("table %s has no PRIMARY KEY", pIter->zTarget); + rc = SQLITE_ERROR; + } + + /* Populate the zCol variable */ + for(i=0; rc==SQLITE_OK && inCol; i++){ + zCol = sqlite3_mprintf("%z%s%s", zCol, (i==0?"":", "), pIter->azCol[i]); + if( zCol==0 ){ + rc = SQLITE_NOMEM; + } + } + + /* Allocate and populate zBindings */ + if( rc==SQLITE_OK ){ + zBindings = (char*)sqlite3_malloc(pIter->nCol * 2); + if( zBindings==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + for(i=0; inCol; i++){ + zBindings[i*2] = '?'; + zBindings[i*2+1] = ','; + } + zBindings[pIter->nCol*2-1] = '\0'; + } + } + + /* Create OtaTblIter.pSelect */ + if( rc==SQLITE_OK ){ + zSql = sqlite3_mprintf("SELECT rowid, %s FROM %Q", zCol, pIter->zSource); + rc = prepareFreeAndCollectError(p->dbOta,zSql,&pIter->pSelect, &p->zErrmsg); + } + + /* Create OtaTblIter.pInsert */ + if( rc==SQLITE_OK ){ + zSql = sqlite3_mprintf("INSERT INTO %Q(%s) VALUES(%s)", + pIter->zTarget, zCol, zBindings + ); + rc = prepareFreeAndCollectError(p->dbDest,zSql,&pIter->pInsert,&p->zErrmsg); + } + + sqlite3_free(zCol); + sqlite3_free(zBindings); + return rc; +} + +static void tblIterFreeAll(OtaTblIter *pIter){ + int i; + + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + for(i=0; inCol; i++) sqlite3_free(pIter->azCol[i]); + sqlite3_free(pIter->azCol); + pIter->azCol = 0; + pIter->pSelect = 0; + pIter->pInsert = 0; + pIter->nCol = 0; +} + +static int tblIterNext(OtaTblIter *pIter){ + int rc; + + tblIterFreeAll(pIter); + assert( pIter->pTblIter ); + rc = sqlite3_step(pIter->pTblIter); + if( rc==SQLITE_ROW ){ + pIter->zSource = (const char*)sqlite3_column_text(pIter->pTblIter, 0); + pIter->zTarget = &pIter->zSource[5]; assert( 5==strlen("data_") ); + pIter->iEntry++; + }else{ + pIter->zSource = 0; + pIter->zTarget = 0; + } + + if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; +} + +static int tblIterFirst(sqlite3 *db, OtaTblIter *pIter){ + int rc; /* return code */ + memset(pIter, 0, sizeof(OtaTblIter)); + rc = sqlite3_prepare_v2(db, + "SELECT name FROM sqlite_master " + "WHERE type='table' AND name LIKE 'data_%'", -1, &pIter->pTblIter, 0 + ); + if( rc==SQLITE_OK ){ + rc = tblIterNext(pIter); + } + return rc; +} + + +static void tblIterFinalize(OtaTblIter *pIter){ + tblIterFreeAll(pIter); + sqlite3_finalize(pIter->pTblIter); + memset(pIter, 0, sizeof(OtaTblIter)); +} + +static void idxIterFreeAll(OtaIdxIter *pIter){ + sqlite3_finalize(pIter->pWriter); + sqlite3_finalize(pIter->pSelect); + pIter->pWriter = 0; + pIter->pSelect = 0; + pIter->aiCol = 0; + pIter->nCol = 0; +} + +static int idxIterPrepareAll(sqlite3ota *p){ + int rc; + int i; /* Iterator variable */ + char *zSql = 0; + char *zCols = 0; /* Columns list */ + OtaIdxIter *pIter = &p->idxiter; + + /* Prepare the writer statement to write (insert) entries into the index. */ + rc = sqlite3_index_writer( + p->dbDest, 0, pIter->zIndex, &pIter->pWriter, &pIter->aiCol, &pIter->nCol + ); + + /* Prepare a SELECT statement to read values from the source table in + ** the same order as they are stored in the current index. The statement + ** is: + ** + ** SELECT rowid, FROM data_ ORDER BY + */ + for(i=0; rc==SQLITE_OK && inCol; i++){ + const char *zQuoted = p->tbliter.azCol[ pIter->aiCol[i] ]; + zCols = sqlite3_mprintf("%z%s%s", zCols, zCols?", ":"", zQuoted); + if( !zCols ){ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + const char *zFmt = "SELECT rowid, %s FROM %Q ORDER BY %s"; + zSql = sqlite3_mprintf(zFmt, zCols, p->tbliter.zSource, zCols); + if( zSql ){ + sqlite3_stmt **pp = &p->idxiter.pSelect; + rc = prepareFreeAndCollectError(p->dbOta, zSql, pp, &p->zErrmsg); + }else{ + rc = SQLITE_NOMEM; + } + } + + sqlite3_free(zCols); + return rc; +} + +static int idxIterNext(OtaIdxIter *pIter){ + int rc; + + idxIterFreeAll(pIter); + assert( pIter->pIdxIter ); + rc = sqlite3_step(pIter->pIdxIter); + if( rc==SQLITE_ROW ){ + pIter->zIndex = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + pIter->iEntry++; + }else{ + pIter->zIndex = 0; + rc = sqlite3_finalize(pIter->pIdxIter); + pIter->pIdxIter = 0; + } + + if( rc==SQLITE_ROW ) rc = SQLITE_OK; + return rc; +} + +static int idxIterFirst(sqlite3 *db, const char *zTable, OtaIdxIter *pIter){ + int rc; /* return code */ + memset(pIter, 0, sizeof(OtaIdxIter)); + rc = sqlite3_prepare_v2(db, + "SELECT name FROM sqlite_master " + "WHERE type='index' AND tbl_name = ?", -1, &pIter->pIdxIter, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_text(pIter->pIdxIter, 1, zTable, -1, SQLITE_TRANSIENT); + } + if( rc==SQLITE_OK ){ + rc = idxIterNext(pIter); + } + return rc; +} + +static void idxIterFinalize(OtaIdxIter *pIter){ + idxIterFreeAll(pIter); + sqlite3_finalize(pIter->pIdxIter); + memset(pIter, 0, sizeof(OtaIdxIter)); +} + +/* +** Call sqlite3_reset() on the SQL statement passed as the second argument. +** If it returns anything other than SQLITE_OK, store the error code and +** error message in the OTA handle. +*/ +static void otaResetStatement(sqlite3ota *p, sqlite3_stmt *pStmt){ + assert( p->rc==SQLITE_OK ); + assert( p->zErrmsg==0 ); + p->rc = sqlite3_reset(pStmt); + if( p->rc!=SQLITE_OK ){ + sqlite3 *db = sqlite3_db_handle(pStmt); + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } +} + +/* +** Check that all SQL statements required to process the current +** table and index have been prepared. If not, prepare them. If +** an error occurs, store the error code and message in the OTA +** handle before returning. +*/ +static int otaPrepareAll(sqlite3ota *p){ + assert( p->rc==SQLITE_OK ); + assert( p->zErrmsg==0 ); + assert( p->tbliter.zTarget ); + + if( p->tbliter.pSelect==0 ){ + p->rc = tblIterPrepareAll(p); + } + if( p->rc==SQLITE_OK && p->idxiter.zIndex && 0==p->idxiter.pSelect ){ + p->rc = idxIterPrepareAll(p); + } + return p->rc; +} + +int sqlite3ota_step(sqlite3ota *p){ + if( p ){ + while( p && p->rc==SQLITE_OK && p->tbliter.zTarget ){ + sqlite3_stmt *pSelect; + int i; + + otaPrepareAll(p); + pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect); + + /* Advance to the next input row. */ + if( p->rc==SQLITE_OK ){ + int rc = sqlite3_step(pSelect); + if( rc!=SQLITE_ROW ){ + otaResetStatement(p, pSelect); + + /* Go to the next index. */ + if( p->rc==SQLITE_OK ){ + if( p->idxiter.zIndex ){ + p->rc = idxIterNext(&p->idxiter); + }else{ + p->rc = idxIterFirst(p->dbDest, p->tbliter.zTarget, &p->idxiter); + } + } + + /* If there is no next index, go to the next table. */ + if( p->rc==SQLITE_OK && p->idxiter.zIndex==0 ){ + p->rc = tblIterNext(&p->tbliter); + } + continue; + } + } + + /* Update the target database PK table according to the row that + ** tbliter.pSelect currently points to. + ** + ** todo: For now, we assume all rows are INSERT commands - this will + ** change. */ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pInsert; + int nCol; + if( p->idxiter.zIndex ){ + pInsert = p->idxiter.pWriter; + nCol = p->idxiter.nCol; + }else{ + pInsert = p->tbliter.pInsert; + nCol = p->tbliter.nCol; + } + + for(i=0; irc==SQLITE_OK && p->tbliter.zTarget==0 ) p->rc = SQLITE_DONE; + } + + return (p ? p->rc : SQLITE_NOMEM); +} + +static void otaOpenDatabase(sqlite3ota *p, sqlite3 **pDb, const char *zFile){ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_open(zFile, pDb); + if( p->rc ){ + const char *zErr = sqlite3_errmsg(*pDb); + p->zErrmsg = sqlite3_mprintf("sqlite3_open(): %s", zErr); + } + } +} + +static void otaSaveTransactionState(sqlite3ota *p){ + sqlite3_stmt *pStmt = 0; + void *pWalState = 0; + int nWalState = 0; + int rc; + + const char *zInsert = + "INSERT INTO ota_state(wal_state, tbl, idx, row, progress)" + "VALUES(:wal_state, :tbl, :idx, :row, :progress)"; + + rc = sqlite3_transaction_save(p->dbDest, &pWalState, &nWalState); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->dbOta, "DELETE FROM ota_state", 0, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbOta, zInsert, &pStmt, &p->zErrmsg); + } + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSelect; + pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect); + sqlite3_bind_blob(pStmt, 1, pWalState, nWalState, SQLITE_STATIC); + sqlite3_bind_text(pStmt, 2, p->tbliter.zTarget, -1, SQLITE_STATIC); + if( p->idxiter.zIndex ){ + sqlite3_bind_text(pStmt, 3, p->idxiter.zIndex, -1, SQLITE_STATIC); + } + sqlite3_bind_int64(pStmt, 4, sqlite3_column_int64(pSelect, 0)); + sqlite3_step(pStmt); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->dbOta, "COMMIT", 0, 0, 0); + } + if( rc!=SQLITE_OK ){ + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->dbOta)); + } + } + sqlite3_free(pWalState); + assert( p->rc==SQLITE_OK ); + p->rc = rc; +} + +static void otaLoadTransactionState(sqlite3ota *p){ + sqlite3_stmt *pStmt = 0; + int rc; + + const char *zSelect = + "SELECT wal_state, tbl, idx, row, progress FROM ota_state"; + + rc = prepareAndCollectError(p->dbOta, zSelect, &pStmt, &p->zErrmsg); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + const void *pWalState = 0; + int nWalState = 0; + const char *zTbl; + const char *zIdx; + sqlite3_int64 iRowid; + + pWalState = sqlite3_column_blob(pStmt, 0); + nWalState = sqlite3_column_bytes(pStmt, 0); + zTbl = (const char*)sqlite3_column_text(pStmt, 1); + zIdx = (const char*)sqlite3_column_text(pStmt, 2); + iRowid = sqlite3_column_int64(pStmt, 3); + rc = sqlite3_transaction_restore(p->dbDest, pWalState, nWalState); + + while( rc==SQLITE_OK + && p->tbliter.zTarget + && sqlite3_stricmp(p->tbliter.zTarget, zTbl) + ){ + rc = tblIterNext(&p->tbliter); + } + if( rc==SQLITE_OK && !p->tbliter.zTarget ){ + rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); + } + + if( rc==SQLITE_OK && zIdx ){ + rc = idxIterFirst(p->dbDest, p->tbliter.zTarget, &p->idxiter); + while( rc==SQLITE_OK + && p->idxiter.zIndex + && sqlite3_stricmp(p->idxiter.zIndex, zIdx) + ){ + rc = idxIterNext(&p->idxiter); + } + if( rc==SQLITE_OK && !p->idxiter.zIndex ){ + rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); + } + } + + if( rc==SQLITE_OK ){ + rc = otaPrepareAll(p); + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSelect; + pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect); + while( sqlite3_column_int64(pSelect, 0)!=iRowid ){ + rc = sqlite3_step(pSelect); + if( rc!=SQLITE_ROW ) break; + } + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + }else{ + rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_finalize(pStmt); + }else{ + sqlite3_finalize(pStmt); + } + } + p->rc = rc; +} + + +/* +** Open and return a new OTA handle. +*/ +sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ + sqlite3ota *p; + + p = (sqlite3ota*)sqlite3_malloc(sizeof(sqlite3ota)); + if( p ){ + + /* Open the target database */ + memset(p, 0, sizeof(sqlite3ota)); + otaOpenDatabase(p, &p->dbDest, zTarget); + otaOpenDatabase(p, &p->dbOta, zOta); + + /* If it has not already been created, create the ota_state table */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbOta, OTA_CREATE_STATE, 0, 0, &p->zErrmsg); + } + + if( p->rc==SQLITE_OK ){ + const char *zScript = + "PRAGMA ota_mode=1;" + "PRAGMA journal_mode=wal;" + "BEGIN IMMEDIATE;" + ; + p->rc = sqlite3_exec(p->dbDest, zScript, 0, 0, &p->zErrmsg); + } + + if( p->rc==SQLITE_OK ){ + const char *zScript = "BEGIN IMMEDIATE"; + p->rc = sqlite3_exec(p->dbOta, zScript, 0, 0, &p->zErrmsg); + } + + /* Point the table iterator at the first table */ + if( p->rc==SQLITE_OK ){ + p->rc = tblIterFirst(p->dbOta, &p->tbliter); + } + + if( p->rc==SQLITE_OK ){ + otaLoadTransactionState(p); + } + } + + return p; +} + +static void otaCloseHandle(sqlite3 *db){ + int rc = sqlite3_close(db); + assert( rc==SQLITE_OK ); +} + +int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ + int rc; + if( p ){ + + /* If the update has not been fully applied, save the state in + ** the ota db. If successful, this call also commits the open + ** transaction on the ota db. */ + assert( p->rc!=SQLITE_ROW ); + if( p->rc==SQLITE_OK ){ + assert( p->zErrmsg==0 ); + otaSaveTransactionState(p); + } + + /* Close all open statement handles. */ + tblIterFinalize(&p->tbliter); + idxIterFinalize(&p->idxiter); + + /* If the ota update has been fully applied, commit the transaction + ** on the target database. */ + if( p->rc==SQLITE_DONE ){ + rc = sqlite3_exec(p->dbDest, "COMMIT", 0, 0, &p->zErrmsg); + if( rc!=SQLITE_OK ) p->rc = rc; + } + + rc = p->rc; + *pzErrmsg = p->zErrmsg; + otaCloseHandle(p->dbDest); + otaCloseHandle(p->dbOta); + sqlite3_free(p); + }else{ + rc = SQLITE_NOMEM; + *pzErrmsg = 0; + } + return rc; +} + + +/**************************************************************************/ + +#ifdef SQLITE_TEST + +#include + +/* From main.c (apparently...) */ +extern const char *sqlite3ErrName(int); + +static int test_sqlite3ota_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int ret = TCL_OK; + sqlite3ota *pOta = (sqlite3ota*)clientData; + const char *azMethod[] = { "step", "close", 0 }; + int iMethod; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "METHOD"); + return TCL_ERROR; + } + if( Tcl_GetIndexFromObj(interp, objv[1], azMethod, "method", 0, &iMethod) ){ + return TCL_ERROR; + } + + switch( iMethod ){ + case 0: /* step */ { + int rc = sqlite3ota_step(pOta); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + } + + case 1: /* close */ { + char *zErrmsg = 0; + int rc; + Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); + rc = sqlite3ota_close(pOta, &zErrmsg); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + assert( zErrmsg==0 ); + }else{ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + if( zErrmsg ){ + Tcl_AppendResult(interp, " - ", zErrmsg, 0); + sqlite3_free(zErrmsg); + } + ret = TCL_ERROR; + } + break; + } + + default: /* seems unlikely */ + assert( !"cannot happen" ); + break; + } + + return ret; +} + +/* +** Tclcmd: sqlite3ota CMD +*/ +static int test_sqlite3ota( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3ota *pOta = 0; + const char *zCmd; + const char *zTarget; + const char *zOta; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB OTA-DB"); + return TCL_ERROR; + } + zCmd = Tcl_GetString(objv[1]); + zTarget = Tcl_GetString(objv[2]); + zOta = Tcl_GetString(objv[3]); + + pOta = sqlite3ota_open(zTarget, zOta); + Tcl_CreateObjCommand(interp, zCmd, test_sqlite3ota_cmd, (ClientData)pOta, 0); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + +int SqliteOta_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "sqlite3ota", test_sqlite3ota, 0, 0); + return TCL_OK; +} + +#endif /* ifdef SQLITE_TEST */ + + + diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h new file mode 100644 index 0000000000..0e577520b2 --- /dev/null +++ b/ext/ota/sqlite3ota.h @@ -0,0 +1,212 @@ +/* +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the public interface for the OTA extension. +*/ + +/* +** SUMMARY +** +** Writing a transaction containing a large number of operations on +** b-tree indexes that are collectively larger than the available cache +** memory can be very inefficient. +** +** The problem is that in order to update a b-tree, the leaf page (at least) +** containing the entry being inserted or deleted must be modified. If the +** working set of leaves is larger than the available cache memory, then a +** single leaf that is modified more than once as part of the transaction +** may be loaded from or written to the persistent media more than once. +** Additionally, because the index updates are likely to be applied in +** random order, access to pages within the databse is also likely to be in +** random order, which is itself quite inefficient. +** +** One way to improve the situation is to sort the operations on each index +** by index key before applying them to the b-tree. This leads to an IO +** pattern that resembles a single linear scan through the index b-tree, +** and all but guarantees each modified leaf page is loaded and stored +** exactly once. SQLite uses this trick to improve the performance of +** CREATE INDEX commands. This extension allows it to be used to improve +** the performance of large transactions on existing databases. +** +** Additionally, this extension allows the work involved in writing the +** large transaction to be broken down into sub-transactions performed +** sequentially by separate processes. This is useful if the system cannot +** guarantee that a single update process may run for long enough to apply +** the entire update, for example because the update is running on a mobile +** device that is frequently rebooted. Even after the writer process has +** committed one or more sub-transactions, other database clients continue +** to read from the original database snapshot. In other words, partially +** applied transactions are not visible to other clients. +** +** "OTA" stands for "Over The Air" update. As in a large database update +** transmitted via a wireless network to a mobile device. A transaction +** applied using this extension is hence refered to as an "OTA update". +** +** +** LIMITATIONS +** +** An "OTA update" transaction is subject to the following limitations: +** +** * The transaction must consist of INSERT, UPDATE and DELETE operations +** only. +** +** * INSERT statements may not use any default values. +** +** * UPDATE and DELETE statements must identify their target rows by +** real PRIMARY KEY values - i.e. INTEGER PRIMARY KEY columns or +** by the PRIMARY KEY columns of WITHOUT ROWID tables. +** +** * UPDATE statements may not modify real PRIMARY KEY columns. +** +** * No triggers will be fired. +** +** * No foreign key violations are detected or reported. +** +** * No constraint handling mode except for "OR ROLLBACK" is supported. +** +** +** PREPARATION +** +** An "OTA update" is stored as a separate SQLite database. A database +** containing an OTA update is an "OTA database". For each table in the +** target database to be updated, the OTA database should contain a table +** named "data_" containing the same set of columns as the +** target table, and one more - "ota_control". The data_% table should +** have no PRIMARY KEY or UNIQUE constraints, but each column should have +** the same type as the corresponding column in the target database. +** The "ota_control" column should have no type at all. For example, if +** the target database contains: +** +** CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c UNIQUE); +** +** Then the OTA database should contain: +** +** CREATE TABLE data_t1(a INTEGER, b TEXT, c, ota_control); +** +** The order of the columns in the data_% table does not matter. +** +** For each row to INSERT into the target database as part of the OTA +** update, the corresponding data_% table should contain a single record +** with the "ota_control" column set to contain integer value 0. The +** other columns should be set to the values that make up the new record +** to insert. +** +** For each row to DELETE from the target database as part of the OTA +** update, the corresponding data_% table should contain a single record +** with the "ota_control" column set to contain integer value 1. The +** real primary key values of the row to delete should be stored in the +** corresponding columns of the data_% table. The values stored in the +** other columns are not used. +** +** For each row to DELETE from the target database as part of the OTA +** update, the corresponding data_% table should contain a single record +** with the "ota_control" column set to contain a value of type text. +** The real primary key values identifying the row to update should be +** stored in the corresponding columns of the data_% table row, as should +** the new values of all columns being update. The text value in the +** "ota_control" column must contain the same number of characters as +** there are column in the target database table, and must consist entirely +** of "x" and "." characters. For each column that is being updated, +** the corresponding character is set to "x". For those that remain as +** they are, the corresponding character of the ota_control value should +** be set to ".". For example, given the tables above, the update +** statement: +** +** UPDATE t1 SET c = 'usa' WHERE a = 4; +** +** is represented by the data_t1 row created by: +** +** INSERT INTO data_t1(a, b, c, ota_control) VALUES(4, NULL, 'usa', '..x'); +** +** +** USAGE +** +** The API declared below allows an application to apply an OTA update +** stored on disk to an existing target database. Essentially, the +** application: +** +** 1) Opens an OTA handle using the sqlite3ota_open() function. +** +** 2) Calls the sqlite3ota_step() function one or more times on +** the new handle. Each call to sqlite3ota_step() performs a single +** b-tree operation, so thousands of calls may be required to apply +** a complete update. +** +** 3) Calls sqlite3ota_close() to close the OTA update handle. If +** sqlite3ota_step() has been called enough times to completely +** apply the update to the target database, then it is committed +** and made visible to other database clients at this point. +** Otherwise, the state of the OTA update application is saved +** in the OTA database for later resumption. +** +** See comments below for more detail on APIs. +** +** If an update is only partially applied to the target database by the +** time sqlite3ota_close() is called, various state information is saved +** within the OTA database. This allows subsequent processes to automatically +** resume the OTA update from where it left off. +** +** To remove all OTA extension state information, returning an OTA database +** to its original contents, it is sufficient to drop all tables that begin +** with the prefix "ota_" +*/ + +#ifndef _SQLITE3OTA_H +#define _SQLITE3OTA_H + +typedef struct sqlite3ota sqlite3ota; + +/* +** Open an OTA handle. +** +** Argument zTarget is the path to the target database. Argument zOta is +** the path to the OTA database. Each call to this function must be matched +** by a call to sqlite3ota_close(). +*/ +sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta); + +/* +** Do some work towards applying the OTA update to the target db. +** +** Return SQLITE_DONE if the update has been completely applied, or +** SQLITE_OK if no error occurs but there remains work to do to apply +** the OTA update. If an error does occur, some other error code is +** returned. +** +** Once a call to sqlite3ota_step() has returned a value other than +** SQLITE_OK, all subsequent calls on the same OTA handle are no-ops +** that immediately return the same value. +*/ +int sqlite3ota_step(sqlite3ota *pOta); + +/* +** Close an OTA handle. +** +** If the OTA update has been completely applied, commit it to the target +** database. Otherwise, assuming no error has occurred, save the current +** state of the OTA update appliation to the OTA database. +** +** If an error has already occurred as part of an sqlite3ota_step() +** or sqlite3ota_open() call, or if one occurs within this function, an +** SQLite error code is returned. Additionally, *pzErrmsg may be set to +** point to a buffer containing a utf-8 formatted English language error +** message. It is the responsibility of the caller to eventually free any +** such buffer using sqlite3_free(). +** +** Otherwise, if no error occurs, this function returns SQLITE_OK if the +** update has been partially applied, or SQLITE_DONE if it has been +** completely applied. +*/ +int sqlite3ota_close(sqlite3ota *pOta, char **pzErrmsg); + +#endif /* _SQLITE3OTA_H */ + diff --git a/main.mk b/main.mk index 7a20373abc..b225e4ecd0 100644 --- a/main.mk +++ b/main.mk @@ -331,7 +331,8 @@ TESTSRC2 = \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ - $(TOP)/ext/async/sqlite3async.c + $(TOP)/ext/async/sqlite3async.c \ + $(TOP)/ext/ota/sqlite3ota.c # Header files used by all library source files. # diff --git a/manifest b/manifest index 58fd42dc62..da360992c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\scomments\sin\sthe\sANALYZE\scommand\sthat\sdescribe\show\sthe\sStat4Accum\nobjecct\sis\spassed\saround\swithin\sthe\sVDBE.\s\sNo\schanges\sto\sfunctional\scode. -D 2014-09-01T23:06:44.401 +C Add\san\sexperimental\sextension\sfor\sapplying\sbulk\supdates\sto\sdatabases. +D 2014-09-02T19:59:40.729 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,6 +121,10 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 +F ext/ota/ota1.test ea2865997ce573fadaf12eb0a0f80ef22d9dd77f +F ext/ota/ota2.test 4f7abfe1dfb7c3709bf45e94f3e65f3839b4f115 +F ext/ota/sqlite3ota.c ad55821883e4110367a30ffca282032d2bf36e45 +F ext/ota/sqlite3ota.h d3187a98fe1e3445c58f7a27d96ac385b78486a1 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -147,7 +151,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 9b5ccf1097050b1f16681f7d4beeea4f7f7ac2c3 +F main.mk 566c36f247d19525264e584466b5f07c0a48302e F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -168,7 +172,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c ec9d3f1295dafeb278c3830211cc5584132468f4 -F src/btree.c 2a483a8045118faa99867a8679da42754b532318 +F src/btree.c c46043fbb09c18a19bdb96eadde6e724901d6fcf F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c c26b233dcdb1e2c8f468d49236c266f9f3de96d8 @@ -185,12 +189,12 @@ F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c d1a104e67b33314d4cc5c1356147446086ab9fc8 +F src/insert.c 62b0ceab1720dc74ed1fbcf953224132245704d8 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab -F src/main.c d2ef03a45552e11813c68326d5edfda992e319d4 +F src/main.c c9802dc99c019fbba516202300d56be2c478fa93 F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -211,30 +215,30 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 8525ca79457c5b4673a5fda2774ee39fe155f40f F src/os_win.c 2aa8aa7780d7cf03e912d2088ab2ec5c32f33dc5 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 3e732d2bbdd8d8d95fed0c5ae7e718d73153c4c5 -F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 +F src/pager.c a3caa08db8227c5a32f388be67f33d8cb44d5e35 +F src/pager.h 1acd367a0ffb63026b0461ea5eaeeb8046414a71 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 3b3791297e8977002e56b4a9b8916f2039abad9b F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c -F src/pragma.c 14bcdb504128a476cce5bbc086d5226c5e46c225 -F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d +F src/pragma.c d252459fb3ce19448d1a2f41000c780fac4c0c26 +F src/prepare.c 314961aa6650cc860394cb2f31931cf2de93baa8 F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 89e569b263535662f54b537eb9118b2c554ae7aa F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 -F src/sqlite.h.in 43852c8b68b4c579948cb37182918078836c5c06 +F src/sqlite.h.in 706b420dc3390532435a3bd196360a940805f02f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 6244ee9052752e26d1275ab20c9b774385aa57d2 +F src/sqliteInt.h 7c090825333d91ca392c2479a9e835e7b6a5eb12 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 7d100e2e7aad614bb3d7026a41a0e3827dbaaebc -F src/test1.c 363a5089230a92cf0aaa7a2945da7f2bf3b0a8d3 -F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 +F src/tclsqlite.c 29357f2be7b0d00e8ea900eaf727e0c5ffeaa660 +F src/test1.c e9a0e5804b078532e69e69ec14c8326bf2cfc318 +F src/test2.c 84f6a786aa7ffa12fff83acb52660e337ffe642a F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 @@ -290,13 +294,13 @@ F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h cdc8e421f85beb1ac9b4669ec5beadab6faa15e0 F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 -F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 +F src/vdbeblob.c 0bc9d22578d87ad9ff1c16e20a36863326f34fd7 F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62 F src/vdbesort.c 02646a9f86421776ae5d7594f620f9ed669d3698 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f -F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a -F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 +F src/wal.c 93b4fcb56a98f435a2cb66024bb2b12d66d1ff53 +F src/wal.h 237bc4484f7c289f094ecb6efb2b6c02005484e1 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 F src/where.c d9eae96b2cbbe4842eac3ee156ccd1b933d802c4 F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 @@ -1193,7 +1197,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4cae93f8ae8fb3fe38fd5dc7d3a5ea0d11552841 -R fe37d23fb1f96547febf2ff7c0ba49b9 -U drh -Z 29e2a2847ff584a00093c5b89ffca704 +P 9779c7a9eb1e2bd36e9286331a9314f064014d80 +R 5df4197c767caad19feca7391f16a42d +T *branch * experimental-bulk-update +T *sym-experimental-bulk-update * +T -sym-trunk * +U dan +Z 195de077202fec58ba90fc725d7305e7 diff --git a/manifest.uuid b/manifest.uuid index af2604f469..4ea970c6b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9779c7a9eb1e2bd36e9286331a9314f064014d80 \ No newline at end of file +2954ab501049968430011b63d046eb42ff37a56c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a04302225d..e9d737b6a0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -151,7 +151,8 @@ static int hasSharedCacheTableLock( ** and has the read-uncommitted flag set, then no lock is required. ** Return true immediately. */ - if( (pBtree->sharable==0) + if( (pBtree->db->flags & SQLITE_OtaMode) + || (pBtree->sharable==0) || (eLockType==READ_LOCK && (pBtree->db->flags & SQLITE_ReadUncommitted)) ){ return 1; @@ -2045,7 +2046,7 @@ int sqlite3BtreeOpen( btree_open_out: if( rc!=SQLITE_OK ){ if( pBt && pBt->pPager ){ - sqlite3PagerClose(pBt->pPager); + sqlite3PagerClose(pBt->pPager, 0); } sqlite3_free(pBt); sqlite3_free(p); @@ -2174,7 +2175,7 @@ int sqlite3BtreeClose(Btree *p){ ** Clean out and delete the BtShared object. */ assert( !pBt->pCursor ); - sqlite3PagerClose(pBt->pPager); + sqlite3PagerClose(pBt->pPager, (p->db->flags & SQLITE_OtaMode)!=0); if( pBt->xFreeSchema && pBt->pSchema ){ pBt->xFreeSchema(pBt->pSchema); } diff --git a/src/insert.c b/src/insert.c index 3e6982d836..2ca212c0cf 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1365,6 +1365,10 @@ void sqlite3GenerateConstraintChecks( int iThisCur; /* Cursor for this UNIQUE index */ int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ + /* If the "ota_mode" flag is set, ignore all indexes except the PK + ** index of WITHOUT ROWID tables. */ + if( (db->flags & SQLITE_OtaMode) && pIdx!=pPk) continue; + if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */ if( bAffinityDone==0 ){ sqlite3TableAffinity(v, pTab, regNewData+1); @@ -1556,6 +1560,15 @@ void sqlite3CompleteInsertion( assert( pTab->pSelect==0 ); /* This table is not a VIEW */ for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ if( aRegIdx[i]==0 ) continue; + + /* If the "ota_mode" flag is set, ignore all indexes except the PK + ** index of WITHOUT ROWID tables. */ + if( (pParse->db->flags & SQLITE_OtaMode) + && (pTab->iPKey>=0 || pIdx->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) + ){ + continue; + } + bAffinityDone = 1; if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); diff --git a/src/main.c b/src/main.c index fd7151b174..f73817e29b 100644 --- a/src/main.c +++ b/src/main.c @@ -3470,3 +3470,15 @@ int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ Btree *pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } + +int sqlite3_transaction_save(sqlite3 *db, void **ppState, int *pnState){ + Pager *pPager = sqlite3BtreePager(db->aDb[0].pBt); + return sqlite3PagerSaveState(pPager, ppState, pnState); +} + +int sqlite3_transaction_restore(sqlite3 *db, const void *pState, int nState){ + Pager *pPager = sqlite3BtreePager(db->aDb[0].pBt); + return sqlite3PagerRestoreState(pPager, pState, nState); +} + + diff --git a/src/pager.c b/src/pager.c index 3ef54d98e0..a54ae816e3 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3947,7 +3947,7 @@ static void pagerFreeMapHdrs(Pager *pPager){ ** a hot journal may be left in the filesystem but no error is returned ** to the caller. */ -int sqlite3PagerClose(Pager *pPager){ +int sqlite3PagerClose(Pager *pPager, int bOtaMode){ u8 *pTmp = (u8 *)pPager->pTmpSpace; assert( assert_pager_state(pPager) ); @@ -3957,7 +3957,9 @@ int sqlite3PagerClose(Pager *pPager){ /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL - sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp); + sqlite3WalClose( + pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (bOtaMode?0:pTmp) + ); pPager->pWal = 0; #endif pager_reset(pPager); @@ -7217,6 +7219,41 @@ int sqlite3PagerCloseWal(Pager *pPager){ return rc; } +int sqlite3PagerSaveState(Pager *pPager, void **ppState, int *pnState){ + int rc = SQLITE_OK; + *ppState = 0; + *pnState = 0; + if( pPager->pWal==0 || pPager->eStatepPCache); + rc = sqlite3WalFrames(pPager->pWal, + pPager->pageSize, pList, 0, 0, pPager->walSyncFlags + ); + if( rc==SQLITE_OK ){ + rc = sqlite3WalSaveState(pPager->pWal, ppState, pnState); + } + } + return rc; +} + +int sqlite3PagerRestoreState(Pager *pPager, const void *pState, int nState){ + int rc = SQLITE_OK; + if( pPager->pWal==0 + || pPager->eStatepPCache) + ){ + rc = SQLITE_ERROR; + }else{ + sqlite3PcacheTruncate(pPager->pPCache, 1); + rc = sqlite3WalRestoreState(pPager->pWal, pState, nState); + pPager->eState = PAGER_WRITER_CACHEMOD; + } + + return rc; +} + #endif /* !SQLITE_OMIT_WAL */ #ifdef SQLITE_ENABLE_ZIPVFS diff --git a/src/pager.h b/src/pager.h index c9ca8553b9..79ffa04db8 100644 --- a/src/pager.h +++ b/src/pager.h @@ -112,7 +112,7 @@ int sqlite3PagerOpen( int, void(*)(DbPage*) ); -int sqlite3PagerClose(Pager *pPager); +int sqlite3PagerClose(Pager *pPager, int); int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ @@ -207,4 +207,7 @@ void *sqlite3PagerCodec(DbPage *); # define enable_simulated_io_errors() #endif +int sqlite3PagerSaveState(Pager *pPager, void **ppState, int *pnState); +int sqlite3PagerRestoreState(Pager *pPager, const void *pState, int nState); + #endif /* _PAGER_H_ */ diff --git a/src/pragma.c b/src/pragma.c index 12446125fb..3f06a51839 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -308,6 +308,12 @@ static const struct sPragmaNames { /* ePragTyp: */ PragTyp_MMAP_SIZE, /* ePragFlag: */ 0, /* iArg: */ 0 }, +#endif + { /* zName: */ "ota_mode", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_OtaMode }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "page_count", /* ePragTyp: */ PragTyp_PAGE_COUNT, /* ePragFlag: */ PragFlag_NeedSchema, @@ -1467,7 +1473,11 @@ void sqlite3Pragma( }else if( pPk==0 ){ k = 1; }else{ - for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} + if( (db->flags & SQLITE_OtaMode) && HasRowid(pTab) ){ + k = 0; + }else{ + for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} + } } sqlite3VdbeAddOp2(v, OP_Integer, k, 6); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); diff --git a/src/prepare.c b/src/prepare.c index 5b92e88513..44efe8c8fb 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -796,7 +796,6 @@ int sqlite3_prepare_v2( return rc; } - #ifndef SQLITE_OMIT_UTF16 /* ** Compile the UTF-16 encoded SQL statement zSql into a statement handle. diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d446d2ea98..7011b67fd4 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7363,7 +7363,79 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* #define SQLITE_ABORT 4 // Also an error code */ #define SQLITE_REPLACE 5 +/* +** Allocate a statement handle that may be used to write directly to an +** index b-tree. This allows the user to create a corrupt database. Once +** the statement handle is allocated, it may be used with the same APIs +** as any statement handle created with sqlite3_prepare(). +** +** The statement writes to the index specified by parameter zIndex, which +** must be in the "main" database. If argument bDelete is false, then each +** time the statement is sqlite3_step()ed, an entry is inserted into the +** b-tree index. If it is true, then an entry may be deleted (or may not, if +** the specified key is not found) each time the statement is +** sqlite3_step()ed. +** +** If statement compilation is successful, *ppStmt is set to point to the +** new statement handle and SQLITE_OK is returned. Otherwise, if an error +** occurs, *ppStmt is set to NULL and an error code returned. An error +** message may be left in the database handle in this case. +** +** If statement compilation succeeds, output variable *pnCol is set to the +** total number of columns in the index, including the primary key columns +** at the end. Variable *paiCol is set to point to an array *pnCol entries +** in size. Each entry is the table column index, numbered from zero from left +** to right, of the corresponding index column. For example, if: +** +** CREATE TABLE t1(a, b, c, d); +** CREATE INDEX i1 ON t1(b, c); +** +** then *pnCol is 3 and *paiCol points to an array containing {1, 2, -1}. +** If table t1 had an explicit INTEGER PRIMARY KEY, then the "-1" in the +** *paiCol array would be replaced by its column index. Or if: +** +** CREATE TABLE t2(a, b, c, d, PRIMARY KEY(d, c)) WITHOUT ROWID; +** CREATE INDEX i2 ON t2(a); +** +** then (*pnCol) is 3 and *paiCol points to an array containing {0, 3, 2}. +** +** The lifetime of the array is the same as that of the statement handle - +** it is automatically freed when the statement handle is passed to +** sqlite3_finalize(). +** +** The statement has (*pnCol) SQL variables that values may be bound to. +** They correspond to the values used to create the index key that is +** inserted or deleted when the statement is stepped. +** +** If the index is a UNIQUE index, the usual checking and error codes apply +** to insert operations. +*/ +int sqlite3_index_writer( + sqlite3 *db, + int bDelete, /* Zero for insert, non-zero for delete */ + const char *zIndex, /* Index to write to */ + sqlite3_stmt**, /* OUT: New statement handle */ + int **paiCol, int *pnCol /* OUT: See above */ +); +/* +** This function is used to save the state of an ongoing WAL mode write +** transaction on the "main" database of the supplied database handle. +** +** If successful, SQLITE_OK is returned and output variable (*ppState) +** is set to point to a buffer containing the transaction state data. +** (*pnState) is set to the size of that buffer in bytes. Otherwise, if +** an error occurs, an SQLite error code is returned and both output +** variables are zeroed. +** +** A transaction state may be saved if: +** +** * the transaction does not contain any schema modifications. +** * there are no open sub-transactions. +*/ +int sqlite3_transaction_save(sqlite3 *db, void **ppState, int *pnState); + +int sqlite3_transaction_restore(sqlite3 *db, const void *pState, int nState); /* ** Undo the hack that converts floating point types to integer for diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7fd999d9ee..11396c651b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1141,6 +1141,8 @@ struct sqlite3 { #define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ #define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_OtaMode 0x08000000 /* True in "ota mode" */ + /* ** Bits of the sqlite3.dbOptFlags field that are used by the @@ -3726,6 +3728,7 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); #define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ #define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ + /* ** Threading interface */ diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 2b98b6aab4..ce88109007 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3699,6 +3699,8 @@ static void init_all(Tcl_Interp *interp){ extern int SqliteSuperlock_Init(Tcl_Interp*); extern int SqlitetestSyscall_Init(Tcl_Interp*); + extern int SqliteOta_Init(Tcl_Interp*); + #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) extern int Sqlitetestfts3_Init(Tcl_Interp *interp); #endif @@ -3740,6 +3742,7 @@ static void init_all(Tcl_Interp *interp){ Sqlitemultiplex_Init(interp); SqliteSuperlock_Init(interp); SqlitetestSyscall_Init(interp); + SqliteOta_Init(interp); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) Sqlitetestfts3_Init(interp); diff --git a/src/test1.c b/src/test1.c index 34faeaadf8..d75f268deb 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6497,6 +6497,70 @@ static int sorter_test_sort4_helper( } +/* +** tclcmd: sqlite3_transaction_save DB +*/ +static int testTransactionSave( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + void *pState; + int nState; + sqlite3 *db; + int rc; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + + rc = sqlite3_transaction_save(db, &pState, &nState); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pState, nState)); + }else{ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + } + + sqlite3_free(pState); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_transaction_restore DB BLOB +*/ +static int testTransactionRestore( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + void *pState; + int nState; + sqlite3 *db; + int rc; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB BLOB"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + pState = (void*)Tcl_GetByteArrayFromObj(objv[2], &nState); + + rc = sqlite3_transaction_restore(db, pState, nState); + if( rc==SQLITE_OK ){ + Tcl_ResetResult(interp); + }else{ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + } + + return TCL_OK; +} + /* ** Register commands with the TCL interpreter. */ @@ -6734,6 +6798,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "load_static_extension", tclLoadStaticExtensionCmd }, { "sorter_test_fakeheap", sorter_test_fakeheap }, { "sorter_test_sort4_helper", sorter_test_sort4_helper }, + { "sqlite3_transaction_save", testTransactionSave }, + { "sqlite3_transaction_restore", testTransactionRestore }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; diff --git a/src/test2.c b/src/test2.c index 58f271ff27..6d655d6473 100644 --- a/src/test2.c +++ b/src/test2.c @@ -89,7 +89,7 @@ static int pager_close( return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); - rc = sqlite3PagerClose(pPager); + rc = sqlite3PagerClose(pPager, 0); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 71bd8816d5..11e018e19c 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -463,4 +463,116 @@ int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ return rc; } +int sqlite3_index_writer( + sqlite3 *db, + int bDelete, + const char *zIndex, + sqlite3_stmt **ppStmt, + int **paiCol, int *pnCol +){ + int rc = SQLITE_OK; + Parse *pParse = 0; + Index *pIdx = 0; /* The index to write to */ + Table *pTab; + int i; /* Used to iterate through index columns */ + Vdbe *v = 0; + int regRec; /* Register to assemble record in */ + int *aiCol = 0; + + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + + /* Allocate the parse context */ + pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); + if( !pParse ) goto index_writer_out; + memset(pParse, 0, sizeof(Parse)); + pParse->db = db; + + /* Allocate the Vdbe */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto index_writer_out; + + /* Find the index to write to */ + pIdx = sqlite3FindIndex(db, zIndex, "main"); + if( pIdx==0 ){ + sqlite3ErrorMsg(pParse, "no such index: %s", zIndex); + goto index_writer_out; + } + pTab = pIdx->pTable; + + /* Populate the two output variables, *pnCol and *pnAiCol. */ + *pnCol = pIdx->nColumn; + *paiCol = aiCol = sqlite3DbMallocZero(db, sizeof(int) * pIdx->nColumn); + if( aiCol==0 ){ + rc = SQLITE_NOMEM; + goto index_writer_out; + } + for(i=0; inKeyCol; i++){ + aiCol[i] = pIdx->aiColumn[i]; + } + if( !HasRowid(pTab) ){ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + assert( pIdx->nColumn==pIdx->nKeyCol+pPk->nKeyCol ); + if( pPk==pIdx ){ + rc = SQLITE_ERROR; + goto index_writer_out; + } + for(i=0; inKeyCol; i++){ + aiCol[pIdx->nKeyCol+i] = pPk->aiColumn[i]; + } + }else{ + assert( pIdx->nColumn==pIdx->nKeyCol+1 ); + aiCol[i] = pTab->iPKey; + } + + /* Add an OP_Noop to the VDBE program. Then store a pointer to the + ** output array *paiCol as its P4 value. This is so that the array + ** is automatically deleted when the user finalizes the statement. The + ** OP_Noop serves no other purpose. */ + sqlite3VdbeAddOp0(v, OP_Noop); + sqlite3VdbeChangeP4(v, -1, (const char*)aiCol, P4_INTARRAY); + + sqlite3BeginWriteOperation(pParse, 0, 0); + + /* Open a write cursor on the index */ + pParse->nTab = 1; + sqlite3VdbeAddOp3(v, OP_OpenWrite, 0, pIdx->tnum, 0); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + + /* Create the record to insert into the index. Store it in register regRec. */ + pParse->nVar = pIdx->nColumn; + pParse->nMem = pIdx->nColumn; + for(i=1; i<=pIdx->nColumn; i++){ + sqlite3VdbeAddOp2(v, OP_Variable, i, i); + } + regRec = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_MakeRecord, 1, pIdx->nColumn, regRec); + + /* If this is a UNIQUE index, check the constraint. */ + if( pIdx->onError ){ + int addr = sqlite3VdbeAddOp4Int(v, OP_NoConflict, 0, 0, 1, pIdx->nKeyCol); + sqlite3UniqueConstraint(pParse, SQLITE_ABORT, pIdx); + sqlite3VdbeJumpHere(v, addr); + } + + /* Code the IdxInsert to write to the b-tree index. */ + sqlite3VdbeAddOp2(v, OP_IdxInsert, 0, regRec); + sqlite3FinishCoding(pParse); + +index_writer_out: + if( rc==SQLITE_OK && db->mallocFailed==0 ){ + *ppStmt = (sqlite3_stmt*)v; + }else{ + *ppStmt = 0; + if( v ) sqlite3VdbeFinalize(v); + } + + sqlite3ParserReset(pParse); + sqlite3StackFree(db, pParse); + sqlite3BtreeLeaveAll(db); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + #endif /* #ifndef SQLITE_OMIT_INCRBLOB */ diff --git a/src/wal.c b/src/wal.c index 70395f8bee..a030824cf7 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1046,6 +1046,62 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ return rc; } +static int walFileReadHdr(Wal *pWal, int *pbValid){ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + int rc; /* Return code */ + u32 magic; /* Magic value read from WAL header */ + int szPage; /* Page size according to the log */ + u32 version; /* Magic value read from WAL header */ + + *pbValid = 0; + + /* Read in the WAL header. */ + rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If the database page size is not a power of two, or is greater than + ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid + ** data. Similarly, if the 'magic' value is invalid, ignore the whole + ** WAL file. + */ + magic = sqlite3Get4byte(&aBuf[0]); + szPage = sqlite3Get4byte(&aBuf[8]); + if( (magic&0xFFFFFFFE)!=WAL_MAGIC + || szPage&(szPage-1) + || szPage>SQLITE_MAX_PAGE_SIZE + || szPage<512 + ){ + return SQLITE_OK; + } + + pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); + pWal->szPage = szPage; + pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); + memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); + + /* Verify that the WAL header checksum is correct */ + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum + ); + if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) + ){ + return SQLITE_OK; + } + + /* Verify that the version number on the WAL format is one that + ** are able to understand */ + version = sqlite3Get4byte(&aBuf[4]); + if( version!=WAL_MAX_VERSION ){ + return SQLITE_CANTOPEN_BKPT; + } + + *pbValid = 1; + return SQLITE_OK; +} + /* ** Recover the wal-index by reading the write-ahead log file. @@ -1090,59 +1146,18 @@ static int walIndexRecover(Wal *pWal){ } if( nSize>WAL_HDRSIZE ){ - u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ int szFrame; /* Number of bytes in buffer aFrame[] */ u8 *aData; /* Pointer to data part of aFrame buffer */ int iFrame; /* Index of last frame read */ i64 iOffset; /* Next offset to read from log file */ int szPage; /* Page size according to the log */ - u32 magic; /* Magic value read from WAL header */ - u32 version; /* Magic value read from WAL header */ int isValid; /* True if this frame is valid */ - /* Read in the WAL header. */ - rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); - if( rc!=SQLITE_OK ){ - goto recovery_error; - } - - /* If the database page size is not a power of two, or is greater than - ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid - ** data. Similarly, if the 'magic' value is invalid, ignore the whole - ** WAL file. - */ - magic = sqlite3Get4byte(&aBuf[0]); - szPage = sqlite3Get4byte(&aBuf[8]); - if( (magic&0xFFFFFFFE)!=WAL_MAGIC - || szPage&(szPage-1) - || szPage>SQLITE_MAX_PAGE_SIZE - || szPage<512 - ){ - goto finished; - } - pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); - pWal->szPage = szPage; - pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); - memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); - - /* Verify that the WAL header checksum is correct */ - walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, - aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum - ); - if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) - || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) - ){ - goto finished; - } - - /* Verify that the version number on the WAL format is one that - ** are able to understand */ - version = sqlite3Get4byte(&aBuf[4]); - if( version!=WAL_MAX_VERSION ){ - rc = SQLITE_CANTOPEN_BKPT; - goto finished; - } + rc = walFileReadHdr(pWal, &isValid); + if( rc!=SQLITE_OK ) goto recovery_error; + if( isValid==0 ) goto finished; + szPage = pWal->szPage; /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; @@ -1837,32 +1852,34 @@ int sqlite3WalClose( ** ** The EXCLUSIVE lock is not released before returning. */ - rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); - if( rc==SQLITE_OK ){ - if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ - pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; - } - rc = sqlite3WalCheckpoint( - pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 - ); + if( zBuf ){ + rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ - int bPersist = -1; - sqlite3OsFileControlHint( - pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist + if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; + } + rc = sqlite3WalCheckpoint( + pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 ); - if( bPersist!=1 ){ - /* Try to delete the WAL file if the checkpoint completed and - ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal - ** mode (!bPersist) */ - isDelete = 1; - }else if( pWal->mxWalSize>=0 ){ - /* Try to truncate the WAL file to zero bytes if the checkpoint - ** completed and fsynced (rc==SQLITE_OK) and we are in persistent - ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a - ** non-negative value (pWal->mxWalSize>=0). Note that we truncate - ** to zero bytes as truncating to the journal_size_limit might - ** leave a corrupt WAL file on disk. */ - walLimitSize(pWal, 0); + if( rc==SQLITE_OK ){ + int bPersist = -1; + sqlite3OsFileControlHint( + pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist + ); + if( bPersist!=1 ){ + /* Try to delete the WAL file if the checkpoint completed and + ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal + ** mode (!bPersist) */ + isDelete = 1; + }else if( pWal->mxWalSize>=0 ){ + /* Try to truncate the WAL file to zero bytes if the checkpoint + ** completed and fsynced (rc==SQLITE_OK) and we are in persistent + ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a + ** non-negative value (pWal->mxWalSize>=0). Note that we truncate + ** to zero bytes as truncating to the journal_size_limit might + ** leave a corrupt WAL file on disk. */ + walLimitSize(pWal, 0); + } } } } @@ -3079,6 +3096,157 @@ int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } +/* +** Save current transaction state. +** +** The transaction state consists of a series of 32-bit big-endian integers: +** +** * initial number of frames in WAL file. +** * initial checksum values (2 integers). +** * current number of frames. +** * current checksum values (2 integers). +*/ +int sqlite3WalSaveState(Wal *pWal, void **ppState, int *pnState){ + int rc = SQLITE_OK; + + *ppState = 0; + *pnState = 0; + if( pWal->writeLock==0 ){ + /* Must be in a write transaction to call this function. */ + rc = SQLITE_ERROR; + }else{ + WalIndexHdr *pOrig = (WalIndexHdr*)walIndexHdr(pWal); + int nBuf = 6 * 4; /* Bytes of space to allocate */ + u8 *aBuf; + + aBuf = sqlite3_malloc(nBuf); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3Put4byte(&aBuf[0], pOrig->mxFrame); + sqlite3Put4byte(&aBuf[4], pOrig->aFrameCksum[0]); + sqlite3Put4byte(&aBuf[8], pOrig->aFrameCksum[1]); + sqlite3Put4byte(&aBuf[12], pWal->hdr.mxFrame); + sqlite3Put4byte(&aBuf[16], pWal->hdr.aFrameCksum[0]); + sqlite3Put4byte(&aBuf[20], pWal->hdr.aFrameCksum[1]); + *ppState = (void*)aBuf; + *pnState = nBuf; + } + } + + return rc; +} + +static int walUndoNoop(void *pUndoCtx, Pgno pgno){ + UNUSED_PARAMETER(pUndoCtx); + UNUSED_PARAMETER(pgno); + return SQLITE_OK; +} + +/* +** If possible, restore the state of the curent transaction to that +** described by the second and third arguments. +*/ +int sqlite3WalRestoreState(Wal *pWal, const void *pState, int nState){ + int rc = SQLITE_OK; + + if( pWal->writeLock==0 ){ + /* Must have opened a write transaction to call this */ + rc = SQLITE_ERROR; + }else{ + u8 *aBuf = (u8*)pState; + int szFrame; /* Size of each frame in WAL file */ + u8 *aFrame = 0; /* Buffer to read data into */ + u8 *aData; /* Data part of aFrame[] buffer */ + u32 mxFrame; /* Maximum frame following restoration */ + int i; /* Iterator variable */ + + WalIndexHdr *pOrig = (WalIndexHdr*)walIndexHdr(pWal); + + /* Check that no dirty pages have been written to the WAL file since + ** the current transaction was opened. */ + if( pOrig->mxFrame!=pWal->hdr.mxFrame + || pOrig->aFrameCksum[0]!=pWal->hdr.aFrameCksum[0] + || pOrig->aFrameCksum[1]!=pWal->hdr.aFrameCksum[1] + ){ + rc = SQLITE_ERROR; + } + + /* Check that the WAL file is in the same state that it was when the + ** transaction was saved. If not, return SQLITE_MISMATCH - cannot + ** resume this transaction */ + if( rc==SQLITE_OK && ( + pWal->hdr.mxFrame!=sqlite3Get4byte(&aBuf[0]) + || pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[4]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[8]) + )){ + rc = SQLITE_MISMATCH; + } + + if( rc==SQLITE_OK && pWal->readLock==0 ){ + int cnt = 0; + walUnlockShared(pWal, WAL_READ_LOCK(0)); + pWal->readLock = -1; + do{ + int notUsed; + rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); + }while( rc==WAL_RETRY ); + + if( rc==SQLITE_OK ){ + int bValid; + rc = walFileReadHdr(pWal, &bValid); + if( rc==SQLITE_OK && bValid==0 ) rc = SQLITE_MISMATCH; + pWal->hdr.szPage = (u16)((pWal->szPage&0xff00) | (pWal->szPage>>16)); + } + } + + /* Malloc a buffer to read frames into. */ + if( rc==SQLITE_OK ){ + szFrame = pWal->szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8*)sqlite3_malloc(szFrame); + if( !aFrame ){ + rc = SQLITE_NOMEM; + }else{ + aData = &aFrame[WAL_FRAME_HDRSIZE]; + } + } + + mxFrame = sqlite3Get4byte(&aBuf[12]); + for(i=pWal->hdr.mxFrame+1; rc==SQLITE_OK && i<=mxFrame; i++){ + sqlite3_int64 iOff = walFrameOffset(i, pWal->szPage); + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOff); + if( rc==SQLITE_OK ){ + u32 iPg; + u32 dummy; + if( 0==walDecodeFrame(pWal, &iPg, &dummy, aData, aFrame) ){ + rc = SQLITE_MISMATCH; + }else{ + rc = walIndexAppend(pWal, i, iPg); + if( iPg>pWal->hdr.nPage ) pWal->hdr.nPage = iPg; + } + pWal->hdr.mxFrame = i; + } + } + sqlite3_free(aFrame); + + if( rc==SQLITE_OK ){ + assert( pWal->hdr.mxFrame==mxFrame ); + if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[16]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[20]) + ){ + rc = SQLITE_MISMATCH; + } + } + + + if( rc!=SQLITE_OK ){ + sqlite3WalUndo(pWal, walUndoNoop, 0); + } + } + + return rc; +} + #ifdef SQLITE_ENABLE_ZIPVFS /* ** If the argument is not NULL, it points to a Wal object that holds a diff --git a/src/wal.h b/src/wal.h index 092546354b..08006fafe9 100644 --- a/src/wal.h +++ b/src/wal.h @@ -126,6 +126,9 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); +int sqlite3WalSaveState(Wal *pWal, void **ppState, int *pnState); +int sqlite3WalRestoreState(Wal *pWal, const void *pState, int nState); + #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). From 98231c054b78e0868b55336389673ab77e968a10 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Sep 2014 08:25:09 +0000 Subject: [PATCH 002/116] Add a command line program that uses the extension. This serves as example code and is also useful for performance testing. FossilOrigin-Name: ffa1524ef2a4c32652183eb4745685f0d1c93af2 --- ext/ota/ota.c | 92 ++++++++++++++++++++++++++++++++++++++++++++ ext/ota/sqlite3ota.h | 4 +- main.mk | 5 +++ manifest | 18 ++++----- manifest.uuid | 2 +- 5 files changed, 109 insertions(+), 12 deletions(-) create mode 100644 ext/ota/ota.c diff --git a/ext/ota/ota.c b/ext/ota/ota.c new file mode 100644 index 0000000000..c9d576f1aa --- /dev/null +++ b/ext/ota/ota.c @@ -0,0 +1,92 @@ +/* +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains a command-line application that uses the OTA +** extension. See the usage() function below for an explanation. +*/ + +#include "sqlite3ota.h" +#include +#include +#include + +/* +** Print a usage message and exit. +*/ +void usage(const char *zArgv0){ + fprintf(stderr, +"Usage: %s [-step NSTEP] TARGET-DB OTA-DB\n" +"\n" +" Argument OTA-DB must be an OTA database containing an update suitable for\n" +" target database TARGET-DB. If NSTEP is set to less than or equal to zero\n" +" (the default value), this program attempts to apply the entire update to\n" +" the target database.\n" +"\n" +" If NSTEP is greater than zero, then a maximum of NSTEP calls are made\n" +" to sqlite3ota_step(). If the OTA update has not been completely applied\n" +" after the NSTEP'th call is made, the state is saved in the database OTA-DB\n" +" and the program exits. Subsequent invocations of this (or any other OTA)\n" +" application will use this state to resume applying the OTA update to the\n" +" target db.\n" +"\n" +, zArgv0); + exit(1); +} + +int main(int argc, char **argv){ + int i; + const char *zTarget; /* Target database to apply OTA to */ + const char *zOta; /* Database containing OTA */ + char *zErrmsg; /* Error message, if any */ + sqlite3ota *pOta; /* OTA handle */ + int nStep = 0; /* Maximum number of step() calls */ + int rc; + + /* Process command line arguments. Following this block local variables + ** zTarget, zOta and nStep are all set. */ + if( argc==5 ){ + int nArg1 = strlen(argv[1]); + if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]); + nStep = atoi(argv[2]); + }else if( argc!=3 ){ + usage(argv[0]); + } + zTarget = argv[argc-2]; + zOta = argv[argc-1]; + + /* Open an OTA handle. If nStep is less than or equal to zero, call + ** sqlite3ota_step() until either the OTA has been completely applied + ** or an error occurs. Or, if nStep is greater than zero, call + ** sqlite3ota_step() a maximum of nStep times. */ + pOta = sqlite3ota_open(zTarget, zOta); + for(i=0; (nStep<=0 || i /* Required for error code definitions */ + typedef struct sqlite3ota sqlite3ota; /* diff --git a/main.mk b/main.mk index b225e4ecd0..0c4a27501f 100644 --- a/main.mk +++ b/main.mk @@ -665,6 +665,11 @@ wordcount$(EXE): $(TOP)/test/wordcount.c sqlite3.c speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.o $(TCC) -I. -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB) +ota$(EXE): $(TOP)/ext/ota/ota.c $(TOP)/ext/ota/sqlite3ota.c sqlite3.o + $(TCC) -I. -o ota$(EXE) \ + $(TOP)/ext/ota/ota.c $(TOP)/ext/ota/sqlite3ota.c sqlite3.o \ + $(THREADLIB) + # This target will fail if the SQLite amalgamation contains any exported # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. diff --git a/manifest b/manifest index da360992c2..ca224c9b1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexperimental\sextension\sfor\sapplying\sbulk\supdates\sto\sdatabases. -D 2014-09-02T19:59:40.729 +C Add\sa\scommand\sline\sprogram\sthat\suses\sthe\sextension.\sThis\sserves\sas\sexample\scode\sand\sis\salso\suseful\sfor\sperformance\stesting. +D 2014-09-03T08:25:09.548 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,10 +121,11 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 +F ext/ota/ota.c 4b48add9494f29144343f513aaac226ca5782189 F ext/ota/ota1.test ea2865997ce573fadaf12eb0a0f80ef22d9dd77f F ext/ota/ota2.test 4f7abfe1dfb7c3709bf45e94f3e65f3839b4f115 F ext/ota/sqlite3ota.c ad55821883e4110367a30ffca282032d2bf36e45 -F ext/ota/sqlite3ota.h d3187a98fe1e3445c58f7a27d96ac385b78486a1 +F ext/ota/sqlite3ota.h 545f0008b5f02f2595899cb9841caddada5c17c0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -151,7 +152,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 566c36f247d19525264e584466b5f07c0a48302e +F main.mk ed2e37f6cd016ecc614165b3392bdbf89a129b4a F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1197,10 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9779c7a9eb1e2bd36e9286331a9314f064014d80 -R 5df4197c767caad19feca7391f16a42d -T *branch * experimental-bulk-update -T *sym-experimental-bulk-update * -T -sym-trunk * +P 2954ab501049968430011b63d046eb42ff37a56c +R 7bee98aca78bdb17f817d7637df95f18 U dan -Z 195de077202fec58ba90fc725d7305e7 +Z 3a37163be54ffa74ee7c4647ff1214ea diff --git a/manifest.uuid b/manifest.uuid index 4ea970c6b0..7d509edecc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2954ab501049968430011b63d046eb42ff37a56c \ No newline at end of file +ffa1524ef2a4c32652183eb4745685f0d1c93af2 \ No newline at end of file From 19b465a0ae4c73ad9f118f85bc134e7a7d515949 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Sep 2014 19:30:32 +0000 Subject: [PATCH 003/116] Split part of "PRAGMA ota_mode" off into "PRAGMA pager_ota_mode". This allows some specialized custom VFS implementations to intercept and implement the expected pager-related effects of this pragma. FossilOrigin-Name: 209f672e588b54dfbfb83c7859cacdc4497f0f2b --- ext/ota/ota2.test | 77 +++++++----- ext/ota/sqlite3ota.c | 285 +++++++++++++++++++++++++++---------------- manifest | 26 ++-- manifest.uuid | 2 +- src/btree.c | 4 +- src/pager.c | 71 +++++++---- src/pager.h | 4 +- src/pragma.c | 55 ++++++--- src/test2.c | 2 +- tool/mkpragmatab.tcl | 6 + 10 files changed, 333 insertions(+), 199 deletions(-) diff --git a/ext/ota/ota2.test b/ext/ota/ota2.test index dfa4a48e9d..d80f7360c9 100644 --- a/ext/ota/ota2.test +++ b/ext/ota/ota2.test @@ -14,49 +14,60 @@ set testdir [file join [file dirname $argv0] .. .. test] source $testdir/tester.tcl set ::testprefix ota2 +forcedelete test.db-oal do_execsql_test 1.0 { - PRAGMA ota_mode = 1; - PRAGMA journal_mode = wal; CREATE TABLE t1(a, b); - BEGIN; - INSERT INTO t1 VALUES(1, 2); -} {wal} - -do_test 1.1 { - set state [sqlite3_transaction_save db] - db close - file exists test.db-wal -} {1} - -do_test 1.2 { - sqlite3 db test.db - db eval {SELECT * FROM t1} + INSERT INTO t1 VALUES(1, 2); } {} +do_test 1.1 { glob test.db* } {test.db} -do_test 1.3 { - execsql {BEGIN IMMEDIATE} - sqlite3_transaction_restore db $::state - db eval {SELECT * FROM t1} -} {1 2} +do_execsql_test 1.2 { + PRAGMA pager_ota_mode = 1; + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + SELECT * FROM t1; +} {1 2 3 4 5 6} + +do_test 1.3 { glob test.db* } {test.db test.db-oal} do_test 1.4 { - execsql { - INSERT INTO t1 VALUES(3, 4); - COMMIT; - SELECT * FROM t1; - } -} {1 2 3 4} + sqlite3 db2 test.db + db2 eval { SELECT * FROM t1 } +} {1 2} do_test 1.5 { - db close - file exists test.db-wal -} {0} + catchsql { INSERT INTO t1 VALUES(7, 8) } db2 +} {1 {database is locked}} + +db2 close +db close + +sqlite3 db test.db +do_execsql_test 1.6 { + PRAGMA pager_ota_mode = 1; + SELECT * FROM t1; +} {1 2 3 4 5 6} + +do_execsql_test 1.7 { + INSERT INTO t1 VALUES(7,8); + SELECT * FROM t1; +} {1 2 3 4 5 6 7 8} + +db close +sqlite3 db2 test.db + +do_test 1.8 { + execsql { BEGIN; SELECT * FROM t1 } db2 +} {1 2} +do_test 1.9 { + file rename test.db-oal test.db-wal + execsql { SELECT * FROM t1 } db2 +} {1 2} +do_test 1.10 { + execsql { COMMIT; SELECT * FROM t1 } db2 +} {1 2 3 4 5 6 7 8} -do_test 1.5 { - sqlite3 db test.db - db eval {SELECT * FROM t1} -} {1 2 3 4} finish_test diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 5483b9049f..2f61c671ac 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -13,6 +13,8 @@ #include #include +#include +#include #include "sqlite3.h" #include "sqlite3ota.h" @@ -23,8 +25,6 @@ ** update so that it can be resumed later. The table contains at most a ** single row: ** -** "wal_state" -> Blob to use with sqlite3_transaction_restore(). -** ** "tbl" -> Table currently being written (target database names). ** ** "idx" -> Index currently being written (target database names). @@ -36,10 +36,11 @@ ** so far as part of this ota update. */ #define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota_state" \ - "(wal_state, tbl, idx, row, progress)" + "(tbl, idx, row, progress)" typedef struct OtaTblIter OtaTblIter; typedef struct OtaIdxIter OtaIdxIter; +typedef struct OtaState OtaState; /* ** Iterator used to iterate through all data tables in the OTA. As follows: @@ -87,10 +88,17 @@ struct OtaIdxIter { sqlite3_stmt *pSelect; /* Select to read values in index order */ }; +struct OtaState { + char *zTbl; + char *zIdx; + sqlite3_int64 iRow; +}; + struct sqlite3ota { sqlite3 *dbDest; /* Target db */ sqlite3 *dbOta; /* Ota db */ + char *zTarget; /* Path to target db */ int rc; /* Value returned by last ota_step() call */ char *zErrmsg; /* Error message if rc!=SQLITE_OK */ @@ -506,133 +514,180 @@ static void otaOpenDatabase(sqlite3ota *p, sqlite3 **pDb, const char *zFile){ } static void otaSaveTransactionState(sqlite3ota *p){ - sqlite3_stmt *pStmt = 0; - void *pWalState = 0; - int nWalState = 0; - int rc; + sqlite3_stmt *pSelect; + char *zInsert; - const char *zInsert = - "INSERT INTO ota_state(wal_state, tbl, idx, row, progress)" - "VALUES(:wal_state, :tbl, :idx, :row, :progress)"; + pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect); + zInsert = sqlite3_mprintf( + "INSERT OR REPLACE INTO ota_state(rowid, tbl, idx, row, progress)" + "VALUES(1, %Q, %Q, %lld, NULL)", + p->tbliter.zTarget, p->idxiter.zIndex, sqlite3_column_int64(pSelect, 0) + ); + if( zInsert==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_exec(p->dbOta, zInsert, 0, 0, &p->zErrmsg); + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbOta, "COMMIT", 0, 0, &p->zErrmsg); + } + } - rc = sqlite3_transaction_save(p->dbDest, &pWalState, &nWalState); - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->dbOta, "DELETE FROM ota_state", 0, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = prepareAndCollectError(p->dbOta, zInsert, &pStmt, &p->zErrmsg); - } - if( rc==SQLITE_OK ){ - sqlite3_stmt *pSelect; - pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect); - sqlite3_bind_blob(pStmt, 1, pWalState, nWalState, SQLITE_STATIC); - sqlite3_bind_text(pStmt, 2, p->tbliter.zTarget, -1, SQLITE_STATIC); - if( p->idxiter.zIndex ){ - sqlite3_bind_text(pStmt, 3, p->idxiter.zIndex, -1, SQLITE_STATIC); - } - sqlite3_bind_int64(pStmt, 4, sqlite3_column_int64(pSelect, 0)); - sqlite3_step(pStmt); - rc = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->dbOta, "COMMIT", 0, 0, 0); - } - if( rc!=SQLITE_OK ){ - p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->dbOta)); - } - } - sqlite3_free(pWalState); - assert( p->rc==SQLITE_OK ); - p->rc = rc; + sqlite3_free(zInsert); } -static void otaLoadTransactionState(sqlite3ota *p){ - sqlite3_stmt *pStmt = 0; +/* +** Allocate an OtaState object and load the contents of the ota_state +** table into it. Return a pointer to the new object. It is the +** responsibility of the caller to eventually free the object using +** sqlite3_free(). +** +** If an error occurs, leave an error code and message in the ota handle +** and return NULL. +*/ +static OtaState *otaLoadState(sqlite3ota *p){ + const char *zSelect = "SELECT tbl, idx, row, progress FROM ota_state"; + OtaState *pRet = 0; + sqlite3_stmt *pStmt; int rc; - const char *zSelect = - "SELECT wal_state, tbl, idx, row, progress FROM ota_state"; - + assert( p->rc==SQLITE_OK ); rc = prepareAndCollectError(p->dbOta, zSelect, &pStmt, &p->zErrmsg); if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - const void *pWalState = 0; - int nWalState = 0; - const char *zTbl; - const char *zIdx; - sqlite3_int64 iRowid; + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zIdx = (const char*)sqlite3_column_text(pStmt, 1); + const char *zTbl = (const char*)sqlite3_column_text(pStmt, 0); + int nIdx = zIdx ? (strlen(zIdx) + 1) : 0; + int nTbl = strlen(zTbl) + 1; + int nByte = sizeof(OtaState) + nTbl + nIdx; - pWalState = sqlite3_column_blob(pStmt, 0); - nWalState = sqlite3_column_bytes(pStmt, 0); - zTbl = (const char*)sqlite3_column_text(pStmt, 1); - zIdx = (const char*)sqlite3_column_text(pStmt, 2); - iRowid = sqlite3_column_int64(pStmt, 3); - rc = sqlite3_transaction_restore(p->dbDest, pWalState, nWalState); - - while( rc==SQLITE_OK - && p->tbliter.zTarget - && sqlite3_stricmp(p->tbliter.zTarget, zTbl) - ){ - rc = tblIterNext(&p->tbliter); + pRet = (OtaState*)sqlite3_malloc(nByte); + if( pRet ){ + pRet->zTbl = (char*)&pRet[1]; + memcpy(pRet->zTbl, sqlite3_column_text(pStmt, 0), nTbl); + if( zIdx ){ + pRet->zIdx = &pRet->zTbl[nTbl]; + memcpy(pRet->zIdx, zIdx, nIdx); + }else{ + pRet->zIdx = 0; + } + pRet->iRow = sqlite3_column_int64(pStmt, 2); } - if( rc==SQLITE_OK && !p->tbliter.zTarget ){ + }else{ + pRet = (OtaState*)sqlite3_malloc(sizeof(OtaState)); + if( pRet ){ + memset(pRet, 0, sizeof(*pRet)); + } + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK && pRet==0 ) rc = SQLITE_NOMEM; + if( rc!=SQLITE_OK ){ + sqlite3_free(pRet); + pRet = 0; + } + } + + p->rc = rc; + return pRet; +} + +static void otaLoadTransactionState(sqlite3ota *p, OtaState *pState){ + assert( p->rc==SQLITE_OK ); + if( pState->zTbl ){ + int rc; + while( rc==SQLITE_OK + && p->tbliter.zTarget + && sqlite3_stricmp(p->tbliter.zTarget, pState->zTbl) + ){ + rc = tblIterNext(&p->tbliter); + } + if( rc==SQLITE_OK && !p->tbliter.zTarget ){ + rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); + } + + if( rc==SQLITE_OK && pState->zIdx ){ + rc = idxIterFirst(p->dbDest, p->tbliter.zTarget, &p->idxiter); + while( rc==SQLITE_OK + && p->idxiter.zIndex + && sqlite3_stricmp(p->idxiter.zIndex, pState->zIdx) + ){ + rc = idxIterNext(&p->idxiter); + } + if( rc==SQLITE_OK && !p->idxiter.zIndex ){ rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); } - - if( rc==SQLITE_OK && zIdx ){ - rc = idxIterFirst(p->dbDest, p->tbliter.zTarget, &p->idxiter); - while( rc==SQLITE_OK - && p->idxiter.zIndex - && sqlite3_stricmp(p->idxiter.zIndex, zIdx) - ){ - rc = idxIterNext(&p->idxiter); - } - if( rc==SQLITE_OK && !p->idxiter.zIndex ){ - rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); - } - } - - if( rc==SQLITE_OK ){ - rc = otaPrepareAll(p); - } - - if( rc==SQLITE_OK ){ - sqlite3_stmt *pSelect; - pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect); - while( sqlite3_column_int64(pSelect, 0)!=iRowid ){ - rc = sqlite3_step(pSelect); - if( rc!=SQLITE_ROW ) break; - } - if( rc==SQLITE_ROW ){ - rc = SQLITE_OK; - }else{ - rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); - } - } } + if( rc==SQLITE_OK ){ - rc = sqlite3_finalize(pStmt); - }else{ - sqlite3_finalize(pStmt); + rc = otaPrepareAll(p); } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSelect; + pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect); + while( sqlite3_column_int64(pSelect, 0)!=pState->iRow ){ + rc = sqlite3_step(pSelect); + if( rc!=SQLITE_ROW ) break; + } + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + }else{ + rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); + } + } + p->rc = rc; } - p->rc = rc; } +/* +** Move the "*-oal" file corresponding to the target database to the +** "*-wal" location. If an error occurs, leave an error code and error +** message in the ota handle. +*/ +static void otaMoveOalFile(sqlite3ota *p){ + char *zWal = sqlite3_mprintf("%s-wal", p->zTarget); + char *zOal = sqlite3_mprintf("%s-oal", p->zTarget); + + assert( p->rc==SQLITE_DONE && p->zErrmsg==0 ); + if( zWal==0 || zOal==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + rename(zOal, zWal); + } + + sqlite3_free(zWal); + sqlite3_free(zOal); +} + +/* +** If there is a "*-oal" file in the file-system corresponding to the +** target database in the file-system, delete it. If an error occurs, +** leave an error code and error message in the ota handle. +*/ +static void otaDeleteOalFile(sqlite3ota *p){ + char *zOal = sqlite3_mprintf("%s-oal", p->zTarget); + assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); + unlink(zOal); + sqlite3_free(zOal); +} /* ** Open and return a new OTA handle. */ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ sqlite3ota *p; + int nTarget = strlen(zTarget); - p = (sqlite3ota*)sqlite3_malloc(sizeof(sqlite3ota)); + p = (sqlite3ota*)sqlite3_malloc(sizeof(sqlite3ota)+nTarget+1); if( p ){ + OtaState *pState = 0; /* Open the target database */ memset(p, 0, sizeof(sqlite3ota)); + p->zTarget = (char*)&p[1]; + memcpy(p->zTarget, zTarget, nTarget+1); otaOpenDatabase(p, &p->dbDest, zTarget); otaOpenDatabase(p, &p->dbOta, zOta); @@ -642,9 +697,18 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ } if( p->rc==SQLITE_OK ){ - const char *zScript = + pState = otaLoadState(p); + if( pState && pState->zTbl==0 ){ + otaDeleteOalFile(p); + } + } + + + if( p->rc==SQLITE_OK ){ + const char *zScript = + "PRAGMA journal_mode=off;" + "PRAGMA pager_ota_mode=1;" "PRAGMA ota_mode=1;" - "PRAGMA journal_mode=wal;" "BEGIN IMMEDIATE;" ; p->rc = sqlite3_exec(p->dbDest, zScript, 0, 0, &p->zErrmsg); @@ -661,8 +725,10 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ } if( p->rc==SQLITE_OK ){ - otaLoadTransactionState(p); + otaLoadTransactionState(p, pState); } + + sqlite3_free(pState); } return p; @@ -690,17 +756,20 @@ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ tblIterFinalize(&p->tbliter); idxIterFinalize(&p->idxiter); - /* If the ota update has been fully applied, commit the transaction - ** on the target database. */ - if( p->rc==SQLITE_DONE ){ + /* Commit the transaction to the *-oal file. */ + if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ rc = sqlite3_exec(p->dbDest, "COMMIT", 0, 0, &p->zErrmsg); if( rc!=SQLITE_OK ) p->rc = rc; } + otaCloseHandle(p->dbDest); + otaCloseHandle(p->dbOta); + + if( p->rc==SQLITE_DONE ){ + otaMoveOalFile(p); + } rc = p->rc; *pzErrmsg = p->zErrmsg; - otaCloseHandle(p->dbDest); - otaCloseHandle(p->dbOta); sqlite3_free(p); }else{ rc = SQLITE_NOMEM; diff --git a/manifest b/manifest index ca224c9b1e..3329e4640b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scommand\sline\sprogram\sthat\suses\sthe\sextension.\sThis\sserves\sas\sexample\scode\sand\sis\salso\suseful\sfor\sperformance\stesting. -D 2014-09-03T08:25:09.548 +C Split\spart\sof\s"PRAGMA\sota_mode"\soff\sinto\s"PRAGMA\spager_ota_mode".\sThis\sallows\ssome\sspecialized\scustom\sVFS\simplementations\sto\sintercept\sand\simplement\sthe\sexpected\spager-related\seffects\sof\sthis\spragma. +D 2014-09-03T19:30:32.283 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -123,8 +123,8 @@ F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/ota.c 4b48add9494f29144343f513aaac226ca5782189 F ext/ota/ota1.test ea2865997ce573fadaf12eb0a0f80ef22d9dd77f -F ext/ota/ota2.test 4f7abfe1dfb7c3709bf45e94f3e65f3839b4f115 -F ext/ota/sqlite3ota.c ad55821883e4110367a30ffca282032d2bf36e45 +F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa +F ext/ota/sqlite3ota.c 3e05e3fa5791977eb88261731a6be6d98935efb3 F ext/ota/sqlite3ota.h 545f0008b5f02f2595899cb9841caddada5c17c0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -173,7 +173,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c ec9d3f1295dafeb278c3830211cc5584132468f4 -F src/btree.c c46043fbb09c18a19bdb96eadde6e724901d6fcf +F src/btree.c 9cb1989073502a9d2f18fbb0e7df8ad89dda2dcf F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c c26b233dcdb1e2c8f468d49236c266f9f3de96d8 @@ -216,13 +216,13 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 8525ca79457c5b4673a5fda2774ee39fe155f40f F src/os_win.c 2aa8aa7780d7cf03e912d2088ab2ec5c32f33dc5 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c a3caa08db8227c5a32f388be67f33d8cb44d5e35 -F src/pager.h 1acd367a0ffb63026b0461ea5eaeeb8046414a71 +F src/pager.c ed122b1346a40d6b53cec28fa63bf9af4a7dc8d7 +F src/pager.h 6a08df06b7edc3684375c0fab40602c695a044f2 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 3b3791297e8977002e56b4a9b8916f2039abad9b F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c -F src/pragma.c d252459fb3ce19448d1a2f41000c780fac4c0c26 +F src/pragma.c e1b8049c059ccab0afc2a483ff2e0dd599fcb124 F src/prepare.c 314961aa6650cc860394cb2f31931cf2de93baa8 F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -239,7 +239,7 @@ F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 29357f2be7b0d00e8ea900eaf727e0c5ffeaa660 F src/test1.c e9a0e5804b078532e69e69ec14c8326bf2cfc318 -F src/test2.c 84f6a786aa7ffa12fff83acb52660e337ffe642a +F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 @@ -1163,7 +1163,7 @@ F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh 5dc5010e2e748a9e1bba67baca5956a2c2deda7b F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl cce51d8f60c7f145d8fccabe6b5dfdedf31c5f5c +F tool/mkpragmatab.tcl 22c85e67987ad7d2e8789c48506ec95b99a90c08 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 88a1e3b0c769773fb7a9ebb363ffc603a4ac21d8 F tool/mksqlite3c.tcl e72c0c97fe1a105fa9616483e652949be2199fe6 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2954ab501049968430011b63d046eb42ff37a56c -R 7bee98aca78bdb17f817d7637df95f18 +P ffa1524ef2a4c32652183eb4745685f0d1c93af2 +R 58c81bcc907452bbe01138bf831636b2 U dan -Z 3a37163be54ffa74ee7c4647ff1214ea +Z 0213bc6d6444959e54104d228d32668b diff --git a/manifest.uuid b/manifest.uuid index 7d509edecc..988b76f2dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffa1524ef2a4c32652183eb4745685f0d1c93af2 \ No newline at end of file +209f672e588b54dfbfb83c7859cacdc4497f0f2b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e9d737b6a0..5557b67b78 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2046,7 +2046,7 @@ int sqlite3BtreeOpen( btree_open_out: if( rc!=SQLITE_OK ){ if( pBt && pBt->pPager ){ - sqlite3PagerClose(pBt->pPager, 0); + sqlite3PagerClose(pBt->pPager); } sqlite3_free(pBt); sqlite3_free(p); @@ -2175,7 +2175,7 @@ int sqlite3BtreeClose(Btree *p){ ** Clean out and delete the BtShared object. */ assert( !pBt->pCursor ); - sqlite3PagerClose(pBt->pPager, (p->db->flags & SQLITE_OtaMode)!=0); + sqlite3PagerClose(pBt->pPager); if( pBt->xFreeSchema && pBt->pSchema ){ pBt->xFreeSchema(pBt->pSchema); } diff --git a/src/pager.c b/src/pager.c index a54ae816e3..9fc88b6930 100644 --- a/src/pager.c +++ b/src/pager.c @@ -630,6 +630,7 @@ struct Pager { u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ + u8 otaMode; /* True if in ota_mode */ /************************************************************************** ** The following block contains those class members that change during @@ -824,6 +825,8 @@ static int pagerUseWal(Pager *pPager){ # define pagerBeginReadTransaction(z) SQLITE_OK #endif +static int pagerOpenWalInternal(Pager*, int*); + #ifndef NDEBUG /* ** Usage: @@ -2006,7 +2009,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; } - if( !pPager->exclusiveMode + if( !pPager->exclusiveMode && !pPager->otaMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ rc2 = pagerUnlockDb(pPager, SHARED_LOCK); @@ -3947,7 +3950,7 @@ static void pagerFreeMapHdrs(Pager *pPager){ ** a hot journal may be left in the filesystem but no error is returned ** to the caller. */ -int sqlite3PagerClose(Pager *pPager, int bOtaMode){ +int sqlite3PagerClose(Pager *pPager){ u8 *pTmp = (u8 *)pPager->pTmpSpace; assert( assert_pager_state(pPager) ); @@ -3957,8 +3960,8 @@ int sqlite3PagerClose(Pager *pPager, int bOtaMode){ /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL - sqlite3WalClose( - pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (bOtaMode?0:pTmp) + sqlite3WalClose(pPager->pWal, + pPager->ckptSyncFlags, pPager->pageSize, (pPager->otaMode?0:pTmp) ); pPager->pWal = 0; #endif @@ -5164,6 +5167,12 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** mode. Otherwise, the following function call is a no-op. */ rc = pagerOpenWalIfPresent(pPager); + if( rc==SQLITE_OK && pPager->otaMode ){ + int nWal = sqlite3Strlen30(pPager->zWal); + pPager->zWal[nWal-3] = 'o'; + rc = pagerOpenWalInternal(pPager, 0); + } + #ifndef SQLITE_OMIT_WAL assert( pPager->pWal==0 || rc==SQLITE_OK ); #endif @@ -7118,7 +7127,7 @@ static int pagerOpenWal(Pager *pPager){ */ if( rc==SQLITE_OK ){ rc = sqlite3WalOpen(pPager->pVfs, - pPager->fd, pPager->zWal, pPager->exclusiveMode, + pPager->fd, pPager->zWal, pPager->exclusiveMode || pPager->otaMode, pPager->journalSizeLimit, &pPager->pWal ); } @@ -7127,23 +7136,7 @@ static int pagerOpenWal(Pager *pPager){ return rc; } - -/* -** The caller must be holding a SHARED lock on the database file to call -** this function. -** -** If the pager passed as the first argument is open on a real database -** file (not a temp file or an in-memory database), and the WAL file -** is not already open, make an attempt to open it now. If successful, -** return SQLITE_OK. If an error occurs or the VFS used by the pager does -** not support the xShmXXX() methods, return an error code. *pbOpen is -** not modified in either case. -** -** If the pager is open on a temp-file (or in-memory database), or if -** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK -** without doing anything. -*/ -int sqlite3PagerOpenWal( +static int pagerOpenWalInternal( Pager *pPager, /* Pager object */ int *pbOpen /* OUT: Set to true if call is a no-op */ ){ @@ -7173,6 +7166,29 @@ int sqlite3PagerOpenWal( return rc; } +/* +** The caller must be holding a SHARED lock on the database file to call +** this function. +** +** If the pager passed as the first argument is open on a real database +** file (not a temp file or an in-memory database), and the WAL file +** is not already open, make an attempt to open it now. If successful, +** return SQLITE_OK. If an error occurs or the VFS used by the pager does +** not support the xShmXXX() methods, return an error code. *pbOpen is +** not modified in either case. +** +** If the pager is open on a temp-file (or in-memory database), or if +** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK +** without doing anything. +*/ +int sqlite3PagerOpenWal( + Pager *pPager, /* Pager object */ + int *pbOpen /* OUT: Set to true if call is a no-op */ +){ + if( pPager->otaMode ) return SQLITE_CANTOPEN; + return pagerOpenWalInternal(pPager, pbOpen); +} + /* ** This function is called to close the connection to the log file prior ** to switching from WAL to rollback mode. @@ -7270,4 +7286,15 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif +/* +** Set or clear the "OTA mode" flag. +*/ +int sqlite3PagerSetOtaMode(Pager *pPager, int bOta){ + if( pPager->pWal || pPager->eState!=PAGER_OPEN ){ + return SQLITE_ERROR; + } + pPager->otaMode = (u8)bOta; + return SQLITE_OK; +} + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index 79ffa04db8..fd2624c3fa 100644 --- a/src/pager.h +++ b/src/pager.h @@ -112,7 +112,7 @@ int sqlite3PagerOpen( int, void(*)(DbPage*) ); -int sqlite3PagerClose(Pager *pPager, int); +int sqlite3PagerClose(Pager *pPager); int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ @@ -210,4 +210,6 @@ void *sqlite3PagerCodec(DbPage *); int sqlite3PagerSaveState(Pager *pPager, void **ppState, int *pnState); int sqlite3PagerRestoreState(Pager *pPager, const void *pState, int nState); +int sqlite3PagerSetOtaMode(Pager *pPager, int bOta); + #endif /* _PAGER_H_ */ diff --git a/src/pragma.c b/src/pragma.c index 3f06a51839..e65593c5ff 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -53,23 +53,24 @@ #define PragTyp_PAGE_COUNT 22 #define PragTyp_MMAP_SIZE 23 #define PragTyp_PAGE_SIZE 24 -#define PragTyp_SECURE_DELETE 25 -#define PragTyp_SHRINK_MEMORY 26 -#define PragTyp_SOFT_HEAP_LIMIT 27 -#define PragTyp_STATS 28 -#define PragTyp_SYNCHRONOUS 29 -#define PragTyp_TABLE_INFO 30 -#define PragTyp_TEMP_STORE 31 -#define PragTyp_TEMP_STORE_DIRECTORY 32 -#define PragTyp_THREADS 33 -#define PragTyp_WAL_AUTOCHECKPOINT 34 -#define PragTyp_WAL_CHECKPOINT 35 -#define PragTyp_ACTIVATE_EXTENSIONS 36 -#define PragTyp_HEXKEY 37 -#define PragTyp_KEY 38 -#define PragTyp_REKEY 39 -#define PragTyp_LOCK_STATUS 40 -#define PragTyp_PARSER_TRACE 41 +#define PragTyp_PAGER_OTA_MODE 25 +#define PragTyp_SECURE_DELETE 26 +#define PragTyp_SHRINK_MEMORY 27 +#define PragTyp_SOFT_HEAP_LIMIT 28 +#define PragTyp_STATS 29 +#define PragTyp_SYNCHRONOUS 30 +#define PragTyp_TABLE_INFO 31 +#define PragTyp_TEMP_STORE 32 +#define PragTyp_TEMP_STORE_DIRECTORY 33 +#define PragTyp_THREADS 34 +#define PragTyp_WAL_AUTOCHECKPOINT 35 +#define PragTyp_WAL_CHECKPOINT 36 +#define PragTyp_ACTIVATE_EXTENSIONS 37 +#define PragTyp_HEXKEY 38 +#define PragTyp_KEY 39 +#define PragTyp_REKEY 40 +#define PragTyp_LOCK_STATUS 41 +#define PragTyp_PARSER_TRACE 42 #define PragFlag_NeedSchema 0x01 static const struct sPragmaNames { const char *const zName; /* Name of pragma */ @@ -323,6 +324,10 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif + { /* zName: */ "pager_ota_mode", + /* ePragTyp: */ PragTyp_PAGER_OTA_MODE, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, #if defined(SQLITE_DEBUG) { /* zName: */ "parser_trace", /* ePragTyp: */ PragTyp_PARSER_TRACE, @@ -476,7 +481,8 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 57 on by default, 70 total. */ +/* Number of pragmas: 59 on by default, 72 total. */ +/* Number of pragmas: 58 on by default, 71 total. */ /* End of the automatically generated pragma table. ***************************************************************************/ @@ -870,6 +876,19 @@ void sqlite3Pragma( } #endif /* !SQLITE_OMIT_PAGER_PRAGMAS && !SQLITE_OMIT_DEPRECATED */ + /* + ** PRAGMA [database.]pager_ota_mode=[01] + */ + case PragTyp_PAGER_OTA_MODE: { + Btree *pBt = pDb->pBt; + assert( pBt!=0 ); + if( zRight ){ + int iArg = !!sqlite3Atoi(zRight); + rc = sqlite3PagerSetOtaMode(sqlite3BtreePager(pBt), iArg); + } + break; + } + #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) /* ** PRAGMA [database.]page_size diff --git a/src/test2.c b/src/test2.c index 6d655d6473..58f271ff27 100644 --- a/src/test2.c +++ b/src/test2.c @@ -89,7 +89,7 @@ static int pager_close( return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); - rc = sqlite3PagerClose(pPager, 0); + rc = sqlite3PagerClose(pPager); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index aa7c8078c5..613b565553 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -296,6 +296,12 @@ set pragma_def { NAME: soft_heap_limit NAME: threads + + NAME: pager_ota_mode + + NAME: ota_mode + TYPE: FLAG + ARG: SQLITE_OtaMode } fconfigure stdout -translation lf set name {} From 8fa7f590cca74ee0f2e6ea933fa5a6f72a117dfd Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 4 Sep 2014 11:03:35 +0000 Subject: [PATCH 004/116] Avoid calling sqlite3OsFetch() on a file-handle for which the xFetch method is NULL. FossilOrigin-Name: 071f7f2decd2f786c0201a4219e9c2cc9d227085 --- ext/ota/ota.c | 7 +++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 6 ++++-- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/ext/ota/ota.c b/ext/ota/ota.c index c9d576f1aa..c00eed4f0b 100644 --- a/ext/ota/ota.c +++ b/ext/ota/ota.c @@ -42,6 +42,11 @@ void usage(const char *zArgv0){ exit(1); } +void report_default_vfs(){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + fprintf(stdout, "using vfs \"%s\"\n", pVfs->zName); +} + int main(int argc, char **argv){ int i; const char *zTarget; /* Target database to apply OTA to */ @@ -63,6 +68,8 @@ int main(int argc, char **argv){ zTarget = argv[argc-2]; zOta = argv[argc-1]; + report_default_vfs(); + /* Open an OTA handle. If nStep is less than or equal to zero, call ** sqlite3ota_step() until either the OTA has been completely applied ** or an error occurs. Or, if nStep is greater than zero, call diff --git a/manifest b/manifest index 3329e4640b..94a42db8e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Split\spart\sof\s"PRAGMA\sota_mode"\soff\sinto\s"PRAGMA\spager_ota_mode".\sThis\sallows\ssome\sspecialized\scustom\sVFS\simplementations\sto\sintercept\sand\simplement\sthe\sexpected\spager-related\seffects\sof\sthis\spragma. -D 2014-09-03T19:30:32.283 +C Avoid\scalling\ssqlite3OsFetch()\son\sa\sfile-handle\sfor\swhich\sthe\sxFetch\smethod\sis\sNULL. +D 2014-09-04T11:03:35.509 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/ota/ota.c 4b48add9494f29144343f513aaac226ca5782189 +F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 F ext/ota/ota1.test ea2865997ce573fadaf12eb0a0f80ef22d9dd77f F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa F ext/ota/sqlite3ota.c 3e05e3fa5791977eb88261731a6be6d98935efb3 @@ -297,7 +297,7 @@ F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 F src/vdbeblob.c 0bc9d22578d87ad9ff1c16e20a36863326f34fd7 F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62 -F src/vdbesort.c 02646a9f86421776ae5d7594f620f9ed669d3698 +F src/vdbesort.c 7c45bfcd823f30d172bbbc1b9f51ef4402fbfe8d F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 93b4fcb56a98f435a2cb66024bb2b12d66d1ff53 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ffa1524ef2a4c32652183eb4745685f0d1c93af2 -R 58c81bcc907452bbe01138bf831636b2 +P 209f672e588b54dfbfb83c7859cacdc4497f0f2b +R 20b7eba6ff5020d8a6df728a74ee9194 U dan -Z 0213bc6d6444959e54104d228d32668b +Z bb146242a51e5fe849d114706a1de049 diff --git a/manifest.uuid b/manifest.uuid index 988b76f2dd..f62eb5ee21 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -209f672e588b54dfbfb83c7859cacdc4497f0f2b \ No newline at end of file +071f7f2decd2f786c0201a4219e9c2cc9d227085 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 7318ea409e..50b82fd119 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -601,7 +601,9 @@ static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ */ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ int rc = SQLITE_OK; - if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ + if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) + && pFile->pFd->pMethods->xFetch + ){ rc = sqlite3OsFetch(pFile->pFd, 0, (int)pFile->iEof, (void**)pp); testcase( rc!=SQLITE_OK ); } @@ -1123,7 +1125,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ if( nByte<=(i64)(db->nMaxSorterMmap) ){ int rc = sqlite3OsTruncate(pFd, nByte); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pFd->pMethods->xFetch ){ void *p = 0; sqlite3OsFetch(pFd, 0, (int)nByte, &p); sqlite3OsUnfetch(pFd, 0, p); From 19f913f26827c67420dd8202964d71854d9fedff Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 4 Sep 2014 18:05:42 +0000 Subject: [PATCH 005/116] Fix showwal.c so that it works with 64KiB pages. FossilOrigin-Name: fc4f7c115207b786e3cac8cb6be4db3131b99a46 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/showwal.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 94a42db8e9..473977bc20 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\ssqlite3OsFetch()\son\sa\sfile-handle\sfor\swhich\sthe\sxFetch\smethod\sis\sNULL. -D 2014-09-04T11:03:35.509 +C Fix\sshowwal.c\sso\sthat\sit\sworks\swith\s64KiB\spages. +D 2014-09-04T18:05:42.326 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1179,7 +1179,7 @@ F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/showdb.c bd073a78bce714a0e42d92ea474b3eb8cb53be5d F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 F tool/showstat4.c c39279d6bd37cb999b634f0064f6f86ad7af008f -F tool/showwal.c 3209120269cdf9380f091459e47b776b4f81dfd3 +F tool/showwal.c 85cb36d4fe3e93e2fbd63e786e0d1ce42d0c4fad F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b F tool/spaceanal.tcl 8e50b217c56a6a086a1b47eac9d09c5cd65b996f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 209f672e588b54dfbfb83c7859cacdc4497f0f2b -R 20b7eba6ff5020d8a6df728a74ee9194 +P 071f7f2decd2f786c0201a4219e9c2cc9d227085 +R 3582d6e2777f5293f1910720c4563e98 U dan -Z bb146242a51e5fe849d114706a1de049 +Z fe920deec8beedfbfd168cc475660b1e diff --git a/manifest.uuid b/manifest.uuid index f62eb5ee21..0e5f1654b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -071f7f2decd2f786c0201a4219e9c2cc9d227085 \ No newline at end of file +fc4f7c115207b786e3cac8cb6be4db3131b99a46 \ No newline at end of file diff --git a/tool/showwal.c b/tool/showwal.c index 6dc1de173f..35810c66a9 100644 --- a/tool/showwal.c +++ b/tool/showwal.c @@ -510,7 +510,7 @@ static void decode_btree_page( int main(int argc, char **argv){ struct stat sbuf; - unsigned char zPgSz[2]; + unsigned char zPgSz[4]; if( argc<2 ){ fprintf(stderr,"Usage: %s FILENAME ?PAGE? ...\n", argv[0]); exit(1); @@ -522,9 +522,9 @@ int main(int argc, char **argv){ } zPgSz[0] = 0; zPgSz[1] = 0; - lseek(fd, 10, SEEK_SET); - read(fd, zPgSz, 2); - pagesize = zPgSz[0]*256 + zPgSz[1]; + lseek(fd, 8, SEEK_SET); + read(fd, zPgSz, 4); + pagesize = zPgSz[1]*65536 + zPgSz[2]*256 + zPgSz[3]; if( pagesize==0 ) pagesize = 1024; printf("Pagesize: %d\n", pagesize); fstat(fd, &sbuf); From 55b40e0ec59732ea61b6a4de022184a9804e1196 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 4 Sep 2014 19:05:31 +0000 Subject: [PATCH 006/116] Avoid ever running a checkpoint in ota mode. FossilOrigin-Name: 9ae44447256b425b5704a1cab3f6796befb92251 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 473977bc20..5c9d321e28 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sshowwal.c\sso\sthat\sit\sworks\swith\s64KiB\spages. -D 2014-09-04T18:05:42.326 +C Avoid\sever\srunning\sa\scheckpoint\sin\sota\smode. +D 2014-09-04T19:05:31.658 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 8525ca79457c5b4673a5fda2774ee39fe155f40f F src/os_win.c 2aa8aa7780d7cf03e912d2088ab2ec5c32f33dc5 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c ed122b1346a40d6b53cec28fa63bf9af4a7dc8d7 +F src/pager.c 9611de7c00ea3cfe35295d88ebda1a096b71b41d F src/pager.h 6a08df06b7edc3684375c0fab40602c695a044f2 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 3b3791297e8977002e56b4a9b8916f2039abad9b @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 071f7f2decd2f786c0201a4219e9c2cc9d227085 -R 3582d6e2777f5293f1910720c4563e98 +P fc4f7c115207b786e3cac8cb6be4db3131b99a46 +R 43c0251327b6a5ac610420013b39a3a5 U dan -Z fe920deec8beedfbfd168cc475660b1e +Z a71ca63f1bbe8e3a7e1c7cbaa054e55c diff --git a/manifest.uuid b/manifest.uuid index 0e5f1654b7..5250b4a025 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc4f7c115207b786e3cac8cb6be4db3131b99a46 \ No newline at end of file +9ae44447256b425b5704a1cab3f6796befb92251 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 9fc88b6930..0658bf7e2c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7060,7 +7060,7 @@ void sqlite3PagerClearCache(Pager *pPager){ */ int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; - if( pPager->pWal ){ + if( pPager->pWal && pPager->otaMode==0 ){ rc = sqlite3WalCheckpoint(pPager->pWal, eMode, pPager->xBusyHandler, pPager->pBusyHandlerArg, pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, From 0c4ba2662efca18411f39c30fff7746c892563ac Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 5 Sep 2014 19:31:15 +0000 Subject: [PATCH 007/116] Reorganize the code in sqlite3ota.c in preparation for adding support for update and delete operations. FossilOrigin-Name: 98387f05697526c7740e91d8a846a31f77639406 --- ext/ota/ota1.test | 2 +- ext/ota/sqlite3ota.c | 863 ++++++++++++++++++++++--------------------- manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 448 insertions(+), 433 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index aa334b9581..12afc6e3f1 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -63,7 +63,7 @@ proc step_ota {target ota} { set rc } -foreach {tn2 cmd} {1 step_ota 2 run_ota} { +foreach {tn2 cmd} {1 run_ota 2 step_ota} { foreach {tn schema} { 1 { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 2f61c671ac..b5f6e2c436 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -30,7 +30,7 @@ ** "idx" -> Index currently being written (target database names). ** Or, if the main table is being written, a NULL value. ** -** "row" -> Last rowid processed from ota database table (i.e. data_%). +** "row" -> Number of rows for this object already processed ** ** "progress" -> total number of key/value b-tree operations performed ** so far as part of this ota update. @@ -38,63 +38,50 @@ #define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota_state" \ "(tbl, idx, row, progress)" -typedef struct OtaTblIter OtaTblIter; -typedef struct OtaIdxIter OtaIdxIter; typedef struct OtaState OtaState; +typedef struct OtaObjIter OtaObjIter; +typedef unsigned char u8; /* -** Iterator used to iterate through all data tables in the OTA. As follows: -** -** OtaTblIter iter; -** for(rc=tblIterFirst(db, &iter); -** rc==SQLITE_OK && iter.zTarget; -** rc=tblIterNext(&iter) -** ){ -** } +** A structure to store values read from the ota_state table in memory. */ -struct OtaTblIter { - sqlite3_stmt *pTblIter; /* Iterate through tables */ - int iEntry; /* Index of current entry (from 1) */ - - /* Output varibles. zTarget==0 implies EOF. */ - const char *zTarget; /* Name of target table */ - const char *zSource; /* Name of source table */ - - /* Useful things populated by a call to tblIterPrepareAll() */ - int nCol; /* Number of columns in this table */ - char **azCol; /* Array of quoted column names */ - sqlite3_stmt *pSelect; /* PK b-tree SELECT statement */ - sqlite3_stmt *pInsert; /* PK b-tree INSERT statement */ -}; - -/* -** API is: -** -** idxIterFirst() -** idxIterNext() -** idxIterFinalize() -** idxIterPrepareAll() -*/ -struct OtaIdxIter { - sqlite3_stmt *pIdxIter; /* Iterate through indexes */ - int iEntry; /* Index of current entry (from 1) */ - - /* Output varibles. zTarget==0 implies EOF. */ - const char *zIndex; /* Name of index */ - - int nCol; /* Number of columns in index */ - int *aiCol; /* Array of column indexes */ - sqlite3_stmt *pWriter; /* Index writer */ - sqlite3_stmt *pSelect; /* Select to read values in index order */ -}; - struct OtaState { char *zTbl; char *zIdx; - sqlite3_int64 iRow; + int nRow; }; +/* +** An iterator of this type is used to iterate through all objects in +** the target database that require updating. For each such table, the +** iterator visits, in order: +** +** * the table itself, +** * each index of the table (zero or more points to visit), and +** * a special "cleanup table" point. +*/ +struct OtaObjIter { + sqlite3_stmt *pTblIter; /* Iterate through tables */ + sqlite3_stmt *pIdxIter; /* Index iterator */ + int nTblCol; /* Size of azTblCol[] array */ + char **azTblCol; /* Array of quoted column names */ + u8 *abTblPk; /* Array of flags - true for PK columns */ + /* Output variables. zTbl==0 implies EOF. */ + int bCleanup; /* True in "cleanup" state */ + const char *zTbl; /* Name of target db table */ + const char *zIdx; /* Name of target db index (or null) */ + int iVisit; /* Number of points visited, incl. current */ + + /* Statements created by otaObjIterPrepareAll() */ + int nCol; /* Number of columns in current object */ + sqlite3_stmt *pSelect; /* Source data */ + sqlite3_stmt *pInsert; /* Statement for INSERT operations */ +}; + +/* +** OTA handle. +*/ struct sqlite3ota { sqlite3 *dbDest; /* Target db */ sqlite3 *dbOta; /* Ota db */ @@ -103,15 +90,25 @@ struct sqlite3ota { int rc; /* Value returned by last ota_step() call */ char *zErrmsg; /* Error message if rc!=SQLITE_OK */ - OtaTblIter tbliter; /* Used to iterate through tables */ - OtaIdxIter idxiter; /* Used to iterate through indexes */ + int nStep; /* Rows processed for current object */ + OtaObjIter objiter; }; +/* +** Prepare the SQL statement in buffer zSql against database handle db. +** If successful, set *ppStmt to point to the new statement and return +** SQLITE_OK. +** +** Otherwise, if an error does occur, set *ppStmt to NULL and return +** an SQLite error code. Additionally, set output variable *pzErrmsg to +** point to a buffer containing an error message. It is the responsibility +** of the caller to (eventually) free this buffer using sqlite3_free(). +*/ static int prepareAndCollectError( sqlite3 *db, - const char *zSql, sqlite3_stmt **ppStmt, - char **pzErrmsg + char **pzErrmsg, + const char *zSql ){ int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); if( rc!=SQLITE_OK ){ @@ -121,6 +118,22 @@ static int prepareAndCollectError( return rc; } +/* +** Reset the SQL statement passed as the first argument. Return a copy +** of the value returned by sqlite3_reset(). +** +** If an error has occurred, then set *pzErrmsg to point to a buffer +** containing an error message. It is the responsibility of the caller +** to eventually free this buffer using sqlite3_free(). +*/ +static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); + } + return rc; +} + /* ** Unless it is NULL, argument zSql points to a buffer allocated using ** sqlite3_malloc containing an SQL statement. This function prepares the SQL @@ -138,9 +151,9 @@ static int prepareAndCollectError( */ static int prepareFreeAndCollectError( sqlite3 *db, - char *zSql, sqlite3_stmt **ppStmt, - char **pzErrmsg + char **pzErrmsg, + char *zSql ){ int rc; assert( *pzErrmsg==0 ); @@ -148,13 +161,137 @@ static int prepareFreeAndCollectError( rc = SQLITE_NOMEM; *ppStmt = 0; }else{ - rc = prepareAndCollectError(db, zSql, ppStmt, pzErrmsg); + rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); sqlite3_free(zSql); } return rc; } -static char *quoteSqlName(const char *zName){ +/* +** Free the OtaObjIter.azTblCol[] and OtaObjIter.abTblPk[] arrays allocated +** by an earlier call to otaObjIterGetCols(). +*/ +static void otaObjIterFreeCols(OtaObjIter *pIter){ + int i; + for(i=0; inTblCol; i++){ + sqlite3_free(pIter->azTblCol[i]); + } + sqlite3_free(pIter->azTblCol); + sqlite3_free(pIter->abTblPk); + pIter->azTblCol = 0; + pIter->abTblPk = 0; + pIter->nTblCol = 0; +} + +/* +** Clean up any resources allocated as part of the iterator object passed +** as the only argument. +*/ +static void otaObjIterFinalize(OtaObjIter *pIter){ + sqlite3_finalize(pIter->pTblIter); + sqlite3_finalize(pIter->pIdxIter); + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + otaObjIterFreeCols(pIter); + memset(pIter, 0, sizeof(OtaObjIter)); +} + +/* +** Advance the iterator to the next position. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the next entry. Otherwise, an error code and message is +** left in the OTA handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int otaObjIterNext(sqlite3ota *p, OtaObjIter *pIter){ + int rc = p->rc; + if( rc==SQLITE_OK ){ + + /* Free any SQLite statements used while processing the previous object */ + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + pIter->pSelect = 0; + pIter->pInsert = 0; + pIter->nCol = 0; + + if( pIter->bCleanup ){ + otaObjIterFreeCols(pIter); + pIter->bCleanup = 0; + rc = sqlite3_step(pIter->pTblIter); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_reset(pIter->pTblIter); + pIter->zTbl = 0; + }else{ + pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); + rc = SQLITE_OK; + } + }else{ + if( pIter->zIdx==0 ){ + sqlite3_bind_text(pIter->pIdxIter, 1, pIter->zTbl, -1, SQLITE_STATIC); + } + rc = sqlite3_step(pIter->pIdxIter); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_reset(pIter->pIdxIter); + pIter->bCleanup = 1; + pIter->zIdx = 0; + }else{ + pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + rc = SQLITE_OK; + } + } + } + + if( rc!=SQLITE_OK ){ + otaObjIterFinalize(pIter); + p->rc = rc; + } + pIter->iVisit++; + return rc; +} + +/* +** Initialize the iterator structure passed as the second argument. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the first entry. Otherwise, an error code and message is +** left in the OTA handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int otaObjIterFirst(sqlite3ota *p, OtaObjIter *pIter){ + int rc; + memset(pIter, 0, sizeof(OtaObjIter)); + + rc = prepareAndCollectError(p->dbOta, &pIter->pTblIter, &p->zErrmsg, + "SELECT substr(name, 6) FROM sqlite_master " + "WHERE type='table' AND name LIKE 'data_%'" + ); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbDest, &pIter->pIdxIter, &p->zErrmsg, + "SELECT name FROM sqlite_master " + "WHERE type='index' AND tbl_name = ?" + ); + } + + pIter->bCleanup = 1; + p->rc = rc; + return otaObjIterNext(p, pIter); +} + +/* +** Allocate a buffer and populate it with the double-quoted version of the +** string in the argument buffer, suitable for use as an SQL identifier. +** For example: +** +** [quick "brown" fox] -> ["quick ""brown"" fox"] +** +** Assuming the allocation is successful, a pointer to the new buffer is +** returned. It is the responsibility of the caller to free it using +** sqlite3_free() at some point in the future. Or, if the allocation fails, +** a NULL pointer is returned. +*/ +static char *otaQuoteName(const char *zName){ int nName = strlen(zName); char *zRet = sqlite3_malloc(nName * 2 + 2 + 1); if( zRet ){ @@ -171,336 +308,230 @@ static char *quoteSqlName(const char *zName){ return zRet; } -static int tblIterPrepareAll(sqlite3ota *p){ - OtaTblIter *pIter = &p->tbliter; - int rc = SQLITE_OK; - char *zCol = 0; - char *zBindings = 0; - char *zSql; - sqlite3_stmt *pPragma = 0; - int i; - int bSeenPk = 0; /* Set to true once PK column seen */ - - /* Allocate and populate the azCol[] array */ - zSql = sqlite3_mprintf("PRAGMA main.table_info(%Q)", pIter->zTarget); - rc = prepareFreeAndCollectError(p->dbDest, zSql, &pPragma, &p->zErrmsg); - pIter->nCol = 0; - if( rc==SQLITE_OK ){ - while( SQLITE_ROW==sqlite3_step(pPragma) ){ - const char *zName = (const char*)sqlite3_column_text(pPragma, 1); - if( (pIter->nCol % 4)==0 ){ - int nByte = sizeof(char*) * (pIter->nCol+4); - char **azNew = (char**)sqlite3_realloc(pIter->azCol, nByte); - if( azNew==0 ){ - rc = SQLITE_NOMEM; - break; - } - pIter->azCol = azNew; - } - pIter->azCol[pIter->nCol] = quoteSqlName(zName); - if( pIter->azCol[pIter->nCol]==0 ){ - rc = SQLITE_NOMEM; - break; - } - pIter->nCol++; - if( sqlite3_column_int(pPragma, 5) ) bSeenPk = 1; - } - if( rc==SQLITE_OK ){ - rc = sqlite3_finalize(pPragma); - }else{ - sqlite3_finalize(pPragma); - } - } - - /* If the table has no PRIMARY KEY, throw an exception. */ - if( bSeenPk==0 ){ - p->zErrmsg = sqlite3_mprintf("table %s has no PRIMARY KEY", pIter->zTarget); - rc = SQLITE_ERROR; - } - - /* Populate the zCol variable */ - for(i=0; rc==SQLITE_OK && inCol; i++){ - zCol = sqlite3_mprintf("%z%s%s", zCol, (i==0?"":", "), pIter->azCol[i]); - if( zCol==0 ){ - rc = SQLITE_NOMEM; - } - } - - /* Allocate and populate zBindings */ - if( rc==SQLITE_OK ){ - zBindings = (char*)sqlite3_malloc(pIter->nCol * 2); - if( zBindings==0 ){ - rc = SQLITE_NOMEM; - }else{ - int i; - for(i=0; inCol; i++){ - zBindings[i*2] = '?'; - zBindings[i*2+1] = ','; - } - zBindings[pIter->nCol*2-1] = '\0'; - } - } - - /* Create OtaTblIter.pSelect */ - if( rc==SQLITE_OK ){ - zSql = sqlite3_mprintf("SELECT rowid, %s FROM %Q", zCol, pIter->zSource); - rc = prepareFreeAndCollectError(p->dbOta,zSql,&pIter->pSelect, &p->zErrmsg); - } - - /* Create OtaTblIter.pInsert */ - if( rc==SQLITE_OK ){ - zSql = sqlite3_mprintf("INSERT INTO %Q(%s) VALUES(%s)", - pIter->zTarget, zCol, zBindings - ); - rc = prepareFreeAndCollectError(p->dbDest,zSql,&pIter->pInsert,&p->zErrmsg); - } - - sqlite3_free(zCol); - sqlite3_free(zBindings); - return rc; -} - -static void tblIterFreeAll(OtaTblIter *pIter){ - int i; - - sqlite3_finalize(pIter->pSelect); - sqlite3_finalize(pIter->pInsert); - for(i=0; inCol; i++) sqlite3_free(pIter->azCol[i]); - sqlite3_free(pIter->azCol); - pIter->azCol = 0; - pIter->pSelect = 0; - pIter->pInsert = 0; - pIter->nCol = 0; -} - -static int tblIterNext(OtaTblIter *pIter){ - int rc; - - tblIterFreeAll(pIter); - assert( pIter->pTblIter ); - rc = sqlite3_step(pIter->pTblIter); - if( rc==SQLITE_ROW ){ - pIter->zSource = (const char*)sqlite3_column_text(pIter->pTblIter, 0); - pIter->zTarget = &pIter->zSource[5]; assert( 5==strlen("data_") ); - pIter->iEntry++; - }else{ - pIter->zSource = 0; - pIter->zTarget = 0; - } - - if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK; - return rc; -} - -static int tblIterFirst(sqlite3 *db, OtaTblIter *pIter){ - int rc; /* return code */ - memset(pIter, 0, sizeof(OtaTblIter)); - rc = sqlite3_prepare_v2(db, - "SELECT name FROM sqlite_master " - "WHERE type='table' AND name LIKE 'data_%'", -1, &pIter->pTblIter, 0 - ); - if( rc==SQLITE_OK ){ - rc = tblIterNext(pIter); - } - return rc; -} - - -static void tblIterFinalize(OtaTblIter *pIter){ - tblIterFreeAll(pIter); - sqlite3_finalize(pIter->pTblIter); - memset(pIter, 0, sizeof(OtaTblIter)); -} - -static void idxIterFreeAll(OtaIdxIter *pIter){ - sqlite3_finalize(pIter->pWriter); - sqlite3_finalize(pIter->pSelect); - pIter->pWriter = 0; - pIter->pSelect = 0; - pIter->aiCol = 0; - pIter->nCol = 0; -} - -static int idxIterPrepareAll(sqlite3ota *p){ - int rc; - int i; /* Iterator variable */ - char *zSql = 0; - char *zCols = 0; /* Columns list */ - OtaIdxIter *pIter = &p->idxiter; - - /* Prepare the writer statement to write (insert) entries into the index. */ - rc = sqlite3_index_writer( - p->dbDest, 0, pIter->zIndex, &pIter->pWriter, &pIter->aiCol, &pIter->nCol - ); - - /* Prepare a SELECT statement to read values from the source table in - ** the same order as they are stored in the current index. The statement - ** is: - ** - ** SELECT rowid, FROM data_ ORDER BY - */ - for(i=0; rc==SQLITE_OK && inCol; i++){ - const char *zQuoted = p->tbliter.azCol[ pIter->aiCol[i] ]; - zCols = sqlite3_mprintf("%z%s%s", zCols, zCols?", ":"", zQuoted); - if( !zCols ){ - rc = SQLITE_NOMEM; - } - } - if( rc==SQLITE_OK ){ - const char *zFmt = "SELECT rowid, %s FROM %Q ORDER BY %s"; - zSql = sqlite3_mprintf(zFmt, zCols, p->tbliter.zSource, zCols); - if( zSql ){ - sqlite3_stmt **pp = &p->idxiter.pSelect; - rc = prepareFreeAndCollectError(p->dbOta, zSql, pp, &p->zErrmsg); - }else{ - rc = SQLITE_NOMEM; - } - } - - sqlite3_free(zCols); - return rc; -} - -static int idxIterNext(OtaIdxIter *pIter){ - int rc; - - idxIterFreeAll(pIter); - assert( pIter->pIdxIter ); - rc = sqlite3_step(pIter->pIdxIter); - if( rc==SQLITE_ROW ){ - pIter->zIndex = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); - pIter->iEntry++; - }else{ - pIter->zIndex = 0; - rc = sqlite3_finalize(pIter->pIdxIter); - pIter->pIdxIter = 0; - } - - if( rc==SQLITE_ROW ) rc = SQLITE_OK; - return rc; -} - -static int idxIterFirst(sqlite3 *db, const char *zTable, OtaIdxIter *pIter){ - int rc; /* return code */ - memset(pIter, 0, sizeof(OtaIdxIter)); - rc = sqlite3_prepare_v2(db, - "SELECT name FROM sqlite_master " - "WHERE type='index' AND tbl_name = ?", -1, &pIter->pIdxIter, 0 - ); - if( rc==SQLITE_OK ){ - rc = sqlite3_bind_text(pIter->pIdxIter, 1, zTable, -1, SQLITE_TRANSIENT); - } - if( rc==SQLITE_OK ){ - rc = idxIterNext(pIter); - } - return rc; -} - -static void idxIterFinalize(OtaIdxIter *pIter){ - idxIterFreeAll(pIter); - sqlite3_finalize(pIter->pIdxIter); - memset(pIter, 0, sizeof(OtaIdxIter)); -} - /* -** Call sqlite3_reset() on the SQL statement passed as the second argument. -** If it returns anything other than SQLITE_OK, store the error code and -** error message in the OTA handle. +** If they are not already populated, populate the pIter->azTblCol[], +** pIter->abTblPk[] and pIter->nTblCol variables according to the table +** that the iterator currently points to. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. If +** an error does occur, an error code and error message are also left in +** the OTA handle. */ -static void otaResetStatement(sqlite3ota *p, sqlite3_stmt *pStmt){ - assert( p->rc==SQLITE_OK ); - assert( p->zErrmsg==0 ); - p->rc = sqlite3_reset(pStmt); - if( p->rc!=SQLITE_OK ){ - sqlite3 *db = sqlite3_db_handle(pStmt); - p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - } -} +static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ + if( pIter->azTblCol==0 ){ + sqlite3_stmt *pStmt; + char *zSql; + int nCol = 0; + int bSeenPk = 0; + int rc2; /* sqlite3_finalize() return value */ -/* -** Check that all SQL statements required to process the current -** table and index have been prepared. If not, prepare them. If -** an error occurs, store the error code and message in the OTA -** handle before returning. -*/ -static int otaPrepareAll(sqlite3ota *p){ - assert( p->rc==SQLITE_OK ); - assert( p->zErrmsg==0 ); - assert( p->tbliter.zTarget ); + zSql = sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl); + p->rc = prepareFreeAndCollectError(p->dbDest, &pStmt, &p->zErrmsg, zSql); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + if( (nCol % 8)==0 ){ + int nByte = sizeof(char*) * (nCol+8); + char **azNew = (char**)sqlite3_realloc(pIter->azTblCol, nByte); + u8 *abNew = (u8*)sqlite3_realloc(pIter->azTblCol, nCol+8); - if( p->tbliter.pSelect==0 ){ - p->rc = tblIterPrepareAll(p); - } - if( p->rc==SQLITE_OK && p->idxiter.zIndex && 0==p->idxiter.pSelect ){ - p->rc = idxIterPrepareAll(p); + if( azNew ) pIter->azTblCol = azNew; + if( abNew ) pIter->abTblPk = abNew; + if( azNew==0 || abNew==0 ) p->rc = SQLITE_NOMEM; + } + + if( p->rc==SQLITE_OK ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + pIter->abTblPk[nCol] = sqlite3_column_int(pStmt, 5); + if( pIter->abTblPk[nCol] ) bSeenPk = 1; + pIter->azTblCol[nCol] = otaQuoteName(zName); + if( pIter->azTblCol[nCol]==0 ) p->rc = SQLITE_NOMEM; + nCol++; + } + } + pIter->nTblCol = nCol; + rc2 = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + if( p->rc==SQLITE_OK && bSeenPk==0 ){ + p->zErrmsg = sqlite3_mprintf("table %s has no PRIMARY KEY", pIter->zTbl); + p->rc = SQLITE_ERROR; + } } + return p->rc; } -int sqlite3ota_step(sqlite3ota *p){ - if( p ){ - while( p && p->rc==SQLITE_OK && p->tbliter.zTarget ){ - sqlite3_stmt *pSelect; +static char *otaObjIterGetCollist( + sqlite3ota *p, + OtaObjIter *pIter, + int nCol, + int *aiCol +){ + char *zList = 0; + if( p->rc==SQLITE_OK ){ + const char *zSep = ""; + int i; + for(i=0; iazTblCol[iCol]); + zSep = ", "; + if( zList==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + } + } + return zList; +} + +static char *otaObjIterGetBindlist(sqlite3ota *p, int nBind){ + char *zRet = 0; + if( p->rc==SQLITE_OK ){ + int nByte = nBind*2 + 1; + zRet = sqlite3_malloc(nByte); + if( zRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ int i; - - otaPrepareAll(p); - pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect); - - /* Advance to the next input row. */ - if( p->rc==SQLITE_OK ){ - int rc = sqlite3_step(pSelect); - if( rc!=SQLITE_ROW ){ - otaResetStatement(p, pSelect); - - /* Go to the next index. */ - if( p->rc==SQLITE_OK ){ - if( p->idxiter.zIndex ){ - p->rc = idxIterNext(&p->idxiter); - }else{ - p->rc = idxIterFirst(p->dbDest, p->tbliter.zTarget, &p->idxiter); - } - } - - /* If there is no next index, go to the next table. */ - if( p->rc==SQLITE_OK && p->idxiter.zIndex==0 ){ - p->rc = tblIterNext(&p->tbliter); - } - continue; - } + for(i=0; irc==SQLITE_OK ){ - sqlite3_stmt *pInsert; - int nCol; - if( p->idxiter.zIndex ){ - pInsert = p->idxiter.pWriter; - nCol = p->idxiter.nCol; - }else{ - pInsert = p->tbliter.pInsert; - nCol = p->tbliter.nCol; - } +/* +** Ensure that the SQLite statement handles required to update the +** target database object currently indicated by the iterator passed +** as the second argument are available. +*/ +static int otaObjIterPrepareAll( + sqlite3ota *p, + OtaObjIter *pIter, + int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ +){ + assert( pIter->bCleanup==0 ); + if( pIter->pSelect==0 && otaObjIterGetCols(p, pIter)==SQLITE_OK ){ + char *zCollist = 0; /* List of indexed columns */ + char **pz = &p->zErrmsg; + const char *zIdx = pIter->zIdx; + char *zLimit = 0; - for(i=0; irc = SQLITE_NOMEM; } - if( p->rc==SQLITE_OK && p->tbliter.zTarget==0 ) p->rc = SQLITE_DONE; + if( zIdx ){ + int *aiCol; /* Column map */ + + /* Create the index writer */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_index_writer( + p->dbDest, 0, zIdx, &pIter->pInsert, &aiCol, &pIter->nCol + ); + } + + /* Create the SELECT statement to read keys in sorted order */ + zCollist = otaObjIterGetCollist(p, pIter, pIter->nCol, aiCol); + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbOta, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s FROM 'data_%q' ORDER BY %s%s", + zCollist, pIter->zTbl, zCollist, zLimit + ) + ); + } + }else{ + char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol); + zCollist = otaObjIterGetCollist(p, pIter, pIter->nTblCol, 0); + pIter->nCol = pIter->nTblCol; + + /* Create the SELECT statement to read keys from data_xxx */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbOta, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s FROM 'data_%q'%s", + zCollist, pIter->zTbl, zLimit) + ); + } + + /* Create the INSERT statement to write to the target PK b-tree */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbDest, &pIter->pInsert, pz, + sqlite3_mprintf( + "INSERT INTO %Q(%s) VALUES(%s)", pIter->zTbl, zCollist, zBindings + ) + ); + } + sqlite3_free(zBindings); + } + sqlite3_free(zCollist); + sqlite3_free(zLimit); + } + + return p->rc; +} + +/* +** This function does the work for an sqlite3ota_step() call. +** +** The object-iterator (p->objiter) currently points to a valid object, +** and the input cursor (p->objiter.pSelect) currently points to a valid +** input row. Perform whatever processing is required and return. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and message is left in the OTA handle and a copy of the error code +** returned. +*/ +static int otaStep(sqlite3ota *p){ + OtaObjIter *pIter = &p->objiter; + int i; + + for(i=0; inCol; i++){ + sqlite3_value *pVal = sqlite3_column_value(pIter->pSelect, i); + sqlite3_bind_value(pIter->pInsert, i+1, pVal); } - return (p ? p->rc : SQLITE_NOMEM); + sqlite3_step(pIter->pInsert); + p->rc = resetAndCollectError(pIter->pInsert, &p->zErrmsg); + return p->rc; +} + +/* +** Step the OTA object. +*/ +int sqlite3ota_step(sqlite3ota *p){ + if( p ){ + OtaObjIter *pIter = &p->objiter; + while( p && p->rc==SQLITE_OK && pIter->zTbl ){ + + if( pIter->bCleanup ){ + /* this is where cleanup of the ota_xxx table will happen... */ + }else{ + otaObjIterPrepareAll(p, pIter, 0); + + /* Advance to the next row to process. */ + if( p->rc==SQLITE_OK ){ + int rc = sqlite3_step(pIter->pSelect); + if( rc==SQLITE_ROW ){ + p->nStep++; + return otaStep(p); + } + p->rc = sqlite3_reset(pIter->pSelect); + p->nStep = 0; + } + } + + otaObjIterNext(p, pIter); + } + + if( p->rc==SQLITE_OK && pIter->zTbl==0 ){ + p->rc = SQLITE_DONE; + } + } + return p->rc; } static void otaOpenDatabase(sqlite3ota *p, sqlite3 **pDb, const char *zFile){ @@ -514,15 +545,14 @@ static void otaOpenDatabase(sqlite3ota *p, sqlite3 **pDb, const char *zFile){ } static void otaSaveTransactionState(sqlite3ota *p){ - sqlite3_stmt *pSelect; char *zInsert; - pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect); zInsert = sqlite3_mprintf( "INSERT OR REPLACE INTO ota_state(rowid, tbl, idx, row, progress)" - "VALUES(1, %Q, %Q, %lld, NULL)", - p->tbliter.zTarget, p->idxiter.zIndex, sqlite3_column_int64(pSelect, 0) + "VALUES(1, %Q, %Q, %d, NULL)", + p->objiter.zTbl, p->objiter.zIdx, p->nStep ); + if( zInsert==0 ){ p->rc = SQLITE_NOMEM; }else{ @@ -551,7 +581,7 @@ static OtaState *otaLoadState(sqlite3ota *p){ int rc; assert( p->rc==SQLITE_OK ); - rc = prepareAndCollectError(p->dbOta, zSelect, &pStmt, &p->zErrmsg); + rc = prepareAndCollectError(p->dbOta, &pStmt, &p->zErrmsg, zSelect); if( rc==SQLITE_OK ){ if( sqlite3_step(pStmt)==SQLITE_ROW ){ const char *zIdx = (const char*)sqlite3_column_text(pStmt, 1); @@ -570,7 +600,7 @@ static OtaState *otaLoadState(sqlite3ota *p){ }else{ pRet->zIdx = 0; } - pRet->iRow = sqlite3_column_int64(pStmt, 2); + pRet->nRow = sqlite3_column_int(pStmt, 2); } }else{ pRet = (OtaState*)sqlite3_malloc(sizeof(OtaState)); @@ -590,53 +620,35 @@ static OtaState *otaLoadState(sqlite3ota *p){ return pRet; } +static int otaStrCompare(const char *z1, const char *z2){ + if( z1==0 && z2==0 ) return 0; + if( z1==0 || z2==0 ) return 1; + return (sqlite3_stricmp(z1, z2)!=0); +} + static void otaLoadTransactionState(sqlite3ota *p, OtaState *pState){ assert( p->rc==SQLITE_OK ); if( pState->zTbl ){ + OtaObjIter *pIter = &p->objiter; int rc; - while( rc==SQLITE_OK - && p->tbliter.zTarget - && sqlite3_stricmp(p->tbliter.zTarget, pState->zTbl) - ){ - rc = tblIterNext(&p->tbliter); + + while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup + || otaStrCompare(pIter->zTbl, pState->zTbl) + || otaStrCompare(pIter->zIdx, pState->zIdx) + )){ + rc = otaObjIterNext(p, &p->objiter); } - if( rc==SQLITE_OK && !p->tbliter.zTarget ){ + + if( rc==SQLITE_OK && !p->objiter.zTbl ){ rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); } - if( rc==SQLITE_OK && pState->zIdx ){ - rc = idxIterFirst(p->dbDest, p->tbliter.zTarget, &p->idxiter); - while( rc==SQLITE_OK - && p->idxiter.zIndex - && sqlite3_stricmp(p->idxiter.zIndex, pState->zIdx) - ){ - rc = idxIterNext(&p->idxiter); - } - if( rc==SQLITE_OK && !p->idxiter.zIndex ){ - rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); - } + if( rc==SQLITE_OK ){ + p->nStep = pState->nRow; + rc = otaObjIterPrepareAll(p, &p->objiter, p->nStep); } - if( rc==SQLITE_OK ){ - rc = otaPrepareAll(p); - } - - if( rc==SQLITE_OK ){ - sqlite3_stmt *pSelect; - pSelect = (p->idxiter.zIndex ? p->idxiter.pSelect : p->tbliter.pSelect); - while( sqlite3_column_int64(pSelect, 0)!=pState->iRow ){ - rc = sqlite3_step(pSelect); - if( rc!=SQLITE_ROW ) break; - } - if( rc==SQLITE_ROW ){ - rc = SQLITE_OK; - }else{ - rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); - } - } p->rc = rc; } } @@ -703,7 +715,6 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ } } - if( p->rc==SQLITE_OK ){ const char *zScript = "PRAGMA journal_mode=off;" @@ -719,9 +730,9 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ p->rc = sqlite3_exec(p->dbOta, zScript, 0, 0, &p->zErrmsg); } - /* Point the table iterator at the first table */ + /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ - p->rc = tblIterFirst(p->dbOta, &p->tbliter); + p->rc = otaObjIterFirst(p, &p->objiter); } if( p->rc==SQLITE_OK ){ @@ -734,11 +745,9 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ return p; } -static void otaCloseHandle(sqlite3 *db){ - int rc = sqlite3_close(db); - assert( rc==SQLITE_OK ); -} - +/* +** Close the OTA handle. +*/ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ int rc; if( p ){ @@ -753,17 +762,23 @@ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ } /* Close all open statement handles. */ - tblIterFinalize(&p->tbliter); - idxIterFinalize(&p->idxiter); + otaObjIterFinalize(&p->objiter); /* Commit the transaction to the *-oal file. */ if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ rc = sqlite3_exec(p->dbDest, "COMMIT", 0, 0, &p->zErrmsg); if( rc!=SQLITE_OK ) p->rc = rc; } - otaCloseHandle(p->dbDest); - otaCloseHandle(p->dbOta); + assert( sqlite3_next_stmt(p->dbDest, 0)==0 ); + assert( sqlite3_next_stmt(p->dbOta, 0)==0 ); + + /* Close the open database handles */ + sqlite3_close(p->dbDest); + sqlite3_close(p->dbOta); + + /* If the OTA has been completely applied and no error occurred, move + ** the *-oal file to *-wal. */ if( p->rc==SQLITE_DONE ){ otaMoveOalFile(p); } diff --git a/manifest b/manifest index 5c9d321e28..6cfb5e1441 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sever\srunning\sa\scheckpoint\sin\sota\smode. -D 2014-09-04T19:05:31.658 +C Reorganize\sthe\scode\sin\ssqlite3ota.c\sin\spreparation\sfor\sadding\ssupport\sfor\supdate\sand\sdelete\soperations. +D 2014-09-05T19:31:15.782 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,9 +122,9 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 -F ext/ota/ota1.test ea2865997ce573fadaf12eb0a0f80ef22d9dd77f +F ext/ota/ota1.test 0bbdffa5cb4c4bc26be5dae55c834830c7e8e5e3 F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa -F ext/ota/sqlite3ota.c 3e05e3fa5791977eb88261731a6be6d98935efb3 +F ext/ota/sqlite3ota.c c400c9e9ef188cedb9bada263145aaad47d90e75 F ext/ota/sqlite3ota.h 545f0008b5f02f2595899cb9841caddada5c17c0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fc4f7c115207b786e3cac8cb6be4db3131b99a46 -R 43c0251327b6a5ac610420013b39a3a5 +P 9ae44447256b425b5704a1cab3f6796befb92251 +R f29d93e4faa0368c9015e32b108d56ba U dan -Z a71ca63f1bbe8e3a7e1c7cbaa054e55c +Z f9daeef70873e6c36444c084d50eac94 diff --git a/manifest.uuid b/manifest.uuid index 5250b4a025..71abb619b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ae44447256b425b5704a1cab3f6796befb92251 \ No newline at end of file +98387f05697526c7740e91d8a846a31f77639406 \ No newline at end of file From 4e9246e9dbf69ff8771a2121782e69b7ad445ead Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 5 Sep 2014 19:52:42 +0000 Subject: [PATCH 008/116] Switch back to using a single database connection in sqlite3ota.c. FossilOrigin-Name: 3c2f4a078132992e33cc675173c84f8385af9cb5 --- ext/ota/sqlite3ota.c | 105 +++++++++++++++++++++---------------------- manifest | 12 ++--- manifest.uuid | 2 +- 3 files changed, 57 insertions(+), 62 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index b5f6e2c436..f0fbd28d1b 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -35,7 +35,7 @@ ** "progress" -> total number of key/value b-tree operations performed ** so far as part of this ota update. */ -#define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota_state" \ +#define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota.ota_state" \ "(tbl, idx, row, progress)" typedef struct OtaState OtaState; @@ -83,13 +83,10 @@ struct OtaObjIter { ** OTA handle. */ struct sqlite3ota { - sqlite3 *dbDest; /* Target db */ - sqlite3 *dbOta; /* Ota db */ + sqlite3 *db; /* "main" -> target db, "ota" -> ota db */ char *zTarget; /* Path to target db */ - int rc; /* Value returned by last ota_step() call */ char *zErrmsg; /* Error message if rc!=SQLITE_OK */ - int nStep; /* Rows processed for current object */ OtaObjIter objiter; }; @@ -262,14 +259,14 @@ static int otaObjIterFirst(sqlite3ota *p, OtaObjIter *pIter){ int rc; memset(pIter, 0, sizeof(OtaObjIter)); - rc = prepareAndCollectError(p->dbOta, &pIter->pTblIter, &p->zErrmsg, - "SELECT substr(name, 6) FROM sqlite_master " + rc = prepareAndCollectError(p->db, &pIter->pTblIter, &p->zErrmsg, + "SELECT substr(name, 6) FROM ota.sqlite_master " "WHERE type='table' AND name LIKE 'data_%'" ); if( rc==SQLITE_OK ){ - rc = prepareAndCollectError(p->dbDest, &pIter->pIdxIter, &p->zErrmsg, - "SELECT name FROM sqlite_master " + rc = prepareAndCollectError(p->db, &pIter->pIdxIter, &p->zErrmsg, + "SELECT name FROM main.sqlite_master " "WHERE type='index' AND tbl_name = ?" ); } @@ -325,8 +322,8 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ int bSeenPk = 0; int rc2; /* sqlite3_finalize() return value */ - zSql = sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl); - p->rc = prepareFreeAndCollectError(p->dbDest, &pStmt, &p->zErrmsg, zSql); + zSql = sqlite3_mprintf("PRAGMA main.table_info(%Q)", pIter->zTbl); + p->rc = prepareFreeAndCollectError(p->db, &pStmt, &p->zErrmsg, zSql); while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ if( (nCol % 8)==0 ){ int nByte = sizeof(char*) * (nCol+8); @@ -429,16 +426,16 @@ static int otaObjIterPrepareAll( /* Create the index writer */ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_index_writer( - p->dbDest, 0, zIdx, &pIter->pInsert, &aiCol, &pIter->nCol + p->db, 0, zIdx, &pIter->pInsert, &aiCol, &pIter->nCol ); } /* Create the SELECT statement to read keys in sorted order */ zCollist = otaObjIterGetCollist(p, pIter, pIter->nCol, aiCol); if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbOta, &pIter->pSelect, pz, + p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, sqlite3_mprintf( - "SELECT %s FROM 'data_%q' ORDER BY %s%s", + "SELECT %s FROM ota.'data_%q' ORDER BY %s%s", zCollist, pIter->zTbl, zCollist, zLimit ) ); @@ -450,18 +447,19 @@ static int otaObjIterPrepareAll( /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbOta, &pIter->pSelect, pz, + p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, sqlite3_mprintf( - "SELECT %s FROM 'data_%q'%s", + "SELECT %s FROM ota.'data_%q'%s", zCollist, pIter->zTbl, zLimit) ); } /* Create the INSERT statement to write to the target PK b-tree */ if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbDest, &pIter->pInsert, pz, + p->rc = prepareFreeAndCollectError(p->db, &pIter->pInsert, pz, sqlite3_mprintf( - "INSERT INTO %Q(%s) VALUES(%s)", pIter->zTbl, zCollist, zBindings + "INSERT INTO main.%Q(%s) VALUES(%s)", + pIter->zTbl, zCollist, zBindings ) ); } @@ -534,35 +532,38 @@ int sqlite3ota_step(sqlite3ota *p){ return p->rc; } -static void otaOpenDatabase(sqlite3ota *p, sqlite3 **pDb, const char *zFile){ +/* +** Argument zFmt is a sqlite3_mprintf() style format string. The trailing +** arguments are the usual subsitution values. This function performs +** the printf() style substitutions and executes the result as an SQL +** statement on the OTA handles database. +** +** If an error occurs, an error code and error message is stored in the +** OTA handle. If an error has already occurred when this function is +** called, it is a no-op. +*/ +static int otaMPrintfExec(sqlite3ota *p, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_open(zFile, pDb); - if( p->rc ){ - const char *zErr = sqlite3_errmsg(*pDb); - p->zErrmsg = sqlite3_mprintf("sqlite3_open(): %s", zErr); + char *zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_exec(p->db, zSql, 0, 0, &p->zErrmsg); + sqlite3_free(zSql); } } + va_end(ap); + return p->rc; } static void otaSaveTransactionState(sqlite3ota *p){ - char *zInsert; - - zInsert = sqlite3_mprintf( - "INSERT OR REPLACE INTO ota_state(rowid, tbl, idx, row, progress)" + otaMPrintfExec(p, + "INSERT OR REPLACE INTO ota.ota_state(rowid, tbl, idx, row, progress)" "VALUES(1, %Q, %Q, %d, NULL)", p->objiter.zTbl, p->objiter.zIdx, p->nStep ); - - if( zInsert==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - p->rc = sqlite3_exec(p->dbOta, zInsert, 0, 0, &p->zErrmsg); - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbOta, "COMMIT", 0, 0, &p->zErrmsg); - } - } - - sqlite3_free(zInsert); } /* @@ -575,13 +576,13 @@ static void otaSaveTransactionState(sqlite3ota *p){ ** and return NULL. */ static OtaState *otaLoadState(sqlite3ota *p){ - const char *zSelect = "SELECT tbl, idx, row, progress FROM ota_state"; + const char *zSelect = "SELECT tbl, idx, row, progress FROM ota.ota_state"; OtaState *pRet = 0; sqlite3_stmt *pStmt; int rc; assert( p->rc==SQLITE_OK ); - rc = prepareAndCollectError(p->dbOta, &pStmt, &p->zErrmsg, zSelect); + rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, zSelect); if( rc==SQLITE_OK ){ if( sqlite3_step(pStmt)==SQLITE_ROW ){ const char *zIdx = (const char*)sqlite3_column_text(pStmt, 1); @@ -700,12 +701,15 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ memset(p, 0, sizeof(sqlite3ota)); p->zTarget = (char*)&p[1]; memcpy(p->zTarget, zTarget, nTarget+1); - otaOpenDatabase(p, &p->dbDest, zTarget); - otaOpenDatabase(p, &p->dbOta, zOta); + p->rc = sqlite3_open(zTarget, &p->db); + if( p->rc ){ + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); + } + otaMPrintfExec(p, "ATTACH %Q AS ota", zOta); /* If it has not already been created, create the ota_state table */ if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbOta, OTA_CREATE_STATE, 0, 0, &p->zErrmsg); + p->rc = sqlite3_exec(p->db, OTA_CREATE_STATE, 0, 0, &p->zErrmsg); } if( p->rc==SQLITE_OK ){ @@ -722,12 +726,7 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ "PRAGMA ota_mode=1;" "BEGIN IMMEDIATE;" ; - p->rc = sqlite3_exec(p->dbDest, zScript, 0, 0, &p->zErrmsg); - } - - if( p->rc==SQLITE_OK ){ - const char *zScript = "BEGIN IMMEDIATE"; - p->rc = sqlite3_exec(p->dbOta, zScript, 0, 0, &p->zErrmsg); + p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); } /* Point the object iterator at the first object */ @@ -766,16 +765,12 @@ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ /* Commit the transaction to the *-oal file. */ if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ - rc = sqlite3_exec(p->dbDest, "COMMIT", 0, 0, &p->zErrmsg); + rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); if( rc!=SQLITE_OK ) p->rc = rc; } - assert( sqlite3_next_stmt(p->dbDest, 0)==0 ); - assert( sqlite3_next_stmt(p->dbOta, 0)==0 ); - /* Close the open database handles */ - sqlite3_close(p->dbDest); - sqlite3_close(p->dbOta); + sqlite3_close(p->db); /* If the OTA has been completely applied and no error occurred, move ** the *-oal file to *-wal. */ diff --git a/manifest b/manifest index 6cfb5e1441..8d8f48b13b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reorganize\sthe\scode\sin\ssqlite3ota.c\sin\spreparation\sfor\sadding\ssupport\sfor\supdate\sand\sdelete\soperations. -D 2014-09-05T19:31:15.782 +C Switch\sback\sto\susing\sa\ssingle\sdatabase\sconnection\sin\ssqlite3ota.c. +D 2014-09-05T19:52:42.359 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 F ext/ota/ota1.test 0bbdffa5cb4c4bc26be5dae55c834830c7e8e5e3 F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa -F ext/ota/sqlite3ota.c c400c9e9ef188cedb9bada263145aaad47d90e75 +F ext/ota/sqlite3ota.c 3ddf5f8122f9ab3270541f61bde5d95ef7b631d5 F ext/ota/sqlite3ota.h 545f0008b5f02f2595899cb9841caddada5c17c0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9ae44447256b425b5704a1cab3f6796befb92251 -R f29d93e4faa0368c9015e32b108d56ba +P 98387f05697526c7740e91d8a846a31f77639406 +R cd4982d2f447b7a01026d09e8721c98f U dan -Z f9daeef70873e6c36444c084d50eac94 +Z 303c253ac0315bace60d825e70dc4fdd diff --git a/manifest.uuid b/manifest.uuid index 71abb619b3..3a256e520f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98387f05697526c7740e91d8a846a31f77639406 \ No newline at end of file +3c2f4a078132992e33cc675173c84f8385af9cb5 \ No newline at end of file From 9295d21bfd8bb3c1f9fcc70ca9f7764c61d70d79 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 6 Sep 2014 20:19:38 +0000 Subject: [PATCH 009/116] Add support for delete operations to the ota extension. FossilOrigin-Name: f988234ba54d7c667f7deef1d04beed4e7fe6182 --- ext/ota/ota1.test | 60 +++++++++- ext/ota/sqlite3ota.c | 258 +++++++++++++++++++++++++++++++++++++------ manifest | 18 +-- manifest.uuid | 2 +- src/delete.c | 3 + src/vdbeblob.c | 24 ++-- 6 files changed, 304 insertions(+), 61 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index 12afc6e3f1..22b8fa66cd 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -32,16 +32,21 @@ proc create_ota1 {filename} { return $filename } -# Create an empty target database suitable for the OTA created by -# [create_ota1]. +# Create a simple OTA database. That expects to write to a table: # # CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); # -proc create_target1 {filename} { +proc create_ota4 {filename} { forcedelete $filename - sqlite3 target1 $filename - target1 eval { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } - target1 close + sqlite3 ota1 $filename + ota1 eval { + CREATE TABLE data_t1(a, b, c, ota_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(2, NULL, 5, 1); + INSERT INTO data_t1 VALUES(3, 8, 9, 0); + INSERT INTO data_t1 VALUES(4, NULL, 11, 1); + } + ota1 close return $filename } @@ -176,6 +181,49 @@ foreach {tn errcode errmsg schema} { do_test 3.$tn.4 { dbcksum db main } $cksum } +#------------------------------------------------------------------------- +# +foreach {tn2 cmd} {1 run_ota 2 step_ota} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + } + 3 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + } { + reset_db + execsql $schema + execsql { + INSERT INTO t1 VALUES(2, 'hello', 'world'); + INSERT INTO t1 VALUES(4, 'hello', 'planet'); + INSERT INTO t1 VALUES(6, 'hello', 'xyz'); + } + + do_test 4.$tn2.$tn.1 { + create_ota4 ota.db + $cmd test.db ota.db + } {SQLITE_DONE} + + do_execsql_test 4.$tn2.$tn.2 { + SELECT * FROM t1 ORDER BY a ASC; + } { + 1 2 3 + 3 8 9 + 6 hello xyz + } + + do_execsql_test 4.$tn2.$tn.3 { PRAGMA integrity_check } ok + } +} + finish_test diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index f0fbd28d1b..62b1c4ab34 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -77,6 +77,7 @@ struct OtaObjIter { int nCol; /* Number of columns in current object */ sqlite3_stmt *pSelect; /* Source data */ sqlite3_stmt *pInsert; /* Statement for INSERT operations */ + sqlite3_stmt *pDelete; /* Statement for DELETE ops */ }; /* @@ -189,6 +190,7 @@ static void otaObjIterFinalize(OtaObjIter *pIter){ sqlite3_finalize(pIter->pIdxIter); sqlite3_finalize(pIter->pSelect); sqlite3_finalize(pIter->pInsert); + sqlite3_finalize(pIter->pDelete); otaObjIterFreeCols(pIter); memset(pIter, 0, sizeof(OtaObjIter)); } @@ -208,8 +210,10 @@ static int otaObjIterNext(sqlite3ota *p, OtaObjIter *pIter){ /* Free any SQLite statements used while processing the previous object */ sqlite3_finalize(pIter->pSelect); sqlite3_finalize(pIter->pInsert); + sqlite3_finalize(pIter->pDelete); pIter->pSelect = 0; pIter->pInsert = 0; + pIter->pDelete = 0; pIter->nCol = 0; if( pIter->bCleanup ){ @@ -305,6 +309,32 @@ static char *otaQuoteName(const char *zName){ return zRet; } +/* +** Argument zFmt is a sqlite3_mprintf() style format string. The trailing +** arguments are the usual subsitution values. This function performs +** the printf() style substitutions and executes the result as an SQL +** statement on the OTA handles database. +** +** If an error occurs, an error code and error message is stored in the +** OTA handle. If an error has already occurred when this function is +** called, it is a no-op. +*/ +static int otaMPrintfExec(sqlite3ota *p, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + if( p->rc==SQLITE_OK ){ + char *zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_exec(p->db, zSql, 0, 0, &p->zErrmsg); + sqlite3_free(zSql); + } + } + va_end(ap); + return p->rc; +} + /* ** If they are not already populated, populate the pIter->azTblCol[], ** pIter->abTblPk[] and pIter->nTblCol variables according to the table @@ -380,6 +410,48 @@ static char *otaObjIterGetCollist( return zList; } +static char *otaObjIterGetOldlist( + sqlite3ota *p, + OtaObjIter *pIter +){ + char *zList = 0; + if( p->rc==SQLITE_OK ){ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + zList = sqlite3_mprintf("%z%sold.%s", zList, zSep, pIter->azTblCol[i]); + zSep = ", "; + if( zList==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + } + } + return zList; +} + +static char *otaObjIterGetWhere( + sqlite3ota *p, + OtaObjIter *pIter +){ + char *zList = 0; + if( p->rc==SQLITE_OK ){ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + zList = sqlite3_mprintf("%z%s%s=?", zList, zSep, pIter->azTblCol[i]); + zSep = " AND "; + if( zList==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + } + } + } + return zList; +} + static char *otaObjIterGetBindlist(sqlite3ota *p, int nBind){ char *zRet = 0; if( p->rc==SQLITE_OK ){ @@ -423,25 +495,37 @@ static int otaObjIterPrepareAll( if( zIdx ){ int *aiCol; /* Column map */ - /* Create the index writer */ + /* Create the index writers */ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_index_writer( p->db, 0, zIdx, &pIter->pInsert, &aiCol, &pIter->nCol ); } + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_index_writer( + p->db, 1, zIdx, &pIter->pDelete, &aiCol, &pIter->nCol + ); + } /* Create the SELECT statement to read keys in sorted order */ zCollist = otaObjIterGetCollist(p, pIter, pIter->nCol, aiCol); if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, sqlite3_mprintf( - "SELECT %s FROM ota.'data_%q' ORDER BY %s%s", - zCollist, pIter->zTbl, zCollist, zLimit + "SELECT %s, ota_control FROM ota.'data_%q' " + "UNION ALL " + "SELECT %s, ota_control FROM ota.'ota_tmp_%q' " + "ORDER BY %s%s", + zCollist, pIter->zTbl, + zCollist, pIter->zTbl, + zCollist, zLimit ) ); } }else{ char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol); + char *zWhere = otaObjIterGetWhere(p, pIter); + char *zOldlist = otaObjIterGetOldlist(p, pIter); zCollist = otaObjIterGetCollist(p, pIter, pIter->nTblCol, 0); pIter->nCol = pIter->nTblCol; @@ -449,7 +533,7 @@ static int otaObjIterPrepareAll( if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, sqlite3_mprintf( - "SELECT %s FROM ota.'data_%q'%s", + "SELECT %s, ota_control FROM ota.'data_%q'%s", zCollist, pIter->zTbl, zLimit) ); } @@ -463,6 +547,31 @@ static int otaObjIterPrepareAll( ) ); } + + /* Create the DELETE statement to write to the target PK b-tree */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->db, &pIter->pDelete, pz, + sqlite3_mprintf( + "DELETE FROM main.%Q WHERE %s", pIter->zTbl, zWhere + ) + ); + } + + if( p->rc==SQLITE_OK ){ + otaMPrintfExec(p, + "CREATE TABLE IF NOT EXISTS ota.'ota_tmp_%q' AS " + "SELECT * FROM ota.'data_%q' WHERE 0;" + "CREATE TEMP TRIGGER ota_delete_%q BEFORE DELETE ON main.%Q " + "BEGIN " + " INSERT INTO 'ota_tmp_%q'(ota_control, %s) VALUES(2, %s);" + "END;" + , pIter->zTbl, pIter->zTbl, pIter->zTbl, pIter->zTbl, pIter->zTbl, + zCollist, zOldlist + ); + } + + sqlite3_free(zWhere); + sqlite3_free(zOldlist); sqlite3_free(zBindings); } sqlite3_free(zCollist); @@ -472,6 +581,72 @@ static int otaObjIterPrepareAll( return p->rc; } +#define OTA_INSERT 1 +#define OTA_DELETE 2 +#define OTA_IDX_DELETE 3 +#define OTA_UPDATE 4 + +/* +** The SELECT statement iterating through the keys for the current object +** (p->objiter.pSelect) currently points to a valid row. However, there +** is something wrong with the ota_control value in the ota_control value +** stored in the (p->nCol+1)'th column. Set the error code and error message +** of the OTA handle to something reflecting this. +*/ +static void otaBadControlError(sqlite3ota *p){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("Invalid ota_control value"); +} + +/* +** The SELECT statement iterating through the keys for the current object +** (p->objiter.pSelect) currently points to a valid row. This function +** determines the type of operation requested by this row and returns +** one of the following values to indicate the result: +** +** * OTA_INSERT +** * OTA_DELETE +** * OTA_IDX_DELETE +** * OTA_UPDATE +** +** If OTA_UPDATE is returned, then output variable *pzMask is set to +** point to the text value indicating the columns to update. +** +** If the ota_control field contains an invalid value, an error code and +** message are left in the OTA handle and zero returned. +*/ +static int otaStepType(sqlite3ota *p, const char **pzMask){ + int iCol = p->objiter.nCol; /* Index of ota_control column */ + int res = 0; /* Return value */ + + switch( sqlite3_column_type(p->objiter.pSelect, iCol) ){ + case SQLITE_INTEGER: { + int iVal = sqlite3_column_int(p->objiter.pSelect, iCol); + if( iVal==0 ){ + res = OTA_INSERT; + }else if( iVal==1 ){ + res = OTA_DELETE; + }else if( iVal==2 ){ + res = OTA_IDX_DELETE; + } + break; + } + + case SQLITE_TEXT: + *pzMask = (const char*)sqlite3_column_text(p->objiter.pSelect, iCol); + res = OTA_UPDATE; + break; + + default: + break; + } + + if( res==0 ){ + otaBadControlError(p); + } + return res; +} + /* ** This function does the work for an sqlite3ota_step() call. ** @@ -485,15 +660,49 @@ static int otaObjIterPrepareAll( */ static int otaStep(sqlite3ota *p){ OtaObjIter *pIter = &p->objiter; + const char *zMask = 0; int i; + int eType = otaStepType(p, &zMask); - for(i=0; inCol; i++){ - sqlite3_value *pVal = sqlite3_column_value(pIter->pSelect, i); - sqlite3_bind_value(pIter->pInsert, i+1, pVal); + if( eType ){ + assert( eType!=OTA_UPDATE || pIter->zIdx==0 ); + + if( pIter->zIdx==0 && eType==OTA_IDX_DELETE ){ + otaBadControlError(p); + } + else if( eType==OTA_INSERT || eType==OTA_IDX_DELETE ){ + sqlite3_stmt *pWriter; + assert( eType!=OTA_UPDATE ); + + pWriter = (eType==OTA_INSERT)?pIter->pInsert:pIter->pDelete; + for(i=0; inCol; i++){ + sqlite3_value *pVal = sqlite3_column_value(pIter->pSelect, i); + sqlite3_bind_value(pWriter, i+1, pVal); + } + sqlite3_step(pWriter); + p->rc = resetAndCollectError(pWriter, &p->zErrmsg); + } + else if( eType==OTA_DELETE && pIter->zIdx==0 ){ + int iVar = 1; + assert( pIter->zIdx==0 ); + assert( pIter->nCol==pIter->nTblCol ); + for(i=0; inCol; i++){ + if( pIter->abTblPk[i] ){ + sqlite3_value *pVal = sqlite3_column_value(pIter->pSelect, i); + sqlite3_bind_value(pIter->pDelete, iVar++, pVal); + } + } + sqlite3_step(pIter->pDelete); + p->rc = resetAndCollectError(pIter->pDelete, &p->zErrmsg); + }else if( eType==OTA_UPDATE ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("not yet"); + }else{ + /* no-op */ + assert( eType==OTA_DELETE && pIter->zIdx ); + } } - sqlite3_step(pIter->pInsert); - p->rc = resetAndCollectError(pIter->pInsert, &p->zErrmsg); return p->rc; } @@ -506,7 +715,10 @@ int sqlite3ota_step(sqlite3ota *p){ while( p && p->rc==SQLITE_OK && pIter->zTbl ){ if( pIter->bCleanup ){ - /* this is where cleanup of the ota_xxx table will happen... */ + /* Clean up the ota_tmp_xxx table for the previous table. It + ** cannot be dropped as there are currently active SQL statements. + ** But the contents can be deleted. */ + otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); }else{ otaObjIterPrepareAll(p, pIter, 0); @@ -532,32 +744,6 @@ int sqlite3ota_step(sqlite3ota *p){ return p->rc; } -/* -** Argument zFmt is a sqlite3_mprintf() style format string. The trailing -** arguments are the usual subsitution values. This function performs -** the printf() style substitutions and executes the result as an SQL -** statement on the OTA handles database. -** -** If an error occurs, an error code and error message is stored in the -** OTA handle. If an error has already occurred when this function is -** called, it is a no-op. -*/ -static int otaMPrintfExec(sqlite3ota *p, const char *zFmt, ...){ - va_list ap; - va_start(ap, zFmt); - if( p->rc==SQLITE_OK ){ - char *zSql = sqlite3_vmprintf(zFmt, ap); - if( zSql==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - p->rc = sqlite3_exec(p->db, zSql, 0, 0, &p->zErrmsg); - sqlite3_free(zSql); - } - } - va_end(ap); - return p->rc; -} - static void otaSaveTransactionState(sqlite3ota *p){ otaMPrintfExec(p, "INSERT OR REPLACE INTO ota.ota_state(rowid, tbl, idx, row, progress)" diff --git a/manifest b/manifest index 8d8f48b13b..46bc695b1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Switch\sback\sto\susing\sa\ssingle\sdatabase\sconnection\sin\ssqlite3ota.c. -D 2014-09-05T19:52:42.359 +C Add\ssupport\sfor\sdelete\soperations\sto\sthe\sota\sextension. +D 2014-09-06T20:19:38.006 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,9 +122,9 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 -F ext/ota/ota1.test 0bbdffa5cb4c4bc26be5dae55c834830c7e8e5e3 +F ext/ota/ota1.test 47317179125b5e65289a9f59753c9f895186e6d5 F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa -F ext/ota/sqlite3ota.c 3ddf5f8122f9ab3270541f61bde5d95ef7b631d5 +F ext/ota/sqlite3ota.c ceb0f77dc6a958d299f532319f6477e5599dc59d F ext/ota/sqlite3ota.h 545f0008b5f02f2595899cb9841caddada5c17c0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -181,7 +181,7 @@ F src/callback.c b97d0695ffcf6a8710ee445ffe56ee387d4d8a6f F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 -F src/delete.c 5adcd322c6b08fc25d215d780ca62cebce66304d +F src/delete.c 3c2a375c0329247d01222170ae19ad8a52ecbf9a F src/expr.c e1691ab0fe6be7247ef073b0038fb8ecd9944fad F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8d81a780ad78d16ec9082585758a8f1d6bf02ca3 @@ -295,7 +295,7 @@ F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h cdc8e421f85beb1ac9b4669ec5beadab6faa15e0 F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 -F src/vdbeblob.c 0bc9d22578d87ad9ff1c16e20a36863326f34fd7 +F src/vdbeblob.c b1b8b2cd86617db009f027f116b335f86e95b617 F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62 F src/vdbesort.c 7c45bfcd823f30d172bbbc1b9f51ef4402fbfe8d F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 98387f05697526c7740e91d8a846a31f77639406 -R cd4982d2f447b7a01026d09e8721c98f +P 3c2f4a078132992e33cc675173c84f8385af9cb5 +R eeedfe1c92df64758f3e97de2d55d43b U dan -Z 303c253ac0315bace60d825e70dc4fdd +Z 3317be7135040205f21e9375db1283ea diff --git a/manifest.uuid b/manifest.uuid index 3a256e520f..457a56e402 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c2f4a078132992e33cc675173c84f8385af9cb5 \ No newline at end of file +f988234ba54d7c667f7deef1d04beed4e7fe6182 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index af83903c42..19c1ed01a8 100644 --- a/src/delete.c +++ b/src/delete.c @@ -730,6 +730,9 @@ void sqlite3GenerateRowIndexDelete( Vdbe *v; /* The prepared statement under construction */ Index *pPk; /* PRIMARY KEY index, or NULL for rowid tables */ + /* Skip this if we are in OTA mode */ + if( pParse->db->flags & SQLITE_OtaMode ) return; + v = pParse->pVdbe; pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 11e018e19c..5e809fc78f 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -546,17 +546,23 @@ int sqlite3_index_writer( sqlite3VdbeAddOp2(v, OP_Variable, i, i); } regRec = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_MakeRecord, 1, pIdx->nColumn, regRec); - /* If this is a UNIQUE index, check the constraint. */ - if( pIdx->onError ){ - int addr = sqlite3VdbeAddOp4Int(v, OP_NoConflict, 0, 0, 1, pIdx->nKeyCol); - sqlite3UniqueConstraint(pParse, SQLITE_ABORT, pIdx); - sqlite3VdbeJumpHere(v, addr); + if( bDelete==0 ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, 1, pIdx->nColumn, regRec); + + /* If this is a UNIQUE index, check the constraint. */ + if( pIdx->onError ){ + int addr = sqlite3VdbeAddOp4Int(v, OP_NoConflict, 0, 0, 1, pIdx->nKeyCol); + sqlite3UniqueConstraint(pParse, SQLITE_ABORT, pIdx); + sqlite3VdbeJumpHere(v, addr); + } + + /* Code the IdxInsert to write to the b-tree index. */ + sqlite3VdbeAddOp2(v, OP_IdxInsert, 0, regRec); + }else{ + /* Code the IdxDelete to remove the entry from the b-tree index. */ + sqlite3VdbeAddOp3(v, OP_IdxDelete, 0, 1, pIdx->nColumn); } - - /* Code the IdxInsert to write to the b-tree index. */ - sqlite3VdbeAddOp2(v, OP_IdxInsert, 0, regRec); sqlite3FinishCoding(pParse); index_writer_out: From a690402b4cab21b9243a206134d1e2eed41f605a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Sep 2014 17:50:35 +0000 Subject: [PATCH 010/116] Add support for update statements to sqlite3ota.c. FossilOrigin-Name: e109b27e4d66b83e1a804e7556d9c91aa37fea28 --- ext/ota/ota1.test | 62 +++++++++++++ ext/ota/sqlite3ota.c | 211 +++++++++++++++++++++++++++++++++---------- manifest | 14 +-- manifest.uuid | 2 +- 4 files changed, 233 insertions(+), 56 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index 22b8fa66cd..917d1d2c9e 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -36,6 +36,8 @@ proc create_ota1 {filename} { # # CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); # +# This OTA includes both insert and delete operations. +# proc create_ota4 {filename} { forcedelete $filename sqlite3 ota1 $filename @@ -50,6 +52,25 @@ proc create_ota4 {filename} { return $filename } +# Create a simple OTA database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); +# +# This OTA includes update statements. +# +proc create_ota5 {filename} { + forcedelete $filename + sqlite3 ota5 $filename + ota5 eval { + CREATE TABLE data_t1(a, b, c, d, ota_control); + INSERT INTO data_t1 VALUES(1, NULL, NULL, 5, '...x'); -- SET d = 5 + INSERT INTO data_t1 VALUES(2, NULL, 10, 5, '..xx'); -- SET c=10, d = 5 + INSERT INTO data_t1 VALUES(3, 11, NULL, NULL, '.x..'); -- SET b=11 + } + ota5 close + return $filename +} + # Run the OTA in file $ota on target database $target until completion. # proc run_ota {target ota} { @@ -224,6 +245,47 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota} { } } +#------------------------------------------------------------------------- +# +foreach {tn2 cmd} {1 run_ota 2 step_ota} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } + } { + reset_db + execsql $schema + execsql { + INSERT INTO t1 VALUES(1, 2, 3, 4); + INSERT INTO t1 VALUES(2, 5, 6, 7); + INSERT INTO t1 VALUES(3, 8, 9, 10); + } + + do_test 5.$tn2.$tn.1 { + create_ota5 ota.db + $cmd test.db ota.db + } {SQLITE_DONE} + + do_execsql_test 5.$tn2.$tn.2 { + SELECT * FROM t1 ORDER BY a ASC; + } { + 1 2 3 5 + 2 5 10 5 + 3 11 9 10 + } + + do_execsql_test 5.$tn2.$tn.3 { PRAGMA integrity_check } ok + } +} finish_test diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 62b1c4ab34..fcf0c8b9fd 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -78,6 +78,10 @@ struct OtaObjIter { sqlite3_stmt *pSelect; /* Source data */ sqlite3_stmt *pInsert; /* Statement for INSERT operations */ sqlite3_stmt *pDelete; /* Statement for DELETE ops */ + + /* Last UPDATE used (for PK b-tree updates only), or NULL. */ + char *zMask; /* Copy of update mask used with pUpdate */ + sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ }; /* @@ -179,6 +183,24 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){ pIter->azTblCol = 0; pIter->abTblPk = 0; pIter->nTblCol = 0; + sqlite3_free(pIter->zMask); + pIter->zMask = 0; +} + +/* +** Finalize all statements and free all allocations that are specific to +** the current object (table/index pair). +*/ +static void otaObjIterClearStatements(OtaObjIter *pIter){ + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + sqlite3_finalize(pIter->pDelete); + sqlite3_finalize(pIter->pUpdate); + pIter->pSelect = 0; + pIter->pInsert = 0; + pIter->pDelete = 0; + pIter->pUpdate = 0; + pIter->nCol = 0; } /* @@ -186,11 +208,9 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){ ** as the only argument. */ static void otaObjIterFinalize(OtaObjIter *pIter){ + otaObjIterClearStatements(pIter); sqlite3_finalize(pIter->pTblIter); sqlite3_finalize(pIter->pIdxIter); - sqlite3_finalize(pIter->pSelect); - sqlite3_finalize(pIter->pInsert); - sqlite3_finalize(pIter->pDelete); otaObjIterFreeCols(pIter); memset(pIter, 0, sizeof(OtaObjIter)); } @@ -208,13 +228,7 @@ static int otaObjIterNext(sqlite3ota *p, OtaObjIter *pIter){ if( rc==SQLITE_OK ){ /* Free any SQLite statements used while processing the previous object */ - sqlite3_finalize(pIter->pSelect); - sqlite3_finalize(pIter->pInsert); - sqlite3_finalize(pIter->pDelete); - pIter->pSelect = 0; - pIter->pInsert = 0; - pIter->pDelete = 0; - pIter->nCol = 0; + otaObjIterClearStatements(pIter); if( pIter->bCleanup ){ otaObjIterFreeCols(pIter); @@ -412,15 +426,16 @@ static char *otaObjIterGetCollist( static char *otaObjIterGetOldlist( sqlite3ota *p, - OtaObjIter *pIter + OtaObjIter *pIter, + const char *zObj ){ char *zList = 0; if( p->rc==SQLITE_OK ){ - const char *zSep = ""; + const char *zS = ""; int i; for(i=0; inTblCol; i++){ - zList = sqlite3_mprintf("%z%sold.%s", zList, zSep, pIter->azTblCol[i]); - zSep = ", "; + zList = sqlite3_mprintf("%z%s%s.%s", zList, zS, zObj, pIter->azTblCol[i]); + zS = ", "; if( zList==0 ){ p->rc = SQLITE_NOMEM; break; @@ -440,7 +455,8 @@ static char *otaObjIterGetWhere( int i; for(i=0; inTblCol; i++){ if( pIter->abTblPk[i] ){ - zList = sqlite3_mprintf("%z%s%s=?", zList, zSep, pIter->azTblCol[i]); + const char *zCol = pIter->azTblCol[i]; + zList = sqlite3_mprintf("%z%s%s=?%d", zList, zSep, zCol, i+1); zSep = " AND "; if( zList==0 ){ p->rc = SQLITE_NOMEM; @@ -452,6 +468,44 @@ static char *otaObjIterGetWhere( return zList; } +/* +** The SELECT statement iterating through the keys for the current object +** (p->objiter.pSelect) currently points to a valid row. However, there +** is something wrong with the ota_control value in the ota_control value +** stored in the (p->nCol+1)'th column. Set the error code and error message +** of the OTA handle to something reflecting this. +*/ +static void otaBadControlError(sqlite3ota *p){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("Invalid ota_control value"); +} + +static char *otaObjIterGetSetlist( + sqlite3ota *p, + OtaObjIter *pIter, + const char *zMask +){ + char *zList = 0; + if( p->rc==SQLITE_OK ){ + int i; + + if( strlen(zMask)!=pIter->nTblCol ){ + otaBadControlError(p); + }else{ + const char *zSep = ""; + for(i=0; inTblCol; i++){ + if( zMask[i]=='x' ){ + zList = sqlite3_mprintf("%z%s%s=?%d", + zList, zSep, pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + } + } + } + return zList; +} + static char *otaObjIterGetBindlist(sqlite3ota *p, int nBind){ char *zRet = 0; if( p->rc==SQLITE_OK ){ @@ -513,6 +567,7 @@ static int otaObjIterPrepareAll( p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, sqlite3_mprintf( "SELECT %s, ota_control FROM ota.'data_%q' " + "WHERE typeof(ota_control)='integer' AND ota_control!=1 " "UNION ALL " "SELECT %s, ota_control FROM ota.'ota_tmp_%q' " "ORDER BY %s%s", @@ -525,7 +580,8 @@ static int otaObjIterPrepareAll( }else{ char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol); char *zWhere = otaObjIterGetWhere(p, pIter); - char *zOldlist = otaObjIterGetOldlist(p, pIter); + char *zOldlist = otaObjIterGetOldlist(p, pIter, "old"); + char *zNewlist = otaObjIterGetOldlist(p, pIter, "new"); zCollist = otaObjIterGetCollist(p, pIter, pIter->nTblCol, 0); pIter->nCol = pIter->nTblCol; @@ -561,17 +617,43 @@ static int otaObjIterPrepareAll( otaMPrintfExec(p, "CREATE TABLE IF NOT EXISTS ota.'ota_tmp_%q' AS " "SELECT * FROM ota.'data_%q' WHERE 0;" + "CREATE TEMP TRIGGER ota_delete_%q BEFORE DELETE ON main.%Q " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s) VALUES(2, %s);" "END;" - , pIter->zTbl, pIter->zTbl, pIter->zTbl, pIter->zTbl, pIter->zTbl, - zCollist, zOldlist + + "CREATE TEMP TRIGGER ota_update1_%q BEFORE UPDATE ON main.%Q " + "BEGIN " + " INSERT INTO 'ota_tmp_%q'(ota_control, %s) VALUES(2, %s);" + "END;" + + "CREATE TEMP TRIGGER ota_update2_%q AFTER UPDATE ON main.%Q " + "BEGIN " + " INSERT INTO 'ota_tmp_%q'(ota_control, %s) VALUES(3, %s);" + "END;" + + , pIter->zTbl, pIter->zTbl, + pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOldlist, + pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOldlist, + pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zNewlist ); } + /* Allocate space required for the zMask field. */ + if( p->rc==SQLITE_OK ){ + int nMask = pIter->nTblCol+1; + pIter->zMask = (char*)sqlite3_malloc(nMask); + if( pIter->zMask==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pIter->zMask, 0, nMask); + } + } + sqlite3_free(zWhere); sqlite3_free(zOldlist); + sqlite3_free(zNewlist); sqlite3_free(zBindings); } sqlite3_free(zCollist); @@ -584,18 +666,39 @@ static int otaObjIterPrepareAll( #define OTA_INSERT 1 #define OTA_DELETE 2 #define OTA_IDX_DELETE 3 -#define OTA_UPDATE 4 +#define OTA_IDX_INSERT 4 +#define OTA_UPDATE 5 -/* -** The SELECT statement iterating through the keys for the current object -** (p->objiter.pSelect) currently points to a valid row. However, there -** is something wrong with the ota_control value in the ota_control value -** stored in the (p->nCol+1)'th column. Set the error code and error message -** of the OTA handle to something reflecting this. -*/ -static void otaBadControlError(sqlite3ota *p){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("Invalid ota_control value"); +static int otaGetUpdateStmt( + sqlite3ota *p, + OtaObjIter *pIter, + const char *zMask, + sqlite3_stmt **ppStmt +){ + if( pIter->pUpdate && strcmp(zMask, pIter->zMask)==0 ){ + *ppStmt = pIter->pUpdate; + }else{ + char *zWhere = otaObjIterGetWhere(p, pIter); + char *zSet = otaObjIterGetSetlist(p, pIter, zMask); + char *zUpdate = 0; + sqlite3_finalize(pIter->pUpdate); + pIter->pUpdate = 0; + if( p->rc==SQLITE_OK ){ + zUpdate = sqlite3_mprintf("UPDATE %Q SET %s WHERE %s", + pIter->zTbl, zSet, zWhere + ); + p->rc = prepareFreeAndCollectError( + p->db, &pIter->pUpdate, &p->zErrmsg, zUpdate + ); + *ppStmt = pIter->pUpdate; + } + if( p->rc==SQLITE_OK ){ + memcpy(pIter->zMask, zMask, pIter->nTblCol); + } + sqlite3_free(zWhere); + sqlite3_free(zSet); + } + return p->rc; } /* @@ -628,6 +731,8 @@ static int otaStepType(sqlite3ota *p, const char **pzMask){ res = OTA_DELETE; }else if( iVal==2 ){ res = OTA_IDX_DELETE; + }else if( iVal==3 ){ + res = OTA_IDX_INSERT; } break; } @@ -670,33 +775,43 @@ static int otaStep(sqlite3ota *p){ if( pIter->zIdx==0 && eType==OTA_IDX_DELETE ){ otaBadControlError(p); } - else if( eType==OTA_INSERT || eType==OTA_IDX_DELETE ){ + else if( + eType==OTA_INSERT + || eType==OTA_DELETE + || eType==OTA_IDX_DELETE + || eType==OTA_IDX_INSERT + ){ sqlite3_stmt *pWriter; - assert( eType!=OTA_UPDATE ); - pWriter = (eType==OTA_INSERT)?pIter->pInsert:pIter->pDelete; + assert( eType!=OTA_UPDATE ); + assert( eType!=OTA_DELETE || pIter->zIdx==0 ); + + if( eType==OTA_IDX_DELETE || eType==OTA_DELETE ){ + pWriter = pIter->pDelete; + }else{ + pWriter = pIter->pInsert; + } + for(i=0; inCol; i++){ + if( eType==SQLITE_DELETE && pIter->zIdx==0 && pIter->abTblPk[i]==0 ){ + continue; + } sqlite3_value *pVal = sqlite3_column_value(pIter->pSelect, i); sqlite3_bind_value(pWriter, i+1, pVal); } sqlite3_step(pWriter); p->rc = resetAndCollectError(pWriter, &p->zErrmsg); - } - else if( eType==OTA_DELETE && pIter->zIdx==0 ){ - int iVar = 1; - assert( pIter->zIdx==0 ); - assert( pIter->nCol==pIter->nTblCol ); - for(i=0; inCol; i++){ - if( pIter->abTblPk[i] ){ - sqlite3_value *pVal = sqlite3_column_value(pIter->pSelect, i); - sqlite3_bind_value(pIter->pDelete, iVar++, pVal); - } - } - sqlite3_step(pIter->pDelete); - p->rc = resetAndCollectError(pIter->pDelete, &p->zErrmsg); }else if( eType==OTA_UPDATE ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("not yet"); + sqlite3_stmt *pUpdate = 0; + otaGetUpdateStmt(p, pIter, zMask, &pUpdate); + if( pUpdate ){ + for(i=0; inCol; i++){ + sqlite3_value *pVal = sqlite3_column_value(pIter->pSelect, i); + sqlite3_bind_value(pUpdate, i+1, pVal); + } + sqlite3_step(pUpdate); + p->rc = resetAndCollectError(pUpdate, &p->zErrmsg); + } }else{ /* no-op */ assert( eType==OTA_DELETE && pIter->zIdx ); @@ -718,7 +833,7 @@ int sqlite3ota_step(sqlite3ota *p){ /* Clean up the ota_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ - otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); + // otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); }else{ otaObjIterPrepareAll(p, pIter, 0); diff --git a/manifest b/manifest index 46bc695b1a..35c6c48a27 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sdelete\soperations\sto\sthe\sota\sextension. -D 2014-09-06T20:19:38.006 +C Add\ssupport\sfor\supdate\sstatements\sto\ssqlite3ota.c. +D 2014-09-08T17:50:35.158 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,9 +122,9 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 -F ext/ota/ota1.test 47317179125b5e65289a9f59753c9f895186e6d5 +F ext/ota/ota1.test fe0bb8acf0caef6c19937b84c6547b788342610d F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa -F ext/ota/sqlite3ota.c ceb0f77dc6a958d299f532319f6477e5599dc59d +F ext/ota/sqlite3ota.c 3697f6db2d51d5f7c0f0306fe94514785361e521 F ext/ota/sqlite3ota.h 545f0008b5f02f2595899cb9841caddada5c17c0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3c2f4a078132992e33cc675173c84f8385af9cb5 -R eeedfe1c92df64758f3e97de2d55d43b +P f988234ba54d7c667f7deef1d04beed4e7fe6182 +R c23b09bc3993f39690d272fa85636507 U dan -Z 3317be7135040205f21e9375db1283ea +Z 28e28dd83293020abb0c0fb47ae5ab2c diff --git a/manifest.uuid b/manifest.uuid index 457a56e402..09501cef43 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f988234ba54d7c667f7deef1d04beed4e7fe6182 \ No newline at end of file +e109b27e4d66b83e1a804e7556d9c91aa37fea28 \ No newline at end of file From d54e06ca5ef35bc5cfbd840881065b819482ec16 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 15 Sep 2014 10:44:24 +0000 Subject: [PATCH 011/116] Add OP_Affinity opcodes to the VMs generated by sqlite3_index_writer(). FossilOrigin-Name: b9b38cb8e21826ddfb4c87a1f166ad33c3c7b6eb --- ext/ota/ota3.test | 54 +++++++++++++++++++++++++++++++++++++++++++++++ manifest | 13 ++++++------ manifest.uuid | 2 +- src/vdbeblob.c | 5 ++++- 4 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 ext/ota/ota3.test diff --git a/ext/ota/ota3.test b/ext/ota/ota3.test new file mode 100644 index 0000000000..7907974767 --- /dev/null +++ b/ext/ota/ota3.test @@ -0,0 +1,54 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file join [file dirname $argv0] .. .. test] +source $testdir/tester.tcl +set ::testprefix ota3 + + +# Run the OTA in file $ota on target database $target until completion. +# +proc run_ota {target ota} { + sqlite3ota ota $target $ota + while { [ota step]=="SQLITE_OK" } {} + ota close +} + +forcedelete test.db-oal ota.db + +#-------------------------------------------------------------------- +# Test that for an OTA to be applied, no corruption results if the +# affinities on the source and target table do not match. +# +do_execsql_test 1.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b TEXT, c REAL); + CREATE INDEX i1 ON x1(b, c); +} {} + +do_test 1.1 { + sqlite3 db2 ota.db + db2 eval { + CREATE TABLE data_x1(a, b, c, ota_control); + INSERT INTO data_x1 VALUES(1, '123', '123', 0); + INSERT INTO data_x1 VALUES(2, 123, 123, 0); + } + db2 close + run_ota test.db ota.db +} {SQLITE_DONE} + +do_execsql_test 1.2 { + PRAGMA integrity_check; +} {ok} + + + +finish_test diff --git a/manifest b/manifest index 35c6c48a27..16fda27393 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\supdate\sstatements\sto\ssqlite3ota.c. -D 2014-09-08T17:50:35.158 +C Add\sOP_Affinity\sopcodes\sto\sthe\sVMs\sgenerated\sby\ssqlite3_index_writer(). +D 2014-09-15T10:44:24.271 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,6 +124,7 @@ F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 F ext/ota/ota1.test fe0bb8acf0caef6c19937b84c6547b788342610d F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa +F ext/ota/ota3.test 9e3e42c6083e92909c408f1fe62aa07b0c9f905c F ext/ota/sqlite3ota.c 3697f6db2d51d5f7c0f0306fe94514785361e521 F ext/ota/sqlite3ota.h 545f0008b5f02f2595899cb9841caddada5c17c0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -295,7 +296,7 @@ F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h cdc8e421f85beb1ac9b4669ec5beadab6faa15e0 F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 -F src/vdbeblob.c b1b8b2cd86617db009f027f116b335f86e95b617 +F src/vdbeblob.c 7a28ade0cae13bd99f5c4c8f008f68e942c23115 F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62 F src/vdbesort.c 7c45bfcd823f30d172bbbc1b9f51ef4402fbfe8d F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 @@ -1198,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f988234ba54d7c667f7deef1d04beed4e7fe6182 -R c23b09bc3993f39690d272fa85636507 +P e109b27e4d66b83e1a804e7556d9c91aa37fea28 +R 75f9e5319b2343a328ed4364426b7add U dan -Z 28e28dd83293020abb0c0fb47ae5ab2c +Z c56c0b8cb355334f7ef68c91cc8c371e diff --git a/manifest.uuid b/manifest.uuid index 09501cef43..6154d2dd6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e109b27e4d66b83e1a804e7556d9c91aa37fea28 \ No newline at end of file +b9b38cb8e21826ddfb4c87a1f166ad33c3c7b6eb \ No newline at end of file diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 5e809fc78f..dd08b0b8aa 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -478,6 +478,7 @@ int sqlite3_index_writer( Vdbe *v = 0; int regRec; /* Register to assemble record in */ int *aiCol = 0; + const char *zAffinity = 0; /* Affinity string for the current index */ sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); @@ -499,6 +500,7 @@ int sqlite3_index_writer( goto index_writer_out; } pTab = pIdx->pTable; + zAffinity = sqlite3IndexAffinityStr(v, pIdx); /* Populate the two output variables, *pnCol and *pnAiCol. */ *pnCol = pIdx->nColumn; @@ -548,7 +550,7 @@ int sqlite3_index_writer( regRec = ++pParse->nMem; if( bDelete==0 ){ - sqlite3VdbeAddOp3(v, OP_MakeRecord, 1, pIdx->nColumn, regRec); + sqlite3VdbeAddOp4(v, OP_MakeRecord, 1, pIdx->nColumn, regRec, zAffinity, 0); /* If this is a UNIQUE index, check the constraint. */ if( pIdx->onError ){ @@ -561,6 +563,7 @@ int sqlite3_index_writer( sqlite3VdbeAddOp2(v, OP_IdxInsert, 0, regRec); }else{ /* Code the IdxDelete to remove the entry from the b-tree index. */ + sqlite3VdbeAddOp4(v, OP_Affinity, 0, pIdx->nColumn, 0, zAffinity, 0); sqlite3VdbeAddOp3(v, OP_IdxDelete, 0, 1, pIdx->nColumn); } sqlite3FinishCoding(pParse); From 13ce7931f70455546fda109d54cc3c5d94585d2d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 15 Sep 2014 12:18:29 +0000 Subject: [PATCH 012/116] Have the sqlite3_index_writer() VMs check that the final values of records inserted into indexes on rowid tables are integers. FossilOrigin-Name: cca376bff3510dc5e99fc5824862c3471ceced16 --- ext/ota/ota3.test | 25 +++++++++++++++++++++++++ ext/ota/sqlite3ota.h | 7 ++++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbeblob.c | 5 +++++ 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/ext/ota/ota3.test b/ext/ota/ota3.test index 7907974767..a4eee32271 100644 --- a/ext/ota/ota3.test +++ b/ext/ota/ota3.test @@ -49,6 +49,31 @@ do_execsql_test 1.2 { PRAGMA integrity_check; } {ok} +#-------------------------------------------------------------------- +# Test that NULL values may not be inserted into INTEGER PRIMARY KEY +# columns. +# +forcedelete ota.db +reset_db + +do_execsql_test 2.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b TEXT, c REAL); + CREATE INDEX i1 ON x1(b, c); +} {} + +do_test 2.1 { + sqlite3 db2 ota.db + db2 eval { + CREATE TABLE data_x1(a, b, c, ota_control); + INSERT INTO data_x1 VALUES(NULL, 'a', 'b', 0); + } + db2 close + list [catch { run_ota test.db ota.db } msg] $msg +} {1 {SQLITE_MISMATCH - datatype mismatch}} + +do_execsql_test 2.2 { + PRAGMA integrity_check; +} {ok} finish_test diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index 12832674c7..9473b359f2 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -98,7 +98,12 @@ ** update, the corresponding data_% table should contain a single record ** with the "ota_control" column set to contain integer value 0. The ** other columns should be set to the values that make up the new record -** to insert. +** to insert. +** +** If the target database table has an INTEGER PRIMARY KEY and there are +** one or more auxiliary indexes, it is not possible to insert a NULL value +** into the IPK column. Attempting to do so results in an SQLITE_MISMATCH +** error. ** ** For each row to DELETE from the target database as part of the OTA ** update, the corresponding data_% table should contain a single record diff --git a/manifest b/manifest index 16fda27393..344ec70e00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sOP_Affinity\sopcodes\sto\sthe\sVMs\sgenerated\sby\ssqlite3_index_writer(). -D 2014-09-15T10:44:24.271 +C Have\sthe\ssqlite3_index_writer()\sVMs\scheck\sthat\sthe\sfinal\svalues\sof\srecords\sinserted\sinto\sindexes\son\srowid\stables\sare\sintegers. +D 2014-09-15T12:18:29.913 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,9 +124,9 @@ F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 F ext/ota/ota1.test fe0bb8acf0caef6c19937b84c6547b788342610d F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa -F ext/ota/ota3.test 9e3e42c6083e92909c408f1fe62aa07b0c9f905c +F ext/ota/ota3.test 7179a90eb64cf8135a6885568653f158fb130872 F ext/ota/sqlite3ota.c 3697f6db2d51d5f7c0f0306fe94514785361e521 -F ext/ota/sqlite3ota.h 545f0008b5f02f2595899cb9841caddada5c17c0 +F ext/ota/sqlite3ota.h 39ce4dffbfcf4ade9e4526369fe2243709345c8e F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -296,7 +296,7 @@ F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h cdc8e421f85beb1ac9b4669ec5beadab6faa15e0 F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 -F src/vdbeblob.c 7a28ade0cae13bd99f5c4c8f008f68e942c23115 +F src/vdbeblob.c 7ddae314db27c39f654fd2af577c9afc33d089d9 F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62 F src/vdbesort.c 7c45bfcd823f30d172bbbc1b9f51ef4402fbfe8d F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e109b27e4d66b83e1a804e7556d9c91aa37fea28 -R 75f9e5319b2343a328ed4364426b7add +P b9b38cb8e21826ddfb4c87a1f166ad33c3c7b6eb +R 6282c8b629d24290f7b2f44befcb6a71 U dan -Z c56c0b8cb355334f7ef68c91cc8c371e +Z 32d433205c33319cd81cbc2b152b81af diff --git a/manifest.uuid b/manifest.uuid index 6154d2dd6b..8423c49a14 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9b38cb8e21826ddfb4c87a1f166ad33c3c7b6eb \ No newline at end of file +cca376bff3510dc5e99fc5824862c3471ceced16 \ No newline at end of file diff --git a/src/vdbeblob.c b/src/vdbeblob.c index dd08b0b8aa..09c519cb59 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -549,6 +549,11 @@ int sqlite3_index_writer( } regRec = ++pParse->nMem; + /* If this is a rowid table, check that the rowid field is an integer. */ + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_MustBeInt, pIdx->nColumn, 0); + } + if( bDelete==0 ){ sqlite3VdbeAddOp4(v, OP_MakeRecord, 1, pIdx->nColumn, regRec, zAffinity, 0); From 02cf6e168103f339c3a8bbca72c612f60f5b3b4b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 15 Sep 2014 14:54:07 +0000 Subject: [PATCH 013/116] Ensure the correct collation sequences are used when sorting data in sqlite3ota.c. FossilOrigin-Name: 473a72d7009a22ea514a98ee8869e7e7bca14cf5 --- ext/ota/sqlite3ota.c | 55 +++++++++++++++++++++---- manifest | 16 ++++---- manifest.uuid | 2 +- src/sqlite.h.in | 1 + src/vdbeblob.c | 96 +++++++++++++++++++++++++++++++------------- 5 files changed, 126 insertions(+), 44 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index fcf0c8b9fd..aca28e2e3b 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -401,11 +401,46 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ return p->rc; } +/* +** This function constructs and returns a pointer to a nul-terminated +** string containing some SQL clause or list based on one or more of the +** column names currently stored in the pIter->azTblCol[] array. +** +** If an OOM error is encountered, NULL is returned and an error code +** left in the OTA handle passed as the first argument. Otherwise, a pointer +** to the allocated string buffer is returned. It is the responsibility +** of the caller to eventually free this buffer using sqlite3_free(). +** +** The number of column names to include in the returned string is passed +** as the third argument. +** +** If arguments aiCol and azCollate are both NULL, then the returned string +** contains the first nCol column names as a comma-separated list. For +** example: +** +** "a", "b", "c" +** +** If argument aiCol is not NULL, it must point to an array containing nCol +** entries - the index of each column name to include in the comma-separated +** list. For example, if aiCol[] contains {2, 0, 1), then the returned +** string is changed to: +** +** "c", "a", "b" +** +** If azCollate is not NULL, it must also point to an array containing nCol +** entries - collation sequence names to associated with each element of +** the comma separated list. For example, ef azCollate[] contains +** {"BINARY", "NOCASE", "REVERSE"}, then the retuned string is: +** +** "c" COLLATE "BINARY", "a" COLLATE "NOCASE", "b" COLLATE "REVERSE" +** +*/ static char *otaObjIterGetCollist( - sqlite3ota *p, - OtaObjIter *pIter, - int nCol, - int *aiCol + sqlite3ota *p, /* OTA object */ + OtaObjIter *pIter, /* Object iterator for column names */ + int nCol, /* Number of column names */ + int *aiCol, /* Array of nCol column indexes */ + const char **azCollate /* Array of nCol collation sequence names */ ){ char *zList = 0; if( p->rc==SQLITE_OK ){ @@ -414,6 +449,9 @@ static char *otaObjIterGetCollist( for(i=0; iazTblCol[iCol]); + if( zList && azCollate ){ + zList = sqlite3_mprintf("%z COLLATE %Q", zList, azCollate[i]); + } zSep = ", "; if( zList==0 ){ p->rc = SQLITE_NOMEM; @@ -548,21 +586,22 @@ static int otaObjIterPrepareAll( if( zIdx ){ int *aiCol; /* Column map */ + const char **azColl; /* Collation sequences */ /* Create the index writers */ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_index_writer( - p->db, 0, zIdx, &pIter->pInsert, &aiCol, &pIter->nCol + p->db, 0, zIdx, &pIter->pInsert, &azColl, &aiCol, &pIter->nCol ); } if( p->rc==SQLITE_OK ){ p->rc = sqlite3_index_writer( - p->db, 1, zIdx, &pIter->pDelete, &aiCol, &pIter->nCol + p->db, 1, zIdx, &pIter->pDelete, &azColl, &aiCol, &pIter->nCol ); } /* Create the SELECT statement to read keys in sorted order */ - zCollist = otaObjIterGetCollist(p, pIter, pIter->nCol, aiCol); + zCollist = otaObjIterGetCollist(p, pIter, pIter->nCol, aiCol, azColl); if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, sqlite3_mprintf( @@ -582,7 +621,7 @@ static int otaObjIterPrepareAll( char *zWhere = otaObjIterGetWhere(p, pIter); char *zOldlist = otaObjIterGetOldlist(p, pIter, "old"); char *zNewlist = otaObjIterGetOldlist(p, pIter, "new"); - zCollist = otaObjIterGetCollist(p, pIter, pIter->nTblCol, 0); + zCollist = otaObjIterGetCollist(p, pIter, pIter->nTblCol, 0, 0); pIter->nCol = pIter->nTblCol; /* Create the SELECT statement to read keys from data_xxx */ diff --git a/manifest b/manifest index 344ec70e00..9496fcd9a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\ssqlite3_index_writer()\sVMs\scheck\sthat\sthe\sfinal\svalues\sof\srecords\sinserted\sinto\sindexes\son\srowid\stables\sare\sintegers. -D 2014-09-15T12:18:29.913 +C Ensure\sthe\scorrect\scollation\ssequences\sare\sused\swhen\ssorting\sdata\sin\ssqlite3ota.c. +D 2014-09-15T14:54:07.328 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,7 +125,7 @@ F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 F ext/ota/ota1.test fe0bb8acf0caef6c19937b84c6547b788342610d F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa F ext/ota/ota3.test 7179a90eb64cf8135a6885568653f158fb130872 -F ext/ota/sqlite3ota.c 3697f6db2d51d5f7c0f0306fe94514785361e521 +F ext/ota/sqlite3ota.c c29b3eb617f37d0e51744508b8f609014244727d F ext/ota/sqlite3ota.h 39ce4dffbfcf4ade9e4526369fe2243709345c8e F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -231,7 +231,7 @@ F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 89e569b263535662f54b537eb9118b2c554ae7aa F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 -F src/sqlite.h.in 706b420dc3390532435a3bd196360a940805f02f +F src/sqlite.h.in 578c42cb3899ee6ad93a0cccff4eb47e1e9a2c80 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 7c090825333d91ca392c2479a9e835e7b6a5eb12 @@ -296,7 +296,7 @@ F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h cdc8e421f85beb1ac9b4669ec5beadab6faa15e0 F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 -F src/vdbeblob.c 7ddae314db27c39f654fd2af577c9afc33d089d9 +F src/vdbeblob.c a8e2c3baa3e7081347c4677185a631bfc43de043 F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62 F src/vdbesort.c 7c45bfcd823f30d172bbbc1b9f51ef4402fbfe8d F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b9b38cb8e21826ddfb4c87a1f166ad33c3c7b6eb -R 6282c8b629d24290f7b2f44befcb6a71 +P cca376bff3510dc5e99fc5824862c3471ceced16 +R 286b5daf6c561394d2eea11438cdac48 U dan -Z 32d433205c33319cd81cbc2b152b81af +Z e26ad3cda7162cfc9e1d6f5aeb5cd971 diff --git a/manifest.uuid b/manifest.uuid index 8423c49a14..4e033cd4fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cca376bff3510dc5e99fc5824862c3471ceced16 \ No newline at end of file +473a72d7009a22ea514a98ee8869e7e7bca14cf5 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7011b67fd4..b543f1345d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7415,6 +7415,7 @@ int sqlite3_index_writer( int bDelete, /* Zero for insert, non-zero for delete */ const char *zIndex, /* Index to write to */ sqlite3_stmt**, /* OUT: New statement handle */ + const char ***pazColl, /* OUT: Collation sequences for each column */ int **paiCol, int *pnCol /* OUT: See above */ ); diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 09c519cb59..4ecd56522f 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -463,12 +463,77 @@ int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ return rc; } +static int indexWriterOutputVars( + sqlite3 *db, + Index *pIdx, + const char ***pazColl, /* OUT: Array of collation sequences */ + int **paiCol, /* OUT: Array of column indexes */ + int *pnCol /* OUT: Total columns in index keys */ +){ + Table *pTbl = pIdx->pTable; /* Table index is attached to */ + Index *pPk = 0; + int nByte = 0; /* Total bytes of space to allocate */ + int i; /* Iterator variable */ + + int *aiCol; + const char **azColl; + char *pCsr; + + if( !HasRowid(pTbl) ){ + pPk = sqlite3PrimaryKeyIndex(pTbl); + } + + for(i=0; inColumn; i++){ + const char *zColl = 0; + if( inKeyCol ){ + zColl = pIdx->azColl[i]; + }else if( pPk ){ + zColl = pPk->azColl[i-pIdx->nKeyCol]; + } + if( zColl==0 ) zColl = "BINARY"; + nByte += sqlite3Strlen30(zColl) + 1; + } + nByte += (pIdx->nColumn) * (sizeof(const char*) + sizeof(int)); + + /* Populate the output variables */ + *pazColl = azColl = (const char**)sqlite3DbMallocZero(db, nByte); + if( azColl==0 ) return SQLITE_NOMEM; + *paiCol = aiCol = (int*)&azColl[pIdx->nColumn]; + *pnCol = pIdx->nColumn; + pCsr = (char*)&aiCol[pIdx->nColumn]; + + for(i=0; inColumn; i++){ + const char *zColl = 0; + int nColl; + int iCol = pTbl->iPKey; + if( inKeyCol ){ + zColl = pIdx->azColl[i]; + iCol = pIdx->aiColumn[i]; + }else if( pPk ){ + zColl = pPk->azColl[i-pIdx->nKeyCol]; + iCol = pPk->aiColumn[i-pIdx->nKeyCol]; + } + if( zColl==0 ) zColl = "BINARY"; + + aiCol[i] = iCol; + azColl[i] = pCsr; + nColl = 1 + sqlite3Strlen30(zColl); + memcpy(pCsr, zColl, nColl); + pCsr += nColl; + } + + return SQLITE_OK; +} + + int sqlite3_index_writer( sqlite3 *db, int bDelete, const char *zIndex, sqlite3_stmt **ppStmt, - int **paiCol, int *pnCol + const char ***pazColl, /* OUT: Array of collation sequences */ + int **paiCol, /* OUT: Array of column indexes */ + int *pnCol /* OUT: Total columns in index keys */ ){ int rc = SQLITE_OK; Parse *pParse = 0; @@ -477,7 +542,6 @@ int sqlite3_index_writer( int i; /* Used to iterate through index columns */ Vdbe *v = 0; int regRec; /* Register to assemble record in */ - int *aiCol = 0; const char *zAffinity = 0; /* Affinity string for the current index */ sqlite3_mutex_enter(db->mutex); @@ -502,37 +566,15 @@ int sqlite3_index_writer( pTab = pIdx->pTable; zAffinity = sqlite3IndexAffinityStr(v, pIdx); - /* Populate the two output variables, *pnCol and *pnAiCol. */ - *pnCol = pIdx->nColumn; - *paiCol = aiCol = sqlite3DbMallocZero(db, sizeof(int) * pIdx->nColumn); - if( aiCol==0 ){ - rc = SQLITE_NOMEM; - goto index_writer_out; - } - for(i=0; inKeyCol; i++){ - aiCol[i] = pIdx->aiColumn[i]; - } - if( !HasRowid(pTab) ){ - Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); - assert( pIdx->nColumn==pIdx->nKeyCol+pPk->nKeyCol ); - if( pPk==pIdx ){ - rc = SQLITE_ERROR; - goto index_writer_out; - } - for(i=0; inKeyCol; i++){ - aiCol[pIdx->nKeyCol+i] = pPk->aiColumn[i]; - } - }else{ - assert( pIdx->nColumn==pIdx->nKeyCol+1 ); - aiCol[i] = pTab->iPKey; - } + rc = indexWriterOutputVars(db, pIdx, pazColl, paiCol, pnCol); + if( rc!=SQLITE_OK ) goto index_writer_out; /* Add an OP_Noop to the VDBE program. Then store a pointer to the ** output array *paiCol as its P4 value. This is so that the array ** is automatically deleted when the user finalizes the statement. The ** OP_Noop serves no other purpose. */ sqlite3VdbeAddOp0(v, OP_Noop); - sqlite3VdbeChangeP4(v, -1, (const char*)aiCol, P4_INTARRAY); + sqlite3VdbeChangeP4(v, -1, (const char*)(*pazColl), P4_INTARRAY); sqlite3BeginWriteOperation(pParse, 0, 0); From ee8d0b4111b47bd91e9d3280e8540d3a560b0cc7 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 15 Sep 2014 15:22:32 +0000 Subject: [PATCH 014/116] Have sqlite3ota.c use grave accents instead of double-quotes to enclose identifiers in generated SQL. To avoid having the SQL engine substitute a literal string if a column reference cannot be resolved. FossilOrigin-Name: 79f2418429aa05c56069c56d51b4d72f662a6970 --- ext/ota/ota3.test | 24 ++++++++++++++++++++++++ ext/ota/sqlite3ota.c | 8 ++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/ext/ota/ota3.test b/ext/ota/ota3.test index a4eee32271..dcf8f817aa 100644 --- a/ext/ota/ota3.test +++ b/ext/ota/ota3.test @@ -75,5 +75,29 @@ do_execsql_test 2.2 { PRAGMA integrity_check; } {ok} +#-------------------------------------------------------------------- +# Test that missing columns are detected. +# +forcedelete ota.db +reset_db + +do_execsql_test 2.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON x1(b, c); +} {} + +do_test 2.1 { + sqlite3 db2 ota.db + db2 eval { + CREATE TABLE data_x1(a, b, ota_control); + INSERT INTO data_x1 VALUES(1, 'a', 0); + } + db2 close + list [catch { run_ota test.db ota.db } msg] $msg +} {1 {SQLITE_ERROR - no such column: c}} + +do_execsql_test 2.2 { + PRAGMA integrity_check; +} {ok} finish_test diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index aca28e2e3b..999bc00cf7 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -299,7 +299,7 @@ static int otaObjIterFirst(sqlite3ota *p, OtaObjIter *pIter){ ** string in the argument buffer, suitable for use as an SQL identifier. ** For example: ** -** [quick "brown" fox] -> ["quick ""brown"" fox"] +** [quick `brown` fox] -> [`quick ``brown`` fox`] ** ** Assuming the allocation is successful, a pointer to the new buffer is ** returned. It is the responsibility of the caller to free it using @@ -312,12 +312,12 @@ static char *otaQuoteName(const char *zName){ if( zRet ){ int i; char *p = zRet; - *p++ = '"'; + *p++ = '`'; for(i=0; i Date: Mon, 15 Sep 2014 19:34:04 +0000 Subject: [PATCH 015/116] Remove the experimental sqlite3_transaction_save() and restore() APIs. FossilOrigin-Name: 48d201cd8b68c0377cf8a2cc6439b893f9462fe2 --- ext/ota/ota1.test | 15 +++ manifest | 30 +++--- manifest.uuid | 2 +- src/main.c | 12 --- src/pager.c | 35 ------- src/pager.h | 3 - src/pragma.c | 1 - src/prepare.c | 1 + src/sqlite.h.in | 19 ---- src/test1.c | 66 ------------ src/wal.c | 259 +++++++++------------------------------------- src/wal.h | 3 - 12 files changed, 80 insertions(+), 366 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index 917d1d2c9e..60cc4fc30e 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -219,6 +219,12 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota} { CREATE INDEX i2 ON t1(c, b); CREATE INDEX i3 ON t1(c, b, c); } + 4 { + CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } } { reset_db execsql $schema @@ -261,6 +267,15 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota} { CREATE INDEX i5 ON t1(c); CREATE INDEX i6 ON t1(c, b); } + 3 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID; + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } } { reset_db execsql $schema diff --git a/manifest b/manifest index e463fef402..7898dc2ecf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\sfixes\sinto\sthis\sbranch. -D 2014-09-15T16:57:10.623 +C Remove\sthe\sexperimental\ssqlite3_transaction_save()\sand\srestore()\sAPIs. +D 2014-09-15T19:34:04.372 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,7 +122,7 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 -F ext/ota/ota1.test fe0bb8acf0caef6c19937b84c6547b788342610d +F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 F ext/ota/sqlite3ota.c 668ed08dd81ff8ae1e8524b2d4bf0f2609cbf907 @@ -199,7 +199,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 6b9fd1867a5575db2119f850302a883e7684009c +F src/main.c d15621461fb0c52675eba2b650492ed1beef69ab F src/malloc.c cc015821ba267ad5c91dc8761d0498a3fc3ce6ce F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -220,21 +220,21 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c addd023b26c623fec4dedc110fc4370a65b4768c F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c b7c625fc92e86fea1971b8a630009eab3ff18997 -F src/pager.h 6a08df06b7edc3684375c0fab40602c695a044f2 +F src/pager.c c1cdf5509386b5c3695082655ce6de6431d920d6 +F src/pager.h 5c13927809e1c35d85e82e14342d817df3019e07 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa -F src/pragma.c 5dba6b386cbfe0b094350b7d6c99fd410958f22d -F src/prepare.c 905c3c601ccadd22bb70b63cd48392f7126c9807 +F src/pragma.c c401b5ddbb5c882e2b9d9e16fad2abd8b4a86564 +F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 0cd6706fd52ae5db229e9041094db6ec27195335 F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d -F src/sqlite.h.in 5bcaca9d5a8403fda0c8677de4c527907b317f1d +F src/sqlite.h.in a98b0429855f023074f44a63dba23d11469ebc6d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 3210f8bd040d1c6d8b1616325b15dd3ff749e48f @@ -242,7 +242,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 218ae2ba022881846741dfc8351aefdf129e0377 F src/tclsqlite.c ac7d1672f69c9d69defeb022f656d04f5cefd198 -F src/test1.c 75655557ebc7138e1d745313635b6b100c90c151 +F src/test1.c 44562bf344a6b12a273f46fd906631ef301a6416 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -304,8 +304,8 @@ F src/vdbemem.c dc36ea9fe26c25550c50085f388167086ef7d73a F src/vdbesort.c a7a40ceca6325b853040ffcc363dcd49a45f201b F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f -F src/wal.c 6f5ff51117293e7b2c75ad21834f51115e59ea96 -F src/wal.h 237bc4484f7c289f094ecb6efb2b6c02005484e1 +F src/wal.c 3c56c85d80a17b51cd8cb0bed5c1ecc4f0771012 +F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 839b5e1db2507e221ad1c308f148a8519ed750be F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 55b8011d5b455927f5b92a3cb911fd909fb0edab 4060efb646c873c4abde7ab9ddf330489a44f274 -R a8de04acab3d1132246bf5d89e41e027 +P 5efafef51d146bcba3adc425561bfa1ac083c0a7 +R 684c9325f89b1d695bf7f6172a6df69f U dan -Z 1d00f387d17c6184cf9c7c454dc66aca +Z 4568916327f8707f39b56bf66a2404a6 diff --git a/manifest.uuid b/manifest.uuid index 607c43815f..282b6874b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5efafef51d146bcba3adc425561bfa1ac083c0a7 \ No newline at end of file +48d201cd8b68c0377cf8a2cc6439b893f9462fe2 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 6da74cbf2a..231890de4b 100644 --- a/src/main.c +++ b/src/main.c @@ -3473,15 +3473,3 @@ int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ Btree *pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } - -int sqlite3_transaction_save(sqlite3 *db, void **ppState, int *pnState){ - Pager *pPager = sqlite3BtreePager(db->aDb[0].pBt); - return sqlite3PagerSaveState(pPager, ppState, pnState); -} - -int sqlite3_transaction_restore(sqlite3 *db, const void *pState, int nState){ - Pager *pPager = sqlite3BtreePager(db->aDb[0].pBt); - return sqlite3PagerRestoreState(pPager, pState, nState); -} - - diff --git a/src/pager.c b/src/pager.c index 3ab29f21b6..7de6a65550 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7237,41 +7237,6 @@ int sqlite3PagerCloseWal(Pager *pPager){ return rc; } -int sqlite3PagerSaveState(Pager *pPager, void **ppState, int *pnState){ - int rc = SQLITE_OK; - *ppState = 0; - *pnState = 0; - if( pPager->pWal==0 || pPager->eStatepPCache); - rc = sqlite3WalFrames(pPager->pWal, - pPager->pageSize, pList, 0, 0, pPager->walSyncFlags - ); - if( rc==SQLITE_OK ){ - rc = sqlite3WalSaveState(pPager->pWal, ppState, pnState); - } - } - return rc; -} - -int sqlite3PagerRestoreState(Pager *pPager, const void *pState, int nState){ - int rc = SQLITE_OK; - if( pPager->pWal==0 - || pPager->eStatepPCache) - ){ - rc = SQLITE_ERROR; - }else{ - sqlite3PcacheTruncate(pPager->pPCache, 1); - rc = sqlite3WalRestoreState(pPager->pWal, pState, nState); - pPager->eState = PAGER_WRITER_CACHEMOD; - } - - return rc; -} - #endif /* !SQLITE_OMIT_WAL */ #ifdef SQLITE_ENABLE_ZIPVFS diff --git a/src/pager.h b/src/pager.h index fd2624c3fa..57948079d2 100644 --- a/src/pager.h +++ b/src/pager.h @@ -207,9 +207,6 @@ void *sqlite3PagerCodec(DbPage *); # define enable_simulated_io_errors() #endif -int sqlite3PagerSaveState(Pager *pPager, void **ppState, int *pnState); -int sqlite3PagerRestoreState(Pager *pPager, const void *pState, int nState); - int sqlite3PagerSetOtaMode(Pager *pPager, int bOta); #endif /* _PAGER_H_ */ diff --git a/src/pragma.c b/src/pragma.c index 734f4aa98e..9785a33c26 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -482,7 +482,6 @@ static const struct sPragmaNames { #endif }; /* Number of pragmas: 59 on by default, 72 total. */ -/* Number of pragmas: 58 on by default, 71 total. */ /* End of the automatically generated pragma table. ***************************************************************************/ diff --git a/src/prepare.c b/src/prepare.c index 2e2be0ec47..a05e619f3e 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -797,6 +797,7 @@ int sqlite3_prepare_v2( return rc; } + #ifndef SQLITE_OMIT_UTF16 /* ** Compile the UTF-16 encoded SQL statement zSql into a statement handle. diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2222451859..2ad571ed0b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7464,25 +7464,6 @@ int sqlite3_index_writer( int **paiCol, int *pnCol /* OUT: See above */ ); -/* -** This function is used to save the state of an ongoing WAL mode write -** transaction on the "main" database of the supplied database handle. -** -** If successful, SQLITE_OK is returned and output variable (*ppState) -** is set to point to a buffer containing the transaction state data. -** (*pnState) is set to the size of that buffer in bytes. Otherwise, if -** an error occurs, an SQLite error code is returned and both output -** variables are zeroed. -** -** A transaction state may be saved if: -** -** * the transaction does not contain any schema modifications. -** * there are no open sub-transactions. -*/ -int sqlite3_transaction_save(sqlite3 *db, void **ppState, int *pnState); - -int sqlite3_transaction_restore(sqlite3 *db, const void *pState, int nState); - /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/test1.c b/src/test1.c index abeb15e8fc..14785c35b3 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6497,70 +6497,6 @@ static int sorter_test_sort4_helper( } -/* -** tclcmd: sqlite3_transaction_save DB -*/ -static int testTransactionSave( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - void *pState; - int nState; - sqlite3 *db; - int rc; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - - rc = sqlite3_transaction_save(db, &pState, &nState); - if( rc==SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pState, nState)); - }else{ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); - return TCL_ERROR; - } - - sqlite3_free(pState); - return TCL_OK; -} - -/* -** tclcmd: sqlite3_transaction_restore DB BLOB -*/ -static int testTransactionRestore( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - void *pState; - int nState; - sqlite3 *db; - int rc; - - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB BLOB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - pState = (void*)Tcl_GetByteArrayFromObj(objv[2], &nState); - - rc = sqlite3_transaction_restore(db, pState, nState); - if( rc==SQLITE_OK ){ - Tcl_ResetResult(interp); - }else{ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); - return TCL_ERROR; - } - - return TCL_OK; -} - #ifdef SQLITE_USER_AUTHENTICATION #include "sqlite3userauth.h" /* @@ -6924,8 +6860,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "load_static_extension", tclLoadStaticExtensionCmd }, { "sorter_test_fakeheap", sorter_test_fakeheap }, { "sorter_test_sort4_helper", sorter_test_sort4_helper }, - { "sqlite3_transaction_save", testTransactionSave }, - { "sqlite3_transaction_restore", testTransactionRestore }, #ifdef SQLITE_USER_AUTHENTICATION { "sqlite3_user_authenticate", test_user_authenticate, 0 }, { "sqlite3_user_add", test_user_add, 0 }, diff --git a/src/wal.c b/src/wal.c index 80539807b1..aa797dfb53 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1046,62 +1046,6 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ return rc; } -static int walFileReadHdr(Wal *pWal, int *pbValid){ - u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ - int rc; /* Return code */ - u32 magic; /* Magic value read from WAL header */ - int szPage; /* Page size according to the log */ - u32 version; /* Magic value read from WAL header */ - - *pbValid = 0; - - /* Read in the WAL header. */ - rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - - /* If the database page size is not a power of two, or is greater than - ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid - ** data. Similarly, if the 'magic' value is invalid, ignore the whole - ** WAL file. - */ - magic = sqlite3Get4byte(&aBuf[0]); - szPage = sqlite3Get4byte(&aBuf[8]); - if( (magic&0xFFFFFFFE)!=WAL_MAGIC - || szPage&(szPage-1) - || szPage>SQLITE_MAX_PAGE_SIZE - || szPage<512 - ){ - return SQLITE_OK; - } - - pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); - pWal->szPage = szPage; - pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); - memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); - - /* Verify that the WAL header checksum is correct */ - walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, - aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum - ); - if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) - || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) - ){ - return SQLITE_OK; - } - - /* Verify that the version number on the WAL format is one that - ** are able to understand */ - version = sqlite3Get4byte(&aBuf[4]); - if( version!=WAL_MAX_VERSION ){ - return SQLITE_CANTOPEN_BKPT; - } - - *pbValid = 1; - return SQLITE_OK; -} - /* ** Recover the wal-index by reading the write-ahead log file. @@ -1146,18 +1090,59 @@ static int walIndexRecover(Wal *pWal){ } if( nSize>WAL_HDRSIZE ){ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ int szFrame; /* Number of bytes in buffer aFrame[] */ u8 *aData; /* Pointer to data part of aFrame buffer */ int iFrame; /* Index of last frame read */ i64 iOffset; /* Next offset to read from log file */ int szPage; /* Page size according to the log */ + u32 magic; /* Magic value read from WAL header */ + u32 version; /* Magic value read from WAL header */ int isValid; /* True if this frame is valid */ - rc = walFileReadHdr(pWal, &isValid); - if( rc!=SQLITE_OK ) goto recovery_error; - if( isValid==0 ) goto finished; - szPage = pWal->szPage; + /* Read in the WAL header. */ + rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + goto recovery_error; + } + + /* If the database page size is not a power of two, or is greater than + ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid + ** data. Similarly, if the 'magic' value is invalid, ignore the whole + ** WAL file. + */ + magic = sqlite3Get4byte(&aBuf[0]); + szPage = sqlite3Get4byte(&aBuf[8]); + if( (magic&0xFFFFFFFE)!=WAL_MAGIC + || szPage&(szPage-1) + || szPage>SQLITE_MAX_PAGE_SIZE + || szPage<512 + ){ + goto finished; + } + pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); + pWal->szPage = szPage; + pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); + memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); + + /* Verify that the WAL header checksum is correct */ + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum + ); + if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) + ){ + goto finished; + } + + /* Verify that the version number on the WAL format is one that + ** are able to understand */ + version = sqlite3Get4byte(&aBuf[4]); + if( version!=WAL_MAX_VERSION ){ + rc = SQLITE_CANTOPEN_BKPT; + goto finished; + } /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; @@ -1833,6 +1818,9 @@ static void walLimitSize(Wal *pWal, i64 nMax){ /* ** Close a connection to a log file. +** +** If parameter zBuf is not NULL, attempt to obtain an exclusive lock +** and run a checkpoint. */ int sqlite3WalClose( Wal *pWal, /* Wal to close */ @@ -3096,157 +3084,6 @@ int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } -/* -** Save current transaction state. -** -** The transaction state consists of a series of 32-bit big-endian integers: -** -** * initial number of frames in WAL file. -** * initial checksum values (2 integers). -** * current number of frames. -** * current checksum values (2 integers). -*/ -int sqlite3WalSaveState(Wal *pWal, void **ppState, int *pnState){ - int rc = SQLITE_OK; - - *ppState = 0; - *pnState = 0; - if( pWal->writeLock==0 ){ - /* Must be in a write transaction to call this function. */ - rc = SQLITE_ERROR; - }else{ - WalIndexHdr *pOrig = (WalIndexHdr*)walIndexHdr(pWal); - int nBuf = 6 * 4; /* Bytes of space to allocate */ - u8 *aBuf; - - aBuf = sqlite3_malloc(nBuf); - if( aBuf==0 ){ - rc = SQLITE_NOMEM; - }else{ - sqlite3Put4byte(&aBuf[0], pOrig->mxFrame); - sqlite3Put4byte(&aBuf[4], pOrig->aFrameCksum[0]); - sqlite3Put4byte(&aBuf[8], pOrig->aFrameCksum[1]); - sqlite3Put4byte(&aBuf[12], pWal->hdr.mxFrame); - sqlite3Put4byte(&aBuf[16], pWal->hdr.aFrameCksum[0]); - sqlite3Put4byte(&aBuf[20], pWal->hdr.aFrameCksum[1]); - *ppState = (void*)aBuf; - *pnState = nBuf; - } - } - - return rc; -} - -static int walUndoNoop(void *pUndoCtx, Pgno pgno){ - UNUSED_PARAMETER(pUndoCtx); - UNUSED_PARAMETER(pgno); - return SQLITE_OK; -} - -/* -** If possible, restore the state of the curent transaction to that -** described by the second and third arguments. -*/ -int sqlite3WalRestoreState(Wal *pWal, const void *pState, int nState){ - int rc = SQLITE_OK; - - if( pWal->writeLock==0 ){ - /* Must have opened a write transaction to call this */ - rc = SQLITE_ERROR; - }else{ - u8 *aBuf = (u8*)pState; - int szFrame; /* Size of each frame in WAL file */ - u8 *aFrame = 0; /* Buffer to read data into */ - u8 *aData; /* Data part of aFrame[] buffer */ - u32 mxFrame; /* Maximum frame following restoration */ - int i; /* Iterator variable */ - - WalIndexHdr *pOrig = (WalIndexHdr*)walIndexHdr(pWal); - - /* Check that no dirty pages have been written to the WAL file since - ** the current transaction was opened. */ - if( pOrig->mxFrame!=pWal->hdr.mxFrame - || pOrig->aFrameCksum[0]!=pWal->hdr.aFrameCksum[0] - || pOrig->aFrameCksum[1]!=pWal->hdr.aFrameCksum[1] - ){ - rc = SQLITE_ERROR; - } - - /* Check that the WAL file is in the same state that it was when the - ** transaction was saved. If not, return SQLITE_MISMATCH - cannot - ** resume this transaction */ - if( rc==SQLITE_OK && ( - pWal->hdr.mxFrame!=sqlite3Get4byte(&aBuf[0]) - || pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[4]) - || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[8]) - )){ - rc = SQLITE_MISMATCH; - } - - if( rc==SQLITE_OK && pWal->readLock==0 ){ - int cnt = 0; - walUnlockShared(pWal, WAL_READ_LOCK(0)); - pWal->readLock = -1; - do{ - int notUsed; - rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); - }while( rc==WAL_RETRY ); - - if( rc==SQLITE_OK ){ - int bValid; - rc = walFileReadHdr(pWal, &bValid); - if( rc==SQLITE_OK && bValid==0 ) rc = SQLITE_MISMATCH; - pWal->hdr.szPage = (u16)((pWal->szPage&0xff00) | (pWal->szPage>>16)); - } - } - - /* Malloc a buffer to read frames into. */ - if( rc==SQLITE_OK ){ - szFrame = pWal->szPage + WAL_FRAME_HDRSIZE; - aFrame = (u8*)sqlite3_malloc(szFrame); - if( !aFrame ){ - rc = SQLITE_NOMEM; - }else{ - aData = &aFrame[WAL_FRAME_HDRSIZE]; - } - } - - mxFrame = sqlite3Get4byte(&aBuf[12]); - for(i=pWal->hdr.mxFrame+1; rc==SQLITE_OK && i<=mxFrame; i++){ - sqlite3_int64 iOff = walFrameOffset(i, pWal->szPage); - rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOff); - if( rc==SQLITE_OK ){ - u32 iPg; - u32 dummy; - if( 0==walDecodeFrame(pWal, &iPg, &dummy, aData, aFrame) ){ - rc = SQLITE_MISMATCH; - }else{ - rc = walIndexAppend(pWal, i, iPg); - if( iPg>pWal->hdr.nPage ) pWal->hdr.nPage = iPg; - } - pWal->hdr.mxFrame = i; - } - } - sqlite3_free(aFrame); - - if( rc==SQLITE_OK ){ - assert( pWal->hdr.mxFrame==mxFrame ); - if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[16]) - || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[20]) - ){ - rc = SQLITE_MISMATCH; - } - } - - - if( rc!=SQLITE_OK ){ - sqlite3WalUndo(pWal, walUndoNoop, 0); - } - } - - return rc; -} - #ifdef SQLITE_ENABLE_ZIPVFS /* ** If the argument is not NULL, it points to a Wal object that holds a diff --git a/src/wal.h b/src/wal.h index 08006fafe9..092546354b 100644 --- a/src/wal.h +++ b/src/wal.h @@ -126,9 +126,6 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); -int sqlite3WalSaveState(Wal *pWal, void **ppState, int *pnState); -int sqlite3WalRestoreState(Wal *pWal, const void *pState, int nState); - #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). From 1b95de09bced91297927bc16c399b05c4dea8645 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 16 Sep 2014 20:02:41 +0000 Subject: [PATCH 016/116] Clarify the effects of the pager_ota_mode pragma. Add tests and fixes for the same. FossilOrigin-Name: decaccc37cbdcd2a663233469efdf4982a810513 --- ext/ota/ota4.test | 121 ++++++++++++++++++++++++++++++++++++++++++++++ manifest | 21 ++++---- manifest.uuid | 2 +- src/pager.c | 36 +++++++++++++- src/pager.h | 1 + src/pragma.c | 26 +++++++++- src/wal.c | 22 ++++++++- src/wal.h | 2 + 8 files changed, 216 insertions(+), 15 deletions(-) create mode 100644 ext/ota/ota4.test diff --git a/ext/ota/ota4.test b/ext/ota/ota4.test new file mode 100644 index 0000000000..d0c9dfae40 --- /dev/null +++ b/ext/ota/ota4.test @@ -0,0 +1,121 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test some properties of the pager_ota_mode pragma. +# + +set testdir [file join [file dirname $argv0] .. .. test] +source $testdir/tester.tcl +set ::testprefix ota4 + +# 1. Cannot set the pager_ota_mode flag on a WAL mode database. +# +# 2. Or if there is an open read transaction. +# +# 3. Cannot start a transaction with pager_ota_mode set if there +# is a WAL file in the file-system. +# +# 4. Or if the wal-mode flag is set in the database file header. +# +# 5. Cannot open a transaction with pager_ota_mode set if the database +# file has been modified by a rollback mode client since the *-oal +# file was started. +# + +do_execsql_test 1.1 { + PRAGMA journal_mode = wal; + SELECT * FROM sqlite_master; +} {wal} +do_catchsql_test 1.2 { + PRAGMA pager_ota_mode = 1 +} {1 {cannot set pager_ota_mode in wal mode}} + + +do_execsql_test 2.1 { + PRAGMA journal_mode = delete; + BEGIN; + SELECT * FROM sqlite_master; +} {delete} +do_catchsql_test 2.2 { + PRAGMA pager_ota_mode = 1 +} {1 {cannot set pager_ota_mode with open transaction}} +do_execsql_test 2.3 { + COMMIT; +} {} + + +do_execsql_test 3.1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} {wal} +do_test 3.2 { + forcecopy test.db-wal test.db-bak + execsql { + PRAGMA journal_mode = delete; + PRAGMA pager_ota_mode = 1; + } + forcecopy test.db-bak test.db-wal + catchsql { + SELECT * FROM sqlite_master + } +} {1 {unable to open database file}} + +do_test 4.1 { + db close + forcedelete test.db-wal test.db-oal + sqlite3 db test.db + execsql { + PRAGMA journal_mode = wal; + PRAGMA pager_ota_mode = 1; + } + catchsql { + SELECT * FROM sqlite_master; + } +} {1 {unable to open database file}} + +do_test 5.1 { + forcedelete test.db-oal + reset_db + execsql { + PRAGMA journal_mode = delete; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } + execsql { + PRAGMA pager_ota_mode = 1; + INSERT INTO t1 VALUES(3, 4); + } + db close + sqlite3 db test.db + execsql { + SELECT * FROM t1; + } +} {1 2} +do_execsql_test 5.2 { + PRAGMA pager_ota_mode = 1; + SELECT * FROM t1; + INSERT INTO t1 VALUES(5, 6); +} {1 2 3 4} +do_test 5.3 { + db close + sqlite3 db test.db + execsql { + INSERT INTO t1 VALUES(7, 8); + SELECT * FROM t1; + } +} {1 2 7 8} +do_catchsql_test 5.4 { + PRAGMA pager_ota_mode = 1; + SELECT * FROM t1; +} {1 {database is locked}} + +finish_test diff --git a/manifest b/manifest index 7898dc2ecf..bcb8678d88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sexperimental\ssqlite3_transaction_save()\sand\srestore()\sAPIs. -D 2014-09-15T19:34:04.372 +C Clarify\sthe\seffects\sof\sthe\spager_ota_mode\spragma.\sAdd\stests\sand\sfixes\sfor\sthe\ssame. +D 2014-09-16T20:02:41.756 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,6 +125,7 @@ F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 +F ext/ota/ota4.test ec01b0d69ad2989559a65fde74560c1cfcca8202 F ext/ota/sqlite3ota.c 668ed08dd81ff8ae1e8524b2d4bf0f2609cbf907 F ext/ota/sqlite3ota.h 39ce4dffbfcf4ade9e4526369fe2243709345c8e F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -220,13 +221,13 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c addd023b26c623fec4dedc110fc4370a65b4768c F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c c1cdf5509386b5c3695082655ce6de6431d920d6 -F src/pager.h 5c13927809e1c35d85e82e14342d817df3019e07 +F src/pager.c 2dafc02d49457084c92472f934fe26a75e6d08f5 +F src/pager.h b62e645e8a19e4f0181253d1663a09f2793d8c94 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa -F src/pragma.c c401b5ddbb5c882e2b9d9e16fad2abd8b4a86564 +F src/pragma.c 5b255c09d6e38a37ec07830b92acceec5cab8c85 F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -304,8 +305,8 @@ F src/vdbemem.c dc36ea9fe26c25550c50085f388167086ef7d73a F src/vdbesort.c a7a40ceca6325b853040ffcc363dcd49a45f201b F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f -F src/wal.c 3c56c85d80a17b51cd8cb0bed5c1ecc4f0771012 -F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 +F src/wal.c 8bd0ced6cf1d3389fd6a73b4f12a1e2bf926e75a +F src/wal.h e25f9d383ffb07986ba20b78dbde2c1d0cb36ab6 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 839b5e1db2507e221ad1c308f148a8519ed750be F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c @@ -1204,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5efafef51d146bcba3adc425561bfa1ac083c0a7 -R 684c9325f89b1d695bf7f6172a6df69f +P 48d201cd8b68c0377cf8a2cc6439b893f9462fe2 +R de185427497b23c6bf5f7a43a97cdd26 U dan -Z 4568916327f8707f39b56bf66a2404a6 +Z de73fdb4c4401b74adf143cdfee4f7e2 diff --git a/manifest.uuid b/manifest.uuid index 282b6874b8..fa8e1e694f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -48d201cd8b68c0377cf8a2cc6439b893f9462fe2 \ No newline at end of file +decaccc37cbdcd2a663233469efdf4982a810513 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 7de6a65550..967f776f09 100644 --- a/src/pager.c +++ b/src/pager.c @@ -615,6 +615,15 @@ struct PagerSavepoint { ** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode ** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX ** sub-codes. +** +** otaMode +** This variable is normally 0. It is set to 1 by the PagerSetOtaMode() +** function - as a result of a "PRAGMA pager_ota_mode=1" command. Once +** the *-oal file has been opened and it has been determined that the +** database file has not been modified since it was created, this variable +** is set to 2. +** +** */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ @@ -630,7 +639,7 @@ struct Pager { u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ - u8 otaMode; /* True if in ota_mode */ + u8 otaMode; /* Non-zero if in ota_mode */ /************************************************************************** ** The following block contains those class members that change during @@ -5183,6 +5192,15 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( pagerUseWal(pPager) ){ assert( rc==SQLITE_OK ); rc = pagerBeginReadTransaction(pPager); + if( rc==SQLITE_OK && pPager->otaMode==1 ){ + rc = sqlite3WalCheckSalt(pPager->pWal, pPager->fd); + if( rc!=SQLITE_OK ){ + sqlite3WalClose(pPager->pWal, 0, 0, 0); + pPager->pWal = 0; + }else{ + pPager->otaMode = 2; + } + } } if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ @@ -7237,6 +7255,20 @@ int sqlite3PagerCloseWal(Pager *pPager){ return rc; } +/* +** This function is called by the wal.c module to obtain the 8 bytes of +** "salt" written into the wal file header. In OTA mode, this is a copy +** of bytes 24-31 of the database file. In non-OTA mode, it is 8 bytes +** of pseudo-random data. +*/ +void sqlite3PagerWalSalt(Pager *pPager, u32 *aSalt){ + if( pPager->otaMode ){ + memcpy(aSalt, pPager->dbFileVers, 8); + }else{ + sqlite3_randomness(8, aSalt); + } +} + #endif /* !SQLITE_OMIT_WAL */ #ifdef SQLITE_ENABLE_ZIPVFS @@ -7260,7 +7292,7 @@ int sqlite3PagerSetOtaMode(Pager *pPager, int bOta){ if( pPager->pWal || pPager->eState!=PAGER_OPEN ){ return SQLITE_ERROR; } - pPager->otaMode = (u8)bOta; + pPager->otaMode = 1; return SQLITE_OK; } diff --git a/src/pager.h b/src/pager.h index 57948079d2..0e928fe64c 100644 --- a/src/pager.h +++ b/src/pager.h @@ -208,5 +208,6 @@ void *sqlite3PagerCodec(DbPage *); #endif int sqlite3PagerSetOtaMode(Pager *pPager, int bOta); +void sqlite3PagerWalSalt(Pager *pPager, u32 *aSalt); #endif /* _PAGER_H_ */ diff --git a/src/pragma.c b/src/pragma.c index 9785a33c26..ee99002274 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -877,13 +877,37 @@ void sqlite3Pragma( /* ** PRAGMA [database.]pager_ota_mode=[01] + ** + ** This pragma sets a flag on the pager associated with the main database + ** only. The flag can only be set when there is no open transaction and + ** the pager does not already have an open WAL file. + ** + ** Once the flag has been set, it is not possible to open a regular WAL + ** file. If, when the next read-transaction is opened, a *-wal file is + ** found or the database header flags indicate that it is a wal-mode + ** database, SQLITE_CANTOPEN is returned. + ** + ** Otherwise, if no WAL file or flags are found, the pager opens the *-oal + ** file and uses it as a write-ahead-log with the *-shm data stored in + ** heap-memory. If the *-oal file already exists but the database file has + ** been modified since it was created, an SQLITE_BUSY_SNAPSHOT error is + ** returned and the read-transaction cannot be opened. + ** + ** Other clients see a rollback-mode database on which the pager_ota_mode + ** client is holding a SHARED lock. */ case PragTyp_PAGER_OTA_MODE: { Btree *pBt = pDb->pBt; assert( pBt!=0 ); if( zRight ){ int iArg = !!sqlite3Atoi(zRight); - rc = sqlite3PagerSetOtaMode(sqlite3BtreePager(pBt), iArg); + if( sqlite3BtreeIsInReadTrans(pBt) ){ + sqlite3ErrorMsg(pParse, + "cannot set pager_ota_mode with open transaction" + ); + }else if( sqlite3PagerSetOtaMode(sqlite3BtreePager(pBt), iArg) ){ + sqlite3ErrorMsg(pParse, "cannot set pager_ota_mode in wal mode"); + } } break; } diff --git a/src/wal.c b/src/wal.c index aa797dfb53..c7ad2bcb2c 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2769,7 +2769,9 @@ int sqlite3WalFrames( sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); sqlite3Put4byte(&aWalHdr[8], szPage); sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); - if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt); + if( pWal->nCkpt==0 ){ + sqlite3PagerWalSalt(pList->pPager, pWal->hdr.aSalt); + } memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); sqlite3Put4byte(&aWalHdr[24], aCksum[0]); @@ -3084,6 +3086,24 @@ int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } +/* +** Unless the wal file is empty, check that the 8 bytes of salt stored in +** the wal header are identical to those in the buffer indicated by the +** second argument. If they are not, return SQLITE_BUSY_SNAPSHOT. Otherwise, +** if the buffers match or the WAL file is empty, return SQLITE_OK. +*/ +int sqlite3WalCheckSalt(Wal *pWal, sqlite3_file *pFd){ + int rc = SQLITE_OK; + if( pWal->hdr.mxFrame>0 ){ + u8 aData[16]; + rc = sqlite3OsRead(pFd, aData, sizeof(aData), 24); + if( rc==SQLITE_OK && memcmp(pWal->hdr.aSalt, aData, 8) ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + } + return rc; +} + #ifdef SQLITE_ENABLE_ZIPVFS /* ** If the argument is not NULL, it points to a Wal object that holds a diff --git a/src/wal.h b/src/wal.h index 092546354b..1c6f27d8f0 100644 --- a/src/wal.h +++ b/src/wal.h @@ -126,6 +126,8 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); +int sqlite3WalCheckSalt(Wal *pWal, sqlite3_file*); + #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). From abc07886636c90fa2e4444b55f06768e68c88dc4 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 17 Sep 2014 15:20:24 +0000 Subject: [PATCH 017/116] Add tests and fixes for "PRAGMA ota_mode". FossilOrigin-Name: 39df35c4ac65ffba76ba2c6f6727cf5e843e7517 --- ext/ota/ota2.test | 2 +- ext/ota/ota4.test | 134 +++++++++++++++++++++++++++++++++++++++++----- manifest | 18 +++---- manifest.uuid | 2 +- src/insert.c | 2 +- src/trigger.c | 8 ++- 6 files changed, 140 insertions(+), 26 deletions(-) diff --git a/ext/ota/ota2.test b/ext/ota/ota2.test index d80f7360c9..6ce6b0852f 100644 --- a/ext/ota/ota2.test +++ b/ext/ota/ota2.test @@ -14,7 +14,7 @@ set testdir [file join [file dirname $argv0] .. .. test] source $testdir/tester.tcl set ::testprefix ota2 -forcedelete test.db-oal +forcedelete test.db-oal test.db-bak do_execsql_test 1.0 { CREATE TABLE t1(a, b); diff --git a/ext/ota/ota4.test b/ext/ota/ota4.test index d0c9dfae40..98bb9330e3 100644 --- a/ext/ota/ota4.test +++ b/ext/ota/ota4.test @@ -9,13 +9,17 @@ # #*********************************************************************** # -# Test some properties of the pager_ota_mode pragma. +# Test some properties of the pager_ota_mode and ota_mode pragmas. # set testdir [file join [file dirname $argv0] .. .. test] source $testdir/tester.tcl set ::testprefix ota4 +#------------------------------------------------------------------------- +# The following tests aim to verify some properties of the pager_ota_mode +# pragma: +# # 1. Cannot set the pager_ota_mode flag on a WAL mode database. # # 2. Or if there is an open read transaction. @@ -30,34 +34,34 @@ set ::testprefix ota4 # file was started. # -do_execsql_test 1.1 { +do_execsql_test 1.1.1 { PRAGMA journal_mode = wal; SELECT * FROM sqlite_master; } {wal} -do_catchsql_test 1.2 { +do_catchsql_test 1.1.2 { PRAGMA pager_ota_mode = 1 } {1 {cannot set pager_ota_mode in wal mode}} -do_execsql_test 2.1 { +do_execsql_test 1.2.1 { PRAGMA journal_mode = delete; BEGIN; SELECT * FROM sqlite_master; } {delete} -do_catchsql_test 2.2 { +do_catchsql_test 1.2.2 { PRAGMA pager_ota_mode = 1 } {1 {cannot set pager_ota_mode with open transaction}} -do_execsql_test 2.3 { +do_execsql_test 1.2.3 { COMMIT; } {} -do_execsql_test 3.1 { +do_execsql_test 1.3.1 { PRAGMA journal_mode = wal; CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 2); } {wal} -do_test 3.2 { +do_test 1.3.2 { forcecopy test.db-wal test.db-bak execsql { PRAGMA journal_mode = delete; @@ -69,7 +73,7 @@ do_test 3.2 { } } {1 {unable to open database file}} -do_test 4.1 { +do_test 1.4.1 { db close forcedelete test.db-wal test.db-oal sqlite3 db test.db @@ -82,7 +86,7 @@ do_test 4.1 { } } {1 {unable to open database file}} -do_test 5.1 { +do_test 1.5.1 { forcedelete test.db-oal reset_db execsql { @@ -100,7 +104,7 @@ do_test 5.1 { SELECT * FROM t1; } } {1 2} -do_execsql_test 5.2 { +do_execsql_test 1.5.2 { PRAGMA pager_ota_mode = 1; SELECT * FROM t1; INSERT INTO t1 VALUES(5, 6); @@ -113,9 +117,115 @@ do_test 5.3 { SELECT * FROM t1; } } {1 2 7 8} -do_catchsql_test 5.4 { +do_catchsql_test 1.5.4 { PRAGMA pager_ota_mode = 1; SELECT * FROM t1; } {1 {database is locked}} +#------------------------------------------------------------------------- +# These tests - ota4-2.* - aim to verify some properties of the ota_mode +# pragma. +# +# 1. Check that UNIQUE constraints are not tested in ota_mode. +# 2. Except for (real) PRIMARY KEY constraints. +# 3. Check that all non-temporary triggers are ignored. +# +reset_db +do_execsql_test 2.1.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE UNIQUE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, 4, 6); +} + +do_execsql_test 2.1.2 { + PRAGMA ota_mode = 1; + INSERT INTO t1 VALUES(3, 2, 6); + UPDATE t1 SET b=2 WHERE a=2; + SELECT * FROM t1; +} { + 1 2 3 + 2 2 6 + 3 2 6 +} + +reset_db +do_execsql_test 2.2.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE t2(x, y, z, PRIMARY KEY(y, z)) WITHOUT ROWID; + + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(4, 5, 6); + PRAGMA ota_mode = 1; +} +do_catchsql_test 2.2.2 { + INSERT INTO t1 VALUES(1, 'two', 'three'); +} {1 {UNIQUE constraint failed: t1.a}} +do_catchsql_test 2.2.3 { + INSERT INTO t2 VALUES('four', 5, 6); +} {1 {UNIQUE constraint failed: t2.y, t2.z}} + +reset_db +do_execsql_test 2.3.1 { + CREATE TABLE t1(a, b, c); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 2, 3); + + CREATE TRIGGER tr1 BEFORE INSERT ON t1 BEGIN + INSERT INTO log VALUES('permanent'); + END; + CREATE TRIGGER tr2 AFTER INSERT ON t1 BEGIN + INSERT INTO log VALUES('permanent'); + END; + CREATE TRIGGER tr3 BEFORE DELETE ON t1 BEGIN + INSERT INTO log VALUES('permanent'); + END; + CREATE TRIGGER tr4 AFTER DELETE ON t1 BEGIN + INSERT INTO log VALUES('permanent'); + END; + CREATE TRIGGER tr5 BEFORE UPDATE ON t1 BEGIN + INSERT INTO log VALUES('permanent'); + END; + CREATE TRIGGER tr6 AFTER UPDATE ON t1 BEGIN + INSERT INTO log VALUES('permanent'); + END; + + CREATE TEMP TRIGGER ttr1 BEFORE INSERT ON t1 BEGIN + INSERT INTO log VALUES('temp'); + END; + CREATE TEMP TRIGGER ttr2 AFTER INSERT ON t1 BEGIN + INSERT INTO log VALUES('temp'); + END; + CREATE TEMP TRIGGER ttr3 BEFORE DELETE ON t1 BEGIN + INSERT INTO log VALUES('temp'); + END; + CREATE TEMP TRIGGER ttr4 AFTER DELETE ON t1 BEGIN + INSERT INTO log VALUES('temp'); + END; + CREATE TEMP TRIGGER ttr5 BEFORE UPDATE ON t1 BEGIN + INSERT INTO log VALUES('temp'); + END; + CREATE TEMP TRIGGER ttr6 AFTER UPDATE ON t1 BEGIN + INSERT INTO log VALUES('temp'); + END; +} +do_execsql_test 2.3.2 { + INSERT INTO t1 VALUES(4, 5, 6); + DELETE FROM t1 WHERE a = 4; + UPDATE t1 SET c = 6; + SELECT x FROM log; +} { + temp permanent temp permanent temp permanent + temp permanent temp permanent temp permanent +} +do_execsql_test 2.3.3 { + DELETE FROM log; + PRAGMA ota_mode = 1; + INSERT INTO t1 VALUES(4, 5, 6); + DELETE FROM t1 WHERE a = 4; + UPDATE t1 SET c = 6; + SELECT x FROM log; +} {temp temp temp temp temp temp} + finish_test + diff --git a/manifest b/manifest index bcb8678d88..d80435f101 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarify\sthe\seffects\sof\sthe\spager_ota_mode\spragma.\sAdd\stests\sand\sfixes\sfor\sthe\ssame. -D 2014-09-16T20:02:41.756 +C Add\stests\sand\sfixes\sfor\s"PRAGMA\sota_mode". +D 2014-09-17T15:20:24.516 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -123,9 +123,9 @@ F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 -F ext/ota/ota2.test 13f76922446c62ed96192e938b8e625ebf0142fa +F ext/ota/ota2.test 716f9c66e8bf8b0ad2fe3a5d8323e6cf460a2e27 F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 -F ext/ota/ota4.test ec01b0d69ad2989559a65fde74560c1cfcca8202 +F ext/ota/ota4.test baf23b47748a5056c713871959cc70fc623c90e9 F ext/ota/sqlite3ota.c 668ed08dd81ff8ae1e8524b2d4bf0f2609cbf907 F ext/ota/sqlite3ota.h 39ce4dffbfcf4ade9e4526369fe2243709345c8e F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -195,7 +195,7 @@ F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 25e63641927530eee0487c5a8bbf6e5414c4c753 +F src/insert.c 887551441b3a46d964733b16a86c614d0152778d F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 @@ -290,7 +290,7 @@ F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff F src/tokenize.c 3df63041994f55afeb168b463ec836e8f1c50e7c -F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f +F src/trigger.c eb921d1292aca83d515bde5881df71df91d962d6 F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 48d201cd8b68c0377cf8a2cc6439b893f9462fe2 -R de185427497b23c6bf5f7a43a97cdd26 +P decaccc37cbdcd2a663233469efdf4982a810513 +R d8d51626052024a717c822f3b6627467 U dan -Z de73fdb4c4401b74adf143cdfee4f7e2 +Z 65720a0642c813fef614e2d54b0a69b7 diff --git a/manifest.uuid b/manifest.uuid index fa8e1e694f..a0c18add6e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -decaccc37cbdcd2a663233469efdf4982a810513 \ No newline at end of file +39df35c4ac65ffba76ba2c6f6727cf5e843e7517 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 356039cbc4..e31bae28f2 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1564,7 +1564,7 @@ void sqlite3CompleteInsertion( /* If the "ota_mode" flag is set, ignore all indexes except the PK ** index of WITHOUT ROWID tables. */ if( (pParse->db->flags & SQLITE_OtaMode) - && (pTab->iPKey>=0 || pIdx->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) + && (HasRowid(pTab) || pIdx->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) ){ continue; } diff --git a/src/trigger.c b/src/trigger.c index d2e7b5a1e6..f837e23b5a 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -43,10 +43,14 @@ void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){ ** To state it another way: This routine returns a list of all triggers ** that fire off of pTab. The list will include any TEMP triggers on ** pTab as well as the triggers lised in pTab->pTrigger. +** +** If the SQLITE_OtaMode flag is set, do not include any non-temporary +** triggers in the returned list. */ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ Schema * const pTmpSchema = pParse->db->aDb[1].pSchema; Trigger *pList = 0; /* List of triggers to return */ + int bOta = !!(pParse->db->flags & SQLITE_OtaMode); if( pParse->disableTriggers ){ return 0; @@ -60,13 +64,13 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ if( pTrig->pTabSchema==pTab->pSchema && 0==sqlite3StrICmp(pTrig->table, pTab->zName) ){ - pTrig->pNext = (pList ? pList : pTab->pTrigger); + pTrig->pNext = ((pList || bOta) ? pList : pTab->pTrigger); pList = pTrig; } } } - return (pList ? pList : pTab->pTrigger); + return ((pList || bOta) ? pList : pTab->pTrigger); } /* From 44ed6bc5c11a04fb1fbf710e904a3b8f28624dd5 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 17 Sep 2014 19:05:46 +0000 Subject: [PATCH 018/116] Fix an unintialized variable problem in sqlite3ota.c. FossilOrigin-Name: 01268607515a30d07559648c43034e522605748a --- ext/ota/sqlite3ota.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 999bc00cf7..a0d282357a 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -971,7 +971,7 @@ static void otaLoadTransactionState(sqlite3ota *p, OtaState *pState){ assert( p->rc==SQLITE_OK ); if( pState->zTbl ){ OtaObjIter *pIter = &p->objiter; - int rc; + int rc = SQLITE_OK; while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup || otaStrCompare(pIter->zTbl, pState->zTbl) diff --git a/manifest b/manifest index d80435f101..8d9373f09f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sand\sfixes\sfor\s"PRAGMA\sota_mode". -D 2014-09-17T15:20:24.516 +C Fix\san\sunintialized\svariable\sproblem\sin\ssqlite3ota.c. +D 2014-09-17T19:05:46.738 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -126,7 +126,7 @@ F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 F ext/ota/ota2.test 716f9c66e8bf8b0ad2fe3a5d8323e6cf460a2e27 F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 F ext/ota/ota4.test baf23b47748a5056c713871959cc70fc623c90e9 -F ext/ota/sqlite3ota.c 668ed08dd81ff8ae1e8524b2d4bf0f2609cbf907 +F ext/ota/sqlite3ota.c 49187d90b40f223b1b618872e71bad44686165d9 F ext/ota/sqlite3ota.h 39ce4dffbfcf4ade9e4526369fe2243709345c8e F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P decaccc37cbdcd2a663233469efdf4982a810513 -R d8d51626052024a717c822f3b6627467 +P 39df35c4ac65ffba76ba2c6f6727cf5e843e7517 +R 11f94a1a393bad4eb750c64571794419 U dan -Z 65720a0642c813fef614e2d54b0a69b7 +Z 16716967f94fb6bc011dff17e7e27c1f diff --git a/manifest.uuid b/manifest.uuid index a0c18add6e..6bd324710a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -39df35c4ac65ffba76ba2c6f6727cf5e843e7517 \ No newline at end of file +01268607515a30d07559648c43034e522605748a \ No newline at end of file From c6c3d00f33fe2732b927837a0396885ca8007fa4 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Sep 2014 11:15:17 +0000 Subject: [PATCH 019/116] Fix an ota bug causing attempts to write to tables with more than 8 columns to fail. FossilOrigin-Name: 7da98ca2422166dc5d68607541707f41b77b5784 --- ext/ota/sqlite3ota.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index a0d282357a..cc968a54bb 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -372,7 +372,7 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ if( (nCol % 8)==0 ){ int nByte = sizeof(char*) * (nCol+8); char **azNew = (char**)sqlite3_realloc(pIter->azTblCol, nByte); - u8 *abNew = (u8*)sqlite3_realloc(pIter->azTblCol, nCol+8); + u8 *abNew = (u8*)sqlite3_realloc(pIter->abTblPk, nCol+8); if( azNew ) pIter->azTblCol = azNew; if( abNew ) pIter->abTblPk = abNew; diff --git a/manifest b/manifest index 8d9373f09f..f4c69d0613 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sunintialized\svariable\sproblem\sin\ssqlite3ota.c. -D 2014-09-17T19:05:46.738 +C Fix\san\sota\sbug\scausing\sattempts\sto\swrite\sto\stables\swith\smore\sthan\s8\scolumns\sto\sfail. +D 2014-09-18T11:15:17.379 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -126,7 +126,7 @@ F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 F ext/ota/ota2.test 716f9c66e8bf8b0ad2fe3a5d8323e6cf460a2e27 F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 F ext/ota/ota4.test baf23b47748a5056c713871959cc70fc623c90e9 -F ext/ota/sqlite3ota.c 49187d90b40f223b1b618872e71bad44686165d9 +F ext/ota/sqlite3ota.c 276c0426e678edc06df5c295515b44f773ad6e93 F ext/ota/sqlite3ota.h 39ce4dffbfcf4ade9e4526369fe2243709345c8e F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 39df35c4ac65ffba76ba2c6f6727cf5e843e7517 -R 11f94a1a393bad4eb750c64571794419 +P 01268607515a30d07559648c43034e522605748a +R 9c5669944e9da69cf3cdc8cdadb20c7b U dan -Z 16716967f94fb6bc011dff17e7e27c1f +Z b82db3d99eb0abfa9c3989bdc212a9ca diff --git a/manifest.uuid b/manifest.uuid index 6bd324710a..9c0feb2429 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01268607515a30d07559648c43034e522605748a \ No newline at end of file +7da98ca2422166dc5d68607541707f41b77b5784 \ No newline at end of file From 4caa8fbc1eb6cf78efadcc148e09d5c83ef1d9ce Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Sep 2014 14:48:38 +0000 Subject: [PATCH 020/116] Add an API to query an ota handle for the total number of key/value operations performed so far. FossilOrigin-Name: e3943fa7bbbfc5e16f73a494d8fa54d19e9cfcf9 --- ext/ota/ota.c | 12 ++++++++++-- ext/ota/sqlite3ota.c | 18 ++++++++++++++++-- ext/ota/sqlite3ota.h | 7 +++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 42 insertions(+), 13 deletions(-) diff --git a/ext/ota/ota.c b/ext/ota/ota.c index c00eed4f0b..febdbfe2da 100644 --- a/ext/ota/ota.c +++ b/ext/ota/ota.c @@ -55,6 +55,7 @@ int main(int argc, char **argv){ sqlite3ota *pOta; /* OTA handle */ int nStep = 0; /* Maximum number of step() calls */ int rc; + sqlite3_int64 nProgress = 0; /* Process command line arguments. Following this block local variables ** zTarget, zOta and nStep are all set. */ @@ -76,16 +77,23 @@ int main(int argc, char **argv){ ** sqlite3ota_step() a maximum of nStep times. */ pOta = sqlite3ota_open(zTarget, zOta); for(i=0; (nStep<=0 || ipSelect); if( rc==SQLITE_ROW ){ p->nStep++; + p->nProgress++; return otaStep(p); } p->rc = sqlite3_reset(pIter->pSelect); @@ -901,8 +904,8 @@ int sqlite3ota_step(sqlite3ota *p){ static void otaSaveTransactionState(sqlite3ota *p){ otaMPrintfExec(p, "INSERT OR REPLACE INTO ota.ota_state(rowid, tbl, idx, row, progress)" - "VALUES(1, %Q, %Q, %d, NULL)", - p->objiter.zTbl, p->objiter.zIdx, p->nStep + "VALUES(1, %Q, %Q, %d, %lld)", + p->objiter.zTbl, p->objiter.zIdx, p->nStep, p->nProgress ); } @@ -942,6 +945,7 @@ static OtaState *otaLoadState(sqlite3ota *p){ pRet->zIdx = 0; } pRet->nRow = sqlite3_column_int(pStmt, 2); + pRet->nProgress = sqlite3_column_int64(pStmt, 3); } }else{ pRet = (OtaState*)sqlite3_malloc(sizeof(OtaState)); @@ -1075,6 +1079,7 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ } if( p->rc==SQLITE_OK ){ + p->nProgress = pState->nProgress; otaLoadTransactionState(p, pState); } @@ -1128,6 +1133,15 @@ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ return rc; } +/* +** Return the total number of key-value operations (inserts, deletes or +** updates) that have been performed on the target database since the +** current OTA update was started. +*/ +sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta){ + return pOta->nProgress; +} + /**************************************************************************/ diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index 9473b359f2..30c84c2c02 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -215,5 +215,12 @@ int sqlite3ota_step(sqlite3ota *pOta); */ int sqlite3ota_close(sqlite3ota *pOta, char **pzErrmsg); +/* +** Return the total number of key-value operations (inserts, deletes or +** updates) that have been performed on the target database since the +** current OTA update was started. +*/ +sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta); + #endif /* _SQLITE3OTA_H */ diff --git a/manifest b/manifest index a16f3db600..8130805f1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2014-09-18T11:31:52.958 +C Add\san\sAPI\sto\squery\san\sota\shandle\sfor\sthe\stotal\snumber\sof\skey/value\soperations\sperformed\sso\sfar. +D 2014-09-18T14:48:38.579 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,13 +121,13 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/ota/ota.c d37097e92a005d3915883adefbb93019ea6f8841 +F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 F ext/ota/ota2.test 716f9c66e8bf8b0ad2fe3a5d8323e6cf460a2e27 F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 F ext/ota/ota4.test baf23b47748a5056c713871959cc70fc623c90e9 -F ext/ota/sqlite3ota.c 276c0426e678edc06df5c295515b44f773ad6e93 -F ext/ota/sqlite3ota.h 39ce4dffbfcf4ade9e4526369fe2243709345c8e +F ext/ota/sqlite3ota.c e5dfb63ac4564139b66acf6f712d6d1bb5007b26 +F ext/ota/sqlite3ota.h 4f9d8c56c673f279167ddc57e8f84c0c406346f0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7da98ca2422166dc5d68607541707f41b77b5784 3bd7c1b2faa2d4cc95b255633204006849bfd5e0 -R 8e63365c95630e53dbe0beb2fbad87f1 +P 67ea2979d5831b6d0d55173bd9413b21644cf6a1 +R 38d6ede7d1ef519b3ce4a32ffbb6512b U dan -Z 4b3913e6deb233dcf8270d9446c1c662 +Z 2cc714eb22b38452d154c6b43dc8544c diff --git a/manifest.uuid b/manifest.uuid index 762f3d0c14..38824d34b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67ea2979d5831b6d0d55173bd9413b21644cf6a1 \ No newline at end of file +e3943fa7bbbfc5e16f73a494d8fa54d19e9cfcf9 \ No newline at end of file From ed18c961074b20f655d9e0afb6eee9a742f4c43a Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Sep 2014 15:22:48 +0000 Subject: [PATCH 021/116] Add new file ext/ota/README.txt, containing notes regarding the implementation of the ota extension. FossilOrigin-Name: 3c6e1cbb4baaebc5958ab7276f27ff0ba14f4fa2 --- ext/ota/README.txt | 142 +++++++++++++++++++++++++++++++++++++++++++++ manifest | 11 ++-- manifest.uuid | 2 +- 3 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 ext/ota/README.txt diff --git a/ext/ota/README.txt b/ext/ota/README.txt new file mode 100644 index 0000000000..2707f22f1d --- /dev/null +++ b/ext/ota/README.txt @@ -0,0 +1,142 @@ + +This file contains notes regarding the implementation of the OTA extension. +User documentation is in sqlite3ota.h. + +SQLite Hacks +------------ + +1) PRAGMA ota_mode: + + This is a new flag pragma. If the flag is set: + + * INSERT/DELETE/UPDATE commands are prevented from updating any but the main + b-tree for each table (the PK index for WITHOUT ROWID tables or the + rowid b-tree for others). + + * The above statements do not check UNIQUE constraints - except those enforced + by the main b-tree. + + * All non-temporary triggers are disabled. + + +2) PRAGMA pager_ota_mode: + + This pragma sets a flag on the pager associated with the main database only. In + a zipvfs system, this pragma is intercepted by zipvfs and the flag is set on + the lower level pager only. + + The flag can only be set when there is no open transaction and the pager does + not already have an open WAL file. Attempting to do so is an error. + + Once the flag has been set, it is not possible to open a regular WAL file. + If, when the next read-transaction is opened, a *-wal file is found or the + database header flags indicate that it is a wal-mode database, + SQLITE_CANTOPEN is returned. + + Otherwise, if no WAL file or flags are found, the pager opens the *-oal file + and uses it as a write-ahead-log with the *-shm data stored in heap-memory. + + The 8-bytes of "salt" at teh start of an *-oal file is a copy of the 8 bytes + starting at offset 24 of the database file header (the change counter and the + number of pages in the file). If the *-oal file already exists when it is + opened, SQLite checks that the salt still matches the database header fields. + If not, it concludes that the database file has been written by a rollback-mode + client since the *-oal wa created and an SQLITE_BUSY_SNAPSHOT error is + returned. No read-transaction can be opened in this case. + + A pager with the pager_ota_mode flag set never runs a checkpoint. + + Other clients see a rollback-mode database on which the pager_ota_mode client + is holding a SHARED lock. There are no locks to arbitrate between multiple + pager_ota_mode connections. If two or more such connections attempt to write + simultaneously, the results are undefined. + + +3) sqlite3_index_writer() + + This new API function is used to create VMs that can insert or delete entries + from individual index b-trees within the database. The VMs apply affinities + and check that UNIQUE constraints are not violated before updating index + b-trees. + + +The OTA extension +----------------- + +The OTA extension requires that the OTA update be packaged as an SQLite +database. The tables it expects to find are described in sqlite3ota.h. +Essentially, for each table xyz in the target database that the user wishes +to write to, a corresponding data_xyz table is created in the OTA database +and populated with one row for each row to update, insert or delete from +the target table. + +The OTA extension opens the target and OTA update databases using a single +database handle (the target database is "main", and the OTA update database is +attached as "ota"). It executes both the "pager_ota_mode" and "ota_mode" +pragmas described above. For each data_xyz table in then: + + * CREATEs an ota_xyz table in the OTA update database. + + * Loops through the data_xyz table, running the INSERT, UPDATE or DELETE + command on the corresponding target database table. Only the main b-tree + is updated by these statements. Modified pages are appended to the *-oal + file. + + Temporary triggers installed on the target database catch the old.* + values associated with any UPDATEd or DELETEd rows and store them in + the ota_xyz table (in the OTA update database). + + * For each index on the data_xyz table in the target database: + + Loop through a union of the data_xyz and ota_xyz tables in the order + specified by the data_xyz index. In other words, if the index is on + columns (a, b), read rows from the OTA update database using: + + SELECT * FROM data_xyz UNION ALL ota_xyz ORDER BY a, b; + + For each row visited, use an sqlite3_index_writer() VM to update the index + in the target database. + + * DROPs the ota_xyz table. + +At any point in the above, the process may be suspended by the user. In this +case the "ota_state" table is created in the OTA database, containing a single +row indicating the current table/index being processed and the number of updates +already performed on it, and the transaction on the target database is committed +to the *-oal file. The next OTA client will use the contents of the ota_state +table to continue the update from where this one left off. + +Alternatively, if the OTA update is completely applied, the transaction is +committed to the *-oal file and the database connection closed. sqlite3ota.c +then uses a rename() call to move the *-oal file to the corresponding *-wal +path. At that point it is finished - it does not take responsibility for +checkpointing the *-wal file. + + +Problems +-------- + +The rename() call might not be portable. And in theory it is unsafe if some +other client starts writing the db file. + +When state is saved, the commit to the *-oal file and the commit to the OTA +update database are not atomic. So if the power fails at the wrong moment they +might get out of sync. As the main database will be committed before the OTA +update database this will likely either just pass unnoticed, or result in +SQLITE_CONSTRAINT errors (due to UNIQUE constraint violations). + +If some client does modify the target database mid OTA update, or some other +error occurs, the OTA extension will keep throwing errors. It's not really +clear how to get out of this state. The system could just by delete the OTA +update database and *-oal file and have the device download the update again +and start over. + +At present, for an UPDATE, both the new.* and old.* records are collected in +the ota_xyz table. And for both UPDATEs and DELETEs all fields are collected. +This means we're probably writing a lot more data to disk when saving the +state of an ongoing update to the OTA update database than is strictly +necessary. + + + + diff --git a/manifest b/manifest index 8130805f1d..ebff66aeaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sAPI\sto\squery\san\sota\shandle\sfor\sthe\stotal\snumber\sof\skey/value\soperations\sperformed\sso\sfar. -D 2014-09-18T14:48:38.579 +C Add\snew\sfile\sext/ota/README.txt,\scontaining\snotes\sregarding\sthe\simplementation\sof\sthe\sota\sextension. +D 2014-09-18T15:22:48.861 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,6 +121,7 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 +F ext/ota/README.txt cb11e39bfeba952ac8896dab860ada9d54731fb8 F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 F ext/ota/ota2.test 716f9c66e8bf8b0ad2fe3a5d8323e6cf460a2e27 @@ -1205,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 67ea2979d5831b6d0d55173bd9413b21644cf6a1 -R 38d6ede7d1ef519b3ce4a32ffbb6512b +P e3943fa7bbbfc5e16f73a494d8fa54d19e9cfcf9 +R fb02b3fe38b193b1aa6184b0012599dc U dan -Z 2cc714eb22b38452d154c6b43dc8544c +Z 4cd279ef72288cc868c417c33c545f0f diff --git a/manifest.uuid b/manifest.uuid index 38824d34b1..d10a6c3486 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3943fa7bbbfc5e16f73a494d8fa54d19e9cfcf9 \ No newline at end of file +3c6e1cbb4baaebc5958ab7276f27ff0ba14f4fa2 \ No newline at end of file From 318e63c60bf87eceda3addac21baae4138ed79c2 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Sep 2014 15:57:22 +0000 Subject: [PATCH 022/116] Remove some c++isms from sqlite3ota.c. FossilOrigin-Name: 0da1862b1b68f15c7c14286a982dda886d1e3b4a --- ext/ota/sqlite3ota.c | 11 ++++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 8d5a5bf761..34f5e3db6e 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -40,7 +40,6 @@ typedef struct OtaState OtaState; typedef struct OtaObjIter OtaObjIter; -typedef unsigned char u8; /* ** A structure to store values read from the ota_state table in memory. @@ -66,7 +65,7 @@ struct OtaObjIter { sqlite3_stmt *pIdxIter; /* Index iterator */ int nTblCol; /* Size of azTblCol[] array */ char **azTblCol; /* Array of quoted column names */ - u8 *abTblPk; /* Array of flags - true for PK columns */ + unsigned char *abTblPk; /* Array of flags - true for PK columns */ /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ @@ -372,9 +371,10 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ p->rc = prepareFreeAndCollectError(p->db, &pStmt, &p->zErrmsg, zSql); while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ if( (nCol % 8)==0 ){ + unsigned char *abNew; int nByte = sizeof(char*) * (nCol+8); char **azNew = (char**)sqlite3_realloc(pIter->azTblCol, nByte); - u8 *abNew = (u8*)sqlite3_realloc(pIter->abTblPk, nCol+8); + abNew = (unsigned char*)sqlite3_realloc(pIter->abTblPk, nCol+8); if( azNew ) pIter->azTblCol = azNew; if( abNew ) pIter->abTblPk = abNew; @@ -834,10 +834,11 @@ static int otaStep(sqlite3ota *p){ } for(i=0; inCol; i++){ + sqlite3_value *pVal; if( eType==SQLITE_DELETE && pIter->zIdx==0 && pIter->abTblPk[i]==0 ){ continue; } - sqlite3_value *pVal = sqlite3_column_value(pIter->pSelect, i); + pVal = sqlite3_column_value(pIter->pSelect, i); sqlite3_bind_value(pWriter, i+1, pVal); } sqlite3_step(pWriter); @@ -874,7 +875,7 @@ int sqlite3ota_step(sqlite3ota *p){ /* Clean up the ota_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ - // otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); + otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); }else{ otaObjIterPrepareAll(p, pIter, 0); diff --git a/manifest b/manifest index ebff66aeaf..b30b820cb2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sfile\sext/ota/README.txt,\scontaining\snotes\sregarding\sthe\simplementation\sof\sthe\sota\sextension. -D 2014-09-18T15:22:48.861 +C Remove\ssome\sc++isms\sfrom\ssqlite3ota.c. +D 2014-09-18T15:57:22.238 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -127,7 +127,7 @@ F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 F ext/ota/ota2.test 716f9c66e8bf8b0ad2fe3a5d8323e6cf460a2e27 F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 F ext/ota/ota4.test baf23b47748a5056c713871959cc70fc623c90e9 -F ext/ota/sqlite3ota.c e5dfb63ac4564139b66acf6f712d6d1bb5007b26 +F ext/ota/sqlite3ota.c 4b7f15662e725bd79c92242293336d74ebeff854 F ext/ota/sqlite3ota.h 4f9d8c56c673f279167ddc57e8f84c0c406346f0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e3943fa7bbbfc5e16f73a494d8fa54d19e9cfcf9 -R fb02b3fe38b193b1aa6184b0012599dc +P 3c6e1cbb4baaebc5958ab7276f27ff0ba14f4fa2 +R 144b17f95f9943a0f5b2a1840d65f3bd U dan -Z 4cd279ef72288cc868c417c33c545f0f +Z 850176a0d2bd1d95bab64ffe8e2adfdc diff --git a/manifest.uuid b/manifest.uuid index d10a6c3486..075b7ba304 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c6e1cbb4baaebc5958ab7276f27ff0ba14f4fa2 \ No newline at end of file +0da1862b1b68f15c7c14286a982dda886d1e3b4a \ No newline at end of file From 6c5984a950072ffac7db89cfdbe2ea73b3413c57 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Sep 2014 16:38:10 +0000 Subject: [PATCH 023/116] Use quotes instead of angle-brackets to include sqlite3.h from sqlite3ota.h. FossilOrigin-Name: fce9c6ccf101d28492a84b85463433f32ef569fb --- ext/ota/sqlite3ota.h | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index 30c84c2c02..8850d54d5b 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -168,7 +168,7 @@ #ifndef _SQLITE3OTA_H #define _SQLITE3OTA_H -#include /* Required for error code definitions */ +#include "sqlite3.h" /* Required for error code definitions */ typedef struct sqlite3ota sqlite3ota; diff --git a/manifest b/manifest index b30b820cb2..98eac4a03c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sc++isms\sfrom\ssqlite3ota.c. -D 2014-09-18T15:57:22.238 +C Use\squotes\sinstead\sof\sangle-brackets\sto\sinclude\ssqlite3.h\sfrom\ssqlite3ota.h. +D 2014-09-18T16:38:10.149 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -128,7 +128,7 @@ F ext/ota/ota2.test 716f9c66e8bf8b0ad2fe3a5d8323e6cf460a2e27 F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 F ext/ota/ota4.test baf23b47748a5056c713871959cc70fc623c90e9 F ext/ota/sqlite3ota.c 4b7f15662e725bd79c92242293336d74ebeff854 -F ext/ota/sqlite3ota.h 4f9d8c56c673f279167ddc57e8f84c0c406346f0 +F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3c6e1cbb4baaebc5958ab7276f27ff0ba14f4fa2 -R 144b17f95f9943a0f5b2a1840d65f3bd +P 0da1862b1b68f15c7c14286a982dda886d1e3b4a +R e1133f29a6887c2dcc50626f2201457f U dan -Z 850176a0d2bd1d95bab64ffe8e2adfdc +Z d4fc012b67241f1149bf53ab8b647500 diff --git a/manifest.uuid b/manifest.uuid index 075b7ba304..497c4aa9aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0da1862b1b68f15c7c14286a982dda886d1e3b4a \ No newline at end of file +fce9c6ccf101d28492a84b85463433f32ef569fb \ No newline at end of file From be758ce3ae490909d9bb12a15b30794fc69b7cbf Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Sep 2014 17:57:46 +0000 Subject: [PATCH 024/116] Update the ota extension to support SQLITE_ENABLE_8_3_NAMES builds. FossilOrigin-Name: 718da6de870231d358384473b40e81c4394b8067 --- ext/ota/sqlite3ota.c | 41 +++++++++++++++++++++++++++++++++++++++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 34f5e3db6e..18a71910e5 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -999,12 +999,46 @@ static void otaLoadTransactionState(sqlite3ota *p, OtaState *pState){ } } +/* +** This routine is a copy of the sqlite3FileSuffix3() routine from the core. +** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. +** +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. +** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa +*/ +static void otaFileSuffix3(const char *zBase, char *z){ +#ifdef SQLITE_ENABLE_8_3_NAMES +#if SQLITE_ENABLE_8_3_NAMES<2 + if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) +#endif + { + int i, sz; + sz = sqlite3Strlen30(z); + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); + } +#endif +} + /* ** Move the "*-oal" file corresponding to the target database to the ** "*-wal" location. If an error occurs, leave an error code and error ** message in the ota handle. */ -static void otaMoveOalFile(sqlite3ota *p){ +static void otaMoveOalFile(const char *zBase, sqlite3ota *p){ char *zWal = sqlite3_mprintf("%s-wal", p->zTarget); char *zOal = sqlite3_mprintf("%s-oal", p->zTarget); @@ -1012,6 +1046,8 @@ static void otaMoveOalFile(sqlite3ota *p){ if( zWal==0 || zOal==0 ){ p->rc = SQLITE_NOMEM; }else{ + otaFileSuffix3(zBase, zWal); + otaFileSuffix3(zBase, zOal); rename(zOal, zWal); } @@ -1096,6 +1132,7 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ int rc; if( p ){ + const char *zBase = sqlite3_db_filename(p->db, "main"); /* If the update has not been fully applied, save the state in ** the ota db. If successful, this call also commits the open @@ -1121,7 +1158,7 @@ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ /* If the OTA has been completely applied and no error occurred, move ** the *-oal file to *-wal. */ if( p->rc==SQLITE_DONE ){ - otaMoveOalFile(p); + otaMoveOalFile(zBase, p); } rc = p->rc; diff --git a/manifest b/manifest index 98eac4a03c..ec203b7073 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\squotes\sinstead\sof\sangle-brackets\sto\sinclude\ssqlite3.h\sfrom\ssqlite3ota.h. -D 2014-09-18T16:38:10.149 +C Update\sthe\sota\sextension\sto\ssupport\sSQLITE_ENABLE_8_3_NAMES\sbuilds. +D 2014-09-18T17:57:46.895 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -127,7 +127,7 @@ F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 F ext/ota/ota2.test 716f9c66e8bf8b0ad2fe3a5d8323e6cf460a2e27 F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 F ext/ota/ota4.test baf23b47748a5056c713871959cc70fc623c90e9 -F ext/ota/sqlite3ota.c 4b7f15662e725bd79c92242293336d74ebeff854 +F ext/ota/sqlite3ota.c b22002105b3b7f3baf63bda2b4e6a00c4973418c F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0da1862b1b68f15c7c14286a982dda886d1e3b4a -R e1133f29a6887c2dcc50626f2201457f +P fce9c6ccf101d28492a84b85463433f32ef569fb +R feab70887c117c061b61fe713ccedbe4 U dan -Z d4fc012b67241f1149bf53ab8b647500 +Z 86cf672a01ea748dc788d9b12ee5fdc1 diff --git a/manifest.uuid b/manifest.uuid index 497c4aa9aa..1b666612e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fce9c6ccf101d28492a84b85463433f32ef569fb \ No newline at end of file +718da6de870231d358384473b40e81c4394b8067 \ No newline at end of file From d1193a98939f17e2ea4bc9f60542a9ab33541bbe Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 19 Sep 2014 15:06:23 +0000 Subject: [PATCH 025/116] Add extra tests for the ota extension. FossilOrigin-Name: 1e468fe1e408e513a1e1bbe72fe2a240f2991b3d --- ext/ota/ota5.test | 244 ++++++++++++++++++++++++++++++++++++++++++++++ manifest | 11 ++- manifest.uuid | 2 +- 3 files changed, 251 insertions(+), 6 deletions(-) create mode 100644 ext/ota/ota5.test diff --git a/ext/ota/ota5.test b/ext/ota/ota5.test new file mode 100644 index 0000000000..4f85c22b94 --- /dev/null +++ b/ext/ota/ota5.test @@ -0,0 +1,244 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test some properties of the pager_ota_mode and ota_mode pragmas. +# + +set testdir [file join [file dirname $argv0] .. .. test] +source $testdir/tester.tcl +set ::testprefix ota5 + + +# Run the OTA in file $ota on target database $target until completion. +# +proc run_ota {target ota} { + sqlite3ota ota $target $ota + while { [ota step]=="SQLITE_OK" } {} + ota close +} + + +# Run the OTA in file $ota on target database $target one step at a +# time until completion. +# +proc step_ota {target ota} { + while 1 { + sqlite3ota ota $target $ota + set rc [ota step] + ota close + if {$rc != "SQLITE_OK"} break + } + set rc +} + +# Return a list of the primary key columns for table $tbl in the database +# opened by database handle $db. +# +proc pkcols {db tbl} { + set ret [list] + $db eval "PRAGMA table_info = '$tbl'" { + if {$pk} { lappend ret $name } + } + return $ret +} + +# Return a list of all columns for table $tbl in the database opened by +# database handle $db. +# +proc allcols {db tbl} { + set ret [list] + $db eval "PRAGMA table_info = '$tbl'" { + lappend ret $name + } + return $ret +} + +# Return a checksum on all tables and data in the main database attached +# to database handle $db. It is possible to add indexes without changing +# the checksum. +# +proc datacksum {db} { + + $db eval { SELECT name FROM sqlite_master WHERE type='table' } { + append txt $name + set cols [list] + set order [list] + set cnt 0 + $db eval "PRAGMA table_info = $name" x { + lappend cols "quote($x(name))" + lappend order [incr cnt] + } + set cols [join $cols ,] + set order [join $order ,] + append txt [$db eval "SELECT $cols FROM $name ORDER BY $order"] + } + return "[string length $txt]-[md5 $txt]" +} + +proc ucontrol {args} { + set ret "" + foreach a $args { + if {$a} { + append ret . + } else { + append ret x + } + } + return $ret +} + +# Argument $target is the name of an SQLite database file. $sql is an SQL +# script containing INSERT, UPDATE and DELETE statements to execute against +# it. This command creates an OTA update database in file $ota that has +# the same effect as the script. The target database is not modified by +# this command. +# +proc generate_ota_db {target ota sql} { + + forcedelete $ota + forcecopy $target copy.db + + # Evaluate the SQL script to modify the contents of copy.db. + # + sqlite3 dbOta copy.db + dbOta eval $sql + + dbOta function ucontrol ucontrol + + # Evaluate the SQL script to modify the contents of copy.db. + set ret [datacksum dbOta] + + dbOta eval { ATTACH $ota AS ota } + dbOta eval { ATTACH $target AS orig } + + dbOta eval { SELECT name AS tbl FROM sqlite_master WHERE type = 'table' } { + set pk [pkcols dbOta $tbl] + set cols [allcols dbOta $tbl] + + # A WHERE clause to test that the PK columns match. + # + set where [list] + foreach c $pk { lappend where "main.$tbl.$c IS orig.$tbl.$c" } + set where [join $where " AND "] + + # A WHERE clause to test that all columns match. + # + set where2 [list] + foreach c $cols { lappend where2 "main.$tbl.$c IS orig.$tbl.$c" } + set ucontrol "ucontrol([join $where2 ,])" + set where2 [join $where2 " AND "] + + # Create a data_xxx table in the OTA update database. + dbOta eval " + CREATE TABLE ota.data_$tbl AS SELECT *, '' AS ota_control + FROM main.$tbl LIMIT 0 + " + + # Find all new rows INSERTed by the script. + dbOta eval " + INSERT INTO ota.data_$tbl + SELECT *, 0 AS ota_control FROM main.$tbl + WHERE NOT EXISTS ( + SELECT 1 FROM orig.$tbl WHERE $where + ) + " + + # Find all old rows DELETEd by the script. + dbOta eval " + INSERT INTO ota.data_$tbl + SELECT *, 1 AS ota_control FROM orig.$tbl + WHERE NOT EXISTS ( + SELECT 1 FROM main.$tbl WHERE $where + ) + " + + # Find all rows UPDATEd by the script. + set origcols [list] + foreach c $cols { lappend origcols "main.$tbl.$c" } + set origcols [join $origcols ,] + dbOta eval " + INSERT INTO ota.data_$tbl + SELECT $origcols, $ucontrol AS ota_control + FROM orig.$tbl, main.$tbl + WHERE $where AND NOT ($where2) + " + + } + + dbOta close + forcedelete copy.db + + return $ret +} + +#------------------------------------------------------------------------- +# +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE t2(x, y, z, PRIMARY KEY(y, z)) WITHOUT ROWID; + + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, 4, 6); + INSERT INTO t1 VALUES(3, 6, 9); + + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(2, 4, 6); + INSERT INTO t2 VALUES(3, 6, 9); +} +db close + +set cksum [generate_ota_db test.db ota.db { + INSERT INTO t1 VALUES(4, 8, 12); + DELETE FROM t1 WHERE a = 2; + UPDATE t1 SET c = 15 WHERE a=3; + + INSERT INTO t2 VALUES(4, 8, 12); + DELETE FROM t2 WHERE x = 2; + UPDATE t2 SET x = 15 WHERE z=9; +}] + +foreach {tn idx} { + 1 { + } + 2 { + CREATE INDEX i1 ON t1(a, b, c); + CREATE INDEX i2 ON t2(x, y, z); + } +} { + foreach cmd {run step} { + forcecopy test.db test.db2 + forcecopy ota.db ota.db2 + + sqlite3 db test.db2 + db eval $idx + + do_test 1.$tn.$cmd.1 { + ${cmd}_ota test.db2 ota.db2 + datacksum db + } $cksum + + do_test 1.$tn.$cmd.2 { + db eval { PRAGMA integrity_check } + } {ok} + + db close + } +} + +#------------------------------------------------------------------------- +# + + +finish_test + + + + diff --git a/manifest b/manifest index ec203b7073..f6a8d1cd9c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sota\sextension\sto\ssupport\sSQLITE_ENABLE_8_3_NAMES\sbuilds. -D 2014-09-18T17:57:46.895 +C Add\sextra\stests\sfor\sthe\sota\sextension. +D 2014-09-19T15:06:23.790 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -127,6 +127,7 @@ F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 F ext/ota/ota2.test 716f9c66e8bf8b0ad2fe3a5d8323e6cf460a2e27 F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 F ext/ota/ota4.test baf23b47748a5056c713871959cc70fc623c90e9 +F ext/ota/ota5.test bd0e0f7609ece4eee2c62c5132935f0c80190cb8 F ext/ota/sqlite3ota.c b22002105b3b7f3baf63bda2b4e6a00c4973418c F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1206,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fce9c6ccf101d28492a84b85463433f32ef569fb -R feab70887c117c061b61fe713ccedbe4 +P 718da6de870231d358384473b40e81c4394b8067 +R 4772700da78a181e555d5cf639b019fb U dan -Z 86cf672a01ea748dc788d9b12ee5fdc1 +Z 93b575ac2dc5f2154ac020bd95667b18 diff --git a/manifest.uuid b/manifest.uuid index 1b666612e7..9eb7806eea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -718da6de870231d358384473b40e81c4394b8067 \ No newline at end of file +1e468fe1e408e513a1e1bbe72fe2a240f2991b3d \ No newline at end of file From 93fdbfa64f79f9699769b94d32f016ff0c3ebc04 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 19 Sep 2014 18:08:39 +0000 Subject: [PATCH 026/116] Add further tests to ota5.test. Add "ota.test", for running all ota tests. FossilOrigin-Name: 95ffdaa542df1c28fac97422e5a4b2c5cb81d50a --- ext/ota/ota1.test | 4 +- ext/ota/ota2.test | 6 ++- ext/ota/ota3.test | 4 +- ext/ota/ota4.test | 4 +- ext/ota/ota5.test | 89 +++++++++++++++++++++++++++++++++++++++++- manifest | 23 +++++------ manifest.uuid | 2 +- test/ota.test | 16 ++++++++ test/permutations.test | 6 +++ 9 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 test/ota.test diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index 60cc4fc30e..c55ec6ee3b 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -10,7 +10,9 @@ #*********************************************************************** # -set testdir [file join [file dirname $argv0] .. .. test] +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} source $testdir/tester.tcl set ::testprefix ota1 diff --git a/ext/ota/ota2.test b/ext/ota/ota2.test index 6ce6b0852f..8830f7461b 100644 --- a/ext/ota/ota2.test +++ b/ext/ota/ota2.test @@ -10,11 +10,13 @@ #*********************************************************************** # -set testdir [file join [file dirname $argv0] .. .. test] +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} source $testdir/tester.tcl set ::testprefix ota2 -forcedelete test.db-oal test.db-bak +forcedelete {*}[glob -nocomplain test.db?*] do_execsql_test 1.0 { CREATE TABLE t1(a, b); diff --git a/ext/ota/ota3.test b/ext/ota/ota3.test index dcf8f817aa..09d1bab083 100644 --- a/ext/ota/ota3.test +++ b/ext/ota/ota3.test @@ -10,7 +10,9 @@ #*********************************************************************** # -set testdir [file join [file dirname $argv0] .. .. test] +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} source $testdir/tester.tcl set ::testprefix ota3 diff --git a/ext/ota/ota4.test b/ext/ota/ota4.test index 98bb9330e3..b14225dc52 100644 --- a/ext/ota/ota4.test +++ b/ext/ota/ota4.test @@ -12,7 +12,9 @@ # Test some properties of the pager_ota_mode and ota_mode pragmas. # -set testdir [file join [file dirname $argv0] .. .. test] +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} source $testdir/tester.tcl set ::testprefix ota4 diff --git a/ext/ota/ota5.test b/ext/ota/ota5.test index 4f85c22b94..8b62eb0f66 100644 --- a/ext/ota/ota5.test +++ b/ext/ota/ota5.test @@ -12,7 +12,9 @@ # Test some properties of the pager_ota_mode and ota_mode pragmas. # -set testdir [file join [file dirname $argv0] .. .. test] +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} source $testdir/tester.tcl set ::testprefix ota5 @@ -235,6 +237,91 @@ foreach {tn idx} { #------------------------------------------------------------------------- # +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d, e); + INSERT INTO t1 VALUES(-750250,'fyetckfaagjkzqjx',-185831,X'FEAD',444258.29); + INSERT INTO t1 VALUES(649081,NULL,X'7DF25BF78778',-342324.63,'akvspktocwozo'); + INSERT INTO t1 VALUES(-133045,-44822.31,X'',287935,NULL); + INSERT INTO t1 VALUES(202132,NULL,X'5399','cujsjtspryqeyovcdpz','m'); + INSERT INTO t1 VALUES(302910,NULL,'dvdhivtfkaedzhdcnn',-717113.41,688487); + INSERT INTO t1 VALUES(-582327,X'7A267A',X'7E6B3CFE5CB9','zacuzilrok',-196478); + INSERT INTO t1 VALUES(-190462,X'D1A087E7D68D9578','lsmleti',NULL,-928094); + INSERT INTO t1 VALUES(-467665,176344.57,-536684.23,828876.22,X'903E'); + INSERT INTO t1 VALUES(-629138,632630.29,X'28D6',-774501,X'819BBBFC65'); + INSERT INTO t1 VALUES(-828110,-54379.24,-881121.44,X'',X'8D5A894F0D'); + + CREATE TABLE t2(a PRIMARY KEY, b, c, d, e) WITHOUT ROWID; + INSERT INTO t2 VALUES(-65174,X'AC1DBFFE27310F',-194471.08,347988,X'84041BA6F9BDDE86A8'); + INSERT INTO t2 VALUES('bzbpi',-952693.69,811628.25,NULL,-817434); + INSERT INTO t2 VALUES(-643830,NULL,'n',NULL,'dio'); + INSERT INTO t2 VALUES('rovoenxxj',NULL,'owupbtdcoxxnvg',-119676,X'55431DFA'); + INSERT INTO t2 VALUES(899770,'jlygdl',X'DBCA4D1A',NULL,-631773); + INSERT INTO t2 VALUES(334698.80,NULL,-697585.58,-89277,-817352); + INSERT INTO t2 VALUES(X'1A9EB7547A4AAF38','aiprdhkpzdz','anw','szvjbwdvzucybpwwqjt',X'53'); + INSERT INTO t2 VALUES(713220,NULL,'hfcqhqzjuqplvkum',X'20B076075649DE','fthgpvqdyy'); + INSERT INTO t2 VALUES(763908,NULL,'xgslzcpvwfknbr',X'75',X'668146'); + INSERT INTO t2 VALUES(X'E1BA2B6BA27278','wjbpd',NULL,139341,-290086.15); +} +db close + +set cksum [generate_ota_db test.db ota.db { +INSERT INTO t2 VALUES(222916.23,'idh',X'472C517405',X'E3',X'7C4F31824669'); +INSERT INTO t2 VALUES('xcndjwafcoxwxizoktd',-319567.21,NULL,-720906.43,-577170); +INSERT INTO t2 VALUES(376369.99,-536058,'yoaiurfqupdscwc',X'29EC8A2542EC3953E9',-740485.22); +INSERT INTO t2 VALUES(X'0EFB4DC50693',-175590.83,X'1779E253CAB5B1789E',X'BC6903',NULL); +INSERT INTO t2 VALUES(-288299,'hfrp',NULL,528477,730676.77); +DELETE FROM t2 WHERE a < -60000; + +UPDATE t2 SET b = 'pgnnaaoflnw' WHERE a = 'bzbpi'; +UPDATE t2 SET c = -675583 WHERE a = 'rovoenxxj'; +UPDATE t2 SET d = X'09CDF2B2C241' WHERE a = 713220; + +INSERT INTO t1 VALUES(224938,'bmruycvfznhhnfmgqys','fr',854381,789143); +INSERT INTO t1 VALUES(-863931,-1386.26,X'2A058540C2FB5C',NULL,X'F9D5990A'); +INSERT INTO t1 VALUES(673696,X'97301F0AC5735F44B5',X'440C',227999.92,-709599.79); +INSERT INTO t1 VALUES(-243640,NULL,-71718.11,X'1EEFEB38',X'8CC7C55D95E142FBA5'); +INSERT INTO t1 VALUES(275893,X'',375606.30,X'0AF9EC334711FB',-468194); +DELETE FROM t1 WHERE a > 200000; + +UPDATE t1 SET b = 'pgnnaaoflnw' WHERE a = -190462; +UPDATE t1 SET c = -675583 WHERE a = -467665; +UPDATE t1 SET d = X'09CDF2B2C241' WHERE a = -133045; + +}] + +foreach {tn idx} { + 1 { + } + 2 { + CREATE UNIQUE INDEX i1 ON t1(b, c, d); + CREATE UNIQUE INDEX i2 ON t1(d, e, a); + CREATE UNIQUE INDEX i3 ON t1(e, d, c, b); + + CREATE UNIQUE INDEX i4 ON t2(b, c, d); + CREATE UNIQUE INDEX i5 ON t2(d, e, a); + CREATE UNIQUE INDEX i6 ON t2(e, d, c, b); + } +} { + foreach cmd {run step} { + forcecopy test.db test.db2 + forcecopy ota.db ota.db2 + + sqlite3 db test.db2 + db eval $idx + + do_test 2.$tn.$cmd.1 { + ${cmd}_ota test.db2 ota.db2 + datacksum db + } $cksum + + do_test 2.$tn.$cmd.2 { + db eval { PRAGMA integrity_check } + } {ok} + + db close + } +} finish_test diff --git a/manifest b/manifest index f6a8d1cd9c..7cd0869d20 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sfor\sthe\sota\sextension. -D 2014-09-19T15:06:23.790 +C Add\sfurther\stests\sto\sota5.test.\sAdd\s"ota.test",\sfor\srunning\sall\sota\stests. +D 2014-09-19T18:08:39.681 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -123,11 +123,11 @@ F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt cb11e39bfeba952ac8896dab860ada9d54731fb8 F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 -F ext/ota/ota1.test 7cbf37a9f6cd29320f47b041cfeb0cc1d7eaa916 -F ext/ota/ota2.test 716f9c66e8bf8b0ad2fe3a5d8323e6cf460a2e27 -F ext/ota/ota3.test 1c48b7476af1c5920db9a43e7b1476d421a463b5 -F ext/ota/ota4.test baf23b47748a5056c713871959cc70fc623c90e9 -F ext/ota/ota5.test bd0e0f7609ece4eee2c62c5132935f0c80190cb8 +F ext/ota/ota1.test 86ff92699aad11e3c80a604832244a043c912a94 +F ext/ota/ota2.test 4568c2671d19dbde789fb9091d727a2e94880128 +F ext/ota/ota3.test 215dd4a8e238567e0f890a5139b6fdf5494ef311 +F ext/ota/ota4.test 60f897f329a6782ef2f24862640acf3c52e48077 +F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/sqlite3ota.c b22002105b3b7f3baf63bda2b4e6a00c4973418c F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -756,6 +756,7 @@ F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 +F test/ota.test d81a211dfbdf9fe02b5ad50c86f8a5df924391f5 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f @@ -768,7 +769,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test cef25f5e8499a15846eccd06785f17f4180407ab +F test/permutations.test 19cc048448340952b08a897167ec7b7bc85142b9 F test/pragma.test 19d0241a007bcdd77fc2606ec60fc60357e7fc8b F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1207,7 +1208,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 718da6de870231d358384473b40e81c4394b8067 -R 4772700da78a181e555d5cf639b019fb +P 1e468fe1e408e513a1e1bbe72fe2a240f2991b3d +R 44fa742e94ebd7f402187be149604025 U dan -Z 93b575ac2dc5f2154ac020bd95667b18 +Z 7bef28ffbee5fdb05a4896f797ef3928 diff --git a/manifest.uuid b/manifest.uuid index 9eb7806eea..788cd2449b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e468fe1e408e513a1e1bbe72fe2a240f2991b3d \ No newline at end of file +95ffdaa542df1c28fac97422e5a4b2c5cb81d50a \ No newline at end of file diff --git a/test/ota.test b/test/ota.test new file mode 100644 index 0000000000..676666d91f --- /dev/null +++ b/test/ota.test @@ -0,0 +1,16 @@ +# 2014 September 20 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file runs all rtree related tests. +# + +set testdir [file dirname $argv0] +source $testdir/permutations.test + +run_test_suite ota + +finish_test diff --git a/test/permutations.test b/test/permutations.test index 54239cc759..b3c9e58934 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -923,6 +923,12 @@ test_suite "rtree" -description { All R-tree related tests. Provides coverage of source file rtree.c. } -files [glob -nocomplain $::testdir/../ext/rtree/*.test] +test_suite "ota" -description { + OTA tests. +} -files [ + test_set [glob -nocomplain $::testdir/../ext/ota/*.test] -exclude ota.test +] + test_suite "no_optimization" -description { Run test scripts with optimizations disabled using the sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS) interface. From d2f5ee20f051e768a4883ce9c01b7725da1fb6a7 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Oct 2014 16:24:23 +0000 Subject: [PATCH 027/116] Have the ota extension perform an incremental checkpoint after generating the wal file. FossilOrigin-Name: 0bf1301aacb3b717b4cc020fbda9fab0bae331c3 --- ext/ota/ota1.test | 6 +- ext/ota/sqlite3ota.c | 535 ++++++++++++++++++++++++++++++------------- manifest | 30 +-- manifest.uuid | 2 +- src/main.c | 18 ++ src/pager.c | 20 +- src/pager.h | 1 + src/pragma.c | 2 +- src/sqlite.h.in | 34 +++ src/wal.c | 416 +++++++++++++++++++++++---------- src/wal.h | 9 + test/wal.test | 1 + 12 files changed, 770 insertions(+), 304 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index c55ec6ee3b..4ac469573f 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -124,12 +124,12 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota} { } { reset_db execsql $schema - + do_test 1.$tn2.$tn.1 { create_ota1 ota.db $cmd test.db ota.db } {SQLITE_DONE} - + do_execsql_test 1.$tn2.$tn.2 { SELECT * FROM t1 ORDER BY a ASC; } { @@ -137,7 +137,7 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota} { 2 two three 3 {} 8.2 } - + do_execsql_test 1.$tn2.$tn.3 { PRAGMA integrity_check } ok } } diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 18a71910e5..1d4ea1f859 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -22,21 +22,54 @@ /* ** The ota_state table is used to save the state of a partially applied -** update so that it can be resumed later. The table contains at most a -** single row: +** update so that it can be resumed later. The table consists of integer +** keys mapped to values as follows: ** -** "tbl" -> Table currently being written (target database names). +** OTA_STATE_STAGE: +** May be set to integer values 1, 2 or 3. As follows: +** 0: Nothing at all has been done. +** 1: the *-ota file is currently under construction. +** 2: the *-ota file has been constructed, but not yet moved +** to the *-wal path. +** 3: the checkpoint is underway. ** -** "idx" -> Index currently being written (target database names). -** Or, if the main table is being written, a NULL value. +** OTA_STATE_TBL: +** Only valid if STAGE==1. The target database name of the table +** currently being written. ** -** "row" -> Number of rows for this object already processed +** OTA_STATE_IDX: +** Only valid if STAGE==1. The target database name of the index +** currently being written, or NULL if the main table is currently being +** updated. +** +** OTA_STATE_ROW: +** Only valid if STAGE==1. Number of rows already processed for the current +** table/index. +** +** OTA_STATE_PROGRESS: +** Total number of sqlite3ota_step() calls made so far as part of this +** ota update. +** +** OTA_STATE_CKPT: +** Valid if STAGE==3. The blob to pass to sqlite3ckpt_start() to resume +** the incremental checkpoint. ** -** "progress" -> total number of key/value b-tree operations performed -** so far as part of this ota update. */ +#define OTA_STATE_STAGE 1 +#define OTA_STATE_TBL 2 +#define OTA_STATE_IDX 3 +#define OTA_STATE_ROW 4 +#define OTA_STATE_PROGRESS 5 +#define OTA_STATE_CKPT 6 + +#define OTA_STAGE_OAL 1 +#define OTA_STAGE_COPY 2 +#define OTA_STAGE_CKPT 3 +#define OTA_STAGE_DONE 4 + + #define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota.ota_state" \ - "(tbl, idx, row, progress)" + "(k INTEGER PRIMARY KEY, v)" typedef struct OtaState OtaState; typedef struct OtaObjIter OtaObjIter; @@ -45,8 +78,11 @@ typedef struct OtaObjIter OtaObjIter; ** A structure to store values read from the ota_state table in memory. */ struct OtaState { + int eStage; char *zTbl; char *zIdx; + unsigned char *pCkptState; + int nCkptState; int nRow; sqlite3_int64 nProgress; }; @@ -88,13 +124,16 @@ struct OtaObjIter { ** OTA handle. */ struct sqlite3ota { + int eStage; /* Value of OTA_STATE_STAGE field */ sqlite3 *db; /* "main" -> target db, "ota" -> ota db */ char *zTarget; /* Path to target db */ + char *zOta; /* Path to ota db */ int rc; /* Value returned by last ota_step() call */ char *zErrmsg; /* Error message if rc!=SQLITE_OK */ int nStep; /* Rows processed for current object */ int nProgress; /* Rows processed for all objects */ - OtaObjIter objiter; + OtaObjIter objiter; /* Iterator for skipping through tbl/idx */ + sqlite3_ckpt *pCkpt; /* Incr-checkpoint handle */ }; /* @@ -742,6 +781,84 @@ static int otaGetUpdateStmt( return p->rc; } +static void otaOpenDatabase(sqlite3ota *p){ + assert( p->rc==SQLITE_OK ); + sqlite3_close(p->db); + p->db = 0; + + p->rc = sqlite3_open(p->zTarget, &p->db); + if( p->rc ){ + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); + } + otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); +} + +/* +** This routine is a copy of the sqlite3FileSuffix3() routine from the core. +** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. +** +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. +** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa +*/ +static void otaFileSuffix3(const char *zBase, char *z){ +#ifdef SQLITE_ENABLE_8_3_NAMES +#if SQLITE_ENABLE_8_3_NAMES<2 + if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) +#endif + { + int i, sz; + sz = sqlite3Strlen30(z); + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); + } +#endif +} + +/* +** Move the "*-oal" file corresponding to the target database to the +** "*-wal" location. If an error occurs, leave an error code and error +** message in the ota handle. +*/ +static void otaMoveOalFile(sqlite3ota *p){ + const char *zBase = sqlite3_db_filename(p->db, "main"); + + char *zWal = sqlite3_mprintf("%s-wal", zBase); + char *zOal = sqlite3_mprintf("%s-oal", zBase); + + assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); + if( zWal==0 || zOal==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + /* Move the *-oal file to *-wal. At this point connection p->db is + ** holding a SHARED lock on the target database file (because it is + ** in WAL mode). So no other connection may be writing the db. */ + otaFileSuffix3(zBase, zWal); + otaFileSuffix3(zBase, zOal); + rename(zOal, zWal); + + /* Re-open the databases. */ + otaObjIterFinalize(&p->objiter); + otaOpenDatabase(p); + p->eStage = OTA_STAGE_CKPT; + } + + sqlite3_free(zWal); + sqlite3_free(zOal); +} + /* ** The SELECT statement iterating through the keys for the current object ** (p->objiter.pSelect) currently points to a valid row. This function @@ -863,51 +980,170 @@ static int otaStep(sqlite3ota *p){ return p->rc; } +/* +** Increment the schema cookie of the main database opened by p->db. +*/ +static void otaIncrSchemaCookie(sqlite3ota *p){ + int iCookie = 1000000; + sqlite3_stmt *pStmt; + + assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); + p->rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, + "PRAGMA schema_version" + ); + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + iCookie = sqlite3_column_int(pStmt, 0); + } + p->rc = sqlite3_finalize(pStmt); + } + if( p->rc==SQLITE_OK ){ + otaMPrintfExec(p, "PRAGMA schema_version = %d", iCookie+1); + } +} + /* ** Step the OTA object. */ int sqlite3ota_step(sqlite3ota *p){ if( p ){ - OtaObjIter *pIter = &p->objiter; - while( p && p->rc==SQLITE_OK && pIter->zTbl ){ + switch( p->eStage ){ + case OTA_STAGE_OAL: { + OtaObjIter *pIter = &p->objiter; + while( p && p->rc==SQLITE_OK && pIter->zTbl ){ - if( pIter->bCleanup ){ - /* Clean up the ota_tmp_xxx table for the previous table. It - ** cannot be dropped as there are currently active SQL statements. - ** But the contents can be deleted. */ - otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); - }else{ - otaObjIterPrepareAll(p, pIter, 0); - - /* Advance to the next row to process. */ - if( p->rc==SQLITE_OK ){ - int rc = sqlite3_step(pIter->pSelect); - if( rc==SQLITE_ROW ){ - p->nStep++; - p->nProgress++; - return otaStep(p); + if( pIter->bCleanup ){ + /* Clean up the ota_tmp_xxx table for the previous table. It + ** cannot be dropped as there are currently active SQL statements. + ** But the contents can be deleted. */ + otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); + }else{ + otaObjIterPrepareAll(p, pIter, 0); + + /* Advance to the next row to process. */ + if( p->rc==SQLITE_OK ){ + int rc = sqlite3_step(pIter->pSelect); + if( rc==SQLITE_ROW ){ + p->nProgress++; + p->nStep++; + return otaStep(p); + } + p->rc = sqlite3_reset(pIter->pSelect); + p->nStep = 0; + } } - p->rc = sqlite3_reset(pIter->pSelect); - p->nStep = 0; + + otaObjIterNext(p, pIter); } + + if( p->rc==SQLITE_OK && pIter->zTbl==0 ){ + p->nProgress++; + otaIncrSchemaCookie(p); + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); + } + if( p->rc==SQLITE_OK ){ + otaMoveOalFile(p); + } + } + break; } - otaObjIterNext(p, pIter); - } + case OTA_STAGE_CKPT: { - if( p->rc==SQLITE_OK && pIter->zTbl==0 ){ - p->rc = SQLITE_DONE; + if( p->rc==SQLITE_OK && p->pCkpt==0 ){ + p->rc = sqlite3_ckpt_open(p->db, 0, 0, &p->pCkpt); + } + if( p->rc==SQLITE_OK ){ + if( SQLITE_OK!=sqlite3_ckpt_step(p->pCkpt) ){ + p->rc = sqlite3_ckpt_close(p->pCkpt, 0, 0); + p->pCkpt = 0; + if( p->rc==SQLITE_OK ){ + p->eStage = OTA_STAGE_DONE; + p->rc = SQLITE_DONE; + } + } + p->nProgress++; + } + + break; + } + + default: + break; } } return p->rc; } static void otaSaveTransactionState(sqlite3ota *p){ - otaMPrintfExec(p, - "INSERT OR REPLACE INTO ota.ota_state(rowid, tbl, idx, row, progress)" - "VALUES(1, %Q, %Q, %d, %lld)", - p->objiter.zTbl, p->objiter.zIdx, p->nStep, p->nProgress + sqlite3_stmt *pInsert; + int rc; + + assert( (p->rc==SQLITE_OK || p->rc==SQLITE_DONE) && p->zErrmsg==0 ); + rc = prepareFreeAndCollectError(p->db, &pInsert, &p->zErrmsg, + sqlite3_mprintf( + "INSERT OR REPLACE INTO ota.ota_state(k, v) VALUES " + "(%d, %d), " + "(%d, %Q), " + "(%d, %Q), " + "(%d, %d), " + "(%d, %lld), " + "(%d, ?) ", + OTA_STATE_STAGE, p->eStage, + OTA_STATE_TBL, p->objiter.zTbl, + OTA_STATE_IDX, p->objiter.zIdx, + OTA_STATE_ROW, p->nStep, + OTA_STATE_PROGRESS, p->nProgress, + OTA_STATE_CKPT + ) ); + assert( pInsert==0 || rc==SQLITE_OK ); + if( rc==SQLITE_OK ){ + if( p->pCkpt ){ + unsigned char *pCkptState = 0; + int nCkptState = 0; + rc = sqlite3_ckpt_close(p->pCkpt, &pCkptState, &nCkptState); + p->pCkpt = 0; + sqlite3_bind_blob(pInsert, 1, pCkptState, nCkptState, SQLITE_TRANSIENT); + sqlite3_free(pCkptState); + } + } + if( rc==SQLITE_OK ){ + sqlite3_step(pInsert); + rc = sqlite3_finalize(pInsert); + }else{ + sqlite3_finalize(pInsert); + } + + if( rc!=SQLITE_OK ){ + p->rc = rc; + } +} + +static char *otaStrndup(char *zStr, int nStr, int *pRc){ + char *zRet = 0; + assert( *pRc==SQLITE_OK ); + + if( zStr ){ + int nCopy = nStr; + if( nCopy<0 ) nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; + } + } + + return zRet; +} + +static void otaFreeState(OtaState *p){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p->pCkptState); + sqlite3_free(p); } /* @@ -920,47 +1156,63 @@ static void otaSaveTransactionState(sqlite3ota *p){ ** and return NULL. */ static OtaState *otaLoadState(sqlite3ota *p){ - const char *zSelect = "SELECT tbl, idx, row, progress FROM ota.ota_state"; + const char *zSelect = "SELECT k, v FROM ota.ota_state"; OtaState *pRet = 0; sqlite3_stmt *pStmt; int rc; + int rc2; assert( p->rc==SQLITE_OK ); - rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, zSelect); - if( rc==SQLITE_OK ){ - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - const char *zIdx = (const char*)sqlite3_column_text(pStmt, 1); - const char *zTbl = (const char*)sqlite3_column_text(pStmt, 0); - int nIdx = zIdx ? (strlen(zIdx) + 1) : 0; - int nTbl = strlen(zTbl) + 1; - int nByte = sizeof(OtaState) + nTbl + nIdx; + pRet = (OtaState*)sqlite3_malloc(sizeof(OtaState)); + if( pRet==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, sizeof(OtaState)); + rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, zSelect); + } - pRet = (OtaState*)sqlite3_malloc(nByte); - if( pRet ){ - pRet->zTbl = (char*)&pRet[1]; - memcpy(pRet->zTbl, sqlite3_column_text(pStmt, 0), nTbl); - if( zIdx ){ - pRet->zIdx = &pRet->zTbl[nTbl]; - memcpy(pRet->zIdx, zIdx, nIdx); - }else{ - pRet->zIdx = 0; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt, 0) ){ + case OTA_STATE_STAGE: + pRet->eStage = sqlite3_column_int(pStmt, 1); + if( pRet->eStage!=OTA_STAGE_OAL + && pRet->eStage!=OTA_STAGE_COPY + && pRet->eStage!=OTA_STAGE_CKPT + ){ + p->rc = SQLITE_CORRUPT; } - pRet->nRow = sqlite3_column_int(pStmt, 2); - pRet->nProgress = sqlite3_column_int64(pStmt, 3); - } - }else{ - pRet = (OtaState*)sqlite3_malloc(sizeof(OtaState)); - if( pRet ){ - memset(pRet, 0, sizeof(*pRet)); - } - } - rc = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK && pRet==0 ) rc = SQLITE_NOMEM; - if( rc!=SQLITE_OK ){ - sqlite3_free(pRet); - pRet = 0; + break; + + case OTA_STATE_TBL: + pRet->zTbl = otaStrndup((char*)sqlite3_column_text(pStmt, 1), -1, &rc); + break; + + case OTA_STATE_IDX: + pRet->zIdx = otaStrndup((char*)sqlite3_column_text(pStmt, 1), -1, &rc); + break; + + case OTA_STATE_ROW: + pRet->nRow = sqlite3_column_int(pStmt, 1); + break; + + case OTA_STATE_PROGRESS: + pRet->nProgress = sqlite3_column_int64(pStmt, 1); + break; + + case OTA_STATE_CKPT: + pRet->nCkptState = sqlite3_column_bytes(pStmt, 1); + pRet->pCkptState = (unsigned char*)otaStrndup( + (char*)sqlite3_column_blob(pStmt, 1), pRet->nCkptState, &rc + ); + break; + + default: + rc = SQLITE_CORRUPT; + break; } } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; p->rc = rc; return pRet; @@ -999,62 +1251,6 @@ static void otaLoadTransactionState(sqlite3ota *p, OtaState *pState){ } } -/* -** This routine is a copy of the sqlite3FileSuffix3() routine from the core. -** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. -** -** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database -** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and -** if filename in z[] has a suffix (a.k.a. "extension") that is longer than -** three characters, then shorten the suffix on z[] to be the last three -** characters of the original suffix. -** -** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always -** do the suffix shortening regardless of URI parameter. -** -** Examples: -** -** test.db-journal => test.nal -** test.db-wal => test.wal -** test.db-shm => test.shm -** test.db-mj7f3319fa => test.9fa -*/ -static void otaFileSuffix3(const char *zBase, char *z){ -#ifdef SQLITE_ENABLE_8_3_NAMES -#if SQLITE_ENABLE_8_3_NAMES<2 - if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) -#endif - { - int i, sz; - sz = sqlite3Strlen30(z); - for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} - if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); - } -#endif -} - -/* -** Move the "*-oal" file corresponding to the target database to the -** "*-wal" location. If an error occurs, leave an error code and error -** message in the ota handle. -*/ -static void otaMoveOalFile(const char *zBase, sqlite3ota *p){ - char *zWal = sqlite3_mprintf("%s-wal", p->zTarget); - char *zOal = sqlite3_mprintf("%s-oal", p->zTarget); - - assert( p->rc==SQLITE_DONE && p->zErrmsg==0 ); - if( zWal==0 || zOal==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - otaFileSuffix3(zBase, zWal); - otaFileSuffix3(zBase, zOal); - rename(zOal, zWal); - } - - sqlite3_free(zWal); - sqlite3_free(zOal); -} - /* ** If there is a "*-oal" file in the file-system corresponding to the ** target database in the file-system, delete it. If an error occurs, @@ -1073,8 +1269,9 @@ static void otaDeleteOalFile(sqlite3ota *p){ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ sqlite3ota *p; int nTarget = strlen(zTarget); + int nOta = strlen(zOta); - p = (sqlite3ota*)sqlite3_malloc(sizeof(sqlite3ota)+nTarget+1); + p = (sqlite3ota*)sqlite3_malloc(sizeof(sqlite3ota)+nTarget+1+nOta+1); if( p ){ OtaState *pState = 0; @@ -1082,11 +1279,9 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ memset(p, 0, sizeof(sqlite3ota)); p->zTarget = (char*)&p[1]; memcpy(p->zTarget, zTarget, nTarget+1); - p->rc = sqlite3_open(zTarget, &p->db); - if( p->rc ){ - p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); - } - otaMPrintfExec(p, "ATTACH %Q AS ota", zOta); + p->zOta = &p->zTarget[nTarget+1]; + memcpy(p->zOta, zOta, nOta+1); + otaOpenDatabase(p); /* If it has not already been created, create the ota_state table */ if( p->rc==SQLITE_OK ){ @@ -1095,32 +1290,45 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ if( p->rc==SQLITE_OK ){ pState = otaLoadState(p); - if( pState && pState->zTbl==0 ){ - otaDeleteOalFile(p); + assert( pState || p->rc!=SQLITE_OK ); + if( pState ){ + if( pState->eStage==0 ){ + otaDeleteOalFile(p); + p->eStage = 1; + }else{ + p->eStage = pState->eStage; + } + p->nProgress = pState->nProgress; } } + assert( p->rc!=SQLITE_OK || p->eStage!=0 ); - if( p->rc==SQLITE_OK ){ - const char *zScript = - "PRAGMA journal_mode=off;" - "PRAGMA pager_ota_mode=1;" - "PRAGMA ota_mode=1;" - "BEGIN IMMEDIATE;" - ; - p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); + if( p->eStage==OTA_STAGE_OAL ){ + if( p->rc==SQLITE_OK ){ + const char *zScript = + "PRAGMA journal_mode=off;" + "PRAGMA pager_ota_mode=1;" + "PRAGMA ota_mode=1;" + "BEGIN IMMEDIATE;" + ; + p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); + } + + /* Point the object iterator at the first object */ + if( p->rc==SQLITE_OK ){ + p->rc = otaObjIterFirst(p, &p->objiter); + } + + if( p->rc==SQLITE_OK ){ + otaLoadTransactionState(p, pState); + } + }else if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_CKPT ){ + p->rc = sqlite3_ckpt_open( + p->db, pState->pCkptState, pState->nCkptState, &p->pCkpt + ); } - /* Point the object iterator at the first object */ - if( p->rc==SQLITE_OK ){ - p->rc = otaObjIterFirst(p, &p->objiter); - } - - if( p->rc==SQLITE_OK ){ - p->nProgress = pState->nProgress; - otaLoadTransactionState(p, pState); - } - - sqlite3_free(pState); + otaFreeState(pState); } return p; @@ -1132,35 +1340,32 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ int rc; if( p ){ - const char *zBase = sqlite3_db_filename(p->db, "main"); /* If the update has not been fully applied, save the state in ** the ota db. If successful, this call also commits the open ** transaction on the ota db. */ assert( p->rc!=SQLITE_ROW ); - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ assert( p->zErrmsg==0 ); otaSaveTransactionState(p); } - /* Close all open statement handles. */ + /* Close any open statement handles. */ otaObjIterFinalize(&p->objiter); /* Commit the transaction to the *-oal file. */ - if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ - rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); - if( rc!=SQLITE_OK ) p->rc = rc; + if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_OAL ){ + p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); } - /* Close the open database handles */ + if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_CKPT ){ + p->rc = sqlite3_exec(p->db, "PRAGMA pager_ota_mode=2", 0, 0, &p->zErrmsg); + } + + /* Close the open database handle */ + if( p->pCkpt ) sqlite3_ckpt_close(p->pCkpt, 0, 0); sqlite3_close(p->db); - /* If the OTA has been completely applied and no error occurred, move - ** the *-oal file to *-wal. */ - if( p->rc==SQLITE_DONE ){ - otaMoveOalFile(zBase, p); - } - rc = p->rc; *pzErrmsg = p->zErrmsg; sqlite3_free(p); diff --git a/manifest b/manifest index 7cd0869d20..373d1c020a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sto\sota5.test.\sAdd\s"ota.test",\sfor\srunning\sall\sota\stests. -D 2014-09-19T18:08:39.681 +C Have\sthe\sota\sextension\sperform\san\sincremental\scheckpoint\safter\sgenerating\sthe\swal\sfile. +D 2014-10-20T16:24:23.616 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -123,12 +123,12 @@ F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt cb11e39bfeba952ac8896dab860ada9d54731fb8 F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 -F ext/ota/ota1.test 86ff92699aad11e3c80a604832244a043c912a94 +F ext/ota/ota1.test 0c8e5ef3d059bebe7872477381424836326a8e0a F ext/ota/ota2.test 4568c2671d19dbde789fb9091d727a2e94880128 F ext/ota/ota3.test 215dd4a8e238567e0f890a5139b6fdf5494ef311 F ext/ota/ota4.test 60f897f329a6782ef2f24862640acf3c52e48077 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb -F ext/ota/sqlite3ota.c b22002105b3b7f3baf63bda2b4e6a00c4973418c +F ext/ota/sqlite3ota.c 1b4e4cdb05e67982865466f4b0aaea8d3648269c F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -202,7 +202,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c d15621461fb0c52675eba2b650492ed1beef69ab +F src/main.c 57cdf37a1bf596829831530fe02f2649d4b721e3 F src/malloc.c 4c1d511157defd7b1d023062cf05a1dc17b8f79b F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -223,13 +223,13 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c addd023b26c623fec4dedc110fc4370a65b4768c F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 348c9da924d2e0064e6a2646bba157e02dcc26cc -F src/pager.h b62e645e8a19e4f0181253d1663a09f2793d8c94 +F src/pager.c 59c1d41bba7dd736f27fdcf74691675aa685a048 +F src/pager.h 997a4aa3bad1638dabe90a0cbb674cc4a7b9c034 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa -F src/pragma.c 5b255c09d6e38a37ec07830b92acceec5cab8c85 +F src/pragma.c 310939bc2fb7e6456edfb4735d004253a4b2505e F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 19e3e81addf593195369ec8d487ed063ad3170bb F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -237,7 +237,7 @@ F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 0cd6706fd52ae5db229e9041094db6ec27195335 F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d -F src/sqlite.h.in a98b0429855f023074f44a63dba23d11469ebc6d +F src/sqlite.h.in f34298ae5de26aebfba0c5ce91590d62ddebc6cb F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 3210f8bd040d1c6d8b1616325b15dd3ff749e48f @@ -307,8 +307,8 @@ F src/vdbemem.c 8b5e1083fed2da94e315858a7edf5604a5b91804 F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f -F src/wal.c 8bd0ced6cf1d3389fd6a73b4f12a1e2bf926e75a -F src/wal.h e25f9d383ffb07986ba20b78dbde2c1d0cb36ab6 +F src/wal.c a5dbbbd8ceccd5e2187b1e7854f359cb5efb7e3b +F src/wal.h 0d3ba0c3f1b4c25796cb213568a84b9f9063f465 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c dc276288039fb45ce23c80e4535980f5a152d8ec F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c @@ -1095,7 +1095,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 -F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc +F test/wal.test d7bb2feeacb74d26f27ebe519334b95adf22c8ae F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1208,7 +1208,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1e468fe1e408e513a1e1bbe72fe2a240f2991b3d -R 44fa742e94ebd7f402187be149604025 +P 95ffdaa542df1c28fac97422e5a4b2c5cb81d50a +R febab191432990759e75444c2ec5fc2f U dan -Z 7bef28ffbee5fdb05a4896f797ef3928 +Z 5341e3998131d826ad56eb5ef99fe7b1 diff --git a/manifest.uuid b/manifest.uuid index 788cd2449b..420e80d994 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95ffdaa542df1c28fac97422e5a4b2c5cb81d50a \ No newline at end of file +0bf1301aacb3b717b4cc020fbda9fab0bae331c3 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 231890de4b..a8f3ee67b9 100644 --- a/src/main.c +++ b/src/main.c @@ -1768,6 +1768,24 @@ int sqlite3_wal_checkpoint_v2( #endif } +int sqlite3_ckpt_open( + sqlite3 *db, + unsigned char *a, int n, + sqlite3_ckpt **ppCkpt +){ + Pager *pPager; + Btree *pBt; + int rc; + + *ppCkpt = 0; + sqlite3_mutex_enter(db->mutex); + pBt = db->aDb[0].pBt; + pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerWalCheckpointStart(db, pPager, a, n, ppCkpt); + sqlite3_mutex_leave(db->mutex); + return rc; +} + /* ** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points diff --git a/src/pager.c b/src/pager.c index f9eba7fd7b..09a132e120 100644 --- a/src/pager.c +++ b/src/pager.c @@ -623,6 +623,8 @@ struct PagerSavepoint { ** database file has not been modified since it was created, this variable ** is set to 2. ** +** noCkptOnClose +** ** */ struct Pager { @@ -7288,12 +7290,24 @@ int sqlite3PagerWalFramesize(Pager *pPager){ /* ** Set or clear the "OTA mode" flag. */ -int sqlite3PagerSetOtaMode(Pager *pPager, int bOta){ - if( pPager->pWal || pPager->eState!=PAGER_OPEN ){ +int sqlite3PagerSetOtaMode(Pager *pPager, int iOta){ + if( iOta==1 && (pPager->pWal || pPager->eState!=PAGER_OPEN) ){ return SQLITE_ERROR; } - pPager->otaMode = 1; + pPager->otaMode = iOta; return SQLITE_OK; } +int sqlite3PagerWalCheckpointStart( + sqlite3 *db, + Pager *pPager, + u8 *a, int n, + sqlite3_ckpt **ppCkpt +){ + return sqlite3WalCheckpointStart(db, pPager->pWal, a, n, + pPager->xBusyHandler, pPager->pBusyHandlerArg, + pPager->ckptSyncFlags, ppCkpt + ); +} + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index 0e928fe64c..282da5765d 100644 --- a/src/pager.h +++ b/src/pager.h @@ -209,5 +209,6 @@ void *sqlite3PagerCodec(DbPage *); int sqlite3PagerSetOtaMode(Pager *pPager, int bOta); void sqlite3PagerWalSalt(Pager *pPager, u32 *aSalt); +int sqlite3PagerWalCheckpointStart(sqlite3*, Pager*, u8*, int, sqlite3_ckpt**); #endif /* _PAGER_H_ */ diff --git a/src/pragma.c b/src/pragma.c index ee99002274..c3566e7fd9 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -900,7 +900,7 @@ void sqlite3Pragma( Btree *pBt = pDb->pBt; assert( pBt!=0 ); if( zRight ){ - int iArg = !!sqlite3Atoi(zRight); + int iArg = sqlite3Atoi(zRight); if( sqlite3BtreeIsInReadTrans(pBt) ){ sqlite3ErrorMsg(pParse, "cannot set pager_ota_mode with open transaction" diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2ad571ed0b..0635f1cae2 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7464,6 +7464,40 @@ int sqlite3_index_writer( int **paiCol, int *pnCol /* OUT: See above */ ); +/* +** Incremental checkpoint API. +** +** An incremental checkpoint handle is opened using the sqlite3_ckpt_open() +** API. To begin a new checkpoint, the second and third arguments should both +** be passed zero. To resume an earlier checkpoint, the second and third +** arguments should specify a buffer returned by an earlier call to +** sqlite3_ckpt_close(). When resuming a checkpoint, if the database or WAL +** file has been modified since the checkpoint was suspended, the +** sqlite3_ckpt_open() call fails with SQLITE_MISMATCH. +** +** Each time sqlite3_ckpt_step() is called on an open checkpoint handle, a +** single page is copied from the WAL file to the database. If no error +** occurs, but the checkpoint is not finished, SQLITE_OK is returned. If the +** checkpoint has been finished (and so sqlite3_ckpt_step() should not be +** called again), SQLITE_DONE is returned. Otherwise, if an error occurs, +** some other SQLite error code is returned. +** +** Calling sqlite3_ckpt_close() closes an open checkpoint handle. If the +** checkpoint has finished and no error has occurred, SQLITE_OK is returned +** and the two output parameters zeroed. Or, if an error has occurred, an +** error code is returned and the two output parameters are zeroed. Finally, +** if the checkpoint is not finished but no error has occurred, SQLITE_OK is +** returned and the first output variable set to point to a buffer allocated +** using sqlite3_malloc() containing the serialized state of the checkpoint. +** The contents of this buffer may be passed to a later call to +** sqlite3_ckpt_open() to restart the checkpoint. The second output variable +** is set to the size of the buffer in bytes. +*/ +typedef struct sqlite3_ckpt sqlite3_ckpt; +int sqlite3_ckpt_open(sqlite3*, unsigned char *a, int n, sqlite3_ckpt **ppCkpt); +int sqlite3_ckpt_step(sqlite3_ckpt*); +int sqlite3_ckpt_close(sqlite3_ckpt*, unsigned char **pa, int *pn); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/wal.c b/src/wal.c index c7ad2bcb2c..dcf88ff7d8 100644 --- a/src/wal.c +++ b/src/wal.c @@ -482,6 +482,24 @@ struct WalIterator { } aSegment[1]; /* One for every 32KB page in the wal-index */ }; +/* +** walCheckpoint +*/ +typedef struct WalCkpt WalCkpt; +struct WalCkpt { + sqlite3 *db; /* Database pointer (incremental only) */ + int szPage; /* Database page-size */ + int sync_flags; /* Flags for OsSync() (or 0) */ + u32 mxSafeFrame; /* Max frame that can be backfilled */ + u32 mxPage; /* Max database page to write */ + volatile WalCkptInfo *pInfo; /* The checkpoint status information */ + WalIterator *pIter; /* Wal iterator context */ + Wal *pWal; /* Pointer to owner object */ + u8 *aBuf; /* Temporary page-sized buffer to use */ + int rc; /* Error code. SQLITE_DONE -> finished */ + int nStep; /* Number of times pIter has been stepped */ +}; + /* ** Define the parameters of the hash tables in the wal-index file. There ** is a hash-table following every HASHTABLE_NPAGE page numbers in the @@ -1623,6 +1641,155 @@ static int walPagesize(Wal *pWal){ return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); } +static int walCheckpointStart( + Wal *pWal, + u8 *aBuf, /* Page-sized temporary buffer */ + int nBuf, /* Size of aBuf[] in bytes */ + int (*xBusy)(void*), /* Function to call when busy (or NULL) */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags for OsSync() (or 0) */ + WalCkpt *p /* Allocated object to populate */ +){ + int rc; /* Return code */ + int i; /* Iterator variable */ + + memset(p, 0, sizeof(WalCkpt)); + + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ + return SQLITE_CORRUPT_BKPT; + } + + p->szPage = walPagesize(pWal); + p->pWal = pWal; + p->aBuf = aBuf; + p->sync_flags = sync_flags; + testcase( p->szPage<=32768 ); + testcase( p->szPage>=65536 ); + p->pInfo = walCkptInfo(pWal); + if( p->pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK; + + /* Allocate the iterator */ + rc = walIteratorInit(pWal, &p->pIter); + if( rc!=SQLITE_OK ) return rc; + assert( p->pIter ); + + /* Compute in mxSafeFrame the index of the last frame of the WAL that is + ** safe to write into the database. Frames beyond mxSafeFrame might + ** overwrite database pages that are in use by active readers and thus + ** cannot be backfilled from the WAL. + */ + p->mxSafeFrame = pWal->hdr.mxFrame; + p->mxPage = pWal->hdr.nPage; + for(i=1; ipInfo->aReadMark[i]; + if( p->mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + p->pInfo->aReadMark[i] = (i==1 ? p->mxSafeFrame : READMARK_NOT_USED); + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + p->mxSafeFrame = y; + xBusy = 0; + }else{ + walIteratorFree(p->pIter); + p->pIter = 0; + return rc; + } + } + } + + if( p->pInfo->nBackfill>=p->mxSafeFrame + || (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0), 1))!=SQLITE_OK + ){ + walIteratorFree(p->pIter); + p->pIter = 0; + } + if( rc==SQLITE_BUSY ) rc = SQLITE_OK; + + if( rc==SQLITE_OK && p->pIter ){ + /* Sync the WAL to disk */ + if( sync_flags ){ + rc = sqlite3OsSync(pWal->pWalFd, sync_flags); + } + + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. */ + if( rc==SQLITE_OK ){ + i64 nSize; /* Current size of database file */ + i64 nReq = ((i64)p->mxPage * p->szPage); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } + } + } + + return rc; +} + +static int walCheckpointStep(WalCkpt *p){ + u32 iDbpage = 0; /* Next database page to write */ + u32 iFrame = 0; /* Wal frame containing data for iDbpage */ + int rc = SQLITE_DONE; + + assert( p->rc==SQLITE_OK ); + while( p->pIter && 0==walIteratorNext(p->pIter, &iDbpage, &iFrame) ){ + i64 iOffset; + assert( walFramePgno(p->pWal, iFrame)==iDbpage ); + p->nStep++; + if( iFrame<=p->pInfo->nBackfill + || iFrame>p->mxSafeFrame + || iDbpage>p->mxPage + ){ + continue; + } + + iOffset = walFrameOffset(iFrame, p->szPage) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ + rc = sqlite3OsRead(p->pWal->pWalFd, p->aBuf, p->szPage, iOffset); + if( rc!=SQLITE_OK ) break; + iOffset = (iDbpage-1)*(i64)p->szPage; + testcase( IS_BIG_INT(iOffset) ); + rc = sqlite3OsWrite(p->pWal->pDbFd, p->aBuf, p->szPage, iOffset); + break; + } + + p->rc = rc; + return rc; +} + +static int walCheckpointFinalize(WalCkpt *p){ + if( p->pIter ){ + int rc = p->rc; + Wal *pWal = p->pWal; + + /* If work was completed */ + if( p->pIter && rc==SQLITE_DONE ){ + rc = SQLITE_OK; + if( p->mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ + i64 szDb = pWal->hdr.nPage*(i64)p->szPage; + testcase( IS_BIG_INT(szDb) ); + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); + if( rc==SQLITE_OK && p->sync_flags ){ + rc = sqlite3OsSync(pWal->pDbFd, p->sync_flags); + } + } + if( rc==SQLITE_OK ){ + p->pInfo->nBackfill = p->mxSafeFrame; + } + p->rc = rc; + } + + /* Release the reader lock held while backfilling */ + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); + walIteratorFree(p->pIter); + p->pIter = 0; + } + + return p->rc; +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -1660,120 +1827,23 @@ static int walCheckpoint( int (*xBusyCall)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ - u8 *zBuf /* Temporary buffer to use */ + u8 *zBuf, /* Temporary buffer to use */ + int nBuf /* Size of zBuf in bytes */ ){ int rc; /* Return code */ - int szPage; /* Database page-size */ - WalIterator *pIter = 0; /* Wal iterator context */ - u32 iDbpage = 0; /* Next database page to write */ - u32 iFrame = 0; /* Wal frame containing data for iDbpage */ - u32 mxSafeFrame; /* Max frame that can be backfilled */ - u32 mxPage; /* Max database page to write */ - int i; /* Loop counter */ - volatile WalCkptInfo *pInfo; /* The checkpoint status information */ int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ - - szPage = walPagesize(pWal); - testcase( szPage<=32768 ); - testcase( szPage>=65536 ); - pInfo = walCkptInfo(pWal); - if( pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK; - - /* Allocate the iterator */ - rc = walIteratorInit(pWal, &pIter); - if( rc!=SQLITE_OK ){ - return rc; - } - assert( pIter ); + WalCkpt sC; if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; + rc = walCheckpointStart(pWal, zBuf, nBuf, xBusy, pBusyArg, sync_flags, &sC); + if( sC.pIter==0 ) goto walcheckpoint_out; + assert( rc==SQLITE_OK ); - /* Compute in mxSafeFrame the index of the last frame of the WAL that is - ** safe to write into the database. Frames beyond mxSafeFrame might - ** overwrite database pages that are in use by active readers and thus - ** cannot be backfilled from the WAL. - */ - mxSafeFrame = pWal->hdr.mxFrame; - mxPage = pWal->hdr.nPage; - for(i=1; iaReadMark[i]; - if( mxSafeFrame>y ){ - assert( y<=pWal->hdr.mxFrame ); - rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); - if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - }else if( rc==SQLITE_BUSY ){ - mxSafeFrame = y; - xBusy = 0; - }else{ - goto walcheckpoint_out; - } - } - } - - if( pInfo->nBackfillnBackfill; - - /* Sync the WAL to disk */ - if( sync_flags ){ - rc = sqlite3OsSync(pWal->pWalFd, sync_flags); - } - - /* If the database may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. - */ - if( rc==SQLITE_OK ){ - i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); - } - } - - - /* Iterate through the contents of the WAL, copying data to the db file. */ - while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ - i64 iOffset; - assert( walFramePgno(pWal, iFrame)==iDbpage ); - if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; - iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ - rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - iOffset = (iDbpage-1)*(i64)szPage; - testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - } - - /* If work was actually accomplished... */ - if( rc==SQLITE_OK ){ - if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ - i64 szDb = pWal->hdr.nPage*(i64)szPage; - testcase( IS_BIG_INT(szDb) ); - rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - if( rc==SQLITE_OK && sync_flags ){ - rc = sqlite3OsSync(pWal->pDbFd, sync_flags); - } - } - if( rc==SQLITE_OK ){ - pInfo->nBackfill = mxSafeFrame; - } - } - - /* Release the reader lock held while backfilling */ - walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); - } - - if( rc==SQLITE_BUSY ){ - /* Reset the return code so as not to report a checkpoint failure - ** just because there are active readers. */ - rc = SQLITE_OK; - } + /* Step the checkpoint object until it reports something other than + ** SQLITE_OK. */ + while( SQLITE_OK==(rc = walCheckpointStep(&sC)) ); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + rc = walCheckpointFinalize(&sC); /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal ** file has been copied into the database file, then block until all @@ -1782,10 +1852,10 @@ static int walCheckpoint( */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); - if( pInfo->nBackfillhdr.mxFrame ){ + if( sC.pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ - assert( mxSafeFrame==pWal->hdr.mxFrame ); + assert( sC.mxSafeFrame==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); @@ -1794,7 +1864,7 @@ static int walCheckpoint( } walcheckpoint_out: - walIteratorFree(pIter); + walIteratorFree(sC.pIter); return rc; } @@ -2971,11 +3041,7 @@ int sqlite3WalCheckpoint( /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ - if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ - rc = SQLITE_CORRUPT_BKPT; - }else{ - rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf); - } + rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf, nBuf); /* If no error occurred, set the output variables. */ if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ @@ -3002,6 +3068,124 @@ int sqlite3WalCheckpoint( return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); } +int sqlite3_ckpt_step(sqlite3_ckpt *pCkpt){ + int rc; + WalCkpt *p = (WalCkpt*)pCkpt; + sqlite3_mutex_enter(p->db->mutex); + rc = walCheckpointStep(p); + sqlite3_mutex_leave(p->db->mutex); + return rc; +} + +int sqlite3_ckpt_close(sqlite3_ckpt *pCkpt, u8 **paState, int *pnState){ + int rc; + WalCkpt *p = (WalCkpt*)pCkpt; + sqlite3 *db = p->db; + Wal *pWal = p->pWal; + sqlite3_mutex_enter(db->mutex); + if( paState ){ + *paState = 0; + *pnState = 0; + if( p->rc==SQLITE_OK ){ + u8 *aState = sqlite3_malloc(sizeof(u32) * 3); + if( aState==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + *pnState = sizeof(u32)*3; + sqlite3Put4byte(&aState[0], p->nStep); + sqlite3Put4byte(&aState[4], p->pWal->hdr.aCksum[0]); + sqlite3Put4byte(&aState[8], p->pWal->hdr.aCksum[1]); + *paState = aState; + } + } + } + rc = walCheckpointFinalize(p); + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + pWal->ckptLock = 0; + sqlite3_free(p); + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +int sqlite3WalCheckpointStart( + sqlite3 *db, /* Database connection */ + Wal *pWal, /* Wal connection */ + u8 *aState, int nState, /* Checkpoint state to restore */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags to sync db file with (or 0) */ + sqlite3_ckpt **ppCkpt /* OUT: Incremental checkpoint object */ +){ + WalCkpt *p = 0; + int isChanged = 0; + int rc; + int pgsz; + + *ppCkpt = 0; + if( pWal->readOnly ) return SQLITE_READONLY; + WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + if( rc ){ + /* Usually this is SQLITE_BUSY meaning that another thread or process + ** is already running a checkpoint, or maybe a recovery. But it might + ** also be SQLITE_IOERR. */ + return rc; + } + pWal->ckptLock = 1; + + /* Read the wal-index header. */ + rc = walIndexReadHdr(pWal, &isChanged); + if( rc!=SQLITE_OK ) goto ckptstart_out; + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } + + pgsz = walPagesize(pWal); + p = sqlite3_malloc(sizeof(WalCkpt) + pgsz); + if( p==0 ){ + rc = SQLITE_NOMEM; + goto ckptstart_out; + } + + rc = walCheckpointStart( + pWal, (u8*)&p[1], pgsz, xBusy, pBusyArg, sync_flags, p + ); + p->db = db; + + if( rc==SQLITE_OK && aState ){ + if( nState!=sizeof(u32)*3 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + int i; + if( pWal->hdr.aCksum[0]!=sqlite3Get4byte(&aState[4]) + || pWal->hdr.aCksum[1]!=sqlite3Get4byte(&aState[8]) + ){ + rc = SQLITE_MISMATCH; + }else{ + p->nStep = (int)sqlite3Get4byte(aState); + sqlite3Put4byte(&aState[4], pWal->hdr.aCksum[0]); + sqlite3Put4byte(&aState[8], pWal->hdr.aCksum[1]); + for(i=0; rc==SQLITE_OK && inStep; i++){ + u32 dummy1, dummy2; + rc = walIteratorNext(p->pIter, &dummy1, &dummy2); + } + } + } + } + + ckptstart_out: + if( rc!=SQLITE_OK ){ + if( p ) walIteratorFree(p->pIter); + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + pWal->ckptLock = 0; + sqlite3_free(p); + p = 0; + } + *ppCkpt = (sqlite3_ckpt*)p; + return rc; +} + /* Return the value to pass to a sqlite3_wal_hook callback, the ** number of frames in the WAL at the point of the last commit since ** sqlite3WalCallback() was called. If no commits have occurred since diff --git a/src/wal.h b/src/wal.h index 1c6f27d8f0..748b6bc277 100644 --- a/src/wal.h +++ b/src/wal.h @@ -128,6 +128,15 @@ int sqlite3WalHeapMemory(Wal *pWal); int sqlite3WalCheckSalt(Wal *pWal, sqlite3_file*); +int sqlite3WalCheckpointStart(sqlite3 *, + Wal *pWal, /* Wal connection */ + u8 *aState, int nState, /* Checkpoint state to restore */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags to sync db file with (or 0) */ + sqlite3_ckpt **ppCkpt /* OUT: Incremental checkpoint object */ +); + #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). diff --git a/test/wal.test b/test/wal.test index 675be73791..63126de58f 100644 --- a/test/wal.test +++ b/test/wal.test @@ -375,6 +375,7 @@ do_test wal-7.2 { # truncation. # do_test wal-8.1 { +breakpoint reopen_db catch { db close } forcedelete test.db test.db-wal From 5e3fc6e99fc2a99ecc971faed7724d06567fa65e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Oct 2014 18:09:58 +0000 Subject: [PATCH 028/116] Add tests for another application writing the database while an ota update is ongoing. FossilOrigin-Name: 2402baa0027ca65e9a5106b8b9c4e10f40d2cbc3 --- ext/ota/README.txt | 30 +++++++--- ext/ota/ota6.test | 128 +++++++++++++++++++++++++++++++++++++++++++ ext/ota/sqlite3ota.c | 38 +++++++------ manifest | 15 ++--- manifest.uuid | 2 +- 5 files changed, 180 insertions(+), 33 deletions(-) create mode 100644 ext/ota/ota6.test diff --git a/ext/ota/README.txt b/ext/ota/README.txt index 2707f22f1d..943d1822af 100644 --- a/ext/ota/README.txt +++ b/ext/ota/README.txt @@ -19,11 +19,11 @@ SQLite Hacks * All non-temporary triggers are disabled. -2) PRAGMA pager_ota_mode: +2) PRAGMA pager_ota_mode=1: - This pragma sets a flag on the pager associated with the main database only. In - a zipvfs system, this pragma is intercepted by zipvfs and the flag is set on - the lower level pager only. + This pragma sets a flag on the pager associated with the main database only. + In a zipvfs system, this pragma is intercepted by zipvfs and the flag is set + on the lower level pager only. The flag can only be set when there is no open transaction and the pager does not already have an open WAL file. Attempting to do so is an error. @@ -36,13 +36,13 @@ SQLite Hacks Otherwise, if no WAL file or flags are found, the pager opens the *-oal file and uses it as a write-ahead-log with the *-shm data stored in heap-memory. - The 8-bytes of "salt" at teh start of an *-oal file is a copy of the 8 bytes + The 8-bytes of "salt" at the start of an *-oal file is a copy of the 8 bytes starting at offset 24 of the database file header (the change counter and the number of pages in the file). If the *-oal file already exists when it is opened, SQLite checks that the salt still matches the database header fields. - If not, it concludes that the database file has been written by a rollback-mode - client since the *-oal wa created and an SQLITE_BUSY_SNAPSHOT error is - returned. No read-transaction can be opened in this case. + If not, it concludes that the database file has been written by a + rollback-mode client since the *-oal wa created and an SQLITE_BUSY_SNAPSHOT + error is returned. No read-transaction can be opened in this case. A pager with the pager_ota_mode flag set never runs a checkpoint. @@ -51,14 +51,26 @@ SQLite Hacks pager_ota_mode connections. If two or more such connections attempt to write simultaneously, the results are undefined. +3) PRAGMA pager_ota_mode=2: -3) sqlite3_index_writer() + The pager_ota_mode pragma may also be set to 2 if the main database is open + in WAL mode. This prevents SQLite from checkpointing the wal file as part + of sqlite3_close(). + + The effects of setting pager_ota_mode=2 if the db is not in WAL mode are + undefined. + +4) sqlite3_index_writer() This new API function is used to create VMs that can insert or delete entries from individual index b-trees within the database. The VMs apply affinities and check that UNIQUE constraints are not violated before updating index b-trees. +5) sqlite3_ckpt_open/step/close() + + API for performing (and resuming) incremental checkpoints. + The OTA extension ----------------- diff --git a/ext/ota/ota6.test b/ext/ota/ota6.test new file mode 100644 index 0000000000..f15873629c --- /dev/null +++ b/ext/ota/ota6.test @@ -0,0 +1,128 @@ +# 2014 October 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the OTA module. Specifically, it tests the +# outcome of some other client writing to the database while an OTA update +# is being applied. + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix ota6 + +proc setup_test {} { + reset_db + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b UNIQUE); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b UNIQUE); + } + db close + + forcedelete ota.db + sqlite3 ota ota.db + ota eval { + CREATE TABLE data_t1(a, b, ota_control); + CREATE TABLE data_t2(a, b, ota_control); + CREATE TABLE data_t3(a, b, ota_control); + INSERT INTO data_t1 VALUES(1, 't1', 0); + INSERT INTO data_t2 VALUES(2, 't2', 0); + INSERT INTO data_t3 VALUES(3, 't3', 0); + } + ota close +} + +# Test the outcome of some other client writing the db while the *-oal +# file is being generated. Once this has happened, the update cannot be +# progressed. +# +for {set nStep 1} {$nStep < 7} {incr nStep} { + do_test 1.$nStep.1 { + setup_test + sqlite3ota ota test.db ota.db + for {set i 0} {$i<$nStep} {incr i} {ota step} + + ota close + sqlite3 db test.db + execsql { INSERT INTO t1 VALUES(5, 'hello') } + sqlite3ota ota test.db ota.db + ota step + } {SQLITE_BUSY} + do_test 1.$nStep.2 { + ota step + } {SQLITE_BUSY} + do_test 1.$nStep.3 { + list [file exists test.db-oal] [file exists test.db-wal] + } {1 0} + do_test 1.$nStep.4 { + list [catch { ota close } msg] $msg + } {1 {SQLITE_BUSY - database is locked}} +} + +for {set nStep 7} {$nStep < 8} {incr nStep} { + do_test 1.$nStep.1 { + setup_test + sqlite3ota ota test.db ota.db + for {set i 0} {$i<$nStep} {incr i} {ota step} + ota close + sqlite3 db test.db + execsql { INSERT INTO t1 VALUES(5, 'hello') } + sqlite3ota ota test.db ota.db + ota step + } {SQLITE_OK} + do_test 1.$nStep.2 { + ota step + } {SQLITE_OK} + do_test 1.$nStep.3 { + list [file exists test.db-oal] [file exists test.db-wal] + } {0 1} + do_test 1.$nStep.4 { + list [catch { ota close } msg] $msg + } {0 SQLITE_OK} +} + + +# Test the outcome of some other client writing the db after the *-oal +# file has been copied to the *-wal path. Once this has happened, any +# other client writing to the db causes OTA to consider its job finished. +# +for {set nStep 8} {$nStep < 20} {incr nStep} { + do_test 1.$nStep.1 { + setup_test + sqlite3ota ota test.db ota.db + for {set i 0} {$i<$nStep} {incr i} {ota step} + ota close + sqlite3 db test.db + execsql { INSERT INTO t1 VALUES(5, 'hello') } + sqlite3ota ota test.db ota.db + ota step + } {SQLITE_DONE} + do_test 1.$nStep.2 { + ota step + } {SQLITE_DONE} + do_test 1.$nStep.3 { + file exists test.db-oal + } {0} + do_test 1.$nStep.4 { + list [catch { ota close } msg] $msg + } {0 SQLITE_DONE} + + do_execsql_test 1.$nStep.5 { + SELECT * FROM t1; + } {1 t1 5 hello} +} + + + +finish_test + + diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 1d4ea1f859..662529831d 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -1303,29 +1303,35 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ } assert( p->rc!=SQLITE_OK || p->eStage!=0 ); - if( p->eStage==OTA_STAGE_OAL ){ - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK ){ + if( p->eStage==OTA_STAGE_OAL ){ const char *zScript = "PRAGMA journal_mode=off;" "PRAGMA pager_ota_mode=1;" "PRAGMA ota_mode=1;" "BEGIN IMMEDIATE;" - ; + ; p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); + + /* Point the object iterator at the first object */ + if( p->rc==SQLITE_OK ){ + p->rc = otaObjIterFirst(p, &p->objiter); + } + + if( p->rc==SQLITE_OK ){ + otaLoadTransactionState(p, pState); + } + }else if( p->eStage==OTA_STAGE_CKPT ){ + p->rc = sqlite3_ckpt_open( + p->db, pState->pCkptState, pState->nCkptState, &p->pCkpt + ); + if( p->rc==SQLITE_MISMATCH ){ + p->eStage = OTA_STAGE_DONE; + p->rc = SQLITE_DONE; + } + }else if( p->eStage==OTA_STAGE_DONE ){ + p->rc = SQLITE_DONE; } - - /* Point the object iterator at the first object */ - if( p->rc==SQLITE_OK ){ - p->rc = otaObjIterFirst(p, &p->objiter); - } - - if( p->rc==SQLITE_OK ){ - otaLoadTransactionState(p, pState); - } - }else if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_CKPT ){ - p->rc = sqlite3_ckpt_open( - p->db, pState->pCkptState, pState->nCkptState, &p->pCkpt - ); } otaFreeState(pState); diff --git a/manifest b/manifest index b1900479f6..36ec26121f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sversion-3.8.7\schanges\swith\sthis\sbranch. -D 2014-10-20T16:34:03.980 +C Add\stests\sfor\sanother\sapplication\swriting\sthe\sdatabase\swhile\san\sota\supdate\sis\songoing. +D 2014-10-21T18:09:58.794 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,14 +122,15 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/ota/README.txt cb11e39bfeba952ac8896dab860ada9d54731fb8 +F ext/ota/README.txt 78d4a9f78f567d4bf826cf0f02df6254902562ca F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 F ext/ota/ota1.test 0c8e5ef3d059bebe7872477381424836326a8e0a F ext/ota/ota2.test 4568c2671d19dbde789fb9091d727a2e94880128 F ext/ota/ota3.test 215dd4a8e238567e0f890a5139b6fdf5494ef311 F ext/ota/ota4.test 60f897f329a6782ef2f24862640acf3c52e48077 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb -F ext/ota/sqlite3ota.c 1b4e4cdb05e67982865466f4b0aaea8d3648269c +F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 +F ext/ota/sqlite3ota.c 2ae05cfc4e45cf847ba5ec79953b9b55bcb9fc8d F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -1214,7 +1215,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0bf1301aacb3b717b4cc020fbda9fab0bae331c3 19fe4a0a475bd94f491031aea7a183f7c0515cf3 -R 348e09d90bfec6d435f7913d476a2e51 +P d380a6482a46478ebdf97e08b2fcf78b5d126dde +R 97e1b4279cf76e9ec95460929d6af6ea U dan -Z d6c8c1f51d897af1aed5284afd8b6a66 +Z f79fd33fec0991480e954c839234840a diff --git a/manifest.uuid b/manifest.uuid index 0ed1c9bb69..d9c4211e19 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d380a6482a46478ebdf97e08b2fcf78b5d126dde \ No newline at end of file +2402baa0027ca65e9a5106b8b9c4e10f40d2cbc3 \ No newline at end of file From d6446a0abebf1ff5c2837476c61a88184bc2e1f1 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Oct 2014 19:35:03 +0000 Subject: [PATCH 029/116] Test that sqlite3ota_open() works with URI paths. Fix some other issues. FossilOrigin-Name: 6fd09854feb46739f42b7e7a5e76680d5f9b8c5a --- ext/ota/ota1.test | 17 ++++++++++++++++- ext/ota/sqlite3ota.c | 13 ++++++++++--- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/main.c | 3 +++ src/pager.c | 14 +++++++++++--- src/wal.c | 3 +++ test/wal.test | 1 - 8 files changed, 55 insertions(+), 20 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index 4ac469573f..deed62f6c8 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -16,6 +16,9 @@ if {![info exists testdir]} { source $testdir/tester.tcl set ::testprefix ota1 +db close +sqlite3_shutdown +sqlite3_config_uri 1 # Create a simple OTA database. That expects to write to a table: # @@ -91,7 +94,19 @@ proc step_ota {target ota} { set rc } -foreach {tn2 cmd} {1 run_ota 2 step_ota} { +# Same as [step_ota], except using a URI to open the target db. +# +proc step_ota_uri {target ota} { + while 1 { + sqlite3ota ota file:$target?xyz=123 $ota + set rc [ota step] + ota close + if {$rc != "SQLITE_OK"} break + } + set rc +} + +foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { foreach {tn schema} { 1 { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 662529831d..6491701852 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -781,6 +781,10 @@ static int otaGetUpdateStmt( return p->rc; } +/* +** Open the database handle and attach the OTA database as "ota". If an +** error occurs, leave an error code and message in the OTA handle. +*/ static void otaOpenDatabase(sqlite3ota *p){ assert( p->rc==SQLITE_OK ); sqlite3_close(p->db); @@ -828,9 +832,11 @@ static void otaFileSuffix3(const char *zBase, char *z){ } /* -** Move the "*-oal" file corresponding to the target database to the -** "*-wal" location. If an error occurs, leave an error code and error -** message in the ota handle. +** The OTA handle is currently in OTA_STAGE_OAL state, with a SHARED lock +** on the database file. This proc moves the *-oal file to the *-wal path, +** then reopens the database file (this time in vanilla, non-oal, WAL mode). +** If an error occurs, leave an error code and error message in the ota +** handle. */ static void otaMoveOalFile(sqlite3ota *p){ const char *zBase = sqlite3_db_filename(p->db, "main"); @@ -838,6 +844,7 @@ static void otaMoveOalFile(sqlite3ota *p){ char *zWal = sqlite3_mprintf("%s-wal", zBase); char *zOal = sqlite3_mprintf("%s-oal", zBase); + assert( p->eStage==OTA_STAGE_OAL ); assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); if( zWal==0 || zOal==0 ){ p->rc = SQLITE_NOMEM; diff --git a/manifest b/manifest index 36ec26121f..e7d8dcf1aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sanother\sapplication\swriting\sthe\sdatabase\swhile\san\sota\supdate\sis\songoing. -D 2014-10-21T18:09:58.794 +C Test\sthat\ssqlite3ota_open()\sworks\swith\sURI\spaths.\sFix\ssome\sother\sissues. +D 2014-10-21T19:35:03.426 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,13 +124,13 @@ F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt 78d4a9f78f567d4bf826cf0f02df6254902562ca F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 -F ext/ota/ota1.test 0c8e5ef3d059bebe7872477381424836326a8e0a +F ext/ota/ota1.test 98deadd0c67ca634bd3ce30ef063a3bb6534a029 F ext/ota/ota2.test 4568c2671d19dbde789fb9091d727a2e94880128 F ext/ota/ota3.test 215dd4a8e238567e0f890a5139b6fdf5494ef311 F ext/ota/ota4.test 60f897f329a6782ef2f24862640acf3c52e48077 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 -F ext/ota/sqlite3ota.c 2ae05cfc4e45cf847ba5ec79953b9b55bcb9fc8d +F ext/ota/sqlite3ota.c a8b4fbe2234964238678ed939e035346efbfeff2 F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -204,7 +204,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c d7a31f8c19c27b046cef381369381bba379c3730 +F src/main.c 627406d9cfee49e666d47a265a5cad8ef402f823 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -225,7 +225,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 93ad94d62e959c2a4d85de17a5636da174e55533 +F src/pager.c e1e385e03df0abf0743a7eea38c193d327ed396c F src/pager.h 997a4aa3bad1638dabe90a0cbb674cc4a7b9c034 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -309,7 +309,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de -F src/wal.c a5dbbbd8ceccd5e2187b1e7854f359cb5efb7e3b +F src/wal.c cbce609ec35e4d170ec0db385880f5f12c9c7fbd F src/wal.h 0d3ba0c3f1b4c25796cb213568a84b9f9063f465 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 @@ -1102,7 +1102,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 -F test/wal.test d7bb2feeacb74d26f27ebe519334b95adf22c8ae +F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1215,7 +1215,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d380a6482a46478ebdf97e08b2fcf78b5d126dde -R 97e1b4279cf76e9ec95460929d6af6ea +P 2402baa0027ca65e9a5106b8b9c4e10f40d2cbc3 +R be41b5fa90bd92c9b4cc1682fc718b4c U dan -Z f79fd33fec0991480e954c839234840a +Z 3c98460b0b6af0c08ea70d54ed77c905 diff --git a/manifest.uuid b/manifest.uuid index d9c4211e19..a140abbb67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2402baa0027ca65e9a5106b8b9c4e10f40d2cbc3 \ No newline at end of file +6fd09854feb46739f42b7e7a5e76680d5f9b8c5a \ No newline at end of file diff --git a/src/main.c b/src/main.c index 436d1b58bf..4d39ff0150 100644 --- a/src/main.c +++ b/src/main.c @@ -1773,6 +1773,9 @@ int sqlite3_wal_checkpoint_v2( #endif } +/* +** Open an incremental checkpoint handle. +*/ int sqlite3_ckpt_open( sqlite3 *db, unsigned char *a, int n, diff --git a/src/pager.c b/src/pager.c index 4a56fb94ea..8df3ad61b6 100644 --- a/src/pager.c +++ b/src/pager.c @@ -623,9 +623,10 @@ struct PagerSavepoint { ** database file has not been modified since it was created, this variable ** is set to 2. ** -** noCkptOnClose -** -** +** It is also possible to use PagerSetOtaMode() to 2 if the database is +** already in WAL mode. In this case the only effect is to prevent the +** connection from checkpointing the db as part of an sqlite3PagerClose() +** call. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ @@ -7293,13 +7294,20 @@ int sqlite3PagerWalFramesize(Pager *pPager){ ** Set or clear the "OTA mode" flag. */ int sqlite3PagerSetOtaMode(Pager *pPager, int iOta){ + assert( iOta==1 || iOta==2 ); if( iOta==1 && (pPager->pWal || pPager->eState!=PAGER_OPEN) ){ return SQLITE_ERROR; } + if( iOta==2 && 0==pPager->pWal ){ + return SQLITE_ERROR; + } pPager->otaMode = iOta; return SQLITE_OK; } +/* +** Open an incremental checkpoint handle. +*/ int sqlite3PagerWalCheckpointStart( sqlite3 *db, Pager *pPager, diff --git a/src/wal.c b/src/wal.c index dcf88ff7d8..fb6d87657f 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3108,6 +3108,9 @@ int sqlite3_ckpt_close(sqlite3_ckpt *pCkpt, u8 **paState, int *pnState){ return rc; } +/* +** Open an incremental checkpoint handle. +*/ int sqlite3WalCheckpointStart( sqlite3 *db, /* Database connection */ Wal *pWal, /* Wal connection */ diff --git a/test/wal.test b/test/wal.test index 63126de58f..675be73791 100644 --- a/test/wal.test +++ b/test/wal.test @@ -375,7 +375,6 @@ do_test wal-7.2 { # truncation. # do_test wal-8.1 { -breakpoint reopen_db catch { db close } forcedelete test.db test.db-wal From 9f0da72b09edfb8112e346060e6cbdc272dfaa05 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Oct 2014 11:30:08 +0000 Subject: [PATCH 030/116] Sync the database file in sqlite3_ckpt_close(), even if the checkpoint has not finished. FossilOrigin-Name: e2729d623c6e9d0b3049049f3f3051bbb479184f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e7d8dcf1aa..90e0ac70d8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\sthat\ssqlite3ota_open()\sworks\swith\sURI\spaths.\sFix\ssome\sother\sissues. -D 2014-10-21T19:35:03.426 +C Sync\sthe\sdatabase\sfile\sin\ssqlite3_ckpt_close(),\seven\sif\sthe\scheckpoint\shas\snot\sfinished. +D 2014-10-22T11:30:08.642 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -309,7 +309,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de -F src/wal.c cbce609ec35e4d170ec0db385880f5f12c9c7fbd +F src/wal.c 99c5b0a68e525d854d86f93be215e9c2bfb7b194 F src/wal.h 0d3ba0c3f1b4c25796cb213568a84b9f9063f465 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 @@ -1215,7 +1215,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2402baa0027ca65e9a5106b8b9c4e10f40d2cbc3 -R be41b5fa90bd92c9b4cc1682fc718b4c +P 6fd09854feb46739f42b7e7a5e76680d5f9b8c5a +R b3f281b9e115a81e4161ab510da7f09d U dan -Z 3c98460b0b6af0c08ea70d54ed77c905 +Z 3d20cbfe4cbde996e6770d4067fcc8a6 diff --git a/manifest.uuid b/manifest.uuid index a140abbb67..b8ae20a48a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6fd09854feb46739f42b7e7a5e76680d5f9b8c5a \ No newline at end of file +e2729d623c6e9d0b3049049f3f3051bbb479184f \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index fb6d87657f..70bd918072 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1764,8 +1764,8 @@ static int walCheckpointFinalize(WalCkpt *p){ int rc = p->rc; Wal *pWal = p->pWal; - /* If work was completed */ - if( p->pIter && rc==SQLITE_DONE ){ + if( rc==SQLITE_DONE ){ + /* If work was completed */ rc = SQLITE_OK; if( p->mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ i64 szDb = pWal->hdr.nPage*(i64)p->szPage; @@ -1779,6 +1779,9 @@ static int walCheckpointFinalize(WalCkpt *p){ p->pInfo->nBackfill = p->mxSafeFrame; } p->rc = rc; + }else if( rc==SQLITE_OK && p->sync_flags ){ + /* If work was not completed, but no error has occured. */ + p->rc = sqlite3OsSync(pWal->pDbFd, p->sync_flags); } /* Release the reader lock held while backfilling */ From 08e06b0b21e1263a88c69b69e8d0d1ded141c8b0 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Oct 2014 15:33:12 +0000 Subject: [PATCH 031/116] Add tests to check error handling in OTA. FossilOrigin-Name: ec7321ae482a8c4d893851a5edd17d67ef1a448b --- ext/ota/otafault.test | 153 ++++++++++++++++++++++++++++++++++++++++++ ext/ota/sqlite3ota.c | 20 ++++-- manifest | 15 +++-- manifest.uuid | 2 +- src/wal.c | 1 + 5 files changed, 177 insertions(+), 14 deletions(-) create mode 100644 ext/ota/otafault.test diff --git a/ext/ota/otafault.test b/ext/ota/otafault.test new file mode 100644 index 0000000000..915d6448d2 --- /dev/null +++ b/ext/ota/otafault.test @@ -0,0 +1,153 @@ +# 2014 October 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix otafault + +do_test 1.1 { + forcedelete ota.db + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX t1cb ON t1(c, b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX t2cb ON t1(c, b); + INSERT INTO t2 VALUES('a', 'a', 'a'); + INSERT INTO t2 VALUES('b', 'b', 'b'); + INSERT INTO t2 VALUES('c', 'c', 'c'); + + ATTACH 'ota.db' AS ota; + CREATE TABLE ota.data_t1(a, b, c, ota_control); + CREATE TABLE ota.data_t2(a, b, c, ota_control); + + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(3, 'three', NULL, '.x.'); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + + INSERT INTO data_t2 VALUES('b', NULL, NULL, 1); + INSERT INTO data_t2 VALUES('c', 'see', NULL, '.x.'); + INSERT INTO data_t2 VALUES('d', 'd', 'd', 0); + } + db close + + forcecopy test.db test.db.bak + forcecopy ota.db ota.db.bak +} {} + +do_faultsim_test 2 -faults oom-trans* -prep { + catch { db close } + forcedelete test.db-journal test.db-wal ota.db-journal ota.db-wal + forcecopy test.db.bak test.db + forcecopy ota.db.bak ota.db +} -body { + sqlite3ota ota test.db ota.db + while {[ota step]=="SQLITE_OK"} {} + ota close +} -test { + faultsim_test_result {0 SQLITE_DONE} \ + {1 {SQLITE_NOMEM - out of memory}} \ + {1 SQLITE_NOMEM} \ + {1 SQLITE_IOERR_NOMEM} \ + {1 {SQLITE_NOMEM - unable to open a temporary database file for storing temporary tables}} + + if {$testrc==0} { + sqlite3 db test.db + faultsim_integrity_check + set res [db eval { + SELECT * FROM t1 UNION ALL SELECT * FROM t2; + }] + set expected [list {*}{ + 1 1 1 3 three 3 4 4 4 + a a a c see c d d d + }] + + if {$res != $expected} { + puts "" + puts "res: $res" + puts "exp: $expected" + error "data not as expected!" + } + } +} + +proc copy_if_exists {src target} { + if {[file exists $src]} { + forcecopy $src $target + } else { + forcedelete $target + } +} + +for {set iStep 0} {$iStep<=21} {incr iStep} { + + forcedelete test.db-journal test.db-wal ota.db-journal ota.db-wal + + copy_if_exists test.db.bak test.db + copy_if_exists ota.db.bak ota.db + + sqlite3ota ota test.db ota.db + for {set x 0} {$x < $::iStep} {incr x} { ota step } + ota close + + copy_if_exists test.db test.db.bak.2 + copy_if_exists test.db-wal test.db.bak.2-wal + copy_if_exists test.db-oal test.db.bak.2-oal + copy_if_exists ota.db ota.db.bak.2 + + do_faultsim_test 3.$iStep -faults oom-trans* -prep { + catch { db close } + forcedelete test.db-journal test.db-wal ota.db-journal ota.db-wal + copy_if_exists test.db.bak.2 test.db + copy_if_exists test.db.bak.2-wal test.db-wal + copy_if_exists test.db.bak.2-oal test.db-oal + copy_if_exists ota.db.bak.2 ota.db + } -body { + sqlite3ota ota test.db ota.db + while {[ota step] == "SQLITE_OK"} {} + ota close + } -test { + faultsim_test_result {0 SQLITE_DONE} \ + {1 {SQLITE_NOMEM - out of memory}} \ + {1 SQLITE_NOMEM} \ + {1 SQLITE_IOERR_NOMEM} \ + {1 {SQLITE_NOMEM - unable to open a temporary database file for storing temporary tables}} + + if {$testrc==0} { + sqlite3 db test.db + faultsim_integrity_check + set res [db eval { + SELECT * FROM t1 UNION ALL SELECT * FROM t2; + }] + set expected [list {*}{ + 1 1 1 3 three 3 4 4 4 + a a a c see c d d d + }] + + if {$res != $expected} { + puts "" + puts "res: $res" + puts "exp: $expected" + error "data not as expected!" + } + } + } + +} + +finish_test + diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 6491701852..149ce63459 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -577,6 +577,10 @@ static char *otaObjIterGetSetlist( zList = sqlite3_mprintf("%z%s%s=?%d", zList, zSep, pIter->azTblCol[i], i+1 ); + if( zList==0 ){ + p->rc = SQLITE_NOMEM; + break; + } zSep = ", "; } } @@ -1079,8 +1083,10 @@ int sqlite3ota_step(sqlite3ota *p){ default: break; } + return p->rc; + }else{ + return SQLITE_NOMEM; } - return p->rc; } static void otaSaveTransactionState(sqlite3ota *p){ @@ -1147,10 +1153,12 @@ static char *otaStrndup(char *zStr, int nStr, int *pRc){ } static void otaFreeState(OtaState *p){ - sqlite3_free(p->zTbl); - sqlite3_free(p->zIdx); - sqlite3_free(p->pCkptState); - sqlite3_free(p); + if( p ){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p->pCkptState); + sqlite3_free(p); + } } /* @@ -1298,7 +1306,7 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ if( p->rc==SQLITE_OK ){ pState = otaLoadState(p); assert( pState || p->rc!=SQLITE_OK ); - if( pState ){ + if( p->rc==SQLITE_OK ){ if( pState->eStage==0 ){ otaDeleteOalFile(p); p->eStage = 1; diff --git a/manifest b/manifest index 90e0ac70d8..255e93be86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Sync\sthe\sdatabase\sfile\sin\ssqlite3_ckpt_close(),\seven\sif\sthe\scheckpoint\shas\snot\sfinished. -D 2014-10-22T11:30:08.642 +C Add\stests\sto\scheck\serror\shandling\sin\sOTA. +D 2014-10-22T15:33:12.681 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -130,7 +130,8 @@ F ext/ota/ota3.test 215dd4a8e238567e0f890a5139b6fdf5494ef311 F ext/ota/ota4.test 60f897f329a6782ef2f24862640acf3c52e48077 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 -F ext/ota/sqlite3ota.c a8b4fbe2234964238678ed939e035346efbfeff2 +F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b +F ext/ota/sqlite3ota.c c7f8cdf55449b5169f79632e78f8e5049abf904c F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -309,7 +310,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de -F src/wal.c 99c5b0a68e525d854d86f93be215e9c2bfb7b194 +F src/wal.c 2787dfceb22d50c45ca51a8f56684516894a135c F src/wal.h 0d3ba0c3f1b4c25796cb213568a84b9f9063f465 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 @@ -1215,7 +1216,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6fd09854feb46739f42b7e7a5e76680d5f9b8c5a -R b3f281b9e115a81e4161ab510da7f09d +P e2729d623c6e9d0b3049049f3f3051bbb479184f +R f1db06b4abcee6f48965e7da46a3387a U dan -Z 3d20cbfe4cbde996e6770d4067fcc8a6 +Z 87316fa0e6665a6aa049c94bffdd7d8b diff --git a/manifest.uuid b/manifest.uuid index b8ae20a48a..41346389d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2729d623c6e9d0b3049049f3f3051bbb479184f \ No newline at end of file +ec7321ae482a8c4d893851a5edd17d67ef1a448b \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 70bd918072..d96ac08403 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1608,6 +1608,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ if( rc!=SQLITE_OK ){ walIteratorFree(p); + p = 0; } *pp = p; return rc; From d8260fd1e3fe1a21dd595b95f91137448d2331ca Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Nov 2014 15:07:40 +0000 Subject: [PATCH 032/116] Fix a problem with the parameters to an OP_Affinity in one of the VM programs generated by sqlite3_index_writer() that was causing an OOB read. FossilOrigin-Name: 447b33b34a9836992f5a8d50cd8647a10435c144 --- ext/ota/ota7.test | 51 +++++++++++++++++++++++++++++++++++++++++++++++ manifest | 13 ++++++------ manifest.uuid | 2 +- src/vdbeblob.c | 2 +- 4 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 ext/ota/ota7.test diff --git a/ext/ota/ota7.test b/ext/ota/ota7.test new file mode 100644 index 0000000000..18bece567a --- /dev/null +++ b/ext/ota/ota7.test @@ -0,0 +1,51 @@ +# 2014 October 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the OTA module. Specifically, it tests the +# that affinities are correctly applied to values within the OTA database. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix ota7 + +do_test 1.0 { + execsql { + CREATE TABLE t1(a INT PRIMARY KEY, b) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 'abc'); + INSERT INTO t1 VALUES(2, 'def'); + } + + forcedelete ota.db + sqlite3 ota ota.db + ota eval { + CREATE TABLE data_t1(a, b, ota_control); + INSERT INTO data_t1 VALUES('1', NULL, 1); + } + ota close +} {} + +do_test 1.1 { + sqlite3ota ota test.db ota.db + while { [ota step]=="SQLITE_OK" } {} + ota close +} {SQLITE_DONE} + +sqlite3 db test.db +do_execsql_test 1.2 { + SELECT * FROM t1 +} {2 def} + +finish_test + + diff --git a/manifest b/manifest index 255e93be86..73890afec0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\scheck\serror\shandling\sin\sOTA. -D 2014-10-22T15:33:12.681 +C Fix\sa\sproblem\swith\sthe\sparameters\sto\san\sOP_Affinity\sin\sone\sof\sthe\sVM\sprograms\sgenerated\sby\ssqlite3_index_writer()\sthat\swas\scausing\san\sOOB\sread. +D 2014-11-17T15:07:40.515 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -130,6 +130,7 @@ F ext/ota/ota3.test 215dd4a8e238567e0f890a5139b6fdf5494ef311 F ext/ota/ota4.test 60f897f329a6782ef2f24862640acf3c52e48077 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 +F ext/ota/ota7.test feba6072af04bc4bee192f7576aac60cef6a4727 F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b F ext/ota/sqlite3ota.c c7f8cdf55449b5169f79632e78f8e5049abf904c F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 @@ -305,7 +306,7 @@ F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 -F src/vdbeblob.c a8e2c3baa3e7081347c4677185a631bfc43de043 +F src/vdbeblob.c 42b1854c42dc7b7f53c9ff1a4b90d24b2daeee5b F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 @@ -1216,7 +1217,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e2729d623c6e9d0b3049049f3f3051bbb479184f -R f1db06b4abcee6f48965e7da46a3387a +P ec7321ae482a8c4d893851a5edd17d67ef1a448b +R ebf92f720e516f33caf61cc1f37b716d U dan -Z 87316fa0e6665a6aa049c94bffdd7d8b +Z c2254e02a36430a8663d836ffe531fb4 diff --git a/manifest.uuid b/manifest.uuid index 41346389d5..e72967c007 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec7321ae482a8c4d893851a5edd17d67ef1a448b \ No newline at end of file +447b33b34a9836992f5a8d50cd8647a10435c144 \ No newline at end of file diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 4ecd56522f..e95688b4f1 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -610,7 +610,7 @@ int sqlite3_index_writer( sqlite3VdbeAddOp2(v, OP_IdxInsert, 0, regRec); }else{ /* Code the IdxDelete to remove the entry from the b-tree index. */ - sqlite3VdbeAddOp4(v, OP_Affinity, 0, pIdx->nColumn, 0, zAffinity, 0); + sqlite3VdbeAddOp4(v, OP_Affinity, 1, pIdx->nColumn, 0, zAffinity, 0); sqlite3VdbeAddOp3(v, OP_IdxDelete, 0, 1, pIdx->nColumn); } sqlite3FinishCoding(pParse); From 0dc3ac5f3dacfc0c8a429e82e43cd40b98fa6d89 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Nov 2014 17:57:06 +0000 Subject: [PATCH 033/116] Changes so that sqlite3_ckpt_open() works with zipvfs databases. FossilOrigin-Name: acbed3380d8a35bf2f1ec133540cb5e9fccb9b42 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 12 ++++++++---- src/sqlite.h.in | 5 +++++ 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 73890afec0..2c61a549c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sthe\sparameters\sto\san\sOP_Affinity\sin\sone\sof\sthe\sVM\sprograms\sgenerated\sby\ssqlite3_index_writer()\sthat\swas\scausing\san\sOOB\sread. -D 2014-11-17T15:07:40.515 +C Changes\sso\sthat\ssqlite3_ckpt_open()\sworks\swith\szipvfs\sdatabases. +D 2014-11-17T17:57:06.217 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 627406d9cfee49e666d47a265a5cad8ef402f823 +F src/main.c 25088f9f0422ef29a12eb1ff57d12a2aac0f8c05 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -241,7 +241,7 @@ F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 5c89ff33315006449119b86b460e0a286758f126 +F src/sqlite.h.in d368351cfb7f98d847ab42c760fbe01ab30d06fd F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h cf72fe02b33f46572c681cc90ad64a5851f10b8f @@ -1217,7 +1217,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ec7321ae482a8c4d893851a5edd17d67ef1a448b -R ebf92f720e516f33caf61cc1f37b716d +P 447b33b34a9836992f5a8d50cd8647a10435c144 +R a385542ae873f1383b1a1f964429fd33 U dan -Z c2254e02a36430a8663d836ffe531fb4 +Z a2dd6eb93ee736daa85bc89fdfbf459a diff --git a/manifest.uuid b/manifest.uuid index e72967c007..f1dfb37c70 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -447b33b34a9836992f5a8d50cd8647a10435c144 \ No newline at end of file +acbed3380d8a35bf2f1ec133540cb5e9fccb9b42 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 4d39ff0150..eb60f7dbaf 100644 --- a/src/main.c +++ b/src/main.c @@ -1781,14 +1781,18 @@ int sqlite3_ckpt_open( unsigned char *a, int n, sqlite3_ckpt **ppCkpt ){ - Pager *pPager; - Btree *pBt; + Pager *pPager = 0; int rc; *ppCkpt = 0; sqlite3_mutex_enter(db->mutex); - pBt = db->aDb[0].pBt; - pPager = sqlite3BtreePager(pBt); + + /* Find the Pager object. */ + rc = sqlite3_file_control(db,"main",SQLITE_FCNTL_ZIPVFS_PAGER,(void*)&pPager); + if( rc!=SQLITE_OK ){ + pPager = sqlite3BtreePager(db->aDb[0].pBt); + } + rc = sqlite3PagerWalCheckpointStart(db, pPager, a, n, ppCkpt); sqlite3_mutex_leave(db->mutex); return rc; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 6439020f4a..e4b9ab0d7c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -941,6 +941,10 @@ struct sqlite3_io_methods { ** pointed to by the pArg argument. This capability is used during testing ** and only needs to be supported when SQLITE_TEST is defined. ** +**
  • [[SQLITE_FCNTL_ZIPVFS_PAGER]] +** The [SQLITE_FCNTL_ZIPVFS_PAGER] opcode is used by the OTA extension when +** updating a zipvfs database. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -965,6 +969,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_SYNC 21 #define SQLITE_FCNTL_COMMIT_PHASETWO 22 #define SQLITE_FCNTL_WIN32_SET_HANDLE 23 +#define SQLITE_FCNTL_ZIPVFS_PAGER 24 /* ** CAPI3REF: Mutex Handle From 8a1e1078116548002f1f7ab76fe2e52df39cebee Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Nov 2014 18:35:30 +0000 Subject: [PATCH 034/116] Add tests for WITHOUT ROWID tables with composite primary keys. FossilOrigin-Name: 712d413d29950b19d4afb18cfcf9d3afb302d0a0 --- ext/ota/ota7.test | 62 +++++++++++++++++++++++++++++++++++++++++++++-- manifest | 12 ++++----- manifest.uuid | 2 +- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/ext/ota/ota7.test b/ext/ota/ota7.test index 18bece567a..099d3da732 100644 --- a/ext/ota/ota7.test +++ b/ext/ota/ota7.test @@ -9,16 +9,24 @@ # #*********************************************************************** # -# This file contains tests for the OTA module. Specifically, it tests the -# that affinities are correctly applied to values within the OTA database. +# This file contains tests for the OTA module. # + if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } source $testdir/tester.tcl set ::testprefix ota7 +# Test index: +# +# 1.*: That affinities are correctly applied to values within the +# OTA database. +# +# 2.*: Tests for multi-column primary keys. +# + do_test 1.0 { execsql { CREATE TABLE t1(a INT PRIMARY KEY, b) WITHOUT ROWID; @@ -46,6 +54,56 @@ do_execsql_test 1.2 { SELECT * FROM t1 } {2 def} +#------------------------------------------------------------------------- +# +foreach {tn tbl} { + 1 { CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID } +} { + reset_db + + execsql $tbl + do_execsql_test 2.$tn.1 { + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 1, 'a'); + INSERT INTO t1 VALUES(1, 2, 'b'); + INSERT INTO t1 VALUES(2, 1, 'c'); + INSERT INTO t1 VALUES(2, 2, 'd'); + } + + do_test 2.$tn.2 { + forcedelete ota.db + sqlite3 ota ota.db + execsql { + CREATE TABLE data_t1(a, b, c, ota_control); + INSERT INTO data_t1 VALUES(3, 1, 'e', 0); + INSERT INTO data_t1 VALUES(3, 2, 'f', 0); + INSERT INTO data_t1 VALUES(1, 2, NULL, 1); + INSERT INTO data_t1 VALUES(2, 1, 'X', '..x'); + } ota + ota close + } {} + + do_test 2.$tn.3 { + set rc "SQLITE_OK" + while {$rc == "SQLITE_OK"} { + sqlite3ota ota test.db ota.db + ota step + set rc [ota close] + } + set rc + } {SQLITE_DONE} + + do_execsql_test 2.$tn.1 { + SELECT * FROM t1 ORDER BY a, b + } { + 1 1 a + 2 1 X + 2 2 d + 3 1 e + 3 2 f + } +} + finish_test diff --git a/manifest b/manifest index 2c61a549c1..dab1b99a41 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sso\sthat\ssqlite3_ckpt_open()\sworks\swith\szipvfs\sdatabases. -D 2014-11-17T17:57:06.217 +C Add\stests\sfor\sWITHOUT\sROWID\stables\swith\scomposite\sprimary\skeys. +D 2014-11-17T18:35:30.003 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -130,7 +130,7 @@ F ext/ota/ota3.test 215dd4a8e238567e0f890a5139b6fdf5494ef311 F ext/ota/ota4.test 60f897f329a6782ef2f24862640acf3c52e48077 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 -F ext/ota/ota7.test feba6072af04bc4bee192f7576aac60cef6a4727 +F ext/ota/ota7.test 36e740da2b67cc086ff9f2975d2929b8beaa1016 F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b F ext/ota/sqlite3ota.c c7f8cdf55449b5169f79632e78f8e5049abf904c F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 @@ -1217,7 +1217,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 447b33b34a9836992f5a8d50cd8647a10435c144 -R a385542ae873f1383b1a1f964429fd33 +P acbed3380d8a35bf2f1ec133540cb5e9fccb9b42 +R 27bbe427f24c751c6dfa08920b459629 U dan -Z a2dd6eb93ee736daa85bc89fdfbf459a +Z 78767b94770b1205744fff578a5ff365 diff --git a/manifest.uuid b/manifest.uuid index f1dfb37c70..6ede1e0d71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -acbed3380d8a35bf2f1ec133540cb5e9fccb9b42 \ No newline at end of file +712d413d29950b19d4afb18cfcf9d3afb302d0a0 \ No newline at end of file From 54ef517012fbca6f098e296c1228bc83755e3de2 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 20 Nov 2014 15:11:12 +0000 Subject: [PATCH 035/116] Updates to support zipvfs in pass-through mode. FossilOrigin-Name: 556c3de53ad33d11d33ec794345c2100aa76f3e1 --- ext/ota/sqlite3ota.c | 7 +++++-- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 16 +++++++++------- src/pragma.c | 5 +++++ 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 149ce63459..88fe45d392 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -1066,7 +1066,10 @@ int sqlite3ota_step(sqlite3ota *p){ p->rc = sqlite3_ckpt_open(p->db, 0, 0, &p->pCkpt); } if( p->rc==SQLITE_OK ){ - if( SQLITE_OK!=sqlite3_ckpt_step(p->pCkpt) ){ + if( p->pCkpt==0 ){ + p->eStage = OTA_STAGE_DONE; + p->rc = SQLITE_DONE; + }else if( SQLITE_OK!=sqlite3_ckpt_step(p->pCkpt) ){ p->rc = sqlite3_ckpt_close(p->pCkpt, 0, 0); p->pCkpt = 0; if( p->rc==SQLITE_OK ){ @@ -1340,7 +1343,7 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ p->rc = sqlite3_ckpt_open( p->db, pState->pCkptState, pState->nCkptState, &p->pCkpt ); - if( p->rc==SQLITE_MISMATCH ){ + if( p->rc==SQLITE_MISMATCH || (p->rc==SQLITE_OK && p->pCkpt==0) ){ p->eStage = OTA_STAGE_DONE; p->rc = SQLITE_DONE; } diff --git a/manifest b/manifest index dab1b99a41..57046725ee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sWITHOUT\sROWID\stables\swith\scomposite\sprimary\skeys. -D 2014-11-17T18:35:30.003 +C Updates\sto\ssupport\szipvfs\sin\spass-through\smode. +D 2014-11-20T15:11:12.852 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -132,7 +132,7 @@ F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 F ext/ota/ota7.test 36e740da2b67cc086ff9f2975d2929b8beaa1016 F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b -F ext/ota/sqlite3ota.c c7f8cdf55449b5169f79632e78f8e5049abf904c +F ext/ota/sqlite3ota.c c2b34913954720200f1f33556df25fe55387c531 F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -227,13 +227,13 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c e1e385e03df0abf0743a7eea38c193d327ed396c +F src/pager.c e5378efabe8479cd8af8194ac71d0bab8bf979f5 F src/pager.h 997a4aa3bad1638dabe90a0cbb674cc4a7b9c034 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 -F src/pragma.c 310939bc2fb7e6456edfb4735d004253a4b2505e +F src/pragma.c 92343541fc50d9fd30eb734f77d43d7636521e7e F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -1217,7 +1217,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P acbed3380d8a35bf2f1ec133540cb5e9fccb9b42 -R 27bbe427f24c751c6dfa08920b459629 +P 712d413d29950b19d4afb18cfcf9d3afb302d0a0 +R 2621b5f3be6f418323e840a9284ff98b U dan -Z 78767b94770b1205744fff578a5ff365 +Z 3bf4b5d8555904656fb3130ef9899e9e diff --git a/manifest.uuid b/manifest.uuid index 6ede1e0d71..ef96f551b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -712d413d29950b19d4afb18cfcf9d3afb302d0a0 \ No newline at end of file +556c3de53ad33d11d33ec794345c2100aa76f3e1 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 8df3ad61b6..833ebdbd8a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7298,9 +7298,6 @@ int sqlite3PagerSetOtaMode(Pager *pPager, int iOta){ if( iOta==1 && (pPager->pWal || pPager->eState!=PAGER_OPEN) ){ return SQLITE_ERROR; } - if( iOta==2 && 0==pPager->pWal ){ - return SQLITE_ERROR; - } pPager->otaMode = iOta; return SQLITE_OK; } @@ -7314,10 +7311,15 @@ int sqlite3PagerWalCheckpointStart( u8 *a, int n, sqlite3_ckpt **ppCkpt ){ - return sqlite3WalCheckpointStart(db, pPager->pWal, a, n, - pPager->xBusyHandler, pPager->pBusyHandlerArg, - pPager->ckptSyncFlags, ppCkpt - ); + if( pPager->pWal==0 ){ + *ppCkpt = 0; + return SQLITE_OK; + }else{ + return sqlite3WalCheckpointStart(db, pPager->pWal, a, n, + pPager->xBusyHandler, pPager->pBusyHandlerArg, + pPager->ckptSyncFlags, ppCkpt + ); + } } #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pragma.c b/src/pragma.c index c3566e7fd9..f1c44bdb13 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -901,10 +901,15 @@ void sqlite3Pragma( assert( pBt!=0 ); if( zRight ){ int iArg = sqlite3Atoi(zRight); + Pager *pPager = sqlite3BtreePager(pBt); if( sqlite3BtreeIsInReadTrans(pBt) ){ sqlite3ErrorMsg(pParse, "cannot set pager_ota_mode with open transaction" ); + }else if( sqlite3PagerWalSupported(pPager)==0 ){ + sqlite3ErrorMsg(pParse, + "cannot set pager_ota_mode without wal support" + ); }else if( sqlite3PagerSetOtaMode(sqlite3BtreePager(pBt), iArg) ){ sqlite3ErrorMsg(pParse, "cannot set pager_ota_mode in wal mode"); } From 268c0f8844440db27c1f455ff45261515c93661e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 20 Nov 2014 17:37:08 +0000 Subject: [PATCH 036/116] Update the ota extension so that it can be used to update tables with external PRIMARY KEY indexes. FossilOrigin-Name: 55066a1171cbd3077f5e6c8ceb2745e810d9476e --- ext/ota/ota1.test | 47 ++++++++++++++++++++-- ext/ota/ota7.test | 1 + ext/ota/sqlite3ota.c | 93 +++++++++++++++++++++++++++++--------------- manifest | 18 ++++----- manifest.uuid | 2 +- src/pragma.c | 5 +-- 6 files changed, 119 insertions(+), 47 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index deed62f6c8..e6b50c5906 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -136,6 +136,18 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { CREATE INDEX i2 ON t1(c, b); CREATE INDEX i3 ON t1(a, b, c, a, b, c); } + + 8 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(a, b, c, a, b, c); + } + + 9 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)); + CREATE INDEX i1 ON t1(b); + } } { reset_db execsql $schema @@ -160,6 +172,11 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { #------------------------------------------------------------------------- # Check that an OTA cannot be applied to a table that has no PK. # +# UPDATE: At one point OTA required that all tables featured either +# explicit IPK columns or were declared WITHOUT ROWID. This has been +# relaxed so that external PRIMARY KEYs on tables with automatic rowids +# are now allowed. +# reset_db create_ota1 ota.db do_execsql_test 2.1 { CREATE TABLE t1(a, b, c) } @@ -170,16 +187,15 @@ do_test 2.2 { do_test 2.3 { list [catch { ota close } msg] $msg } {1 {SQLITE_ERROR - table t1 has no PRIMARY KEY}} - reset_db do_execsql_test 2.4 { CREATE TABLE t1(a PRIMARY KEY, b, c) } do_test 2.5 { sqlite3ota ota test.db ota.db ota step -} {SQLITE_ERROR} +} {SQLITE_OK} do_test 2.6 { list [catch { ota close } msg] $msg -} {1 {SQLITE_ERROR - table t1 has no PRIMARY KEY}} +} {0 SQLITE_OK} #------------------------------------------------------------------------- # Check that if a UNIQUE constraint is violated the current and all @@ -198,6 +214,16 @@ foreach {tn errcode errmsg schema} { INSERT INTO t1 VALUES(4, 2, 'three'); } + 3 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(3, 2, 1); + } + + 4 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { + CREATE TABLE t1(a PRIMARY KEY, b, c UNIQUE); + INSERT INTO t1 VALUES(4, 2, 'three'); + } + } { reset_db execsql $schema @@ -242,6 +268,12 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota} { CREATE INDEX i2 ON t1(c, b); CREATE INDEX i3 ON t1(c, b, c); } + 5 { + CREATE TABLE t1(a INT PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } } { reset_db execsql $schema @@ -293,6 +325,15 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota} { CREATE INDEX i5 ON t1(c); CREATE INDEX i6 ON t1(c, b); } + 4 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d); + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } } { reset_db execsql $schema diff --git a/ext/ota/ota7.test b/ext/ota/ota7.test index 099d3da732..a4ee6b41ad 100644 --- a/ext/ota/ota7.test +++ b/ext/ota/ota7.test @@ -58,6 +58,7 @@ do_execsql_test 1.2 { # foreach {tn tbl} { 1 { CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID } + 2 { CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)) } } { reset_db diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 88fe45d392..3b02750866 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -102,6 +102,7 @@ struct OtaObjIter { int nTblCol; /* Size of azTblCol[] array */ char **azTblCol; /* Array of quoted column names */ unsigned char *abTblPk; /* Array of flags - true for PK columns */ + unsigned char bRowid; /* True for implicit IPK tables */ /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ @@ -225,6 +226,7 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){ pIter->nTblCol = 0; sqlite3_free(pIter->zMask); pIter->zMask = 0; + pIter->bRowid = 0; } /* @@ -391,8 +393,8 @@ static int otaMPrintfExec(sqlite3ota *p, const char *zFmt, ...){ /* ** If they are not already populated, populate the pIter->azTblCol[], -** pIter->abTblPk[] and pIter->nTblCol variables according to the table -** that the iterator currently points to. +** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to +** the table that the iterator currently points to. ** ** Return SQLITE_OK if successful, or an SQLite error code otherwise. If ** an error does occur, an error code and error message are also left in @@ -406,6 +408,7 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ int bSeenPk = 0; int rc2; /* sqlite3_finalize() return value */ + assert( pIter->bRowid==0 ); zSql = sqlite3_mprintf("PRAGMA main.table_info(%Q)", pIter->zTbl); p->rc = prepareFreeAndCollectError(p->db, &pStmt, &p->zErrmsg, zSql); while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ @@ -422,8 +425,10 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ if( p->rc==SQLITE_OK ){ const char *zName = (const char*)sqlite3_column_text(pStmt, 1); - pIter->abTblPk[nCol] = sqlite3_column_int(pStmt, 5); - if( pIter->abTblPk[nCol] ) bSeenPk = 1; + int iPk = sqlite3_column_int(pStmt, 5); + pIter->abTblPk[nCol] = (iPk!=0); + if( iPk ) bSeenPk = 1; + if( iPk<0 ) pIter->bRowid = 1; pIter->azTblCol[nCol] = otaQuoteName(zName); if( pIter->azTblCol[nCol]==0 ) p->rc = SQLITE_NOMEM; nCol++; @@ -489,7 +494,8 @@ static char *otaObjIterGetCollist( int i; for(i=0; iazTblCol[iCol]); + char *zCol = (iCol>=0 ? pIter->azTblCol[iCol] : "ota_rowid"); + zList = sqlite3_mprintf("%z%s%s", zList, zSep, zCol); if( zList && azCollate ){ zList = sqlite3_mprintf("%z COLLATE %Q", zList, azCollate[i]); } @@ -520,6 +526,11 @@ static char *otaObjIterGetOldlist( break; } } + + /* For a table with implicit rowids, append "old._rowid_" to the list. */ + if( pIter->bRowid ){ + zList = sqlite3_mprintf("%z, %s._rowid_", zList, zObj); + } } return zList; } @@ -648,20 +659,29 @@ static int otaObjIterPrepareAll( /* Create the SELECT statement to read keys in sorted order */ zCollist = otaObjIterGetCollist(p, pIter, pIter->nCol, aiCol, azColl); if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, - sqlite3_mprintf( + char *zSql; + if( pIter->bRowid ){ + zSql = sqlite3_mprintf( + "SELECT %s, ota_control FROM ota.'ota_tmp_%q' ORDER BY %s%s", + zCollist, pIter->zTbl, + zCollist, zLimit + ); + }else{ + zSql = sqlite3_mprintf( "SELECT %s, ota_control FROM ota.'data_%q' " "WHERE typeof(ota_control)='integer' AND ota_control!=1 " - "UNION ALL " + "UNION ALL " "SELECT %s, ota_control FROM ota.'ota_tmp_%q' " "ORDER BY %s%s", zCollist, pIter->zTbl, zCollist, pIter->zTbl, zCollist, zLimit - ) - ); + ); + } + p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, zSql); } }else{ + const char *zOtaRowid = (pIter->bRowid ? ", ota_rowid" : ""); char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol); char *zWhere = otaObjIterGetWhere(p, pIter); char *zOldlist = otaObjIterGetOldlist(p, pIter, "old"); @@ -697,33 +717,44 @@ static int otaObjIterPrepareAll( ); } - if( p->rc==SQLITE_OK ){ + /* Create the ota_tmp_xxx table and the triggers to populate it. */ + otaMPrintfExec(p, + "CREATE TABLE IF NOT EXISTS ota.'ota_tmp_%q' AS " + "SELECT *%s FROM ota.'data_%q' WHERE 0;" + + "CREATE TEMP TRIGGER ota_delete_%q BEFORE DELETE ON main.%Q " + "BEGIN " + " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(2, %s);" + "END;" + + "CREATE TEMP TRIGGER ota_update1_%q BEFORE UPDATE ON main.%Q " + "BEGIN " + " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(2, %s);" + "END;" + + "CREATE TEMP TRIGGER ota_update2_%q AFTER UPDATE ON main.%Q " + "BEGIN " + " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(3, %s);" + "END;" + , pIter->zTbl, (pIter->bRowid ? ", 0 AS ota_rowid" : ""), + pIter->zTbl, + pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOtaRowid, zOldlist, + pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOtaRowid, zOldlist, + pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOtaRowid, zNewlist + ); + if( pIter->bRowid ){ otaMPrintfExec(p, - "CREATE TABLE IF NOT EXISTS ota.'ota_tmp_%q' AS " - "SELECT * FROM ota.'data_%q' WHERE 0;" - - "CREATE TEMP TRIGGER ota_delete_%q BEFORE DELETE ON main.%Q " + "CREATE TEMP TRIGGER ota_insert_%q AFTER INSERT ON main.%Q " "BEGIN " - " INSERT INTO 'ota_tmp_%q'(ota_control, %s) VALUES(2, %s);" + " INSERT INTO 'ota_tmp_%q'(ota_control, %s, ota_rowid)" + " VALUES(0, %s);" "END;" - - "CREATE TEMP TRIGGER ota_update1_%q BEFORE UPDATE ON main.%Q " - "BEGIN " - " INSERT INTO 'ota_tmp_%q'(ota_control, %s) VALUES(2, %s);" - "END;" - - "CREATE TEMP TRIGGER ota_update2_%q AFTER UPDATE ON main.%Q " - "BEGIN " - " INSERT INTO 'ota_tmp_%q'(ota_control, %s) VALUES(3, %s);" - "END;" - - , pIter->zTbl, pIter->zTbl, - pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOldlist, - pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOldlist, - pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zNewlist + , pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zNewlist ); } + + /* Allocate space required for the zMask field. */ if( p->rc==SQLITE_OK ){ int nMask = pIter->nTblCol+1; diff --git a/manifest b/manifest index 57046725ee..78a7f550e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\ssupport\szipvfs\sin\spass-through\smode. -D 2014-11-20T15:11:12.852 +C Update\sthe\sota\sextension\sso\sthat\sit\scan\sbe\sused\sto\supdate\stables\swith\sexternal\sPRIMARY\sKEY\sindexes. +D 2014-11-20T17:37:08.997 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,15 +124,15 @@ F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt 78d4a9f78f567d4bf826cf0f02df6254902562ca F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 -F ext/ota/ota1.test 98deadd0c67ca634bd3ce30ef063a3bb6534a029 +F ext/ota/ota1.test a8f9d89c9b2d381a663bcedaa5dd5952cdbd1231 F ext/ota/ota2.test 4568c2671d19dbde789fb9091d727a2e94880128 F ext/ota/ota3.test 215dd4a8e238567e0f890a5139b6fdf5494ef311 F ext/ota/ota4.test 60f897f329a6782ef2f24862640acf3c52e48077 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 -F ext/ota/ota7.test 36e740da2b67cc086ff9f2975d2929b8beaa1016 +F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b -F ext/ota/sqlite3ota.c c2b34913954720200f1f33556df25fe55387c531 +F ext/ota/sqlite3ota.c 2c31a56890e915e13eb5d6ced02325e1f4db7487 F ext/ota/sqlite3ota.h 7b20abe9247d292429d00f0a5c237ff6e0dc0196 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -233,7 +233,7 @@ F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 -F src/pragma.c 92343541fc50d9fd30eb734f77d43d7636521e7e +F src/pragma.c 272b122a873fc756e999c319f8e81de55ef39d5c F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -1217,7 +1217,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 712d413d29950b19d4afb18cfcf9d3afb302d0a0 -R 2621b5f3be6f418323e840a9284ff98b +P 556c3de53ad33d11d33ec794345c2100aa76f3e1 +R c3a23811c2315fa65ee1bcdda07f2567 U dan -Z 3bf4b5d8555904656fb3130ef9899e9e +Z eebecee2ceb200b37a4bd343197c1b04 diff --git a/manifest.uuid b/manifest.uuid index ef96f551b3..8e90b7e445 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -556c3de53ad33d11d33ec794345c2100aa76f3e1 \ No newline at end of file +55066a1171cbd3077f5e6c8ceb2745e810d9476e \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index f1c44bdb13..5b89237014 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1526,10 +1526,9 @@ void sqlite3Pragma( }else if( pPk==0 ){ k = 1; }else{ + for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} if( (db->flags & SQLITE_OtaMode) && HasRowid(pTab) ){ - k = 0; - }else{ - for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} + k = -1 * k; } } sqlite3VdbeAddOp2(v, OP_Integer, k, 6); From f8bd49ee22fcd900013e9cdbfd374ab51b211e88 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 20 Nov 2014 19:19:02 +0000 Subject: [PATCH 037/116] Add the "ota_delta()" feature for delta-compressed updates. FossilOrigin-Name: c64dcd1788f5cc7db197a0ec4ab0981f34a72c6b --- ext/ota/ota8.test | 75 +++++++++++++++++++++++++++++++++++++++++++ ext/ota/sqlite3ota.c | 76 ++++++++++++++++++++++++++++++++++++++++---- ext/ota/sqlite3ota.h | 34 ++++++++++++++++++++ manifest | 15 +++++---- manifest.uuid | 2 +- 5 files changed, 187 insertions(+), 15 deletions(-) create mode 100644 ext/ota/ota8.test diff --git a/ext/ota/ota8.test b/ext/ota/ota8.test new file mode 100644 index 0000000000..24a6e72248 --- /dev/null +++ b/ext/ota/ota8.test @@ -0,0 +1,75 @@ +# 2014 November 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test the ota_delta() feature. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix ota8 + +do_execsql_test 1.0 { + CREATE TABLE t1(x, y PRIMARY KEY, z); + INSERT INTO t1 VALUES(NULL, 1, 'one'); + INSERT INTO t1 VALUES(NULL, 2, 'two'); + INSERT INTO t1 VALUES(NULL, 3, 'three'); + CREATE INDEX i1z ON t1(z, x); +} + +do_test 1.1 { + forcedelete ota.db + sqlite3 db2 ota.db + db2 eval { + CREATE TABLE data_t1(x, y, z, ota_control); + INSERT INTO data_t1 VALUES('a', 1, '_i' , 'x.d'); + INSERT INTO data_t1 VALUES('b', 2, 2 , '..x'); + INSERT INTO data_t1 VALUES('_iii', 3, '-III' , 'd.d'); + } + db2 close +} {} + +do_test 1.2.1 { + sqlite3ota ota test.db ota.db + ota step +} {SQLITE_ERROR} +do_test 1.2.2 { + list [catch {ota close} msg] $msg +} {1 {SQLITE_ERROR - no such function: ota_delta}} + +proc ota_delta {orig new} { + return "${orig}${new}" +} + +do_test 1.3.1 { + while 1 { + sqlite3ota ota test.db ota.db + ota create_ota_delta + set rc [ota step] + if {$rc != "SQLITE_OK"} break + ota close + } + ota close +} {SQLITE_DONE} + +do_execsql_test 1.3.2 { + SELECT * FROM t1 +} { + a 1 one_i + {} 2 2 + _iii 3 three-III +} +integrity_check 1.3.3 + + +finish_test + diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 3b02750866..40091d965b 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -447,6 +447,22 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ return p->rc; } +/* +** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, +** an error code is stored in the OTA handle passed as the first argument. +*/ +static char *otaMPrintfAndCollectError(sqlite3ota *p, const char *zFmt, ...){ + char *zSql = 0; + va_list ap; + va_start(ap, zFmt); + if( p->rc==SQLITE_OK ){ + zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ) p->rc = SQLITE_NOMEM; + } + va_end(ap); + return zSql; +} + /* ** This function constructs and returns a pointer to a nul-terminated ** string containing some SQL clause or list based on one or more of the @@ -570,6 +586,7 @@ static void otaBadControlError(sqlite3ota *p){ p->zErrmsg = sqlite3_mprintf("Invalid ota_control value"); } + static char *otaObjIterGetSetlist( sqlite3ota *p, OtaObjIter *pIter, @@ -584,14 +601,17 @@ static char *otaObjIterGetSetlist( }else{ const char *zSep = ""; for(i=0; inTblCol; i++){ - if( zMask[i]=='x' ){ - zList = sqlite3_mprintf("%z%s%s=?%d", + char c = zMask[i]; + if( c=='x' ){ + zList = otaMPrintfAndCollectError(p, "%z%s%s=?%d", zList, zSep, pIter->azTblCol[i], i+1 ); - if( zList==0 ){ - p->rc = SQLITE_NOMEM; - break; - } + zSep = ", "; + } + if( c=='d' ){ + zList = otaMPrintfAndCollectError(p, "%z%s%s=ota_delta(%s, ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); zSep = ", "; } } @@ -1389,6 +1409,13 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ return p; } +/* +** Return the database handle used by pOta. +*/ +sqlite3 *sqlite3ota_db(sqlite3ota *pOta){ + return (pOta ? pOta->db : 0); +} + /* ** Close the OTA handle. */ @@ -1450,6 +1477,31 @@ sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta){ /* From main.c (apparently...) */ extern const char *sqlite3ErrName(int); +void test_ota_delta(sqlite3_context *pCtx, int nArg, sqlite3_value **apVal){ + Tcl_Interp *interp = (Tcl_Interp*)sqlite3_user_data(pCtx); + Tcl_Obj *pScript; + int i; + + pScript = Tcl_NewObj(); + Tcl_IncrRefCount(pScript); + Tcl_ListObjAppendElement(0, pScript, Tcl_NewStringObj("ota_delta", -1)); + for(i=0; i Date: Fri, 21 Nov 2014 10:46:23 +0000 Subject: [PATCH 038/116] Add support for updating virtual tables via ota. FossilOrigin-Name: 4dfcfe543945aa60a7ac397a3bdb0ac9e20ef7b6 --- ext/ota/ota9.test | 66 ++++++++++++++ ext/ota/sqlite3ota.c | 199 +++++++++++++++++++++++++++++-------------- ext/ota/sqlite3ota.h | 34 +++++++- manifest | 15 ++-- manifest.uuid | 2 +- 5 files changed, 240 insertions(+), 76 deletions(-) create mode 100644 ext/ota/ota9.test diff --git a/ext/ota/ota9.test b/ext/ota/ota9.test new file mode 100644 index 0000000000..18791b479e --- /dev/null +++ b/ext/ota/ota9.test @@ -0,0 +1,66 @@ +# 2014 November 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test OTA with virtual tables +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix ota9 + +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE f1 USING fts4(a, b, c); + INSERT INTO f1(rowid, a, b, c) VALUES(11, 'a', 'b', 'c'); + INSERT INTO f1(rowid, a, b, c) VALUES(12, 'd', 'e', 'f'); + INSERT INTO f1(rowid, a, b, c) VALUES(13, 'g', 'h', 'i'); +} + +do_test 1.1 { + forcedelete ota.db + sqlite3 db2 ota.db + db2 eval { + CREATE TABLE data_f1(ota_rowid, a, b, c, ota_control); + INSERT INTO data_f1 VALUES(14, 'x', 'y', 'z', 0); -- INSERT + INSERT INTO data_f1 VALUES(11, NULL, NULL, NULL, 1); -- DELETE + INSERT INTO data_f1 VALUES(13, NULL, NULL, 'X', '..x'); -- UPDATE + } + db2 close +} {} + +do_test 1.2.1 { + while 1 { + sqlite3ota ota test.db ota.db + set rc [ota step] + if {$rc != "SQLITE_OK"} break + ota close + } + ota close +} {SQLITE_DONE} + +do_execsql_test 1.2.2 { SELECT rowid, * FROM f1 } { + 12 d e f + 13 g h X + 14 x y z +} +do_execsql_test 1.2.3 { INSERT INTO f1(f1) VALUES('integrity-check') } +integrity_check 1.2.4 + + + +finish_test + diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 40091d965b..8613c179f6 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -103,6 +103,7 @@ struct OtaObjIter { char **azTblCol; /* Array of quoted column names */ unsigned char *abTblPk; /* Array of flags - true for PK columns */ unsigned char bRowid; /* True for implicit IPK tables */ + unsigned char bVtab; /* True for a virtual table */ /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ @@ -227,6 +228,7 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){ sqlite3_free(pIter->zMask); pIter->zMask = 0; pIter->bRowid = 0; + pIter->bVtab = 0; } /* @@ -391,6 +393,51 @@ static int otaMPrintfExec(sqlite3ota *p, const char *zFmt, ...){ return p->rc; } +/* +** Increase the size of the pIter->azTblCol[] and abTblPk[] arrays so that +** there is room for at least nCol elements. If an OOM occurs, store an +** error code in the OTA handle passed as the first argument. +*/ +static void otaExtendIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ + assert( p->rc==SQLITE_OK ); + if( (nCol % 8)==0 ){ + unsigned char *abNew; + int nByte = sizeof(char*) * (nCol+8); + char **azNew = (char**)sqlite3_realloc(pIter->azTblCol, nByte); + abNew = (unsigned char*)sqlite3_realloc(pIter->abTblPk, nCol+8); + + if( azNew ) pIter->azTblCol = azNew; + if( abNew ) pIter->abTblPk = abNew; + if( azNew==0 || abNew==0 ) p->rc = SQLITE_NOMEM; + } +} + +/* +** Return true if zTab is the name of a virtual table within the target +** database. +*/ +static int otaIsVtab(sqlite3ota *p, const char *zTab){ + int res = 0; + sqlite3_stmt *pSelect = 0; + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->db, &pSelect, &p->zErrmsg, + "SELECT count(*) FROM sqlite_master WHERE name = ? AND type='table' " + "AND sql LIKE 'CREATE VIRTUAL TABLE%'" + ); + } + + if( p->rc==SQLITE_OK ){ + sqlite3_bind_text(pSelect, 1, zTab, -1, SQLITE_STATIC); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + res = sqlite3_column_int(pSelect, 0); + } + p->rc = sqlite3_finalize(pSelect); + } + + return res; +} + /* ** If they are not already populated, populate the pIter->azTblCol[], ** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to @@ -408,21 +455,12 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ int bSeenPk = 0; int rc2; /* sqlite3_finalize() return value */ - assert( pIter->bRowid==0 ); + assert( pIter->bRowid==0 && pIter->bVtab==0 ); + zSql = sqlite3_mprintf("PRAGMA main.table_info(%Q)", pIter->zTbl); p->rc = prepareFreeAndCollectError(p->db, &pStmt, &p->zErrmsg, zSql); while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - if( (nCol % 8)==0 ){ - unsigned char *abNew; - int nByte = sizeof(char*) * (nCol+8); - char **azNew = (char**)sqlite3_realloc(pIter->azTblCol, nByte); - abNew = (unsigned char*)sqlite3_realloc(pIter->abTblPk, nCol+8); - - if( azNew ) pIter->azTblCol = azNew; - if( abNew ) pIter->abTblPk = abNew; - if( azNew==0 || abNew==0 ) p->rc = SQLITE_NOMEM; - } - + otaExtendIterArrays(p, pIter, nCol); if( p->rc==SQLITE_OK ){ const char *zName = (const char*)sqlite3_column_text(pStmt, 1); int iPk = sqlite3_column_int(pStmt, 5); @@ -439,8 +477,13 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ if( p->rc==SQLITE_OK ) p->rc = rc2; if( p->rc==SQLITE_OK && bSeenPk==0 ){ - p->zErrmsg = sqlite3_mprintf("table %s has no PRIMARY KEY", pIter->zTbl); - p->rc = SQLITE_ERROR; + const char *zTab = pIter->zTbl; + if( otaIsVtab(p, zTab) ){ + pIter->bVtab = 1; + }else{ + p->zErrmsg = sqlite3_mprintf("table %s has no PRIMARY KEY", zTab); + p->rc = SQLITE_ERROR; + } } } @@ -557,16 +600,18 @@ static char *otaObjIterGetWhere( ){ char *zList = 0; if( p->rc==SQLITE_OK ){ - const char *zSep = ""; - int i; - for(i=0; inTblCol; i++){ - if( pIter->abTblPk[i] ){ - const char *zCol = pIter->azTblCol[i]; - zList = sqlite3_mprintf("%z%s%s=?%d", zList, zSep, zCol, i+1); - zSep = " AND "; - if( zList==0 ){ - p->rc = SQLITE_NOMEM; - break; + if( pIter->bVtab ){ + zList = otaMPrintfAndCollectError(p, "rowid = ?%d", pIter->nTblCol+1); + }else{ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = otaMPrintfAndCollectError( + p, "%z%s%s=?%d", zList, zSep, zCol, i+1 + ); + zSep = " AND "; } } } @@ -664,6 +709,8 @@ static int otaObjIterPrepareAll( int *aiCol; /* Column map */ const char **azColl; /* Collation sequences */ + assert( pIter->bVtab==0 ); + /* Create the index writers */ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_index_writer( @@ -701,8 +748,8 @@ static int otaObjIterPrepareAll( p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, zSql); } }else{ - const char *zOtaRowid = (pIter->bRowid ? ", ota_rowid" : ""); - char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol); + const char *zTbl = pIter->zTbl; + char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol+pIter->bVtab); char *zWhere = otaObjIterGetWhere(p, pIter); char *zOldlist = otaObjIterGetOldlist(p, pIter, "old"); char *zNewlist = otaObjIterGetOldlist(p, pIter, "new"); @@ -713,8 +760,9 @@ static int otaObjIterPrepareAll( if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, sqlite3_mprintf( - "SELECT %s, ota_control FROM ota.'data_%q'%s", - zCollist, pIter->zTbl, zLimit) + "SELECT %s, ota_control%s FROM ota.'data_%q'%s", + zCollist, (pIter->bVtab ? ", ota_rowid" : ""), zTbl, zLimit + ) ); } @@ -722,8 +770,8 @@ static int otaObjIterPrepareAll( if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pInsert, pz, sqlite3_mprintf( - "INSERT INTO main.%Q(%s) VALUES(%s)", - pIter->zTbl, zCollist, zBindings + "INSERT INTO main.%Q(%s%s) VALUES(%s)", + zTbl, zCollist, (pIter->bVtab ? ", rowid" : ""), zBindings ) ); } @@ -732,49 +780,53 @@ static int otaObjIterPrepareAll( if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pDelete, pz, sqlite3_mprintf( - "DELETE FROM main.%Q WHERE %s", pIter->zTbl, zWhere + "DELETE FROM main.%Q WHERE %s", zTbl, zWhere ) ); } - /* Create the ota_tmp_xxx table and the triggers to populate it. */ - otaMPrintfExec(p, - "CREATE TABLE IF NOT EXISTS ota.'ota_tmp_%q' AS " - "SELECT *%s FROM ota.'data_%q' WHERE 0;" + if( pIter->bVtab==0 ){ + const char *zOtaRowid = (pIter->bRowid ? ", ota_rowid" : ""); - "CREATE TEMP TRIGGER ota_delete_%q BEFORE DELETE ON main.%Q " - "BEGIN " - " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(2, %s);" - "END;" - - "CREATE TEMP TRIGGER ota_update1_%q BEFORE UPDATE ON main.%Q " - "BEGIN " - " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(2, %s);" - "END;" - - "CREATE TEMP TRIGGER ota_update2_%q AFTER UPDATE ON main.%Q " - "BEGIN " - " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(3, %s);" - "END;" - , pIter->zTbl, (pIter->bRowid ? ", 0 AS ota_rowid" : ""), - pIter->zTbl, - pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOtaRowid, zOldlist, - pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOtaRowid, zOldlist, - pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOtaRowid, zNewlist - ); - if( pIter->bRowid ){ + /* Create the ota_tmp_xxx table and the triggers to populate it. */ otaMPrintfExec(p, - "CREATE TEMP TRIGGER ota_insert_%q AFTER INSERT ON main.%Q " + "PRAGMA ota_mode = 1;" + "CREATE TABLE IF NOT EXISTS ota.'ota_tmp_%q' AS " + "SELECT *%s FROM ota.'data_%q' WHERE 0;" + + "CREATE TEMP TRIGGER ota_delete_%q BEFORE DELETE ON main.%Q " "BEGIN " - " INSERT INTO 'ota_tmp_%q'(ota_control, %s, ota_rowid)" - " VALUES(0, %s);" + " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(2, %s);" "END;" - , pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zNewlist + + "CREATE TEMP TRIGGER ota_update1_%q BEFORE UPDATE ON main.%Q " + "BEGIN " + " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(2, %s);" + "END;" + + "CREATE TEMP TRIGGER ota_update2_%q AFTER UPDATE ON main.%Q " + "BEGIN " + " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(3, %s);" + "END;" + , zTbl, (pIter->bRowid ? ", 0 AS ota_rowid" : ""), zTbl, + zTbl, zTbl, zTbl, zCollist, zOtaRowid, zOldlist, + zTbl, zTbl, zTbl, zCollist, zOtaRowid, zOldlist, + zTbl, zTbl, zTbl, zCollist, zOtaRowid, zNewlist ); + if( pIter->bRowid ){ + otaMPrintfExec(p, + "CREATE TEMP TRIGGER ota_insert_%q AFTER INSERT ON main.%Q " + "BEGIN " + " INSERT INTO 'ota_tmp_%q'(ota_control, %s, ota_rowid)" + " VALUES(0, %s);" + "END;" + , zTbl, zTbl, zTbl, zCollist, zNewlist + ); + } + }else if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->db, "PRAGMA ota_mode = 0", 0, 0, &p->zErrmsg); } - - /* Allocate space required for the zMask field. */ if( p->rc==SQLITE_OK ){ int nMask = pIter->nTblCol+1; @@ -1001,6 +1053,7 @@ static int otaStep(sqlite3ota *p){ || eType==OTA_IDX_DELETE || eType==OTA_IDX_INSERT ){ + sqlite3_value *pVal; sqlite3_stmt *pWriter; assert( eType!=OTA_UPDATE ); @@ -1013,23 +1066,37 @@ static int otaStep(sqlite3ota *p){ } for(i=0; inCol; i++){ - sqlite3_value *pVal; if( eType==SQLITE_DELETE && pIter->zIdx==0 && pIter->abTblPk[i]==0 ){ continue; } pVal = sqlite3_column_value(pIter->pSelect, i); sqlite3_bind_value(pWriter, i+1, pVal); } + if( pIter->bVtab ){ + /* For a virtual table, the SELECT statement is: + ** + ** SELECT , ota_control, ota_rowid FROM .... + ** + ** Hence column_value(pIter->nCol+1). + */ + pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); + sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + } sqlite3_step(pWriter); p->rc = resetAndCollectError(pWriter, &p->zErrmsg); }else if( eType==OTA_UPDATE ){ + sqlite3_value *pVal; sqlite3_stmt *pUpdate = 0; otaGetUpdateStmt(p, pIter, zMask, &pUpdate); if( pUpdate ){ for(i=0; inCol; i++){ - sqlite3_value *pVal = sqlite3_column_value(pIter->pSelect, i); + pVal = sqlite3_column_value(pIter->pSelect, i); sqlite3_bind_value(pUpdate, i+1, pVal); } + if( pIter->bVtab ){ + pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); + sqlite3_bind_value(pUpdate, pIter->nCol+1, pVal); + } sqlite3_step(pUpdate); p->rc = resetAndCollectError(pUpdate, &p->zErrmsg); } @@ -1078,7 +1145,9 @@ int sqlite3ota_step(sqlite3ota *p){ /* Clean up the ota_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ - otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); + if( pIter->bVtab==0 ){ + otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); + } }else{ otaObjIterPrepareAll(p, pIter, 0); diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index 022f3e3f3d..4d7fa2a2ba 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -94,6 +94,18 @@ ** ** The order of the columns in the data_% table does not matter. ** +** If the target database table is a virtual table, the data_% table should +** also contain a column named "ota_rowid". This column is mapped to the +** virtual tables implicit primary key column - "rowid". Virtual tables +** for which the "rowid" column does not function like a primary key value +** can not be updated using OTA. For example, if the target db contains: +** +** CREATE VIRTUAL TABLE ft1 USING fts3(a, b); +** +** then the OTA database should contain: +** +** CREATE TABLE data_ft1(a, b, ota_rowid, ota_control); +** ** For each row to INSERT into the target database as part of the OTA ** update, the corresponding data_% table should contain a single record ** with the "ota_control" column set to contain integer value 0. The @@ -119,7 +131,7 @@ ** stored in the corresponding columns of the data_% table row, as should ** the new values of all columns being update. The text value in the ** "ota_control" column must contain the same number of characters as -** there are column in the target database table, and must consist entirely +** there are columns in the target database table, and must consist entirely ** of "x" and "." characters. For each column that is being updated, ** the corresponding character is set to "x". For those that remain as ** they are, the corresponding character of the ota_control value should @@ -148,6 +160,18 @@ ** ** UPDATE t1 SET c = ota_delta(c, 'usa') WHERE a = 4; ** +** If the target database table is a virtual table, the ota_control value +** should not include a character corresponding to the ota_rowid value. +** For example, this: +** +** INSERT INTO data_ft1(a, b, ota_rowid, ota_control) +** VALUES(NULL, 'usa', 12, '..d'); +** +** causes a result similar to: +** +** UPDATE ft1 SET b = 'usa' WHERE rowid = 12; +** +** ** USAGE ** ** The API declared below allows an application to apply an OTA update @@ -156,12 +180,16 @@ ** ** 1) Opens an OTA handle using the sqlite3ota_open() function. ** -** 2) Calls the sqlite3ota_step() function one or more times on +** 2) Registers any required virtual table modules with the database +** handle returned by sqlite3ota_db(). Also, if required, register +** the ota_delta() implementation. +** +** 3) Calls the sqlite3ota_step() function one or more times on ** the new handle. Each call to sqlite3ota_step() performs a single ** b-tree operation, so thousands of calls may be required to apply ** a complete update. ** -** 3) Calls sqlite3ota_close() to close the OTA update handle. If +** 4) Calls sqlite3ota_close() to close the OTA update handle. If ** sqlite3ota_step() has been called enough times to completely ** apply the update to the target database, then it is committed ** and made visible to other database clients at this point. diff --git a/manifest b/manifest index 505e6916bc..f64d688e27 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"ota_delta()"\sfeature\sfor\sdelta-compressed\supdates. -D 2014-11-20T19:19:02.502 +C Add\ssupport\sfor\supdating\svirtual\stables\svia\sota. +D 2014-11-21T10:46:23.101 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -132,9 +132,10 @@ F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda +F ext/ota/ota9.test d9ad30ccb4e08f878e382876fe67752309538af9 F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b -F ext/ota/sqlite3ota.c edeea10871d1307ff9ee9ccc765ba4031b507509 -F ext/ota/sqlite3ota.h 08b276fc9f56c04cdb454cf7aefa41c29361ed7a +F ext/ota/sqlite3ota.c 07ef7b72358ed422b69a10e4702ab131041e2896 +F ext/ota/sqlite3ota.h 8c09973d27fba7fa34f2d1bf0606d2bd420fe123 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1218,7 +1219,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 55066a1171cbd3077f5e6c8ceb2745e810d9476e -R 97efd71a02c88dd0da29f19b3d083930 +P c64dcd1788f5cc7db197a0ec4ab0981f34a72c6b +R 5a2fe82c5b487561e46e520bd6653b40 U dan -Z 1557aa5ff604851a6b4517de0e4ff6aa +Z b7e67cf183a3b3e32028fa6f7bcf265c diff --git a/manifest.uuid b/manifest.uuid index d045752a5f..2901801d1e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c64dcd1788f5cc7db197a0ec4ab0981f34a72c6b \ No newline at end of file +4dfcfe543945aa60a7ac397a3bdb0ac9e20ef7b6 \ No newline at end of file From 54f0dec3e65be2362530678c367cfc88db965dec Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 21 Nov 2014 11:22:11 +0000 Subject: [PATCH 039/116] Changes to comments in sqlite3ota.h. FossilOrigin-Name: 14139542b68fbf01632a1b149cd6fca4bb01efa6 --- ext/ota/sqlite3ota.h | 8 ++++---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index 4d7fa2a2ba..8a201f25bd 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -61,11 +61,11 @@ ** ** * INSERT statements may not use any default values. ** -** * UPDATE and DELETE statements must identify their target rows by -** real PRIMARY KEY values - i.e. INTEGER PRIMARY KEY columns or -** by the PRIMARY KEY columns of WITHOUT ROWID tables. +** * Non-virtual tables that do not have declared primary keys may not +** be updated. UPDATE and DELETE statements must identify their target +** rows by PRIMARY KEY values. ** -** * UPDATE statements may not modify real PRIMARY KEY columns. +** * UPDATE statements may not modify PRIMARY KEY columns. ** ** * No triggers will be fired. ** diff --git a/manifest b/manifest index f64d688e27..3c75754c18 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\supdating\svirtual\stables\svia\sota. -D 2014-11-21T10:46:23.101 +C Changes\sto\scomments\sin\ssqlite3ota.h. +D 2014-11-21T11:22:11.959 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -135,7 +135,7 @@ F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d9ad30ccb4e08f878e382876fe67752309538af9 F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b F ext/ota/sqlite3ota.c 07ef7b72358ed422b69a10e4702ab131041e2896 -F ext/ota/sqlite3ota.h 8c09973d27fba7fa34f2d1bf0606d2bd420fe123 +F ext/ota/sqlite3ota.h 04577b00c456aacb99be9c8b55572a6e3ca9aa27 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1219,7 +1219,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c64dcd1788f5cc7db197a0ec4ab0981f34a72c6b -R 5a2fe82c5b487561e46e520bd6653b40 +P 4dfcfe543945aa60a7ac397a3bdb0ac9e20ef7b6 +R ce2cc26848888f59033084d0ff8b06d9 U dan -Z b7e67cf183a3b3e32028fa6f7bcf265c +Z 059883a58f80da6dc602fa5f1d52e4f7 diff --git a/manifest.uuid b/manifest.uuid index 2901801d1e..ba0bea7466 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4dfcfe543945aa60a7ac397a3bdb0ac9e20ef7b6 \ No newline at end of file +14139542b68fbf01632a1b149cd6fca4bb01efa6 \ No newline at end of file From 7bf9ec1c26d1d09f6b65ce1d8f11d967f6f81227 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 22 Nov 2014 09:09:50 +0000 Subject: [PATCH 040/116] Add SQLITE_ENABLE_OTA pre-processor directives so that this branch may be compiled with or without OTA. FossilOrigin-Name: 600cefdd4d29c1de4d107fa7ddeb76a18edce4f5 --- ext/ota/sqlite3ota.c | 10 +++- manifest | 34 ++++++------ manifest.uuid | 2 +- src/main.c | 2 + src/pager.c | 40 +++++++++++--- src/pragma.c | 8 ++- src/sqlite.h.in | 122 +++++++++++++++++++++++-------------------- src/sqliteInt.h | 7 ++- src/test_config.c | 6 +++ src/trigger.c | 2 +- src/vdbeblob.c | 11 +++- src/wal.c | 107 +++++++++++++++++++++++++++---------- test/ota.test | 4 +- tool/mkpragmatab.tcl | 2 + 14 files changed, 239 insertions(+), 118 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 8613c179f6..6c3c99434e 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -17,6 +17,8 @@ #include #include "sqlite3.h" + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_OTA) #include "sqlite3ota.h" @@ -1662,12 +1664,18 @@ static int test_sqlite3ota( return TCL_OK; } + int SqliteOta_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "sqlite3ota", test_sqlite3ota, 0, 0); return TCL_OK; } - #endif /* ifdef SQLITE_TEST */ +#else /* !SQLITE_CORE || SQLITE_ENABLE_OTA */ +# ifdef SQLITE_TEST +#include +int SqliteOta_Init(Tcl_Interp *interp){ return TCL_OK; } +# endif +#endif diff --git a/manifest b/manifest index bb2073b96b..5d142cd5c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2014-11-21T14:37:24.898 +C Add\sSQLITE_ENABLE_OTA\spre-processor\sdirectives\sso\sthat\sthis\sbranch\smay\sbe\scompiled\swith\sor\swithout\sOTA. +D 2014-11-22T09:09:50.320 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -135,7 +135,7 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d9ad30ccb4e08f878e382876fe67752309538af9 F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b -F ext/ota/sqlite3ota.c 07ef7b72358ed422b69a10e4702ab131041e2896 +F ext/ota/sqlite3ota.c accfada2ab182dc52225e7345f656520a4e8db22 F ext/ota/sqlite3ota.h 04577b00c456aacb99be9c8b55572a6e3ca9aa27 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -209,7 +209,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c e2adfa1e7bef8b553a1e7c250cc02bcb109d15f2 +F src/main.c cd819123ed552a15c37bd2fd5360db25015dc461 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -230,13 +230,13 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a9e500dd963fb1f67d7860e58b5772abe6123862 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c c2a4795e65b5794248dadd7c8bb65a8d9145995e +F src/pager.c 9fe27a768be53dbe0a818ae10791dc36100a0e76 F src/pager.h c6157af66a9999797629968921133f67716f8f9f F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c facbdd3ecc09c8f750089d941305694301328e98 -F src/pragma.c 272b122a873fc756e999c319f8e81de55ef39d5c +F src/pragma.c 8e0087a5ae6e60ac9ed48df19025cb423e3c8c34 F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 @@ -244,10 +244,10 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in 972125a6b4ea1891d2428cc55302c71f0fb1adb3 +F src/sqlite.h.in f60a24616a6a7e622266e723ed141f0c6131514e F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h d96b5f3ee07d35b52aa54a279028105b33a9592c +F src/sqliteInt.h 32d7becef5cbd9a1118608921ec99f00f96c6e5d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -266,7 +266,7 @@ F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_blob.c 1f2e3e25255b731c4fcf15ee7990d06347cb6c09 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c 035c17a173937d019b8dfc1d524f9d3fc8123504 +F src/test_config.c 7d50e35f5e94235863c9bac448f63b0d141119e5 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -300,7 +300,7 @@ F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 -F src/trigger.c eb921d1292aca83d515bde5881df71df91d962d6 +F src/trigger.c 6dcdf46a21acf4d4e011c809b2c971e63f797a1a F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 @@ -310,12 +310,12 @@ F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 F src/vdbeaux.c 5ce4f414147a3bc3cbcf00ec57f2606c25791629 -F src/vdbeblob.c 845c24601a4f107f4829725fa0c672b93b816bab +F src/vdbeblob.c 317c71482ed73b0966db2d1c4e20839be3e9fe79 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 -F src/wal.c e8fe2d73c40066872f97db5809ea62f285175825 +F src/wal.c d5c581b635951cf5513ec9699d118b32323443f3 F src/wal.h 0d3ba0c3f1b4c25796cb213568a84b9f9063f465 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c e275cb74731a3351a9da6ba8280bd5054db6192d @@ -777,7 +777,7 @@ F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 -F test/ota.test d81a211dfbdf9fe02b5ad50c86f8a5df924391f5 +F test/ota.test 3a8d97cbf8f7210dc6a638797c4e4cd674036927 F test/ovfl.test 4f7ca651cba5c059a12d8c67dddd49bec5747799 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1200,7 +1200,7 @@ F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh 5dc5010e2e748a9e1bba67baca5956a2c2deda7b F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 22c85e67987ad7d2e8789c48506ec95b99a90c08 +F tool/mkpragmatab.tcl 7c9f48bfe61ba0e4018868bf34b2450026c24ae1 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 88a1e3b0c769773fb7a9ebb363ffc603a4ac21d8 F tool/mksqlite3c.tcl e72c0c97fe1a105fa9616483e652949be2199fe6 @@ -1236,7 +1236,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 14139542b68fbf01632a1b149cd6fca4bb01efa6 b1e6c02f8b9a2afaa12ac15a33e3f698c3be27d6 -R 0115f003ae5ca012270e27f3f42f3f8e +P 7ef44c5b5bd30bcc4ef59ed172b9ce9ac6a843f6 +R a969a1e9314cd2ef179f3839efd0b8ec U dan -Z 7ed6e391881cc7d739c6ad1cb9d3b777 +Z bba2f4cd69d571c58d439710133b8bb2 diff --git a/manifest.uuid b/manifest.uuid index ab5ea12dd7..3c2a061073 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ef44c5b5bd30bcc4ef59ed172b9ce9ac6a843f6 \ No newline at end of file +600cefdd4d29c1de4d107fa7ddeb76a18edce4f5 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 2ba085d9ea..12dd56e2a1 100644 --- a/src/main.c +++ b/src/main.c @@ -1960,6 +1960,7 @@ int sqlite3_wal_checkpoint_v2( #endif } +#ifdef SQLITE_ENABLE_OTA /* ** Open an incremental checkpoint handle. */ @@ -1984,6 +1985,7 @@ int sqlite3_ckpt_open( sqlite3_mutex_leave(db->mutex); return rc; } +#endif /* SQLITE_ENABLE_OTA */ /* diff --git a/src/pager.c b/src/pager.c index db511c3f20..3508a9f937 100644 --- a/src/pager.c +++ b/src/pager.c @@ -642,7 +642,9 @@ struct Pager { u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ +#ifdef SQLITE_ENABLE_OTA u8 otaMode; /* Non-zero if in ota_mode */ +#endif /************************************************************************** ** The following block contains those class members that change during @@ -716,6 +718,16 @@ struct Pager { #endif }; +/* +** Return the value of the pager otaMode flag (0, 1 or 2). Or, if +** SQLITE_ENABLE_OTA is not defined, return constant value 0. +*/ +#ifdef SQLITE_ENABLE_OTA +# define PagerOtaMode(pPager) ((pPager)->otaMode) +#else +# define PagerOtaMode(pPager) 0 +#endif + /* ** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains ** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS @@ -2029,7 +2041,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; } - if( !pPager->exclusiveMode && !pPager->otaMode + if( !pPager->exclusiveMode && !PagerOtaMode(pPager) && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ rc2 = pagerUnlockDb(pPager, SHARED_LOCK); @@ -3985,7 +3997,7 @@ int sqlite3PagerClose(Pager *pPager){ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL sqlite3WalClose(pPager->pWal, - pPager->ckptSyncFlags, pPager->pageSize, (pPager->otaMode?0:pTmp) + pPager->ckptSyncFlags, pPager->pageSize, (PagerOtaMode(pPager)?0:pTmp) ); pPager->pWal = 0; #endif @@ -5191,7 +5203,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** mode. Otherwise, the following function call is a no-op. */ rc = pagerOpenWalIfPresent(pPager); - if( rc==SQLITE_OK && pPager->otaMode ){ + if( rc==SQLITE_OK && PagerOtaMode(pPager) ){ int nWal = sqlite3Strlen30(pPager->zWal); pPager->zWal[nWal-3] = 'o'; rc = pagerOpenWalInternal(pPager, 0); @@ -5205,13 +5217,15 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( pagerUseWal(pPager) ){ assert( rc==SQLITE_OK ); rc = pagerBeginReadTransaction(pPager); - if( rc==SQLITE_OK && pPager->otaMode==1 ){ + if( rc==SQLITE_OK && PagerOtaMode(pPager)==1 ){ rc = sqlite3WalCheckSalt(pPager->pWal, pPager->fd); if( rc!=SQLITE_OK ){ sqlite3WalClose(pPager->pWal, 0, 0, 0); pPager->pWal = 0; }else{ +#ifdef SQLITE_ENABLE_OTA pPager->otaMode = 2; +#endif } } } @@ -7105,7 +7119,7 @@ void sqlite3PagerClearCache(Pager *pPager){ */ int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; - if( pPager->pWal && pPager->otaMode==0 ){ + if( pPager->pWal && PagerOtaMode(pPager)==0 ){ rc = sqlite3WalCheckpoint(pPager->pWal, eMode, pPager->xBusyHandler, pPager->pBusyHandlerArg, pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, @@ -7172,7 +7186,7 @@ static int pagerOpenWal(Pager *pPager){ */ if( rc==SQLITE_OK ){ rc = sqlite3WalOpen(pPager->pVfs, - pPager->fd, pPager->zWal, pPager->exclusiveMode || pPager->otaMode, + pPager->fd, pPager->zWal, pPager->exclusiveMode || PagerOtaMode(pPager), pPager->journalSizeLimit, &pPager->pWal ); } @@ -7181,6 +7195,14 @@ static int pagerOpenWal(Pager *pPager){ return rc; } +/* +** Open the WAL file if it is not open. If it is already open, set *pbOpen +** to 1 before returning. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** The difference between this function and sqlite3PagerOpenWal() is that +** PagerOpenWal() does not open the WAL file if the pager is in OTA mode. +*/ static int pagerOpenWalInternal( Pager *pPager, /* Pager object */ int *pbOpen /* OUT: Set to true if call is a no-op */ @@ -7230,7 +7252,7 @@ int sqlite3PagerOpenWal( Pager *pPager, /* Pager object */ int *pbOpen /* OUT: Set to true if call is a no-op */ ){ - if( pPager->otaMode ) return SQLITE_CANTOPEN; + if( PagerOtaMode(pPager) ) return SQLITE_CANTOPEN_BKPT; return pagerOpenWalInternal(pPager, pbOpen); } @@ -7287,7 +7309,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ ** of pseudo-random data. */ void sqlite3PagerWalSalt(Pager *pPager, u32 *aSalt){ - if( pPager->otaMode ){ + if( PagerOtaMode(pPager) ){ memcpy(aSalt, pPager->dbFileVers, 8); }else{ sqlite3_randomness(8, aSalt); @@ -7310,6 +7332,7 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif +#ifdef SQLITE_ENABLE_OTA /* ** Set or clear the "OTA mode" flag. */ @@ -7341,5 +7364,6 @@ int sqlite3PagerWalCheckpointStart( ); } } +#endif /* !SQLITE_ENABLE_OTA */ #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pragma.c b/src/pragma.c index 5b89237014..140d496abb 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -310,10 +310,12 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif +#if defined(SQLITE_ENABLE_OTA) { /* zName: */ "ota_mode", /* ePragTyp: */ PragTyp_FLAG, /* ePragFlag: */ 0, /* iArg: */ SQLITE_OtaMode }, +#endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "page_count", /* ePragTyp: */ PragTyp_PAGE_COUNT, @@ -324,10 +326,12 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif +#if defined(SQLITE_ENABLE_OTA) { /* zName: */ "pager_ota_mode", /* ePragTyp: */ PragTyp_PAGER_OTA_MODE, /* ePragFlag: */ 0, /* iArg: */ 0 }, +#endif #if defined(SQLITE_DEBUG) { /* zName: */ "parser_trace", /* ePragTyp: */ PragTyp_PARSER_TRACE, @@ -481,7 +485,7 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 59 on by default, 72 total. */ +/* Number of pragmas: 57 on by default, 72 total. */ /* End of the automatically generated pragma table. ***************************************************************************/ @@ -896,6 +900,7 @@ void sqlite3Pragma( ** Other clients see a rollback-mode database on which the pager_ota_mode ** client is holding a SHARED lock. */ +#ifdef SQLITE_ENABLE_OTA case PragTyp_PAGER_OTA_MODE: { Btree *pBt = pDb->pBt; assert( pBt!=0 ); @@ -916,6 +921,7 @@ void sqlite3Pragma( } break; } +#endif /* SQLITE_ENABLE_OTA */ #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) /* diff --git a/src/sqlite.h.in b/src/sqlite.h.in index e5e98d2a8a..2d8d5fd773 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7445,62 +7445,6 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* #define SQLITE_ABORT 4 // Also an error code */ #define SQLITE_REPLACE 5 -/* -** Allocate a statement handle that may be used to write directly to an -** index b-tree. This allows the user to create a corrupt database. Once -** the statement handle is allocated, it may be used with the same APIs -** as any statement handle created with sqlite3_prepare(). -** -** The statement writes to the index specified by parameter zIndex, which -** must be in the "main" database. If argument bDelete is false, then each -** time the statement is sqlite3_step()ed, an entry is inserted into the -** b-tree index. If it is true, then an entry may be deleted (or may not, if -** the specified key is not found) each time the statement is -** sqlite3_step()ed. -** -** If statement compilation is successful, *ppStmt is set to point to the -** new statement handle and SQLITE_OK is returned. Otherwise, if an error -** occurs, *ppStmt is set to NULL and an error code returned. An error -** message may be left in the database handle in this case. -** -** If statement compilation succeeds, output variable *pnCol is set to the -** total number of columns in the index, including the primary key columns -** at the end. Variable *paiCol is set to point to an array *pnCol entries -** in size. Each entry is the table column index, numbered from zero from left -** to right, of the corresponding index column. For example, if: -** -** CREATE TABLE t1(a, b, c, d); -** CREATE INDEX i1 ON t1(b, c); -** -** then *pnCol is 3 and *paiCol points to an array containing {1, 2, -1}. -** If table t1 had an explicit INTEGER PRIMARY KEY, then the "-1" in the -** *paiCol array would be replaced by its column index. Or if: -** -** CREATE TABLE t2(a, b, c, d, PRIMARY KEY(d, c)) WITHOUT ROWID; -** CREATE INDEX i2 ON t2(a); -** -** then (*pnCol) is 3 and *paiCol points to an array containing {0, 3, 2}. -** -** The lifetime of the array is the same as that of the statement handle - -** it is automatically freed when the statement handle is passed to -** sqlite3_finalize(). -** -** The statement has (*pnCol) SQL variables that values may be bound to. -** They correspond to the values used to create the index key that is -** inserted or deleted when the statement is stepped. -** -** If the index is a UNIQUE index, the usual checking and error codes apply -** to insert operations. -*/ -int sqlite3_index_writer( - sqlite3 *db, - int bDelete, /* Zero for insert, non-zero for delete */ - const char *zIndex, /* Index to write to */ - sqlite3_stmt**, /* OUT: New statement handle */ - const char ***pazColl, /* OUT: Collation sequences for each column */ - int **paiCol, int *pnCol /* OUT: See above */ -); - /* ** CAPI3REF: Prepared Statement Scan Status Opcodes ** KEYWORDS: {scanstatus options} @@ -7594,6 +7538,66 @@ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( */ SQLITE_EXPERIMENTAL void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); +/* +** Allocate a statement handle that may be used to write directly to an +** index b-tree. This allows the user to create a corrupt database. Once +** the statement handle is allocated, it may be used with the same APIs +** as any statement handle created with sqlite3_prepare(). +** +** The statement writes to the index specified by parameter zIndex, which +** must be in the "main" database. If argument bDelete is false, then each +** time the statement is sqlite3_step()ed, an entry is inserted into the +** b-tree index. If it is true, then an entry may be deleted (or may not, if +** the specified key is not found) each time the statement is +** sqlite3_step()ed. +** +** If statement compilation is successful, *ppStmt is set to point to the +** new statement handle and SQLITE_OK is returned. Otherwise, if an error +** occurs, *ppStmt is set to NULL and an error code returned. An error +** message may be left in the database handle in this case. +** +** If statement compilation succeeds, output variable *pnCol is set to the +** total number of columns in the index, including the primary key columns +** at the end. Variable *paiCol is set to point to an array *pnCol entries +** in size. Each entry is the table column index, numbered from zero from left +** to right, of the corresponding index column. For example, if: +** +** CREATE TABLE t1(a, b, c, d); +** CREATE INDEX i1 ON t1(b, c); +** +** then *pnCol is 3 and *paiCol points to an array containing {1, 2, -1}. +** If table t1 had an explicit INTEGER PRIMARY KEY, then the "-1" in the +** *paiCol array would be replaced by its column index. Or if: +** +** CREATE TABLE t2(a, b, c, d, PRIMARY KEY(d, c)) WITHOUT ROWID; +** CREATE INDEX i2 ON t2(a); +** +** then (*pnCol) is 3 and *paiCol points to an array containing {0, 3, 2}. +** +** The lifetime of the array is the same as that of the statement handle - +** it is automatically freed when the statement handle is passed to +** sqlite3_finalize(). +** +** The statement has (*pnCol) SQL variables that values may be bound to. +** They correspond to the values used to create the index key that is +** inserted or deleted when the statement is stepped. +** +** If the index is a UNIQUE index, the usual checking and error codes apply +** to insert operations. +** +** This API is only available if SQLITE_ENABLE_OTA is defined at compile +** time. It is intended for use by the OTA extension only. As such, it is +** subject to change or removal at any point. +*/ +int sqlite3_index_writer( + sqlite3 *db, + int bDelete, /* Zero for insert, non-zero for delete */ + const char *zIndex, /* Index to write to */ + sqlite3_stmt**, /* OUT: New statement handle */ + const char ***pazColl, /* OUT: Collation sequence for each column */ + int **paiCol, int *pnCol /* OUT: See above */ +); + /* ** Incremental checkpoint API. ** @@ -7622,9 +7626,13 @@ SQLITE_EXPERIMENTAL void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); ** The contents of this buffer may be passed to a later call to ** sqlite3_ckpt_open() to restart the checkpoint. The second output variable ** is set to the size of the buffer in bytes. +** +** These APIs are only available if SQLITE_ENABLE_OTA is defined at compile +** time. They are intended for use by the OTA extension only. As such, they +** are subject to change or removal at any point. */ typedef struct sqlite3_ckpt sqlite3_ckpt; -int sqlite3_ckpt_open(sqlite3*, unsigned char *a, int n, sqlite3_ckpt **ppCkpt); +int sqlite3_ckpt_open(sqlite3*, unsigned char*, int n, sqlite3_ckpt **ppCkpt); int sqlite3_ckpt_step(sqlite3_ckpt*); int sqlite3_ckpt_close(sqlite3_ckpt*, unsigned char **pa, int *pn); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8bc737d199..b89214354e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1196,8 +1196,11 @@ struct sqlite3 { #define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ #define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ -#define SQLITE_OtaMode 0x08000000 /* True in "ota mode" */ - +#ifdef SQLITE_ENABLE_OTA +# define SQLITE_OtaMode 0x08000000 /* True in "ota mode" */ +#else +# define SQLITE_OtaMode 0x00000000 +#endif /* ** Bits of the sqlite3.dbOptFlags field that are used by the diff --git a/src/test_config.c b/src/test_config.c index 834113b33b..93f6083838 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -430,6 +430,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "or_opt", "1", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_ENABLE_OTA + Tcl_SetVar2(interp, "sqlite_options", "ota", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "ota", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_PAGER_PRAGMAS Tcl_SetVar2(interp, "sqlite_options", "pager_pragmas", "0", TCL_GLOBAL_ONLY); #else diff --git a/src/trigger.c b/src/trigger.c index f837e23b5a..44b9b89fcc 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -50,7 +50,7 @@ void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ Schema * const pTmpSchema = pParse->db->aDb[1].pSchema; Trigger *pList = 0; /* List of triggers to return */ - int bOta = !!(pParse->db->flags & SQLITE_OtaMode); + const int bOta = !!(pParse->db->flags & SQLITE_OtaMode); if( pParse->disableTriggers ){ return 0; diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 8565844f21..e033468e96 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -467,6 +467,11 @@ int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ return rc; } +#ifdef SQLITE_ENABLE_OTA +/* +** Allocate and populate the output arrays returned by the +** sqlite3_index_writer() function. +*/ static int indexWriterOutputVars( sqlite3 *db, Index *pIdx, @@ -529,7 +534,10 @@ static int indexWriterOutputVars( return SQLITE_OK; } - +/* +** Prepare and return an SQL statement handle that can be used to write +** directly to an index b-tree. +*/ int sqlite3_index_writer( sqlite3 *db, int bDelete, @@ -634,5 +642,6 @@ index_writer_out: sqlite3_mutex_leave(db->mutex); return rc; } +#endif /* SQLITE_ENABLE_OTA */ #endif /* #ifndef SQLITE_OMIT_INCRBLOB */ diff --git a/src/wal.c b/src/wal.c index ba86d84b73..bb8c0f806e 100644 --- a/src/wal.c +++ b/src/wal.c @@ -483,7 +483,14 @@ struct WalIterator { }; /* -** walCheckpoint +** An object of the following type is used to store state information for +** an ongoing checkpoint operation. For normal checkpoints, the instance +** is allocated on the stack by the walCheckpoint() function. For the special +** incremental checkpoints performed by OTA clients, it is allocated in +** heap memory by sqlite3WalCheckpointStart(). +** +** See the implementations of walCheckpointStart(), walCheckpointStep() and +** walCheckpointFinalize() for further details. */ typedef struct WalCkpt WalCkpt; struct WalCkpt { @@ -1642,8 +1649,15 @@ static int walPagesize(Wal *pWal){ return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); } +/* +** Initialize the contents of the WalCkpt object indicated by the final +** argument and begin a checkpoint operation. The CKPT lock must already +** be held when this function is called. +** +** Return SQLITE_OK if successful or an error code otherwise. +*/ static int walCheckpointStart( - Wal *pWal, + Wal *pWal, /* Wal connection */ u8 *aBuf, /* Page-sized temporary buffer */ int nBuf, /* Size of aBuf[] in bytes */ int (*xBusy)(void*), /* Function to call when busy (or NULL) */ @@ -1655,7 +1669,6 @@ static int walCheckpointStart( int i; /* Iterator variable */ memset(p, 0, sizeof(WalCkpt)); - if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ return SQLITE_CORRUPT_BKPT; } @@ -1729,6 +1742,12 @@ static int walCheckpointStart( return rc; } +/* +** Attempt to copy the next frame from the wal file to the database file. If +** there are no more frames to copy to the database file return SQLITE_DONE. +** If the frame is successfully copied, return SQLITE_OK. Or, if an error +** occurs, return an SQLite error code. +*/ static int walCheckpointStep(WalCkpt *p){ u32 iDbpage = 0; /* Next database page to write */ u32 iFrame = 0; /* Wal frame containing data for iDbpage */ @@ -1760,6 +1779,16 @@ static int walCheckpointStep(WalCkpt *p){ return rc; } +/* +** The current round of checkpointing work using the object indicated by +** the only argument is now finished. If no error occcurred, this function +** saves the results to shared memory (i.e. updates the WalCkptInfo.nBackfill +** variable), and truncates and syncs the database file as required. +** +** All dynamic resources currently held by the WalCkpt object are released. +** It is the responsibility of the caller to delete the WalCkpt itself if +** required. +*/ static int walCheckpointFinalize(WalCkpt *p){ if( p->pIter ){ int rc = p->rc; @@ -1780,9 +1809,15 @@ static int walCheckpointFinalize(WalCkpt *p){ p->pInfo->nBackfill = p->mxSafeFrame; } p->rc = rc; - }else if( rc==SQLITE_OK && p->sync_flags ){ - /* If work was not completed, but no error has occured. */ - p->rc = sqlite3OsSync(pWal->pDbFd, p->sync_flags); + }else{ +#ifdef SQLITE_ENABLE_OTA + if( rc==SQLITE_OK && p->sync_flags ){ + /* If work was not completed, but no error has occured. */ + p->rc = sqlite3OsSync(pWal->pDbFd, p->sync_flags); + } +#else + assert( rc!=SQLITE_OK ); +#endif } /* Release the reader lock held while backfilling */ @@ -1846,7 +1881,6 @@ static int walCheckpoint( /* Step the checkpoint object until it reports something other than ** SQLITE_OK. */ while( SQLITE_OK==(rc = walCheckpointStep(&sC)) ); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; rc = walCheckpointFinalize(&sC); /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal @@ -1893,8 +1927,8 @@ static void walLimitSize(Wal *pWal, i64 nMax){ /* ** Close a connection to a log file. ** -** If parameter zBuf is not NULL, attempt to obtain an exclusive lock -** and run a checkpoint. +** If parameter zBuf is not NULL, also attempt to obtain an exclusive +** lock and run a checkpoint. */ int sqlite3WalClose( Wal *pWal, /* Wal to close */ @@ -1914,7 +1948,10 @@ int sqlite3WalClose( ** ** The EXCLUSIVE lock is not released before returning. */ - if( zBuf ){ +#ifdef SQLITE_ENABLE_OTA + if( zBuf ) /* In non-OTA builds, zBuf is always non-NULL */ +#endif + { rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ @@ -3071,6 +3108,11 @@ int sqlite3WalCheckpoint( return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); } +#ifdef SQLITE_ENABLE_OTA + +/* +** Step the checkpoint object passed as the first argument. +*/ int sqlite3_ckpt_step(sqlite3_ckpt *pCkpt){ int rc; WalCkpt *p = (WalCkpt*)pCkpt; @@ -3080,6 +3122,14 @@ int sqlite3_ckpt_step(sqlite3_ckpt *pCkpt){ return rc; } +/* +** Close the checkpoint object passed as the first argument. If the checkpoint +** was completed, zero the two output variables. Otherwise, set *paState to +** point to a buffer containing data that may be passed to a subsequent +** call to ckpt_open() to resume the checkpoint. In this case *pnState is +** set to the size of the buffer in bytes. The buffer should be eventually +** freed by the caller using sqlite3_free(). +*/ int sqlite3_ckpt_close(sqlite3_ckpt *pCkpt, u8 **paState, int *pnState){ int rc; WalCkpt *p = (WalCkpt*)pCkpt; @@ -3192,6 +3242,25 @@ int sqlite3WalCheckpointStart( return rc; } +/* +** Unless the wal file is empty, check that the 8 bytes of salt stored in +** the wal header are identical to those in the buffer indicated by the +** second argument. If they are not, return SQLITE_BUSY_SNAPSHOT. Otherwise, +** if the buffers match or the WAL file is empty, return SQLITE_OK. +*/ +int sqlite3WalCheckSalt(Wal *pWal, sqlite3_file *pFd){ + int rc = SQLITE_OK; + if( pWal->hdr.mxFrame>0 ){ + u8 aData[16]; + rc = sqlite3OsRead(pFd, aData, sizeof(aData), 24); + if( rc==SQLITE_OK && memcmp(pWal->hdr.aSalt, aData, 8) ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + } + return rc; +} +#endif /* SQLITE_ENABLE_OTA */ + /* Return the value to pass to a sqlite3_wal_hook callback, the ** number of frames in the WAL at the point of the last commit since ** sqlite3WalCallback() was called. If no commits have occurred since @@ -3276,24 +3345,6 @@ int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } -/* -** Unless the wal file is empty, check that the 8 bytes of salt stored in -** the wal header are identical to those in the buffer indicated by the -** second argument. If they are not, return SQLITE_BUSY_SNAPSHOT. Otherwise, -** if the buffers match or the WAL file is empty, return SQLITE_OK. -*/ -int sqlite3WalCheckSalt(Wal *pWal, sqlite3_file *pFd){ - int rc = SQLITE_OK; - if( pWal->hdr.mxFrame>0 ){ - u8 aData[16]; - rc = sqlite3OsRead(pFd, aData, sizeof(aData), 24); - if( rc==SQLITE_OK && memcmp(pWal->hdr.aSalt, aData, 8) ){ - rc = SQLITE_BUSY_SNAPSHOT; - } - } - return rc; -} - #ifdef SQLITE_ENABLE_ZIPVFS /* ** If the argument is not NULL, it points to a Wal object that holds a diff --git a/test/ota.test b/test/ota.test index 676666d91f..9dc01c2b3a 100644 --- a/test/ota.test +++ b/test/ota.test @@ -11,6 +11,8 @@ set testdir [file dirname $argv0] source $testdir/permutations.test -run_test_suite ota +ifcapable !ota { finish_test ; return } +run_test_suite ota finish_test + diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 613b565553..3c36a6c011 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -298,10 +298,12 @@ set pragma_def { NAME: threads NAME: pager_ota_mode + IF: defined(SQLITE_ENABLE_OTA) NAME: ota_mode TYPE: FLAG ARG: SQLITE_OtaMode + IF: defined(SQLITE_ENABLE_OTA) } fconfigure stdout -translation lf set name {} From ca3d648f9c26b911a9e412e66d459984e4c89878 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Nov 2014 18:09:46 +0000 Subject: [PATCH 041/116] Update ota so that the hidden columns of virtual tables may be written. FossilOrigin-Name: ccee999649d0fa1d48e53847542f4cbe05e3d694 --- ext/ota/ota10.test | 122 +++++++++++++++++++++++++++++++++++++++++++ ext/ota/ota3.test | 2 +- ext/ota/sqlite3ota.c | 104 ++++++++++++++++++++++++++---------- ext/ota/sqlite3ota.h | 13 +++++ manifest | 17 +++--- manifest.uuid | 2 +- 6 files changed, 222 insertions(+), 38 deletions(-) create mode 100644 ext/ota/ota10.test diff --git a/ext/ota/ota10.test b/ext/ota/ota10.test new file mode 100644 index 0000000000..bca34d68cb --- /dev/null +++ b/ext/ota/ota10.test @@ -0,0 +1,122 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix ota10 + + +#-------------------------------------------------------------------- +# Test that UPDATE commands work even if the input columns are in a +# different order to the output columns. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 'b', 'c'); +} + +proc apply_ota {sql} { + forcedelete ota.db + sqlite3 db2 ota.db + db2 eval $sql + db2 close + sqlite3ota ota test.db ota.db + while { [ota step]=="SQLITE_OK" } {} + ota close +} + +do_test 1.1 { + apply_ota { + CREATE TABLE data_t1(a, c, b, ota_control); + INSERT INTO data_t1 VALUES(1, 'xxx', NULL, '.x.'); + } + db eval { SELECT * FROM t1 } +} {1 b xxx} + +#-------------------------------------------------------------------- +# Test that the hidden languageid column of an fts4 table can be +# written. +# +ifcapable fts3 { + do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts4(a, b, languageid='langid'); + } + do_test 2.1 { + apply_ota { + CREATE TABLE data_ft(a, b, ota_rowid, langid, ota_control); + INSERT INTO data_ft VALUES('a', 'b', 22, 1, 0); -- insert + INSERT INTO data_ft VALUES('a', 'b', 23, 10, 0); -- insert + INSERT INTO data_ft VALUES('a', 'b', 24, 100, 0); -- insert + } + db eval { SELECT a, b, rowid, langid FROM ft } + } [list {*}{ + a b 22 1 + a b 23 10 + a b 24 100 + }] + + # Or not - this data_xxx table has no langid column, so langid + # defaults to 0. + # + do_test 2.2 { + apply_ota { + CREATE TABLE data_ft(a, b, ota_rowid, ota_control); + INSERT INTO data_ft VALUES('a', 'b', 25, 0); -- insert + } + db eval { SELECT a, b, rowid, langid FROM ft } + } [list {*}{ + a b 22 1 + a b 23 10 + a b 24 100 + a b 25 0 + }] + + # Update langid. + # + do_test 2.3 { + apply_ota { + CREATE TABLE data_ft(a, b, ota_rowid, langid, ota_control); + INSERT INTO data_ft VALUES(NULL, NULL, 23, 50, '..x'); + INSERT INTO data_ft VALUES(NULL, NULL, 25, 500, '..x'); + } + db eval { SELECT a, b, rowid, langid FROM ft } + } [list {*}{ + a b 22 1 + a b 23 50 + a b 24 100 + a b 25 500 + }] +} + +#-------------------------------------------------------------------- +# Test that if writing a hidden virtual table column is an error, +# attempting to do so via ota is also an error. +# +ifcapable fts3 { + do_execsql_test 3.0 { + CREATE VIRTUAL TABLE xt USING fts4(a); + } + do_test 3.1 { + list [catch { + apply_ota { + CREATE TABLE data_xt(a, xt, ota_rowid, ota_control); + INSERT INTO data_xt VALUES('a', 'b', 1, 0); + } + } msg] $msg + } {1 {SQLITE_ERROR - SQL logic error or missing database}} +} + + + +finish_test diff --git a/ext/ota/ota3.test b/ext/ota/ota3.test index 09d1bab083..8921c12b35 100644 --- a/ext/ota/ota3.test +++ b/ext/ota/ota3.test @@ -96,7 +96,7 @@ do_test 2.1 { } db2 close list [catch { run_ota test.db ota.db } msg] $msg -} {1 {SQLITE_ERROR - no such column: c}} +} {1 {SQLITE_ERROR - column missing from data_x1: c}} do_execsql_test 2.2 { PRAGMA integrity_check; diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 6c3c99434e..b1bc1d10cc 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -223,7 +223,6 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){ sqlite3_free(pIter->azTblCol[i]); } sqlite3_free(pIter->azTblCol); - sqlite3_free(pIter->abTblPk); pIter->azTblCol = 0; pIter->abTblPk = 0; pIter->nTblCol = 0; @@ -369,6 +368,24 @@ static char *otaQuoteName(const char *zName){ return zRet; } +/* +** Argument zName points to a column name. Argument zQuoted also points +** to a column name, but one that has been quoted using otaQuoteName(). +** Return true if the column names are the same, or false otherwise. +*/ +static int otaMatchName(const char *zName, const char *zQuoted){ + const char *p = zName; + const char *q = &zQuoted[1]; + while( 1 ){ + if( *q=='`' ) q++; + if( sqlite3_strnicmp(q, p, 1) ) return 0; + if( !*q ) break; + p++; + q++; + } + return 1; +} + /* ** Argument zFmt is a sqlite3_mprintf() style format string. The trailing ** arguments are the usual subsitution values. This function performs @@ -396,21 +413,22 @@ static int otaMPrintfExec(sqlite3ota *p, const char *zFmt, ...){ } /* -** Increase the size of the pIter->azTblCol[] and abTblPk[] arrays so that +** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that ** there is room for at least nCol elements. If an OOM occurs, store an ** error code in the OTA handle passed as the first argument. */ -static void otaExtendIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ - assert( p->rc==SQLITE_OK ); - if( (nCol % 8)==0 ){ - unsigned char *abNew; - int nByte = sizeof(char*) * (nCol+8); - char **azNew = (char**)sqlite3_realloc(pIter->azTblCol, nByte); - abNew = (unsigned char*)sqlite3_realloc(pIter->abTblPk, nCol+8); +static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ + int nByte = sizeof(char*) * nCol + sizeof(unsigned char*) * nCol; + char **azNew; - if( azNew ) pIter->azTblCol = azNew; - if( abNew ) pIter->abTblPk = abNew; - if( azNew==0 || abNew==0 ) p->rc = SQLITE_NOMEM; + assert( p->rc==SQLITE_OK ); + azNew = (char**)sqlite3_malloc(nByte); + if( azNew ){ + memset(azNew, 0, nByte); + pIter->azTblCol = azNew; + pIter->abTblPk = (unsigned char*)&pIter->azTblCol[nCol]; + }else{ + p->rc = SQLITE_NOMEM; } } @@ -451,30 +469,60 @@ static int otaIsVtab(sqlite3ota *p, const char *zTab){ */ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ if( pIter->azTblCol==0 ){ - sqlite3_stmt *pStmt; - char *zSql; + sqlite3_stmt *pStmt = 0; int nCol = 0; int bSeenPk = 0; + int i; /* for() loop iterator variable */ int rc2; /* sqlite3_finalize() return value */ assert( pIter->bRowid==0 && pIter->bVtab==0 ); - zSql = sqlite3_mprintf("PRAGMA main.table_info(%Q)", pIter->zTbl); - p->rc = prepareFreeAndCollectError(p->db, &pStmt, &p->zErrmsg, zSql); - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - otaExtendIterArrays(p, pIter, nCol); - if( p->rc==SQLITE_OK ){ - const char *zName = (const char*)sqlite3_column_text(pStmt, 1); - int iPk = sqlite3_column_int(pStmt, 5); - pIter->abTblPk[nCol] = (iPk!=0); - if( iPk ) bSeenPk = 1; - if( iPk<0 ) pIter->bRowid = 1; - pIter->azTblCol[nCol] = otaQuoteName(zName); - if( pIter->azTblCol[nCol]==0 ) p->rc = SQLITE_NOMEM; - nCol++; + /* Populate the azTblCol[] and nTblCol variables based on the columns + ** of the input table. Ignore any input table columns that begin with + ** "ota_". */ + p->rc = prepareFreeAndCollectError(p->db, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT * FROM 'data_%q'", pIter->zTbl) + ); + if( p->rc==SQLITE_OK ){ + nCol = sqlite3_column_count(pStmt); + otaAllocateIterArrays(p, pIter, nCol); + } + for(i=0; p->rc==SQLITE_OK && iazTblCol[pIter->nTblCol++] = zCopy; + if( zCopy==0 ) p->rc = SQLITE_NOMEM; + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + + /* Check that all non-HIDDEN columns in the destination table are also + ** present in the input table. Populate the abTblPk[] array at the + ** same time. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->db, &pStmt, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.table_info(%Q)", pIter->zTbl) + ); + } + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + for(i=0; inTblCol; i++){ + if( otaMatchName(zName, pIter->azTblCol[i]) ) break; + } + if( i==pIter->nTblCol ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("column missing from data_%q: %s", + pIter->zTbl, zName + ); + }else{ + int iPk = sqlite3_column_int(pStmt, 5); + pIter->abTblPk[i] = (iPk!=0); + if( iPk ) bSeenPk = 1; + if( iPk<0 ) pIter->bRowid = 1; } } - pIter->nTblCol = nCol; rc2 = sqlite3_finalize(pStmt); if( p->rc==SQLITE_OK ) p->rc = rc2; diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index 8a201f25bd..a652316ca2 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -106,6 +106,19 @@ ** ** CREATE TABLE data_ft1(a, b, ota_rowid, ota_control); ** +** All non-hidden columns (i.e. all columns matched by "SELECT *") of the +** target table must be present in the input table. For virtual tables, +** hidden columns are optional - they are updated by OTA if present in +** the input table, or not otherwise. For example, to write to an fts4 +** table with a hidden languageid column such as: +** +** CREATE VIRTUAL TABLE ft1 USING fts4(a, b, languageid='langid'); +** +** Either of the following input table schemas may be used: +** +** CREATE TABLE data_ft1(a, b, langid, ota_rowid, ota_control); +** CREATE TABLE data_ft1(a, b, ota_rowid, ota_control); +** ** For each row to INSERT into the target database as part of the OTA ** update, the corresponding data_% table should contain a single record ** with the "ota_control" column set to contain integer value 0. The diff --git a/manifest b/manifest index 5d142cd5c1..00072e021c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_ENABLE_OTA\spre-processor\sdirectives\sso\sthat\sthis\sbranch\smay\sbe\scompiled\swith\sor\swithout\sOTA. -D 2014-11-22T09:09:50.320 +C Update\sota\sso\sthat\sthe\shidden\scolumns\sof\svirtual\stables\smay\sbe\swritten. +D 2014-11-27T18:09:46.630 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -126,8 +126,9 @@ F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt 78d4a9f78f567d4bf826cf0f02df6254902562ca F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 F ext/ota/ota1.test a8f9d89c9b2d381a663bcedaa5dd5952cdbd1231 +F ext/ota/ota10.test ab815dff9cef7248c504f06b888627d236f25e9c F ext/ota/ota2.test 4568c2671d19dbde789fb9091d727a2e94880128 -F ext/ota/ota3.test 215dd4a8e238567e0f890a5139b6fdf5494ef311 +F ext/ota/ota3.test 71bd8cc0cf8d7e7d9bb11a1fcc238320a5a9d8c8 F ext/ota/ota4.test 60f897f329a6782ef2f24862640acf3c52e48077 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 @@ -135,8 +136,8 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d9ad30ccb4e08f878e382876fe67752309538af9 F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b -F ext/ota/sqlite3ota.c accfada2ab182dc52225e7345f656520a4e8db22 -F ext/ota/sqlite3ota.h 04577b00c456aacb99be9c8b55572a6e3ca9aa27 +F ext/ota/sqlite3ota.c f1e930690ec2bcb72138301ebf05ea2ffd4450de +F ext/ota/sqlite3ota.h 8dc9c812e61f47f497336a45f820bee88b33a2c5 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1236,7 +1237,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7ef44c5b5bd30bcc4ef59ed172b9ce9ac6a843f6 -R a969a1e9314cd2ef179f3839efd0b8ec +P 600cefdd4d29c1de4d107fa7ddeb76a18edce4f5 +R 19cf180a791a327f1f4fb68723a586ef U dan -Z bba2f4cd69d571c58d439710133b8bb2 +Z 7646a22500cce7c7ef211b057e34ff27 diff --git a/manifest.uuid b/manifest.uuid index 3c2a061073..0c798bdfcc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -600cefdd4d29c1de4d107fa7ddeb76a18edce4f5 \ No newline at end of file +ccee999649d0fa1d48e53847542f4cbe05e3d694 \ No newline at end of file From 7ca1d07aff15812cab4adc9fd403632ad662dcf6 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 6 Dec 2014 19:30:41 +0000 Subject: [PATCH 042/116] Allow the ota extension to write to tables with no PRIMARY KEY declaration. FossilOrigin-Name: ba59a7e2ba97244492cbca9247456df0f3f19248 --- ext/ota/ota1.test | 5 +- ext/ota/ota9.test | 64 +++++++++++++++++++++- ext/ota/sqlite3ota.c | 126 +++++++++++++++++++++++++++++++++---------- manifest | 16 +++--- manifest.uuid | 2 +- 5 files changed, 173 insertions(+), 40 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index e6b50c5906..74600ff6ec 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -177,6 +177,9 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { # relaxed so that external PRIMARY KEYs on tables with automatic rowids # are now allowed. # +# UPDATE 2: Tables without any PRIMARY KEY declaration are now allowed. +# However the input table must feature an "ota_rowid" column. +# reset_db create_ota1 ota.db do_execsql_test 2.1 { CREATE TABLE t1(a, b, c) } @@ -186,7 +189,7 @@ do_test 2.2 { } {SQLITE_ERROR} do_test 2.3 { list [catch { ota close } msg] $msg -} {1 {SQLITE_ERROR - table t1 has no PRIMARY KEY}} +} {1 {SQLITE_ERROR - table data_t1 requires ota_rowid column}} reset_db do_execsql_test 2.4 { CREATE TABLE t1(a PRIMARY KEY, b, c) } do_test 2.5 { diff --git a/ext/ota/ota9.test b/ext/ota/ota9.test index 18791b479e..e746d363f1 100644 --- a/ext/ota/ota9.test +++ b/ext/ota/ota9.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# Test OTA with virtual tables +# Test OTA with virtual tables. And tables with no PRIMARY KEY declarations. # if {![info exists testdir]} { @@ -60,6 +60,68 @@ do_execsql_test 1.2.2 { SELECT rowid, * FROM f1 } { do_execsql_test 1.2.3 { INSERT INTO f1(f1) VALUES('integrity-check') } integrity_check 1.2.4 +#------------------------------------------------------------------------- +# Tables with no PK declaration. +# + +# Run the OTA in file $ota on target database $target until completion. +# +proc run_ota {target ota} { + sqlite3ota ota $target $ota + while { [ota step]=="SQLITE_OK" } {} + ota close +} + +foreach {tn idx} { + 1 { } + 2 { + CREATE INDEX i1 ON t1(a); + } + 3 { + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(a, a, a, b, b, b, c, c, c); + } +} { + + reset_db + do_execsql_test 2.$tn.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1(rowid, a, b, c) VALUES(-1, 'a', 'b', 'c'); + INSERT INTO t1(rowid, a, b, c) VALUES(-2, 'd', 'e', 'f'); + } + + db eval $idx + + do_test 2.$tn.2 { + forcedelete ota.db + sqlite3 db2 ota.db + db2 eval { + CREATE TABLE data_t1(ota_rowid, a, b, c, ota_control); + INSERT INTO data_t1 VALUES(3, 'x', 'y', 'z', 0); + INSERT INTO data_t1 VALUES(NULL, 'X', 'Y', 'Z', 0); + INSERT INTO data_t1 VALUES('1', NULL, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(-2, NULL, NULL, 'fff', '..x'); + } + db2 close + } {} + + run_ota test.db ota.db + + do_execsql_test 2.$tn.3 { + SELECT rowid, a, b, c FROM t1 ORDER BY rowid; + } { + -2 d e fff + -1 a b c + 2 4 5 6 + 3 x y z + 4 X Y Z + } + + integrity_check 2.$tn.4 +} finish_test diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index b1bc1d10cc..a3d73ae67b 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -104,8 +104,11 @@ struct OtaObjIter { int nTblCol; /* Size of azTblCol[] array */ char **azTblCol; /* Array of quoted column names */ unsigned char *abTblPk; /* Array of flags - true for PK columns */ + int eType; +#if 0 unsigned char bRowid; /* True for implicit IPK tables */ unsigned char bVtab; /* True for a virtual table */ +#endif /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ @@ -124,6 +127,14 @@ struct OtaObjIter { sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ }; +/* +** Values for OtaObjIter.eType +*/ +#define OTA_PK_REAL 1 /* Table has a real primary key */ +#define OTA_PK_EXTERNAL 2 /* Table has an external primary key index */ +#define OTA_PK_NONE 3 /* Table has no PK (use rowid) */ +#define OTA_PK_VTAB 4 /* Table is a virtual table (use rowid) */ + /* ** OTA handle. */ @@ -228,8 +239,7 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){ pIter->nTblCol = 0; sqlite3_free(pIter->zMask); pIter->zMask = 0; - pIter->bRowid = 0; - pIter->bVtab = 0; + pIter->eType = 0; /* Invalid value */ } /* @@ -471,11 +481,11 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ if( pIter->azTblCol==0 ){ sqlite3_stmt *pStmt = 0; int nCol = 0; - int bSeenPk = 0; int i; /* for() loop iterator variable */ int rc2; /* sqlite3_finalize() return value */ + int bOtaRowid = 0; /* If input table has column "ota_rowid" */ - assert( pIter->bRowid==0 && pIter->bVtab==0 ); + assert( pIter->eType==0 ); /* Populate the azTblCol[] and nTblCol variables based on the columns ** of the input table. Ignore any input table columns that begin with @@ -494,6 +504,9 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ pIter->azTblCol[pIter->nTblCol++] = zCopy; if( zCopy==0 ) p->rc = SQLITE_NOMEM; } + else if( 0==sqlite3_stricmp("ota_rowid", zName) ){ + bOtaRowid = 1; + } } sqlite3_finalize(pStmt); pStmt = 0; @@ -519,20 +532,33 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ }else{ int iPk = sqlite3_column_int(pStmt, 5); pIter->abTblPk[i] = (iPk!=0); - if( iPk ) bSeenPk = 1; - if( iPk<0 ) pIter->bRowid = 1; + if( iPk ){ + pIter->eType = (iPk<0) ? OTA_PK_EXTERNAL : OTA_PK_REAL; + } } } rc2 = sqlite3_finalize(pStmt); if( p->rc==SQLITE_OK ) p->rc = rc2; - if( p->rc==SQLITE_OK && bSeenPk==0 ){ - const char *zTab = pIter->zTbl; - if( otaIsVtab(p, zTab) ){ - pIter->bVtab = 1; - }else{ - p->zErrmsg = sqlite3_mprintf("table %s has no PRIMARY KEY", zTab); + if( p->rc==SQLITE_OK ){ + if( pIter->eType==0 ){ + /* This must either be a virtual table, or a regular table with no + ** PRIMARY KEY declaration whatsoever. */ + if( bOtaRowid==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf( + "table data_%q requires ota_rowid column", pIter->zTbl + ); + }else if( otaIsVtab(p, pIter->zTbl) ){ + pIter->eType = OTA_PK_VTAB; + }else{ + pIter->eType = OTA_PK_NONE; + } + }else if( bOtaRowid ){ p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf( + "table data_%q may not have ota_rowid column", pIter->zTbl + ); } } } @@ -618,6 +644,17 @@ static char *otaObjIterGetCollist( return zList; } +/* +** Assuming the current table columns are "a", "b" and "c", and the zObj +** paramter is passed "old", return a string of the form: +** +** "old.a, old.b, old.b" +** +** With the column names escaped. +** +** For tables with implicit rowids - OTA_PK_EXTERNAL and OTA_PK_NONE, append +** the text ", old._rowid_" to the returned value. +*/ static char *otaObjIterGetOldlist( sqlite3ota *p, OtaObjIter *pIter, @@ -637,21 +674,31 @@ static char *otaObjIterGetOldlist( } /* For a table with implicit rowids, append "old._rowid_" to the list. */ - if( pIter->bRowid ){ + if( pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_NONE ){ zList = sqlite3_mprintf("%z, %s._rowid_", zList, zObj); } } return zList; } +/* +** Return an expression that can be used in a WHERE clause to match the +** primary key of the current table. For example, if the table is: +** +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)); +** +** Return the string: +** +** "b = ?1 AND c = ?2" +*/ static char *otaObjIterGetWhere( sqlite3ota *p, OtaObjIter *pIter ){ char *zList = 0; if( p->rc==SQLITE_OK ){ - if( pIter->bVtab ){ - zList = otaMPrintfAndCollectError(p, "rowid = ?%d", pIter->nTblCol+1); + if( pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE ){ + zList = otaMPrintfAndCollectError(p, "_rowid_ = ?%d", pIter->nTblCol+1); }else{ const char *zSep = ""; int i; @@ -759,7 +806,7 @@ static int otaObjIterPrepareAll( int *aiCol; /* Column map */ const char **azColl; /* Collation sequences */ - assert( pIter->bVtab==0 ); + assert( pIter->eType!=OTA_PK_VTAB ); /* Create the index writers */ if( p->rc==SQLITE_OK ){ @@ -777,10 +824,10 @@ static int otaObjIterPrepareAll( zCollist = otaObjIterGetCollist(p, pIter, pIter->nCol, aiCol, azColl); if( p->rc==SQLITE_OK ){ char *zSql; - if( pIter->bRowid ){ + if( pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_NONE ){ zSql = sqlite3_mprintf( "SELECT %s, ota_control FROM ota.'ota_tmp_%q' ORDER BY %s%s", - zCollist, pIter->zTbl, + zCollist, pIter->zTbl, zCollist, zLimit ); }else{ @@ -798,11 +845,13 @@ static int otaObjIterPrepareAll( p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, zSql); } }else{ + int bOtaRowid = (pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE); const char *zTbl = pIter->zTbl; - char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol+pIter->bVtab); char *zWhere = otaObjIterGetWhere(p, pIter); char *zOldlist = otaObjIterGetOldlist(p, pIter, "old"); char *zNewlist = otaObjIterGetOldlist(p, pIter, "new"); + char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol + bOtaRowid); + zCollist = otaObjIterGetCollist(p, pIter, pIter->nTblCol, 0, 0); pIter->nCol = pIter->nTblCol; @@ -811,7 +860,7 @@ static int otaObjIterPrepareAll( p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, sqlite3_mprintf( "SELECT %s, ota_control%s FROM ota.'data_%q'%s", - zCollist, (pIter->bVtab ? ", ota_rowid" : ""), zTbl, zLimit + zCollist, (bOtaRowid ? ", ota_rowid" : ""), zTbl, zLimit ) ); } @@ -821,7 +870,7 @@ static int otaObjIterPrepareAll( p->rc = prepareFreeAndCollectError(p->db, &pIter->pInsert, pz, sqlite3_mprintf( "INSERT INTO main.%Q(%s%s) VALUES(%s)", - zTbl, zCollist, (pIter->bVtab ? ", rowid" : ""), zBindings + zTbl, zCollist, (bOtaRowid ? ", _rowid_" : ""), zBindings ) ); } @@ -835,8 +884,11 @@ static int otaObjIterPrepareAll( ); } - if( pIter->bVtab==0 ){ - const char *zOtaRowid = (pIter->bRowid ? ", ota_rowid" : ""); + if( pIter->eType!=OTA_PK_VTAB ){ + const char *zOtaRowid = ""; + if( pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_NONE ){ + zOtaRowid = ", ota_rowid"; + } /* Create the ota_tmp_xxx table and the triggers to populate it. */ otaMPrintfExec(p, @@ -858,12 +910,13 @@ static int otaObjIterPrepareAll( "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(3, %s);" "END;" - , zTbl, (pIter->bRowid ? ", 0 AS ota_rowid" : ""), zTbl, + , zTbl, (pIter->eType==OTA_PK_EXTERNAL ? ", 0 AS ota_rowid" : "") + , zTbl, zTbl, zTbl, zTbl, zCollist, zOtaRowid, zOldlist, zTbl, zTbl, zTbl, zCollist, zOtaRowid, zOldlist, zTbl, zTbl, zTbl, zCollist, zOtaRowid, zNewlist ); - if( pIter->bRowid ){ + if( pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_NONE ){ otaMPrintfExec(p, "CREATE TEMP TRIGGER ota_insert_%q AFTER INSERT ON main.%Q " "BEGIN " @@ -1074,6 +1127,15 @@ static int otaStepType(sqlite3ota *p, const char **pzMask){ return res; } +#ifdef SQLITE_DEBUG +static void assertColumnName(sqlite3_stmt *pStmt, int iCol, const char *zName){ + const char *zCol = sqlite3_column_name(pStmt, iCol); + assert( 0==sqlite3_stricmp(zName, zCol) ); +} +#else +# define assertColumnName(x,y,z) +#endif + /* ** This function does the work for an sqlite3ota_step() call. ** @@ -1122,13 +1184,17 @@ static int otaStep(sqlite3ota *p){ pVal = sqlite3_column_value(pIter->pSelect, i); sqlite3_bind_value(pWriter, i+1, pVal); } - if( pIter->bVtab ){ - /* For a virtual table, the SELECT statement is: + if( pIter->zIdx==0 + && (pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE) + ){ + /* For a virtual table, or a table with no primary key, the + ** SELECT statement is: ** ** SELECT , ota_control, ota_rowid FROM .... ** ** Hence column_value(pIter->nCol+1). */ + assertColumnName(pIter->pSelect, pIter->nCol+1, "ota_rowid"); pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); } @@ -1143,7 +1209,9 @@ static int otaStep(sqlite3ota *p){ pVal = sqlite3_column_value(pIter->pSelect, i); sqlite3_bind_value(pUpdate, i+1, pVal); } - if( pIter->bVtab ){ + if( pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE ){ + /* Bind the ota_rowid value to column _rowid_ */ + assertColumnName(pIter->pSelect, pIter->nCol+1, "ota_rowid"); pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); sqlite3_bind_value(pUpdate, pIter->nCol+1, pVal); } @@ -1195,7 +1263,7 @@ int sqlite3ota_step(sqlite3ota *p){ /* Clean up the ota_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ - if( pIter->bVtab==0 ){ + if( pIter->eType!=OTA_PK_VTAB ){ otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); } }else{ diff --git a/manifest b/manifest index 00072e021c..9e9bafcb13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sota\sso\sthat\sthe\shidden\scolumns\sof\svirtual\stables\smay\sbe\swritten. -D 2014-11-27T18:09:46.630 +C Allow\sthe\sota\sextension\sto\swrite\sto\stables\swith\sno\sPRIMARY\sKEY\sdeclaration. +D 2014-12-06T19:30:41.673 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,7 +125,7 @@ F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt 78d4a9f78f567d4bf826cf0f02df6254902562ca F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 -F ext/ota/ota1.test a8f9d89c9b2d381a663bcedaa5dd5952cdbd1231 +F ext/ota/ota1.test 64770d76d3dc00c24f9a78ac69e4448708bde985 F ext/ota/ota10.test ab815dff9cef7248c504f06b888627d236f25e9c F ext/ota/ota2.test 4568c2671d19dbde789fb9091d727a2e94880128 F ext/ota/ota3.test 71bd8cc0cf8d7e7d9bb11a1fcc238320a5a9d8c8 @@ -134,9 +134,9 @@ F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda -F ext/ota/ota9.test d9ad30ccb4e08f878e382876fe67752309538af9 +F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b -F ext/ota/sqlite3ota.c f1e930690ec2bcb72138301ebf05ea2ffd4450de +F ext/ota/sqlite3ota.c 84cab0f965144772068ec0183252ae5e5278f0be F ext/ota/sqlite3ota.h 8dc9c812e61f47f497336a45f820bee88b33a2c5 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b @@ -1237,7 +1237,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 600cefdd4d29c1de4d107fa7ddeb76a18edce4f5 -R 19cf180a791a327f1f4fb68723a586ef +P ccee999649d0fa1d48e53847542f4cbe05e3d694 +R 23b9af0814440584d410f9d611aafb5a U dan -Z 7646a22500cce7c7ef211b057e34ff27 +Z b3bc69908e00e34016dd9eb2dbbe05c2 diff --git a/manifest.uuid b/manifest.uuid index 0c798bdfcc..b63e4f6272 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ccee999649d0fa1d48e53847542f4cbe05e3d694 \ No newline at end of file +ba59a7e2ba97244492cbca9247456df0f3f19248 \ No newline at end of file From 61826cdb6c22c9ed59a2f689d06933cff76e5c70 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Dec 2014 07:22:34 +0000 Subject: [PATCH 043/116] Extra tests for the ota_rowid column. FossilOrigin-Name: 46069393b3141ab198f0fcc4f6c05229f06bf1e8 --- ext/ota/ota1.test | 47 +++++++++++++++++++++++++++++++++++++++++++++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index 74600ff6ec..0b2da1a60e 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -363,5 +363,52 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota} { } } +#------------------------------------------------------------------------- +# Test some error cases: +# +# * A virtual table with no ota_rowid column. +# * A no-PK table with no ota_rowid column. +# * A PK table with an ota_rowid column. +# +ifcapable fts3 { + foreach {tn schema error} { + 1 { + CREATE TABLE t1(a, b); + CREATE TABLE ota.data_t1(a, b, ota_control); + } {SQLITE_ERROR - table data_t1 requires ota_rowid column} + + 2 { + CREATE VIRTUAL TABLE t1 USING fts4(a, b); + CREATE TABLE ota.data_t1(a, b, ota_control); + } {SQLITE_ERROR - table data_t1 requires ota_rowid column} + + 3 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); + } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} + + 4 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); + } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} + + 5 { + CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); + } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} + + } { + reset_db + forcedelete ota.db + execsql { ATTACH 'ota.db' AS ota } + execsql $schema + + do_test 6.$tn { + list [catch { run_ota test.db ota.db } msg] $msg + } [list 1 $error] + } +} + + finish_test diff --git a/manifest b/manifest index 9e9bafcb13..4cb31a632e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\sota\sextension\sto\swrite\sto\stables\swith\sno\sPRIMARY\sKEY\sdeclaration. -D 2014-12-06T19:30:41.673 +C Extra\stests\sfor\sthe\sota_rowid\scolumn. +D 2014-12-08T07:22:34.951 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,7 +125,7 @@ F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt 78d4a9f78f567d4bf826cf0f02df6254902562ca F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 -F ext/ota/ota1.test 64770d76d3dc00c24f9a78ac69e4448708bde985 +F ext/ota/ota1.test d76b9ec77437759e9da0ff4abe9c070bb9f4eae1 F ext/ota/ota10.test ab815dff9cef7248c504f06b888627d236f25e9c F ext/ota/ota2.test 4568c2671d19dbde789fb9091d727a2e94880128 F ext/ota/ota3.test 71bd8cc0cf8d7e7d9bb11a1fcc238320a5a9d8c8 @@ -1237,7 +1237,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ccee999649d0fa1d48e53847542f4cbe05e3d694 -R 23b9af0814440584d410f9d611aafb5a +P ba59a7e2ba97244492cbca9247456df0f3f19248 +R 06c52933f7ce4a2123403894b1b52edd U dan -Z b3bc69908e00e34016dd9eb2dbbe05c2 +Z 372aa0d2cbd560795c244eab3953e926 diff --git a/manifest.uuid b/manifest.uuid index b63e4f6272..2086ff8c78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba59a7e2ba97244492cbca9247456df0f3f19248 \ No newline at end of file +46069393b3141ab198f0fcc4f6c05229f06bf1e8 \ No newline at end of file From fdf3a8970d71e1a2e51d47c2c391264cad834952 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Dec 2014 07:28:26 +0000 Subject: [PATCH 044/116] Update comments in sqlite3ota.h to remove the "must have PRIMARY KEY" restriction. FossilOrigin-Name: 088a41eb8c18886a260cf53fa0cca3bd1958dc05 --- ext/ota/sqlite3ota.h | 30 ++++++++++++++++-------------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index a652316ca2..44aff2e30d 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -61,9 +61,9 @@ ** ** * INSERT statements may not use any default values. ** -** * Non-virtual tables that do not have declared primary keys may not -** be updated. UPDATE and DELETE statements must identify their target -** rows by PRIMARY KEY values. +** * UPDATE and DELETE statements must identify their target rows by +** PRIMARY KEY values. If the table being written has no PRIMARY KEY +** declaration, affected rows must be identified by rowid. ** ** * UPDATE statements may not modify PRIMARY KEY columns. ** @@ -94,17 +94,19 @@ ** ** The order of the columns in the data_% table does not matter. ** -** If the target database table is a virtual table, the data_% table should -** also contain a column named "ota_rowid". This column is mapped to the -** virtual tables implicit primary key column - "rowid". Virtual tables -** for which the "rowid" column does not function like a primary key value -** can not be updated using OTA. For example, if the target db contains: +** If the target database table is a virtual table or a table that has no +** PRIMARY KEY declaration, the data_% table must also contain a column +** named "ota_rowid". This column is mapped to the tables implicit primary +** key column - "rowid". Virtual tables for which the "rowid" column does +** not function like a primary key value cannot be updated using OTA. For +** example, if the target db contains either of the following: ** -** CREATE VIRTUAL TABLE ft1 USING fts3(a, b); +** CREATE VIRTUAL TABLE x1 USING fts3(a, b); +** CREATE TABLE x1(a, b) ** ** then the OTA database should contain: ** -** CREATE TABLE data_ft1(a, b, ota_rowid, ota_control); +** CREATE TABLE data_x1(a, b, ota_rowid, ota_control); ** ** All non-hidden columns (i.e. all columns matched by "SELECT *") of the ** target table must be present in the input table. For virtual tables, @@ -173,12 +175,12 @@ ** ** UPDATE t1 SET c = ota_delta(c, 'usa') WHERE a = 4; ** -** If the target database table is a virtual table, the ota_control value -** should not include a character corresponding to the ota_rowid value. -** For example, this: +** If the target database table is a virtual table or a table with no PRIMARY +** KEY, the ota_control value should not include a character corresponding +** to the ota_rowid value. For example, this: ** ** INSERT INTO data_ft1(a, b, ota_rowid, ota_control) -** VALUES(NULL, 'usa', 12, '..d'); +** VALUES(NULL, 'usa', 12, '.x'); ** ** causes a result similar to: ** diff --git a/manifest b/manifest index 4cb31a632e..61039ed68d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\stests\sfor\sthe\sota_rowid\scolumn. -D 2014-12-08T07:22:34.951 +C Update\scomments\sin\ssqlite3ota.h\sto\sremove\sthe\s"must\shave\sPRIMARY\sKEY"\srestriction. +D 2014-12-08T07:28:26.753 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -137,7 +137,7 @@ F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b F ext/ota/sqlite3ota.c 84cab0f965144772068ec0183252ae5e5278f0be -F ext/ota/sqlite3ota.h 8dc9c812e61f47f497336a45f820bee88b33a2c5 +F ext/ota/sqlite3ota.h ce378c0c503f625611713133f9c79704ea4ee7a4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1237,7 +1237,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ba59a7e2ba97244492cbca9247456df0f3f19248 -R 06c52933f7ce4a2123403894b1b52edd +P 46069393b3141ab198f0fcc4f6c05229f06bf1e8 +R 2ff45547a2c264d0b419c476d87c1792 U dan -Z 372aa0d2cbd560795c244eab3953e926 +Z 50a00d12069d2404879b86db79315481 diff --git a/manifest.uuid b/manifest.uuid index 2086ff8c78..85a4152dfb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46069393b3141ab198f0fcc4f6c05229f06bf1e8 \ No newline at end of file +088a41eb8c18886a260cf53fa0cca3bd1958dc05 \ No newline at end of file From 25876016a3e8c1dcb5530c377bf76ad3ea6c0b7b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Jan 2015 12:27:35 +0000 Subject: [PATCH 045/116] Correctly detect errors returned by walCheckpointStart() due to OOM. FossilOrigin-Name: 8e878c20238ecf60e5d98d088a955e834ffd35c1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index e60c8e7d07..749e2793c3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sin\sall\schanges\sfrom\strunk. -D 2015-01-28T12:00:40.078 +C Correctly\sdetect\serrors\sreturned\sby\swalCheckpointStart()\sdue\sto\sOOM. +D 2015-01-28T12:27:35.926 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -26,7 +26,7 @@ F autoconf/missing d7c9981a81af13370d4ed152b24c0a82b7028585 x F autoconf/tea/Makefile.in d55bcc63832caf0309c2ff80358756116618cfca F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.ac 93d43c79e936fb16556e22498177d7e8571efa04 w autoconf/tea/configure.in +F autoconf/tea/configure.ac 93d43c79e936fb16556e22498177d7e8571efa04 F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in 3ef61715cf1c7bdcff56947ffadb26bc991ca39d @@ -317,7 +317,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 6d64c5448b64851b99931ede980addc3af70d5e2 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 -F src/wal.c 0d9591fdec673f8402cc604b81dfeec4a150b3d0 +F src/wal.c e4779a5ce3b0cd01ebf17f92b1fab618181ce4cf F src/wal.h 0d3ba0c3f1b4c25796cb213568a84b9f9063f465 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c d46de821bc604a4fd36fa3928c086950e91aafb1 @@ -1253,7 +1253,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 69a312ad3fe5b39bc394b9ce958cb63d734518c7 e7d2ec048c88237c124fbe598f8f7e950d43d90f -R 8ac69e11eda6b72259b7582e363c58d6 +P 17c69be80542c5f84e21d60df3edc49422b087d9 +R 0f7f7a39f5a064489a7f2265717c296f U drh -Z a99a35d018da82adaec9a86ad728c987 +Z a4be37c8322775eddaaa905b716d3639 diff --git a/manifest.uuid b/manifest.uuid index 2bbf8096d1..7c72edba88 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17c69be80542c5f84e21d60df3edc49422b087d9 \ No newline at end of file +8e878c20238ecf60e5d98d088a955e834ffd35c1 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index fcffa0f281..07c47c5b60 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1909,8 +1909,7 @@ static int walCheckpoint( assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); rc = walCheckpointStart(pWal, zBuf, nBuf, xBusy, pBusyArg, sync_flags, &sC); - if( sC.pIter==0 ) goto walcheckpoint_out; - assert( rc==SQLITE_OK ); + if( sC.pIter==0 || rc!=SQLITE_OK ) goto walcheckpoint_out; /* Step the checkpoint object until it reports something other than ** SQLITE_OK. */ From 5c08eb143f2b0961eb9f307819bde4de26a57011 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 28 Jan 2015 15:30:03 +0000 Subject: [PATCH 046/116] When a "full", "restart" or "truncate" checkpoint is attempted, return SQLITE_BUSY if a reader prevents the checkpointer from copying any frames at all into the database file. FossilOrigin-Name: f270004016ef20f1963e0f1ad32e76bcbaeb6798 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 749e2793c3..e2ba9a8a01 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correctly\sdetect\serrors\sreturned\sby\swalCheckpointStart()\sdue\sto\sOOM. -D 2015-01-28T12:27:35.926 +C When\sa\s"full",\s"restart"\sor\s"truncate"\scheckpoint\sis\sattempted,\sreturn\sSQLITE_BUSY\sif\sa\sreader\sprevents\sthe\scheckpointer\sfrom\scopying\sany\sframes\sat\sall\sinto\sthe\sdatabase\sfile. +D 2015-01-28T15:30:03.968 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -317,7 +317,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 6d64c5448b64851b99931ede980addc3af70d5e2 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 -F src/wal.c e4779a5ce3b0cd01ebf17f92b1fab618181ce4cf +F src/wal.c 4d1a785a9e90b715b5f5e0ae62bd61fe953265c4 F src/wal.h 0d3ba0c3f1b4c25796cb213568a84b9f9063f465 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c d46de821bc604a4fd36fa3928c086950e91aafb1 @@ -1253,7 +1253,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 17c69be80542c5f84e21d60df3edc49422b087d9 -R 0f7f7a39f5a064489a7f2265717c296f -U drh -Z a4be37c8322775eddaaa905b716d3639 +P 8e878c20238ecf60e5d98d088a955e834ffd35c1 +R e292091de00d2dc719cd8b0d6e22bcff +U dan +Z f2e16431401ec30e3df3be2f77db3946 diff --git a/manifest.uuid b/manifest.uuid index 7c72edba88..8596526d7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e878c20238ecf60e5d98d088a955e834ffd35c1 \ No newline at end of file +f270004016ef20f1963e0f1ad32e76bcbaeb6798 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 07c47c5b60..db56b14d20 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1824,6 +1824,8 @@ static int walCheckpointFinalize(WalCkpt *p){ walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); walIteratorFree(p->pIter); p->pIter = 0; + }else if( p->rc==SQLITE_DONE ){ + p->rc = SQLITE_OK; } return p->rc; @@ -1909,7 +1911,7 @@ static int walCheckpoint( assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); rc = walCheckpointStart(pWal, zBuf, nBuf, xBusy, pBusyArg, sync_flags, &sC); - if( sC.pIter==0 || rc!=SQLITE_OK ) goto walcheckpoint_out; + if( rc!=SQLITE_OK ) goto walcheckpoint_out; /* Step the checkpoint object until it reports something other than ** SQLITE_OK. */ From 3e6bc3977f8c879e997259ef3c8582e2f4f493b3 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Jan 2015 20:37:03 +0000 Subject: [PATCH 047/116] Add missing VdbeCoverage() macros to the sqlite3_index_writer() implementation. FossilOrigin-Name: 6f8cda26e93f09eadf0b084131a1d4002d94d959 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeblob.c | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e2ba9a8a01..fcbd84871c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sa\s"full",\s"restart"\sor\s"truncate"\scheckpoint\sis\sattempted,\sreturn\sSQLITE_BUSY\sif\sa\sreader\sprevents\sthe\scheckpointer\sfrom\scopying\sany\sframes\sat\sall\sinto\sthe\sdatabase\sfile. -D 2015-01-28T15:30:03.968 +C Add\smissing\sVdbeCoverage()\smacros\sto\sthe\ssqlite3_index_writer()\simplementation. +D 2015-01-28T20:37:03.629 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -312,7 +312,7 @@ F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 4bc511a46b9839392ae0e90844a71dc96d9dbd71 F src/vdbeaux.c 97911edb61074b871ec4aa2d6bb779071643dee5 -F src/vdbeblob.c 317c71482ed73b0966db2d1c4e20839be3e9fe79 +F src/vdbeblob.c ad7787440295e43c12248dc48cde4b13e5df4ca0 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 6d64c5448b64851b99931ede980addc3af70d5e2 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 @@ -1253,7 +1253,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8e878c20238ecf60e5d98d088a955e834ffd35c1 -R e292091de00d2dc719cd8b0d6e22bcff -U dan -Z f2e16431401ec30e3df3be2f77db3946 +P f270004016ef20f1963e0f1ad32e76bcbaeb6798 +R b0398084d8fe2aeca2bc9ad972629a75 +U drh +Z 8d857ec25793e6618479325cb81d1be3 diff --git a/manifest.uuid b/manifest.uuid index 8596526d7e..c4540655ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f270004016ef20f1963e0f1ad32e76bcbaeb6798 \ No newline at end of file +6f8cda26e93f09eadf0b084131a1d4002d94d959 \ No newline at end of file diff --git a/src/vdbeblob.c b/src/vdbeblob.c index e033468e96..93a17ed384 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -606,6 +606,7 @@ int sqlite3_index_writer( /* If this is a rowid table, check that the rowid field is an integer. */ if( HasRowid(pTab) ){ sqlite3VdbeAddOp2(v, OP_MustBeInt, pIdx->nColumn, 0); + VdbeCoverageNeverTaken(v); } if( bDelete==0 ){ @@ -614,6 +615,7 @@ int sqlite3_index_writer( /* If this is a UNIQUE index, check the constraint. */ if( pIdx->onError ){ int addr = sqlite3VdbeAddOp4Int(v, OP_NoConflict, 0, 0, 1, pIdx->nKeyCol); + VdbeCoverage(v); sqlite3UniqueConstraint(pParse, SQLITE_ABORT, pIdx); sqlite3VdbeJumpHere(v, addr); } From 3b660d7da3faf36d4bbe6537cd13b7911590741b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 31 Jan 2015 20:42:04 +0000 Subject: [PATCH 048/116] Have ota use imposter tables to write to indexes instead of the sqlite3_index_writer() interface. The error handling in this version is broken in a few small ways. FossilOrigin-Name: cdaeab467f6aa3217be161377a9b78a4eec37093 --- ext/ota/otafault.test | 1 - ext/ota/sqlite3ota.c | 214 +++++++++++++++++++++++++++++++++++------- manifest | 18 ++-- manifest.uuid | 2 +- src/main.c | 2 + 5 files changed, 190 insertions(+), 47 deletions(-) diff --git a/ext/ota/otafault.test b/ext/ota/otafault.test index 915d6448d2..5c641dc675 100644 --- a/ext/ota/otafault.test +++ b/ext/ota/otafault.test @@ -64,7 +64,6 @@ do_faultsim_test 2 -faults oom-trans* -prep { {1 SQLITE_NOMEM} \ {1 SQLITE_IOERR_NOMEM} \ {1 {SQLITE_NOMEM - unable to open a temporary database file for storing temporary tables}} - if {$testrc==0} { sqlite3 db test.db faultsim_integrity_check diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index a3d73ae67b..1727e51f1f 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -96,24 +96,23 @@ struct OtaState { ** ** * the table itself, ** * each index of the table (zero or more points to visit), and -** * a special "cleanup table" point. +** * a special "cleanup table" state. */ struct OtaObjIter { sqlite3_stmt *pTblIter; /* Iterate through tables */ sqlite3_stmt *pIdxIter; /* Index iterator */ int nTblCol; /* Size of azTblCol[] array */ char **azTblCol; /* Array of quoted column names */ + char **azTblType; /* Array of column types */ unsigned char *abTblPk; /* Array of flags - true for PK columns */ int eType; -#if 0 - unsigned char bRowid; /* True for implicit IPK tables */ - unsigned char bVtab; /* True for a virtual table */ -#endif /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ const char *zTbl; /* Name of target db table */ const char *zIdx; /* Name of target db index (or null) */ + int tnum; /* Root page of index (not table) */ + int bUnique; /* Current index is unique */ int iVisit; /* Number of points visited, incl. current */ /* Statements created by otaObjIterPrepareAll() */ @@ -232,9 +231,11 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){ int i; for(i=0; inTblCol; i++){ sqlite3_free(pIter->azTblCol[i]); + sqlite3_free(pIter->azTblType[i]); } sqlite3_free(pIter->azTblCol); pIter->azTblCol = 0; + pIter->azTblType = 0; pIter->abTblPk = 0; pIter->nTblCol = 0; sqlite3_free(pIter->zMask); @@ -307,6 +308,8 @@ static int otaObjIterNext(sqlite3ota *p, OtaObjIter *pIter){ pIter->zIdx = 0; }else{ pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + pIter->tnum = sqlite3_column_int(pIter->pIdxIter, 1); + pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); rc = SQLITE_OK; } } @@ -339,8 +342,9 @@ static int otaObjIterFirst(sqlite3ota *p, OtaObjIter *pIter){ if( rc==SQLITE_OK ){ rc = prepareAndCollectError(p->db, &pIter->pIdxIter, &p->zErrmsg, - "SELECT name FROM main.sqlite_master " - "WHERE type='index' AND tbl_name = ?" + "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " + " FROM main.sqlite_master " + " WHERE type='index' AND tbl_name = ?" ); } @@ -428,7 +432,7 @@ static int otaMPrintfExec(sqlite3ota *p, const char *zFmt, ...){ ** error code in the OTA handle passed as the first argument. */ static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ - int nByte = sizeof(char*) * nCol + sizeof(unsigned char*) * nCol; + int nByte = sizeof(char*) * nCol * 2 + sizeof(unsigned char*) * nCol; char **azNew; assert( p->rc==SQLITE_OK ); @@ -436,12 +440,32 @@ static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ if( azNew ){ memset(azNew, 0, nByte); pIter->azTblCol = azNew; - pIter->abTblPk = (unsigned char*)&pIter->azTblCol[nCol]; + pIter->azTblType = &azNew[nCol]; + pIter->abTblPk = (unsigned char*)&pIter->azTblType[nCol]; }else{ p->rc = SQLITE_NOMEM; } } +static char *otaStrndup(const char *zStr, int nStr, int *pRc){ + char *zRet = 0; + assert( *pRc==SQLITE_OK ); + + if( zStr ){ + int nCopy = nStr; + if( nCopy<0 ) nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; + } + } + + return zRet; +} + + /* ** Return true if zTab is the name of a virtual table within the target ** database. @@ -531,6 +555,8 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ ); }else{ int iPk = sqlite3_column_int(pStmt, 5); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + pIter->azTblType[i] = otaStrndup(zType, -1, &p->rc); pIter->abTblPk[i] = (iPk!=0); if( iPk ){ pIter->eType = (iPk<0) ? OTA_PK_EXTERNAL : OTA_PK_REAL; @@ -644,6 +670,115 @@ static char *otaObjIterGetCollist( return zList; } +/* +** This function is used to create a SELECT list (the list of SQL +** expressions that follows a SELECT keyword) for a SELECT statement +** used to read from an ota_xxx table while updating the index object +** currently indicated by the iterator object passed as the second +** argument. A "PRAGMA index_xinfo = " statement is used to +** obtain the required information. +** +** If the index is of the following form: +** +** CREATE INDEX i1 ON t1(c, b COLLATE nocase); +** +** and "t1" is a table with an explicit INTEGER PRIMARY KEY column +** "ipk", the returned string is: +** +** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'" +** +** As well as the returned string, three other malloc'd strings are +** returned via output parameters. As follows: +** +** pzImposterCols: ... +** pzImposterPk: ... +** pzWhere: ... +*/ +static char *otaObjIterGetIndexCols( + sqlite3ota *p, /* OTA object */ + OtaObjIter *pIter, /* Object iterator for column names */ + char **pzImposterCols, /* OUT: Columns for imposter table */ + char **pzImposterPk, /* OUT: Imposter PK clause */ + char **pzWhere, /* OUT: WHERE clause */ + int *pnBind /* OUT: Total number of columns */ +){ + int rc = p->rc; /* Error code */ + int rc2; /* sqlite3_finalize() return code */ + char *zRet = 0; /* String to return */ + char *zImpCols = 0; /* String to return via *pzImposterCols */ + char *zImpPK = 0; /* String to return via *pzImposterPK */ + char *zWhere = 0; /* String to return via *pzWhere */ + int nBind = 0; /* Value to return via *pnBind */ + const char *zComma = ""; /* Set to ", " later on */ + const char *zAnd = ""; /* Set to " AND " later on */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */ + + if( rc==SQLITE_OK ){ + assert( p->zErrmsg==0 ); + rc = prepareFreeAndCollectError(p->db, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + int iCid = sqlite3_column_int(pXInfo, 1); + const char *zCol; + const char *zType; + + if( iCid<0 ){ + /* An integer primary key. If the table has an explicit IPK, use + ** its name. Otherwise, use "ota_rowid". */ + if( pIter->eType==OTA_PK_REAL ){ + int i; + for(i=0; inTblCol && pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else{ + zCol = "ota_rowid"; + } + zType = "INTEGER"; + }else{ + zCol = pIter->azTblCol[iCid]; + zType = pIter->azTblType[iCid]; + } + + zRet = sqlite3_mprintf("%z%s%s COLLATE %Q", zRet, zComma, zCol, zCollate); + if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ + zImpPK = sqlite3_mprintf("%z%sc%d", zImpPK, zComma, nBind); + } + zImpCols = sqlite3_mprintf( + "%z%sc%d %s COLLATE %Q", zImpCols, zComma, nBind, zType, zCollate + ); + zWhere = sqlite3_mprintf("%z%sc%d IS ?", zWhere, zAnd, nBind); + if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; + zComma = ", "; + zAnd = " AND "; + nBind++; + } + + rc2 = sqlite3_finalize(pXInfo); + if( rc==SQLITE_OK ) rc = rc2; + + if( rc!=SQLITE_OK ){ + sqlite3_free(zRet); + sqlite3_free(zImpCols); + sqlite3_free(zImpPK); + sqlite3_free(zWhere); + zRet = 0; + zImpCols = 0; + zImpPK = 0; + zWhere = 0; + p->rc = rc; + } + + *pzImposterCols = zImpCols; + *pzImposterPk = zImpPK; + *pzWhere = zWhere; + *pnBind = nBind; + return zRet; +} + /* ** Assuming the current table columns are "a", "b" and "c", and the zObj ** paramter is passed "old", return a string of the form: @@ -792,6 +927,7 @@ static int otaObjIterPrepareAll( ){ assert( pIter->bCleanup==0 ); if( pIter->pSelect==0 && otaObjIterGetCols(p, pIter)==SQLITE_OK ){ + const int tnum = pIter->tnum; char *zCollist = 0; /* List of indexed columns */ char **pz = &p->zErrmsg; const char *zIdx = pIter->zIdx; @@ -803,25 +939,43 @@ static int otaObjIterPrepareAll( } if( zIdx ){ - int *aiCol; /* Column map */ - const char **azColl; /* Collation sequences */ + char *zImposterCols = 0; + char *zImposterPK = 0; + char *zWhere = 0; + char *zBind = 0; + int nBind = 0; assert( pIter->eType!=OTA_PK_VTAB ); + zCollist = otaObjIterGetIndexCols( + p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind + ); + zBind = otaObjIterGetBindlist(p, nBind); - /* Create the index writers */ + /* Create the imposter table used to write to this index. */ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum); + otaMPrintfExec(p, + "CREATE TABLE ota_imposter( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", + zImposterCols, zImposterPK + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); + + /* Create the statement to insert index entries */ + pIter->nCol = nBind; if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_index_writer( - p->db, 0, zIdx, &pIter->pInsert, &azColl, &aiCol, &pIter->nCol + p->rc = prepareFreeAndCollectError(p->db, &pIter->pInsert, &p->zErrmsg, + sqlite3_mprintf("INSERT INTO ota_imposter VALUES(%s)", zBind) ); } + + /* And to delete index entries */ if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_index_writer( - p->db, 1, zIdx, &pIter->pDelete, &azColl, &aiCol, &pIter->nCol + p->rc = prepareFreeAndCollectError(p->db, &pIter->pDelete, &p->zErrmsg, + sqlite3_mprintf("DELETE FROM ota_imposter WHERE %s", zWhere) ); } /* Create the SELECT statement to read keys in sorted order */ - zCollist = otaObjIterGetCollist(p, pIter, pIter->nCol, aiCol, azColl); if( p->rc==SQLITE_OK ){ char *zSql; if( pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_NONE ){ @@ -844,6 +998,11 @@ static int otaObjIterPrepareAll( } p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, zSql); } + + sqlite3_free(zImposterCols); + sqlite3_free(zImposterPK); + sqlite3_free(zWhere); + sqlite3_free(zBind); }else{ int bOtaRowid = (pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE); const char *zTbl = pIter->zTbl; @@ -1205,7 +1364,7 @@ static int otaStep(sqlite3ota *p){ sqlite3_stmt *pUpdate = 0; otaGetUpdateStmt(p, pIter, zMask, &pUpdate); if( pUpdate ){ - for(i=0; inCol; i++){ + for(i=0; p->rc==SQLITE_OK && inCol; i++){ pVal = sqlite3_column_value(pIter->pSelect, i); sqlite3_bind_value(pUpdate, i+1, pVal); } @@ -1375,24 +1534,6 @@ static void otaSaveTransactionState(sqlite3ota *p){ } } -static char *otaStrndup(char *zStr, int nStr, int *pRc){ - char *zRet = 0; - assert( *pRc==SQLITE_OK ); - - if( zStr ){ - int nCopy = nStr; - if( nCopy<0 ) nCopy = strlen(zStr) + 1; - zRet = (char*)sqlite3_malloc(nCopy); - if( zRet ){ - memcpy(zRet, zStr, nCopy); - }else{ - *pRc = SQLITE_NOMEM; - } - } - - return zRet; -} - static void otaFreeState(OtaState *p){ if( p ){ sqlite3_free(p->zTbl); @@ -1740,6 +1881,7 @@ static int test_sqlite3ota_cmd( db, "ota_delta", -1, SQLITE_UTF8, (void*)interp, test_ota_delta, 0, 0 ); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + sqlite3_exec(db, "PRAGMA vdbe_trace = 1", 0, 0, 0); ret = (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); break; } diff --git a/manifest b/manifest index 109bbd6537..3c7bd57c80 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sin\ssupport\sfor\sthe\sindex_xinfo\spragma. -D 2015-01-31T02:34:23.039 +C Have\sota\suse\simposter\stables\sto\swrite\sto\sindexes\sinstead\sof\sthe\ssqlite3_index_writer()\sinterface.\sThe\serror\shandling\sin\sthis\sversion\sis\sbroken\sin\sa\sfew\ssmall\sways. +D 2015-01-31T20:42:04.027 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -135,8 +135,8 @@ F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b -F ext/ota/otafault.test be02466863015a583cc0ceb6aca871a5e6f7a71b -F ext/ota/sqlite3ota.c 84cab0f965144772068ec0183252ae5e5278f0be +F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd +F ext/ota/sqlite3ota.c 975ccfe032ee81ee39368ed5e9cb33cbb6edc603 F ext/ota/sqlite3ota.h ce378c0c503f625611713133f9c79704ea4ee7a4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -210,7 +210,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 -F src/main.c d6c5fd51a719fcdcf1cc9ef08349dbd4454cf2f3 +F src/main.c c4cb192ebf0bcc975648ae05ac40bc1f40018c52 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987 @@ -1254,7 +1254,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3ed6eb2fab5d95709ef392170339e6dd5ba13971 30f51d7b3b292191e8351223242e708bb7f3dfa6 -R 50758dde054ce12d446bde117108ee7b -U drh -Z ec35f224c29e962e36a2b69233e35843 +P f9b6dc77021ee421bffd5697d5d337d3bbd07eb9 +R 907483269529ba471a525b7f8d05bc2e +U dan +Z 87f206a3330dd276f74650f9e68af8ed diff --git a/manifest.uuid b/manifest.uuid index 6038dc4c5d..ecf8a3c6bb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9b6dc77021ee421bffd5697d5d337d3bbd07eb9 \ No newline at end of file +cdaeab467f6aa3217be161377a9b78a4eec37093 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 6ccc6d68f6..7bca1e5777 100644 --- a/src/main.c +++ b/src/main.c @@ -3644,12 +3644,14 @@ int sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_IMPOSTER: { sqlite3 *db = va_arg(ap, sqlite3*); + sqlite3_mutex_enter(db->mutex); db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); db->init.busy = db->init.imposterTable = va_arg(ap,int); db->init.newTnum = va_arg(ap,int); if( db->init.busy==0 && db->init.newTnum>0 ){ sqlite3ResetAllSchemasOfConnection(db); } + sqlite3_mutex_leave(db->mutex); break; } } From 75fda9b39558eb826aa1d3623e7f4bd4e4923ba7 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 3 Feb 2015 15:56:08 +0000 Subject: [PATCH 049/116] Remove "PRAGMA ota_mode". FossilOrigin-Name: 1c111447a07687c30ed4ad5a6c27a169c85b7ea6 --- ext/ota/ota3.test | 52 ++++- ext/ota/ota4.test | 105 ---------- ext/ota/sqlite3ota.c | 478 ++++++++++++++++++++++--------------------- manifest | 38 ++-- manifest.uuid | 2 +- src/btree.c | 3 +- src/delete.c | 3 - src/insert.c | 13 -- src/main.c | 39 ++++ src/pragma.c | 13 +- src/sqlite.h.in | 63 +----- src/sqliteInt.h | 6 - src/test1.c | 2 +- src/trigger.c | 8 +- src/vdbeblob.c | 179 ---------------- tool/mkpragmatab.tcl | 5 - 16 files changed, 358 insertions(+), 651 deletions(-) diff --git a/ext/ota/ota3.test b/ext/ota/ota3.test index 8921c12b35..74dba1f166 100644 --- a/ext/ota/ota3.test +++ b/ext/ota/ota3.test @@ -63,19 +63,29 @@ do_execsql_test 2.0 { CREATE INDEX i1 ON x1(b, c); } {} -do_test 2.1 { - sqlite3 db2 ota.db - db2 eval { +foreach {tn otadb} { + 1 { CREATE TABLE data_x1(a, b, c, ota_control); INSERT INTO data_x1 VALUES(NULL, 'a', 'b', 0); } - db2 close - list [catch { run_ota test.db ota.db } msg] $msg -} {1 {SQLITE_MISMATCH - datatype mismatch}} -do_execsql_test 2.2 { - PRAGMA integrity_check; -} {ok} + 2 { + CREATE TABLE data_x1(c, b, a, ota_control); + INSERT INTO data_x1 VALUES('b', 'a', NULL, 0); + } +} { + do_test 2.$tn.1 { + forcedelete ota.db + sqlite3 db2 ota.db + db2 eval $otadb + db2 close + list [catch { run_ota test.db ota.db } msg] $msg + } {1 {SQLITE_MISMATCH - datatype mismatch}} + + do_execsql_test 2.1.2 { + PRAGMA integrity_check; + } {ok} +} #-------------------------------------------------------------------- # Test that missing columns are detected. @@ -102,4 +112,28 @@ do_execsql_test 2.2 { PRAGMA integrity_check; } {ok} +# Also extra columns. +# +do_execsql_test 2.3 { + CREATE TABLE x2(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i2 ON x2(b, c); +} {} + +do_test 2.4 { + forcedelete ota.db + sqlite3 db2 ota.db + db2 eval { + CREATE TABLE data_x2(a, b, c, d, ota_control); + INSERT INTO data_x2 VALUES(1, 'a', 2, 3, 0); + } + db2 close + breakpoint + list [catch { run_ota test.db ota.db } msg] $msg +} {1 SQLITE_ERROR} + +do_execsql_test 2.5 { + PRAGMA integrity_check; +} {ok} + finish_test + diff --git a/ext/ota/ota4.test b/ext/ota/ota4.test index b14225dc52..a12cbd8b7b 100644 --- a/ext/ota/ota4.test +++ b/ext/ota/ota4.test @@ -124,110 +124,5 @@ do_catchsql_test 1.5.4 { SELECT * FROM t1; } {1 {database is locked}} -#------------------------------------------------------------------------- -# These tests - ota4-2.* - aim to verify some properties of the ota_mode -# pragma. -# -# 1. Check that UNIQUE constraints are not tested in ota_mode. -# 2. Except for (real) PRIMARY KEY constraints. -# 3. Check that all non-temporary triggers are ignored. -# -reset_db -do_execsql_test 2.1.1 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE UNIQUE INDEX i1 ON t1(b); - INSERT INTO t1 VALUES(1, 2, 3); - INSERT INTO t1 VALUES(2, 4, 6); -} - -do_execsql_test 2.1.2 { - PRAGMA ota_mode = 1; - INSERT INTO t1 VALUES(3, 2, 6); - UPDATE t1 SET b=2 WHERE a=2; - SELECT * FROM t1; -} { - 1 2 3 - 2 2 6 - 3 2 6 -} - -reset_db -do_execsql_test 2.2.1 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE TABLE t2(x, y, z, PRIMARY KEY(y, z)) WITHOUT ROWID; - - INSERT INTO t1 VALUES(1, 2, 3); - INSERT INTO t2 VALUES(4, 5, 6); - PRAGMA ota_mode = 1; -} -do_catchsql_test 2.2.2 { - INSERT INTO t1 VALUES(1, 'two', 'three'); -} {1 {UNIQUE constraint failed: t1.a}} -do_catchsql_test 2.2.3 { - INSERT INTO t2 VALUES('four', 5, 6); -} {1 {UNIQUE constraint failed: t2.y, t2.z}} - -reset_db -do_execsql_test 2.3.1 { - CREATE TABLE t1(a, b, c); - CREATE TABLE log(x); - INSERT INTO t1 VALUES(1, 2, 3); - - CREATE TRIGGER tr1 BEFORE INSERT ON t1 BEGIN - INSERT INTO log VALUES('permanent'); - END; - CREATE TRIGGER tr2 AFTER INSERT ON t1 BEGIN - INSERT INTO log VALUES('permanent'); - END; - CREATE TRIGGER tr3 BEFORE DELETE ON t1 BEGIN - INSERT INTO log VALUES('permanent'); - END; - CREATE TRIGGER tr4 AFTER DELETE ON t1 BEGIN - INSERT INTO log VALUES('permanent'); - END; - CREATE TRIGGER tr5 BEFORE UPDATE ON t1 BEGIN - INSERT INTO log VALUES('permanent'); - END; - CREATE TRIGGER tr6 AFTER UPDATE ON t1 BEGIN - INSERT INTO log VALUES('permanent'); - END; - - CREATE TEMP TRIGGER ttr1 BEFORE INSERT ON t1 BEGIN - INSERT INTO log VALUES('temp'); - END; - CREATE TEMP TRIGGER ttr2 AFTER INSERT ON t1 BEGIN - INSERT INTO log VALUES('temp'); - END; - CREATE TEMP TRIGGER ttr3 BEFORE DELETE ON t1 BEGIN - INSERT INTO log VALUES('temp'); - END; - CREATE TEMP TRIGGER ttr4 AFTER DELETE ON t1 BEGIN - INSERT INTO log VALUES('temp'); - END; - CREATE TEMP TRIGGER ttr5 BEFORE UPDATE ON t1 BEGIN - INSERT INTO log VALUES('temp'); - END; - CREATE TEMP TRIGGER ttr6 AFTER UPDATE ON t1 BEGIN - INSERT INTO log VALUES('temp'); - END; -} -do_execsql_test 2.3.2 { - INSERT INTO t1 VALUES(4, 5, 6); - DELETE FROM t1 WHERE a = 4; - UPDATE t1 SET c = 6; - SELECT x FROM log; -} { - temp permanent temp permanent temp permanent - temp permanent temp permanent temp permanent -} -do_execsql_test 2.3.3 { - DELETE FROM log; - PRAGMA ota_mode = 1; - INSERT INTO t1 VALUES(4, 5, 6); - DELETE FROM t1 WHERE a = 4; - UPDATE t1 SET c = 6; - SELECT x FROM log; -} {temp temp temp temp temp temp} - finish_test diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 1727e51f1f..ae45db032b 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -102,8 +102,9 @@ struct OtaObjIter { sqlite3_stmt *pTblIter; /* Iterate through tables */ sqlite3_stmt *pIdxIter; /* Index iterator */ int nTblCol; /* Size of azTblCol[] array */ - char **azTblCol; /* Array of quoted column names */ + char **azTblCol; /* Array of unquoted column names */ char **azTblType; /* Array of column types */ + int *aiTblOrder; /* Order of columns in target table */ unsigned char *abTblPk; /* Array of flags - true for PK columns */ int eType; @@ -128,11 +129,18 @@ struct OtaObjIter { /* ** Values for OtaObjIter.eType +** +** 1: Table has an implicit rowid. +** 2: Table has an explicit IPK column. +** 3: Table has an external PK index. +** 4: Table is WITHOUT ROWID. +** 5: Table is a virtual table. */ -#define OTA_PK_REAL 1 /* Table has a real primary key */ -#define OTA_PK_EXTERNAL 2 /* Table has an external primary key index */ -#define OTA_PK_NONE 3 /* Table has no PK (use rowid) */ -#define OTA_PK_VTAB 4 /* Table is a virtual table (use rowid) */ +#define OTA_PK_NONE 1 +#define OTA_PK_IPK 2 +#define OTA_PK_EXTERNAL 3 +#define OTA_PK_WITHOUT_ROWID 4 +#define OTA_PK_VTAB 5 /* ** OTA handle. @@ -225,7 +233,7 @@ static int prepareFreeAndCollectError( /* ** Free the OtaObjIter.azTblCol[] and OtaObjIter.abTblPk[] arrays allocated -** by an earlier call to otaObjIterGetCols(). +** by an earlier call to otaObjIterCacheTableInfo(). */ static void otaObjIterFreeCols(OtaObjIter *pIter){ int i; @@ -236,6 +244,7 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){ sqlite3_free(pIter->azTblCol); pIter->azTblCol = 0; pIter->azTblType = 0; + pIter->aiTblOrder = 0; pIter->abTblPk = 0; pIter->nTblCol = 0; sqlite3_free(pIter->zMask); @@ -285,32 +294,44 @@ static int otaObjIterNext(sqlite3ota *p, OtaObjIter *pIter){ /* Free any SQLite statements used while processing the previous object */ otaObjIterClearStatements(pIter); + if( pIter->zIdx==0 ){ + rc = sqlite3_exec(p->db, + "DROP TRIGGER IF EXISTS temp.ota_insert_tr;" + "DROP TRIGGER IF EXISTS temp.ota_update1_tr;" + "DROP TRIGGER IF EXISTS temp.ota_update2_tr;" + "DROP TRIGGER IF EXISTS temp.ota_delete_tr;" + , 0, 0, &p->zErrmsg + ); + } - if( pIter->bCleanup ){ - otaObjIterFreeCols(pIter); - pIter->bCleanup = 0; - rc = sqlite3_step(pIter->pTblIter); - if( rc!=SQLITE_ROW ){ - rc = sqlite3_reset(pIter->pTblIter); - pIter->zTbl = 0; + if( rc==SQLITE_OK ){ + if( pIter->bCleanup ){ + otaObjIterFreeCols(pIter); + pIter->bCleanup = 0; + rc = sqlite3_step(pIter->pTblIter); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_reset(pIter->pTblIter); + pIter->zTbl = 0; + }else{ + pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); + pIter->tnum = sqlite3_column_int(pIter->pTblIter, 1); + rc = SQLITE_OK; + } }else{ - pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); - rc = SQLITE_OK; - } - }else{ - if( pIter->zIdx==0 ){ - sqlite3_bind_text(pIter->pIdxIter, 1, pIter->zTbl, -1, SQLITE_STATIC); - } - rc = sqlite3_step(pIter->pIdxIter); - if( rc!=SQLITE_ROW ){ - rc = sqlite3_reset(pIter->pIdxIter); - pIter->bCleanup = 1; - pIter->zIdx = 0; - }else{ - pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); - pIter->tnum = sqlite3_column_int(pIter->pIdxIter, 1); - pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); - rc = SQLITE_OK; + if( pIter->zIdx==0 ){ + sqlite3_bind_text(pIter->pIdxIter, 1, pIter->zTbl, -1, SQLITE_STATIC); + } + rc = sqlite3_step(pIter->pIdxIter); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_reset(pIter->pIdxIter); + pIter->bCleanup = 1; + pIter->zIdx = 0; + }else{ + pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + pIter->tnum = sqlite3_column_int(pIter->pIdxIter, 1); + pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); + rc = SQLITE_OK; + } } } } @@ -336,8 +357,10 @@ static int otaObjIterFirst(sqlite3ota *p, OtaObjIter *pIter){ memset(pIter, 0, sizeof(OtaObjIter)); rc = prepareAndCollectError(p->db, &pIter->pTblIter, &p->zErrmsg, - "SELECT substr(name, 6) FROM ota.sqlite_master " - "WHERE type='table' AND name LIKE 'data_%'" + "SELECT substr(a.name, 6), b.rootpage FROM ota.sqlite_master AS a " + "LEFT JOIN main.sqlite_master AS b ON " + "(substr(a.name, 6)==b.name) " + "WHERE a.type='table' AND a.name LIKE 'data_%'" ); if( rc==SQLITE_OK ){ @@ -353,53 +376,6 @@ static int otaObjIterFirst(sqlite3ota *p, OtaObjIter *pIter){ return otaObjIterNext(p, pIter); } -/* -** Allocate a buffer and populate it with the double-quoted version of the -** string in the argument buffer, suitable for use as an SQL identifier. -** For example: -** -** [quick `brown` fox] -> [`quick ``brown`` fox`] -** -** Assuming the allocation is successful, a pointer to the new buffer is -** returned. It is the responsibility of the caller to free it using -** sqlite3_free() at some point in the future. Or, if the allocation fails, -** a NULL pointer is returned. -*/ -static char *otaQuoteName(const char *zName){ - int nName = strlen(zName); - char *zRet = sqlite3_malloc(nName * 2 + 2 + 1); - if( zRet ){ - int i; - char *p = zRet; - *p++ = '`'; - for(i=0; irc==SQLITE_OK ); @@ -441,7 +417,8 @@ static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ memset(azNew, 0, nByte); pIter->azTblCol = azNew; pIter->azTblType = &azNew[nCol]; - pIter->abTblPk = (unsigned char*)&pIter->azTblType[nCol]; + pIter->aiTblOrder = (int*)&pIter->azTblType[nCol]; + pIter->abTblPk = (unsigned char*)&pIter->aiTblOrder[nCol]; }else{ p->rc = SQLITE_NOMEM; } @@ -466,50 +443,33 @@ static char *otaStrndup(const char *zStr, int nStr, int *pRc){ } -/* -** Return true if zTab is the name of a virtual table within the target -** database. -*/ -static int otaIsVtab(sqlite3ota *p, const char *zTab){ - int res = 0; - sqlite3_stmt *pSelect = 0; - - if( p->rc==SQLITE_OK ){ - p->rc = prepareAndCollectError(p->db, &pSelect, &p->zErrmsg, - "SELECT count(*) FROM sqlite_master WHERE name = ? AND type='table' " - "AND sql LIKE 'CREATE VIRTUAL TABLE%'" - ); - } - - if( p->rc==SQLITE_OK ){ - sqlite3_bind_text(pSelect, 1, zTab, -1, SQLITE_STATIC); - if( sqlite3_step(pSelect)==SQLITE_ROW ){ - res = sqlite3_column_int(pSelect, 0); - } - p->rc = sqlite3_finalize(pSelect); - } - - return res; -} - /* ** If they are not already populated, populate the pIter->azTblCol[], ** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to -** the table that the iterator currently points to. +** the table (not index) that the iterator currently points to. ** ** Return SQLITE_OK if successful, or an SQLite error code otherwise. If ** an error does occur, an error code and error message are also left in ** the OTA handle. */ -static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ +static int otaObjIterCacheTableInfo(sqlite3ota *p, OtaObjIter *pIter){ if( pIter->azTblCol==0 ){ sqlite3_stmt *pStmt = 0; int nCol = 0; int i; /* for() loop iterator variable */ int rc2; /* sqlite3_finalize() return value */ int bOtaRowid = 0; /* If input table has column "ota_rowid" */ + int iOrder = 0; + /* Figure out the type of table this step will deal with. */ assert( pIter->eType==0 ); + sqlite3_test_control( + SQLITE_TESTCTRL_TBLTYPE, p->db, "main", pIter->zTbl, &pIter->eType + ); + assert( pIter->eType==OTA_PK_NONE || pIter->eType==OTA_PK_IPK + || pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_WITHOUT_ROWID + || pIter->eType==OTA_PK_VTAB + ); /* Populate the azTblCol[] and nTblCol variables based on the columns ** of the input table. Ignore any input table columns that begin with @@ -524,9 +484,8 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ for(i=0; p->rc==SQLITE_OK && irc); pIter->azTblCol[pIter->nTblCol++] = zCopy; - if( zCopy==0 ) p->rc = SQLITE_NOMEM; } else if( 0==sqlite3_stricmp("ota_rowid", zName) ){ bOtaRowid = 1; @@ -535,9 +494,19 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ sqlite3_finalize(pStmt); pStmt = 0; + if( p->rc==SQLITE_OK + && bOtaRowid!=(pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE) + ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf( + "table data_%q %s ota_rowid column", pIter->zTbl, + (bOtaRowid ? "may not have" : "requires") + ); + } + /* Check that all non-HIDDEN columns in the destination table are also - ** present in the input table. Populate the abTblPk[] array at the - ** same time. */ + ** present in the input table. Populate the abTblPk[], azTblType[] and + ** aiTblOrder[] arrays at the same time. */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pStmt, &p->zErrmsg, sqlite3_mprintf("PRAGMA main.table_info(%Q)", pIter->zTbl) @@ -546,7 +515,7 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zName = (const char*)sqlite3_column_text(pStmt, 1); for(i=0; inTblCol; i++){ - if( otaMatchName(zName, pIter->azTblCol[i]) ) break; + if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; } if( i==pIter->nTblCol ){ p->rc = SQLITE_ERROR; @@ -556,37 +525,34 @@ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ }else{ int iPk = sqlite3_column_int(pStmt, 5); const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + pIter->aiTblOrder[i] = iOrder++; pIter->azTblType[i] = otaStrndup(zType, -1, &p->rc); pIter->abTblPk[i] = (iPk!=0); - if( iPk ){ - pIter->eType = (iPk<0) ? OTA_PK_EXTERNAL : OTA_PK_REAL; + } + } + while( iOrdernTblCol ){ + for(i=0; inTblCol; i++){ + if( pIter->aiTblOrder[i]==0 ){ + pIter->aiTblOrder[i] = iOrder++; + continue; } } } + + /* Check that there were no extra columns in the data_xxx table that + ** are not present in the target table. If there are, an error. */ +#if 0 + assert( iOrder<=pIter->nTblCol ); + if( p->rc==SQLITE_OK && iOrder!=pIter->nTblCol ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("data_%q has %d columns, expected %d", + pIter->zTbl, iOrder, pIter->nTblCol + ); + } +#endif + rc2 = sqlite3_finalize(pStmt); if( p->rc==SQLITE_OK ) p->rc = rc2; - - if( p->rc==SQLITE_OK ){ - if( pIter->eType==0 ){ - /* This must either be a virtual table, or a regular table with no - ** PRIMARY KEY declaration whatsoever. */ - if( bOtaRowid==0 ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf( - "table data_%q requires ota_rowid column", pIter->zTbl - ); - }else if( otaIsVtab(p, pIter->zTbl) ){ - pIter->eType = OTA_PK_VTAB; - }else{ - pIter->eType = OTA_PK_NONE; - } - }else if( bOtaRowid ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf( - "table data_%q may not have ota_rowid column", pIter->zTbl - ); - } - } } return p->rc; @@ -612,60 +578,18 @@ static char *otaMPrintfAndCollectError(sqlite3ota *p, const char *zFmt, ...){ ** This function constructs and returns a pointer to a nul-terminated ** string containing some SQL clause or list based on one or more of the ** column names currently stored in the pIter->azTblCol[] array. -** -** If an OOM error is encountered, NULL is returned and an error code -** left in the OTA handle passed as the first argument. Otherwise, a pointer -** to the allocated string buffer is returned. It is the responsibility -** of the caller to eventually free this buffer using sqlite3_free(). -** -** The number of column names to include in the returned string is passed -** as the third argument. -** -** If arguments aiCol and azCollate are both NULL, then the returned string -** contains the first nCol column names as a comma-separated list. For -** example: -** -** "a", "b", "c" -** -** If argument aiCol is not NULL, it must point to an array containing nCol -** entries - the index of each column name to include in the comma-separated -** list. For example, if aiCol[] contains {2, 0, 1), then the returned -** string is changed to: -** -** "c", "a", "b" -** -** If azCollate is not NULL, it must also point to an array containing nCol -** entries - collation sequence names to associated with each element of -** the comma separated list. For example, ef azCollate[] contains -** {"BINARY", "NOCASE", "REVERSE"}, then the retuned string is: -** -** "c" COLLATE "BINARY", "a" COLLATE "NOCASE", "b" COLLATE "REVERSE" -** */ static char *otaObjIterGetCollist( sqlite3ota *p, /* OTA object */ - OtaObjIter *pIter, /* Object iterator for column names */ - int nCol, /* Number of column names */ - int *aiCol, /* Array of nCol column indexes */ - const char **azCollate /* Array of nCol collation sequence names */ + OtaObjIter *pIter /* Object iterator for column names */ ){ char *zList = 0; - if( p->rc==SQLITE_OK ){ - const char *zSep = ""; - int i; - for(i=0; i=0 ? pIter->azTblCol[iCol] : "ota_rowid"); - zList = sqlite3_mprintf("%z%s%s", zList, zSep, zCol); - if( zList && azCollate ){ - zList = sqlite3_mprintf("%z COLLATE %Q", zList, azCollate[i]); - } - zSep = ", "; - if( zList==0 ){ - p->rc = SQLITE_NOMEM; - break; - } - } + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + const char *z = pIter->azTblCol[i]; + zList = otaMPrintfAndCollectError(p, "%z%s\"%w\"", zList, zSep, z); + zSep = ", "; } return zList; } @@ -709,7 +633,7 @@ static char *otaObjIterGetIndexCols( char *zImpPK = 0; /* String to return via *pzImposterPK */ char *zWhere = 0; /* String to return via *pzWhere */ int nBind = 0; /* Value to return via *pnBind */ - const char *zComma = ""; /* Set to ", " later on */ + const char *zCom = ""; /* Set to ", " later on */ const char *zAnd = ""; /* Set to " AND " later on */ sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */ @@ -729,7 +653,7 @@ static char *otaObjIterGetIndexCols( if( iCid<0 ){ /* An integer primary key. If the table has an explicit IPK, use ** its name. Otherwise, use "ota_rowid". */ - if( pIter->eType==OTA_PK_REAL ){ + if( pIter->eType==OTA_PK_IPK ){ int i; for(i=0; inTblCol && pIter->abTblPk[i]==0; i++); assert( inTblCol ); @@ -743,16 +667,16 @@ static char *otaObjIterGetIndexCols( zType = pIter->azTblType[iCid]; } - zRet = sqlite3_mprintf("%z%s%s COLLATE %Q", zRet, zComma, zCol, zCollate); + zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate); if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ - zImpPK = sqlite3_mprintf("%z%sc%d", zImpPK, zComma, nBind); + zImpPK = sqlite3_mprintf("%z%sc%d", zImpPK, zCom, nBind); } zImpCols = sqlite3_mprintf( - "%z%sc%d %s COLLATE %Q", zImpCols, zComma, nBind, zType, zCollate + "%z%sc%d %s COLLATE %Q", zImpCols, zCom, nBind, zType, zCollate ); zWhere = sqlite3_mprintf("%z%sc%d IS ?", zWhere, zAnd, nBind); if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; - zComma = ", "; + zCom = ", "; zAnd = " AND "; nBind++; } @@ -800,7 +724,8 @@ static char *otaObjIterGetOldlist( const char *zS = ""; int i; for(i=0; inTblCol; i++){ - zList = sqlite3_mprintf("%z%s%s.%s", zList, zS, zObj, pIter->azTblCol[i]); + const char *zCol = pIter->azTblCol[i]; + zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol); zS = ", "; if( zList==0 ){ p->rc = SQLITE_NOMEM; @@ -831,20 +756,18 @@ static char *otaObjIterGetWhere( OtaObjIter *pIter ){ char *zList = 0; - if( p->rc==SQLITE_OK ){ - if( pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE ){ - zList = otaMPrintfAndCollectError(p, "_rowid_ = ?%d", pIter->nTblCol+1); - }else{ - const char *zSep = ""; - int i; - for(i=0; inTblCol; i++){ - if( pIter->abTblPk[i] ){ - const char *zCol = pIter->azTblCol[i]; - zList = otaMPrintfAndCollectError( - p, "%z%s%s=?%d", zList, zSep, zCol, i+1 - ); - zSep = " AND "; - } + if( pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE ){ + zList = otaMPrintfAndCollectError(p, "_rowid_ = ?%d", pIter->nTblCol+1); + }else{ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = otaMPrintfAndCollectError( + p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1 + ); + zSep = " AND "; } } } @@ -880,13 +803,14 @@ static char *otaObjIterGetSetlist( for(i=0; inTblCol; i++){ char c = zMask[i]; if( c=='x' ){ - zList = otaMPrintfAndCollectError(p, "%z%s%s=?%d", + zList = otaMPrintfAndCollectError(p, "%z%s\"%w\"=?%d", zList, zSep, pIter->azTblCol[i], i+1 ); zSep = ", "; } if( c=='d' ){ - zList = otaMPrintfAndCollectError(p, "%z%s%s=ota_delta(%s, ?%d)", + zList = otaMPrintfAndCollectError(p, + "%z%s\"%w\"=ota_delta(\"%w\", ?%d)", zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 ); zSep = ", "; @@ -915,6 +839,81 @@ static char *otaObjIterGetBindlist(sqlite3ota *p, int nBind){ return zRet; } +/* +** If an error has already occurred when this function is called, it +** immediately returns zero (without doing any work). Or, if an error +** occurs during the execution of this function, it sets the error code +** in the sqlite3ota object indicated by the first argument and returns +** zero. +** +** The iterator passed as the second argument is guaranteed to point to +** a table (not an index) when this function is called. This function +** attempts to create any imposter tables required to write to the main +** table b-tree of the table before returning. Non-zero is returned if +** imposter tables are created, or zero otherwise. +** +** The required imposter tables depend on the type of table that the +** iterator currently points to. +** +** OTA_PK_NONE, OTA_PK_IPK, OTA_PK_WITHOUT_ROWID: +** A single imposter table is required. With the same schema as +** the actual target table (less any UNIQUE constraints). More +** precisely, the "same schema" means the same columns, types, collation +** sequences and primary key declaration. +** +** OTA_PK_VTAB: +** No imposters required. +** +** OTA_PK_EXTERNAL: +** Two imposters are required (TODO!!) +*/ +static void otaCreateImposterTable(sqlite3ota *p, OtaObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType!=OTA_PK_VTAB ){ + int tnum = pIter->tnum; + const char *zComma = ""; + char *zSql = 0; + int iCol; + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1); + + for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ + int iDataCol = pIter->aiTblOrder[iCol]; + const char *zCol = pIter->azTblCol[iDataCol]; + const char *zColl = 0; + p->rc = sqlite3_table_column_metadata( + p->db, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0 + ); + zSql = otaMPrintfAndCollectError(p, "%z%s\"%w\" %s COLLATE %s", + zSql, zComma, zCol, pIter->azTblType[iDataCol], zColl + ); + zComma = ", "; + } + + if( pIter->eType==OTA_PK_IPK || pIter->eType==OTA_PK_WITHOUT_ROWID ){ + zSql = otaMPrintfAndCollectError(p, "%z, PRIMARY KEY(", zSql); + zComma = ""; + for(iCol=0; iColnTblCol; iCol++){ + if( pIter->abTblPk[iCol] ){ + zSql = otaMPrintfAndCollectError(p, "%z%s\"%w\"", + zSql, zComma, pIter->azTblCol[iCol] + ); + zComma = ", "; + } + } + zSql = otaMPrintfAndCollectError(p, "%z)", zSql); + } + + zSql = otaMPrintfAndCollectError(p, "CREATE TABLE ota_imposter(%z)%s", + zSql, (pIter->eType==OTA_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") + ); + if( p->rc==SQLITE_OK ){ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum); + p->rc = sqlite3_exec(p->db, zSql, 0, 0, &p->zErrmsg); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); + } + sqlite3_free(zSql); + } +} + /* ** Ensure that the SQLite statement handles required to update the ** target database object currently indicated by the iterator passed @@ -926,7 +925,7 @@ static int otaObjIterPrepareAll( int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ ){ assert( pIter->bCleanup==0 ); - if( pIter->pSelect==0 && otaObjIterGetCols(p, pIter)==SQLITE_OK ){ + if( pIter->pSelect==0 && otaObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){ const int tnum = pIter->tnum; char *zCollist = 0; /* List of indexed columns */ char **pz = &p->zErrmsg; @@ -939,9 +938,9 @@ static int otaObjIterPrepareAll( } if( zIdx ){ - char *zImposterCols = 0; - char *zImposterPK = 0; - char *zWhere = 0; + char *zImposterCols = 0; /* Columns for imposter table */ + char *zImposterPK = 0; /* Primary key declaration for imposter */ + char *zWhere = 0; /* WHERE clause on PK columns */ char *zBind = 0; int nBind = 0; @@ -1005,13 +1004,15 @@ static int otaObjIterPrepareAll( sqlite3_free(zBind); }else{ int bOtaRowid = (pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE); - const char *zTbl = pIter->zTbl; + const char *zTbl = pIter->zTbl; /* Table this step applies to */ + const char *zWrite; /* Imposter table name */ + + char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol + bOtaRowid); char *zWhere = otaObjIterGetWhere(p, pIter); char *zOldlist = otaObjIterGetOldlist(p, pIter, "old"); char *zNewlist = otaObjIterGetOldlist(p, pIter, "new"); - char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol + bOtaRowid); - zCollist = otaObjIterGetCollist(p, pIter, pIter->nTblCol, 0, 0); + zCollist = otaObjIterGetCollist(p, pIter); pIter->nCol = pIter->nTblCol; /* Create the SELECT statement to read keys from data_xxx */ @@ -1024,12 +1025,16 @@ static int otaObjIterPrepareAll( ); } + /* Create the imposter table or tables (if required). */ + otaCreateImposterTable(p, pIter); + zWrite = (pIter->eType==OTA_PK_VTAB ? zTbl : "ota_imposter"); + /* Create the INSERT statement to write to the target PK b-tree */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pInsert, pz, sqlite3_mprintf( "INSERT INTO main.%Q(%s%s) VALUES(%s)", - zTbl, zCollist, (bOtaRowid ? ", _rowid_" : ""), zBindings + zWrite, zCollist, (bOtaRowid ? ", _rowid_" : ""), zBindings ) ); } @@ -1038,7 +1043,7 @@ static int otaObjIterPrepareAll( if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pDelete, pz, sqlite3_mprintf( - "DELETE FROM main.%Q WHERE %s", zTbl, zWhere + "DELETE FROM main.%Q WHERE %s", zWrite, zWhere ) ); } @@ -1055,34 +1060,34 @@ static int otaObjIterPrepareAll( "CREATE TABLE IF NOT EXISTS ota.'ota_tmp_%q' AS " "SELECT *%s FROM ota.'data_%q' WHERE 0;" - "CREATE TEMP TRIGGER ota_delete_%q BEFORE DELETE ON main.%Q " + "CREATE TEMP TRIGGER ota_delete_tr BEFORE DELETE ON ota_imposter " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(2, %s);" "END;" - "CREATE TEMP TRIGGER ota_update1_%q BEFORE UPDATE ON main.%Q " + "CREATE TEMP TRIGGER ota_update1_tr BEFORE UPDATE ON ota_imposter " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(2, %s);" "END;" - "CREATE TEMP TRIGGER ota_update2_%q AFTER UPDATE ON main.%Q " + "CREATE TEMP TRIGGER ota_update2_tr AFTER UPDATE ON ota_imposter " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(3, %s);" "END;" , zTbl, (pIter->eType==OTA_PK_EXTERNAL ? ", 0 AS ota_rowid" : "") , zTbl, - zTbl, zTbl, zTbl, zCollist, zOtaRowid, zOldlist, - zTbl, zTbl, zTbl, zCollist, zOtaRowid, zOldlist, - zTbl, zTbl, zTbl, zCollist, zOtaRowid, zNewlist + zTbl, zCollist, zOtaRowid, zOldlist, + zTbl, zCollist, zOtaRowid, zOldlist, + zTbl, zCollist, zOtaRowid, zNewlist ); if( pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_NONE ){ otaMPrintfExec(p, - "CREATE TEMP TRIGGER ota_insert_%q AFTER INSERT ON main.%Q " + "CREATE TEMP TRIGGER ota_insert_tr AFTER INSERT ON ota_imposter " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s, ota_rowid)" " VALUES(0, %s);" "END;" - , zTbl, zTbl, zTbl, zCollist, zNewlist + , zTbl, zCollist, zNewlist ); } }else if( p->rc==SQLITE_OK ){ @@ -1133,8 +1138,9 @@ static int otaGetUpdateStmt( sqlite3_finalize(pIter->pUpdate); pIter->pUpdate = 0; if( p->rc==SQLITE_OK ){ - zUpdate = sqlite3_mprintf("UPDATE %Q SET %s WHERE %s", - pIter->zTbl, zSet, zWhere + zUpdate = sqlite3_mprintf("UPDATE \"%w\" SET %s WHERE %s", + (pIter->eType==OTA_PK_VTAB ? pIter->zTbl : "ota_imposter"), + zSet, zWhere ); p->rc = prepareFreeAndCollectError( p->db, &pIter->pUpdate, &p->zErrmsg, zUpdate @@ -1337,9 +1343,22 @@ static int otaStep(sqlite3ota *p){ } for(i=0; inCol; i++){ + /* If this is an INSERT into a table b-tree and the table has an + ** explicit INTEGER PRIMARY KEY, check that this is not an attempt + ** to write a NULL into the IPK column. That is not permitted. */ + if( eType==OTA_INSERT + && pIter->zIdx==0 && pIter->eType==OTA_PK_IPK && pIter->abTblPk[i] + && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL + ){ + p->rc = SQLITE_MISMATCH; + p->zErrmsg = sqlite3_mprintf("datatype mismatch"); + goto step_out; + } + if( eType==SQLITE_DELETE && pIter->zIdx==0 && pIter->abTblPk[i]==0 ){ continue; } + pVal = sqlite3_column_value(pIter->pSelect, i); sqlite3_bind_value(pWriter, i+1, pVal); } @@ -1383,6 +1402,7 @@ static int otaStep(sqlite3ota *p){ } } + step_out: return p->rc; } diff --git a/manifest b/manifest index 3c7bd57c80..0a49c2b2a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sota\suse\simposter\stables\sto\swrite\sto\sindexes\sinstead\sof\sthe\ssqlite3_index_writer()\sinterface.\sThe\serror\shandling\sin\sthis\sversion\sis\sbroken\sin\sa\sfew\ssmall\sways. -D 2015-01-31T20:42:04.027 +C Remove\s"PRAGMA\sota_mode". +D 2015-02-03T15:56:08.271 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -128,15 +128,15 @@ F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 F ext/ota/ota1.test d76b9ec77437759e9da0ff4abe9c070bb9f4eae1 F ext/ota/ota10.test ab815dff9cef7248c504f06b888627d236f25e9c F ext/ota/ota2.test 2829bc08ffbb71b605392a68fedfd554763356a7 -F ext/ota/ota3.test 71bd8cc0cf8d7e7d9bb11a1fcc238320a5a9d8c8 -F ext/ota/ota4.test 60f897f329a6782ef2f24862640acf3c52e48077 +F ext/ota/ota3.test a77efbce7723332eb688d2b28bf18204fc9614d7 +F ext/ota/ota4.test 82434aa39c9acca6cd6317f6b0ab07b0ec6c2e7d F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c 975ccfe032ee81ee39368ed5e9cb33cbb6edc603 +F ext/ota/sqlite3ota.c 52c91eec41b8fbb5ed12a8f0a2159bc5ec16498f F ext/ota/sqlite3ota.h ce378c0c503f625611713133f9c79704ea4ee7a4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -188,7 +188,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c ddca0ae681c49813a4bfd88d5f922fac71e0ffaa +F src/btree.c 2a1245df0356a229bcd0fd87a8536b5067f16e82 F src/btree.h 94277c1d30c0b75705974bcc8b0c05e79c03d474 F src/btreeInt.h a3d0ae1d511365e1a2b76ad10960dbe55c286f34 F src/build.c eefaa4f1d86bc3c08023a61fdd1e695b47796975 @@ -196,7 +196,7 @@ F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 198a0066ba60ab06fc00fba1998d870a4d575463 F src/ctime.c 98f89724adc891a1a4c655bee04e33e716e05887 F src/date.c e4d50b3283696836ec1036b695ead9a19e37a5ac -F src/delete.c e68b70ac41dcf6e92a813d860fa984fcd9aec042 +F src/delete.c bd1a91ddd247ce13004075251e0b7fe2bf9925ef F src/expr.c abe930897ccafae3819fd2855cbc1b00c262fd12 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e0444b61bed271a76840cbe6182df93a9baa3f12 @@ -205,12 +205,12 @@ F src/global.c 12561d70a1b25f67b21154622bb1723426724f75 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c fef86ab8218cf0d926db93280b9eb5b583981353 +F src/insert.c 5b9243a33726008cc4132897d2be371db12a13be F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 -F src/main.c c4cb192ebf0bcc975648ae05ac40bc1f40018c52 +F src/main.c 55d548a2c2f32d27366968c394d091475f7ea00a F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987 @@ -238,7 +238,7 @@ F src/parse.y c5d0d964f9ac023e8154cad512e54b0b6058e086 F src/pcache.c d210cf90d04365a74f85d21374dded65af67b0cb F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c 1e77432b40b7d3288327d9cdf399dcdfd2b6d3bf -F src/pragma.c e49831e54c72894cce08702fe2e127e5c53d90f0 +F src/pragma.c 26fc55619109828c9b7add4cfa8a961b6f4c456d F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 F src/printf.c 05edc41450d0eb2c05ef7db113bf32742ae65325 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 @@ -246,15 +246,15 @@ F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 1f2087523007c42900ffcbdeaef06a23ad9329fc F src/shell.c 22b4406b0b59efd14b3b351a5809dda517df6d30 -F src/sqlite.h.in 78e493f94202d8083dd270e257786a6311d1fb3b +F src/sqlite.h.in 8913937ba11415bf369818431700adf3a921fb18 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 66180aa8f81155a7f391bbf759ee5a3b61d2f89f +F src/sqliteInt.h 57f8f45028598cc2877fc08ac03b402242242c68 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c e7a09215315a978057fb42c640f890160dbcc45e F src/tclsqlite.c b321464aba1fff1ed9317ebc82a1a94887f97af8 -F src/test1.c 313567541c980e45220d6faed393b6ad454f8ecd +F src/test1.c ce8ea168800d129acb2c0afdf2831ddf8667e082 F src/test2.c 577961fe48961b2f2e5c8b56ee50c3f459d3359d F src/test3.c 64d2afdd68feac1bb5e2ffb8226c8c639f798622 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e @@ -302,7 +302,7 @@ F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 F src/tokenize.c e00458c9938072b0ea711c850b8dcf4ddcb5fe18 -F src/trigger.c 6dcdf46a21acf4d4e011c809b2c971e63f797a1a +F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 98a7627ca48ad3265b6940915a1d08355eb3fc7e @@ -312,7 +312,7 @@ F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 4bc511a46b9839392ae0e90844a71dc96d9dbd71 F src/vdbeaux.c 97911edb61074b871ec4aa2d6bb779071643dee5 -F src/vdbeblob.c ad7787440295e43c12248dc48cde4b13e5df4ca0 +F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 6d64c5448b64851b99931ede980addc3af70d5e2 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 @@ -1218,7 +1218,7 @@ F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 5a401a3f5461fedb2e3230eb9908a18d485fefea +F tool/mkpragmatab.tcl aea392b69f8e72715760629cd50411d37bc85de4 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 9ef48e1748dce7b844f67e2450ff9dfeb0fb4ab5 F tool/mksqlite3c.tcl cfde806851c413db7689b9cb74a4eeb92539c601 @@ -1254,7 +1254,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f9b6dc77021ee421bffd5697d5d337d3bbd07eb9 -R 907483269529ba471a525b7f8d05bc2e +P cdaeab467f6aa3217be161377a9b78a4eec37093 +R 2703aba199c695992db82956950982ca U dan -Z 87f206a3330dd276f74650f9e68af8ed +Z fae9dc4e5e5c1a2d2a162b422a05ccf9 diff --git a/manifest.uuid b/manifest.uuid index ecf8a3c6bb..0e07d68cdf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cdaeab467f6aa3217be161377a9b78a4eec37093 \ No newline at end of file +1c111447a07687c30ed4ad5a6c27a169c85b7ea6 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4f752066ac..eb5151351c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -151,8 +151,7 @@ static int hasSharedCacheTableLock( ** and has the read-uncommitted flag set, then no lock is required. ** Return true immediately. */ - if( (pBtree->db->flags & SQLITE_OtaMode) - || (pBtree->sharable==0) + if( (pBtree->sharable==0) || (eLockType==READ_LOCK && (pBtree->db->flags & SQLITE_ReadUncommitted)) ){ return 1; diff --git a/src/delete.c b/src/delete.c index 17f7a87459..011fb80dee 100644 --- a/src/delete.c +++ b/src/delete.c @@ -730,9 +730,6 @@ void sqlite3GenerateRowIndexDelete( Vdbe *v; /* The prepared statement under construction */ Index *pPk; /* PRIMARY KEY index, or NULL for rowid tables */ - /* Skip this if we are in OTA mode */ - if( pParse->db->flags & SQLITE_OtaMode ) return; - v = pParse->pVdbe; pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ diff --git a/src/insert.c b/src/insert.c index 0cc2078179..a5c3f3e92d 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1365,10 +1365,6 @@ void sqlite3GenerateConstraintChecks( int iThisCur; /* Cursor for this UNIQUE index */ int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ - /* If the "ota_mode" flag is set, ignore all indexes except the PK - ** index of WITHOUT ROWID tables. */ - if( (db->flags & SQLITE_OtaMode) && pIdx!=pPk) continue; - if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */ if( bAffinityDone==0 ){ sqlite3TableAffinity(v, pTab, regNewData+1); @@ -1560,15 +1556,6 @@ void sqlite3CompleteInsertion( assert( pTab->pSelect==0 ); /* This table is not a VIEW */ for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ if( aRegIdx[i]==0 ) continue; - - /* If the "ota_mode" flag is set, ignore all indexes except the PK - ** index of WITHOUT ROWID tables. */ - if( (pParse->db->flags & SQLITE_OtaMode) - && (HasRowid(pTab) || pIdx->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) - ){ - continue; - } - bAffinityDone = 1; if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); diff --git a/src/main.c b/src/main.c index 7bca1e5777..710d1dfb84 100644 --- a/src/main.c +++ b/src/main.c @@ -3654,6 +3654,45 @@ int sqlite3_test_control(int op, ...){ sqlite3_mutex_leave(db->mutex); break; } + + /* sqlite3_test_control(SQLITE_TESTCTRL_TBLTYPE, db, dbName, zTbl, peType) + ** + ** peType is of type (int*), a pointer to an output parameter of type + ** (int). This call sets the output parameter as follows, depending + ** on the type of the table specified by parameters dbName and zTbl. + ** + ** 0: No such table. + ** 1: Table has an implicit rowid. + ** 2: Table has an explicit IPK column. + ** 3: Table has an external PK index. + ** 4: Table is WITHOUT ROWID. + ** 5: Table is a virtual table. + */ + case SQLITE_TESTCTRL_TBLTYPE: { + sqlite3 *db = va_arg(ap, sqlite3*); + const char *zDb = va_arg(ap, const char*); + const char *zTab = va_arg(ap, const char*); + int *peType = va_arg(ap, int*); + Table *pTab; + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + pTab = sqlite3FindTable(db, zTab, zDb); + if( pTab==0 ){ + *peType = 0; + }else if( IsVirtual(pTab) ){ + *peType = 5; + }else if( HasRowid(pTab)==0 ){ + *peType = 4; + }else if( pTab->iPKey>=0 ){ + *peType = 2; + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + *peType = (pPk ? 3 : 1); + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + break; + } } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ diff --git a/src/pragma.c b/src/pragma.c index a2913abdbd..61c721f4d2 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -320,14 +320,6 @@ static const struct sPragmaNames { /* ePragTyp: */ PragTyp_MMAP_SIZE, /* ePragFlag: */ 0, /* iArg: */ 0 }, -#endif -#if defined(SQLITE_ENABLE_OTA) - { /* zName: */ "ota_mode", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_OtaMode }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "page_count", /* ePragTyp: */ PragTyp_PAGE_COUNT, /* ePragFlag: */ PragFlag_NeedSchema, @@ -496,7 +488,7 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 59 on by default, 74 total. */ +/* Number of pragmas: 59 on by default, 73 total. */ /* End of the automatically generated pragma table. ***************************************************************************/ @@ -1545,9 +1537,6 @@ void sqlite3Pragma( k = 1; }else{ for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} - if( (db->flags & SQLITE_OtaMode) && HasRowid(pTab) ){ - k = -1 * k; - } } sqlite3VdbeAddOp2(v, OP_Integer, k, 6); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 6e02a98688..1cba5b1772 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6266,7 +6266,8 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ISINIT 23 #define SQLITE_TESTCTRL_SORTER_MMAP 24 #define SQLITE_TESTCTRL_IMPOSTER 25 -#define SQLITE_TESTCTRL_LAST 25 +#define SQLITE_TESTCTRL_TBLTYPE 26 +#define SQLITE_TESTCTRL_LAST 26 /* ** CAPI3REF: SQLite Runtime Status @@ -7585,66 +7586,6 @@ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( */ SQLITE_EXPERIMENTAL void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); -/* -** Allocate a statement handle that may be used to write directly to an -** index b-tree. This allows the user to create a corrupt database. Once -** the statement handle is allocated, it may be used with the same APIs -** as any statement handle created with sqlite3_prepare(). -** -** The statement writes to the index specified by parameter zIndex, which -** must be in the "main" database. If argument bDelete is false, then each -** time the statement is sqlite3_step()ed, an entry is inserted into the -** b-tree index. If it is true, then an entry may be deleted (or may not, if -** the specified key is not found) each time the statement is -** sqlite3_step()ed. -** -** If statement compilation is successful, *ppStmt is set to point to the -** new statement handle and SQLITE_OK is returned. Otherwise, if an error -** occurs, *ppStmt is set to NULL and an error code returned. An error -** message may be left in the database handle in this case. -** -** If statement compilation succeeds, output variable *pnCol is set to the -** total number of columns in the index, including the primary key columns -** at the end. Variable *paiCol is set to point to an array *pnCol entries -** in size. Each entry is the table column index, numbered from zero from left -** to right, of the corresponding index column. For example, if: -** -** CREATE TABLE t1(a, b, c, d); -** CREATE INDEX i1 ON t1(b, c); -** -** then *pnCol is 3 and *paiCol points to an array containing {1, 2, -1}. -** If table t1 had an explicit INTEGER PRIMARY KEY, then the "-1" in the -** *paiCol array would be replaced by its column index. Or if: -** -** CREATE TABLE t2(a, b, c, d, PRIMARY KEY(d, c)) WITHOUT ROWID; -** CREATE INDEX i2 ON t2(a); -** -** then (*pnCol) is 3 and *paiCol points to an array containing {0, 3, 2}. -** -** The lifetime of the array is the same as that of the statement handle - -** it is automatically freed when the statement handle is passed to -** sqlite3_finalize(). -** -** The statement has (*pnCol) SQL variables that values may be bound to. -** They correspond to the values used to create the index key that is -** inserted or deleted when the statement is stepped. -** -** If the index is a UNIQUE index, the usual checking and error codes apply -** to insert operations. -** -** This API is only available if SQLITE_ENABLE_OTA is defined at compile -** time. It is intended for use by the OTA extension only. As such, it is -** subject to change or removal at any point. -*/ -int sqlite3_index_writer( - sqlite3 *db, - int bDelete, /* Zero for insert, non-zero for delete */ - const char *zIndex, /* Index to write to */ - sqlite3_stmt**, /* OUT: New statement handle */ - const char ***pazColl, /* OUT: Collation sequence for each column */ - int **paiCol, int *pnCol /* OUT: See above */ -); - /* ** Incremental checkpoint API. ** diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9de37160bf..5fdbc6d65d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1207,12 +1207,6 @@ struct sqlite3 { #define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ #define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ -#ifdef SQLITE_ENABLE_OTA -# define SQLITE_OtaMode 0x08000000 /* True in "ota mode" */ -#else -# define SQLITE_OtaMode 0x00000000 -#endif - /* ** Bits of the sqlite3.dbOptFlags field that are used by the ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to diff --git a/src/test1.c b/src/test1.c index ae6e7bc0ea..d3c3d314bf 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6922,11 +6922,11 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_user_change", test_user_change, 0 }, { "sqlite3_user_delete", test_user_delete, 0 }, #endif - #ifdef SQLITE_ENABLE_STMT_SCANSTATUS { "sqlite3_stmt_scanstatus", test_stmt_scanstatus, 0 }, { "sqlite3_stmt_scanstatus_reset", test_stmt_scanstatus_reset, 0 }, #endif + }; static int bitmask_size = sizeof(Bitmask)*8; diff --git a/src/trigger.c b/src/trigger.c index 44b9b89fcc..d2e7b5a1e6 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -43,14 +43,10 @@ void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){ ** To state it another way: This routine returns a list of all triggers ** that fire off of pTab. The list will include any TEMP triggers on ** pTab as well as the triggers lised in pTab->pTrigger. -** -** If the SQLITE_OtaMode flag is set, do not include any non-temporary -** triggers in the returned list. */ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ Schema * const pTmpSchema = pParse->db->aDb[1].pSchema; Trigger *pList = 0; /* List of triggers to return */ - const int bOta = !!(pParse->db->flags & SQLITE_OtaMode); if( pParse->disableTriggers ){ return 0; @@ -64,13 +60,13 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ if( pTrig->pTabSchema==pTab->pSchema && 0==sqlite3StrICmp(pTrig->table, pTab->zName) ){ - pTrig->pNext = ((pList || bOta) ? pList : pTab->pTrigger); + pTrig->pNext = (pList ? pList : pTab->pTrigger); pList = pTrig; } } } - return ((pList || bOta) ? pList : pTab->pTrigger); + return (pList ? pList : pTab->pTrigger); } /* diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 93a17ed384..cf1eb59054 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -467,183 +467,4 @@ int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ return rc; } -#ifdef SQLITE_ENABLE_OTA -/* -** Allocate and populate the output arrays returned by the -** sqlite3_index_writer() function. -*/ -static int indexWriterOutputVars( - sqlite3 *db, - Index *pIdx, - const char ***pazColl, /* OUT: Array of collation sequences */ - int **paiCol, /* OUT: Array of column indexes */ - int *pnCol /* OUT: Total columns in index keys */ -){ - Table *pTbl = pIdx->pTable; /* Table index is attached to */ - Index *pPk = 0; - int nByte = 0; /* Total bytes of space to allocate */ - int i; /* Iterator variable */ - - int *aiCol; - const char **azColl; - char *pCsr; - - if( !HasRowid(pTbl) ){ - pPk = sqlite3PrimaryKeyIndex(pTbl); - } - - for(i=0; inColumn; i++){ - const char *zColl = 0; - if( inKeyCol ){ - zColl = pIdx->azColl[i]; - }else if( pPk ){ - zColl = pPk->azColl[i-pIdx->nKeyCol]; - } - if( zColl==0 ) zColl = "BINARY"; - nByte += sqlite3Strlen30(zColl) + 1; - } - nByte += (pIdx->nColumn) * (sizeof(const char*) + sizeof(int)); - - /* Populate the output variables */ - *pazColl = azColl = (const char**)sqlite3DbMallocZero(db, nByte); - if( azColl==0 ) return SQLITE_NOMEM; - *paiCol = aiCol = (int*)&azColl[pIdx->nColumn]; - *pnCol = pIdx->nColumn; - pCsr = (char*)&aiCol[pIdx->nColumn]; - - for(i=0; inColumn; i++){ - const char *zColl = 0; - int nColl; - int iCol = pTbl->iPKey; - if( inKeyCol ){ - zColl = pIdx->azColl[i]; - iCol = pIdx->aiColumn[i]; - }else if( pPk ){ - zColl = pPk->azColl[i-pIdx->nKeyCol]; - iCol = pPk->aiColumn[i-pIdx->nKeyCol]; - } - if( zColl==0 ) zColl = "BINARY"; - - aiCol[i] = iCol; - azColl[i] = pCsr; - nColl = 1 + sqlite3Strlen30(zColl); - memcpy(pCsr, zColl, nColl); - pCsr += nColl; - } - - return SQLITE_OK; -} - -/* -** Prepare and return an SQL statement handle that can be used to write -** directly to an index b-tree. -*/ -int sqlite3_index_writer( - sqlite3 *db, - int bDelete, - const char *zIndex, - sqlite3_stmt **ppStmt, - const char ***pazColl, /* OUT: Array of collation sequences */ - int **paiCol, /* OUT: Array of column indexes */ - int *pnCol /* OUT: Total columns in index keys */ -){ - int rc = SQLITE_OK; - Parse *pParse = 0; - Index *pIdx = 0; /* The index to write to */ - Table *pTab; - int i; /* Used to iterate through index columns */ - Vdbe *v = 0; - int regRec; /* Register to assemble record in */ - const char *zAffinity = 0; /* Affinity string for the current index */ - - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - - /* Allocate the parse context */ - pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); - if( !pParse ) goto index_writer_out; - memset(pParse, 0, sizeof(Parse)); - pParse->db = db; - - /* Allocate the Vdbe */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto index_writer_out; - - /* Find the index to write to */ - pIdx = sqlite3FindIndex(db, zIndex, "main"); - if( pIdx==0 ){ - sqlite3ErrorMsg(pParse, "no such index: %s", zIndex); - goto index_writer_out; - } - pTab = pIdx->pTable; - zAffinity = sqlite3IndexAffinityStr(v, pIdx); - - rc = indexWriterOutputVars(db, pIdx, pazColl, paiCol, pnCol); - if( rc!=SQLITE_OK ) goto index_writer_out; - - /* Add an OP_Noop to the VDBE program. Then store a pointer to the - ** output array *paiCol as its P4 value. This is so that the array - ** is automatically deleted when the user finalizes the statement. The - ** OP_Noop serves no other purpose. */ - sqlite3VdbeAddOp0(v, OP_Noop); - sqlite3VdbeChangeP4(v, -1, (const char*)(*pazColl), P4_INTARRAY); - - sqlite3BeginWriteOperation(pParse, 0, 0); - - /* Open a write cursor on the index */ - pParse->nTab = 1; - sqlite3VdbeAddOp3(v, OP_OpenWrite, 0, pIdx->tnum, 0); - sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - - /* Create the record to insert into the index. Store it in register regRec. */ - pParse->nVar = pIdx->nColumn; - pParse->nMem = pIdx->nColumn; - for(i=1; i<=pIdx->nColumn; i++){ - sqlite3VdbeAddOp2(v, OP_Variable, i, i); - } - regRec = ++pParse->nMem; - - /* If this is a rowid table, check that the rowid field is an integer. */ - if( HasRowid(pTab) ){ - sqlite3VdbeAddOp2(v, OP_MustBeInt, pIdx->nColumn, 0); - VdbeCoverageNeverTaken(v); - } - - if( bDelete==0 ){ - sqlite3VdbeAddOp4(v, OP_MakeRecord, 1, pIdx->nColumn, regRec, zAffinity, 0); - - /* If this is a UNIQUE index, check the constraint. */ - if( pIdx->onError ){ - int addr = sqlite3VdbeAddOp4Int(v, OP_NoConflict, 0, 0, 1, pIdx->nKeyCol); - VdbeCoverage(v); - sqlite3UniqueConstraint(pParse, SQLITE_ABORT, pIdx); - sqlite3VdbeJumpHere(v, addr); - } - - /* Code the IdxInsert to write to the b-tree index. */ - sqlite3VdbeAddOp2(v, OP_IdxInsert, 0, regRec); - }else{ - /* Code the IdxDelete to remove the entry from the b-tree index. */ - sqlite3VdbeAddOp4(v, OP_Affinity, 1, pIdx->nColumn, 0, zAffinity, 0); - sqlite3VdbeAddOp3(v, OP_IdxDelete, 0, 1, pIdx->nColumn); - } - sqlite3FinishCoding(pParse); - -index_writer_out: - if( rc==SQLITE_OK && db->mallocFailed==0 ){ - *ppStmt = (sqlite3_stmt*)v; - }else{ - *ppStmt = 0; - if( v ) sqlite3VdbeFinalize(v); - } - - sqlite3ParserReset(pParse); - sqlite3StackFree(db, pParse); - sqlite3BtreeLeaveAll(db); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} -#endif /* SQLITE_ENABLE_OTA */ - #endif /* #ifndef SQLITE_OMIT_INCRBLOB */ diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 149a1bfff7..0d93704d70 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -318,11 +318,6 @@ set pragma_def { NAME: pager_ota_mode IF: defined(SQLITE_ENABLE_OTA) - - NAME: ota_mode - TYPE: FLAG - ARG: SQLITE_OtaMode - IF: defined(SQLITE_ENABLE_OTA) } fconfigure stdout -translation lf set name {} From e2863b607463c1270a3dcd3a77340af37492a8b2 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 3 Feb 2015 18:43:42 +0000 Subject: [PATCH 050/116] Fix some problems surrounding WITHOUT ROWID tables with DESC primary key indexes . FossilOrigin-Name: a21fefb79a161e6bb057ae4f6ba554f659706ac1 --- ext/ota/ota1.test | 62 ++++++++++++++++++++++++++++++++++--- ext/ota/sqlite3ota.c | 73 +++++++++++++++++++++++++++++++++++--------- manifest | 16 +++++----- manifest.uuid | 2 +- src/pragma.c | 4 +++ 5 files changed, 129 insertions(+), 28 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index 0b2da1a60e..afb5fd5865 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -148,6 +148,39 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)); CREATE INDEX i1 ON t1(b); } + + 10 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b DESC); + } + + 11 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b DESC, a ASC, c DESC); + } + + 12 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; + } + + 13 { + CREATE TABLE t1(a INT, b, c, PRIMARY KEY(a DESC)) WITHOUT ROWID; + } + + 14 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a DESC, c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + + 15 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c DESC)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + + 16 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(c DESC, a)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b DESC, c, a); + } } { reset_db execsql $schema @@ -157,15 +190,23 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { $cmd test.db ota.db } {SQLITE_DONE} - do_execsql_test 1.$tn2.$tn.2 { - SELECT * FROM t1 ORDER BY a ASC; - } { + do_execsql_test 1.$tn2.$tn.2 { SELECT * FROM t1 ORDER BY a ASC } { 1 2 3 2 two three 3 {} 8.2 } + do_execsql_test 1.$tn2.$tn.3 { SELECT * FROM t1 ORDER BY b ASC } { + 3 {} 8.2 + 1 2 3 + 2 two three + } + do_execsql_test 1.$tn2.$tn.4 { SELECT * FROM t1 ORDER BY c ASC } { + 1 2 3 + 3 {} 8.2 + 2 two three + } - do_execsql_test 1.$tn2.$tn.3 { PRAGMA integrity_check } ok + do_execsql_test 1.$tn2.$tn.5 { PRAGMA integrity_check } ok } } @@ -277,6 +318,19 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota} { CREATE INDEX i2 ON t1(c, b); CREATE INDEX i3 ON t1(c, b, c); } + + 6 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c); + CREATE INDEX i1 ON t1(b DESC); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c DESC, b, c); + } + 7 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } } { reset_db execsql $schema diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index ae45db032b..c7f3afec12 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -645,8 +645,9 @@ static char *otaObjIterGetIndexCols( } while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ - const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); const char *zCol; const char *zType; @@ -669,7 +670,8 @@ static char *otaObjIterGetIndexCols( zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate); if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ - zImpPK = sqlite3_mprintf("%z%sc%d", zImpPK, zCom, nBind); + const char *zOrder = (bDesc ? " DESC" : ""); + zImpPK = sqlite3_mprintf("%z%sc%d%s", zImpPK, zCom, nBind, zOrder); } zImpCols = sqlite3_mprintf( "%z%sc%d %s COLLATE %Q", zImpCols, zCom, nBind, zType, zCollate @@ -839,6 +841,46 @@ static char *otaObjIterGetBindlist(sqlite3ota *p, int nBind){ return zRet; } +/* +** The iterator currently points to a table (not index) of type +** OTA_PK_WITHOUT_ROWID. This function creates the PRIMARY KEY +** declaration for the corresponding imposter table. For example, +** if the iterator points to a table created as: +** +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, a DESC)) WITHOUT ROWID +** +** this function returns: +** +** PRIMARY KEY("b", "a" DESC) +*/ +static char *otaWithoutRowidPK(sqlite3ota *p, OtaObjIter *pIter){ + char *z = 0; + assert( pIter->zIdx==0 ); + if( p->rc==SQLITE_OK ){ + const char *zSep = "PRIMARY KEY("; + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = (pIter->zTbl) */ + int rc; /* sqlite3_finalize() return code */ + + p->rc = prepareFreeAndCollectError(p->db, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zTbl) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + if( sqlite3_column_int(pXInfo, 5) ){ + /* int iCid = sqlite3_column_int(pXInfo, 0); */ + const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); + const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : ""; + z = otaMPrintfAndCollectError(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); + zSep = ", "; + } + } + z = otaMPrintfAndCollectError(p, "%z)", z); + + rc = sqlite3_finalize(pXInfo); + if( p->rc==SQLITE_OK ) p->rc = rc; + } + return z; +} + /* ** If an error has already occurred when this function is called, it ** immediately returns zero (without doing any work). Or, if an error @@ -876,30 +918,31 @@ static void otaCreateImposterTable(sqlite3ota *p, OtaObjIter *pIter){ sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1); for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ + const char *zPk = ""; int iDataCol = pIter->aiTblOrder[iCol]; const char *zCol = pIter->azTblCol[iDataCol]; const char *zColl = 0; + p->rc = sqlite3_table_column_metadata( p->db, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0 ); - zSql = otaMPrintfAndCollectError(p, "%z%s\"%w\" %s COLLATE %s", - zSql, zComma, zCol, pIter->azTblType[iDataCol], zColl + + if( pIter->eType==OTA_PK_IPK && pIter->abTblPk[iCol] ){ + /* If the target table column is an "INTEGER PRIMARY KEY", add + ** "PRIMARY KEY" to the imposter table column declaration. */ + zPk = "PRIMARY KEY "; + } + zSql = otaMPrintfAndCollectError(p, "%z%s\"%w\" %s %sCOLLATE %s", + zSql, zComma, zCol, pIter->azTblType[iDataCol], zPk, zColl ); zComma = ", "; } - if( pIter->eType==OTA_PK_IPK || pIter->eType==OTA_PK_WITHOUT_ROWID ){ - zSql = otaMPrintfAndCollectError(p, "%z, PRIMARY KEY(", zSql); - zComma = ""; - for(iCol=0; iColnTblCol; iCol++){ - if( pIter->abTblPk[iCol] ){ - zSql = otaMPrintfAndCollectError(p, "%z%s\"%w\"", - zSql, zComma, pIter->azTblCol[iCol] - ); - zComma = ", "; - } + if( pIter->eType==OTA_PK_WITHOUT_ROWID ){ + char *zPk = otaWithoutRowidPK(p, pIter); + if( zPk ){ + zSql = otaMPrintfAndCollectError(p, "%z, %z", zSql, zPk); } - zSql = otaMPrintfAndCollectError(p, "%z)", zSql); } zSql = otaMPrintfAndCollectError(p, "CREATE TABLE ota_imposter(%z)%s", diff --git a/manifest b/manifest index 0a49c2b2a1..39ded5aeaa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\s"PRAGMA\sota_mode". -D 2015-02-03T15:56:08.271 +C Fix\ssome\sproblems\ssurrounding\sWITHOUT\sROWID\stables\swith\sDESC\sprimary\skey\sindexes\n. +D 2015-02-03T18:43:42.428 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,7 +125,7 @@ F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt 78d4a9f78f567d4bf826cf0f02df6254902562ca F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 -F ext/ota/ota1.test d76b9ec77437759e9da0ff4abe9c070bb9f4eae1 +F ext/ota/ota1.test 719854e444dff2ead58ff6b62d8315954bd7762a F ext/ota/ota10.test ab815dff9cef7248c504f06b888627d236f25e9c F ext/ota/ota2.test 2829bc08ffbb71b605392a68fedfd554763356a7 F ext/ota/ota3.test a77efbce7723332eb688d2b28bf18204fc9614d7 @@ -136,7 +136,7 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c 52c91eec41b8fbb5ed12a8f0a2159bc5ec16498f +F ext/ota/sqlite3ota.c de2e605697871de02fd9ea717a57c00bf022a27e F ext/ota/sqlite3ota.h ce378c0c503f625611713133f9c79704ea4ee7a4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -238,7 +238,7 @@ F src/parse.y c5d0d964f9ac023e8154cad512e54b0b6058e086 F src/pcache.c d210cf90d04365a74f85d21374dded65af67b0cb F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c 1e77432b40b7d3288327d9cdf399dcdfd2b6d3bf -F src/pragma.c 26fc55619109828c9b7add4cfa8a961b6f4c456d +F src/pragma.c e39954ff47e287f88bdf9e9adb123da029e83d9f F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 F src/printf.c 05edc41450d0eb2c05ef7db113bf32742ae65325 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 @@ -1254,7 +1254,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cdaeab467f6aa3217be161377a9b78a4eec37093 -R 2703aba199c695992db82956950982ca +P 1c111447a07687c30ed4ad5a6c27a169c85b7ea6 +R 544935f49f95af3db3c849fcc1fc392c U dan -Z fae9dc4e5e5c1a2d2a162b422a05ccf9 +Z 5f7217c8a4b7a144888c5aa7b5260598 diff --git a/manifest.uuid b/manifest.uuid index 0e07d68cdf..66e6bbaaef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c111447a07687c30ed4ad5a6c27a169c85b7ea6 \ No newline at end of file +a21fefb79a161e6bb057ae4f6ba554f659706ac1 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 61c721f4d2..fc39a5476c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1581,6 +1581,10 @@ void sqlite3Pragma( Index *pIdx; Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); + if( pIdx==0 ){ + pTab = sqlite3FindTable(db, zRight, zDb); + if( pTab && !HasRowid(pTab) ) pIdx = sqlite3PrimaryKeyIndex(pTab); + } if( pIdx ){ int i; int mx = pPragma->iArg ? pIdx->nColumn : pIdx->nKeyCol; From 1e35850229af982d5341c945b43cb442113f8757 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 4 Feb 2015 11:08:47 +0000 Subject: [PATCH 051/116] Fix a memory leak that could follow an OOM error in ota. FossilOrigin-Name: 0d5415f26427d0af89663c5a3e944e488143b648 --- ext/ota/sqlite3ota.c | 99 +++++++++++++++++++++----------------------- manifest | 12 +++--- manifest.uuid | 2 +- 3 files changed, 55 insertions(+), 58 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index c7f3afec12..c3ac806d64 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -21,6 +21,12 @@ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_OTA) #include "sqlite3ota.h" +/* +** Swap two objects of type TYPE. +*/ +#if !defined(SQLITE_AMALGAMATION) +# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} +#endif /* ** The ota_state table is used to save the state of a partially applied @@ -102,11 +108,11 @@ struct OtaObjIter { sqlite3_stmt *pTblIter; /* Iterate through tables */ sqlite3_stmt *pIdxIter; /* Index iterator */ int nTblCol; /* Size of azTblCol[] array */ - char **azTblCol; /* Array of unquoted column names */ - char **azTblType; /* Array of column types */ - int *aiTblOrder; /* Order of columns in target table */ - unsigned char *abTblPk; /* Array of flags - true for PK columns */ - int eType; + char **azTblCol; /* Array of unquoted target column names */ + char **azTblType; /* Array of target column types */ + int *aiSrcOrder; /* src table col -> target table col */ + unsigned char *abTblPk; /* Array of flags, set on target PK columns */ + int eType; /* Table type - an OTA_PK_XXX value */ /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ @@ -244,7 +250,7 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){ sqlite3_free(pIter->azTblCol); pIter->azTblCol = 0; pIter->azTblType = 0; - pIter->aiTblOrder = 0; + pIter->aiSrcOrder = 0; pIter->abTblPk = 0; pIter->nTblCol = 0; sqlite3_free(pIter->zMask); @@ -417,8 +423,8 @@ static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ memset(azNew, 0, nByte); pIter->azTblCol = azNew; pIter->azTblType = &azNew[nCol]; - pIter->aiTblOrder = (int*)&pIter->azTblType[nCol]; - pIter->abTblPk = (unsigned char*)&pIter->aiTblOrder[nCol]; + pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; + pIter->abTblPk = (unsigned char*)&pIter->aiSrcOrder[nCol]; }else{ p->rc = SQLITE_NOMEM; } @@ -485,6 +491,7 @@ static int otaObjIterCacheTableInfo(sqlite3ota *p, OtaObjIter *pIter){ const char *zName = (const char*)sqlite3_column_name(pStmt, i); if( sqlite3_strnicmp("ota_", zName, 4) ){ char *zCopy = otaStrndup(zName, -1, &p->rc); + pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol; pIter->azTblCol[pIter->nTblCol++] = zCopy; } else if( 0==sqlite3_stricmp("ota_rowid", zName) ){ @@ -514,7 +521,7 @@ static int otaObjIterCacheTableInfo(sqlite3ota *p, OtaObjIter *pIter){ } while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zName = (const char*)sqlite3_column_text(pStmt, 1); - for(i=0; inTblCol; i++){ + for(i=iOrder; inTblCol; i++){ if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; } if( i==pIter->nTblCol ){ @@ -525,31 +532,17 @@ static int otaObjIterCacheTableInfo(sqlite3ota *p, OtaObjIter *pIter){ }else{ int iPk = sqlite3_column_int(pStmt, 5); const char *zType = (const char*)sqlite3_column_text(pStmt, 2); - pIter->aiTblOrder[i] = iOrder++; - pIter->azTblType[i] = otaStrndup(zType, -1, &p->rc); - pIter->abTblPk[i] = (iPk!=0); - } - } - while( iOrdernTblCol ){ - for(i=0; inTblCol; i++){ - if( pIter->aiTblOrder[i]==0 ){ - pIter->aiTblOrder[i] = iOrder++; - continue; - } - } - } - /* Check that there were no extra columns in the data_xxx table that - ** are not present in the target table. If there are, an error. */ -#if 0 - assert( iOrder<=pIter->nTblCol ); - if( p->rc==SQLITE_OK && iOrder!=pIter->nTblCol ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("data_%q has %d columns, expected %d", - pIter->zTbl, iOrder, pIter->nTblCol - ); + if( i!=iOrder ){ + SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]); + SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]); + } + + pIter->azTblType[iOrder] = otaStrndup(zType, -1, &p->rc); + pIter->abTblPk[iOrder] = (iPk!=0); + iOrder++; + } } -#endif rc2 = sqlite3_finalize(pStmt); if( p->rc==SQLITE_OK ) p->rc = rc2; @@ -561,14 +554,22 @@ static int otaObjIterCacheTableInfo(sqlite3ota *p, OtaObjIter *pIter){ /* ** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, ** an error code is stored in the OTA handle passed as the first argument. +** +** If an error has already occurred (p->rc is already set to something other +** than SQLITE_OK), then this function returns NULL without modifying the +** stored error code. In this case it still calls sqlite3_free() on any +** printf() parameters associated with %z conversions. */ -static char *otaMPrintfAndCollectError(sqlite3ota *p, const char *zFmt, ...){ +static char *otaMPrintf(sqlite3ota *p, const char *zFmt, ...){ char *zSql = 0; va_list ap; va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); if( p->rc==SQLITE_OK ){ - zSql = sqlite3_vmprintf(zFmt, ap); if( zSql==0 ) p->rc = SQLITE_NOMEM; + }else{ + sqlite3_free(zSql); + zSql = 0; } va_end(ap); return zSql; @@ -588,7 +589,7 @@ static char *otaObjIterGetCollist( int i; for(i=0; inTblCol; i++){ const char *z = pIter->azTblCol[i]; - zList = otaMPrintfAndCollectError(p, "%z%s\"%w\"", zList, zSep, z); + zList = otaMPrintf(p, "%z%s\"%w\"", zList, zSep, z); zSep = ", "; } return zList; @@ -759,16 +760,14 @@ static char *otaObjIterGetWhere( ){ char *zList = 0; if( pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE ){ - zList = otaMPrintfAndCollectError(p, "_rowid_ = ?%d", pIter->nTblCol+1); + zList = otaMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1); }else{ const char *zSep = ""; int i; for(i=0; inTblCol; i++){ if( pIter->abTblPk[i] ){ const char *zCol = pIter->azTblCol[i]; - zList = otaMPrintfAndCollectError( - p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1 - ); + zList = otaMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1); zSep = " AND "; } } @@ -803,16 +802,15 @@ static char *otaObjIterGetSetlist( }else{ const char *zSep = ""; for(i=0; inTblCol; i++){ - char c = zMask[i]; + char c = zMask[pIter->aiSrcOrder[i]]; if( c=='x' ){ - zList = otaMPrintfAndCollectError(p, "%z%s\"%w\"=?%d", + zList = otaMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, pIter->azTblCol[i], i+1 ); zSep = ", "; } if( c=='d' ){ - zList = otaMPrintfAndCollectError(p, - "%z%s\"%w\"=ota_delta(\"%w\", ?%d)", + zList = otaMPrintf(p, "%z%s\"%w\"=ota_delta(\"%w\", ?%d)", zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 ); zSep = ", "; @@ -869,11 +867,11 @@ static char *otaWithoutRowidPK(sqlite3ota *p, OtaObjIter *pIter){ /* int iCid = sqlite3_column_int(pXInfo, 0); */ const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : ""; - z = otaMPrintfAndCollectError(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); + z = otaMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); zSep = ", "; } } - z = otaMPrintfAndCollectError(p, "%z)", z); + z = otaMPrintf(p, "%z)", z); rc = sqlite3_finalize(pXInfo); if( p->rc==SQLITE_OK ) p->rc = rc; @@ -919,8 +917,7 @@ static void otaCreateImposterTable(sqlite3ota *p, OtaObjIter *pIter){ for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ const char *zPk = ""; - int iDataCol = pIter->aiTblOrder[iCol]; - const char *zCol = pIter->azTblCol[iDataCol]; + const char *zCol = pIter->azTblCol[iCol]; const char *zColl = 0; p->rc = sqlite3_table_column_metadata( @@ -932,8 +929,8 @@ static void otaCreateImposterTable(sqlite3ota *p, OtaObjIter *pIter){ ** "PRIMARY KEY" to the imposter table column declaration. */ zPk = "PRIMARY KEY "; } - zSql = otaMPrintfAndCollectError(p, "%z%s\"%w\" %s %sCOLLATE %s", - zSql, zComma, zCol, pIter->azTblType[iDataCol], zPk, zColl + zSql = otaMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s", + zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl ); zComma = ", "; } @@ -941,11 +938,11 @@ static void otaCreateImposterTable(sqlite3ota *p, OtaObjIter *pIter){ if( pIter->eType==OTA_PK_WITHOUT_ROWID ){ char *zPk = otaWithoutRowidPK(p, pIter); if( zPk ){ - zSql = otaMPrintfAndCollectError(p, "%z, %z", zSql, zPk); + zSql = otaMPrintf(p, "%z, %z", zSql, zPk); } } - zSql = otaMPrintfAndCollectError(p, "CREATE TABLE ota_imposter(%z)%s", + zSql = otaMPrintf(p, "CREATE TABLE ota_imposter(%z)%s", zSql, (pIter->eType==OTA_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") ); if( p->rc==SQLITE_OK ){ diff --git a/manifest b/manifest index 39ded5aeaa..f5510c93ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\ssurrounding\sWITHOUT\sROWID\stables\swith\sDESC\sprimary\skey\sindexes\n. -D 2015-02-03T18:43:42.428 +C Fix\sa\smemory\sleak\sthat\scould\sfollow\san\sOOM\serror\sin\sota. +D 2015-02-04T11:08:47.037 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -136,7 +136,7 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c de2e605697871de02fd9ea717a57c00bf022a27e +F ext/ota/sqlite3ota.c d8910eb0f88d8641fd14c3aa7497d06cb9007745 F ext/ota/sqlite3ota.h ce378c0c503f625611713133f9c79704ea4ee7a4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -1254,7 +1254,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1c111447a07687c30ed4ad5a6c27a169c85b7ea6 -R 544935f49f95af3db3c849fcc1fc392c +P a21fefb79a161e6bb057ae4f6ba554f659706ac1 +R d3231c518b6e7b2eec719e82b7a1a42f U dan -Z 5f7217c8a4b7a144888c5aa7b5260598 +Z 7434d64dd874abe2fbb928ef046bb349 diff --git a/manifest.uuid b/manifest.uuid index 66e6bbaaef..d86e85b7ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a21fefb79a161e6bb057ae4f6ba554f659706ac1 \ No newline at end of file +0d5415f26427d0af89663c5a3e944e488143b648 \ No newline at end of file From 738a3b2fcb261b723968077181ddd7750c2a1178 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 4 Feb 2015 16:32:47 +0000 Subject: [PATCH 052/116] Avoid a linear scan of the entire table when ota updates or deletes a row from a table with an external primary key index. FossilOrigin-Name: 1db198ccca1e5c5a922cefe3daeff8d2e5d3a7f7 --- ext/ota/sqlite3ota.c | 102 +++++++++++++++++++++++++++++++++++++++---- manifest | 14 +++--- manifest.uuid | 2 +- src/main.c | 18 +++++++- 4 files changed, 118 insertions(+), 18 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index c3ac806d64..6c7437d5e1 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -118,7 +118,8 @@ struct OtaObjIter { int bCleanup; /* True in "cleanup" state */ const char *zTbl; /* Name of target db table */ const char *zIdx; /* Name of target db index (or null) */ - int tnum; /* Root page of index (not table) */ + int iTnum; /* Root page of current object */ + int iPkTnum; /* If eType==EXTERNAL, root of PK index */ int bUnique; /* Current index is unique */ int iVisit; /* Number of points visited, incl. current */ @@ -320,7 +321,7 @@ static int otaObjIterNext(sqlite3ota *p, OtaObjIter *pIter){ pIter->zTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); - pIter->tnum = sqlite3_column_int(pIter->pTblIter, 1); + pIter->iTnum = sqlite3_column_int(pIter->pTblIter, 1); rc = SQLITE_OK; } }else{ @@ -334,7 +335,7 @@ static int otaObjIterNext(sqlite3ota *p, OtaObjIter *pIter){ pIter->zIdx = 0; }else{ pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); - pIter->tnum = sqlite3_column_int(pIter->pIdxIter, 1); + pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1); pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); rc = SQLITE_OK; } @@ -470,7 +471,8 @@ static int otaObjIterCacheTableInfo(sqlite3ota *p, OtaObjIter *pIter){ /* Figure out the type of table this step will deal with. */ assert( pIter->eType==0 ); sqlite3_test_control( - SQLITE_TESTCTRL_TBLTYPE, p->db, "main", pIter->zTbl, &pIter->eType + SQLITE_TESTCTRL_TBLTYPE, p->db, "main", pIter->zTbl, &pIter->eType, + &pIter->iPkTnum ); assert( pIter->eType==OTA_PK_NONE || pIter->eType==OTA_PK_IPK || pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_WITHOUT_ROWID @@ -761,6 +763,19 @@ static char *otaObjIterGetWhere( char *zList = 0; if( pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE ){ zList = otaMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1); + }else if( pIter->eType==OTA_PK_EXTERNAL ){ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + zList = otaMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1); + zSep = " AND "; + } + } + zList = otaMPrintf(p, + "_rowid_ = (SELECT id FROM ota_imposter2 WHERE %z)", zList + ); + }else{ const char *zSep = ""; int i; @@ -879,6 +894,75 @@ static char *otaWithoutRowidPK(sqlite3ota *p, OtaObjIter *pIter){ return z; } +static void otaCreateImposterTable2(sqlite3ota *p, OtaObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType==OTA_PK_EXTERNAL ){ + int tnum = pIter->iPkTnum; /* Root page of PK index */ + sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */ + const char *zIdx = 0; /* Name of PK index */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */ + int rc; + + const char *zComma = ""; + + char *zCols = 0; /* Used to build up list of table cols */ + char *zPk = 0; /* Used to build up table PK declaration */ + char *zSql = 0; /* CREATE TABLE statement */ + + /* Figure out the name of the primary key index for the current table. + ** This is needed for the argument to "PRAGMA index_xinfo". Set + ** zIdx to point to a nul-terminated string containing this name. */ + p->rc = prepareAndCollectError(p->db, &pQuery, &p->zErrmsg, + "SELECT name FROM sqlite_master WHERE rootpage = ?" + ); + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(pQuery, 1, tnum); + if( SQLITE_ROW==sqlite3_step(pQuery) ){ + zIdx = (const char*)sqlite3_column_text(pQuery, 0); + } + if( zIdx==0 ){ + p->rc = SQLITE_CORRUPT; + } + } + assert( (zIdx==0)==(p->rc!=SQLITE_OK) ); + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->db, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + sqlite3_finalize(pQuery); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int bKey = sqlite3_column_int(pXInfo, 5); + if( bKey ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + zCols = otaMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma, + iCid, pIter->azTblType[iCid], zCollate + ); + zPk = otaMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":""); + zComma = ", "; + } + } + zCols = otaMPrintf(p, "%z, id INTEGER", zCols); + rc = sqlite3_finalize(pXInfo); + if( p->rc==SQLITE_OK ) p->rc = rc; + + zSql = otaMPrintf(p, + "CREATE TABLE ota_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", + zCols, zPk + ); + assert( (zSql==0)==(p->rc!=SQLITE_OK) ); + if( zSql ){ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum); + p->rc = sqlite3_exec(p->db, zSql, 0, 0, &p->zErrmsg); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); + } + sqlite3_free(zSql); + } +} + /* ** If an error has already occurred when this function is called, it ** immediately returns zero (without doing any work). Or, if an error @@ -905,11 +989,13 @@ static char *otaWithoutRowidPK(sqlite3ota *p, OtaObjIter *pIter){ ** No imposters required. ** ** OTA_PK_EXTERNAL: -** Two imposters are required (TODO!!) +** Two imposters are required. The first has the same schema as the +** target database table, with no PRIMARY KEY or UNIQUE clauses. The +** second is used to access the PK b-tree index on disk. */ static void otaCreateImposterTable(sqlite3ota *p, OtaObjIter *pIter){ if( p->rc==SQLITE_OK && pIter->eType!=OTA_PK_VTAB ){ - int tnum = pIter->tnum; + int tnum = pIter->iTnum; const char *zComma = ""; char *zSql = 0; int iCol; @@ -966,7 +1052,7 @@ static int otaObjIterPrepareAll( ){ assert( pIter->bCleanup==0 ); if( pIter->pSelect==0 && otaObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){ - const int tnum = pIter->tnum; + const int tnum = pIter->iTnum; char *zCollist = 0; /* List of indexed columns */ char **pz = &p->zErrmsg; const char *zIdx = pIter->zIdx; @@ -1067,6 +1153,7 @@ static int otaObjIterPrepareAll( /* Create the imposter table or tables (if required). */ otaCreateImposterTable(p, pIter); + otaCreateImposterTable2(p, pIter); zWrite = (pIter->eType==OTA_PK_VTAB ? zTbl : "ota_imposter"); /* Create the INSERT statement to write to the target PK b-tree */ @@ -1941,7 +2028,6 @@ static int test_sqlite3ota_cmd( db, "ota_delta", -1, SQLITE_UTF8, (void*)interp, test_ota_delta, 0, 0 ); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); - sqlite3_exec(db, "PRAGMA vdbe_trace = 1", 0, 0, 0); ret = (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); break; } diff --git a/manifest b/manifest index f5510c93ec..b19a7886de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sthat\scould\sfollow\san\sOOM\serror\sin\sota. -D 2015-02-04T11:08:47.037 +C Avoid\sa\slinear\sscan\sof\sthe\sentire\stable\swhen\sota\supdates\sor\sdeletes\sa\srow\sfrom\sa\stable\swith\san\sexternal\sprimary\skey\sindex. +D 2015-02-04T16:32:47.267 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -136,7 +136,7 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c d8910eb0f88d8641fd14c3aa7497d06cb9007745 +F ext/ota/sqlite3ota.c 8d1f7f1643b85bd442cc99a919b1bbbac5b1d785 F ext/ota/sqlite3ota.h ce378c0c503f625611713133f9c79704ea4ee7a4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -210,7 +210,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 -F src/main.c 55d548a2c2f32d27366968c394d091475f7ea00a +F src/main.c 893ca8955539f5dcff8c798b508e94a6d08b6d99 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987 @@ -1254,7 +1254,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a21fefb79a161e6bb057ae4f6ba554f659706ac1 -R d3231c518b6e7b2eec719e82b7a1a42f +P 0d5415f26427d0af89663c5a3e944e488143b648 +R f9a2d13b5d717bd34c88d8b13812d5f5 U dan -Z 7434d64dd874abe2fbb928ef046bb349 +Z 54797f4495fbc966460447f60ea02156 diff --git a/manifest.uuid b/manifest.uuid index d86e85b7ec..e98583da50 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d5415f26427d0af89663c5a3e944e488143b648 \ No newline at end of file +1db198ccca1e5c5a922cefe3daeff8d2e5d3a7f7 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 710d1dfb84..0ff49f289b 100644 --- a/src/main.c +++ b/src/main.c @@ -3655,7 +3655,7 @@ int sqlite3_test_control(int op, ...){ break; } - /* sqlite3_test_control(SQLITE_TESTCTRL_TBLTYPE, db, dbName, zTbl, peType) + /* sqlite3_test_control(TESTCTRL_TBLTYPE, db, dbName, zTbl, peType, piPk) ** ** peType is of type (int*), a pointer to an output parameter of type ** (int). This call sets the output parameter as follows, depending @@ -3667,13 +3667,22 @@ int sqlite3_test_control(int op, ...){ ** 3: Table has an external PK index. ** 4: Table is WITHOUT ROWID. ** 5: Table is a virtual table. + ** + ** Argument *piPk is also of type (int*), and also points to an output + ** parameter. Unless the table has an external primary key index + ** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, + ** if the table does have an external primary key index, then *piPk + ** is set to the root page number of the primary key index before + ** returning. */ case SQLITE_TESTCTRL_TBLTYPE: { sqlite3 *db = va_arg(ap, sqlite3*); const char *zDb = va_arg(ap, const char*); const char *zTab = va_arg(ap, const char*); int *peType = va_arg(ap, int*); + int *piPk = va_arg(ap, int*); Table *pTab; + *piPk = 0; sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); pTab = sqlite3FindTable(db, zTab, zDb); @@ -3687,7 +3696,12 @@ int sqlite3_test_control(int op, ...){ *peType = 2; }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); - *peType = (pPk ? 3 : 1); + if( pPk ){ + *peType = 3; + *piPk = pPk->tnum; + }else{ + *peType = 1; + } } sqlite3BtreeLeaveAll(db); sqlite3_mutex_leave(db->mutex); From 06c97225641821330871db5efdd964a5e2793087 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 4 Feb 2015 19:20:42 +0000 Subject: [PATCH 053/116] Fix the error messages returned by ota if an update violates a unique constraint. FossilOrigin-Name: c4845a3b591723ff60b7b1da198f44908c12bdfc --- ext/ota/sqlite3ota.c | 86 ++++++++++++++++++++++++++++---------------- manifest | 12 +++---- manifest.uuid | 2 +- 3 files changed, 63 insertions(+), 37 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 6c7437d5e1..e6882d855d 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -674,12 +674,16 @@ static char *otaObjIterGetIndexCols( zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate); if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ const char *zOrder = (bDesc ? " DESC" : ""); - zImpPK = sqlite3_mprintf("%z%sc%d%s", zImpPK, zCom, nBind, zOrder); + zImpPK = sqlite3_mprintf("%z%s\"ota_imp_%d%w\"%s", + zImpPK, zCom, nBind, zCol, zOrder + ); } - zImpCols = sqlite3_mprintf( - "%z%sc%d %s COLLATE %Q", zImpCols, zCom, nBind, zType, zCollate + zImpCols = sqlite3_mprintf("%z%s\"ota_imp_%d%w\" %s COLLATE %Q", + zImpCols, zCom, nBind, zCol, zType, zCollate + ); + zWhere = sqlite3_mprintf( + "%z%s\"ota_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol ); - zWhere = sqlite3_mprintf("%z%sc%d IS ?", zWhere, zAnd, nBind); if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; zCom = ", "; zAnd = " AND "; @@ -1028,8 +1032,9 @@ static void otaCreateImposterTable(sqlite3ota *p, OtaObjIter *pIter){ } } - zSql = otaMPrintf(p, "CREATE TABLE ota_imposter(%z)%s", - zSql, (pIter->eType==OTA_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") + zSql = otaMPrintf(p, "CREATE TABLE \"ota_imp_%w\"(%z)%s", + pIter->zTbl, zSql, + (pIter->eType==OTA_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") ); if( p->rc==SQLITE_OK ){ sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum); @@ -1064,6 +1069,7 @@ static int otaObjIterPrepareAll( } if( zIdx ){ + const char *zTbl = pIter->zTbl; char *zImposterCols = 0; /* Columns for imposter table */ char *zImposterPK = 0; /* Primary key declaration for imposter */ char *zWhere = 0; /* WHERE clause on PK columns */ @@ -1080,8 +1086,8 @@ static int otaObjIterPrepareAll( sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum); otaMPrintfExec(p, - "CREATE TABLE ota_imposter( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", - zImposterCols, zImposterPK + "CREATE TABLE \"ota_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", + zTbl, zImposterCols, zImposterPK ); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); @@ -1089,14 +1095,14 @@ static int otaObjIterPrepareAll( pIter->nCol = nBind; if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pInsert, &p->zErrmsg, - sqlite3_mprintf("INSERT INTO ota_imposter VALUES(%s)", zBind) + sqlite3_mprintf("INSERT INTO \"ota_imp_%w\" VALUES(%s)", zTbl, zBind) ); } /* And to delete index entries */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pDelete, &p->zErrmsg, - sqlite3_mprintf("DELETE FROM ota_imposter WHERE %s", zWhere) + sqlite3_mprintf("DELETE FROM \"ota_imp_%w\" WHERE %s", zTbl, zWhere) ); } @@ -1154,14 +1160,14 @@ static int otaObjIterPrepareAll( /* Create the imposter table or tables (if required). */ otaCreateImposterTable(p, pIter); otaCreateImposterTable2(p, pIter); - zWrite = (pIter->eType==OTA_PK_VTAB ? zTbl : "ota_imposter"); + zWrite = (pIter->eType==OTA_PK_VTAB ? "" : "ota_imp_"); /* Create the INSERT statement to write to the target PK b-tree */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pInsert, pz, sqlite3_mprintf( - "INSERT INTO main.%Q(%s%s) VALUES(%s)", - zWrite, zCollist, (bOtaRowid ? ", _rowid_" : ""), zBindings + "INSERT INTO main.\"%s%w\"(%s%s) VALUES(%s)", + zWrite, zTbl, zCollist, (bOtaRowid ? ", _rowid_" : ""), zBindings ) ); } @@ -1170,7 +1176,7 @@ static int otaObjIterPrepareAll( if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pDelete, pz, sqlite3_mprintf( - "DELETE FROM main.%Q WHERE %s", zWrite, zWhere + "DELETE FROM main.\"%s%w\" WHERE %s", zWrite, zTbl, zWhere ) ); } @@ -1183,42 +1189,39 @@ static int otaObjIterPrepareAll( /* Create the ota_tmp_xxx table and the triggers to populate it. */ otaMPrintfExec(p, - "PRAGMA ota_mode = 1;" "CREATE TABLE IF NOT EXISTS ota.'ota_tmp_%q' AS " "SELECT *%s FROM ota.'data_%q' WHERE 0;" - "CREATE TEMP TRIGGER ota_delete_tr BEFORE DELETE ON ota_imposter " + "CREATE TEMP TRIGGER ota_delete_tr BEFORE DELETE ON \"%s%w\" " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(2, %s);" "END;" - "CREATE TEMP TRIGGER ota_update1_tr BEFORE UPDATE ON ota_imposter " + "CREATE TEMP TRIGGER ota_update1_tr BEFORE UPDATE ON \"%s%w\" " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(2, %s);" "END;" - "CREATE TEMP TRIGGER ota_update2_tr AFTER UPDATE ON ota_imposter " + "CREATE TEMP TRIGGER ota_update2_tr AFTER UPDATE ON \"%s%w\" " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s%s) VALUES(3, %s);" "END;" , zTbl, (pIter->eType==OTA_PK_EXTERNAL ? ", 0 AS ota_rowid" : "") , zTbl, - zTbl, zCollist, zOtaRowid, zOldlist, - zTbl, zCollist, zOtaRowid, zOldlist, - zTbl, zCollist, zOtaRowid, zNewlist + zWrite, zTbl, zTbl, zCollist, zOtaRowid, zOldlist, + zWrite, zTbl, zTbl, zCollist, zOtaRowid, zOldlist, + zWrite, zTbl, zTbl, zCollist, zOtaRowid, zNewlist ); if( pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_NONE ){ otaMPrintfExec(p, - "CREATE TEMP TRIGGER ota_insert_tr AFTER INSERT ON ota_imposter " + "CREATE TEMP TRIGGER ota_insert_tr AFTER INSERT ON \"%s%w\" " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s, ota_rowid)" " VALUES(0, %s);" - "END;" - , zTbl, zCollist, zNewlist + "END;", + zWrite, zTbl, zTbl, zCollist, zNewlist ); } - }else if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->db, "PRAGMA ota_mode = 0", 0, 0, &p->zErrmsg); } /* Allocate space required for the zMask field. */ @@ -1265,9 +1268,11 @@ static int otaGetUpdateStmt( sqlite3_finalize(pIter->pUpdate); pIter->pUpdate = 0; if( p->rc==SQLITE_OK ){ - zUpdate = sqlite3_mprintf("UPDATE \"%w\" SET %s WHERE %s", - (pIter->eType==OTA_PK_VTAB ? pIter->zTbl : "ota_imposter"), - zSet, zWhere + const char *zPrefix = ""; + + if( pIter->eType!=OTA_PK_VTAB ) zPrefix = "ota_imp_"; + zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", + zPrefix, pIter->zTbl, zSet, zWhere ); p->rc = prepareFreeAndCollectError( p->db, &pIter->pUpdate, &p->zErrmsg, zUpdate @@ -1852,7 +1857,6 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ const char *zScript = "PRAGMA journal_mode=off;" "PRAGMA pager_ota_mode=1;" - "PRAGMA ota_mode=1;" "BEGIN IMMEDIATE;" ; p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); @@ -1891,6 +1895,27 @@ sqlite3 *sqlite3ota_db(sqlite3ota *pOta){ return (pOta ? pOta->db : 0); } + +/* +** If the error code currently stored in the OTA handle is SQLITE_CONSTRAINT, +** then edit any error message string so as to remove all occurrences of +** the pattern "ota_imp_[0-9]*". +*/ +static void otaEditErrmsg(sqlite3ota *p){ + if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){ + int i; + int nErrmsg = strlen(p->zErrmsg); + for(i=0; i<(nErrmsg-8); i++){ + if( memcmp(&p->zErrmsg[i], "ota_imp_", 8)==0 ){ + int nDel = 8; + while( p->zErrmsg[i+nDel]>='0' && p->zErrmsg[i+nDel]<='9' ) nDel++; + memmove(&p->zErrmsg[i], &p->zErrmsg[i+nDel], nErrmsg + 1 - i - nDel); + nErrmsg -= nDel; + } + } + } +} + /* ** Close the OTA handle. */ @@ -1923,6 +1948,7 @@ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ if( p->pCkpt ) sqlite3_ckpt_close(p->pCkpt, 0, 0); sqlite3_close(p->db); + otaEditErrmsg(p); rc = p->rc; *pzErrmsg = p->zErrmsg; sqlite3_free(p); diff --git a/manifest b/manifest index b19a7886de..bbae196aaa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\slinear\sscan\sof\sthe\sentire\stable\swhen\sota\supdates\sor\sdeletes\sa\srow\sfrom\sa\stable\swith\san\sexternal\sprimary\skey\sindex. -D 2015-02-04T16:32:47.267 +C Fix\sthe\serror\smessages\sreturned\sby\sota\sif\san\supdate\sviolates\sa\sunique\sconstraint. +D 2015-02-04T19:20:42.631 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -136,7 +136,7 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c 8d1f7f1643b85bd442cc99a919b1bbbac5b1d785 +F ext/ota/sqlite3ota.c 41df903106a0107aa70068090b263c93a502da72 F ext/ota/sqlite3ota.h ce378c0c503f625611713133f9c79704ea4ee7a4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -1254,7 +1254,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0d5415f26427d0af89663c5a3e944e488143b648 -R f9a2d13b5d717bd34c88d8b13812d5f5 +P 1db198ccca1e5c5a922cefe3daeff8d2e5d3a7f7 +R d14aafac71cb07666d32346674c8200d U dan -Z 54797f4495fbc966460447f60ea02156 +Z f2b3565f17adbd359615b6fecd58bae9 diff --git a/manifest.uuid b/manifest.uuid index e98583da50..a3e199e74c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1db198ccca1e5c5a922cefe3daeff8d2e5d3a7f7 \ No newline at end of file +c4845a3b591723ff60b7b1da198f44908c12bdfc \ No newline at end of file From d4ef7c50bb5b270e30437448e75a153c5ce03b60 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 5 Feb 2015 01:49:31 +0000 Subject: [PATCH 054/116] Figure out the primary-key type of a table using queries of sqlite_master and the table_info and index_list pragmas, obviating the need for SQLITE_TESTCTRL_TBLTYPE. FossilOrigin-Name: 50ecdfc443b51e3569c6add2fba5132f959c61cb --- ext/ota/sqlite3ota.c | 135 ++++++++++++++++++++++++++++++++++++++++--- manifest | 16 ++--- manifest.uuid | 2 +- src/main.c | 53 ----------------- src/sqlite.h.in | 3 +- 5 files changed, 137 insertions(+), 72 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index e6882d855d..959622bcf4 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -137,12 +137,14 @@ struct OtaObjIter { /* ** Values for OtaObjIter.eType ** +** 0: Table does not exist (error) ** 1: Table has an implicit rowid. ** 2: Table has an explicit IPK column. ** 3: Table has an external PK index. ** 4: Table is WITHOUT ROWID. ** 5: Table is a virtual table. */ +#define OTA_PK_NOTABLE 0 #define OTA_PK_NONE 1 #define OTA_PK_IPK 2 #define OTA_PK_EXTERNAL 3 @@ -449,6 +451,127 @@ static char *otaStrndup(const char *zStr, int nStr, int *pRc){ return zRet; } +/* Determine the type of a table. +** +** peType is of type (int*), a pointer to an output parameter of type +** (int). This call sets the output parameter as follows, depending +** on the type of the table specified by parameters dbName and zTbl. +** +** OTA_PK_NOTABLE: No such table. +** OTA_PK_NONE: Table has an implicit rowid. +** OTA_PK_IPK: Table has an explicit IPK column. +** OTA_PK_EXTERNAL: Table has an external PK index. +** OTA_PK_WITHOUT_ROWID: Table is WITHOUT ROWID. +** OTA_PK_VTAB: Table is a virtual table. +** +** Argument *piPk is also of type (int*), and also points to an output +** parameter. Unless the table has an external primary key index +** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, +** if the table does have an external primary key index, then *piPk +** is set to the root page number of the primary key index before +** returning. +** +** ALGORITHM: +** +** if( no entry exists in sqlite_master ){ +** return OTA_PK_NOTABLE +** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ +** return OTA_PK_VTAB +** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ +** if( the index that is the pk exists in sqlite_master ){ +** *piPK = rootpage of that index. +** return OTA_PK_EXTERNAL +** }else{ +** return OTA_PK_WITHOUT_ROWID +** } +** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){ +** return OTA_PK_IPK +** }else{ +** return OTA_PK_NONE +** } +*/ +static int otaTableType( + sqlite3 *db, + const char *zTab, + int *peType, + int *piPk +){ + sqlite3_stmt *pStmt = 0; + int rc = SQLITE_OK; + int rc2; + char *zSql = 0; + + *peType = OTA_PK_NOTABLE; + *piPk = 0; + zSql = sqlite3_mprintf( + "SELECT (sql LIKE 'create virtual%%')" + " FROM main.sqlite_master" + " WHERE name=%Q", zTab); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( pStmt==0 ) goto otaTableType_end; + if( sqlite3_step(pStmt)!=SQLITE_ROW ){ + goto otaTableType_end; /* no such table */ + } + if( sqlite3_column_int(pStmt,0) ){ + *peType = OTA_PK_VTAB; /* virtual table */ + goto otaTableType_end; + } + rc = sqlite3_finalize(pStmt); + if( rc ) return rc; + zSql = sqlite3_mprintf("PRAGMA index_list=%Q",zTab); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( pStmt==0 ) goto otaTableType_end; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const unsigned char *zOrig = sqlite3_column_text(pStmt,3); + if( zOrig && zOrig[0]=='p' ){ + zSql = sqlite3_mprintf("SELECT rootpage FROM main.sqlite_master" + " WHERE name=%Q", sqlite3_column_text(pStmt,1)); + if( zSql==0 ){ rc = SQLITE_NOMEM; goto otaTableType_end; } + break; + } + } + rc = sqlite3_finalize(pStmt); + pStmt = 0; + if( rc ) return rc; + if( zSql ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( pStmt==0 ) goto otaTableType_end; + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + *piPk = sqlite3_column_int(pStmt, 0); + *peType = OTA_PK_EXTERNAL; /* external PK index */ + }else{ + *peType = OTA_PK_WITHOUT_ROWID; /* WITHOUT ROWID table */ + } + }else{ + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( pStmt==0 ) goto otaTableType_end; + *peType = OTA_PK_NONE; /* (default) implicit ROWID */ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( sqlite3_column_int(pStmt,5)>0 ){ + *peType = OTA_PK_IPK; /* explicit IPK column */ + break; + } + } + } + +otaTableType_end: + sqlite3_free(zSql); + rc2 = sqlite3_finalize(pStmt); + return rc ? rc : rc2; +} + /* ** If they are not already populated, populate the pIter->azTblCol[], @@ -470,10 +593,9 @@ static int otaObjIterCacheTableInfo(sqlite3ota *p, OtaObjIter *pIter){ /* Figure out the type of table this step will deal with. */ assert( pIter->eType==0 ); - sqlite3_test_control( - SQLITE_TESTCTRL_TBLTYPE, p->db, "main", pIter->zTbl, &pIter->eType, - &pIter->iPkTnum - ); + p->rc = otaTableType(p->db, pIter->zTbl, &pIter->eType, &pIter->iPkTnum); + if( p->rc ) return p->rc; + assert( pIter->eType==OTA_PK_NONE || pIter->eType==OTA_PK_IPK || pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_WITHOUT_ROWID || pIter->eType==OTA_PK_VTAB @@ -2105,7 +2227,4 @@ int SqliteOta_Init(Tcl_Interp *interp){ #include int SqliteOta_Init(Tcl_Interp *interp){ return TCL_OK; } # endif -#endif - - - +#endif diff --git a/manifest b/manifest index 6512a12554..44587374cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\schanges\sfrom\sindex_xinfo,\sincluding\sthe\smove\sof\sthe\spragma\stable\ninto\sthe\sseparate\spragma.h\sfile. -D 2015-02-04T23:51:59.867 +C Figure\sout\sthe\sprimary-key\stype\sof\sa\stable\susing\squeries\sof\ssqlite_master\nand\sthe\stable_info\sand\sindex_list\spragmas,\sobviating\sthe\sneed\sfor\nSQLITE_TESTCTRL_TBLTYPE. +D 2015-02-05T01:49:31.187 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -136,7 +136,7 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c 41df903106a0107aa70068090b263c93a502da72 +F ext/ota/sqlite3ota.c 0721c7dae5d16624d708ec13ad372c4738f2a23b F ext/ota/sqlite3ota.h ce378c0c503f625611713133f9c79704ea4ee7a4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -210,7 +210,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 -F src/main.c 893ca8955539f5dcff8c798b508e94a6d08b6d99 +F src/main.c c4cb192ebf0bcc975648ae05ac40bc1f40018c52 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987 @@ -247,7 +247,7 @@ F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 1f2087523007c42900ffcbdeaef06a23ad9329fc F src/shell.c 22b4406b0b59efd14b3b351a5809dda517df6d30 -F src/sqlite.h.in 8913937ba11415bf369818431700adf3a921fb18 +F src/sqlite.h.in 4807b024e8d257af774cde0cf178f721ff2406ec F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 57f8f45028598cc2877fc08ac03b402242242c68 @@ -1255,7 +1255,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c4845a3b591723ff60b7b1da198f44908c12bdfc 3af19f84446ba5fc1ed754d0d73f6a6d7fb2f365 -R f8693fbefe5770ef46252d8013588714 +P 21e95d28a51e56b45a9d4166475972535e53f954 +R 20b83842609fc0bd353b0fba0ba7b9e9 U drh -Z 86d4960217df24559dfd1328c13955b1 +Z ce11446a64cadc5d0eb91c8707e9ad41 diff --git a/manifest.uuid b/manifest.uuid index f5ba861c45..6416dfb803 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21e95d28a51e56b45a9d4166475972535e53f954 \ No newline at end of file +50ecdfc443b51e3569c6add2fba5132f959c61cb \ No newline at end of file diff --git a/src/main.c b/src/main.c index 0ff49f289b..7bca1e5777 100644 --- a/src/main.c +++ b/src/main.c @@ -3654,59 +3654,6 @@ int sqlite3_test_control(int op, ...){ sqlite3_mutex_leave(db->mutex); break; } - - /* sqlite3_test_control(TESTCTRL_TBLTYPE, db, dbName, zTbl, peType, piPk) - ** - ** peType is of type (int*), a pointer to an output parameter of type - ** (int). This call sets the output parameter as follows, depending - ** on the type of the table specified by parameters dbName and zTbl. - ** - ** 0: No such table. - ** 1: Table has an implicit rowid. - ** 2: Table has an explicit IPK column. - ** 3: Table has an external PK index. - ** 4: Table is WITHOUT ROWID. - ** 5: Table is a virtual table. - ** - ** Argument *piPk is also of type (int*), and also points to an output - ** parameter. Unless the table has an external primary key index - ** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, - ** if the table does have an external primary key index, then *piPk - ** is set to the root page number of the primary key index before - ** returning. - */ - case SQLITE_TESTCTRL_TBLTYPE: { - sqlite3 *db = va_arg(ap, sqlite3*); - const char *zDb = va_arg(ap, const char*); - const char *zTab = va_arg(ap, const char*); - int *peType = va_arg(ap, int*); - int *piPk = va_arg(ap, int*); - Table *pTab; - *piPk = 0; - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - pTab = sqlite3FindTable(db, zTab, zDb); - if( pTab==0 ){ - *peType = 0; - }else if( IsVirtual(pTab) ){ - *peType = 5; - }else if( HasRowid(pTab)==0 ){ - *peType = 4; - }else if( pTab->iPKey>=0 ){ - *peType = 2; - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); - if( pPk ){ - *peType = 3; - *piPk = pPk->tnum; - }else{ - *peType = 1; - } - } - sqlite3BtreeLeaveAll(db); - sqlite3_mutex_leave(db->mutex); - break; - } } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1cba5b1772..cdf5a8c179 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6266,8 +6266,7 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ISINIT 23 #define SQLITE_TESTCTRL_SORTER_MMAP 24 #define SQLITE_TESTCTRL_IMPOSTER 25 -#define SQLITE_TESTCTRL_TBLTYPE 26 -#define SQLITE_TESTCTRL_LAST 26 +#define SQLITE_TESTCTRL_LAST 25 /* ** CAPI3REF: SQLite Runtime Status From a8ccf3ed740faa517d5dc06e649bafb94677f0a3 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 5 Feb 2015 17:36:30 +0000 Subject: [PATCH 055/116] Prevent ota updates from violating NOT NULL constraints. Add a comment to the "limitations" section of sqlite3ota.h saying that CHECK constraints are not enforced. FossilOrigin-Name: 74e073dd604142212f3d3e1931065d124daabd80 --- ext/ota/ota10.test | 53 ++++++++++++++++++++++++++++++++++++++++++++ ext/ota/sqlite3ota.c | 12 +++++++--- ext/ota/sqlite3ota.h | 2 ++ manifest | 18 +++++++-------- manifest.uuid | 2 +- 5 files changed, 74 insertions(+), 13 deletions(-) diff --git a/ext/ota/ota10.test b/ext/ota/ota10.test index bca34d68cb..02931a3bf5 100644 --- a/ext/ota/ota10.test +++ b/ext/ota/ota10.test @@ -117,6 +117,59 @@ ifcapable fts3 { } {1 {SQLITE_ERROR - SQL logic error or missing database}} } +#-------------------------------------------------------------------- +# Test that it is not possible to violate a NOT NULL constraint by +# applying an OTA update. +# +do_execsql_test 4.1 { + CREATE TABLE t2(a INTEGER NOT NULL, b TEXT NOT NULL, c PRIMARY KEY); + CREATE TABLE t3(a INTEGER NOT NULL, b TEXT NOT NULL, c INTEGER PRIMARY KEY); + CREATE TABLE t4(a, b, PRIMARY KEY(a, b)) WITHOUT ROWID; + + INSERT INTO t2 VALUES(10, 10, 10); + INSERT INTO t3 VALUES(10, 10, 10); + INSERT INTO t4 VALUES(10, 10); + +} + +foreach {tn error ota} { + 2 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t2.a} { + INSERT INTO data_t2 VALUES(NULL, 'abc', 1, 0); + } + 3 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t2.b} { + INSERT INTO data_t2 VALUES(2, NULL, 1, 0); + } + 4 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t2.c} { + INSERT INTO data_t2 VALUES(1, 'abc', NULL, 0); + } + 5 {SQLITE_MISMATCH - datatype mismatch} { + INSERT INTO data_t3 VALUES(1, 'abc', NULL, 0); + } + 6 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t4.b} { + INSERT INTO data_t4 VALUES('a', NULL, 0); + } + 7 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t4.a} { + INSERT INTO data_t4 VALUES(NULL, 'a', 0); + } + 8 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t2.a} { + INSERT INTO data_t2 VALUES(NULL, 0, 10, 'x..'); + } + 9 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t3.b} { + INSERT INTO data_t3 VALUES(10, NULL, 10, '.x.'); + } +} { + set ota " + CREATE TABLE data_t2(a, b, c, ota_control); + CREATE TABLE data_t3(a, b, c, ota_control); + CREATE TABLE data_t4(a, b, ota_control); + $ota + " + do_test 4.$tn { + list [catch { apply_ota $ota } msg] $msg + } [list 1 $error] +} + finish_test + diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 959622bcf4..dd742c4ab2 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -112,6 +112,7 @@ struct OtaObjIter { char **azTblType; /* Array of target column types */ int *aiSrcOrder; /* src table col -> target table col */ unsigned char *abTblPk; /* Array of flags, set on target PK columns */ + unsigned char *abNotNull; /* Array of flags, set on NOT NULL columns */ int eType; /* Table type - an OTA_PK_XXX value */ /* Output variables. zTbl==0 implies EOF. */ @@ -255,6 +256,7 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){ pIter->azTblType = 0; pIter->aiSrcOrder = 0; pIter->abTblPk = 0; + pIter->abNotNull = 0; pIter->nTblCol = 0; sqlite3_free(pIter->zMask); pIter->zMask = 0; @@ -417,7 +419,7 @@ static int otaMPrintfExec(sqlite3ota *p, const char *zFmt, ...){ ** error code in the OTA handle passed as the first argument. */ static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ - int nByte = (sizeof(char*) * 2 + sizeof(int) + sizeof(unsigned char)) * nCol; + int nByte = (2*sizeof(char*) + sizeof(int) + 2*sizeof(unsigned char)) * nCol; char **azNew; assert( p->rc==SQLITE_OK ); @@ -428,6 +430,7 @@ static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ pIter->azTblType = &azNew[nCol]; pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; pIter->abTblPk = (unsigned char*)&pIter->aiSrcOrder[nCol]; + pIter->abNotNull = (unsigned char*)&pIter->abTblPk[nCol]; }else{ p->rc = SQLITE_NOMEM; } @@ -655,6 +658,7 @@ static int otaObjIterCacheTableInfo(sqlite3ota *p, OtaObjIter *pIter){ ); }else{ int iPk = sqlite3_column_int(pStmt, 5); + int bNotNull = sqlite3_column_int(pStmt, 3); const char *zType = (const char*)sqlite3_column_text(pStmt, 2); if( i!=iOrder ){ @@ -664,6 +668,7 @@ static int otaObjIterCacheTableInfo(sqlite3ota *p, OtaObjIter *pIter){ pIter->azTblType[iOrder] = otaStrndup(zType, -1, &p->rc); pIter->abTblPk[iOrder] = (iPk!=0); + pIter->abNotNull[iOrder] = (unsigned char)bNotNull || (iPk!=0); iOrder++; } } @@ -1141,8 +1146,9 @@ static void otaCreateImposterTable(sqlite3ota *p, OtaObjIter *pIter){ ** "PRIMARY KEY" to the imposter table column declaration. */ zPk = "PRIMARY KEY "; } - zSql = otaMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s", - zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl + zSql = otaMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s", + zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl, + (pIter->abNotNull[iCol] ? " NOT NULL" : "") ); zComma = ", "; } diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index 44aff2e30d..0749ee9aea 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -71,6 +71,8 @@ ** ** * No foreign key violations are detected or reported. ** +** * CHECK constraints are not enforced. +** ** * No constraint handling mode except for "OR ROLLBACK" is supported. ** ** diff --git a/manifest b/manifest index 44587374cc..3f58be9e62 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Figure\sout\sthe\sprimary-key\stype\sof\sa\stable\susing\squeries\sof\ssqlite_master\nand\sthe\stable_info\sand\sindex_list\spragmas,\sobviating\sthe\sneed\sfor\nSQLITE_TESTCTRL_TBLTYPE. -D 2015-02-05T01:49:31.187 +C Prevent\sota\supdates\sfrom\sviolating\sNOT\sNULL\sconstraints.\sAdd\sa\scomment\sto\sthe\s"limitations"\ssection\sof\ssqlite3ota.h\ssaying\sthat\sCHECK\sconstraints\sare\snot\senforced. +D 2015-02-05T17:36:30.528 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -126,7 +126,7 @@ F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt 78d4a9f78f567d4bf826cf0f02df6254902562ca F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 F ext/ota/ota1.test 719854e444dff2ead58ff6b62d8315954bd7762a -F ext/ota/ota10.test ab815dff9cef7248c504f06b888627d236f25e9c +F ext/ota/ota10.test 69bded736f5dd1e07b96e5c51c081e2fd051cb12 F ext/ota/ota2.test 2829bc08ffbb71b605392a68fedfd554763356a7 F ext/ota/ota3.test a77efbce7723332eb688d2b28bf18204fc9614d7 F ext/ota/ota4.test 82434aa39c9acca6cd6317f6b0ab07b0ec6c2e7d @@ -136,8 +136,8 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c 0721c7dae5d16624d708ec13ad372c4738f2a23b -F ext/ota/sqlite3ota.h ce378c0c503f625611713133f9c79704ea4ee7a4 +F ext/ota/sqlite3ota.c 6f4f4e07d48ad51b745a5d30ba601a6895fd15ba +F ext/ota/sqlite3ota.h 22423135fcdf7b242af8039b2ebf8437685e87c9 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1255,7 +1255,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 21e95d28a51e56b45a9d4166475972535e53f954 -R 20b83842609fc0bd353b0fba0ba7b9e9 -U drh -Z ce11446a64cadc5d0eb91c8707e9ad41 +P 50ecdfc443b51e3569c6add2fba5132f959c61cb +R 39225d5cd1d4a3167fd9b3981c9a3ddd +U dan +Z 95e5525c749de3fda07c16abd9286707 diff --git a/manifest.uuid b/manifest.uuid index 6416dfb803..189550920b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -50ecdfc443b51e3569c6add2fba5132f959c61cb \ No newline at end of file +74e073dd604142212f3d3e1931065d124daabd80 \ No newline at end of file From 8800b53bc511349d4013a7f3e0cb5e41cb3a9484 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 5 Feb 2015 17:46:19 +0000 Subject: [PATCH 056/116] Change a comment in sqlite3ota.h to make it clear that it is not possible to insert a NULL value into an INTEGER PRIMARY KEY column using ota. FossilOrigin-Name: a5e86bea4ad51bbb988a2a2215961706894f4a02 --- ext/ota/ota10.test | 17 +++++++++++++++-- ext/ota/sqlite3ota.h | 7 +++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/ext/ota/ota10.test b/ext/ota/ota10.test index 02931a3bf5..52cc27a8f6 100644 --- a/ext/ota/ota10.test +++ b/ext/ota/ota10.test @@ -129,7 +129,6 @@ do_execsql_test 4.1 { INSERT INTO t2 VALUES(10, 10, 10); INSERT INTO t3 VALUES(10, 10, 10); INSERT INTO t4 VALUES(10, 10); - } foreach {tn error ota} { @@ -142,9 +141,11 @@ foreach {tn error ota} { 4 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t2.c} { INSERT INTO data_t2 VALUES(1, 'abc', NULL, 0); } + 5 {SQLITE_MISMATCH - datatype mismatch} { INSERT INTO data_t3 VALUES(1, 'abc', NULL, 0); } + 6 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t4.b} { INSERT INTO data_t4 VALUES('a', NULL, 0); } @@ -157,6 +158,10 @@ foreach {tn error ota} { 9 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t3.b} { INSERT INTO data_t3 VALUES(10, NULL, 10, '.x.'); } + + 10 {SQLITE_MISMATCH - datatype mismatch} { + INSERT INTO data_t3 VALUES(1, 'abc', 'text', 0); + } } { set ota " CREATE TABLE data_t2(a, b, c, ota_control); @@ -164,11 +169,19 @@ foreach {tn error ota} { CREATE TABLE data_t4(a, b, ota_control); $ota " - do_test 4.$tn { + do_test 4.2.$tn { list [catch { apply_ota $ota } msg] $msg } [list 1 $error] } +do_test 4.3 { + set ota { + CREATE TABLE data_t3(a, b, c, ota_control); + INSERT INTO data_t3 VALUES(1, 'abc', '5', 0); + INSERT INTO data_t3 VALUES(1, 'abc', '-6.0', 0); + } + list [catch { apply_ota $ota } msg] $msg +} {0 SQLITE_DONE} finish_test diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index 0749ee9aea..e549022f9a 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -129,10 +129,9 @@ ** other columns should be set to the values that make up the new record ** to insert. ** -** If the target database table has an INTEGER PRIMARY KEY and there are -** one or more auxiliary indexes, it is not possible to insert a NULL value -** into the IPK column. Attempting to do so results in an SQLITE_MISMATCH -** error. +** If the target database table has an INTEGER PRIMARY KEY, it is not +** possible to insert a NULL value into the IPK column. Attempting to +** do so results in an SQLITE_MISMATCH error. ** ** For each row to DELETE from the target database as part of the OTA ** update, the corresponding data_% table should contain a single record diff --git a/manifest b/manifest index 3f58be9e62..40f75346a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Prevent\sota\supdates\sfrom\sviolating\sNOT\sNULL\sconstraints.\sAdd\sa\scomment\sto\sthe\s"limitations"\ssection\sof\ssqlite3ota.h\ssaying\sthat\sCHECK\sconstraints\sare\snot\senforced. -D 2015-02-05T17:36:30.528 +C Change\sa\scomment\sin\ssqlite3ota.h\sto\smake\sit\sclear\sthat\sit\sis\snot\spossible\sto\sinsert\sa\sNULL\svalue\sinto\san\sINTEGER\sPRIMARY\sKEY\scolumn\susing\sota. +D 2015-02-05T17:46:19.663 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -126,7 +126,7 @@ F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt 78d4a9f78f567d4bf826cf0f02df6254902562ca F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 F ext/ota/ota1.test 719854e444dff2ead58ff6b62d8315954bd7762a -F ext/ota/ota10.test 69bded736f5dd1e07b96e5c51c081e2fd051cb12 +F ext/ota/ota10.test 85e0f6e7964db5007590c1b299e75211ed4240d4 F ext/ota/ota2.test 2829bc08ffbb71b605392a68fedfd554763356a7 F ext/ota/ota3.test a77efbce7723332eb688d2b28bf18204fc9614d7 F ext/ota/ota4.test 82434aa39c9acca6cd6317f6b0ab07b0ec6c2e7d @@ -137,7 +137,7 @@ F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd F ext/ota/sqlite3ota.c 6f4f4e07d48ad51b745a5d30ba601a6895fd15ba -F ext/ota/sqlite3ota.h 22423135fcdf7b242af8039b2ebf8437685e87c9 +F ext/ota/sqlite3ota.h b4c54c7df5d223f2ee40efa5ba363188daa3ad37 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1255,7 +1255,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 50ecdfc443b51e3569c6add2fba5132f959c61cb -R 39225d5cd1d4a3167fd9b3981c9a3ddd +P 74e073dd604142212f3d3e1931065d124daabd80 +R e48869d5f9b5af46af5b0a2d65cd95eb U dan -Z 95e5525c749de3fda07c16abd9286707 +Z 20d9f4ca66bd6d3a4217ce480de94529 diff --git a/manifest.uuid b/manifest.uuid index 189550920b..5e25ab7cca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -74e073dd604142212f3d3e1931065d124daabd80 \ No newline at end of file +a5e86bea4ad51bbb988a2a2215961706894f4a02 \ No newline at end of file From 2f8109cef1d8c5df3b3d7e7c6f879e20dfedb981 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 6 Feb 2015 00:31:45 +0000 Subject: [PATCH 057/116] Revise the way that the index structure for a WITHOUT ROWID table is discovered. FossilOrigin-Name: 7f10a0eaf1fedfa020cbd7019ec9342ffdc3b9b0 --- ext/ota/sqlite3ota.c | 21 +++++++++++++++++---- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pragma.c | 4 ---- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index dd742c4ab2..20b9ede75b 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -1002,13 +1002,26 @@ static char *otaWithoutRowidPK(sqlite3ota *p, OtaObjIter *pIter){ assert( pIter->zIdx==0 ); if( p->rc==SQLITE_OK ){ const char *zSep = "PRIMARY KEY("; - sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = (pIter->zTbl) */ + sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = */ int rc; /* sqlite3_finalize() return code */ - p->rc = prepareFreeAndCollectError(p->db, &pXInfo, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zTbl) + + p->rc = prepareFreeAndCollectError(p->db, &pXList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) ); - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){ + const char *zOrig = (const char*)sqlite3_column_text(pXList,3); + if( zOrig && strcmp(zOrig,"pk")==0 ){ + p->rc = prepareFreeAndCollectError(p->db, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", + sqlite3_column_text(pXList,1)) + ); + break; + } + } + sqlite3_finalize(pXList); + while( p->rc==SQLITE_OK && pXInfo && SQLITE_ROW==sqlite3_step(pXInfo) ){ if( sqlite3_column_int(pXInfo, 5) ){ /* int iCid = sqlite3_column_int(pXInfo, 0); */ const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); diff --git a/manifest b/manifest index 40f75346a0..a07f7ba90c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sa\scomment\sin\ssqlite3ota.h\sto\smake\sit\sclear\sthat\sit\sis\snot\spossible\sto\sinsert\sa\sNULL\svalue\sinto\san\sINTEGER\sPRIMARY\sKEY\scolumn\susing\sota. -D 2015-02-05T17:46:19.663 +C Revise\sthe\sway\sthat\sthe\sindex\sstructure\sfor\sa\sWITHOUT\sROWID\stable\sis\ndiscovered. +D 2015-02-06T00:31:45.481 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -136,7 +136,7 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c 6f4f4e07d48ad51b745a5d30ba601a6895fd15ba +F ext/ota/sqlite3ota.c bf417242a191617841cc1ab0815071b49444c9c8 F ext/ota/sqlite3ota.h b4c54c7df5d223f2ee40efa5ba363188daa3ad37 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -238,7 +238,7 @@ F src/parse.y c5d0d964f9ac023e8154cad512e54b0b6058e086 F src/pcache.c d210cf90d04365a74f85d21374dded65af67b0cb F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c 1e77432b40b7d3288327d9cdf399dcdfd2b6d3bf -F src/pragma.c 6e376554d9574416559101369fad2207c1e73299 +F src/pragma.c 8042d2b202140c49ffccb267aaa2012b50e337e4 F src/pragma.h d2f776d719d156544638fe3f87f9627d8e16222f F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 F src/printf.c 05edc41450d0eb2c05ef7db113bf32742ae65325 @@ -1255,7 +1255,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 74e073dd604142212f3d3e1931065d124daabd80 -R e48869d5f9b5af46af5b0a2d65cd95eb -U dan -Z 20d9f4ca66bd6d3a4217ce480de94529 +P a5e86bea4ad51bbb988a2a2215961706894f4a02 +R cf9fe799f99a90eb0a6f043505cb4c1a +U drh +Z c0c25c22927eb0667b2fe51c91fe6714 diff --git a/manifest.uuid b/manifest.uuid index 5e25ab7cca..2cf1a922f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a5e86bea4ad51bbb988a2a2215961706894f4a02 \ No newline at end of file +7f10a0eaf1fedfa020cbd7019ec9342ffdc3b9b0 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 3bb330c899..d4c5ba51dc 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1119,10 +1119,6 @@ void sqlite3Pragma( Index *pIdx; Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); - if( pIdx==0 ){ - pTab = sqlite3FindTable(db, zRight, zDb); - if( pTab && !HasRowid(pTab) ) pIdx = sqlite3PrimaryKeyIndex(pTab); - } if( pIdx ){ int i; int mx = pPragma->iArg ? pIdx->nColumn : pIdx->nKeyCol; From dd0ad3785b8124abdd23e9b97be8b8b971b4fdaa Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 7 Feb 2015 19:17:36 +0000 Subject: [PATCH 058/116] Remove "PRAGMA pager_ota_mode". FossilOrigin-Name: 8ac58e46782bd6b81c06fdf1cb5b316b8a4e1ddf --- ext/ota/README.txt | 26 +- ext/ota/ota1.test | 1 + ext/ota/ota2.test | 74 ----- ext/ota/ota4.test | 128 --------- ext/ota/ota6.test | 6 +- ext/ota/sqlite3ota.c | 647 +++++++++++++++++++++++++++++++++++++++++-- manifest | 37 +-- manifest.uuid | 2 +- src/pager.c | 65 +---- src/pager.h | 2 - src/pragma.c | 44 --- src/pragma.h | 43 ++- src/wal.c | 4 +- tool/mkpragmatab.tcl | 3 - 14 files changed, 678 insertions(+), 404 deletions(-) delete mode 100644 ext/ota/ota2.test delete mode 100644 ext/ota/ota4.test diff --git a/ext/ota/README.txt b/ext/ota/README.txt index 943d1822af..42ac29064c 100644 --- a/ext/ota/README.txt +++ b/ext/ota/README.txt @@ -5,21 +5,8 @@ User documentation is in sqlite3ota.h. SQLite Hacks ------------ -1) PRAGMA ota_mode: - This is a new flag pragma. If the flag is set: - - * INSERT/DELETE/UPDATE commands are prevented from updating any but the main - b-tree for each table (the PK index for WITHOUT ROWID tables or the - rowid b-tree for others). - - * The above statements do not check UNIQUE constraints - except those enforced - by the main b-tree. - - * All non-temporary triggers are disabled. - - -2) PRAGMA pager_ota_mode=1: +1) PRAGMA pager_ota_mode=1: This pragma sets a flag on the pager associated with the main database only. In a zipvfs system, this pragma is intercepted by zipvfs and the flag is set @@ -51,7 +38,7 @@ SQLite Hacks pager_ota_mode connections. If two or more such connections attempt to write simultaneously, the results are undefined. -3) PRAGMA pager_ota_mode=2: +2) PRAGMA pager_ota_mode=2: The pager_ota_mode pragma may also be set to 2 if the main database is open in WAL mode. This prevents SQLite from checkpointing the wal file as part @@ -60,14 +47,7 @@ SQLite Hacks The effects of setting pager_ota_mode=2 if the db is not in WAL mode are undefined. -4) sqlite3_index_writer() - - This new API function is used to create VMs that can insert or delete entries - from individual index b-trees within the database. The VMs apply affinities - and check that UNIQUE constraints are not violated before updating index - b-trees. - -5) sqlite3_ckpt_open/step/close() +3) sqlite3_ckpt_open/step/close() API for performing (and resuming) incremental checkpoints. diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index afb5fd5865..265a3d2a2f 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -187,6 +187,7 @@ foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { do_test 1.$tn2.$tn.1 { create_ota1 ota.db + breakpoint $cmd test.db ota.db } {SQLITE_DONE} diff --git a/ext/ota/ota2.test b/ext/ota/ota2.test deleted file mode 100644 index 3c273e348a..0000000000 --- a/ext/ota/ota2.test +++ /dev/null @@ -1,74 +0,0 @@ -# 2014 August 30 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# - -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} -source $testdir/tester.tcl -set ::testprefix ota2 - -forcedelete {*}[glob -nocomplain test.db?*] - -do_execsql_test 1.0 { - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, 2); -} {} -do_test 1.1 { glob test.db* } {test.db} - -do_execsql_test 1.2 { - PRAGMA pager_ota_mode = 1; - INSERT INTO t1 VALUES(3, 4); - INSERT INTO t1 VALUES(5, 6); - SELECT * FROM t1; -} {1 2 3 4 5 6} - -do_test 1.3 { lsort [glob test.db*] } {test.db test.db-oal} - -do_test 1.4 { - sqlite3 db2 test.db - db2 eval { SELECT * FROM t1 } -} {1 2} - -do_test 1.5 { - catchsql { INSERT INTO t1 VALUES(7, 8) } db2 -} {1 {database is locked}} - -db2 close -db close - -sqlite3 db test.db -do_execsql_test 1.6 { - PRAGMA pager_ota_mode = 1; - SELECT * FROM t1; -} {1 2 3 4 5 6} - -do_execsql_test 1.7 { - INSERT INTO t1 VALUES(7,8); - SELECT * FROM t1; -} {1 2 3 4 5 6 7 8} - -db close -sqlite3 db2 test.db - -do_test 1.8 { - execsql { BEGIN; SELECT * FROM t1 } db2 -} {1 2} -do_test 1.9 { - file rename test.db-oal test.db-wal - execsql { SELECT * FROM t1 } db2 -} {1 2} -do_test 1.10 { - execsql { COMMIT; SELECT * FROM t1 } db2 -} {1 2 3 4 5 6 7 8} - - -finish_test diff --git a/ext/ota/ota4.test b/ext/ota/ota4.test deleted file mode 100644 index a12cbd8b7b..0000000000 --- a/ext/ota/ota4.test +++ /dev/null @@ -1,128 +0,0 @@ -# 2014 August 30 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test some properties of the pager_ota_mode and ota_mode pragmas. -# - -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} -source $testdir/tester.tcl -set ::testprefix ota4 - -#------------------------------------------------------------------------- -# The following tests aim to verify some properties of the pager_ota_mode -# pragma: -# -# 1. Cannot set the pager_ota_mode flag on a WAL mode database. -# -# 2. Or if there is an open read transaction. -# -# 3. Cannot start a transaction with pager_ota_mode set if there -# is a WAL file in the file-system. -# -# 4. Or if the wal-mode flag is set in the database file header. -# -# 5. Cannot open a transaction with pager_ota_mode set if the database -# file has been modified by a rollback mode client since the *-oal -# file was started. -# - -do_execsql_test 1.1.1 { - PRAGMA journal_mode = wal; - SELECT * FROM sqlite_master; -} {wal} -do_catchsql_test 1.1.2 { - PRAGMA pager_ota_mode = 1 -} {1 {cannot set pager_ota_mode in wal mode}} - - -do_execsql_test 1.2.1 { - PRAGMA journal_mode = delete; - BEGIN; - SELECT * FROM sqlite_master; -} {delete} -do_catchsql_test 1.2.2 { - PRAGMA pager_ota_mode = 1 -} {1 {cannot set pager_ota_mode with open transaction}} -do_execsql_test 1.2.3 { - COMMIT; -} {} - - -do_execsql_test 1.3.1 { - PRAGMA journal_mode = wal; - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, 2); -} {wal} -do_test 1.3.2 { - forcecopy test.db-wal test.db-bak - execsql { - PRAGMA journal_mode = delete; - PRAGMA pager_ota_mode = 1; - } - forcecopy test.db-bak test.db-wal - catchsql { - SELECT * FROM sqlite_master - } -} {1 {unable to open database file}} - -do_test 1.4.1 { - db close - forcedelete test.db-wal test.db-oal - sqlite3 db test.db - execsql { - PRAGMA journal_mode = wal; - PRAGMA pager_ota_mode = 1; - } - catchsql { - SELECT * FROM sqlite_master; - } -} {1 {unable to open database file}} - -do_test 1.5.1 { - forcedelete test.db-oal - reset_db - execsql { - PRAGMA journal_mode = delete; - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, 2); - } - execsql { - PRAGMA pager_ota_mode = 1; - INSERT INTO t1 VALUES(3, 4); - } - db close - sqlite3 db test.db - execsql { - SELECT * FROM t1; - } -} {1 2} -do_execsql_test 1.5.2 { - PRAGMA pager_ota_mode = 1; - SELECT * FROM t1; - INSERT INTO t1 VALUES(5, 6); -} {1 2 3 4} -do_test 5.3 { - db close - sqlite3 db test.db - execsql { - INSERT INTO t1 VALUES(7, 8); - SELECT * FROM t1; - } -} {1 2 7 8} -do_catchsql_test 1.5.4 { - PRAGMA pager_ota_mode = 1; - SELECT * FROM t1; -} {1 {database is locked}} - -finish_test - diff --git a/ext/ota/ota6.test b/ext/ota/ota6.test index f15873629c..8027f36c5f 100644 --- a/ext/ota/ota6.test +++ b/ext/ota/ota6.test @@ -50,7 +50,7 @@ for {set nStep 1} {$nStep < 7} {incr nStep} { setup_test sqlite3ota ota test.db ota.db for {set i 0} {$i<$nStep} {incr i} {ota step} - + ota close sqlite3 db test.db execsql { INSERT INTO t1 VALUES(5, 'hello') } @@ -65,7 +65,7 @@ for {set nStep 1} {$nStep < 7} {incr nStep} { } {1 0} do_test 1.$nStep.4 { list [catch { ota close } msg] $msg - } {1 {SQLITE_BUSY - database is locked}} + } {1 {SQLITE_BUSY - database modified during ota update}} } for {set nStep 7} {$nStep < 8} {incr nStep} { @@ -122,7 +122,5 @@ for {set nStep 8} {$nStep < 20} {incr nStep} { } - finish_test - diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 20b9ede75b..2e23d5658c 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -62,6 +62,9 @@ ** Valid if STAGE==3. The blob to pass to sqlite3ckpt_start() to resume ** the incremental checkpoint. ** +** OTA_STATE_COOKIE: +** Valid if STAGE==1. The current change-counter cookie value in the +** target db file. */ #define OTA_STATE_STAGE 1 #define OTA_STATE_TBL 2 @@ -69,6 +72,7 @@ #define OTA_STATE_ROW 4 #define OTA_STATE_PROGRESS 5 #define OTA_STATE_CKPT 6 +#define OTA_STATE_COOKIE 7 #define OTA_STAGE_OAL 1 #define OTA_STAGE_COPY 2 @@ -166,8 +170,13 @@ struct sqlite3ota { int nProgress; /* Rows processed for all objects */ OtaObjIter objiter; /* Iterator for skipping through tbl/idx */ sqlite3_ckpt *pCkpt; /* Incr-checkpoint handle */ + sqlite3_vfs *pVfs; /* Special ota VFS object */ + unsigned int iCookie; }; +static void otaCreateVfs(sqlite3ota*, const char*); +static void otaDeleteVfs(sqlite3ota*); + /* ** Prepare the SQL statement in buffer zSql against database handle db. ** If successful, set *ppStmt to point to the new statement and return @@ -704,6 +713,19 @@ static char *otaMPrintf(sqlite3ota *p, const char *zFmt, ...){ return zSql; } +static void *otaMalloc(sqlite3ota *p, int nByte){ + void *pRet = 0; + if( p->rc==SQLITE_OK ){ + pRet = sqlite3_malloc(nByte); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + /* ** This function constructs and returns a pointer to a nul-terminated ** string containing some SQL clause or list based on one or more of the @@ -1434,11 +1456,11 @@ static int otaGetUpdateStmt( ** error occurs, leave an error code and message in the OTA handle. */ static void otaOpenDatabase(sqlite3ota *p){ + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; assert( p->rc==SQLITE_OK ); - sqlite3_close(p->db); - p->db = 0; + assert( p->db==0 ); - p->rc = sqlite3_open(p->zTarget, &p->db); + p->rc = sqlite3_open_v2(p->zTarget, &p->db, flags, p->pVfs->zName); if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); } @@ -1506,8 +1528,10 @@ static void otaMoveOalFile(sqlite3ota *p){ /* Re-open the databases. */ otaObjIterFinalize(&p->objiter); - otaOpenDatabase(p); + sqlite3_close(p->db); + p->db = 0; p->eStage = OTA_STAGE_CKPT; + otaOpenDatabase(p); } sqlite3_free(zWal); @@ -1795,13 +1819,15 @@ static void otaSaveTransactionState(sqlite3ota *p){ "(%d, %Q), " "(%d, %d), " "(%d, %lld), " - "(%d, ?) ", + "(%d, ?), " + "(%d, %lld) ", OTA_STATE_STAGE, p->eStage, OTA_STATE_TBL, p->objiter.zTbl, OTA_STATE_IDX, p->objiter.zIdx, OTA_STATE_ROW, p->nStep, OTA_STATE_PROGRESS, p->nProgress, - OTA_STATE_CKPT + OTA_STATE_CKPT, + OTA_STATE_COOKIE, (sqlite3_int64)p->iCookie ) ); assert( pInsert==0 || rc==SQLITE_OK ); @@ -1896,6 +1922,19 @@ static OtaState *otaLoadState(sqlite3ota *p){ ); break; + case OTA_STATE_COOKIE: + /* At this point (p->iCookie) contains the value of the change-counter + ** cookie (the thing that gets incremented when a transaction is + ** committed in rollback mode) currently stored on page 1 of the + ** database file. */ + if( pRet->eStage==OTA_STAGE_OAL + && p->iCookie!=(unsigned int)sqlite3_column_int64(pStmt, 1) + ){ + rc = SQLITE_BUSY; + p->zErrmsg = sqlite3_mprintf("database modified during ota update"); + } + break; + default: rc = SQLITE_CORRUPT; break; @@ -1965,13 +2004,18 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ if( p ){ OtaState *pState = 0; - /* Open the target database */ + /* Create the custom VFS */ memset(p, 0, sizeof(sqlite3ota)); - p->zTarget = (char*)&p[1]; - memcpy(p->zTarget, zTarget, nTarget+1); - p->zOta = &p->zTarget[nTarget+1]; - memcpy(p->zOta, zOta, nOta+1); - otaOpenDatabase(p); + otaCreateVfs(p, 0); + + /* Open the target database */ + if( p->rc==SQLITE_OK ){ + p->zTarget = (char*)&p[1]; + memcpy(p->zTarget, zTarget, nTarget+1); + p->zOta = &p->zTarget[nTarget+1]; + memcpy(p->zOta, zOta, nOta+1); + otaOpenDatabase(p); + } /* If it has not already been created, create the ota_state table */ if( p->rc==SQLITE_OK ){ @@ -1997,7 +2041,6 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ if( p->eStage==OTA_STAGE_OAL ){ const char *zScript = "PRAGMA journal_mode=off;" - "PRAGMA pager_ota_mode=1;" "BEGIN IMMEDIATE;" ; p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); @@ -2081,13 +2124,10 @@ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); } - if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_CKPT ){ - p->rc = sqlite3_exec(p->db, "PRAGMA pager_ota_mode=2", 0, 0, &p->zErrmsg); - } - - /* Close the open database handle */ + /* Close the open database handle and VFS object. */ if( p->pCkpt ) sqlite3_ckpt_close(p->pCkpt, 0, 0); sqlite3_close(p->db); + otaDeleteVfs(p); otaEditErrmsg(p); rc = p->rc; @@ -2109,6 +2149,577 @@ sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta){ return pOta->nProgress; } +/************************************************************************** +** Beginning of OTA VFS shim methods. The VFS shim modifies the behaviour +** of a standard VFS in the following ways: +** +** TODO +*/ + +#if 0 +#define OTA_FILE_VANILLA 0 +#define OTA_FILE_TARGET_DB 1 +#define OTA_FILE_TARGET_WAL 2 +#endif + +typedef struct ota_file ota_file; +typedef struct ota_vfs ota_vfs; + +struct ota_file { + sqlite3_file base; /* sqlite3_file methods */ + sqlite3_file *pReal; /* Underlying file handle */ + ota_vfs *pOtaVfs; /* Pointer to the ota_vfs object */ + + int nShm; /* Number of entries in apShm[] array */ + char **apShm; /* Array of mmap'd *-shm regions */ + char *zFilename; /* Filename for *-oal file only */ +}; + +struct ota_vfs { + sqlite3_vfs base; /* ota VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3ota *pOta; + ota_file *pTargetDb; /* Target database file descriptor */ + const char *zTargetDb; /* Path that pTargetDb was opened with */ +}; + +/* +** Close an ota file. +*/ +static int otaVfsClose(sqlite3_file *pFile){ + ota_file *p = (ota_file*)pFile; + ota_vfs *pOtaVfs = p->pOtaVfs; + int rc; + int i; + + /* Free the contents of the apShm[] array. And the array itself. */ + for(i=0; inShm; i++){ + sqlite3_free(p->apShm[i]); + } + sqlite3_free(p->apShm); + p->apShm = 0; + sqlite3_free(p->zFilename); + + if( p==pOtaVfs->pTargetDb ){ + pOtaVfs->pTargetDb = 0; + pOtaVfs->zTargetDb = 0; + } + + rc = p->pReal->pMethods->xClose(p->pReal); + return rc; +} + + +/* +** Read and return an unsigned 32-bit big-endian integer from the buffer +** passed as the only argument. +*/ +static unsigned int otaGetU32(unsigned char *aBuf){ + return ((unsigned int)aBuf[0] << 24) + + ((unsigned int)aBuf[1] << 16) + + ((unsigned int)aBuf[2] << 8) + + ((unsigned int)aBuf[3]); +} + +/* +** Read data from an otaVfs-file. +*/ +static int otaVfsRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ota_file *p = (ota_file*)pFile; + ota_vfs *pOtaVfs = p->pOtaVfs; + int rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK && p==pOtaVfs->pTargetDb && iOfst==0 ){ + unsigned char *pBuf = (unsigned char*)zBuf; + assert( iAmt>=100 ); + pOtaVfs->pOta->iCookie = otaGetU32(&pBuf[24]); + } + return rc; +} + +/* +** Write data to an otaVfs-file. +*/ +static int otaVfsWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ota_file *p = (ota_file*)pFile; + ota_vfs *pOtaVfs = p->pOtaVfs; + int rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK && p==pOtaVfs->pTargetDb && iOfst==0 ){ + unsigned char *pBuf = (unsigned char*)zBuf; + assert( iAmt>=100 ); + pOtaVfs->pOta->iCookie = otaGetU32(&pBuf[24]); + } + return rc; +} + +/* +** Truncate an otaVfs-file. +*/ +static int otaVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){ + ota_file *p = (ota_file*)pFile; + return p->pReal->pMethods->xTruncate(p->pReal, size); +} + +/* +** Sync an otaVfs-file. +*/ +static int otaVfsSync(sqlite3_file *pFile, int flags){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xSync(p->pReal, flags); +} + +/* +** Return the current file-size of an otaVfs-file. +*/ +static int otaVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xFileSize(p->pReal, pSize); +} + +/* +** Lock an otaVfs-file. +*/ +static int otaVfsLock(sqlite3_file *pFile, int eLock){ + ota_file *p = (ota_file*)pFile; + ota_vfs *pOtaVfs = p->pOtaVfs; + int rc = SQLITE_OK; + int eStage = pOtaVfs->pOta->eStage; + + if( pOtaVfs->pTargetDb==p + && (eStage==OTA_STAGE_OAL || eStage==OTA_STAGE_CKPT) + && eLock==SQLITE_LOCK_EXCLUSIVE + ){ + /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this + ** prevents it from checkpointing the database from sqlite3_close(). */ + rc = SQLITE_BUSY; + }else{ + rc = p->pReal->pMethods->xLock(p->pReal, eLock); + } + + return rc; +} + +/* +** Unlock an otaVfs-file. +*/ +static int otaVfsUnlock(sqlite3_file *pFile, int eLock){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xUnlock(p->pReal, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on an otaVfs-file. +*/ +static int otaVfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut); +} + +/* +** File control method. For custom operations on an otaVfs-file. +*/ +static int otaVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xFileControl(p->pReal, op, pArg); +} + +/* +** Return the sector-size in bytes for an otaVfs-file. +*/ +static int otaVfsSectorSize(sqlite3_file *pFile){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xSectorSize(p->pReal); +} + +/* +** Return the device characteristic flags supported by an otaVfs-file. +*/ +static int otaVfsDeviceCharacteristics(sqlite3_file *pFile){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xDeviceCharacteristics(p->pReal); +} + +/* +** Shared-memory methods are all pass-thrus. +*/ +static int otaVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ + ota_file *p = (ota_file*)pFile; + ota_vfs *pOtaVfs = p->pOtaVfs; + int rc = SQLITE_OK; + +#ifdef SQLITE_AMALGAMATION + assert( WAL_WRITE_CKPT==1 ); +#endif + + if( pOtaVfs->pTargetDb==p && pOtaVfs->pOta->eStage==OTA_STAGE_OAL ){ + /* Magic number 1 is the WAL_WRITE_CKPT lock. Preventing SQLite from + ** taking this lock also prevents any checkpoints from occurring. + ** todo: really, it's not clear why this might occur, as + ** wal_autocheckpoint ought to be turned off. */ + if( ofst==1 && n==1 ) rc = SQLITE_BUSY; + }else{ + assert( p->nShm==0 ); + return p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags); + } + + return rc; +} + +static int otaVfsShmMap( + sqlite3_file *pFile, + int iRegion, + int szRegion, + int isWrite, + void volatile **pp +){ + ota_file *p = (ota_file*)pFile; + ota_vfs *pOtaVfs = p->pOtaVfs; + int rc = SQLITE_OK; + + /* If not in OTA_STAGE_OAL, allow this call to pass through. Or, if this + ** ota is in the OTA_STAGE_OAL state, use heap memory for *-shm space + ** instead of a file on disk. */ + if( pOtaVfs->pTargetDb==p && pOtaVfs->pOta->eStage==OTA_STAGE_OAL ){ + if( iRegion<=p->nShm ){ + int nByte = (iRegion+1) * sizeof(char*); + char **apNew = (char**)sqlite3_realloc(p->apShm, nByte); + if( apNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm)); + p->apShm = apNew; + p->nShm = iRegion+1; + } + } + + if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){ + char *pNew = (char*)sqlite3_malloc(szRegion); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + p->apShm[iRegion] = pNew; + } + } + + if( rc==SQLITE_OK ){ + *pp = p->apShm[iRegion]; + }else{ + *pp = 0; + } + }else{ + assert( p->apShm==0 ); + rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp); + } + + return rc; +} + +/* +** Memory barrier. +*/ +static void otaVfsShmBarrier(sqlite3_file *pFile){ + ota_file *p = (ota_file *)pFile; + p->pReal->pMethods->xShmBarrier(p->pReal); +} + +static int otaVfsShmUnmap(sqlite3_file *pFile, int delFlag){ + ota_file *p = (ota_file*)pFile; + ota_vfs *pOtaVfs = p->pOtaVfs; + int rc = SQLITE_OK; + + if( pOtaVfs->pTargetDb==p && pOtaVfs->pOta->eStage==OTA_STAGE_OAL ){ + /* no-op */ + }else{ + rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag); + } + return rc; +} + + +static int otaVfsIswal(ota_vfs *pOtaVfs, const char *zPath){ + int nPath = strlen(zPath); + int nTargetDb = strlen(pOtaVfs->zTargetDb); + return ( nPath==(nTargetDb+4) + && 0==memcmp(zPath, pOtaVfs->zTargetDb, nTargetDb) + && 0==memcmp(&zPath[nTargetDb], "-wal", 4) + ); +} + + +/* +** Open an ota file handle. +*/ +static int otaVfsOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + static sqlite3_io_methods otavfs_io_methods = { + 2, /* iVersion */ + otaVfsClose, /* xClose */ + otaVfsRead, /* xRead */ + otaVfsWrite, /* xWrite */ + otaVfsTruncate, /* xTruncate */ + otaVfsSync, /* xSync */ + otaVfsFileSize, /* xFileSize */ + otaVfsLock, /* xLock */ + otaVfsUnlock, /* xUnlock */ + otaVfsCheckReservedLock, /* xCheckReservedLock */ + otaVfsFileControl, /* xFileControl */ + otaVfsSectorSize, /* xSectorSize */ + otaVfsDeviceCharacteristics, /* xDeviceCharacteristics */ + otaVfsShmMap, /* xShmMap */ + otaVfsShmLock, /* xShmLock */ + otaVfsShmBarrier, /* xShmBarrier */ + otaVfsShmUnmap /* xShmUnmap */ + }; + ota_vfs *pOtaVfs = (ota_vfs*)pVfs; + sqlite3_vfs *pRealVfs = pOtaVfs->pRealVfs; + sqlite3ota *p = pOtaVfs->pOta; + ota_file *pFd = (ota_file *)pFile; + int rc = SQLITE_OK; + const char *zOpen = zName; + + memset(pFd, 0, sizeof(ota_file)); + pFd->pReal = (sqlite3_file*)&pFd[1]; + pFd->pOtaVfs = pOtaVfs; + + if( zName && p->eStage==OTA_STAGE_OAL && otaVfsIswal(pOtaVfs, zName) ){ + char *zCopy = otaStrndup(zName, -1, &rc); + if( zCopy ){ + int nCopy = strlen(zCopy); + zCopy[nCopy-3] = 'o'; + zOpen = (const char*)(pFd->zFilename = zCopy); + } + } + + if( rc==SQLITE_OK ){ + rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, flags, pOutFlags); + } + if( pFd->pReal->pMethods ){ + pFile->pMethods = &otavfs_io_methods; + if( pOtaVfs->pTargetDb==0 ){ + /* This is the target db file. */ + assert( (flags & SQLITE_OPEN_MAIN_DB) ); + assert( zOpen==zName ); + pOtaVfs->pTargetDb = pFd; + pOtaVfs->zTargetDb = zName; + } + } + + return rc; +} + +/* +** Delete the file located at zPath. +*/ +static int otaVfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xDelete(pRealVfs, zPath, dirSync); +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int otaVfsAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + ota_vfs *pOtaVfs = (ota_vfs*)pVfs; + sqlite3_vfs *pRealVfs = pOtaVfs->pRealVfs; + int rc; + + rc = pRealVfs->xAccess(pRealVfs, zPath, flags, pResOut); + + if( rc==SQLITE_OK + && flags==SQLITE_ACCESS_EXISTS + && pOtaVfs->pOta->eStage==OTA_STAGE_OAL + && otaVfsIswal(pOtaVfs, zPath) + ){ + if( *pResOut ){ + rc = SQLITE_CANTOPEN; + }else{ + *pResOut = 1; + } + } + + return rc; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (DEVSYM_MAX_PATHNAME+1) bytes. +*/ +static int otaVfsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xFullPathname(pRealVfs, zPath, nOut, zOut); +} + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *otaVfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xDlOpen(pRealVfs, zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void otaVfsDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + pRealVfs->xDlError(pRealVfs, nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*otaVfsDlSym( + sqlite3_vfs *pVfs, + void *pArg, + const char *zSym +))(void){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xDlSym(pRealVfs, pArg, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void otaVfsDlClose(sqlite3_vfs *pVfs, void *pHandle){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xDlClose(pRealVfs, pHandle); +} +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int otaVfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xRandomness(pRealVfs, nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int otaVfsSleep(sqlite3_vfs *pVfs, int nMicro){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xSleep(pRealVfs, nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int otaVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xCurrentTime(pRealVfs, pTimeOut); +} + +static int otaVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return 0; +} + +static void otaCreateVfs(sqlite3ota *p, const char *zParent){ + + /* Template for VFS */ + static sqlite3_vfs vfs_template = { + 1, /* iVersion */ + 0, /* szOsFile */ + 0, /* mxPathname */ + 0, /* pNext */ + 0, /* zName */ + 0, /* pAppData */ + otaVfsOpen, /* xOpen */ + otaVfsDelete, /* xDelete */ + otaVfsAccess, /* xAccess */ + otaVfsFullPathname, /* xFullPathname */ + + otaVfsDlOpen, /* xDlOpen */ + otaVfsDlError, /* xDlError */ + otaVfsDlSym, /* xDlSym */ + otaVfsDlClose, /* xDlClose */ + + otaVfsRandomness, /* xRandomness */ + otaVfsSleep, /* xSleep */ + otaVfsCurrentTime, /* xCurrentTime */ + otaVfsGetLastError, /* xGetLastError */ + 0, /* xCurrentTimeInt64 (version 2) */ + 0, 0, 0 /* Unimplemented version 3 methods */ + }; + + sqlite3_vfs *pParent; /* Parent VFS */ + ota_vfs *pNew = 0; /* Newly allocated VFS */ + + assert( p->rc==SQLITE_OK ); + pParent = sqlite3_vfs_find(zParent); + if( pParent==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("no such vfs: %s", zParent); + }else{ + int nByte = sizeof(ota_vfs) + 64; + pNew = (ota_vfs*)otaMalloc(p, nByte); + } + + if( pNew ){ + int rnd; + char *zName; + memcpy(&pNew->base, &vfs_template, sizeof(sqlite3_vfs)); + pNew->base.mxPathname = pParent->mxPathname; + pNew->base.szOsFile = sizeof(ota_file) + pParent->szOsFile; + pNew->pOta = p; + pNew->pRealVfs = pParent; + + /* Give the new VFS a unique name */ + sqlite3_randomness(sizeof(int), (void*)&rnd); + pNew->base.zName = (const char*)(zName = (char*)&pNew[1]); + sprintf(zName, "ota_vfs_%d", rnd); + + /* Register the new VFS (not as the default) */ + assert( p->rc==SQLITE_OK ); + p->rc = sqlite3_vfs_register(&pNew->base, 0); + if( p->rc ){ + p->zErrmsg = sqlite3_mprintf("error in sqlite3_vfs_register()"); + sqlite3_free(pNew); + }else{ + p->pVfs = &pNew->base; + } + } +} + +static void otaDeleteVfs(sqlite3ota *p){ + if( p->pVfs ){ + sqlite3_vfs_unregister(p->pVfs); + sqlite3_free(p->pVfs); + p->pVfs = 0; + } +} + /**************************************************************************/ diff --git a/manifest b/manifest index 4d69256b43..93b2368df6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\scommand-line\sshell\senhancements\sfrom\strunk. -D 2015-02-06T15:03:45.342 +C Remove\s"PRAGMA\spager_ota_mode". +D 2015-02-07T19:17:36.157 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -123,20 +123,18 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/ota/README.txt 78d4a9f78f567d4bf826cf0f02df6254902562ca +F ext/ota/README.txt 2ce4ffbb0aaa6731b041c27a7359f9a5f1c69152 F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 -F ext/ota/ota1.test 719854e444dff2ead58ff6b62d8315954bd7762a +F ext/ota/ota1.test e6b64d6ffb23dcae72386da153626b40566a69e9 F ext/ota/ota10.test 85e0f6e7964db5007590c1b299e75211ed4240d4 -F ext/ota/ota2.test 2829bc08ffbb71b605392a68fedfd554763356a7 F ext/ota/ota3.test a77efbce7723332eb688d2b28bf18204fc9614d7 -F ext/ota/ota4.test 82434aa39c9acca6cd6317f6b0ab07b0ec6c2e7d F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb -F ext/ota/ota6.test 82f1f757ec9b2ad07d6de4060b8e3ba8e44dfdd3 +F ext/ota/ota6.test 1fbba5fd46e3e0bfa5ae1d0caf9da27d15cb7cdf F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c bf417242a191617841cc1ab0815071b49444c9c8 +F ext/ota/sqlite3ota.c 7015400382d1d6655626046f2c1763634dd8a168 F ext/ota/sqlite3ota.h b4c54c7df5d223f2ee40efa5ba363188daa3ad37 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -232,14 +230,14 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c aefeaf915aaef9f81aa2645e0d5d06fa1bd83beb F src/os_win.c 8223e7db5b7c4a81d8b161098ac3959400434cdb F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 90b164ac8fefed940cd50fad6938cd18b55af8f3 -F src/pager.h 19d83e2782fe978976cb1acf474d09d9a6124ac3 +F src/pager.c 46bc7849b02c51e13f0165fa6d6faa452e91a957 +F src/pager.h 20954a3fa1bbf05d39063d94e789ad9efd15e5d1 F src/parse.y c5d0d964f9ac023e8154cad512e54b0b6058e086 F src/pcache.c d210cf90d04365a74f85d21374dded65af67b0cb F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c 1e77432b40b7d3288327d9cdf399dcdfd2b6d3bf -F src/pragma.c 8042d2b202140c49ffccb267aaa2012b50e337e4 -F src/pragma.h d2f776d719d156544638fe3f87f9627d8e16222f +F src/pragma.c ea0be138a99784b14e87bd4522fea40e7b979e9c +F src/pragma.h 09c89bca58e9a44de2116cc8272b8d454657129f F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 F src/printf.c 05edc41450d0eb2c05ef7db113bf32742ae65325 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 @@ -318,7 +316,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 6d64c5448b64851b99931ede980addc3af70d5e2 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 -F src/wal.c 735d081f736fd7fecbf8f2aa213484e641ba35ff +F src/wal.c 7a8a4e7a40d693d44dbfc4d1f2bcb7e2b620f530 F src/wal.h 0d3ba0c3f1b4c25796cb213568a84b9f9063f465 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c d46de821bc604a4fd36fa3928c086950e91aafb1 @@ -1219,7 +1217,7 @@ F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl a5cb9b20ad7abb2ffd519c85f1f8f99bcbfa6823 +F tool/mkpragmatab.tcl 94f196c9961e0ca3513e29f57125a3197808be2d F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 9ef48e1748dce7b844f67e2450ff9dfeb0fb4ab5 F tool/mksqlite3c.tcl 6b8e572a90eb4e0086e3ba90d88b76c085919863 @@ -1255,7 +1253,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7f10a0eaf1fedfa020cbd7019ec9342ffdc3b9b0 0f65a7e2e09f801b66897479d501607caeae4abf -R 1ef390b8e8775fda79ba88277a4a2044 -U drh -Z f716caa3559c7f56ae8a7035846f96fb +P c3931db560ab4a2601c7f7318fb02c8d5e6862b1 +R 8cb076bc2c2cc0b3b28786011832c69d +T *branch * ota-update-no-pager_ota_mode +T *sym-ota-update-no-pager_ota_mode * +T -sym-ota-update * +U dan +Z b552e61eecb5b9b2a0bf7b3905abae17 diff --git a/manifest.uuid b/manifest.uuid index 67420b8cdc..78b4daf1f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3931db560ab4a2601c7f7318fb02c8d5e6862b1 \ No newline at end of file +8ac58e46782bd6b81c06fdf1cb5b316b8a4e1ddf \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 7cc35505f6..4b8e31d513 100644 --- a/src/pager.c +++ b/src/pager.c @@ -642,9 +642,6 @@ struct Pager { u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ -#ifdef SQLITE_ENABLE_OTA - u8 otaMode; /* Non-zero if in ota_mode */ -#endif /************************************************************************** ** The following block contains those class members that change during @@ -720,16 +717,6 @@ struct Pager { #endif }; -/* -** Return the value of the pager otaMode flag (0, 1 or 2). Or, if -** SQLITE_ENABLE_OTA is not defined, return constant value 0. -*/ -#ifdef SQLITE_ENABLE_OTA -# define PagerOtaMode(pPager) ((pPager)->otaMode) -#else -# define PagerOtaMode(pPager) 0 -#endif - /* ** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains ** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS @@ -2052,7 +2039,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; } - if( !pPager->exclusiveMode && !PagerOtaMode(pPager) + if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ rc2 = pagerUnlockDb(pPager, SHARED_LOCK); @@ -4007,9 +3994,7 @@ int sqlite3PagerClose(Pager *pPager){ /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL - sqlite3WalClose(pPager->pWal, - pPager->ckptSyncFlags, pPager->pageSize, (PagerOtaMode(pPager)?0:pTmp) - ); + sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp); pPager->pWal = 0; #endif pager_reset(pPager); @@ -5210,11 +5195,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** mode. Otherwise, the following function call is a no-op. */ rc = pagerOpenWalIfPresent(pPager); - if( rc==SQLITE_OK && PagerOtaMode(pPager) ){ - int nWal = sqlite3Strlen30(pPager->zWal); - pPager->zWal[nWal-3] = 'o'; - rc = pagerOpenWalInternal(pPager, 0); - } #ifndef SQLITE_OMIT_WAL assert( pPager->pWal==0 || rc==SQLITE_OK ); @@ -5224,17 +5204,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( pagerUseWal(pPager) ){ assert( rc==SQLITE_OK ); rc = pagerBeginReadTransaction(pPager); - if( rc==SQLITE_OK && PagerOtaMode(pPager)==1 ){ - rc = sqlite3WalCheckSalt(pPager->pWal, pPager->fd); - if( rc!=SQLITE_OK ){ - sqlite3WalClose(pPager->pWal, 0, 0, 0); - pPager->pWal = 0; - }else{ -#ifdef SQLITE_ENABLE_OTA - pPager->otaMode = 2; -#endif - } - } } if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ @@ -7129,7 +7098,7 @@ void sqlite3PagerClearCache(Pager *pPager){ */ int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; - if( pPager->pWal && PagerOtaMode(pPager)==0 ){ + if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, eMode, (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, @@ -7197,7 +7166,7 @@ static int pagerOpenWal(Pager *pPager){ */ if( rc==SQLITE_OK ){ rc = sqlite3WalOpen(pPager->pVfs, - pPager->fd, pPager->zWal, pPager->exclusiveMode || PagerOtaMode(pPager), + pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); } @@ -7263,7 +7232,6 @@ int sqlite3PagerOpenWal( Pager *pPager, /* Pager object */ int *pbOpen /* OUT: Set to true if call is a no-op */ ){ - if( PagerOtaMode(pPager) ) return SQLITE_CANTOPEN_BKPT; return pagerOpenWalInternal(pPager, pbOpen); } @@ -7313,20 +7281,6 @@ int sqlite3PagerCloseWal(Pager *pPager){ return rc; } -/* -** This function is called by the wal.c module to obtain the 8 bytes of -** "salt" written into the wal file header. In OTA mode, this is a copy -** of bytes 24-31 of the database file. In non-OTA mode, it is 8 bytes -** of pseudo-random data. -*/ -void sqlite3PagerWalSalt(Pager *pPager, u32 *aSalt){ - if( PagerOtaMode(pPager) ){ - memcpy(aSalt, pPager->dbFileVers, 8); - }else{ - sqlite3_randomness(8, aSalt); - } -} - #endif /* !SQLITE_OMIT_WAL */ #ifdef SQLITE_ENABLE_ZIPVFS @@ -7344,17 +7298,6 @@ int sqlite3PagerWalFramesize(Pager *pPager){ #endif #ifdef SQLITE_ENABLE_OTA -/* -** Set or clear the "OTA mode" flag. -*/ -int sqlite3PagerSetOtaMode(Pager *pPager, int iOta){ - assert( iOta==1 || iOta==2 ); - if( iOta==1 && (pPager->pWal || pPager->eState!=PAGER_OPEN) ){ - return SQLITE_ERROR; - } - pPager->otaMode = iOta; - return SQLITE_OK; -} /* ** Open an incremental checkpoint handle. diff --git a/src/pager.h b/src/pager.h index 289ef9d42b..10df8e0deb 100644 --- a/src/pager.h +++ b/src/pager.h @@ -210,8 +210,6 @@ void *sqlite3PagerCodec(DbPage *); # define enable_simulated_io_errors() #endif -int sqlite3PagerSetOtaMode(Pager *pPager, int bOta); -void sqlite3PagerWalSalt(Pager *pPager, u32 *aSalt); int sqlite3PagerWalCheckpointStart(sqlite3*, Pager*, u8*, int, sqlite3_ckpt**); #endif /* _PAGER_H_ */ diff --git a/src/pragma.c b/src/pragma.c index d4c5ba51dc..ac217d4597 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -422,50 +422,6 @@ void sqlite3Pragma( } #endif /* !SQLITE_OMIT_PAGER_PRAGMAS && !SQLITE_OMIT_DEPRECATED */ - /* - ** PRAGMA [database.]pager_ota_mode=[01] - ** - ** This pragma sets a flag on the pager associated with the main database - ** only. The flag can only be set when there is no open transaction and - ** the pager does not already have an open WAL file. - ** - ** Once the flag has been set, it is not possible to open a regular WAL - ** file. If, when the next read-transaction is opened, a *-wal file is - ** found or the database header flags indicate that it is a wal-mode - ** database, SQLITE_CANTOPEN is returned. - ** - ** Otherwise, if no WAL file or flags are found, the pager opens the *-oal - ** file and uses it as a write-ahead-log with the *-shm data stored in - ** heap-memory. If the *-oal file already exists but the database file has - ** been modified since it was created, an SQLITE_BUSY_SNAPSHOT error is - ** returned and the read-transaction cannot be opened. - ** - ** Other clients see a rollback-mode database on which the pager_ota_mode - ** client is holding a SHARED lock. - */ -#ifdef SQLITE_ENABLE_OTA - case PragTyp_PAGER_OTA_MODE: { - Btree *pBt = pDb->pBt; - assert( pBt!=0 ); - if( zRight ){ - int iArg = sqlite3Atoi(zRight); - Pager *pPager = sqlite3BtreePager(pBt); - if( sqlite3BtreeIsInReadTrans(pBt) ){ - sqlite3ErrorMsg(pParse, - "cannot set pager_ota_mode with open transaction" - ); - }else if( sqlite3PagerWalSupported(pPager)==0 ){ - sqlite3ErrorMsg(pParse, - "cannot set pager_ota_mode without wal support" - ); - }else if( sqlite3PagerSetOtaMode(sqlite3BtreePager(pBt), iArg) ){ - sqlite3ErrorMsg(pParse, "cannot set pager_ota_mode in wal mode"); - } - } - break; - } -#endif /* SQLITE_ENABLE_OTA */ - #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) /* ** PRAGMA [database.]page_size diff --git a/src/pragma.h b/src/pragma.h index b2cf43911b..c9ae8e6eb8 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -28,24 +28,23 @@ #define PragTyp_PAGE_COUNT 22 #define PragTyp_MMAP_SIZE 23 #define PragTyp_PAGE_SIZE 24 -#define PragTyp_PAGER_OTA_MODE 25 -#define PragTyp_SECURE_DELETE 26 -#define PragTyp_SHRINK_MEMORY 27 -#define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_STATS 29 -#define PragTyp_SYNCHRONOUS 30 -#define PragTyp_TABLE_INFO 31 -#define PragTyp_TEMP_STORE 32 -#define PragTyp_TEMP_STORE_DIRECTORY 33 -#define PragTyp_THREADS 34 -#define PragTyp_WAL_AUTOCHECKPOINT 35 -#define PragTyp_WAL_CHECKPOINT 36 -#define PragTyp_ACTIVATE_EXTENSIONS 37 -#define PragTyp_HEXKEY 38 -#define PragTyp_KEY 39 -#define PragTyp_REKEY 40 -#define PragTyp_LOCK_STATUS 41 -#define PragTyp_PARSER_TRACE 42 +#define PragTyp_SECURE_DELETE 25 +#define PragTyp_SHRINK_MEMORY 26 +#define PragTyp_SOFT_HEAP_LIMIT 27 +#define PragTyp_STATS 28 +#define PragTyp_SYNCHRONOUS 29 +#define PragTyp_TABLE_INFO 30 +#define PragTyp_TEMP_STORE 31 +#define PragTyp_TEMP_STORE_DIRECTORY 32 +#define PragTyp_THREADS 33 +#define PragTyp_WAL_AUTOCHECKPOINT 34 +#define PragTyp_WAL_CHECKPOINT 35 +#define PragTyp_ACTIVATE_EXTENSIONS 36 +#define PragTyp_HEXKEY 37 +#define PragTyp_KEY 38 +#define PragTyp_REKEY 39 +#define PragTyp_LOCK_STATUS 40 +#define PragTyp_PARSER_TRACE 41 #define PragFlag_NeedSchema 0x01 #define PragFlag_ReadOnly 0x02 static const struct sPragmaNames { @@ -304,12 +303,6 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif -#if defined(SQLITE_ENABLE_OTA) - { /* zName: */ "pager_ota_mode", - /* ePragTyp: */ PragTyp_PAGER_OTA_MODE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, -#endif #if defined(SQLITE_DEBUG) { /* zName: */ "parser_trace", /* ePragTyp: */ PragTyp_PARSER_TRACE, @@ -463,4 +456,4 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 59 on by default, 73 total. */ +/* Number of pragmas: 59 on by default, 72 total. */ diff --git a/src/wal.c b/src/wal.c index bf2ef44a46..fb51ac9fd2 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2920,9 +2920,7 @@ int sqlite3WalFrames( sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); sqlite3Put4byte(&aWalHdr[8], szPage); sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); - if( pWal->nCkpt==0 ){ - sqlite3PagerWalSalt(pList->pPager, pWal->hdr.aSalt); - } + if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt); memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); sqlite3Put4byte(&aWalHdr[24], aCksum[0]); diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 5627a19d35..964946e788 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -315,9 +315,6 @@ set pragma_def { NAME: soft_heap_limit NAME: threads - - NAME: pager_ota_mode - IF: defined(SQLITE_ENABLE_OTA) } # Open the output file From de1f6a270b60c05bc037d34b860a26ffd6bf6bfa Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 7 Feb 2015 20:20:32 +0000 Subject: [PATCH 059/116] Add comments to explain the role of the ota vfs. FossilOrigin-Name: 7bb633639d1e41067903a49653f09a823054e213 --- ext/ota/sqlite3ota.c | 35 ++++++++++++++++++++++++++++------- manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 2e23d5658c..cf408b8b9f 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -2153,15 +2153,36 @@ sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta){ ** Beginning of OTA VFS shim methods. The VFS shim modifies the behaviour ** of a standard VFS in the following ways: ** -** TODO +** 1. Whenever the first page of an OTA target database file is read or +** written, the value of the change-counter cookie is stored in +** sqlite3ota.iCookie. This ensures that, so long as a read transaction +** is held on the db file, the value of sqlite3ota.iCookie matches +** that stored on disk. +** +** 2. When the ota handle is in OTA_STAGE_OAL or OTA_STAGE_CKPT state, all +** EXCLUSIVE lock attempts on the target database fail. This prevents +** sqlite3_close() from running an automatic checkpoint. Until the +** ota handle reaches OTA_STAGE_DONE - at that point the automatic +** checkpoint may be required to delete the *-wal file. +** +** 3. In OTA_STAGE_OAL, the *-shm file is stored in memory. All xShmLock() +** calls are noops. +** +** 4. In OTA_STAGE_OAL mode, when SQLite calls xAccess() to check if a +** *-wal file associated with the target database exists, the following +** special handling applies: +** +** a) if the *-wal file does exist, return SQLITE_CANTOPEN. An OTA +** target database may not be in wal mode already. +** +** b) if the *-wal file does not exist, set the output parameter to +** non-zero (to tell SQLite that it does exist) anyway. +** +** 5. In OTA_STAGE_OAL mode, if SQLite tries to open a *-wal file +** associated with a target database, open the corresponding *-oal file +** instead. */ -#if 0 -#define OTA_FILE_VANILLA 0 -#define OTA_FILE_TARGET_DB 1 -#define OTA_FILE_TARGET_WAL 2 -#endif - typedef struct ota_file ota_file; typedef struct ota_vfs ota_vfs; diff --git a/manifest b/manifest index 93b2368df6..777601fb6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\s"PRAGMA\spager_ota_mode". -D 2015-02-07T19:17:36.157 +C Add\scomments\sto\sexplain\sthe\srole\sof\sthe\sota\svfs. +D 2015-02-07T20:20:32.171 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -134,7 +134,7 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c 7015400382d1d6655626046f2c1763634dd8a168 +F ext/ota/sqlite3ota.c a5cea44e3d186ae53c1a0dbdc1c3ca73e9d1dc62 F ext/ota/sqlite3ota.h b4c54c7df5d223f2ee40efa5ba363188daa3ad37 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -1253,10 +1253,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c3931db560ab4a2601c7f7318fb02c8d5e6862b1 -R 8cb076bc2c2cc0b3b28786011832c69d -T *branch * ota-update-no-pager_ota_mode -T *sym-ota-update-no-pager_ota_mode * -T -sym-ota-update * +P 8ac58e46782bd6b81c06fdf1cb5b316b8a4e1ddf +R f5474ed197582e342ea21c91b796b802 U dan -Z b552e61eecb5b9b2a0bf7b3905abae17 +Z 4e62cdaf7cd09d75059d0dc1ee3d7cb0 diff --git a/manifest.uuid b/manifest.uuid index 78b4daf1f5..489302fa8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8ac58e46782bd6b81c06fdf1cb5b316b8a4e1ddf \ No newline at end of file +7bb633639d1e41067903a49653f09a823054e213 \ No newline at end of file From faa8c4fdcb6cdc6bc7d156a0f687d937117f443b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 9 Feb 2015 20:07:35 +0000 Subject: [PATCH 060/116] Add the sqlite3ota_create_vfs() and sqlite3ota_destroy_vfs() functions. FossilOrigin-Name: 96443ecb6909141aa621a16e628455857d036482 --- ext/ota/ota1.test | 2 +- ext/ota/sqlite3ota.c | 305 +++++++++++++++++++++++-------------------- ext/ota/sqlite3ota.h | 16 +++ manifest | 18 +-- manifest.uuid | 2 +- src/sqlite.h.in | 1 + 6 files changed, 195 insertions(+), 149 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index 265a3d2a2f..6b8ed971da 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -98,7 +98,7 @@ proc step_ota {target ota} { # proc step_ota_uri {target ota} { while 1 { - sqlite3ota ota file:$target?xyz=123 $ota + sqlite3ota ota file:$target?xyz=&abc=123 $ota set rc [ota step] ota close if {$rc != "SQLITE_OK"} break diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index cf408b8b9f..41207501cf 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -85,6 +85,8 @@ typedef struct OtaState OtaState; typedef struct OtaObjIter OtaObjIter; +typedef struct ota_vfs ota_vfs; +typedef struct ota_file ota_file; /* ** A structure to store values read from the ota_state table in memory. @@ -156,6 +158,7 @@ struct OtaObjIter { #define OTA_PK_WITHOUT_ROWID 4 #define OTA_PK_VTAB 5 + /* ** OTA handle. */ @@ -170,10 +173,31 @@ struct sqlite3ota { int nProgress; /* Rows processed for all objects */ OtaObjIter objiter; /* Iterator for skipping through tbl/idx */ sqlite3_ckpt *pCkpt; /* Incr-checkpoint handle */ - sqlite3_vfs *pVfs; /* Special ota VFS object */ + ota_file *pTargetFd; /* File handle open on target db */ + const char *zVfsName; /* Name of automatically created ota vfs */ unsigned int iCookie; }; +struct ota_vfs { + sqlite3_vfs base; /* ota VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3_mutex *mutex; + const char *zOtaWal; +}; + +struct ota_file { + sqlite3_file base; /* sqlite3_file methods */ + sqlite3_file *pReal; /* Underlying file handle */ + ota_vfs *pOtaVfs; /* Pointer to the ota_vfs object */ + sqlite3ota *pOta; /* Pointer to ota object (ota target only) */ + + int nShm; /* Number of entries in apShm[] array */ + char **apShm; /* Array of mmap'd *-shm regions */ + const char *zWal; /* Wal filename for this db file */ + char *zDel; /* Delete this when closing file */ +}; + + static void otaCreateVfs(sqlite3ota*, const char*); static void otaDeleteVfs(sqlite3ota*); @@ -422,6 +446,20 @@ static int otaMPrintfExec(sqlite3ota *p, const char *zFmt, ...){ return p->rc; } +static void *otaMalloc(sqlite3ota *p, int nByte){ + void *pRet = 0; + if( p->rc==SQLITE_OK ){ + pRet = sqlite3_malloc(nByte); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + + /* ** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that ** there is room for at least nCol elements. If an OOM occurs, store an @@ -431,17 +469,13 @@ static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ int nByte = (2*sizeof(char*) + sizeof(int) + 2*sizeof(unsigned char)) * nCol; char **azNew; - assert( p->rc==SQLITE_OK ); - azNew = (char**)sqlite3_malloc(nByte); + azNew = (char**)otaMalloc(p, nByte); if( azNew ){ - memset(azNew, 0, nByte); pIter->azTblCol = azNew; pIter->azTblType = &azNew[nCol]; pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; pIter->abTblPk = (unsigned char*)&pIter->aiSrcOrder[nCol]; pIter->abNotNull = (unsigned char*)&pIter->abTblPk[nCol]; - }else{ - p->rc = SQLITE_NOMEM; } } @@ -713,19 +747,6 @@ static char *otaMPrintf(sqlite3ota *p, const char *zFmt, ...){ return zSql; } -static void *otaMalloc(sqlite3ota *p, int nByte){ - void *pRet = 0; - if( p->rc==SQLITE_OK ){ - pRet = sqlite3_malloc(nByte); - if( pRet==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - memset(pRet, 0, nByte); - } - } - return pRet; -} - /* ** This function constructs and returns a pointer to a nul-terminated ** string containing some SQL clause or list based on one or more of the @@ -991,17 +1012,14 @@ static char *otaObjIterGetSetlist( static char *otaObjIterGetBindlist(sqlite3ota *p, int nBind){ char *zRet = 0; - if( p->rc==SQLITE_OK ){ - int nByte = nBind*2 + 1; - zRet = sqlite3_malloc(nByte); - if( zRet==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - int i; - for(i=0; irc==SQLITE_OK ); assert( p->db==0 ); - p->rc = sqlite3_open_v2(p->zTarget, &p->db, flags, p->pVfs->zName); + p->rc = sqlite3_open_v2(p->zTarget, &p->db, flags, p->zVfsName); if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); + }else{ + /* Mark the database file just opened as an OTA target database. If + ** this call returns SQLITE_NOTFOUND, then the OTA vfs is not in use. + ** This is an error. */ + p->rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_OTA, (void*)p); + if( p->rc==SQLITE_NOTFOUND ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("ota vfs not found"); + }else{ + otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); + } } - otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); } /* @@ -2039,11 +2067,14 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ if( p->rc==SQLITE_OK ){ if( p->eStage==OTA_STAGE_OAL ){ - const char *zScript = - "PRAGMA journal_mode=off;" - "BEGIN IMMEDIATE;" - ; - p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); + ota_vfs *pOtaVfs = p->pTargetFd->pOtaVfs; + + sqlite3_mutex_enter(pOtaVfs->mutex); + assert( pOtaVfs->zOtaWal==0 ); + pOtaVfs->zOtaWal = p->pTargetFd->zWal; + p->rc = sqlite3_exec(p->db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + pOtaVfs->zOtaWal = 0; + sqlite3_mutex_leave(pOtaVfs->mutex); /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ @@ -2183,33 +2214,11 @@ sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta){ ** instead. */ -typedef struct ota_file ota_file; -typedef struct ota_vfs ota_vfs; - -struct ota_file { - sqlite3_file base; /* sqlite3_file methods */ - sqlite3_file *pReal; /* Underlying file handle */ - ota_vfs *pOtaVfs; /* Pointer to the ota_vfs object */ - - int nShm; /* Number of entries in apShm[] array */ - char **apShm; /* Array of mmap'd *-shm regions */ - char *zFilename; /* Filename for *-oal file only */ -}; - -struct ota_vfs { - sqlite3_vfs base; /* ota VFS shim methods */ - sqlite3_vfs *pRealVfs; /* Underlying VFS */ - sqlite3ota *pOta; - ota_file *pTargetDb; /* Target database file descriptor */ - const char *zTargetDb; /* Path that pTargetDb was opened with */ -}; - /* ** Close an ota file. */ static int otaVfsClose(sqlite3_file *pFile){ ota_file *p = (ota_file*)pFile; - ota_vfs *pOtaVfs = p->pOtaVfs; int rc; int i; @@ -2219,12 +2228,7 @@ static int otaVfsClose(sqlite3_file *pFile){ } sqlite3_free(p->apShm); p->apShm = 0; - sqlite3_free(p->zFilename); - - if( p==pOtaVfs->pTargetDb ){ - pOtaVfs->pTargetDb = 0; - pOtaVfs->zTargetDb = 0; - } + sqlite3_free(p->zDel); rc = p->pReal->pMethods->xClose(p->pReal); return rc; @@ -2252,12 +2256,11 @@ static int otaVfsRead( sqlite_int64 iOfst ){ ota_file *p = (ota_file*)pFile; - ota_vfs *pOtaVfs = p->pOtaVfs; int rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); - if( rc==SQLITE_OK && p==pOtaVfs->pTargetDb && iOfst==0 ){ + if( rc==SQLITE_OK && p->pOta && iOfst==0 ){ unsigned char *pBuf = (unsigned char*)zBuf; assert( iAmt>=100 ); - pOtaVfs->pOta->iCookie = otaGetU32(&pBuf[24]); + p->pOta->iCookie = otaGetU32(&pBuf[24]); } return rc; } @@ -2272,12 +2275,11 @@ static int otaVfsWrite( sqlite_int64 iOfst ){ ota_file *p = (ota_file*)pFile; - ota_vfs *pOtaVfs = p->pOtaVfs; int rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst); - if( rc==SQLITE_OK && p==pOtaVfs->pTargetDb && iOfst==0 ){ + if( rc==SQLITE_OK && p->pOta && iOfst==0 ){ unsigned char *pBuf = (unsigned char*)zBuf; assert( iAmt>=100 ); - pOtaVfs->pOta->iCookie = otaGetU32(&pBuf[24]); + p->pOta->iCookie = otaGetU32(&pBuf[24]); } return rc; } @@ -2311,13 +2313,11 @@ static int otaVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ */ static int otaVfsLock(sqlite3_file *pFile, int eLock){ ota_file *p = (ota_file*)pFile; - ota_vfs *pOtaVfs = p->pOtaVfs; + sqlite3ota *pOta = p->pOta; int rc = SQLITE_OK; - int eStage = pOtaVfs->pOta->eStage; - if( pOtaVfs->pTargetDb==p - && (eStage==OTA_STAGE_OAL || eStage==OTA_STAGE_CKPT) - && eLock==SQLITE_LOCK_EXCLUSIVE + if( pOta && eLock==SQLITE_LOCK_EXCLUSIVE + && (pOta->eStage==OTA_STAGE_OAL || pOta->eStage==OTA_STAGE_CKPT) ){ /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this ** prevents it from checkpointing the database from sqlite3_close(). */ @@ -2350,6 +2350,12 @@ static int otaVfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){ */ static int otaVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ ota_file *p = (ota_file *)pFile; + if( op==SQLITE_FCNTL_OTA ){ + sqlite3ota *pOta = (sqlite3ota*)pArg; + pOta->pTargetFd = p; + p->pOta = pOta; + return SQLITE_OK; + } return p->pReal->pMethods->xFileControl(p->pReal, op, pArg); } @@ -2374,14 +2380,13 @@ static int otaVfsDeviceCharacteristics(sqlite3_file *pFile){ */ static int otaVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ ota_file *p = (ota_file*)pFile; - ota_vfs *pOtaVfs = p->pOtaVfs; int rc = SQLITE_OK; #ifdef SQLITE_AMALGAMATION assert( WAL_WRITE_CKPT==1 ); #endif - if( pOtaVfs->pTargetDb==p && pOtaVfs->pOta->eStage==OTA_STAGE_OAL ){ + if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){ /* Magic number 1 is the WAL_WRITE_CKPT lock. Preventing SQLite from ** taking this lock also prevents any checkpoints from occurring. ** todo: really, it's not clear why this might occur, as @@ -2389,7 +2394,7 @@ static int otaVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ if( ofst==1 && n==1 ) rc = SQLITE_BUSY; }else{ assert( p->nShm==0 ); - return p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags); + rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags); } return rc; @@ -2403,13 +2408,12 @@ static int otaVfsShmMap( void volatile **pp ){ ota_file *p = (ota_file*)pFile; - ota_vfs *pOtaVfs = p->pOtaVfs; int rc = SQLITE_OK; /* If not in OTA_STAGE_OAL, allow this call to pass through. Or, if this ** ota is in the OTA_STAGE_OAL state, use heap memory for *-shm space ** instead of a file on disk. */ - if( pOtaVfs->pTargetDb==p && pOtaVfs->pOta->eStage==OTA_STAGE_OAL ){ + if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){ if( iRegion<=p->nShm ){ int nByte = (iRegion+1) * sizeof(char*); char **apNew = (char**)sqlite3_realloc(p->apShm, nByte); @@ -2454,10 +2458,9 @@ static void otaVfsShmBarrier(sqlite3_file *pFile){ static int otaVfsShmUnmap(sqlite3_file *pFile, int delFlag){ ota_file *p = (ota_file*)pFile; - ota_vfs *pOtaVfs = p->pOtaVfs; int rc = SQLITE_OK; - if( pOtaVfs->pTargetDb==p && pOtaVfs->pOta->eStage==OTA_STAGE_OAL ){ + if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){ /* no-op */ }else{ rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag); @@ -2465,17 +2468,6 @@ static int otaVfsShmUnmap(sqlite3_file *pFile, int delFlag){ return rc; } - -static int otaVfsIswal(ota_vfs *pOtaVfs, const char *zPath){ - int nPath = strlen(zPath); - int nTargetDb = strlen(pOtaVfs->zTargetDb); - return ( nPath==(nTargetDb+4) - && 0==memcmp(zPath, pOtaVfs->zTargetDb, nTargetDb) - && 0==memcmp(&zPath[nTargetDb], "-wal", 4) - ); -} - - /* ** Open an ota file handle. */ @@ -2507,7 +2499,6 @@ static int otaVfsOpen( }; ota_vfs *pOtaVfs = (ota_vfs*)pVfs; sqlite3_vfs *pRealVfs = pOtaVfs->pRealVfs; - sqlite3ota *p = pOtaVfs->pOta; ota_file *pFd = (ota_file *)pFile; int rc = SQLITE_OK; const char *zOpen = zName; @@ -2515,13 +2506,38 @@ static int otaVfsOpen( memset(pFd, 0, sizeof(ota_file)); pFd->pReal = (sqlite3_file*)&pFd[1]; pFd->pOtaVfs = pOtaVfs; - - if( zName && p->eStage==OTA_STAGE_OAL && otaVfsIswal(pOtaVfs, zName) ){ - char *zCopy = otaStrndup(zName, -1, &rc); - if( zCopy ){ - int nCopy = strlen(zCopy); - zCopy[nCopy-3] = 'o'; - zOpen = (const char*)(pFd->zFilename = zCopy); + if( zName ){ + if( flags & SQLITE_OPEN_MAIN_DB ){ + /* A main database has just been opened. The following block sets + ** (pFd->zWal) to point to a buffer owned by SQLite that contains + ** the name of the *-wal file this db connection will use. SQLite + ** happens to pass a pointer to this buffer when using xAccess() + ** or xOpen() to operate on the *-wal file. */ + int n = strlen(zName); + const char *z = &zName[n]; + if( flags & SQLITE_OPEN_URI ){ + int odd = 0; + while( 1 ){ + if( z[0]==0 ){ + odd = 1 - odd; + if( odd && z[1]==0 ) break; + } + z++; + } + z += 2; + }else{ + while( *z==0 ) z++; + } + z += (n + 8 + 1); + pFd->zWal = z; + } + else if( (flags & SQLITE_OPEN_WAL) && zName==pOtaVfs->zOtaWal ){ + char *zCopy = otaStrndup(zName, -1, &rc); + if( zCopy ){ + int nCopy = strlen(zCopy); + zCopy[nCopy-3] = 'o'; + zOpen = (const char*)(pFd->zDel = zCopy); + } } } @@ -2530,13 +2546,6 @@ static int otaVfsOpen( } if( pFd->pReal->pMethods ){ pFile->pMethods = &otavfs_io_methods; - if( pOtaVfs->pTargetDb==0 ){ - /* This is the target db file. */ - assert( (flags & SQLITE_OPEN_MAIN_DB) ); - assert( zOpen==zName ); - pOtaVfs->pTargetDb = pFd; - pOtaVfs->zTargetDb = zName; - } } return rc; @@ -2566,11 +2575,7 @@ static int otaVfsAccess( rc = pRealVfs->xAccess(pRealVfs, zPath, flags, pResOut); - if( rc==SQLITE_OK - && flags==SQLITE_ACCESS_EXISTS - && pOtaVfs->pOta->eStage==OTA_STAGE_OAL - && otaVfsIswal(pOtaVfs, zPath) - ){ + if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS && pOtaVfs->zOtaWal==zPath ){ if( *pResOut ){ rc = SQLITE_CANTOPEN; }else{ @@ -2666,7 +2671,15 @@ static int otaVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){ return 0; } -static void otaCreateVfs(sqlite3ota *p, const char *zParent){ +void sqlite3ota_destroy_vfs(const char *zName){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zName); + if( pVfs ){ + sqlite3_vfs_unregister(pVfs); + sqlite3_free(pVfs); + } +} + +int sqlite3ota_create_vfs(const char *zName, const char *zParent){ /* Template for VFS */ static sqlite3_vfs vfs_template = { @@ -2696,48 +2709,64 @@ static void otaCreateVfs(sqlite3ota *p, const char *zParent){ sqlite3_vfs *pParent; /* Parent VFS */ ota_vfs *pNew = 0; /* Newly allocated VFS */ + int nName; + int rc = SQLITE_OK; - assert( p->rc==SQLITE_OK ); + nName = strlen(zName); pParent = sqlite3_vfs_find(zParent); if( pParent==0 ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("no such vfs: %s", zParent); + rc = SQLITE_NOTFOUND; }else{ - int nByte = sizeof(ota_vfs) + 64; - pNew = (ota_vfs*)otaMalloc(p, nByte); + int nByte = sizeof(ota_vfs) + nName + 1; + pNew = (ota_vfs*)sqlite3_malloc(nByte); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, nByte); + } } - if( pNew ){ - int rnd; - char *zName; + if( rc==SQLITE_OK ){ + char *zSpace; memcpy(&pNew->base, &vfs_template, sizeof(sqlite3_vfs)); pNew->base.mxPathname = pParent->mxPathname; pNew->base.szOsFile = sizeof(ota_file) + pParent->szOsFile; - pNew->pOta = p; pNew->pRealVfs = pParent; - /* Give the new VFS a unique name */ - sqlite3_randomness(sizeof(int), (void*)&rnd); - pNew->base.zName = (const char*)(zName = (char*)&pNew[1]); - sprintf(zName, "ota_vfs_%d", rnd); + pNew->base.zName = (const char*)(zSpace = (char*)&pNew[1]); + memcpy(zSpace, zName, nName); /* Register the new VFS (not as the default) */ - assert( p->rc==SQLITE_OK ); - p->rc = sqlite3_vfs_register(&pNew->base, 0); - if( p->rc ){ - p->zErrmsg = sqlite3_mprintf("error in sqlite3_vfs_register()"); + rc = sqlite3_vfs_register(&pNew->base, 0); + if( rc ){ sqlite3_free(pNew); - }else{ - p->pVfs = &pNew->base; } } + + return rc; +} + +static void otaCreateVfs(sqlite3ota *p, const char *zParent){ + int rnd; + char zRnd[64]; + + assert( p->rc==SQLITE_OK ); + sqlite3_randomness(sizeof(int), (void*)&rnd); + sprintf(zRnd, "ota_vfs_%d", rnd); + p->rc = sqlite3ota_create_vfs(zRnd, zParent); + if( p->rc==SQLITE_NOTFOUND ){ + p->zErrmsg = sqlite3_mprintf("no such vfs: %s", zParent); + }else if( p->rc==SQLITE_OK ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd); + assert( pVfs ); + p->zVfsName = pVfs->zName; + } } static void otaDeleteVfs(sqlite3ota *p){ - if( p->pVfs ){ - sqlite3_vfs_unregister(p->pVfs); - sqlite3_free(p->pVfs); - p->pVfs = 0; + if( p->zVfsName ){ + sqlite3ota_destroy_vfs(p->zVfsName); + p->zVfsName = 0; } } diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index e549022f9a..f6890f4508 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -300,5 +300,21 @@ int sqlite3ota_close(sqlite3ota *pOta, char **pzErrmsg); */ sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta); +/* +** Create an OTA VFS named zName. Use existing VFS zParent to interact +** with the file-system. +*/ +int sqlite3ota_create_vfs(const char *zName, const char *zParent); + +/* +** Deregister and destroy an OTA vfs previously created by +** sqlite3ota_create_vfs(). +** +** VFS objects are not reference counted. If a VFS object is destroyed +** before all database handles that use it have been closed, the results +** are undefined. +*/ +void sqlite3ota_destroy_vfs(const char *zName); + #endif /* _SQLITE3OTA_H */ diff --git a/manifest b/manifest index 777601fb6b..fe890ddfd3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scomments\sto\sexplain\sthe\srole\sof\sthe\sota\svfs. -D 2015-02-07T20:20:32.171 +C Add\sthe\ssqlite3ota_create_vfs()\sand\ssqlite3ota_destroy_vfs()\sfunctions. +D 2015-02-09T20:07:35.066 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,7 +125,7 @@ F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/ota/README.txt 2ce4ffbb0aaa6731b041c27a7359f9a5f1c69152 F ext/ota/ota.c c11a85af71dccc45976622fe7a51169a481caa91 -F ext/ota/ota1.test e6b64d6ffb23dcae72386da153626b40566a69e9 +F ext/ota/ota1.test a4779d5fe9710d607ebae360eebee3d82a1c14e6 F ext/ota/ota10.test 85e0f6e7964db5007590c1b299e75211ed4240d4 F ext/ota/ota3.test a77efbce7723332eb688d2b28bf18204fc9614d7 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb @@ -134,8 +134,8 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c a5cea44e3d186ae53c1a0dbdc1c3ca73e9d1dc62 -F ext/ota/sqlite3ota.h b4c54c7df5d223f2ee40efa5ba363188daa3ad37 +F ext/ota/sqlite3ota.c 3d3179fb9bcb0edf88b3391f480e84b451f1275e +F ext/ota/sqlite3ota.h 58af0ab6361c76e0eda7aede72e7de42abf83605 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -245,7 +245,7 @@ F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 1f2087523007c42900ffcbdeaef06a23ad9329fc F src/shell.c 82c25508dac802b32198af6f5256ca1597c6a1af -F src/sqlite.h.in 4807b024e8d257af774cde0cf178f721ff2406ec +F src/sqlite.h.in c49acd2daa6e54110ab0cc607eb73ff32720a269 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 57f8f45028598cc2877fc08ac03b402242242c68 @@ -1253,7 +1253,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8ac58e46782bd6b81c06fdf1cb5b316b8a4e1ddf -R f5474ed197582e342ea21c91b796b802 +P 7bb633639d1e41067903a49653f09a823054e213 +R 290f53a2e90389d88a1ff448f97370f1 U dan -Z 4e62cdaf7cd09d75059d0dc1ee3d7cb0 +Z 09bf3544f840f7fca63db224a647bd64 diff --git a/manifest.uuid b/manifest.uuid index 489302fa8e..ae4cdfe82d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7bb633639d1e41067903a49653f09a823054e213 \ No newline at end of file +96443ecb6909141aa621a16e628455857d036482 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index cdf5a8c179..23681cc6f3 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -970,6 +970,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_COMMIT_PHASETWO 22 #define SQLITE_FCNTL_WIN32_SET_HANDLE 23 #define SQLITE_FCNTL_ZIPVFS_PAGER 24 +#define SQLITE_FCNTL_OTA 25 /* ** CAPI3REF: Mutex Handle From 07427d03507750417c21eba5f4dee339a2b0def3 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 10 Feb 2015 17:08:17 +0000 Subject: [PATCH 061/116] Add documentation and test cases for sqlite3ota_create_vfs(). Also code to detect errors in zipvfs/ota setup. FossilOrigin-Name: e729668168f00325459bc2e9b515aa95e57f2754 --- ext/ota/ota1.test | 690 ++++++++++++++++++++++--------------------- ext/ota/sqlite3ota.c | 98 +++++- ext/ota/sqlite3ota.h | 50 +++- manifest | 16 +- manifest.uuid | 2 +- 5 files changed, 498 insertions(+), 358 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index 6b8ed971da..dec5f14a99 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -106,362 +106,376 @@ proc step_ota_uri {target ota} { set rc } -foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { - foreach {tn schema} { - 1 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - } - 2 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b); - } - 3 { - CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; - } - 4 { - CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - } - 5 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - } - 6 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(c)) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b, a); - } - 7 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b, c); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(a, b, c, a, b, c); - } - - 8 { - CREATE TABLE t1(a PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b, c); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(a, b, c, a, b, c); - } - - 9 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)); - CREATE INDEX i1 ON t1(b); - } - - 10 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b DESC); - } - - 11 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b DESC, a ASC, c DESC); - } - - 12 { - CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; - } - - 13 { - CREATE TABLE t1(a INT, b, c, PRIMARY KEY(a DESC)) WITHOUT ROWID; - } - - 14 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(a DESC, c)) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - } - - 15 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c DESC)) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - } - - 16 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(c DESC, a)) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b DESC, c, a); - } +foreach {tn3 create_vfs destroy_vfs} { + 1 {} {} + 2 { + sqlite3ota_create_vfs -default myota "" } { - reset_db - execsql $schema - - do_test 1.$tn2.$tn.1 { - create_ota1 ota.db - breakpoint - $cmd test.db ota.db - } {SQLITE_DONE} - - do_execsql_test 1.$tn2.$tn.2 { SELECT * FROM t1 ORDER BY a ASC } { - 1 2 3 - 2 two three - 3 {} 8.2 - } - do_execsql_test 1.$tn2.$tn.3 { SELECT * FROM t1 ORDER BY b ASC } { - 3 {} 8.2 - 1 2 3 - 2 two three - } - do_execsql_test 1.$tn2.$tn.4 { SELECT * FROM t1 ORDER BY c ASC } { - 1 2 3 - 3 {} 8.2 - 2 two three - } - - do_execsql_test 1.$tn2.$tn.5 { PRAGMA integrity_check } ok + sqlite3ota_destroy_vfs myota } -} - -#------------------------------------------------------------------------- -# Check that an OTA cannot be applied to a table that has no PK. -# -# UPDATE: At one point OTA required that all tables featured either -# explicit IPK columns or were declared WITHOUT ROWID. This has been -# relaxed so that external PRIMARY KEYs on tables with automatic rowids -# are now allowed. -# -# UPDATE 2: Tables without any PRIMARY KEY declaration are now allowed. -# However the input table must feature an "ota_rowid" column. -# -reset_db -create_ota1 ota.db -do_execsql_test 2.1 { CREATE TABLE t1(a, b, c) } -do_test 2.2 { - sqlite3ota ota test.db ota.db - ota step -} {SQLITE_ERROR} -do_test 2.3 { - list [catch { ota close } msg] $msg -} {1 {SQLITE_ERROR - table data_t1 requires ota_rowid column}} -reset_db -do_execsql_test 2.4 { CREATE TABLE t1(a PRIMARY KEY, b, c) } -do_test 2.5 { - sqlite3ota ota test.db ota.db - ota step -} {SQLITE_OK} -do_test 2.6 { - list [catch { ota close } msg] $msg -} {0 SQLITE_OK} - -#------------------------------------------------------------------------- -# Check that if a UNIQUE constraint is violated the current and all -# subsequent [ota step] calls return SQLITE_CONSTRAINT. And that the OTA -# transaction is rolled back by the [ota close] that deletes the ota -# handle. -# -foreach {tn errcode errmsg schema} { - 1 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - INSERT INTO t1 VALUES(3, 2, 1); - } - - 2 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c UNIQUE); - INSERT INTO t1 VALUES(4, 2, 'three'); - } - - 3 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { - CREATE TABLE t1(a PRIMARY KEY, b, c); - INSERT INTO t1 VALUES(3, 2, 1); - } - - 4 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { - CREATE TABLE t1(a PRIMARY KEY, b, c UNIQUE); - INSERT INTO t1 VALUES(4, 2, 'three'); - } - } { + + eval $create_vfs + + foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + } + 3 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + } + 4 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + 5 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + 6 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b, a); + } + 7 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(a, b, c, a, b, c); + } + + 8 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(a, b, c, a, b, c); + } + + 9 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)); + CREATE INDEX i1 ON t1(b); + } + + 10 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b DESC); + } + + 11 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b DESC, a ASC, c DESC); + } + + 12 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; + } + + 13 { + CREATE TABLE t1(a INT, b, c, PRIMARY KEY(a DESC)) WITHOUT ROWID; + } + + 14 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a DESC, c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + + 15 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c DESC)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + + 16 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(c DESC, a)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b DESC, c, a); + } + } { + reset_db + execsql $schema + + do_test $tn3.1.$tn2.$tn.1 { + create_ota1 ota.db + breakpoint + $cmd test.db ota.db + } {SQLITE_DONE} + + do_execsql_test $tn3.1.$tn2.$tn.2 { SELECT * FROM t1 ORDER BY a ASC } { + 1 2 3 + 2 two three + 3 {} 8.2 + } + do_execsql_test $tn3.1.$tn2.$tn.3 { SELECT * FROM t1 ORDER BY b ASC } { + 3 {} 8.2 + 1 2 3 + 2 two three + } + do_execsql_test $tn3.1.$tn2.$tn.4 { SELECT * FROM t1 ORDER BY c ASC } { + 1 2 3 + 3 {} 8.2 + 2 two three + } + + do_execsql_test $tn3.1.$tn2.$tn.5 { PRAGMA integrity_check } ok + } + } + + #------------------------------------------------------------------------- + # Check that an OTA cannot be applied to a table that has no PK. + # + # UPDATE: At one point OTA required that all tables featured either + # explicit IPK columns or were declared WITHOUT ROWID. This has been + # relaxed so that external PRIMARY KEYs on tables with automatic rowids + # are now allowed. + # + # UPDATE 2: Tables without any PRIMARY KEY declaration are now allowed. + # However the input table must feature an "ota_rowid" column. + # reset_db - execsql $schema - set cksum [dbcksum db main] - - do_test 3.$tn.1 { - create_ota1 ota.db + create_ota1 ota.db + do_execsql_test $tn3.2.1 { CREATE TABLE t1(a, b, c) } + do_test $tn3.2.2 { sqlite3ota ota test.db ota.db - while {[set res [ota step]]=="SQLITE_OK"} {} - set res - } $errcode - - do_test 3.$tn.2 { ota step } $errcode - - do_test 3.$tn.3 { + ota step + } {SQLITE_ERROR} + do_test $tn3.2.3 { list [catch { ota close } msg] $msg - } [list 1 "$errcode - $errmsg"] + } {1 {SQLITE_ERROR - table data_t1 requires ota_rowid column}} + reset_db + do_execsql_test $tn3.2.4 { CREATE TABLE t1(a PRIMARY KEY, b, c) } + do_test $tn3.2.5 { + sqlite3ota ota test.db ota.db + ota step + } {SQLITE_OK} + do_test $tn3.2.6 { + list [catch { ota close } msg] $msg + } {0 SQLITE_OK} - do_test 3.$tn.4 { dbcksum db main } $cksum -} + #------------------------------------------------------------------------- + # Check that if a UNIQUE constraint is violated the current and all + # subsequent [ota step] calls return SQLITE_CONSTRAINT. And that the OTA + # transaction is rolled back by the [ota close] that deletes the ota + # handle. + # + foreach {tn errcode errmsg schema} { + 1 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(3, 2, 1); + } -#------------------------------------------------------------------------- -# -foreach {tn2 cmd} {1 run_ota 2 step_ota} { - foreach {tn schema} { - 1 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - } - 2 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b); - } - 3 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(c, b, c); - } - 4 { - CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(c, b, c); - } - 5 { - CREATE TABLE t1(a INT PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(c, b, c); - } + 2 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c UNIQUE); + INSERT INTO t1 VALUES(4, 2, 'three'); + } + + 3 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(3, 2, 1); + } + + 4 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { + CREATE TABLE t1(a PRIMARY KEY, b, c UNIQUE); + INSERT INTO t1 VALUES(4, 2, 'three'); + } - 6 { - CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c); - CREATE INDEX i1 ON t1(b DESC); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(c DESC, b, c); - } - 7 { - CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(c, b, c); - } } { reset_db execsql $schema - execsql { - INSERT INTO t1 VALUES(2, 'hello', 'world'); - INSERT INTO t1 VALUES(4, 'hello', 'planet'); - INSERT INTO t1 VALUES(6, 'hello', 'xyz'); - } - - do_test 4.$tn2.$tn.1 { - create_ota4 ota.db - $cmd test.db ota.db - } {SQLITE_DONE} - - do_execsql_test 4.$tn2.$tn.2 { - SELECT * FROM t1 ORDER BY a ASC; + set cksum [dbcksum db main] + + do_test $tn3.3.$tn.1 { + create_ota1 ota.db + sqlite3ota ota test.db ota.db + while {[set res [ota step]]=="SQLITE_OK"} {} + set res + } $errcode + + do_test $tn3.3.$tn.2 { ota step } $errcode + + do_test $tn3.3.$tn.3 { + list [catch { ota close } msg] $msg + } [list 1 "$errcode - $errmsg"] + + do_test $tn3.3.$tn.4 { dbcksum db main } $cksum + } + + #------------------------------------------------------------------------- + # + foreach {tn2 cmd} {1 run_ota 2 step_ota} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + } + 3 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + 4 { + CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + 5 { + CREATE TABLE t1(a INT PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + + 6 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c); + CREATE INDEX i1 ON t1(b DESC); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c DESC, b, c); + } + 7 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } } { - 1 2 3 - 3 8 9 - 6 hello xyz - } - - do_execsql_test 4.$tn2.$tn.3 { PRAGMA integrity_check } ok - } -} - -#------------------------------------------------------------------------- -# -foreach {tn2 cmd} {1 run_ota 2 step_ota} { - foreach {tn schema} { - 1 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); - } - 2 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); - CREATE INDEX i1 ON t1(d); - CREATE INDEX i2 ON t1(d, c); - CREATE INDEX i3 ON t1(d, c, b); - CREATE INDEX i4 ON t1(b); - CREATE INDEX i5 ON t1(c); - CREATE INDEX i6 ON t1(c, b); - } - 3 { - CREATE TABLE t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID; - CREATE INDEX i1 ON t1(d); - CREATE INDEX i2 ON t1(d, c); - CREATE INDEX i3 ON t1(d, c, b); - CREATE INDEX i4 ON t1(b); - CREATE INDEX i5 ON t1(c); - CREATE INDEX i6 ON t1(c, b); - } - 4 { - CREATE TABLE t1(a PRIMARY KEY, b, c, d); - CREATE INDEX i1 ON t1(d); - CREATE INDEX i2 ON t1(d, c); - CREATE INDEX i3 ON t1(d, c, b); - CREATE INDEX i4 ON t1(b); - CREATE INDEX i5 ON t1(c); - CREATE INDEX i6 ON t1(c, b); - } - } { - reset_db - execsql $schema - execsql { - INSERT INTO t1 VALUES(1, 2, 3, 4); - INSERT INTO t1 VALUES(2, 5, 6, 7); - INSERT INTO t1 VALUES(3, 8, 9, 10); - } - - do_test 5.$tn2.$tn.1 { - create_ota5 ota.db - $cmd test.db ota.db - } {SQLITE_DONE} + reset_db + execsql $schema + execsql { + INSERT INTO t1 VALUES(2, 'hello', 'world'); + INSERT INTO t1 VALUES(4, 'hello', 'planet'); + INSERT INTO t1 VALUES(6, 'hello', 'xyz'); + } - do_execsql_test 5.$tn2.$tn.2 { - SELECT * FROM t1 ORDER BY a ASC; - } { - 1 2 3 5 - 2 5 10 5 - 3 11 9 10 + do_test $tn3.4.$tn2.$tn.1 { + create_ota4 ota.db + $cmd test.db ota.db + } {SQLITE_DONE} + + do_execsql_test $tn3.4.$tn2.$tn.2 { + SELECT * FROM t1 ORDER BY a ASC; + } { + 1 2 3 + 3 8 9 + 6 hello xyz + } + + do_execsql_test $tn3.4.$tn2.$tn.3 { PRAGMA integrity_check } ok } - - do_execsql_test 5.$tn2.$tn.3 { PRAGMA integrity_check } ok } -} -#------------------------------------------------------------------------- -# Test some error cases: -# -# * A virtual table with no ota_rowid column. -# * A no-PK table with no ota_rowid column. -# * A PK table with an ota_rowid column. -# -ifcapable fts3 { - foreach {tn schema error} { - 1 { - CREATE TABLE t1(a, b); - CREATE TABLE ota.data_t1(a, b, ota_control); - } {SQLITE_ERROR - table data_t1 requires ota_rowid column} - - 2 { - CREATE VIRTUAL TABLE t1 USING fts4(a, b); - CREATE TABLE ota.data_t1(a, b, ota_control); - } {SQLITE_ERROR - table data_t1 requires ota_rowid column} - - 3 { - CREATE TABLE t1(a PRIMARY KEY, b); - CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); - } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} - - 4 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b); - CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); - } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} - - 5 { - CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; - CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); - } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} - - } { - reset_db - forcedelete ota.db - execsql { ATTACH 'ota.db' AS ota } - execsql $schema - - do_test 6.$tn { - list [catch { run_ota test.db ota.db } msg] $msg - } [list 1 $error] + #------------------------------------------------------------------------- + # + foreach {tn2 cmd} {1 run_ota 2 step_ota} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } + 3 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID; + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } + 4 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d); + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } + } { + reset_db + execsql $schema + execsql { + INSERT INTO t1 VALUES(1, 2, 3, 4); + INSERT INTO t1 VALUES(2, 5, 6, 7); + INSERT INTO t1 VALUES(3, 8, 9, 10); + } + + do_test $tn3.5.$tn2.$tn.1 { + create_ota5 ota.db + $cmd test.db ota.db + } {SQLITE_DONE} + + do_execsql_test $tn3.5.$tn2.$tn.2 { + SELECT * FROM t1 ORDER BY a ASC; + } { + 1 2 3 5 + 2 5 10 5 + 3 11 9 10 + } + + do_execsql_test $tn3.5.$tn2.$tn.3 { PRAGMA integrity_check } ok + } } + + #------------------------------------------------------------------------- + # Test some error cases: + # + # * A virtual table with no ota_rowid column. + # * A no-PK table with no ota_rowid column. + # * A PK table with an ota_rowid column. + # + ifcapable fts3 { + foreach {tn schema error} { + 1 { + CREATE TABLE t1(a, b); + CREATE TABLE ota.data_t1(a, b, ota_control); + } {SQLITE_ERROR - table data_t1 requires ota_rowid column} + + 2 { + CREATE VIRTUAL TABLE t1 USING fts4(a, b); + CREATE TABLE ota.data_t1(a, b, ota_control); + } {SQLITE_ERROR - table data_t1 requires ota_rowid column} + + 3 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); + } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} + + 4 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); + } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} + + 5 { + CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); + } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} + + } { + reset_db + forcedelete ota.db + execsql { ATTACH 'ota.db' AS ota } + execsql $schema + + do_test $tn3.6.$tn { + list [catch { run_ota test.db ota.db } msg] $msg + } [list 1 $error] + } + } + + eval $destroy_vfs } diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 41207501cf..72c4333ed4 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -2350,13 +2350,34 @@ static int otaVfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){ */ static int otaVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ ota_file *p = (ota_file *)pFile; + int (*xControl)(sqlite3_file*,int,void*) = p->pReal->pMethods->xFileControl; + if( op==SQLITE_FCNTL_OTA ){ + int rc; sqlite3ota *pOta = (sqlite3ota*)pArg; - pOta->pTargetFd = p; - p->pOta = pOta; - return SQLITE_OK; + + /* First try to find another OTA vfs lower down in the vfs stack. If + ** one is found, this vfs will operate in pass-through mode. The lower + ** level vfs will do the special OTA handling. */ + rc = xControl(p->pReal, op, pArg); + + if( rc==SQLITE_NOTFOUND ){ + /* Now search for a zipvfs instance lower down in the VFS stack. If + ** one is found, this is an error. */ + void *dummy = 0; + rc = xControl(p->pReal, SQLITE_FCNTL_ZIPVFS_PAGER, &dummy); + if( rc==SQLITE_OK ){ + rc = SQLITE_ERROR; + pOta->zErrmsg = sqlite3_mprintf("ota/zipvfs setup error"); + }else if( rc==SQLITE_NOTFOUND ){ + pOta->pTargetFd = p; + p->pOta = pOta; + rc = SQLITE_OK; + } + } + return rc; } - return p->pReal->pMethods->xFileControl(p->pReal, op, pArg); + return xControl(p->pReal, op, pArg); } /* @@ -2896,9 +2917,76 @@ static int test_sqlite3ota( return TCL_OK; } +/* +** Tclcmd: sqlite3ota_create_vfs ?-default? NAME PARENT +*/ +static int test_sqlite3ota_create_vfs( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zName; + const char *zParent; + int rc; + + if( objc!=3 && objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?-default? NAME PARENT"); + return TCL_ERROR; + } + + zName = Tcl_GetString(objv[objc-2]); + zParent = Tcl_GetString(objv[objc-1]); + if( zParent[0]=='\0' ) zParent = 0; + + rc = sqlite3ota_create_vfs(zName, zParent); + if( rc!=SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + }else if( objc==4 ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zName); + sqlite3_vfs_register(pVfs, 1); + } + + Tcl_ResetResult(interp); + return TCL_OK; +} + +/* +** Tclcmd: sqlite3ota_destroy_vfs NAME +*/ +static int test_sqlite3ota_destroy_vfs( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zName; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NAME"); + return TCL_ERROR; + } + + zName = Tcl_GetString(objv[1]); + sqlite3ota_destroy_vfs(zName); + return TCL_OK; +} + int SqliteOta_Init(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "sqlite3ota", test_sqlite3ota, 0, 0); + static struct { + char *zName; + Tcl_ObjCmdProc *xProc; + } aObjCmd[] = { + { "sqlite3ota", test_sqlite3ota }, + { "sqlite3ota_create_vfs", test_sqlite3ota_create_vfs }, + { "sqlite3ota_destroy_vfs", test_sqlite3ota_destroy_vfs }, + }; + int i; + for(i=0; i Date: Tue, 10 Feb 2015 20:00:38 +0000 Subject: [PATCH 062/116] Further tweaks to work with zipvfs. FossilOrigin-Name: 0f152416be792457c52417aeb531ac860d12a5bd --- ext/ota/sqlite3ota.c | 48 ++++++++++++++++++++++++-------------------- main.mk | 3 +++ manifest | 16 +++++++-------- manifest.uuid | 2 +- tool/mksqlite3c.tcl | 4 +++- 5 files changed, 41 insertions(+), 32 deletions(-) diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 72c4333ed4..39f2042b96 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -75,7 +75,6 @@ #define OTA_STATE_COOKIE 7 #define OTA_STAGE_OAL 1 -#define OTA_STAGE_COPY 2 #define OTA_STAGE_CKPT 3 #define OTA_STAGE_DONE 4 @@ -175,7 +174,6 @@ struct sqlite3ota { sqlite3_ckpt *pCkpt; /* Incr-checkpoint handle */ ota_file *pTargetFd; /* File handle open on target db */ const char *zVfsName; /* Name of automatically created ota vfs */ - unsigned int iCookie; }; struct ota_vfs { @@ -190,6 +188,8 @@ struct ota_file { sqlite3_file *pReal; /* Underlying file handle */ ota_vfs *pOtaVfs; /* Pointer to the ota_vfs object */ sqlite3ota *pOta; /* Pointer to ota object (ota target only) */ + int openFlags; /* Flags this file was opened with */ + unsigned int iCookie; /* Cookie value for main db files */ int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ @@ -1469,6 +1469,10 @@ static int otaGetUpdateStmt( return p->rc; } +static void otaSqlTrace(void *pCtx, const char *zSql){ + /* printf("SQL: %s\n", zSql); */ +} + /* ** Open the database handle and attach the OTA database as "ota". If an ** error occurs, leave an error code and message in the OTA handle. @@ -1482,15 +1486,18 @@ static void otaOpenDatabase(sqlite3ota *p){ if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); }else{ + otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); + /* sqlite3_trace(p->db, otaSqlTrace, 0); */ + /* Mark the database file just opened as an OTA target database. If ** this call returns SQLITE_NOTFOUND, then the OTA vfs is not in use. ** This is an error. */ - p->rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_OTA, (void*)p); - if( p->rc==SQLITE_NOTFOUND ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("ota vfs not found"); - }else{ - otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_OTA, (void*)p); + if( p->rc==SQLITE_NOTFOUND ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("ota vfs not found"); + } } } } @@ -1855,7 +1862,7 @@ static void otaSaveTransactionState(sqlite3ota *p){ OTA_STATE_ROW, p->nStep, OTA_STATE_PROGRESS, p->nProgress, OTA_STATE_CKPT, - OTA_STATE_COOKIE, (sqlite3_int64)p->iCookie + OTA_STATE_COOKIE, (sqlite3_int64)p->pTargetFd->iCookie ) ); assert( pInsert==0 || rc==SQLITE_OK ); @@ -1902,7 +1909,7 @@ static void otaFreeState(OtaState *p){ static OtaState *otaLoadState(sqlite3ota *p){ const char *zSelect = "SELECT k, v FROM ota.ota_state"; OtaState *pRet = 0; - sqlite3_stmt *pStmt; + sqlite3_stmt *pStmt = 0; int rc; int rc2; @@ -1920,7 +1927,6 @@ static OtaState *otaLoadState(sqlite3ota *p){ case OTA_STATE_STAGE: pRet->eStage = sqlite3_column_int(pStmt, 1); if( pRet->eStage!=OTA_STAGE_OAL - && pRet->eStage!=OTA_STAGE_COPY && pRet->eStage!=OTA_STAGE_CKPT ){ p->rc = SQLITE_CORRUPT; @@ -1956,7 +1962,7 @@ static OtaState *otaLoadState(sqlite3ota *p){ ** committed in rollback mode) currently stored on page 1 of the ** database file. */ if( pRet->eStage==OTA_STAGE_OAL - && p->iCookie!=(unsigned int)sqlite3_column_int64(pStmt, 1) + && p->pTargetFd->iCookie!=(unsigned int)sqlite3_column_int64(pStmt, 1) ){ rc = SQLITE_BUSY; p->zErrmsg = sqlite3_mprintf("database modified during ota update"); @@ -2257,10 +2263,8 @@ static int otaVfsRead( ){ ota_file *p = (ota_file*)pFile; int rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); - if( rc==SQLITE_OK && p->pOta && iOfst==0 ){ - unsigned char *pBuf = (unsigned char*)zBuf; - assert( iAmt>=100 ); - p->pOta->iCookie = otaGetU32(&pBuf[24]); + if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ + p->iCookie = otaGetU32((unsigned char*)&zBuf[24]); } return rc; } @@ -2276,10 +2280,8 @@ static int otaVfsWrite( ){ ota_file *p = (ota_file*)pFile; int rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst); - if( rc==SQLITE_OK && p->pOta && iOfst==0 ){ - unsigned char *pBuf = (unsigned char*)zBuf; - assert( iAmt>=100 ); - p->pOta->iCookie = otaGetU32(&pBuf[24]); + if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ + p->iCookie = otaGetU32((unsigned char*)&zBuf[24]); } return rc; } @@ -2404,11 +2406,11 @@ static int otaVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ int rc = SQLITE_OK; #ifdef SQLITE_AMALGAMATION - assert( WAL_WRITE_CKPT==1 ); + assert( WAL_CKPT_LOCK==1 ); #endif if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){ - /* Magic number 1 is the WAL_WRITE_CKPT lock. Preventing SQLite from + /* Magic number 1 is the WAL_CKPT_LOCK lock. Preventing SQLite from ** taking this lock also prevents any checkpoints from occurring. ** todo: really, it's not clear why this might occur, as ** wal_autocheckpoint ought to be turned off. */ @@ -2452,6 +2454,7 @@ static int otaVfsShmMap( if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ + memset(pNew, 0, szRegion); p->apShm[iRegion] = pNew; } } @@ -2527,6 +2530,7 @@ static int otaVfsOpen( memset(pFd, 0, sizeof(ota_file)); pFd->pReal = (sqlite3_file*)&pFd[1]; pFd->pOtaVfs = pOtaVfs; + pFd->openFlags = flags; if( zName ){ if( flags & SQLITE_OPEN_MAIN_DB ){ /* A main database has just been opened. The following block sets diff --git a/main.mk b/main.mk index b9d613a78d..4727430718 100644 --- a/main.mk +++ b/main.mk @@ -219,6 +219,9 @@ SRC += \ SRC += \ $(TOP)/ext/userauth/userauth.c \ $(TOP)/ext/userauth/sqlite3userauth.h +SRC += \ + $(TOP)/ext/ota/sqlite3ota.c \ + $(TOP)/ext/ota/sqlite3ota.h # Generated source code files # diff --git a/manifest b/manifest index 7a19969c20..24b4a814d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sdocumentation\sand\stest\scases\sfor\ssqlite3ota_create_vfs().\sAlso\scode\sto\sdetect\serrors\sin\szipvfs/ota\ssetup. -D 2015-02-10T17:08:17.934 +C Further\stweaks\sto\swork\swith\szipvfs. +D 2015-02-10T20:00:38.125 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -134,7 +134,7 @@ F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c c73855939e124005f5c91fb50987297d50a81405 +F ext/ota/sqlite3ota.c 1f96966839c57e6a6f4ca8e8a771b23fbf79b8f6 F ext/ota/sqlite3ota.h 1cc7201086fe65a36957740381485a24738c4077 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -165,7 +165,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 1de9f345052b7cf631e3323b42bd35064cdfcf0a +F main.mk 57c115aba023c1988564edb80ac87c3e07472b05 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1220,7 +1220,7 @@ F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 94f196c9961e0ca3513e29f57125a3197808be2d F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 9ef48e1748dce7b844f67e2450ff9dfeb0fb4ab5 -F tool/mksqlite3c.tcl 6b8e572a90eb4e0086e3ba90d88b76c085919863 +F tool/mksqlite3c.tcl d8b0b0cc5f0e912058c9300f052769c62404d2d9 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mkvsix.tcl 52a4c613707ac34ae9c226e5ccc69cb948556105 @@ -1253,7 +1253,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 96443ecb6909141aa621a16e628455857d036482 -R 5e91280662dadb1373699ffdcf091961 +P e729668168f00325459bc2e9b515aa95e57f2754 +R 245c10d0f1613c672374bc7bdeb81ab9 U dan -Z 228647d512212dceaeb5d71668e39aca +Z 8b6e26dd39381b8c315cf4c234f64cec diff --git a/manifest.uuid b/manifest.uuid index 64d574f4ff..5c028557bb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e729668168f00325459bc2e9b515aa95e57f2754 \ No newline at end of file +0f152416be792457c52417aeb531ac860d12a5bd \ No newline at end of file diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 4ab8b12b45..4034128d5b 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -112,8 +112,9 @@ foreach hdr { pcache.h pragma.h rtree.h - sqlite3ext.h sqlite3.h + sqlite3ext.h + sqlite3ota.h sqliteicu.h sqliteInt.h sqliteLimit.h @@ -334,6 +335,7 @@ foreach file { rtree.c icu.c fts3_icu.c + sqlite3ota.c } { copy_file tsrc/$file } From 2e24971f2aeee5886f47efe196850116e6fb69d7 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 11 Feb 2015 16:25:27 +0000 Subject: [PATCH 063/116] Ensure that an error is reported if an attempt is made to update a wal mode database via ota. FossilOrigin-Name: 6fc5d4d26a603b3906f02ceea0f507780d0c35eb --- ext/ota/otaA.test | 82 ++++++++++++++++++++++++++++++++++++++++++++ ext/ota/sqlite3ota.c | 19 ++++++++-- manifest | 13 +++---- manifest.uuid | 2 +- 4 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 ext/ota/otaA.test diff --git a/ext/ota/otaA.test b/ext/ota/otaA.test new file mode 100644 index 0000000000..9ba9606e30 --- /dev/null +++ b/ext/ota/otaA.test @@ -0,0 +1,82 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the OTA module. More specifically, it +# contains tests to ensure that it is an error to attempt to update +# a wal mode database via OTA. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix otaA + +set db_sql { + CREATE TABLE t1(a PRIMARY KEY, b, c); +} +set ota_sql { + CREATE TABLE data_t1(a, b, c, ota_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(4, 5, 6, 0); + INSERT INTO data_t1 VALUES(7, 8, 9, 0); +} + +do_test 1.0 { + forcedelete test.db ota.db + + sqlite3 db test.db + db eval $db_sql + db eval { PRAGMA journal_mode = wal } + db close + + sqlite3 db ota.db + db eval $ota_sql + db close + + sqlite3ota ota test.db ota.db + ota step +} {SQLITE_ERROR} +do_test 1.1 { + list [catch { ota close } msg] $msg +} {1 {SQLITE_ERROR - cannot update wal mode database}} + +do_test 2.0 { + forcedelete test.db ota.db + + sqlite3 db test.db + db eval $db_sql + db close + + sqlite3 db ota.db + db eval $ota_sql + db close + + sqlite3ota ota test.db ota.db + ota step + ota close +} {SQLITE_OK} + +do_test 2.1 { + sqlite3 db test.db + db eval {PRAGMA journal_mode = wal} + db close + sqlite3ota ota test.db ota.db + ota step +} {SQLITE_ERROR} + +do_test 2.2 { + list [catch { ota close } msg] $msg +} {1 {SQLITE_ERROR - cannot update wal mode database}} + + +finish_test + diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 39f2042b96..5a97a9a57c 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -188,8 +188,10 @@ struct ota_file { sqlite3_file *pReal; /* Underlying file handle */ ota_vfs *pOtaVfs; /* Pointer to the ota_vfs object */ sqlite3ota *pOta; /* Pointer to ota object (ota target only) */ + int openFlags; /* Flags this file was opened with */ unsigned int iCookie; /* Cookie value for main db files */ + unsigned char iWriteVer; /* "write-version" value for main db files */ int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ @@ -2056,6 +2058,11 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ p->rc = sqlite3_exec(p->db, OTA_CREATE_STATE, 0, 0, &p->zErrmsg); } + if( p->rc==SQLITE_OK && p->pTargetFd->iWriteVer>1 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot update wal mode database"); + } + if( p->rc==SQLITE_OK ){ pState = otaLoadState(p); assert( pState || p->rc!=SQLITE_OK ); @@ -2264,7 +2271,11 @@ static int otaVfsRead( ota_file *p = (ota_file*)pFile; int rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ - p->iCookie = otaGetU32((unsigned char*)&zBuf[24]); + /* These look like magic numbers. But they are stable, as they are part + ** of the definition of the SQLite file format, which may not change. */ + unsigned char *pBuf = (unsigned char*)zBuf; + p->iCookie = otaGetU32(&pBuf[24]); + p->iWriteVer = pBuf[19]; } return rc; } @@ -2281,7 +2292,11 @@ static int otaVfsWrite( ota_file *p = (ota_file*)pFile; int rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst); if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ - p->iCookie = otaGetU32((unsigned char*)&zBuf[24]); + /* These look like magic numbers. But they are stable, as they are part + ** of the definition of the SQLite file format, which may not change. */ + unsigned char *pBuf = (unsigned char*)zBuf; + p->iCookie = otaGetU32(&pBuf[24]); + p->iWriteVer = pBuf[19]; } return rc; } diff --git a/manifest b/manifest index 24b4a814d1..5f28a04b52 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stweaks\sto\swork\swith\szipvfs. -D 2015-02-10T20:00:38.125 +C Ensure\sthat\san\serror\sis\sreported\sif\san\sattempt\sis\smade\sto\supdate\sa\swal\smode\sdatabase\svia\sota. +D 2015-02-11T16:25:27.816 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,8 +133,9 @@ F ext/ota/ota6.test 1fbba5fd46e3e0bfa5ae1d0caf9da27d15cb7cdf F ext/ota/ota7.test 1fe2c5761705374530e29f70c39693076028221a F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b +F ext/ota/otaA.test 95566a8d193113867b960eadf85b310937f2fe03 F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd -F ext/ota/sqlite3ota.c 1f96966839c57e6a6f4ca8e8a771b23fbf79b8f6 +F ext/ota/sqlite3ota.c 7400075206e6cb8cbb32fc7268cb2fcf6321ea66 F ext/ota/sqlite3ota.h 1cc7201086fe65a36957740381485a24738c4077 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -1253,7 +1254,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e729668168f00325459bc2e9b515aa95e57f2754 -R 245c10d0f1613c672374bc7bdeb81ab9 +P 0f152416be792457c52417aeb531ac860d12a5bd +R d9bfc5d0f25bbcabd7abf7a0a51ab0ed U dan -Z 8b6e26dd39381b8c315cf4c234f64cec +Z 85c9979bf4d433f3d16c5b8b0ec7a03f diff --git a/manifest.uuid b/manifest.uuid index 5c028557bb..d68e5af259 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f152416be792457c52417aeb531ac860d12a5bd \ No newline at end of file +6fc5d4d26a603b3906f02ceea0f507780d0c35eb \ No newline at end of file From 52800df9fcc2946ba507f64cafade052f62d7b7b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 14 Feb 2015 18:58:22 +0000 Subject: [PATCH 064/116] Change the way the "incremental checkpoint" function of OTA works in order to reduce the effect on the SQLite core code. FossilOrigin-Name: b64a11a754dc56f3406d3b703531ebe9e4af4908 --- ext/ota/ota1.test | 2 +- ext/ota/ota6.test | 24 +- ext/ota/sqlite3ota.c | 442 +++++++++++++++++++++++++--------- manifest | 28 +-- manifest.uuid | 2 +- src/main.c | 27 --- src/pager.c | 23 -- src/pager.h | 2 - src/sqlite.h.in | 37 --- src/wal.c | 556 ++++++++++++------------------------------- src/wal.h | 11 - 11 files changed, 492 insertions(+), 662 deletions(-) diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index dec5f14a99..addcde9446 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -198,7 +198,6 @@ foreach {tn3 create_vfs destroy_vfs} { do_test $tn3.1.$tn2.$tn.1 { create_ota1 ota.db - breakpoint $cmd test.db ota.db } {SQLITE_DONE} @@ -475,6 +474,7 @@ foreach {tn3 create_vfs destroy_vfs} { } } + catch { db close } eval $destroy_vfs } diff --git a/ext/ota/ota6.test b/ext/ota/ota6.test index 8027f36c5f..3b794ee77c 100644 --- a/ext/ota/ota6.test +++ b/ext/ota/ota6.test @@ -68,34 +68,12 @@ for {set nStep 1} {$nStep < 7} {incr nStep} { } {1 {SQLITE_BUSY - database modified during ota update}} } -for {set nStep 7} {$nStep < 8} {incr nStep} { - do_test 1.$nStep.1 { - setup_test - sqlite3ota ota test.db ota.db - for {set i 0} {$i<$nStep} {incr i} {ota step} - ota close - sqlite3 db test.db - execsql { INSERT INTO t1 VALUES(5, 'hello') } - sqlite3ota ota test.db ota.db - ota step - } {SQLITE_OK} - do_test 1.$nStep.2 { - ota step - } {SQLITE_OK} - do_test 1.$nStep.3 { - list [file exists test.db-oal] [file exists test.db-wal] - } {0 1} - do_test 1.$nStep.4 { - list [catch { ota close } msg] $msg - } {0 SQLITE_OK} -} - # Test the outcome of some other client writing the db after the *-oal # file has been copied to the *-wal path. Once this has happened, any # other client writing to the db causes OTA to consider its job finished. # -for {set nStep 8} {$nStep < 20} {incr nStep} { +for {set nStep 7} {$nStep < 20} {incr nStep} { do_test 1.$nStep.1 { setup_test sqlite3ota ota test.db ota.db diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index de7f1250c5..652c21a5f5 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -75,6 +75,7 @@ #define OTA_STATE_COOKIE 7 #define OTA_STAGE_OAL 1 +#define OTA_STAGE_CAPTURE 2 #define OTA_STAGE_CKPT 3 #define OTA_STAGE_DONE 4 @@ -87,6 +88,21 @@ typedef struct OtaObjIter OtaObjIter; typedef struct ota_vfs ota_vfs; typedef struct ota_file ota_file; +#if !defined(SQLITE_AMALGAMATION) +typedef unsigned int u32; +typedef unsigned char u8; +typedef sqlite3_int64 i64; +#endif + +/* +** These values must match the values defined in wal.c for the equivalent +** locks. These are not magic numbers as they are part of the SQLite file +** format. +*/ +#define WAL_LOCK_WRITE 0 +#define WAL_LOCK_CKPT 1 +#define WAL_LOCK_READ0 3 + /* ** A structure to store values read from the ota_state table in memory. */ @@ -94,10 +110,9 @@ struct OtaState { int eStage; char *zTbl; char *zIdx; - unsigned char *pCkptState; - int nCkptState; + i64 iWalCksum; int nRow; - sqlite3_int64 nProgress; + i64 nProgress; }; /* @@ -116,8 +131,8 @@ struct OtaObjIter { char **azTblCol; /* Array of unquoted target column names */ char **azTblType; /* Array of target column types */ int *aiSrcOrder; /* src table col -> target table col */ - unsigned char *abTblPk; /* Array of flags, set on target PK columns */ - unsigned char *abNotNull; /* Array of flags, set on NOT NULL columns */ + u8 *abTblPk; /* Array of flags, set on target PK columns */ + u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ int eType; /* Table type - an OTA_PK_XXX value */ /* Output variables. zTbl==0 implies EOF. */ @@ -158,6 +173,12 @@ struct OtaObjIter { #define OTA_PK_VTAB 5 +typedef struct OtaFrame OtaFrame; +struct OtaFrame { + u32 iDbPage; + u32 iWalFrame; +}; + /* ** OTA handle. */ @@ -171,16 +192,27 @@ struct sqlite3ota { int nStep; /* Rows processed for current object */ int nProgress; /* Rows processed for all objects */ OtaObjIter objiter; /* Iterator for skipping through tbl/idx */ - sqlite3_ckpt *pCkpt; /* Incr-checkpoint handle */ - ota_file *pTargetFd; /* File handle open on target db */ const char *zVfsName; /* Name of automatically created ota vfs */ + ota_file *pTargetFd; /* File handle open on target db */ + + /* The following state variables are used as part of the incremental + ** checkpoint stage (eStage==OTA_STAGE_CKPT). See function otaSetupCkpt() + ** for details. */ + u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ + u32 mLock; + int nFrame; /* Entries in aFrame[] array */ + int nFrameAlloc; /* Allocated size of aFrame[] array */ + OtaFrame *aFrame; + int pgsz; + u8 *aBuf; + i64 iWalCksum; }; struct ota_vfs { - sqlite3_vfs base; /* ota VFS shim methods */ - sqlite3_vfs *pRealVfs; /* Underlying VFS */ - sqlite3_mutex *mutex; - const char *zOtaWal; + sqlite3_vfs base; /* ota VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3_mutex *mutex; /* Mutex to protect pMain */ + ota_file *pMain; /* Linked list of main db files */ }; struct ota_file { @@ -190,13 +222,16 @@ struct ota_file { sqlite3ota *pOta; /* Pointer to ota object (ota target only) */ int openFlags; /* Flags this file was opened with */ - unsigned int iCookie; /* Cookie value for main db files */ - unsigned char iWriteVer; /* "write-version" value for main db files */ + u32 iCookie; /* Cookie value for main db files */ + u8 iWriteVer; /* "write-version" value for main db files */ int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ - const char *zWal; /* Wal filename for this db file */ char *zDel; /* Delete this when closing file */ + + const char *zWal; /* Wal filename for this main db file */ + ota_file *pWalFd; /* Wal file descriptor for this main db */ + ota_file *pMainNext; /* Next MAIN_DB file */ }; @@ -468,7 +503,7 @@ static void *otaMalloc(sqlite3ota *p, int nByte){ ** error code in the OTA handle passed as the first argument. */ static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ - int nByte = (2*sizeof(char*) + sizeof(int) + 2*sizeof(unsigned char)) * nCol; + int nByte = (2*sizeof(char*) + sizeof(int) + 2*sizeof(u8)) * nCol; char **azNew; azNew = (char**)otaMalloc(p, nByte); @@ -476,8 +511,8 @@ static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ pIter->azTblCol = azNew; pIter->azTblType = &azNew[nCol]; pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; - pIter->abTblPk = (unsigned char*)&pIter->aiSrcOrder[nCol]; - pIter->abNotNull = (unsigned char*)&pIter->abTblPk[nCol]; + pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol]; + pIter->abNotNull = (u8*)&pIter->abTblPk[nCol]; } } @@ -576,7 +611,7 @@ static int otaTableType( zSql = 0; if( pStmt==0 ) goto otaTableType_end; while( sqlite3_step(pStmt)==SQLITE_ROW ){ - const unsigned char *zOrig = sqlite3_column_text(pStmt,3); + const u8 *zOrig = sqlite3_column_text(pStmt,3); if( zOrig && zOrig[0]=='p' ){ zSql = sqlite3_mprintf("SELECT rootpage FROM main.sqlite_master" " WHERE name=%Q", sqlite3_column_text(pStmt,1)); @@ -713,7 +748,7 @@ static int otaObjIterCacheTableInfo(sqlite3ota *p, OtaObjIter *pIter){ pIter->azTblType[iOrder] = otaStrndup(zType, -1, &p->rc); pIter->abTblPk[iOrder] = (iPk!=0); - pIter->abNotNull[iOrder] = (unsigned char)bNotNull || (iPk!=0); + pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); iOrder++; } } @@ -1480,21 +1515,26 @@ static void otaOpenDatabase(sqlite3ota *p){ assert( p->rc==SQLITE_OK ); assert( p->db==0 ); + p->eStage = 0; p->rc = sqlite3_open_v2(p->zTarget, &p->db, flags, p->zVfsName); if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); }else{ - otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); + p->rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_OTA, (void*)p); + if( p->rc==SQLITE_OK ){ + otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); + } /* Mark the database file just opened as an OTA target database. If ** this call returns SQLITE_NOTFOUND, then the OTA vfs is not in use. ** This is an error. */ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_OTA, (void*)p); - if( p->rc==SQLITE_NOTFOUND ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("ota vfs not found"); - } + } + + if( p->rc==SQLITE_NOTFOUND ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("ota vfs not found"); } } } @@ -1533,6 +1573,101 @@ static void otaFileSuffix3(const char *zBase, char *z){ #endif } +/* +** Return the current wal-index header checksum for the target database +** as a 64-bit integer. +** +** The checksum is store in the first page of xShmMap memory as an 8-byte +** blob starting at byte offset 40. +*/ +static i64 otaShmChecksum(sqlite3ota *p){ + i64 iRet; + if( p->rc==SQLITE_OK ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + u32 volatile *ptr; + p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); + if( p->rc==SQLITE_OK ){ + iRet = ((i64)ptr[10] << 32) + ptr[11]; + } + } + return iRet; +} + +static void otaSetupCheckpoint(sqlite3ota *p, OtaState *pState){ + + if( pState==0 ){ + p->eStage = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->db, "SELECT * FROM sqlite_master", 0, 0, 0); + } + } + + if( p->rc==SQLITE_OK ){ + int rc2; + p->eStage = OTA_STAGE_CAPTURE; + rc2 = sqlite3_exec(p->db, "PRAGMA main.wal_checkpoint=restart", 0, 0, 0); + if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + } + + if( p->rc==SQLITE_OK ){ + p->eStage = OTA_STAGE_CKPT; + p->nStep = 0; + p->aBuf = otaMalloc(p, p->pgsz); + p->iWalCksum = otaShmChecksum(p); + } + + if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){ + p->rc = SQLITE_DONE; + p->eStage = OTA_STAGE_DONE; + } +} + +static int otaCaptureWalRead(sqlite3ota *pOta, i64 iOff, int iAmt){ + const u32 mReq = (1<mLock!=mReq ){ + return SQLITE_BUSY; + } + + pOta->pgsz = iAmt; + if( pOta->nFrame==pOta->nFrameAlloc ){ + int nNew = (pOta->nFrameAlloc ? pOta->nFrameAlloc : 64) * 2; + OtaFrame *aNew; + aNew = (OtaFrame*)sqlite3_realloc(pOta->aFrame, nNew * sizeof(OtaFrame)); + if( aNew==0 ) return SQLITE_NOMEM; + pOta->aFrame = aNew; + pOta->nFrameAlloc = nNew; + } + + iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; + if( pOta->iMaxFrame