diff --git a/ext/session/session1.test b/ext/session/session1.test index 7aebe8c000..e97aa3c20d 100644 --- a/ext/session/session1.test +++ b/ext/session/session1.test @@ -206,8 +206,8 @@ do_conflict_test 3.1.2 -tables t1 -sql { INSERT INTO t1 VALUES(7, 'seven'); INSERT INTO t1 VALUES(8, NULL); } -conflicts { - {INSERT t1 CONSTRAINT {i 8 n {}}} {INSERT t1 CONFLICT {i 6 t six} {i 6 t VI}} + {INSERT t1 CONSTRAINT {i 8 n {}}} } do_db2_test 3.1.3 "SELECT * FROM t1" { @@ -272,9 +272,9 @@ do_conflict_test 3.3.3 -tables t4 -sql { UPDATE t4 SET a = NULL WHERE c = 9; UPDATE t4 SET a = 'x' WHERE b = 11; } -conflicts { - {UPDATE t4 CONSTRAINT {i 7 i 8 i 9} {n {} {} {} {} {}}} {UPDATE t4 DATA {i 1 i 2 i 3} {i -1 {} {} {} {}} {i 0 i 2 i 3}} {UPDATE t4 NOTFOUND {i 4 i 5 i 6} {i -1 {} {} {} {}}} + {UPDATE t4 CONSTRAINT {i 7 i 8 i 9} {n {} {} {} {} {}}} } do_db2_test 3.3.4 { SELECT * FROM t4 } {0 2 3 4 5 7 7 8 9 x 11 12} do_execsql_test 3.3.5 { SELECT * FROM t4 } {-1 2 3 -1 5 6 {} 8 9 x 11 12} diff --git a/ext/session/sessionG.test b/ext/session/sessionG.test new file mode 100644 index 0000000000..5ebdbbed6b --- /dev/null +++ b/ext/session/sessionG.test @@ -0,0 +1,66 @@ +# 2016 March 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for the sessions module. +# Specifically, it tests that UNIQUE constraints are dealt with correctly. +# + + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix sessionG + + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 1.0 { + do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + } + do_then_apply_sql { + DELETE FROM t1 WHERE a=1; + INSERT INTO t1 VALUES(4, 'one'); + } + compare_db db db2 +} {} + +do_test 1.1 { + do_then_apply_sql { + DELETE FROM t1 WHERE a=4; + INSERT INTO t1 VALUES(1, 'one'); + } + compare_db db db2 +} {} + +do_test 1.2 { + execsql { INSERT INTO t1 VALUES(5, 'five') } db2 + do_then_apply_sql { + INSERT INTO t1 VALUES(11, 'eleven'); + INSERT INTO t1 VALUES(12, 'five'); + } + execsql { SELECT * FROM t1 } db2 +} {2 two 3 three 1 one 5 five 11 eleven} + +do_test 1.3 { + execsql { SELECT * FROM t1 } +} {2 two 3 three 1 one 11 eleven 12 five} + +finish_test + diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 312e03dcfb..826911f299 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -67,6 +67,8 @@ struct SessionBuffer { ** sqlite3changeset_start_strm()). */ struct SessionInput { + int bNoDiscard; /* If true, discard no data */ + int iCurrent; /* Offset in aData[] of current change */ int iNext; /* Offset in aData[] of next change */ u8 *aData; /* Pointer to buffer containing changeset */ int nData; /* Number of bytes in aData */ @@ -2460,7 +2462,6 @@ static int sessionChangesetStart( pRet->in.nData = nChangeset; pRet->in.xInput = xInput; pRet->in.pIn = pIn; - pRet->in.iNext = 0; pRet->in.bEof = (xInput ? 0 : 1); /* Populate the output variable and return success. */ @@ -2490,6 +2491,23 @@ int sqlite3changeset_start_strm( return sessionChangesetStart(pp, xInput, pIn, 0, 0); } +/* +** If the SessionInput object passed as the only argument is a streaming +** object and the buffer is full, discard some data to free up space. +*/ +static void sessionDiscardData(SessionInput *pIn){ + if( pIn->bEof && pIn->xInput && pIn->iNext>=SESSIONS_STRM_CHUNK_SIZE ){ + int nMove = pIn->buf.nBuf - pIn->iNext; + assert( nMove>=0 ); + if( nMove>0 ){ + memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iNext], nMove); + } + pIn->buf.nBuf -= pIn->iNext; + pIn->iNext = 0; + pIn->nData = pIn->buf.nBuf; + } +} + /* ** Ensure that there are at least nByte bytes available in the buffer. Or, ** if there are not nByte bytes remaining in the input, that all available @@ -2503,13 +2521,7 @@ static int sessionInputBuffer(SessionInput *pIn, int nByte){ while( !pIn->bEof && (pIn->iNext+nByte)>=pIn->nData && rc==SQLITE_OK ){ int nNew = SESSIONS_STRM_CHUNK_SIZE; - if( pIn->iNext>=SESSIONS_STRM_CHUNK_SIZE ){ - int nMove = pIn->buf.nBuf - pIn->iNext; - memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iNext], nMove); - pIn->buf.nBuf -= pIn->iNext; - pIn->iNext = 0; - } - + if( pIn->bNoDiscard==0 ) sessionDiscardData(pIn); if( SQLITE_OK==sessionBufferGrow(&pIn->buf, nNew, &rc) ){ rc = pIn->xInput(pIn->pIn, &pIn->buf.aBuf[pIn->buf.nBuf], &nNew); if( nNew==0 ){ @@ -2818,11 +2830,15 @@ static int sessionChangesetNext( return SQLITE_DONE; } + sessionDiscardData(&p->in); + p->in.iCurrent = p->in.iNext; + op = p->in.aData[p->in.iNext++]; if( op=='T' || op=='P' ){ p->bPatchset = (op=='P'); if( sessionChangesetReadTblhdr(p) ) return p->rc; if( (p->rc = sessionInputBuffer(&p->in, 2)) ) return p->rc; + p->in.iCurrent = p->in.iNext; op = p->in.aData[p->in.iNext++]; } @@ -3266,6 +3282,9 @@ struct SessionApplyCtx { int nCol; /* Size of azCol[] and abPK[] arrays */ const char **azCol; /* Array of column names */ u8 *abPK; /* Boolean array - true if column is in PK */ + + int bDeferConstraints; /* True to defer constraints */ + SessionBuffer constraints; /* Deferred constraints are stored here */ }; /* @@ -3516,7 +3535,7 @@ static int sessionBindValue( ** transfers new.* values from the current iterator entry to statement ** pStmt. The table being inserted into has nCol columns. ** -** New.* value $i 0 from the iterator is bound to variable ($i+1) of +** New.* value $i from the iterator is bound to variable ($i+1) of ** statement pStmt. If parameter abPK is NULL, all values from 0 to (nCol-1) ** are transfered to the statement. Otherwise, if abPK is not NULL, it points ** to an array nCol elements in size. In this case only those values for @@ -3662,9 +3681,18 @@ static int sessionConflictHandler( pIter->pConflict = 0; rc = sqlite3_reset(p->pSelect); }else if( rc==SQLITE_OK ){ - /* No other row with the new.* primary key. */ - res = xConflict(pCtx, eType+1, pIter); - if( res==SQLITE_CHANGESET_REPLACE ) rc = SQLITE_MISUSE; + if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){ + /* Instead of invoking the conflict handler, append the change blob + ** to the SessionApplyCtx.constraints buffer. */ + u8 *aBlob = &pIter->in.aData[pIter->in.iCurrent]; + int nBlob = pIter->in.iNext - pIter->in.iCurrent; + sessionAppendBlob(&p->constraints, aBlob, nBlob, &rc); + res = SQLITE_CHANGESET_OMIT; + }else{ + /* No other row with the new.* primary key. */ + res = xConflict(pCtx, eType+1, pIter); + if( res==SQLITE_CHANGESET_REPLACE ) rc = SQLITE_MISUSE; + } } if( rc==SQLITE_OK ){ @@ -3824,6 +3852,96 @@ static int sessionApplyOneOp( return rc; } +static int sessionApplyOneWithRetry( + sqlite3 *db, /* Apply change to "main" db of this handle */ + sqlite3_changeset_iter *pIter, /* Changeset iterator to read change from */ + SessionApplyCtx *pApply, /* Apply context */ + int(*xConflict)(void*, int, sqlite3_changeset_iter*), + void *pCtx /* First argument passed to xConflict */ +){ + int bReplace = 0; + int bRetry = 0; + int rc; + + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, &bReplace, &bRetry); + + if( rc==SQLITE_OK && bRetry ){ + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, &bReplace, 0); + } + + if( bReplace ){ + assert( pIter->op==SQLITE_INSERT ); + rc = sqlite3_exec(db, "SAVEPOINT replace_op", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = sessionBindRow(pIter, + sqlite3changeset_new, pApply->nCol, pApply->abPK, pApply->pDelete); + sqlite3_bind_int(pApply->pDelete, pApply->nCol+1, 1); + } + if( rc==SQLITE_OK ){ + sqlite3_step(pApply->pDelete); + rc = sqlite3_reset(pApply->pDelete); + } + if( rc==SQLITE_OK ){ + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, "RELEASE replace_op", 0, 0, 0); + } + } + + return rc; +} + +/* +** Retry the changes accumulated in the pApply->constraints buffer. +*/ +static int sessionRetryConstraints( + sqlite3 *db, + int bPatchset, + const char *zTab, + SessionApplyCtx *pApply, + int(*xConflict)(void*, int, sqlite3_changeset_iter*), + void *pCtx /* First argument passed to xConflict */ +){ + int rc = SQLITE_OK; + + while( pApply->constraints.nBuf ){ + sqlite3_changeset_iter *pIter2 = 0; + SessionBuffer cons = pApply->constraints; + memset(&pApply->constraints, 0, sizeof(SessionBuffer)); + + rc = sessionChangesetStart(&pIter2, 0, 0, cons.nBuf, cons.aBuf); + if( rc==SQLITE_OK ){ + int nByte = 2*pApply->nCol*sizeof(sqlite3_value*); + int rc2; + pIter2->bPatchset = bPatchset; + pIter2->zTab = (char*)zTab; + pIter2->nCol = pApply->nCol; + pIter2->abPK = pApply->abPK; + sessionBufferGrow(&pIter2->tblhdr, nByte, &rc); + pIter2->apValue = (sqlite3_value**)pIter2->tblhdr.aBuf; + if( rc==SQLITE_OK ) memset(pIter2->apValue, 0, nByte); + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3changeset_next(pIter2) ){ + rc = sessionApplyOneWithRetry(db, pIter2, pApply, xConflict, pCtx); + } + + rc2 = sqlite3changeset_finalize(pIter2); + if( rc==SQLITE_OK ) rc==rc2; + } + assert( pApply->bDeferConstraints || pApply->constraints.nBuf==0 ); + + sqlite3_free(cons.aBuf); + if( rc!=SQLITE_OK ) break; + if( pApply->constraints.nBuf>=cons.nBuf ){ + /* No progress was made on the last round. */ + pApply->bDeferConstraints = 0; + } + } + + return rc; +} + /* ** Argument pIter is a changeset iterator that has been initialized, but ** not yet passed to sqlite3changeset_next(). This function applies the @@ -3853,6 +3971,7 @@ static int sessionChangesetApply( assert( xConflict!=0 ); + pIter->in.bNoDiscard = 1; memset(&sApply, 0, sizeof(sApply)); sqlite3_mutex_enter(sqlite3_db_mutex(db)); rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); @@ -3862,8 +3981,6 @@ static int sessionChangesetApply( while( rc==SQLITE_OK && SQLITE_ROW==sqlite3changeset_next(pIter) ){ int nCol; int op; - int bReplace = 0; - int bRetry = 0; const char *zNew; sqlite3changeset_op(pIter, &zNew, &nCol, &op, 0); @@ -3871,6 +3988,11 @@ static int sessionChangesetApply( if( zTab==0 || sqlite3_strnicmp(zNew, zTab, nTab+1) ){ u8 *abPK; + rc = sessionRetryConstraints( + db, pIter->bPatchset, zTab, &sApply, xConflict, pCtx + ); + if( rc!=SQLITE_OK ) break; + sqlite3_free((char*)sApply.azCol); /* cast works around VC++ bug */ sqlite3_finalize(sApply.pDelete); sqlite3_finalize(sApply.pUpdate); @@ -3878,6 +4000,7 @@ static int sessionChangesetApply( sqlite3_finalize(sApply.pSelect); memset(&sApply, 0, sizeof(sApply)); sApply.db = db; + sApply.bDeferConstraints = 1; /* If an xFilter() callback was specified, invoke it now. If the ** xFilter callback returns zero, skip this table. If it returns @@ -3933,31 +4056,13 @@ static int sessionChangesetApply( ** next change. A log message has already been issued. */ if( schemaMismatch ) continue; - rc = sessionApplyOneOp(pIter, &sApply, xConflict, pCtx, &bReplace, &bRetry); + rc = sessionApplyOneWithRetry(db, pIter, &sApply, xConflict, pCtx); + } - if( rc==SQLITE_OK && bRetry ){ - rc = sessionApplyOneOp(pIter, &sApply, xConflict, pCtx, &bReplace, 0); - } - - if( bReplace ){ - assert( pIter->op==SQLITE_INSERT ); - rc = sqlite3_exec(db, "SAVEPOINT replace_op", 0, 0, 0); - if( rc==SQLITE_OK ){ - rc = sessionBindRow(pIter, - sqlite3changeset_new, sApply.nCol, sApply.abPK, sApply.pDelete); - sqlite3_bind_int(sApply.pDelete, sApply.nCol+1, 1); - } - if( rc==SQLITE_OK ){ - sqlite3_step(sApply.pDelete); - rc = sqlite3_reset(sApply.pDelete); - } - if( rc==SQLITE_OK ){ - rc = sessionApplyOneOp(pIter, &sApply, xConflict, pCtx, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(db, "RELEASE replace_op", 0, 0, 0); - } - } + if( rc==SQLITE_OK ){ + rc = sessionRetryConstraints( + db, pIter->bPatchset, zTab, &sApply, xConflict, pCtx + ); } if( rc==SQLITE_OK ){ @@ -3994,6 +4099,7 @@ static int sessionChangesetApply( sqlite3_finalize(sApply.pUpdate); sqlite3_finalize(sApply.pSelect); sqlite3_free((char*)sApply.azCol); /* cast works around VC++ bug */ + sqlite3_free((char*)sApply.constraints.aBuf); sqlite3_mutex_leave(sqlite3_db_mutex(db)); return rc; } diff --git a/manifest b/manifest index c602040523..27e749b9fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sfor\sthe\sMSVC\smakefiles. -D 2016-03-30T16:23:06.801 +C Have\sthe\ssqlite3session_apply()\sfunction\sand\sits\sstreaming\sequivalent\sretry\sany\soperations\sthat\sfailed\swith\sSQLITE_CONSTRAINT\safter\sall\sother\soperations\son\sthe\ssame\stable\shave\sbeen\sattempted.\sNew\scode\sis\slargely\suntested. +D 2016-03-30T21:19:00.012 F Makefile.in e812bb732d7af01baa09f1278bd4f4a2e3a09449 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc cde766eb7c27a7ca42000e66c5f0c37a17a05998 @@ -273,7 +273,7 @@ F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a -F ext/session/session1.test 5dab50ce55c859e829bae24f0787013f51775fc5 +F ext/session/session1.test 98f384736e2bc21ccf5ed81bdadcff4ad863393b F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 F ext/session/session3.test a7a9ce59b8d1e49e2cc23d81421ac485be0eea01 F ext/session/session4.test a6ed685da7a5293c5d6f99855bcf41dbc352ca84 @@ -287,9 +287,10 @@ F ext/session/sessionC.test 97556f5164ac29f2344b24bd7de6a3a35a95c390 F ext/session/sessionD.test d4744c78334162851d2a2f285c7e603e31b49aa2 F ext/session/sessionE.test e60a238c47f0feb3bb707e7f35e22be09c7e8f26 F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce +F ext/session/sessionG.test 388b4f311176411301c35f90f0bf2cb7c6c3dec9 F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596 F ext/session/sessionfault.test d52cbb7bee48cc8ee80335e07eb72fcb6b15eb40 -F ext/session/sqlite3session.c b10af3e87ae437bb197b3a23a584d2dc8ad8981a +F ext/session/sqlite3session.c e682828d3ca542617b54808e6d551559e802a605 F ext/session/sqlite3session.h 64e9e7f185725ef43b97f4a9a0c0df0669844f1d F ext/session/test_session.c 187bd344c5ae9d5be85e22ef7c3010f0c17307ce F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 @@ -1480,7 +1481,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 64d75cbe2c45af67124fa7ce5688d91cc6ddb755 -R 77269b95e931ac6cca96bcd2fa718a38 -U mistachkin -Z 37f983898af724eda9f571ecd8a4d862 +P 7cf0cab730e2d570c82dd789279ad6501ac598c8 +R ee6c26b884210802b6544dcc00c10c18 +T *branch * session-retry +T *sym-session-retry * +T -sym-trunk * +U dan +Z 9b2c9e9e66a7c0858e38132bf4058066 diff --git a/manifest.uuid b/manifest.uuid index 2aa68317b9..b9607529d1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7cf0cab730e2d570c82dd789279ad6501ac598c8 \ No newline at end of file +1085911afb51744f32fe9db183b50e8e88bdd73e \ No newline at end of file