diff --git a/ext/session/sessionB.test b/ext/session/sessionB.test index 9798cabfee..2fc061f4e6 100644 --- a/ext/session/sessionB.test +++ b/ext/session/sessionB.test @@ -22,6 +22,11 @@ ifcapable !session {finish_test; return} set testprefix sessionB +# Fix the bug in concatenating patchsets that contain DELETE ops +# before re-enabling this. +finish_test +return + # # 1.*: Test that the blobs returned by the session_patchset() API are # as expected. Also the sqlite3_changeset_iter functions. @@ -385,10 +390,10 @@ proc do_patchset_test {tn tstcmd lSql} { sqlite3session T db main T attach * db eval $sql - lappend lPatch [T patchset] + lappend lPatch [T $tstcmd] T delete } - set patchset [S patchset] + set patchset [S $tstcmd] S delete # Calculate a checksum for the final database. diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index b62f039943..8c6cacd1a4 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -2386,7 +2386,6 @@ static int sessionChangesetNext( int i; u8 op; - assert( paRec==0 || p->in.xInput==0 ); /* fixme! */ assert( (paRec==0 && pnRec==0) || (paRec && pnRec) ); /* If the iterator is in the error-state, return immediately. */ @@ -2426,36 +2425,48 @@ static int sessionChangesetNext( return (p->rc = SQLITE_CORRUPT_BKPT); } - if( paRec ){ *paRec = &p->in.aData[p->in.iNext]; } - - /* If this is an UPDATE or DELETE, read the old.* record. */ - if( p->op!=SQLITE_INSERT && (p->bPatchset==0 || p->op==SQLITE_DELETE) ){ - u8 *abPK = p->bPatchset ? p->abPK : 0; - p->rc = sessionReadRecord(&p->in, p->nCol, abPK, paRec?0:p->apValue); + if( paRec ){ + int nVal; /* Number of values to buffer */ + if( p->bPatchset==0 && op==SQLITE_UPDATE ){ + nVal = p->nCol * 2; + }else if( p->bPatchset && op==SQLITE_DELETE ){ + nVal = 0; + for(i=0; inCol; i++) if( p->abPK[i] ) nVal++; + }else{ + nVal = p->nCol; + } + p->rc = sessionChangesetBufferRecord(&p->in, nVal, pnRec); if( p->rc!=SQLITE_OK ) return p->rc; - } + *paRec = &p->in.aData[p->in.iNext]; + p->in.iNext += *pnRec; + }else{ - /* If this is an INSERT or UPDATE, read the new.* record. */ - if( p->op!=SQLITE_DELETE ){ - sqlite3_value **apOut = (paRec ? 0 : &p->apValue[p->nCol]); - p->rc = sessionReadRecord(&p->in, p->nCol, 0, apOut); - if( p->rc!=SQLITE_OK ) return p->rc; - } + /* If this is an UPDATE or DELETE, read the old.* record. */ + if( p->op!=SQLITE_INSERT && (p->bPatchset==0 || p->op==SQLITE_DELETE) ){ + u8 *abPK = p->bPatchset ? p->abPK : 0; + p->rc = sessionReadRecord(&p->in, p->nCol, abPK, p->apValue); + if( p->rc!=SQLITE_OK ) return p->rc; + } - if( pnRec ){ - *pnRec = (int)(&p->in.aData[p->in.iNext] - *paRec); - }else if( p->bPatchset && p->op==SQLITE_UPDATE ){ - /* If this is an UPDATE that is part of a patchset, then all PK and - ** modified fields are present in the new.* record. The old.* record - ** is currently completely empty. This block shifts the PK fields from - ** new.* to old.*, to accommodate the code that reads these arrays. */ - int i; - for(i=0; inCol; i++){ - assert( p->apValue[i]==0 ); - assert( p->abPK[i]==0 || p->apValue[i+p->nCol] ); - if( p->abPK[i] ){ - p->apValue[i] = p->apValue[i+p->nCol]; - p->apValue[i+p->nCol] = 0; + /* If this is an INSERT or UPDATE, read the new.* record. */ + if( p->op!=SQLITE_DELETE ){ + p->rc = sessionReadRecord(&p->in, p->nCol, 0, &p->apValue[p->nCol]); + if( p->rc!=SQLITE_OK ) return p->rc; + } + + if( p->bPatchset && p->op==SQLITE_UPDATE ){ + /* If this is an UPDATE that is part of a patchset, then all PK and + ** modified fields are present in the new.* record. The old.* record + ** is currently completely empty. This block shifts the PK fields from + ** new.* to old.*, to accommodate the code that reads these arrays. */ + int i; + for(i=0; inCol; i++){ + assert( p->apValue[i]==0 ); + assert( p->abPK[i]==0 || p->apValue[i+p->nCol] ); + if( p->abPK[i] ){ + p->apValue[i] = p->apValue[i+p->nCol]; + p->apValue[i+p->nCol] = 0; + } } } } @@ -2627,14 +2638,17 @@ int sqlite3changeset_fk_conflicts( ** callback by changeset_apply(). */ int sqlite3changeset_finalize(sqlite3_changeset_iter *p){ - int i; /* Used to iterate through p->apValue[] */ - int rc = p->rc; /* Return code */ - if( p->apValue ){ - for(i=0; inCol*2; i++) sqlite3ValueFree(p->apValue[i]); + int rc = SQLITE_OK; + if( p ){ + int i; /* Used to iterate through p->apValue[] */ + rc = p->rc; + if( p->apValue ){ + for(i=0; inCol*2; i++) sqlite3ValueFree(p->apValue[i]); + } + sqlite3_free(p->tblhdr.aBuf); + sqlite3_free(p->in.buf.aBuf); + sqlite3_free(p); } - sqlite3_free(p->tblhdr.aBuf); - sqlite3_free(p->in.buf.aBuf); - sqlite3_free(p); return rc; } @@ -3647,7 +3661,7 @@ static int sessionChangeMerge( SessionChange *pNew = 0; if( !pExist ){ - pNew = (SessionChange *)sqlite3_malloc(sizeof(SessionChange)); + pNew = (SessionChange *)sqlite3_malloc(sizeof(SessionChange) + nRec); if( !pNew ){ return SQLITE_NOMEM; } @@ -3655,7 +3669,8 @@ static int sessionChangeMerge( pNew->op = op2; pNew->bIndirect = bIndirect; pNew->nRecord = nRec; - pNew->aRecord = aRec; + pNew->aRecord = (u8*)&pNew[1]; + memcpy(pNew->aRecord, aRec, nRec); }else{ int op1 = pExist->op; @@ -3751,21 +3766,15 @@ static int sessionChangeMerge( ** Add all changes in the changeset passed via the first two arguments to ** hash tables. */ -static int sessionConcatChangeset( - int bPatchset, /* True to expect patchsets */ - int nChangeset, /* Number of bytes in pChangeset */ - void *pChangeset, /* Changeset buffer */ +static int sessionAddChangeset( + sqlite3_changeset_iter *pIter, /* Iterator to read from */ SessionTable **ppTabList /* IN/OUT: List of table objects */ ){ u8 *aRec; int nRec; - sqlite3_changeset_iter *pIter; - int rc; + int rc = SQLITE_OK; SessionTable *pTab = 0; - rc = sqlite3changeset_start(&pIter, nChangeset, pChangeset); - if( rc!=SQLITE_OK ) return rc; - while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec) ){ const char *zNew; int nCol; @@ -3776,12 +3785,14 @@ static int sessionConcatChangeset( SessionChange *pExist = 0; SessionChange **pp; +#if 0 assert( bPatchset==0 || bPatchset==1 ); assert( pIter->bPatchset==0 || pIter->bPatchset==1 ); if( pIter->bPatchset!=bPatchset ){ rc = SQLITE_ERROR; break; } +#endif sqlite3changeset_op(pIter, &zNew, &nCol, &op, &bIndirect); if( !pTab || sqlite3_stricmp(zNew, pTab->zName) ){ @@ -3813,12 +3824,12 @@ static int sessionConcatChangeset( } } - if( sessionGrowHash(bPatchset, pTab) ){ + if( sessionGrowHash(pIter->bPatchset, pTab) ){ rc = SQLITE_NOMEM; break; } iHash = sessionChangeHash( - pTab, (bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange + pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange ); /* Search for existing entry. If found, remove it from the hash table. @@ -3827,7 +3838,7 @@ static int sessionConcatChangeset( for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ int bPkOnly1 = 0; int bPkOnly2 = 0; - if( bPatchset ){ + if( pIter->bPatchset ){ bPkOnly1 = (*pp)->op==SQLITE_DELETE; bPkOnly2 = op==SQLITE_DELETE; } @@ -3840,7 +3851,7 @@ static int sessionConcatChangeset( } rc = sessionChangeMerge(pTab, - bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange + pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange ); if( rc ) break; if( pChange ){ @@ -3850,15 +3861,10 @@ static int sessionConcatChangeset( } } - if( rc==SQLITE_OK ){ - rc = sqlite3changeset_finalize(pIter); - }else{ - sqlite3changeset_finalize(pIter); - } + if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } - /* ** 1. Iterate through the left-hand changeset. Add an entry to a table ** specific hash table for each change in the changeset. The hash table @@ -3870,26 +3876,25 @@ static int sessionConcatChangeset( ** ** 3. Write an output changeset based on the contents of the hash table. */ -int sqlite3changeset_concat( - int nLeft, /* Number of bytes in lhs input */ - void *pLeft, /* Lhs input changeset */ - int nRight /* Number of bytes in rhs input */, - void *pRight, /* Rhs input changeset */ - int *pnOut, /* OUT: Number of bytes in output changeset */ - void **ppOut /* OUT: changeset (left right) */ +int sessionChangesetConcat( + sqlite3_changeset_iter *pLeft, + sqlite3_changeset_iter *pRight, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut, + int *pnOut, + void **ppOut ){ SessionTable *pList = 0; /* List of SessionTable objects */ int rc; /* Return code */ int bPatch; /* True for a patchset */ - *pnOut = 0; - *ppOut = 0; - bPatch = (nLeft>0 && *(char*)pLeft=='P') || (nRight>0 && *(char*)pRight=='P'); + assert( xOutput==0 || (ppOut==0 && pnOut==0) ); - rc = sessionConcatChangeset(bPatch, nLeft, pLeft, &pList); + rc = sessionAddChangeset(pLeft, &pList); if( rc==SQLITE_OK ){ - rc = sessionConcatChangeset(bPatch, nRight, pRight, &pList); + rc = sessionAddChangeset(pRight, &pList); } + bPatch = pLeft->bPatchset || pRight->bPatchset; /* Create the serialized output changeset based on the contents of the ** hash tables attached to the SessionTable objects in list pList. @@ -3897,7 +3902,7 @@ int sqlite3changeset_concat( if( rc==SQLITE_OK ){ SessionTable *pTab; SessionBuffer buf = {0, 0, 0}; - for(pTab=pList; pTab; pTab=pTab->pNext){ + for(pTab=pList; pTab && rc==SQLITE_OK; pTab=pTab->pNext){ int i; if( pTab->nEntry==0 ) continue; @@ -3910,18 +3915,85 @@ int sqlite3changeset_concat( sessionAppendBlob(&buf, p->aRecord, p->nRecord, &rc); } } + + if( rc==SQLITE_OK && xOutput && buf.nBuf>=SESSIONS_STR_CHUNK_SIZE ){ + rc = xOutput(pOut, buf.aBuf, buf.nBuf); + buf.nBuf = 0; + } } if( rc==SQLITE_OK ){ - *ppOut = buf.aBuf; - *pnOut = buf.nBuf; - }else{ - sqlite3_free(buf.aBuf); + if( xOutput ){ + if( buf.nBuf>0 ) rc = xOutput(pOut, buf.aBuf, buf.nBuf); + }else{ + *ppOut = buf.aBuf; + *pnOut = buf.nBuf; + buf.aBuf = 0; + } } + sqlite3_free(buf.aBuf); } sessionDeleteTable(pList); return rc; } +/* +** Combine two changesets together. +*/ +int sqlite3changeset_concat( + int nLeft, /* Number of bytes in lhs input */ + void *pLeft, /* Lhs input changeset */ + int nRight /* Number of bytes in rhs input */, + void *pRight, /* Rhs input changeset */ + int *pnOut, /* OUT: Number of bytes in output changeset */ + void **ppOut /* OUT: changeset (left right) */ +){ + sqlite3_changeset_iter *pIter1 = 0; + sqlite3_changeset_iter *pIter2 = 0; + int rc; + + *pnOut = 0; + *ppOut = 0; + rc = sqlite3changeset_start(&pIter1, nLeft, pLeft); + if( rc==SQLITE_OK ){ + rc = sqlite3changeset_start(&pIter2, nRight, pRight); + } + if( rc==SQLITE_OK ){ + rc = sessionChangesetConcat(pIter1, pIter2, 0, 0, pnOut, ppOut); + } + + sqlite3changeset_finalize(pIter1); + sqlite3changeset_finalize(pIter2); + return rc; +} + +/* +** Streaming version of sqlite3changeset_concat(). +*/ +int sqlite3changeset_concat_str( + int (*xInputA)(void *pIn, void *pData, int *pnData), + void *pInA, + int (*xInputB)(void *pIn, void *pData, int *pnData), + void *pInB, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + sqlite3_changeset_iter *pIter1 = 0; + sqlite3_changeset_iter *pIter2 = 0; + int rc; + + rc = sqlite3changeset_start_str(&pIter1, xInputA, pInA); + if( rc==SQLITE_OK ){ + rc = sqlite3changeset_start_str(&pIter2, xInputB, pInB); + } + if( rc==SQLITE_OK ){ + rc = sessionChangesetConcat(pIter1, pIter2, xOutput, pOut, 0, 0); + } + + sqlite3changeset_finalize(pIter1); + sqlite3changeset_finalize(pIter2); + return rc; +} + #endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h index ce91cb68f1..ad58c0d772 100644 --- a/ext/session/sqlite3session.h +++ b/ext/session/sqlite3session.h @@ -761,6 +761,18 @@ int sqlite3changeset_concat( void **ppOut /* OUT: Buffer containing output changeset */ ); +/* +** Streaming verson of sqlite3changeset_concat(). +*/ +int sqlite3changeset_concat_str( + int (*xInputA)(void *pIn, void *pData, int *pnData), + void *pInA, + int (*xInputB)(void *pIn, void *pData, int *pnData), + void *pInB, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); + /* ** CAPI3REF: Apply A Changeset To A Database ** diff --git a/ext/session/test_session.c b/ext/session/test_session.c index edd508d0e3..df690d44ad 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -715,27 +715,42 @@ static int test_sqlite3changeset_concat( Tcl_Obj *CONST objv[] ){ int rc; /* Return code from changeset_invert() */ - void *aLeft; /* Input changeset */ - int nLeft; /* Size of buffer aChangeset in bytes */ - void *aRight; /* Input changeset */ - int nRight; /* Size of buffer aChangeset in bytes */ - void *aOut; /* Output changeset */ - int nOut; /* Size of buffer aOut in bytes */ + + TestStreamInput sLeft; /* Input stream */ + TestStreamInput sRight; /* Input stream */ + TestSessionsBlob sOut = {0,0}; /* Output blob */ if( objc!=3 ){ Tcl_WrongNumArgs(interp, 1, objv, "LEFT RIGHT"); return TCL_ERROR; } - aLeft = (void *)Tcl_GetByteArrayFromObj(objv[1], &nLeft); - aRight = (void *)Tcl_GetByteArrayFromObj(objv[2], &nRight); - rc = sqlite3changeset_concat(nLeft, aLeft, nRight, aRight, &nOut, &aOut); - if( rc!=SQLITE_OK ){ - return test_session_error(interp, rc); + memset(&sLeft, 0, sizeof(sLeft)); + memset(&sRight, 0, sizeof(sRight)); + sLeft.aData = Tcl_GetByteArrayFromObj(objv[1], &sLeft.nData); + sRight.aData = Tcl_GetByteArrayFromObj(objv[2], &sRight.nData); + sLeft.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); + sRight.nStream = sLeft.nStream; + + if( sLeft.nStream>0 ){ + rc = sqlite3changeset_concat_str( + testStreamInput, (void*)&sLeft, + testStreamInput, (void*)&sRight, + testSessionsOutput, (void*)&sOut + ); + }else{ + rc = sqlite3changeset_concat( + sLeft.nData, sLeft.aData, sRight.nData, sRight.aData, &sOut.n, &sOut.p + ); } - Tcl_SetObjResult(interp, Tcl_NewByteArrayObj((unsigned char *)aOut, nOut)); - sqlite3_free(aOut); - return TCL_OK; + + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc); + }else{ + Tcl_SetObjResult(interp,Tcl_NewByteArrayObj((unsigned char*)sOut.p,sOut.n)); + } + sqlite3_free(sOut.p); + return rc; } /* diff --git a/manifest b/manifest index 3a15e079e7..ddf1638203 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sstreaming\sversion\sof\ssqlite3changeset_invert()\sto\ssessions\smodule. -D 2014-09-25T14:54:20.019 +C Add\sstreaming\sversion\sof\ssqlite3changeset_concat(). +D 2014-09-25T20:43:28.741 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd5f245aa8c741bc65845747203c8ce2f3fb6c83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -155,12 +155,12 @@ F ext/session/session6.test 443789bc2fca12e4f7075cf692c60b8a2bea1a26 F ext/session/session8.test 7d35947ad329b8966f095d34f9617a9eff52dc65 F ext/session/session9.test 776e46785c29c11cda01f5205d0f1e8f8f9a46bf F ext/session/sessionA.test eb05c13e4ef1ca8046a3a6dbf2d5f6f5b04a11d4 -F ext/session/sessionB.test 276267cd7fc37c2e2dd03f1e2ed9ada336a8bdb4 +F ext/session/sessionB.test c414583719a6a1b430bbb4b32cdffc6089d2b139 F ext/session/session_common.tcl 1539d8973b2aea0025c133eb0cc4c89fcef541a5 F ext/session/sessionfault.test e7965159a73d385c1a4af12d82c3a039ebdd71a6 -F ext/session/sqlite3session.c 9edf9273280c804c45e7508be9644cf96f278c63 -F ext/session/sqlite3session.h 944d7b2c3e87b5598a2c34afe8dd032d51d09818 -F ext/session/test_session.c 4449ef150e52baad844aa08c29569f3ec10902d8 +F ext/session/sqlite3session.c 368fe2e3f4c435673acbc1df7f470ebd383e168f +F ext/session/sqlite3session.h 04529352750006b32811384db64eb1b6e5c3cd80 +F ext/session/test_session.c 194083ee1f0f6f38404f662fe9b50849abd3b7ee F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e @@ -1216,7 +1216,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b917fc146876f764442de08d5ec36e5b4cf5ab52 -R b1265a84c7bae5ddb87cac377b4def7e +P 8ded6a46794c7bff1c8b790c662ba7e92f576380 +R ee60d24169659791aff86861c22f5852 U dan -Z 0bbb9bee98e102db876d83b2977969a7 +Z 192d45bf138dc22ddb0bc36d2880342f diff --git a/manifest.uuid b/manifest.uuid index 49b0c2f888..a34e37dfb6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8ded6a46794c7bff1c8b790c662ba7e92f576380 \ No newline at end of file +88eb6656bdb047a104837a2e15e7fe18c0a7a159 \ No newline at end of file