Allow sqlite3session_apply() to apply changesets to tables that have been

extended using ALTER TABLE ADD COLUMN.

FossilOrigin-Name: b20ff81ff9c8af500ea96e0ba9d34524220a89f1

ext/session/session3.test

@@ -63,7 +63,10 @@ do_test 1.2.1 {
     INSERT INTO t1 VALUES(7, 8);
   }
   set ::log
-} {SQLITE_SCHEMA {sqlite3changeset_apply(): table t1 has 3 columns, expected 2}}
+} {}
+do_test 1.2.2 {
+  db2 eval { SELECT * FROM t1 }
+} {5 6 {} 7 8 {}}
 
 do_test 1.3.0 {
   execsql { 

ext/session/session_common.tcl

@@ -33,16 +33,11 @@ proc do_changeset_invert_test {tn session res} {
 
 
 proc do_conflict_test {tn args} {
-  proc xConflict {args} { 
-    lappend ::xConflict $args
-    return "" 
-  }
-  proc bgerror {args} { set ::background_error $args }
-
 
   set O(-tables)    [list]
   set O(-sql)       [list]
   set O(-conflicts) [list]
+  set O(-policy)    "OMIT"
 
   array set V $args
   foreach key [array names V] {
@@ -50,6 +45,12 @@ proc do_conflict_test {tn args} {
   }
   array set O $args
 
+  proc xConflict {args} [subst -nocommands { 
+    lappend ::xConflict [set args]
+    return $O(-policy) 
+  }]
+  proc bgerror {args} { set ::background_error $args }
+
   sqlite3session S db main
   foreach t $O(-tables) { S attach $t }
   execsql $O(-sql)

ext/session/sessionat.test

@@ -0,0 +1,216 @@
+# 2017 February 04
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for the sessions module. Specifically, that a changeset can
+# be applied after ALTER TABLE ADD COLUMN has been used to add 
+# columns to tables.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source [file join [file dirname [info script]] session_common.tcl]
+source $testdir/tester.tcl
+ifcapable !session {finish_test; return}
+
+set testprefix sessionat
+
+db close
+sqlite3_shutdown
+test_sqlite3_log log
+proc log {code msg} { lappend ::log $code $msg }
+
+proc reset_test {} {
+  catch { db  close }
+  catch { db2 close }
+  forcedelete test.db test.db2
+  sqlite3 db test.db
+  sqlite3 db2 test.db2
+}
+
+
+# Run all tests in this file twice. Once with "WITHOUT ROWID", and once
+# with regular rowid tables.
+#
+# ?.1.*: Test that PK inconsistencies are detected if one or more of the PK
+#        columns are not present in the changeset.
+#
+# ?.2.*: Test that it is not possible to apply a changeset with N columns
+#        to a db with fewer than N columns.
+#
+# ?.3.*: Test some INSERT, UPDATE and DELETE operations that do not
+#        require conflict handling.
+#
+# ?.4.*: Test some INSERT, UPDATE and DELETE operations that do require 
+#        conflict handling.
+#
+# ?.5.*: Test that attempting to concat two changesets with different
+#        numbers of columns for the same table is an error.
+#
+foreach {tn trailing} {
+  sessionat-ipk ""
+  sessionat-wor " WITHOUT ROWID "
+} {
+eval [string map [list %WR% $trailing] {
+  reset_test
+
+  #-----------------------------------------------------------------------
+  do_execsql_test $tn.1.0 {
+    CREATE TABLE t1(a, b, PRIMARY KEY(a)) %WR%;
+  }
+  do_execsql_test -db db2 $tn.1.1 {
+    CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)) %WR%;
+  }
+  do_test $tn.1.2 {
+    set ::log {}
+    do_then_apply_sql { INSERT INTO t1 VALUES('one', 'two') }
+    set ::log
+  } [list \
+    SQLITE_SCHEMA {sqlite3changeset_apply(): primary key mismatch for table t1}
+  ]
+  do_execsql_test $tn.1.3 { SELECT * FROM t1 } {one two}
+  do_execsql_test -db db2 $tn.1.4 { SELECT * FROM t1 } {}
+
+  #-----------------------------------------------------------------------
+  do_execsql_test $tn.2.0 {
+    CREATE TABLE t2(x, y, z, PRIMARY KEY(x)) %WR%;
+  }
+  do_execsql_test -db db2 $tn.2.1 {
+    CREATE TABLE t2(x, y, PRIMARY KEY(x)) %WR%;
+  }
+  do_test $tn.2.2 {
+    db cache flush
+    set ::log {}
+    do_then_apply_sql { INSERT INTO t2 VALUES(1, 2, 3) }
+    set ::log
+  } [list SQLITE_SCHEMA \
+    {sqlite3changeset_apply(): table t2 has 2 columns, expected 3 or more}
+  ]
+  do_execsql_test $tn.2.3 { SELECT * FROM t2 } {1 2 3}
+  do_execsql_test -db db2 $tn.2.4 { SELECT * FROM t2 } {}
+
+  #-----------------------------------------------------------------------
+  do_execsql_test $tn.3.0 {
+    CREATE TABLE t3(a, b, PRIMARY KEY(b)) %WR%;
+  }
+  do_execsql_test -db db2 $tn.3.1 {
+    CREATE TABLE t3(a, b, c DEFAULT 'D', PRIMARY KEY(b)) %WR%;
+  }
+  do_test $tn.3.2 {
+    do_then_apply_sql {
+      INSERT INTO t3 VALUES(1, 2);
+      INSERT INTO t3 VALUES(3, 4);
+      INSERT INTO t3 VALUES(5, 6);
+    };
+    db2 eval {SELECT * FROM t3}
+  } {1 2 D 3 4 D 5 6 D}
+  do_test $tn.3.3 {
+    do_then_apply_sql {
+      UPDATE t3 SET a=45 WHERE b=4;
+      DELETE FROM t3 WHERE a=5;
+    };
+    db2 eval {SELECT * FROM t3}
+  } {1 2 D 45 4 D}
+
+  #-----------------------------------------------------------------------
+  # 4.1: INSERT statements
+  # 4.2: DELETE statements
+  # 4.3: UPDATE statements
+  #  
+  do_execsql_test $tn.4.1.0 {
+    CREATE TABLE t4(x INTEGER PRIMARY KEY, y) %WR%;
+  }
+  do_execsql_test -db db2 $tn.4.1.1 {
+    CREATE TABLE t4(x INTEGER PRIMARY KEY, y, z) %WR%;
+    INSERT INTO t4 VALUES(1, 2, 3);
+    INSERT INTO t4 VALUES(4, 5, 6);
+  }
+  do_conflict_test $tn.4.1.2 -tables t4 -sql {
+    INSERT INTO t4 VALUES(10, 20);
+    INSERT INTO t4 VALUES(4, 11);
+  } -conflicts {
+    {INSERT t4 CONFLICT {i 4 i 11} {i 4 i 5}}
+  }
+  do_execsql_test -db db2 $tn.4.1.3 {
+    SELECT * FROM t4 ORDER BY x
+  } {1 2 3 4 5 6 10 20 {}}
+  do_conflict_test $tn.4.1.4 -policy REPLACE -tables t4 -sql {
+    INSERT INTO t4 VALUES(1, 11);
+  } -conflicts {
+    {INSERT t4 CONFLICT {i 1 i 11} {i 1 i 2}}
+  }
+  do_execsql_test -db db2 $tn.4.1.5 {
+    SELECT * FROM t4 ORDER BY x
+  } {1 11 {} 4 5 6 10 20 {}}
+
+  do_execsql_test $tn.4.2.0 {
+    DELETE FROM t4;
+    INSERT INTO t4 VALUES(1, 'A');
+    INSERT INTO t4 VALUES(2, 'B');
+    INSERT INTO t4 VALUES(3, 'C');
+    INSERT INTO t4 VALUES(4, 'D');
+  }
+  do_execsql_test -db db2 $tn.4.2.1 {
+    DELETE FROM t4;
+    INSERT INTO t4 VALUES(1, 'A', 'a');
+    INSERT INTO t4 VALUES(3, 'C', 'c');
+    INSERT INTO t4 VALUES(4, 'E', 'd');
+  }
+  do_conflict_test $tn.4.2.2 -tables t4 -sql {
+    DELETE FROM t4 WHERE x=2;
+    DELETE FROM t4 WHERE x=4;
+  } -conflicts {
+    {DELETE t4 NOTFOUND {i 2 t B}}
+    {DELETE t4 DATA {i 4 t D} {i 4 t E}}
+  }
+
+  do_execsql_test $tn.4.3.0 {
+    CREATE TABLE t5(a, b, c PRIMARY KEY) %WR%;
+    INSERT INTO t5 VALUES(1,1,1), (2,2,2), (3,3,3), (4,4,4);
+  }
+  do_execsql_test -db db2 $tn.4.3.1 {
+    CREATE TABLE t5(a, b, c PRIMARY KEY, d CHECK(b!=10)) %WR%;
+    INSERT INTO t5 VALUES (2,2,2,2), (3,8,3,3), (4,4,4,4);
+  }
+  do_conflict_test $tn.4.3.2 -tables t5 -sql {
+    UPDATE t5 SET a=4 WHERE c=1;
+    UPDATE t5 SET b=9 WHERE c=3;
+    UPDATE t5 SET b=10 WHERE c=2;
+  } -conflicts {
+    {UPDATE t5 NOTFOUND {i 1 {} {} i 1} {i 4 {} {} {} {}}}
+    {UPDATE t5 DATA {{} {} i 3 i 3} {{} {} i 9 {} {}} {i 3 i 8 i 3}}
+    {UPDATE t5 CONSTRAINT {{} {} i 2 i 2} {{} {} i 10 {} {}}}
+  }
+  
+  #-----------------------------------------------------------------------
+  do_execsql_test $tn.5.0 {
+    CREATE TABLE t6(a, b, c, PRIMARY KEY(a, b)) %WR%;
+  }
+  do_execsql_test -db db2 $tn.5.1 {
+    CREATE TABLE t6(a, b, c, d, e, PRIMARY KEY(a, b)) %WR%;
+  }
+  do_test $tn.5.2 {
+    set c1 [sql_exec_changeset db {
+      INSERT INTO t6 VALUES(1, 1, 1);
+      INSERT INTO t6 VALUES(2, 2, 2);
+    }]
+    set c2 [sql_exec_changeset db2 {
+      INSERT INTO t6 VALUES(3, 3, 3, 3, 3);
+      INSERT INTO t6 VALUES(4, 4, 4, 4, 4);
+    }]
+    list [catch { sqlite3changeset_concat $c1 $c2} msg] $msg
+  } {1 SQLITE_SCHEMA}
+
+}]
+}
+
+
+finish_test

ext/session/sqlite3session.c

@@ -3028,7 +3028,7 @@ int sqlite3changeset_conflict(
   if( !pIter->pConflict ){
     return SQLITE_MISUSE;
   }
-  if( iVal<0 || iVal>=sqlite3_column_count(pIter->pConflict) ){
+  if( iVal<0 || iVal>=pIter->nCol ){
     return SQLITE_RANGE;
   }
   *ppValue = sqlite3_column_value(pIter->pConflict, iVal);
@@ -3495,7 +3495,13 @@ static int sessionInsertRow(
 
   sessionAppendStr(&buf, "INSERT INTO main.", &rc);
   sessionAppendIdent(&buf, zTab, &rc);
-  sessionAppendStr(&buf, " VALUES(?", &rc);
+  sessionAppendStr(&buf, "(", &rc);
+  for(i=0; i<p->nCol; i++){
+    if( i!=0 ) sessionAppendStr(&buf, ", ", &rc);
+    sessionAppendIdent(&buf, p->azCol[i], &rc);
+  }
+
+  sessionAppendStr(&buf, ") VALUES(?", &rc);
   for(i=1; i<p->nCol; i++){
     sessionAppendStr(&buf, ", ?", &rc);
   }
@@ -4041,11 +4047,17 @@ static int sessionChangesetApply(
         nTab = (int)strlen(zTab);
         sApply.azCol = (const char **)zTab;
       }else{
+        int nMinCol = 0;
+        int i;
+
         sqlite3changeset_pk(pIter, &abPK, 0);
         rc = sessionTableInfo(
             db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK
         );
         if( rc!=SQLITE_OK ) break;
+        for(i=0; i<sApply.nCol; i++){
+          if( sApply.abPK[i] ) nMinCol = i+1;
+        }
   
         if( sApply.nCol==0 ){
           schemaMismatch = 1;
@@ -4053,26 +4065,29 @@ static int sessionChangesetApply(
               "sqlite3changeset_apply(): no such table: %s", zTab
           );
         }
-        else if( sApply.nCol!=nCol ){
+        else if( sApply.nCol<nCol ){
           schemaMismatch = 1;
           sqlite3_log(SQLITE_SCHEMA, 
-              "sqlite3changeset_apply(): table %s has %d columns, expected %d", 
+              "sqlite3changeset_apply(): table %s has %d columns, "
+              "expected %d or more", 
               zTab, sApply.nCol, nCol
           );
         }
-        else if( memcmp(sApply.abPK, abPK, nCol)!=0 ){
+        else if( nCol<nMinCol || memcmp(sApply.abPK, abPK, nCol)!=0 ){
           schemaMismatch = 1;
           sqlite3_log(SQLITE_SCHEMA, "sqlite3changeset_apply(): "
               "primary key mismatch for table %s", zTab
           );
         }
-        else if( 
+        else{
-            (rc = sessionSelectRow(db, zTab, &sApply))
+          sApply.nCol = nCol;
-         || (rc = sessionUpdateRow(db, zTab, &sApply))
+          if((rc = sessionSelectRow(db, zTab, &sApply))
-         || (rc = sessionDeleteRow(db, zTab, &sApply))
+          || (rc = sessionUpdateRow(db, zTab, &sApply))
-         || (rc = sessionInsertRow(db, zTab, &sApply))
+          || (rc = sessionDeleteRow(db, zTab, &sApply))
-        ){
+          || (rc = sessionInsertRow(db, zTab, &sApply))
-          break;
+          ){
+            break;
+          }
         }
         nTab = sqlite3Strlen30(zTab);
       }

ext/session/sqlite3session.h

@@ -319,7 +319,8 @@ int sqlite3session_changeset(
 **     the from-table, a DELETE record is added to the session object.
 **
 **   <li> For each row (primary key) that exists in both tables, but features 
-**     different in each, an UPDATE record is added to the session.
+**     different non-PK values in each, an UPDATE record is added to the
+**     session.  
 ** </ul>
 **
 ** To clarify, if this function is called and then a changeset constructed
@@ -904,7 +905,7 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
 ** <ul>
 **   <li> The table has the same name as the name recorded in the 
 **        changeset, and
-**   <li> The table has the same number of columns as recorded in the 
+**   <li> The table has at least as many columns as recorded in the 
 **        changeset, and
 **   <li> The table has primary key columns in the same position as 
 **        recorded in the changeset.
@@ -949,7 +950,11 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
 **   If a row with matching primary key values is found, but one or more of
 **   the non-primary key fields contains a value different from the original
 **   row value stored in the changeset, the conflict-handler function is
-**   invoked with [SQLITE_CHANGESET_DATA] as the second argument.
+**   invoked with [SQLITE_CHANGESET_DATA] as the second argument. If the
+**   database table has more columns than are recorded in the changeset,
+**   only the values of those non-primary key fields are compared against
+**   the current database contents - any trailing database table columns
+**   are ignored.
 **
 **   If no row with matching primary key values is found in the database,
 **   the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND]
@@ -964,7 +969,9 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
 **
 ** <dt>INSERT Changes<dd>
 **   For each INSERT change, an attempt is made to insert the new row into
-**   the database.
+**   the database. If the changeset row contains fewer fields than the
+**   database table, the trailing fields are populated with their default
+**   values.
 **
 **   If the attempt to insert the row fails because the database already 
 **   contains a row with the same primary key values, the conflict handler
@@ -982,13 +989,13 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
 **   For each UPDATE change, this function checks if the target database 
 **   contains a row with the same primary key value (or values) as the 
 **   original row values stored in the changeset. If it does, and the values 
-**   stored in all non-primary key columns also match the values stored in 
+**   stored in all modified non-primary key columns also match the values
-**   the changeset the row is updated within the target database.
+**   stored in the changeset the row is updated within the target database.
 **
 **   If a row with matching primary key values is found, but one or more of
-**   the non-primary key fields contains a value different from an original
+**   the modified non-primary key fields contains a value different from an
-**   row value stored in the changeset, the conflict-handler function is
+**   original row value stored in the changeset, the conflict-handler function
-**   invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since
+**   is invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since
 **   UPDATE changes only contain values for non-primary key fields that are
 **   to be modified, only those fields need to match the original values to
 **   avoid the SQLITE_CHANGESET_DATA conflict-handler callback.

manifest

@@ -1,5 +1,5 @@
-C In\sRTREE,\suse\san\ssqlite3_blob\sobject\srather\sthan\san\ssqlite3_stmt\sobject\nfor\sreading\scontent\sout\sof\sthe\s%_node\sshadow\stable.
+C Allow\ssqlite3session_apply()\sto\sapply\schangesets\sto\stables\sthat\shave\sbeen\nextended\susing\sALTER\sTABLE\sADD\sCOLUMN.
-D 2017-02-04T14:24:05.401
+D 2017-02-04T17:33:30.792
 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964
@@ -290,7 +290,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
 F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a
 F ext/session/session1.test c8a50e0e8581dc1a00e832aa59bb61f180404d44
 F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0
-F ext/session/session3.test a7a9ce59b8d1e49e2cc23d81421ac485be0eea01
+F ext/session/session3.test ce9ce3dfa489473987f899e9f6a0f2db9bde3479
 F ext/session/session4.test 457b02bdc349eb01151e54de014df77abd3c08c8
 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169
 F ext/session/session6.test 443789bc2fca12e4f7075cf692c60b8a2bea1a26
@@ -303,13 +303,14 @@ F ext/session/sessionD.test d4744c78334162851d2a2f285c7e603e31b49aa2
 F ext/session/sessionE.test e60a238c47f0feb3bb707e7f35e22be09c7e8f26
 F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce
 F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60
-F ext/session/session_common.tcl 9b696a341cf1d3744823715ed92bb19749b6c3d4
+F ext/session/session_common.tcl 7776eda579773113b30c7abfd4545c445228cb73
 F ext/session/session_speed_test.c edc1f96fd5e0e4b16eb03e2a73041013d59e8723
+F ext/session/sessionat.test b25d61d663ebc795506bf74079dc4ba0092fad25
 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c c61a43396368ec00dc127f7bc647e9bd6a4ee5fb
+F ext/session/sqlite3session.c 13642d9c754cc18f17e141f82860d269e2adf920
-F ext/session/sqlite3session.h 9345166bd8f80562145586cf817f707de5ecada2
+F ext/session/sqlite3session.h d4db650adfcc7a4360e9f12a09c2d117b1db6b53
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
@@ -1167,7 +1168,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl 2a49c1aff731f380ea640106377e19611a1443ae
+F test/tester.tcl 67835ac17e90055f24a9cf52e5c5bce0dd511c74
 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5
 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -1552,8 +1553,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5706d4708a30eb54da0ecbb6eb02f54746c390d9 95ee745fceb4a48c683f34c404c380fe5e7d684a
+P 97ccf3e4de11ffea46993cb7fb7ab559b9810705
-R ad6ed02dcc60e4bca51c672c64ac9b20
+R cad52333a5be0dfe0e9c0b2fb42ed068
-T +closed 95ee745fceb4a48c683f34c404c380fe5e7d684a
+U dan
-U drh
+Z 4713f44978189d862833bfda87ecea6f
-Z c59d505acee8b5f741c72c39e5c6cdac

manifest.uuid

@@ -1 +1 @@
-97ccf3e4de11ffea46993cb7fb7ab559b9810705
+b20ff81ff9c8af500ea96e0ba9d34524220a89f1

test/tester.tcl

@@ -919,10 +919,37 @@ proc normalize_list {L} {
   set L2
 }
 
-proc do_execsql_test {testname sql {result {}}} {
+# Either:
+#
+#   do_execsql_test TESTNAME SQL ?RES?
+#   do_execsql_test -db DB TESTNAME SQL ?RES?
+#
+proc do_execsql_test {args} {
+  set db db
+  if {[lindex $args 0]=="-db"} {
+    set db [lindex $args 1]
+    set args [lrange $args 2 end]
+  }
+
+  if {[llength $args]==2} {
+    foreach {testname sql} $args {}
+    set result ""
+  } elseif {[llength $args]==3} {
+    foreach {testname sql result} $args {}
+  } else {
+    error [string trim {
+      wrong # args: should be "do_execsql_test ?-db DB? testname sql ?result?"
+    }]
+  }
+
   fix_testname testname
-  uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]]
+
+  uplevel do_test                 \
+      [list $testname]            \
+      [list "execsql {$sql} $db"] \
+      [list [list {*}$result]]
 }
+
 proc do_catchsql_test {testname sql result} {
   fix_testname testname
   uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result]