Add the --rbu switch to the "sqldiff" utility.

FossilOrigin-Name: 098bea26da4533d9ad97a85687cca56fb0d764a4

ext/rbu/rbudiff.test

@@ -0,0 +1,100 @@
+# 2015-07-31
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for the [sqldiff --rbu] command.
+#
+#
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
+set testprefix rbudiff
+
+if {$tcl_platform(platform)=="windows"} {
+  set PROG "sqldiff.exe"
+} else {
+  set PROG "./sqldiff"
+}
+if {![file exe $PROG]} {
+  puts "rbudiff.test cannot run because $PROG is not available"
+  finish_test
+  return
+}
+db close
+
+proc get_rbudiff_sql {db1 db2} {
+  exec $::PROG --rbu $db1 $db2
+}
+
+proc step_rbu {target rbu} {
+  while 1 {
+    sqlite3rbu rbu $target $rbu
+    set rc [rbu step]
+    rbu close
+    if {$rc != "SQLITE_OK"} break
+  }
+  set rc
+}
+
+proc apply_rbudiff {sql target} {
+  forcedelete rbu.db
+  sqlite3 rbudb rbu.db
+  rbudb eval $sql
+  rbudb close
+  step_rbu $target rbu.db
+}
+
+proc rbudiff_cksum {db1} {
+  sqlite3 dbtmp $db1
+  set txt [dbtmp eval {
+    SELECT a || '.' || b || '.' || c FROM t1 ORDER BY 1;
+    SELECT a || '.' || b || '.' || c FROM t2 ORDER BY 1;
+  }]
+  dbtmp close
+  md5 $txt
+}
+
+sqlite3 db test.db
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a PRIMARY KEY, b, c);
+  INSERT INTO t1 VALUES(1, 2, 3);
+  INSERT INTO t1 VALUES(4, 5, 6);
+
+  CREATE TABLE t2(a, b, c, PRIMARY KEY(b, c));
+  INSERT INTO t2 VALUES(1, 2, 3);
+  INSERT INTO t2 VALUES(4, 5, 6);
+}
+
+db close
+forcedelete test.db2
+forcecopy test.db test.db2
+
+sqlite3 db test.db
+do_execsql_test 1.1 {
+  INSERT INTO t1 VALUES(7, 8, 9);
+  DELETE FROM t1 WHERE a=4;
+  UPDATE t1 SET c = 11 WHERE a = 1;
+
+  INSERT INTO t2 VALUES(7, 8, 9);
+  DELETE FROM t2 WHERE a=4;
+  UPDATE t2 SET c = 11 WHERE a = 1;
+}
+db close
+
+do_test 1.2 {
+  set sql [get_rbudiff_sql test.db test.db2]
+  apply_rbudiff $sql test.db
+} {SQLITE_DONE}
+
+do_test 1.3 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
+
+finish_test
+

manifest

@@ -1,5 +1,5 @@
-C Allow\sRBU\stables\sto\sbe\snamed\s"data[0-9]*_<target>"\sinstead\sof\sstrictly\s"data_<target>".\sAlso\supdate\sRBU\sso\sthat\sit\salways\sprocesses\sdata\stables\sin\sorder\ssorted\sby\sname.
+C Add\sthe\s--rbu\sswitch\sto\sthe\s"sqldiff"\sutility.
-D 2015-07-30T11:38:19.207
+D 2015-07-30T20:26:16.763
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 4de3ef40c8b3b75c0c55ff4242a43c8ce1ad90ee
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -215,6 +215,7 @@ F ext/rbu/rbu8.test 3bbf2c35d71a843c463efe93946f14ad10c3ede0
 F ext/rbu/rbu9.test 0806d1772c9f4981774ff028de6656e4183082af
 F ext/rbu/rbuA.test c1a7b3e2d926b8f8448bb3b4ae787e314ee4b2b3
 F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695
+F ext/rbu/rbudiff.test deba11b97a41163e39de49d7455eb6429d0bd510
 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89
 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufts.test 17db7d968b3d073788bcba044c498d09e830726b
@@ -1356,7 +1357,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
-F tool/sqldiff.c 0748c0daed08f31e5a8eab6de98ca57500e61ecf
+F tool/sqldiff.c 7ed3a042126755e87266c3e1dbbce39a20455a1f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
 F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f
@@ -1367,7 +1368,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 8e818b8985c0196cd9671a6491796faaeebeb16e
+P 287aa30601506f168d355c35176a3383474444ca
-R d2e717e7df793b66cba7be5b1c5545b9
+R 4b1cd3413787b049b8357f11d80a75d6
 U dan
-Z b15734d6bf397de4fcabb387cb6f82da
+Z 5d367f3cdab2e7ca1b015457b0b5954e

manifest.uuid

@@ -1 +1 @@
-287aa30601506f168d355c35176a3383474444ca
+098bea26da4533d9ad97a85687cca56fb0d764a4

tool/sqldiff.c

@@ -259,7 +259,12 @@ static void namelistFree(char **az){
 **    CREATE TABLE t5(rowid,_rowid_,oid);
 **    az = 0     // The rowid is not accessible
 */
-static char **columnNames(const char *zDb, const char *zTab, int *pnPKey){
+static char **columnNames(
+  const char *zDb,                /* Database ("main" or "aux") to query */
+  const char *zTab,               /* Name of table to return details of */
+  int *pnPKey,                    /* OUT: Number of PK columns */
+  int *pbRowid                    /* OUT: True if PK is an implicit rowid */
+){
   char **az = 0;           /* List of column names to be returned */
   int naz = 0;             /* Number of entries in az[] */
   sqlite3_stmt *pStmt;     /* SQL statement being run */
@@ -338,6 +343,15 @@ static char **columnNames(const char *zDb, const char *zTab, int *pnPKey){
   }
   sqlite3_finalize(pStmt);
   if( az ) az[naz] = 0;
+
+  /* If it is non-NULL, set *pbRowid to indicate whether or not the PK of 
+  ** this table is an implicit rowid (*pbRowid==1) or not (*pbRowid==0).  */
+  if( pbRowid ) *pbRowid = (az[0]==0);
+
+  /* If this table has an implicit rowid for a PK, figure out how to refer
+  ** to it. There are three options - "rowid", "_rowid_" and "oid". Any
+  ** of these will work, unless the table has an explicit column of the
+  ** same name.  */
   if( az[0]==0 ){
     const char *azRowid[] = { "rowid", "_rowid_", "oid" };
     for(i=0; i<sizeof(azRowid)/sizeof(azRowid[0]); i++){
@@ -434,7 +448,7 @@ static void dump_table(const char *zTab, FILE *out){
   }
   sqlite3_finalize(pStmt);
   if( !g.bSchemaOnly ){
-    az = columnNames("aux", zTab, &nPk);
+    az = columnNames("aux", zTab, &nPk, 0);
     strInit(&ins);
     if( az==0 ){
       pStmt = db_prepare("SELECT * FROM aux.%s", zId);
@@ -511,7 +525,7 @@ static void diff_one_table(const char *zTab, FILE *out){
     ** database and show the results.  This is used for testing
     ** and debugging of the columnNames() function.
     */
-    az = columnNames("aux",zTab, &nPk);
+    az = columnNames("aux",zTab, &nPk, 0);
     if( az==0 ){
       printf("Rowid not accessible for %s\n", zId);
     }else{
@@ -540,8 +554,8 @@ static void diff_one_table(const char *zTab, FILE *out){
     goto end_diff_one_table;
   }
 
-  az = columnNames("main", zTab, &nPk);
+  az = columnNames("main", zTab, &nPk, 0);
-  az2 = columnNames("aux", zTab, &nPk2);
+  az2 = columnNames("aux", zTab, &nPk2, 0);
   if( az && az2 ){
     for(n=0; az[n]; n++){
       if( sqlite3_stricmp(az[n],az2[n])!=0 ) break;
@@ -719,6 +733,164 @@ end_diff_one_table:
   return;
 }
 
+/*
+** Check that table zTab exists and has the same schema in both the "main"
+** and "aux" databases currently opened by the global db handle. If they
+** do not, output an error message on stderr and exit(1). Otherwise, if
+** the schemas do match, return control to the caller.
+*/
+static void checkSchemasMatch(const char *zTab){
+  sqlite3_stmt *pStmt = db_prepare(
+      "SELECT A.sql=B.sql FROM main.sqlite_master A, aux.sqlite_master B"
+      " WHERE A.name=%Q AND B.name=%Q", zTab, zTab
+  );
+  if( SQLITE_ROW==sqlite3_step(pStmt) ){
+    if( sqlite3_column_int(pStmt,0)==0 ){
+      runtimeError("schema changes for table %s", safeId(zTab));
+    }
+  }else{
+    runtimeError("table %s missing from one or both databases", safeId(zTab));
+  }
+  sqlite3_finalize(pStmt);
+}
+
+static void strPrintfArray(
+  Str *pStr,                      /* String object to append to */
+  const char *zSep,               /* Separator string */
+  const char *zFmt,               /* Format for each entry */
+  char **az, int n                /* Array of strings & its size (or -1) */
+){
+  int i;
+  for(i=0; az[i] && (i<n || n<0); i++){
+    if( i!=0 ) strPrintf(pStr, "%s", zSep);
+    strPrintf(pStr, zFmt, az[i], az[i], az[i]);
+  }
+}
+
+static void getRbudiffQuery(
+  const char *zTab,
+  char **azCol,
+  int nPK,
+  int bOtaRowid,
+  Str *pSql
+){
+  int i;
+
+  /* First the newly inserted rows: **/ 
+  strPrintf(pSql, "SELECT ");
+  strPrintfArray(pSql, ", ", "%s", azCol, -1);
+  strPrintf(pSql, ", 0");         /* Set ota_control to 0 for an insert */
+  strPrintf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab);
+  strPrintf(pSql, "    SELECT 1 FROM ", zTab);
+  strPrintf(pSql, " main.%Q AS o WHERE ", zTab);
+  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
+  strPrintf(pSql, "\n)");
+
+  /* Deleted rows: */
+  strPrintf(pSql, "\nUNION ALL\nSELECT ");
+  strPrintfArray(pSql, ", ", "%s", azCol, nPK);
+  strPrintf(pSql, ", ");
+  strPrintfArray(pSql, ", ", "NULL", &azCol[nPK], -1);
+  strPrintf(pSql, ", 1");         /* Set ota_control to 1 for a delete */
+  strPrintf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab);
+  strPrintf(pSql, "    SELECT 1 FROM ", zTab);
+  strPrintf(pSql, " aux.%Q AS o WHERE ", zTab);
+  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
+  strPrintf(pSql, "\n) ");
+
+  /* Updated rows: */
+  strPrintf(pSql, "\nUNION ALL\nSELECT ");
+  strPrintfArray(pSql, ", ", "n.%s", azCol, nPK);
+  strPrintf(pSql, ",\n");
+  strPrintfArray(pSql, " ,\n", 
+      "    CASE WHEN n.%s IS o.%s THEN NULL ELSE n.%s END", &azCol[nPK], -1
+  );
+
+  if( bOtaRowid==0 ){
+    strPrintf(pSql, ", '");
+    strPrintfArray(pSql, "", ".", azCol, nPK);
+    strPrintf(pSql, "' ||\n");
+  }else{
+    strPrintf(pSql, ",\n");
+  }
+  strPrintfArray(pSql, " ||\n", 
+      "    CASE WHEN n.%s IS o.%s THEN '.' ELSE 'x' END", &azCol[nPK], -1
+  );
+  strPrintf(pSql, "\nAS ota_control");
+
+  strPrintf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab, zTab);
+  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
+  strPrintf(pSql, " AND ota_control LIKE '%%x%%'");
+
+  /* Now add an ORDER BY clause to sort everything by PK. */
+  strPrintf(pSql, "\nORDER BY ");
+  for(i=1; i<=nPK; i++) strPrintf(pSql, "%s%d", ((i>1)?", ":""), i);
+}
+
+static void rbudiff_one_table(const char *zTab, FILE *out){
+  int bOtaRowid;                  /* True to use an ota_rowid column */
+  int nPK;                        /* Number of primary key columns in table */
+  char **azCol;                   /* NULL terminated array of col names */
+  int i;
+  int nCol;
+  Str ct = {0, 0, 0};             /* The "CREATE TABLE data_xxx" statement */
+  Str sql = {0, 0, 0};            /* Query to find differences */
+  Str insert = {0, 0, 0};         /* First part of output INSERT statement */
+  sqlite3_stmt *pStmt = 0;
+
+  /* --rbu mode must use real primary keys. */
+  g.bSchemaPK = 1;
+
+  /* Check that the schemas of the two tables match. Exit early otherwise. */
+  checkSchemasMatch(zTab);
+
+  /* Grab the column names and PK details for the table(s). If no usable PK
+  ** columns are found, bail out early.  */
+  azCol = columnNames("main", zTab, &nPK, &bOtaRowid);
+  if( azCol==0 ){
+    runtimeError("table %s has no usable PK columns", zTab);
+  }
+
+  /* Build and output the CREATE TABLE statement for the data_xxx table */
+  strPrintf(&ct, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab);
+  if( bOtaRowid ) strPrintf(&ct, "rbu_rowid, ");
+  strPrintfArray(&ct, ", ", "%s", &azCol[bOtaRowid], -1);
+  strPrintf(&ct, ", rbu_control);");
+
+
+  /* Get the SQL for the query to retrieve data from the two databases */
+  getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, &sql);
+
+  /* Build the first part of the INSERT statement output for each row
+  ** in the data_xxx table. */
+  strPrintf(&insert, "INSERT INTO 'data_%q' (", zTab);
+  if( bOtaRowid ) strPrintf(&insert, "rbu_rowid, ");
+  strPrintfArray(&insert, ", ", "%s", &azCol[bOtaRowid], -1);
+  strPrintf(&insert, ", rbu_control) VALUES(");
+
+  pStmt = db_prepare("%s", sql.z);
+  nCol = sqlite3_column_count(pStmt);
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    if( ct.z ){
+      fprintf(out, "%s\n", ct.z);
+      strFree(&ct);
+    }
+
+    fprintf(out, "%s", insert.z);
+    for(i=0; i<nCol; i++){
+      if( i>0 ) fprintf(out, ", ");
+      printQuoted(out, sqlite3_column_value(pStmt, i));
+    }
+    fprintf(out, ");\n");
+  }
+
+  sqlite3_finalize(pStmt);
+
+  strFree(&ct);
+  strFree(&sql);
+  strFree(&insert);
+}
+
 /*
 ** Display a summary of differences between two versions of the same
 ** table table.
@@ -760,8 +932,8 @@ static void summarize_one_table(const char *zTab, FILE *out){
     goto end_summarize_one_table;
   }
 
-  az = columnNames("main", zTab, &nPk);
+  az = columnNames("main", zTab, &nPk, 0);
-  az2 = columnNames("aux", zTab, &nPk2);
+  az2 = columnNames("aux", zTab, &nPk2, 0);
   if( az && az2 ){
     for(n=0; az[n]; n++){
       if( sqlite3_stricmp(az[n],az2[n])!=0 ) break;
@@ -931,18 +1103,9 @@ static void changeset_one_table(const char *zTab, FILE *out){
   int i, k;                     /* Loop counters */
   const char *zSep;             /* List separator */
 
-  pStmt = db_prepare(
+  /* Check that the schemas of the two tables match. Exit early otherwise. */
-      "SELECT A.sql=B.sql FROM main.sqlite_master A, aux.sqlite_master B"
+  checkSchemasMatch(zTab);
-      " WHERE A.name=%Q AND B.name=%Q", zTab, zTab
+
-  );
-  if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    if( sqlite3_column_int(pStmt,0)==0 ){
-      runtimeError("schema changes for table %s", safeId(zTab));
-    }
-  }else{
-    runtimeError("table %s missing from one or both databases", safeId(zTab));
-  }
-  sqlite3_finalize(pStmt);
   pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab);
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
     nCol++;
@@ -1118,6 +1281,7 @@ static void showHelp(void){
 "  --changeset FILE      Write a CHANGESET into FILE\n"
 "  -L|--lib LIBRARY      Load an SQLite extension library\n"
 "  --primarykey          Use schema-defined PRIMARY KEYs\n"
+"  --rbu                 Output SQL to create/populate RBU table(s)\n"
 "  --schema              Show only differences in the schema\n"
 "  --summary             Show only a summary of the differences\n"
 "  --table TAB           Show only differences in table TAB\n"
@@ -1170,6 +1334,9 @@ int main(int argc, char **argv){
       if( strcmp(z,"primarykey")==0 ){
         g.bSchemaPK = 1;
       }else
+      if( strcmp(z,"rbu")==0 ){
+        xDiff = rbudiff_one_table;
+      }else
       if( strcmp(z,"schema")==0 ){
         g.bSchemaOnly = 1;
       }else