Change the name of the new API on this branch to "sqlite3_bp_progress". Add tests and documentation for the same.

FossilOrigin-Name: 1a1b69e87eb7d18f76f5b733e44da75136a686b6

ext/rbu/rbuprogress.test

@@ -13,14 +13,20 @@
 source [file join [file dirname [info script]] rbu_common.tcl]
 set ::testprefix rbuprogress
 
+
+proc create_db_file {filename sql} {
+  forcedelete $filename
+  sqlite3 tmpdb $filename  
+  tmpdb eval $sql
+  tmpdb close
+}
+
 # Create a simple RBU database. That expects to write to a table:
 #
 #   CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
 #
 proc create_rbu1 {filename} {
-  forcedelete $filename
+  create_db_file $filename {
-  sqlite3 rbu1 $filename  
-  rbu1 eval {
     CREATE TABLE data_t1(a, b, c, rbu_control);
     INSERT INTO data_t1 VALUES(1, 2, 3, 0);
     INSERT INTO data_t1 VALUES(2, 'two', 'three', 0);
@@ -29,7 +35,6 @@ proc create_rbu1 {filename} {
     CREATE TABLE rbu_count(tbl, cnt);
     INSERT INTO rbu_count VALUES('data_t1', 3);
   }
-  rbu1 close
   return $filename
 }
 
@@ -41,16 +46,16 @@ do_execsql_test 1.0 {
 do_test 1.1 {
   create_rbu1 rbu.db
   sqlite3rbu rbu test.db rbu.db
-  rbu stage_progress
+  rbu bp_progress
 } {0 0}
-do_test 1.2 { rbu step ; rbu stage_progress } {3333 0}
+do_test 1.2 { rbu step ; rbu bp_progress } {3333 0}
-do_test 1.3 { rbu step ; rbu stage_progress } {6666 0}
+do_test 1.3 { rbu step ; rbu bp_progress } {6666 0}
-do_test 1.4 { rbu step ; rbu stage_progress } {10000 0}
+do_test 1.4 { rbu step ; rbu bp_progress } {10000 0}
-do_test 1.5 { rbu step ; rbu stage_progress } {10000 0}
+do_test 1.5 { rbu step ; rbu bp_progress } {10000 0}
-do_test 1.6 { rbu step ; rbu stage_progress } {10000 0}
+do_test 1.6 { rbu step ; rbu bp_progress } {10000 0}
-do_test 1.7 { rbu step ; rbu stage_progress } {10000 5000}
+do_test 1.7 { rbu step ; rbu bp_progress } {10000 5000}
-do_test 1.8 { rbu step ; rbu stage_progress } {10000 10000}
+do_test 1.8 { rbu step ; rbu bp_progress } {10000 10000}
-do_test 1.9 { rbu step ; rbu stage_progress } {10000 10000}
+do_test 1.9 { rbu step ; rbu bp_progress } {10000 10000}
 
 do_test 1.10 {
   rbu close
@@ -66,7 +71,7 @@ proc do_sp_test {tn bReopen target rbu reslist} {
       if {$bReopen} { sqlite3rbu rbu $target $rbu }
       set rc [rbu step]
       if {[set rc] != "SQLITE_OK"} { error "error 1" }
-      lappend res [lindex [rbu stage_progress] 0]
+      lappend res [lindex [rbu bp_progress] 0]
       if {[lindex [set res] end]==10000} break
       if {$bReopen} { rbu close }
     }
@@ -79,26 +84,29 @@ proc do_sp_test {tn bReopen target rbu reslist} {
     # file to *-wal. After each of these steps, the progress remains
     # at "10000 0".
     #
-    rbu step
+    if {[lindex [list $reslist] 0]!=-1} {
-    set res [rbu stage_progress]
+      rbu step
-    if {[set res] != [list 10000 0]} {
+      set res [rbu bp_progress]
-      error "2. reslist incorrect (expect=10000 0 got=[set res])"
+      if {[set res] != [list 10000 0]} {
+        error "2. reslist incorrect (expect=10000 0 got=[set res])"
+      }
     }
+
     rbu step
-    set res [rbu stage_progress]
+    set res [rbu bp_progress]
     if {[set res] != [list 10000 0]} {
       error "3. reslist incorrect (expect=10000 0 got=[set res])"
     }
 
     # Do the checkpoint.
     while {[rbu step]=="SQLITE_OK"} { 
-      foreach {a b} [rbu stage_progress] {}
+      foreach {a b} [rbu bp_progress] {}
       if {[set a]!=10000 || [set b]<=0 || [set b]>10000} {
         error "4. reslist incorrect (expect=10000 1..10000 got=[set a] [set b])"
       }
     }
 
-    set res [rbu stage_progress]
+    set res [rbu bp_progress]
     if {[set res] != [list 10000 10000]} {
       error "5. reslist is incorrect (expect=10000 10000 got=[set res])"
     }
@@ -107,13 +115,6 @@ proc do_sp_test {tn bReopen target rbu reslist} {
   }] {SQLITE_DONE}]
 }
 
-proc create_db_file {filename sql} {
-  forcedelete $filename
-  sqlite3 tmpdb $filename  
-  tmpdb eval $sql
-  tmpdb close
-}
-
 foreach {bReopen} { 0 1 } {
 
   reset_db
@@ -197,7 +198,95 @@ foreach {bReopen} { 0 1 } {
     }
   } {}
   do_sp_test 2.$bReopen.5.1 $bReopen test.db rbu.db {10000}
+
+  reset_db
+  do_test 2.$bReopen.6.0 {
+    execsql { 
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
+      CREATE INDEX i1 ON t1(b);
+      INSERT INTO t1 VALUES(1, 1, 1);
+      INSERT INTO t1 VALUES(2, 2, 2);
+      INSERT INTO t1 VALUES(3, 3, 3);
+    }
+    create_db_file rbu.db {
+      CREATE TABLE data_t1(a, b, c, rbu_control);
+      INSERT INTO data_t1 VALUES(4, 4, 4, 0);
+      INSERT INTO data_t1 VALUES(2, NULL, NULL, 1);
+      INSERT INTO data_t1 VALUES(5, NULL, NULL, 1);
+    }
+  } {}
+  do_sp_test 2.$bReopen.6.1 $bReopen test.db rbu.db {-1 -1 -1 -1 -1 10000}
 }
 
+#-------------------------------------------------------------------------
+# The following tests verify that the API works when resuming an update
+# during the incremental checkpoint stage.
+#
+proc do_phase2_test {tn bReopen target rbu nStep} {
+  uplevel [list do_test $tn [subst -nocommands {
+
+    # Build the OAL/WAL file:
+    sqlite3rbu rbu $target $rbu
+    while {[lindex [rbu bp_progress] 0]<10000} { 
+      set rc [rbu step]
+      if {"SQLITE_OK" != [set rc]} { rbu close }
+    }
+
+    # Clean up the temp tables and move the *-oal file to *-wal.
+    rbu step
+    rbu step
+
+    for {set i 0} {[set i] < $nStep} {incr i} {
+      if {$bReopen} {
+        rbu close
+        sqlite3rbu rbu $target $rbu
+      }
+      rbu step
+      set res [rbu bp_progress]
+      set expect [expr (1 + [set i]) * 10000 / $nStep]
+      if {[lindex [set res] 1] != [set expect]} {
+        error "Have [set res], expected 10000 [set expect]"
+      }
+    }
+
+    set rc [rbu step]
+    if {[set rc] != "SQLITE_DONE"} {
+      error "Have [set rc], expected SQLITE_DONE" 
+    }
+
+    rbu close
+  }] {SQLITE_DONE}]
+}
+
+foreach bReopen {0 1} {
+  do_test 3.$bReopen.1.0 {
+    reset_db
+    execsql {
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+      CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
+      CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
+    }
+    create_db_file rbu.db {
+      CREATE TABLE data_t1(a, b, rbu_control);
+      CREATE TABLE data_t2(a, b, rbu_control);
+      CREATE TABLE data_t3(a, b, rbu_control);
+      CREATE TABLE data_t4(a, b, rbu_control);
+      INSERT INTO data_t1 VALUES(1, 2, 0);
+      INSERT INTO data_t2 VALUES(1, 2, 0);
+      INSERT INTO data_t3 VALUES(1, 2, 0);
+      INSERT INTO data_t4 VALUES(1, 2, 0);
+  
+      CREATE TABLE rbu_count(tbl, cnt);
+      INSERT INTO rbu_count VALUES('data_t1', 1);
+      INSERT INTO rbu_count VALUES('data_t2', 1);
+      INSERT INTO rbu_count VALUES('data_t3', 1);
+      INSERT INTO rbu_count VALUES('data_t4', 1);
+    }
+  } {}
+  do_phase2_test 3.$bReopen.1.1 $bReopen test.db rbu.db 5
+}
+
+
 finish_test
 

ext/rbu/sqlite3rbu.c

@@ -300,6 +300,43 @@ struct RbuFrame {
 
 /*
 ** RBU handle.
+**
+** nPhaseOneStep:
+**   If the RBU database contains an rbu_count table, this value is set to
+**   a running estimate of the number of b-tree operations required to 
+**   finish populating the *-oal file. This allows the sqlite3_bp_progress()
+**   API to calculate the permyriadage progress of populating the *-oal file
+**   using the formula:
+**
+**     permyriadage = (10000 * nProgress) / nPhaseOneStep
+**
+**   nPhaseOneStep is initialized to the sum of:
+**
+**     nRow * (nIndex + 1)
+**
+**   for all source tables in the RBU database, where nRow is the number
+**   of rows in the source table and nIndex the number of indexes on the
+**   corresponding target database table.
+**
+**   This estimate is accurate if the RBU update consists entirely of
+**   INSERT operations. However, it is inaccurate if:
+**
+**     * the RBU update contains any UPDATE operations. If the PK specified
+**       for an UPDATE operation does not exist in the target table, then
+**       no b-tree operations are required on index b-trees. Or if the 
+**       specified PK does exist, then (nIndex*2) such operations are
+**       required (one delete and one insert on each index b-tree).
+**
+**     * the RBU update contains any DELETE operations for which the specified
+**       PK does not exist. In this case no operations are required on index
+**       b-trees.
+**
+**     * the RBU update contains REPLACE operations. These are similar to
+**       UPDATE operations.
+**
+**   nPhaseOneStep is updated to account for the conditions above during the
+**   first pass of each source table. The updated nPhaseOneStep value is
+**   stored in the rbu_state table if the RBU update is suspended.
 */
 struct sqlite3rbu {
   int eStage;                     /* Value of RBU_STATE_STAGE field */
@@ -2957,7 +2994,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){
         break;
 
       case RBU_STATE_PHASEONESTEP:
-        pRet->nPhaseOneStep = (u32)sqlite3_column_int64(pStmt, 1);
+        pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
         break;
 
       default:
@@ -3068,7 +3105,9 @@ static void rbuDeleteVfs(sqlite3rbu *p){
 }
 
 /*
-**
+** This user-defined SQL function is invoked with a single argument - the
+** name of a table expected to appear in the target database. It returns
+** the number of auxilliary indexes on the table.
 */
 static void rbuIndexCntFunc(
   sqlite3_context *pCtx, 
@@ -3371,7 +3410,11 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){
   return pRbu->nProgress;
 }
 
-void sqlite3rbu_stage_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
+/*
+** Return permyriadage progress indications for the two main stages of
+** an RBU update.
+*/
+void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
   const int MAX_PROGRESS = 10000;
   switch( p->eStage ){
     case RBU_STAGE_OAL:

ext/rbu/sqlite3rbu.h

@@ -400,7 +400,47 @@ int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg);
 */
 sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu);
 
-void sqlite3rbu_stage_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo);
+/*
+** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) 
+** progress indications for the two stages of an RBU update. This API may
+** be useful for driving GUI progress indicators and similar.
+**
+** An RBU update is divided into two stages:
+**
+**   * Stage 1, in which changes are accumulated in an oal/wal file, and
+**   * Stage 2, in which the contents of the wal file are copied into the
+**     main database.
+**
+** The update is visible to non-RBU clients during stage 2. During stage 1
+** non-RBU reader clients may see the original database.
+**
+** If this API is called during stage 2 of the update, output variable 
+** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo)
+** to a value between 0 and 10000 to indicate the permyriadage progress of
+** stage 2. A value of 5000 indicates that stage 2 is half finished, 
+** 9000 indicates that it is 90% finished, and so on.
+**
+** If this API is called during stage 1 of the update, output variable 
+** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The
+** value to which (*pnOne) is set depends on whether or not the RBU 
+** database contains an "rbu_count" table. The rbu_count table, if it 
+** exists, must contain the same columns as the following:
+**
+**   CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID;
+**
+** There must be one row in the table for each source (data_xxx) table within
+** the RBU database. The 'tbl' column should contain the name of the source
+** table. The 'cnt' column should contain the number of rows within the
+** source table.
+**
+** If the rbu_count table is present and populated correctly and this
+** API is called during stage 1, the *pnOne output variable is set to the
+** permyriadage progress of the same stage. If the rbu_count table does
+** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count
+** table exists but is not correctly populated, the value of the *pnOne
+** output variable during stage 1 is undefined.
+*/
+void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo);
 
 /*
 ** Create an RBU VFS named zName that accesses the underlying file-system

ext/rbu/test_rbu.c

@@ -66,7 +66,7 @@ static int test_sqlite3rbu_cmd(
     {"create_rbu_delta", 2, ""},  /* 2 */
     {"savestate", 2, ""},         /* 3 */
     {"dbMain_eval", 3, "SQL"},    /* 4 */
-    {"stage_progress", 2, ""},    /* 5 */
+    {"bp_progress", 2, ""},    /* 5 */
     {0,0,0}
   };
   int iCmd;
@@ -137,10 +137,10 @@ static int test_sqlite3rbu_cmd(
       break;
     }
 
-    case 5: /* stage_progress */ {
+    case 5: /* bp_progress */ {
       int one, two;
       Tcl_Obj *pObj;
-      sqlite3rbu_stage_progress(pRbu, &one, &two);
+      sqlite3rbu_bp_progress(pRbu, &one, &two);
 
       pObj = Tcl_NewObj();
       Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one));

manifest

@@ -1,5 +1,5 @@
-C Add\stests\sfor\sthe\schanges\son\sthis\sbranch.\sFix\sa\sproblem\swith\scalls\sto\sthe\snew\sprogress\sindicator\sAPI\smade\safter\san\srbu\supdate\shas\sbeen\sresumed.
+C Change\sthe\sname\sof\sthe\snew\sAPI\son\sthis\sbranch\sto\s"sqlite3_bp_progress".\sAdd\stests\sand\sdocumentation\sfor\sthe\ssame.
-D 2016-03-18T10:29:47.529
+D 2016-03-18T18:56:45.343
 F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66
@@ -242,11 +242,11 @@ F ext/rbu/rbudiff.test 6cc806dc36389292f2a8f5842d0103721df4a07d
 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89
 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda
-F ext/rbu/rbuprogress.test c4a9b3262bc0cafbf19709b56fbda0c3a9e69ac2
+F ext/rbu/rbuprogress.test 77fe3cd10d3c408bef364bc2e529310495ddd07e
 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
-F ext/rbu/sqlite3rbu.c c89f1e59eb09257e126d2cdcb3c5588e16a44fee
+F ext/rbu/sqlite3rbu.c 6b7dc899b3980d4236bffa5048218f8dba85ac0a
-F ext/rbu/sqlite3rbu.h f8ee94f95fc80a35b7cb7ef3f2f5ea740b662477
+F ext/rbu/sqlite3rbu.h d7cc99350c10134f358fe1a8997d9225b3f712b2
-F ext/rbu/test_rbu.c 5b6d31af188193d929234d1cd08ee967df092f66
+F ext/rbu/test_rbu.c 3505641a78b723589b8780d5f9b2faeeb73e037d
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -1457,7 +1457,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ffc58d2c2576a5b6e1c2c7112612c5760e711afd
+P bf82321724d3b0feb51e26d9b76090e03cc3964a
-R b5c08658920cbd4a66ad9300ad32e622
+R 14faa32cca0db38c921d7bc3da7dea4c
 U dan
-Z 25ebcfe6c5065c93d73006dcdc7ea2c0
+Z 40a2b2dbafc34d98f3493240ec44ac36

manifest.uuid

@@ -1 +1 @@
-bf82321724d3b0feb51e26d9b76090e03cc3964a
+1a1b69e87eb7d18f76f5b733e44da75136a686b6