Add sqlite_progress_handler() API for specifying an progress callback (CVS 1111)

FossilOrigin-Name: ddb364635a207658664ea92fc677cf16a143a938

manifest

@@ -1,5 +1,5 @@
-C Update\sto\sthe\sdate\sfunctions.\s(CVS\s1110)
+C Add\ssqlite_progress_handler()\sAPI\sfor\sspecifying\san\sprogress\scallback\s(CVS\s1111)
-D 2003-10-10T02:09:57
+D 2003-10-18T09:37:26
 F Makefile.in ab585a91e34bc33928a1b6181fa2f6ebd4fb17e1
 F Makefile.linux-gcc b86a99c493a5bfb402d1d9178dcdc4bd4b32f906
 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
@@ -35,7 +35,7 @@ F src/func.c fce558b4c1d895e81091d6d5e7d86a192fc8e84c
 F src/hash.c 058f077c1f36f266581aa16f907a3903abf64aa3
 F src/hash.h cd0433998bc1a3759d244e1637fe5a3c13b53bf8
 F src/insert.c dc200ae04a36bd36e575272a069e20c528b7fbdf
-F src/main.c ae92469674db9987de2848e373cd41a394621e32
+F src/main.c 9422005bb4411cc08c2986fde3278ac5b87068a0
 F src/md5.c fe4f9c9c6f71dfc26af8da63e4d04489b1430565
 F src/os.c 97df440bc71f65e22df5d3d920ce39551c0a5f5a
 F src/os.h 729395fefcca4b81ae056aa9ff67b72bb40dd9e0
@@ -48,10 +48,10 @@ F src/random.c 19e8e00fe0df32a742f115773f57651be327cabe
 F src/select.c d79ac60ba1595ff3c94b12892e87098329776482
 F src/shell.c c2ba26c850874964f5ec1ebf6c43406f28e44c4a
 F src/shell.tcl 27ecbd63dd88396ad16d81ab44f73e6c0ea9d20e
-F src/sqlite.h.in f8ae61546942e5a81df0ce3118048bec8dc87be4
+F src/sqlite.h.in e6cfff01fafc8a82ce82cd8c932af421dc9adb54
-F src/sqliteInt.h 5f706313beafcc2da8102c807c35e18b2b0a3572
+F src/sqliteInt.h 74dc7989c9f2b46b50485d0455a8ef8d4f178708
 F src/table.c 4301926464d88d2c2c7cd21c3360aa75bf068b95
-F src/tclsqlite.c ec9e5b796bf9ec1483927e986828a205d4a7422a
+F src/tclsqlite.c 3efac6b5861ac149c41251d4d4c420c94be5ba6a
 F src/test1.c f9d5816610f7ec4168ab7b098d5207a5708712b6
 F src/test2.c 5014337d8576b731cce5b5a14bec4f0daf432700
 F src/test3.c 30985ebdfaf3ee1462a9b0652d3efbdc8d9798f5
@@ -61,7 +61,7 @@ F src/trigger.c ce83e017b407d046e909d05373d7f8ee70f9f7f9
 F src/update.c 24260b4fda00c9726d27699a0561d53c0dccc397
 F src/util.c f16efa2d60bfd4e31ae06b07ed149557e828d294
 F src/vacuum.c e4724eade07e4cf8897060a8cf632dbd92408eeb
-F src/vdbe.c a9923a38a24ee86dd2e237c9f7e9d0116e329394
+F src/vdbe.c 0928a242ced0b5d26292f3949fdab26fa4dc327d
 F src/vdbe.h 3957844e46fea71fd030e78f6a3bd2f7e320fb43
 F src/vdbeInt.h 2824bf88895b901b3a8c9e44527c67530e1c0dcb
 F src/vdbeaux.c 31abb8e3e57866913360381947e267a51fed92c6
@@ -109,6 +109,7 @@ F test/null.test c14d0f4739f21e929b8115b72bf0c765b6bb1721
 F test/pager.test dd31da9bee94a82e2e87e58cf286cfe809f8fc5f
 F test/pragma.test e7cb7ffd765c9158868b0b7a3771d54a0d5f5072
 F test/printf.test 3ed02f1361402c0767492cd5cef4650e61df8308
+F test/progress.test 701b6115c2613128ececdfe1398a1bd0e1a4cfb3 x
 F test/quick.test c527bdb899b12a8cd8ceecce45f72922099f4095
 F test/quote.test 08f23385c685d3dc7914ec760d492cacea7f6e3d
 F test/rowid.test 1936d0d866a8105ab53cf6cb40a549b6664d06ce
@@ -153,7 +154,7 @@ F www/arch.fig d5f9752a4dbf242e9cfffffd3f5762b6c63b3bcf
 F www/arch.png 82ef36db1143828a7abc88b1e308a5f55d4336f4
 F www/arch.tcl 44b589fc01d6829d43447ab40588b00aec5b9734
 F www/audit.tcl 90e09d580f79c7efec0c7d6f447b7ec5c2dce5c0
-F www/c_interface.tcl acacd31d4441de900e09ee48b5ffdef0162d8dc3
+F www/c_interface.tcl 17d8bd9e7b4fbdca47c30c8b9bcb728c351d55c0
 F www/changes.tcl 1188dd0e79f9a8c48996ff44e4d9e81789bf1503
 F www/conflict.tcl 81dd21f9a679e60aae049e9dd8ab53d59570cda2
 F www/datatypes.tcl 0cb28565580554fa7e03e8fcb303e87ce57757ae
@@ -173,7 +174,7 @@ F www/speed.tcl 2f6b1155b99d39adb185f900456d1d592c4832b3
 F www/sqlite.tcl 3c83b08cf9f18aa2d69453ff441a36c40e431604
 F www/tclsqlite.tcl b9271d44dcf147a93c98f8ecf28c927307abd6da
 F www/vdbe.tcl 9b9095d4495f37697fd1935d10e14c6015e80aa1
-P 54aa0fb236d17b53b194a667d68c71007c8e7687
+P 06d4e88394217fb1390b069bad82d6ac71981f72
-R 78b719cb974e90d3bbfc839a1c83b68b
+R cde37f606921798e43de3b54f3499f6d
-U drh
+U danielk1977
-Z 6362f853c8fee8f874044b89744bb342
+Z a955533f397af1a9a7db7d26a39908ea

manifest.uuid

@@ -1 +1 @@
-06d4e88394217fb1390b069bad82d6ac71981f72
+ddb364635a207658664ea92fc677cf16a143a938

src/main.c

@@ -14,7 +14,7 @@
 ** other files are for internal use by SQLite and should not be
 ** accessed by users of the library.
 **
-** $Id: main.c,v 1.142 2003/09/06 22:18:08 drh Exp $
+** $Id: main.c,v 1.143 2003/10/18 09:37:26 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -826,6 +826,31 @@ void sqlite_busy_handler(
   db->pBusyArg = pArg;
 }
 
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+/*
+** This routine sets the progress callback for an Sqlite database to the
+** given callback function with the given argument. The progress callback will
+** be invoked every nOps opcodes.
+*/
+void sqlite_progress_handler(
+  sqlite *db, 
+  int nOps,
+  int (*xProgress)(void*), 
+  void *pArg
+){
+  if( nOps>0 ){
+    db->xProgress = xProgress;
+    db->nProgressOps = nOps;
+    db->pProgressArg = pArg;
+  }else{
+    db->xProgress = 0;
+    db->nProgressOps = 0;
+    db->pProgressArg = 0;
+  }
+}
+#endif
+
+
 /*
 ** This routine installs a default busy handler that waits for the
 ** specified number of milliseconds before returning 0.

src/sqlite.h.in

@@ -12,7 +12,7 @@
 ** This header file defines the interface that the SQLite library
 ** presents to client programs.
 **
-** @(#) $Id: sqlite.h.in,v 1.52 2003/09/06 22:18:08 drh Exp $
+** @(#) $Id: sqlite.h.in,v 1.53 2003/10/18 09:37:26 danielk1977 Exp $
 */
 #ifndef _SQLITE_H_
 #define _SQLITE_H_
@@ -729,6 +729,33 @@ int sqlite_reset(sqlite_vm*, char **pzErrMsg);
 */
 int sqlite_bind(sqlite_vm*, int idx, const char *value, int len, int copy);
 
+/*
+** This routine configures a callback function - the progress callback - that
+** is invoked periodically during long running calls to sqlite_exec(),
+** sqlite_step() and sqlite_get_table(). An example use for this API is to keep
+** a GUI updated during a large query.
+**
+** The progress callback is invoked once for every N virtual machine opcodes,
+** where N is the second argument to this function. The progress callback
+** itself is identified by the third argument to this function. The fourth
+** argument to this function is a void pointer passed to the progress callback
+** function each time it is invoked.
+**
+** If a call to sqlite_exec(), sqlite_step() or sqlite_get_table() results 
+** in less than N opcodes being executed, then the progress callback is not
+** invoked.
+** 
+** Calling this routine overwrites any previously installed progress callback.
+** To remove the progress callback altogether, pass NULL as the third
+** argument to this function.
+**
+** If the progress callback returns a result other than 0, then the current 
+** query is immediately terminated and any database changes rolled back. If the
+** query was part of a larger transaction, then the transaction is not rolled
+** back and remains active. The sqlite_exec() call returns SQLITE_ABORT. 
+*/
+void sqlite_progress_handler(sqlite*, int, int(*)(void*), void*);
+
 #ifdef __cplusplus
 }  /* End of the 'extern "C"' block */
 #endif

src/sqliteInt.h

@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.199 2003/09/27 13:39:39 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.200 2003/10/18 09:37:26 danielk1977 Exp $
 */
 #include "config.h"
 #include "sqlite.h"
@@ -88,6 +88,7 @@
 /* #define SQLITE_OMIT_INMEMORYDB     1 */
 /* #define SQLITE_OMIT_VACUUM         1 */
 /* #define SQLITE_OMIT_TIMEDATE_FUNCS 1 */
+/* #define SQLITE_OMIT_PROGRESS_CALLBACK 1 */
 
 /*
 ** Integers of known sizes.  These typedefs might change for architectures
@@ -326,6 +327,11 @@ struct sqlite {
                                 /* Access authorization function */
   void *pAuthArg;               /* 1st argument to the access auth function */
 #endif
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+  int (*xProgress)(void *);     /* The progress callback */
+  void *pProgressArg;           /* Argument to the progress callback */
+  int nProgressOps;             /* Number of opcodes for progress callback */
+#endif
 };
 
 /*

src/tclsqlite.c

@@ -11,7 +11,7 @@
 *************************************************************************
 ** A TCL Interface to SQLite
 **
-** $Id: tclsqlite.c,v 1.50 2003/08/19 14:31:02 drh Exp $
+** $Id: tclsqlite.c,v 1.51 2003/10/18 09:37:26 danielk1977 Exp $
 */
 #ifndef NO_TCL     /* Omit this whole file if TCL is unavailable */
 
@@ -52,6 +52,7 @@ struct SqliteDb {
   Tcl_Interp *interp;   /* The interpreter used for this database */
   char *zBusy;          /* The busy callback routine */
   char *zTrace;         /* The trace callback routine */
+  char *zProgress;      /* The progress callback routine */
   char *zAuth;          /* The authorization callback routine */
   SqlFunc *pFunc;       /* List of SQL functions */
   int rc;               /* Return code of most recent sqlite_exec() */
@@ -325,6 +326,21 @@ static int DbBusyHandler(void *cd, const char *zTable, int nTries){
   return 1;
 }
 
+/*
+** This routine is invoked as the 'progress callback' for the database.
+*/
+static int DbProgressHandler(void *cd){
+  SqliteDb *pDb = (SqliteDb*)cd;
+  int rc;
+
+  assert( pDb->zProgress );
+  rc = Tcl_Eval(pDb->interp, pDb->zProgress);
+  if( rc!=TCL_OK || atoi(Tcl_GetStringResult(pDb->interp)) ){
+    return 1;
+  }
+  return 0;
+}
+
 /*
 ** This routine is called by the SQLite trace handler whenever a new
 ** block of SQL is executed.  The TCL script in pDb->zTrace is executed.
@@ -457,13 +473,14 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     "close",              "complete",          "errorcode",
     "eval",               "function",          "last_insert_rowid",
     "onecolumn",          "timeout",            "trace",
-    0
+    "progress",           0
   };
   enum DB_enum {
     DB_AUTHORIZER,        DB_BUSY,             DB_CHANGES,
     DB_CLOSE,             DB_COMPLETE,         DB_ERRORCODE,
     DB_EVAL,              DB_FUNCTION,         DB_LAST_INSERT_ROWID,
     DB_ONECOLUMN,         DB_TIMEOUT,          DB_TRACE,            
+    DB_PROGRESS,
   };
 
   if( objc<2 ){
@@ -562,6 +579,48 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     break;
   }
 
+  /*    $db progress ?N CALLBACK?
+  ** 
+  ** Invoke the given callback every N virtual machine opcodes while executing
+  ** queries.
+  */
+  case DB_PROGRESS: {
+    if( objc==2 ){
+      if( pDb->zProgress ){
+        Tcl_AppendResult(interp, pDb->zProgress, 0);
+      }
+    }else if( objc==4 ){
+      char *zProgress;
+      int len;
+      int N;
+      if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &N) ){
+	return TCL_ERROR;
+      };
+      if( pDb->zProgress ){
+        Tcl_Free(pDb->zProgress);
+      }
+      zProgress = Tcl_GetStringFromObj(objv[3], &len);
+      if( zProgress && len>0 ){
+        pDb->zProgress = Tcl_Alloc( len + 1 );
+        strcpy(pDb->zProgress, zProgress);
+      }else{
+        pDb->zProgress = 0;
+      }
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+      if( pDb->zProgress ){
+        pDb->interp = interp;
+        sqlite_progress_handler(pDb->db, N, DbProgressHandler, pDb);
+      }else{
+        sqlite_progress_handler(pDb->db, 0, 0, 0);
+      }
+#endif
+    }else{
+      Tcl_WrongNumArgs(interp, 2, objv, "N CALLBACK");
+      return TCL_ERROR;
+    }
+    break;
+  }
+
   /*
   **     $db changes
   **

src/vdbe.c

@@ -43,7 +43,7 @@
 ** in this file for details.  If in doubt, do not deviate from existing
 ** commenting and indentation practices when changing or adding code.
 **
-** $Id: vdbe.c,v 1.241 2003/09/27 00:56:32 drh Exp $
+** $Id: vdbe.c,v 1.242 2003/10/18 09:37:26 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -529,6 +529,9 @@ int sqliteVdbeExec(
   unsigned long long start;  /* CPU clock count at start of opcode */
   int origPc;                /* Program counter at start of opcode */
 #endif
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+  int nProgressOps = 0;      /* Opcodes executed since progress callback. */
+#endif
 
   if( p->magic!=VDBE_MAGIC_RUN ) return SQLITE_MISUSE;
   assert( db->magic==SQLITE_MAGIC_BUSY );
@@ -556,6 +559,23 @@ int sqliteVdbeExec(
     }
 #endif
 
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+    /* Call the progress callback if it is configured and the required number
+    ** of VDBE ops have been executed (either since this invocation of
+    ** sqliteVdbeExec() or since last time the progress callback was called).
+    ** If the progress callback returns non-zero, exit the virtual machine with
+    ** a return code SQLITE_ABORT.
+    */
+    if( db->xProgress && (db->nProgressOps==nProgressOps) ){
+      if( db->xProgress(db->pProgressArg)!=0 ){
+        rc = SQLITE_ABORT;
+        continue; /* skip to the next iteration of the for loop */
+      }
+      nProgressOps = 0;
+    }
+    nProgressOps++;
+#endif
+
     switch( pOp->opcode ){
 
 /*****************************************************************************

test/progress.test

@@ -0,0 +1,118 @@
+# 2001 September 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the 'progress callback'.
+#
+# $Id: progress.test,v 1.1 2003/10/18 09:37:27 danielk1977 Exp $
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Build some test data
+#
+execsql {
+  BEGIN;
+  CREATE TABLE t1(a);
+  INSERT INTO t1 VALUES(1);
+  INSERT INTO t1 VALUES(2);
+  INSERT INTO t1 VALUES(3);
+  INSERT INTO t1 VALUES(4);
+  INSERT INTO t1 VALUES(5);
+  INSERT INTO t1 VALUES(6);
+  INSERT INTO t1 VALUES(7);
+  INSERT INTO t1 VALUES(8);
+  INSERT INTO t1 VALUES(9);
+  INSERT INTO t1 VALUES(10);
+  COMMIT;
+}
+
+
+# Test that the progress callback is invoked.
+do_test progress-1.0 {
+  set counter 0
+  db progress 1 "[namespace code {incr counter}] ; expr 0"
+  execsql {
+    SELECT * FROM t1
+  }
+  expr $counter > 1
+} 1
+
+# Test that the query is abandoned when the progress callback returns non-zero
+do_test progress1.1 {
+  set counter 0
+  db progress 1 "[namespace code {incr counter}] ; expr 1"
+  execsql {
+    SELECT * FROM t1
+  }
+  set counter 
+} 1
+
+# Test that the query is rolled back when the progress callback returns
+# non-zero.
+do_test progress1.2 {
+
+  # This figures out how many opcodes it takes to copy 5 extra rows into t1.
+  db progress 1 "[namespace code {incr five_rows}] ; expr 0"
+  set five_rows 0
+  execsql {
+    INSERT INTO t1 SELECT a+10 FROM t1 WHERE a < 6
+  }
+  db progress 0 ""
+  execsql {
+    DELETE FROM t1 WHERE a > 10
+  }
+
+  # Now set up the progress callback to abandon the query after the number of
+  # opcodes to copy 5 rows. That way, when we try to copy 6 rows, we know
+  # some data will have been inserted into the table by the time the progress
+  # callback abandons the query.
+  db progress $five_rows "expr 1"
+  execsql {
+    INSERT INTO t1 SELECT a+10 FROM t1 WHERE a < 7
+  }
+  execsql {
+    SELECT count(*) FROM t1
+  }
+} 10
+
+# Test that an active transaction remains active and not rolled back after the
+# progress query abandons a query. 
+do_test progress1.3 {
+
+  db progress 0 ""
+  execsql BEGIN
+  execsql {
+    INSERT INTO t1 VALUES(11)
+  }
+  db progress 1 "expr 1"
+  execsql {
+    INSERT INTO t1 VALUES(12)
+  }
+  db progress 0 ""
+  execsql COMMIT
+  execsql {
+    SELECT count(*) FROM t1
+  }
+} 11
+
+# Check that a value of 0 for N means no progress callback
+do_test progress1.4 {
+  set counter 0
+  db progress 0 "[namespace code {incr counter}] ; expr 0"
+  execsql {
+    SELECT * FROM t1;
+  }
+  set counter
+} 0
+
+db progress 0 ""
+
+finish_test

www/c_interface.tcl

@@ -1,7 +1,7 @@
 #
 # Run this Tcl script to generate the sqlite.html file.
 #
-set rcsid {$Id: c_interface.tcl,v 1.38 2003/07/08 23:42:25 drh Exp $}
+set rcsid {$Id: c_interface.tcl,v 1.39 2003/10/18 09:37:27 danielk1977 Exp $}
 
 puts {<html>
 <head>
@@ -635,6 +635,8 @@ char *sqlite_vmprintf(const char *zFormat, va_list);
 
 void sqlite_freemem(char*);
 
+void sqlite_progress_handler(sqlite*, int, int (*)(void*), void*);
+
 </pre></blockquote>
 
 <p>All of the above definitions are included in the "sqlite.h"
@@ -979,6 +981,30 @@ routine.  The string pointer that these routines return should be freed
 by passing it to <b>sqlite_freemem()</b>.
 </p>
 
+<h3>3.10 Performing background jobs during large queries </h2>
+
+<p>The <b>sqlite_progress_handler()</b> routine can be used to register a
+callback routine with an SQLite database to be invoked periodically during long
+running calls to <b>sqlite_exec()</b>, <b>sqlite_step()</b> and the various
+wrapper functions.
+</p>
+
+<p>The callback is invoked every N virtual machine operations, where N is
+supplied as the second argument to <b>sqlite_progress_handler()</b>. The third
+and fourth arguments to <b>sqlite_progress_handler()</b> are a pointer to the
+routine to be invoked and a void pointer to be passed as the first argument to
+it.
+</p>
+
+<p>The time taken to execute each virtual machine operation can vary based on
+many factors.  A typical value for a 1 GHz PC is between half and three million
+per second but may be much higher or lower, depending on the query.  As such it
+is difficult to schedule background operations based on virtual machine
+operations. Instead, it is recommended that a callback be scheduled relatively
+frequently (say every 1000 instructions) and external timer routines used to
+determine whether or not background jobs need to be run.  
+</p>
+
 <a name="cfunc">
 <h2>4.0 Adding New SQL Functions</h2>