Merge in all the latest trunk enhancements.

FossilOrigin-Name: 7f48f6ab2af071ec20204105ed703c60e9fa92602558b910d14b09cc634bed7d

Makefile.in

@@ -181,7 +181,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
          pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
          random.lo resolve.lo rowset.lo rtree.lo \
-         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
+         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
          table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
          update.lo util.lo vacuum.lo \
          vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
@@ -353,7 +353,8 @@ SRC += \
   $(TOP)/ext/rbu/sqlite3rbu.h \
   $(TOP)/ext/rbu/sqlite3rbu.c
 SRC += \
-  $(TOP)/ext/misc/json1.c
+  $(TOP)/ext/misc/json1.c \
+  $(TOP)/ext/misc/stmt.c
 
 # Generated source code files
 #
@@ -578,6 +579,7 @@ SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
 # SHELL_OPT += -DSQLITE_ENABLE_FTS5
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
+SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
@@ -1047,6 +1049,9 @@ sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
 json1.lo:	$(TOP)/ext/misc/json1.c
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
 
+stmt.lo:	$(TOP)/ext/misc/stmt.c
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c
+
 # FTS5 things
 #
 FTS5_SRC = \
@@ -1096,6 +1101,7 @@ TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
+TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
 
 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
 TESTFIXTURE_SRC1 = sqlite3.c

Makefile.msc

@@ -1294,7 +1294,8 @@ SRC07 = \
   $(TOP)\ext\rtree\rtree.c \
   $(TOP)\ext\session\sqlite3session.c \
   $(TOP)\ext\rbu\sqlite3rbu.c \
-  $(TOP)\ext\misc\json1.c
+  $(TOP)\ext\misc\json1.c \
+  $(TOP)\ext\misc\stmt.c
 
 # Extension header files, part 1.
 #
@@ -1505,7 +1506,7 @@ FUZZDATA = \
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
 # SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS5
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
@@ -2092,6 +2093,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
 
 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
@@ -2245,6 +2247,9 @@ rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H
 	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \
 		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+LSMDIR=$(TOP)\ext\lsm1
+!INCLUDE $(LSMDIR)\Makefile.msc
+
 moreclean:	clean
 	del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL
 # <</mark>>
@@ -2264,6 +2269,7 @@ clean:
 	-rmdir /Q/S tsrc 2>NUL
 	del /Q .target_source 2>NUL
 	del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL
+	del /Q lsm.dll lsmtest.exe 2>NUL
 	del /Q testloadext.dll 2>NUL
 	del /Q testfixture.exe test.db 2>NUL
 	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL

ext/fts3/fts3.c

@@ -1707,6 +1707,19 @@ static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){
   sqlite3_finalize(pCsr->pStmt);
 }
 
+/*
+** Free all resources currently held by the cursor passed as the only
+** argument.
+*/
+static void fts3ClearCursor(Fts3Cursor *pCsr){
+  fts3CursorFinalizeStmt(pCsr);
+  sqlite3Fts3FreeDeferredTokens(pCsr);
+  sqlite3_free(pCsr->aDoclist);
+  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
+  sqlite3Fts3ExprFree(pCsr->pExpr);
+  memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
+}
+
 /*
 ** Close the cursor.  For additional information see the documentation
 ** on the xClose method of the virtual table interface.
@@ -1714,11 +1727,7 @@ static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){
 static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
   Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
   assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
-  fts3CursorFinalizeStmt(pCsr);
-  sqlite3Fts3ExprFree(pCsr->pExpr);
-  sqlite3Fts3FreeDeferredTokens(pCsr);
-  sqlite3_free(pCsr->aDoclist);
-  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
+  fts3ClearCursor(pCsr);
   assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
   sqlite3_free(pCsr);
   return SQLITE_OK;
@@ -1744,7 +1753,7 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr){
     }else{
       zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
       if( !zSql ) return SQLITE_NOMEM;
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+      rc = sqlite3_prepare_v3(p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
       sqlite3_free(zSql);
     }
     if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1;
@@ -3219,11 +3228,7 @@ static int fts3FilterMethod(
   assert( iIdx==nVal );
 
   /* In case the cursor has been used before, clear it now. */
-  fts3CursorFinalizeStmt(pCsr);
-  sqlite3_free(pCsr->aDoclist);
-  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
-  sqlite3Fts3ExprFree(pCsr->pExpr);
-  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
+  fts3ClearCursor(pCsr);
 
   /* Set the lower and upper bounds on docids to return */
   pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
@@ -3281,7 +3286,7 @@ static int fts3FilterMethod(
       );
     }
     if( zSql ){
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+      rc = sqlite3_prepare_v3(p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
       sqlite3_free(zSql);
     }else{
       rc = SQLITE_NOMEM;
@@ -3302,7 +3307,12 @@ static int fts3FilterMethod(
 ** routine to find out if it has reached the end of a result set.
 */
 static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
-  return ((Fts3Cursor *)pCursor)->isEof;
+  Fts3Cursor *pCsr = (Fts3Cursor*)pCursor;
+  if( pCsr->isEof ){
+    fts3ClearCursor(pCsr);
+    pCsr->isEof = 1;
+  }
+  return pCsr->isEof;
 }
 
 /* 

ext/fts3/fts3_write.c

@@ -407,7 +407,8 @@ static int fts3SqlStmt(
     if( !zSql ){
       rc = SQLITE_NOMEM;
     }else{
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
+      rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
+                              &pStmt, NULL);
       sqlite3_free(zSql);
       assert( rc==SQLITE_OK || pStmt==0 );
       p->aStmt[eStmt] = pStmt;

ext/fts5/fts5_index.c

@@ -728,7 +728,8 @@ static int fts5IndexPrepareStmt(
 ){
   if( p->rc==SQLITE_OK ){
     if( zSql ){
-      p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0);
+      p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1,
+                                 SQLITE_PREPARE_PERSISTENT, ppStmt, 0);
     }else{
       p->rc = SQLITE_NOMEM;
     }
@@ -777,7 +778,8 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){
     if( zSql==0 ){
       rc = SQLITE_NOMEM;
     }else{
-      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0);
+      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
+                              SQLITE_PREPARE_PERSISTENT, &p->pDeleter, 0);
       sqlite3_free(zSql);
     }
     if( rc!=SQLITE_OK ){

ext/fts5/fts5_main.c

@@ -883,7 +883,8 @@ static int fts5PrepareStatement(
   if( zSql==0 ){
     rc = SQLITE_NOMEM; 
   }else{
-    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
+    rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, 
+                            SQLITE_PREPARE_PERSISTENT, &pRet, 0);
     if( rc!=SQLITE_OK ){
       *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
     }
@@ -1019,7 +1020,8 @@ static int fts5FindRankFunction(Fts5Cursor *pCsr){
     char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs);
     if( zSql ){
       sqlite3_stmt *pStmt = 0;
-      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0);
+      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
+                              SQLITE_PREPARE_PERSISTENT, &pStmt, 0);
       sqlite3_free(zSql);
       assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 );
       if( rc==SQLITE_OK ){

ext/fts5/fts5_storage.c

@@ -136,7 +136,8 @@ static int fts5StorageGetStmt(
     if( zSql==0 ){
       rc = SQLITE_NOMEM;
     }else{
-      rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
+      rc = sqlite3_prepare_v3(pC->db, zSql, -1,
+                              SQLITE_PREPARE_PERSISTENT, &p->aStmt[eStmt], 0);
       sqlite3_free(zSql);
       if( rc!=SQLITE_OK && pzErrMsg ){
         *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));

ext/fts5/test/fts5aa.test

@@ -441,7 +441,7 @@ db func funk funk
 do_catchsql_test 16.2 {
   SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
 } {0 {{} -1e-06 {}}}
-# {1 {SQL logic error or missing database}}
+# {1 {SQL logic error}}
 
 #-------------------------------------------------------------------------
 #
@@ -595,5 +595,3 @@ do_execsql_test 22.1 {
 
 
 finish_test
-
-

ext/fts5/test/fts5ab.test

@@ -294,4 +294,3 @@ do_execsql_test 7.0 {
 
 
 finish_test
-

ext/fts5/test/fts5ac.test

@@ -276,4 +276,3 @@ foreach {tn expr tclexpr} {
 }
 
 finish_test
-

ext/fts5/test/fts5ad.test

@@ -242,4 +242,3 @@ foreach {T create} {
 }
 
 finish_test
-

ext/fts5/test/fts5ae.test

@@ -309,4 +309,3 @@ foreach {tn q cnt} {
 }
 
 finish_test
-

ext/fts5/test/fts5af.test

@@ -178,4 +178,3 @@ do_execsql_test 5.1 {
 } ;# foreach_detail_mode 
 
 finish_test
-

ext/fts5/test/fts5ag.test

@@ -142,4 +142,3 @@ if {[detail_is_full]} {
 
 
 finish_test
-

ext/fts5/test/fts5ah.test

@@ -167,4 +167,3 @@ do_execsql_test 1.8.2 {
 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
 
 finish_test
-

ext/fts5/test/fts5ai.test

@@ -55,4 +55,3 @@ do_execsql_test 1.2 {
 
 
 finish_test
-

ext/fts5/test/fts5aj.test

@@ -66,4 +66,3 @@ do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }
 
 
 finish_test
-

ext/fts5/test/fts5ak.test

@@ -147,4 +147,3 @@ do_execsql_test 3.1 {
 }
 
 finish_test
-

ext/fts5/test/fts5al.test

@@ -77,7 +77,7 @@ foreach {tn defn} {
 } {
   do_test 2.2.$tn {
     catchsql { INSERT INTO ft1(ft1, rank) VALUES('rank', $defn) }
-  } {1 {SQL logic error or missing database}}
+  } {1 {SQL logic error}}
 }
 
 #-------------------------------------------------------------------------
@@ -297,4 +297,3 @@ do_catchsql_test 4.4.4 {
 
 
 finish_test
-

ext/fts5/test/fts5alter.test

@@ -100,4 +100,3 @@ do_execsql_test 3.3 {
 } {1 2}
 
 finish_test
-

ext/fts5/test/fts5auto.test

@@ -342,4 +342,3 @@ foreach {tn expr} {
 
 
 finish_test
-

ext/fts5/test/fts5aux.test

@@ -279,4 +279,3 @@ do_execsql_test 9.3 {
 
 
 finish_test
-

ext/fts5/test/fts5auxdata.test

@@ -112,4 +112,3 @@ db eval {
 }
 
 finish_test
-

ext/fts5/test/fts5bigpl.test

@@ -61,4 +61,3 @@ do_test 2.1...slow {
 } {}
 
 finish_test
-

ext/fts5/test/fts5bigtok.test

@@ -64,5 +64,3 @@ foreach_detail_mode $::testprefix {
 }
 
 finish_test
-
-

ext/fts5/test/fts5colset.test

@@ -84,5 +84,3 @@ foreach_detail_mode $::testprefix {
 
 
 finish_test
-
-

ext/fts5/test/fts5config.test

@@ -66,7 +66,7 @@ foreach {tn val} {
 } {
   do_catchsql_test 3.$tn {
     INSERT INTO t1(t1, rank) VALUES('rank', $val);
-  } {1 {SQL logic error or missing database}}
+  } {1 {SQL logic error}}
 }
 
 #-------------------------------------------------------------------------
@@ -169,33 +169,33 @@ do_execsql_test 9.0 {
 } {}
 do_catchsql_test 9.1.1 {
   INSERT INTO abc(abc, rank) VALUES('pgsz', -5);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.1.2 {
   INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.1.3 {
   INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 
 do_catchsql_test 9.2.1 {
   INSERT INTO abc(abc, rank) VALUES('automerge', -5);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.2.2 {
   INSERT INTO abc(abc, rank) VALUES('automerge', 50000000);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.2.3 {
   INSERT INTO abc(abc, rank) VALUES('automerge', 66.67);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_execsql_test 9.2.4 {
   INSERT INTO abc(abc, rank) VALUES('automerge', 1);
 } {}
 
 do_catchsql_test 9.3.1 {
   INSERT INTO abc(abc, rank) VALUES('crisismerge', -5);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.3.2 {
   INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_execsql_test 9.3.3 {
   INSERT INTO abc(abc, rank) VALUES('crisismerge', 1);
 } {}
@@ -205,14 +205,14 @@ do_execsql_test 9.3.4 {
 
 do_catchsql_test 9.4.1 {
   INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 
 do_catchsql_test 9.5.1 {
   INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.5.2 {
   INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 do_catchsql_test 9.5.3 {
   INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
 } {0 {}}
@@ -245,7 +245,7 @@ foreach {tn opt} {
 
 do_catchsql_test 12.1 {
   INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 
 #-------------------------------------------------------------------------
 # errors in the 'usermerge' option
@@ -260,8 +260,7 @@ foreach {tn val} {
   4     1
 } {
   set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)"
-  do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}}
+  do_catchsql_test 13.$tn $sql {1 {SQL logic error}}
 }
 
 finish_test
-

ext/fts5/test/fts5conflict.test

@@ -66,5 +66,3 @@ do_execsql_test 2.1 {
 }
 
 finish_test
-
-

ext/fts5/test/fts5content.test

@@ -255,4 +255,3 @@ do_execsql_test 6.2 {
 
 
 finish_test
-

ext/fts5/test/fts5corrupt.test

@@ -96,4 +96,3 @@ do_catchsql_test 3.1 {
 } {1 {database disk image is malformed}}
 
 finish_test
-

ext/fts5/test/fts5corrupt2.test

@@ -269,4 +269,3 @@ do_catchsql_test 6.2 {
 
 sqlite3_fts5_may_be_corrupt 0
 finish_test
-

ext/fts5/test/fts5corrupt3.test

@@ -409,4 +409,3 @@ do_catchsql_test 9.2.2 {
 
 sqlite3_fts5_may_be_corrupt 0
 finish_test
-

ext/fts5/test/fts5delete.test

@@ -51,4 +51,3 @@ do_test 1.2 {
 } {}
 
 finish_test
-

ext/fts5/test/fts5detail.test

@@ -241,4 +241,3 @@ do_execsql_test 5.3 {
 
 
 finish_test
-

ext/fts5/test/fts5determin.test

@@ -63,5 +63,3 @@ foreach_detail_mode $::testprefix {
 
 
 finish_test
-
-

ext/fts5/test/fts5dlidx.test

@@ -197,4 +197,3 @@ foreach v $vocab {
 
 
 finish_test
-

ext/fts5/test/fts5doclist.test

@@ -44,4 +44,3 @@ do_execsql_test 1.2 {
 
 
 finish_test
-

ext/fts5/test/fts5eb.test

@@ -81,6 +81,3 @@ do_execsql_test 3.3 {
 
 
 finish_test
-
-
-

ext/fts5/test/fts5fault1.test

@@ -351,4 +351,3 @@ do_faultsim_test 9.1 -faults oom-* -prep {
 
 
 finish_test
-

ext/fts5/test/fts5fault2.test

@@ -137,4 +137,3 @@ do_faultsim_test 5.0 -faults oom-* -prep {
 }
 
 finish_test
-

ext/fts5/test/fts5fault3.test

@@ -110,4 +110,3 @@ do_faultsim_test 3.2 -faults oom-* -prep {
 
 
 finish_test
-

ext/fts5/test/fts5fault4.test

@@ -395,4 +395,3 @@ do_faultsim_test 14.1 -faults oom-t* -prep {
 }
 
 finish_test
-

ext/fts5/test/fts5fault5.test

@@ -130,4 +130,3 @@ do_faultsim_test 3.3 -faults oom-t* -body {
 
 
 finish_test
-

ext/fts5/test/fts5fault6.test

@@ -292,4 +292,3 @@ do_faultsim_test 6 -faults oom* -prep {
   db close
 }
 finish_test
-

ext/fts5/test/fts5fault7.test

@@ -116,4 +116,3 @@ do_faultsim_test 2.2 -faults oom-* -body {
 }
 
 finish_test
-

ext/fts5/test/fts5fault8.test

@@ -82,4 +82,3 @@ do_faultsim_test 4 -faults oom-* -prep {
 
 
 finish_test
-

ext/fts5/test/fts5fault9.test

@@ -153,4 +153,3 @@ do_faultsim_test 6 -faults oom-* -body {
 } ;# foreach_detail_mode...
 
 finish_test
-

ext/fts5/test/fts5faultA.test

@@ -61,4 +61,3 @@ do_faultsim_test 2 -faults oom* -prep {
   faultsim_test_result {0 {1 2}}
 }
 finish_test
-

ext/fts5/test/fts5faultB.test

@@ -132,4 +132,3 @@ do_faultsim_test 4.2 -faults oom* -body {
 
 
 finish_test
-

ext/fts5/test/fts5full.test

@@ -40,4 +40,3 @@ do_test 1.1 {
 
 
 finish_test
-

ext/fts5/test/fts5fuzz1.test

@@ -90,4 +90,3 @@ do_catchsql_test 4.1 {
 } {1 {fts5: syntax error near "`"}}
 
 finish_test
-

ext/fts5/test/fts5hash.test

@@ -130,4 +130,3 @@ breakpoint
 } ;# foreach_detail_mode
 
 finish_test
-

ext/fts5/test/fts5integrity.test

@@ -210,4 +210,3 @@ foreach {tn pgsz} {
 }
 
 finish_test
-

ext/fts5/test/fts5lastrowid.test

@@ -70,4 +70,3 @@ do_execsql_test 1.6 {
 
 
 finish_test
-

ext/fts5/test/fts5leftjoin.test

@@ -41,5 +41,3 @@ do_execsql_test 1.2 {
 } {1 abc 2 abc}
 
 finish_test
-
-

ext/fts5/test/fts5matchinfo.test

@@ -492,4 +492,3 @@ do_catchsql_test 14.2 {
 } {1 {unrecognized matchinfo flag: d}}
 
 finish_test
-

ext/fts5/test/fts5merge.test

@@ -241,4 +241,3 @@ do_execsql_test 6.3 {
 
 
 finish_test
-

ext/fts5/test/fts5merge2.test

@@ -55,4 +55,3 @@ do_execsql_test 1.2 {
 }
 
 finish_test
-

ext/fts5/test/fts5multiclient.test

@@ -45,4 +45,3 @@ do_multiclient_test tn {
 };# do_multiclient_test
 };# foreach_detail_mode
 finish_test
-

ext/fts5/test/fts5near.test

@@ -68,4 +68,3 @@ do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0
 
 
 finish_test
-

ext/fts5/test/fts5onepass.test

@@ -178,4 +178,3 @@ do_execsql_test 4.3.1 {
 do_test 4.2.2 { fts5_level_segs ttt } {3}
 
 finish_test
-

ext/fts5/test/fts5optimize.test

@@ -106,4 +106,3 @@ foreach {tn nStep} {
   do_test 2.$tn.6 { fts5_segcount t1 } 1
 }
 finish_test
-

ext/fts5/test/fts5phrase.test

@@ -116,4 +116,3 @@ do_execsql_test 2.0 {
 }
 
 finish_test
-

ext/fts5/test/fts5plan.test

@@ -64,4 +64,3 @@ do_eqp_test 1.5 {
 
 
 finish_test
-

ext/fts5/test/fts5porter.test

@@ -11803,4 +11803,3 @@ foreach {in out} $test_vocab {
 
 
 finish_test
-

ext/fts5/test/fts5porter2.test

@@ -67,4 +67,3 @@ foreach {in out} $test_vocab {
 
 
 finish_test
-

ext/fts5/test/fts5prefix.test

@@ -341,5 +341,3 @@ foreach {tn create} {
 }
 
 finish_test
-
-

ext/fts5/test/fts5query.test

@@ -79,5 +79,3 @@ for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
 
 
 finish_test
-
-

ext/fts5/test/fts5rank.test

@@ -152,4 +152,3 @@ do_execsql_test 4.1 {
 
 
 finish_test
-

ext/fts5/test/fts5rebuild.test

@@ -64,4 +64,3 @@ do_catchsql_test 2.2 {
   INSERT INTO nc(nc) VALUES('rebuild');
 } {1 {'rebuild' may not be used with a contentless fts5 table}}
 finish_test
-

ext/fts5/test/fts5restart.test

@@ -149,4 +149,3 @@ do_test 4.3 {
 
 
 finish_test
-

ext/fts5/test/fts5rowid.test

@@ -216,4 +216,3 @@ do_execsql_test 6.2 {
 
 
 finish_test
-

ext/fts5/test/fts5simple2.test

@@ -370,4 +370,3 @@ do_execsql_test 17.6 {
 #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r}
   
 finish_test
-

ext/fts5/test/fts5simple3.test

@@ -116,4 +116,3 @@ do_execsql_test 4.6 {
 
 
 finish_test
-

ext/fts5/test/fts5synonym.test

@@ -421,4 +421,3 @@ do_execsql_test 7.1.2 {
 } ;# foreach_detail_mode
 
 finish_test
-

ext/fts5/test/fts5synonym2.test

@@ -161,4 +161,3 @@ foreach {tn expr} {
 }
 
 finish_test
-

ext/fts5/test/fts5tok1.test

@@ -109,7 +109,7 @@ do_catchsql_test 2.0 {
 do_catchsql_test 2.1 {
   CREATE VIRTUAL TABLE t4 USING fts5tokenize;
   SELECT * FROM t4;
-} {1 {SQL logic error or missing database}}
+} {1 {SQL logic error}}
 
 
 finish_test

ext/fts5/test/fts5tokenizer.test

@@ -302,4 +302,3 @@ do_test 9.5.2 { set ::flags } {query}
 
 
 finish_test
-

ext/fts5/test/fts5unicode.test

@@ -59,4 +59,3 @@ do_execsql_test 2.1 "
 
 
 finish_test
-

ext/fts5/test/fts5unicode3.test

@@ -126,4 +126,3 @@ do_test 1.5 {
 
 
 finish_test
-

ext/fts5/test/fts5unindexed.test

@@ -76,4 +76,3 @@ do_execsql_test 3.2 {
 
 
 finish_test
-

ext/fts5/test/fts5update.test

@@ -117,5 +117,3 @@ do_execsql_test 2.2.integrity {
 
 }
 finish_test
-
-

ext/fts5/test/fts5version.test

@@ -61,4 +61,3 @@ do_test 1.7 {
 
 
 finish_test
-

ext/lsm1/Makefile

@@ -0,0 +1,56 @@
+#
+# This Makefile is designed for use with main.mk in the root directory of
+# this project. After including main.mk, the users makefile should contain:
+#
+#    LSMDIR=$(TOP)/ext/lsm1/
+#    include $(LSMDIR)/Makefile
+#
+# The most useful targets are [lsmtest] and [lsm.so].
+#
+
+LSMOBJ    = \
+  lsm_ckpt.o \
+  lsm_file.o \
+  lsm_log.o \
+  lsm_main.o \
+  lsm_mem.o \
+  lsm_mutex.o \
+  lsm_shared.o \
+  lsm_sorted.o \
+  lsm_str.o \
+  lsm_tree.o \
+  lsm_unix.o \
+  lsm_win32.o \
+  lsm_varint.o \
+  lsm_vtab.o
+
+LSMHDR   = \
+  $(LSMDIR)/lsm.h \
+  $(LSMDIR)/lsmInt.h
+
+LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c     \
+             $(LSMDIR)/lsm-test/lsmtest3.c $(LSMDIR)/lsm-test/lsmtest4.c     \
+             $(LSMDIR)/lsm-test/lsmtest5.c $(LSMDIR)/lsm-test/lsmtest6.c     \
+             $(LSMDIR)/lsm-test/lsmtest7.c $(LSMDIR)/lsm-test/lsmtest8.c     \
+             $(LSMDIR)/lsm-test/lsmtest9.c                                   \
+             $(LSMDIR)/lsm-test/lsmtest_datasource.c \
+             $(LSMDIR)/lsm-test/lsmtest_func.c $(LSMDIR)/lsm-test/lsmtest_io.c  \
+             $(LSMDIR)/lsm-test/lsmtest_main.c $(LSMDIR)/lsm-test/lsmtest_mem.c \
+             $(LSMDIR)/lsm-test/lsmtest_tdb.c $(LSMDIR)/lsm-test/lsmtest_tdb3.c \
+             $(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c
+
+
+# all: lsm.so
+
+LSMOPTS = -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR)
+
+lsm.so:	$(LSMOBJ)
+	$(TCCX) -shared -o lsm.so $(LSMOBJ)
+
+%.o:	$(LSMDIR)/%.c $(LSMHDR) sqlite3.h
+	$(TCCX) $(LSMOPTS) -c $<
+	
+lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o
+	# $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
+	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) 
+

ext/lsm1/Makefile.msc

@@ -0,0 +1,100 @@
+#
+# This Makefile is designed for use with main.mk in the root directory of
+# this project. After including main.mk, the users makefile should contain:
+#
+#    LSMDIR=$(TOP)\ext\lsm1\
+#    include $(LSMDIR)\Makefile.msc
+#
+# The most useful targets are [lsmtest.exe] and [lsm.dll].
+#
+
+LSMOBJ    = \
+  lsm_ckpt.lo \
+  lsm_file.lo \
+  lsm_log.lo \
+  lsm_main.lo \
+  lsm_mem.lo \
+  lsm_mutex.lo \
+  lsm_shared.lo \
+  lsm_sorted.lo \
+  lsm_str.lo \
+  lsm_tree.lo \
+  lsm_unix.lo \
+  lsm_win32.lo \
+  lsm_varint.lo \
+  lsm_vtab.lo
+
+LSMHDR   = \
+  $(LSMDIR)\lsm.h \
+  $(LSMDIR)\lsmInt.h
+
+LSMTESTSRC = $(LSMDIR)\lsm-test\lsmtest1.c $(LSMDIR)\lsm-test\lsmtest2.c     \
+             $(LSMDIR)\lsm-test\lsmtest3.c $(LSMDIR)\lsm-test\lsmtest4.c     \
+             $(LSMDIR)\lsm-test\lsmtest5.c $(LSMDIR)\lsm-test\lsmtest6.c     \
+             $(LSMDIR)\lsm-test\lsmtest7.c $(LSMDIR)\lsm-test\lsmtest8.c     \
+             $(LSMDIR)\lsm-test\lsmtest9.c                                   \
+             $(LSMDIR)\lsm-test\lsmtest_datasource.c \
+             $(LSMDIR)\lsm-test\lsmtest_func.c $(LSMDIR)\lsm-test\lsmtest_io.c  \
+             $(LSMDIR)\lsm-test\lsmtest_main.c $(LSMDIR)\lsm-test\lsmtest_mem.c \
+             $(LSMDIR)\lsm-test\lsmtest_tdb.c $(LSMDIR)\lsm-test\lsmtest_tdb3.c \
+             $(LSMDIR)\lsm-test\lsmtest_util.c $(LSMDIR)\lsm-test\lsmtest_win32.c
+
+# all: lsm.dll
+
+LSMOPTS = -DLSM_MUTEX_WIN32=1 -I$(LSMDIR)
+
+!IF $(DEBUG)>2
+LSMOPTS = $(LSMOPTS) -DLSM_DEBUG=1
+!ENDIF
+
+!IF $(MEMDEBUG)!=0
+LSMOPTS = $(LSMOPTS) -DLSM_DEBUG_MEM=1
+!ENDIF
+
+lsm_ckpt.lo:	$(LSMDIR)\lsm_ckpt.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_ckpt.c
+
+lsm_file.lo:	$(LSMDIR)\lsm_file.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_file.c
+
+lsm_log.lo:	$(LSMDIR)\lsm_log.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_log.c
+
+lsm_main.lo:	$(LSMDIR)\lsm_main.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_main.c
+
+lsm_mem.lo:	$(LSMDIR)\lsm_mem.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mem.c
+
+lsm_mutex.lo:	$(LSMDIR)\lsm_mutex.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mutex.c
+
+lsm_shared.lo:	$(LSMDIR)\lsm_shared.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_shared.c
+
+lsm_sorted.lo:	$(LSMDIR)\lsm_sorted.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_sorted.c
+
+lsm_str.lo:	$(LSMDIR)\lsm_str.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_str.c
+
+lsm_tree.lo:	$(LSMDIR)\lsm_tree.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_tree.c
+
+lsm_unix.lo:	$(LSMDIR)\lsm_unix.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_unix.c
+
+lsm_win32.lo:	$(LSMDIR)\lsm_win32.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_win32.c
+
+lsm_varint.lo:	$(LSMDIR)\lsm_varint.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_varint.c
+
+lsm_vtab.lo:	$(LSMDIR)\lsm_vtab.c $(LSMHDR) $(SQLITE3H)
+	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_vtab.c
+
+lsm.dll:	$(LSMOBJ)
+	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ $(LSMOBJ)
+
+lsmtest.exe: $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) $(LIBOBJS1)
+	$(LTLINK) $(LSMOPTS) $(LSMTESTSRC) /link $(LSMOBJ) $(LIBOBJS1)

ext/lsm1/lsm-test/README

@@ -0,0 +1,40 @@
+
+
+Organization of test case files:
+
+  lsmtest1.c: Data tests. Tests that perform many inserts and deletes on a 
+              database file, then verify that the contents of the database can
+              be queried.
+
+  lsmtest2.c: Crash tests. Tests that attempt to verify that the database 
+              recovers correctly following an application or system crash.
+
+  lsmtest3.c: Rollback tests. Tests that focus on the explicit rollback of
+              transactions and sub-transactions.
+
+  lsmtest4.c: Multi-client tests.
+
+  lsmtest5.c: Multi-client tests with a different thread for each client.
+
+  lsmtest6.c: OOM injection tests.
+
+  lsmtest7.c: API tests.
+
+  lsmtest8.c: Writer crash tests. Tests in this file attempt to verify that
+              the system recovers and other clients proceed unaffected if
+              a process fails in the middle of a write transaction.
+
+              The difference from lsmtest2.c is that this file tests
+              live-recovery (recovery from a failure that occurs while other
+              clients are still running) whereas lsmtest2.c tests recovery
+              from a system or power failure.
+
+  lsmtest9.c: More data tests. These focus on testing that calling
+              lsm_work(nMerge=1) to compact the database does not corrupt it.
+              In other words, that databases containing block-redirects
+              can be read and written.
+
+
+
+
+

ext/lsm1/lsm-test/lsmtest.h

@@ -0,0 +1,292 @@
+
+#ifndef __WRAPPER_INT_H_
+#define __WRAPPER_INT_H_
+
+#include "lsmtest_tdb.h"
+#include "sqlite3.h"
+#include "lsm.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef _O_BINARY
+# define _O_BINARY (0)
+#endif
+
+#ifdef _WIN32
+# include "windows.h"
+# define gettimeofday win32GetTimeOfDay
+# define F_OK  (0)
+# define sleep(sec) Sleep(1000 * (sec))
+# define usleep(usec) Sleep(((usec) + 999) / 1000)
+# ifdef _MSC_VER
+#  include <io.h>
+#  define snprintf _snprintf
+#  define fsync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd)))
+#  define fdatasync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd)))
+#  define __va_copy(dst,src) ((dst) = (src))
+#  define ftruncate(fd,sz) ((_chsize_s((fd), (sz))==0) ? 0 : -1)
+# else
+#  error Unsupported C compiler for Windows.
+# endif
+int win32GetTimeOfDay(struct timeval *, void *);
+#endif
+
+#ifndef _LSM_INT_H
+typedef unsigned int  u32;
+typedef unsigned char u8;
+typedef long long int i64;
+typedef unsigned long long int u64;
+#endif
+
+
+#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0])))
+
+#define MIN(x,y) ((x)<(y) ? (x) : (y))
+#define MAX(x,y) ((x)>(y) ? (x) : (y))
+
+#define unused_parameter(x) (void)(x)
+
+#define TESTDB_DEFAULT_PAGE_SIZE   4096
+#define TESTDB_DEFAULT_CACHE_SIZE  2048
+
+/*
+** Ideally, these should be in wrapper.c. But they are here instead so that 
+** they can be used by the C++ database wrappers in wrapper2.cc.
+*/
+typedef struct DatabaseMethods DatabaseMethods;
+struct TestDb {
+  DatabaseMethods const *pMethods;          /* Database methods */
+  const char *zLibrary;                     /* Library name for tdb_open() */
+};
+struct DatabaseMethods {
+  int (*xClose)(TestDb *);
+  int (*xWrite)(TestDb *, void *, int , void *, int);
+  int (*xDelete)(TestDb *, void *, int);
+  int (*xDeleteRange)(TestDb *, void *, int, void *, int);
+  int (*xFetch)(TestDb *, void *, int, void **, int *);
+  int (*xScan)(TestDb *, void *, int, void *, int, void *, int,
+    void (*)(void *, void *, int , void *, int)
+  );
+  int (*xBegin)(TestDb *, int);
+  int (*xCommit)(TestDb *, int);
+  int (*xRollback)(TestDb *, int);
+};
+
+/* 
+** Functions in wrapper2.cc (a C++ source file). wrapper2.cc contains the
+** wrapper for Kyoto Cabinet. Kyoto cabinet has a C API, but
+** the primary interface is the C++ API.
+*/
+int test_kc_open(const char*, const char *zFilename, int bClear, TestDb **ppDb);
+int test_kc_close(TestDb *);
+int test_kc_write(TestDb *, void *, int , void *, int);
+int test_kc_delete(TestDb *, void *, int);
+int test_kc_delete_range(TestDb *, void *, int, void *, int);
+int test_kc_fetch(TestDb *, void *, int, void **, int *);
+int test_kc_scan(TestDb *, void *, int, void *, int, void *, int,
+  void (*)(void *, void *, int , void *, int)
+);
+
+int test_mdb_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
+int test_mdb_close(TestDb *);
+int test_mdb_write(TestDb *, void *, int , void *, int);
+int test_mdb_delete(TestDb *, void *, int);
+int test_mdb_fetch(TestDb *, void *, int, void **, int *);
+int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int,
+  void (*)(void *, void *, int , void *, int)
+);
+
+/* 
+** Functions in wrapper3.c. This file contains the tdb wrapper for lsm.
+** The wrapper for lsm is a bit more involved than the others, as it 
+** includes code for a couple of different lsm configurations, and for
+** various types of fault injection and robustness testing.
+*/
+int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
+int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
+int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
+int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
+int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);
+int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb);
+
+int tdb_lsm_configure(lsm_db *, const char *);
+
+/* Functions in lsmtest_tdb4.c */
+int test_bt_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
+int test_fbt_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
+int test_fbts_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
+
+
+/* Functions in testutil.c. */
+int  testPrngInit(void);
+u32  testPrngValue(u32 iVal);
+void testPrngArray(u32 iVal, u32 *aOut, int nOut);
+void testPrngString(u32 iVal, char *aOut, int nOut);
+
+void testErrorInit(int argc, char **);
+void testPrintError(const char *zFormat, ...);
+void testPrintUsage(const char *zArgs);
+void testPrintFUsage(const char *zFormat, ...);
+void testTimeInit(void);
+int  testTimeGet(void);
+
+/* Functions in testmem.c. */
+void testMallocInstall(lsm_env *pEnv);
+void testMallocUninstall(lsm_env *pEnv);
+void testMallocCheck(lsm_env *pEnv, int *, int *, FILE *);
+void testMallocOom(lsm_env *pEnv, int, int, void(*)(void*), void *);
+void testMallocOomEnable(lsm_env *pEnv, int);
+
+/* lsmtest.c */
+TestDb *testOpen(const char *zSystem, int, int *pRc);
+void testReopen(TestDb **ppDb, int *pRc);
+void testClose(TestDb **ppDb);
+
+void testFetch(TestDb *, void *, int, void *, int, int *);
+void testWrite(TestDb *, void *, int, void *, int, int *);
+void testDelete(TestDb *, void *, int, int *);
+void testDeleteRange(TestDb *, void *, int, void *, int, int *);
+void testWriteStr(TestDb *, const char *, const char *zVal, int *pRc);
+void testFetchStr(TestDb *, const char *, const char *, int *pRc);
+
+void testBegin(TestDb *pDb, int iTrans, int *pRc);
+void testCommit(TestDb *pDb, int iTrans, int *pRc);
+
+void test_failed(void);
+
+char *testMallocPrintf(const char *zFormat, ...);
+char *testMallocVPrintf(const char *zFormat, va_list ap);
+int testGlobMatch(const char *zPattern, const char *zStr);
+
+void testScanCompare(TestDb *, TestDb *, int, void *, int, void *, int, int *);
+void testFetchCompare(TestDb *, TestDb *, void *, int, int *);
+
+void *testMalloc(int);
+void *testMallocCopy(void *pCopy, int nByte);
+void *testRealloc(void *, int);
+void testFree(void *);
+
+/* lsmtest_bt.c */
+int do_bt(int nArg, char **azArg);
+
+/* testio.c */
+int testVfsConfigureDb(TestDb *pDb);
+
+/* testfunc.c */
+int do_show(int nArg, char **azArg);
+int do_work(int nArg, char **azArg);
+
+/* testio.c */
+int do_io(int nArg, char **azArg);
+
+/* lsmtest2.c */
+void do_crash_test(const char *zPattern, int *pRc);
+int do_rollback_test(int nArg, char **azArg);
+
+/* test3.c */
+void test_rollback(const char *zSystem, const char *zPattern, int *pRc);
+
+/* test4.c */
+void test_mc(const char *zSystem, const char *zPattern, int *pRc);
+
+/* test5.c */
+void test_mt(const char *zSystem, const char *zPattern, int *pRc);
+
+/* lsmtest6.c */
+void test_oom(const char *zPattern, int *pRc);
+void testDeleteLsmdb(const char *zFile);
+
+void testSaveDb(const char *zFile, const char *zAuxExt);
+void testRestoreDb(const char *zFile, const char *zAuxExt);
+void testCopyLsmdb(const char *zFrom, const char *zTo);
+
+/* lsmtest7.c */
+void test_api(const char *zPattern, int *pRc);
+
+/* lsmtest8.c */
+void do_writer_crash_test(const char *zPattern, int *pRc);
+
+/*************************************************************************
+** Interface to functionality in test_datasource.c.
+*/
+typedef struct Datasource Datasource;
+typedef struct DatasourceDefn DatasourceDefn;
+
+struct DatasourceDefn {
+  int eType;                      /* A TEST_DATASOURCE_* value */
+  int nMinKey;                    /* Minimum key size */
+  int nMaxKey;                    /* Maximum key size */
+  int nMinVal;                    /* Minimum value size */
+  int nMaxVal;                    /* Maximum value size */
+};
+
+#define TEST_DATASOURCE_RANDOM    1
+#define TEST_DATASOURCE_SEQUENCE  2
+
+char *testDatasourceName(const DatasourceDefn *);
+Datasource *testDatasourceNew(const DatasourceDefn *);
+void testDatasourceFree(Datasource *);
+void testDatasourceEntry(Datasource *, int, void **, int *, void **, int *);
+/* End of test_datasource.c interface.
+*************************************************************************/
+void testDatasourceFetch(
+  TestDb *pDb,                    /* Database handle */
+  Datasource *pData,
+  int iKey,
+  int *pRc                        /* IN/OUT: Error code */
+);
+
+void testWriteDatasource(TestDb *, Datasource *, int, int *);
+void testWriteDatasourceRange(TestDb *, Datasource *, int, int, int *);
+void testDeleteDatasource(TestDb *, Datasource *, int, int *);
+void testDeleteDatasourceRange(TestDb *, Datasource *, int, int, int *);
+
+
+/* test1.c */
+void test_data_1(const char *, const char *, int *pRc);
+void test_data_2(const char *, const char *, int *pRc);
+void test_data_3(const char *, const char *, int *pRc);
+void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *);
+void testCaseProgress(int, int, int, int *);
+int testCaseNDot(void);
+
+void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *);
+int testControlDb(TestDb **ppDb);
+
+typedef struct CksumDb CksumDb;
+CksumDb *testCksumArrayNew(Datasource *, int, int, int);
+char *testCksumArrayGet(CksumDb *, int);
+void testCksumArrayFree(CksumDb *);
+void testCaseStart(int *pRc, char *zFmt, ...);
+void testCaseFinish(int rc);
+void testCaseSkip(void);
+int testCaseBegin(int *, const char *, const char *, ...);
+
+#define TEST_CKSUM_BYTES 29
+int testCksumDatabase(TestDb *pDb, char *zOut);
+int testCountDatabase(TestDb *pDb);
+void testCompareInt(int, int, int *);
+void testCompareStr(const char *z1, const char *z2, int *pRc);
+
+/* lsmtest9.c */
+void test_data_4(const char *, const char *, int *pRc);
+
+
+/*
+** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function.
+*/
+#define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z)
+int testArgSelectX(void *, const char *, int, const char *, int *);
+
+#ifdef __cplusplus
+}  /* End of the 'extern "C"' block */
+#endif
+
+#endif

ext/lsm1/lsm-test/lsmtest1.c

@@ -0,0 +1,654 @@
+
+#include "lsmtest.h"
+
+#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE
+#define DATA_RANDOM     TEST_DATASOURCE_RANDOM
+
+typedef struct Datatest1 Datatest1;
+typedef struct Datatest2 Datatest2;
+
+/*
+** An instance of the following structure contains parameters used to
+** customize the test function in this file. Test procedure:
+**
+**   1. Create a data-source based on the "datasource definition" vars.
+**
+**   2. Insert nRow key value pairs into the database.
+**
+**   3. Delete all keys from the database. Deletes are done in the same 
+**      order as the inserts.
+**
+** During steps 2 and 3 above, after each Datatest1.nVerify inserts or
+** deletes, the following:
+**
+**   a. Run Datasource.nTest key lookups and check the results are as expected.
+**
+**   b. If Datasource.bTestScan is true, run a handful (8) of range
+**      queries (scanning forwards and backwards). Check that the results
+**      are as expected.
+**
+**   c. Close and reopen the database. Then run (a) and (b) again.
+*/
+struct Datatest1 {
+  /* Datasource definition */
+  DatasourceDefn defn;
+
+  /* Test procedure parameters */
+  int nRow;                       /* Number of rows to insert then delete */
+  int nVerify;                    /* How often to verify the db contents */
+  int nTest;                      /* Number of keys to test (0==all) */
+  int bTestScan;                  /* True to do scan tests */
+};
+
+/*
+** An instance of the following data structure is used to describe the
+** second type of test case in this file. The chief difference between 
+** these tests and those described by Datatest1 is that these tests also
+** experiment with range-delete operations. Tests proceed as follows:
+**
+**     1. Open the datasource described by Datatest2.defn. 
+**
+**     2. Open a connection on an empty database.
+**
+**     3. Do this Datatest2.nIter times:
+**
+**        a) Insert Datatest2.nWrite key-value pairs from the datasource.
+**
+**        b) Select two pseudo-random keys and use them as the start
+**           and end points of a range-delete operation.
+**
+**        c) Verify that the contents of the database are as expected (see
+**           below for details).
+**
+**        d) Close and then reopen the database handle.
+**
+**        e) Verify that the contents of the database are still as expected.
+**
+** The inserts and range deletes are run twice - once on the database being
+** tested and once using a control system (sqlite3, kc etc. - something that 
+** works). In order to verify that the contents of the db being tested are
+** correct, the test runs a bunch of scans and lookups on both the test and
+** control databases. If the results are the same, the test passes.
+*/
+struct Datatest2 {
+  DatasourceDefn defn;
+  int nRange;
+  int nWrite;                     /* Number of writes per iteration */
+  int nIter;                      /* Total number of iterations to run */
+};
+
+/*
+** Generate a unique name for the test case pTest with database system
+** zSystem.
+*/
+static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){
+  char *zRet;
+  char *zData;
+  zData = testDatasourceName(&pTest->defn);
+  zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", 
+      zSystem, zData, bRecover, pTest->nRow, pTest->nVerify
+  );
+  testFree(zData);
+  return zRet;
+}
+
+int testControlDb(TestDb **ppDb){
+#ifdef HAVE_KYOTOCABINET
+  return tdb_open("kyotocabinet", "tmp.db", 1, ppDb);
+#else
+  return tdb_open("sqlite3", "", 1, ppDb);
+#endif
+}
+
+void testDatasourceFetch(
+  TestDb *pDb,                    /* Database handle */
+  Datasource *pData,
+  int iKey,
+  int *pRc                        /* IN/OUT: Error code */
+){
+  void *pKey; int nKey;           /* Database key to query for */
+  void *pVal; int nVal;           /* Expected result of query */
+
+  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
+  testFetch(pDb, pKey, nKey, pVal, nVal, pRc);
+}
+
+/*
+** This function is called to test that the contents of database pDb
+** are as expected. In this case, expected is defined as containing
+** key-value pairs iFirst through iLast, inclusive, from data source 
+** pData. In other words, a loop like the following could be used to
+** construct a database with identical contents from scratch.
+**
+**   for(i=iFirst; i<=iLast; i++){
+**     testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
+**     // insert (pKey, nKey) -> (pVal, nVal) into database
+**   }
+**
+** The key domain consists of keys 0 to (nRow-1), inclusive, from
+** data source pData. For both scan and lookup tests, keys are selected
+** pseudo-randomly from within this set.
+**
+** This function runs nLookupTest lookup tests and nScanTest scan tests.
+**
+** A lookup test consists of selecting a key from the domain and querying
+** pDb for it. The test fails if the presence of the key and, if present,
+** the associated value do not match the expectations defined above.
+**
+** A scan test involves selecting a key from the domain and running
+** the following queries:
+**
+**   1. Scan all keys equal to or greater than the key, in ascending order.
+**   2. Scan all keys equal to or smaller than the key, in descending order.
+**
+** Additionally, if nLookupTest is greater than zero, the following are
+** run once:
+**
+**   1. Scan all keys in the db, in ascending order.
+**   2. Scan all keys in the db, in descending order.
+**
+** As you would assume, the test fails if the returned values do not match
+** expectations.
+*/
+void testDbContents(
+  TestDb *pDb,                    /* Database handle being tested */
+  Datasource *pData,              /* pDb contains data from here */
+  int nRow,                       /* Size of key domain */
+  int iFirst,                     /* Index of first key from pData in pDb */
+  int iLast,                      /* Index of last key from pData in pDb */
+  int nLookupTest,                /* Number of lookup tests to run */
+  int nScanTest,                  /* Number of scan tests to run */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  int j;
+  int rc = *pRc;
+
+  if( rc==0 && nScanTest ){
+    TestDb *pDb2 = 0;
+
+    /* Open a control db (i.e. one that we assume works) */
+    rc = testControlDb(&pDb2);
+
+    for(j=iFirst; rc==0 && j<=iLast; j++){
+      void *pKey; int nKey;         /* Database key to insert */
+      void *pVal; int nVal;         /* Database value to insert */
+      testDatasourceEntry(pData, j, &pKey, &nKey, &pVal, &nVal);
+      rc = tdb_write(pDb2, pKey, nKey, pVal, nVal);
+    }
+
+    if( rc==0 ){
+      int iKey1;
+      int iKey2;
+      void *pKey1; int nKey1;       /* Start key */
+      void *pKey2; int nKey2;       /* Final key */
+
+      iKey1 = testPrngValue((iFirst<<8) + (iLast<<16)) % nRow;
+      iKey2 = testPrngValue((iLast<<8) + (iFirst<<16)) % nRow;
+      testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0);
+      pKey1 = testMalloc(nKey1+1);
+      memcpy(pKey1, pKey2, nKey1+1);
+      testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0);
+
+      testScanCompare(pDb2, pDb, 0, 0, 0,         0, 0,         &rc);
+      testScanCompare(pDb2, pDb, 0, 0, 0,         pKey2, nKey2, &rc);
+      testScanCompare(pDb2, pDb, 0, pKey1, nKey1, 0, 0,         &rc);
+      testScanCompare(pDb2, pDb, 0, pKey1, nKey1, pKey2, nKey2, &rc);
+      testScanCompare(pDb2, pDb, 1, 0, 0,         0, 0,         &rc);
+      testScanCompare(pDb2, pDb, 1, 0, 0,         pKey2, nKey2, &rc);
+      testScanCompare(pDb2, pDb, 1, pKey1, nKey1, 0, 0,         &rc);
+      testScanCompare(pDb2, pDb, 1, pKey1, nKey1, pKey2, nKey2, &rc);
+      testFree(pKey1);
+    }
+    tdb_close(pDb2);
+  }
+
+  /* Test some lookups. */
+  for(j=0; rc==0 && j<nLookupTest; j++){
+    int iKey;                     /* Datasource key to test */
+    void *pKey; int nKey;         /* Database key to query for */
+    void *pVal; int nVal;         /* Expected result of query */
+
+    if( nLookupTest>=nRow ){
+      iKey = j;
+    }else{
+      iKey = testPrngValue(j + (iFirst<<8) + (iLast<<16)) % nRow;
+    }
+
+    testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
+    if( iFirst>iKey || iKey>iLast ){
+      pVal = 0;
+      nVal = -1;
+    }
+
+    testFetch(pDb, pKey, nKey, pVal, nVal, &rc);
+  }
+
+  *pRc = rc;
+}
+
+/*
+** This function should be called during long running test cases to output
+** the progress dots (...) to stdout.
+*/
+void testCaseProgress(int i, int n, int nDot, int *piDot){
+  int iDot = *piDot;
+  while( iDot < ( ((nDot*2+1) * i) / (n*2) ) ){
+    printf(".");
+    fflush(stdout);
+    iDot++;
+  }
+  *piDot = iDot;
+}
+
+int testCaseNDot(void){ return 20; }
+
+#if 0
+static void printScanCb(
+    void *pCtx, void *pKey, int nKey, void *pVal, int nVal
+){
+  printf("%s\n", (char *)pKey);
+  fflush(stdout);
+}
+#endif
+
+void testReopenRecover(TestDb **ppDb, int *pRc){
+  if( *pRc==0 ){
+    const char *zLib = tdb_library_name(*ppDb);
+    const char *zDflt = tdb_default_db(zLib);
+    testCopyLsmdb(zDflt, "bak.db");
+    testClose(ppDb);
+    testCopyLsmdb("bak.db", zDflt);
+    *pRc = tdb_open(zLib, 0, 0, ppDb);
+  }
+}
+
+
+static void doDataTest1(
+  const char *zSystem,            /* Database system to test */
+  int bRecover,
+  Datatest1 *p,                   /* Structure containing test parameters */
+  int *pRc                        /* OUT: Error code */
+){
+  int i;
+  int iDot;
+  int rc = LSM_OK;
+  Datasource *pData;
+  TestDb *pDb;
+
+  /* Start the test case, open a database and allocate the datasource. */
+  pDb = testOpen(zSystem, 1, &rc);
+  pData = testDatasourceNew(&p->defn);
+
+  i = 0;
+  iDot = 0;
+  while( rc==LSM_OK && i<p->nRow ){
+
+    /* Insert some data */
+    testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
+    i += p->nVerify;
+
+    /* Check that the db content is correct. */
+    testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
+
+    if( bRecover ){
+      testReopenRecover(&pDb, &rc);
+    }else{
+      testReopen(&pDb, &rc);
+    }
+
+    /* Check that the db content is still correct. */
+    testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
+
+    /* Update the progress dots... */
+    testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot);
+  }
+
+  i = 0;
+  iDot = 0;
+  while( rc==LSM_OK && i<p->nRow ){
+
+    /* Delete some entries */
+    testDeleteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
+    i += p->nVerify;
+
+    /* Check that the db content is correct. */
+    testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
+
+    /* Close and reopen the database. */
+    if( bRecover ){
+      testReopenRecover(&pDb, &rc);
+    }else{
+      testReopen(&pDb, &rc);
+    }
+
+    /* Check that the db content is still correct. */
+    testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
+
+    /* Update the progress dots... */
+    testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot);
+  }
+
+  /* Free the datasource, close the database and finish the test case. */
+  testDatasourceFree(pData);
+  tdb_close(pDb);
+  testCaseFinish(rc);
+  *pRc = rc;
+}
+
+
+void test_data_1(
+  const char *zSystem,            /* Database system name */
+  const char *zPattern,           /* Run test cases that match this pattern */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  Datatest1 aTest[] = {
+    { {DATA_RANDOM,     500,600,   1000,2000},     1000,  100,  10,  0},
+    { {DATA_RANDOM,     20,25,     100,200},       1000,  250, 1000, 1},
+    { {DATA_RANDOM,     8,10,      100,200},       1000,  250, 1000, 1},
+    { {DATA_RANDOM,     8,10,      10,20},         1000,  250, 1000, 1},
+    { {DATA_RANDOM,     8,10,      1000,2000},     1000,  250, 1000, 1},
+    { {DATA_RANDOM,     8,100,     10000,20000},    100,   25,  100, 1},
+    { {DATA_RANDOM,     80,100,    10,20},         1000,  250, 1000, 1},
+    { {DATA_RANDOM,     5000,6000, 10,20},          100,   25,  100, 1},
+    { {DATA_SEQUENTIAL, 5,10,      10,20},         1000,  250, 1000, 1},
+    { {DATA_SEQUENTIAL, 5,10,      100,200},       1000,  250, 1000, 1},
+    { {DATA_SEQUENTIAL, 5,10,      1000,2000},     1000,  250, 1000, 1},
+    { {DATA_SEQUENTIAL, 5,100,     10000,20000},    100,   25,  100, 1},
+    { {DATA_RANDOM,     10,10,     100,100},     100000, 1000,  100, 0},
+    { {DATA_SEQUENTIAL, 10,10,     100,100},     100000, 1000,  100, 0},
+  };
+
+  int i;
+  int bRecover;
+
+  for(bRecover=0; bRecover<2; bRecover++){
+    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
+    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
+      char *zName = getName(zSystem, bRecover, &aTest[i]);
+      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
+        doDataTest1(zSystem, bRecover, &aTest[i], pRc);
+      }
+      testFree(zName);
+    }
+  }
+}
+
+void testCompareDb(
+  Datasource *pData,
+  int nData,
+  int iSeed,
+  TestDb *pControl,
+  TestDb *pDb,
+  int *pRc
+){
+  int i;
+
+  static int nCall = 0;
+  nCall++;
+
+  testScanCompare(pControl, pDb, 0, 0, 0,         0, 0,         pRc);
+  testScanCompare(pControl, pDb, 1, 0, 0,         0, 0,         pRc);
+
+  if( *pRc==0 ){
+    int iKey1;
+    int iKey2;
+    void *pKey1; int nKey1;       /* Start key */
+    void *pKey2; int nKey2;       /* Final key */
+
+    iKey1 = testPrngValue(iSeed) % nData;
+    iKey2 = testPrngValue(iSeed+1) % nData;
+    testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0);
+    pKey1 = testMalloc(nKey1+1);
+    memcpy(pKey1, pKey2, nKey1+1);
+    testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0);
+
+    testScanCompare(pControl, pDb, 0, 0, 0,         pKey2, nKey2, pRc);
+    testScanCompare(pControl, pDb, 0, pKey1, nKey1, 0, 0,         pRc);
+    testScanCompare(pControl, pDb, 0, pKey1, nKey1, pKey2, nKey2, pRc);
+    testScanCompare(pControl, pDb, 1, 0, 0,         pKey2, nKey2, pRc);
+    testScanCompare(pControl, pDb, 1, pKey1, nKey1, 0, 0,         pRc);
+    testScanCompare(pControl, pDb, 1, pKey1, nKey1, pKey2, nKey2, pRc);
+    testFree(pKey1);
+  }
+
+  for(i=0; i<nData && *pRc==0; i++){
+    void *pKey; int nKey;
+    testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0);
+    testFetchCompare(pControl, pDb, pKey, nKey, pRc);
+  }
+}
+
+static void doDataTest2(
+  const char *zSystem,            /* Database system to test */
+  int bRecover,
+  Datatest2 *p,                   /* Structure containing test parameters */
+  int *pRc                        /* OUT: Error code */
+){
+  TestDb *pDb;
+  TestDb *pControl;
+  Datasource *pData;
+  int i;
+  int rc = LSM_OK;
+  int iDot = 0;
+
+  /* Start the test case, open a database and allocate the datasource. */
+  pDb = testOpen(zSystem, 1, &rc);
+  pData = testDatasourceNew(&p->defn);
+  rc = testControlDb(&pControl);
+
+  if( tdb_lsm(pDb) ){
+    int nBuf = 32 * 1024 * 1024;
+    lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf);
+  }
+
+  for(i=0; rc==0 && i<p->nIter; i++){
+    void *pKey1; int nKey1;
+    void *pKey2; int nKey2;
+    int ii;
+    int nRange = MIN(p->nIter*p->nWrite, p->nRange);
+
+    for(ii=0; rc==0 && ii<p->nWrite; ii++){
+      int iKey = (i*p->nWrite + ii) % p->nRange;
+      testWriteDatasource(pControl, pData, iKey, &rc);
+      testWriteDatasource(pDb, pData, iKey, &rc);
+    }
+
+    testDatasourceEntry(pData, i+1000000, &pKey1, &nKey1, 0, 0);
+    pKey1 = testMallocCopy(pKey1, nKey1);
+    testDatasourceEntry(pData, i+2000000, &pKey2, &nKey2, 0, 0);
+
+    testDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2, &rc);
+    testDeleteRange(pControl, pKey1, nKey1, pKey2, nKey2, &rc);
+    testFree(pKey1);
+
+    testCompareDb(pData, nRange, i, pControl, pDb, &rc);
+    if( bRecover ){
+      testReopenRecover(&pDb, &rc);
+    }else{
+      testReopen(&pDb, &rc);
+    }
+    testCompareDb(pData, nRange, i, pControl, pDb, &rc);
+
+    /* Update the progress dots... */
+    testCaseProgress(i, p->nIter, testCaseNDot(), &iDot);
+  }
+
+  testClose(&pDb);
+  testClose(&pControl);
+  testDatasourceFree(pData);
+  testCaseFinish(rc);
+  *pRc = rc;
+}
+
+static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){
+  char *zRet;
+  char *zData;
+  zData = testDatasourceName(&pTest->defn);
+  zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", 
+      zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter
+  );
+  testFree(zData);
+  return zRet;
+}
+
+void test_data_2(
+  const char *zSystem,            /* Database system name */
+  const char *zPattern,           /* Run test cases that match this pattern */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  Datatest2 aTest[] = {
+      /* defn,                                 nRange, nWrite, nIter */
+    { {DATA_RANDOM,     20,25,     100,200},   10000,  10,     50   },
+    { {DATA_RANDOM,     20,25,     100,200},   10000,  200,    50   },
+    { {DATA_RANDOM,     20,25,     100,200},   100,    10,     1000 },
+    { {DATA_RANDOM,     20,25,     100,200},   100,    200,    50   },
+  };
+
+  int i;
+  int bRecover;
+
+  for(bRecover=0; bRecover<2; bRecover++){
+    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
+    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
+      char *zName = getName2(zSystem, bRecover, &aTest[i]);
+      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
+        doDataTest2(zSystem, bRecover, &aTest[i], pRc);
+      }
+      testFree(zName);
+    }
+  }
+}
+
+/*************************************************************************
+** Test case data3.*
+*/
+
+typedef struct Datatest3 Datatest3;
+struct Datatest3 {
+  int nRange;                     /* Keys are between 1 and this value, incl. */
+  int nIter;                      /* Number of iterations */
+  int nWrite;                     /* Number of writes per iteration */
+  int nDelete;                    /* Number of deletes per iteration */
+
+  int nValMin;                    /* Minimum value size for writes */
+  int nValMax;                    /* Maximum value size for writes */
+};
+
+void testPutU32(u8 *aBuf, u32 iVal){
+  aBuf[0] = (iVal >> 24) & 0xFF;
+  aBuf[1] = (iVal >> 16) & 0xFF;
+  aBuf[2] = (iVal >>  8) & 0xFF;
+  aBuf[3] = (iVal >>  0) & 0xFF;
+}
+
+void dt3PutKey(u8 *aBuf, int iKey){
+  assert( iKey<100000 && iKey>=0 );
+  sprintf((char *)aBuf, "%.5d", iKey);
+}
+
+static void doDataTest3(
+  const char *zSystem,            /* Database system to test */
+  Datatest3 *p,                   /* Structure containing test parameters */
+  int *pRc                        /* OUT: Error code */
+){
+  int iDot = 0;
+  int rc = *pRc;
+  TestDb *pDb;
+  u8 *abPresent;                  /* Array of boolean */
+  char *aVal;                     /* Buffer to hold values */
+  int i;
+  u32 iSeq = 10;                  /* prng counter */
+
+  abPresent = (u8 *)testMalloc(p->nRange+1);
+  aVal = (char *)testMalloc(p->nValMax+1);
+  pDb = testOpen(zSystem, 1, &rc);
+
+  for(i=0; i<p->nIter && rc==0; i++){
+    int ii;
+
+    testCaseProgress(i, p->nIter, testCaseNDot(), &iDot);
+
+    /* Perform nWrite inserts */
+    for(ii=0; ii<p->nWrite; ii++){
+      u8 aKey[6];
+      u32 iKey;
+      int nVal;
+
+      iKey = (testPrngValue(iSeq++) % p->nRange) + 1;
+      nVal = (testPrngValue(iSeq++) % (p->nValMax - p->nValMin)) + p->nValMin;
+      testPrngString(testPrngValue(iSeq++), aVal, nVal);
+      dt3PutKey(aKey, iKey);
+
+      testWrite(pDb, aKey, sizeof(aKey)-1, aVal, nVal, &rc);
+      abPresent[iKey] = 1;
+    }
+
+    /* Perform nDelete deletes */
+    for(ii=0; ii<p->nDelete; ii++){
+      u8 aKey1[6];
+      u8 aKey2[6];
+      u32 iKey;
+
+      iKey = (testPrngValue(iSeq++) % p->nRange) + 1;
+      dt3PutKey(aKey1, iKey-1);
+      dt3PutKey(aKey2, iKey+1);
+
+      testDeleteRange(pDb, aKey1, sizeof(aKey1)-1, aKey2, sizeof(aKey2)-1, &rc);
+      abPresent[iKey] = 0;
+    }
+
+    testReopen(&pDb, &rc);
+
+    for(ii=1; rc==0 && ii<=p->nRange; ii++){
+      int nDbVal;
+      void *pDbVal;
+      u8 aKey[6];
+      int dbrc;
+
+      dt3PutKey(aKey, ii);
+      dbrc = tdb_fetch(pDb, aKey, sizeof(aKey)-1, &pDbVal, &nDbVal);
+      testCompareInt(0, dbrc, &rc);
+
+      if( abPresent[ii] ){
+        testCompareInt(1, (nDbVal>0), &rc);
+      }else{
+        testCompareInt(1, (nDbVal<0), &rc);
+      }
+    }
+  }
+
+  testClose(&pDb);
+  testCaseFinish(rc);
+  *pRc = rc;
+}
+
+static char *getName3(const char *zSystem, Datatest3 *p){
+  return testMallocPrintf("data3.%s.%d.%d.%d.%d.(%d..%d)",
+      zSystem, p->nRange, p->nIter, p->nWrite, p->nDelete, 
+      p->nValMin, p->nValMax
+  );
+}
+
+void test_data_3(
+  const char *zSystem,            /* Database system name */
+  const char *zPattern,           /* Run test cases that match this pattern */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  Datatest3 aTest[] = {
+    /* nRange, nIter, nWrite, nDelete, nValMin, nValMax */
+    {  100,    1000,  5,      5,       50,      100 },
+    {  100,    1000,  2,      2,        5,       10 },
+  };
+
+  int i;
+
+  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
+    char *zName = getName3(zSystem, &aTest[i]);
+    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
+      doDataTest3(zSystem, &aTest[i], pRc);
+    }
+    testFree(zName);
+  }
+}
+
+

ext/lsm1/lsm-test/lsmtest2.c

@@ -0,0 +1,488 @@
+
+/*
+** This file contains tests related to recovery following application 
+** and system crashes (power failures) while writing to the database.
+*/
+
+#include "lsmtest.h"
+
+/*
+** Structure used by testCksumDatabase() to accumulate checksum values in.
+*/
+typedef struct Cksum Cksum;
+struct Cksum {
+  int nRow;
+  int cksum1;
+  int cksum2;
+};
+
+/*
+** tdb_scan() callback used by testCksumDatabase()
+*/
+static void scanCksumDb(
+  void *pCtx, 
+  void *pKey, int nKey,
+  void *pVal, int nVal
+){
+  Cksum *p = (Cksum *)pCtx;
+  int i;
+
+  p->nRow++;
+  for(i=0; i<nKey; i++){
+    p->cksum1 += ((u8 *)pKey)[i];
+    p->cksum2 += p->cksum1;
+  }
+  for(i=0; i<nVal; i++){
+    p->cksum1 += ((u8 *)pVal)[i];
+    p->cksum2 += p->cksum1;
+  }
+}
+
+/*
+** tdb_scan() callback used by testCountDatabase()
+*/
+static void scanCountDb(
+  void *pCtx, 
+  void *pKey, int nKey,
+  void *pVal, int nVal
+){
+  Cksum *p = (Cksum *)pCtx;
+  p->nRow++;
+
+  unused_parameter(pKey);
+  unused_parameter(nKey);
+  unused_parameter(pVal);
+  unused_parameter(nVal);
+}
+
+
+/*
+** Iterate through the entire contents of database pDb. Write a checksum
+** string based on the db contents into buffer zOut before returning. A
+** checksum string is at most 29 (TEST_CKSUM_BYTES) bytes in size:
+**
+**    * 32-bit integer (10 bytes)
+**    * 1 space        (1 byte)
+**    * 32-bit hex     (8 bytes)
+**    * 1 space        (1 byte)
+**    * 32-bit hex     (8 bytes)
+**    * nul-terminator (1 byte)
+**
+** The number of entries in the database is returned.
+*/
+int testCksumDatabase(
+  TestDb *pDb,                    /* Database handle */
+  char *zOut                      /* Buffer to write checksum to */
+){
+  Cksum cksum;
+  memset(&cksum, 0, sizeof(Cksum));
+  tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCksumDb);
+  sprintf(zOut, "%d %x %x", 
+      cksum.nRow, (u32)cksum.cksum1, (u32)cksum.cksum2
+  );
+  assert( strlen(zOut)<TEST_CKSUM_BYTES );
+  return cksum.nRow;
+}
+
+int testCountDatabase(TestDb *pDb){
+  Cksum cksum;
+  memset(&cksum, 0, sizeof(Cksum));
+  tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCountDb);
+  return cksum.nRow;
+}
+
+/*
+** This function is a no-op if *pRc is not 0 when it is called.
+**
+** Otherwise, the two nul-terminated strings z1 and z1 are compared. If
+** they are the same, the function returns without doing anything. Otherwise,
+** an error message is printed, *pRc is set to 1 and the test_failed()
+** function called.
+*/
+void testCompareStr(const char *z1, const char *z2, int *pRc){
+  if( *pRc==0 ){
+    if( strcmp(z1, z2) ){
+      testPrintError("testCompareStr: \"%s\" != \"%s\"\n", z1, z2);
+      *pRc = 1;
+      test_failed();
+    }
+  }
+}
+
+/*
+** This function is a no-op if *pRc is not 0 when it is called.
+**
+** Otherwise, the two integers i1 and i2 are compared. If they are equal,
+** the function returns without doing anything. Otherwise, an error message 
+** is printed, *pRc is set to 1 and the test_failed() function called.
+*/
+void testCompareInt(int i1, int i2, int *pRc){
+  if( *pRc==0 && i1!=i2 ){
+    testPrintError("testCompareInt: %d != %d\n", i1, i2);
+    *pRc = 1;
+    test_failed();
+  }
+}
+
+void testCaseStart(int *pRc, char *zFmt, ...){
+  va_list ap;
+  va_start(ap, zFmt);
+  vprintf(zFmt, ap);
+  printf(" ...");
+  va_end(ap);
+  *pRc = 0;
+  fflush(stdout);
+}
+
+/*
+** This function is a no-op if *pRc is non-zero when it is called. Zero
+** is returned in this case.
+**
+** Otherwise, the zFmt (a printf style format string) and following arguments 
+** are used to create a test case name. If zPattern is NULL or a glob pattern
+** that matches the test case name, 1 is returned and the test case started.
+** Otherwise, zero is returned and the test case does not start.
+*/
+int testCaseBegin(int *pRc, const char *zPattern, const char *zFmt, ...){
+  int res = 0;
+  if( *pRc==0 ){
+    char *zTest;
+    va_list ap;
+
+    va_start(ap, zFmt);
+    zTest = testMallocVPrintf(zFmt, ap);
+    va_end(ap);
+    if( zPattern==0 || testGlobMatch(zPattern, zTest) ){
+      printf("%-50s ...", zTest);
+      res = 1;
+    }
+    testFree(zTest);
+    fflush(stdout);
+  }
+
+  return res;
+}
+
+void testCaseFinish(int rc){
+  if( rc==0 ){
+    printf("Ok\n");
+  }else{
+    printf("FAILED\n");
+  }
+  fflush(stdout);
+}
+
+void testCaseSkip(){
+  printf("Skipped\n");
+}
+
+void testSetupSavedLsmdb(
+  const char *zCfg,
+  const char *zFile,
+  Datasource *pData,
+  int nRow,
+  int *pRc
+){
+  if( *pRc==0 ){
+    int rc;
+    TestDb *pDb;
+    rc = tdb_lsm_open(zCfg, zFile, 1, &pDb);
+    if( rc==0 ){
+      testWriteDatasourceRange(pDb, pData, 0, nRow, &rc);
+      testClose(&pDb);
+      if( rc==0 ) testSaveDb(zFile, "log");
+    }
+    *pRc = rc;
+  }
+}
+
+/*
+** This function is a no-op if *pRc is non-zero when it is called.
+**
+** Open the LSM database identified by zFile and compute its checksum
+** (a string, as returned by testCksumDatabase()). If the checksum is
+** identical to zExpect1 or, if it is not NULL, zExpect2, the test passes.
+** Otherwise, print an error message and set *pRc to 1.
+*/
+static void testCompareCksumLsmdb(
+  const char *zFile,              /* Path to LSM database */
+  int bCompress,                  /* True if db is compressed */
+  const char *zExpect1,           /* Expected checksum 1 */
+  const char *zExpect2,           /* Expected checksum 2 (or NULL) */
+  int *pRc                        /* IN/OUT: Test case error code */
+){
+  if( *pRc==0 ){
+    char zCksum[TEST_CKSUM_BYTES];
+    TestDb *pDb;
+
+    *pRc = tdb_lsm_open((bCompress?"compression=1 mmap=0":""), zFile, 0, &pDb);
+    testCksumDatabase(pDb, zCksum);
+    testClose(&pDb);
+
+    if( *pRc==0 ){
+      int r1 = 0;
+      int r2 = -1;
+
+      r1 = strcmp(zCksum, zExpect1);
+      if( zExpect2 ) r2 = strcmp(zCksum, zExpect2);
+      if( r1 && r2 ){
+        if( zExpect2 ){
+          testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")",
+              zCksum, zExpect1, zExpect2
+          );
+        }else{
+          testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"",
+              zCksum, zExpect1
+          );
+        }
+        *pRc = 1;
+        test_failed();
+      }
+    }
+  }
+}
+
+#if 0 /* not used */
+static void testCompareCksumBtdb(
+  const char *zFile,              /* Path to LSM database */
+  const char *zExpect1,           /* Expected checksum 1 */
+  const char *zExpect2,           /* Expected checksum 2 (or NULL) */
+  int *pRc                        /* IN/OUT: Test case error code */
+){
+  if( *pRc==0 ){
+    char zCksum[TEST_CKSUM_BYTES];
+    TestDb *pDb;
+
+    *pRc = tdb_open("bt", zFile, 0, &pDb);
+    testCksumDatabase(pDb, zCksum);
+    testClose(&pDb);
+
+    if( *pRc==0 ){
+      int r1 = 0;
+      int r2 = -1;
+
+      r1 = strcmp(zCksum, zExpect1);
+      if( zExpect2 ) r2 = strcmp(zCksum, zExpect2);
+      if( r1 && r2 ){
+        if( zExpect2 ){
+          testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")",
+              zCksum, zExpect1, zExpect2
+          );
+        }else{
+          testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"",
+              zCksum, zExpect1
+          );
+        }
+        *pRc = 1;
+        test_failed();
+      }
+    }
+  }
+}
+#endif /* not used */
+
+/* Above this point are reusable test routines. Not clear that they
+** should really be in this file.
+*************************************************************************/
+
+/*
+** This test verifies that if a system crash occurs while doing merge work
+** on the db, no data is lost.
+*/
+static void crash_test1(int bCompress, int *pRc){
+  const char *DBNAME = "testdb.lsm";
+  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 200, 200};
+
+  const int nRow = 5000;          /* Database size */
+  const int nIter = 200;          /* Number of test iterations */
+  const int nWork = 20;           /* Maximum lsm_work() calls per iteration */
+  const int nPage = 15;           /* Pages per lsm_work call */
+
+  int i;
+  int iDot = 0;
+  Datasource *pData;
+  CksumDb *pCksumDb;
+  TestDb *pDb;
+  char *zCfg;
+
+  const char *azConfig[2] = {
+    "page_size=1024 block_size=65536 autoflush=16384 safety=2 mmap=0", 
+    "page_size=1024 block_size=65536 autoflush=16384 safety=2 "
+    " compression=1 mmap=0"
+  };
+  assert( bCompress==0 || bCompress==1 );
+
+  /* Allocate datasource. And calculate the expected checksums. */
+  pData = testDatasourceNew(&defn);
+  pCksumDb = testCksumArrayNew(pData, nRow, nRow, 1);
+
+  /* Setup and save the initial database. */
+
+  zCfg = testMallocPrintf("%s automerge=7", azConfig[bCompress]);
+  testSetupSavedLsmdb(zCfg, DBNAME, pData, 5000, pRc);
+  testFree(zCfg);
+
+  for(i=0; i<nIter && *pRc==0; i++){
+    int iWork;
+    int testrc = 0;
+
+    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
+
+    /* Restore and open the database. */
+    testRestoreDb(DBNAME, "log");
+    testrc = tdb_lsm_open(azConfig[bCompress], DBNAME, 0, &pDb);
+    assert( testrc==0 );
+
+    /* Call lsm_work() on the db */
+    tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nWork*2)));
+    for(iWork=0; testrc==0 && iWork<nWork; iWork++){
+      int nWrite = 0;
+      lsm_db *db = tdb_lsm(pDb);
+      testrc = lsm_work(db, 0, nPage, &nWrite);
+      /* assert( testrc!=0 || nWrite>0 ); */
+      if( testrc==0 ) testrc = lsm_checkpoint(db, 0);
+    }
+    tdb_close(pDb);
+
+    /* Check that the database content is still correct */
+    testCompareCksumLsmdb(DBNAME, 
+        bCompress, testCksumArrayGet(pCksumDb, nRow), 0, pRc);
+  }
+
+  testCksumArrayFree(pCksumDb);
+  testDatasourceFree(pData);
+}
+
+/*
+** This test verifies that if a system crash occurs while committing a
+** transaction to the log file, no earlier transactions are lost or damaged.
+*/
+static void crash_test2(int bCompress, int *pRc){
+  const char *DBNAME = "testdb.lsm";
+  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000};
+
+  const int nIter = 200;
+  const int nInsert = 20;
+
+  int i;
+  int iDot = 0;
+  Datasource *pData;
+  CksumDb *pCksumDb;
+  TestDb *pDb;
+
+  /* Allocate datasource. And calculate the expected checksums. */
+  pData = testDatasourceNew(&defn);
+  pCksumDb = testCksumArrayNew(pData, 100, 100+nInsert, 1);
+
+  /* Setup and save the initial database. */
+  testSetupSavedLsmdb("", DBNAME, pData, 100, pRc);
+
+  for(i=0; i<nIter && *pRc==0; i++){
+    int iIns;
+    int testrc = 0;
+
+    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
+
+    /* Restore and open the database. */
+    testRestoreDb(DBNAME, "log");
+    testrc = tdb_lsm_open("safety=2", DBNAME, 0, &pDb);
+    assert( testrc==0 );
+
+    /* Insert nInsert records into the database. Crash midway through. */
+    tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nInsert+2)));
+    for(iIns=0; iIns<nInsert; iIns++){
+      void *pKey; int nKey;
+      void *pVal; int nVal;
+
+      testDatasourceEntry(pData, 100+iIns, &pKey, &nKey, &pVal, &nVal);
+      testrc = tdb_write(pDb, pKey, nKey, pVal, nVal);
+      if( testrc ) break;
+    }
+    tdb_close(pDb);
+
+    /* Check that no data was lost when the system crashed. */
+    testCompareCksumLsmdb(DBNAME, bCompress,
+      testCksumArrayGet(pCksumDb, 100 + iIns),
+      testCksumArrayGet(pCksumDb, 100 + iIns + 1),
+      pRc
+    );
+  }
+
+  testDatasourceFree(pData);
+  testCksumArrayFree(pCksumDb);
+}
+
+
+/*
+** This test verifies that if a system crash occurs when checkpointing
+** the database, data is not lost (assuming that any writes not synced
+** to the db have been synced into the log file).
+*/
+static void crash_test3(int bCompress, int *pRc){
+  const char *DBNAME = "testdb.lsm";
+  const int nIter = 100;
+  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000};
+
+  int i;
+  int iDot = 0;
+  Datasource *pData;
+  CksumDb *pCksumDb;
+  TestDb *pDb;
+
+  /* Allocate datasource. And calculate the expected checksums. */
+  pData = testDatasourceNew(&defn);
+  pCksumDb = testCksumArrayNew(pData, 110, 150, 10);
+
+  /* Setup and save the initial database. */
+  testSetupSavedLsmdb("", DBNAME, pData, 100, pRc);
+
+  for(i=0; i<nIter && *pRc==0; i++){
+    int iOpen;
+    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
+    testRestoreDb(DBNAME, "log");
+
+    for(iOpen=0; iOpen<5; iOpen++){
+      /* Open the database. Insert 10 more records. */
+      pDb = testOpen("lsm", 0, pRc);
+      testWriteDatasourceRange(pDb, pData, 100+iOpen*10, 10, pRc);
+
+      /* Schedule a crash simulation then close the db. */
+      tdb_lsm_prepare_sync_crash(pDb, 1 + (i%2));
+      tdb_close(pDb);
+
+      /* Open the database and check that the crash did not cause any
+      ** data loss.  */
+      testCompareCksumLsmdb(DBNAME, bCompress,
+        testCksumArrayGet(pCksumDb, 110 + iOpen*10), 0,
+        pRc
+      );
+    }
+  }
+
+  testDatasourceFree(pData);
+  testCksumArrayFree(pCksumDb);
+}
+
+void do_crash_test(const char *zPattern, int *pRc){
+  struct Test {
+    const char *zTest;
+    void (*x)(int, int *);
+    int bCompress;
+  } aTest [] = {
+    { "crash.lsm.1",     crash_test1, 0 },
+#ifdef HAVE_ZLIB
+    { "crash.lsm_zip.1", crash_test1, 1 },
+#endif
+    { "crash.lsm.2",     crash_test2, 0 },
+    { "crash.lsm.3",     crash_test3, 0 },
+  };
+  int i;
+
+  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
+    struct Test *p = &aTest[i];
+    if( testCaseBegin(pRc, zPattern, "%s", p->zTest) ){
+      p->x(p->bCompress, pRc);
+      testCaseFinish(*pRc);
+    }
+  }
+}

ext/lsm1/lsm-test/lsmtest3.c

@@ -0,0 +1,238 @@
+
+
+/*
+** This file contains tests related to the explicit rollback of database
+** transactions and sub-transactions.
+*/
+
+
+/*
+** Repeat 2000 times (until the db contains 100,000 entries):
+**
+**   1. Open a transaction and insert 500 rows, opening a nested 
+**      sub-transaction each 100 rows.
+**
+**   2. Roll back to each sub-transaction savepoint. Check the database
+**      checksum looks Ok.
+**
+**   3. Every second iteration, roll back the main transaction. Check the
+**      db checksum is correct. Every other iteration, commit the main
+**      transaction (increasing the size of the db by 100 rows).
+*/
+
+
+#include "lsmtest.h"
+
+struct CksumDb {
+  int nFirst;
+  int nLast;
+  int nStep;
+  char **azCksum;
+};
+
+CksumDb *testCksumArrayNew(
+  Datasource *pData, 
+  int nFirst, 
+  int nLast, 
+  int nStep
+){
+  TestDb *pDb;
+  CksumDb *pRet;
+  int i;
+  int nEntry;
+  int rc = 0;
+
+  assert( nLast>=nFirst && ((nLast-nFirst)%nStep)==0 );
+ 
+  pRet = malloc(sizeof(CksumDb));
+  memset(pRet, 0, sizeof(CksumDb));
+  pRet->nFirst = nFirst;
+  pRet->nLast = nLast;
+  pRet->nStep = nStep;
+  nEntry = 1 + ((nLast - nFirst) / nStep);
+
+  /* Allocate space so that azCksum is an array of nEntry pointers to
+  ** buffers each TEST_CKSUM_BYTES in size.  */
+  pRet->azCksum = (char **)malloc(nEntry * (sizeof(char *) + TEST_CKSUM_BYTES));
+  for(i=0; i<nEntry; i++){
+    char *pStart = (char *)(&pRet->azCksum[nEntry]);
+    pRet->azCksum[i] = &pStart[i * TEST_CKSUM_BYTES];
+  }
+
+  tdb_open("lsm", "tempdb.lsm", 1, &pDb);
+  testWriteDatasourceRange(pDb, pData, 0, nFirst, &rc);
+  for(i=0; i<nEntry; i++){
+    testCksumDatabase(pDb, pRet->azCksum[i]);
+    if( i==nEntry ) break;
+    testWriteDatasourceRange(pDb, pData, nFirst+i*nStep, nStep, &rc);
+  }
+
+  tdb_close(pDb);
+
+  return pRet;
+}
+
+char *testCksumArrayGet(CksumDb *p, int nRow){
+  int i;
+  assert( nRow>=p->nFirst );
+  assert( nRow<=p->nLast );
+  assert( ((nRow-p->nFirst) % p->nStep)==0 );
+
+  i = (nRow - p->nFirst) / p->nStep;
+  return p->azCksum[i];
+}
+
+void testCksumArrayFree(CksumDb *p){
+  free(p->azCksum);
+  memset(p, 0x55, sizeof(*p));
+  free(p);
+}
+
+/* End of CksumDb code.
+**************************************************************************/
+
+/*
+** Test utility function. Write key-value pair $i from datasource pData 
+** into database pDb.
+*/
+void testWriteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){
+  void *pKey; int nKey;
+  void *pVal; int nVal;
+  testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
+  testWrite(pDb, pKey, nKey, pVal, nVal, pRc);
+}
+
+/*
+** Test utility function. Delete datasource pData key $i from database pDb.
+*/
+void testDeleteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){
+  void *pKey; int nKey;
+  testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0);
+  testDelete(pDb, pKey, nKey, pRc);
+}
+
+/*
+** This function inserts nWrite key/value pairs into database pDb - the
+** nWrite key value pairs starting at iFirst from data source pData.
+*/
+void testWriteDatasourceRange(
+  TestDb *pDb,                    /* Database to write to */
+  Datasource *pData,              /* Data source to read values from */
+  int iFirst,                     /* Index of first key/value pair */
+  int nWrite,                     /* Number of key/value pairs to write */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  int i;
+  for(i=0; i<nWrite; i++){
+    testWriteDatasource(pDb, pData, iFirst+i, pRc);
+  }
+}
+
+void testDeleteDatasourceRange(
+  TestDb *pDb,                    /* Database to write to */
+  Datasource *pData,              /* Data source to read keys from */
+  int iFirst,                     /* Index of first key */
+  int nWrite,                     /* Number of keys to delete */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  int i;
+  for(i=0; i<nWrite; i++){
+    testDeleteDatasource(pDb, pData, iFirst+i, pRc);
+  }
+}
+
+static char *getName(const char *zSystem){ 
+  char *zRet; 
+  zRet = testMallocPrintf("rollback.%s", zSystem);
+  return zRet;
+}
+
+static int rollback_test_1(
+  const char *zSystem,
+  Datasource *pData
+){
+  const int nRepeat = 100;
+
+  TestDb *pDb;
+  int rc;
+  int i;
+  CksumDb *pCksum;
+  char *zName;
+
+  zName = getName(zSystem);
+  testCaseStart(&rc, zName);
+  testFree(zName);
+
+  pCksum = testCksumArrayNew(pData, 0, nRepeat*100, 100);
+  pDb = 0;
+  rc = tdb_open(zSystem, 0, 1, &pDb);
+  if( pDb && tdb_transaction_support(pDb)==0 ){
+    testCaseSkip();
+    goto skip_rollback_test;
+  }
+
+  for(i=0; i<nRepeat && rc==0; i++){
+    char zCksum[TEST_CKSUM_BYTES];
+    int nCurrent = (((i+1)/2) * 100);
+    int nDbRow;
+    int iTrans;
+
+    /* Check that the database is the expected size. */
+    nDbRow = testCountDatabase(pDb);
+    testCompareInt(nCurrent, nDbRow, &rc);
+
+    for(iTrans=2; iTrans<=6 && rc==0; iTrans++){
+      tdb_begin(pDb, iTrans);
+      testWriteDatasourceRange(pDb, pData, nCurrent, 100, &rc);
+      nCurrent += 100;
+    }
+
+    testCksumDatabase(pDb, zCksum);
+    testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
+
+    for(iTrans=6; iTrans>2 && rc==0; iTrans--){
+      tdb_rollback(pDb, iTrans);
+      nCurrent -= 100;
+      testCksumDatabase(pDb, zCksum);
+      testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
+    }
+
+    if( i%2 ){
+      tdb_rollback(pDb, 0);
+      nCurrent -= 100;
+      testCksumDatabase(pDb, zCksum);
+      testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
+    }else{
+      tdb_commit(pDb, 0);
+    }
+  }
+  testCaseFinish(rc);
+
+ skip_rollback_test:
+  tdb_close(pDb);
+  testCksumArrayFree(pCksum);
+  return rc;
+}
+
+void test_rollback(
+  const char *zSystem, 
+  const char *zPattern, 
+  int *pRc
+){
+  if( *pRc==0 ){
+    int bRun = 1;
+
+    if( zPattern ){
+      char *zName = getName(zSystem);
+      bRun = testGlobMatch(zPattern, zName);
+      testFree(zName);
+    }
+
+    if( bRun ){
+      DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 50, 100 };
+      Datasource *pData = testDatasourceNew(&defn);
+      *pRc = rollback_test_1(zSystem, pData);
+      testDatasourceFree(pData);
+    }
+  }
+}

ext/lsm1/lsm-test/lsmtest4.c

@@ -0,0 +1,127 @@
+
+/*
+** This file contains test cases involving multiple database clients.
+*/
+
+#include "lsmtest.h"
+
+/*
+** The following code implements test cases "mc1.*".
+**
+** This test case uses one writer and $nReader readers. All connections
+** are driven by a single thread. All connections are opened at the start
+** of the test and remain open until the test is finished.
+**
+** The test consists of $nStep steps. Each step the following is performed:
+**
+**   1. The writer inserts $nWriteStep records into the db.
+**
+**   2. The writer checks that the contents of the db are as expected.
+**
+**   3. Each reader that currently has an open read transaction also checks
+**      that the contents of the db are as expected (according to the snapshot
+**      the read transaction is reading - see below).
+**
+** After step 1, reader 1 opens a read transaction. After step 2, reader
+** 2 opens a read transaction, and so on. At step ($nReader+1), reader 1
+** closes the current read transaction and opens a new one. And so on.
+** The result is that at step N (for N > $nReader), there exists a reader
+** with an open read transaction reading the snapshot committed following
+** steps (N-$nReader-1) to N. 
+*/
+typedef struct Mctest Mctest;
+struct Mctest {
+  DatasourceDefn defn;            /* Datasource to use */
+  int nStep;                      /* Total number of steps in test */
+  int nWriteStep;                 /* Number of rows to insert each step */
+  int nReader;                    /* Number of read connections */
+};
+static void do_mc_test(
+  const char *zSystem,            /* Database system to test */
+  Mctest *pTest,
+  int *pRc                        /* IN/OUT: return code */
+){
+  const int nDomain = pTest->nStep * pTest->nWriteStep;
+  Datasource *pData;              /* Source of data */
+  TestDb *pDb;                    /* First database connection (writer) */
+  int iReader;                    /* Used to iterate through aReader */
+  int iStep;                      /* Current step in test */
+  int iDot = 0;                   /* Current step in test */
+
+  /* Array of reader connections */
+  struct Reader {
+    TestDb *pDb;                  /* Connection handle */
+    int iLast;                    /* Current snapshot contains keys 0..iLast */
+  } *aReader;
+
+  /* Create a data source */
+  pData = testDatasourceNew(&pTest->defn);
+
+  /* Open the writer connection */
+  pDb = testOpen(zSystem, 1, pRc);
+
+  /* Allocate aReader */
+  aReader = (struct Reader *)testMalloc(sizeof(aReader[0]) * pTest->nReader);
+  for(iReader=0; iReader<pTest->nReader; iReader++){
+    aReader[iReader].pDb = testOpen(zSystem, 0, pRc);
+  }
+
+  for(iStep=0; iStep<pTest->nStep; iStep++){
+    int iLast;
+    int iBegin;                   /* Start read trans using aReader[iBegin] */
+
+    /* Insert nWriteStep more records into the database */
+    int iFirst = iStep*pTest->nWriteStep;
+    testWriteDatasourceRange(pDb, pData, iFirst, pTest->nWriteStep, pRc);
+
+    /* Check that the db is Ok according to the writer */
+    iLast = (iStep+1) * pTest->nWriteStep - 1;
+    testDbContents(pDb, pData, nDomain, 0, iLast, iLast, 1, pRc);
+
+    /* Have reader (iStep % nReader) open a read transaction here. */
+    iBegin = (iStep % pTest->nReader);
+    if( iBegin<iStep ) tdb_commit(aReader[iBegin].pDb, 0);
+    tdb_begin(aReader[iBegin].pDb, 1);
+    aReader[iBegin].iLast = iLast;
+
+    /* Check that the db is Ok for each open reader */
+    for(iReader=0; iReader<pTest->nReader && aReader[iReader].iLast; iReader++){
+      iLast = aReader[iReader].iLast;
+      testDbContents(
+          aReader[iReader].pDb, pData, nDomain, 0, iLast, iLast, 1, pRc
+      );
+    }
+
+    /* Report progress */
+    testCaseProgress(iStep, pTest->nStep, testCaseNDot(), &iDot);
+  }
+
+  /* Close all readers */
+  for(iReader=0; iReader<pTest->nReader; iReader++){
+    testClose(&aReader[iReader].pDb);
+  }
+  testFree(aReader);
+
+  /* Close the writer-connection and free the datasource */
+  testClose(&pDb);
+  testDatasourceFree(pData);
+}
+
+
+void test_mc(
+  const char *zSystem,            /* Database system name */
+  const char *zPattern,           /* Run test cases that match this pattern */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  int i;
+  Mctest aTest[] = {
+    { { TEST_DATASOURCE_RANDOM, 10,10, 100,100 }, 100, 10, 5 },
+  };
+
+  for(i=0; i<ArraySize(aTest); i++){
+    if( testCaseBegin(pRc, zPattern, "mc1.%s.%d", zSystem, i) ){
+      do_mc_test(zSystem, &aTest[i], pRc);
+      testCaseFinish(*pRc);
+    }
+  }
+}

ext/lsm1/lsm-test/lsmtest5.c

@@ -0,0 +1,633 @@
+
+/*
+** This file is broken into three semi-autonomous parts:
+**
+**   1. The database functions.
+**   2. The thread wrappers.
+**   3. The implementation of the mt1.* tests.
+*/
+
+/*************************************************************************
+** DATABASE CONTENTS:
+**
+**   The database contains up to N key/value pairs, where N is some large 
+**   number (say 10,000,000). Keys are integer values between 0 and (N-1).
+**   The value associated with each key is a pseudo-random blob of data.
+**
+**   Key/value pair keys are encoded as the two bytes "k." followed by a 
+**   10-digit decimal number. i.e. key 45 -> "k.0000000045".
+**
+**   As well as the key/value pairs, the database also contains checksum 
+**   entries. The checksums form a hierarchy - for every F key/value
+**   entries there is one level 1 checksum. And for each F level 1 checksums
+**   there is one level 2 checksum. And so on.
+**
+**   Checksum keys are encoded as the two byte "c." followed by the 
+**   checksum level, followed by a 10 digit decimal number containing
+**   the value of the first key that contributes to the checksum value.
+**   For example, assuming F==10, the level 1 checksum that spans keys
+**   10 to 19 is "c.1.0000000010".
+**
+**   Clients may perform one of two operations on the database: a read
+**   or a write.
+** 
+** READ OPERATIONS:
+**
+**   A read operation scans a range of F key/value pairs. It computes
+**   the expected checksum and then compares the computed value to the
+**   actual value stored in the level 1 checksum entry. It then scans 
+**   the group of F level 1 checksums, and compares the computed checksum 
+**   to the associated level 2 checksum value, and so on until the 
+**   highest level checksum value has been verified.
+**
+**   If a checksum ever fails to match the expected value, the test 
+**   has failed.
+**
+** WRITE OPERATIONS:
+**
+**   A write operation involves writing (possibly clobbering) a single
+**   key/value pair. The associated level 1 checksum is then recalculated
+**   updated. Then the level 2 checksum, and so on until the highest
+**   level checksum has been modified.
+**
+**   All updates occur inside a single transaction.
+**
+** INTERFACE:
+**
+**   The interface used by test cases to read and write the db consists
+**   of type DbParameters and the following functions:
+**
+**       dbReadOperation()
+**       dbWriteOperation()
+*/
+
+#include "lsmtest.h"
+
+typedef struct DbParameters DbParameters;
+struct DbParameters {
+  int nFanout;                    /* Checksum fanout (F) */
+  int nKey;                       /* Size of key space (N) */
+};
+
+#define DB_KEY_BYTES          (2+5+10+1)
+
+/*
+** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size.
+** This function populates the buffer with a nul-terminated key string 
+** corresponding to key iKey.
+*/
+static void dbFormatKey(
+  DbParameters *pParam,
+  int iLevel,
+  int iKey,                       /* Key value */
+  char *aBuf                      /* Write key string here */
+){
+  if( iLevel==0 ){
+    snprintf(aBuf, DB_KEY_BYTES, "k.%.10d", iKey);
+  }else{
+    int f = 1;
+    int i;
+    for(i=0; i<iLevel; i++) f = f * pParam->nFanout;
+    snprintf(aBuf, DB_KEY_BYTES, "c.%d.%.10d", iLevel, f*(iKey/f));
+  }
+}
+
+/*
+** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size.
+** This function populates the buffer with the string representation of
+** checksum value iVal.
+*/
+static void dbFormatCksumValue(u32 iVal, char *aBuf){
+  snprintf(aBuf, DB_KEY_BYTES, "%.10u", iVal);
+}
+
+/*
+** Return the highest level of checksum in the database described
+** by *pParam.
+*/
+static int dbMaxLevel(DbParameters *pParam){
+  int iMax;
+  int n = 1;
+  for(iMax=0; n<pParam->nKey; iMax++){
+    n = n * pParam->nFanout;
+  }
+  return iMax;
+}
+
+static void dbCksum(
+  void *pCtx,                     /* IN/OUT: Pointer to u32 containing cksum */
+  void *pKey, int nKey,           /* Database key. Unused. */
+  void *pVal, int nVal            /* Database value. Checksum this. */
+){
+  u8 *aVal = (u8 *)pVal;
+  u32 *pCksum = (u32 *)pCtx;
+  u32 cksum = *pCksum;
+  int i;
+
+  unused_parameter(pKey);
+  unused_parameter(nKey);
+
+  for(i=0; i<nVal; i++){
+    cksum += (cksum<<3) + (int)aVal[i];
+  }
+
+  *pCksum = cksum;
+}
+
+/*
+** Compute the value of the checksum stored on level iLevel that contains
+** data from key iKey by scanning the pParam->nFanout entries at level 
+** iLevel-1.
+*/
+static u32 dbComputeCksum(
+  DbParameters *pParam,           /* Database parameters */
+  TestDb *pDb,                    /* Database connection handle */
+  int iLevel,                     /* Level of checksum to compute */
+  int iKey,                       /* Compute checksum for this key */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  u32 cksum = 0;
+  if( *pRc==0 ){
+    int nFirst;
+    int nLast;
+    int iFirst = 0;
+    int iLast = 0;
+    int i;
+    int f = 1;
+    char zFirst[DB_KEY_BYTES];
+    char zLast[DB_KEY_BYTES];
+
+    assert( iLevel>=1 );
+    for(i=0; i<iLevel; i++) f = f * pParam->nFanout;
+
+    iFirst = f*(iKey/f);
+    iLast = iFirst + f - 1;
+    dbFormatKey(pParam, iLevel-1, iFirst, zFirst);
+    dbFormatKey(pParam, iLevel-1, iLast, zLast);
+    nFirst = strlen(zFirst);
+    nLast = strlen(zLast);
+
+    *pRc = tdb_scan(pDb, (u32*)&cksum, 0, zFirst, nFirst, zLast, nLast,dbCksum);
+  }
+
+  return cksum;
+}
+
+static void dbReadOperation(
+  DbParameters *pParam,           /* Database parameters */
+  TestDb *pDb,                    /* Database connection handle */
+  void (*xDelay)(void *),
+  void *pDelayCtx,
+  int iKey,                       /* Key to read */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  const int iMax = dbMaxLevel(pParam);
+  int i;
+
+  if( tdb_transaction_support(pDb) ) testBegin(pDb, 1, pRc);
+  for(i=1; *pRc==0 && i<=iMax; i++){
+    char zCksum[DB_KEY_BYTES];
+    char zKey[DB_KEY_BYTES];
+    u32 iCksum = 0;
+
+    iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc);
+    if( iCksum ){
+      if( xDelay && i==1 ) xDelay(pDelayCtx);
+      dbFormatCksumValue(iCksum, zCksum);
+      dbFormatKey(pParam, i, iKey, zKey);
+      testFetchStr(pDb, zKey, zCksum, pRc);
+    }
+  }
+  if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc);
+}
+
+static int dbWriteOperation(
+  DbParameters *pParam,           /* Database parameters */
+  TestDb *pDb,                    /* Database connection handle */
+  int iKey,                       /* Key to write to */
+  const char *zValue,             /* Nul-terminated value to write */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  const int iMax = dbMaxLevel(pParam);
+  char zKey[DB_KEY_BYTES];
+  int i;
+  int rc;
+
+  assert( iKey>=0 && iKey<pParam->nKey );
+  dbFormatKey(pParam, 0, iKey, zKey);
+
+  /* Open a write transaction. This may fail - SQLITE4_BUSY */
+  if( *pRc==0 && tdb_transaction_support(pDb) ){
+    rc = tdb_begin(pDb, 2);
+    if( rc==5 ) return 0;
+    *pRc = rc;
+  }
+
+  testWriteStr(pDb, zKey, zValue, pRc);
+  for(i=1; i<=iMax; i++){
+    char zCksum[DB_KEY_BYTES];
+    u32 iCksum = 0;
+
+    iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc);
+    dbFormatCksumValue(iCksum, zCksum);
+    dbFormatKey(pParam, i, iKey, zKey);
+    testWriteStr(pDb, zKey, zCksum, pRc);
+  }
+  if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc);
+  return 1;
+}
+
+/*************************************************************************
+** The following block contains testXXX() functions that implement a
+** wrapper around the systems native multi-thread support. There are no
+** synchronization primitives - just functions to launch and join 
+** threads. Wrapper functions are:
+**
+**    testThreadSupport()
+**
+**    testThreadInit()
+**    testThreadShutdown()
+**    testThreadLaunch()
+**    testThreadWait()
+**
+**    testThreadSetHalt()
+**    testThreadGetHalt()
+**    testThreadSetResult()
+**    testThreadGetResult()
+**
+**    testThreadEnterMutex()
+**    testThreadLeaveMutex()
+*/
+typedef struct ThreadSet ThreadSet;
+#ifdef LSM_MUTEX_PTHREADS
+
+#include <pthread.h>
+#include <unistd.h>
+
+typedef struct Thread Thread;
+struct Thread {
+  int rc;
+  char *zMsg;
+  pthread_t id;
+  void (*xMain)(ThreadSet *, int, void *);
+  void *pCtx;
+  ThreadSet *pThreadSet;
+};
+
+struct ThreadSet {
+  int bHalt;                      /* Halt flag */
+  int nThread;                    /* Number of threads */
+  Thread *aThread;                /* Array of Thread structures */
+  pthread_mutex_t mutex;          /* Mutex used for cheating */
+};
+
+/*
+** Return true if this build supports threads, or false otherwise. If
+** this function returns false, no other testThreadXXX() functions should
+** be called.
+*/
+static int testThreadSupport(){ return 1; }
+
+/*
+** Allocate and return a thread-set handle with enough space allocated
+** to handle up to nMax threads. Each call to this function should be
+** matched by a call to testThreadShutdown() to delete the object.
+*/
+static ThreadSet *testThreadInit(int nMax){
+  int nByte;                      /* Total space to allocate */
+  ThreadSet *p;                   /* Return value */
+
+  nByte = sizeof(ThreadSet) + sizeof(struct Thread) * nMax;
+  p = (ThreadSet *)testMalloc(nByte);
+  p->nThread = nMax;
+  p->aThread = (Thread *)&p[1];
+  pthread_mutex_init(&p->mutex, 0);
+
+  return p;
+}
+
+/*
+** Delete a thread-set object and release all resources held by it.
+*/
+static void testThreadShutdown(ThreadSet *p){
+  int i;
+  for(i=0; i<p->nThread; i++){
+    testFree(p->aThread[i].zMsg);
+  }
+  pthread_mutex_destroy(&p->mutex);
+  testFree(p);
+}
+
+static void *ttMain(void *pArg){
+  Thread *pThread = (Thread *)pArg;
+  int iThread;
+  iThread = (pThread - pThread->pThreadSet->aThread);
+  pThread->xMain(pThread->pThreadSet, iThread, pThread->pCtx);
+  return 0;
+}
+
+/*
+** Launch a new thread.
+*/
+static int testThreadLaunch(
+  ThreadSet *p,
+  int iThread,
+  void (*xMain)(ThreadSet *, int, void *),
+  void *pCtx
+){
+  int rc;
+  Thread *pThread;
+
+  assert( iThread>=0 && iThread<p->nThread );
+
+  pThread = &p->aThread[iThread];
+  assert( pThread->pThreadSet==0 );
+  pThread->xMain = xMain;
+  pThread->pCtx = pCtx;
+  pThread->pThreadSet = p;
+  rc = pthread_create(&pThread->id, 0, ttMain, (void *)pThread);
+
+  return rc;
+}
+
+/*
+** Set the thread-set "halt" flag.
+*/
+static void testThreadSetHalt(ThreadSet *pThreadSet){
+  pThreadSet->bHalt = 1;
+}
+
+/*
+** Return the current value of the thread-set "halt" flag.
+*/
+static int testThreadGetHalt(ThreadSet *pThreadSet){
+  return pThreadSet->bHalt;
+}
+
+static void testThreadSleep(ThreadSet *pThreadSet, int nMs){
+  int nRem = nMs;
+  while( nRem>0 && testThreadGetHalt(pThreadSet)==0 ){
+    usleep(50000);
+    nRem -= 50;
+  }
+}
+
+/*
+** Wait for all threads launched to finish before returning. If nMs
+** is greater than zero, set the "halt" flag to tell all threads
+** to halt after waiting nMs milliseconds.
+*/
+static void testThreadWait(ThreadSet *pThreadSet, int nMs){
+  int i;
+
+  testThreadSleep(pThreadSet, nMs);
+  testThreadSetHalt(pThreadSet);
+  for(i=0; i<pThreadSet->nThread; i++){
+    Thread *pThread = &pThreadSet->aThread[i];
+    if( pThread->xMain ){
+      pthread_join(pThread->id, 0);
+    }
+  }
+}
+
+/*
+** Set the result for thread iThread. 
+*/
+static void testThreadSetResult(
+  ThreadSet *pThreadSet,          /* Thread-set handle */
+  int iThread,                    /* Set result for this thread */
+  int rc,                         /* Result error code */
+  char *zFmt,                     /* Result string format */
+  ...                             /* Result string formatting args... */
+){
+  va_list ap;
+
+  testFree(pThreadSet->aThread[iThread].zMsg);
+  pThreadSet->aThread[iThread].rc = rc;
+  pThreadSet->aThread[iThread].zMsg = 0;
+  if( zFmt ){
+    va_start(ap, zFmt);
+    pThreadSet->aThread[iThread].zMsg = testMallocVPrintf(zFmt, ap);
+    va_end(ap);
+  }
+}
+
+/*
+** Retrieve the result for thread iThread. 
+*/
+static int testThreadGetResult(
+  ThreadSet *pThreadSet,          /* Thread-set handle */
+  int iThread,                    /* Get result for this thread */
+  const char **pzRes              /* OUT: Pointer to result string */
+){
+  if( pzRes ) *pzRes = pThreadSet->aThread[iThread].zMsg;
+  return pThreadSet->aThread[iThread].rc;
+}
+
+/*
+** Enter and leave the test case mutex.
+*/
+#if 0
+static void testThreadEnterMutex(ThreadSet *p){
+  pthread_mutex_lock(&p->mutex);
+}
+static void testThreadLeaveMutex(ThreadSet *p){
+  pthread_mutex_unlock(&p->mutex);
+}
+#endif
+#endif
+
+#if !defined(LSM_MUTEX_PTHREADS)
+static int testThreadSupport(){ return 0; }
+
+#define testThreadInit(a) 0
+#define testThreadShutdown(a)
+#define testThreadLaunch(a,b,c,d) 0
+#define testThreadWait(a,b)
+#define testThreadSetHalt(a)
+#define testThreadGetHalt(a) 0
+#define testThreadGetResult(a,b,c) 0
+#define testThreadSleep(a,b) 0
+
+static void testThreadSetResult(ThreadSet *a, int b, int c, char *d, ...){
+  unused_parameter(a);
+  unused_parameter(b);
+  unused_parameter(c);
+  unused_parameter(d);
+}
+#endif
+/* End of threads wrapper.
+*************************************************************************/
+
+/*************************************************************************
+** Below this point is the third part of this file - the implementation
+** of the mt1.* tests.
+*/
+typedef struct Mt1Test Mt1Test;
+struct Mt1Test {
+  DbParameters param;             /* Description of database to read/write */
+  int nReadwrite;                 /* Number of read/write threads */
+  int nFastReader;                /* Number of fast reader threads */
+  int nSlowReader;                /* Number of slow reader threads */
+  int nMs;                        /* How long to run for */
+  const char *zSystem;            /* Database system to test */
+};
+
+typedef struct Mt1DelayCtx Mt1DelayCtx;
+struct Mt1DelayCtx {
+  ThreadSet *pSet;                /* Threadset to sleep within */
+  int nMs;                        /* Sleep in ms */
+};
+
+static void xMt1Delay(void *pCtx){
+  Mt1DelayCtx *p = (Mt1DelayCtx *)pCtx;
+  testThreadSleep(p->pSet, p->nMs);
+}
+
+#define MT1_THREAD_RDWR 0
+#define MT1_THREAD_SLOW 1
+#define MT1_THREAD_FAST 2
+
+static void xMt1Work(lsm_db *pDb, void *pCtx){
+#if 0
+  char *z = 0;
+  lsm_info(pDb, LSM_INFO_DB_STRUCTURE, &z);
+  printf("%s\n", z);
+  fflush(stdout);
+#endif
+}
+
+/*
+** This is the main() proc for all threads in test case "mt1".
+*/
+static void mt1Main(ThreadSet *pThreadSet, int iThread, void *pCtx){
+  Mt1Test *p = (Mt1Test *)pCtx;   /* Test parameters */
+  Mt1DelayCtx delay;
+  int nRead = 0;                  /* Number of calls to dbReadOperation() */
+  int nWrite = 0;                 /* Number of completed database writes */
+  int rc = 0;                     /* Error code */
+  int iPrng;                      /* Prng argument variable */
+  TestDb *pDb;                    /* Database handle */
+  int eType;
+
+  delay.pSet = pThreadSet;
+  delay.nMs = 0;
+  if( iThread<p->nReadwrite ){
+    eType = MT1_THREAD_RDWR;
+  }else if( iThread<(p->nReadwrite+p->nFastReader) ){
+    eType = MT1_THREAD_FAST;
+  }else{
+    eType = MT1_THREAD_SLOW;
+    delay.nMs = (p->nMs / 20);
+  }
+
+  /* Open a new database connection. Initialize the pseudo-random number
+  ** argument based on the thread number.  */
+  iPrng = testPrngValue(iThread);
+  pDb = testOpen(p->zSystem, 0, &rc);
+
+  if( rc==0 ){
+    tdb_lsm_config_work_hook(pDb, xMt1Work, 0);
+  }
+
+  /* Loop until either an error occurs or some other thread sets the
+  ** halt flag.  */
+  while( rc==0 && testThreadGetHalt(pThreadSet)==0 ){
+    int iKey;
+
+    /* Perform a read operation on an arbitrarily selected key. */
+    iKey = (testPrngValue(iPrng++) % p->param.nKey);
+    dbReadOperation(&p->param, pDb, xMt1Delay, (void *)&delay, iKey, &rc);
+    if( rc ) continue;
+    nRead++;
+
+    /* Attempt to write an arbitrary key value pair (and update the associated
+    ** checksum entries). dbWriteOperation() returns 1 if the write is
+    ** successful, or 0 if it failed with an LSM_BUSY error.  */
+    if( eType==MT1_THREAD_RDWR ){
+      char aValue[50];
+      char aRnd[25];
+
+      iKey = (testPrngValue(iPrng++) % p->param.nKey);
+      testPrngString(iPrng, aRnd, sizeof(aRnd));
+      iPrng += sizeof(aRnd);
+      snprintf(aValue, sizeof(aValue), "%d.%s", iThread, aRnd);
+      nWrite += dbWriteOperation(&p->param, pDb, iKey, aValue, &rc);
+    }
+  }
+  testClose(&pDb);
+
+  /* If an error has occured, set the thread error code and the threadset 
+  ** halt flag to tell the other test threads to halt. Otherwise, set the
+  ** thread error code to 0 and post a message with the number of read
+  ** and write operations completed.  */
+  if( rc ){
+    testThreadSetResult(pThreadSet, iThread, rc, 0);
+    testThreadSetHalt(pThreadSet);
+  }else{
+    testThreadSetResult(pThreadSet, iThread, 0, "r/w: %d/%d", nRead, nWrite);
+  }
+}
+
+static void do_test_mt1(
+  const char *zSystem,            /* Database system name */
+  const char *zPattern,           /* Run test cases that match this pattern */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  Mt1Test aTest[] = {
+    /* param, nReadwrite, nFastReader, nSlowReader, nMs, zSystem */
+    { {10, 1000},     4, 0, 0,   10000,   0 },
+    { {10, 1000},     4, 4, 2,   100000,  0 },
+    { {10, 100000},   4, 0, 0,   10000,   0 },
+    { {10, 100000},   4, 4, 2,   100000,  0 },
+  };
+  int i;
+
+  for(i=0; *pRc==0 && i<ArraySize(aTest); i++){
+    Mt1Test *p = &aTest[i];
+    int bRun = testCaseBegin(pRc, zPattern, 
+        "mt1.%s.db=%d,%d.ms=%d.rdwr=%d.fast=%d.slow=%d", 
+        zSystem, p->param.nFanout, p->param.nKey, 
+        p->nMs, p->nReadwrite, p->nFastReader, p->nSlowReader
+    );
+    if( bRun ){
+      TestDb *pDb;
+      ThreadSet *pSet;
+      int iThread;
+      int nThread;
+
+      p->zSystem = zSystem;
+      pDb = testOpen(zSystem, 1, pRc);
+
+      nThread = p->nReadwrite + p->nFastReader + p->nSlowReader;
+      pSet = testThreadInit(nThread);
+      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
+        testThreadLaunch(pSet, iThread, mt1Main, (void *)p);
+      }
+
+      testThreadWait(pSet, p->nMs);
+      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
+        *pRc = testThreadGetResult(pSet, iThread, 0);
+      }
+      testCaseFinish(*pRc);
+
+      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
+        const char *zMsg = 0;
+        *pRc = testThreadGetResult(pSet, iThread, &zMsg);
+        printf("  Info: thread %d (%d): %s\n", iThread, *pRc, zMsg);
+      }
+
+      testThreadShutdown(pSet);
+      testClose(&pDb);
+    }
+  }
+}
+
+void test_mt(
+  const char *zSystem,            /* Database system name */
+  const char *zPattern,           /* Run test cases that match this pattern */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  if( testThreadSupport()==0 ) return;
+  do_test_mt1(zSystem, zPattern, pRc);
+}

ext/lsm1/lsm-test/lsmtest6.c

@@ -0,0 +1,668 @@
+
+#include "lsmtest.h"
+
+typedef struct OomTest OomTest;
+struct OomTest {
+  lsm_env *pEnv;
+  int iNext;                      /* Next value to pass to testMallocOom() */
+  int nFail;                      /* Number of OOM events injected */
+  int bEnable;
+  int rc;                         /* Test case error code */
+};
+
+static void testOomStart(OomTest *p){
+  memset(p, 0, sizeof(OomTest));
+  p->iNext = 1;
+  p->bEnable = 1;
+  p->nFail = 1;
+  p->pEnv = tdb_lsm_env();
+}
+
+static void xOomHook(OomTest *p){
+  p->nFail++;
+}
+
+static int testOomContinue(OomTest *p){
+  if( p->rc!=0 || (p->iNext>1 && p->nFail==0) ){
+    return 0;
+  }
+  p->nFail = 0;
+  testMallocOom(p->pEnv, p->iNext, 0, (void (*)(void*))xOomHook, (void *)p);
+  return 1;
+}
+
+static void testOomEnable(OomTest *p, int bEnable){
+  p->bEnable = bEnable;
+  testMallocOomEnable(p->pEnv, bEnable);
+}
+
+static void testOomNext(OomTest *p){
+  p->iNext++;
+}
+
+static int testOomHit(OomTest *p){
+  return (p->nFail>0);
+}
+
+static int testOomFinish(OomTest *p){
+  return p->rc;
+}
+
+static void testOomAssert(OomTest *p, int bVal){
+  if( bVal==0 ){
+    test_failed();
+    p->rc = 1;
+  }
+}
+
+/*
+** Test that the error code matches the state of the OomTest object passed
+** as the first argument. Specifically, check that rc is LSM_NOMEM if an 
+** OOM error has already been injected, or LSM_OK if not.
+*/
+static void testOomAssertRc(OomTest *p, int rc){
+  testOomAssert(p, rc==LSM_OK || rc==LSM_NOMEM);
+  testOomAssert(p, testOomHit(p)==(rc==LSM_NOMEM) || p->bEnable==0 );
+}
+
+static void testOomOpen(
+  OomTest *pOom,
+  const char *zName,
+  lsm_db **ppDb,
+  int *pRc
+){
+  if( *pRc==LSM_OK ){
+    int rc;
+    rc = lsm_new(tdb_lsm_env(), ppDb);
+    if( rc==LSM_OK ) rc = lsm_open(*ppDb, zName);
+    testOomAssertRc(pOom, rc);
+    *pRc = rc;
+  }
+}
+
+static void testOomFetch(
+  OomTest *pOom,
+  lsm_db *pDb,
+  void *pKey, int nKey,
+  void *pVal, int nVal,
+  int *pRc
+){
+  testOomAssertRc(pOom, *pRc);
+  if( *pRc==LSM_OK ){
+    lsm_cursor *pCsr;
+    int rc;
+
+    rc = lsm_csr_open(pDb, &pCsr);
+    if( rc==LSM_OK ) rc = lsm_csr_seek(pCsr, pKey, nKey, 0);
+    testOomAssertRc(pOom, rc);
+
+    if( rc==LSM_OK ){
+      const void *p; int n;
+      testOomAssert(pOom, lsm_csr_valid(pCsr));
+
+      rc = lsm_csr_key(pCsr, &p, &n);
+      testOomAssertRc(pOom, rc);
+      testOomAssert(pOom, rc!=LSM_OK || (n==nKey && memcmp(pKey, p, nKey)==0) );
+    }
+
+    if( rc==LSM_OK ){
+      const void *p; int n;
+      testOomAssert(pOom, lsm_csr_valid(pCsr));
+
+      rc = lsm_csr_value(pCsr, &p, &n);
+      testOomAssertRc(pOom, rc);
+      testOomAssert(pOom, rc!=LSM_OK || (n==nVal && memcmp(pVal, p, nVal)==0) );
+    }
+
+    lsm_csr_close(pCsr);
+    *pRc = rc;
+  }
+}
+
+static void testOomWrite(
+  OomTest *pOom,
+  lsm_db *pDb,
+  void *pKey, int nKey,
+  void *pVal, int nVal,
+  int *pRc
+){
+  testOomAssertRc(pOom, *pRc);
+  if( *pRc==LSM_OK ){
+    int rc;
+
+    rc = lsm_insert(pDb, pKey, nKey, pVal, nVal);
+    testOomAssertRc(pOom, rc);
+
+    *pRc = rc;
+  }
+}
+
+
+static void testOomFetchStr(
+  OomTest *pOom,
+  lsm_db *pDb,
+  const char *zKey,
+  const char *zVal,
+  int *pRc
+){
+  int nKey = strlen(zKey);
+  int nVal = strlen(zVal);
+  testOomFetch(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc);
+}
+
+static void testOomFetchData(
+  OomTest *pOom,
+  lsm_db *pDb,
+  Datasource *pData,
+  int iKey,
+  int *pRc
+){
+  void *pKey; int nKey;
+  void *pVal; int nVal;
+  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
+  testOomFetch(pOom, pDb, pKey, nKey, pVal, nVal, pRc);
+}
+
+static void testOomWriteStr(
+  OomTest *pOom,
+  lsm_db *pDb,
+  const char *zKey,
+  const char *zVal,
+  int *pRc
+){
+  int nKey = strlen(zKey);
+  int nVal = strlen(zVal);
+  testOomWrite(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc);
+}
+
+static void testOomWriteData(
+  OomTest *pOom,
+  lsm_db *pDb,
+  Datasource *pData,
+  int iKey,
+  int *pRc
+){
+  void *pKey; int nKey;
+  void *pVal; int nVal;
+  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
+  testOomWrite(pOom, pDb, pKey, nKey, pVal, nVal, pRc);
+}
+
+static void testOomScan(
+  OomTest *pOom, 
+  lsm_db *pDb, 
+  int bReverse,
+  const void *pKey, int nKey,
+  int nScan,
+  int *pRc
+){
+  if( *pRc==0 ){
+    int rc;
+    int iScan = 0;
+    lsm_cursor *pCsr;
+    int (*xAdvance)(lsm_cursor *);
+    
+
+    rc = lsm_csr_open(pDb, &pCsr);
+    testOomAssertRc(pOom, rc);
+
+    if( rc==LSM_OK ){
+      if( bReverse ){
+        rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_LE);
+        xAdvance = lsm_csr_prev;
+      }else{
+        rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_GE);
+        xAdvance = lsm_csr_next;
+      }
+    }
+    testOomAssertRc(pOom, rc);
+
+    while( rc==LSM_OK && lsm_csr_valid(pCsr) && iScan<nScan ){
+      const void *p; int n;
+
+      rc = lsm_csr_key(pCsr, &p, &n);
+      testOomAssertRc(pOom, rc);
+      if( rc==LSM_OK ){
+        rc = lsm_csr_value(pCsr, &p, &n);
+        testOomAssertRc(pOom, rc);
+      }
+      if( rc==LSM_OK ){
+        rc = xAdvance(pCsr);
+        testOomAssertRc(pOom, rc);
+      }
+      iScan++;
+    }
+
+    lsm_csr_close(pCsr);
+    *pRc = rc;
+  }
+}
+
+#define LSMTEST6_TESTDB "testdb.lsm" 
+
+#ifndef _WIN32
+# include <unistd.h>
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+void testDeleteLsmdb(const char *zFile){
+  char *zLog = testMallocPrintf("%s-log", zFile);
+  char *zShm = testMallocPrintf("%s-shm", zFile);
+  unlink(zFile);
+  unlink(zLog);
+  unlink(zShm);
+  testFree(zLog);
+  testFree(zShm);
+}
+
+static void copy_file(const char *zFrom, const char *zTo, int isDatabase){
+
+  if( access(zFrom, F_OK) ){
+    unlink(zTo);
+  }else{
+    int fd1;
+    int fd2;
+    off_t sz;
+    off_t i;
+    struct stat buf;
+    u8 *aBuf;
+
+    fd1 = open(zFrom, O_RDONLY | _O_BINARY, 0644);
+    fd2 = open(zTo, O_RDWR | O_CREAT | _O_BINARY, 0644);
+
+    fstat(fd1, &buf);
+    sz = buf.st_size;
+    ftruncate(fd2, sz);
+
+    aBuf = testMalloc(4096);
+    for(i=0; i<sz; i+=4096){
+      int bLockPage = isDatabase && i == 0;
+      int nByte = MIN((bLockPage ? 4066 : 4096), sz - i);
+      memset(aBuf, 0, 4096);
+      read(fd1, aBuf, nByte);
+      write(fd2, aBuf, nByte);
+      if( bLockPage ){
+        lseek(fd1, 4096, SEEK_SET);
+        lseek(fd2, 4096, SEEK_SET);
+      }
+    }
+    testFree(aBuf);
+
+    close(fd1);
+    close(fd2);
+  }
+}
+
+void testCopyLsmdb(const char *zFrom, const char *zTo){
+  char *zLog1 = testMallocPrintf("%s-log", zFrom);
+  char *zLog2 = testMallocPrintf("%s-log", zTo);
+  char *zShm1 = testMallocPrintf("%s-shm", zFrom);
+  char *zShm2 = testMallocPrintf("%s-shm", zTo);
+
+  unlink(zShm2);
+  unlink(zLog2);
+  unlink(zTo);
+  copy_file(zFrom, zTo, 1);
+  copy_file(zLog1, zLog2, 0);
+  copy_file(zShm1, zShm2, 0);
+
+  testFree(zLog1); testFree(zLog2); testFree(zShm1); testFree(zShm2);
+}
+
+/*
+** File zFile is the path to a database. This function makes backups
+** of the database file and its log as follows:
+**
+**     cp $(zFile)         $(zFile)-save
+**     cp $(zFile)-$(zAux) $(zFile)-save-$(zAux)
+**
+** Function testRestoreDb() can be used to copy the files back in the
+** other direction.
+*/
+void testSaveDb(const char *zFile, const char *zAux){
+  char *zLog = testMallocPrintf("%s-%s", zFile, zAux);
+  char *zFileSave = testMallocPrintf("%s-save", zFile);
+  char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux);
+
+  unlink(zFileSave);
+  unlink(zLogSave);
+  copy_file(zFile, zFileSave, 1);
+  copy_file(zLog, zLogSave, 0);
+
+  testFree(zLog); testFree(zFileSave); testFree(zLogSave);
+}
+
+/*
+** File zFile is the path to a database. This function restores
+** a backup of the database made by a previous call to testSaveDb().
+** Specifically, it does the equivalent of:
+**
+**     cp $(zFile)-save         $(zFile)
+**     cp $(zFile)-save-$(zAux) $(zFile)-$(zAux)
+*/
+void testRestoreDb(const char *zFile, const char *zAux){
+  char *zLog = testMallocPrintf("%s-%s", zFile, zAux);
+  char *zFileSave = testMallocPrintf("%s-save", zFile);
+  char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux);
+
+  copy_file(zFileSave, zFile, 1);
+  copy_file(zLogSave, zLog, 0);
+
+  testFree(zLog); testFree(zFileSave); testFree(zLogSave);
+}
+
+
+static int lsmWriteStr(lsm_db *pDb, const char *zKey, const char *zVal){
+  int nKey = strlen(zKey);
+  int nVal = strlen(zVal);
+  return lsm_insert(pDb, (void *)zKey, nKey, (void *)zVal, nVal);
+}
+
+static void setup_delete_db(){
+  testDeleteLsmdb(LSMTEST6_TESTDB);
+}
+
+/*
+** Create a small database. With the following content:
+**
+**    "one"   -> "one"
+**    "two"   -> "four"
+**    "three" -> "nine"
+**    "four"  -> "sixteen"
+**    "five"  -> "twentyfive"
+**    "six"   -> "thirtysix"
+**    "seven" -> "fourtynine"
+**    "eight" -> "sixtyfour"
+*/
+static void setup_populate_db(){
+  const char *azStr[] = {
+    "one",   "one",
+    "two",   "four",
+    "three", "nine",
+    "four",  "sixteen",
+    "five",  "twentyfive",
+    "six",   "thirtysix",
+    "seven", "fourtynine",
+    "eight", "sixtyfour",
+  };
+  int rc;
+  int ii;
+  lsm_db *pDb;
+
+  testDeleteLsmdb(LSMTEST6_TESTDB);
+
+  rc = lsm_new(tdb_lsm_env(), &pDb);
+  if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB);
+
+  for(ii=0; rc==LSM_OK && ii<ArraySize(azStr); ii+=2){
+    rc = lsmWriteStr(pDb, azStr[ii], azStr[ii+1]);
+  }
+  lsm_close(pDb);
+
+  testSaveDb(LSMTEST6_TESTDB, "log");
+  assert( rc==LSM_OK );
+}
+
+static Datasource *getDatasource(void){
+  const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 };
+  return testDatasourceNew(&defn);
+}
+
+/*
+** Set up a database file with the following properties:
+**
+**   * Page size is 1024 bytes.
+**   * Block size is 64 KB.
+**   * Contains 5000 key-value pairs starting at 0 from the
+**     datasource returned getDatasource().
+*/
+static void setup_populate_db2(){
+  Datasource *pData;
+  int ii;
+  int rc;
+  int nBlocksize = 64*1024;
+  int nPagesize = 1024;
+  int nWritebuffer = 4*1024;
+  lsm_db *pDb;
+
+  testDeleteLsmdb(LSMTEST6_TESTDB);
+  rc = lsm_new(tdb_lsm_env(), &pDb);
+  if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB);
+
+  lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &nBlocksize); 
+  lsm_config(pDb, LSM_CONFIG_PAGE_SIZE, &nPagesize); 
+  lsm_config(pDb, LSM_CONFIG_AUTOFLUSH, &nWritebuffer); 
+
+  pData = getDatasource();
+  for(ii=0; rc==LSM_OK && ii<5000; ii++){
+    void *pKey; int nKey;
+    void *pVal; int nVal;
+    testDatasourceEntry(pData, ii, &pKey, &nKey, &pVal, &nVal);
+    lsm_insert(pDb, pKey, nKey, pVal, nVal);
+  }
+  testDatasourceFree(pData);
+  lsm_close(pDb);
+
+  testSaveDb(LSMTEST6_TESTDB, "log");
+  assert( rc==LSM_OK );
+}
+
+/*
+** Test the results of OOM conditions in lsm_new().
+*/
+static void simple_oom_1(OomTest *pOom){
+  int rc;
+  lsm_db *pDb;
+
+  rc = lsm_new(tdb_lsm_env(), &pDb);
+  testOomAssertRc(pOom, rc);
+
+  lsm_close(pDb);
+}
+
+/*
+** Test the results of OOM conditions in lsm_open().
+*/
+static void simple_oom_2(OomTest *pOom){
+  int rc;
+  lsm_db *pDb;
+
+  rc = lsm_new(tdb_lsm_env(), &pDb);
+  if( rc==LSM_OK ){
+    rc = lsm_open(pDb, "testdb.lsm");
+  }
+  testOomAssertRc(pOom, rc);
+
+  lsm_close(pDb);
+}
+
+/*
+** Test the results of OOM conditions in simple fetch operations.
+*/
+static void simple_oom_3(OomTest *pOom){
+  int rc = LSM_OK;
+  lsm_db *pDb;
+
+  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
+
+  testOomFetchStr(pOom, pDb, "four",  "sixteen",    &rc);
+  testOomFetchStr(pOom, pDb, "seven", "fourtynine", &rc);
+  testOomFetchStr(pOom, pDb, "one",   "one",        &rc);
+  testOomFetchStr(pOom, pDb, "eight", "sixtyfour",  &rc);
+
+  lsm_close(pDb);
+}
+
+/*
+** Test the results of OOM conditions in simple write operations.
+*/
+static void simple_oom_4(OomTest *pOom){
+  int rc = LSM_OK;
+  lsm_db *pDb;
+
+  testDeleteLsmdb(LSMTEST6_TESTDB);
+  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
+
+  testOomWriteStr(pOom, pDb, "123", "onetwothree", &rc);
+  testOomWriteStr(pOom, pDb, "456", "fourfivesix", &rc);
+  testOomWriteStr(pOom, pDb, "789", "seveneightnine", &rc);
+  testOomWriteStr(pOom, pDb, "123", "teneleventwelve", &rc);
+  testOomWriteStr(pOom, pDb, "456", "fourteenfifteensixteen", &rc);
+
+  lsm_close(pDb);
+}
+
+static void simple_oom_5(OomTest *pOom){
+  Datasource *pData = getDatasource();
+  int rc = LSM_OK;
+  lsm_db *pDb;
+
+  testRestoreDb(LSMTEST6_TESTDB, "log");
+  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
+
+  testOomFetchData(pOom, pDb, pData, 3333, &rc);
+  testOomFetchData(pOom, pDb, pData, 0, &rc);
+  testOomFetchData(pOom, pDb, pData, 4999, &rc);
+
+  lsm_close(pDb);
+  testDatasourceFree(pData);
+}
+
+static void simple_oom_6(OomTest *pOom){
+  Datasource *pData = getDatasource();
+  int rc = LSM_OK;
+  lsm_db *pDb;
+
+  testRestoreDb(LSMTEST6_TESTDB, "log");
+  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
+
+  testOomWriteData(pOom, pDb, pData, 5000, &rc);
+  testOomWriteData(pOom, pDb, pData, 5001, &rc);
+  testOomWriteData(pOom, pDb, pData, 5002, &rc);
+  testOomFetchData(pOom, pDb, pData, 5001, &rc);
+  testOomFetchData(pOom, pDb, pData, 1234, &rc);
+
+  lsm_close(pDb);
+  testDatasourceFree(pData);
+}
+
+static void simple_oom_7(OomTest *pOom){
+  Datasource *pData = getDatasource();
+  int rc = LSM_OK;
+  lsm_db *pDb;
+
+  testRestoreDb(LSMTEST6_TESTDB, "log");
+  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
+  testOomScan(pOom, pDb, 0, "abc", 3, 20, &rc);
+  lsm_close(pDb);
+  testDatasourceFree(pData);
+}
+
+static void simple_oom_8(OomTest *pOom){
+  Datasource *pData = getDatasource();
+  int rc = LSM_OK;
+  lsm_db *pDb;
+  testRestoreDb(LSMTEST6_TESTDB, "log");
+  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
+  testOomScan(pOom, pDb, 1, "xyz", 3, 20, &rc);
+  lsm_close(pDb);
+  testDatasourceFree(pData);
+}
+
+/*
+** This test case has two clients connected to a database. The first client
+** hits an OOM while writing to the database. Check that the second 
+** connection is still able to query the db following the OOM.
+*/
+static void simple_oom2_1(OomTest *pOom){
+  const int nRecord = 100;        /* Number of records initially in db */
+  const int nIns = 10;            /* Number of records inserted with OOM */
+
+  Datasource *pData = getDatasource();
+  int rc = LSM_OK;
+  lsm_db *pDb1;
+  lsm_db *pDb2;
+  int i;
+
+  testDeleteLsmdb(LSMTEST6_TESTDB);
+
+  /* Open the two connections. Initialize the in-memory tree so that it
+  ** contains 100 records. Do all this with OOM injection disabled. */
+  testOomEnable(pOom, 0);
+  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb1, &rc);
+  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb2, &rc);
+  for(i=0; i<nRecord; i++){
+    testOomWriteData(pOom, pDb1, pData, i, &rc);
+  }
+  testOomEnable(pOom, 1);
+  assert( rc==0 );
+
+  /* Insert 10 more records using pDb1. Stop when an OOM is encountered. */
+  for(i=nRecord; i<nRecord+nIns; i++){
+    testOomWriteData(pOom, pDb1, pData, i, &rc);
+    if( rc ) break;
+  }
+  testOomAssertRc(pOom, rc);
+
+  /* Switch off OOM injection. Write a few rows using pDb2. Then check
+  ** that the database may be successfully queried.  */
+  testOomEnable(pOom, 0);
+  rc = 0;
+  for(; i<nRecord+nIns && rc==0; i++){
+    testOomWriteData(pOom, pDb2, pData, i, &rc);
+  }
+  for(i=0; i<nRecord+nIns; i++) testOomFetchData(pOom, pDb2, pData, i, &rc);
+  testOomEnable(pOom, 1);
+
+  lsm_close(pDb1);
+  lsm_close(pDb2);
+  testDatasourceFree(pData);
+}
+
+
+static void do_test_oom1(const char *zPattern, int *pRc){
+  struct SimpleOom {
+    const char *zName;
+    void (*xSetup)(void);
+    void (*xFunc)(OomTest *);
+  } aSimple[] = {
+    { "oom1.lsm.1", setup_delete_db,    simple_oom_1 },
+    { "oom1.lsm.2", setup_delete_db,    simple_oom_2 },
+    { "oom1.lsm.3", setup_populate_db,  simple_oom_3 },
+    { "oom1.lsm.4", setup_delete_db,    simple_oom_4 },
+    { "oom1.lsm.5", setup_populate_db2, simple_oom_5 },
+    { "oom1.lsm.6", setup_populate_db2, simple_oom_6 },
+    { "oom1.lsm.7", setup_populate_db2, simple_oom_7 },
+    { "oom1.lsm.8", setup_populate_db2, simple_oom_8 },
+
+    { "oom2.lsm.1", setup_delete_db,    simple_oom2_1 },
+  };
+  int i;
+
+  for(i=0; i<ArraySize(aSimple); i++){
+    if( *pRc==0 && testCaseBegin(pRc, zPattern, "%s", aSimple[i].zName) ){
+      OomTest t;
+
+      if( aSimple[i].xSetup ){
+        aSimple[i].xSetup();
+      }
+
+      for(testOomStart(&t); testOomContinue(&t); testOomNext(&t)){
+        aSimple[i].xFunc(&t);
+      }
+
+      printf("(%d injections).", t.iNext-2);
+      testCaseFinish( (*pRc = testOomFinish(&t)) );
+      testMallocOom(tdb_lsm_env(), 0, 0, 0, 0);
+    }
+  }
+}
+
+void test_oom(
+  const char *zPattern,           /* Run test cases that match this pattern */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  do_test_oom1(zPattern, pRc);
+}

ext/lsm1/lsm-test/lsmtest7.c

@@ -0,0 +1,206 @@
+
+
+#include "lsmtest.h"
+
+
+/*
+** Test that the rules for when lsm_csr_next() and lsm_csr_prev() are
+** enforced. Specifically:
+**
+**   * Both functions always return LSM_MISUSE if the cursor is at EOF
+**     when they are called.
+**
+**   * lsm_csr_next() may only be used after lsm_csr_seek(LSM_SEEK_GE) or 
+**     lsm_csr_first(). 
+**
+**   * lsm_csr_prev() may only be used after lsm_csr_seek(LSM_SEEK_LE) or 
+**     lsm_csr_last().
+*/
+static void do_test_api1_lsm(lsm_db *pDb, int *pRc){
+  int ret;
+  lsm_cursor *pCsr;
+  lsm_cursor *pCsr2;
+  int nKey;
+  const void *pKey;
+
+  ret = lsm_csr_open(pDb, &pCsr);
+  testCompareInt(LSM_OK, ret, pRc);
+
+  ret = lsm_csr_next(pCsr);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+  ret = lsm_csr_prev(pCsr);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+
+  ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_GE);
+  testCompareInt(LSM_OK, ret, pRc);
+  ret = lsm_csr_next(pCsr);
+  testCompareInt(LSM_OK, ret, pRc);
+  ret = lsm_csr_prev(pCsr);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+
+  ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_LE);
+  testCompareInt(LSM_OK, ret, pRc);
+  ret = lsm_csr_next(pCsr);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+  ret = lsm_csr_prev(pCsr);
+  testCompareInt(LSM_OK, ret, pRc);
+
+  ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_LEFAST);
+  testCompareInt(LSM_OK, ret, pRc);
+  ret = lsm_csr_next(pCsr);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+  ret = lsm_csr_prev(pCsr);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+
+  ret = lsm_csr_key(pCsr, &pKey, &nKey);
+  testCompareInt(LSM_OK, ret, pRc);
+
+  ret = lsm_csr_open(pDb, &pCsr2);
+  testCompareInt(LSM_OK, ret, pRc);
+
+  ret = lsm_csr_seek(pCsr2, pKey, nKey, LSM_SEEK_EQ);
+  testCompareInt(LSM_OK, ret, pRc);
+  testCompareInt(1, lsm_csr_valid(pCsr2), pRc);
+  ret = lsm_csr_next(pCsr2);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+  ret = lsm_csr_prev(pCsr2);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+
+  lsm_csr_close(pCsr2);
+
+  ret = lsm_csr_first(pCsr);
+  testCompareInt(LSM_OK, ret, pRc);
+  ret = lsm_csr_next(pCsr);
+  testCompareInt(LSM_OK, ret, pRc);
+  ret = lsm_csr_prev(pCsr);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+
+  ret = lsm_csr_last(pCsr);
+  testCompareInt(LSM_OK, ret, pRc);
+  ret = lsm_csr_prev(pCsr);
+  testCompareInt(LSM_OK, ret, pRc);
+  ret = lsm_csr_next(pCsr);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+
+  ret = lsm_csr_first(pCsr);
+  while( lsm_csr_valid(pCsr) ){
+    ret = lsm_csr_next(pCsr);
+    testCompareInt(LSM_OK, ret, pRc);
+  }
+  ret = lsm_csr_next(pCsr);
+  testCompareInt(LSM_OK, ret, pRc);
+  ret = lsm_csr_prev(pCsr);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+
+  ret = lsm_csr_last(pCsr);
+  while( lsm_csr_valid(pCsr) ){
+    ret = lsm_csr_prev(pCsr);
+    testCompareInt(LSM_OK, ret, pRc);
+  }
+  ret = lsm_csr_prev(pCsr);
+  testCompareInt(LSM_OK, ret, pRc);
+  ret = lsm_csr_next(pCsr);
+  testCompareInt(LSM_MISUSE, ret, pRc);
+
+  lsm_csr_close(pCsr);
+}
+
+static void do_test_api1(const char *zPattern, int *pRc){
+  if( testCaseBegin(pRc, zPattern, "api1.lsm") ){
+    const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 };
+    Datasource *pData;
+    TestDb *pDb;
+    int rc = 0;
+
+    pDb = testOpen("lsm_lomem", 1, &rc);
+    pData = testDatasourceNew(&defn);
+    testWriteDatasourceRange(pDb, pData, 0, 1000, pRc);
+
+    do_test_api1_lsm(tdb_lsm(pDb), pRc);
+
+    testDatasourceFree(pData);
+    testClose(&pDb);
+
+    testCaseFinish(*pRc);
+  }
+}
+
+static lsm_db *newLsmConnection(
+  const char *zDb, 
+  int nPgsz, 
+  int nBlksz,
+  int *pRc
+){
+  lsm_db *db = 0;
+  if( *pRc==0 ){
+    int n1 = nPgsz;
+    int n2 = nBlksz;
+    *pRc = lsm_new(tdb_lsm_env(), &db);
+    if( *pRc==0 ){
+      if( n1 ) lsm_config(db, LSM_CONFIG_PAGE_SIZE, &n1);
+      if( n2 ) lsm_config(db, LSM_CONFIG_BLOCK_SIZE, &n2);
+      *pRc = lsm_open(db, "testdb.lsm");
+    }
+  }
+  return db;
+}
+
+static void testPagesize(lsm_db *db, int nPgsz, int nBlksz, int *pRc){
+  if( *pRc==0 ){
+    int n1 = 0;
+    int n2 = 0;
+
+    lsm_config(db, LSM_CONFIG_PAGE_SIZE, &n1);
+    lsm_config(db, LSM_CONFIG_BLOCK_SIZE, &n2);
+
+    testCompareInt(n1, nPgsz, pRc);
+    testCompareInt(n2, nBlksz, pRc);
+  }
+}
+
+/*
+** Test case "api2" tests that the default page and block sizes of a 
+** database may only be modified before lsm_open() is called. And that
+** after lsm_open() is called lsm_config() may be used to read the 
+** actual page and block size of the db.
+*/
+static void do_test_api2(const char *zPattern, int *pRc){
+  if( *pRc==0 && testCaseBegin(pRc, zPattern, "api2.lsm") ){
+    lsm_db *db1 = 0;
+    lsm_db *db2 = 0;
+
+    testDeleteLsmdb("testdb.lsm");
+    db1 = newLsmConnection("testdb.lsm", 0, 0, pRc);
+    testPagesize(db1, 4096, 1024, pRc);
+    db2 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc);
+    testPagesize(db2, 4096, 1024, pRc);
+    lsm_close(db1);
+    lsm_close(db2);
+
+    testDeleteLsmdb("testdb.lsm");
+    db1 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc);
+    testPagesize(db1, 1024, 64*1024, pRc);
+    db2 = newLsmConnection("testdb.lsm", 0, 0, pRc);
+    testPagesize(db2, 1024, 64*1024, pRc);
+    lsm_close(db1);
+    lsm_close(db2);
+
+    testDeleteLsmdb("testdb.lsm");
+    db1 = newLsmConnection("testdb.lsm", 8192, 2*1024, pRc);
+    testPagesize(db1, 8192, 2*1024, pRc);
+    db2 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc);
+    testPagesize(db2, 8192, 2*1024, pRc);
+    lsm_close(db1);
+    lsm_close(db2);
+
+    testCaseFinish(*pRc);
+  }
+}
+
+void test_api(
+  const char *zPattern,           /* Run test cases that match this pattern */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  do_test_api1(zPattern, pRc);
+  do_test_api2(zPattern, pRc);
+}

ext/lsm1/lsm-test/lsmtest8.c

@@ -0,0 +1,322 @@
+
+/*
+** This file contains test cases to verify that "live-recovery" following
+** a mid-transaction failure of a writer process.
+*/
+
+
+/* 
+** This test file includes lsmInt.h to get access to the definition of the
+** ShmHeader structure. This is required to cause strategic damage to the
+** shared memory header as part of recovery testing.
+*/
+#include "lsmInt.h"
+
+#include "lsmtest.h"
+
+typedef struct SetupStep SetupStep;
+struct SetupStep {
+  int bFlush;                     /* Flush to disk and checkpoint */
+  int iInsStart;                  /* First key-value from ds to insert */
+  int nIns;                       /* Number of rows to insert */
+  int iDelStart;                  /* First key from ds to delete */
+  int nDel;                       /* Number of rows to delete */
+};
+
+static void doSetupStep(
+  TestDb *pDb, 
+  Datasource *pData, 
+  const SetupStep *pStep, 
+  int *pRc
+){
+  testWriteDatasourceRange(pDb, pData, pStep->iInsStart, pStep->nIns, pRc);
+  testDeleteDatasourceRange(pDb, pData, pStep->iDelStart, pStep->nDel, pRc);
+  if( *pRc==0 ){
+    int nSave = -1;
+    int nBuf = 64;
+    lsm_db *db = tdb_lsm(pDb);
+
+    lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave);
+    lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf);
+    lsm_begin(db, 1);
+    lsm_commit(db, 0);
+    lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave);
+
+    *pRc = lsm_work(db, 0, 0, 0);
+    if( *pRc==0 ){
+      *pRc = lsm_checkpoint(db, 0);
+    }
+  }
+}
+
+static void doSetupStepArray(
+  TestDb *pDb, 
+  Datasource *pData, 
+  const SetupStep *aStep, 
+  int nStep
+){
+  int i;
+  for(i=0; i<nStep; i++){
+    int rc = 0;
+    doSetupStep(pDb, pData, &aStep[i], &rc);
+    assert( rc==0 );
+  }
+}
+
+static void setupDatabase1(TestDb *pDb, Datasource **ppData){
+  const SetupStep aStep[] = {
+    { 0,                                  1,     2000, 0, 0 },
+    { 1,                                  0,     0, 0, 0 },
+    { 0,                                  10001, 1000, 0, 0 },
+  };
+  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 100, 500};
+  Datasource *pData;
+
+  pData = testDatasourceNew(&defn);
+  doSetupStepArray(pDb, pData, aStep, ArraySize(aStep));
+  if( ppData ){
+    *ppData = pData;
+  }else{
+    testDatasourceFree(pData);
+  }
+}
+
+#include <stdio.h>
+void testReadFile(const char *zFile, int iOff, void *pOut, int nByte, int *pRc){
+  if( *pRc==0 ){
+    FILE *fd;
+    fd = fopen(zFile, "r");
+    if( fd==0 ){
+      *pRc = 1;
+    }else{
+      if( 0!=fseek(fd, iOff, SEEK_SET) ){
+        *pRc = 1;
+      }else{
+        if( nByte!=fread(pOut, 1, nByte, fd) ){
+          *pRc = 1;
+        }
+      }
+      fclose(fd);
+    }
+  }
+}
+
+void testWriteFile(
+  const char *zFile, 
+  int iOff, 
+  void *pOut, 
+  int nByte, 
+  int *pRc
+){
+  if( *pRc==0 ){
+    FILE *fd;
+    fd = fopen(zFile, "r+");
+    if( fd==0 ){
+      *pRc = 1;
+    }else{
+      if( 0!=fseek(fd, iOff, SEEK_SET) ){
+        *pRc = 1;
+      }else{
+        if( nByte!=fwrite(pOut, 1, nByte, fd) ){
+          *pRc = 1;
+        }
+      }
+      fclose(fd);
+    }
+  }
+}
+
+static ShmHeader *getShmHeader(const char *zDb){
+  int rc = 0;
+  char *zShm = testMallocPrintf("%s-shm", zDb);
+  ShmHeader *pHdr;
+
+  pHdr = testMalloc(sizeof(ShmHeader));
+  testReadFile(zShm, 0, (void *)pHdr, sizeof(ShmHeader), &rc);
+  assert( rc==0 );
+
+  return pHdr;
+}
+
+/*
+** This function makes a copy of the three files associated with LSM 
+** database zDb (i.e. if zDb is "test.db", it makes copies of "test.db",
+** "test.db-log" and "test.db-shm").
+**
+** It then opens a new database connection to the copy with the xLock() call
+** instrumented so that it appears that some other process already connected
+** to the db (holding a shared lock on DMS2). This prevents recovery from
+** running. Then:
+**
+**    1) Check that the checksum of the database is zCksum. 
+**    2) Write a few keys to the database. Then delete the same keys. 
+**    3) Check that the checksum is zCksum.
+**    4) Flush the db to disk and run a checkpoint. 
+**    5) Check once more that the checksum is still zCksum.
+*/
+static void doLiveRecovery(const char *zDb, const char *zCksum, int *pRc){
+  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 20, 25, 100, 500};
+  Datasource *pData;
+  const char *zCopy = "testcopy.lsm";
+  char zCksum2[TEST_CKSUM_BYTES];
+  TestDb *pDb = 0;
+  int rc;
+
+  pData = testDatasourceNew(&defn);
+
+  testCopyLsmdb(zDb, zCopy);
+  rc = tdb_lsm_open("test_no_recovery=1", zCopy, 0, &pDb);
+  if( rc==0 ){
+    ShmHeader *pHdr;
+    lsm_db *db;
+    testCksumDatabase(pDb, zCksum2);
+    testCompareStr(zCksum, zCksum2, &rc);
+
+    testWriteDatasourceRange(pDb, pData, 1, 10, &rc);
+    testDeleteDatasourceRange(pDb, pData, 1, 10, &rc);
+
+    /* Test that the two tree-headers are now consistent. */
+    pHdr = getShmHeader(zCopy);
+    if( rc==0 && memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(pHdr->hdr1)) ){
+      rc = 1;
+    }
+    testFree(pHdr);
+
+    if( rc==0 ){
+      int nBuf = 64;
+      db = tdb_lsm(pDb);
+      lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf);
+      lsm_begin(db, 1);
+      lsm_commit(db, 0);
+      rc = lsm_work(db, 0, 0, 0);
+    }
+
+    testCksumDatabase(pDb, zCksum2);
+    testCompareStr(zCksum, zCksum2, &rc);
+  }
+
+  testDatasourceFree(pData);
+  testClose(&pDb);
+  testDeleteLsmdb(zCopy);
+  *pRc = rc;
+}
+
+static void doWriterCrash1(int *pRc){
+  const int nWrite = 2000;
+  const int nStep = 10;
+  const int iWriteStart = 20000;
+  int rc = 0;
+  TestDb *pDb = 0;
+  Datasource *pData = 0;
+
+  rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb);
+  if( rc==0 ){
+    int iDot = 0;
+    char zCksum[TEST_CKSUM_BYTES];
+    int i;
+    setupDatabase1(pDb, &pData);
+    testCksumDatabase(pDb, zCksum);
+    testBegin(pDb, 2, &rc);
+    for(i=0; rc==0 && i<nWrite; i+=nStep){
+      testCaseProgress(i, nWrite, testCaseNDot(), &iDot);
+      testWriteDatasourceRange(pDb, pData, iWriteStart+i, nStep, &rc);
+      doLiveRecovery("testdb.lsm", zCksum, &rc);
+    }
+  }
+  testCommit(pDb, 0, &rc);
+  testClose(&pDb);
+  testDatasourceFree(pData);
+  *pRc = rc;
+}
+
+/*
+** This test case verifies that inconsistent tree-headers in shared-memory
+** are resolved correctly. 
+*/
+static void doWriterCrash2(int *pRc){
+  int rc = 0;
+  TestDb *pDb = 0;
+  Datasource *pData = 0;
+
+  rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb);
+  if( rc==0 ){
+    ShmHeader *pHdr1;
+    ShmHeader *pHdr2;
+    char zCksum1[TEST_CKSUM_BYTES];
+    char zCksum2[TEST_CKSUM_BYTES];
+
+    pHdr1 = testMalloc(sizeof(ShmHeader));
+    pHdr2 = testMalloc(sizeof(ShmHeader));
+    setupDatabase1(pDb, &pData);
+
+    /* Grab a copy of the shared-memory header. And the db checksum */
+    testReadFile("testdb.lsm-shm", 0, (void *)pHdr1, sizeof(ShmHeader), &rc);
+    testCksumDatabase(pDb, zCksum1);
+
+    /* Modify the database */
+    testBegin(pDb, 2, &rc);
+    testWriteDatasourceRange(pDb, pData, 30000, 200, &rc);
+    testCommit(pDb, 0, &rc);
+
+    /* Grab a second copy of the shared-memory header. And the db checksum */
+    testReadFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
+    testCksumDatabase(pDb, zCksum2);
+    doLiveRecovery("testdb.lsm", zCksum2, &rc);
+
+    /* If both tree-headers are valid, tree-header-1 is used. */
+    memcpy(&pHdr2->hdr1, &pHdr1->hdr1, sizeof(pHdr1->hdr1));
+    pHdr2->bWriter = 1;
+    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
+    doLiveRecovery("testdb.lsm", zCksum1, &rc);
+
+    /* If both tree-headers are valid, tree-header-1 is used. */
+    memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1));
+    memcpy(&pHdr2->hdr2, &pHdr1->hdr1, sizeof(pHdr1->hdr1));
+    pHdr2->bWriter = 1;
+    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
+    doLiveRecovery("testdb.lsm", zCksum2, &rc);
+
+    /* If tree-header 1 is invalid, tree-header-2 is used */
+    memcpy(&pHdr2->hdr2, &pHdr2->hdr1, sizeof(pHdr1->hdr1));
+    pHdr2->hdr1.aCksum[0] = 5;
+    pHdr2->hdr1.aCksum[0] = 6;
+    pHdr2->bWriter = 1;
+    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
+    doLiveRecovery("testdb.lsm", zCksum2, &rc);
+
+    /* If tree-header 2 is invalid, tree-header-1 is used */
+    memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1));
+    pHdr2->hdr2.aCksum[0] = 5;
+    pHdr2->hdr2.aCksum[0] = 6;
+    pHdr2->bWriter = 1;
+    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
+    doLiveRecovery("testdb.lsm", zCksum2, &rc);
+
+    testFree(pHdr1);
+    testFree(pHdr2);
+    testClose(&pDb);
+  }
+
+  *pRc = rc;
+}
+
+void do_writer_crash_test(const char *zPattern, int *pRc){
+  struct Test {
+    const char *zName;
+    void (*xFunc)(int *);
+  } aTest[] = {
+    { "writercrash1.lsm", doWriterCrash1 },
+    { "writercrash2.lsm", doWriterCrash2 },
+  };
+  int i;
+  for(i=0; i<ArraySize(aTest); i++){
+    struct Test *p = &aTest[i];
+    if( testCaseBegin(pRc, zPattern, p->zName) ){
+      p->xFunc(pRc);
+      testCaseFinish(*pRc);
+    }
+  }
+
+}
+
+

ext/lsm1/lsm-test/lsmtest9.c

@@ -0,0 +1,143 @@
+
+#include "lsmtest.h"
+
+#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE
+#define DATA_RANDOM     TEST_DATASOURCE_RANDOM
+
+typedef struct Datatest4 Datatest4;
+
+/*
+** Test overview:
+**
+**   1. Insert (Datatest4.nRec) records into a database.
+**
+**   2. Repeat (Datatest4.nRepeat) times:
+**
+**      2a. Delete 2/3 of the records in the database.
+**
+**      2b. Run lsm_work(nMerge=1).
+**
+**      2c. Insert as many records as were deleted in 2a.
+**
+**      2d. Check database content is as expected.
+**
+**      2e. If (Datatest4.bReopen) is true, close and reopen the database.
+*/
+struct Datatest4 {
+  /* Datasource definition */
+  DatasourceDefn defn;
+
+  int nRec;
+  int nRepeat;
+  int bReopen;
+};
+
+static void doDataTest4(
+  const char *zSystem,            /* Database system to test */
+  Datatest4 *p,                   /* Structure containing test parameters */
+  int *pRc                        /* OUT: Error code */
+){
+  lsm_db *db = 0;
+  TestDb *pDb;
+  TestDb *pControl;
+  Datasource *pData;
+  int i;
+  int rc = 0;
+  int iDot = 0;
+  int bMultiThreaded = 0;         /* True for MT LSM database */
+
+  int nRecOn3 = (p->nRec / 3);
+  int iData = 0;
+
+  /* Start the test case, open a database and allocate the datasource. */
+  rc = testControlDb(&pControl);
+  pDb = testOpen(zSystem, 1, &rc);
+  pData = testDatasourceNew(&p->defn);
+  if( rc==0 ){
+    db = tdb_lsm(pDb);
+    bMultiThreaded = tdb_lsm_multithread(pDb);
+  }
+
+  testWriteDatasourceRange(pControl, pData, iData, nRecOn3*3, &rc);
+  testWriteDatasourceRange(pDb,      pData, iData, nRecOn3*3, &rc);
+
+  for(i=0; rc==0 && i<p->nRepeat; i++){
+
+    testDeleteDatasourceRange(pControl, pData, iData, nRecOn3*2, &rc);
+    testDeleteDatasourceRange(pDb,      pData, iData, nRecOn3*2, &rc);
+
+    if( db ){
+      int nDone;
+#if 0
+      fprintf(stderr, "lsm_work() start...\n"); fflush(stderr);
+#endif
+      do {
+        nDone = 0;
+        rc = lsm_work(db, 1, (1<<30), &nDone);
+      }while( rc==0 && nDone>0 );
+      if( bMultiThreaded && rc==LSM_BUSY ) rc = LSM_OK;
+#if 0 
+      fprintf(stderr, "lsm_work() done...\n"); fflush(stderr);
+#endif
+    }
+
+if( i+1<p->nRepeat ){
+    iData += (nRecOn3*2);
+    testWriteDatasourceRange(pControl, pData, iData+nRecOn3, nRecOn3*2, &rc);
+    testWriteDatasourceRange(pDb,      pData, iData+nRecOn3, nRecOn3*2, &rc);
+
+    testCompareDb(pData, nRecOn3*3, iData, pControl, pDb, &rc);
+
+    /* If Datatest4.bReopen is true, close and reopen the database */
+    if( p->bReopen ){
+      testReopen(&pDb, &rc);
+      if( rc==0 ) db = tdb_lsm(pDb);
+    }
+}
+
+    /* Update the progress dots... */
+    testCaseProgress(i, p->nRepeat, testCaseNDot(), &iDot);
+  }
+
+  testClose(&pDb);
+  testClose(&pControl);
+  testDatasourceFree(pData);
+  testCaseFinish(rc);
+  *pRc = rc;
+}
+
+static char *getName4(const char *zSystem, Datatest4 *pTest){
+  char *zRet;
+  char *zData;
+  zData = testDatasourceName(&pTest->defn);
+  zRet = testMallocPrintf("data4.%s.%s.%d.%d.%d", 
+      zSystem, zData, pTest->nRec, pTest->nRepeat, pTest->bReopen
+  );
+  testFree(zData);
+  return zRet;
+}
+
+void test_data_4(
+  const char *zSystem,            /* Database system name */
+  const char *zPattern,           /* Run test cases that match this pattern */
+  int *pRc                        /* IN/OUT: Error code */
+){
+  Datatest4 aTest[] = {
+      /* defn,                                 nRec, nRepeat, bReopen */
+    { {DATA_RANDOM,     20,25,     500,600}, 10000,      10,       0   },
+    { {DATA_RANDOM,     20,25,     500,600}, 10000,      10,       1   },
+  };
+
+  int i;
+
+  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
+    char *zName = getName4(zSystem, &aTest[i]);
+    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
+      doDataTest4(zSystem, &aTest[i], pRc);
+    }
+    testFree(zName);
+  }
+}
+
+
+

ext/lsm1/lsm-test/lsmtest_bt.c

@@ -0,0 +1,75 @@
+
+#include "lsmtest.h"
+#include "bt.h"
+
+int do_bt(int nArg, char **azArg){
+  struct Option {
+    const char *zName;
+    int bPgno;
+    int eOpt;
+  } aOpt [] = { 
+    { "dbhdr",          0, BT_INFO_HDRDUMP },
+    { "filename",       0, BT_INFO_FILENAME },
+    { "block_freelist", 0, BT_INFO_BLOCK_FREELIST },
+    { "page_freelist",  0, BT_INFO_PAGE_FREELIST },
+    { "filename",       0, BT_INFO_FILENAME },
+    { "page",           1, BT_INFO_PAGEDUMP },
+    { "page_ascii",     1, BT_INFO_PAGEDUMP_ASCII },
+    { "leaks",          0, BT_INFO_PAGE_LEAKS },
+    { 0, 0 } 
+  };
+  int iOpt;
+  int rc;
+  bt_info buf;
+  char *zOpt;
+  char *zFile;
+
+  bt_db *db = 0;
+
+  if( nArg<2 ){
+    testPrintUsage("FILENAME OPTION ...");
+    return -1;
+  }
+  zFile = azArg[0];
+  zOpt = azArg[1];
+
+  rc = testArgSelect(aOpt, "option", zOpt, &iOpt);
+  if( rc!=0 ) return rc;
+  if( nArg!=2+aOpt[iOpt].bPgno ){
+    testPrintFUsage("FILENAME %s %s", zOpt, aOpt[iOpt].bPgno ? "PGNO" : "");
+    return -4;
+  }
+
+  rc = sqlite4BtNew(sqlite4_env_default(), 0, &db);
+  if( rc!=SQLITE4_OK ){
+    testPrintError("sqlite4BtNew() failed: %d", rc);
+    return -2;
+  }
+  rc = sqlite4BtOpen(db, zFile);
+  if( rc!=SQLITE4_OK ){
+    testPrintError("sqlite4BtOpen() failed: %d", rc);
+    return -3;
+  }
+
+  buf.eType = aOpt[iOpt].eOpt;
+  buf.pgno = 0;
+  sqlite4_buffer_init(&buf.output, 0);
+
+  if( aOpt[iOpt].bPgno ){
+    buf.pgno = (u32)atoi(azArg[2]);
+  }
+
+  rc = sqlite4BtControl(db, BT_CONTROL_INFO, &buf);
+  if( rc!=SQLITE4_OK ){
+    testPrintError("sqlite4BtControl() failed: %d\n", rc);
+    return -4;
+  }
+
+  printf("%s\n", (char*)buf.output.p);
+  sqlite4_buffer_clear(&buf.output);
+  return 0;
+}
+
+
+
+

ext/lsm1/lsm-test/lsmtest_datasource.c

@@ -0,0 +1,96 @@
+
+
+#include "lsmtest.h"
+
+struct Datasource {
+  int eType;
+
+  int nMinKey;
+  int nMaxKey;
+  int nMinVal;
+  int nMaxVal;
+
+  char *aKey;
+  char *aVal;
+};
+
+void testDatasourceEntry(
+  Datasource *p, 
+  int iData, 
+  void **ppKey, int *pnKey,
+  void **ppVal, int *pnVal
+){
+  assert( (ppKey==0)==(pnKey==0) );
+  assert( (ppVal==0)==(pnVal==0) );
+
+  if( ppKey ){
+    int nKey = 0;
+    switch( p->eType ){
+      case TEST_DATASOURCE_RANDOM: {
+        int nRange = (1 + p->nMaxKey - p->nMinKey);
+        nKey = (int)( testPrngValue((u32)iData) % nRange ) + p->nMinKey; 
+        testPrngString((u32)iData, p->aKey, nKey);
+        break;
+      }
+      case TEST_DATASOURCE_SEQUENCE:
+        nKey = sprintf(p->aKey, "%012d", iData);
+        break;
+    }
+    *ppKey = p->aKey;
+    *pnKey = nKey;
+  }
+  if( ppVal ){
+    u32 nVal = testPrngValue((u32)iData)%(1+p->nMaxVal-p->nMinVal)+p->nMinVal;
+    testPrngString((u32)~iData, p->aVal, (int)nVal);
+    *ppVal = p->aVal;
+    *pnVal = (int)nVal;
+  }
+}
+
+void testDatasourceFree(Datasource *p){
+  testFree(p);
+}
+
+/*
+** Return a pointer to a nul-terminated string that corresponds to the
+** contents of the datasource-definition passed as the first argument.
+** The caller should eventually free the returned pointer using testFree().
+*/
+char *testDatasourceName(const DatasourceDefn *p){
+  char *zRet;
+  zRet = testMallocPrintf("%s.(%d-%d).(%d-%d)",
+      (p->eType==TEST_DATASOURCE_SEQUENCE ? "seq" : "rnd"),
+      p->nMinKey, p->nMaxKey,
+      p->nMinVal, p->nMaxVal
+  );
+  return zRet;
+}
+
+Datasource *testDatasourceNew(const DatasourceDefn *pDefn){
+  Datasource *p;
+  int nMinKey; 
+  int nMaxKey;
+  int nMinVal;
+  int nMaxVal; 
+
+  if( pDefn->eType==TEST_DATASOURCE_SEQUENCE ){
+    nMinKey = 128;
+    nMaxKey = 128;
+  }else{
+    nMinKey = MAX(0, pDefn->nMinKey);
+    nMaxKey = MAX(nMinKey, pDefn->nMaxKey);
+  }
+  nMinVal = MAX(0, pDefn->nMinVal);
+  nMaxVal = MAX(nMinVal, pDefn->nMaxVal);
+
+  p = (Datasource *)testMalloc(sizeof(Datasource) + nMaxKey + nMaxVal + 1);
+  p->eType = pDefn->eType;
+  p->nMinKey = nMinKey;
+  p->nMinVal = nMinVal;
+  p->nMaxKey = nMaxKey;
+  p->nMaxVal = nMaxVal;
+  
+  p->aKey = (char *)&p[1];
+  p->aVal = &p->aKey[nMaxKey];
+  return p;
+};

ext/lsm1/lsm-test/lsmtest_func.c

@@ -0,0 +1,177 @@
+
+#include "lsmtest.h"
+
+
+int do_work(int nArg, char **azArg){
+  struct Option {
+    const char *zName;
+  } aOpt [] = {
+    { "-nmerge" },
+    { "-nkb" },
+    { 0 }
+  };
+
+  lsm_db *pDb;
+  int rc;
+  int i;
+  const char *zDb;
+  int nMerge = 1;
+  int nKB = (1<<30);
+
+  if( nArg==0 ) goto usage;
+  zDb = azArg[nArg-1];
+  for(i=0; i<(nArg-1); i++){
+    int iSel;
+    rc = testArgSelect(aOpt, "option", azArg[i], &iSel);
+    if( rc ) return rc;
+    switch( iSel ){
+      case 0:
+        i++;
+        if( i==(nArg-1) ) goto usage;
+        nMerge = atoi(azArg[i]);
+        break;
+      case 1:
+        i++;
+        if( i==(nArg-1) ) goto usage;
+        nKB = atoi(azArg[i]);
+        break;
+    }
+  }
+
+  rc = lsm_new(0, &pDb);
+  if( rc!=LSM_OK ){
+    testPrintError("lsm_open(): rc=%d\n", rc);
+  }else{
+    rc = lsm_open(pDb, zDb);
+    if( rc!=LSM_OK ){
+      testPrintError("lsm_open(): rc=%d\n", rc);
+    }else{
+      int n = -1;
+      lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n);
+      n = n*2;
+      lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n);
+
+      rc = lsm_work(pDb, nMerge, nKB, 0);
+      if( rc!=LSM_OK ){
+        testPrintError("lsm_work(): rc=%d\n", rc);
+      }
+    }
+  }
+  if( rc==LSM_OK ){
+    rc = lsm_checkpoint(pDb, 0);
+  }
+
+  lsm_close(pDb);
+  return rc;
+
+ usage:
+  testPrintUsage("?-optimize? ?-n N? DATABASE");
+  return -1;
+}
+
+
+/*
+**   lsmtest show ?-config LSM-CONFIG? DATABASE ?COMMAND ?PGNO??
+*/
+int do_show(int nArg, char **azArg){
+  lsm_db *pDb;
+  int rc;
+  const char *zDb;
+
+  int eOpt = LSM_INFO_DB_STRUCTURE;
+  unsigned int iPg = 0;
+  int bConfig = 0;
+  const char *zConfig = "";
+
+  struct Option {
+    const char *zName;
+    int bConfig;
+    int eOpt;
+  } aOpt [] = { 
+    { "array",       0, LSM_INFO_ARRAY_STRUCTURE },
+    { "array-pages", 0, LSM_INFO_ARRAY_PAGES },
+    { "blocksize",   1, LSM_CONFIG_BLOCK_SIZE },
+    { "pagesize",    1, LSM_CONFIG_PAGE_SIZE },
+    { "freelist",    0, LSM_INFO_FREELIST },
+    { "page-ascii",  0, LSM_INFO_PAGE_ASCII_DUMP },
+    { "page-hex",    0, LSM_INFO_PAGE_HEX_DUMP },
+    { 0, 0 } 
+  };
+
+  char *z = 0; 
+  int iDb = 0;                    /* Index of DATABASE in azArg[] */
+
+  /* Check if there is a "-config" option: */
+  if( nArg>2 && strlen(azArg[0])>1 
+   && memcmp(azArg[0], "-config", strlen(azArg[0]))==0
+  ){
+    zConfig = azArg[1];
+    iDb = 2;
+  }
+  if( nArg<(iDb+1) ) goto usage;
+
+  if( nArg>(iDb+1) ){
+    rc = testArgSelect(aOpt, "option", azArg[iDb+1], &eOpt);
+    if( rc!=0 ) return rc;
+    bConfig = aOpt[eOpt].bConfig;
+    eOpt = aOpt[eOpt].eOpt;
+    if( (bConfig==0 && eOpt==LSM_INFO_FREELIST)
+     || (bConfig==1 && eOpt==LSM_CONFIG_BLOCK_SIZE)
+     || (bConfig==1 && eOpt==LSM_CONFIG_PAGE_SIZE)
+    ){
+      if( nArg!=(iDb+2) ) goto usage;
+    }else{
+      if( nArg!=(iDb+3) ) goto usage;
+      iPg = atoi(azArg[iDb+2]);
+    }
+  }
+  zDb = azArg[iDb];
+
+  rc = lsm_new(0, &pDb);
+  tdb_lsm_configure(pDb, zConfig);
+  if( rc!=LSM_OK ){
+    testPrintError("lsm_new(): rc=%d\n", rc);
+  }else{
+    rc = lsm_open(pDb, zDb);
+    if( rc!=LSM_OK ){
+      testPrintError("lsm_open(): rc=%d\n", rc);
+    }
+  }
+
+  if( rc==LSM_OK ){
+    if( bConfig==0 ){
+      switch( eOpt ){
+        case LSM_INFO_DB_STRUCTURE:
+        case LSM_INFO_FREELIST:
+          rc = lsm_info(pDb, eOpt, &z);
+          break;
+        case LSM_INFO_ARRAY_STRUCTURE:
+        case LSM_INFO_ARRAY_PAGES:
+        case LSM_INFO_PAGE_ASCII_DUMP:
+        case LSM_INFO_PAGE_HEX_DUMP:
+          rc = lsm_info(pDb, eOpt, iPg, &z);
+          break;
+        default:
+          assert( !"no chance" );
+      }
+
+      if( rc==LSM_OK ){
+        printf("%s\n", z ? z : "");
+        fflush(stdout);
+      }
+      lsm_free(lsm_get_env(pDb), z);
+    }else{
+      int iRes = -1;
+      lsm_config(pDb, eOpt, &iRes);
+      printf("%d\n", iRes);
+      fflush(stdout);
+    }
+  }
+
+  lsm_close(pDb);
+  return rc;
+
+ usage:
+  testPrintUsage("DATABASE ?array|page-ascii|page-hex PGNO?");
+  return -1;
+}

ext/lsm1/lsm-test/lsmtest_io.c

@@ -0,0 +1,256 @@
+
+/*
+** SUMMARY
+**
+**   This file implements the 'io' subcommand of the test program. It is used
+**   for testing the performance of various combinations of write() and fsync()
+**   system calls. All operations occur on a single file, which may or may not
+**   exist when a test is started.
+**
+**   A test consists of a series of commands. Each command is either a write
+**   or an fsync. A write is specified as "<amount>@<offset>", where <amount>
+**   is the amount of data written, and <offset> is the offset of the file
+**   to write to. An <amount> or an <offset> is specified as an integer number
+**   of bytes. Or, if postfixed with a "K", "M" or "G", an integer number of
+**   KB, MB or GB, respectively. An fsync is simply "S". All commands are
+**   case-insensitive.
+**
+**   Example test program:
+**
+**        2M@6M 1492K@4M S 4096@4K S
+**
+**   This program writes 2 MB of data starting at the offset 6MB offset of
+**   the file, followed by 1492 KB of data written at the 4MB offset of the
+**   file, followed by a call to fsync(), a write of 4KB of data at byte
+**   offset 4096, and finally another call to fsync().
+**
+**   Commands may either be specified on the command line (one command per
+**   command line argument) or read from stdin. Commands read from stdin
+**   must be separated by white-space.
+**
+** COMMAND LINE INVOCATION
+**
+**   The sub-command implemented in this file must be invoked with at least
+**   two arguments - the path to the file to write to and the page-size to
+**   use for writing. If there are more than two arguments, then each
+**   subsequent argument is assumed to be a test command. If there are exactly
+**   two arguments, the test commands are read from stdin.
+**
+**   A write command does not result in a single call to system call write().
+**   Instead, the specified region is written sequentially using one or
+**   more calls to write(), each of which writes not more than one page of
+**   data. For example, if the page-size is 4KB, the command "2M@6M" results
+**   in 512 calls to write(), each of which writes 4KB of data.
+**
+** EXAMPLES
+**
+**   Two equivalent examples:
+**
+**     $ lsmtest io testfile.db 4KB 2M@6M 1492K@4M S 4096@4K S
+**     3544K written in 129 ms
+**     $ echo "2M@6M 1492K@4M S 4096@4K S" | lsmtest io testfile.db 4096 
+**     3544K written in 127 ms
+**
+*/
+
+#include "lsmtest.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#ifndef _WIN32
+# include <unistd.h>
+#endif
+#include <ctype.h>
+
+typedef struct IoContext IoContext;
+
+struct IoContext {
+  int fd;
+  int nWrite;
+};
+
+/*
+** As isspace(3)
+*/
+static int safe_isspace(char c){
+  if( c&0x80) return 0;
+  return isspace(c);
+}
+
+/*
+** As isdigit(3)
+*/
+static int safe_isdigit(char c){
+  if( c&0x80) return 0;
+  return isdigit(c);
+}
+
+static i64 getNextSize(char *zIn, char **pzOut, int *pRc){
+  i64 iRet = 0;
+  if( *pRc==0 ){
+    char *z = zIn;
+
+    if( !safe_isdigit(*z) ){
+      *pRc = 1;
+      return 0;
+    }
+
+    /* Process digits */
+    while( safe_isdigit(*z) ){
+      iRet = iRet*10 + (*z - '0');
+      z++;
+    }
+
+    /* Process suffix */
+    switch( *z ){
+      case 'k': case 'K':
+        iRet = iRet * 1024;
+        z++;
+        break;
+
+      case 'm': case 'M':
+        iRet = iRet * 1024 * 1024;
+        z++;
+        break;
+
+      case 'g': case 'G':
+        iRet = iRet * 1024 * 1024 * 1024;
+        z++;
+        break;
+    }
+
+    if( pzOut ) *pzOut = z;
+  }
+  return iRet;
+}
+
+static int doOneCmd(
+  IoContext *pCtx,
+  u8 *aData,
+  int pgsz,
+  char *zCmd,
+  char **pzOut
+){
+  char c;
+  char *z = zCmd;
+
+  while( safe_isspace(*z) ) z++;
+  c = *z;
+
+  if( c==0 ){
+    if( pzOut ) *pzOut = z;
+    return 0;
+  }
+
+  if( c=='s' || c=='S' ){
+    if( pzOut ) *pzOut = &z[1];
+    return fdatasync(pCtx->fd);
+  }
+
+  if( safe_isdigit(c) ){
+    i64 iOff = 0;
+    int nByte = 0;
+    int rc = 0;
+    int nPg;
+    int iPg;
+
+    nByte = getNextSize(z, &z, &rc);
+    if( rc || *z!='@' ) goto bad_command;
+    z++;
+    iOff = getNextSize(z, &z, &rc);
+    if( rc || (safe_isspace(*z)==0 && *z!='\0') ) goto bad_command;
+    if( pzOut ) *pzOut = z;
+
+    nPg = (nByte+pgsz-1) / pgsz;
+    lseek(pCtx->fd, iOff, SEEK_SET);
+    for(iPg=0; iPg<nPg; iPg++){
+      write(pCtx->fd, aData, pgsz);
+    }
+    pCtx->nWrite += nByte/1024;
+
+    return 0;
+  }
+
+ bad_command:
+  testPrintError("unrecognized command: %s", zCmd);
+  return 1;
+}
+
+static int readStdin(char **pzOut){
+  int nAlloc = 128;
+  char *zOut = 0;
+  int nOut = 0;
+
+  while( !feof(stdin) ){
+    int nRead;
+
+    nAlloc = nAlloc*2;
+    zOut = realloc(zOut, nAlloc);
+    nRead = fread(&zOut[nOut], 1, nAlloc-nOut-1, stdin);
+
+    if( nRead==0 ) break;
+    nOut += nRead;
+    zOut[nOut] = '\0';
+  }
+
+  *pzOut = zOut;
+  return 0;
+}
+
+int do_io(int nArg, char **azArg){
+  IoContext ctx;
+  int pgsz;
+  char *zFile;
+  char *zPgsz;
+  int i;
+  int rc = 0;
+
+  char *zStdin = 0;
+  char *z;
+
+  u8 *aData;
+
+  memset(&ctx, 0, sizeof(IoContext));
+  if( nArg<2 ){
+    testPrintUsage("FILE PGSZ ?CMD-1 ...?");
+    return -1;
+  }
+  zFile = azArg[0];
+  zPgsz = azArg[1];
+
+  pgsz = getNextSize(zPgsz, 0, &rc);
+  if( pgsz<=0 ){
+    testPrintError("Ridiculous page size: %d", pgsz);
+    return -1;
+  }
+  aData = malloc(pgsz);
+  memset(aData, 0x77, pgsz);
+
+  ctx.fd = open(zFile, O_RDWR|O_CREAT|_O_BINARY, 0644);
+  if( ctx.fd<0 ){
+    perror("open: ");
+    return -1;
+  }
+
+  if( nArg==2 ){
+    readStdin(&zStdin);
+    testTimeInit();
+    z = zStdin;
+    while( *z && rc==0 ){
+      rc = doOneCmd(&ctx, aData, pgsz, z, &z);
+    }
+  }else{
+    testTimeInit();
+    for(i=2; i<nArg; i++){
+      rc = doOneCmd(&ctx, aData, pgsz, azArg[i], 0);
+    }
+  }
+
+  printf("%dK written in %d ms\n", ctx.nWrite, testTimeGet());
+
+  free(zStdin);
+  close(ctx.fd);
+
+  return 0;
+}