merge:

git://github.com/Tokutek/ft-index.git
git://github.com/Tokutek/ft-engine.git
at the tag tokudb-7.1.6

storage/tokudb/CMakeLists.txt

@@ -40,17 +40,22 @@ IF(DEFINED TOKUDB_VERSION)
     ADD_DEFINITIONS("-DTOKUDB_VERSION=\"${TOKUDB_VERSION}\"")
 ENDIF()
 
+IF(DEFINED TOKUDB_NOPATCH_CONFIG)
+    ADD_DEFINITIONS("-DTOKUDB_NOPATCH_CONFIG=${TOKUDB_NOPATCH_CONFIG}")
+ENDIF()
+
+IF(DEFINED TOKUDB_CHECK_JEMALLOC)
+    ADD_DEFINITIONS("-DTOKUDB_CHECK_JEMALLOC=${TOKUDB_CHECK_JEMALLOC}")
+ENDIF()
+
 ADD_SUBDIRECTORY(ft-index)
 
-# TODO: clean up includes in ft-index
 INCLUDE_DIRECTORIES(ft-index)
-INCLUDE_DIRECTORIES(ft-index/include)
 INCLUDE_DIRECTORIES(ft-index/portability)
-INCLUDE_DIRECTORIES(ft-index/toku_include)
 INCLUDE_DIRECTORIES(ft-index/util)
 INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index)
 INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index/buildheader)
-INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index/toku_include)
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index/portability)
 
 SET(TOKUDB_PLUGIN_DYNAMIC "ha_tokudb")
 SET(TOKUDB_SOURCES ha_tokudb.cc)

storage/tokudb/README.md

@@ -24,19 +24,20 @@ working MySQL or MariaDB with Tokutek patches, and with the TokuDB storage
 engine, called `make.mysql.bash`.  This script will download copies of the
 needed source code from github and build everything.
 
-To build MySQL with TokuDB 7.1.0:
+To build MySQL 5.5.36 with TokuDB 7.1.5:
 ```sh
-scripts/make.mysql.bash --mysqlbuild=mysql-5.5.30-tokudb-7.1.0-linux-x86_64
+scripts/make.mysql.bash --mysqlbuild=mysql-5.5.36-tokudb-7.1.5-linux-x86_64
 ```
 
-To build MariaDB with TokuDB 7.1.0:
+To build MariaDB 5.5.36 with TokuDB 7.1.5:
 ```sh
-scripts/make.mysql.bash --mysqlbuild=mariadb-5.5.30-tokudb-7.1.0-linux-x86_64
+scripts/make.mysql.bash --mysqlbuild=mariadb-5.5.36-tokudb-7.1.5-linux-x86_64
 ```
 
 Before you start, make sure you have a C++11-compatible compiler (GCC >=
 4.7 is recommended), as well as CMake >=2.8.8, and the libraries and
-header files for valgrind,zlib, and Berkeley DB.
+header files for valgrind,zlib, and Berkeley DB.  We are using the gcc 4.7
+in devtoolset-1.1.
 
 On CentOS, `yum install valgrind-devel zlib-devel libdb-devel`
 
@@ -76,5 +77,5 @@ TokuDB is available under the GPL version 2.  See [COPYING][copying]
 The TokuKV component of TokuDB is available under the GPL version 2, with
 slight modifications.  See [README-TOKUDB][license].
 
-[copying]: http://github.com/Tokutek/ft-engine/blob/master/COPYING
+[copying]: http://github.com/Tokutek/tokudb-engine/blob/master/COPYING
-[license]: http://github.com/Tokutek/ft-index/blob/master/README-TOKUDB
+[license]: http://github.com/Tokutek/tokudb-index/blob/master/README-TOKUDB

storage/tokudb/ft-index/CMakeLists.txt

@@ -26,14 +26,12 @@ if (USE_VALGRIND AND NOT VALGRIND_INCLUDE_DIR MATCHES NOTFOUND)
     )
 endif()
 include_directories(
-  ${CMAKE_CURRENT_SOURCE_DIR}/include
-  ${CMAKE_CURRENT_SOURCE_DIR}/toku_include
   ${CMAKE_CURRENT_SOURCE_DIR}/portability
   ${CMAKE_CURRENT_SOURCE_DIR}  ## so you can include <ft/ft-ops.h> from inside src/
   ${CMAKE_CURRENT_BINARY_DIR}  ## for logging code
   )
 ## include where config.h will be generated
-include_directories(${CMAKE_CURRENT_BINARY_DIR}/toku_include)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/portability)
 
 ## build db.h and include where it will be generated
 add_subdirectory(buildheader)
@@ -51,11 +49,9 @@ add_subdirectory(portability)
 add_subdirectory(ft)
 add_subdirectory(locktree)
 add_subdirectory(src)
-add_subdirectory(utils)
+add_subdirectory(tools)
 
 ## subdirectories that just install things
-add_subdirectory(include)
-add_subdirectory(toku_include)
 #add_subdirectory(examples)
 
 INSTALL_DOCUMENTATION(README.md README-TOKUDB COMPONENT Server)

storage/tokudb/ft-index/buildheader/make_tdb.cc

@@ -464,6 +464,7 @@ static void print_db_env_struct (void) {
                              "void (*set_loader_memory_size)(DB_ENV *env, uint64_t (*get_loader_memory_size_callback)(void))",
                              "uint64_t (*get_loader_memory_size)(DB_ENV *env)",
                              "void (*set_killed_callback)(DB_ENV *env, uint64_t default_killed_time_msec, uint64_t (*get_killed_time_callback)(uint64_t default_killed_time_msec), int (*killed_callback)(void))",
+                             "void (*do_backtrace)                        (DB_ENV *env)",
                              NULL};
 
         sort_and_dump_fields("db_env", true, extra);
@@ -575,7 +576,6 @@ static void print_db_txn_struct (void) {
     STRUCT_SETUP(DB_TXN, parent,      "DB_TXN *%s");
     const char *extra[] = {
 	"int (*txn_stat)(DB_TXN *, struct txn_stat **)", 
-	"struct toku_list open_txns",
 	"int (*commit_with_progress)(DB_TXN*, uint32_t, TXN_PROGRESS_POLL_FUNCTION, void*)",
 	"int (*abort_with_progress)(DB_TXN*, TXN_PROGRESS_POLL_FUNCTION, void*)",
 	"int (*xa_prepare) (DB_TXN*, TOKU_XA_XID *)",
@@ -608,8 +608,9 @@ static void print_dbc_struct (void) {
 	"int (*c_getf_set)(DBC *, uint32_t, DBT *, YDB_CALLBACK_FUNCTION, void *)",
 	"int (*c_getf_set_range)(DBC *, uint32_t, DBT *, YDB_CALLBACK_FUNCTION, void *)",
 	"int (*c_getf_set_range_reverse)(DBC *, uint32_t, DBT *, YDB_CALLBACK_FUNCTION, void *)",
+	"int (*c_getf_set_range_with_bound)(DBC *, uint32_t, DBT *k, DBT *k_bound, YDB_CALLBACK_FUNCTION, void *)",
 	"int (*c_set_bounds)(DBC*, const DBT*, const DBT*, bool pre_acquire, int out_of_range_error)",
-    "void (*c_set_check_interrupt_callback)(DBC*, bool (*)(void*), void *)",
+        "void (*c_set_check_interrupt_callback)(DBC*, bool (*)(void*), void *)",
 	"void (*c_remove_restriction)(DBC*)",
 	NULL};
     sort_and_dump_fields("dbc", false, extra);
@@ -632,7 +633,6 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
     //printf("#include <inttypes.h>\n");
     printf("#if defined(__cplusplus) || defined(__cilkplusplus)\nextern \"C\" {\n#endif\n");
 
-    printf("#define TOKUDB 1\n");
     printf("#define DB_VERSION_MAJOR %d\n", DB_VERSION_MAJOR);
     printf("#define DB_VERSION_MINOR %d\n", DB_VERSION_MINOR);
     printf("/* As of r40364 (post TokuDB 5.2.7), the patch version number is 100+ the BDB header patch version number.*/\n");
@@ -651,7 +651,6 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
            "    char data[DB_GID_SIZE];\n"
            "} TOKU_XA_XID;\n");
 
-   //Typedef toku_off_t
     printf("#ifndef TOKU_OFF_T_DEFINED\n"
            "#define TOKU_OFF_T_DEFINED\n"
            "typedef int64_t toku_off_t;\n"
@@ -670,7 +669,10 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
     printf("typedef uint32_t db_recno_t;\n");
     printf("typedef int(*YDB_CALLBACK_FUNCTION)(DBT const*, DBT const*, void*);\n");
 
-    printf("#include <tdb-internal.h>\n");
+    printf("struct simple_dbt {\n");
+    printf("    uint32_t len;\n");
+    printf("    void     *data;\n");
+    printf("};\n");
     
     //stat64
     printf("typedef struct __toku_db_btree_stat64 {\n");

storage/tokudb/ft-index/cmake/merge_archives_unix.cmake.in

@@ -26,15 +26,52 @@ SET(CMAKE_RANLIB "@CMAKE_RANLIB@")
 
 SET(TEMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/merge_archives_${TARGET})
 MAKE_DIRECTORY(${TEMP_DIR})
-# Extract each archive to its own subdirectory(avoid object filename clashes)
+# Extract each archive to its own subdirectory(avoid object filename
+# clashes) Since the lib may contain objects with the same name, we first
+# list the archive contents, then uniquify the object names as we extract
+# them.
 FOREACH(LIB ${STATIC_LIBS})
   GET_FILENAME_COMPONENT(NAME_NO_EXT ${LIB} NAME_WE)
   SET(TEMP_SUBDIR ${TEMP_DIR}/${NAME_NO_EXT})
   MAKE_DIRECTORY(${TEMP_SUBDIR})
   EXECUTE_PROCESS(
-    COMMAND ${CMAKE_AR} -x ${LIB}
+    COMMAND ${CMAKE_AR} -t ${LIB}
-    WORKING_DIRECTORY ${TEMP_SUBDIR}
+    OUTPUT_VARIABLE LIB_OBJS
     )
+  STRING(REGEX REPLACE "\n" ";" LIB_OBJ_LIST "${LIB_OBJS}")
+  STRING(REGEX REPLACE ";$" "" LIB_OBJ_LIST "${LIB_OBJ_LIST}")
+
+  LIST(LENGTH LIB_OBJ_LIST LENGTH_WITH_DUPS)
+  SET(LIB_OBJ_LIST_NO_DUPS ${LIB_OBJ_LIST})
+  LIST(REMOVE_DUPLICATES LIB_OBJ_LIST_NO_DUPS)
+  LIST(LENGTH LIB_OBJ_LIST_NO_DUPS LENGTH_WITHOUT_DUPS)
+
+  IF(LENGTH_WITH_DUPS EQUAL LENGTH_WITHOUT_DUPS)
+    # Optimization for when lib doesn't actually have duplicate object
+    # names, we can just extract everything.
+    EXECUTE_PROCESS(
+      COMMAND ${CMAKE_AR} -x ${LIB}
+      WORKING_DIRECTORY ${TEMP_SUBDIR}
+      )
+  ELSE()
+    LIST(SORT LIB_OBJ_LIST)
+    SET(SAME_OBJ_COUNT 1)
+    SET(LAST_OBJ_NAME)
+    FOREACH(OBJ ${LIB_OBJ_LIST})
+      IF(OBJ STREQUAL LAST_OBJ_NAME)
+        GET_FILENAME_COMPONENT(OBJ_NO_EXT ${OBJ} NAME_WE)
+        FILE(RENAME "${TEMP_SUBDIR}/${OBJ}" "${TEMP_SUBDIR}/${OBJ_NO_EXT}.${SAME_OBJ_COUNT}.o")
+        MATH(EXPR SAME_OBJ_COUNT "${SAME_OBJ_COUNT}+1")
+      ELSE()
+        SET(SAME_OBJ_COUNT 1)
+      ENDIF()
+      SET(LAST_OBJ_NAME "${OBJ}")
+      EXECUTE_PROCESS(
+        COMMAND ${CMAKE_AR} -xN ${SAME_OBJ_COUNT} ${LIB} ${OBJ}
+        WORKING_DIRECTORY ${TEMP_SUBDIR}
+        )
+    ENDFOREACH()
+  ENDIF()
 
   FILE(GLOB_RECURSE LIB_OBJECTS "${TEMP_SUBDIR}/*.o")
   SET(OBJECTS ${OBJECTS} ${LIB_OBJECTS})
@@ -51,11 +88,7 @@ ENDFOREACH()
 FILE(TO_NATIVE_PATH ${TARGET_LOCATION} ${TARGET_LOCATION})
 # Now pack the objects into library with ar.
 EXECUTE_PROCESS(
-  COMMAND ${CMAKE_AR} -r ${TARGET_LOCATION} ${ALL_OBJECTS}
+  COMMAND ${CMAKE_AR} rcs ${TARGET_LOCATION} ${ALL_OBJECTS}
-  WORKING_DIRECTORY ${TEMP_DIR}
-)
-EXECUTE_PROCESS(
-  COMMAND ${CMAKE_RANLIB} ${TARGET_LOCATION}
   WORKING_DIRECTORY ${TEMP_DIR}
 )
 

storage/tokudb/ft-index/cmake_modules/TokuBuildTagDatabases.cmake

@@ -7,7 +7,6 @@ file(GLOB_RECURSE all_srcs
   locktree/*.cc
   portability/*.cc
   src/*.cc
-  toku_include/*.cc
   utils/*.cc
   util/*.cc
   db-benchmark-test/*.cc
@@ -24,13 +23,12 @@ file(GLOB_RECURSE all_hdrs
   locktree/*.h
   portability/*.h
   src/*.h
-  toku_include/*.h
   utils/*.h
   util/*.h
   db-benchmark-test/*.h
   )
 list(APPEND all_hdrs
-  ${CMAKE_CURRENT_BINARY_DIR}/toku_include/toku_config.h
+  ${CMAKE_CURRENT_BINARY_DIR}/portability/toku_config.h
   ${CMAKE_CURRENT_BINARY_DIR}/buildheader/db.h
   ${CMAKE_CURRENT_BINARY_DIR}/ft/log_header.h
   )
@@ -79,7 +77,7 @@ if (USE_CSCOPE)
       OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/cscope.out"
       OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/cscope.in.out"
       OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/cscope.po.out"
-      COMMAND ${CSCOPE} -b -q -R -i"${CMAKE_CURRENT_BINARY_DIR}/cscope.files" -I"${CMAKE_CURRENT_SOURCE_DIR}" -I"${CMAKE_CURRENT_SOURCE_DIR}/include" -I"${CMAKE_CURRENT_SOURCE_DIR}/toku_include" -I"${CMAKE_CURRENT_SOURCE_DIR}/portability" -I"${CMAKE_CURRENT_SOURCE_DIR}/ft" -I"${CMAKE_CURRENT_SOURCE_DIR}/src" -I"${CMAKE_CURRENT_SOURCE_DIR}/locktree" -I"${CMAKE_CURRENT_SOURCE_DIR}/utils" -I"${CMAKE_CURRENT_SOURCE_DIR}/db-benchmark-test" -I"${CMAKE_CURRENT_BINARY_DIR}" -I"${CMAKE_CURRENT_BINARY_DIR}/toku_include" -I"${CMAKE_CURRENT_BINARY_DIR}/buildheader"
+      COMMAND ${CSCOPE} -b -q -R -i"${CMAKE_CURRENT_BINARY_DIR}/cscope.files" -I"${CMAKE_CURRENT_SOURCE_DIR}" -I"${CMAKE_CURRENT_SOURCE_DIR}/include" -I"${CMAKE_CURRENT_SOURCE_DIR}/portability" -I"${CMAKE_CURRENT_SOURCE_DIR}/portability" -I"${CMAKE_CURRENT_SOURCE_DIR}/ft" -I"${CMAKE_CURRENT_SOURCE_DIR}/src" -I"${CMAKE_CURRENT_SOURCE_DIR}/locktree" -I"${CMAKE_CURRENT_SOURCE_DIR}/utils" -I"${CMAKE_CURRENT_SOURCE_DIR}/db-benchmark-test" -I"${CMAKE_CURRENT_BINARY_DIR}" -I"${CMAKE_CURRENT_BINARY_DIR}/portability" -I"${CMAKE_CURRENT_BINARY_DIR}/buildheader"
       DEPENDS ${all_srcs} ${all_hdrs} install_tdb_h generate_config_h generate_log_code
       WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
     add_custom_target(build_cscope.out ALL DEPENDS

storage/tokudb/ft-index/cmake_modules/TokuMergeLibs.cmake

@@ -50,7 +50,7 @@ MACRO(TOKU_MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE)
   ENDFOREACH()
   IF(OSLIBS)
     #LIST(REMOVE_DUPLICATES OSLIBS)
-    TARGET_LINK_LIBRARIES(${TARGET} ${OSLIBS})
+    TARGET_LINK_LIBRARIES(${TARGET} LINK_PUBLIC ${OSLIBS})
   ENDIF()
 
   # Make the generated dummy source file depended on all static input

storage/tokudb/ft-index/cmake_modules/TokuSetupCTest.cmake

@@ -94,8 +94,6 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
   ## set up full valgrind suppressions file (concatenate the suppressions files)
   file(READ ft/valgrind.suppressions valgrind_suppressions)
   file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${valgrind_suppressions}")
-  file(READ src/tests/bdb.suppressions bdb_suppressions)
-  file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${bdb_suppressions}")
   file(READ bash.suppressions bash_suppressions)
   file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${bash_suppressions}")
 

storage/tokudb/ft-index/ft/CMakeLists.txt

@@ -52,9 +52,7 @@ set(FT_SOURCES
   logfilemgr
   logger
   log_upgrade
-  memarena
   minicron
-  omt
   pqueue
   queue
   quicklz
@@ -69,7 +67,6 @@ set(FT_SOURCES
   txn_child_manager
   txn_manager
   ule
-  x1764
   xids
   ybt
   "${CMAKE_CURRENT_BINARY_DIR}/log_code"

storage/tokudb/ft-index/ft/background_job_manager.cc

@@ -89,11 +89,12 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2011-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "background_job_manager.h"
+#include <portability/toku_config.h>
-#include "toku_config.h"
 #include <memory.h>
 #include <toku_pthread.h>
 
+#include "background_job_manager.h"
+
 struct background_job_manager_struct {
     bool accepting_jobs;
     uint32_t num_jobs;

storage/tokudb/ft-index/ft/block_table.cc

@@ -427,7 +427,7 @@ toku_ft_unlock (FT ft) {
     unlock_for_blocktable(bt);
 }
 
-// Also used only in brt-serialize-test.
+// Also used only in ft-serialize-test.
 void
 toku_block_free(BLOCK_TABLE bt, uint64_t offset) {
     lock_for_blocktable(bt);
@@ -580,7 +580,7 @@ void toku_serialize_translation_to_wbuf(BLOCK_TABLE bt, int fd, struct wbuf *w,
         wbuf_DISKOFF(w, t->block_translation[i].u.diskoff);
         wbuf_DISKOFF(w, t->block_translation[i].size);
     }
-    uint32_t checksum = x1764_finish(&w->checksum);
+    uint32_t checksum = toku_x1764_finish(&w->checksum);
     wbuf_int(w, checksum);
     *address = t->block_translation[b.b].u.diskoff;
     *size    = size_translation;
@@ -915,7 +915,7 @@ translation_deserialize_from_buffer(struct translation *t,    // destination int
     t->type = TRANSLATION_CHECKPOINTED;
     {
         // check the checksum
-        uint32_t x1764 = x1764_memory(translation_buffer, size_on_disk - 4);
+        uint32_t x1764 = toku_x1764_memory(translation_buffer, size_on_disk - 4);
         uint64_t offset = size_on_disk - 4;
         //printf("%s:%d read from %ld (x1764 offset=%ld) size=%ld\n", __FILE__, __LINE__, block_translation_address_on_disk, offset, block_translation_size_on_disk);
         uint32_t stored_x1764 = toku_dtoh32(*(int*)(translation_buffer + offset));

storage/tokudb/ft-index/ft/bndata.cc

@@ -90,46 +90,197 @@ PATENT RIGHTS GRANT:
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
 #include <bndata.h>
+#include <ft-ops.h>
 
-static uint32_t klpair_size(KLPAIR klpair){
+using namespace toku;
-    return sizeof(*klpair) + klpair->keylen + leafentry_memsize(get_le_from_klpair(klpair));
+uint32_t bn_data::klpair_disksize(const uint32_t klpair_len, const klpair_struct *klpair) const {
-}
+    return sizeof(*klpair) + keylen_from_klpair_len(klpair_len) + leafentry_disksize(get_le_from_klpair(klpair));
-
-static uint32_t klpair_disksize(KLPAIR klpair){
-    return sizeof(*klpair) + klpair->keylen + leafentry_disksize(get_le_from_klpair(klpair));
 }
 
 void bn_data::init_zero() {
     toku_mempool_zero(&m_buffer_mempool);
+    m_disksize_of_keys = 0;
 }
 
 void bn_data::initialize_empty() {
-    toku_mempool_zero(&m_buffer_mempool);
+    init_zero();
-    m_buffer.create_no_array();
+    m_buffer.create();
 }
 
-void bn_data::initialize_from_data(uint32_t num_entries, unsigned char *buf, uint32_t data_size) {
+void bn_data::add_key(uint32_t keylen) {
+    m_disksize_of_keys += sizeof(keylen) + keylen;
+}
+
+void bn_data::add_keys(uint32_t n_keys, uint32_t combined_klpair_len) {
+    invariant(n_keys * sizeof(uint32_t) <= combined_klpair_len);
+    m_disksize_of_keys += combined_klpair_len;
+}
+
+void bn_data::remove_key(uint32_t keylen) {
+    m_disksize_of_keys -= sizeof(keylen) + keylen;
+}
+
+// Deserialize from format optimized for keys being inlined.
+// Currently only supports fixed-length keys.
+void bn_data::initialize_from_separate_keys_and_vals(uint32_t num_entries, struct rbuf *rb, uint32_t data_size, uint32_t version UU(),
+                                                     uint32_t key_data_size, uint32_t val_data_size, bool all_keys_same_length,
+                                                     uint32_t fixed_klpair_length) {
+    paranoid_invariant(version >= FT_LAYOUT_VERSION_26);  // Support was added @26
+    uint32_t ndone_before = rb->ndone;
+    init_zero();
+    invariant(all_keys_same_length);  // Until otherwise supported.
+    bytevec keys_src;
+    rbuf_literal_bytes(rb, &keys_src, key_data_size);
+    //Generate dmt
+    this->m_buffer.create_from_sorted_memory_of_fixed_size_elements(
+            keys_src, num_entries, key_data_size, fixed_klpair_length);
+    toku_mempool_construct(&this->m_buffer_mempool, val_data_size);
+
+    bytevec vals_src;
+    rbuf_literal_bytes(rb, &vals_src, val_data_size);
+
+    if (num_entries > 0) {
+        void *vals_dest = toku_mempool_malloc(&this->m_buffer_mempool, val_data_size, 1);
+        paranoid_invariant_notnull(vals_dest);
+        memcpy(vals_dest, vals_src, val_data_size);
+    }
+
+    add_keys(num_entries, num_entries * fixed_klpair_length);
+
+    toku_note_deserialized_basement_node(all_keys_same_length);
+
+    invariant(rb->ndone - ndone_before == data_size);
+}
+
+static int
+wbufwriteleafentry(const void* key, const uint32_t keylen, const LEAFENTRY &le, const uint32_t UU(idx), struct wbuf * const wb) {
+    // need to pack the leafentry as it was in versions
+    // where the key was integrated into it (< 26)
+    uint32_t begin_spot UU() = wb->ndone;
+    uint32_t le_disk_size = leafentry_disksize(le);
+    wbuf_nocrc_uint8_t(wb, le->type);
+    wbuf_nocrc_uint32_t(wb, keylen);
+    if (le->type == LE_CLEAN) {
+        wbuf_nocrc_uint32_t(wb, le->u.clean.vallen);
+        wbuf_nocrc_literal_bytes(wb, key, keylen);
+        wbuf_nocrc_literal_bytes(wb, le->u.clean.val, le->u.clean.vallen);
+    }
+    else {
+        paranoid_invariant(le->type == LE_MVCC);
+        wbuf_nocrc_uint32_t(wb, le->u.mvcc.num_cxrs);
+        wbuf_nocrc_uint8_t(wb, le->u.mvcc.num_pxrs);
+        wbuf_nocrc_literal_bytes(wb, key, keylen);
+        wbuf_nocrc_literal_bytes(wb, le->u.mvcc.xrs, le_disk_size - (1 + 4 + 1));
+    }
+    uint32_t end_spot UU() = wb->ndone;
+    paranoid_invariant((end_spot - begin_spot) == keylen + sizeof(keylen) + le_disk_size);
+    return 0;
+}
+
+void bn_data::serialize_to_wbuf(struct wbuf *const wb) {
+    prepare_to_serialize();
+    serialize_header(wb);
+    if (m_buffer.value_length_is_fixed()) {
+        serialize_rest(wb);
+    } else {
+        //
+        // iterate over leafentries and place them into the buffer
+        //
+        iterate<struct wbuf, wbufwriteleafentry>(wb);
+    }
+}
+
+// If we have fixed-length keys, we prepare the dmt and mempool.
+// The mempool is prepared by removing any fragmented space and ordering leafentries in the same order as their keys.
+void bn_data::prepare_to_serialize(void) {
+    if (m_buffer.value_length_is_fixed()) {
+        m_buffer.prepare_for_serialize();
+        dmt_compress_kvspace(0, nullptr, true);  // Gets it ready for easy serialization.
+    }
+}
+
+void bn_data::serialize_header(struct wbuf *wb) const {
+    bool fixed = m_buffer.value_length_is_fixed();
+
+    //key_data_size
+    wbuf_nocrc_uint(wb, m_disksize_of_keys);
+    //val_data_size
+    wbuf_nocrc_uint(wb, toku_mempool_get_used_size(&m_buffer_mempool));
+    //fixed_klpair_length
+    wbuf_nocrc_uint(wb, m_buffer.get_fixed_length());
+    // all_keys_same_length
+    wbuf_nocrc_uint8_t(wb, fixed);
+    // keys_vals_separate
+    wbuf_nocrc_uint8_t(wb, fixed);
+}
+
+void bn_data::serialize_rest(struct wbuf *wb) const {
+    //Write keys
+    invariant(m_buffer.value_length_is_fixed()); //Assumes prepare_to_serialize was called
+    m_buffer.serialize_values(m_disksize_of_keys, wb);
+
+    //Write leafentries
+    //Just ran dmt_compress_kvspace so there is no fragmentation and also leafentries are in sorted order.
+    paranoid_invariant(toku_mempool_get_frag_size(&m_buffer_mempool) == 0);
+    uint32_t val_data_size = toku_mempool_get_used_size(&m_buffer_mempool);
+    wbuf_nocrc_literal_bytes(wb, toku_mempool_get_base(&m_buffer_mempool), val_data_size);
+}
+
+// Deserialize from rbuf
+void bn_data::deserialize_from_rbuf(uint32_t num_entries, struct rbuf *rb, uint32_t data_size, uint32_t version) {
+    uint32_t key_data_size = data_size;  // overallocate if < version 26 (best guess that is guaranteed not too small)
+    uint32_t val_data_size = data_size;  // overallocate if < version 26 (best guess that is guaranteed not too small)
+
+    bool all_keys_same_length = false;
+    bool keys_vals_separate = false;
+    uint32_t fixed_klpair_length = 0;
+
+    // In version 25 and older there is no header.  Skip reading header for old version.
+    if (version >= FT_LAYOUT_VERSION_26) {
+        uint32_t ndone_before = rb->ndone;
+        key_data_size = rbuf_int(rb);
+        val_data_size = rbuf_int(rb);
+        fixed_klpair_length = rbuf_int(rb);  // 0 if !all_keys_same_length
+        all_keys_same_length = rbuf_char(rb);
+        keys_vals_separate = rbuf_char(rb);
+        invariant(all_keys_same_length == keys_vals_separate);  // Until we support otherwise
+        uint32_t header_size = rb->ndone - ndone_before;
+        data_size -= header_size;
+        invariant(header_size == HEADER_LENGTH);
+        if (keys_vals_separate) {
+            invariant(fixed_klpair_length >= sizeof(klpair_struct) || num_entries == 0);
+            initialize_from_separate_keys_and_vals(num_entries, rb, data_size, version,
+                                                   key_data_size, val_data_size, all_keys_same_length,
+                                                   fixed_klpair_length);
+            return;
+        }
+    }
+    // Version >= 26 and version 25 deserialization are now identical except that <= 25 might allocate too much memory.
+    bytevec bytes;
+    rbuf_literal_bytes(rb, &bytes, data_size);
+    const unsigned char *CAST_FROM_VOIDP(buf, bytes);
     if (data_size == 0) {
         invariant_zero(num_entries);
     }
-    KLPAIR *XMALLOC_N(num_entries, array); // create array of pointers to leafentries
+    init_zero();
-    unsigned char *newmem = NULL;
+    klpair_dmt_t::builder dmt_builder;
-    // add same wiggle room that toku_mempool_construct would, 25% extra
+    dmt_builder.create(num_entries, key_data_size);
-    uint32_t allocated_bytes = data_size + data_size/4;
+
-    CAST_FROM_VOIDP(newmem, toku_xmalloc(allocated_bytes)); 
+    // TODO(leif): clean this up (#149)
-    unsigned char* curr_src_pos = buf;
+    unsigned char *newmem = nullptr;
+    // add 25% extra wiggle room
+    uint32_t allocated_bytes_vals = val_data_size + (val_data_size / 4);
+    CAST_FROM_VOIDP(newmem, toku_xmalloc(allocated_bytes_vals));
+    const unsigned char* curr_src_pos = buf;
     unsigned char* curr_dest_pos = newmem;
     for (uint32_t i = 0; i < num_entries; i++) {
-        KLPAIR curr_kl = (KLPAIR)curr_dest_pos;
-        array[i] = curr_kl;
-
         uint8_t curr_type = curr_src_pos[0];
         curr_src_pos++;
         // first thing we do is lay out the key,
         // to do so, we must extract it from the leafentry
         // and write it in
         uint32_t keylen = 0;
-        void* keyp = NULL;
+        const void* keyp = nullptr;
         keylen = *(uint32_t *)curr_src_pos;
         curr_src_pos += sizeof(uint32_t);
         uint32_t clean_vallen = 0;
@@ -150,12 +301,10 @@ void bn_data::initialize_from_data(uint32_t num_entries, unsigned char *buf, uin
             keyp = curr_src_pos;
             curr_src_pos += keylen;
         }
-        // now that we have the keylen and the key, we can copy it
+        uint32_t le_offset = curr_dest_pos - newmem;
-        // into the destination
+        dmt_builder.append(klpair_dmtwriter(keylen, le_offset, keyp));
-        *(uint32_t *)curr_dest_pos = keylen;
+        add_key(keylen);
-        curr_dest_pos += sizeof(keylen);
+
-        memcpy(curr_dest_pos, keyp, keylen);
-        curr_dest_pos += keylen;
         // now curr_dest_pos is pointing to where the leafentry should be packed
         curr_dest_pos[0] = curr_type;
         curr_dest_pos++;
@@ -173,31 +322,44 @@ void bn_data::initialize_from_data(uint32_t num_entries, unsigned char *buf, uin
             *(uint8_t *)curr_dest_pos = num_pxrs;
             curr_dest_pos += sizeof(num_pxrs);
             // now we need to pack the rest of the data
-            uint32_t num_rest_bytes = leafentry_rest_memsize(num_pxrs, num_cxrs, curr_src_pos);
+            uint32_t num_rest_bytes = leafentry_rest_memsize(num_pxrs, num_cxrs, const_cast<uint8_t*>(curr_src_pos));
             memcpy(curr_dest_pos, curr_src_pos, num_rest_bytes);
             curr_dest_pos += num_rest_bytes;
             curr_src_pos += num_rest_bytes;
         }
     }
-    uint32_t num_bytes_read UU() = (uint32_t)(curr_src_pos - buf);
+    dmt_builder.build(&this->m_buffer);
-    paranoid_invariant( num_bytes_read == data_size);
+    toku_note_deserialized_basement_node(m_buffer.value_length_is_fixed());
-    uint32_t num_bytes_written = curr_dest_pos - newmem;
-    paranoid_invariant( num_bytes_written == data_size);
-    toku_mempool_init(&m_buffer_mempool, newmem, (size_t)(num_bytes_written), allocated_bytes);
 
-    // destroy old omt that was created by toku_create_empty_bn(), so we can create a new one
+    uint32_t num_bytes_read = (uint32_t)(curr_src_pos - buf);
-    m_buffer.destroy();
+    invariant(num_bytes_read == data_size);
-    m_buffer.create_steal_sorted_array(&array, num_entries, num_entries);
+
+    uint32_t num_bytes_written = curr_dest_pos - newmem + m_disksize_of_keys;
+    invariant(num_bytes_written == data_size);
+    toku_mempool_init(&m_buffer_mempool, newmem, (size_t)(curr_dest_pos - newmem), allocated_bytes_vals);
+
+    invariant(get_disk_size() == data_size);
+    // Versions older than 26 might have allocated too much memory.  Try to shrink the mempool now that we
+    // know how much memory we need.
+    if (version < FT_LAYOUT_VERSION_26) {
+        // Unnecessary after version 26
+        // Reallocate smaller mempool to save memory
+        invariant_zero(toku_mempool_get_frag_size(&m_buffer_mempool));
+        toku_mempool_realloc_larger(&m_buffer_mempool, toku_mempool_get_used_size(&m_buffer_mempool));
+    }
 }
 
 uint64_t bn_data::get_memory_size() {
     uint64_t retval = 0;
+    //TODO: Maybe ask for memory_size instead of mempool_footprint (either this todo or the next)
     // include fragmentation overhead but do not include space in the
     // mempool that has not yet been allocated for leaf entries
     size_t poolsize = toku_mempool_footprint(&m_buffer_mempool);
-    invariant(poolsize >= get_disk_size());
     retval += poolsize;
+    // This one includes not-yet-allocated for nodes (just like old constant-key omt)
+    //TODO: Maybe ask for mempool_footprint instead of memory_size.
     retval += m_buffer.memory_size();
+    invariant(retval >= get_disk_size());
     return retval;
 }
 
@@ -205,169 +367,263 @@ void bn_data::delete_leafentry (
     uint32_t idx,
     uint32_t keylen,
     uint32_t old_le_size
-    ) 
+    )
 {
+    remove_key(keylen);
     m_buffer.delete_at(idx);
-    toku_mempool_mfree(&m_buffer_mempool, 0, old_le_size + keylen + sizeof(keylen)); // Must pass 0, since le is no good any more.
+    toku_mempool_mfree(&m_buffer_mempool, nullptr, old_le_size);
 }
 
 /* mempool support */
 
-struct omt_compressor_state {
+struct dmt_compressor_state {
     struct mempool *new_kvspace;
-    KLPAIR *newvals;
+    class bn_data *bd;
 };
 
-static int move_it (const KLPAIR &klpair, const uint32_t idx, struct omt_compressor_state * const oc) {
+static int move_it (const uint32_t, klpair_struct *klpair, const uint32_t idx UU(), struct dmt_compressor_state * const oc) {
-    uint32_t size = klpair_size(klpair);
+    LEAFENTRY old_le = oc->bd->get_le_from_klpair(klpair);
-    KLPAIR CAST_FROM_VOIDP(newdata, toku_mempool_malloc(oc->new_kvspace, size, 1));
+    uint32_t size = leafentry_memsize(old_le);
+    void* newdata = toku_mempool_malloc(oc->new_kvspace, size, 1);
     paranoid_invariant_notnull(newdata); // we do this on a fresh mempool, so nothing bad should happen
-    memcpy(newdata, klpair, size);
+    memcpy(newdata, old_le, size);
-    oc->newvals[idx] = newdata;
+    klpair->le_offset = toku_mempool_get_offset_from_pointer_and_base(oc->new_kvspace, newdata);
     return 0;
 }
 
-// Compress things, and grow the mempool if needed.
+// Compress things, and grow or shrink the mempool if needed.
-void bn_data::omt_compress_kvspace(size_t added_size, void **maybe_free) {
+// May (always if force_compress) have a side effect of putting contents of mempool in sorted order.
-    uint32_t total_size_needed = toku_mempool_get_used_space(&m_buffer_mempool) + added_size;
+void bn_data::dmt_compress_kvspace(size_t added_size, void **maybe_free, bool force_compress) {
-    // set the new mempool size to be twice of the space we actually need.
+    uint32_t total_size_needed = toku_mempool_get_used_size(&m_buffer_mempool) + added_size;
-    // On top of the 25% that is padded within toku_mempool_construct (which we
+
-    // should consider getting rid of), that should be good enough.
+    // If there is no fragmentation, e.g. in serial inserts, we can just increase the size
+    // of the mempool and move things over with a cheap memcpy. If force_compress is true, 
+    // the caller needs the side effect that all contents are put in sorted order.
+    bool do_compress = toku_mempool_get_frag_size(&m_buffer_mempool) > 0 || force_compress;
+
+    void *old_mempool_base = toku_mempool_get_base(&m_buffer_mempool);
     struct mempool new_kvspace;
-    toku_mempool_construct(&new_kvspace, 2*total_size_needed);
+    if (do_compress) {
-    uint32_t numvals = omt_size();
+        size_t requested_size = force_compress ? total_size_needed : ((total_size_needed * 3) / 2);
-    KLPAIR *XMALLOC_N(numvals, newvals);
+        toku_mempool_construct(&new_kvspace, requested_size);
-    struct omt_compressor_state oc = { &new_kvspace, newvals };
+        struct dmt_compressor_state oc = { &new_kvspace, this };
-
+        m_buffer.iterate_ptr< decltype(oc), move_it >(&oc);
-    m_buffer.iterate_on_range< decltype(oc), move_it >(0, omt_size(), &oc);
+    } else {
-
+        toku_mempool_construct(&new_kvspace, total_size_needed);
-    m_buffer.destroy();
+        size_t old_offset_limit = toku_mempool_get_offset_limit(&m_buffer_mempool);
-    m_buffer.create_steal_sorted_array(&newvals, numvals, numvals);
+        void *new_mempool_base = toku_mempool_malloc(&new_kvspace, old_offset_limit, 1);
+        memcpy(new_mempool_base, old_mempool_base, old_offset_limit);
+    }
 
     if (maybe_free) {
-        *maybe_free = m_buffer_mempool.base;
+        *maybe_free = old_mempool_base;
     } else {
-        toku_free(m_buffer_mempool.base);
+        toku_free(old_mempool_base);
     }
     m_buffer_mempool = new_kvspace;
 }
 
 // Effect: Allocate a new object of size SIZE in MP.  If MP runs out of space, allocate new a new mempool space, and copy all the items
 //  from the OMT (which items refer to items in the old mempool) into the new mempool.
-//  If MAYBE_FREE is NULL then free the old mempool's space.
+//  If MAYBE_FREE is nullptr then free the old mempool's space.
 //  Otherwise, store the old mempool's space in maybe_free.
-KLPAIR bn_data::mempool_malloc_from_omt(size_t size, void **maybe_free) {
+LEAFENTRY bn_data::mempool_malloc_and_update_dmt(size_t size, void **maybe_free) {
     void *v = toku_mempool_malloc(&m_buffer_mempool, size, 1);
-    if (v == NULL) {
+    if (v == nullptr) {
-        omt_compress_kvspace(size, maybe_free);
+        dmt_compress_kvspace(size, maybe_free, false);
         v = toku_mempool_malloc(&m_buffer_mempool, size, 1);
         paranoid_invariant_notnull(v);
     }
-    return (KLPAIR)v;
+    return (LEAFENTRY)v;
 }
 
-//TODO: probably not free the "maybe_free" right away?
 void bn_data::get_space_for_overwrite(
     uint32_t idx,
-    const void* keyp,
+    const void* keyp UU(),
-    uint32_t keylen,
+    uint32_t keylen UU(),
     uint32_t old_le_size,
     uint32_t new_size,
-    LEAFENTRY* new_le_space
+    LEAFENTRY* new_le_space,
+    void **const maybe_free
     )
 {
-    void* maybe_free = nullptr;
+    *maybe_free = nullptr;
-    uint32_t size_alloc = new_size + keylen + sizeof(keylen);
+    LEAFENTRY new_le = mempool_malloc_and_update_dmt(new_size, maybe_free);
-    KLPAIR new_kl = mempool_malloc_from_omt(
+    toku_mempool_mfree(&m_buffer_mempool, nullptr, old_le_size);
-        size_alloc,
+    klpair_struct* klp = nullptr;
-        &maybe_free
+    uint32_t klpair_len;
-        );
+    int r = m_buffer.fetch(idx, &klpair_len, &klp);
-    uint32_t size_freed = old_le_size + keylen + sizeof(keylen);
+    invariant_zero(r);
-    toku_mempool_mfree(&m_buffer_mempool, nullptr, size_freed);  // Must pass nullptr, since le is no good any more.
+    paranoid_invariant(klp!=nullptr);
-    new_kl->keylen = keylen;
+    // Key never changes.
-    memcpy(new_kl->key_le, keyp, keylen);
+    paranoid_invariant(keylen_from_klpair_len(klpair_len) == keylen);
-    m_buffer.set_at(new_kl, idx);
+
-    *new_le_space = get_le_from_klpair(new_kl);
+    size_t new_le_offset = toku_mempool_get_offset_from_pointer_and_base(&this->m_buffer_mempool, new_le);
-    // free at end, so that the keyp and keylen
+    paranoid_invariant(new_le_offset <= UINT32_MAX - new_size);  // Not using > 4GB
-    // passed in is still valid
+    klp->le_offset = new_le_offset;
-    if (maybe_free) {
+
-        toku_free(maybe_free);
+    paranoid_invariant(new_le == get_le_from_klpair(klp));
-    }
+    *new_le_space = new_le;
 }
 
-//TODO: probably not free the "maybe_free" right away?
 void bn_data::get_space_for_insert(
     uint32_t idx,
     const void* keyp,
     uint32_t keylen,
     size_t size,
-    LEAFENTRY* new_le_space
+    LEAFENTRY* new_le_space,
+    void **const maybe_free
     )
 {
-    void* maybe_free = nullptr;
+    add_key(keylen);
-    uint32_t size_alloc = size + keylen + sizeof(keylen);
+
-    KLPAIR new_kl = mempool_malloc_from_omt(
+    *maybe_free = nullptr;
-        size_alloc,
+    LEAFENTRY new_le = mempool_malloc_and_update_dmt(size, maybe_free);
-        &maybe_free
+    size_t new_le_offset = toku_mempool_get_offset_from_pointer_and_base(&this->m_buffer_mempool, new_le);
-        );
+
-    new_kl->keylen = keylen;
+    klpair_dmtwriter kl(keylen, new_le_offset, keyp);
-    memcpy(new_kl->key_le, keyp, keylen);
+    m_buffer.insert_at(kl, idx);
-    m_buffer.insert_at(new_kl, idx);
+
-    *new_le_space = get_le_from_klpair(new_kl);
+    *new_le_space = new_le;
-    // free at end, so that the keyp and keylen
-    // passed in is still valid (you never know if
-    // it was part of the old mempool, this is just
-    // safer).
-    if (maybe_free) {
-        toku_free(maybe_free);
-    }
 }
 
-void bn_data::move_leafentries_to(
+class split_klpairs_extra {
-     BN_DATA dest_bd,
+    bn_data *const m_left_bn;
-     uint32_t lbi, //lower bound inclusive
+    bn_data *const m_right_bn;
-     uint32_t ube //upper bound exclusive
+    klpair_dmt_t::builder *const m_left_builder;
+    klpair_dmt_t::builder *const m_right_builder;
+    struct mempool *const m_left_dest_mp;
+    uint32_t m_split_at;
+
+    struct mempool *left_dest_mp(void) const { return m_left_dest_mp; }
+    struct mempool *right_dest_mp(void) const { return &m_right_bn->m_buffer_mempool; }
+
+    void copy_klpair(const uint32_t klpair_len, const klpair_struct &klpair,
+                     klpair_dmt_t::builder *const builder,
+                     struct mempool *const dest_mp,
+                     bn_data *const bn) {
+        LEAFENTRY old_le = m_left_bn->get_le_from_klpair(&klpair);
+        size_t le_size = leafentry_memsize(old_le);
+
+        void *new_le = toku_mempool_malloc(dest_mp, le_size, 1);
+        paranoid_invariant_notnull(new_le);
+        memcpy(new_le, old_le, le_size);
+        size_t le_offset = toku_mempool_get_offset_from_pointer_and_base(dest_mp, new_le);
+        size_t keylen = keylen_from_klpair_len(klpair_len);
+        builder->append(klpair_dmtwriter(keylen, le_offset, klpair.key));
+
+        bn->add_key(keylen);
+    }
+
+    int move_leafentry(const uint32_t klpair_len, const klpair_struct &klpair, const uint32_t idx) {
+        m_left_bn->remove_key(keylen_from_klpair_len(klpair_len));
+
+        if (idx < m_split_at) {
+            copy_klpair(klpair_len, klpair, m_left_builder, left_dest_mp(), m_left_bn);
+        } else {
+            copy_klpair(klpair_len, klpair, m_right_builder, right_dest_mp(), m_right_bn);
+        }
+        return 0;
+    }
+
+  public:
+    split_klpairs_extra(bn_data *const left_bn, bn_data *const right_bn,
+                        klpair_dmt_t::builder *const left_builder,
+                        klpair_dmt_t::builder *const right_builder,
+                        struct mempool *const left_new_mp,
+                        uint32_t split_at)
+        : m_left_bn(left_bn),
+          m_right_bn(right_bn),
+          m_left_builder(left_builder),
+          m_right_builder(right_builder),
+          m_left_dest_mp(left_new_mp),
+          m_split_at(split_at) {}
+    static int cb(const uint32_t klpair_len, const klpair_struct &klpair, const uint32_t idx, split_klpairs_extra *const thisp) {
+        return thisp->move_leafentry(klpair_len, klpair, idx);
+    }
+};
+
+void bn_data::split_klpairs(
+     bn_data* right_bd,
+     uint32_t split_at //lower bound inclusive for right_bd
      )
-//Effect: move leafentries in the range [lbi, ube) from this to src_omt to newly created dest_omt
 {
-    paranoid_invariant(lbi < ube);
+    // We use move_leafentries_to during a split, and the split algorithm should never call this
-    paranoid_invariant(ube <= omt_size());
+    // if it's splitting on a boundary, so there must be some leafentries in the range to move.
-    KLPAIR *XMALLOC_N(ube-lbi, newklpointers);    // create new omt
+    paranoid_invariant(split_at < num_klpairs());
 
-    size_t mpsize = toku_mempool_get_used_space(&m_buffer_mempool);   // overkill, but safe
+    right_bd->init_zero();
-    struct mempool *dest_mp = &dest_bd->m_buffer_mempool;
-    struct mempool *src_mp  = &m_buffer_mempool;
-    toku_mempool_construct(dest_mp, mpsize);
 
-    uint32_t i = 0;
+    size_t mpsize = toku_mempool_get_used_size(&m_buffer_mempool);   // overkill, but safe
-    for (i = lbi; i < ube; i++) {
-        KLPAIR curr_kl;
-        m_buffer.fetch(i, &curr_kl);
 
-        size_t kl_size = klpair_size(curr_kl);
+    struct mempool new_left_mp;
-        KLPAIR new_kl = NULL;
+    toku_mempool_construct(&new_left_mp, mpsize);
-        CAST_FROM_VOIDP(new_kl, toku_mempool_malloc(dest_mp, kl_size, 1));
-        memcpy(new_kl, curr_kl, kl_size);
-        newklpointers[i-lbi] = new_kl;
-        toku_mempool_mfree(src_mp, curr_kl, kl_size);
-    }
 
-    dest_bd->m_buffer.create_steal_sorted_array(&newklpointers, ube-lbi, ube-lbi);
+    struct mempool *right_mp = &right_bd->m_buffer_mempool;
-    // now remove the elements from src_omt
+    toku_mempool_construct(right_mp, mpsize);
-    for (i=ube-1; i >= lbi; i--) {
+
-        m_buffer.delete_at(i);
+    klpair_dmt_t::builder left_dmt_builder;
-    }
+    left_dmt_builder.create(split_at, m_disksize_of_keys);  // overkill, but safe (builder will realloc at the end)
+
+    klpair_dmt_t::builder right_dmt_builder;
+    right_dmt_builder.create(num_klpairs() - split_at, m_disksize_of_keys);  // overkill, but safe (builder will realloc at the end)
+
+    split_klpairs_extra extra(this, right_bd, &left_dmt_builder, &right_dmt_builder, &new_left_mp, split_at);
+
+    int r = m_buffer.iterate<split_klpairs_extra, split_klpairs_extra::cb>(&extra);
+    invariant_zero(r);
+
+    m_buffer.destroy();
+    toku_mempool_destroy(&m_buffer_mempool);
+
+    m_buffer_mempool = new_left_mp;
+
+    left_dmt_builder.build(&m_buffer);
+    right_dmt_builder.build(&right_bd->m_buffer);
+
+    // Potentially shrink memory pool for destination.
+    // We overallocated ("overkill") above
+    struct mempool *const left_mp = &m_buffer_mempool;
+    paranoid_invariant_zero(toku_mempool_get_frag_size(left_mp));
+    toku_mempool_realloc_larger(left_mp, toku_mempool_get_used_size(left_mp));
+    paranoid_invariant_zero(toku_mempool_get_frag_size(right_mp));
+    toku_mempool_realloc_larger(right_mp, toku_mempool_get_used_size(right_mp));
 }
 
 uint64_t bn_data::get_disk_size() {
-    return toku_mempool_get_used_space(&m_buffer_mempool);
+    return m_disksize_of_keys +
+           toku_mempool_get_used_size(&m_buffer_mempool);
 }
 
+struct verify_le_in_mempool_state {
+    size_t offset_limit;
+    class bn_data *bd;
+};
+
+static int verify_le_in_mempool (const uint32_t, klpair_struct *klpair, const uint32_t idx UU(), struct verify_le_in_mempool_state * const state) {
+    invariant(klpair->le_offset < state->offset_limit);
+
+    LEAFENTRY le = state->bd->get_le_from_klpair(klpair);
+    uint32_t size = leafentry_memsize(le);
+
+    size_t end_offset = klpair->le_offset+size;
+
+    invariant(end_offset <= state->offset_limit);
+    return 0;
+}
+
+//This is a debug-only (paranoid) verification.
+//Verifies the dmt is valid, and all leafentries are entirely in the mempool's memory.
 void bn_data::verify_mempool(void) {
-    // TODO: implement something
+    //Verify the dmt itself <- paranoid and slow
+    m_buffer.verify();
+
+    verify_le_in_mempool_state state = { .offset_limit = toku_mempool_get_offset_limit(&m_buffer_mempool), .bd = this };
+    //Verify every leafentry pointed to by the keys in the dmt are fully inside the mempool
+    m_buffer.iterate_ptr< decltype(state), verify_le_in_mempool >(&state);
 }
 
-uint32_t bn_data::omt_size(void) const {
+uint32_t bn_data::num_klpairs(void) const {
     return m_buffer.size();
 }
 
@@ -375,40 +631,54 @@ void bn_data::destroy(void) {
     // The buffer may have been freed already, in some cases.
     m_buffer.destroy();
     toku_mempool_destroy(&m_buffer_mempool);
+    m_disksize_of_keys = 0;
 }
 
-//TODO: Splitting key/val requires changing this
+void bn_data::set_contents_as_clone_of_sorted_array(
-void bn_data::replace_contents_with_clone_of_sorted_array(
     uint32_t num_les,
     const void** old_key_ptrs,
     uint32_t* old_keylens,
-    LEAFENTRY* old_les, 
+    LEAFENTRY* old_les,
-    size_t *le_sizes, 
+    size_t *le_sizes,
-    size_t mempool_size
+    size_t total_key_size,
-    ) 
+    size_t total_le_size
+    )
 {
-    toku_mempool_construct(&m_buffer_mempool, mempool_size);
+    //Enforce "just created" invariant.
-    KLPAIR *XMALLOC_N(num_les, le_array);
+    paranoid_invariant_zero(m_disksize_of_keys);
-    for (uint32_t idx = 0; idx < num_les; idx++) {
+    paranoid_invariant_zero(num_klpairs());
-        KLPAIR new_kl = (KLPAIR)toku_mempool_malloc(
+    paranoid_invariant_null(toku_mempool_get_base(&m_buffer_mempool));
-            &m_buffer_mempool,
+    paranoid_invariant_zero(toku_mempool_get_size(&m_buffer_mempool));
-            le_sizes[idx] + old_keylens[idx] + sizeof(uint32_t),
+
-            1); // point to new location
+    toku_mempool_construct(&m_buffer_mempool, total_le_size);
-        new_kl->keylen = old_keylens[idx];
-        memcpy(new_kl->key_le, old_key_ptrs[idx], new_kl->keylen);
-        memcpy(get_le_from_klpair(new_kl), old_les[idx], le_sizes[idx]);
-        CAST_FROM_VOIDP(le_array[idx], new_kl);
-    }
-    //TODO: Splitting key/val requires changing this; keys are stored in old omt.. cannot delete it yet?
     m_buffer.destroy();
-    m_buffer.create_steal_sorted_array(&le_array, num_les, num_les);
+    m_disksize_of_keys = 0;
+
+    klpair_dmt_t::builder dmt_builder;
+    dmt_builder.create(num_les, total_key_size);
+
+    for (uint32_t idx = 0; idx < num_les; idx++) {
+        void* new_le = toku_mempool_malloc(&m_buffer_mempool, le_sizes[idx], 1);
+        paranoid_invariant_notnull(new_le);
+        memcpy(new_le, old_les[idx], le_sizes[idx]);
+        size_t le_offset = toku_mempool_get_offset_from_pointer_and_base(&m_buffer_mempool, new_le);
+        dmt_builder.append(klpair_dmtwriter(old_keylens[idx], le_offset, old_key_ptrs[idx]));
+        add_key(old_keylens[idx]);
+    }
+    dmt_builder.build(&this->m_buffer);
+}
+
+LEAFENTRY bn_data::get_le_from_klpair(const klpair_struct *klpair) const {
+    void * ptr = toku_mempool_get_pointer_from_base_and_offset(&this->m_buffer_mempool, klpair->le_offset);
+    LEAFENTRY CAST_FROM_VOIDP(le, ptr);
+    return le;
 }
 
 
 // get info about a single leafentry by index
 int bn_data::fetch_le(uint32_t idx, LEAFENTRY *le) {
-    KLPAIR klpair = NULL;
+    klpair_struct* klpair = nullptr;
-    int r = m_buffer.fetch(idx, &klpair);
+    int r = m_buffer.fetch(idx, nullptr, &klpair);
     if (r == 0) {
         *le = get_le_from_klpair(klpair);
     }
@@ -416,59 +686,41 @@ int bn_data::fetch_le(uint32_t idx, LEAFENTRY *le) {
 }
 
 int bn_data::fetch_klpair(uint32_t idx, LEAFENTRY *le, uint32_t *len, void** key) {
-    KLPAIR klpair = NULL;
+    klpair_struct* klpair = nullptr;
-    int r = m_buffer.fetch(idx, &klpair);
+    uint32_t klpair_len;
+    int r = m_buffer.fetch(idx, &klpair_len, &klpair);
     if (r == 0) {
-        *len = klpair->keylen;
+        *len = keylen_from_klpair_len(klpair_len);
-        *key = klpair->key_le;
+        *key = klpair->key;
         *le = get_le_from_klpair(klpair);
     }
     return r;
 }
 
 int bn_data::fetch_klpair_disksize(uint32_t idx, size_t *size) {
-    KLPAIR klpair = NULL;
+    klpair_struct* klpair = nullptr;
-    int r = m_buffer.fetch(idx, &klpair);
+    uint32_t klpair_len;
+    int r = m_buffer.fetch(idx, &klpair_len, &klpair);
     if (r == 0) {
-        *size = klpair_disksize(klpair);
+        *size = klpair_disksize(klpair_len, klpair);
     }
     return r;
 }
 
-int bn_data::fetch_le_key_and_len(uint32_t idx, uint32_t *len, void** key) {
+int bn_data::fetch_key_and_len(uint32_t idx, uint32_t *len, void** key) {
-    KLPAIR klpair = NULL;
+    klpair_struct* klpair = nullptr;
-    int r = m_buffer.fetch(idx, &klpair);
+    uint32_t klpair_len;
+    int r = m_buffer.fetch(idx, &klpair_len, &klpair);
     if (r == 0) {
-        *len = klpair->keylen;
+        *len = keylen_from_klpair_len(klpair_len);
-        *key = klpair->key_le;
+        *key = klpair->key;
     }
     return r;
 }
 
-
-struct mp_pair {
-    void* orig_base;
-    void* new_base;
-    klpair_omt_t* omt;
-};
-
-static int fix_mp_offset(const KLPAIR &klpair, const uint32_t idx,  struct mp_pair * const p) {
-    char* old_value = (char *) klpair;
-    char *new_value = old_value - (char *)p->orig_base + (char *)p->new_base;
-    p->omt->set_at((KLPAIR)new_value, idx);
-    return 0;
-}
-
 void bn_data::clone(bn_data* orig_bn_data) {
     toku_mempool_clone(&orig_bn_data->m_buffer_mempool, &m_buffer_mempool);
     m_buffer.clone(orig_bn_data->m_buffer);
-    struct mp_pair p;
+    this->m_disksize_of_keys = orig_bn_data->m_disksize_of_keys;
-    p.orig_base = toku_mempool_get_base(&orig_bn_data->m_buffer_mempool);
-    p.new_base = toku_mempool_get_base(&m_buffer_mempool);
-    p.omt = &m_buffer;
-
-    int r = m_buffer.iterate_on_range<decltype(p), fix_mp_offset>(0, omt_size(), &p);
-    invariant_zero(r);
 }
 
-

storage/tokudb/ft-index/ft/bndata.h

@@ -91,166 +91,296 @@ PATENT RIGHTS GRANT:
 
 #pragma once
 
-#include <util/omt.h>
-#include "leafentry.h"
 #include <util/mempool.h>
+#include "wbuf.h"
+#include <util/dmt.h>
+#include "leafentry.h"
 
-#if 0 //for implementation
+// Key/leafentry pair stored in a dmt.  The key is inlined, the offset (in leafentry mempool) is stored for the leafentry.
-static int
-UU() verify_in_mempool(OMTVALUE lev, uint32_t UU(idx), void *mpv)
-{
-    LEAFENTRY CAST_FROM_VOIDP(le, lev);
-    struct mempool *CAST_FROM_VOIDP(mp, mpv);
-    int r = toku_mempool_inrange(mp, le, leafentry_memsize(le));
-    lazy_assert(r);
-    return 0;
-}
-            toku_omt_iterate(bn->buffer, verify_in_mempool, &bn->buffer_mempool);
-
-#endif
-
 struct klpair_struct {
-    uint32_t keylen;
+    uint32_t le_offset;  //Offset of leafentry (in leafentry mempool)
-    uint8_t key_le[0]; // key, followed by le
+    uint8_t key[0]; // key, followed by le
 };
 
-typedef struct klpair_struct *KLPAIR;
+static constexpr uint32_t keylen_from_klpair_len(const uint32_t klpair_len) {
-
+    return klpair_len - __builtin_offsetof(klpair_struct, key);
-static inline LEAFENTRY get_le_from_klpair(KLPAIR klpair){
-    uint32_t keylen = klpair->keylen;
-    LEAFENTRY le = (LEAFENTRY)(klpair->key_le + keylen);
-    return le;
 }
 
-template<typename omtcmp_t,
+
-         int (*h)(const DBT &, const omtcmp_t &)>
+static_assert(__builtin_offsetof(klpair_struct, key) == 1*sizeof(uint32_t), "klpair alignment issues");
-static int wrappy_fun_find(const KLPAIR &klpair, const omtcmp_t &extra) {
+static_assert(__builtin_offsetof(klpair_struct, key) == sizeof(klpair_struct), "klpair size issues");
-    //TODO: kill this function when we split, and/or use toku_fill_dbt
+
+// A wrapper for the heaviside function provided to dmt->find*.
+// Needed because the heaviside functions provided to bndata do not know about the internal types.
+// Alternative to this wrapper is to expose accessor functions and rewrite all the external heaviside functions.
+template<typename dmtcmp_t,
+         int (*h)(const DBT &, const dmtcmp_t &)>
+static int klpair_find_wrapper(const uint32_t klpair_len, const klpair_struct &klpair, const dmtcmp_t &extra) {
     DBT kdbt;
-    kdbt.data = klpair->key_le;
+    kdbt.data = const_cast<void*>(reinterpret_cast<const void*>(klpair.key));
-    kdbt.size = klpair->keylen;
+    kdbt.size = keylen_from_klpair_len(klpair_len);
     return h(kdbt, extra);
 }
 
+template<typename inner_iterate_extra_t>
+struct klpair_iterate_extra {
+    public:
+    inner_iterate_extra_t *inner;
+    const class bn_data * bd;
+};
+
+// A wrapper for the high-order function provided to dmt->iterate*
+// Needed because the heaviside functions provided to bndata do not know about the internal types.
+// Alternative to this wrapper is to expose accessor functions and rewrite all the external heaviside functions.
 template<typename iterate_extra_t,
-         int (*h)(const void * key, const uint32_t keylen, const LEAFENTRY &, const uint32_t idx, iterate_extra_t *const)>
+         int (*f)(const void * key, const uint32_t keylen, const LEAFENTRY &, const uint32_t idx, iterate_extra_t *const)>
-static int wrappy_fun_iterate(const KLPAIR &klpair, const uint32_t idx, iterate_extra_t *const extra) {
+static int klpair_iterate_wrapper(const uint32_t klpair_len, const klpair_struct &klpair, const uint32_t idx, klpair_iterate_extra<iterate_extra_t> *const extra) {
-    uint32_t keylen = klpair->keylen;
+    const void* key = &klpair.key;
-    void* key = klpair->key_le;
+    LEAFENTRY le = extra->bd->get_le_from_klpair(&klpair);
-    LEAFENTRY le = get_le_from_klpair(klpair);
+    return f(key, keylen_from_klpair_len(klpair_len), le, idx, extra->inner);
-    return h(key, keylen, le, idx, extra);
 }
 
-typedef toku::omt<KLPAIR> klpair_omt_t;
+
+namespace toku {
+// dmt writer for klpair_struct
+class klpair_dmtwriter {
+    public:
+        // Return the size needed for the klpair_struct that this dmtwriter represents
+        size_t get_size(void) const {
+            return sizeof(klpair_struct) + this->keylen;
+        }
+        // Write the klpair_struct this dmtwriter represents to a destination
+        void write_to(klpair_struct *const dest) const {
+            dest->le_offset = this->le_offset;
+            memcpy(dest->key, this->keyp, this->keylen);
+        }
+
+        klpair_dmtwriter(uint32_t _keylen, uint32_t _le_offset, const void* _keyp)
+            : keylen(_keylen), le_offset(_le_offset), keyp(_keyp) {}
+        klpair_dmtwriter(const uint32_t klpair_len, klpair_struct *const src)
+            : keylen(keylen_from_klpair_len(klpair_len)), le_offset(src->le_offset), keyp(src->key) {}
+    private:
+        const uint32_t keylen;
+        const uint32_t le_offset;
+        const void* keyp;
+};
+}
+
+typedef toku::dmt<klpair_struct, klpair_struct*, toku::klpair_dmtwriter> klpair_dmt_t;
 // This class stores the data associated with a basement node
 class bn_data {
 public:
+    // Initialize an empty bn_data _without_ a dmt backing.
+    // Externally only used for deserialization.
     void init_zero(void);
+
+    // Initialize an empty bn_data _with_ a dmt
     void initialize_empty(void);
-    void initialize_from_data(uint32_t num_entries, unsigned char *buf, uint32_t data_size);
+
-    // globals
+    // Deserialize a bn_data from rbuf.
+    // This is the entry point for deserialization.
+    void deserialize_from_rbuf(uint32_t num_entries, struct rbuf *rb, uint32_t data_size, uint32_t version);
+
+    // Retrieve the memory footprint of this basement node.
+    // May over or under count: see Tokutek/ft-index#136
+    // Also see dmt's implementation.
     uint64_t get_memory_size(void);
+
+    // Get the serialized size of this basement node.
     uint64_t get_disk_size(void);
+
+    // Perform (paranoid) verification that all leafentries are fully contained within the mempool
     void verify_mempool(void);
 
-    // Interact with "omt"
+    // size() of key dmt
-    uint32_t omt_size(void) const;
+    uint32_t num_klpairs(void) const;
 
+    // iterate() on key dmt (and associated leafentries)
     template<typename iterate_extra_t,
              int (*f)(const void * key, const uint32_t keylen, const LEAFENTRY &, const uint32_t, iterate_extra_t *const)>
-    int omt_iterate(iterate_extra_t *const iterate_extra) const {
+    int iterate(iterate_extra_t *const iterate_extra) const {
-        return omt_iterate_on_range<iterate_extra_t, f>(0, omt_size(), iterate_extra);
+        return iterate_on_range<iterate_extra_t, f>(0, num_klpairs(), iterate_extra);
     }
 
+    // iterate_on_range() on key dmt (and associated leafentries)
     template<typename iterate_extra_t,
              int (*f)(const void * key, const uint32_t keylen, const LEAFENTRY &, const uint32_t, iterate_extra_t *const)>
-    int omt_iterate_on_range(const uint32_t left, const uint32_t right, iterate_extra_t *const iterate_extra) const {
+    int iterate_on_range(const uint32_t left, const uint32_t right, iterate_extra_t *const iterate_extra) const {
-        return m_buffer.iterate_on_range< iterate_extra_t, wrappy_fun_iterate<iterate_extra_t, f> >(left, right, iterate_extra);
+        klpair_iterate_extra<iterate_extra_t> klpair_extra = { iterate_extra, this };
+        return m_buffer.iterate_on_range< klpair_iterate_extra<iterate_extra_t>, klpair_iterate_wrapper<iterate_extra_t, f> >(left, right, &klpair_extra);
     }
 
-    template<typename omtcmp_t,
+    // find_zero() on key dmt
-             int (*h)(const DBT &, const omtcmp_t &)>
+    template<typename dmtcmp_t,
-    int find_zero(const omtcmp_t &extra, LEAFENTRY *const value, void** key, uint32_t* keylen, uint32_t *const idxp) const {
+             int (*h)(const DBT &, const dmtcmp_t &)>
-        KLPAIR klpair = NULL;
+    int find_zero(const dmtcmp_t &extra, LEAFENTRY *const value, void** key, uint32_t* keylen, uint32_t *const idxp) const {
-        int r = m_buffer.find_zero< omtcmp_t, wrappy_fun_find<omtcmp_t, h> >(extra, &klpair, idxp);
+        klpair_struct* klpair = nullptr;
+        uint32_t klpair_len;
+        int r = m_buffer.find_zero< dmtcmp_t, klpair_find_wrapper<dmtcmp_t, h> >(extra, &klpair_len, &klpair, idxp);
         if (r == 0) {
             if (value) {
                 *value = get_le_from_klpair(klpair);
             }
             if (key) {
-                paranoid_invariant(keylen != NULL);
+                paranoid_invariant_notnull(keylen);
-                *key = klpair->key_le;
+                *key = klpair->key;
-                *keylen = klpair->keylen;
+                *keylen = keylen_from_klpair_len(klpair_len);
             }
             else {
-                paranoid_invariant(keylen == NULL);
+                paranoid_invariant_null(keylen);
             }
         }
         return r;
     }
 
-    template<typename omtcmp_t,
+    // find() on key dmt (and associated leafentries)
-             int (*h)(const DBT &, const omtcmp_t &)>
+    template<typename dmtcmp_t,
-    int find(const omtcmp_t &extra, int direction, LEAFENTRY *const value, void** key, uint32_t* keylen, uint32_t *const idxp) const {
+             int (*h)(const DBT &, const dmtcmp_t &)>
-        KLPAIR klpair = NULL;
+    int find(const dmtcmp_t &extra, int direction, LEAFENTRY *const value, void** key, uint32_t* keylen, uint32_t *const idxp) const {
-        int r = m_buffer.find< omtcmp_t, wrappy_fun_find<omtcmp_t, h> >(extra, direction, &klpair, idxp);
+        klpair_struct* klpair = nullptr;
+        uint32_t klpair_len;
+        int r = m_buffer.find< dmtcmp_t, klpair_find_wrapper<dmtcmp_t, h> >(extra, direction, &klpair_len, &klpair, idxp);
         if (r == 0) {
             if (value) {
                 *value = get_le_from_klpair(klpair);
             }
             if (key) {
-                paranoid_invariant(keylen != NULL);
+                paranoid_invariant_notnull(keylen);
-                *key = klpair->key_le;
+                *key = klpair->key;
-                *keylen = klpair->keylen;
+                *keylen = keylen_from_klpair_len(klpair_len);
             }
             else {
-                paranoid_invariant(keylen == NULL);
+                paranoid_invariant_null(keylen);
             }
         }
         return r;
     }
 
-    // get info about a single leafentry by index
+    // Fetch leafentry by index
+    __attribute__((__nonnull__))
     int fetch_le(uint32_t idx, LEAFENTRY *le);
+    // Fetch (leafentry, key, keylen) by index
+    __attribute__((__nonnull__))
     int fetch_klpair(uint32_t idx, LEAFENTRY *le, uint32_t *len, void** key);
+    // Fetch (serialized size of leafentry, key, and keylen) by index
+    __attribute__((__nonnull__))
     int fetch_klpair_disksize(uint32_t idx, size_t *size);
-    int fetch_le_key_and_len(uint32_t idx, uint32_t *len, void** key);
+    // Fetch (key, keylen) by index
+    __attribute__((__nonnull__))
+    int fetch_key_and_len(uint32_t idx, uint32_t *len, void** key);
 
-    // Interact with another bn_data
+    // Move leafentries (and associated key/keylens) from this basement node to dest_bd
-    void move_leafentries_to(BN_DATA dest_bd,
+    // Moves indexes [lbi-ube)
-                                      uint32_t lbi, //lower bound inclusive
+    __attribute__((__nonnull__))
-                                      uint32_t ube //upper bound exclusive
+    void split_klpairs(bn_data* dest_bd, uint32_t first_index_for_dest);
-                                      );
 
+    // Destroy this basement node and free memory.
     void destroy(void);
 
-    // Replaces contents, into brand new mempool.
+    // Uses sorted array as input for this basement node.
-    // Returns old mempool base, expects caller to free it.
+    // Expects this to be a basement node just initialized with initialize_empty()
-    void replace_contents_with_clone_of_sorted_array(
+    void set_contents_as_clone_of_sorted_array(
         uint32_t num_les,
         const void** old_key_ptrs,
         uint32_t* old_keylens,
         LEAFENTRY* old_les,
         size_t *le_sizes,
-        size_t mempool_size
+        size_t total_key_size,
+        size_t total_le_size
         );
 
+    // Make this basement node a clone of orig_bn_data.
+    // orig_bn_data still owns all its memory (dmt, mempool)
+    // this basement node will have a new dmt, mempool containing same data.
     void clone(bn_data* orig_bn_data);
+
+    // Delete klpair index idx with provided keylen and old leafentry with size old_le_size
     void delete_leafentry (
         uint32_t idx,
         uint32_t keylen,
         uint32_t old_le_size
         );
-    void get_space_for_overwrite(uint32_t idx, const void* keyp, uint32_t keylen, uint32_t old_size, uint32_t new_size, LEAFENTRY* new_le_space);
-    void get_space_for_insert(uint32_t idx, const void* keyp, uint32_t keylen, size_t size, LEAFENTRY* new_le_space);
-private:
-    // Private functions
-    KLPAIR mempool_malloc_from_omt(size_t size, void **maybe_free);
-    void omt_compress_kvspace(size_t added_size, void **maybe_free);
 
-    klpair_omt_t m_buffer;                     // pointers to individual leaf entries
+    // Allocates space in the mempool to store a new leafentry.
+    // This may require reorganizing the mempool and updating the dmt.
+    __attribute__((__nonnull__))
+    void get_space_for_overwrite(uint32_t idx, const void* keyp, uint32_t keylen, uint32_t old_size, uint32_t new_size, LEAFENTRY* new_le_space, void **const maybe_free);
+
+    // Allocates space in the mempool to store a new leafentry
+    // and inserts a new key into the dmt
+    // This may require reorganizing the mempool and updating the dmt.
+    __attribute__((__nonnull__))
+    void get_space_for_insert(uint32_t idx, const void* keyp, uint32_t keylen, size_t size, LEAFENTRY* new_le_space, void **const maybe_free);
+
+    // Gets a leafentry given a klpair from this basement node.
+    LEAFENTRY get_le_from_klpair(const klpair_struct *klpair) const;
+
+    void serialize_to_wbuf(struct wbuf *const wb);
+
+    // Prepares this basement node for serialization.
+    // Must be called before serializing this basement node.
+    // Between calling prepare_to_serialize and actually serializing, the basement node may not be modified
+    void prepare_to_serialize(void);
+
+    // Serialize the basement node header to a wbuf
+    // Requires prepare_to_serialize() to have been called first.
+    void serialize_header(struct wbuf *wb) const;
+
+    // Serialize all keys and leafentries to a wbuf
+    // Requires prepare_to_serialize() (and serialize_header()) has been called first.
+    // Currently only supported when all keys are fixed-length.
+    void serialize_rest(struct wbuf *wb) const;
+
+    static const uint32_t HEADER_LENGTH = 0
+        + sizeof(uint32_t) // key_data_size
+        + sizeof(uint32_t) // val_data_size
+        + sizeof(uint32_t) // fixed_key_length
+        + sizeof(uint8_t) // all_keys_same_length
+        + sizeof(uint8_t) // keys_vals_separate
+        + 0;
+private:
+
+    // split_klpairs_extra should be a local class in split_klpairs, but
+    // the dmt template parameter for iterate needs linkage, so it has to be a
+    // separate class, but we want it to be able to call e.g. add_key
+    friend class split_klpairs_extra;
+
+    // Allocates space in the mempool.
+    // If there is insufficient space, the mempool is enlarged and leafentries may be shuffled to reduce fragmentation.
+    // If shuffling happens, the offsets stored in the dmt are updated.
+    LEAFENTRY mempool_malloc_and_update_dmt(size_t size, void **maybe_free);
+
+    // Change the size of the mempool to support what is already in it, plus added_size.
+    // possibly "compress" by shuffling leafentries around to reduce fragmentation to 0.
+    // If fragmentation is already 0 and force_compress is not true, shuffling may be skipped.
+    // If shuffling happens, leafentries will be stored in the mempool in sorted order.
+    void dmt_compress_kvspace(size_t added_size, void **maybe_free, bool force_compress);
+
+    // Note that a key was added (for maintaining disk-size of this basement node)
+    void add_key(uint32_t keylen);
+
+    // Note that multiple keys were added (for maintaining disk-size of this basement node)
+    void add_keys(uint32_t n_keys, uint32_t combined_klpair_len);
+
+    // Note that a key was removed (for maintaining disk-size of this basement node)
+    void remove_key(uint32_t keylen);
+
+    klpair_dmt_t m_buffer;                     // pointers to individual leaf entries
     struct mempool m_buffer_mempool;  // storage for all leaf entries
 
     friend class bndata_bugfix_test;
+
+    // Get the serialized size of a klpair.
+    // As of Jan 14, 2014, serialized size of a klpair is independent of whether this basement node has fixed-length keys.
+    uint32_t klpair_disksize(const uint32_t klpair_len, const klpair_struct *klpair) const;
+
+    // The disk/memory size of all keys.  (Note that the size of memory for the leafentries is maintained by m_buffer_mempool)
+    size_t m_disksize_of_keys;
+
+    // Deserialize this basement node from rbuf
+    // all keys will be first followed by all leafentries (both in sorted order)
+    void initialize_from_separate_keys_and_vals(uint32_t num_entries, struct rbuf *rb, uint32_t data_size, uint32_t version,
+                                                uint32_t key_data_size, uint32_t val_data_size, bool all_keys_same_length,
+                                                uint32_t fixed_klpair_length);
 };
 

storage/tokudb/ft-index/ft/cachetable-internal.h

@@ -446,6 +446,8 @@ public:
     FILENUM m_next_filenum_to_use;
     uint32_t m_next_hash_id_to_use;
     toku_pthread_rwlock_t m_lock; // this field is publoc so we are still POD
+    toku::omt<CACHEFILE> m_active_filenum;
+    toku::omt<CACHEFILE> m_active_fileid;
 private:    
     CACHEFILE find_cachefile_in_list_unlocked(CACHEFILE start, struct fileid* fileid);
 };

storage/tokudb/ft-index/ft/cachetable.cc

@@ -370,7 +370,7 @@ toku_cachetable_set_env_dir(CACHETABLE ct, const char *env_dir) {
 
 // What cachefile goes with particular iname (iname relative to env)?
 // The transaction that is adding the reference might not have a reference
-// to the brt, therefore the cachefile might be closing.
+// to the ft, therefore the cachefile might be closing.
 // If closing, we want to return that it is not there, but must wait till after
 // the close has finished.
 // Once the close has finished, there must not be a cachefile with that name
@@ -380,7 +380,7 @@ int toku_cachefile_of_iname_in_env (CACHETABLE ct, const char *iname_in_env, CAC
 }
 
 // What cachefile goes with particular fd?
-// This function can only be called if the brt is still open, so file must 
+// This function can only be called if the ft is still open, so file must 
 // still be open
 int toku_cachefile_of_filenum (CACHETABLE ct, FILENUM filenum, CACHEFILE *cf) {
     return ct->cf_list.cachefile_of_filenum(filenum, cf);
@@ -564,10 +564,10 @@ void toku_cachefile_close(CACHEFILE *cfp, bool oplsn_valid, LSN oplsn) {
     // that do not persist across opens/closes
     bjm_destroy(cf->bjm);
     cf->bjm = NULL;
-    cf->filenum = FILENUM_NONE;
 
     // remove the cf from the list of active cachefiles
     ct->cf_list.remove_cf(cf);
+    cf->filenum = FILENUM_NONE;
 
     // Unlink the file if the bit was set
     if (cf->unlink_on_close) {
@@ -642,7 +642,7 @@ static void cachetable_free_pair(PAIR p) {
     cachetable_evictions++;
     PAIR_ATTR new_attr = p->attr;
     // Note that flush_callback is called with write_me false, so the only purpose of this 
-    // call is to tell the brt layer to evict the node (keep_me is false).
+    // call is to tell the ft layer to evict the node (keep_me is false).
     // Also, because we have already removed the PAIR from the cachetable in 
     // cachetable_remove_pair, we cannot pass in p->cachefile and p->cachefile->fd
     // for the first two parameters, as these may be invalid (#5171), so, we
@@ -1302,8 +1302,6 @@ void toku_cachetable_pf_pinned_pair(
     pair_unlock(p);
 }
 
-
-// NOW A TEST ONLY FUNCTION!!!
 int toku_cachetable_get_and_pin (
     CACHEFILE cachefile, 
     CACHEKEY key, 
@@ -1573,7 +1571,7 @@ exit:
     return try_again;
 }
 
-int toku_cachetable_get_and_pin_with_dep_pairs_batched (
+int toku_cachetable_get_and_pin_with_dep_pairs (
     CACHEFILE cachefile,
     CACHEKEY key,
     uint32_t fullhash,
@@ -1766,43 +1764,6 @@ got_value:
     return 0;
 }
 
-int toku_cachetable_get_and_pin_with_dep_pairs (
-    CACHEFILE cachefile,
-    CACHEKEY key,
-    uint32_t fullhash,
-    void**value,
-    long *sizep,
-    CACHETABLE_WRITE_CALLBACK write_callback,
-    CACHETABLE_FETCH_CALLBACK fetch_callback,
-    CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback,
-    CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback,
-    pair_lock_type lock_type,
-    void* read_extraargs, // parameter for fetch_callback, pf_req_callback, and pf_callback
-    uint32_t num_dependent_pairs, // number of dependent pairs that we may need to checkpoint
-    PAIR* dependent_pairs,
-    enum cachetable_dirty* dependent_dirty // array stating dirty/cleanness of dependent pairs
-    )
-// See cachetable.h
-{
-    int r = toku_cachetable_get_and_pin_with_dep_pairs_batched(
-        cachefile,
-        key,
-        fullhash,
-        value,
-        sizep,
-        write_callback,
-        fetch_callback,
-        pf_req_callback,
-        pf_callback,
-        lock_type,
-        read_extraargs,
-        num_dependent_pairs,
-        dependent_pairs,
-        dependent_dirty
-        );
-    return r;
-}
-
 // Lookup a key in the cachetable.  If it is found and it is not being written, then
 // acquire a read lock on the pair, update the LRU list, and return sucess.
 //
@@ -2048,7 +2009,7 @@ maybe_pin_pair(
     return retval;
 }
 
-int toku_cachetable_get_and_pin_nonblocking_batched(
+int toku_cachetable_get_and_pin_nonblocking(
     CACHEFILE cf,
     CACHEKEY key,
     uint32_t fullhash,
@@ -2200,40 +2161,6 @@ try_again:
     abort();
 }
 
-int toku_cachetable_get_and_pin_nonblocking (
-    CACHEFILE cf,
-    CACHEKEY key,
-    uint32_t fullhash,
-    void**value,
-    long* sizep,
-    CACHETABLE_WRITE_CALLBACK write_callback,
-    CACHETABLE_FETCH_CALLBACK fetch_callback,
-    CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback,
-    CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback,
-    pair_lock_type lock_type,
-    void *read_extraargs,
-    UNLOCKERS unlockers
-    )
-// See cachetable.h.
-{
-    int r = 0;
-    r = toku_cachetable_get_and_pin_nonblocking_batched(
-        cf,
-        key,
-        fullhash,
-        value,
-        sizep,
-        write_callback,
-        fetch_callback,
-        pf_req_callback,
-        pf_callback,
-        lock_type,
-        read_extraargs,
-        unlockers
-    );
-    return r;
-}
-
 struct cachefile_prefetch_args {
     PAIR p;
     CACHETABLE_FETCH_CALLBACK fetch_callback;
@@ -4750,9 +4677,13 @@ void cachefile_list::init() {
     m_next_filenum_to_use.fileid = 0;
     m_next_hash_id_to_use = 0;
     toku_pthread_rwlock_init(&m_lock, NULL);
+    m_active_filenum.create();
+    m_active_fileid.create();
 }
 
 void cachefile_list::destroy() {
+    m_active_filenum.destroy();
+    m_active_fileid.destroy();
     toku_pthread_rwlock_destroy(&m_lock);
 }
 
@@ -4804,6 +4735,21 @@ int cachefile_list::cachefile_of_filenum(FILENUM filenum, CACHEFILE *cf) {
     return r;
 }
 
+static int cachefile_find_by_filenum(const CACHEFILE &a_cf, const FILENUM &b) {
+    const FILENUM a = a_cf->filenum;
+    if (a.fileid < b.fileid) {
+        return -1;
+    } else if (a.fileid == b.fileid) {
+        return 0;
+    } else {
+        return 1;
+    }
+}
+
+static int cachefile_find_by_fileid(const CACHEFILE &a_cf, const struct fileid &b) {
+    return toku_fileid_cmp(a_cf->fileid, b);
+}
+
 void cachefile_list::add_cf_unlocked(CACHEFILE cf) {
     invariant(cf->next == NULL);
     invariant(cf->prev == NULL);
@@ -4813,6 +4759,12 @@ void cachefile_list::add_cf_unlocked(CACHEFILE cf) {
         m_active_head->prev = cf;
     }
     m_active_head = cf;
+
+    int r;
+    r = m_active_filenum.insert<FILENUM, cachefile_find_by_filenum>(cf, cf->filenum, nullptr);
+    assert_zero(r);
+    r = m_active_fileid.insert<struct fileid, cachefile_find_by_fileid>(cf, cf->fileid, nullptr);
+    assert_zero(r);
 }
 
 void cachefile_list::add_stale_cf(CACHEFILE cf) {
@@ -4847,6 +4799,19 @@ void cachefile_list::remove_cf(CACHEFILE cf) {
     }
     cf->prev = NULL;
     cf->next = NULL;
+
+    uint32_t idx;
+    int r;
+    r = m_active_filenum.find_zero<FILENUM, cachefile_find_by_filenum>(cf->filenum, nullptr, &idx);
+    assert_zero(r);
+    r = m_active_filenum.delete_at(idx);
+    assert_zero(r);
+
+    r = m_active_fileid.find_zero<struct fileid, cachefile_find_by_fileid>(cf->fileid, nullptr, &idx);
+    assert_zero(r);
+    r = m_active_fileid.delete_at(idx);
+    assert_zero(r);
+
     write_unlock();
 }
 
@@ -4872,18 +4837,23 @@ void cachefile_list::remove_stale_cf_unlocked(CACHEFILE cf) {
 }
 
 FILENUM cachefile_list::reserve_filenum() {
-    CACHEFILE extant;
-    FILENUM filenum;
     // taking a write lock because we are modifying next_filenum_to_use
     write_lock();
-try_again:
+    while (1) {
-    for (extant = m_active_head; extant; extant = extant->next) {
+        int r = m_active_filenum.find_zero<FILENUM, cachefile_find_by_filenum>(m_next_filenum_to_use, nullptr, nullptr);
-        if (m_next_filenum_to_use.fileid==extant->filenum.fileid) {
+        if (r == 0) {
             m_next_filenum_to_use.fileid++;
-            goto try_again;
+            continue;
         }
+        assert(r == DB_NOTFOUND);
+        break;
     }
-    filenum = m_next_filenum_to_use;
+    FILENUM filenum = m_next_filenum_to_use;
+#if TOKU_DEBUG_PARANOID
+    for (CACHEFILE extant = m_active_head; extant; extant = extant->next) {
+        assert(filenum.fileid != extant->filenum.fileid);
+    }
+#endif
     m_next_filenum_to_use.fileid++;
     write_unlock();
     return filenum;
@@ -4916,7 +4886,15 @@ exit:
 }
 
 CACHEFILE cachefile_list::find_cachefile_unlocked(struct fileid* fileid) {
-    return find_cachefile_in_list_unlocked(m_active_head, fileid);
+    CACHEFILE cf = nullptr;
+    int r = m_active_fileid.find_zero<struct fileid, cachefile_find_by_fileid>(*fileid, &cf, nullptr);
+    if (r == 0) {
+        assert(!cf->unlink_on_close);
+    }
+#if TOKU_DEBUG_PARANOID
+    assert(cf == find_cachefile_in_list_unlocked(m_active_head, fileid));
+#endif
+    return cf;
 }
 
 CACHEFILE cachefile_list::find_stale_cachefile_unlocked(struct fileid* fileid) {
@@ -4924,9 +4902,13 @@ CACHEFILE cachefile_list::find_stale_cachefile_unlocked(struct fileid* fileid) {
 }
 
 void cachefile_list::verify_unused_filenum(FILENUM filenum) {
+    int r = m_active_filenum.find_zero<FILENUM, cachefile_find_by_filenum>(filenum, nullptr, nullptr);
+    assert(r == DB_NOTFOUND);
+#if TOKU_DEBUG_PARANOID
     for (CACHEFILE extant = m_active_head; extant; extant = extant->next) {
         invariant(extant->filenum.fileid != filenum.fileid);
     }
+#endif
 }
 
 // returns true if some eviction ran, false otherwise

storage/tokudb/ft-index/ft/cachetable.h

@@ -344,28 +344,6 @@ void toku_cachetable_put(CACHEFILE cf, CACHEKEY key, uint32_t fullhash,
 // then the required PAIRs are written to disk for checkpoint.
 // KEY PROPERTY OF DEPENDENT PAIRS: They are already locked by the client
 // Returns: 0 if the memory object is in memory, otherwise an error number.
-// Rationale:
-//   begin_batched_pin and end_batched_pin take and release a read lock on the pair list.
-//   Normally, that would be done within this get_and_pin, but we want to pin multiple nodes with a single acquisition of the read lock.
-int toku_cachetable_get_and_pin_with_dep_pairs_batched (
-    CACHEFILE cachefile,
-    CACHEKEY key,
-    uint32_t fullhash,
-    void**value,
-    long *sizep,
-    CACHETABLE_WRITE_CALLBACK write_callback,
-    CACHETABLE_FETCH_CALLBACK fetch_callback,
-    CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback,
-    CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback,
-    pair_lock_type lock_type,
-    void* read_extraargs, // parameter for fetch_callback, pf_req_callback, and pf_callback
-    uint32_t num_dependent_pairs, // number of dependent pairs that we may need to checkpoint
-    PAIR* dependent_pairs,
-    enum cachetable_dirty* dependent_dirty // array stating dirty/cleanness of dependent pairs
-    );
-
-// Effect: call toku_cachetable_get_and_pin_with_dep_pairs_batched once,
-//  wrapped in begin_batched_pin and end_batched_pin calls.
 int toku_cachetable_get_and_pin_with_dep_pairs (
     CACHEFILE cachefile,
     CACHEKEY key,
@@ -383,7 +361,6 @@ int toku_cachetable_get_and_pin_with_dep_pairs (
     enum cachetable_dirty* dependent_dirty // array stating dirty/cleanness of dependent pairs
     );
 
-
 // Get and pin a memory object.
 // Effects: If the memory object is in the cachetable acquire the PAIR lock on it.
 // Otherwise, fetch it from storage by calling the fetch callback.  If the fetch
@@ -422,10 +399,7 @@ struct unlockers {
 
 // Effect:  If the block is in the cachetable, then return it.
 //   Otherwise call the functions in unlockers, fetch the data (but don't pin it, since we'll just end up pinning it again later), and return TOKUDB_TRY_AGAIN.
-// Rationale:
+int toku_cachetable_get_and_pin_nonblocking (
-//   begin_batched_pin and end_batched_pin take and release a read lock on the pair list.
-//   Normally, that would be done within this get_and_pin, but we want to pin multiple nodes with a single acquisition of the read lock.
-int toku_cachetable_get_and_pin_nonblocking_batched (
     CACHEFILE cf,
     CACHEKEY key,
     uint32_t fullhash,
@@ -440,23 +414,6 @@ int toku_cachetable_get_and_pin_nonblocking_batched (
     UNLOCKERS unlockers
     );
 
-// Effect: call toku_cachetable_get_and_pin_nonblocking_batched once,
-//  wrapped in begin_batched_pin and end_batched_pin calls.
-int toku_cachetable_get_and_pin_nonblocking (
-    CACHEFILE cf,
-    CACHEKEY key,
-    uint32_t fullhash,
-    void**value,
-    long *sizep,
-    CACHETABLE_WRITE_CALLBACK write_callback,
-    CACHETABLE_FETCH_CALLBACK fetch_callback,
-    CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback  __attribute__((unused)),
-    CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback  __attribute__((unused)),
-    pair_lock_type lock_type,
-    void *read_extraargs, // parameter for fetch_callback, pf_req_callback, and pf_callback
-    UNLOCKERS unlockers
-    );
-
 int toku_cachetable_maybe_get_and_pin (CACHEFILE, CACHEKEY, uint32_t /*fullhash*/, pair_lock_type, void**);
 // Effect: Maybe get and pin a memory object.
 //  This function is similar to the get_and_pin function except that it

storage/tokudb/ft-index/ft/checkpoint.cc

@@ -158,6 +158,8 @@ status_init(void) {
     STATUS_INIT(CP_TIME_LAST_CHECKPOINT_BEGIN,          CHECKPOINT_LAST_BEGAN, UNIXTIME, "last checkpoint began ", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
     STATUS_INIT(CP_TIME_LAST_CHECKPOINT_BEGIN_COMPLETE, CHECKPOINT_LAST_COMPLETE_BEGAN, UNIXTIME, "last complete checkpoint began ", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
     STATUS_INIT(CP_TIME_LAST_CHECKPOINT_END,            CHECKPOINT_LAST_COMPLETE_ENDED, UNIXTIME, "last complete checkpoint ended", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
+    STATUS_INIT(CP_TIME_CHECKPOINT_DURATION,            CHECKPOINT_DURATION, UNIXTIME, "time spent during checkpoint (begin and end phases)", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
+    STATUS_INIT(CP_TIME_CHECKPOINT_DURATION_LAST,       CHECKPOINT_DURATION_LAST, UNIXTIME, "time spent during last checkpoint (begin and end phases)", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
     STATUS_INIT(CP_LAST_LSN,                            nullptr, UINT64,   "last complete checkpoint LSN", TOKU_ENGINE_STATUS);
     STATUS_INIT(CP_CHECKPOINT_COUNT,                    CHECKPOINT_TAKEN, UINT64,   "checkpoints taken ", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
     STATUS_INIT(CP_CHECKPOINT_COUNT_FAIL,               CHECKPOINT_FAILED, UINT64,   "checkpoints failed", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
@@ -379,6 +381,8 @@ toku_checkpoint(CHECKPOINTER cp, TOKULOGGER logger,
         STATUS_VALUE(CP_LONG_BEGIN_TIME) += duration;
         STATUS_VALUE(CP_LONG_BEGIN_COUNT) += 1;
     }
+    STATUS_VALUE(CP_TIME_CHECKPOINT_DURATION) += ((time_t) STATUS_VALUE(CP_TIME_LAST_CHECKPOINT_END)) - ((time_t) STATUS_VALUE(CP_TIME_LAST_CHECKPOINT_BEGIN));
+    STATUS_VALUE(CP_TIME_CHECKPOINT_DURATION_LAST) = ((time_t) STATUS_VALUE(CP_TIME_LAST_CHECKPOINT_END)) - ((time_t) STATUS_VALUE(CP_TIME_LAST_CHECKPOINT_BEGIN));
     STATUS_VALUE(CP_FOOTPRINT) = 0;
 
     checkpoint_safe_checkpoint_unlock();

storage/tokudb/ft-index/ft/checkpoint.h

@@ -179,6 +179,8 @@ typedef enum {
     CP_TIME_LAST_CHECKPOINT_BEGIN,
     CP_TIME_LAST_CHECKPOINT_BEGIN_COMPLETE,
     CP_TIME_LAST_CHECKPOINT_END,
+    CP_TIME_CHECKPOINT_DURATION,
+    CP_TIME_CHECKPOINT_DURATION_LAST,
     CP_LAST_LSN,
     CP_CHECKPOINT_COUNT,
     CP_CHECKPOINT_COUNT_FAIL,

storage/tokudb/ft-index/ft/checksum-benchmarks/Makefile

@@ -1,3 +0,0 @@
-CFLAGS = -O2 -Wall -W -Werror -g
-LDFLAGS = -lz -lssl -g
-adler32:

storage/tokudb/ft-index/ft/checksum-benchmarks/adler32.cc

@@ -1,236 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-#include <toku_assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/time.h>
-#include <zlib.h>
-#include <openssl/md2.h>
-#include <openssl/md4.h>
-#include <openssl/md5.h>
-
-const unsigned int prime = 2000000011;
-
-unsigned int karprabin (unsigned char *datac, int N) {
-    assert(N%4==0);
-    unsigned int *data=(unsigned int*)datac;
-    N=N/4;
-    int i;
-    unsigned int result=0;
-    for (i=0; i<N; i++) {
-	result=(result*prime)+data[i];
-    }
-    return result;
-}
-
-// According to
-//  P. L'Ecuyer, "Tables of Linear Congruential Generators of
-//  Different Sizes and Good Lattice Structure", Mathematics of
-//  Computation 68:225, 249--260 (1999).
-// m=2^{32}-5  a=1588635695 is good.
-
-const unsigned int mkr = 4294967291U;
-const unsigned int akr = 1588635695U;
-
-
-// But this is slower
-unsigned int karprabinP (unsigned char *datac, int N) {
-    assert(N%4==0);
-    unsigned int *data=(unsigned int*)datac;
-    N=N/4;
-    int i;
-    unsigned long long result=0;
-    for (i=0; i<N; i++) {
-	result=((result*akr)+data[i])%mkr;
-    }
-    return result;
-}
-
-float tdiff (struct timeval *start, struct timeval *end) {
-    return (end->tv_sec-start->tv_sec) +1e-6*(end->tv_usec - start->tv_usec);
-}
-
-int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
-    struct timeval start, end;
-    const int N=2<<20;
-    unsigned char *data=malloc(N);
-    int i;
-    assert(data);
-    for (i=0; i<N; i++) data[i]=random();
-
-    // adler32
-    {
-	uLong a32 = adler32(0L, Z_NULL, 0);
-	for (i=0; i<3; i++) {
-	    gettimeofday(&start, 0);
-	    a32 = adler32(a32, data, N);
-	    gettimeofday(&end,   0);
-	    float tm = tdiff(&start, &end);
-	    printf("adler32=%lu, time=%9.6fs %9.6fns/b\n", a32, tm, 1e9*tm/N);
-	}
-    }
-
-    // crc32
-    {
-	uLong c32 = crc32(0L, Z_NULL, 0);
-	for (i=0; i<3; i++) {
-	    gettimeofday(&start, 0);
-	    c32 = crc32(c32, data, N);
-	    gettimeofday(&end,   0);
-	    float tm = tdiff(&start, &end);
-	    printf("crc32=%lu, time=%9.6fs %9.6fns/b\n", c32, tm, 1e9*tm/N);
-	}
-    }
-
-    // MD2
-    {
-	unsigned char buf[MD2_DIGEST_LENGTH];
-	int j;
-	for (i=0; i<3; i++) {
-	    gettimeofday(&start, 0);
-	    MD2(data, N, buf);
-	    gettimeofday(&end,   0);
-	    float tm = tdiff(&start, &end);
-	    printf("md2=");
-	    for (j=0; j<MD2_DIGEST_LENGTH; j++) {
-		printf("%02x", buf[j]);
-	    }
-	    printf(" time=%9.6fs %9.6fns/b\n", tm, 1e9*tm/N);
-	}
-    }
-
-    // MD4
-    {
-	unsigned char buf[MD4_DIGEST_LENGTH];
-	int j;
-	for (i=0; i<3; i++) {
-	    gettimeofday(&start, 0);
-	    MD4(data, N, buf);
-	    gettimeofday(&end,   0);
-	    float tm = tdiff(&start, &end);
-	    printf("md4=");
-	    for (j=0; j<MD4_DIGEST_LENGTH; j++) {
-		printf("%02x", buf[j]);
-	    }
-	    printf(" time=%9.6fs %9.6fns/b\n", tm, 1e9*tm/N);
-	}
-    }
-
-    // MD5
-    {
-	unsigned char buf[MD5_DIGEST_LENGTH];
-	int j;
-	for (i=0; i<3; i++) {
-	    gettimeofday(&start, 0);
-	    MD5(data, N, buf);
-	    gettimeofday(&end,   0);
-	    float tm = tdiff(&start, &end);
-	    printf("md5=");
-	    for (j=0; j<MD5_DIGEST_LENGTH; j++) {
-		printf("%02x", buf[j]);
-	    }
-	    printf(" time=%9.6fs %9.6fns/b\n", tm, 1e9*tm/N);
-	}
-    }
-
-    // karp rabin
-    {
-	for (i=0; i<3; i++) {
-	    gettimeofday(&start, 0);
-	    unsigned int kr = karprabin(data, N);
-	    gettimeofday(&end,   0);
-	    float tm = tdiff(&start, &end);
-	    printf("kr=%ud time=%9.6fs %9.6fns/b\n", kr, tm, 1e9*tm/N);
-	}
-    }
-    free(data);
-    return 0;
-}

storage/tokudb/ft-index/ft/compression-ratio/Makefile

@@ -1,3 +0,0 @@
-CFLAGS = -O -Wall -W -Werror
-LDFLAGS = -lz
-cratio:

storage/tokudb/ft-index/ft/compression-ratio/cratio.cc

@@ -1,195 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-/* Measure the extent to which we can compress a file.
- * Works on version 8. */
-
-#define _XOPEN_SOURCE 500
-#include <arpa/inet.h>
-#include <toku_assert.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <zlib.h>
-#include <toku_portability.h>
-
-
-toku_off_t fd_size (int fd) {
-    int64_t file_size;
-    int r = toku_os_get_file_size(fd, &file_size);
-    assert(r==0);
-    return file_size;
-}
-
-#define NSIZE (1<<20)
-unsigned char fbuf[NSIZE];
-unsigned char cbuf[NSIZE+500];
-
-void
-measure_header (int fd, toku_off_t off, // read header from this offset
-		toku_off_t *usize,      // size uncompressed (but not including any padding)
-		toku_off_t *csize)      // compressed size
-{
-    int r;
-    r=pread(fd, fbuf, 12, off);
-    assert(r==12);
-    assert(memcmp(fbuf,"tokudata",8)==0);
-    int bsize = toku_dtoh32(*(uint32_t*)(fbuf+8));
-    //printf("Bsize=%d\n", bsize);
-    (*usize)+=bsize;
-    assert(bsize<=NSIZE);
-    r=pread(fd, fbuf, bsize, off);
-    assert(r==bsize);
-    uLongf destLen=sizeof(cbuf);
-    r=compress2(cbuf, &destLen,
-		fbuf+20, bsize-=20, // skip magic nodesize and version
-		1);
-    assert(r==Z_OK);
-    destLen+=16; // account for the size and magic and version
-    //printf("Csize=%ld\n", destLen);
-    (*csize)+=destLen;
-}
-
-void
-measure_node (int fd, toku_off_t off, // read header from this offset
-	      toku_off_t *usize,      // size uncompressed (but not including any padding)
-	      toku_off_t *csize)      // compressed size
-{
-    int r;
-    r=pread(fd, fbuf, 24, off);
-    assert(r==24);
-    //printf("fbuf[0..7]=%c%c%c%c%c%c%c%c\n", fbuf[0], fbuf[1], fbuf[2], fbuf[3], fbuf[4], fbuf[5], fbuf[6], fbuf[7]);
-    assert(memcmp(fbuf,"tokuleaf",8)==0 || memcmp(fbuf, "tokunode", 8)==0);
-    assert(8==toku_dtoh32(*(uint32_t*)(fbuf+8))); // check file version
-    int bsize = toku_dtoh32(*(uint32_t*)(fbuf+20));
-    //printf("Bsize=%d\n", bsize);
-    (*usize)+=bsize;
-
-    assert(bsize<=NSIZE);
-    r=pread(fd, fbuf, bsize, off);
-    assert(r==bsize);
-    uLongf destLen=sizeof(cbuf);
-    r=compress2(cbuf, &destLen,
-		fbuf+28, bsize-=28, // skip constant header stuff
-		1);
-    destLen += 24; // add in magic (8), version(4), lsn (8), and size (4).  Actually lsn will be compressed, but ignore that for now.
-    assert(r==Z_OK);
-    //printf("Csize=%ld\n", destLen);
-    (*csize)+=destLen;
-
-}
-
-
-
-/* The single argument is the filename to measure. */
-int main (int argc, const char *argv[]) {
-    assert(argc==2);
-    const char *fname=argv[1];
-    int fd = open(fname, O_RDONLY);
-    assert(fd>=0);
-    toku_off_t fsize = fd_size(fd);
-    printf("fsize (uncompressed with   padding)=%lld\n", (long long)fsize);
-
-    toku_off_t usize=0, csize=0;
-    measure_header(fd, 0, &usize, &csize);
-
-    toku_off_t i;
-    for (i=NSIZE; i+24<fsize; i+=NSIZE) {
-	measure_node(fd, i, &usize, &csize);
-    }
-
-    printf("usize (uncompressed with no padding)=%10lld  (ratio=%5.2f)\n", (long long)usize, (double)fsize/(double)usize);
-    printf("csize (compressed)                  =%10lld  (ratio=%5.2f)\n", (long long)csize, (double)fsize/(double)csize);
-
-    close(fd);
-    return 0;
-}

storage/tokudb/ft-index/ft/fifo.cc

@@ -102,7 +102,6 @@ struct fifo {
     int   memory_used;  // How many bytes are in use?
 };
 
-const int fifo_initial_size = 4096;
 static void fifo_init(struct fifo *fifo) {
     fifo->n_items_in_fifo = 0;
     fifo->memory       = 0;
@@ -118,12 +117,12 @@ static int fifo_entry_size(struct fifo_entry *entry) {
 }
 
 __attribute__((const,nonnull))
-size_t toku_ft_msg_memsize_in_fifo(FT_MSG cmd) {
+size_t toku_ft_msg_memsize_in_fifo(FT_MSG msg) {
     // This must stay in sync with fifo_entry_size because that's what we
     // really trust.  But sometimes we only have an in-memory FT_MSG, not
     // a serialized fifo_entry so we have to fake it.
-    return sizeof (struct fifo_entry) + cmd->u.id.key->size + cmd->u.id.val->size
+    return sizeof (struct fifo_entry) + msg->u.id.key->size + msg->u.id.val->size
-        + xids_get_size(cmd->xids)
+        + xids_get_size(msg->xids)
         - sizeof(XIDS_S);
 }
 

storage/tokudb/ft-index/ft/fifo.h

@@ -97,10 +97,6 @@ PATENT RIGHTS GRANT:
 
 
 // If the fifo_entry is unpacked, the compiler aligns the xids array and we waste a lot of space
-#if TOKU_WINDOWS
-#pragma pack(push, 1)
-#endif
-
 struct __attribute__((__packed__)) fifo_entry {
     unsigned int keylen;
     unsigned int vallen;
@@ -110,7 +106,7 @@ struct __attribute__((__packed__)) fifo_entry {
     XIDS_S        xids_s;
 };
 
-// get and set the brt message type for a fifo entry.
+// get and set the ft message type for a fifo entry.
 // it is internally stored as a single unsigned char.
 static inline enum ft_msg_type 
 fifo_entry_get_msg_type(const struct fifo_entry * entry)
@@ -128,10 +124,6 @@ fifo_entry_set_msg_type(struct fifo_entry * entry,
     entry->type = type;
 }
 
-#if TOKU_WINDOWS
-#pragma pack(pop)
-#endif
-
 typedef struct fifo *FIFO;
 
 int toku_fifo_create(FIFO *);
@@ -149,9 +141,6 @@ unsigned long toku_fifo_memory_size_in_use(FIFO fifo);  // return how much memor
 
 unsigned long toku_fifo_memory_footprint(FIFO fifo);  // return how much memory the fifo occupies
 
-//These two are problematic, since I don't want to malloc() the bytevecs, but dequeueing the fifo frees the memory.
-//int toku_fifo_peek_deq (FIFO, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen, uint32_t *type, TXNID *xid);
-//int toku_fifo_peek_deq_cmdstruct (FIFO, FT_MSG, DBT*, DBT*); // fill in the FT_MSG, using the two DBTs for the DBT part.
 void toku_fifo_iterate(FIFO, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,ITEMLEN datalen, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, void*), void*);
 
 #define FIFO_ITERATE(fifo,keyvar,keylenvar,datavar,datalenvar,typevar,msnvar,xidsvar,is_freshvar,body) ({ \
@@ -178,7 +167,7 @@ int toku_fifo_iterate_internal_has_more(FIFO fifo, int off);
 int toku_fifo_iterate_internal_next(FIFO fifo, int off);
 struct fifo_entry * toku_fifo_iterate_internal_get_entry(FIFO fifo, int off);
 size_t toku_fifo_internal_entry_memsize(struct fifo_entry *e) __attribute__((const,nonnull));
-size_t toku_ft_msg_memsize_in_fifo(FT_MSG cmd) __attribute__((const,nonnull));
+size_t toku_ft_msg_memsize_in_fifo(FT_MSG msg) __attribute__((const,nonnull));
 
 DBT *fill_dbt_for_fifo_entry(DBT *dbt, const struct fifo_entry *entry);
 struct fifo_entry *toku_fifo_get_entry(FIFO fifo, int off);

storage/tokudb/ft-index/ft/ft-cachetable-wrappers.cc

@@ -201,8 +201,8 @@ toku_create_new_ftnode (
 // then a PL_WRITE_CHEAP lock is grabbed
 //
 int
-toku_pin_ftnode_batched(
+toku_pin_ftnode_for_query(
-    FT_HANDLE brt,
+    FT_HANDLE ft_handle,
     BLOCKNUM blocknum,
     uint32_t fullhash,
     UNLOCKERS unlockers,
@@ -225,13 +225,13 @@ toku_pin_ftnode_batched(
         paranoid_invariant(bfe->type == ftnode_fetch_subset);
     }
     
-    int r = toku_cachetable_get_and_pin_nonblocking_batched(
+    int r = toku_cachetable_get_and_pin_nonblocking(
-            brt->ft->cf,
+            ft_handle->ft->cf,
             blocknum,
             fullhash,
             &node_v,
             NULL,
-            get_write_callbacks_for_node(brt->ft),
+            get_write_callbacks_for_node(ft_handle->ft),
             toku_ftnode_fetch_callback,
             toku_ftnode_pf_req_callback,
             toku_ftnode_pf_callback,
@@ -245,7 +245,7 @@ toku_pin_ftnode_batched(
     node = static_cast<FTNODE>(node_v);
     if (apply_ancestor_messages && node->height == 0) {
         needs_ancestors_messages = toku_ft_leaf_needs_ancestors_messages(
-            brt->ft, 
+            ft_handle->ft, 
             node, 
             ancestors, 
             bounds, 
@@ -255,20 +255,20 @@ toku_pin_ftnode_batched(
         if (needs_ancestors_messages) {
             toku::context apply_messages_ctx(CTX_MESSAGE_APPLICATION);
 
-            toku_unpin_ftnode_read_only(brt->ft, node);
+            toku_unpin_ftnode_read_only(ft_handle->ft, node);
-            int rr = toku_cachetable_get_and_pin_nonblocking_batched(
+            int rr = toku_cachetable_get_and_pin_nonblocking(
-                    brt->ft->cf,
+                 ft_handle->ft->cf,
-                    blocknum,
+                 blocknum,
-                    fullhash,
+                 fullhash,
-                    &node_v,
+                 &node_v,
-                    NULL,
+                 NULL,
-                    get_write_callbacks_for_node(brt->ft),
+                 get_write_callbacks_for_node(ft_handle->ft),
-                    toku_ftnode_fetch_callback,
+                 toku_ftnode_fetch_callback,
-                    toku_ftnode_pf_req_callback,
+                 toku_ftnode_pf_req_callback,
-                    toku_ftnode_pf_callback,
+                 toku_ftnode_pf_callback,
-                    PL_WRITE_CHEAP,
+                 PL_WRITE_CHEAP,
-                    bfe, //read_extraargs
+                 bfe, //read_extraargs
-                    unlockers);
+                 unlockers);
             if (rr != 0) {
                 assert(rr == TOKUDB_TRY_AGAIN); // Any other error and we should bomb out ASAP.
                 r = TOKUDB_TRY_AGAIN;
@@ -276,7 +276,7 @@ toku_pin_ftnode_batched(
             }
             node = static_cast<FTNODE>(node_v);
             toku_apply_ancestors_messages_to_node(
-                brt, 
+                ft_handle, 
                 node, 
                 ancestors, 
                 bounds, 
@@ -317,54 +317,14 @@ exit:
 }
 
 void
-toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
+toku_pin_ftnode_with_dep_nodes(
     FT h,
     BLOCKNUM blocknum,
     uint32_t fullhash,
     FTNODE_FETCH_EXTRA bfe,
     pair_lock_type lock_type,
     uint32_t num_dependent_nodes,
-    FTNODE* dependent_nodes,
+    FTNODE *dependent_nodes,
-    FTNODE *node_p,
-    bool move_messages)
-{
-    toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
-        h,
-        blocknum,
-        fullhash,
-        bfe,
-        lock_type,
-        num_dependent_nodes,
-        dependent_nodes,
-        node_p,
-        move_messages
-        );
-}
-
-void
-toku_pin_ftnode_off_client_thread(
-    FT h,
-    BLOCKNUM blocknum,
-    uint32_t fullhash,
-    FTNODE_FETCH_EXTRA bfe,
-    pair_lock_type lock_type,
-    uint32_t num_dependent_nodes,
-    FTNODE* dependent_nodes,
-    FTNODE *node_p)
-{
-    toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
-            h, blocknum, fullhash, bfe, lock_type, num_dependent_nodes, dependent_nodes, node_p, true);
-}
-
-void
-toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
-    FT h,
-    BLOCKNUM blocknum,
-    uint32_t fullhash,
-    FTNODE_FETCH_EXTRA bfe,
-    pair_lock_type lock_type,
-    uint32_t num_dependent_nodes,
-    FTNODE* dependent_nodes,
     FTNODE *node_p,
     bool move_messages)
 {
@@ -376,7 +336,7 @@ toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
         dependent_dirty_bits[i] = (enum cachetable_dirty) dependent_nodes[i]->dirty;
     }
 
-    int r = toku_cachetable_get_and_pin_with_dep_pairs_batched(
+    int r = toku_cachetable_get_and_pin_with_dep_pairs(
         h->cf,
         blocknum,
         fullhash,
@@ -392,27 +352,22 @@ toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
         dependent_pairs,
         dependent_dirty_bits
         );
-    assert(r==0);
+    invariant_zero(r);
     FTNODE node = (FTNODE) node_v;
-    if ((lock_type != PL_READ) && node->height > 0 && move_messages) {
+    if (lock_type != PL_READ && node->height > 0 && move_messages) {
         toku_move_ftnode_messages_to_stale(h, node);
     }
     *node_p = node;
 }
 
-void
+void toku_pin_ftnode(FT ft,
-toku_pin_ftnode_off_client_thread_batched(
+                     BLOCKNUM blocknum,
-    FT h,
+                     uint32_t fullhash,
-    BLOCKNUM blocknum,
+                     FTNODE_FETCH_EXTRA bfe,
-    uint32_t fullhash,
+                     pair_lock_type lock_type,
-    FTNODE_FETCH_EXTRA bfe,
+                     FTNODE *node_p,
-    pair_lock_type lock_type,
+                     bool move_messages) {
-    uint32_t num_dependent_nodes,
+    toku_pin_ftnode_with_dep_nodes(ft, blocknum, fullhash, bfe, lock_type, 0, nullptr, node_p, move_messages);
-    FTNODE* dependent_nodes,
-    FTNODE *node_p)
-{
-    toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
-            h, blocknum, fullhash, bfe, lock_type, num_dependent_nodes, dependent_nodes, node_p, true);
 }
 
 int toku_maybe_pin_ftnode_clean(FT ft, BLOCKNUM blocknum, uint32_t fullhash, pair_lock_type lock_type, FTNODE *nodep) {
@@ -429,24 +384,12 @@ cleanup:
     return r;
 }
 
-void
+void toku_unpin_ftnode(FT ft, FTNODE node) {
-toku_unpin_ftnode_off_client_thread(FT ft, FTNODE node)
+    int r = toku_cachetable_unpin(ft->cf,
-{
+                                  node->ct_pair,
-    int r = toku_cachetable_unpin(
+                                  static_cast<enum cachetable_dirty>(node->dirty),
-        ft->cf,
+                                  make_ftnode_pair_attr(node));
-        node->ct_pair,
+    invariant_zero(r);
-        (enum cachetable_dirty) node->dirty,
-        make_ftnode_pair_attr(node)
-        );
-    assert(r==0);
-}
-
-void
-toku_unpin_ftnode(FT ft, FTNODE node)
-{
-    // printf("%*sUnpin %ld\n", 8-node->height, "", node->thisnodename.b);
-    //VERIFY_NODE(brt,node);
-    toku_unpin_ftnode_off_client_thread(ft, node);
 }
 
 void

storage/tokudb/ft-index/ft/ft-cachetable-wrappers.h

@@ -138,13 +138,10 @@ toku_create_new_ftnode (
     int n_children
     );
 
-/**
+// This function returns a pinned ftnode to the caller.
- * Batched version of toku_pin_ftnode, see cachetable batched API for more
- * details.
- */
 int
-toku_pin_ftnode_batched(
+toku_pin_ftnode_for_query(
-    FT_HANDLE brt,
+    FT_HANDLE ft_h,
     BLOCKNUM blocknum,
     uint32_t fullhash,
     UNLOCKERS unlockers,
@@ -156,34 +153,27 @@ toku_pin_ftnode_batched(
     bool* msgs_applied
     );
 
-/**
+// Pins an ftnode without dependent pairs
- * Unfortunately, this function is poorly named
+void toku_pin_ftnode(
- * as over time, client threads have also started
- * calling this function.
- * This function returns a pinned ftnode to the caller.
- * Unlike toku_pin_ftnode, this function blocks until the node is pinned.
- */
-void
-toku_pin_ftnode_off_client_thread(
     FT h,
     BLOCKNUM blocknum,
     uint32_t fullhash,
     FTNODE_FETCH_EXTRA bfe,
     pair_lock_type lock_type,
-    uint32_t num_dependent_nodes,
+    FTNODE *node_p,
-    FTNODE* dependent_nodes,
+    bool move_messages
-    FTNODE *node_p
     );
 
-void
+// Pins an ftnode with dependent pairs
-toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
+// Unlike toku_pin_ftnode_for_query, this function blocks until the node is pinned.
+void toku_pin_ftnode_with_dep_nodes(
     FT h,
     BLOCKNUM blocknum,
     uint32_t fullhash,
     FTNODE_FETCH_EXTRA bfe,
     pair_lock_type lock_type,
     uint32_t num_dependent_nodes,
-    FTNODE* dependent_nodes,
+    FTNODE *dependent_nodes,
     FTNODE *node_p,
     bool move_messages
     );
@@ -195,53 +185,9 @@ toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
 int toku_maybe_pin_ftnode_clean(FT ft, BLOCKNUM blocknum, uint32_t fullhash, pair_lock_type lock_type, FTNODE *nodep);
 
 /**
- * Batched version of toku_pin_ftnode_off_client_thread, see cachetable
+ * Effect: Unpin an ftnode.
- * batched API for more details.
  */
-void
+void toku_unpin_ftnode(FT h, FTNODE node);
-toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
+void toku_unpin_ftnode_read_only(FT ft, FTNODE node);
-    FT h,
-    BLOCKNUM blocknum,
-    uint32_t fullhash,
-    FTNODE_FETCH_EXTRA bfe,
-    pair_lock_type lock_type,
-    uint32_t num_dependent_nodes,
-    FTNODE* dependent_nodes,
-    FTNODE *node_p,
-    bool move_messages
-    );
-
-/**
- * Batched version of toku_pin_ftnode_off_client_thread, see cachetable
- * batched API for more details.
- */
-void
-toku_pin_ftnode_off_client_thread_batched(
-    FT h,
-    BLOCKNUM blocknum,
-    uint32_t fullhash,
-    FTNODE_FETCH_EXTRA bfe,
-    pair_lock_type lock_type,
-    uint32_t num_dependent_nodes,
-    FTNODE* dependent_nodes,
-    FTNODE *node_p
-    );
-
-/**
- * Effect: Unpin a brt node. Used for
- * nodes that were pinned off client thread.
- */
-void
-toku_unpin_ftnode_off_client_thread(FT h, FTNODE node);
-
-/**
- * Effect: Unpin a brt node.
- * Used for nodes pinned on a client thread
- */
-void
-toku_unpin_ftnode(FT h, FTNODE node);
-
-void
-toku_unpin_ftnode_read_only(FT ft, FTNODE node);
 
 #endif

storage/tokudb/ft-index/ft/ft-flusher.cc

@@ -104,7 +104,7 @@ PATENT RIGHTS GRANT:
  */
 static FT_FLUSHER_STATUS_S ft_flusher_status;
 
-#define STATUS_INIT(k,c,t,l,inc) TOKUDB_STATUS_INIT(ft_flusher_status, k, c, t, "brt flusher: " l, inc)
+#define STATUS_INIT(k,c,t,l,inc) TOKUDB_STATUS_INIT(ft_flusher_status, k, c, t, "ft flusher: " l, inc)
 
 #define STATUS_VALUE(x) ft_flusher_status.status[x].value.num
 void toku_ft_flusher_status_init(void) {
@@ -334,7 +334,7 @@ default_merge_child(struct flusher_advice *fa,
     // we are just going to unpin child and
     // let ft_merge_child pin it again
     //
-    toku_unpin_ftnode_off_client_thread(h, child);
+    toku_unpin_ftnode(h, child);
     //
     //
     // it is responsibility of ft_merge_child to unlock parent
@@ -486,8 +486,8 @@ ct_maybe_merge_child(struct flusher_advice *fa,
             default_pick_child_after_split,
             &ctme);
 
-        toku_unpin_ftnode_off_client_thread(h, parent);
+        toku_unpin_ftnode(h, parent);
-        toku_unpin_ftnode_off_client_thread(h, child);
+        toku_unpin_ftnode(h, child);
 
         FTNODE root_node = NULL;
         {
@@ -496,7 +496,7 @@ ct_maybe_merge_child(struct flusher_advice *fa,
             toku_calculate_root_offset_pointer(h, &root, &fullhash);
             struct ftnode_fetch_extra bfe;
             fill_bfe_for_full_read(&bfe, h);
-            toku_pin_ftnode_off_client_thread(h, root, fullhash, &bfe, PL_WRITE_EXPENSIVE, 0, NULL, &root_node);
+            toku_pin_ftnode(h, root, fullhash, &bfe, PL_WRITE_EXPENSIVE, &root_node, true);
             toku_assert_entire_node_in_memory(root_node);
         }
 
@@ -689,16 +689,16 @@ ftleaf_get_split_loc(
     switch (split_mode) {
     case SPLIT_LEFT_HEAVY: {
         *num_left_bns = node->n_children;
-        *num_left_les = BLB_DATA(node, *num_left_bns - 1)->omt_size();
+        *num_left_les = BLB_DATA(node, *num_left_bns - 1)->num_klpairs();
         if (*num_left_les == 0) {
             *num_left_bns = node->n_children - 1;
-            *num_left_les = BLB_DATA(node, *num_left_bns - 1)->omt_size();
+            *num_left_les = BLB_DATA(node, *num_left_bns - 1)->num_klpairs();
         }
         goto exit;
     }
     case SPLIT_RIGHT_HEAVY: {
         *num_left_bns = 1;
-        *num_left_les = BLB_DATA(node, 0)->omt_size() ? 1 : 0;
+        *num_left_les = BLB_DATA(node, 0)->num_klpairs() ? 1 : 0;
         goto exit;
     }
     case SPLIT_EVENLY: {
@@ -707,8 +707,8 @@ ftleaf_get_split_loc(
         uint64_t sumlesizes = ftleaf_disk_size(node);
         uint32_t size_so_far = 0;
         for (int i = 0; i < node->n_children; i++) {
-            BN_DATA bd = BLB_DATA(node, i);
+            bn_data* bd = BLB_DATA(node, i);
-            uint32_t n_leafentries = bd->omt_size();
+            uint32_t n_leafentries = bd->num_klpairs();
             for (uint32_t j=0; j < n_leafentries; j++) {
                 size_t size_this_le;
                 int rr = bd->fetch_klpair_disksize(j, &size_this_le);
@@ -725,7 +725,7 @@ ftleaf_get_split_loc(
                             (*num_left_les)--;
                         } else if (*num_left_bns > 1) {
                             (*num_left_bns)--;
-                            *num_left_les = BLB_DATA(node, *num_left_bns - 1)->omt_size();
+                            *num_left_les = BLB_DATA(node, *num_left_bns - 1)->num_klpairs();
                         } else {
                             // we are trying to split a leaf with only one
                             // leafentry in it
@@ -743,8 +743,6 @@ exit:
     return;
 }
 
-// TODO: (Zardosht) possibly get rid of this function and use toku_omt_split_at in
-// ftleaf_split
 static void
 move_leafentries(
     BASEMENTNODE dest_bn,
@@ -754,7 +752,8 @@ move_leafentries(
     )
 //Effect: move leafentries in the range [lbi, upe) from src_omt to newly created dest_omt
 {
-    src_bn->data_buffer.move_leafentries_to(&dest_bn->data_buffer, lbi, ube);
+    invariant(ube == src_bn->data_buffer.num_klpairs());
+    src_bn->data_buffer.split_klpairs(&dest_bn->data_buffer, lbi);
 }
 
 static void ftnode_finalize_split(FTNODE node, FTNODE B, MSN max_msn_applied_to_node) {
@@ -851,7 +850,7 @@ ftleaf_split(
     ftleaf_get_split_loc(node, split_mode, &num_left_bns, &num_left_les);
     {
         // did we split right on the boundary between basement nodes?
-        const bool split_on_boundary = (num_left_les == 0) || (num_left_les == (int) BLB_DATA(node, num_left_bns - 1)->omt_size());
+        const bool split_on_boundary = (num_left_les == 0) || (num_left_les == (int) BLB_DATA(node, num_left_bns - 1)->num_klpairs());
         // Now we know where we are going to break it
         // the two nodes will have a total of n_children+1 basement nodes
         // and n_children-1 pivots
@@ -912,7 +911,7 @@ ftleaf_split(
             move_leafentries(BLB(B, curr_dest_bn_index),
                              BLB(node, curr_src_bn_index),
                              num_left_les,         // first row to be moved to B
-                             BLB_DATA(node, curr_src_bn_index)->omt_size()  // number of rows in basement to be split
+                             BLB_DATA(node, curr_src_bn_index)->num_klpairs()  // number of rows in basement to be split
                              );
             BLB_MAX_MSN_APPLIED(B, curr_dest_bn_index) = BLB_MAX_MSN_APPLIED(node, curr_src_bn_index);
             curr_dest_bn_index++;
@@ -954,10 +953,10 @@ ftleaf_split(
                 toku_destroy_dbt(&node->childkeys[num_left_bns - 1]);
             }
         } else if (splitk) {
-            BN_DATA bd = BLB_DATA(node, num_left_bns - 1);
+            bn_data* bd = BLB_DATA(node, num_left_bns - 1);
             uint32_t keylen;
             void *key;
-            int rr = bd->fetch_le_key_and_len(bd->omt_size() - 1, &keylen, &key);
+            int rr = bd->fetch_key_and_len(bd->num_klpairs() - 1, &keylen, &key);
             invariant_zero(rr);
             toku_memdup_dbt(splitk, key, keylen);
         }
@@ -1082,20 +1081,20 @@ ft_split_child(
     // and possibly continue
     // flushing one of the children
     int picked_child = fa->pick_child_after_split(h, node, childnum, childnum + 1, fa->extra);
-    toku_unpin_ftnode_off_client_thread(h, node);
+    toku_unpin_ftnode(h, node);
     if (picked_child == childnum ||
         (picked_child < 0 && nodea->height > 0 && fa->should_recursively_flush(nodea, fa->extra))) {
-        toku_unpin_ftnode_off_client_thread(h, nodeb);
+        toku_unpin_ftnode(h, nodeb);
         toku_ft_flush_some_child(h, nodea, fa);
     }
     else if (picked_child == childnum + 1 ||
              (picked_child < 0 && nodeb->height > 0 && fa->should_recursively_flush(nodeb, fa->extra))) {
-        toku_unpin_ftnode_off_client_thread(h, nodea);
+        toku_unpin_ftnode(h, nodea);
         toku_ft_flush_some_child(h, nodeb, fa);
     }
     else {
-        toku_unpin_ftnode_off_client_thread(h, nodea);
+        toku_unpin_ftnode(h, nodea);
-        toku_unpin_ftnode_off_client_thread(h, nodeb);
+        toku_unpin_ftnode(h, nodeb);
     }
 }
 
@@ -1168,11 +1167,11 @@ merge_leaf_nodes(FTNODE a, FTNODE b)
     a->dirty = 1;
     b->dirty = 1;
 
-    BN_DATA a_last_bd = BLB_DATA(a, a->n_children-1);
+    bn_data* a_last_bd = BLB_DATA(a, a->n_children-1);
     // this bool states if the last basement node in a has any items or not
     // If it does, then it stays in the merge. If it does not, the last basement node
     // of a gets eliminated because we do not have a pivot to store for it (because it has no elements)
-    const bool a_has_tail = a_last_bd->omt_size() > 0;
+    const bool a_has_tail = a_last_bd->num_klpairs() > 0;
 
     // move each basement node from b to a
     // move the pivots, adding one of what used to be max(a)
@@ -1199,7 +1198,7 @@ merge_leaf_nodes(FTNODE a, FTNODE b)
     if (a_has_tail) {
         uint32_t keylen;
         void *key;
-        int rr = a_last_bd->fetch_le_key_and_len(a_last_bd->omt_size() - 1, &keylen, &key);
+        int rr = a_last_bd->fetch_key_and_len(a_last_bd->num_klpairs() - 1, &keylen, &key);
         invariant_zero(rr);
         toku_memdup_dbt(&a->childkeys[a->n_children-1], key, keylen);
         a->totalchildkeylens += keylen;
@@ -1343,7 +1342,7 @@ maybe_merge_pinned_nodes(
 //    For nonleaf nodes, we distribute the children evenly.  That may leave one or both of the nodes overfull, but that's OK.
 //  If we distribute, we set *splitk to a malloced pivot key.
 // Parameters:
-//  t			The BRT.
+//  t			The FT.
 //  parent		The parent of the two nodes to be split.
 //  parent_splitk	The pivot key between a and b.	 This is either free()'d or returned in *splitk.
 //  a			The first node to merge.
@@ -1426,7 +1425,7 @@ ft_merge_child(
         uint32_t childfullhash = compute_child_fullhash(h->cf, node, childnuma);
         struct ftnode_fetch_extra bfe;
         fill_bfe_for_full_read(&bfe, h);
-        toku_pin_ftnode_off_client_thread(h, BP_BLOCKNUM(node, childnuma), childfullhash, &bfe, PL_WRITE_EXPENSIVE, 1, &node, &childa);
+        toku_pin_ftnode_with_dep_nodes(h, BP_BLOCKNUM(node, childnuma), childfullhash, &bfe, PL_WRITE_EXPENSIVE, 1, &node, &childa, true);
     }
     // for test
     call_flusher_thread_callback(flt_flush_before_pin_second_node_for_merge);
@@ -1437,7 +1436,7 @@ ft_merge_child(
         uint32_t childfullhash = compute_child_fullhash(h->cf, node, childnumb);
         struct ftnode_fetch_extra bfe;
         fill_bfe_for_full_read(&bfe, h);
-        toku_pin_ftnode_off_client_thread(h, BP_BLOCKNUM(node, childnumb), childfullhash, &bfe, PL_WRITE_EXPENSIVE, 2, dep_nodes, &childb);
+        toku_pin_ftnode_with_dep_nodes(h, BP_BLOCKNUM(node, childnumb), childfullhash, &bfe, PL_WRITE_EXPENSIVE, 2, dep_nodes, &childb, true);
     }
 
     if (toku_bnc_n_entries(BNC(node,childnuma))>0) {
@@ -1525,7 +1524,7 @@ ft_merge_child(
 
         // unlock the parent
         paranoid_invariant(node->dirty);
-        toku_unpin_ftnode_off_client_thread(h, node);
+        toku_unpin_ftnode(h, node);
     }
     else {
         // for test
@@ -1533,14 +1532,14 @@ ft_merge_child(
 
         // unlock the parent
         paranoid_invariant(node->dirty);
-        toku_unpin_ftnode_off_client_thread(h, node);
+        toku_unpin_ftnode(h, node);
-        toku_unpin_ftnode_off_client_thread(h, childb);
+        toku_unpin_ftnode(h, childb);
     }
     if (childa->height > 0 && fa->should_recursively_flush(childa, fa->extra)) {
         toku_ft_flush_some_child(h, childa, fa);
     }
     else {
-        toku_unpin_ftnode_off_client_thread(h, childa);
+        toku_unpin_ftnode(h, childa);
     }
 }
 
@@ -1575,7 +1574,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
     // Note that we don't read the entire node into memory yet.
     // The idea is let's try to do the minimum work before releasing the parent lock
     fill_bfe_for_min_read(&bfe, ft);
-    toku_pin_ftnode_off_client_thread(ft, targetchild, childfullhash, &bfe, PL_WRITE_EXPENSIVE, 1, &parent, &child);
+    toku_pin_ftnode_with_dep_nodes(ft, targetchild, childfullhash, &bfe, PL_WRITE_EXPENSIVE, 1, &parent, &child, true);
 
     // for test
     call_flusher_thread_callback(ft_flush_aflter_child_pin);
@@ -1591,7 +1590,6 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
     bool may_child_be_reactive = may_node_be_reactive(ft, child);
 
     paranoid_invariant(child->thisnodename.b!=0);
-    //VERIFY_NODE(brt, child);
 
     // only do the following work if there is a flush to perform
     if (toku_bnc_n_entries(BNC(parent, childnum)) > 0 || parent->height == 1) {
@@ -1614,7 +1612,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
     // reactive, we can unpin the parent
     //
     if (!may_child_be_reactive) {
-        toku_unpin_ftnode_off_client_thread(ft, parent);
+        toku_unpin_ftnode(ft, parent);
         parent = NULL;
     }
 
@@ -1632,7 +1630,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
     // for the root with a fresh one
     enum reactivity child_re = get_node_reactivity(ft, child);
     if (parent && child_re == RE_STABLE) {
-        toku_unpin_ftnode_off_client_thread(ft, parent);
+        toku_unpin_ftnode(ft, parent);
         parent = NULL;
     }
 
@@ -1671,7 +1669,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
         )
     {
         if (parent) {
-            toku_unpin_ftnode_off_client_thread(ft, parent);
+            toku_unpin_ftnode(ft, parent);
             parent = NULL;
         }
         //
@@ -1681,7 +1679,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
             toku_ft_flush_some_child(ft, child, fa);
         }
         else {
-            toku_unpin_ftnode_off_client_thread(ft, child);
+            toku_unpin_ftnode(ft, child);
         }
     }
     else if (child_re == RE_FISSIBLE) {
@@ -1837,7 +1835,7 @@ toku_ftnode_cleaner_callback(
         ct_flusher_advice_init(&fa, &fste, h->h->nodesize);
         toku_ft_flush_some_child(h, node, &fa);
     } else {
-        toku_unpin_ftnode_off_client_thread(h, node);
+        toku_unpin_ftnode(h, node);
     }
     return 0;
 }
@@ -1897,7 +1895,7 @@ static void flush_node_fun(void *fe_v)
             toku_ft_flush_some_child(fe->h, fe->node, &fa);
         }
         else {
-            toku_unpin_ftnode_off_client_thread(fe->h,fe->node);
+            toku_unpin_ftnode(fe->h,fe->node);
         }
     }
     else {

storage/tokudb/ft-index/ft/ft-flusher.h

@@ -124,7 +124,7 @@ typedef enum {
     FT_FLUSHER_SPLIT_NONLEAF,               // number of nonleaf nodes split
     FT_FLUSHER_MERGE_LEAF,                  // number of times leaf nodes are merged
     FT_FLUSHER_MERGE_NONLEAF,               // number of times nonleaf nodes are merged
-    FT_FLUSHER_BALANCE_LEAF,                // number of times a leaf node is balanced inside brt
+    FT_FLUSHER_BALANCE_LEAF,                // number of times a leaf node is balanced
     FT_FLUSHER_STATUS_NUM_ROWS
 } ft_flusher_status_entry;
 
@@ -230,7 +230,7 @@ void toku_ft_hot_get_status(FT_HOT_STATUS);
  * we go until the end of the FT.
  */
 int
-toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
+toku_ft_hot_optimize(FT_HANDLE ft_h, DBT* left, DBT* right,
                       int (*progress_callback)(void *extra, float progress),
                       void *progress_extra, uint64_t* loops_run);
 

storage/tokudb/ft-index/ft/ft-hot-flusher.cc

@@ -298,9 +298,9 @@ hot_flusher_destroy(struct hot_flusher_extra *flusher)
 // Entry point for Hot Optimize Table (HOT).  Note, this function is
 // not recursive.  It iterates over root-to-leaf paths.
 int
-toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
+toku_ft_hot_optimize(FT_HANDLE ft_handle, DBT* left, DBT* right,
-                      int (*progress_callback)(void *extra, float progress),
+                     int (*progress_callback)(void *extra, float progress),
-                      void *progress_extra, uint64_t* loops_run)
+                     void *progress_extra, uint64_t* loops_run)
 {
     toku::context flush_ctx(CTX_FLUSH);
 
@@ -316,7 +316,7 @@ toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
                                          // start of HOT operation
     (void) toku_sync_fetch_and_add(&STATUS_VALUE(FT_HOT_NUM_STARTED), 1);
 
-    toku_ft_note_hot_begin(brt);
+    toku_ft_note_hot_begin(ft_handle);
 
     // Higher level logic prevents a dictionary from being deleted or
     // truncated during a hot optimize operation.  Doing so would violate
@@ -329,17 +329,16 @@ toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
         {
             // Get root node (the first parent of each successive HOT
             // call.)
-            toku_calculate_root_offset_pointer(brt->ft, &root_key, &fullhash);
+            toku_calculate_root_offset_pointer(ft_handle->ft, &root_key, &fullhash);
             struct ftnode_fetch_extra bfe;
-            fill_bfe_for_full_read(&bfe, brt->ft);
+            fill_bfe_for_full_read(&bfe, ft_handle->ft);
-            toku_pin_ftnode_off_client_thread(brt->ft,
+            toku_pin_ftnode(ft_handle->ft,
-                                               (BLOCKNUM) root_key,
+                            (BLOCKNUM) root_key,
-                                               fullhash,
+                            fullhash,
-                                               &bfe,
+                            &bfe,
-                                               PL_WRITE_EXPENSIVE, 
+                            PL_WRITE_EXPENSIVE, 
-                                               0,
+                            &root,
-                                               NULL,
+                            true);
-                                               &root);
             toku_assert_entire_node_in_memory(root);
         }
 
@@ -365,12 +364,12 @@ toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
         // This should recurse to the bottom of the tree and then
         // return.
         if (root->height > 0) {
-            toku_ft_flush_some_child(brt->ft, root, &advice);
+            toku_ft_flush_some_child(ft_handle->ft, root, &advice);
         } else {
             // Since there are no children to flush, we should abort
             // the HOT call.
             flusher.rightmost_leaf_seen = 1;
-            toku_unpin_ftnode_off_client_thread(brt->ft, root);
+            toku_unpin_ftnode(ft_handle->ft, root);
         }
 
         // Set the highest pivot key seen here, since the parent may
@@ -386,8 +385,8 @@ toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
         else if (right) {
             // if we have flushed past the bounds set for us,
             // set rightmost_leaf_seen so we exit
-            FAKE_DB(db, &brt->ft->cmp_descriptor);
+            FAKE_DB(db, &ft_handle->ft->cmp_descriptor);
-            int cmp = brt->ft->compare_fun(&db, &flusher.max_current_key, right);
+            int cmp = ft_handle->ft->compare_fun(&db, &flusher.max_current_key, right);
             if (cmp > 0) {
                 flusher.rightmost_leaf_seen = 1;
             }
@@ -417,7 +416,7 @@ toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
         if (r == 0) { success = true; }
 
         {
-            toku_ft_note_hot_complete(brt, success, msn_at_start_of_hot);
+            toku_ft_note_hot_complete(ft_handle, success, msn_at_start_of_hot);
         }
 
         if (success) {

storage/tokudb/ft-index/ft/ft-internal.h

@@ -92,7 +92,7 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "toku_config.h"
+#include <portability/toku_config.h>
 #include <toku_race_tools.h>
 
 // Symbol TOKUDB_REVISION is not defined by fractal-tree makefiles, so
@@ -118,7 +118,7 @@ PATENT RIGHTS GRANT:
 #include "bndata.h"
 
 enum { KEY_VALUE_OVERHEAD = 8 }; /* Must store the two lengths. */
-enum { FT_CMD_OVERHEAD = (2 + sizeof(MSN)) };   // the type plus freshness plus MSN
+enum { FT_MSG_OVERHEAD = (2 + sizeof(MSN)) };   // the type plus freshness plus MSN
 enum { FT_DEFAULT_FANOUT = 16 };
 enum { FT_DEFAULT_NODE_SIZE = 4 * 1024 * 1024 };
 enum { FT_DEFAULT_BASEMENT_NODE_SIZE = 128 * 1024 };
@@ -454,7 +454,7 @@ enum {
 
 uint32_t compute_child_fullhash (CACHEFILE cf, FTNODE node, int childnum);
 
-// The brt_header is not managed by the cachetable.  Instead, it hangs off the cachefile as userdata.
+// The ft_header is not managed by the cachetable.  Instead, it hangs off the cachefile as userdata.
 
 enum ft_type {FT_CURRENT=1, FT_CHECKPOINT_INPROGRESS};
 
@@ -470,7 +470,7 @@ struct ft_header {
     // LSN of creation of "checkpoint-begin" record in log.
     LSN checkpoint_lsn;
 
-    // see brt_layout_version.h.  maybe don't need this if we assume
+    // see ft_layout_version.h.  maybe don't need this if we assume
     // it's always the current version after deserializing
     const int layout_version;
     // different (<) from layout_version if upgraded from a previous
@@ -504,7 +504,7 @@ struct ft_header {
     enum toku_compression_method compression_method;
     unsigned int fanout;
 
-    // Current Minimum MSN to be used when upgrading pre-MSN BRT's.
+    // Current Minimum MSN to be used when upgrading pre-MSN FT's.
     // This is decremented from our currnt MIN_MSN so as not to clash
     // with any existing 'normal' MSN's.
     MSN highest_unused_msn_for_upgrade;
@@ -526,7 +526,7 @@ struct ft_header {
     STAT64INFO_S on_disk_stats;
 };
 
-// brt_header is always the current version.
+// ft_header is always the current version.
 struct ft {
     FT_HEADER h;
     FT_HEADER checkpoint_header;
@@ -708,7 +708,7 @@ void toku_assert_entire_node_in_memory(FTNODE node);
 // append a child node to a parent node
 void toku_ft_nonleaf_append_child(FTNODE node, FTNODE child, const DBT *pivotkey);
 
-// append a cmd to a nonleaf node child buffer
+// append a message to a nonleaf node child buffer
 void toku_ft_append_to_child_buffer(ft_compare_func compare_fun, DESCRIPTOR desc, FTNODE node, int childnum, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, const DBT *key, const DBT *val);
 
 STAT64INFO_S toku_get_and_clear_basement_stats(FTNODE leafnode);
@@ -768,7 +768,7 @@ static inline CACHETABLE_WRITE_CALLBACK get_write_callbacks_for_node(FT h) {
 
 static const FTNODE null_ftnode=0;
 
-/* a brt cursor is represented as a kv pair in a tree */
+/* an ft cursor is represented as a kv pair in a tree */
 struct ft_cursor {
     struct toku_list cursors_link;
     FT_HANDLE ft_handle;
@@ -969,7 +969,7 @@ __attribute__((nonnull))
 void toku_ft_bn_update_max_msn(FTNODE node, MSN max_msn_applied, int child_to_read);
 
 __attribute__((const,nonnull))
-size_t toku_ft_msg_memsize_in_fifo(FT_MSG cmd);
+size_t toku_ft_msg_memsize_in_fifo(FT_MSG msg);
 
 int
 toku_ft_search_which_child(
@@ -1018,26 +1018,25 @@ int toku_ftnode_hot_next_child(FTNODE node,
 /* Stuff for testing */
 // toku_testsetup_initialize() must be called before any other test_setup_xxx() functions are called.
 void toku_testsetup_initialize(void);
-int toku_testsetup_leaf(FT_HANDLE brt, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens);
+int toku_testsetup_leaf(FT_HANDLE ft_h, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens);
-int toku_testsetup_nonleaf (FT_HANDLE brt, int height, BLOCKNUM *diskoff, int n_children, BLOCKNUM *children, char **keys, int *keylens);
+int toku_testsetup_nonleaf (FT_HANDLE ft_h, int height, BLOCKNUM *diskoff, int n_children, BLOCKNUM *children, char **keys, int *keylens);
-int toku_testsetup_root(FT_HANDLE brt, BLOCKNUM);
+int toku_testsetup_root(FT_HANDLE ft_h, BLOCKNUM);
-int toku_testsetup_get_sersize(FT_HANDLE brt, BLOCKNUM); // Return the size on disk.
+int toku_testsetup_get_sersize(FT_HANDLE ft_h, BLOCKNUM); // Return the size on disk.
-int toku_testsetup_insert_to_leaf (FT_HANDLE brt, BLOCKNUM, const char *key, int keylen, const char *val, int vallen);
+int toku_testsetup_insert_to_leaf (FT_HANDLE ft_h, BLOCKNUM, const char *key, int keylen, const char *val, int vallen);
-int toku_testsetup_insert_to_nonleaf (FT_HANDLE brt, BLOCKNUM, enum ft_msg_type, const char *key, int keylen, const char *val, int vallen);
+int toku_testsetup_insert_to_nonleaf (FT_HANDLE ft_h, BLOCKNUM, enum ft_msg_type, const char *key, int keylen, const char *val, int vallen);
 void toku_pin_node_with_min_bfe(FTNODE* node, BLOCKNUM b, FT_HANDLE t);
 
-// toku_ft_root_put_cmd() accepts non-constant cmd because this is where we set the msn
+void toku_ft_root_put_msg(FT h, FT_MSG msg, txn_gc_info *gc_info);
-void toku_ft_root_put_cmd(FT h, FT_MSG_S * cmd, txn_gc_info *gc_info);
 
 void
 toku_get_node_for_verify(
     BLOCKNUM blocknum,
-    FT_HANDLE brt,
+    FT_HANDLE ft_h,
     FTNODE* nodep
     );
 
 int
-toku_verify_ftnode (FT_HANDLE brt,
+toku_verify_ftnode (FT_HANDLE ft_h,
                     MSN rootmsn, MSN parentmsn, bool messages_exist_above,
                      FTNODE node, int height,
                      const DBT *lesser_pivot,               // Everything in the subtree should be > lesser_pivot.  (lesser_pivot==NULL if there is no lesser pivot.)
@@ -1185,6 +1184,8 @@ typedef enum {
     FT_PRO_NUM_STOP_LOCK_CHILD,
     FT_PRO_NUM_STOP_CHILD_INMEM,
     FT_PRO_NUM_DIDNT_WANT_PROMOTE,
+    FT_BASEMENT_DESERIALIZE_FIXED_KEYSIZE, // how many basement nodes were deserialized with a fixed keysize
+    FT_BASEMENT_DESERIALIZE_VARIABLE_KEYSIZE, // how many basement nodes were deserialized with a variable keysize
     FT_STATUS_NUM_ROWS
 } ft_status_entry;
 
@@ -1196,9 +1197,9 @@ typedef struct {
 void toku_ft_get_status(FT_STATUS);
 
 void
-toku_ft_bn_apply_cmd_once (
+toku_ft_bn_apply_msg_once(
     BASEMENTNODE bn,
-    const FT_MSG cmd,
+    const FT_MSG msg,
     uint32_t idx,
     LEAFENTRY le,
     txn_gc_info *gc_info,
@@ -1207,38 +1208,38 @@ toku_ft_bn_apply_cmd_once (
     );
 
 void
-toku_ft_bn_apply_cmd (
+toku_ft_bn_apply_msg(
     ft_compare_func compare_fun,
     ft_update_func update_fun,
     DESCRIPTOR desc,
     BASEMENTNODE bn,
-    FT_MSG cmd,
+    FT_MSG msg,
     txn_gc_info *gc_info,
     uint64_t *workdone,
     STAT64INFO stats_to_update
     );
 
 void
-toku_ft_leaf_apply_cmd (
+toku_ft_leaf_apply_msg(
     ft_compare_func compare_fun,
     ft_update_func update_fun,
     DESCRIPTOR desc,
     FTNODE node,
     int target_childnum,
-    FT_MSG cmd,
+    FT_MSG msg,
     txn_gc_info *gc_info,
     uint64_t *workdone,
     STAT64INFO stats_to_update
     );
 
 void
-toku_ft_node_put_cmd (
+toku_ft_node_put_msg(
     ft_compare_func compare_fun,
     ft_update_func update_fun,
     DESCRIPTOR desc,
     FTNODE node,
     int target_childnum,
-    FT_MSG cmd,
+    FT_MSG msg,
     bool is_fresh,
     txn_gc_info *gc_info,
     size_t flow_deltas[],

storage/tokudb/ft-index/ft/ft-node-deserialize.cc

@@ -193,7 +193,7 @@ check_node_info_checksum(struct rbuf *rb)
 {
     int r = 0;
     // Verify checksum of header stored.
-    uint32_t checksum = x1764_memory(rb->buf, rb->ndone);
+    uint32_t checksum = toku_x1764_memory(rb->buf, rb->ndone);
     uint32_t stored_checksum = rbuf_int(rb);
 
     if (stored_checksum != checksum) {
@@ -229,7 +229,7 @@ check_legacy_end_checksum(struct rbuf *rb)
 {
     int r = 0;
     uint32_t expected_xsum = rbuf_int(rb);
-    uint32_t actual_xsum = x1764_memory(rb->buf, rb->size - 4);
+    uint32_t actual_xsum = toku_x1764_memory(rb->buf, rb->size - 4);
     if (expected_xsum != actual_xsum) {
         r = TOKUDB_BAD_CHECKSUM;
     }

storage/tokudb/ft-index/ft/ft-ops.h

@@ -103,7 +103,7 @@ PATENT RIGHTS GRANT:
 
 // A callback function is invoked with the key, and the data.
 // The pointers (to the bytevecs) must not be modified.  The data must be copied out before the callback function returns.
-// Note: In the thread-safe version, the brt node remains locked while the callback function runs.  So return soon, and don't call the BRT code from the callback function.
+// Note: In the thread-safe version, the ftnode remains locked while the callback function runs.  So return soon, and don't call the ft code from the callback function.
 // If the callback function returns a nonzero value (an error code), then that error code is returned from the get function itself.
 // The cursor object will have been updated (so that if result==0 the current value is the value being passed)
 //  (If r!=0 then the cursor won't have been updated.)
@@ -141,9 +141,9 @@ void toku_ft_handle_set_fanout(FT_HANDLE, unsigned int fanout);
 void toku_ft_handle_get_fanout(FT_HANDLE, unsigned int *fanout);
 
 void toku_ft_set_bt_compare(FT_HANDLE, ft_compare_func);
-ft_compare_func toku_ft_get_bt_compare (FT_HANDLE brt);
+ft_compare_func toku_ft_get_bt_compare (FT_HANDLE ft_h);
 
-void toku_ft_set_redirect_callback(FT_HANDLE brt, on_redirect_callback redir_cb, void* extra);
+void toku_ft_set_redirect_callback(FT_HANDLE ft_h, on_redirect_callback redir_cb, void* extra);
 
 // How updates (update/insert/deletes) work:
 // There are two flavers of upsertdels:  Singleton and broadcast.
@@ -181,7 +181,7 @@ void toku_ft_set_redirect_callback(FT_HANDLE brt, on_redirect_callback redir_cb,
 // Implementation note: Acquires a write lock on the entire database.
 //  This function works by sending an BROADCAST-UPDATE message containing
 //   the key and the extra.
-void toku_ft_set_update(FT_HANDLE brt, ft_update_func update_fun);
+void toku_ft_set_update(FT_HANDLE ft_h, ft_update_func update_fun);
 
 int toku_ft_handle_open(FT_HANDLE, const char *fname_in_env,
 		  int is_create, int only_create, CACHETABLE ct, TOKUTXN txn)  __attribute__ ((warn_unused_result));
@@ -208,60 +208,60 @@ toku_ft_handle_open_with_dict_id(
     DICTIONARY_ID use_dictionary_id
     )  __attribute__ ((warn_unused_result));
 
-int toku_ft_lookup (FT_HANDLE brt, DBT *k, FT_GET_CALLBACK_FUNCTION getf, void *getf_v)  __attribute__ ((warn_unused_result));
+int toku_ft_lookup (FT_HANDLE ft_h, DBT *k, FT_GET_CALLBACK_FUNCTION getf, void *getf_v)  __attribute__ ((warn_unused_result));
 
-// Effect: Insert a key and data pair into a brt
+// Effect: Insert a key and data pair into an ft
-void toku_ft_insert (FT_HANDLE brt, DBT *k, DBT *v, TOKUTXN txn);
+void toku_ft_insert (FT_HANDLE ft_h, DBT *k, DBT *v, TOKUTXN txn);
 
 // Effect: Optimize the ft
-void toku_ft_optimize (FT_HANDLE brt);
+void toku_ft_optimize (FT_HANDLE ft_h);
 
-// Effect: Insert a key and data pair into a brt if the oplsn is newer than the brt lsn.  This function is called during recovery.
+// Effect: Insert a key and data pair into an ft if the oplsn is newer than the ft's lsn.  This function is called during recovery.
-void toku_ft_maybe_insert (FT_HANDLE brt, DBT *k, DBT *v, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging, enum ft_msg_type type);
+void toku_ft_maybe_insert (FT_HANDLE ft_h, DBT *k, DBT *v, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging, enum ft_msg_type type);
 
-// Effect: Send an update message into a brt.  This function is called
+// Effect: Send an update message into an ft.  This function is called
 // during recovery.
-void toku_ft_maybe_update(FT_HANDLE brt, const DBT *key, const DBT *update_function_extra, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging);
+void toku_ft_maybe_update(FT_HANDLE ft_h, const DBT *key, const DBT *update_function_extra, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging);
 
-// Effect: Send a broadcasting update message into a brt.  This function
+// Effect: Send a broadcasting update message into an ft.  This function
 // is called during recovery.
-void toku_ft_maybe_update_broadcast(FT_HANDLE brt, const DBT *update_function_extra, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging, bool is_resetting_op);
+void toku_ft_maybe_update_broadcast(FT_HANDLE ft_h, const DBT *update_function_extra, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging, bool is_resetting_op);
 
 void toku_ft_load_recovery(TOKUTXN txn, FILENUM old_filenum, char const * new_iname, int do_fsync, int do_log, LSN *load_lsn);
-void toku_ft_load(FT_HANDLE brt, TOKUTXN txn, char const * new_iname, int do_fsync, LSN *get_lsn);
+void toku_ft_load(FT_HANDLE ft_h, TOKUTXN txn, char const * new_iname, int do_fsync, LSN *get_lsn);
 void toku_ft_hot_index_recovery(TOKUTXN txn, FILENUMS filenums, int do_fsync, int do_log, LSN *hot_index_lsn);
-void toku_ft_hot_index(FT_HANDLE brt, TOKUTXN txn, FILENUMS filenums, int do_fsync, LSN *lsn);
+void toku_ft_hot_index(FT_HANDLE ft_h, TOKUTXN txn, FILENUMS filenums, int do_fsync, LSN *lsn);
 
-void toku_ft_log_put_multiple (TOKUTXN txn, FT_HANDLE src_ft, FT_HANDLE *brts, uint32_t num_fts, const DBT *key, const DBT *val);
+void toku_ft_log_put_multiple (TOKUTXN txn, FT_HANDLE src_ft, FT_HANDLE *fts, uint32_t num_fts, const DBT *key, const DBT *val);
-void toku_ft_log_put (TOKUTXN txn, FT_HANDLE brt, const DBT *key, const DBT *val);
+void toku_ft_log_put (TOKUTXN txn, FT_HANDLE ft_h, const DBT *key, const DBT *val);
-void toku_ft_log_del_multiple (TOKUTXN txn, FT_HANDLE src_ft, FT_HANDLE *brts, uint32_t num_fts, const DBT *key, const DBT *val);
+void toku_ft_log_del_multiple (TOKUTXN txn, FT_HANDLE src_ft, FT_HANDLE *fts, uint32_t num_fts, const DBT *key, const DBT *val);
-void toku_ft_log_del (TOKUTXN txn, FT_HANDLE brt, const DBT *key);
+void toku_ft_log_del (TOKUTXN txn, FT_HANDLE ft_h, const DBT *key);
 
-// Effect: Delete a key from a brt
+// Effect: Delete a key from an ft
-void toku_ft_delete (FT_HANDLE brt, DBT *k, TOKUTXN txn);
+void toku_ft_delete (FT_HANDLE ft_h, DBT *k, TOKUTXN txn);
 
-// Effect: Delete a key from a brt if the oplsn is newer than the brt lsn.  This function is called during recovery.
+// Effect: Delete a key from an ft if the oplsn is newer than the ft lsn.  This function is called during recovery.
-void toku_ft_maybe_delete (FT_HANDLE brt, DBT *k, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging);
+void toku_ft_maybe_delete (FT_HANDLE ft_h, DBT *k, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging);
 
 TXNID toku_ft_get_oldest_referenced_xid_estimate(FT_HANDLE ft_h);
 TXN_MANAGER toku_ft_get_txn_manager(FT_HANDLE ft_h);
 
-void toku_ft_send_insert(FT_HANDLE brt, DBT *key, DBT *val, XIDS xids, enum ft_msg_type type, txn_gc_info *gc_info);
+void toku_ft_send_insert(FT_HANDLE ft_h, DBT *key, DBT *val, XIDS xids, enum ft_msg_type type, txn_gc_info *gc_info);
-void toku_ft_send_delete(FT_HANDLE brt, DBT *key, XIDS xids, txn_gc_info *gc_info);
+void toku_ft_send_delete(FT_HANDLE ft_h, DBT *key, XIDS xids, txn_gc_info *gc_info);
-void toku_ft_send_commit_any(FT_HANDLE brt, DBT *key, XIDS xids, txn_gc_info *gc_info);
+void toku_ft_send_commit_any(FT_HANDLE ft_h, DBT *key, XIDS xids, txn_gc_info *gc_info);
 
 int toku_close_ft_handle_nolsn (FT_HANDLE, char **error_string)  __attribute__ ((warn_unused_result));
 
-int toku_dump_ft (FILE *,FT_HANDLE brt)  __attribute__ ((warn_unused_result));
+int toku_dump_ft (FILE *,FT_HANDLE ft_h)  __attribute__ ((warn_unused_result));
 
 extern int toku_ft_debug_mode;
-int toku_verify_ft (FT_HANDLE brt)  __attribute__ ((warn_unused_result));
+int toku_verify_ft (FT_HANDLE ft_h)  __attribute__ ((warn_unused_result));
-int toku_verify_ft_with_progress (FT_HANDLE brt, int (*progress_callback)(void *extra, float progress), void *extra, int verbose, int keep_going)  __attribute__ ((warn_unused_result));
+int toku_verify_ft_with_progress (FT_HANDLE ft_h, int (*progress_callback)(void *extra, float progress), void *extra, int verbose, int keep_going)  __attribute__ ((warn_unused_result));
 
 typedef struct ft_cursor *FT_CURSOR;
 int toku_ft_cursor (FT_HANDLE, FT_CURSOR*, TOKUTXN, bool, bool)  __attribute__ ((warn_unused_result));
 void toku_ft_cursor_set_leaf_mode(FT_CURSOR);
-// Sets a boolean on the brt cursor that prevents uncessary copying of
+// Sets a boolean on the ft cursor that prevents uncessary copying of
 // the cursor duing a one query.
 void toku_ft_cursor_set_temporary(FT_CURSOR);
 void toku_ft_cursor_remove_restriction(FT_CURSOR);
@@ -278,7 +278,7 @@ int toku_ft_cursor_next(FT_CURSOR cursor, FT_GET_CALLBACK_FUNCTION getf, void *g
 int toku_ft_cursor_prev(FT_CURSOR cursor, FT_GET_CALLBACK_FUNCTION getf, void *getf_v)  __attribute__ ((warn_unused_result));
 int toku_ft_cursor_current(FT_CURSOR cursor, int op, FT_GET_CALLBACK_FUNCTION getf, void *getf_v)  __attribute__ ((warn_unused_result));
 int toku_ft_cursor_set(FT_CURSOR cursor, DBT *key, FT_GET_CALLBACK_FUNCTION getf, void *getf_v)  __attribute__ ((warn_unused_result));
-int toku_ft_cursor_set_range(FT_CURSOR cursor, DBT *key, FT_GET_CALLBACK_FUNCTION getf, void *getf_v)  __attribute__ ((warn_unused_result));
+int toku_ft_cursor_set_range(FT_CURSOR cursor, DBT *key, DBT *key_bound, FT_GET_CALLBACK_FUNCTION getf, void *getf_v)  __attribute__ ((warn_unused_result));
 int toku_ft_cursor_set_range_reverse(FT_CURSOR cursor, DBT *key, FT_GET_CALLBACK_FUNCTION getf, void *getf_v)  __attribute__ ((warn_unused_result));
 int toku_ft_cursor_get_both_range(FT_CURSOR cursor, DBT *key, DBT *val, FT_GET_CALLBACK_FUNCTION getf, void *getf_v)  __attribute__ ((warn_unused_result));
 int toku_ft_cursor_get_both_range_reverse(FT_CURSOR cursor, DBT *key, DBT *val, FT_GET_CALLBACK_FUNCTION getf, void *getf_v)  __attribute__ ((warn_unused_result));
@@ -298,8 +298,8 @@ enum ft_flags {
     TOKU_DB_VALCMP_BUILTIN_13  = (1<<3),
 };
 
-void toku_ft_keyrange(FT_HANDLE brt, DBT *key, uint64_t *less,  uint64_t *equal,  uint64_t *greater);
+void toku_ft_keyrange(FT_HANDLE ft_h, DBT *key, uint64_t *less,  uint64_t *equal,  uint64_t *greater);
-void toku_ft_keysrange(FT_HANDLE brt, DBT* key_left, DBT* key_right, uint64_t *less_p, uint64_t* equal_left_p, uint64_t* middle_p, uint64_t* equal_right_p, uint64_t* greater_p, bool* middle_3_exact_p);
+void toku_ft_keysrange(FT_HANDLE ft_h, DBT* key_left, DBT* key_right, uint64_t *less_p, uint64_t* equal_left_p, uint64_t* middle_p, uint64_t* equal_right_p, uint64_t* greater_p, bool* middle_3_exact_p);
 
 int toku_ft_get_key_after_bytes(FT_HANDLE ft_h, const DBT *start_key, uint64_t skip_len, void (*callback)(const DBT *end_key, uint64_t actually_skipped, void *extra), void *cb_extra);
 
@@ -341,9 +341,9 @@ void toku_maybe_preallocate_in_file (int fd, int64_t size, int64_t expected_size
 // Effect: make the file bigger by either doubling it or growing by 16MiB whichever is less, until it is at least size
 // Return 0 on success, otherwise an error number.
 
-int toku_ft_get_fragmentation(FT_HANDLE brt, TOKU_DB_FRAGMENTATION report) __attribute__ ((warn_unused_result));
+int toku_ft_get_fragmentation(FT_HANDLE ft_h, TOKU_DB_FRAGMENTATION report) __attribute__ ((warn_unused_result));
 
-bool toku_ft_is_empty_fast (FT_HANDLE brt) __attribute__ ((warn_unused_result));
+bool toku_ft_is_empty_fast (FT_HANDLE ft_h) __attribute__ ((warn_unused_result));
 // Effect: Return true if there are no messages or leaf entries in the tree.  If so, it's empty.  If there are messages  or leaf entries, we say it's not empty
 // even though if we were to optimize the tree it might turn out that they are empty.
 
@@ -358,4 +358,6 @@ extern bool garbage_collection_debug;
 void toku_ft_set_direct_io(bool direct_io_on);
 void toku_ft_set_compress_buffers_before_eviction(bool compress_buffers);
 
+void toku_note_deserialized_basement_node(bool fixed_key_size);
+
 #endif

storage/tokudb/ft-index/ft/ft-search.h

@@ -136,15 +136,18 @@ typedef struct ft_search {
     //   no guarantee that we will get everything pinned again.  We ought to keep nodes pinned when we retry, except that on the
     //   way out with a DB_NOTFOUND we ought to unpin those nodes.  See #3528.
     DBT pivot_bound;
+    const DBT *k_bound;
 } ft_search_t;
 
 /* initialize the search compare object */
-static inline ft_search_t *ft_search_init(ft_search_t *so, ft_search_compare_func_t compare, enum ft_search_direction_e direction, const DBT *k, void *context) {
+static inline ft_search_t *ft_search_init(ft_search_t *so, ft_search_compare_func_t compare, enum ft_search_direction_e direction, 
+                                          const DBT *k, const DBT *k_bound, void *context) {
     so->compare = compare;
     so->direction = direction;
     so->k = k;
     so->context = context;
     toku_init_dbt(&so->pivot_bound);
+    so->k_bound = k_bound;
     return so;
 }
 

storage/tokudb/ft-index/ft/ft-serialize.cc

@@ -132,7 +132,7 @@ toku_serialize_descriptor_contents_to_fd(int fd, const DESCRIPTOR desc, DISKOFF
     toku_serialize_descriptor_contents_to_wbuf(&w, desc);
     {
         //Add checksum
-        uint32_t checksum = x1764_finish(&w.checksum);
+        uint32_t checksum = toku_x1764_finish(&w.checksum);
         wbuf_int(&w, checksum);
     }
     lazy_assert(w.ndone==w.size);
@@ -185,7 +185,7 @@ deserialize_descriptor_from(int fd, BLOCK_TABLE bt, DESCRIPTOR desc, int layout_
             }
             {
                 // check the checksum
-                uint32_t x1764 = x1764_memory(dbuf, size-4);
+                uint32_t x1764 = toku_x1764_memory(dbuf, size-4);
                 //printf("%s:%d read from %ld (x1764 offset=%ld) size=%ld\n", __FILE__, __LINE__, block_translation_address_on_disk, offset, block_translation_size_on_disk);
                 uint32_t stored_x1764 = toku_dtoh32(*(int*)(dbuf + size-4));
                 if (x1764 != stored_x1764) {
@@ -210,7 +210,7 @@ exit:
 
 int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
 // Effect: Deserialize the ft header.
-//   We deserialize brt header only once and then share everything with all the brts.
+//   We deserialize ft_header only once and then share everything with all the FTs.
 {
     int r;
     FT ft = NULL;
@@ -462,6 +462,7 @@ serialize_ft_min_size (uint32_t version) {
     size_t size = 0;
 
     switch(version) {
+    case FT_LAYOUT_VERSION_26:
     case FT_LAYOUT_VERSION_25:
     case FT_LAYOUT_VERSION_24:
     case FT_LAYOUT_VERSION_23:
@@ -623,7 +624,7 @@ int deserialize_ft_from_fd_into_rbuf(int fd,
 
     //Verify checksum (FT_LAYOUT_VERSION_13 or later, when checksum function changed)
     uint32_t calculated_x1764;
-    calculated_x1764 = x1764_memory(rb->buf, rb->size-4);
+    calculated_x1764 = toku_x1764_memory(rb->buf, rb->size-4);
     uint32_t stored_x1764;
     stored_x1764 = toku_dtoh32(*(int*)(rb->buf+rb->size-4));
     if (calculated_x1764 != stored_x1764) {
@@ -803,7 +804,7 @@ void toku_serialize_ft_to_wbuf (
     wbuf_char(wbuf, (unsigned char) h->compression_method);
     wbuf_MSN(wbuf, h->highest_unused_msn_for_upgrade);
     wbuf_MSN(wbuf, h->max_msn_in_ft);
-    uint32_t checksum = x1764_finish(&wbuf->checksum);
+    uint32_t checksum = toku_x1764_finish(&wbuf->checksum);
     wbuf_int(wbuf, checksum);
     lazy_assert(wbuf->ndone == wbuf->size);
 }

storage/tokudb/ft-index/ft/ft-test-helpers.cc

@@ -96,7 +96,7 @@ PATENT RIGHTS GRANT:
 #include "fttypes.h"
 #include "ule.h"
 
-// dummymsn needed to simulate msn because messages are injected at a lower level than toku_ft_root_put_cmd()
+// dummymsn needed to simulate msn because messages are injected at a lower level than toku_ft_root_put_msg()
 #define MIN_DUMMYMSN ((MSN) {(uint64_t)1 << 62})
 static MSN dummymsn;      
 static int testsetup_initialized = 0;
@@ -119,10 +119,10 @@ next_dummymsn(void) {
 
 
 bool ignore_if_was_already_open;
-int toku_testsetup_leaf(FT_HANDLE brt, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens) {
+int toku_testsetup_leaf(FT_HANDLE ft_handle, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens) {
     FTNODE node;
     assert(testsetup_initialized);
-    toku_create_new_ftnode(brt, &node, 0, n_children);
+    toku_create_new_ftnode(ft_handle, &node, 0, n_children);
     int i;
     for (i=0; i<n_children; i++) {
         BP_STATE(node,i) = PT_AVAIL;
@@ -134,15 +134,15 @@ int toku_testsetup_leaf(FT_HANDLE brt, BLOCKNUM *blocknum, int n_children, char
     }
 
     *blocknum = node->thisnodename;
-    toku_unpin_ftnode(brt->ft, node);
+    toku_unpin_ftnode(ft_handle->ft, node);
     return 0;
 }
 
 // Don't bother to clean up carefully if something goes wrong.  (E.g., it's OK to have malloced stuff that hasn't been freed.)
-int toku_testsetup_nonleaf (FT_HANDLE brt, int height, BLOCKNUM *blocknum, int n_children, BLOCKNUM *children, char **keys, int *keylens) {
+int toku_testsetup_nonleaf (FT_HANDLE ft_handle, int height, BLOCKNUM *blocknum, int n_children, BLOCKNUM *children, char **keys, int *keylens) {
     FTNODE node;
     assert(testsetup_initialized);
-    toku_create_new_ftnode(brt, &node, height, n_children);
+    toku_create_new_ftnode(ft_handle, &node, height, n_children);
     int i;
     for (i=0; i<n_children; i++) {
         BP_BLOCKNUM(node, i) = children[i];
@@ -153,28 +153,28 @@ int toku_testsetup_nonleaf (FT_HANDLE brt, int height, BLOCKNUM *blocknum, int n
         node->totalchildkeylens += keylens[i];
     }
     *blocknum = node->thisnodename;
-    toku_unpin_ftnode(brt->ft, node);
+    toku_unpin_ftnode(ft_handle->ft, node);
     return 0;
 }
 
-int toku_testsetup_root(FT_HANDLE brt, BLOCKNUM blocknum) {
+int toku_testsetup_root(FT_HANDLE ft_handle, BLOCKNUM blocknum) {
     assert(testsetup_initialized);
-    brt->ft->h->root_blocknum = blocknum;
+    ft_handle->ft->h->root_blocknum = blocknum;
     return 0;
 }
 
-int toku_testsetup_get_sersize(FT_HANDLE brt, BLOCKNUM diskoff) // Return the size on disk
+int toku_testsetup_get_sersize(FT_HANDLE ft_handle, BLOCKNUM diskoff) // Return the size on disk
 {
     assert(testsetup_initialized);
     void *node_v;
     struct ftnode_fetch_extra bfe;
-    fill_bfe_for_full_read(&bfe, brt->ft);
+    fill_bfe_for_full_read(&bfe, ft_handle->ft);
     int r  = toku_cachetable_get_and_pin(
-        brt->ft->cf, diskoff,
+        ft_handle->ft->cf, diskoff,
-        toku_cachetable_hash(brt->ft->cf, diskoff),
+        toku_cachetable_hash(ft_handle->ft->cf, diskoff),
         &node_v,
         NULL,
-        get_write_callbacks_for_node(brt->ft),
+        get_write_callbacks_for_node(ft_handle->ft),
         toku_ftnode_fetch_callback,
         toku_ftnode_pf_req_callback,
         toku_ftnode_pf_callback,
@@ -184,25 +184,25 @@ int toku_testsetup_get_sersize(FT_HANDLE brt, BLOCKNUM diskoff) // Return the si
     assert(r==0);
     FTNODE CAST_FROM_VOIDP(node, node_v);
     int size = toku_serialize_ftnode_size(node);
-    toku_unpin_ftnode(brt->ft, node);
+    toku_unpin_ftnode(ft_handle->ft, node);
     return size;
 }
 
-int toku_testsetup_insert_to_leaf (FT_HANDLE brt, BLOCKNUM blocknum, const char *key, int keylen, const char *val, int vallen) {
+int toku_testsetup_insert_to_leaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, const char *key, int keylen, const char *val, int vallen) {
     void *node_v;
     int r;
 
     assert(testsetup_initialized);
 
     struct ftnode_fetch_extra bfe;
-    fill_bfe_for_full_read(&bfe, brt->ft);
+    fill_bfe_for_full_read(&bfe, ft_handle->ft);
     r = toku_cachetable_get_and_pin(
-        brt->ft->cf,
+        ft_handle->ft->cf,
         blocknum,
-        toku_cachetable_hash(brt->ft->cf, blocknum),
+        toku_cachetable_hash(ft_handle->ft->cf, blocknum),
         &node_v,
         NULL,
-        get_write_callbacks_for_node(brt->ft),
+        get_write_callbacks_for_node(ft_handle->ft),
 	toku_ftnode_fetch_callback,
         toku_ftnode_pf_req_callback,
         toku_ftnode_pf_callback,
@@ -216,19 +216,19 @@ int toku_testsetup_insert_to_leaf (FT_HANDLE brt, BLOCKNUM blocknum, const char
 
     DBT keydbt,valdbt;
     MSN msn = next_dummymsn();
-    FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(),
+    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(),
                      .u = { .id = { toku_fill_dbt(&keydbt, key, keylen),
                                     toku_fill_dbt(&valdbt, val, vallen) } } };
 
     static size_t zero_flow_deltas[] = { 0, 0 };
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true);
-    toku_ft_node_put_cmd (
+    toku_ft_node_put_msg (
-        brt->ft->compare_fun,
+        ft_handle->ft->compare_fun,
-        brt->ft->update_fun,
+        ft_handle->ft->update_fun,
-        &brt->ft->cmp_descriptor,
+        &ft_handle->ft->cmp_descriptor,
         node,
         -1,
-        &cmd,
+        &msg,
         true,
         &gc_info,
         zero_flow_deltas,
@@ -237,7 +237,7 @@ int toku_testsetup_insert_to_leaf (FT_HANDLE brt, BLOCKNUM blocknum, const char
 
     toku_verify_or_set_counts(node);
 
-    toku_unpin_ftnode(brt->ft, node);
+    toku_unpin_ftnode(ft_handle->ft, node);
     return 0;
 }
 
@@ -254,33 +254,32 @@ toku_pin_node_with_min_bfe(FTNODE* node, BLOCKNUM b, FT_HANDLE t)
 {
     struct ftnode_fetch_extra bfe;
     fill_bfe_for_min_read(&bfe, t->ft);
-    toku_pin_ftnode_off_client_thread(
+    toku_pin_ftnode(
         t->ft, 
         b,
         toku_cachetable_hash(t->ft->cf, b),
         &bfe,
         PL_WRITE_EXPENSIVE,
-        0,
+        node,
-        NULL,
+        true
-        node
         );
 }
 
-int toku_testsetup_insert_to_nonleaf (FT_HANDLE brt, BLOCKNUM blocknum, enum ft_msg_type cmdtype, const char *key, int keylen, const char *val, int vallen) {
+int toku_testsetup_insert_to_nonleaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, enum ft_msg_type msgtype, const char *key, int keylen, const char *val, int vallen) {
     void *node_v;
     int r;
 
     assert(testsetup_initialized);
 
     struct ftnode_fetch_extra bfe;
-    fill_bfe_for_full_read(&bfe, brt->ft);
+    fill_bfe_for_full_read(&bfe, ft_handle->ft);
     r = toku_cachetable_get_and_pin(
-        brt->ft->cf,
+        ft_handle->ft->cf,
         blocknum,
-        toku_cachetable_hash(brt->ft->cf, blocknum),
+        toku_cachetable_hash(ft_handle->ft->cf, blocknum),
         &node_v,
         NULL,
-        get_write_callbacks_for_node(brt->ft),
+        get_write_callbacks_for_node(ft_handle->ft),
 	toku_ftnode_fetch_callback,
         toku_ftnode_pf_req_callback,
         toku_ftnode_pf_callback,
@@ -294,19 +293,19 @@ int toku_testsetup_insert_to_nonleaf (FT_HANDLE brt, BLOCKNUM blocknum, enum ft_
     DBT k;
     int childnum = toku_ftnode_which_child(node,
                                             toku_fill_dbt(&k, key, keylen),
-                                            &brt->ft->cmp_descriptor, brt->ft->compare_fun);
+                                            &ft_handle->ft->cmp_descriptor, ft_handle->ft->compare_fun);
 
     XIDS xids_0 = xids_get_root_xids();
     MSN msn = next_dummymsn();
-    toku_bnc_insert_msg(BNC(node, childnum), key, keylen, val, vallen, cmdtype, msn, xids_0, true, NULL, testhelper_string_key_cmp);
+    toku_bnc_insert_msg(BNC(node, childnum), key, keylen, val, vallen, msgtype, msn, xids_0, true, NULL, testhelper_string_key_cmp);
     // Hack to get the test working. The problem is that this test
     // is directly queueing something in a FIFO instead of 
-    // using brt APIs.
+    // using ft APIs.
     node->max_msn_applied_to_node_on_disk = msn;
     node->dirty = 1;
     // Also hack max_msn_in_ft
-    brt->ft->h->max_msn_in_ft = msn;
+    ft_handle->ft->h->max_msn_in_ft = msn;
 
-    toku_unpin_ftnode(brt->ft, node);
+    toku_unpin_ftnode(ft_handle->ft, node);
     return 0;
 }

storage/tokudb/ft-index/ft/ft-verify.cc

@@ -89,7 +89,7 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-/* Verify a BRT. */
+/* Verify an FT. */
 /* Check:
  *   The tree is of uniform depth (and the height is correct at every node)
  *   For each pivot key:  the max of the stuff to the left is <= the pivot key < the min of the stuff to the right.
@@ -102,26 +102,26 @@ PATENT RIGHTS GRANT:
 #include "ft.h"
 
 static int 
-compare_pairs (FT_HANDLE brt, const DBT *a, const DBT *b) {
+compare_pairs (FT_HANDLE ft_handle, const DBT *a, const DBT *b) {
-    FAKE_DB(db, &brt->ft->cmp_descriptor);
+    FAKE_DB(db, &ft_handle->ft->cmp_descriptor);
-    int cmp = brt->ft->compare_fun(&db, a, b);
+    int cmp = ft_handle->ft->compare_fun(&db, a, b);
     return cmp;
 }
 
 static int 
-compare_pair_to_key (FT_HANDLE brt, const DBT *a, bytevec key, ITEMLEN keylen) {
+compare_pair_to_key (FT_HANDLE ft_handle, const DBT *a, bytevec key, ITEMLEN keylen) {
     DBT y;
-    FAKE_DB(db, &brt->ft->cmp_descriptor);
+    FAKE_DB(db, &ft_handle->ft->cmp_descriptor);
-    int cmp = brt->ft->compare_fun(&db, a, toku_fill_dbt(&y, key, keylen));
+    int cmp = ft_handle->ft->compare_fun(&db, a, toku_fill_dbt(&y, key, keylen));
     return cmp;
 }
 
 static int
-verify_msg_in_child_buffer(FT_HANDLE brt, enum ft_msg_type type, MSN msn, bytevec key, ITEMLEN keylen, bytevec UU(data), ITEMLEN UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot)
+verify_msg_in_child_buffer(FT_HANDLE ft_handle, enum ft_msg_type type, MSN msn, bytevec key, ITEMLEN keylen, bytevec UU(data), ITEMLEN UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot)
     __attribute__((warn_unused_result));
 
 static int
-verify_msg_in_child_buffer(FT_HANDLE brt, enum ft_msg_type type, MSN msn, bytevec key, ITEMLEN keylen, bytevec UU(data), ITEMLEN UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot) {
+verify_msg_in_child_buffer(FT_HANDLE ft_handle, enum ft_msg_type type, MSN msn, bytevec key, ITEMLEN keylen, bytevec UU(data), ITEMLEN UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot) {
     int result = 0;
     if (msn.msn == ZERO_MSN.msn)
         result = EINVAL;
@@ -135,12 +135,12 @@ verify_msg_in_child_buffer(FT_HANDLE brt, enum ft_msg_type type, MSN msn, byteve
     case FT_COMMIT_ANY:
         // verify key in bounds
         if (lesser_pivot) {
-            int compare = compare_pair_to_key(brt, lesser_pivot, key, keylen);
+            int compare = compare_pair_to_key(ft_handle, lesser_pivot, key, keylen);
             if (compare >= 0)
                 result = EINVAL;
         }
         if (result == 0 && greatereq_pivot) {
-            int compare = compare_pair_to_key(brt, greatereq_pivot, key, keylen);
+            int compare = compare_pair_to_key(ft_handle, greatereq_pivot, key, keylen);
             if (compare < 0)
                 result = EINVAL;
         }
@@ -152,7 +152,7 @@ verify_msg_in_child_buffer(FT_HANDLE brt, enum ft_msg_type type, MSN msn, byteve
 static DBT
 get_ith_key_dbt (BASEMENTNODE bn, int i) {
     DBT kdbt;
-    int r = bn->data_buffer.fetch_le_key_and_len(i, &kdbt.size, &kdbt.data);
+    int r = bn->data_buffer.fetch_key_and_len(i, &kdbt.size, &kdbt.data);
     invariant_zero(r); // this is a bad failure if it happens.
     return kdbt;
 }
@@ -243,7 +243,7 @@ int verify_marked_messages(const int32_t &offset, const uint32_t UU(idx), struct
 
 template<typename verify_omt_t>
 static int
-verify_sorted_by_key_msn(FT_HANDLE brt, FIFO fifo, const verify_omt_t &mt) {
+verify_sorted_by_key_msn(FT_HANDLE ft_handle, FIFO fifo, const verify_omt_t &mt) {
     int result = 0;
     size_t last_offset = 0;
     for (uint32_t i = 0; i < mt.size(); i++) {
@@ -253,8 +253,8 @@ verify_sorted_by_key_msn(FT_HANDLE brt, FIFO fifo, const verify_omt_t &mt) {
         if (i > 0) {
             struct toku_fifo_entry_key_msn_cmp_extra extra;
             ZERO_STRUCT(extra);
-            extra.desc = &brt->ft->cmp_descriptor;
+            extra.desc = &ft_handle->ft->cmp_descriptor;
-            extra.cmp = brt->ft->compare_fun;
+            extra.cmp = ft_handle->ft->compare_fun;
             extra.fifo = fifo;
             if (toku_fifo_entry_key_msn_cmp(extra, last_offset, offset) >= 0) {
                 result = TOKUDB_NEEDS_REPAIR;
@@ -268,11 +268,11 @@ verify_sorted_by_key_msn(FT_HANDLE brt, FIFO fifo, const verify_omt_t &mt) {
 
 template<typename count_omt_t>
 static int
-count_eq_key_msn(FT_HANDLE brt, FIFO fifo, const count_omt_t &mt, const DBT *key, MSN msn) {
+count_eq_key_msn(FT_HANDLE ft_handle, FIFO fifo, const count_omt_t &mt, const DBT *key, MSN msn) {
     struct toku_fifo_entry_key_msn_heaviside_extra extra;
     ZERO_STRUCT(extra);
-    extra.desc = &brt->ft->cmp_descriptor;
+    extra.desc = &ft_handle->ft->cmp_descriptor;
-    extra.cmp = brt->ft->compare_fun;
+    extra.cmp = ft_handle->ft->compare_fun;
     extra.fifo = fifo;
     extra.key = key;
     extra.msn = msn;
@@ -290,28 +290,26 @@ count_eq_key_msn(FT_HANDLE brt, FIFO fifo, const count_omt_t &mt, const DBT *key
 void
 toku_get_node_for_verify(
     BLOCKNUM blocknum,
-    FT_HANDLE brt,
+    FT_HANDLE ft_handle,
     FTNODE* nodep
     )
 {
-    uint32_t fullhash = toku_cachetable_hash(brt->ft->cf, blocknum);
+    uint32_t fullhash = toku_cachetable_hash(ft_handle->ft->cf, blocknum);
     struct ftnode_fetch_extra bfe;
-    fill_bfe_for_full_read(&bfe, brt->ft);
+    fill_bfe_for_full_read(&bfe, ft_handle->ft);
-    toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
+    toku_pin_ftnode(
-        brt->ft,
+        ft_handle->ft,
         blocknum,
         fullhash,
         &bfe,
         PL_WRITE_EXPENSIVE, // may_modify_node
-        0,
-        NULL,
         nodep,
         false
         );
 }
 
 static int
-toku_verify_ftnode_internal(FT_HANDLE brt,
+toku_verify_ftnode_internal(FT_HANDLE ft_handle,
                             MSN rootmsn, MSN parentmsn, bool messages_exist_above,
                             FTNODE node, int height,
                             const DBT *lesser_pivot,               // Everything in the subtree should be > lesser_pivot.  (lesser_pivot==NULL if there is no lesser pivot.)
@@ -334,17 +332,17 @@ toku_verify_ftnode_internal(FT_HANDLE brt,
     }
     // Verify that all the pivot keys are in order.
     for (int i = 0; i < node->n_children-2; i++) {
-        int compare = compare_pairs(brt, &node->childkeys[i], &node->childkeys[i+1]);
+        int compare = compare_pairs(ft_handle, &node->childkeys[i], &node->childkeys[i+1]);
         VERIFY_ASSERTION(compare < 0, i, "Value is >= the next value");
     }
     // Verify that all the pivot keys are lesser_pivot < pivot <= greatereq_pivot
     for (int i = 0; i < node->n_children-1; i++) {
         if (lesser_pivot) {
-            int compare = compare_pairs(brt, lesser_pivot, &node->childkeys[i]);
+            int compare = compare_pairs(ft_handle, lesser_pivot, &node->childkeys[i]);
             VERIFY_ASSERTION(compare < 0, i, "Pivot is >= the lower-bound pivot");
         }
         if (greatereq_pivot) {
-            int compare = compare_pairs(brt, greatereq_pivot, &node->childkeys[i]);
+            int compare = compare_pairs(ft_handle, greatereq_pivot, &node->childkeys[i]);
             VERIFY_ASSERTION(compare >= 0, i, "Pivot is < the upper-bound pivot");
         }
     }
@@ -356,12 +354,12 @@ toku_verify_ftnode_internal(FT_HANDLE brt,
             MSN last_msn = ZERO_MSN;
             // Verify that messages in the buffers are in the right place.
             NONLEAF_CHILDINFO bnc = BNC(node, i);
-            VERIFY_ASSERTION(verify_sorted_by_key_msn(brt, bnc->buffer, bnc->fresh_message_tree) == 0, i, "fresh_message_tree");
+            VERIFY_ASSERTION(verify_sorted_by_key_msn(ft_handle, bnc->buffer, bnc->fresh_message_tree) == 0, i, "fresh_message_tree");
-            VERIFY_ASSERTION(verify_sorted_by_key_msn(brt, bnc->buffer, bnc->stale_message_tree) == 0, i, "stale_message_tree");
+            VERIFY_ASSERTION(verify_sorted_by_key_msn(ft_handle, bnc->buffer, bnc->stale_message_tree) == 0, i, "stale_message_tree");
             FIFO_ITERATE(bnc->buffer, key, keylen, data, datalen, itype, msn, xid, is_fresh,
                          ({
                              enum ft_msg_type type = (enum ft_msg_type) itype;
-                             int r = verify_msg_in_child_buffer(brt, type, msn, key, keylen, data, datalen, xid,
+                             int r = verify_msg_in_child_buffer(ft_handle, type, msn, key, keylen, data, datalen, xid,
                                                                 curr_less_pivot,
                                                                 curr_geq_pivot);
                              VERIFY_ASSERTION(r==0, i, "A message in the buffer is out of place");
@@ -372,7 +370,7 @@ toku_verify_ftnode_internal(FT_HANDLE brt,
                                  DBT keydbt;
                                  toku_fill_dbt(&keydbt, key, keylen);
                                  int total_count = 0;
-                                 count = count_eq_key_msn(brt, bnc->buffer, bnc->fresh_message_tree, toku_fill_dbt(&keydbt, key, keylen), msn);
+                                 count = count_eq_key_msn(ft_handle, bnc->buffer, bnc->fresh_message_tree, toku_fill_dbt(&keydbt, key, keylen), msn);
                                  total_count += count;
                                  if (is_fresh) {
                                      VERIFY_ASSERTION(count == 1, i, "a fresh message was not found in the fresh message tree");
@@ -380,7 +378,7 @@ toku_verify_ftnode_internal(FT_HANDLE brt,
                                      VERIFY_ASSERTION(count == 0, i, "a stale message was found in the fresh message tree");
                                  }
                                  VERIFY_ASSERTION(count <= 1, i, "a message was found multiple times in the fresh message tree");
-                                 count = count_eq_key_msn(brt, bnc->buffer, bnc->stale_message_tree, &keydbt, msn);
+                                 count = count_eq_key_msn(ft_handle, bnc->buffer, bnc->stale_message_tree, &keydbt, msn);
 
                                  total_count += count;
                                  if (is_fresh) {
@@ -424,20 +422,20 @@ toku_verify_ftnode_internal(FT_HANDLE brt,
         }
         else {
             BASEMENTNODE bn = BLB(node, i);
-            for (uint32_t j = 0; j < bn->data_buffer.omt_size(); j++) {
+            for (uint32_t j = 0; j < bn->data_buffer.num_klpairs(); j++) {
                 VERIFY_ASSERTION((rootmsn.msn >= this_msn.msn), 0, "leaf may have latest msn, but cannot be greater than root msn");
                 DBT kdbt = get_ith_key_dbt(bn, j);
                 if (curr_less_pivot) {
-                    int compare = compare_pairs(brt, curr_less_pivot, &kdbt);
+                    int compare = compare_pairs(ft_handle, curr_less_pivot, &kdbt);
                     VERIFY_ASSERTION(compare < 0, j, "The leafentry is >= the lower-bound pivot");
                 }
                 if (curr_geq_pivot) {
-                    int compare = compare_pairs(brt, curr_geq_pivot, &kdbt);
+                    int compare = compare_pairs(ft_handle, curr_geq_pivot, &kdbt);
                     VERIFY_ASSERTION(compare >= 0, j, "The leafentry is < the upper-bound pivot");
                 }
                 if (0 < j) {
                     DBT prev_key_dbt = get_ith_key_dbt(bn, j-1);
-                    int compare = compare_pairs(brt, &prev_key_dbt, &kdbt);
+                    int compare = compare_pairs(ft_handle, &prev_key_dbt, &kdbt);
                     VERIFY_ASSERTION(compare < 0, j, "Adjacent leafentries are out of order");
                 }
             }
@@ -451,7 +449,7 @@ done:
 
 // input is a pinned node, on exit, node is unpinned
 int
-toku_verify_ftnode (FT_HANDLE brt,
+toku_verify_ftnode (FT_HANDLE ft_handle,
                     MSN rootmsn, MSN parentmsn, bool messages_exist_above,
                      FTNODE node, int height,
                      const DBT *lesser_pivot,               // Everything in the subtree should be > lesser_pivot.  (lesser_pivot==NULL if there is no lesser pivot.)
@@ -471,15 +469,15 @@ toku_verify_ftnode (FT_HANDLE brt,
         // Otherwise we'll just do the next call
 
         result = toku_verify_ftnode_internal(
-                brt, rootmsn, parentmsn, messages_exist_above, node, height, lesser_pivot, greatereq_pivot,
+                ft_handle, rootmsn, parentmsn, messages_exist_above, node, height, lesser_pivot, greatereq_pivot,
                 verbose, keep_going_on_failure, false);
         if (result != 0 && (!keep_going_on_failure || result != TOKUDB_NEEDS_REPAIR)) goto done;
     }
     if (node->height > 0) {
-        toku_move_ftnode_messages_to_stale(brt->ft, node);
+        toku_move_ftnode_messages_to_stale(ft_handle->ft, node);
     }
     result2 = toku_verify_ftnode_internal(
-            brt, rootmsn, parentmsn, messages_exist_above, node, height, lesser_pivot, greatereq_pivot,
+            ft_handle, rootmsn, parentmsn, messages_exist_above, node, height, lesser_pivot, greatereq_pivot,
             verbose, keep_going_on_failure, true);
     if (result == 0) {
         result = result2;
@@ -490,8 +488,8 @@ toku_verify_ftnode (FT_HANDLE brt,
     if (recurse && node->height > 0) {
         for (int i = 0; i < node->n_children; i++) {
             FTNODE child_node;
-            toku_get_node_for_verify(BP_BLOCKNUM(node, i), brt, &child_node);
+            toku_get_node_for_verify(BP_BLOCKNUM(node, i), ft_handle, &child_node);
-            int r = toku_verify_ftnode(brt, rootmsn, this_msn, messages_exist_above || toku_bnc_n_entries(BNC(node, i)) > 0,
+            int r = toku_verify_ftnode(ft_handle, rootmsn, this_msn, messages_exist_above || toku_bnc_n_entries(BNC(node, i)) > 0,
                                         child_node, node->height-1,
                                         (i==0)                  ? lesser_pivot        : &node->childkeys[i-1],
                                         (i==node->n_children-1) ? greatereq_pivot     : &node->childkeys[i],
@@ -504,7 +502,7 @@ toku_verify_ftnode (FT_HANDLE brt,
         }
     }
 done:
-    toku_unpin_ftnode(brt->ft, node);
+    toku_unpin_ftnode(ft_handle->ft, node);
     
     if (result == 0 && progress_callback) 
     result = progress_callback(progress_extra, 0.0);
@@ -513,26 +511,26 @@ done:
 }
 
 int 
-toku_verify_ft_with_progress (FT_HANDLE brt, int (*progress_callback)(void *extra, float progress), void *progress_extra, int verbose, int keep_on_going) {
+toku_verify_ft_with_progress (FT_HANDLE ft_handle, int (*progress_callback)(void *extra, float progress), void *progress_extra, int verbose, int keep_on_going) {
-    assert(brt->ft);
+    assert(ft_handle->ft);
     FTNODE root_node = NULL;
     {
         uint32_t root_hash;
         CACHEKEY root_key;
-        toku_calculate_root_offset_pointer(brt->ft, &root_key, &root_hash);
+        toku_calculate_root_offset_pointer(ft_handle->ft, &root_key, &root_hash);
-        toku_get_node_for_verify(root_key, brt, &root_node);
+        toku_get_node_for_verify(root_key, ft_handle, &root_node);
     }
-    int r = toku_verify_ftnode(brt, brt->ft->h->max_msn_in_ft, brt->ft->h->max_msn_in_ft, false, root_node, -1, NULL, NULL, progress_callback, progress_extra, 1, verbose, keep_on_going);
+    int r = toku_verify_ftnode(ft_handle, ft_handle->ft->h->max_msn_in_ft, ft_handle->ft->h->max_msn_in_ft, false, root_node, -1, NULL, NULL, progress_callback, progress_extra, 1, verbose, keep_on_going);
     if (r == 0) {
-        toku_ft_lock(brt->ft);
+        toku_ft_lock(ft_handle->ft);
-        brt->ft->h->time_of_last_verification = time(NULL);
+        ft_handle->ft->h->time_of_last_verification = time(NULL);
-        brt->ft->h->dirty = 1;
+        ft_handle->ft->h->dirty = 1;
-        toku_ft_unlock(brt->ft);
+        toku_ft_unlock(ft_handle->ft);
     }
     return r;
 }
 
 int 
-toku_verify_ft (FT_HANDLE brt) {
+toku_verify_ft (FT_HANDLE ft_handle) {
-    return toku_verify_ft_with_progress(brt, NULL, NULL, 0, 0);
+    return toku_verify_ft_with_progress(ft_handle, NULL, NULL, 0, 0);
 }

storage/tokudb/ft-index/ft/ft.cc

@@ -465,8 +465,8 @@ void toku_ft_create(FT *ftp, FT_OPTIONS options, CACHEFILE cf, TOKUTXN txn) {
     *ftp = ft;
 }
 
-// TODO: (Zardosht) get rid of brt parameter
+// TODO: (Zardosht) get rid of ft parameter
-int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_acceptable_lsn, FT *header)
+int toku_read_ft_and_store_in_cachefile (FT_HANDLE ft_handle, CACHEFILE cf, LSN max_acceptable_lsn, FT *header)
 // If the cachefile already has the header, then just get it.
 // If the cachefile has not been initialized, then don't modify anything.
 // max_acceptable_lsn is the latest acceptable checkpointed version of the file.
@@ -475,8 +475,8 @@ int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_ac
         FT h;
         if ((h = (FT) toku_cachefile_get_userdata(cf))!=0) {
             *header = h;
-            assert(brt->options.update_fun == h->update_fun);
+            assert(ft_handle->options.update_fun == h->update_fun);
-            assert(brt->options.compare_fun == h->compare_fun);
+            assert(ft_handle->options.compare_fun == h->compare_fun);
             return 0;
         }
     }
@@ -494,8 +494,8 @@ int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_ac
     // GCC 4.8 seems to get confused by the gotos in the deserialize code and think h is maybe uninitialized.
     invariant_notnull(h);
     h->cf = cf;
-    h->compare_fun = brt->options.compare_fun;
+    h->compare_fun = ft_handle->options.compare_fun;
-    h->update_fun = brt->options.update_fun;
+    h->update_fun = ft_handle->options.update_fun;
     toku_cachefile_set_userdata(cf,
                                 (void*)h,
                                 ft_log_fassociate_during_checkpoint,
@@ -557,13 +557,13 @@ FT_HANDLE toku_ft_get_only_existing_ft_handle(FT h) {
     return ft_handle_ret;
 }
 
-// Purpose: set fields in brt_header to capture accountability info for start of HOT optimize.
+// Purpose: set fields in ft_header to capture accountability info for start of HOT optimize.
 // Note: HOT accountability variables in header are modified only while holding header lock.
 //       (Header lock is really needed for touching the dirty bit, but it's useful and 
 //       convenient here for keeping the HOT variables threadsafe.)
 void
-toku_ft_note_hot_begin(FT_HANDLE brt) {
+toku_ft_note_hot_begin(FT_HANDLE ft_handle) {
-    FT ft = brt->ft;
+    FT ft = ft_handle->ft;
     time_t now = time(NULL);
 
     // hold lock around setting and clearing of dirty bit
@@ -576,11 +576,11 @@ toku_ft_note_hot_begin(FT_HANDLE brt) {
 }
 
 
-// Purpose: set fields in brt_header to capture accountability info for end of HOT optimize.
+// Purpose: set fields in ft_header to capture accountability info for end of HOT optimize.
 // Note: See note for toku_ft_note_hot_begin().
 void
-toku_ft_note_hot_complete(FT_HANDLE brt, bool success, MSN msn_at_start_of_hot) {
+toku_ft_note_hot_complete(FT_HANDLE ft_handle, bool success, MSN msn_at_start_of_hot) {
-    FT ft = brt->ft;
+    FT ft = ft_handle->ft;
     time_t now = time(NULL);
 
     toku_ft_lock(ft);
@@ -626,7 +626,7 @@ toku_ft_init(FT ft,
     ft->h->checkpoint_lsn   = checkpoint_lsn;
 }
 
-// Open a brt for use by redirect.  The new brt must have the same dict_id as the old_ft passed in.  (FILENUM is assigned by the ft_handle_open() function.)
+// Open an ft for use by redirect.  The new ft must have the same dict_id as the old_ft passed in.  (FILENUM is assigned by the ft_handle_open() function.)
 static int
 ft_handle_open_for_redirect(FT_HANDLE *new_ftp, const char *fname_in_env, TOKUTXN txn, FT old_h) {
     FT_HANDLE t;
@@ -664,9 +664,9 @@ dictionary_redirect_internal(const char *dst_fname_in_env, FT src_h, TOKUTXN txn
 
     FT dst_h = NULL;
     struct toku_list *list;
-    // open a dummy brt based off of 
+    // open a dummy ft based off of 
     // dst_fname_in_env to get the header
-    // then we will change all the brt's to have
+    // then we will change all the ft's to have
     // their headers point to dst_h instead of src_h
     FT_HANDLE tmp_dst_ft = NULL;
     r = ft_handle_open_for_redirect(&tmp_dst_ft, dst_fname_in_env, txn, src_h);
@@ -680,7 +680,7 @@ dictionary_redirect_internal(const char *dst_fname_in_env, FT src_h, TOKUTXN txn
     assert(dst_filenum.fileid!=FILENUM_NONE.fileid);
     assert(dst_filenum.fileid!=src_filenum.fileid); //Cannot be same file.
 
-    // for each live brt, brt->ft is currently src_h
+    // for each live ft_handle, ft_handle->ft is currently src_h
     // we want to change it to dummy_dst
     toku_ft_grab_reflock(src_h);
     while (!toku_list_empty(&src_h->live_ft_handles)) {
@@ -720,7 +720,7 @@ toku_dictionary_redirect_abort(FT old_h, FT new_h, TOKUTXN txn) {
         FILENUM new_filenum = toku_cachefile_filenum(new_h->cf);
         assert(old_filenum.fileid!=new_filenum.fileid); //Cannot be same file.
 
-        //No living brts in old header.
+        //No living fts in old header.
         toku_ft_grab_reflock(old_h);
         assert(toku_list_empty(&old_h->live_ft_handles));
         toku_ft_release_reflock(old_h);
@@ -738,13 +738,13 @@ toku_dictionary_redirect_abort(FT old_h, FT new_h, TOKUTXN txn) {
 /****
  * on redirect or abort:
  *  if redirect txn_note_doing_work(txn)
- *  if redirect connect src brt to txn (txn modified this brt)
+ *  if redirect connect src ft to txn (txn modified this ft)
- *  for each src brt
+ *  for each src ft
- *    open brt to dst file (create new brt struct)
+ *    open ft to dst file (create new ft struct)
- *    if redirect connect dst brt to txn 
+ *    if redirect connect dst ft to txn 
- *    redirect db to new brt
+ *    redirect db to new ft
- *    redirect cursors to new brt
+ *    redirect cursors to new ft
- *  close all src brts
+ *  close all src fts
  *  if redirect make rollback log entry
  * 
  * on commit:
@@ -756,21 +756,21 @@ int
 toku_dictionary_redirect (const char *dst_fname_in_env, FT_HANDLE old_ft_h, TOKUTXN txn) {
 // Input args:
 //   new file name for dictionary (relative to env)
-//   old_ft_h is a live brt of open handle ({DB, BRT} pair) that currently refers to old dictionary file.
+//   old_ft_h is a live ft of open handle ({DB, FT_HANDLE} pair) that currently refers to old dictionary file.
 //   (old_ft_h may be one of many handles to the dictionary.)
 //   txn that created the loader
 // Requires: 
 //   multi operation lock is held.
-//   The brt is open.  (which implies there can be no zombies.)
+//   The ft is open.  (which implies there can be no zombies.)
 //   The new file must be a valid dictionary.
-//   The block size and flags in the new file must match the existing BRT.
+//   The block size and flags in the new file must match the existing FT.
 //   The new file must already have its descriptor in it (and it must match the existing descriptor).
 // Effect:   
 //   Open new FTs (and related header and cachefile) to the new dictionary file with a new FILENUM.
-//   Redirect all DBs that point to brts that point to the old file to point to brts that point to the new file.
+//   Redirect all DBs that point to fts that point to the old file to point to fts that point to the new file.
 //   Copy the dictionary id (dict_id) from the header of the original file to the header of the new file.
 //   Create a rollback log entry.
-//   The original BRT, header, cachefile and file remain unchanged.  They will be cleaned up on commmit.
+//   The original FT, header, cachefile and file remain unchanged.  They will be cleaned up on commmit.
 //   If the txn aborts, then this operation will be undone
     int r;
 
@@ -1077,8 +1077,8 @@ garbage_helper(BLOCKNUM blocknum, int64_t UU(size), int64_t UU(address), void *e
         goto exit;
     }
     for (int i = 0; i < node->n_children; ++i) {
-        BN_DATA bd = BLB_DATA(node, i);
+        bn_data* bd = BLB_DATA(node, i);
-        r = bd->omt_iterate<struct garbage_helper_extra, garbage_leafentry_helper>(info);
+        r = bd->iterate<struct garbage_helper_extra, garbage_leafentry_helper>(info);
         if (r != 0) {
             goto exit;
         }

storage/tokudb/ft-index/ft/ft.h

@@ -113,7 +113,7 @@ void toku_ft_release_reflock(FT ft);
 void toku_ft_create(FT *ftp, FT_OPTIONS options, CACHEFILE cf, TOKUTXN txn);
 void toku_ft_free (FT h);
 
-int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_acceptable_lsn, FT *header);
+int toku_read_ft_and_store_in_cachefile (FT_HANDLE ft_h, CACHEFILE cf, LSN max_acceptable_lsn, FT *header);
 void toku_ft_note_ft_handle_open(FT ft, FT_HANDLE live);
 
 bool toku_ft_needed_unlocked(FT ft);
@@ -125,8 +125,8 @@ void toku_ft_evict_from_memory(FT ft, bool oplsn_valid, LSN oplsn);
 
 FT_HANDLE toku_ft_get_only_existing_ft_handle(FT h);
 
-void toku_ft_note_hot_begin(FT_HANDLE brt);
+void toku_ft_note_hot_begin(FT_HANDLE ft_h);
-void toku_ft_note_hot_complete(FT_HANDLE brt, bool success, MSN msn_at_start_of_hot);
+void toku_ft_note_hot_complete(FT_HANDLE ft_h, bool success, MSN msn_at_start_of_hot);
 
 void
 toku_ft_init(

storage/tokudb/ft-index/ft/ft_layout_version.h

@@ -99,7 +99,7 @@ enum ft_layout_version_e {
     FT_LAYOUT_VERSION_7 = 7,   // Diff from 6 to 7:  Add exact-bit to leafentry_estimate #818, add magic to header #22, add per-subdatase flags #333
     FT_LAYOUT_VERSION_8 = 8,   // Diff from 7 to 8:  Use murmur instead of crc32.  We are going to make a simplification and stop supporting version 7 and before.  Current As of Beta 1.0.6
     FT_LAYOUT_VERSION_9 = 9,   // Diff from 8 to 9:  Variable-sized blocks and compression.
-    FT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates, translation table, dictionary descriptors, checksum in header, subdb support removed from brt layer
+    FT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates, translation table, dictionary descriptors, checksum in header, subdb support removed from ft layer
     FT_LAYOUT_VERSION_11 = 11, // Diff from 10 to 11: Nested transaction leafentries (completely redesigned).  FT_CMDs on disk now support XIDS (multiple txnids) instead of exactly one.
     FT_LAYOUT_VERSION_12 = 12, // Diff from 11 to 12: Added FT_CMD 'FT_INSERT_NO_OVERWRITE', compressed block format, num old blocks
     FT_LAYOUT_VERSION_13 = 13, // Diff from 12 to 13: Fixed loader pivot bug, added build_id to every node, timestamps to ft 
@@ -107,8 +107,8 @@ enum ft_layout_version_e {
     FT_LAYOUT_VERSION_15 = 15, // Diff from 14 to 15: basement nodes, last verification time
     FT_LAYOUT_VERSION_16 = 16, // Dr. No:  No subtree estimates, partition layout information represented more transparently. 
                                 // ALERT ALERT ALERT: version 16 never released to customers, internal and beta use only
-    FT_LAYOUT_VERSION_17 = 17, // Dr. No:  Add STAT64INFO_S to brt_header
+    FT_LAYOUT_VERSION_17 = 17, // Dr. No:  Add STAT64INFO_S to ft header
-    FT_LAYOUT_VERSION_18 = 18, // Dr. No:  Add HOT info to brt_header
+    FT_LAYOUT_VERSION_18 = 18, // Dr. No:  Add HOT info to ft header
     FT_LAYOUT_VERSION_19 = 19, // Doofenshmirtz: Add compression method, highest_unused_msn_for_upgrade
     FT_LAYOUT_VERSION_20 = 20, // Deadshot: Add compression method to log_fcreate,
                                // mgr_last_xid after begin checkpoint,
@@ -119,6 +119,7 @@ enum ft_layout_version_e {
     FT_LAYOUT_VERSION_23 = 23, // Ming: Fix upgrade path #5902
     FT_LAYOUT_VERSION_24 = 24, // Riddler: change logentries that log transactions to store TXNID_PAIRs instead of TXNIDs
     FT_LAYOUT_VERSION_25 = 25, // SecretSquirrel: ROLLBACK_LOG_NODES (on disk and in memory) now just use blocknum (instead of blocknum + hash) to point to other log nodes.  same for xstillopen log entry
+    FT_LAYOUT_VERSION_26 = 26, // Hojo: basements store key/vals separately on disk for fixed klpair length BNs
     FT_NEXT_VERSION,           // the version after the current version
     FT_LAYOUT_VERSION   = FT_NEXT_VERSION-1, // A hack so I don't have to change this line.
     FT_LAYOUT_MIN_SUPPORTED_VERSION = FT_LAYOUT_VERSION_13, // Minimum version supported

storage/tokudb/ft-index/ft/ft_node-serialize.cc

@@ -101,7 +101,7 @@ PATENT RIGHTS GRANT:
 
 static FT_UPGRADE_STATUS_S ft_upgrade_status;
 
-#define STATUS_INIT(k,c,t,l,inc) TOKUDB_STATUS_INIT(ft_upgrade_status, k, c, t, "brt upgrade: " l, inc)
+#define STATUS_INIT(k,c,t,l,inc) TOKUDB_STATUS_INIT(ft_upgrade_status, k, c, t, "ft upgrade: " l, inc)
 
 static void
 status_init(void)
@@ -279,37 +279,12 @@ serialize_node_header(FTNODE node, FTNODE_DISK_DATA ndd, struct wbuf *wbuf) {
         wbuf_nocrc_int(wbuf, BP_SIZE (ndd, i));         // and the size
     }
     // checksum the header
-    uint32_t end_to_end_checksum = x1764_memory(wbuf->buf, wbuf_get_woffset(wbuf));
+    uint32_t end_to_end_checksum = toku_x1764_memory(wbuf->buf, wbuf_get_woffset(wbuf));
     wbuf_nocrc_int(wbuf, end_to_end_checksum);
     invariant(wbuf->ndone == wbuf->size);
 }
 
-static int
+static uint32_t
-wbufwriteleafentry(const void* key, const uint32_t keylen, const LEAFENTRY &le, const uint32_t UU(idx), struct wbuf * const wb) {
-    // need to pack the leafentry as it was in versions
-    // where the key was integrated into it
-    uint32_t begin_spot UU() = wb->ndone;
-    uint32_t le_disk_size = leafentry_disksize(le);
-    wbuf_nocrc_uint8_t(wb, le->type);
-    wbuf_nocrc_uint32_t(wb, keylen);
-    if (le->type == LE_CLEAN) {
-        wbuf_nocrc_uint32_t(wb, le->u.clean.vallen);
-        wbuf_nocrc_literal_bytes(wb, key, keylen);
-        wbuf_nocrc_literal_bytes(wb, le->u.clean.val, le->u.clean.vallen);
-    }
-    else {
-        paranoid_invariant(le->type == LE_MVCC);
-        wbuf_nocrc_uint32_t(wb, le->u.mvcc.num_cxrs);
-        wbuf_nocrc_uint8_t(wb, le->u.mvcc.num_pxrs);
-        wbuf_nocrc_literal_bytes(wb, key, keylen);
-        wbuf_nocrc_literal_bytes(wb, le->u.mvcc.xrs, le_disk_size - (1 + 4 + 1));
-    }
-    uint32_t end_spot UU() = wb->ndone;
-    paranoid_invariant((end_spot - begin_spot) == keylen + sizeof(keylen) + le_disk_size);
-    return 0;
-}
-
-static uint32_t 
 serialize_ftnode_partition_size (FTNODE node, int i)
 {
     uint32_t result = 0;
@@ -320,14 +295,14 @@ serialize_ftnode_partition_size (FTNODE node, int i)
         result += toku_bnc_nbytesinbuf(BNC(node, i));
     }
     else {
-        result += 4; // n_entries in buffer table
+        result += 4 + bn_data::HEADER_LENGTH; // n_entries in buffer table + basement header
         result += BLB_NBYTESINDATA(node, i);
     }
     result += 4; // checksum
     return result;
 }
 
-#define FTNODE_PARTITION_OMT_LEAVES 0xaa
+#define FTNODE_PARTITION_DMT_LEAVES 0xaa
 #define FTNODE_PARTITION_FIFO_MSG 0xbb
 
 static void
@@ -340,9 +315,9 @@ serialize_nonleaf_childinfo(NONLEAF_CHILDINFO bnc, struct wbuf *wb)
     FIFO_ITERATE(
         bnc->buffer, key, keylen, data, datalen, type, msn, xids, is_fresh,
         {
-            paranoid_invariant((int)type>=0 && type<256);
+            paranoid_invariant((int) type >= 0 && (int) type < 256);
-            wbuf_nocrc_char(wb, (unsigned char)type);
+            wbuf_nocrc_char(wb, (unsigned char) type);
-            wbuf_nocrc_char(wb, (unsigned char)is_fresh);
+            wbuf_nocrc_char(wb, (unsigned char) is_fresh);
             wbuf_MSN(wb, msn);
             wbuf_nocrc_xids(wb, xids);
             wbuf_nocrc_bytes(wb, key, keylen);
@@ -374,18 +349,15 @@ serialize_ftnode_partition(FTNODE node, int i, struct sub_block *sb) {
         serialize_nonleaf_childinfo(BNC(node, i), &wb);
     }
     else {
-        unsigned char ch = FTNODE_PARTITION_OMT_LEAVES;
+        unsigned char ch = FTNODE_PARTITION_DMT_LEAVES;
-        BN_DATA bd = BLB_DATA(node, i);
+        bn_data* bd = BLB_DATA(node, i);
 
         wbuf_nocrc_char(&wb, ch);
-        wbuf_nocrc_uint(&wb, bd->omt_size());
+        wbuf_nocrc_uint(&wb, bd->num_klpairs());
 
-        //
+        bd->serialize_to_wbuf(&wb);
-        // iterate over leafentries and place them into the buffer
-        //
-        bd->omt_iterate<struct wbuf, wbufwriteleafentry>(&wb);
     }
-    uint32_t end_to_end_checksum = x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
+    uint32_t end_to_end_checksum = toku_x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
     wbuf_nocrc_int(&wb, end_to_end_checksum);
     invariant(wb.ndone == wb.size);
     invariant(sb->uncompressed_size==wb.ndone);
@@ -429,7 +401,7 @@ compress_ftnode_sub_block(struct sub_block *sb, enum toku_compression_method met
     extra[1] = toku_htod32(sb->uncompressed_size);
     // now checksum the entire thing
     sb->compressed_size += 8; // now add the eight bytes that we saved for the sizes
-    sb->xsum = x1764_memory(sb->compressed_ptr,sb->compressed_size);
+    sb->xsum = toku_x1764_memory(sb->compressed_ptr,sb->compressed_size);
 
     //
     // This is the end result for Dr. No and forward. For ftnodes, sb->compressed_ptr contains
@@ -493,7 +465,7 @@ static void serialize_ftnode_info(FTNODE node,
         }
     }
 
-    uint32_t end_to_end_checksum = x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
+    uint32_t end_to_end_checksum = toku_x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
     wbuf_nocrc_int(&wb, end_to_end_checksum);
     invariant(wb.ndone == wb.size);
     invariant(sb->uncompressed_size==wb.ndone);
@@ -546,7 +518,7 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
     // Count number of leaf entries in this leaf (num_le).
     uint32_t num_le = 0;
     for (uint32_t i = 0; i < num_orig_basements; i++) {
-        num_le += BLB_DATA(node, i)->omt_size();
+        num_le += BLB_DATA(node, i)->num_klpairs();
     }
 
     uint32_t num_alloc = num_le ? num_le : 1;  // simplify logic below by always having at least one entry per array
@@ -571,10 +543,10 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
 
     uint32_t curr_le = 0;
     for (uint32_t i = 0; i < num_orig_basements; i++) {
-        BN_DATA bd = BLB_DATA(node, i);
+        bn_data* bd = BLB_DATA(node, i);
         struct array_info ai {.offset = curr_le, .le_array = leafpointers, .key_sizes_array = key_sizes, .key_ptr_array = key_pointers };
-        bd->omt_iterate<array_info, array_item>(&ai);
+        bd->iterate<array_info, array_item>(&ai);
-        curr_le += bd->omt_size();
+        curr_le += bd->num_klpairs();
     }
 
     // Create an array that will store indexes of new pivots.
@@ -592,9 +564,14 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
     // Create an array that will store the size of each basement.
     // This is the sum of the leaf sizes of all the leaves in that basement.
     // We don't know how many basements there will be, so we use num_le as the upper bound.
-    toku::scoped_malloc bn_sizes_buf(sizeof(size_t) * num_alloc);
+
-    size_t *bn_sizes = reinterpret_cast<size_t *>(bn_sizes_buf.get());
+    // Sum of all le sizes in a single basement
-    bn_sizes[0] = 0;
+    toku::scoped_calloc bn_le_sizes_buf(sizeof(size_t) * num_alloc);
+    size_t *bn_le_sizes = reinterpret_cast<size_t *>(bn_le_sizes_buf.get());
+
+    // Sum of all key sizes in a single basement
+    toku::scoped_calloc bn_key_sizes_buf(sizeof(size_t) * num_alloc);
+    size_t *bn_key_sizes = reinterpret_cast<size_t *>(bn_key_sizes_buf.get());
 
     // TODO 4050: All these arrays should be combined into a single array of some bn_info struct (pivot, msize, num_les).
     // Each entry is the number of leafentries in this basement.  (Again, num_le is overkill upper baound.)
@@ -611,7 +588,7 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
     for (uint32_t i = 0; i < num_le; i++) {
         uint32_t curr_le_size = leafentry_disksize((LEAFENTRY) leafpointers[i]); 
         le_sizes[i] = curr_le_size;
-        if ((bn_size_so_far + curr_le_size > basementnodesize) && (num_le_in_curr_bn != 0)) {
+        if ((bn_size_so_far + curr_le_size + sizeof(uint32_t) + key_sizes[i] > basementnodesize) && (num_le_in_curr_bn != 0)) {
             // cap off the current basement node to end with the element before i
             new_pivots[curr_pivot] = i-1;
             curr_pivot++;
@@ -620,8 +597,9 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
         }
         num_le_in_curr_bn++;
         num_les_this_bn[curr_pivot] = num_le_in_curr_bn;
+        bn_le_sizes[curr_pivot] += curr_le_size;
+        bn_key_sizes[curr_pivot] += sizeof(uint32_t) + key_sizes[i];  // uint32_t le_offset
         bn_size_so_far += curr_le_size + sizeof(uint32_t) + key_sizes[i];
-        bn_sizes[curr_pivot] = bn_size_so_far;
     }
     // curr_pivot is now the total number of pivot keys in the leaf node
     int num_pivots   = curr_pivot;
@@ -688,17 +666,15 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
         uint32_t num_les_to_copy = num_les_this_bn[i];
         invariant(num_les_to_copy == num_in_bn); 
 
-        // construct mempool for this basement
+        bn_data* bd = BLB_DATA(node, i);
-        size_t size_this_bn = bn_sizes[i];
+        bd->set_contents_as_clone_of_sorted_array(
-
-        BN_DATA bd = BLB_DATA(node, i);
-        bd->replace_contents_with_clone_of_sorted_array(
             num_les_to_copy,
             &key_pointers[baseindex_this_bn],
             &key_sizes[baseindex_this_bn],
             &leafpointers[baseindex_this_bn],
             &le_sizes[baseindex_this_bn],
-            size_this_bn
+            bn_key_sizes[i],  // Total key sizes
+            bn_le_sizes[i]  // total le sizes
             );
 
         BP_STATE(node,i) = PT_AVAIL;
@@ -1272,7 +1248,7 @@ read_compressed_sub_block(struct rbuf *rb, struct sub_block *sb)
     rbuf_literal_bytes(rb, cp, sb->compressed_size);
     sb->xsum = rbuf_int(rb);
     // let's check the checksum
-    uint32_t actual_xsum = x1764_memory((char *)sb->compressed_ptr-8, 8+sb->compressed_size);
+    uint32_t actual_xsum = toku_x1764_memory((char *)sb->compressed_ptr-8, 8+sb->compressed_size);
     if (sb->xsum != actual_xsum) {
         r = TOKUDB_BAD_CHECKSUM;
     }
@@ -1317,7 +1293,7 @@ verify_ftnode_sub_block (struct sub_block *sb)
     // first verify the checksum
     uint32_t data_size = sb->uncompressed_size - 4; // checksum is 4 bytes at end
     uint32_t stored_xsum = toku_dtoh32(*((uint32_t *)((char *)sb->uncompressed_ptr + data_size)));
-    uint32_t actual_xsum = x1764_memory(sb->uncompressed_ptr, data_size);
+    uint32_t actual_xsum = toku_x1764_memory(sb->uncompressed_ptr, data_size);
     if (stored_xsum != actual_xsum) {
         dump_bad_block((Bytef *) sb->uncompressed_ptr, sb->uncompressed_size);
         r = TOKUDB_BAD_CHECKSUM;
@@ -1412,7 +1388,7 @@ setup_available_ftnode_partition(FTNODE node, int i) {
     }
 }
 
-// Assign the child_to_read member of the bfe from the given brt node
+// Assign the child_to_read member of the bfe from the given ftnode
 // that has been brought into memory.
 static void
 update_bfe_using_ftnode(FTNODE node, struct ftnode_fetch_extra *bfe)
@@ -1447,7 +1423,7 @@ update_bfe_using_ftnode(FTNODE node, struct ftnode_fetch_extra *bfe)
 }
 
 // Using the search parameters in the bfe, this function will
-// initialize all of the given brt node's partitions.
+// initialize all of the given ftnode's partitions.
 static void
 setup_partitions_using_bfe(FTNODE node,
                            struct ftnode_fetch_extra *bfe,
@@ -1541,15 +1517,14 @@ deserialize_ftnode_partition(
         BP_WORKDONE(node, childnum) = 0;
     }
     else {
-        assert(ch == FTNODE_PARTITION_OMT_LEAVES);
+        assert(ch == FTNODE_PARTITION_DMT_LEAVES);
         BLB_SEQINSERT(node, childnum) = 0;
         uint32_t num_entries = rbuf_int(&rb);
         // we are now at the first byte of first leafentry
         data_size -= rb.ndone; // remaining bytes of leafentry data
-        
+
         BASEMENTNODE bn = BLB(node, childnum);
-        bn->data_buffer.initialize_from_data(num_entries, &rb.buf[rb.ndone], data_size);
+        bn->data_buffer.deserialize_from_rbuf(num_entries, &rb, data_size, node->layout_version_read_from_disk);
-        rb.ndone += data_size;
     }
     assert(rb.ndone == rb.size);
 exit:
@@ -1681,7 +1656,7 @@ deserialize_ftnode_header_from_rbuf_if_small_enough (FTNODE *ftnode,
     }
 
     uint32_t checksum;
-    checksum = x1764_memory(rb->buf, rb->ndone);
+    checksum = toku_x1764_memory(rb->buf, rb->ndone);
     uint32_t stored_checksum;
     stored_checksum = rbuf_int(rb);
     if (stored_checksum != checksum) {
@@ -1707,7 +1682,7 @@ deserialize_ftnode_header_from_rbuf_if_small_enough (FTNODE *ftnode,
     sb_node_info.xsum = rbuf_int(rb);
     // let's check the checksum
     uint32_t actual_xsum;
-    actual_xsum = x1764_memory((char *)sb_node_info.compressed_ptr-8, 8+sb_node_info.compressed_size);
+    actual_xsum = toku_x1764_memory((char *)sb_node_info.compressed_ptr-8, 8+sb_node_info.compressed_size);
     if (sb_node_info.xsum != actual_xsum) {
         r = TOKUDB_BAD_CHECKSUM;
         goto cleanup;
@@ -1792,7 +1767,7 @@ cleanup:
 // also creates MSN's for older messages created in older versions
 // that did not generate MSN's for messages.  These new MSN's are
 // generated from the root downwards, counting backwards from MIN_MSN
-// and persisted in the brt header.
+// and persisted in the ft header.
 static int
 deserialize_and_upgrade_internal_node(FTNODE node,
                                       struct rbuf *rb,
@@ -1978,7 +1953,7 @@ deserialize_and_upgrade_internal_node(FTNODE node,
     // still have the pointer to the buffer).
     if (version >= FT_FIRST_LAYOUT_VERSION_WITH_END_TO_END_CHECKSUM) {
         uint32_t expected_xsum = toku_dtoh32(*(uint32_t*)(rb->buf+rb->size-4)); // 27. checksum
-        uint32_t actual_xsum   = x1764_memory(rb->buf, rb->size-4);
+        uint32_t actual_xsum   = toku_x1764_memory(rb->buf, rb->size-4);
         if (expected_xsum != actual_xsum) {
             fprintf(stderr, "%s:%d: Bad checksum: expected = %" PRIx32 ", actual= %" PRIx32 "\n",
                     __FUNCTION__,
@@ -2050,7 +2025,7 @@ deserialize_and_upgrade_leaf_node(FTNODE node,
     setup_partitions_using_bfe(node, &temp_bfe, true);
 
     // 11. Deserialize the partition maps, though they are not used in the
-    // newer versions of brt nodes.
+    // newer versions of ftnodes.
     struct sub_block_map part_map[npartitions];
     for (int i = 0; i < npartitions; ++i) {
         sub_block_map_deserialize(&part_map[i], rb);
@@ -2086,13 +2061,18 @@ deserialize_and_upgrade_leaf_node(FTNODE node,
             assert_zero(r);
             // Copy the pointer value straight into the OMT
             LEAFENTRY new_le_in_bn = nullptr;
+            void *maybe_free;
             bn->data_buffer.get_space_for_insert(
                 i,
                 key,
                 keylen,
                 new_le_size,
-                &new_le_in_bn
+                &new_le_in_bn,
+                &maybe_free
                 );
+            if (maybe_free) {
+                toku_free(maybe_free);
+            }
             memcpy(new_le_in_bn, new_le, new_le_size);
             toku_free(new_le);
         }
@@ -2101,8 +2081,7 @@ deserialize_and_upgrade_leaf_node(FTNODE node,
         if (has_end_to_end_checksum) {
             data_size -= sizeof(uint32_t);
         }
-        bn->data_buffer.initialize_from_data(n_in_buf, &rb->buf[rb->ndone], data_size);
+        bn->data_buffer.deserialize_from_rbuf(n_in_buf, rb, data_size, node->layout_version_read_from_disk);
-        rb->ndone += data_size;
     }
 
     // Whatever this is must be less than the MSNs of every message above
@@ -2114,7 +2093,7 @@ deserialize_and_upgrade_leaf_node(FTNODE node,
     // Checksum (end to end) is only on version 14
     if (has_end_to_end_checksum) {
         uint32_t expected_xsum = rbuf_int(rb);             // 17. checksum 
-        uint32_t actual_xsum = x1764_memory(rb->buf, rb->size - 4);
+        uint32_t actual_xsum = toku_x1764_memory(rb->buf, rb->size - 4);
         if (expected_xsum != actual_xsum) {
             fprintf(stderr, "%s:%d: Bad checksum: expected = %" PRIx32 ", actual= %" PRIx32 "\n",
                     __FUNCTION__,
@@ -2313,7 +2292,7 @@ deserialize_ftnode_from_rbuf(
     }
     // verify checksum of header stored
     uint32_t checksum;
-    checksum = x1764_memory(rb->buf, rb->ndone);
+    checksum = toku_x1764_memory(rb->buf, rb->ndone);
     uint32_t stored_checksum;
     stored_checksum = rbuf_int(rb);
     if (stored_checksum != checksum) {
@@ -2420,7 +2399,7 @@ cleanup:
         // NOTE: Right now, callers higher in the stack will assert on
         // failure, so this is OK for production.  However, if we
         // create tools that use this function to search for errors in
-        // the BRT, then we will leak memory.
+        // the FT, then we will leak memory.
         if (node) {
             toku_free(node);
         }
@@ -2579,7 +2558,7 @@ deserialize_ftnode_from_fd(int fd,
     return r;
 }
 
-// Read brt node from file into struct.  Perform version upgrade if necessary.
+// Read ftnode from file into struct.  Perform version upgrade if necessary.
 int
 toku_deserialize_ftnode_from (int fd,
                                BLOCKNUM blocknum,
@@ -2628,7 +2607,7 @@ serialize_rollback_log_size(ROLLBACK_LOG_NODE log) {
                  +8 //blocknum
                  +8 //previous (blocknum)
                  +8 //resident_bytecount
-                 +8 //memarena_size_needed_to_load
+                 +8 //memarena size
                  +log->rollentry_resident_bytecount;
     return size;
 }
@@ -2649,7 +2628,7 @@ serialize_rollback_log_node_to_buf(ROLLBACK_LOG_NODE log, char *buf, size_t calc
         wbuf_nocrc_BLOCKNUM(&wb, log->previous);
         wbuf_nocrc_ulonglong(&wb, log->rollentry_resident_bytecount);
         //Write down memarena size needed to restore
-        wbuf_nocrc_ulonglong(&wb, memarena_total_size_in_use(log->rollentry_arena));
+        wbuf_nocrc_ulonglong(&wb, toku_memarena_total_size_in_use(log->rollentry_arena));
 
         {
             //Store rollback logs
@@ -2705,7 +2684,7 @@ serialize_uncompressed_block_to_memory(char * uncompressed_buf,
 
     // compute the header checksum and serialize it
     uint32_t header_length = (char *)ptr - (char *)compressed_buf;
-    uint32_t xsum = x1764_memory(compressed_buf, header_length);
+    uint32_t xsum = toku_x1764_memory(compressed_buf, header_length);
     *ptr = toku_htod32(xsum);
 
     uint32_t padded_len = roundup_to_multiple(512, header_len + compressed_len);
@@ -2813,8 +2792,8 @@ deserialize_rollback_log_from_rbuf (BLOCKNUM blocknum, ROLLBACK_LOG_NODE *log_p,
     result->rollentry_resident_bytecount = rbuf_ulonglong(rb);
 
     size_t arena_initial_size = rbuf_ulonglong(rb);
-    result->rollentry_arena = memarena_create_presized(arena_initial_size);
+    result->rollentry_arena = toku_memarena_create_presized(arena_initial_size);
-    if (0) { died1: memarena_close(&result->rollentry_arena); goto died0; }
+    if (0) { died1: toku_memarena_destroy(&result->rollentry_arena); goto died0; }
 
     //Load rollback entries
     lazy_assert(rb->size > 4);
@@ -2877,7 +2856,7 @@ decompress_from_raw_block_into_rbuf(uint8_t *raw_block, size_t raw_block_size, s
     { // verify the header checksum
         uint32_t header_length = node_header_overhead + sub_block_header_size(n_sub_blocks);
         invariant(header_length <= raw_block_size);
-        uint32_t xsum = x1764_memory(raw_block, header_length);
+        uint32_t xsum = toku_x1764_memory(raw_block, header_length);
         uint32_t stored_xsum = toku_dtoh32(*(uint32_t *)(raw_block + header_length));
         if (xsum != stored_xsum) {
             r = TOKUDB_BAD_CHECKSUM;

storage/tokudb/ft-index/ft/ftloader-internal.h

@@ -256,7 +256,7 @@ struct ft_loader_s {
 
     int progress_callback_result; // initially zero, if any call to the poll function callback returns nonzero, we save the result here (and don't call the poll callback function again).
 
-    LSN load_lsn; //LSN of the fsynced 'load' log entry.  Write this LSN (as checkpoint_lsn) in brt headers made by this loader.
+    LSN load_lsn; //LSN of the fsynced 'load' log entry.  Write this LSN (as checkpoint_lsn) in ft headers made by this loader.
     TXNID load_root_xid; //(Root) transaction that performed the load.
 
     QUEUE *fractal_queues; // an array of work queues, one for each secondary index.
@@ -279,7 +279,7 @@ uint64_t toku_ft_loader_get_n_rows(FTLOADER bl);
 struct fractal_thread_args {
     FTLOADER                bl;
     const DESCRIPTOR descriptor;
-    int                      fd; // write the brt into tfd.
+    int                      fd; // write the ft into fd.
     int                      progress_allocation;
     QUEUE                    q;
     uint64_t                 total_disksize_estimate;
@@ -311,17 +311,17 @@ int toku_merge_some_files_using_dbufio (const bool to_q, FIDX dest_data, QUEUE q
 int ft_loader_sort_and_write_rows (struct rowset *rows, struct merge_fileset *fs, FTLOADER bl, int which_db, DB *dest_db, ft_compare_func);
 
 // This is probably only for testing.
-int toku_loader_write_brt_from_q_in_C (FTLOADER                bl,
+int toku_loader_write_ft_from_q_in_C (FTLOADER                 bl,
-				       const DESCRIPTOR descriptor,
+				      const DESCRIPTOR         descriptor,
-				       int                      fd, // write to here
+				      int                      fd, // write to here
-				       int                      progress_allocation,
+				      int                      progress_allocation,
-				       QUEUE                    q,
+				      QUEUE                    q,
-				       uint64_t                 total_disksize_estimate,
+				      uint64_t                 total_disksize_estimate,
-                                       int                      which_db,
+                                      int                      which_db,
-                                       uint32_t                 target_nodesize,
+                                      uint32_t                 target_nodesize,
-                                       uint32_t                 target_basementnodesize,
+                                      uint32_t                 target_basementnodesize,
-                                       enum toku_compression_method target_compression_method,
+                                      enum toku_compression_method target_compression_method,
-                                       uint32_t                 fanout);
+                                      uint32_t                 fanout);
 
 int ft_loader_mergesort_row_array (struct row rows[/*n*/], int n, int which_db, DB *dest_db, ft_compare_func, FTLOADER, struct rowset *);
 
@@ -338,7 +338,7 @@ int toku_ft_loader_internal_init (/* out */ FTLOADER *blp,
 				   CACHETABLE cachetable,
 				   generate_row_for_put_func g,
 				   DB *src_db,
-				   int N, FT_HANDLE brts[/*N*/], DB* dbs[/*N*/],
+				   int N, FT_HANDLE ft_hs[/*N*/], DB* dbs[/*N*/],
 				   const char *new_fnames_in_env[/*N*/],
 				   ft_compare_func bt_compare_functions[/*N*/],
 				   const char *temp_file_template,

storage/tokudb/ft-index/ft/ftloader.cc

@@ -91,9 +91,7 @@ PATENT RIGHTS GRANT:
 
 #include <toku_portability.h>
 
-#if !TOKU_WINDOWS
 #include <arpa/inet.h>
-#endif
 
 #include <stdio.h>
 #include <memory.h>
@@ -101,7 +99,9 @@ PATENT RIGHTS GRANT:
 #include <toku_assert.h>
 #include <string.h>
 #include <fcntl.h>
-#include "x1764.h"
+
+#include <util/x1764.h>
+
 #include "ftloader-internal.h"
 #include "ft-internal.h"
 #include "sub_block.h"
@@ -535,7 +535,7 @@ int toku_ft_loader_internal_init (/* out */ FTLOADER *blp,
                                    CACHETABLE cachetable,
                                    generate_row_for_put_func g,
                                    DB *src_db,
-                                   int N, FT_HANDLE brts[/*N*/], DB* dbs[/*N*/],
+                                   int N, FT_HANDLE fts[/*N*/], DB* dbs[/*N*/],
                                    const char *new_fnames_in_env[/*N*/],
                                    ft_compare_func bt_compare_functions[/*N*/],
                                    const char *temp_file_template,
@@ -581,11 +581,11 @@ int toku_ft_loader_internal_init (/* out */ FTLOADER *blp,
 #define SET_TO_MY_STRDUP(lval, s) do { char *v = toku_strdup(s); if (!v) { int r = get_error_errno(); toku_ft_loader_internal_destroy(bl, true); return r; } lval = v; } while (0)
 
     MY_CALLOC_N(N, bl->root_xids_that_created);
-    for (int i=0; i<N; i++) if (brts[i]) bl->root_xids_that_created[i]=brts[i]->ft->h->root_xid_that_created;
+    for (int i=0; i<N; i++) if (fts[i]) bl->root_xids_that_created[i]=fts[i]->ft->h->root_xid_that_created;
     MY_CALLOC_N(N, bl->dbs);
-    for (int i=0; i<N; i++) if (brts[i]) bl->dbs[i]=dbs[i];
+    for (int i=0; i<N; i++) if (fts[i]) bl->dbs[i]=dbs[i];
     MY_CALLOC_N(N, bl->descriptors);
-    for (int i=0; i<N; i++) if (brts[i]) bl->descriptors[i]=&brts[i]->ft->descriptor;
+    for (int i=0; i<N; i++) if (fts[i]) bl->descriptors[i]=&fts[i]->ft->descriptor;
     MY_CALLOC_N(N, bl->new_fnames_in_env);
     for (int i=0; i<N; i++) SET_TO_MY_STRDUP(bl->new_fnames_in_env[i], new_fnames_in_env[i]);
     MY_CALLOC_N(N, bl->extracted_datasizes); // the calloc_n zeroed everything, which is what we want
@@ -642,7 +642,7 @@ int toku_ft_loader_open (/* out */ FTLOADER *blp,
                           CACHETABLE cachetable,
                           generate_row_for_put_func g,
                           DB *src_db,
-                          int N, FT_HANDLE brts[/*N*/], DB* dbs[/*N*/],
+                          int N, FT_HANDLE fts[/*N*/], DB* dbs[/*N*/],
                           const char *new_fnames_in_env[/*N*/],
                           ft_compare_func bt_compare_functions[/*N*/],
                           const char *temp_file_template,
@@ -651,9 +651,9 @@ int toku_ft_loader_open (/* out */ FTLOADER *blp,
                           bool reserve_memory,
                           uint64_t reserve_memory_size,
                           bool compress_intermediates)
-/* Effect: called by DB_ENV->create_loader to create a brt loader.
+/* Effect: called by DB_ENV->create_loader to create an ft loader.
  * Arguments:
- *   blp                  Return the brt loader here.
+ *   blp                  Return the ft loader here.
  *   g                    The function for generating a row
  *   src_db               The source database.  Needed by g.  May be NULL if that's ok with g.
  *   N                    The number of dbs to create.
@@ -666,15 +666,15 @@ int toku_ft_loader_open (/* out */ FTLOADER *blp,
     int result = 0;
     {
         int r = toku_ft_loader_internal_init(blp, cachetable, g, src_db,
-                                              N, brts, dbs,
+                                             N, fts, dbs,
-                                              new_fnames_in_env,
+                                             new_fnames_in_env,
-                                              bt_compare_functions,
+                                             bt_compare_functions,
-                                              temp_file_template,
+                                             temp_file_template,
-                                              load_lsn,
+                                             load_lsn,
-                                              txn,
+                                             txn,
-                                              reserve_memory,
+                                             reserve_memory,
-                                              reserve_memory_size,
+                                             reserve_memory_size,
-                                              compress_intermediates);
+                                             compress_intermediates);
         if (r!=0) result = r;
     }
     if (result==0) {
@@ -1370,7 +1370,7 @@ static int process_primary_rows (FTLOADER bl, struct rowset *primary_rowset) {
 }
  
 int toku_ft_loader_put (FTLOADER bl, DBT *key, DBT *val)
-/* Effect: Put a key-value pair into the brt loader.  Called by DB_LOADER->put().
+/* Effect: Put a key-value pair into the ft loader.  Called by DB_LOADER->put().
  * Return value: 0 on success, an error number otherwise.
  */
 {
@@ -2621,7 +2621,7 @@ static int toku_loader_write_ft_from_q (FTLOADER bl,
             char *XMALLOC_N(desc_size, buf);
             wbuf_init(&wbuf, buf, desc_size);
             toku_serialize_descriptor_contents_to_wbuf(&wbuf, descriptor);
-            uint32_t checksum = x1764_finish(&wbuf.checksum);
+            uint32_t checksum = toku_x1764_finish(&wbuf.checksum);
             wbuf_int(&wbuf, checksum);
             invariant(wbuf.ndone==desc_size);
             r = toku_os_write(out.fd, wbuf.buf, wbuf.ndone);
@@ -2672,17 +2672,17 @@ static int toku_loader_write_ft_from_q (FTLOADER bl,
     return result;
 }
 
-int toku_loader_write_brt_from_q_in_C (FTLOADER                bl,
+int toku_loader_write_ft_from_q_in_C (FTLOADER                bl,
-                                       const DESCRIPTOR descriptor,
+                                      const DESCRIPTOR descriptor,
-                                       int                      fd, // write to here
+                                      int                      fd, // write to here
-                                       int                      progress_allocation,
+                                      int                      progress_allocation,
-                                       QUEUE                    q,
+                                      QUEUE                    q,
-                                       uint64_t                 total_disksize_estimate,
+                                      uint64_t                 total_disksize_estimate,
-                                       int                      which_db,
+                                      int                      which_db,
-                                       uint32_t                 target_nodesize,
+                                      uint32_t                 target_nodesize,
-                                       uint32_t                 target_basementnodesize,
+                                      uint32_t                 target_basementnodesize,
-                                       enum toku_compression_method target_compression_method,
+                                      enum toku_compression_method target_compression_method,
-                                       uint32_t                 target_fanout)
+                                      uint32_t                 target_fanout)
 // This is probably only for testing.
 {
     target_nodesize = target_nodesize == 0 ? default_loader_nodesize : target_nodesize;
@@ -2917,17 +2917,17 @@ static void add_pair_to_leafnode (struct leaf_buf *lbuf, unsigned char *key, int
     // #3588 TODO just make a clean ule and append it to the omt
     // #3588 TODO can do the rebalancing here and avoid a lot of work later
     FTNODE leafnode = lbuf->node;
-    uint32_t idx = BLB_DATA(leafnode, 0)->omt_size();
+    uint32_t idx = BLB_DATA(leafnode, 0)->num_klpairs();
     DBT thekey = { .data = key, .size = (uint32_t) keylen }; 
     DBT theval = { .data = val, .size = (uint32_t) vallen };
-    FT_MSG_S cmd = { .type = FT_INSERT,
+    FT_MSG_S msg = { .type = FT_INSERT,
                      .msn = ZERO_MSN,
                      .xids = lbuf->xids,
                      .u = { .id = { &thekey, &theval } } };
     uint64_t workdone=0;
     // there's no mvcc garbage in a bulk-loaded FT, so there's no need to pass useful gc info
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true);
-    toku_ft_bn_apply_cmd_once(BLB(leafnode,0), &cmd, idx, NULL, &gc_info, &workdone, stats_to_update);
+    toku_ft_bn_apply_msg_once(BLB(leafnode,0), &msg, idx, NULL, &gc_info, &workdone, stats_to_update);
 }
 
 static int write_literal(struct dbout *out, void*data,  size_t len) {
@@ -2992,7 +2992,7 @@ static int write_translation_table (struct dbout *out, long long *off_of_transla
         putbuf_int64(&ttable, out->translation[i].off);
         putbuf_int64(&ttable, out->translation[i].size);
     }
-    unsigned int checksum = x1764_memory(ttable.buf, ttable.off);
+    unsigned int checksum = toku_x1764_memory(ttable.buf, ttable.off);
     putbuf_int32(&ttable, checksum);
     // pad it to 512 zeros
     long long encoded_length = ttable.off;

storage/tokudb/ft-index/ft/ftloader.h

@@ -105,7 +105,7 @@ int toku_ft_loader_open (FTLOADER *bl,
 			  generate_row_for_put_func g,
 			  DB *src_db,
 			  int N,
-			  FT_HANDLE brts[/*N*/], DB* dbs[/*N*/],
+			  FT_HANDLE ft_hs[/*N*/], DB* dbs[/*N*/],
 			  const char * new_fnames_in_env[/*N*/],
 			  ft_compare_func bt_compare_functions[/*N*/],
 			  const char *temp_file_template,

storage/tokudb/ft-index/ft/fttypes.h

@@ -198,7 +198,7 @@ typedef struct {
 
 static const STAT64INFO_S ZEROSTATS = {0,0};
 
-/* At the brt layer, a FILENUM uniquely identifies an open file.
+/* At the ft layer, a FILENUM uniquely identifies an open file.
  * At the ydb layer, a DICTIONARY_ID uniquely identifies an open dictionary.
  * With the introduction of the loader (ticket 2216), it is possible for the file that holds
  * an open dictionary to change, so these are now separate and independent unique identifiers.
@@ -230,7 +230,7 @@ typedef struct cachetable *CACHETABLE;
 typedef struct cachefile *CACHEFILE;
 typedef struct ctpair *PAIR;
 typedef class checkpointer *CHECKPOINTER;
-typedef class bn_data *BN_DATA;
+class bn_data;
 
 /* tree command types */
 enum ft_msg_type {
@@ -323,14 +323,14 @@ struct ft_msg {
     XIDS         xids;
     union {
         /* insert or delete */
-        struct ft_cmd_insert_delete {
+        struct ft_msg_insert_delete {
             const DBT *key;   // for insert, delete, upsertdel
             const DBT *val;   // for insert, delete, (and it is the "extra" for upsertdel, upsertdel_broadcast_all)
         } id;
     } u;
 };
-// Message sent into brt to implement command (insert, delete, etc.)
+
-// This structure supports nested transactions, and obsoletes ft_msg.
+// Message sent into the ft to implement insert, delete, update, etc
 typedef struct ft_msg FT_MSG_S;
 typedef struct ft_msg *FT_MSG;
 

storage/tokudb/ft-index/ft/ftverify.cc

@@ -98,7 +98,6 @@ PATENT RIGHTS GRANT:
 #include "ft-internal.h"
 #include "ft_layout_version.h"
 #include "block_table.h"
-#include "x1764.h"
 #include "rbuf.h"
 #include "sub_block.h"
 

storage/tokudb/ft-index/ft/hash-benchmarks/Makefile

@@ -1,4 +0,0 @@
-LDFLAGS = -lz
-CFLAGS = -O4 -Wall -W
-hash-benchmark:
-

storage/tokudb/ft-index/ft/hash-benchmarks/hash-benchmark-manually-open.cc

@@ -1,378 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-/* Benchmark various hash functions. */
-
-#include <sys/time.h>
-#include <zlib.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <toku_assert.h>
-
-#define N 200000000
-char *buf;
-
-static double tdiff (struct timeval *a, struct timeval *b) {
-    return a->tv_sec - b->tv_sec + (1e-6)*(a->tv_usec - b->tv_usec);
-}
-
-#define measure_bandwidth(str, body) ({		        \
-    int c; 				  	        \
-    struct timeval start,end;                           \
-    gettimeofday(&start, 0);                            \
-    body;                                               \
-    gettimeofday(&end, 0);                              \
-    double diff = tdiff(&end, &start);                  \
-    printf("%s=%08x %d bytes in %8.6fs for %8.3fMB/s\n", str, c, N, diff, N*(1e-6)/diff); \
-	})
-
-int sum32 (int start, void *buf, int bytecount)  {
-    int *ibuf = buf;
-    assert(bytecount%4==0);
-    while (bytecount>0) {
-	start+=*ibuf;
-	ibuf++;
-	bytecount-=4;
-    }
-    return start;
-}
-
-static const uint32_t m = 0x5bd1e995;
-static const int r = 24;
-static const uint32_t seed = 0x3dd3b51a;
-
-#define USE_ZERO_CHECKSUM 0
-
-static uint32_t MurmurHash2 ( const void * key, int len)
-{
-    if (USE_ZERO_CHECKSUM) return 0;
-
-    // 'm' and 'r' are mixing constants generated offline.
-    // They're not really 'magic', they just happen to work well.
-
-
-    // Initialize the hash to a 'random' value
-
-    uint32_t h = seed;
-
-    // Mix 4 bytes at a time into the hash
-
-    const unsigned char * data = (const unsigned char *)key;
-
-    while(len >= 4)
-	{
-	    uint32_t k = *(uint32_t *)data;
-
-	    k *= m; 
-	    k ^= k >> r; 
-	    k *= m; 
-		
-	    h *= m; 
-	    h ^= k;
-
-	    data += 4;
-	    len -= 4;
-	}
-	
-    // Handle the last few bytes of the input array
-
-    switch(len)
-	{
-	case 3: h ^= data[2] << 16;
-	case 2: h ^= data[1] << 8;
-	case 1: h ^= data[0];
-	    h *= m;
-	};
-
-    // Do a few final mixes of the hash to ensure the last few
-    // bytes are well-incorporated.
-
-    h ^= h >> 29;
-    h *= m;
-    h ^= h >> 31;
-
-    return h;
-} 
-
-struct murmur {
-    int n_bytes_in_k;  // How many bytes in k
-    uint32_t k;       // These are the extra bytes.   Bytes are shifted into the low-order bits.
-    uint32_t h;       // The hash so far (up to the most recent 4-byte boundary)
-};
-
-void murmur_init (struct murmur *mm) {
-    mm->n_bytes_in_k=0;
-    mm->k  =0;
-    mm->h = seed;
-}
-
-#define MIX() ({ k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; })
-#define LD1() data[0]
-#define LD2() ((data[0]<<8)  | data[1])
-#define LD3() ((data[0]<<16) | (data[1]<<8) | data[2])
-#define ADD1_0()  (mm->k =           LD1())
-#define ADD1()    (mm->k = (k<<8)  | LD1())
-#define ADD2_0()  (mm->k =           LD2())
-#define ADD2()    (mm->k = (k<<16) | LD2())
-#define ADD3_0()  (mm->k =           LD3())
-#define ADD3()    (mm->k = (k<<24) | LD3())
-
-void murmur_add (struct murmur *mm, const void * key, unsigned int len) {
-    if (USE_ZERO_CHECKSUM) return;
-    if (len==0) return;
-    const int n_bytes_in_k =  mm->n_bytes_in_k;
-    uint32_t k = mm->k;
-    const unsigned char *data = key;
-    uint32_t h = mm->h;
-    switch (n_bytes_in_k) {
-    case 0:
-	switch (len) {
-	case 1:  ADD1_0(); mm->n_bytes_in_k = 1;                       mm->h=h; return;
-	case 2:  ADD2_0(); mm->n_bytes_in_k = 2;                       mm->h=h; return;
-	case 3:  ADD3_0(); mm->n_bytes_in_k = 3;                       mm->h=h; return;
-	default: break;
-	}
-	break;
-    case 1:
-	switch (len) {
-	case 1:  ADD1(); mm->n_bytes_in_k = 2;                         mm->h=h; return;
-	case 2:  ADD2(); mm->n_bytes_in_k = 3;                         mm->h=h; return;
-	case 3:  ADD3(); mm->n_bytes_in_k = 0; MIX();                  mm->h=h; return;
-	default: ADD3(); mm->n_bytes_in_k = 0; MIX(); len-=3; data+=3; break;
-	}
-	break;
-    case 2:
-	switch (len) {
-	case 1:  ADD1(); mm->n_bytes_in_k = 3;                         mm->h=h; return;
-	case 2:  ADD2(); mm->n_bytes_in_k = 0; MIX();                  mm->h=h; return;
-	default: ADD2(); mm->n_bytes_in_k = 0; MIX(); len-=2; data+=2; break;
-	}
-	break;
-    case 3:
-	switch (len) {
-	case 1:  ADD1(); mm->n_bytes_in_k = 0; MIX();                  mm->h=h; return;
-	default: ADD1(); mm->n_bytes_in_k = 0; MIX(); len--; data++;   break;
-	}
-	break;
-    default: assert(0);
-    }
-
-    // We've used up the partial bytes at the beginning of k.
-    assert(mm->n_bytes_in_k==0);
-    while (len >= 4) {
-	uint32_t k = toku_dtoh32(*(uint32_t *)data);
-	//printf(" oldh=%08x k=%08x", h, k);
-
-	k *= m; 
-	k ^= k >> r; 
-	k *= m; 
-		
-	h *= m; 
-	h ^= k;
-
-	data += 4;
-	len -= 4;
-	//printf(" h=%08x\n", h);
-    }
-    mm->h=h;
-    //printf("%s:%d h=%08x\n", __FILE__, __LINE__, h);
-    {
-	uint32_t k=0;
-	switch (len) {
-	case 3: k =  *data << 16;  data++;
-	case 2: k |= *data << 8;   data++;
-	case 1: k |= *data;
-	}
-	mm->k = k;
-	mm->n_bytes_in_k = len;
-	//printf("now extra=%08x (%d bytes) n_bytes=%d\n", mm->k, len, mm->n_bytes_in_k);
-
-    }
-}
-
-uint32_t murmur_finish (struct murmur *mm) {
-    if (USE_ZERO_CHECKSUM) return 0;
-    uint32_t h = mm->h;
-    if (mm->n_bytes_in_k>0) {
-	h ^= mm->k;
-	h *= m;
-    }
-    if (0) {
-	// The real murmur function does this extra mixing at the end.  We don't need that for fingerprint.
-	h ^= h >> 29;
-	h *= m;
-	h ^= h >> 31;
-    }
-    return h;
-}
-
-struct sum84 {
-    uint32_t sum;
-    int i;
-};
-void sum84_init (struct sum84 *s) { s->sum=0; s->i=0; };
-void sum84_add  (struct sum84 *s, char *buf, int count) {
-    while (s->i%4!=0 && count>0) {
-	char v = *buf;
-	s->sum ^= v << (s->i%4)*8;
-	buf++; count--; s->i++;
-    }
-    while (count>4) {
-	s->sum ^= *(int*)buf;
-	buf+=4; count-=4;
-    }
-    while (count>0) {
-	char v = *buf;
-	s->sum ^= v << (s->i%4)*8;
-	buf++; count--; s->i++;
-    }
-}
-int sum84_finish (struct sum84 *s) {
-    return s->sum;
-}
-
-uint32_t xor8_add  (uint32_t x, char *buf, int count) {
-    while (count>4) {
-	x ^= *(int*)buf;
-	buf+=4; count-=4;
-    }
-    while (count>0) {
-	char v = *buf;
-	x ^= v;
-	buf++; count--;
-    }
-    return x;
-}
-uint32_t xor8_finish (uint32_t x) {
-    return (x ^ (x>>8) ^ (x>>16) ^ (x>>24))&0xff;
-}
-
-uint64_t xor8_64_add  (uint64_t x, char *buf, int count) {
-    while (count>8) {
-	x ^= *(uint64_t*)buf;
-	buf+=8; count-=8;
-    }
-    while (count>0) {
-	char v = *buf;
-	x ^= v;
-	buf++; count--;
-    }
-    return x;
-}
-uint32_t xor8_64_finish (uint64_t x) {
-    return (x ^ (x>>8) ^ (x>>16) ^ (x>>24) ^ (x>>32) ^ (x>>40) ^ (x>>48) ^ (x>>56))&0xff;
-}
-
-static void measure_bandwidths (void) {
-    measure_bandwidth("crc32    ", c=crc32(0, buf, N));
-    measure_bandwidth("sum32    ", c=sum32(0, buf, N));
-    measure_bandwidth("murmur   ", c=MurmurHash2(buf, N));
-    measure_bandwidth("murmurf  ", ({ struct murmur mm; murmur_init(&mm); murmur_add(&mm, buf, N); c=murmur_finish(&mm); }));
-    measure_bandwidth("sum84    ", ({ struct sum84 s; sum84_init(&s); sum84_add(&s, buf, N); c=sum84_finish(&s); }));
-    measure_bandwidth("xor32    ", ({ c=0; int j; for(j=0; j<N/4; j++) c^=*(int*)buf+j*4; }));
-    measure_bandwidth("xor8     ", c=xor8_finish(xor8_add(0, buf, N)));
-    measure_bandwidth("xor8_64  ", c=xor8_64_finish(xor8_64_add(0, buf, N)));
-    measure_bandwidth("crc32by1 ", ({ c=0; int j; for(j=0; j<N; j++) c=crc32(c, buf+j, 1); }));
-    measure_bandwidth("crc32by2 ", ({ c=0; int j; for(j=0; j<N; j+=2) c=crc32(c, buf+j, 2); }));
-    measure_bandwidth("sum8by1  ", ({ c=0; int j; for(j=0; j<N; j++) c+=buf[j]; }));
-    measure_bandwidth("murmurby1", ({ struct murmur mm; murmur_init(&mm); int j; for(j=0; j<N; j++) murmur_add(&mm, buf+j, 1); c=murmur_finish(&mm); }));
-    measure_bandwidth("murmurby2", ({ struct murmur mm; murmur_init(&mm); int j; for(j=0; j<N; j+=2) murmur_add(&mm, buf+j, 2); c=murmur_finish(&mm); }));
-    measure_bandwidth("sum84by1 ", ({ struct sum84 s; sum84_init(&s); int j; for(j=0; j<N; j++) sum84_add(&s, buf+j, 1); c=sum84_finish(&s); }));
-    measure_bandwidth("xor8by1  ", ({ int j; c=0; for(j=0; j<N; j++) c=xor8_add(c, buf+j, 1); c=xor8_finish(c); }));
-    measure_bandwidth("xor864by1", ({ int j; uint64_t x=0; for(j=0; j<N; j++) x=xor8_64_add(x, buf+j, 1); c=xor8_64_finish(x); }));
-}
-
-int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
-    buf = malloc(N);
-    int i;
-    for (i=0; i<N; i++) buf[i]=random();
-    measure_bandwidths();
-    return 0;
-}

storage/tokudb/ft-index/ft/le-cursor.h

@@ -104,10 +104,10 @@ PATENT RIGHTS GRANT:
 
 typedef struct le_cursor *LE_CURSOR;
 
-// Create a leaf cursor for a tree (brt) within a transaction (txn)
+// Create a leaf cursor for a tree (ft_h) within a transaction (txn)
 // Success: returns 0, stores the LE_CURSOR in the le_cursor_result
 // Failure: returns a non-zero error number
-int toku_le_cursor_create(LE_CURSOR *le_cursor_result, FT_HANDLE brt, TOKUTXN txn);
+int toku_le_cursor_create(LE_CURSOR *le_cursor_result, FT_HANDLE ft_h, TOKUTXN txn);
 
 // Close and free the LE_CURSOR
 void toku_le_cursor_close(LE_CURSOR le_cursor);

storage/tokudb/ft-index/ft/leafentry.h

@@ -96,11 +96,10 @@ PATENT RIGHTS GRANT:
 #include <toku_portability.h>
 
 #include <util/mempool.h>
+#include <util/omt.h>
 
 #include "txn_manager.h"
 #include "rbuf.h"
-#include "x1764.h"
-#include "omt.h"
 
 /*
     Memory format of packed leaf entry

storage/tokudb/ft-index/ft/locking-benchmarks/Makefile

@@ -1,4 +0,0 @@
-CPPFLAGS= -D_GNU_SOURCE -I../../toku_include
-CFLAGS=-O2 -Wall -W -Werror
-LDFLAGS=-lpthread
-default: trylock trylock-rdtsc mfence-benchmark process_locks pthread-locks

storage/tokudb/ft-index/ft/locking-benchmarks/mfence-benchmark.cc

@@ -1,217 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-/* Time {m,l,s}fence vs.xchgl for a memory barrier. */
-
-/* Timing numbers:
- * Intel T2500 2GHZ
-
-do1       9.0ns/loop
-mfence:  29.0ns/loop  (marginal cost=  20.0ns)
-sfence:  17.3ns/loop  (marginal cost=   8.3ns)
-lfence:  23.6ns/loop  (marginal cost=  14.6ns)
- xchgl:  35.8ns/loop  (marginal cost=  26.8ns)
-
-* AMD Athlon 64 X2 Dual Core Processor 4200+
-  Timings are more crazy
-
-do1      20.6ns/loop
-mfence:  12.9ns/loop  (marginal cost=  -7.6ns)
-sfence:   8.4ns/loop  (marginal cost= -12.1ns)
-lfence:  20.2ns/loop  (marginal cost=  -0.3ns)
- xchgl:  16.6ns/loop  (marginal cost=  -3.9ns)
-
-do1      13.0ns/loop
-mfence:  25.6ns/loop  (marginal cost=  12.6ns)
-sfence:  21.0ns/loop  (marginal cost=   8.1ns)
-lfence:  12.9ns/loop  (marginal cost=  -0.1ns)
- xchgl:  29.3ns/loop  (marginal cost=  16.3ns)
-
-*/
-
-
-#include <sys/time.h>
-#include <stdio.h>
-#include <portability/toku_atomic.h>
-
-enum { COUNT = 100000000 };
-
-static inline void xchgl (void) {
-    {
-	/*
-	 * According to the Intel Architecture Software Developer's
-	 * Manual, Volume 3: System Programming Guide
-	 * (http://www.intel.com/design/pro/manuals/243192.htm), page
-	 * 7-6, "For the P6 family processors, locked operations
-	 * serialize all outstanding load and store operations (that
-	 * is, wait for them to complete)."  
-	 * Since xchg is locked by default, it is one way to do membar.
-	 */
-	int x=0, y;
-	asm volatile ("xchgl %0,%1" :"=r" (x) :"m" (y), "0" (x) :"memory");
-   }
-}
-
-static inline void mfence (void) {
-    asm volatile ("mfence":::"memory");
-}
-
-static inline void lfence (void) {
-    asm volatile ("lfence":::"memory");
-}
-
-static inline void sfence (void) {
-    asm volatile ("sfence":::"memory");
-}
-
-int lock_for_lock_and_unlock;
-static inline void lock_and_unlock (void) {
-    (void)toku_sync_lock_test_and_set(&lock_for_lock_and_unlock, 1);
-    toku_sync_lock_release(&lock_for_lock_and_unlock);
-}
-
-
-double tdiff (struct timeval *start, struct timeval *end) {
-    return ((end->tv_sec-start->tv_sec + 1e-6*(end->tv_usec + start->tv_usec))/COUNT)*1e9;
-}
-
-double nop_cost;
-
-void do1 (volatile int *x) {
-    int i;
-    struct timeval start, end;
-    gettimeofday(&start, 0);
-    for (i=0; i<COUNT; i++) {
-	x[0]++;
-	x[1]++;
-	x[2]++;
-	x[3]++;
-    }
-    gettimeofday(&end, 0);
-    printf("do1    %6.1fns/loop\n", nop_cost=tdiff(&start, &end));
-}
-
-#define doit(name) void do ##name (volatile int *x) { \
-    int i;                      \
-    struct timeval start, end;  \
-    gettimeofday(&start, 0);    \
-    for (i=0; i<COUNT; i++) {   \
-	x[0]++;                 \
-	x[1]++;                 \
-	name();                 \
-	x[2]++;                 \
-	x[3]++;                 \
-    }                           \
-    gettimeofday(&end, 0);      \
-    double this_cost = tdiff(&start, &end); \
-    printf("%15s:%6.1fns/loop  (marginal cost=%6.1fns)\n",  #name, this_cost, this_cost-nop_cost); \
-}
-
-
-doit(mfence)
-doit(lfence)
-doit(sfence)
-doit(xchgl)
-doit(lock_and_unlock);
-
-int main (int argc __attribute__((__unused__)), 
-	  char *argv[] __attribute__((__unused__))) {
-    int x[4];
-    int i;
-    for (i=0; i<4; i++) {
-	do1(x);
-	domfence(x);
-	dosfence(x);
-	dolfence(x);
-	doxchgl(x);
-	dolock_and_unlock(x);
-    }
-    return 0;
-}

storage/tokudb/ft-index/ft/locking-benchmarks/process_locks.cc

@@ -1,233 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-/* Test pthread rwlocks in multiprocess environment. */
-
-/* How expensive is
- *   - Obtaining a read-only lock for the first obtainer.
- *   - Obtaining it for the second one?
- *   - The third one? */
-
-#include <toku_assert.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <unistd.h>
-
-
-float tdiff (struct timeval *start, struct timeval *end) {
-    return 1e6*(end->tv_sec-start->tv_sec) +(end->tv_usec - start->tv_usec);
-}
-
-#define FILE "process.data"
-
-int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
-    int r;
-    int fd;
-    void *p;
-    fd=open(FILE, O_CREAT|O_RDWR|O_TRUNC, 0666); assert(fd>=0);
-    int i;
-    for (i=0; i<4096; i++) {
-	r=write(fd, "\000", 1);
-	assert(r==1);
-    }
-    p=mmap(0, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
-    if (p==MAP_FAILED) {
-	printf("err=%d %s (EPERM=%d)\n", errno, strerror(errno), EPERM);
-    }
-    assert(p!=MAP_FAILED);
-    r=close(fd); assert(r==0);
-
-    pthread_rwlockattr_t attr;
-    pthread_rwlock_t *lock=p;
-    r=pthread_rwlockattr_init(&attr);                               assert(r==0);
-    r=pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); assert(r==0);
-    r=pthread_rwlock_init(lock, &attr);                             assert(r==0);
-    r=pthread_rwlock_init(lock+1, &attr);                             assert(r==0);
-
-    r=pthread_rwlock_wrlock(lock);
-
-    pid_t pid;
-    if ((pid=fork())==0) {
-	// I'm the child
-	r = munmap(p, 4096); assert(r==0);
-	fd = open(FILE, O_RDWR, 0666); assert(fd>=0);
-	p=mmap(0, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
-	assert(p!=MAP_FAILED);
-	r=close(fd); assert(r==0);
-	
-	printf("A0\n");
-	r=pthread_rwlock_wrlock(lock);
-	printf("C\n");
-	sleep(1);
-    
-	r=pthread_rwlock_unlock(lock);
-	printf("D\n");
-
-	r=pthread_rwlock_rdlock(lock);
-	printf("E0\n");
-	sleep(1);
-
-    } else {
-	printf("A1\n");
-	sleep(1);
-	printf("B\n");
-	r=pthread_rwlock_unlock(lock); // release the lock grabbed before the fork
-	assert(r==0);
-
-	sleep(1);
-	r=pthread_rwlock_rdlock(lock);
-	assert(r==0);
-	printf("E1\n");
-	sleep(1);
-
-	int status;
-	pid_t waited=wait(&status);
-	assert(waited==pid);
-    }
-    return 0;    
-    
-    
-#if 0    
-
-    int j;
-    int i;
-    int r;
-    struct timeval start, end;
-    for (j=0; j<3; j++) {
-	for (i=0; i<K; i++) {
-	    r=pthread_rwlock_init(&rwlocks[i], NULL);
-	    assert(r==0);
-	}
-	gettimeofday(&start, 0);
-	for (i=0; i<K; i++) {
-	    r = pthread_rwlock_tryrdlock(&rwlocks[i]);
-	    assert(r==0);
-	}
-	gettimeofday(&end, 0);
-	printf("pthread_rwlock_tryrdlock took %9.3fus for %d ops:  %9.3fus/lock (%9.3fMops/s)\n", tdiff(&start,&end), K, tdiff(&start,&end)/K, K/tdiff(&start,&end));
-    }
-
-    for (j=0; j<3; j++) {
-	for (i=0; i<K; i++) {
-	    r=pthread_rwlock_init(&rwlocks[i], NULL);
-	    assert(r==0);
-	}
-	gettimeofday(&start, 0);
-	for (i=0; i<K; i++) {
-	    r = pthread_rwlock_rdlock(&rwlocks[i]);
-	    assert(r==0);
-	}
-	gettimeofday(&end, 0);
-	printf("pthread_rwlock_rdlock    took %9.3fus for %d ops:  %9.3fus/lock (%9.3fMops/s)\n", tdiff(&start,&end), K, tdiff(&start,&end)/K, K/tdiff(&start,&end));
-    }
-
-    for (j=0; j<3; j++) {
-	for (i=0; i<K; i++) {
-	    blocks[i].state=0;
-	    blocks[i].mutex=0;
-	}
-	gettimeofday(&start, 0);
-	for (i=0; i<K; i++) {
-	    brwl_rlock(&blocks[i]);
-	}
-	gettimeofday(&end, 0);
-	printf("brwl_rlock               took %9.3fus for %d ops:  %9.3fus/lock (%9.3fMops/s)\n", tdiff(&start,&end), K, tdiff(&start,&end)/K, K/tdiff(&start,&end));
-    }
-    return 0;
-#endif
-}

storage/tokudb/ft-index/ft/locking-benchmarks/pthread-locks.cc

@@ -1,272 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-/* How expensive is
- *   - Obtaining a read-only lock for the first obtainer.
- *   - Obtaining it for the second one?
- *   - The third one? */
-
-#include <toku_assert.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <sys/time.h>
-#include <pthread.h>
-#include <portability/toku_atomic.h>
-
-float tdiff (struct timeval *start, struct timeval *end) {
-    return 1e6*(end->tv_sec-start->tv_sec) +(end->tv_usec - start->tv_usec);
-}
-
-/* My own rwlock implementation. */
-struct brwl {
-    int mutex;
-    int state; // 0 for unlocked, -1 for a writer, otherwise many readers
-};
-
-static inline int xchg(volatile int *ptr, int x)
-{
-    __asm__("xchgl %0,%1" :"=r" (x) :"m" (*(ptr)), "0" (x) :"memory");
-    return x;
-}
-
-static inline void sfence (void) {
-    asm volatile ("sfence":::"memory");
-}
-
-static inline void brwl_rlock_fence (struct brwl *l) {
-    while (xchg(&l->mutex, 1)) ;
-    l->state++;
-    sfence();
-    l->mutex=0;
-}
-
-static inline void brwl_rlock_xchg (struct brwl *l) {
-    while (xchg(&l->mutex, 1)) ;
-    l->state++;
-    xchg(&l->mutex, 0);
-}
-
-// Something wrong with the compiler  for longs
-static inline long
-fetch_and_add (volatile long *p, long incr)
-{
-  long result = incr;
-
-  __asm__ __volatile__ ("lock; xaddl %0, %1" :
-			"+r" (result), "+m" (*p) : : "memory");
-  return result;
-}
-
-static inline int
-fetch_and_add_i (volatile int *p, int incr)
-{
-  int result = incr;
-
-  __asm__ __volatile__ ("lock; xadd %0, %1" :
-			"+r" (result), "+m" (*p) : : "memory");
-  return result;
-}
-
-static inline int
-gcc_fetch_and_add_i (volatile int *p, int incr)
-{
-  return toku_sync_fetch_and_add(p, incr);
-}
-
-static inline long
-gcc_fetch_and_add_l (volatile long *p, long incr)
-{
-  return toku_sync_fetch_and_add(p, incr);
-}
-
-// Something wrong with the compiler  for longs
-/* Returns nonzero if the comparison succeeded. */
-static inline long
-compare_and_swap_full(volatile long *addr,
-		  	     long old, long new_val) 
-{
-  char result;
-  __asm__ __volatile__("lock; cmpxchgl %2, %0; setz %1"
-	    	       : "+m"(*(addr)), "=q"(result)
-		       : "r" (new_val), "a"(old) : "memory");
-  return (int) result;
-}
-
-/* Returns nonzero if the comparison succeeded. */
-//	Atomically compare *addr to old_val, and replace *addr by new_val
-//	if the first comparison succeeds.  Returns nonzero if the comparison
-//	succeeded and *addr was updated.
-static inline int
-compare_and_swap_full_i(volatile int *addr,
-			int old, int new_val) 
-{
-  char result;
-  __asm__ __volatile__("lock; cmpxchg %2, %0; setz %1"
-	    	       : "+m"(*(addr)), "=q"(result)
-		       : "r" (new_val), "a"(old) : "memory");
-  return (int) result;
-}
-
-enum {K=100000};
-pthread_rwlock_t rwlocks[K];
-struct brwl blocks[K];
-pthread_mutex_t mlocks[K];
-long lvals[K];
-int ivals[K];
-
-#define TIME(s, i, init, body) ({    \
-   int j_tmp;          		     \
-   printf("%-24s", s);               \
-   for (j_tmp=0; j_tmp<3; j_tmp++) { \
-       struct timeval start,end;     \
-       int i;                        \
-       for (i=0; i<K; i++) {         \
-           init;                     \
-       }                             \
-       gettimeofday(&start, 0);      \
-       for (i=0; i<K; i++) {         \
-	   body;	             \
-       }                             \
-       gettimeofday(&end, 0);        \
-       printf(" %9.3fus", tdiff(&start,&end)/K); \
-   }                                 \
-   printf("\n");                     \
-	})
-
-int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
-
-    printf("sizeof (pthread_mutex_t) %lu\n", sizeof (pthread_mutex_t));
-    printf("sizeof (pthread_cond_t) %lu\n", sizeof (pthread_cond_t));
-
-    TIME("pthread_mutex_lock_errorcheck", i,
-	 ({ int r; pthread_mutexattr_t mattr; 
-	   r = pthread_mutexattr_init(&mattr); assert(r == 0);
-	   r = pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_ERRORCHECK_NP); assert(r == 0);
-	   r = pthread_mutex_init(&mlocks[i], &mattr); assert(r==0); 
-	   r = pthread_mutexattr_destroy(&mattr); assert(r == 0); }),
-	 ({ int r = pthread_mutex_lock(&mlocks[i]);       assert(r==0); }));
-    TIME("pthread_mutex_lock", i,
-	 ({ int r = pthread_mutex_init(&mlocks[i], NULL); assert(r==0); }),
-	 ({ int r = pthread_mutex_lock(&mlocks[i]);       assert(r==0); }));
-    TIME("pthread_mutex_unlock", i,
-	 ({ int r = pthread_mutex_init(&mlocks[i], NULL); assert(r==0); r = pthread_mutex_lock(&mlocks[i]); assert(r==0); }),
-	 ({ int r = pthread_mutex_unlock(&mlocks[i]);     assert(r==0); }));	 
-    TIME("pthread_rwlock_tryrdlock", i,
-	 ({ int r = pthread_rwlock_init(&rwlocks[i], NULL);  assert(r==0); }),
-	 ({ int r = pthread_rwlock_tryrdlock(&rwlocks[i]); assert(r==0); }));
-    TIME("pthread_rwlock_rdlock", i,
-	 ({ int r = pthread_rwlock_init(&rwlocks[i], NULL); assert(r==0); }),
-	 ({ int r = pthread_rwlock_rdlock(&rwlocks[i]);	    assert(r==0); }));
-    TIME("brwl_rlock_xchg", i,
-	 (blocks[i].state=0, blocks[i].mutex=0),
-	 brwl_rlock_xchg(&blocks[i]));
-    TIME("brwl_rlock_fence", i,
-	 (blocks[i].state=0, blocks[i].mutex=0),
-	 brwl_rlock_fence(&blocks[i]));
-    int fa=0;
-    TIME("fetchadd", i,
-	 (void)0,
-	 fetch_and_add_i(&fa, i));
-    // printf("fa=%d\n", fa);
-    fa=0;
-    TIME("gcc_fetchadd", i,
-	 (void)0,
-	 gcc_fetch_and_add_i(&fa, i));
-    // printf("fa=%d\n", fa);
-    long fal = 0;
-    TIME("gcc_fetchaddlong", i,
-	 (void)0,
-	 gcc_fetch_and_add_l(&fal, i));
-    // printf("fa=%d\n", fa);
-    TIME("compare_and_swap", i,
-	 ivals[i]=0,
-	 ({ int r=compare_and_swap_full_i(&ivals[i], 0, 1); assert(r==1); }));
-    return 0;
-}

storage/tokudb/ft-index/ft/locking-benchmarks/trylock-rdtsc.cc

@@ -1,247 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-/* Like trylock, except use rdstc */
-#define _MULTI_THREADED
-#include <pthread.h>
-#include <stdio.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <sys/time.h>
-#include <unistd.h>
-#include <rdtsc.h>
-#include <portability/toku_atomic.h>
-
-float tdiff (struct timeval *start, struct timeval *end) {
-    return 1e6*(end->tv_sec-start->tv_sec) +(end->tv_usec - start->tv_usec);
-}
-
-unsigned long long rtdiff (unsigned long long a, unsigned long long b) {
-    return (b-a);
-}
-
-/* Simple function to check the return code and exit the program
-   if the function call failed
-   */
-static void compResults(char *string, int rc) {
-  if (rc) {
-    printf("Error on : %s, rc=%d",
-           string, rc);
-    exit(EXIT_FAILURE);
-  }
-  return;
-}
-
-pthread_rwlock_t       rwlock = PTHREAD_RWLOCK_INITIALIZER;
-
-void *rdlockThread(void *arg __attribute__((unused)))
-{
-  int             rc;
-  int             count=0;
-
-  unsigned long long t_start, t_end;
-
-  printf("Entered thread, getting read lock with mp wait\n");
-  Retry:
-
-  t_start = rdtsc();
-  rc = pthread_rwlock_tryrdlock(&rwlock);
-  t_end = rdtsc();
-  printf("pthread_rwlock_tryrdlock took %llu clocks\n", rtdiff(t_start,t_end));
-  if (rc == EBUSY) {
-    if (count >= 10) {
-      printf("Retried too many times, failure!\n");
-
-      exit(EXIT_FAILURE);
-    }
-    ++count;
-    printf("Could not get lock, do other work, then RETRY...\n");
-    sleep(1);
-    goto Retry;
-  }
-  compResults("pthread_rwlock_tryrdlock() 1\n", rc);
-
-  sleep(2);
-
-  printf("unlock the read lock\n");
-  t_start = rdtsc();
-  rc = pthread_rwlock_unlock(&rwlock);
-  t_end = rdtsc();
-  compResults("pthread_rwlock_unlock()\n", rc);
-  printf("Took %llu clocks\n", rtdiff(t_start, t_end));
-
-  printf("Secondary thread complete\n");
-  return NULL;
-}
-
-int main(int argc __attribute__((unused)), char **argv)
-{
-  int                   rc=0;
-  pthread_t             thread;
-  unsigned long long t_start, t_end;
-
-  printf("Enter Testcase - %s\n", argv[0]);
-
-  t_start = rdtsc();
-  t_end   = rdtsc();
-  printf("nop Took %llu clocks\n", rtdiff(t_start, t_end));
-
-  {
-      int N=1000;
-      int i;
-      printf("Main, get and release the write lock %d times\n", N);
-      t_start = rdtsc();
-      for (i=0; i<N; i++) {
-	  rc = pthread_rwlock_wrlock(&rwlock);
-	  rc = pthread_rwlock_unlock(&rwlock);
-      }
-      t_end = rdtsc();
-      compResults("pthread_rwlock_wrlock()\n", rc);
-      printf("Took %5.2f clocks/op\n", ((double)(t_end-t_start))/N);
-  }
-
-  printf("Main, get the write lock\n");
-  t_start = rdtsc();
-  rc = pthread_rwlock_wrlock(&rwlock);
-  t_end   = rdtsc();
-  compResults("pthread_rwlock_wrlock()\n", rc);
-  printf("Took %llu clocks\n", rtdiff(t_start, t_end));
-
-  printf("Main, create the try read lock thread\n");
-  rc = pthread_create(&thread, NULL, rdlockThread, NULL);
-  compResults("pthread_create\n", rc);
-
-  printf("Main, wait a bit holding the write lock\n");
-  sleep(5);
-
-  printf("Main, Now unlock the write lock\n");
-  t_start = rdtsc();
-  rc = pthread_rwlock_unlock(&rwlock);
-  t_end   = rdtsc();
-  compResults("pthread_rwlock_unlock()\n", rc);
-  printf("Took %llu clocks\n", rtdiff(t_start, t_end));
-
-  printf("Main, wait for the thread to end\n");
-  rc = pthread_join(thread, NULL);
-  compResults("pthread_join\n", rc);
-
-  rc = pthread_rwlock_destroy(&rwlock);
-  compResults("pthread_rwlock_destroy()\n", rc);
-  printf("Main completed\n");
-
-  
-  {
-      static int lock_for_lock_and_unlock;
-      t_start = rdtsc();
-      (void)toku_sync_lock_test_and_set(&lock_for_lock_and_unlock, 1);
-      t_end   = rdtsc();
-      printf("sync_lock_test_and_set took %llu clocks\n", t_end-t_start);
-
-      t_start = rdtsc();
-      toku_sync_lock_release(&lock_for_lock_and_unlock);
-      t_end   = rdtsc();
-      printf("sync_lock_release      took %llu clocks\n", t_end-t_start);
-  }
-
-
-  {
-      t_start = rdtsc();
-      (void)toku_sync_synchronize();
-      t_end   = rdtsc();
-      printf("sync_synchornize took %llu clocks\n", t_end-t_start);
-  }
-
-  t_start = rdtsc();
-  sleep(1);
-  t_end   = rdtsc();
-  printf("sleep(1) took %llu clocks\n", t_end-t_start);
-  return 0;
-}

storage/tokudb/ft-index/ft/locking-benchmarks/trylock.cc

@@ -1,213 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-#define _MULTI_THREADED
-#include <pthread.h>
-#include <stdio.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <sys/time.h>
-#include <unistd.h>
-
-float tdiff (struct timeval *start, struct timeval *end) {
-    return 1e6*(end->tv_sec-start->tv_sec) +(end->tv_usec - start->tv_usec);
-}
-
-/* Simple function to check the return code and exit the program
-   if the function call failed
-   */
-static void compResults(char *string, int rc) {
-  if (rc) {
-    printf("Error on : %s, rc=%d",
-           string, rc);
-    exit(EXIT_FAILURE);
-  }
-  return;
-}
-
-pthread_rwlock_t       rwlock = PTHREAD_RWLOCK_INITIALIZER;
-
-void *rdlockThread(void *arg __attribute__((unused)))
-{
-  int             rc;
-  int             count=0;
-
-  struct timeval start, end;
-
-  printf("Entered thread, getting read lock with mp wait\n");
-  Retry:
-
-  gettimeofday(&start, 0);
-  rc = pthread_rwlock_tryrdlock(&rwlock);
-  gettimeofday(&end, 0);
-  printf("pthread_rwlock_tryrdlock took %9.3fus\n", tdiff(&start,&end));
-  if (rc == EBUSY) {
-    if (count >= 10) {
-      printf("Retried too many times, failure!\n");
-
-      exit(EXIT_FAILURE);
-    }
-    ++count;
-    printf("Could not get lock, do other work, then RETRY...\n");
-    sleep(1);
-    goto Retry;
-  }
-  compResults("pthread_rwlock_tryrdlock() 1\n", rc);
-
-  sleep(2);
-
-  printf("unlock the read lock\n");
-  gettimeofday(&start, 0);
-  rc = pthread_rwlock_unlock(&rwlock);
-  gettimeofday(&end, 0);
-  compResults("pthread_rwlock_unlock()\n", rc);
-  printf("%lu.%6lu to %lu.%6lu is %9.2f\n", start.tv_sec, start.tv_usec, end.tv_sec, end.tv_usec, tdiff(&start, &end));
-
-  printf("Secondary thread complete\n");
-  return NULL;
-}
-
-int main(int argc __attribute__((unused)), char **argv)
-{
-  int                   rc=0;
-  pthread_t             thread;
-  struct timeval start, end;
-
-  printf("Enter Testcase - %s\n", argv[0]);
-
-  gettimeofday(&start, 0);
-  gettimeofday(&end, 0);
-  printf("nop Took %9.2f\n", tdiff(&start, &end));
-
-  {
-      int N=1000;
-      int i;
-      printf("Main, get and release the write lock %d times\n", N);
-      gettimeofday(&start, 0);
-      for (i=0; i<N; i++) {
-	  rc = pthread_rwlock_wrlock(&rwlock);
-	  rc = pthread_rwlock_unlock(&rwlock);
-      }
-      gettimeofday(&end, 0);
-      compResults("pthread_rwlock_wrlock()\n", rc);
-      printf("Took %9.2fns/op\n", 1000*tdiff(&start, &end)/N);
-  }
-
-  printf("Main, get the write lock\n");
-  gettimeofday(&start, 0);
-  rc = pthread_rwlock_wrlock(&rwlock);
-  gettimeofday(&end, 0);
-  compResults("pthread_rwlock_wrlock()\n", rc);
-  printf("Took %9.2f\n", tdiff(&start, &end));
-
-  printf("Main, create the try read lock thread\n");
-  rc = pthread_create(&thread, NULL, rdlockThread, NULL);
-  compResults("pthread_create\n", rc);
-
-  printf("Main, wait a bit holding the write lock\n");
-  sleep(5);
-
-  printf("Main, Now unlock the write lock\n");
-  gettimeofday(&start, 0);
-  rc = pthread_rwlock_unlock(&rwlock);
-  gettimeofday(&end, 0);
-  compResults("pthread_rwlock_unlock()\n", rc);
-  printf("Took %9.2f\n", tdiff(&start, &end));
-
-  printf("Main, wait for the thread to end\n");
-  rc = pthread_join(thread, NULL);
-  compResults("pthread_join\n", rc);
-
-  rc = pthread_rwlock_destroy(&rwlock);
-  compResults("pthread_rwlock_destroy()\n", rc);
-  printf("Main completed\n");
-  return 0;
-}

storage/tokudb/ft-index/ft/log-internal.h

@@ -99,15 +99,17 @@ PATENT RIGHTS GRANT:
 #include "ft-internal.h"
 #include "log.h"
 #include "toku_list.h"
-#include "memarena.h"
 #include "logfilemgr.h"
 #include "txn.h"
 #include "txn_manager.h"
-#include <portability/toku_pthread.h>
-#include <util/omt.h>
 #include "rollback_log_node_cache.h"
 #include "txn_child_manager.h"
 
+#include <portability/toku_pthread.h>
+
+#include <util/memarena.h>
+#include <util/omt.h>
+
 using namespace toku;
 // Locking for the logger
 //  For most purposes we use the big ydb lock.

storage/tokudb/ft-index/ft/log.h

@@ -94,19 +94,17 @@ PATENT RIGHTS GRANT:
 
 #include <toku_portability.h>
 #include <errno.h>
-
 #include <db.h>
+
 #include "fttypes.h"
 #include "memory.h"
-#include "x1764.h"
-
-struct roll_entry;
-
 #include "logger.h"
 #include "rollback.h"
 #include "recover.h"
 #include "txn.h"
 
+struct roll_entry;
+
 static inline int toku_copy_BYTESTRING(BYTESTRING *target, BYTESTRING val) {
     target->len = val.len;
     target->data = (char *) toku_memdup(val.data, (size_t)val.len);
@@ -117,6 +115,7 @@ static inline int toku_copy_BYTESTRING(BYTESTRING *target, BYTESTRING val) {
 }
 static inline void toku_free_TXNID(TXNID txnid __attribute__((__unused__))) {}
 static inline void toku_free_TXNID_PAIR(TXNID_PAIR txnid __attribute__((__unused__))) {}
+
 static inline void toku_free_LSN(LSN lsn __attribute__((__unused__))) {}
 static inline void toku_free_uint64_t(uint64_t u __attribute__((__unused__))) {}
 static inline void toku_free_uint32_t(uint32_t u __attribute__((__unused__))) {}

storage/tokudb/ft-index/ft/logcursor.cc

@@ -167,11 +167,8 @@ static int lc_open_logfile(TOKULOGCURSOR lc, int index) {
     lc->cur_fp = fopen(lc->logfiles[index], "rb");
     if ( lc->cur_fp == NULL ) 
         return DB_NOTFOUND;
-    // debug printf("%s:%d %s %p %u\n", __FUNCTION__, __LINE__, lc->logfiles[index], lc->buffer, (unsigned) lc->buffer_size);
-#if !TOKU_WINDOWS //Windows reads logs fastest if we use default settings (not use setvbuf to change buffering)
     r = setvbuf(lc->cur_fp, (char *) lc->buffer, _IOFBF, lc->buffer_size);
     assert(r == 0);
-#endif
     // position fp past header, ignore 0 length file (t:2384)
     unsigned int version=0;
     if ( lc_file_len(lc->logfiles[index]) >= 12 ) {

storage/tokudb/ft-index/ft/logformat.cc

@@ -536,7 +536,7 @@ generate_log_writer (void) {
                                   if (strcmp(field_type->name, "timestamp") == 0)
                                       fprintf(cf, "  if (timestamp == 0) timestamp = toku_get_timestamp();\n");
                                   fprintf(cf, "  wbuf_nocrc_%s(&wbuf, %s);\n", field_type->type, field_type->name));
-                        fprintf(cf, "  wbuf_nocrc_int(&wbuf, x1764_memory(wbuf.buf, wbuf.ndone));\n");
+                        fprintf(cf, "  wbuf_nocrc_int(&wbuf, toku_x1764_memory(wbuf.buf, wbuf.ndone));\n");
                         fprintf(cf, "  wbuf_nocrc_int(&wbuf, buflen);\n");
                         fprintf(cf, "  assert(wbuf.ndone==buflen);\n");
                         fprintf(cf, "  logger->inbuf.n_in_buf += buflen;\n");
@@ -558,7 +558,7 @@ generate_log_reader (void) {
                         fprintf(cf, "  uint32_t checksum_in_file, len_in_file;\n");
                         fprintf(cf, "  r=toku_fread_uint32_t_nocrclen(infile, &checksum_in_file); actual_len+=4;   if (r!=0) return r;\n");
                         fprintf(cf, "  r=toku_fread_uint32_t_nocrclen(infile, &len_in_file);    actual_len+=4;   if (r!=0) return r;\n");
-                        fprintf(cf, "  if (checksum_in_file!=x1764_finish(checksum) || len_in_file!=actual_len || len1 != len_in_file) return DB_BADFORMAT;\n");
+                        fprintf(cf, "  if (checksum_in_file!=toku_x1764_finish(checksum) || len_in_file!=actual_len || len1 != len_in_file) return DB_BADFORMAT;\n");
                         fprintf(cf, "  return 0;\n");
                         fprintf(cf, "}\n\n");
                     });
@@ -568,12 +568,12 @@ generate_log_reader (void) {
     fprintf(cf, "  uint32_t len1; int r;\n");
     fprintf(cf, "  uint32_t ignorelen=0;\n");
     fprintf(cf, "  struct x1764 checksum;\n");
-    fprintf(cf, "  x1764_init(&checksum);\n");
+    fprintf(cf, "  toku_x1764_init(&checksum);\n");
     fprintf(cf, "  r = toku_fread_uint32_t(infile, &len1, &checksum, &ignorelen); if (r!=0) return r;\n");
     fprintf(cf, "  int cmd=fgetc(infile);\n");
     fprintf(cf, "  if (cmd==EOF) return EOF;\n");
     fprintf(cf, "  char cmdchar = (char)cmd;\n");
-    fprintf(cf, "  x1764_add(&checksum, &cmdchar, 1);\n");
+    fprintf(cf, "  toku_x1764_add(&checksum, &cmdchar, 1);\n");
     fprintf(cf, "  le->cmd=(enum lt_cmd)cmd;\n");
     fprintf(cf, "  switch ((enum lt_cmd)cmd) {\n");
     DO_LOGTYPES(lt, {
@@ -639,14 +639,14 @@ generate_logprint (void) {
     fprintf(pf, "    uint32_t len1, crc_in_file;\n");
     fprintf(pf, "    uint32_t ignorelen=0;\n");
     fprintf(pf, "    struct x1764 checksum;\n");
-    fprintf(pf, "    x1764_init(&checksum);\n");
+    fprintf(pf, "    toku_x1764_init(&checksum);\n");
     fprintf(pf, "    r=toku_fread_uint32_t(f, &len1, &checksum, &ignorelen);\n");
     fprintf(pf, "    if (r==EOF) return EOF;\n");
     fprintf(pf, "    cmd=fgetc(f);\n");
     fprintf(pf, "    if (cmd==EOF) return DB_BADFORMAT;\n");
     fprintf(pf, "    uint32_t len_in_file, len=1+4; // cmd + len1\n");
     fprintf(pf, "    char charcmd = (char)cmd;\n");
-    fprintf(pf, "    x1764_add(&checksum, &charcmd, 1);\n");
+    fprintf(pf, "    toku_x1764_add(&checksum, &charcmd, 1);\n");
     fprintf(pf, "    switch ((enum lt_cmd)cmd) {\n");
     DO_LOGTYPES(lt, { if (strlen(lt->name)>maxnamelen) maxnamelen=strlen(lt->name); });
     DO_LOGTYPES(lt, {
@@ -664,7 +664,7 @@ generate_logprint (void) {
                             fprintf(pf, "); if (r!=0) return r;\n");
                         });
                 fprintf(pf, "        {\n");
-                fprintf(pf, "          uint32_t actual_murmur = x1764_finish(&checksum);\n");
+                fprintf(pf, "          uint32_t actual_murmur = toku_x1764_finish(&checksum);\n");
                 fprintf(pf, "          r = toku_fread_uint32_t_nocrclen (f, &crc_in_file); len+=4; if (r!=0) return r;\n");
                 fprintf(pf, "          fprintf(outf, \" crc=%%08x\", crc_in_file);\n");
                 fprintf(pf, "          if (crc_in_file!=actual_murmur) fprintf(outf, \" checksum=%%08x\", actual_murmur);\n");
@@ -806,7 +806,7 @@ generate_rollbacks (void) {
     DO_ROLLBACKS(lt, {
                 fprintf(cf, "  case RT_%s:\n", lt->name);
                 fprintf(cf, "    mem_needed = sizeof(item->u.%s) + __builtin_offsetof(struct roll_entry, u.%s);\n", lt->name, lt->name);
-                fprintf(cf, "    CAST_FROM_VOIDP(item, malloc_in_memarena(ma, mem_needed));\n");
+                fprintf(cf, "    CAST_FROM_VOIDP(item, toku_memarena_malloc(ma, mem_needed));\n");
                 fprintf(cf, "    item->cmd = cmd;\n");
                 DO_FIELDS(field_type, lt, fprintf(cf, "    rbuf_ma_%s(&rc, ma, &item->u.%s.%s);\n", field_type->type, lt->name, field_type->name));
                 fprintf(cf, "    *itemp = item;\n");
@@ -858,7 +858,7 @@ int main (int argc, const char *const argv[]) {
     fprintf2(cf, pf, "#include <ft/fttypes.h>\n");
     fprintf2(cf, pf, "#include <ft/log-internal.h>\n");
     fprintf(hf, "#include <ft/ft-internal.h>\n");
-    fprintf(hf, "#include <ft/memarena.h>\n");
+    fprintf(hf, "#include <util/memarena.h>\n");
     generate_enum();
     generate_log_struct();
     generate_dispatch();

storage/tokudb/ft-index/ft/logger.cc

@@ -98,7 +98,7 @@ PATENT RIGHTS GRANT:
 #include "log-internal.h"
 #include "txn_manager.h"
 #include "rollback_log_node_cache.h"
-#include "huge_page_detection.h"
+
 #include <util/status.h>
 
 static const int log_format_version=TOKU_LOG_VERSION;
@@ -164,11 +164,6 @@ static bool is_a_logfile (const char *name, long long *number_result) {
 
 // TODO: can't fail
 int toku_logger_create (TOKULOGGER *resultp) {
-    if (complain_and_return_true_if_huge_pages_are_enabled()) {
-        *resultp = NULL;
-        errno = TOKUDB_HUGE_PAGES_ENABLED;
-        return TOKUDB_HUGE_PAGES_ENABLED;
-    }
     TOKULOGGER CALLOC(result);
     if (result==0) return get_error_errno();
     result->is_open=false;
@@ -187,7 +182,7 @@ int toku_logger_create (TOKULOGGER *resultp) {
     result->last_completed_checkpoint_lsn = ZERO_LSN;
     // next_log_file_number is uninitialized
     // n_in_file is uninitialized
-    result->write_block_size = FT_DEFAULT_NODE_SIZE; // default logging size is the same as the default brt block size
+    result->write_block_size = FT_DEFAULT_NODE_SIZE; // default logging size is the same as the default ft block size
     toku_logfilemgr_create(&result->logfilemgr);
     *resultp=result;
     ml_init(&result->input_lock);
@@ -234,7 +229,7 @@ toku_logger_open_with_last_xid(const char *directory, TOKULOGGER logger, TXNID l
     if (logger->is_open) return EINVAL;
 
     int r;
-    TXNID last_xid_if_clean_shutdown;
+    TXNID last_xid_if_clean_shutdown = TXNID_NONE;
     r = toku_logfilemgr_init(logger->logfilemgr, directory, &last_xid_if_clean_shutdown);
     if ( r!=0 )
         return r;
@@ -285,7 +280,7 @@ toku_logger_open_rollback(TOKULOGGER logger, CACHETABLE cachetable, bool create)
     assert(logger->is_open);
     assert(!logger->rollback_cachefile);
 
-    FT_HANDLE t = NULL;   // Note, there is no DB associated with this BRT.
+    FT_HANDLE t = NULL;   // Note, there is no DB associated with this FT.
     toku_ft_handle_create(&t);
     int r = toku_ft_handle_open(t, toku_product_name_strings.rollback_cachefile, create, create, cachetable, NULL_TXN);
     if (r == 0) {
@@ -313,7 +308,7 @@ void toku_logger_close_rollback(TOKULOGGER logger) {
     CACHEFILE cf = logger->rollback_cachefile;  // stored in logger at rollback cachefile open
     if (cf) {
         FT_HANDLE ft_to_close;
-        {   //Find "brt"
+        {   //Find "ft_to_close"
             logger->rollback_cache.destroy();
             FT CAST_FROM_VOIDP(ft, toku_cachefile_get_userdata(cf));
             //Verify it is safe to close it.
@@ -938,7 +933,7 @@ int toku_fread_uint8_t (FILE *f, uint8_t *v, struct x1764 *mm, uint32_t *len) {
     int vi=fgetc(f);
     if (vi==EOF) return -1;
     uint8_t vc=(uint8_t)vi;
-    x1764_add(mm, &vc, 1);
+    toku_x1764_add(mm, &vc, 1);
     (*len)++;
     *v = vc;
     return 0;
@@ -1286,7 +1281,7 @@ static int peek_at_log (TOKULOGGER logger, char* filename, LSN *first_lsn) {
         if (logger->write_log_files) printf("couldn't open: %s\n", strerror(er));
         return er;
     }
-    enum { SKIP = 12+1+4 }; // read the 12 byte header, the first cmd, and the first len
+    enum { SKIP = 12+1+4 }; // read the 12 byte header, the first message, and the first len
     unsigned char header[SKIP+8];
     int r = read(fd, header, SKIP+8);
     if (r!=SKIP+8) return 0; // cannot determine that it's archivable, so we'll assume no.  If a later-log is archivable is then this one will be too.

storage/tokudb/ft-index/ft/omt.cc

@@ -1,230 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-
-#include <toku_portability.h>
-#include <memory.h>
-#include <string.h>
-#include <db.h>
-
-#include "omt.h"
-
-int
-toku_omt_create_steal_sorted_array(OMT *omtp, OMTVALUE **valuesp, uint32_t numvalues, uint32_t capacity) {
-    OMT XMALLOC(omt);
-    omt->create_steal_sorted_array(valuesp, numvalues, capacity);
-    *omtp = omt;
-    return 0;
-}
-
-//TODO: Put all omt API functions here.
-int toku_omt_create (OMT *omtp) {
-    OMT XMALLOC(omt);
-    omt->create();
-    *omtp = omt;
-    return 0;
-}
-
-void toku_omt_destroy(OMT *omtp) {
-    OMT omt=*omtp;
-    omt->destroy();
-    toku_free(omt);
-    *omtp=NULL;
-}
-
-uint32_t toku_omt_size(OMT V) {
-    return V->size();
-}
-
-int toku_omt_create_from_sorted_array(OMT *omtp, OMTVALUE *values, uint32_t numvalues) {
-    OMT XMALLOC(omt);
-    omt->create_from_sorted_array(values, numvalues);
-    *omtp=omt;
-    return 0;
-}
-
-int toku_omt_insert_at(OMT omt, OMTVALUE value, uint32_t index) {
-    return omt->insert_at(value, index);
-}
-
-int toku_omt_set_at (OMT omt, OMTVALUE value, uint32_t index) {
-    return omt->set_at(value, index);
-}
-
-int toku_omt_delete_at(OMT omt, uint32_t index) {
-    return omt->delete_at(index);
-}
-
-int toku_omt_fetch(OMT omt, uint32_t i, OMTVALUE *v) {
-    return omt->fetch(i, v);
-}
-
-struct functor {
-    int (*f)(OMTVALUE, uint32_t, void *);
-    void *v;
-};
-static_assert(std::is_pod<functor>::value, "not POD");
-
-int call_functor(const OMTVALUE &v, uint32_t idx, functor *const ftor);
-int call_functor(const OMTVALUE &v, uint32_t idx, functor *const ftor) {
-    return ftor->f(const_cast<OMTVALUE>(v), idx, ftor->v);
-}
-
-int toku_omt_iterate(OMT omt, int (*f)(OMTVALUE, uint32_t, void*), void*v) {
-    struct functor ftor = { .f = f, .v = v };
-    return omt->iterate<functor, call_functor>(&ftor);
-}
-
-int toku_omt_iterate_on_range(OMT omt, uint32_t left, uint32_t right, int (*f)(OMTVALUE, uint32_t, void*), void*v) {
-    struct functor ftor = { .f = f, .v = v };
-    return omt->iterate_on_range<functor, call_functor>(left, right, &ftor);
-}
-
-struct heftor {
-    int (*h)(OMTVALUE, void *v);
-    void *v;
-};
-static_assert(std::is_pod<heftor>::value, "not POD");
-
-int call_heftor(const OMTVALUE &v, const heftor &htor);
-int call_heftor(const OMTVALUE &v, const heftor &htor) {
-    return htor.h(const_cast<OMTVALUE>(v), htor.v);
-}
-
-int toku_omt_insert(OMT omt, OMTVALUE value, int(*h)(OMTVALUE, void*v), void *v, uint32_t *index) {
-    struct heftor htor = { .h = h, .v = v };
-    return omt->insert<heftor, call_heftor>(value, htor, index);
-}
-
-int toku_omt_find_zero(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, OMTVALUE *value, uint32_t *index) {
-    struct heftor htor = { .h = h, .v = extra };
-    return V->find_zero<heftor, call_heftor>(htor, value, index);
-}
-
-int toku_omt_find(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, int direction, OMTVALUE *value, uint32_t *index) {
-    struct heftor htor = { .h = h, .v = extra };
-    return V->find<heftor, call_heftor>(htor, direction, value, index);
-}
-
-int toku_omt_split_at(OMT omt, OMT *newomtp, uint32_t index) {
-    OMT XMALLOC(newomt);
-    int r = omt->split_at(newomt, index);
-    if (r != 0) {
-        toku_free(newomt);
-    } else {
-        *newomtp = newomt;
-    }
-    return r;
-}
-
-int toku_omt_merge(OMT leftomt, OMT rightomt, OMT *newomtp) {
-    OMT XMALLOC(newomt);
-    newomt->merge(leftomt, rightomt);
-    toku_free(leftomt);
-    toku_free(rightomt);
-    *newomtp = newomt;
-    return 0;
-}
-
-int toku_omt_clone_noptr(OMT *dest, OMT src) {
-    OMT XMALLOC(omt);
-    omt->clone(*src);
-    *dest = omt;
-    return 0;
-}
-
-void toku_omt_clear(OMT omt) {
-    omt->clear();
-}
-
-size_t toku_omt_memory_size (OMT omt) {
-    return omt->memory_size();
-}
-

storage/tokudb/ft-index/ft/omt.h

@@ -1,416 +0,0 @@
-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#if !defined(TOKU_OMT_H)
-#define TOKU_OMT_H
-
-#ident "$Id$"
-/*
-COPYING CONDITIONS NOTICE:
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation, and provided that the
-  following conditions are met:
-
-      * Redistributions of source code must retain this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below).
-
-      * Redistributions in binary form must reproduce this COPYING
-        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
-        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
-        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
-        GRANT (below) in the documentation and/or other materials
-        provided with the distribution.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-  02110-1301, USA.
-
-COPYRIGHT NOTICE:
-
-  TokuDB, Tokutek Fractal Tree Indexing Library.
-  Copyright (C) 2007-2013 Tokutek, Inc.
-
-DISCLAIMER:
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-UNIVERSITY PATENT NOTICE:
-
-  The technology is licensed by the Massachusetts Institute of
-  Technology, Rutgers State University of New Jersey, and the Research
-  Foundation of State University of New York at Stony Brook under
-  United States of America Serial No. 11/760379 and to the patents
-  and/or patent applications resulting from it.
-
-PATENT MARKING NOTICE:
-
-  This software is covered by US Patent No. 8,185,551.
-  This software is covered by US Patent No. 8,489,638.
-
-PATENT RIGHTS GRANT:
-
-  "THIS IMPLEMENTATION" means the copyrightable works distributed by
-  Tokutek as part of the Fractal Tree project.
-
-  "PATENT CLAIMS" means the claims of patents that are owned or
-  licensable by Tokutek, both currently or in the future; and that in
-  the absence of this license would be infringed by THIS
-  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
-
-  "PATENT CHALLENGE" shall mean a challenge to the validity,
-  patentability, enforceability and/or non-infringement of any of the
-  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
-
-  Tokutek hereby grants to you, for the term and geographical scope of
-  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
-  irrevocable (except as stated in this section) patent license to
-  make, have made, use, offer to sell, sell, import, transfer, and
-  otherwise run, modify, and propagate the contents of THIS
-  IMPLEMENTATION, where such license applies only to the PATENT
-  CLAIMS.  This grant does not include claims that would be infringed
-  only as a consequence of further modifications of THIS
-  IMPLEMENTATION.  If you or your agent or licensee institute or order
-  or agree to the institution of patent litigation against any entity
-  (including a cross-claim or counterclaim in a lawsuit) alleging that
-  THIS IMPLEMENTATION constitutes direct or contributory patent
-  infringement, or inducement of patent infringement, then any rights
-  granted to you under this License shall terminate as of the date
-  such litigation is filed.  If you or your agent or exclusive
-  licensee institute or order or agree to the institution of a PATENT
-  CHALLENGE, then Tokutek may terminate any rights granted to you
-  under this License.
-*/
-
-#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-
-
-// Order Maintenance Tree (OMT)
-//
-// Maintains a collection of totally ordered values, where each value has an integer weight.
-// The OMT is a mutable datatype.
-//
-// The Abstraction:
-//
-// An OMT is a vector of values, $V$, where $|V|$ is the length of the vector.
-// The vector is numbered from $0$ to $|V|-1$.
-// Each value has a weight.  The weight of the $i$th element is denoted $w(V_i)$.
-//
-// We can create a new OMT, which is the empty vector.
-//
-// We can insert a new element $x$ into slot $i$, changing $V$ into $V'$ where
-//  $|V'|=1+|V|$       and
-//
-//   V'_j = V_j       if $j<i$
-//          x         if $j=i$
-//          V_{j-1}   if $j>i$.
-//
-// We can specify $i$ using a kind of function instead of as an integer.
-// Let $b$ be a function mapping from values to nonzero integers, such that
-// the signum of $b$ is monotically increasing.
-// We can specify $i$ as the minimum integer such that $b(V_i)>0$.
-//
-// We look up a value using its index, or using a Heaviside function.
-// For lookups, we allow $b$ to be zero for some values, and again the signum of $b$ must be monotonically increasing.
-// When lookup up values, we can look up
-//  $V_i$ where $i$ is the minimum integer such that $b(V_i)=0$.   (With a special return code if no such value exists.)
-//      (Rationale:  Ordinarily we want $i$ to be unique.  But for various reasons we want to allow multiple zeros, and we want the smallest $i$ in that case.)
-//  $V_i$ where $i$ is the minimum integer such that $b(V_i)>0$.   (Or an indication that no such value exists.)
-//  $V_i$ where $i$ is the maximum integer such that $b(V_i)<0$.   (Or an indication that no such value exists.)
-//
-// When looking up a value using a Heaviside function, we get the value and its index.
-//
-// We can also split an OMT into two OMTs, splitting the weight of the values evenly.
-// Find a value $j$ such that the values to the left of $j$ have about the same total weight as the values to the right of $j$.
-// The resulting two OMTs contain the values to the left of $j$ and the values to the right of $j$ respectively.
-// All of the values from the original OMT go into one of the new OMTs.
-// If the weights of the values don't split exactly evenly, then the implementation has the freedom to choose whether
-//  the new left OMT or the new right OMT is larger.
-//
-// Performance:
-//  Insertion and deletion should run with $O(\log |V|)$ time and $O(\log |V|)$ calls to the Heaviside function.
-//  The memory required is O(|V|).
-//
-// The programming API:
-
-//typedef struct value *OMTVALUE; // A slight improvement over using void*.
-#include <util/omt.h>
-typedef void *OMTVALUE;
-typedef toku::omt<OMTVALUE> *OMT;
-
-
-int toku_omt_create (OMT *omtp);
-// Effect: Create an empty OMT.  Stores it in *omtp.
-// Requires: omtp != NULL
-// Returns:
-//   0        success
-//   ENOMEM   out of memory (and doesn't modify *omtp)
-// Performance: constant time.
-
-int toku_omt_create_from_sorted_array(OMT *omtp, OMTVALUE *values, uint32_t numvalues);
-// Effect: Create a OMT containing values.  The number of values is in numvalues.
-//  Stores the new OMT in *omtp.
-// Requires: omtp != NULL
-// Requires: values != NULL
-// Requires: values is sorted
-// Returns:
-//   0        success
-//   ENOMEM   out of memory (and doesn't modify *omtp)
-// Performance:  time=O(numvalues)
-// Rational:     Normally to insert N values takes O(N lg N) amortized time.
-//               If the N values are known in advance, are sorted, and
-//               the structure is empty, we can batch insert them much faster.
-
-int toku_omt_create_steal_sorted_array(OMT *omtp, OMTVALUE **valuesp, uint32_t numvalues, uint32_t steal_capacity);
-// Effect: Create an OMT containing values.  The number of values is in numvalues.
-//         On success the OMT takes ownership of *valuesp array, and sets valuesp=NULL.
-// Requires: omtp != NULL
-// Requires: valuesp != NULL
-// Requires: *valuesp is sorted
-// Requires: *valuesp was allocated with toku_malloc
-// Requires: Capacity of the *valuesp array is <= steal_capacity
-// Requires: On success, *valuesp may not be accessed again by the caller.
-// Returns:
-//   0        success
-//   ENOMEM   out of memory (and doesn't modify *omtp)
-//   EINVAL   *valuesp == NULL or numvalues > capacity
-// Performance:  time=O(1)
-// Rational:     toku_omt_create_from_sorted_array takes O(numvalues) time.
-//               By taking ownership of the array, we save a malloc and memcpy,
-//               and possibly a free (if the caller is done with the array).
-
-void toku_omt_destroy(OMT *omtp);
-// Effect:  Destroy an OMT, freeing all its memory.
-//   Does not free the OMTVALUEs stored in the OMT.
-//   Those values may be freed before or after calling toku_omt_destroy.
-//   Also sets *omtp=NULL.
-// Requires: omtp != NULL
-// Requires: *omtp != NULL
-// Rationale:  The usage is to do something like
-//   toku_omt_destroy(&s->omt);
-// and now s->omt will have a NULL pointer instead of a dangling freed pointer.
-// Rationale: Returns no values since free() cannot fail.
-// Rationale: Does not free the OMTVALUEs to reduce complexity.
-// Performance:  time=O(toku_omt_size(*omtp))
-
-uint32_t toku_omt_size(OMT V);
-// Effect: return |V|.
-// Requires: V != NULL
-// Performance:  time=O(1)
-
-int toku_omt_iterate_on_range(OMT omt, uint32_t left, uint32_t right, int (*f)(OMTVALUE, uint32_t, void*), void*v);
-// Effect:  Iterate over the values of the omt, from left to right, calling f on each value.
-//  The second argument passed to f is the index of the value.
-//  The third argument passed to f is v.
-//  The indices run from 0 (inclusive) to toku_omt_size(omt) (exclusive).
-//  We will iterate only over [left,right)
-//
-// Requires: omt != NULL
-// left <= right
-// Requires: f != NULL
-// Returns:
-//  If f ever returns nonzero, then the iteration stops, and the value returned by f is returned by toku_omt_iterate.
-//  If f always returns zero, then toku_omt_iterate returns 0.
-// Requires:  Don't modify omt while running.  (E.g., f may not insert or delete values form omt.)
-// Performance: time=O(i+\log N) where i is the number of times f is called, and N is the number of elements in omt.
-// Rational: Although the functional iterator requires defining another function (as opposed to C++ style iterator), it is much easier to read.
-
-int toku_omt_iterate(OMT omt, int (*f)(OMTVALUE, uint32_t, void*), void*v);
-// Effect:  Iterate over the values of the omt, from left to right, calling f on each value.
-//  The second argument passed to f is the index of the value.
-//  The third argument passed to f is v.
-//  The indices run from 0 (inclusive) to toku_omt_size(omt) (exclusive).
-// Requires: omt != NULL
-// Requires: f != NULL
-// Returns:
-//  If f ever returns nonzero, then the iteration stops, and the value returned by f is returned by toku_omt_iterate.
-//  If f always returns zero, then toku_omt_iterate returns 0.
-// Requires:  Don't modify omt while running.  (E.g., f may not insert or delete values form omt.)
-// Performance: time=O(i+\log N) where i is the number of times f is called, and N is the number of elements in omt.
-// Rational: Although the functional iterator requires defining another function (as opposed to C++ style iterator), it is much easier to read.
-
-int toku_omt_insert_at(OMT omt, OMTVALUE value, uint32_t idx);
-// Effect: Increases indexes of all items at slot >= index by 1.
-//         Insert value into the position at index.
-//
-// Returns:
-//   0         success
-//   EINVAL    if index>toku_omt_size(omt)
-//   ENOMEM
-// On error, omt is unchanged.
-// Performance: time=O(\log N) amortized time.
-// Rationale: Some future implementation may be O(\log N) worst-case time, but O(\log N) amortized is good enough for now.
-
-int toku_omt_set_at (OMT omt, OMTVALUE value, uint32_t idx);
-// Effect:  Replaces the item at index with value.
-// Returns:
-//   0       success
-//   EINVAL  if index>=toku_omt_size(omt)
-// On error, omt i sunchanged.
-// Performance: time=O(\log N)
-// Rationale: The BRT needs to be able to replace a value with another copy of the same value (allocated in a different location)
-
-int toku_omt_insert(OMT omt, OMTVALUE value, int(*h)(OMTVALUE, void*v), void *v, uint32_t *idx);
-// Effect:  Insert value into the OMT.
-//   If there is some i such that $h(V_i, v)=0$ then returns DB_KEYEXIST.
-//   Otherwise, let i be the minimum value such that $h(V_i, v)>0$.
-//      If no such i exists, then let i be |V|
-//   Then this has the same effect as
-//    omt_insert_at(tree, value, i);
-//   If index!=NULL then i is stored in *index
-// Requires:  The signum of h must be monotonically increasing.
-// Returns:
-//    0            success
-//    DB_KEYEXIST  the key is present (h was equal to zero for some value)
-//    ENOMEM
-// On nonzero return, omt is unchanged.
-// On nonzero non-DB_KEYEXIST return, *index is unchanged.
-// Performance: time=O(\log N) amortized.
-// Rationale: Some future implementation may be O(\log N) worst-case time, but O(\log N) amortized is good enough for now.
-
-int toku_omt_delete_at(OMT omt, uint32_t idx);
-// Effect: Delete the item in slot index.
-//         Decreases indexes of all items at slot >= index by 1.
-// Returns
-//     0            success
-//     EINVAL       if index>=toku_omt_size(omt)
-// On error, omt is unchanged.
-// Rationale: To delete an item, first find its index using toku_omt_find, then delete it.
-// Performance: time=O(\log N) amortized.
-
-int toku_omt_fetch (OMT V, uint32_t i, OMTVALUE *v);
-// Effect: Set *v=V_i
-//   If c!=NULL then set c's abstract offset to i.
-// Requires: v   != NULL
-// Returns
-//    0             success
-//    EINVAL        if index>=toku_omt_size(omt)
-// On nonzero return, *v is unchanged, and c (if nonnull) is either
-//   invalidated or unchanged.
-// Performance: time=O(\log N)
-// Implementation Notes: It is possible that c was previously valid and was
-//   associated with a different OMT.   If c is changed by this
-//   function, the function must remove c's association with the old
-//   OMT, and associate it with the new OMT.
-
-int toku_omt_find_zero(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, OMTVALUE *value, uint32_t *idx);
-// Effect:  Find the smallest i such that h(V_i, extra)>=0
-//  If there is such an i and h(V_i,extra)==0 then set *index=i and return 0.
-//  If there is such an i and h(V_i,extra)>0  then set *index=i and return DB_NOTFOUND.
-//  If there is no such i then set *index=toku_omt_size(V) and return DB_NOTFOUND.
-// Requires: index!=NULL
-
-int toku_omt_find(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, int direction, OMTVALUE *value, uint32_t *idx);
-//   Effect:
-//    If direction >0 then find the smallest i such that h(V_i,extra)>0.
-//    If direction <0 then find the largest  i such that h(V_i,extra)<0.
-//    (Direction may not be equal to zero.)
-//    If value!=NULL then store V_i in *value
-//    If index!=NULL then store i in *index.
-//   Requires: The signum of h is monotically increasing.
-//   Returns
-//      0             success
-//      DB_NOTFOUND   no such value is found.
-//   On nonzero return, *value and *index are unchanged, and c (if nonnull) is either invalidated
-//      or unchanged.
-//   Performance: time=O(\log N)
-//   Rationale:
-//     Here's how to use the find function to find various things
-//       Cases for find:
-//        find first value:         ( h(v)=+1, direction=+1 )
-//        find last value           ( h(v)=-1, direction=-1 )
-//        find first X              ( h(v)=(v< x) ? -1 : 1    direction=+1 )
-//        find last X               ( h(v)=(v<=x) ? -1 : 1    direction=-1 )
-//        find X or successor to X  ( same as find first X. )
-//
-//   Rationale: To help understand heaviside functions and behavor of find:
-//    There are 7 kinds of heaviside functions.
-//    The signus of the h must be monotonically increasing.
-//    Given a function of the following form, A is the element
-//    returned for direction>0, B is the element returned
-//    for direction<0, C is the element returned for
-//    direction==0 (see find_zero) (with a return of 0), and D is the element
-//    returned for direction==0 (see find_zero) with a return of DB_NOTFOUND.
-//    If any of A, B, or C are not found, then asking for the
-//    associated direction will return DB_NOTFOUND.
-//    See find_zero for more information.
-//
-//    Let the following represent the signus of the heaviside function.
-//
-//    -...-
-//        A
-//         D
-//
-//    +...+
-//    B
-//    D
-//
-//    0...0
-//    C
-//
-//    -...-0...0
-//        AC
-//
-//    0...0+...+
-//    C    B
-//
-//    -...-+...+
-//        AB
-//         D
-//
-//    -...-0...0+...+
-//        AC    B
-
-int toku_omt_split_at(OMT omt, OMT *newomt, uint32_t idx);
-// Effect: Create a new OMT, storing it in *newomt.
-//  The values to the right of index (starting at index) are moved to *newomt.
-// Requires: omt != NULL
-// Requires: newomt != NULL
-// Returns
-//    0             success,
-//    EINVAL        if index > toku_omt_size(omt)
-//    ENOMEM
-// On nonzero return, omt and *newomt are unmodified.
-// Performance: time=O(n)
-// Rationale:  We don't need a split-evenly operation.  We need to split items so that their total sizes
-//  are even, and other similar splitting criteria.  It's easy to split evenly by calling toku_omt_size(), and dividing by two.
-
-int toku_omt_merge(OMT leftomt, OMT rightomt, OMT *newomt);
-// Effect: Appends leftomt and rightomt to produce a new omt.
-//  Sets *newomt to the new omt.
-//  On success, leftomt and rightomt destroyed,.
-// Returns 0 on success
-//   ENOMEM on out of memory.
-// On error, nothing is modified.
-// Performance: time=O(n) is acceptable, but one can imagine implementations that are O(\log n) worst-case.
-
-int toku_omt_clone_noptr(OMT *dest, OMT src);
-// Effect: Creates a copy of an omt.
-//  Sets *dest to the clone
-//  Each element is assumed to be stored directly in the omt, that is, the OMTVALUEs are not pointers, they are data.  Thus no extra memory allocation is required.
-// Returns 0 on success
-//  ENOMEM on out of memory.
-// On error, nothing is modified.
-// Performance: time between O(n) and O(n log n), depending how long it
-//  takes to traverse src.
-
-void toku_omt_clear(OMT omt);
-// Effect: Set the tree to be empty.
-//  Note: Will not reallocate or resize any memory, since returning void precludes calling malloc.
-// Performance: time=O(1)
-
-size_t toku_omt_memory_size (OMT omt);
-// Effect: Return the size (in bytes) of the omt, as it resides in main memory.  Don't include any of the OMTVALUES.
-
-
-
-#endif  /* #ifndef TOKU_OMT_H */
-

storage/tokudb/ft-index/ft/quicklz.cc

@@ -311,6 +311,7 @@ static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui3
 #endif
 }
 
+__attribute__((unused))
 static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s)
 {
 #if QLZ_COMPRESSION_LEVEL == 1

storage/tokudb/ft-index/ft/rbuf.h

@@ -93,12 +93,12 @@ PATENT RIGHTS GRANT:
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
 #include <toku_portability.h>
-#include "memarena.h"
 #include "toku_assert.h"
 #include "fttypes.h"
 #include "memory.h"
 #include <toku_htonl.h>
 
+#include <util/memarena.h>
 
 struct rbuf {
     unsigned char *buf;
@@ -250,7 +250,7 @@ static inline void rbuf_FILENUMS(struct rbuf *r, FILENUMS *filenums) {
 // 2954
 static inline void rbuf_ma_FILENUMS (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), FILENUMS *filenums) {
     rbuf_ma_uint32_t(r, ma, &(filenums->num));
-    filenums->filenums = (FILENUM *) malloc_in_memarena(ma, filenums->num * sizeof(FILENUM) );
+    filenums->filenums = (FILENUM *) toku_memarena_malloc(ma, filenums->num * sizeof(FILENUM) );
     assert(filenums->filenums != NULL);
     for (uint32_t i=0; i < filenums->num; i++) {
         rbuf_ma_FILENUM(r, ma, &(filenums->filenums[i]));
@@ -271,7 +271,7 @@ static inline void rbuf_ma_BYTESTRING  (struct rbuf *r, MEMARENA ma, BYTESTRING
     bs->len  = rbuf_int(r);
     uint32_t newndone = r->ndone + bs->len;
     assert(newndone <= r->size);
-    bs->data = (char *) memarena_memdup(ma, &r->buf[r->ndone], (size_t)bs->len);
+    bs->data = (char *) toku_memarena_memdup(ma, &r->buf[r->ndone], (size_t)bs->len);
     assert(bs->data);
     r->ndone = newndone;
 }

storage/tokudb/ft-index/ft/recover.cc

@@ -98,6 +98,8 @@ PATENT RIGHTS GRANT:
 #include "checkpoint.h"
 #include "txn_manager.h"
 
+#include <util/omt.h>
+
 int tokudb_recovery_trace = 0;                    // turn on recovery tracing, default off.
 
 //#define DO_VERIFY_COUNTS
@@ -154,9 +156,9 @@ struct file_map_tuple {
     struct __toku_db fake_db;
 };
 
-static void file_map_tuple_init(struct file_map_tuple *tuple, FILENUM filenum, FT_HANDLE brt, char *iname) {
+static void file_map_tuple_init(struct file_map_tuple *tuple, FILENUM filenum, FT_HANDLE ft_handle, char *iname) {
     tuple->filenum = filenum;
-    tuple->ft_handle = brt;
+    tuple->ft_handle = ft_handle;
     tuple->iname = iname;
     // use a fake DB for comparisons, using the ft's cmp descriptor
     memset(&tuple->fake_db, 0, sizeof(tuple->fake_db));
@@ -171,9 +173,9 @@ static void file_map_tuple_destroy(struct file_map_tuple *tuple) {
     }
 }
 
-// Map filenum to brt
+// Map filenum to ft_handle
 struct file_map {
-    OMT filenums;
+    toku::omt<struct file_map_tuple *> *filenums;
 };
 
 // The recovery environment
@@ -199,31 +201,33 @@ typedef struct recover_env *RECOVER_ENV;
 
 
 static void file_map_init(struct file_map *fmap) {
-    int r = toku_omt_create(&fmap->filenums);
+    XMALLOC(fmap->filenums);
-    assert(r == 0);
+    fmap->filenums->create();
 }
 
 static void file_map_destroy(struct file_map *fmap) {
-    toku_omt_destroy(&fmap->filenums);
+    fmap->filenums->destroy();
+    toku_free(fmap->filenums);
+    fmap->filenums = nullptr;
 }
 
 static uint32_t file_map_get_num_dictionaries(struct file_map *fmap) {
-    return toku_omt_size(fmap->filenums);
+    return fmap->filenums->size();
 }
 
 static void file_map_close_dictionaries(struct file_map *fmap, LSN oplsn) {
     int r;
 
     while (1) {
-        uint32_t n = toku_omt_size(fmap->filenums);
+        uint32_t n = fmap->filenums->size();
-        if (n == 0)
+        if (n == 0) {
             break;
-        OMTVALUE v;
+        }
-        r = toku_omt_fetch(fmap->filenums, n-1, &v);
+        struct file_map_tuple *tuple;
+        r = fmap->filenums->fetch(n - 1, &tuple);
         assert(r == 0);
-        r = toku_omt_delete_at(fmap->filenums, n-1);
+        r = fmap->filenums->delete_at(n - 1);
         assert(r == 0);
-        struct file_map_tuple *CAST_FROM_VOIDP(tuple, v);
         assert(tuple->ft_handle);
         // Logging is on again, but we must pass the right LSN into close.
         if (tuple->ft_handle) { // it's a DB, not a rollback file
@@ -234,27 +238,29 @@ static void file_map_close_dictionaries(struct file_map *fmap, LSN oplsn) {
     }
 }
 
-static int file_map_h(OMTVALUE omtv, void *v) {
+static int file_map_h(struct file_map_tuple *const &a, const FILENUM &b) {
-    struct file_map_tuple *CAST_FROM_VOIDP(a, omtv);
+    if (a->filenum.fileid < b.fileid) {
-    FILENUM *CAST_FROM_VOIDP(b, v);
+        return -1;
-    if (a->filenum.fileid < b->fileid) return -1;
+    } else if (a->filenum.fileid > b.fileid) {
-    if (a->filenum.fileid > b->fileid) return +1;
+        return 1;
-    return 0;
+    } else {
+        return 0;
+    }
 }
 
-static int file_map_insert (struct file_map *fmap, FILENUM fnum, FT_HANDLE brt, char *iname) {
+static int file_map_insert (struct file_map *fmap, FILENUM fnum, FT_HANDLE ft_handle, char *iname) {
     struct file_map_tuple *XMALLOC(tuple);
-    file_map_tuple_init(tuple, fnum, brt, iname);
+    file_map_tuple_init(tuple, fnum, ft_handle, iname);
-    int r = toku_omt_insert(fmap->filenums, tuple, file_map_h, &fnum, NULL);
+    int r = fmap->filenums->insert<FILENUM, file_map_h>(tuple, fnum, nullptr);
     return r;
 }
 
 static void file_map_remove(struct file_map *fmap, FILENUM fnum) {
-    OMTVALUE v; uint32_t idx;
+    uint32_t idx;
-    int r = toku_omt_find_zero(fmap->filenums, file_map_h, &fnum, &v, &idx);
+    struct file_map_tuple *tuple;
+    int r = fmap->filenums->find_zero<FILENUM, file_map_h>(fnum, &tuple, &idx);
     if (r == 0) {
-        struct file_map_tuple *CAST_FROM_VOIDP(tuple, v);
+        r = fmap->filenums->delete_at(idx);
-        r = toku_omt_delete_at(fmap->filenums, idx);
         file_map_tuple_destroy(tuple);
         toku_free(tuple);
     }
@@ -262,14 +268,15 @@ static void file_map_remove(struct file_map *fmap, FILENUM fnum) {
 
 // Look up file info: given FILENUM, return file_map_tuple (or DB_NOTFOUND)
 static int file_map_find(struct file_map *fmap, FILENUM fnum, struct file_map_tuple **file_map_tuple) {
-    OMTVALUE v; uint32_t idx;
+    uint32_t idx;
-    int r = toku_omt_find_zero(fmap->filenums, file_map_h, &fnum, &v, &idx);
+    struct file_map_tuple *tuple;
+    int r = fmap->filenums->find_zero<FILENUM, file_map_h>(fnum, &tuple, &idx);
     if (r == 0) {
-        struct file_map_tuple *CAST_FROM_VOIDP(tuple, v);
         assert(tuple->filenum.fileid == fnum.fileid);
         *file_map_tuple = tuple;
+    } else {
+        assert(r == DB_NOTFOUND);
     }
-    else assert(r==DB_NOTFOUND);
     return r;
 }
 
@@ -319,7 +326,7 @@ static int recover_env_init (RECOVER_ENV renv,
 static void recover_env_cleanup (RECOVER_ENV renv) {
     int r;
 
-    assert(toku_omt_size(renv->fmap.filenums)==0);
+    invariant_zero(renv->fmap.filenums->size());
     file_map_destroy(&renv->fmap);
 
     if (renv->destroy_logger_at_end) {
@@ -350,48 +357,48 @@ static const char *recover_state(RECOVER_ENV renv) {
 static int internal_recover_fopen_or_fcreate (RECOVER_ENV renv, bool must_create, int UU(mode), BYTESTRING *bs_iname, FILENUM filenum, uint32_t treeflags,
                                               TOKUTXN txn, uint32_t nodesize, uint32_t basementnodesize, enum toku_compression_method compression_method, LSN max_acceptable_lsn) {
     int r = 0;
-    FT_HANDLE brt = NULL;
+    FT_HANDLE ft_handle = NULL;
     char *iname = fixup_fname(bs_iname);
 
-    toku_ft_handle_create(&brt);
+    toku_ft_handle_create(&ft_handle);
-    toku_ft_set_flags(brt, treeflags);
+    toku_ft_set_flags(ft_handle, treeflags);
 
     if (nodesize != 0) {
-        toku_ft_handle_set_nodesize(brt, nodesize);
+        toku_ft_handle_set_nodesize(ft_handle, nodesize);
     }
 
     if (basementnodesize != 0) {
-        toku_ft_handle_set_basementnodesize(brt, basementnodesize);
+        toku_ft_handle_set_basementnodesize(ft_handle, basementnodesize);
     }
 
     if (compression_method != TOKU_DEFAULT_COMPRESSION_METHOD) {
-        toku_ft_handle_set_compression_method(brt, compression_method);
+        toku_ft_handle_set_compression_method(ft_handle, compression_method);
     }
 
     // set the key compare functions
     if (!(treeflags & TOKU_DB_KEYCMP_BUILTIN) && renv->bt_compare) {
-        toku_ft_set_bt_compare(brt, renv->bt_compare);
+        toku_ft_set_bt_compare(ft_handle, renv->bt_compare);
     }
 
     if (renv->update_function) {
-        toku_ft_set_update(brt, renv->update_function);
+        toku_ft_set_update(ft_handle, renv->update_function);
     }
 
     // TODO mode (FUTURE FEATURE)
     //mode = mode;
 
-    r = toku_ft_handle_open_recovery(brt, iname, must_create, must_create, renv->ct, txn, filenum, max_acceptable_lsn);
+    r = toku_ft_handle_open_recovery(ft_handle, iname, must_create, must_create, renv->ct, txn, filenum, max_acceptable_lsn);
     if (r != 0) {
         //Note:  If ft_handle_open fails, then close_ft will NOT write a header to disk.
         //No need to provide lsn, so use the regular toku_ft_handle_close function
-        toku_ft_handle_close(brt);
+        toku_ft_handle_close(ft_handle);
         toku_free(iname);
         if (r == ENOENT) //Not an error to simply be missing.
             r = 0;
         return r;
     }
 
-    file_map_insert(&renv->fmap, filenum, brt, iname);
+    file_map_insert(&renv->fmap, filenum, ft_handle, iname);
     return 0;
 }
 

storage/tokudb/ft-index/ft/recover.h

@@ -96,10 +96,10 @@ PATENT RIGHTS GRANT:
 #include <errno.h>
 
 #include <db.h>
+#include <util/x1764.h>
+
 #include "fttypes.h"
 #include "memory.h"
-#include "x1764.h"
-
 
 typedef void (*prepared_txn_callback_t)(DB_ENV*, TOKUTXN);
 typedef void (*keep_cachetable_callback_t)(DB_ENV*, CACHETABLE);

storage/tokudb/ft-index/ft/roll.cc

@@ -258,7 +258,7 @@ static int do_insertion (enum ft_msg_type type, FILENUM filenum, BYTESTRING key,
     XIDS xids;
     xids = toku_txn_get_xids(txn);
     {
-        FT_MSG_S ftcmd = { type, ZERO_MSN, xids,
+        FT_MSG_S ftmsg = { type, ZERO_MSN, xids,
                            .u = { .id = { (key.len > 0)
                                           ? toku_fill_dbt(&key_dbt,  key.data,  key.len)
                                           : toku_init_dbt(&key_dbt),
@@ -275,7 +275,7 @@ static int do_insertion (enum ft_msg_type type, FILENUM filenum, BYTESTRING key,
                             // no messages above us, we can implicitly promote uxrs based on this xid
                             oldest_referenced_xid_estimate,
                             !txn->for_recovery);
-        toku_ft_root_put_cmd(h, &ftcmd, &gc_info);
+        toku_ft_root_put_msg(h, &ftmsg, &gc_info);
         if (reset_root_xid_that_created) {
             TXNID new_root_xid_that_created = xids_get_outermost_xid(xids);
             toku_reset_root_xid_that_created(h, new_root_xid_that_created);

storage/tokudb/ft-index/ft/rollback-apply.cc

@@ -258,9 +258,9 @@ int toku_rollback_commit(TOKUTXN txn, LSN lsn) {
             }
             child_log->newest_logentry = child_log->oldest_logentry = 0;
             // Put all the memarena data into the parent.
-            if (memarena_total_size_in_use(child_log->rollentry_arena) > 0) {
+            if (toku_memarena_total_size_in_use(child_log->rollentry_arena) > 0) {
                 // If there are no bytes to move, then just leave things alone, and let the memory be reclaimed on txn is closed.
-                memarena_move_buffers(parent_log->rollentry_arena, child_log->rollentry_arena);
+                toku_memarena_move_buffers(parent_log->rollentry_arena, child_log->rollentry_arena);
             }
             // each txn tries to give back at most one rollback log node
             // to the cache. All other rollback log nodes for this child
@@ -281,7 +281,7 @@ int toku_rollback_commit(TOKUTXN txn, LSN lsn) {
             toku_txn_unlock(txn->parent);
         }
 
-        // Note the open brts, the omts must be merged
+        // Note the open FTs, the omts must be merged
         r = txn->open_fts.iterate<struct tokutxn, note_ft_used_in_txns_parent>(txn);
         assert(r==0);
 

storage/tokudb/ft-index/ft/rollback-ct-callbacks.cc

@@ -89,15 +89,15 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "rollback-ct-callbacks.h"
-
 #include <toku_portability.h>
 #include <memory.h>
+
 #include "ft-internal.h"
 #include "fttypes.h"
-#include "memarena.h"
 #include "rollback.h"
+#include "rollback-ct-callbacks.h"
 
+#include <util/memarena.h>
 
 // Address used as a sentinel. Otherwise unused.
 static struct serialized_rollback_log_node cloned_rollback;

storage/tokudb/ft-index/ft/rollback.cc

@@ -121,7 +121,7 @@ toku_find_xid_by_xid (const TXNID &xid, const TXNID &xidfind) {
 }
 
 void *toku_malloc_in_rollback(ROLLBACK_LOG_NODE log, size_t size) {
-    return malloc_in_memarena(log->rollentry_arena, size);
+    return toku_memarena_malloc(log->rollentry_arena, size);
 }
 
 void *toku_memdup_in_rollback(ROLLBACK_LOG_NODE log, const void *v, size_t len) {
@@ -146,7 +146,7 @@ PAIR_ATTR
 rollback_memory_size(ROLLBACK_LOG_NODE log) {
     size_t size = sizeof(*log);
     if (log->rollentry_arena) {
-        size += memarena_total_memory_size(log->rollentry_arena);
+        size += toku_memarena_total_footprint(log->rollentry_arena);
     }
     return make_rollback_pair_attr(size);
 }
@@ -192,13 +192,13 @@ static void rollback_initialize_for_txn(
     log->previous = previous;
     log->oldest_logentry = NULL;
     log->newest_logentry = NULL;
-    log->rollentry_arena = memarena_create();
+    log->rollentry_arena = toku_memarena_create();
     log->rollentry_resident_bytecount = 0;
     log->dirty = true;
 }
 
 void make_rollback_log_empty(ROLLBACK_LOG_NODE log) {
-    memarena_close(&log->rollentry_arena);
+    toku_memarena_destroy(&log->rollentry_arena);
     rollback_empty_log_init(log);
 }
 
@@ -267,7 +267,7 @@ int find_filenum (const FT &h, const FT &hfind) {
     return 0;
 }
 
-//Notify a transaction that it has touched a brt.
+//Notify a transaction that it has touched an ft.
 void toku_txn_maybe_note_ft (TOKUTXN txn, FT ft) {
     toku_txn_lock(txn);
     FT ftv;

storage/tokudb/ft-index/ft/rollback_log_node_cache.cc

@@ -90,7 +90,7 @@ PATENT RIGHTS GRANT:
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
 #include <memory.h>
-#include <toku_include/toku_portability.h>
+#include <portability/toku_portability.h>
 
 #include "rollback_log_node_cache.h"
 

storage/tokudb/ft-index/ft/sub_block.cc

@@ -89,16 +89,15 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "sub_block.h"
-
 #include "compress.h"
+#include "sub_block.h"
 #include "quicklz.h"
-#include "x1764.h"
 
 #include <memory.h>
 #include <toku_assert.h>
 #include <toku_portability.h>
 #include <util/threadpool.h>
+#include <util/x1764.h>
 
 #include <stdio.h>
 #include <string.h>
@@ -268,7 +267,7 @@ compress_sub_block(struct sub_block *sub_block, enum toku_compression_method met
         method
         );
     // checksum it
-    sub_block->xsum = x1764_memory(sub_block->compressed_ptr, sub_block->compressed_size);
+    sub_block->xsum = toku_x1764_memory(sub_block->compressed_ptr, sub_block->compressed_size);
 }
 
 void *
@@ -365,7 +364,7 @@ decompress_sub_block(void *compress_ptr, uint32_t compress_size, void *uncompres
     int result = 0;
 
     // verify checksum
-    uint32_t xsum = x1764_memory(compress_ptr, compress_size);
+    uint32_t xsum = toku_x1764_memory(compress_ptr, compress_size);
     if (xsum != expected_xsum) {
         if (verbose_decompress_sub_block) fprintf(stderr, "%s:%d xsum %u expected %u\n", __FUNCTION__, __LINE__, xsum, expected_xsum);
         result = EINVAL;

storage/tokudb/ft-index/ft/tests/cachetable-checkpoint-pending.cc

@@ -108,13 +108,9 @@ static volatile int n_flush, n_write_me, n_keep_me, n_fetch;
 static void
 sleep_random (void)
 {
-#if TOKU_WINDOWS
-    usleep(random() % 1000); //Will turn out to be almost always 1ms.
-#else
     toku_timespec_t req = {.tv_sec  = 0,
 			   .tv_nsec = random()%1000000}; //Max just under 1ms
     nanosleep(&req, NULL);
-#endif
 }
 
 int expect_value = 42; // initially 42, later 43

storage/tokudb/ft-index/ft/tests/cachetable-checkpointer-class.cc

@@ -165,6 +165,7 @@ void checkpointer_test::test_begin_checkpoint() {
     }
     ctbl.list.destroy();
     m_cp.destroy();
+    cfl.destroy();
 }
 
 //------------------------------------------------------------------------------
@@ -257,6 +258,7 @@ void checkpointer_test::test_pending_bits() {
 
     ctbl.list.destroy();
     m_cp.destroy();
+    cfl.destroy();
 }
 
 //------------------------------------------------------------------------------
@@ -394,6 +396,7 @@ void checkpointer_test::test_end_checkpoint() {
     }
     m_cp.destroy();
     ctbl.list.destroy();
+    cfl.destroy();
 }
 
 

storage/tokudb/ft-index/ft/tests/cachetable-test.cc

@@ -241,8 +241,6 @@ PAIR_ATTR *sizep __attribute__((__unused__)), int * dirtyp, void*extraargs) {
     return 0;
 }
 
-#if !TOKU_WINDOWS
-
 static void test_multi_filehandles (void) {
     CACHETABLE t;
     CACHEFILE f1,f2,f3;
@@ -291,8 +289,6 @@ static void test_multi_filehandles (void) {
     toku_cachetable_close(&t);
 }
 
-#endif
-
 static void test_dirty_flush(CACHEFILE f,
                              int UU(fd),
 			     CACHEKEY key,
@@ -587,15 +583,11 @@ test_main (int argc, const char *argv[]) {
     test_mutex_init();
 
     // run tests
-#if !TOKU_WINDOWS
     test_multi_filehandles();
-#endif
     test_cachetable_create();
     for (i=0; i<1; i++) {
         test_nested_pin();
-#if !TOKU_WINDOWS
         test_multi_filehandles ();
-#endif
         test_dirty();
         test_size_resize();
         //test_size_flush();

storage/tokudb/ft-index/ft/tests/dmt-test.cc

@@ -90,10 +90,103 @@ PATENT RIGHTS GRANT:
 
 #include "test.h"
 
-#include "omt.h"
+#include <util/dmt.h>
-#include <util/omt.h>
 
-typedef OMTVALUE TESTVALUE;
+typedef void *DMTVALUE;
+
+class dmtvalue_writer {
+public:
+    size_t get_size(void) const {
+        return sizeof(DMTVALUE);
+    }
+    void write_to(DMTVALUE *const dest) const {
+        *dest = value;
+    }
+
+    dmtvalue_writer(DMTVALUE _value)
+        : value(_value) {
+    }
+    dmtvalue_writer(const uint32_t size UU(), DMTVALUE *const src)
+        : value(*src) {
+        paranoid_invariant(size == sizeof(DMTVALUE));
+    }
+private:
+    const DMTVALUE value;
+};
+
+typedef toku::dmt<DMTVALUE, DMTVALUE, dmtvalue_writer> *DMT;
+
+static int dmt_insert_at(DMT dmt, DMTVALUE value, uint32_t index) {
+    dmtvalue_writer functor(value);
+    return dmt->insert_at(functor, index);
+}
+
+static DMT dmt_create_from_sorted_array(DMTVALUE *values, uint32_t numvalues) {
+    DMT XMALLOC(dmt);
+    dmt->create();
+    for (uint32_t i = 0; i < numvalues; i++) {
+        dmt_insert_at(dmt, values[i], i);
+    }
+    return dmt;
+}
+
+struct heftor {
+    int (*h)(DMTVALUE, void *v);
+    void *v;
+};
+
+int call_heftor(const uint32_t size, const DMTVALUE &v, const heftor &htor);
+int call_heftor(const uint32_t size, const DMTVALUE &v, const heftor &htor) {
+    invariant(size == sizeof(DMTVALUE));
+    return htor.h(const_cast<DMTVALUE>(v), htor.v);
+}
+
+static int dmt_insert(DMT dmt, DMTVALUE value, int(*h)(DMTVALUE, void*v), void *v, uint32_t *index) {
+    struct heftor htor = { .h = h, .v = v };
+    dmtvalue_writer functor(value);
+    return dmt->insert<heftor, call_heftor>(functor, htor, index);
+}
+
+static int dmt_find_zero(DMT V, int (*h)(DMTVALUE, void*extra), void*extra, DMTVALUE *value, uint32_t *index) {
+    struct heftor htor = { .h = h, .v = extra };
+    uint32_t ignore;
+    return V->find_zero<heftor, call_heftor>(htor, &ignore, value, index);
+}
+
+static int dmt_find(DMT V, int (*h)(DMTVALUE, void*extra), void*extra, int direction, DMTVALUE *value, uint32_t *index) {
+    struct heftor htor = { .h = h, .v = extra };
+    uint32_t ignore;
+    return V->find<heftor, call_heftor>(htor, direction, &ignore, value, index);
+}
+
+static int dmt_split_at(DMT dmt, DMT *newdmtp, uint32_t index) {
+    if (index > dmt->size()) { return EINVAL; }
+    DMT XMALLOC(newdmt);
+    newdmt->create();
+    int r;
+
+    for (uint32_t i = index; i < dmt->size(); i++) {
+        DMTVALUE v;
+        r = dmt->fetch(i, nullptr, &v);
+        invariant_zero(r);
+        r = dmt_insert_at(newdmt, v, i-index);
+        invariant_zero(r);
+    }
+    if (dmt->size() > 0) {
+        for (uint32_t i = dmt->size(); i > index; i--) {
+            r = dmt->delete_at(i - 1);
+            invariant_zero(r);
+        }
+    }
+    r = 0;
+
+    if (r != 0) {
+        toku_free(newdmt);
+    } else {
+        *newdmtp = newdmt;
+    }
+    return r;
+}
 
 static void
 parse_args (int argc, const char *argv[]) {
@@ -133,26 +226,25 @@ enum close_when_done {
     KEEP_WHEN_DONE
 };
 enum create_type {
-    STEAL_ARRAY,
     BATCH_INSERT,
     INSERT_AT,
     INSERT_AT_ALMOST_RANDOM,
 };
 
 /* Globals */
-OMT global_omt;
+DMT global_dmt;
-TESTVALUE*       values = NULL;
+DMTVALUE*       values = nullptr;
-struct value*   nums   = NULL;
+struct value*   nums   = nullptr;
 uint32_t       length;
 
 static void
 cleanup_globals (void) {
     assert(values);
     toku_free(values);
-    values = NULL;
+    values = nullptr;
     assert(nums);
     toku_free(nums);
-    nums = NULL;
+    nums = nullptr;
 }
 
 const unsigned int random_seed = 0xFEADACBA;
@@ -178,7 +270,7 @@ init_identity_values (unsigned int seed, uint32_t num_elements) {
 
     for (i = 0; i < length; i++) {
         nums[i].number   = i;
-        values[i]        = (TESTVALUE)&nums[i];
+        values[i]        = (DMTVALUE)&nums[i];
     }
 }
 
@@ -193,7 +285,7 @@ init_distinct_sorted_values (unsigned int seed, uint32_t num_elements) {
     for (i = 0; i < length; i++) {
         number          += (uint32_t)(random() % 32) + 1;
         nums[i].number   = number;
-        values[i]        = (TESTVALUE)&nums[i];
+        values[i]        = (DMTVALUE)&nums[i];
     }
 }
 
@@ -229,25 +321,22 @@ static void
 test_close (enum close_when_done do_close) {
     if (do_close == KEEP_WHEN_DONE) return;
     assert(do_close == CLOSE_WHEN_DONE);
-    toku_omt_destroy(&global_omt);
+    global_dmt->destroy();
-    assert(global_omt==NULL);
+    toku_free(global_dmt);
+    global_dmt = nullptr;
 }
 
 static void
 test_create (enum close_when_done do_close) {
-    int r;
+    XMALLOC(global_dmt);
-    global_omt = NULL;
+    global_dmt->create();
-
-    r = toku_omt_create(&global_omt);
-    CKERR(r);
-    assert(global_omt!=NULL);
     test_close(do_close);
 }
 
 static void
 test_create_size (enum close_when_done do_close) {
     test_create(KEEP_WHEN_DONE);
-    assert(toku_omt_size(global_omt) == 0);
+    assert(global_dmt->size() == 0);
     test_close(do_close);
 }
 
@@ -258,24 +347,24 @@ test_create_insert_at_almost_random (enum close_when_done do_close) {
     uint32_t size = 0;
 
     test_create(KEEP_WHEN_DONE);
-    r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+1);
+    r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+1);
     CKERR2(r, EINVAL);
-    r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+2);
+    r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+2);
     CKERR2(r, EINVAL);
     for (i = 0; i < length/2; i++) {
-        assert(size==toku_omt_size(global_omt));
+        assert(size==global_dmt->size());
-        r = toku_omt_insert_at(global_omt, values[i], i);
+        r = dmt_insert_at(global_dmt, values[i], i);
         CKERR(r);
-        assert(++size==toku_omt_size(global_omt));
+        assert(++size==global_dmt->size());
-        r = toku_omt_insert_at(global_omt, values[length-1-i], i+1);
+        r = dmt_insert_at(global_dmt, values[length-1-i], i+1);
         CKERR(r);
-        assert(++size==toku_omt_size(global_omt));
+        assert(++size==global_dmt->size());
     }
-    r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+1);
+    r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+1);
     CKERR2(r, EINVAL);
-    r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+2);
+    r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+2);
     CKERR2(r, EINVAL);
-    assert(size==toku_omt_size(global_omt));
+    assert(size==global_dmt->size());
     test_close(do_close);
 }
 
@@ -286,39 +375,30 @@ test_create_insert_at_sequential (enum close_when_done do_close) {
     uint32_t size = 0;
 
     test_create(KEEP_WHEN_DONE);
-    r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+1);
+    r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+1);
     CKERR2(r, EINVAL);
-    r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+2);
+    r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+2);
     CKERR2(r, EINVAL);
     for (i = 0; i < length; i++) {
-        assert(size==toku_omt_size(global_omt));
+        assert(size==global_dmt->size());
-        r = toku_omt_insert_at(global_omt, values[i], i);
+        r = dmt_insert_at(global_dmt, values[i], i);
         CKERR(r);
-        assert(++size==toku_omt_size(global_omt));
+        assert(++size==global_dmt->size());
     }
-    r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+1);
+    r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+1);
     CKERR2(r, EINVAL);
-    r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+2);
+    r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+2);
     CKERR2(r, EINVAL);
-    assert(size==toku_omt_size(global_omt));
+    assert(size==global_dmt->size());
     test_close(do_close);
 }
 
 static void
 test_create_from_sorted_array (enum create_type create_choice, enum close_when_done do_close) {
-    int r;
+    global_dmt = nullptr;
-    global_omt = NULL;
 
     if (create_choice == BATCH_INSERT) {
-        r = toku_omt_create_from_sorted_array(&global_omt, values, length);
+        global_dmt = dmt_create_from_sorted_array(values, length);
-        CKERR(r);
-    }
-    else if (create_choice == STEAL_ARRAY) {
-        TESTVALUE* MALLOC_N(length, values_copy);
-        memcpy(values_copy, values, length*sizeof(*values));
-        r = toku_omt_create_steal_sorted_array(&global_omt, &values_copy, length, length);
-        CKERR(r);
-        assert(values_copy==NULL);
     }
     else if (create_choice == INSERT_AT) {
         test_create_insert_at_sequential(KEEP_WHEN_DONE);
@@ -326,33 +406,35 @@ test_create_from_sorted_array (enum create_type create_choice, enum close_when_d
     else if (create_choice == INSERT_AT_ALMOST_RANDOM) {
         test_create_insert_at_almost_random(KEEP_WHEN_DONE);
     }
-    else assert(false);
+    else {
+        assert(false);
+    }
 
-    assert(global_omt!=NULL);
+    assert(global_dmt!=nullptr);
     test_close(do_close);
 }
 
 static void
 test_create_from_sorted_array_size (enum create_type create_choice, enum close_when_done do_close) {
     test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
-    assert(toku_omt_size(global_omt)==length);
+    assert(global_dmt->size()==length);
     test_close(do_close);
 }    
 
 static void
-test_fetch_verify (OMT omtree, TESTVALUE* val, uint32_t len ) {
+test_fetch_verify (DMT dmtree, DMTVALUE* val, uint32_t len ) {
     uint32_t i;
     int r;
-    TESTVALUE v = (TESTVALUE)&i;
+    DMTVALUE v = (DMTVALUE)&i;
-    TESTVALUE oldv = v;
+    DMTVALUE oldv = v;
 
-    assert(len == toku_omt_size(omtree));
+    assert(len == dmtree->size());
     for (i = 0; i < len; i++) {
         assert(oldv!=val[i]);
-        v = NULL;
+        v = nullptr;
-        r = toku_omt_fetch(omtree, i, &v);
+        r = dmtree->fetch(i, nullptr, &v);
         CKERR(r);
-        assert(v != NULL);
+        assert(v != nullptr);
         assert(v != oldv);
         assert(v == val[i]);
         assert(V(v)->number == V(val[i])->number);
@@ -361,7 +443,7 @@ test_fetch_verify (OMT omtree, TESTVALUE* val, uint32_t len ) {
 
     for (i = len; i < len*2; i++) {
         v = oldv;
-        r = toku_omt_fetch(omtree, i, &v);
+        r = dmtree->fetch(i, nullptr, &v);
         CKERR2(r, EINVAL);
         assert(v == oldv);
     }
@@ -371,30 +453,45 @@ test_fetch_verify (OMT omtree, TESTVALUE* val, uint32_t len ) {
 static void
 test_create_fetch_verify (enum create_type create_choice, enum close_when_done do_close) {
     test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
-    test_fetch_verify(global_omt, values, length);
+    test_fetch_verify(global_dmt, values, length);
     test_close(do_close);
 }
 
 static int iterate_helper_error_return = 1;
 
 static int
-iterate_helper (TESTVALUE v, uint32_t idx, void* extra) {
+iterate_helper (DMTVALUE v, uint32_t idx, void* extra) {
-    if (extra == NULL) return iterate_helper_error_return;
+    if (extra == nullptr) return iterate_helper_error_return;
-    TESTVALUE* vals = (TESTVALUE *)extra;
+    DMTVALUE* vals = (DMTVALUE *)extra;
-    assert(v != NULL);
+    assert(v != nullptr);
     assert(v == vals[idx]);
     assert(V(v)->number == V(vals[idx])->number);
     return 0;
 }
+struct functor {
+    int (*f)(DMTVALUE, uint32_t, void *);
+    void *v;
+};
+
+int call_functor(const uint32_t size, const DMTVALUE &v, uint32_t idx, functor *const ftor);
+int call_functor(const uint32_t size, const DMTVALUE &v, uint32_t idx, functor *const ftor) {
+    invariant(size == sizeof(DMTVALUE));
+    return ftor->f(const_cast<DMTVALUE>(v), idx, ftor->v);
+}
+
+static int dmt_iterate(DMT dmt, int (*f)(DMTVALUE, uint32_t, void*), void*v) {
+    struct functor ftor = { .f = f, .v = v };
+    return dmt->iterate<functor, call_functor>(&ftor);
+}
 
 static void
-test_iterate_verify (OMT omtree, TESTVALUE* vals, uint32_t len) {
+test_iterate_verify (DMT dmtree, DMTVALUE* vals, uint32_t len) {
     int r;
     iterate_helper_error_return = 0;
-    r = toku_omt_iterate(omtree, iterate_helper, (void*)vals);
+    r = dmt_iterate(dmtree, iterate_helper, (void*)vals);
     CKERR(r);
     iterate_helper_error_return = 0xFEEDABBA;
-    r = toku_omt_iterate(omtree, iterate_helper, NULL);
+    r = dmt_iterate(dmtree, iterate_helper, nullptr);
     if (!len) {
         CKERR2(r, 0);
     }
@@ -406,7 +503,7 @@ test_iterate_verify (OMT omtree, TESTVALUE* vals, uint32_t len) {
 static void
 test_create_iterate_verify (enum create_type create_choice, enum close_when_done do_close) {
     test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
-    test_iterate_verify(global_omt, values, length);
+    test_iterate_verify(global_dmt, values, length);
     test_close(do_close);
 }
 
@@ -431,19 +528,26 @@ permute_array (uint32_t* arr, uint32_t len) {
     }
 }
 
+static int
+dmt_set_at (DMT dmt, DMTVALUE value, uint32_t index) {
+    int r = dmt->delete_at(index);
+    if (r!=0) return r;
+    return dmt_insert_at(dmt, value, index);
+}
+
 static void
 test_create_set_at (enum create_type create_choice, enum close_when_done do_close) {
     uint32_t i = 0;
 
-    struct value*   old_nums   = NULL;
+    struct value*   old_nums   = nullptr;
     MALLOC_N(length, old_nums);
     assert(nums);
 
-    uint32_t* perm = NULL;
+    uint32_t* perm = nullptr;
     MALLOC_N(length, perm);
     assert(perm);
 
-    TESTVALUE* old_values = NULL;
+    DMTVALUE* old_values = nullptr;
     MALLOC_N(length, old_values);
     assert(old_values);
     
@@ -459,22 +563,22 @@ test_create_set_at (enum create_type create_choice, enum close_when_done do_clos
     }
     test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
     int r;
-    r = toku_omt_set_at (global_omt, values[0], length);
+    r = dmt_set_at (global_dmt, values[0], length);
     CKERR2(r,EINVAL);    
-    r = toku_omt_set_at (global_omt, values[0], length+1);
+    r = dmt_set_at (global_dmt, values[0], length+1);
     CKERR2(r,EINVAL);    
     for (i = 0; i < length; i++) {
         uint32_t choice = perm[i];
         values[choice] = &nums[choice];
         nums[choice].number = (uint32_t)random();
-        r = toku_omt_set_at (global_omt, values[choice], choice);
+        r = dmt_set_at (global_dmt, values[choice], choice);
         CKERR(r);
-        test_iterate_verify(global_omt, values, length);
+        test_iterate_verify(global_dmt, values, length);
-        test_fetch_verify(global_omt, values, length);
+        test_fetch_verify(global_dmt, values, length);
     }
-    r = toku_omt_set_at (global_omt, values[0], length);
+    r = dmt_set_at (global_dmt, values[0], length);
     CKERR2(r,EINVAL);    
-    r = toku_omt_set_at (global_omt, values[0], length+1);
+    r = dmt_set_at (global_dmt, values[0], length+1);
     CKERR2(r,EINVAL);    
 
     toku_free(perm);
@@ -485,8 +589,8 @@ test_create_set_at (enum create_type create_choice, enum close_when_done do_clos
 }
 
 static int
-insert_helper (TESTVALUE value, void* extra_insert) {
+insert_helper (DMTVALUE value, void* extra_insert) {
-    TESTVALUE to_insert = (OMTVALUE)extra_insert;
+    DMTVALUE to_insert = (DMTVALUE)extra_insert;
     assert(to_insert);
 
     if (V(value)->number < V(to_insert)->number) return -1;
@@ -498,7 +602,7 @@ static void
 test_create_insert (enum close_when_done do_close) {
     uint32_t i = 0;
 
-    uint32_t* perm = NULL;
+    uint32_t* perm = nullptr;
     MALLOC_N(length, perm);
     assert(perm);
 
@@ -510,11 +614,11 @@ test_create_insert (enum close_when_done do_close) {
     length = 0;
     while (length < size) {
         uint32_t choice = perm[length];
-        TESTVALUE to_insert = &nums[choice];
+        DMTVALUE to_insert = &nums[choice];
         uint32_t idx = UINT32_MAX;
 
-        assert(length==toku_omt_size(global_omt));
+        assert(length==global_dmt->size());
-        r = toku_omt_insert(global_omt, to_insert, insert_helper, to_insert, &idx);
+        r = dmt_insert(global_dmt, to_insert, insert_helper, to_insert, &idx);
         CKERR(r);
         assert(idx <= length);
         if (idx > 0) {
@@ -524,24 +628,24 @@ test_create_insert (enum close_when_done do_close) {
             assert(V(to_insert)->number < V(values[idx])->number);
         }
         length++;
-        assert(length==toku_omt_size(global_omt));
+        assert(length==global_dmt->size());
         /* Make room */
         for (i = length-1; i > idx; i--) {
             values[i] = values[i-1];
         }
         values[idx] = to_insert;
-        test_fetch_verify(global_omt, values, length);
+        test_fetch_verify(global_dmt, values, length);
-        test_iterate_verify(global_omt, values, length);
+        test_iterate_verify(global_dmt, values, length);
 
         idx = UINT32_MAX;
-        r = toku_omt_insert(global_omt, to_insert, insert_helper, to_insert, &idx);
+        r = dmt_insert(global_dmt, to_insert, insert_helper, to_insert, &idx);
         CKERR2(r, DB_KEYEXIST);
         assert(idx < length);
         assert(V(values[idx])->number == V(to_insert)->number);
-        assert(length==toku_omt_size(global_omt));
+        assert(length==global_dmt->size());
 
-        test_iterate_verify(global_omt, values, length);
+        test_iterate_verify(global_dmt, values, length);
-        test_fetch_verify(global_omt, values, length);
+        test_fetch_verify(global_dmt, values, length);
     }
 
     toku_free(perm);
@@ -555,91 +659,118 @@ test_create_delete_at (enum create_type create_choice, enum close_when_done do_c
     int r = ENOSYS;
     test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
 
-    assert(length == toku_omt_size(global_omt));
+    assert(length == global_dmt->size());
-    r = toku_omt_delete_at(global_omt, length);
+    r = global_dmt->delete_at(length);
     CKERR2(r,EINVAL);
-    assert(length == toku_omt_size(global_omt));
+    assert(length == global_dmt->size());
-    r = toku_omt_delete_at(global_omt, length+1);
+    r = global_dmt->delete_at(length+1);
     CKERR2(r,EINVAL);
     while (length > 0) {
-        assert(length == toku_omt_size(global_omt));
+        assert(length == global_dmt->size());
         uint32_t index_to_delete = random()%length;
-        r = toku_omt_delete_at(global_omt, index_to_delete);
+        r = global_dmt->delete_at(index_to_delete);
         CKERR(r);
         for (i = index_to_delete+1; i < length; i++) {
             values[i-1] = values[i];
         }
         length--;
-        test_fetch_verify(global_omt, values, length);
+        test_fetch_verify(global_dmt, values, length);
-        test_iterate_verify(global_omt, values, length);
+        test_iterate_verify(global_dmt, values, length);
     }
     assert(length == 0);
-    assert(length == toku_omt_size(global_omt));
+    assert(length == global_dmt->size());
-    r = toku_omt_delete_at(global_omt, length);
+    r = global_dmt->delete_at(length);
     CKERR2(r, EINVAL);
-    assert(length == toku_omt_size(global_omt));
+    assert(length == global_dmt->size());
-    r = toku_omt_delete_at(global_omt, length+1);
+    r = global_dmt->delete_at(length+1);
     CKERR2(r, EINVAL);
     test_close(do_close);
 }
 
+static int dmt_merge(DMT leftdmt, DMT rightdmt, DMT *newdmtp) {
+    DMT XMALLOC(newdmt);
+    newdmt->create();
+    int r;
+    for (uint32_t i = 0; i < leftdmt->size(); i++) {
+        DMTVALUE v;
+        r = leftdmt->fetch(i, nullptr, &v);
+        invariant_zero(r);
+        r = newdmt->insert_at(v, i);
+        invariant_zero(r);
+    }
+    uint32_t offset = leftdmt->size();
+    for (uint32_t i = 0; i < rightdmt->size(); i++) {
+        DMTVALUE v;
+        r = rightdmt->fetch(i, nullptr, &v);
+        invariant_zero(r);
+        r = newdmt->insert_at(v, i+offset);
+        invariant_zero(r);
+    }
+    leftdmt->destroy();
+    rightdmt->destroy();
+    toku_free(leftdmt);
+    toku_free(rightdmt);
+    *newdmtp = newdmt;
+    return 0;
+}
+
 static void
 test_split_merge (enum create_type create_choice, enum close_when_done do_close) {
     int r = ENOSYS;
     uint32_t i = 0;
-    OMT left_split = NULL;
+    DMT left_split = nullptr;
-    OMT right_split = NULL;
+    DMT right_split = nullptr;
     test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
 
     for (i = 0; i <= length; i++) {
-        r = toku_omt_split_at(global_omt, &right_split, length+1);
+        r = dmt_split_at(global_dmt, &right_split, length+1);
         CKERR2(r,EINVAL);
-        r = toku_omt_split_at(global_omt, &right_split, length+2);
+        r = dmt_split_at(global_dmt, &right_split, length+2);
         CKERR2(r,EINVAL);
 
         //
         // test successful split
         //
-        r = toku_omt_split_at(global_omt, &right_split, i);
+        r = dmt_split_at(global_dmt, &right_split, i);
         CKERR(r);
-        left_split = global_omt;
+        left_split = global_dmt;
-        global_omt = NULL;
+        global_dmt = nullptr;
-        assert(toku_omt_size(left_split) == i);
+        assert(left_split->size() == i);
-        assert(toku_omt_size(right_split) == length - i);
+        assert(right_split->size() == length - i);
         test_fetch_verify(left_split, values, i);
         test_iterate_verify(left_split, values, i);
         test_fetch_verify(right_split, &values[i], length - i);
         test_iterate_verify(right_split, &values[i], length - i);
         //
-        // verify that new global_omt's cannot do bad splits
+        // verify that new global_dmt's cannot do bad splits
         //
-        r = toku_omt_split_at(left_split, &global_omt, i+1);
+        r = dmt_split_at(left_split, &global_dmt, i+1);
         CKERR2(r,EINVAL);
-        assert(toku_omt_size(left_split) == i);
+        assert(left_split->size() == i);
-        assert(toku_omt_size(right_split) == length - i);
+        assert(right_split->size() == length - i);
-        r = toku_omt_split_at(left_split, &global_omt, i+2);
+        r = dmt_split_at(left_split, &global_dmt, i+2);
         CKERR2(r,EINVAL);
-        assert(toku_omt_size(left_split) == i);
+        assert(left_split->size() == i);
-        assert(toku_omt_size(right_split) == length - i);
+        assert(right_split->size() == length - i);
-        r = toku_omt_split_at(right_split, &global_omt, length - i + 1);
+        r = dmt_split_at(right_split, &global_dmt, length - i + 1);
         CKERR2(r,EINVAL);
-        assert(toku_omt_size(left_split) == i);
+        assert(left_split->size() == i);
-        assert(toku_omt_size(right_split) == length - i);
+        assert(right_split->size() == length - i);
-        r = toku_omt_split_at(right_split, &global_omt, length - i + 1);
+        r = dmt_split_at(right_split, &global_dmt, length - i + 1);
         CKERR2(r,EINVAL);
-        assert(toku_omt_size(left_split) == i);
+        assert(left_split->size() == i);
-        assert(toku_omt_size(right_split) == length - i);
+        assert(right_split->size() == length - i);
 
         //
         // test merge
         //
-        r = toku_omt_merge(left_split,right_split,&global_omt);
+        r = dmt_merge(left_split,right_split,&global_dmt);
         CKERR(r);
-        left_split = NULL;
+        left_split = nullptr;
-        right_split = NULL;
+        right_split = nullptr;
-        assert(toku_omt_size(global_omt) == length);
+        assert(global_dmt->size() == length);
-        test_fetch_verify(global_omt, values, length);
+        test_fetch_verify(global_dmt, values, length);
-        test_iterate_verify(global_omt, values, length);
+        test_iterate_verify(global_dmt, values, length);
     }
     test_close(do_close);
 }
@@ -693,8 +824,8 @@ typedef struct {
 
 
 static int
-test_heaviside (OMTVALUE v_omt, void* x) {
+test_heaviside (DMTVALUE v_dmt, void* x) {
-    TESTVALUE v = (OMTVALUE) v_omt;
+    DMTVALUE v = (DMTVALUE) v_dmt;
     h_extra* extra = (h_extra*)x;
     assert(v && x);
     assert(extra->first_zero <= extra->first_pos);
@@ -712,24 +843,24 @@ heavy_extra (h_extra* extra, uint32_t first_zero, uint32_t first_pos) {
 }
 
 static void
-test_find_dir (int dir, void* extra, int (*h)(OMTVALUE, void*),
+test_find_dir (int dir, void* extra, int (*h)(DMTVALUE, void*),
 	       int r_expect, bool idx_will_change, uint32_t idx_expect,
 	       uint32_t number_expect, bool UU(cursor_valid)) {
     uint32_t idx     = UINT32_MAX;
     uint32_t old_idx = idx;
-    TESTVALUE omt_val;
+    DMTVALUE dmt_val;
     int r;
 
-    omt_val = NULL;
+    dmt_val = nullptr;
 
-    /* Verify we can pass NULL value. */
+    /* Verify we can pass nullptr value. */
-    omt_val = NULL;
+    dmt_val = nullptr;
     idx      = old_idx;
     if (dir == 0) {
-        r = toku_omt_find_zero(global_omt, h, extra,      NULL, &idx);
+        r = dmt_find_zero(global_dmt, h, extra,      nullptr, &idx);
     }
     else {
-        r = toku_omt_find(     global_omt, h, extra, dir, NULL, &idx);
+        r = dmt_find(     global_dmt, h, extra, dir, nullptr, &idx);
     }
     CKERR2(r, r_expect);
     if (idx_will_change) {
@@ -738,38 +869,38 @@ test_find_dir (int dir, void* extra, int (*h)(OMTVALUE, void*),
     else {
         assert(idx == old_idx);
     }
-    assert(omt_val == NULL);
+    assert(dmt_val == nullptr);
     
-    /* Verify we can pass NULL idx. */
+    /* Verify we can pass nullptr idx. */
-    omt_val  = NULL;
+    dmt_val  = nullptr;
     idx      = old_idx;
     if (dir == 0) {
-        r = toku_omt_find_zero(global_omt, h, extra,      &omt_val, 0);
+        r = dmt_find_zero(global_dmt, h, extra,      &dmt_val, 0);
     }
     else {
-        r = toku_omt_find(     global_omt, h, extra, dir, &omt_val, 0);
+        r = dmt_find(     global_dmt, h, extra, dir, &dmt_val, 0);
     }
     CKERR2(r, r_expect);
     assert(idx == old_idx);
     if (r == DB_NOTFOUND) {
-        assert(omt_val == NULL);
+        assert(dmt_val == nullptr);
     }
     else {
-        assert(V(omt_val)->number == number_expect);
+        assert(V(dmt_val)->number == number_expect);
     }
 
-    /* Verify we can pass NULL both. */
+    /* Verify we can pass nullptr both. */
-    omt_val  = NULL;
+    dmt_val  = nullptr;
     idx      = old_idx;
     if (dir == 0) {
-        r = toku_omt_find_zero(global_omt, h, extra,      NULL, 0);
+        r = dmt_find_zero(global_dmt, h, extra,      nullptr, 0);
     }
     else {
-        r = toku_omt_find(     global_omt, h, extra, dir, NULL, 0);
+        r = dmt_find(     global_dmt, h, extra, dir, nullptr, 0);
     }
     CKERR2(r, r_expect);
     assert(idx == old_idx);
-    assert(omt_val == NULL);
+    assert(dmt_val == nullptr);
 }
 
 static void
@@ -857,33 +988,34 @@ runtests_create_choice (enum create_type create_choice) {
 static void
 test_clone(uint32_t nelts)
 // Test that each clone operation gives the right data back.  If nelts is
-// zero, also tests that you still get a valid OMT back and that the way
+// zero, also tests that you still get a valid DMT back and that the way
 // to deallocate it still works.
 {
-    OMT src = NULL, dest = NULL;
+    DMT src = nullptr, dest = nullptr;
-    int r;
+    int r = 0;
 
-    r = toku_omt_create(&src);
+    XMALLOC(src);
-    assert_zero(r);
+    src->create();
     for (long i = 0; i < nelts; ++i) {
-        r = toku_omt_insert_at(src, (OMTVALUE) i, i);
+        r = dmt_insert_at(src, (DMTVALUE) i, i);
         assert_zero(r);
     }
 
-    r = toku_omt_clone_noptr(&dest, src);
+    XMALLOC(dest);
-    assert_zero(r);
+    dest->clone(*src);
-    assert(dest != NULL);
+    assert(dest->size() == nelts);
-    assert(toku_omt_size(dest) == nelts);
     for (long i = 0; i < nelts; ++i) {
-        OMTVALUE v;
+        DMTVALUE v;
         long l;
-        r = toku_omt_fetch(dest, i, &v);
+        r = dest->fetch(i, nullptr, &v);
         assert_zero(r);
         l = (long) v;
         assert(l == i);
     }
-    toku_omt_destroy(&dest);
+    dest->destroy();
-    toku_omt_destroy(&src);
+    toku_free(dest);
+    src->destroy();
+    toku_free(src);
 }
 
 int
@@ -893,7 +1025,6 @@ test_main(int argc, const char *argv[]) {
     test_create(      CLOSE_WHEN_DONE);
     test_create_size( CLOSE_WHEN_DONE);
     runtests_create_choice(BATCH_INSERT);
-    runtests_create_choice(STEAL_ARRAY);
     runtests_create_choice(INSERT_AT);
     runtests_create_choice(INSERT_AT_ALMOST_RANDOM);
     test_clone(0);

storage/tokudb/ft-index/ft/tests/dmt-test2.cc

@@ -0,0 +1,373 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*
+COPYING CONDITIONS NOTICE:
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation, and provided that the
+  following conditions are met:
+
+      * Redistributions of source code must retain this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below).
+
+      * Redistributions in binary form must reproduce this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below) in the documentation and/or other materials
+        provided with the distribution.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+  02110-1301, USA.
+
+COPYRIGHT NOTICE:
+
+  TokuDB, Tokutek Fractal Tree Indexing Library.
+  Copyright (C) 2007-2013 Tokutek, Inc.
+
+DISCLAIMER:
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+UNIVERSITY PATENT NOTICE:
+
+  The technology is licensed by the Massachusetts Institute of
+  Technology, Rutgers State University of New Jersey, and the Research
+  Foundation of State University of New York at Stony Brook under
+  United States of America Serial No. 11/760379 and to the patents
+  and/or patent applications resulting from it.
+
+PATENT MARKING NOTICE:
+
+  This software is covered by US Patent No. 8,185,551.
+  This software is covered by US Patent No. 8,489,638.
+
+PATENT RIGHTS GRANT:
+
+  "THIS IMPLEMENTATION" means the copyrightable works distributed by
+  Tokutek as part of the Fractal Tree project.
+
+  "PATENT CLAIMS" means the claims of patents that are owned or
+  licensable by Tokutek, both currently or in the future; and that in
+  the absence of this license would be infringed by THIS
+  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
+
+  "PATENT CHALLENGE" shall mean a challenge to the validity,
+  patentability, enforceability and/or non-infringement of any of the
+  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
+
+  Tokutek hereby grants to you, for the term and geographical scope of
+  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
+  irrevocable (except as stated in this section) patent license to
+  make, have made, use, offer to sell, sell, import, transfer, and
+  otherwise run, modify, and propagate the contents of THIS
+  IMPLEMENTATION, where such license applies only to the PATENT
+  CLAIMS.  This grant does not include claims that would be infringed
+  only as a consequence of further modifications of THIS
+  IMPLEMENTATION.  If you or your agent or licensee institute or order
+  or agree to the institution of patent litigation against any entity
+  (including a cross-claim or counterclaim in a lawsuit) alleging that
+  THIS IMPLEMENTATION constitutes direct or contributory patent
+  infringement, or inducement of patent infringement, then any rights
+  granted to you under this License shall terminate as of the date
+  such litigation is filed.  If you or your agent or exclusive
+  licensee institute or order or agree to the institution of a PATENT
+  CHALLENGE, then Tokutek may terminate any rights granted to you
+  under this License.
+*/
+
+#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
+
+#include "test.h"
+
+#include <util/dmt.h>
+
+static void
+parse_args (int argc, const char *argv[]) {
+    const char *argv0=argv[0];
+    while (argc>1) {
+        int resultcode=0;
+        if (strcmp(argv[1], "-v")==0) {
+            verbose++;
+        } else if (strcmp(argv[1], "-q")==0) {
+            verbose = 0;
+        } else if (strcmp(argv[1], "-h")==0) {
+        do_usage:
+            fprintf(stderr, "Usage:\n%s [-v|-h]\n", argv0);
+            exit(resultcode);
+        } else {
+            resultcode=1;
+            goto do_usage;
+        }
+        argc--;
+        argv++;
+    }
+}
+/* End ".h like" stuff. */
+
+struct value {
+    uint32_t number;
+};
+#define V(x) ((struct value *)(x))
+
+
+
+const uint32_t MAXNUM = 1024;
+const uint32_t MAXLEN = 32;
+char data[MAXNUM][MAXLEN];
+
+struct val_type {
+    char c[MAXLEN];
+};
+
+namespace toku {
+class vwriter {
+    public:
+        size_t get_size(void) const {
+            size_t len = strlen(v.c);
+            invariant(len < sizeof(val_type));
+            return len + 1;
+        }
+        void write_to(val_type *const dest) const {
+            strcpy(dest->c, v.c);
+        }
+
+        vwriter(const char* c) {
+            invariant(strlen(c) < sizeof(val_type));
+            strcpy(v.c, c);
+        }
+
+        vwriter(const uint32_t klpair_len, val_type *const src) {
+            invariant(strlen(src->c) < sizeof(val_type));
+            strcpy(v.c, src->c);
+            invariant(klpair_len == get_size());
+        }
+    private:
+        val_type v;
+};
+}
+
+/* Globals */
+typedef toku::dmt<val_type, val_type*, toku::vwriter> vdmt;
+
+const unsigned int random_seed = 0xFEADACBA;
+
+///////////////
+
+
+static void fail_one_verify(uint32_t len, uint32_t num, vdmt *v) {
+    val_type* fetched_data;
+    int count = 0;
+    v->verify();
+    for (uint32_t i = 0; i < num; i++) {
+        uint32_t fetched_len;
+        int r = v->fetch(i-count, &fetched_len, &fetched_data);
+        if (r != 0 || fetched_len != len || strcmp(fetched_data->c, data[i])) {
+            count++;
+            continue;
+        }
+    }
+    invariant(count == 1);
+}
+
+static void verify(uint32_t len, uint32_t num, vdmt *v) {
+    v->verify();
+    val_type* fetched_data;
+    for (uint32_t i = 0; i < num; i++) {
+        uint32_t fetched_len;
+        int r = v->fetch(i, &fetched_len, &fetched_data);
+        CKERR(r);
+        invariant(fetched_len == len);
+        invariant(!strcmp(fetched_data->c, data[i]));
+    }
+}
+
+
+static void test_builder_fixed(uint32_t len, uint32_t num) {
+    srandom(random_seed);
+    assert(len > 1);
+    assert(len <= MAXLEN);
+    assert(num <= MAXNUM);
+    for (uint32_t i = 0; i < num; i++) {
+        for (uint32_t j = 0; j < len-1; j++) {
+            data[i][j] = random() % 255 + 1; //This way it doesn't end up being 0 and thought of as NUL
+        }
+        data[i][len-1] = '\0'; //cap it
+    }
+
+    vdmt::builder builder;
+    builder.create(num, num * len);
+
+    for (uint32_t i = 0; i < num; i++) {
+        vwriter vfun(data[i]);
+        builder.append(vfun);
+    }
+    invariant(builder.value_length_is_fixed());
+    vdmt v;
+    builder.build(&v);
+    invariant(v.value_length_is_fixed());
+    invariant(v.get_fixed_length() == len || num == 0);
+
+    invariant(v.size() == num);
+
+    verify(len, num, &v);
+
+    for (uint32_t change = 0; change < num; change++) {
+        vdmt v2;
+        v2.clone(v);
+        v2.delete_at(change);
+        fail_one_verify(len, num, &v2);
+
+        vwriter vfun(data[change]);
+        v2.insert_at(vfun, change);
+        verify(len, num, &v2);
+        v2.destroy();
+    }
+
+    v.destroy();
+}
+
+static void test_builder_variable(uint32_t len, uint32_t len2, uint32_t num) {
+    srandom(random_seed);
+    assert(len > 1);
+    assert(len <= MAXLEN);
+    assert(num <= MAXNUM);
+    assert(num > 3);
+    uint32_t which2 = random() % num;
+    for (uint32_t i = 0; i < num; i++) {
+        uint32_t thislen = i == which2 ? len2 : len;
+        for (uint32_t j = 0; j < thislen-1; j++) {
+            data[i][j] = random() % 255 + 1; //This way it doesn't end up being 0 and thought of as NUL
+        }
+        data[i][thislen-1] = '\0'; //cap it
+    }
+
+    vdmt::builder builder;
+    builder.create(num, (num-1) * len + len2);
+
+    for (uint32_t i = 0; i < num; i++) {
+        vwriter vfun(data[i]);
+        builder.append(vfun);
+    }
+    invariant(!builder.value_length_is_fixed());
+    vdmt v;
+    builder.build(&v);
+    invariant(!v.value_length_is_fixed());
+
+    invariant(v.size() == num);
+
+    val_type* fetched_data;
+    for (uint32_t i = 0; i < num; i++) {
+        uint32_t fetched_len;
+        int r = v.fetch(i, &fetched_len, &fetched_data);
+        CKERR(r);
+        if (i == which2) {
+            invariant(fetched_len == len2);
+            invariant(!strcmp(fetched_data->c, data[i]));
+        } else {
+            invariant(fetched_len == len);
+            invariant(!strcmp(fetched_data->c, data[i]));
+        }
+    }
+
+    v.destroy();
+}
+
+static void test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(uint32_t len, uint32_t num) {
+    srandom(random_seed);
+    assert(len <= MAXLEN);
+    assert(num <= MAXNUM);
+    for (uint32_t i = 0; i < num; i++) {
+        for (uint32_t j = 0; j < len-1; j++) {
+            data[i][j] = random() % 255 + 1; //This way it doesn't end up being 0 and thought of as NUL
+        }
+        data[i][len-1] = '\0'; //cap it
+    }
+
+    char *flat = (char*)toku_xmalloc(len * num);
+    char *p = flat;
+    for (uint32_t i = 0; i < num; i++) {
+        memcpy(p, data[i], len);
+        p += len;
+    }
+    vdmt v;
+
+    v.create_from_sorted_memory_of_fixed_size_elements(flat, num, len*num, len);
+    invariant(v.value_length_is_fixed());
+    invariant(v.get_fixed_length() == len);
+
+    invariant(v.size() == num);
+
+    val_type* fetched_data;
+    for (uint32_t i = 0; i < num; i++) {
+        uint32_t fetched_len;
+        int r = v.fetch(i, &fetched_len, &fetched_data);
+        CKERR(r);
+        invariant(fetched_len == len);
+        invariant(!strcmp(fetched_data->c, data[i]));
+    }
+
+    char *serialized_flat = (char*)toku_xmalloc(len*num);
+    struct wbuf wb;
+    wbuf_nocrc_init(&wb, serialized_flat, len*num);
+    v.prepare_for_serialize();
+    v.serialize_values(len*num, &wb);
+    invariant(!memcmp(serialized_flat, flat, len*num));
+
+    
+    if (num > 1) {
+        //Currently converting to dtree treats the entire thing as NOT fixed length.
+        //Optional additional perf here.
+        uint32_t which = (random() % (num-1)) + 1;  // Not last, not first
+        invariant(which > 0 && which < num-1);
+        v.delete_at(which);
+
+        memmove(flat + which*len, flat+(which+1)*len, (num-which-1) * len);
+        v.prepare_for_serialize();
+        wbuf_nocrc_init(&wb, serialized_flat, len*(num-1));
+        v.serialize_values(len*(num-1), &wb);
+        invariant(!memcmp(serialized_flat, flat, len*(num-1)));
+    }
+
+    toku_free(flat);
+    toku_free(serialized_flat);
+
+    v.destroy();
+}
+
+int
+test_main(int argc, const char *argv[]) {
+    parse_args(argc, argv);
+    // Do test with size divisible by 4 and not
+    test_builder_fixed(4, 0);
+    test_builder_fixed(5, 0);
+    test_builder_fixed(4, 1);
+    test_builder_fixed(5, 1);
+    test_builder_fixed(4, 100);
+    test_builder_fixed(5, 100);
+    // Do test with zero, one, or both sizes divisible
+    test_builder_variable(4, 8, 100);
+    test_builder_variable(4, 5, 100);
+    test_builder_variable(5, 8, 100);
+    test_builder_variable(5, 10, 100);
+
+    test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(4, 0);
+    test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(5, 0);
+    test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(4, 1);
+    test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(5, 1);
+    test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(4, 100);
+    test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(5, 100);
+
+    return 0;
+}
+

storage/tokudb/ft-index/ft/tests/ft-bfe-query.cc

@@ -101,9 +101,9 @@ int64_key_cmp (DB *db UU(), const DBT *a, const DBT *b) {
 }
 
 static void
-test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
+test_prefetch_read(int fd, FT_HANDLE UU(ft), FT ft_h) {
     int r;
-    brt_h->compare_fun = int64_key_cmp;    
+    ft_h->compare_fun = int64_key_cmp;    
     FT_CURSOR XMALLOC(cursor);
     FTNODE dn = NULL;
     PAIR_ATTR attr;
@@ -120,7 +120,7 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
     // quick test to see that we have the right behavior when we set
     // disable_prefetching to true
     cursor->disable_prefetching = true;
-    fill_bfe_for_prefetch(&bfe, brt_h, cursor);
+    fill_bfe_for_prefetch(&bfe, ft_h, cursor);
     FTNODE_DISK_DATA ndd = NULL;
     r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
     assert(r==0);
@@ -139,14 +139,14 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
     // now enable prefetching again
     cursor->disable_prefetching = false;
     
-    fill_bfe_for_prefetch(&bfe, brt_h, cursor);
+    fill_bfe_for_prefetch(&bfe, ft_h, cursor);
     r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
     assert(r==0);
     assert(dn->n_children == 3);
     assert(BP_STATE(dn,0) == PT_AVAIL);
     assert(BP_STATE(dn,1) == PT_AVAIL);
     assert(BP_STATE(dn,2) == PT_AVAIL);
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_COMPRESSED);
     assert(BP_STATE(dn,1) == PT_COMPRESSED);
     assert(BP_STATE(dn,2) == PT_COMPRESSED);
@@ -161,14 +161,14 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
     uint64_t left_key = 150;
     toku_fill_dbt(&cursor->range_lock_left_key, &left_key, sizeof(uint64_t));
     cursor->left_is_neg_infty = false;
-    fill_bfe_for_prefetch(&bfe, brt_h, cursor);
+    fill_bfe_for_prefetch(&bfe, ft_h, cursor);
     r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
     assert(r==0);
     assert(dn->n_children == 3);
     assert(BP_STATE(dn,0) == PT_ON_DISK);
     assert(BP_STATE(dn,1) == PT_AVAIL);
     assert(BP_STATE(dn,2) == PT_AVAIL);
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_ON_DISK);
     assert(BP_STATE(dn,1) == PT_COMPRESSED);
     assert(BP_STATE(dn,2) == PT_COMPRESSED);
@@ -183,14 +183,14 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
     uint64_t right_key = 151;
     toku_fill_dbt(&cursor->range_lock_right_key, &right_key, sizeof(uint64_t));
     cursor->right_is_pos_infty = false;
-    fill_bfe_for_prefetch(&bfe, brt_h, cursor);
+    fill_bfe_for_prefetch(&bfe, ft_h, cursor);
     r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
     assert(r==0);
     assert(dn->n_children == 3);
     assert(BP_STATE(dn,0) == PT_ON_DISK);
     assert(BP_STATE(dn,1) == PT_AVAIL);
     assert(BP_STATE(dn,2) == PT_ON_DISK);
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_ON_DISK);
     assert(BP_STATE(dn,1) == PT_COMPRESSED);
     assert(BP_STATE(dn,2) == PT_ON_DISK);
@@ -204,14 +204,14 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
 
     left_key = 100000;
     right_key = 100000;
-    fill_bfe_for_prefetch(&bfe, brt_h, cursor);
+    fill_bfe_for_prefetch(&bfe, ft_h, cursor);
     r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
     assert(r==0);
     assert(dn->n_children == 3);
     assert(BP_STATE(dn,0) == PT_ON_DISK);
     assert(BP_STATE(dn,1) == PT_ON_DISK);
     assert(BP_STATE(dn,2) == PT_AVAIL);
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_ON_DISK);
     assert(BP_STATE(dn,1) == PT_ON_DISK);
     assert(BP_STATE(dn,2) == PT_COMPRESSED);
@@ -225,14 +225,14 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
 
     left_key = 100;
     right_key = 100;
-    fill_bfe_for_prefetch(&bfe, brt_h, cursor);
+    fill_bfe_for_prefetch(&bfe, ft_h, cursor);
     r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
     assert(r==0);
     assert(dn->n_children == 3);
     assert(BP_STATE(dn,0) == PT_AVAIL);
     assert(BP_STATE(dn,1) == PT_ON_DISK);
     assert(BP_STATE(dn,2) == PT_ON_DISK);
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_COMPRESSED);
     assert(BP_STATE(dn,1) == PT_ON_DISK);
     assert(BP_STATE(dn,2) == PT_ON_DISK);
@@ -248,9 +248,9 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
 }
 
 static void
-test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
+test_subset_read(int fd, FT_HANDLE UU(ft), FT ft_h) {
     int r;
-    brt_h->compare_fun = int64_key_cmp;    
+    ft_h->compare_fun = int64_key_cmp;    
     FT_CURSOR XMALLOC(cursor);
     FTNODE dn = NULL;
     FTNODE_DISK_DATA ndd = NULL;
@@ -271,7 +271,7 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
     toku_fill_dbt(&right, &right_key, sizeof(right_key));
     fill_bfe_for_subset_read(
         &bfe,
-        brt_h,
+        ft_h,
         NULL, 
         &left,
         &right,
@@ -292,11 +292,11 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
     assert(BP_STATE(dn,1) == PT_ON_DISK);
     assert(BP_STATE(dn,2) == PT_AVAIL);
     // need to call this twice because we had a subset read before, that touched the clock
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_ON_DISK);
     assert(BP_STATE(dn,1) == PT_ON_DISK);
     assert(BP_STATE(dn,2) == PT_AVAIL);
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_ON_DISK);
     assert(BP_STATE(dn,1) == PT_ON_DISK);
     assert(BP_STATE(dn,2) == PT_COMPRESSED);
@@ -317,11 +317,11 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
     assert(BP_STATE(dn,1) == PT_AVAIL);
     assert(BP_STATE(dn,2) == PT_AVAIL);
     // need to call this twice because we had a subset read before, that touched the clock
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_ON_DISK);
     assert(BP_STATE(dn,1) == PT_COMPRESSED);
     assert(BP_STATE(dn,2) == PT_AVAIL);
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_ON_DISK);
     assert(BP_STATE(dn,1) == PT_COMPRESSED);
     assert(BP_STATE(dn,2) == PT_COMPRESSED);
@@ -341,11 +341,11 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
     assert(BP_STATE(dn,1) == PT_AVAIL);
     assert(BP_STATE(dn,2) == PT_ON_DISK);
     // need to call this twice because we had a subset read before, that touched the clock
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_AVAIL);
     assert(BP_STATE(dn,1) == PT_COMPRESSED);
     assert(BP_STATE(dn,2) == PT_ON_DISK);
-    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(dn,0) == PT_COMPRESSED);
     assert(BP_STATE(dn,1) == PT_COMPRESSED);
     assert(BP_STATE(dn,2) == PT_ON_DISK);
@@ -412,9 +412,9 @@ test_prefetching(void) {
     xids_destroy(&xids_123);
     xids_destroy(&xids_234);
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -422,32 +422,32 @@ test_prefetching(void) {
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
     FTNODE_DISK_DATA ndd = NULL;
-    r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, brt->ft, false);
+    r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false);
     assert(r==0);
 
-    test_prefetch_read(fd, brt, brt_h);    
+    test_prefetch_read(fd, ft, ft_h);    
-    test_subset_read(fd, brt, brt_h);
+    test_subset_read(fd, ft, ft_h);
 
     toku_free(sn.childkeys[0].data);
     toku_free(sn.childkeys[1].data);
@@ -457,11 +457,11 @@ test_prefetching(void) {
     toku_free(sn.bp);
     toku_free(sn.childkeys);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
     toku_free(ndd);
 
     r = close(fd); assert(r != -1);

storage/tokudb/ft-index/ft/tests/ft-clock-test.cc

@@ -115,13 +115,18 @@ le_add_to_bn(bn_data* bn, uint32_t idx, const  char *key, int keylen, const char
 {
     LEAFENTRY r = NULL;
     uint32_t size_needed = LE_CLEAN_MEMSIZE(vallen);
+    void *maybe_free = nullptr;
     bn->get_space_for_insert(
         idx, 
         key,
         keylen,
         size_needed,
-        &r
+        &r,
+        &maybe_free
         );
+    if (maybe_free) {
+        toku_free(maybe_free);
+    }
     resource_assert(r);
     r->type = LE_CLEAN;
     r->u.clean.vallen = vallen;
@@ -139,11 +144,11 @@ le_malloc(bn_data* bn, uint32_t idx, const char *key, const char *val)
 
 
 static void
-test1(int fd, FT brt_h, FTNODE *dn) {
+test1(int fd, FT ft_h, FTNODE *dn) {
     int r;
     struct ftnode_fetch_extra bfe_all;
-    brt_h->compare_fun = string_key_cmp;
+    ft_h->compare_fun = string_key_cmp;
-    fill_bfe_for_full_read(&bfe_all, brt_h);
+    fill_bfe_for_full_read(&bfe_all, ft_h);
     FTNODE_DISK_DATA ndd = NULL;
     r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, &ndd, &bfe_all);
     bool is_leaf = ((*dn)->height == 0);
@@ -154,12 +159,12 @@ test1(int fd, FT brt_h, FTNODE *dn) {
     // should sweep and NOT get rid of anything
     PAIR_ATTR attr;
     memset(&attr,0,sizeof(attr));
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
     for (int i = 0; i < (*dn)->n_children; i++) {
         assert(BP_STATE(*dn,i) == PT_AVAIL);
     }
     // should sweep and get compress all
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
     for (int i = 0; i < (*dn)->n_children; i++) {
         if (!is_leaf) {
             assert(BP_STATE(*dn,i) == PT_COMPRESSED);
@@ -172,12 +177,12 @@ test1(int fd, FT brt_h, FTNODE *dn) {
     bool req = toku_ftnode_pf_req_callback(*dn, &bfe_all);
     assert(req);
     toku_ftnode_pf_callback(*dn, ndd, &bfe_all, fd, &size);
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
     for (int i = 0; i < (*dn)->n_children; i++) {
         assert(BP_STATE(*dn,i) == PT_AVAIL);
     }
     // should sweep and get compress all
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
     for (int i = 0; i < (*dn)->n_children; i++) {
         if (!is_leaf) {
             assert(BP_STATE(*dn,i) == PT_COMPRESSED);
@@ -190,15 +195,15 @@ test1(int fd, FT brt_h, FTNODE *dn) {
     req = toku_ftnode_pf_req_callback(*dn, &bfe_all);
     assert(req);
     toku_ftnode_pf_callback(*dn, ndd, &bfe_all, fd, &size);
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
     for (int i = 0; i < (*dn)->n_children; i++) {
         assert(BP_STATE(*dn,i) == PT_AVAIL);
     }
     (*dn)->dirty = 1;
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
     for (int i = 0; i < (*dn)->n_children; i++) {
         assert(BP_STATE(*dn,i) == PT_AVAIL);
     }
@@ -212,7 +217,7 @@ static int search_cmp(const struct ft_search& UU(so), const DBT* UU(key)) {
 }
 
 static void
-test2(int fd, FT brt_h, FTNODE *dn) {
+test2(int fd, FT ft_h, FTNODE *dn) {
     struct ftnode_fetch_extra bfe_subset;
     DBT left, right;
     DB dummy_db;
@@ -221,17 +226,11 @@ test2(int fd, FT brt_h, FTNODE *dn) {
     memset(&right, 0, sizeof(right));
     ft_search_t search_t;
     
-    brt_h->compare_fun = string_key_cmp;
+    ft_h->compare_fun = string_key_cmp;
     fill_bfe_for_subset_read(
         &bfe_subset,
-        brt_h,
+        ft_h,
-        ft_search_init(
+        ft_search_init(&search_t, search_cmp, FT_SEARCH_LEFT, nullptr, nullptr, nullptr),
-            &search_t, 
-            search_cmp, 
-            FT_SEARCH_LEFT, 
-            NULL, 
-            NULL
-            ),
         &left,
         &right,
         true,
@@ -252,11 +251,11 @@ test2(int fd, FT brt_h, FTNODE *dn) {
     assert(!BP_SHOULD_EVICT(*dn, 1));
     PAIR_ATTR attr;
     memset(&attr,0,sizeof(attr));
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(*dn, 0) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED);
     assert(BP_STATE(*dn, 1) == PT_AVAIL);
     assert(BP_SHOULD_EVICT(*dn, 1));
-    toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
+    toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
     assert(BP_STATE(*dn, 1) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED);
 
     bool req = toku_ftnode_pf_req_callback(*dn, &bfe_subset);
@@ -272,7 +271,7 @@ test2(int fd, FT brt_h, FTNODE *dn) {
 }
 
 static void
-test3_leaf(int fd, FT brt_h, FTNODE *dn) {
+test3_leaf(int fd, FT ft_h, FTNODE *dn) {
     struct ftnode_fetch_extra bfe_min;
     DBT left, right;
     DB dummy_db;
@@ -280,10 +279,10 @@ test3_leaf(int fd, FT brt_h, FTNODE *dn) {
     memset(&left, 0, sizeof(left));
     memset(&right, 0, sizeof(right));
     
-    brt_h->compare_fun = string_key_cmp;
+    ft_h->compare_fun = string_key_cmp;
     fill_bfe_for_min_read(
         &bfe_min,
-        brt_h
+        ft_h
         );
     FTNODE_DISK_DATA ndd = NULL;
     int r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, &ndd, &bfe_min);
@@ -347,9 +346,9 @@ test_serialize_nonleaf(void) {
     xids_destroy(&xids_123);
     xids_destroy(&xids_234);
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -357,33 +356,33 @@ test_serialize_nonleaf(void) {
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
     
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
     FTNODE_DISK_DATA ndd = NULL;
-    r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, brt->ft, false);
+    r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false);
     assert(r==0);
 
-    test1(fd, brt_h, &dn);
+    test1(fd, ft_h, &dn);
-    test2(fd, brt_h, &dn);
+    test2(fd, ft_h, &dn);
 
     toku_free(hello_string);
     destroy_nonleaf_childinfo(BNC(&sn, 0));
@@ -392,11 +391,11 @@ test_serialize_nonleaf(void) {
     toku_free(sn.childkeys);
     toku_free(ndd);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
 
     r = close(fd); assert(r != -1);
 }
@@ -431,9 +430,9 @@ test_serialize_leaf(void) {
     le_malloc(BLB_DATA(&sn, 0), 1, "b", "bval");
     le_malloc(BLB_DATA(&sn, 1), 0, "x", "xval");
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -441,33 +440,33 @@ test_serialize_leaf(void) {
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
     
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
     FTNODE_DISK_DATA ndd = NULL;
-    r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, brt->ft, false);
+    r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false);
     assert(r==0);
 
-    test1(fd, brt_h, &dn);
+    test1(fd, ft_h, &dn);
-    test3_leaf(fd, brt_h,&dn);
+    test3_leaf(fd, ft_h,&dn);
 
     for (int i = 0; i < sn.n_children-1; ++i) {
         toku_free(sn.childkeys[i].data);
@@ -478,11 +477,11 @@ test_serialize_leaf(void) {
     toku_free(sn.bp);
     toku_free(sn.childkeys);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
     toku_free(ndd);
     r = close(fd); assert(r != -1);
 }

storage/tokudb/ft-index/ft/tests/ft-serialize-benchmark.cc

@@ -105,13 +105,18 @@ le_add_to_bn(bn_data* bn, uint32_t idx, char *key, int keylen, char *val, int va
 {
     LEAFENTRY r = NULL;
     uint32_t size_needed = LE_CLEAN_MEMSIZE(vallen);
+    void *maybe_free = nullptr;
     bn->get_space_for_insert(
         idx, 
         key,
         keylen,
         size_needed,
-        &r
+        &r,
+        &maybe_free
         );
+    if (maybe_free) {
+        toku_free(maybe_free);
+    }
     resource_assert(r);
     r->type = LE_CLEAN;
     r->u.clean.vallen = vallen;
@@ -127,7 +132,7 @@ long_key_cmp(DB *UU(e), const DBT *a, const DBT *b)
 }
 
 static void
-test_serialize_leaf(int valsize, int nelts, double entropy) {
+test_serialize_leaf(int valsize, int nelts, double entropy, int ser_runs, int deser_runs) {
     //    struct ft_handle source_ft;
     struct ftnode *sn, *dn;
 
@@ -181,9 +186,9 @@ test_serialize_leaf(int valsize, int nelts, double entropy) {
         }
     }
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -191,70 +196,99 @@ test_serialize_leaf(int valsize, int nelts, double entropy) {
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
     
-    brt_h->compare_fun = long_key_cmp;
+    ft_h->compare_fun = long_key_cmp;
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
 
+    struct timeval total_start;
+    struct timeval total_end;
+    total_start.tv_sec = total_start.tv_usec = 0;
+    total_end.tv_sec = total_end.tv_usec = 0;
     struct timeval t[2];
-    gettimeofday(&t[0], NULL);
     FTNODE_DISK_DATA ndd = NULL;
-    r = toku_serialize_ftnode_to(fd, make_blocknum(20), sn, &ndd, true, brt->ft, false);
+    for (int i = 0; i < ser_runs; i++) {
-    assert(r==0);
+        gettimeofday(&t[0], NULL);
-    gettimeofday(&t[1], NULL);
+        ndd = NULL;
+        sn->dirty = 1;
+        r = toku_serialize_ftnode_to(fd, make_blocknum(20), sn, &ndd, true, ft->ft, false);
+        assert(r==0);
+        gettimeofday(&t[1], NULL);
+        total_start.tv_sec += t[0].tv_sec;
+        total_start.tv_usec += t[0].tv_usec;
+        total_end.tv_sec += t[1].tv_sec;
+        total_end.tv_usec += t[1].tv_usec;
+        toku_free(ndd);
+    }
     double dt;
-    dt = (t[1].tv_sec - t[0].tv_sec) + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC);
+    dt = (total_end.tv_sec - total_start.tv_sec) + ((total_end.tv_usec - total_start.tv_usec) / USECS_PER_SEC);
-    printf("serialize leaf:   %0.05lf\n", dt);
+    dt *= 1000;
+    dt /= ser_runs;
+    printf("serialize leaf(ms):   %0.05lf (average of %d runs)\n", dt, ser_runs);
+
+    //reset 
+    total_start.tv_sec = total_start.tv_usec = 0;
+    total_end.tv_sec = total_end.tv_usec = 0;
 
     struct ftnode_fetch_extra bfe;
-    fill_bfe_for_full_read(&bfe, brt_h);
+    for (int i = 0; i < deser_runs; i++) {
-    gettimeofday(&t[0], NULL);
+        fill_bfe_for_full_read(&bfe, ft_h);
-    FTNODE_DISK_DATA ndd2 = NULL;
+        gettimeofday(&t[0], NULL);
-    r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd2, &bfe);
+        FTNODE_DISK_DATA ndd2 = NULL;
-    assert(r==0);
+        r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd2, &bfe);
-    gettimeofday(&t[1], NULL);
+        assert(r==0);
-    dt = (t[1].tv_sec - t[0].tv_sec) + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC);
+        gettimeofday(&t[1], NULL);
-    printf("deserialize leaf: %0.05lf\n", dt);
+
-    printf("io time %lf decompress time %lf deserialize time %lf\n",
+        total_start.tv_sec += t[0].tv_sec;
-           tokutime_to_seconds(bfe.io_time),
+        total_start.tv_usec += t[0].tv_usec;
-           tokutime_to_seconds(bfe.decompress_time),
+        total_end.tv_sec += t[1].tv_sec;
-           tokutime_to_seconds(bfe.deserialize_time)
+        total_end.tv_usec += t[1].tv_usec;
+
+        toku_ftnode_free(&dn);
+        toku_free(ndd2);
+    }
+    dt = (total_end.tv_sec - total_start.tv_sec) + ((total_end.tv_usec - total_start.tv_usec) / USECS_PER_SEC);
+    dt *= 1000;
+    dt /= deser_runs;
+    printf("deserialize leaf(ms): %0.05lf (average of %d runs)\n", dt, deser_runs);
+    printf("io time(ms) %lf decompress time(ms) %lf deserialize time(ms) %lf (average of %d runs)\n",
+           tokutime_to_seconds(bfe.io_time)*1000,
+           tokutime_to_seconds(bfe.decompress_time)*1000,
+           tokutime_to_seconds(bfe.deserialize_time)*1000,
+           deser_runs
            );
 
-    toku_ftnode_free(&dn);
     toku_ftnode_free(&sn);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
-    toku_free(ndd);
-    toku_free(ndd2);
 
     r = close(fd); assert(r != -1);
 }
 
 static void
-test_serialize_nonleaf(int valsize, int nelts, double entropy) {
+test_serialize_nonleaf(int valsize, int nelts, double entropy, int ser_runs, int deser_runs) {
     //    struct ft_handle source_ft;
     struct ftnode sn, *dn;
 
@@ -312,9 +346,9 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
     xids_destroy(&xids_0);
     xids_destroy(&xids_123);
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -322,25 +356,25 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
     
-    brt_h->compare_fun = long_key_cmp;
+    ft_h->compare_fun = long_key_cmp;
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
@@ -348,26 +382,29 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
     struct timeval t[2];
     gettimeofday(&t[0], NULL);
     FTNODE_DISK_DATA ndd = NULL;
-    r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, brt->ft, false);
+    r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false);
     assert(r==0);
     gettimeofday(&t[1], NULL);
     double dt;
     dt = (t[1].tv_sec - t[0].tv_sec) + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC);
-    printf("serialize nonleaf:   %0.05lf\n", dt);
+    dt *= 1000;
+    printf("serialize nonleaf(ms):   %0.05lf (IGNORED RUNS=%d)\n", dt, ser_runs);
 
     struct ftnode_fetch_extra bfe;
-    fill_bfe_for_full_read(&bfe, brt_h);
+    fill_bfe_for_full_read(&bfe, ft_h);
     gettimeofday(&t[0], NULL);
     FTNODE_DISK_DATA ndd2 = NULL;
     r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd2, &bfe);
     assert(r==0);
     gettimeofday(&t[1], NULL);
     dt = (t[1].tv_sec - t[0].tv_sec) + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC);
-    printf("deserialize nonleaf: %0.05lf\n", dt);
+    dt *= 1000;
-    printf("io time %lf decompress time %lf deserialize time %lf\n",
+    printf("deserialize nonleaf(ms): %0.05lf (IGNORED RUNS=%d)\n", dt, deser_runs);
-           tokutime_to_seconds(bfe.io_time),
+    printf("io time(ms) %lf decompress time(ms) %lf deserialize time(ms) %lf (IGNORED RUNS=%d)\n",
-           tokutime_to_seconds(bfe.decompress_time),
+           tokutime_to_seconds(bfe.io_time)*1000,
-           tokutime_to_seconds(bfe.deserialize_time)
+           tokutime_to_seconds(bfe.decompress_time)*1000,
+           tokutime_to_seconds(bfe.deserialize_time)*1000,
+           deser_runs
            );
 
     toku_ftnode_free(&dn);
@@ -381,11 +418,11 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
     toku_free(sn.bp);
     toku_free(sn.childkeys);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
     toku_free(ndd);
     toku_free(ndd2);
 
@@ -394,19 +431,32 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
 
 int
 test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
-    long valsize, nelts;
+    const int DEFAULT_RUNS = 5;
+    long valsize, nelts, ser_runs = DEFAULT_RUNS, deser_runs = DEFAULT_RUNS;
     double entropy = 0.3;
 
-    if (argc != 3) {
+    if (argc != 3 && argc != 5) {
-        fprintf(stderr, "Usage: %s <valsize> <nelts>\n", argv[0]);
+        fprintf(stderr, "Usage: %s <valsize> <nelts> [<serialize_runs> <deserialize_runs>]\n", argv[0]);
+        fprintf(stderr, "Default (and min) runs is %d\n", DEFAULT_RUNS);
         return 2;
     }
     valsize = strtol(argv[1], NULL, 0);
     nelts = strtol(argv[2], NULL, 0);
+    if (argc == 5) {
+        ser_runs = strtol(argv[3], NULL, 0);
+        deser_runs = strtol(argv[4], NULL, 0);
+    }
+
+    if (ser_runs <= 0) {
+        ser_runs = DEFAULT_RUNS;
+    }
+    if (deser_runs <= 0) {
+        deser_runs = DEFAULT_RUNS;
+    }
 
     initialize_dummymsn();
-    test_serialize_leaf(valsize, nelts, entropy);
+    test_serialize_leaf(valsize, nelts, entropy, ser_runs, deser_runs);
-    test_serialize_nonleaf(valsize, nelts, entropy);
+    test_serialize_nonleaf(valsize, nelts, entropy, ser_runs, deser_runs);
 
     return 0;
 }

storage/tokudb/ft-index/ft/tests/ft-serialize-sub-block-test.cc

@@ -92,9 +92,9 @@ PATENT RIGHTS GRANT:
 #include "test.h"
 
 
-// create a brt and put n rows into it
+// create a ft and put n rows into it
-// write the brt to the file
+// write the ft to the file
-// verify the rows in the brt
+// verify the rows in the ft
 static void test_sub_block(int n) {
     if (verbose) printf("%s:%d %d\n", __FUNCTION__, __LINE__, n);
 
@@ -107,14 +107,14 @@ static void test_sub_block(int n) {
 
     int error;
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int i;
 
     unlink(fname);
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    error = toku_open_ft_handle(fname, true, &brt, nodesize, basementnodesize, compression_method, ct, null_txn, toku_builtin_compare_fun);
+    error = toku_open_ft_handle(fname, true, &ft, nodesize, basementnodesize, compression_method, ct, null_txn, toku_builtin_compare_fun);
     assert(error == 0);
 
     // insert keys 0, 1, 2, .. (n-1)
@@ -124,20 +124,20 @@ static void test_sub_block(int n) {
 	DBT key, val;
         toku_fill_dbt(&key, &k, sizeof k);
         toku_fill_dbt(&val, &v, sizeof v);
-        toku_ft_insert(brt, &key, &val, 0);
+        toku_ft_insert(ft, &key, &val, 0);
         assert(error == 0);
     }
 
     // write to the file
-    error = toku_close_ft_handle_nolsn(brt, 0);
+    error = toku_close_ft_handle_nolsn(ft, 0);
     assert(error == 0);
 
-    // verify the brt by walking a cursor through the rows
+    // verify the ft by walking a cursor through the rows
-    error = toku_open_ft_handle(fname, false, &brt, nodesize, basementnodesize, compression_method, ct, null_txn, toku_builtin_compare_fun);
+    error = toku_open_ft_handle(fname, false, &ft, nodesize, basementnodesize, compression_method, ct, null_txn, toku_builtin_compare_fun);
     assert(error == 0);
 
     FT_CURSOR cursor;
-    error = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    error = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(error == 0);
 
     for (i=0; ; i++) {
@@ -155,7 +155,7 @@ static void test_sub_block(int n) {
 
     toku_ft_cursor_close(cursor);
 
-    error = toku_close_ft_handle_nolsn(brt, 0);
+    error = toku_close_ft_handle_nolsn(ft, 0);
     assert(error == 0);
 
     toku_cachetable_close(&ct);

storage/tokudb/ft-index/ft/tests/ft-serialize-test.cc

@@ -98,57 +98,53 @@ PATENT RIGHTS GRANT:
 #endif
 
 static size_t
-calc_le_size(int keylen, int vallen) {
-    return LE_CLEAN_MEMSIZE(vallen) + keylen + sizeof(uint32_t);
-}
-
-static void
 le_add_to_bn(bn_data* bn, uint32_t idx, const  char *key, int keysize, const char *val, int valsize)
 {
     LEAFENTRY r = NULL;
     uint32_t size_needed = LE_CLEAN_MEMSIZE(valsize);
+    void *maybe_free = nullptr;
     bn->get_space_for_insert(
-        idx, 
+        idx,
         key,
         keysize,
         size_needed,
-        &r
+        &r,
+        &maybe_free
         );
+    if (maybe_free) {
+        toku_free(maybe_free);
+    }
     resource_assert(r);
     r->type = LE_CLEAN;
     r->u.clean.vallen = valsize;
     memcpy(r->u.clean.val, val, valsize);
+    return size_needed + keysize + sizeof(uint32_t);
 }
 
-static KLPAIR
+class test_key_le_pair {
-le_fastmalloc(struct mempool * mp, const char *key, int keylen, const char *val, int vallen)
+    public:
-{
+    uint32_t keylen;
-    KLPAIR kl;
+    char* keyp;
-    size_t le_size = calc_le_size(keylen, vallen);
+    LEAFENTRY le;
-    CAST_FROM_VOIDP(kl, toku_mempool_malloc(mp, le_size, 1));
-    resource_assert(kl);
-    kl->keylen = keylen;
-    memcpy(kl->key_le, key, keylen);
-    LEAFENTRY le = get_le_from_klpair(kl);
-    le->type = LE_CLEAN;
-    le->u.clean.vallen = vallen;
-    memcpy(le->u.clean.val, val, vallen);
-    return kl;
-}
 
-static KLPAIR
+    test_key_le_pair() : keylen(), keyp(), le() {}
-le_malloc(struct mempool * mp, const char *key, const char *val)
+    void init(const char *_keyp, const char *_val) {
-{
+        init(_keyp, strlen(_keyp) + 1, _val, strlen(_val) + 1);
-    int keylen = strlen(key) + 1;
+    }
-    int vallen = strlen(val) + 1;
+    void init(const char * _keyp, uint32_t _keylen, const char*_val, uint32_t _vallen) {
-    return le_fastmalloc(mp, key, keylen, val, vallen);
+        keylen = _keylen;
-}
 
-struct check_leafentries_struct {
+        CAST_FROM_VOIDP(le, toku_malloc(LE_CLEAN_MEMSIZE(_vallen)));
-    int nelts;
+        le->type = LE_CLEAN;
-    LEAFENTRY *elts;
+        le->u.clean.vallen = _vallen;
-    int i;
+        memcpy(le->u.clean.val, _val, _vallen);
-    int (*cmp)(OMTVALUE, void *);
+
+        CAST_FROM_VOIDP(keyp, toku_xmemdup(_keyp, keylen));
+    }
+    ~test_key_le_pair() {
+        toku_free(le);
+        toku_free(keyp);
+    }
 };
 
 enum ftnode_verify_type {
@@ -166,18 +162,18 @@ string_key_cmp(DB *UU(e), const DBT *a, const DBT *b)
 }
 
 static void
-setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_DATA* ndd) {
+setup_dn(enum ftnode_verify_type bft, int fd, FT ft_h, FTNODE *dn, FTNODE_DISK_DATA* ndd) {
     int r;
-    brt_h->compare_fun = string_key_cmp;
+    ft_h->compare_fun = string_key_cmp;
     if (bft == read_all) {
         struct ftnode_fetch_extra bfe;
-        fill_bfe_for_full_read(&bfe, brt_h);
+        fill_bfe_for_full_read(&bfe, ft_h);
         r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, ndd, &bfe);
         assert(r==0);
     }
     else if (bft == read_compressed || bft == read_none) {
         struct ftnode_fetch_extra bfe;
-        fill_bfe_for_min_read(&bfe, brt_h);
+        fill_bfe_for_min_read(&bfe, ft_h);
         r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, ndd, &bfe);
         assert(r==0);
         // assert all bp's are compressed or on disk.
@@ -187,7 +183,7 @@ setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_
         // if read_none, get rid of the compressed bp's
         if (bft == read_none) {
             if ((*dn)->height == 0) {
-                toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+                toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
                 // assert all bp's are on disk
                 for (int i = 0; i < (*dn)->n_children; i++) {
                     if ((*dn)->height == 0) {
@@ -204,7 +200,7 @@ setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_
                 // that it is available
                 // then run partial eviction to get it compressed
                 PAIR_ATTR attr;
-                fill_bfe_for_full_read(&bfe, brt_h);
+                fill_bfe_for_full_read(&bfe, ft_h);
                 assert(toku_ftnode_pf_req_callback(*dn, &bfe));
                 r = toku_ftnode_pf_callback(*dn, *ndd, &bfe, fd, &attr);
                 assert(r==0);
@@ -212,21 +208,21 @@ setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_
                 for (int i = 0; i < (*dn)->n_children; i++) {
                     assert(BP_STATE(*dn,i) == PT_AVAIL);
                 }
-                toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+                toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
                 for (int i = 0; i < (*dn)->n_children; i++) {
                     // assert all bp's are still available, because we touched the clock
                     assert(BP_STATE(*dn,i) == PT_AVAIL);
                     // now assert all should be evicted
                     assert(BP_SHOULD_EVICT(*dn, i));
                 }
-                toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
+                toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
                 for (int i = 0; i < (*dn)->n_children; i++) {
                     assert(BP_STATE(*dn,i) == PT_COMPRESSED);
                 }
             }
         }
         // now decompress them
-        fill_bfe_for_full_read(&bfe, brt_h);
+        fill_bfe_for_full_read(&bfe, ft_h);
         assert(toku_ftnode_pf_req_callback(*dn, &bfe));
         PAIR_ATTR attr;
         r = toku_ftnode_pf_callback(*dn, *ndd, &bfe, fd, &attr);
@@ -243,20 +239,20 @@ setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_
     }
 }
 
-static void write_sn_to_disk(int fd, FT_HANDLE brt, FTNODE sn, FTNODE_DISK_DATA* src_ndd, bool do_clone) {
+static void write_sn_to_disk(int fd, FT_HANDLE ft, FTNODE sn, FTNODE_DISK_DATA* src_ndd, bool do_clone) {
     int r;
     if (do_clone) {
         void* cloned_node_v = NULL;
         PAIR_ATTR attr;
         long clone_size;
-        toku_ftnode_clone_callback(sn, &cloned_node_v, &clone_size, &attr, false, brt->ft);
+        toku_ftnode_clone_callback(sn, &cloned_node_v, &clone_size, &attr, false, ft->ft);
         FTNODE CAST_FROM_VOIDP(cloned_node, cloned_node_v);
-        r = toku_serialize_ftnode_to(fd, make_blocknum(20), cloned_node, src_ndd, false, brt->ft, false);
+        r = toku_serialize_ftnode_to(fd, make_blocknum(20), cloned_node, src_ndd, false, ft->ft, false);
         assert(r==0);        
         toku_ftnode_free(&cloned_node);
     }
     else {
-        r = toku_serialize_ftnode_to(fd, make_blocknum(20), sn, src_ndd, true, brt->ft, false);
+        r = toku_serialize_ftnode_to(fd, make_blocknum(20), sn, src_ndd, true, ft->ft, false);
         assert(r==0);
     }
 }
@@ -290,16 +286,15 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) {
     BP_STATE(&sn,1) = PT_AVAIL;
     set_BLB(&sn, 0, toku_create_empty_bn());
     set_BLB(&sn, 1, toku_create_empty_bn());
-    KLPAIR elts[3];
     le_add_to_bn(BLB_DATA(&sn, 0), 0, "a", 2, "aval", 5);
     le_add_to_bn(BLB_DATA(&sn, 0), 1, "b", 2, "bval", 5);
     le_add_to_bn(BLB_DATA(&sn, 1), 0, "x", 2, "xval", 5);
     BLB_MAX_MSN_APPLIED(&sn, 0) = ((MSN) { MIN_MSN.msn + 73 });
     BLB_MAX_MSN_APPLIED(&sn, 1) = POSTSERIALIZE_MSN_ON_DISK;
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -307,33 +302,33 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) {
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
 
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
     FTNODE_DISK_DATA src_ndd = NULL;
     FTNODE_DISK_DATA dest_ndd = NULL;
 
-    write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
+    write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
 
-    setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
+    setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
 
     assert(dn->thisnodename.b==20);
 
@@ -346,11 +341,10 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) {
     {
         // Man, this is way too ugly.  This entire test suite needs to be refactored.
         // Create a dummy mempool and put the leaves there.  Ugh.
-        struct mempool dummy_mp;
+        test_key_le_pair elts[3];
-        toku_mempool_construct(&dummy_mp, 1024);
+        elts[0].init("a", "aval");
-        elts[0] = le_malloc(&dummy_mp, "a", "aval");
+        elts[1].init("b", "bval");
-        elts[1] = le_malloc(&dummy_mp, "b", "bval");
+        elts[2].init("x", "xval");
-        elts[2] = le_malloc(&dummy_mp, "x", "xval");
         const uint32_t npartitions = dn->n_children;
         assert(dn->totalchildkeylens==(2*(npartitions-1)));
         uint32_t last_i = 0;
@@ -361,22 +355,21 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) {
             if (bn > 0) {
                 assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size);
             }
-            for (uint32_t i = 0; i < BLB_DATA(dn, bn)->omt_size(); i++) {
+            for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) {
                 LEAFENTRY curr_le;
                 uint32_t curr_keylen;
                 void* curr_key;
                 BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key);
-                assert(leafentry_memsize(curr_le) == leafentry_memsize(get_le_from_klpair(elts[last_i])));
+                assert(leafentry_memsize(curr_le) == leafentry_memsize(elts[last_i].le));
-                assert(memcmp(curr_le, get_le_from_klpair(elts[last_i]), leafentry_memsize(curr_le)) == 0);
+                assert(memcmp(curr_le, elts[last_i].le, leafentry_memsize(curr_le)) == 0);
                 if (bn < npartitions-1) {
-                    assert(strcmp((char*)dn->childkeys[bn].data, (char*)(elts[last_i]->key_le)) <= 0);
+                    assert(strcmp((char*)dn->childkeys[bn].data, elts[last_i].keyp) <= 0);
                 }
                 // TODO for later, get a key comparison here as well
                 last_i++;
             }
 
         }
-        toku_mempool_destroy(&dummy_mp);
         assert(last_i == 3);
     }
     toku_ftnode_free(&dn);
@@ -390,11 +383,11 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) {
     toku_free(sn.bp);
     toku_free(sn.childkeys);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
     toku_free(src_ndd);
     toku_free(dest_ndd);
 
@@ -436,14 +429,14 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone
         if (i < nrows-1) {
             uint32_t keylen;
             void* curr_key;
-            BLB_DATA(&sn, i)->fetch_le_key_and_len(0, &keylen, &curr_key);
+            BLB_DATA(&sn, i)->fetch_key_and_len(0, &keylen, &curr_key);
             toku_memdup_dbt(&sn.childkeys[i], curr_key, keylen);
         }
     }
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -451,32 +444,32 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
     FTNODE_DISK_DATA src_ndd = NULL;
     FTNODE_DISK_DATA dest_ndd = NULL;
 
-    write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
+    write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
 
-    setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
+    setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
     
     assert(dn->thisnodename.b==20);
 
@@ -485,18 +478,14 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone
     {
         // Man, this is way too ugly.  This entire test suite needs to be refactored.
         // Create a dummy mempool and put the leaves there.  Ugh.
-        struct mempool dummy_mp;
+        test_key_le_pair *les = new test_key_le_pair[nrows];
-        size_t le_size = calc_le_size(keylens, vallens);
-        size_t mpsize = nrows * le_size;
-        toku_mempool_construct(&dummy_mp, mpsize);
-        KLPAIR les[nrows];
         {
             char key[keylens], val[vallens];
             key[keylens-1] = '\0';
             for (uint32_t i = 0; i < nrows; ++i) {
                 char c = 'a' + i;
                 memset(key, c, keylens-1);
-                les[i] = le_fastmalloc(&dummy_mp, (char *) &key, sizeof(key), (char *) &val, sizeof(val));
+                les[i].init((char *) &key, sizeof(key), (char *) &val, sizeof(val));
             }
         }
         const uint32_t npartitions = dn->n_children;
@@ -508,23 +497,23 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone
             if (bn > 0) {
                 assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size);
             }
-            assert(BLB_DATA(dn, bn)->omt_size() > 0);
+            assert(BLB_DATA(dn, bn)->num_klpairs() > 0);
-            for (uint32_t i = 0; i < BLB_DATA(dn, bn)->omt_size(); i++) {
+            for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) {
                 LEAFENTRY curr_le;
                 uint32_t curr_keylen;
                 void* curr_key;
                 BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key);
-                assert(leafentry_memsize(curr_le) == leafentry_memsize(get_le_from_klpair(les[last_i])));
+                assert(leafentry_memsize(curr_le) == leafentry_memsize(les[last_i].le));
-                assert(memcmp(curr_le, get_le_from_klpair(les[last_i]), leafentry_memsize(curr_le)) == 0);
+                assert(memcmp(curr_le, les[last_i].le, leafentry_memsize(curr_le)) == 0);
                 if (bn < npartitions-1) {
-                    assert(strcmp((char*)dn->childkeys[bn].data, (char*)(les[last_i]->key_le)) <= 0);
+                    assert(strcmp((char*)dn->childkeys[bn].data, les[last_i].keyp) <= 0);
                 }
                 // TODO for later, get a key comparison here as well
                 last_i++;
             }
         }
-        toku_mempool_destroy(&dummy_mp);
         assert(last_i == nrows);
+        delete[] les;
     }
 
     toku_ftnode_free(&dn);
@@ -537,11 +526,11 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone
     }
     toku_free(sn.bp);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
     toku_free(src_ndd);
     toku_free(dest_ndd);
 
@@ -552,7 +541,6 @@ static void
 test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
     int r;
     struct ftnode sn, *dn;
-    const int keylens = sizeof(int), vallens = sizeof(int);
     const uint32_t nrows = 196*1024;
     int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0);
 
@@ -566,22 +554,23 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
     sn.dirty = 1;
     sn.oldest_referenced_xid_known = TXNID_NONE;
 
-    MALLOC_N(sn.n_children, sn.bp);
+    XMALLOC_N(sn.n_children, sn.bp);
-    MALLOC_N(sn.n_children-1, sn.childkeys);
+    XMALLOC_N(sn.n_children-1, sn.childkeys);
     sn.totalchildkeylens = (sn.n_children-1)*sizeof(int);
     for (int i = 0; i < sn.n_children; ++i) {
         BP_STATE(&sn,i) = PT_AVAIL;
         set_BLB(&sn, i, toku_create_empty_bn()); 
     }
+    size_t total_size = 0;
     for (uint32_t i = 0; i < nrows; ++i) {
         uint32_t key = i;
         uint32_t val = i;
-        le_add_to_bn(BLB_DATA(&sn, 0), i, (char *) &key, sizeof(key), (char *) &val, sizeof(val));
+        total_size += le_add_to_bn(BLB_DATA(&sn, 0), i, (char *) &key, sizeof(key), (char *) &val, sizeof(val));
     }
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -589,33 +578,33 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
     
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
 
     FTNODE_DISK_DATA src_ndd = NULL;
     FTNODE_DISK_DATA dest_ndd = NULL;
-    write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
+    write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
 
-    setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
+    setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
 
     assert(dn->thisnodename.b==20);
 
@@ -624,15 +613,11 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
     {
         // Man, this is way too ugly.  This entire test suite needs to be refactored.
         // Create a dummy mempool and put the leaves there.  Ugh.
-        struct mempool dummy_mp;
+        test_key_le_pair *les = new test_key_le_pair[nrows];
-        size_t le_size = calc_le_size(keylens, vallens);
-        size_t mpsize = nrows * le_size;
-        toku_mempool_construct(&dummy_mp, mpsize);
-        KLPAIR les[nrows];
         {
             int key = 0, val = 0;
             for (uint32_t i = 0; i < nrows; ++i, key++, val++) {
-                les[i] = le_fastmalloc(&dummy_mp, (char *) &key, sizeof(key), (char *) &val, sizeof(val));
+                les[i].init((char *) &key, sizeof(key), (char *) &val, sizeof(val));
             }
         }
         const uint32_t npartitions = dn->n_children;
@@ -644,17 +629,17 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
             if (bn > 0) {
                 assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size);
             }
-            assert(BLB_DATA(dn, bn)->omt_size() > 0);
+            assert(BLB_DATA(dn, bn)->num_klpairs() > 0);
-            for (uint32_t i = 0; i < BLB_DATA(dn, bn)->omt_size(); i++) {
+            for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) {
                 LEAFENTRY curr_le;
                 uint32_t curr_keylen;
                 void* curr_key;
                 BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key);
-                assert(leafentry_memsize(curr_le) == leafentry_memsize(get_le_from_klpair(les[last_i])));
+                assert(leafentry_memsize(curr_le) == leafentry_memsize(les[last_i].le));
-                assert(memcmp(curr_le, get_le_from_klpair(les[last_i]), leafentry_memsize(curr_le)) == 0);
+                assert(memcmp(curr_le, les[last_i].le, leafentry_memsize(curr_le)) == 0);
                 if (bn < npartitions-1) {
                     uint32_t *CAST_FROM_VOIDP(pivot, dn->childkeys[bn].data);
-                    void* tmp = les[last_i]->key_le;
+                    void* tmp = les[last_i].keyp;
                     uint32_t *CAST_FROM_VOIDP(item, tmp);
                     assert(*pivot >= *item);
                 }
@@ -664,8 +649,8 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
             // don't check soft_copy_is_up_to_date or seqinsert
             assert(BLB_DATA(dn, bn)->get_disk_size() < 128*1024);  // BN_MAX_SIZE, apt to change
         }
-        toku_mempool_destroy(&dummy_mp);
         assert(last_i == nrows);
+        delete[] les;
     }
 
     toku_ftnode_free(&dn);
@@ -678,11 +663,11 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
     toku_free(sn.bp);
     toku_free(sn.childkeys);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
     toku_free(src_ndd);
     toku_free(dest_ndd);
 
@@ -727,9 +712,9 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
         le_add_to_bn(BLB_DATA(&sn, 0), i,key, 8, val, val_size);
     }
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -737,33 +722,33 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
     
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
 
     FTNODE_DISK_DATA src_ndd = NULL;
     FTNODE_DISK_DATA dest_ndd = NULL;
-    write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
+    write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
 
-    setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
+    setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
 
     assert(dn->thisnodename.b==20);
 
@@ -772,11 +757,7 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
     {
         // Man, this is way too ugly.  This entire test suite needs to be refactored.
         // Create a dummy mempool and put the leaves there.  Ugh.
-        struct mempool dummy_mp;
+        test_key_le_pair *les = new test_key_le_pair[nrows];
-        size_t le_size = calc_le_size(key_size, val_size);
-        size_t mpsize = nrows * le_size;
-        toku_mempool_construct(&dummy_mp, mpsize);
-        KLPAIR les[nrows];
         {
             char key[key_size], val[val_size];
             key[key_size-1] = '\0';
@@ -785,7 +766,7 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
                 char c = 'a' + i;
                 memset(key, c, key_size-1);
                 memset(val, c, val_size-1);
-                les[i] = le_fastmalloc(&dummy_mp, key, key_size, val, val_size);
+                les[i].init(key, key_size, val, val_size);
             }
         }
         const uint32_t npartitions = dn->n_children;
@@ -798,24 +779,24 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
             if (bn > 0) {
                 assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size);
             }
-            assert(BLB_DATA(dn, bn)->omt_size() > 0);
+            assert(BLB_DATA(dn, bn)->num_klpairs() > 0);
-            for (uint32_t i = 0; i < BLB_DATA(dn, bn)->omt_size(); i++) {
+            for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) {
                 LEAFENTRY curr_le;
                 uint32_t curr_keylen;
                 void* curr_key;
                 BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key);
-                assert(leafentry_memsize(curr_le) == leafentry_memsize(get_le_from_klpair(les[last_i])));
+                assert(leafentry_memsize(curr_le) == leafentry_memsize(les[last_i].le));
-                assert(memcmp(curr_le, get_le_from_klpair(les[last_i]), leafentry_memsize(curr_le)) == 0);
+                assert(memcmp(curr_le, les[last_i].le, leafentry_memsize(curr_le)) == 0);
                 if (bn < npartitions-1) {
-                    assert(strcmp((char*)dn->childkeys[bn].data, (char*)(les[last_i]->key_le)) <= 0);
+                    assert(strcmp((char*)dn->childkeys[bn].data, (char*)(les[last_i].keyp)) <= 0);
                 }
                 // TODO for later, get a key comparison here as well
                 last_i++;
             }
             // don't check soft_copy_is_up_to_date or seqinsert
         }
-        toku_mempool_destroy(&dummy_mp);
         assert(last_i == 7);
+        delete[] les;
     }
 
     toku_ftnode_free(&dn);
@@ -828,11 +809,11 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
     toku_free(sn.bp);
     toku_free(sn.childkeys);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
     toku_free(src_ndd);
     toku_free(dest_ndd);
 
@@ -871,14 +852,13 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool
         set_BLB(&sn, i, toku_create_empty_bn());
         BLB_SEQINSERT(&sn, i) = 0;
     }
-    KLPAIR elts[3];
     le_add_to_bn(BLB_DATA(&sn, 1), 0, "a", 2, "aval", 5);
     le_add_to_bn(BLB_DATA(&sn, 3), 0, "b", 2, "bval", 5);
     le_add_to_bn(BLB_DATA(&sn, 5), 0, "x", 2, "xval", 5);
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -886,32 +866,32 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
     
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
     FTNODE_DISK_DATA src_ndd = NULL;
     FTNODE_DISK_DATA dest_ndd = NULL;
-    write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
+    write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
 
-    setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
+    setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
 
     assert(dn->thisnodename.b==20);
 
@@ -921,13 +901,13 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool
     assert(dn->height == 0);
     assert(dn->n_children>0);
     {
+        test_key_le_pair elts[3];
+
         // Man, this is way too ugly.  This entire test suite needs to be refactored.
         // Create a dummy mempool and put the leaves there.  Ugh.
-        struct mempool dummy_mp;
+        elts[0].init("a", "aval");
-        toku_mempool_construct(&dummy_mp, 1024);
+        elts[1].init("b", "bval");
-        elts[0] = le_malloc(&dummy_mp, "a", "aval");
+        elts[2].init("x", "xval");
-        elts[1] = le_malloc(&dummy_mp, "b", "bval");
-        elts[2] = le_malloc(&dummy_mp, "x", "xval");
         const uint32_t npartitions = dn->n_children;
         assert(dn->totalchildkeylens==(2*(npartitions-1)));
         uint32_t last_i = 0;
@@ -937,22 +917,21 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool
             if (bn > 0) {
                 assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size);
             }
-            for (uint32_t i = 0; i < BLB_DATA(dn, bn)->omt_size(); i++) {
+            for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) {
                 LEAFENTRY curr_le;
                 uint32_t curr_keylen;
                 void* curr_key;
                 BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key);
-                assert(leafentry_memsize(curr_le) == leafentry_memsize(get_le_from_klpair(elts[last_i])));
+                assert(leafentry_memsize(curr_le) == leafentry_memsize(elts[last_i].le));
-                assert(memcmp(curr_le, get_le_from_klpair(elts[last_i]), leafentry_memsize(curr_le)) == 0);
+                assert(memcmp(curr_le, elts[last_i].le, leafentry_memsize(curr_le)) == 0);
                 if (bn < npartitions-1) {
-                    assert(strcmp((char*)dn->childkeys[bn].data, (char*)(elts[last_i]->key_le)) <= 0);
+                    assert(strcmp((char*)dn->childkeys[bn].data, (char*)(elts[last_i].keyp)) <= 0);
                 }
                 // TODO for later, get a key comparison here as well
                 last_i++;
             }
 
         }
-        toku_mempool_destroy(&dummy_mp);
         assert(last_i == 3);
     }
     toku_ftnode_free(&dn);
@@ -966,11 +945,11 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool
     toku_free(sn.bp);
     toku_free(sn.childkeys);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
     toku_free(src_ndd);
     toku_free(dest_ndd);
 
@@ -1005,9 +984,9 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b
         set_BLB(&sn, i, toku_create_empty_bn());
     }
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -1015,33 +994,33 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
     
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
 
     FTNODE_DISK_DATA src_ndd = NULL;
     FTNODE_DISK_DATA dest_ndd = NULL;
-    write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
+    write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
 
-    setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
+    setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
 
     assert(dn->thisnodename.b==20);
 
@@ -1059,7 +1038,7 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b
             if (i > 0) {
                 assert(dest_ndd[i].start >= dest_ndd[i-1].start + dest_ndd[i-1].size);
             }
-            assert(BLB_DATA(dn, i)->omt_size() == 0);
+            assert(BLB_DATA(dn, i)->num_klpairs() == 0);
         }
     }
     toku_ftnode_free(&dn);
@@ -1073,11 +1052,11 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b
     toku_free(sn.bp);
     toku_free(sn.childkeys);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
     toku_free(src_ndd);
     toku_free(dest_ndd);
 
@@ -1131,9 +1110,9 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) {
     xids_destroy(&xids_123);
     xids_destroy(&xids_234);
 
-    FT_HANDLE XMALLOC(brt);
+    FT_HANDLE XMALLOC(ft);
-    FT XCALLOC(brt_h);
+    FT XCALLOC(ft_h);
-    toku_ft_init(brt_h,
+    toku_ft_init(ft_h,
                  make_blocknum(0),
                  ZERO_LSN,
                  TXNID_NONE,
@@ -1141,32 +1120,32 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) {
                  128*1024,
                  TOKU_DEFAULT_COMPRESSION_METHOD,
                  16);
-    brt->ft = brt_h;
+    ft->ft = ft_h;
     
-    toku_blocktable_create_new(&brt_h->blocktable);
+    toku_blocktable_create_new(&ft_h->blocktable);
     { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
     //Want to use block #20
     BLOCKNUM b = make_blocknum(0);
     while (b.b < 20) {
-        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+        toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
     }
     assert(b.b == 20);
 
     {
         DISKOFF offset;
         DISKOFF size;
-        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
+        toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
         assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
 
-        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
         assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
         assert(size   == 100);
     }
     FTNODE_DISK_DATA src_ndd = NULL;
     FTNODE_DISK_DATA dest_ndd = NULL;
-    write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
+    write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
 
-    setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
+    setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
 
     assert(dn->thisnodename.b==20);
 
@@ -1197,11 +1176,11 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) {
     toku_free(sn.bp);
     toku_free(sn.childkeys);
 
-    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+    toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
-    toku_blocktable_destroy(&brt_h->blocktable);
+    toku_blocktable_destroy(&ft_h->blocktable);
-    toku_free(brt_h->h);
+    toku_free(ft_h->h);
-    toku_free(brt_h);
+    toku_free(ft_h);
-    toku_free(brt);
+    toku_free(ft);
     toku_free(src_ndd);
     toku_free(dest_ndd);
 

storage/tokudb/ft-index/ft/tests/ft-test-cursor-2.cc

@@ -113,14 +113,14 @@ static void test_multiple_ft_cursor_dbts(int n) {
 
     int r;
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     FT_CURSOR cursors[n];
 
     unlink(fname);
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
     assert(r==0);
 
     int i;
@@ -129,14 +129,14 @@ static void test_multiple_ft_cursor_dbts(int n) {
 	char key[10],val[10];
 	snprintf(key, sizeof key, "k%04d", i);
 	snprintf(val, sizeof val, "v%04d", i);
-	toku_ft_insert(brt,
+	toku_ft_insert(ft,
                        toku_fill_dbt(&kbt, key, 1+strlen(key)),
                        toku_fill_dbt(&vbt, val, 1+strlen(val)),
                        0);
     }
 
     for (i=0; i<n; i++) {
-        r = toku_ft_cursor(brt, &cursors[i], NULL, false, false);
+        r = toku_ft_cursor(ft, &cursors[i], NULL, false, false);
         assert(r == 0);
     }
 
@@ -166,7 +166,7 @@ static void test_multiple_ft_cursor_dbts(int n) {
 	toku_free(ptrs[i]);
     }
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);

storage/tokudb/ft-index/ft/tests/ft-test-cursor.cc

@@ -102,11 +102,11 @@ static int test_ft_cursor_keycompare(DB *desc __attribute__((unused)), const DBT
     return toku_keycompare(a->data, a->size, b->data, b->size);
 }
 
-static void assert_cursor_notfound(FT_HANDLE brt, int position) {
+static void assert_cursor_notfound(FT_HANDLE ft, int position) {
     FT_CURSOR cursor=0;
     int r;
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(r==0);
 
     struct check_pair pair = {0,0,0,0,0};
@@ -117,11 +117,11 @@ static void assert_cursor_notfound(FT_HANDLE brt, int position) {
     toku_ft_cursor_close(cursor);
 }
 
-static void assert_cursor_value(FT_HANDLE brt, int position, long long value) {
+static void assert_cursor_value(FT_HANDLE ft, int position, long long value) {
     FT_CURSOR cursor=0;
     int r;
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(r==0);
 
     if (test_cursor_debug && verbose) printf("key: ");
@@ -133,11 +133,11 @@ static void assert_cursor_value(FT_HANDLE brt, int position, long long value) {
     toku_ft_cursor_close(cursor);
 }
 
-static void assert_cursor_first_last(FT_HANDLE brt, long long firstv, long long lastv) {
+static void assert_cursor_first_last(FT_HANDLE ft, long long firstv, long long lastv) {
     FT_CURSOR cursor=0;
     int r;
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(r==0);
 
     if (test_cursor_debug && verbose) printf("first key: ");
@@ -162,7 +162,7 @@ static void assert_cursor_first_last(FT_HANDLE brt, long long firstv, long long
 
 static void test_ft_cursor_first(int n) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int r;
     int i;
 
@@ -172,7 +172,7 @@ static void test_ft_cursor_first(int n) {
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     /* insert a bunch of kv pairs */
@@ -184,15 +184,15 @@ static void test_ft_cursor_first(int n) {
         toku_fill_dbt(&kbt, key, strlen(key)+1);
         v = i;
         toku_fill_dbt(&vbt, &v, sizeof v);
-        toku_ft_insert(brt, &kbt, &vbt, 0);
+        toku_ft_insert(ft, &kbt, &vbt, 0);
     }
 
     if (n == 0)
-        assert_cursor_notfound(brt, DB_FIRST);
+        assert_cursor_notfound(ft, DB_FIRST);
     else
-        assert_cursor_value(brt, DB_FIRST, 0);
+        assert_cursor_value(ft, DB_FIRST, 0);
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
@@ -200,7 +200,7 @@ static void test_ft_cursor_first(int n) {
 
 static void test_ft_cursor_last(int n) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int r;
     int i;
 
@@ -210,7 +210,7 @@ static void test_ft_cursor_last(int n) {
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     /* insert keys 0, 1, .. (n-1) */
@@ -222,16 +222,16 @@ static void test_ft_cursor_last(int n) {
         toku_fill_dbt(&kbt, key, strlen(key)+1);
         v = i;
         toku_fill_dbt(&vbt, &v, sizeof v);
-        toku_ft_insert(brt, &kbt, &vbt, 0);
+        toku_ft_insert(ft, &kbt, &vbt, 0);
         assert(r==0);
     }
 
     if (n == 0)
-        assert_cursor_notfound(brt, DB_LAST);
+        assert_cursor_notfound(ft, DB_LAST);
     else
-        assert_cursor_value(brt, DB_LAST, n-1);
+        assert_cursor_value(ft, DB_LAST, n-1);
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
@@ -239,7 +239,7 @@ static void test_ft_cursor_last(int n) {
 
 static void test_ft_cursor_first_last(int n) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int r;
     int i;
 
@@ -249,7 +249,7 @@ static void test_ft_cursor_first_last(int n) {
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     /* insert a bunch of kv pairs */
@@ -262,16 +262,16 @@ static void test_ft_cursor_first_last(int n) {
         v = i;
         toku_fill_dbt(&vbt, &v, sizeof v);
 
-        toku_ft_insert(brt, &kbt, &vbt, 0);
+        toku_ft_insert(ft, &kbt, &vbt, 0);
     }
 
     if (n == 0) {
-        assert_cursor_notfound(brt, DB_FIRST);
+        assert_cursor_notfound(ft, DB_FIRST);
-        assert_cursor_notfound(brt, DB_LAST);
+        assert_cursor_notfound(ft, DB_LAST);
     } else
-        assert_cursor_first_last(brt, 0, n-1);
+        assert_cursor_first_last(ft, 0, n-1);
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
@@ -281,7 +281,7 @@ static void test_ft_cursor_first_last(int n) {
 
 static void test_ft_cursor_rfirst(int n) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int r;
     int i;
 
@@ -291,7 +291,7 @@ static void test_ft_cursor_rfirst(int n) {
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     /* insert keys n-1, n-2, ... , 0 */
@@ -304,26 +304,26 @@ static void test_ft_cursor_rfirst(int n) {
         toku_fill_dbt(&kbt, key, strlen(key)+1);
         v = i;
         toku_fill_dbt(&vbt, &v, sizeof v);
-        toku_ft_insert(brt, &kbt, &vbt, 0);
+        toku_ft_insert(ft, &kbt, &vbt, 0);
     }
 
     if (n == 0)
-        assert_cursor_notfound(brt, DB_FIRST);
+        assert_cursor_notfound(ft, DB_FIRST);
     else
-        assert_cursor_value(brt, DB_FIRST, 0);
+        assert_cursor_value(ft, DB_FIRST, 0);
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
 }
 
-static void assert_cursor_walk(FT_HANDLE brt, int n) {
+static void assert_cursor_walk(FT_HANDLE ft, int n) {
     FT_CURSOR cursor=0;
     int i;
     int r;
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(r==0);
 
     if (test_cursor_debug && verbose) printf("key: ");
@@ -345,7 +345,7 @@ static void assert_cursor_walk(FT_HANDLE brt, int n) {
 
 static void test_ft_cursor_walk(int n) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int r;
     int i;
 
@@ -355,7 +355,7 @@ static void test_ft_cursor_walk(int n) {
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     /* insert a bunch of kv pairs */
@@ -367,25 +367,25 @@ static void test_ft_cursor_walk(int n) {
         toku_fill_dbt(&kbt, key, strlen(key)+1);
         v = i;
         toku_fill_dbt(&vbt, &v, sizeof v);
-        toku_ft_insert(brt, &kbt, &vbt, 0);
+        toku_ft_insert(ft, &kbt, &vbt, 0);
     }
 
     /* walk the tree */
-    assert_cursor_walk(brt, n);
+    assert_cursor_walk(ft, n);
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
 
 }
 
-static void assert_cursor_rwalk(FT_HANDLE brt, int n) {
+static void assert_cursor_rwalk(FT_HANDLE ft, int n) {
     FT_CURSOR cursor=0;
     int i;
     int r;
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(r==0);
 
     if (test_cursor_debug && verbose) printf("key: ");
@@ -407,7 +407,7 @@ static void assert_cursor_rwalk(FT_HANDLE brt, int n) {
 
 static void test_ft_cursor_rwalk(int n) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int r;
     int i;
 
@@ -417,7 +417,7 @@ static void test_ft_cursor_rwalk(int n) {
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     /* insert a bunch of kv pairs */
@@ -429,13 +429,13 @@ static void test_ft_cursor_rwalk(int n) {
         toku_fill_dbt(&kbt, &k, sizeof k);
         v = i;
         toku_fill_dbt(&vbt, &v, sizeof v);
-        toku_ft_insert(brt, &kbt, &vbt, 0);
+        toku_ft_insert(ft, &kbt, &vbt, 0);
     }
 
     /* walk the tree */
-    assert_cursor_rwalk(brt, n);
+    assert_cursor_rwalk(ft, n);
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
@@ -462,13 +462,13 @@ ascending_key_string_checkf (ITEMLEN keylen, bytevec key, ITEMLEN UU(vallen), by
 }
 
 // The keys are strings (null terminated)
-static void assert_cursor_walk_inorder(FT_HANDLE brt, int n) {
+static void assert_cursor_walk_inorder(FT_HANDLE ft, int n) {
     FT_CURSOR cursor=0;
     int i;
     int r;
     char *prevkey = 0;
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(r==0);
 
     if (test_cursor_debug && verbose) printf("key: ");
@@ -488,7 +488,7 @@ static void assert_cursor_walk_inorder(FT_HANDLE brt, int n) {
 
 static void test_ft_cursor_rand(int n) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int r;
     int i;
 
@@ -498,7 +498,7 @@ static void test_ft_cursor_rand(int n) {
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     /* insert a bunch of kv pairs */
@@ -513,22 +513,22 @@ static void test_ft_cursor_rand(int n) {
 	    v = i;
 	    toku_fill_dbt(&vbt, &v, sizeof v);
 	    struct check_pair pair = {kbt.size, key, len_ignore, 0, 0};
-	    r = toku_ft_lookup(brt, &kbt, lookup_checkf, &pair);
+	    r = toku_ft_lookup(ft, &kbt, lookup_checkf, &pair);
 	    if (r == 0) {
 		assert(pair.call_count==1);
                 if (verbose) printf("dup");
                 continue;
 	    }
 	    assert(pair.call_count==0);
-	    toku_ft_insert(brt, &kbt, &vbt, 0);
+	    toku_ft_insert(ft, &kbt, &vbt, 0);
 	    break;
         }
     }
 
     /* walk the tree */
-    assert_cursor_walk_inorder(brt, n);
+    assert_cursor_walk_inorder(ft, n);
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
@@ -536,7 +536,7 @@ static void test_ft_cursor_rand(int n) {
 
 static void test_ft_cursor_split(int n) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     FT_CURSOR cursor=0;
     int r;
     int keyseqnum;
@@ -548,7 +548,7 @@ static void test_ft_cursor_split(int n) {
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     /* insert a bunch of kv pairs */
@@ -560,10 +560,10 @@ static void test_ft_cursor_split(int n) {
         toku_fill_dbt(&kbt, key, strlen(key)+1);
         v = keyseqnum;
         toku_fill_dbt(&vbt, &v, sizeof v);
-        toku_ft_insert(brt, &kbt, &vbt, 0);
+        toku_ft_insert(ft, &kbt, &vbt, 0);
     }
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(r==0);
 
     if (test_cursor_debug && verbose) printf("key: ");
@@ -583,7 +583,7 @@ static void test_ft_cursor_split(int n) {
         toku_fill_dbt(&kbt, key, strlen(key)+1);
         v = keyseqnum;
         toku_fill_dbt(&vbt, &v, sizeof v);
-        toku_ft_insert(brt, &kbt, &vbt, 0);
+        toku_ft_insert(ft, &kbt, &vbt, 0);
     }
 
     if (test_cursor_debug && verbose) printf("key: ");
@@ -601,7 +601,7 @@ static void test_ft_cursor_split(int n) {
 
     toku_ft_cursor_close(cursor);
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
@@ -612,19 +612,19 @@ static void test_multiple_ft_cursors(int n) {
 
     int r;
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     FT_CURSOR cursors[n];
 
     unlink(fname);
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     int i;
     for (i=0; i<n; i++) {
-        r = toku_ft_cursor(brt, &cursors[i], NULL, false, false);
+        r = toku_ft_cursor(ft, &cursors[i], NULL, false, false);
         assert(r == 0);
     }
 
@@ -632,7 +632,7 @@ static void test_multiple_ft_cursors(int n) {
         toku_ft_cursor_close(cursors[i]);
     }
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
@@ -653,7 +653,7 @@ static void test_multiple_ft_cursor_walk(int n) {
 
     int r;
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     const int cursor_gap = 1000;
     const int ncursors = n/cursor_gap;
     FT_CURSOR cursors[ncursors];
@@ -665,13 +665,13 @@ static void test_multiple_ft_cursor_walk(int n) {
     int cachesize = 2 * h * ncursors * nodesize;
     toku_cachetable_create(&ct, cachesize, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     int c;
     /* create the cursors */
     for (c=0; c<ncursors; c++) {
-        r = toku_ft_cursor(brt, &cursors[c], NULL, false, false);
+        r = toku_ft_cursor(ft, &cursors[c], NULL, false, false);
         assert(r == 0);
     }
 
@@ -686,7 +686,7 @@ static void test_multiple_ft_cursor_walk(int n) {
 	    toku_fill_dbt(&key, &k, sizeof k);
 	    toku_fill_dbt(&val, &v, sizeof v);
 
-	    toku_ft_insert(brt, &key, &val, 0);
+	    toku_ft_insert(ft, &key, &val, 0);
 	}
 
         /* point cursor i / cursor_gap to the current last key i */
@@ -720,7 +720,7 @@ static void test_multiple_ft_cursor_walk(int n) {
         toku_ft_cursor_close(cursors[i]);
     }
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
@@ -731,14 +731,14 @@ static void test_ft_cursor_set(int n, int cursor_op) {
 
     int r;
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     FT_CURSOR cursor=0;
 
     unlink(fname);
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     int i;
@@ -750,10 +750,10 @@ static void test_ft_cursor_set(int n, int cursor_op) {
 	DBT key,val;
         toku_fill_dbt(&key, &k, sizeof k);
         toku_fill_dbt(&val, &v, sizeof v);
-        toku_ft_insert(brt, &key, &val, 0);
+        toku_ft_insert(ft, &key, &val, 0);
     }
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(r==0);
 
     /* set cursor to random keys in set { 0, 10, 20, .. 10*(n-1) } */
@@ -788,7 +788,7 @@ static void test_ft_cursor_set(int n, int cursor_op) {
 
     toku_ft_cursor_close(cursor);
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
@@ -799,14 +799,14 @@ static void test_ft_cursor_set_range(int n) {
 
     int r;
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     FT_CURSOR cursor=0;
 
     unlink(fname);
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(r==0);
 
     int i;
@@ -819,10 +819,10 @@ static void test_ft_cursor_set_range(int n) {
 	DBT key, val;
         toku_fill_dbt(&key, &k, sizeof k);
         toku_fill_dbt(&val, &v, sizeof v);
-        toku_ft_insert(brt, &key, &val, 0);
+        toku_ft_insert(ft, &key, &val, 0);
     }
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(r==0);
 
     /* pick random keys v in 0 <= v < 10*n, the cursor should point
@@ -848,7 +848,7 @@ static void test_ft_cursor_set_range(int n) {
 
     toku_ft_cursor_close(cursor);
 
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     assert(r==0);
 
     toku_cachetable_close(&ct);
@@ -859,17 +859,17 @@ static void test_ft_cursor_delete(int n) {
 
     int error;
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     FT_CURSOR cursor=0;
 
     unlink(fname);
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
+    error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
     assert(error == 0);
 
-    error = toku_ft_cursor(brt, &cursor, NULL, false, false);
+    error = toku_ft_cursor(ft, &cursor, NULL, false, false);
     assert(error == 0);
 
     DBT key, val;
@@ -882,7 +882,7 @@ static void test_ft_cursor_delete(int n) {
         v = i;
         toku_fill_dbt(&key, &k, sizeof k);
         toku_fill_dbt(&val, &v, sizeof v);
-        toku_ft_insert(brt, &key, &val, 0);
+        toku_ft_insert(ft, &key, &val, 0);
     }
 
     /* walk the tree and delete under the cursor */
@@ -905,7 +905,7 @@ static void test_ft_cursor_delete(int n) {
 
     toku_ft_cursor_close(cursor);
 
-    error = toku_close_ft_handle_nolsn(brt, 0);
+    error = toku_close_ft_handle_nolsn(ft, 0);
     assert(error == 0);
 
     toku_cachetable_close(&ct);

storage/tokudb/ft-index/ft/tests/ft-test-header.cc

@@ -92,7 +92,7 @@ PATENT RIGHTS GRANT:
 #include "test.h"
 
 // The purpose of this test is to verify that certain information in the 
-// brt_header is properly serialized and deserialized. 
+// ft_header is properly serialized and deserialized. 
 
 
 static TOKUTXN const null_txn = 0;

storage/tokudb/ft-index/ft/tests/ft-test.cc

@@ -214,7 +214,7 @@ static void test_multiple_ft_handles_one_db_one_file (void) {
 /* Check to see if data can be read that was written. */
 static void  test_read_what_was_written (void) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int r;
     const int NVALS=10000;
 
@@ -224,34 +224,34 @@ static void  test_read_what_was_written (void) {
     
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);  assert(r==0);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);  assert(r==0);
-    r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
+    r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
     toku_cachetable_close(&ct);
 
     
 
     /* Now see if we can read an empty tree in. */
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
-    r = toku_open_ft_handle(fname, 0, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);  assert(r==0);
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);  assert(r==0);
 
     /* See if we can put something in it. */
     {
 	DBT k,v;
-	toku_ft_insert(brt, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
+	toku_ft_insert(ft, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
     }
 
-    r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
+    r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
     toku_cachetable_close(&ct);
 
     
 
     /* Now see if we can read it in and get the value. */
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
-    r = toku_open_ft_handle(fname, 0, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
 
-    ft_lookup_and_check_nodup(brt, "hello", "there");
+    ft_lookup_and_check_nodup(ft, "hello", "there");
 
-    assert(toku_verify_ft(brt)==0);
+    assert(toku_verify_ft(ft)==0);
 
     /* Now put a bunch (NVALS) of things in. */
     {
@@ -262,14 +262,14 @@ static void  test_read_what_was_written (void) {
 	    snprintf(key, 100, "key%d", i);
 	    snprintf(val, 100, "val%d", i);
 	    if (i<600) {
-		int verify_result=toku_verify_ft(brt);;
+		int verify_result=toku_verify_ft(ft);;
 		assert(verify_result==0);
 	    }
-	    toku_ft_insert(brt, toku_fill_dbt(&k, key, strlen(key)+1), toku_fill_dbt(&v, val, strlen(val)+1), null_txn);
+	    toku_ft_insert(ft, toku_fill_dbt(&k, key, strlen(key)+1), toku_fill_dbt(&v, val, strlen(val)+1), null_txn);
 	    if (i<600) {
-		int verify_result=toku_verify_ft(brt);
+		int verify_result=toku_verify_ft(ft);
 		if (verify_result) {
-		    r = toku_dump_ft(stdout, brt);
+		    r = toku_dump_ft(stdout, ft);
 		    assert(r==0);
 		    assert(0);
 		}
@@ -279,7 +279,7 @@ static void  test_read_what_was_written (void) {
 			char expectedval[100];
 			snprintf(key, 100, "key%d", j);
 			snprintf(expectedval, 100, "val%d", j);
-			ft_lookup_and_check_nodup(brt, key, expectedval);
+			ft_lookup_and_check_nodup(ft, key, expectedval);
 		    }
 		}
 	    }
@@ -287,9 +287,9 @@ static void  test_read_what_was_written (void) {
     }
     if (verbose) printf("Now read them out\n");
 
-    r = toku_verify_ft(brt);
+    r = toku_verify_ft(ft);
     assert(r==0);
-    //dump_ft(brt);
+    //dump_ft(ft);
 
     /* See if we can read them all out again. */
     {
@@ -298,31 +298,31 @@ static void  test_read_what_was_written (void) {
 	    char key[100],expectedval[100];
 	    snprintf(key, 100, "key%d", i);
 	    snprintf(expectedval, 100, "val%d", i);
-	    ft_lookup_and_check_nodup(brt, key, expectedval);
+	    ft_lookup_and_check_nodup(ft, key, expectedval);
 	}
     }
 
-    r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
+    r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
     if (verbose) printf("%s:%d About to close %p\n", __FILE__, __LINE__, ct);
     toku_cachetable_close(&ct);
 
     
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
-    r = toku_open_ft_handle(fname, 0, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
 
-    ft_lookup_and_check_nodup(brt, "hello", "there");
+    ft_lookup_and_check_nodup(ft, "hello", "there");
     {
 	int i;
 	for (i=0; i<NVALS; i++) {
 	    char key[100],expectedval[100];
 	    snprintf(key, 100, "key%d", i);
 	    snprintf(expectedval, 100, "val%d", i);
-	    ft_lookup_and_check_nodup(brt, key, expectedval);
+	    ft_lookup_and_check_nodup(ft, key, expectedval);
 	}
     }
 
-    r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
+    r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
     toku_cachetable_close(&ct);
 
     
@@ -334,7 +334,7 @@ static void  test_read_what_was_written (void) {
 /* Test c_get(DB_LAST) on an empty tree */
 static void test_cursor_last_empty(void) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     FT_CURSOR cursor=0;
     int r;
     if (verbose) printf("%s", __FUNCTION__);
@@ -343,9 +343,9 @@ static void test_cursor_last_empty(void) {
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);  assert(r==0);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);  assert(r==0);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);            assert(r==0);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);            assert(r==0);
     {
 	struct check_pair pair = {0,0,0,0,0};
 	r = toku_ft_cursor_get(cursor, NULL, lookup_checkf, &pair, DB_LAST);
@@ -359,7 +359,7 @@ static void test_cursor_last_empty(void) {
 	assert(r==DB_NOTFOUND);
     }
     toku_ft_cursor_close(cursor);
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
     toku_cachetable_close(&ct);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
@@ -368,7 +368,7 @@ static void test_cursor_last_empty(void) {
 
 static void test_cursor_next (void) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     FT_CURSOR cursor=0;
     int r;
     DBT kbt, vbt;
@@ -377,12 +377,12 @@ static void test_cursor_next (void) {
     
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);  assert(r==0);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);  assert(r==0);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
-    toku_ft_insert(brt, toku_fill_dbt(&kbt, "hello", 6), toku_fill_dbt(&vbt, "there", 6), null_txn);
+    toku_ft_insert(ft, toku_fill_dbt(&kbt, "hello", 6), toku_fill_dbt(&vbt, "there", 6), null_txn);
-    toku_ft_insert(brt, toku_fill_dbt(&kbt, "byebye", 7), toku_fill_dbt(&vbt, "byenow", 7), null_txn);
+    toku_ft_insert(ft, toku_fill_dbt(&kbt, "byebye", 7), toku_fill_dbt(&vbt, "byenow", 7), null_txn);
     if (verbose) printf("%s:%d calling toku_ft_cursor(...)\n", __FILE__, __LINE__);
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false);            assert(r==0);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false);            assert(r==0);
     toku_init_dbt(&kbt);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
     toku_init_dbt(&vbt);
@@ -411,7 +411,7 @@ static void test_cursor_next (void) {
     }
 
     toku_ft_cursor_close(cursor);
-    r = toku_close_ft_handle_nolsn(brt, 0);
+    r = toku_close_ft_handle_nolsn(ft, 0);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
     toku_cachetable_close(&ct);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
@@ -436,7 +436,7 @@ static int wrong_compare_fun(DB* UU(desc), const DBT *a, const DBT *b) {
 
 static void test_wrongendian_compare (int wrong_p, unsigned int N) {
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int r;
     unsigned int i;
 
@@ -455,7 +455,7 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
     //printf("%s:%d WRONG=%d\n", __FILE__, __LINE__, wrong_p);
 
     if (0) { // ???? Why is this commented out?
-        r = toku_open_ft_handle(fname, 1, &brt, 1<<20, 1<<17, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, wrong_p ? wrong_compare_fun : toku_builtin_compare_fun);  assert(r==0);
+        r = toku_open_ft_handle(fname, 1, &ft, 1<<20, 1<<17, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, wrong_p ? wrong_compare_fun : toku_builtin_compare_fun);  assert(r==0);
     for (i=1; i<257; i+=255) {
 	unsigned char a[4],b[4];
 	b[3] = a[0] = (unsigned char)(i&255);
@@ -470,11 +470,11 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
 	    printf("%s:%d insert: %02x%02x%02x%02x -> %02x%02x%02x%02x\n", __FILE__, __LINE__,
 		   ((char*)kbt.data)[0], ((char*)kbt.data)[1], ((char*)kbt.data)[2], ((char*)kbt.data)[3],
 		   ((char*)vbt.data)[0], ((char*)vbt.data)[1], ((char*)vbt.data)[2], ((char*)vbt.data)[3]);
-	toku_ft_insert(brt, &kbt, &vbt, null_txn);
+	toku_ft_insert(ft, &kbt, &vbt, null_txn);
     }
     {
 	FT_CURSOR cursor=0;
-	r = toku_ft_cursor(brt, &cursor, NULL, false, false);            assert(r==0);
+	r = toku_ft_cursor(ft, &cursor, NULL, false, false);            assert(r==0);
 
 	for (i=0; i<2; i++) {
 	    unsigned char a[4],b[4];
@@ -489,13 +489,13 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
 	}
 
 
-        r = toku_close_ft_handle_nolsn(brt, 0);
+        r = toku_close_ft_handle_nolsn(ft, 0);
     }
     }
 
     {
 	toku_cachetable_verify(ct);
-	r = toku_open_ft_handle(fname, 1, &brt, 1<<20, 1<<17, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, wrong_p ? wrong_compare_fun : toku_builtin_compare_fun);  assert(r==0);
+	r = toku_open_ft_handle(fname, 1, &ft, 1<<20, 1<<17, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, wrong_p ? wrong_compare_fun : toku_builtin_compare_fun);  assert(r==0);
 	toku_cachetable_verify(ct);
 
 	for (i=0; i<N; i++) {
@@ -511,11 +511,11 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
 	    if (0) printf("%s:%d insert: %02x%02x%02x%02x -> %02x%02x%02x%02x\n", __FILE__, __LINE__,
 			  ((unsigned char*)kbt.data)[0], ((unsigned char*)kbt.data)[1], ((unsigned char*)kbt.data)[2], ((unsigned char*)kbt.data)[3],
 			  ((unsigned char*)vbt.data)[0], ((unsigned char*)vbt.data)[1], ((unsigned char*)vbt.data)[2], ((unsigned char*)vbt.data)[3]);
-	    toku_ft_insert(brt, &kbt, &vbt, null_txn);
+	    toku_ft_insert(ft, &kbt, &vbt, null_txn);
 	    toku_cachetable_verify(ct);
 	}
 	FT_CURSOR cursor=0;
-	r = toku_ft_cursor(brt, &cursor, NULL, false, false);            assert(r==0);
+	r = toku_ft_cursor(ft, &cursor, NULL, false, false);            assert(r==0);
 	
 	for (i=0; i<N; i++) {
 	    unsigned char a[4],b[4];
@@ -530,7 +530,7 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
 	    toku_cachetable_verify(ct);
 	}
         toku_ft_cursor_close(cursor);
-	r = toku_close_ft_handle_nolsn(brt, 0);
+	r = toku_close_ft_handle_nolsn(ft, 0);
 	assert(r==0);
     }
     toku_cachetable_close(&ct);
@@ -806,8 +806,8 @@ static void test_ft_delete_cursor_first(int n) {
     toku_cachetable_close(&ct);
 }
 
-/* test for bug: insert cmd in a nonleaf node, delete removes the
+/* test for bug: insert message in a nonleaf node, delete removes the
-   insert cmd, but lookup finds the insert cmd
+   insert message, but lookup finds the insert message
 
    build a 2 level tree, and expect the last insertion to be
    buffered. then delete and lookup. */
@@ -881,22 +881,22 @@ static void test_ft_delete(void) {
 
 static void test_new_ft_cursor_create_close (void) {
     int r;
-    FT_HANDLE brt=0;
+    FT_HANDLE ft=0;
     int n = 8;
     FT_CURSOR cursors[n];
 
-    toku_ft_handle_create(&brt);
+    toku_ft_handle_create(&ft);
 
     int i;
     for (i=0; i<n; i++) {
-        r = toku_ft_cursor(brt, &cursors[i], NULL, false, false); assert(r == 0);
+        r = toku_ft_cursor(ft, &cursors[i], NULL, false, false); assert(r == 0);
     }
 
     for (i=0; i<n; i++) {
         toku_ft_cursor_close(cursors[i]);
     }
 
-    r = toku_close_ft_handle_nolsn(brt, 0); assert(r == 0);
+    r = toku_close_ft_handle_nolsn(ft, 0); assert(r == 0);
 }
 
 static void test_new_ft_cursor_first(int n) {
@@ -1177,14 +1177,14 @@ static void test_new_ft_cursor_set_range(int n) {
 
     int r;
     CACHETABLE ct;
-    FT_HANDLE brt=0;
+    FT_HANDLE ft=0;
     FT_CURSOR cursor=0;
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
     unlink(fname);
-    toku_ft_handle_create(&brt);
+    toku_ft_handle_create(&ft);
-    toku_ft_handle_set_nodesize(brt, 4096);
+    toku_ft_handle_set_nodesize(ft, 4096);
-    r = toku_ft_handle_open(brt, fname, 1, 1, ct, null_txn); assert(r==0);
+    r = toku_ft_handle_open(ft, fname, 1, 1, ct, null_txn); assert(r==0);
 
     int i;
 
@@ -1194,10 +1194,10 @@ static void test_new_ft_cursor_set_range(int n) {
 	DBT key, val;
         int k = toku_htonl(10*i);
         int v = 10*i;
-        toku_ft_insert(brt, toku_fill_dbt(&key, &k, sizeof k), toku_fill_dbt(&val, &v, sizeof v), 0); assert(r == 0);
+        toku_ft_insert(ft, toku_fill_dbt(&key, &k, sizeof k), toku_fill_dbt(&val, &v, sizeof v), 0); assert(r == 0);
     }
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false); assert(r==0);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false); assert(r==0);
 
     /* pick random keys v in 0 <= v < 10*n, the cursor should point
        to the smallest key in the tree that is >= v */
@@ -1226,7 +1226,7 @@ static void test_new_ft_cursor_set_range(int n) {
 
     toku_ft_cursor_close(cursor);
 
-    r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
+    r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
 
     toku_cachetable_close(&ct);
 }
@@ -1236,14 +1236,14 @@ static void test_new_ft_cursor_set(int n, int cursor_op, DB *db) {
 
     int r;
     CACHETABLE ct;
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     FT_CURSOR cursor=0;
 
     unlink(fname);
 
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare); assert(r==0);
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare); assert(r==0);
 
     int i;
 
@@ -1252,10 +1252,10 @@ static void test_new_ft_cursor_set(int n, int cursor_op, DB *db) {
 	DBT key, val;
         int k = toku_htonl(10*i);
         int v = 10*i;
-        toku_ft_insert(brt, toku_fill_dbt(&key, &k, sizeof k), toku_fill_dbt(&val, &v, sizeof v), 0); assert(r == 0);
+        toku_ft_insert(ft, toku_fill_dbt(&key, &k, sizeof k), toku_fill_dbt(&val, &v, sizeof v), 0); assert(r == 0);
     }
 
-    r = toku_ft_cursor(brt, &cursor, NULL, false, false); assert(r==0);
+    r = toku_ft_cursor(ft, &cursor, NULL, false, false); assert(r==0);
 
     /* set cursor to random keys in set { 0, 10, 20, .. 10*(n-1) } */
     for (i=0; i<n; i++) {
@@ -1287,7 +1287,7 @@ static void test_new_ft_cursor_set(int n, int cursor_op, DB *db) {
 
     toku_ft_cursor_close(cursor);
 
-    r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
+    r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
 
     toku_cachetable_close(&ct);
 }

storage/tokudb/ft-index/ft/tests/ftloader-test-bad-generate.cc

@@ -158,19 +158,19 @@ static void test_extractor(int nrows, int nrowsets, bool expect_fail) {
 
     // open the ft_loader. this runs the extractor.
     const int N = 1;
-    FT_HANDLE brts[N];
+    FT_HANDLE fts[N];
     DB* dbs[N];
     const char *fnames[N];
     ft_compare_func compares[N];
     for (int i = 0; i < N; i++) {
-        brts[i] = NULL;
+        fts[i] = NULL;
         dbs[i] = NULL;
         fnames[i] = "";
         compares[i] = compare_int;
     }
 
     FTLOADER loader;
-    r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, nullptr, true, 0, false);
+    r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, fts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, nullptr, true, 0, false);
     assert(r == 0);
 
     struct rowset *rowset[nrowsets];

storage/tokudb/ft-index/ft/tests/ftloader-test-extractor-errors.cc

@@ -165,12 +165,12 @@ static void test_extractor(int nrows, int nrowsets, bool expect_fail, const char
 
     // open the ft_loader. this runs the extractor.
     const int N = 1;
-    FT_HANDLE brts[N];
+    FT_HANDLE fts[N];
     DB* dbs[N];
     const char *fnames[N];
     ft_compare_func compares[N];
     for (int i = 0; i < N; i++) {
-        brts[i] = NULL;
+        fts[i] = NULL;
         dbs[i] = NULL;
         fnames[i] = "";
         compares[i] = compare_int;
@@ -180,7 +180,7 @@ static void test_extractor(int nrows, int nrowsets, bool expect_fail, const char
     sprintf(temp, "%s/%s", testdir, "tempXXXXXX");
 
     FTLOADER loader;
-    r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, nullptr, true, 0, false);
+    r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, fts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, nullptr, true, 0, false);
     assert(r == 0);
 
     struct rowset *rowset[nrowsets];

storage/tokudb/ft-index/ft/tests/ftloader-test-extractor.cc

@@ -89,7 +89,7 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2010-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-// The purpose of this test is to test the extractor component of the brt loader.  We insert rowsets into the extractor queue and verify temp files 
+// The purpose of this test is to test the extractor component of the ft loader.  We insert rowsets into the extractor queue and verify temp files 
 // after the extractor is finished.
 
 #define DONT_DEPRECATE_MALLOC
@@ -387,12 +387,12 @@ static void test_extractor(int nrows, int nrowsets, const char *testdir) {
 
     // open the ft_loader. this runs the extractor.
     const int N = 1;
-    FT_HANDLE brts[N];
+    FT_HANDLE fts[N];
     DB* dbs[N];
     const char *fnames[N];
     ft_compare_func compares[N];
     for (int i = 0; i < N; i++) {
-        brts[i] = NULL;
+        fts[i] = NULL;
         dbs[i] = NULL;
         fnames[i] = "";
         compares[i] = compare_int;
@@ -402,7 +402,7 @@ static void test_extractor(int nrows, int nrowsets, const char *testdir) {
     sprintf(temp, "%s/%s", testdir, "tempXXXXXX");
 
     FTLOADER loader;
-    r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, temp, ZERO_LSN, nullptr, true, 0, false);
+    r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, fts, dbs, fnames, compares, temp, ZERO_LSN, nullptr, true, 0, false);
     assert(r == 0);
 
     struct rowset *rowset[nrowsets];

storage/tokudb/ft-index/ft/tests/ftloader-test-merge-files-dbufio.cc

@@ -385,7 +385,7 @@ static void test (const char *directory, bool is_error) {
     }
 
     FTLOADER bl;
-    FT_HANDLE *XCALLOC_N(N_DEST_DBS, brts);
+    FT_HANDLE *XCALLOC_N(N_DEST_DBS, fts);
     DB* *XCALLOC_N(N_DEST_DBS, dbs);
     const char **XMALLOC_N(N_DEST_DBS, new_fnames_in_env);
     for (int i=0; i<N_DEST_DBS; i++) {
@@ -407,7 +407,7 @@ static void test (const char *directory, bool is_error) {
 					       ct,
 					       (generate_row_for_put_func)NULL,
 					       (DB*)NULL,
-					       N_DEST_DBS, brts, dbs,
+					       N_DEST_DBS, fts, dbs,
 					       new_fnames_in_env,
 					       bt_compare_functions,
 					       "tempxxxxxx",
@@ -522,7 +522,7 @@ static void test (const char *directory, bool is_error) {
     destroy_dbufio_fileset(bfs);
     toku_free(fnames);
     toku_free(fds);
-    toku_free(brts);
+    toku_free(fts);
     toku_free(dbs);
     toku_free(new_fnames_in_env);
     toku_free(bt_compare_functions);

storage/tokudb/ft-index/ft/tests/ftloader-test-open.cc

@@ -126,12 +126,12 @@ static void test_loader_open(int ndbs) {
     FTLOADER loader;
 
     // open the ft_loader. this runs the extractor.
-    FT_HANDLE brts[ndbs];
+    FT_HANDLE fts[ndbs];
     DB* dbs[ndbs];
     const char *fnames[ndbs];
     ft_compare_func compares[ndbs];
     for (int i = 0; i < ndbs; i++) {
-        brts[i] = NULL;
+        fts[i] = NULL;
         dbs[i] = NULL;
         fnames[i] = "";
         compares[i] = my_compare;
@@ -143,7 +143,7 @@ static void test_loader_open(int ndbs) {
     for (i = 0; ; i++) {
         set_my_malloc_trigger(i+1);
 
-        r = toku_ft_loader_open(&loader, NULL, NULL, NULL, ndbs, brts, dbs, fnames, compares, "", ZERO_LSN, nullptr, true, 0, false);
+        r = toku_ft_loader_open(&loader, NULL, NULL, NULL, ndbs, fts, dbs, fnames, compares, "", ZERO_LSN, nullptr, true, 0, false);
         if (r == 0)
             break;
     }

storage/tokudb/ft-index/ft/tests/ftloader-test-writer-errors.cc

@@ -213,7 +213,7 @@ static int write_dbfile (char *tf_template, int n, char *output_name, bool expec
     ft_loader_set_error_function(&bl.error_callback, NULL, NULL);
     ft_loader_set_poll_function(&bl.poll_callback, loader_poll_callback, NULL);
 
-    result = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
+    result = toku_loader_write_ft_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
 
     toku_set_func_malloc_only(NULL);
     toku_set_func_realloc_only(NULL);

storage/tokudb/ft-index/ft/tests/ftloader-test-writer.cc

@@ -137,9 +137,9 @@ static void verify_dbfile(int n, const char *name) {
     toku_ft_set_bt_compare(t, compare_ints);
     r = toku_ft_handle_open(t, name, 0, 0, ct, null_txn); assert(r==0);
 
-    if (verbose) traceit("Verifying brt internals");
+    if (verbose) traceit("Verifying ft internals");
     r = toku_verify_ft(t);
-    if (verbose) traceit("Verified brt internals");
+    if (verbose) traceit("Verified ft internals");
 
     FT_CURSOR cursor = NULL;
     r = toku_ft_cursor(t, &cursor, NULL, false, false); assert(r == 0);
@@ -262,7 +262,7 @@ static void test_write_dbfile (char *tf_template, int n, char *output_name, TXNI
     assert(fd>=0);
 
     if (verbose) traceit("write to file");
-    r = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
+    r = toku_loader_write_ft_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
     assert(r==0);
 
     r = queue_destroy(q2);

storage/tokudb/ft-index/ft/tests/ftloader-test.cc

@@ -425,7 +425,7 @@ static void test_merge_files (const char *tf_template, const char *output_name)
     int fd = open(output_name, O_RDWR | O_CREAT | O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(fd>=0);
     
-    r = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
+    r = toku_loader_write_ft_from_q_in_C(&bl, &desc, fd, 1000, q, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
     assert(r==0);
 
     destroy_merge_fileset(&fs);

storage/tokudb/ft-index/ft/tests/is_empty.cc

@@ -98,7 +98,7 @@ PATENT RIGHTS GRANT:
 #define FILENAME "test0.ft"
 
 static void test_it (int N) {
-    FT_HANDLE brt;
+    FT_HANDLE ft;
     int r;
     toku_os_recursive_delete(TOKU_TEST_FILENAME);
     r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU);                                                                    CKERR(r);
@@ -119,18 +119,18 @@ static void test_it (int N) {
     TOKUTXN txn;
     r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT, false);                     CKERR(r);
 
-    r = toku_open_ft_handle(FILENAME, 1, &brt, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun);                    CKERR(r);
+    r = toku_open_ft_handle(FILENAME, 1, &ft, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun);                    CKERR(r);
 
     r = toku_txn_commit_txn(txn, false, NULL, NULL);                                 CKERR(r);
     toku_txn_close_txn(txn);
     CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct);
     r = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);                             CKERR(r);
-    r = toku_close_ft_handle_nolsn(brt, NULL);                                                                          CKERR(r);
+    r = toku_close_ft_handle_nolsn(ft, NULL);                                                                          CKERR(r);
 
     unsigned int rands[N];
     for (int i=0; i<N; i++) {
 	r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT, false);                 CKERR(r);
-	r = toku_open_ft_handle(FILENAME, 0, &brt, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun);                CKERR(r);
+	r = toku_open_ft_handle(FILENAME, 0, &ft, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun);                CKERR(r);
 	r = toku_txn_commit_txn(txn, false, NULL, NULL);                             CKERR(r);
 	toku_txn_close_txn(txn);
 
@@ -141,19 +141,19 @@ static void test_it (int N) {
 	snprintf(key, sizeof(key), "key%x.%x", rands[i], i);
 	memset(val, 'v', sizeof(val));
 	val[sizeof(val)-1]=0;
-	toku_ft_insert(brt, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), txn);
+	toku_ft_insert(ft, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), txn);
 	r = toku_txn_commit_txn(txn, false, NULL, NULL);                                 CKERR(r);
 	toku_txn_close_txn(txn);
 
 
 	r = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);                             CKERR(r);
-	r = toku_close_ft_handle_nolsn(brt, NULL);                                                                          CKERR(r);
+	r = toku_close_ft_handle_nolsn(ft, NULL);                                                                          CKERR(r);
 
 	if (verbose) printf("i=%d\n", i);
     }
     for (int i=0; i<N; i++) {
 	r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT, false);                     CKERR(r);
-	r = toku_open_ft_handle(FILENAME, 0, &brt, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun);                CKERR(r);
+	r = toku_open_ft_handle(FILENAME, 0, &ft, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun);                CKERR(r);
 	r = toku_txn_commit_txn(txn, false, NULL, NULL);                                 CKERR(r);
 	toku_txn_close_txn(txn);
 
@@ -161,11 +161,11 @@ static void test_it (int N) {
 	char key[100];
 	DBT k;
 	snprintf(key, sizeof(key), "key%x.%x", rands[i], i);
-	toku_ft_delete(brt, toku_fill_dbt(&k, key, 1+strlen(key)), txn);
+	toku_ft_delete(ft, toku_fill_dbt(&k, key, 1+strlen(key)), txn);
 
 	if (0) {
 	bool is_empty;
-        is_empty = toku_ft_is_empty_fast(brt);
+        is_empty = toku_ft_is_empty_fast(ft);
 	assert(!is_empty);
 	}
 	
@@ -173,23 +173,23 @@ static void test_it (int N) {
 	toku_txn_close_txn(txn);
 
 	r = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);                             CKERR(r);
-	r = toku_close_ft_handle_nolsn(brt, NULL);                                                                          CKERR(r);
+	r = toku_close_ft_handle_nolsn(ft, NULL);                                                                          CKERR(r);
 
 	if (verbose) printf("d=%d\n", i);
     }
     r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT, false);                        CKERR(r);
-    r = toku_open_ft_handle(FILENAME, 0, &brt, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun);                       CKERR(r);
+    r = toku_open_ft_handle(FILENAME, 0, &ft, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun);                       CKERR(r);
     r = toku_txn_commit_txn(txn, false, NULL, NULL);                                     CKERR(r);
     toku_txn_close_txn(txn);
 
     if (0) {
     bool is_empty;
-    is_empty = toku_ft_is_empty_fast(brt);
+    is_empty = toku_ft_is_empty_fast(ft);
     assert(is_empty);
     }
 
     r = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);                                CKERR(r);
-    r = toku_close_ft_handle_nolsn(brt, NULL);                                                                             CKERR(r);
+    r = toku_close_ft_handle_nolsn(ft, NULL);                                                                             CKERR(r);
 
     r = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);                                CKERR(r);
     toku_logger_close_rollback(logger);

storage/tokudb/ft-index/ft/tests/keyrange.cc

@@ -132,8 +132,8 @@ static void reload (uint64_t limit) {
 
 enum memory_state {
     LEAVE_IN_MEMORY,       // leave the state in main memory
-    CLOSE_AND_RELOAD,      // close the brts and reload them into main memory (that will cause >1 partitio in many leaves.)
+    CLOSE_AND_RELOAD,      // close the fts and reload them into main memory (that will cause >1 partitio in many leaves.)
-    CLOSE_AND_REOPEN_LEAVE_ON_DISK   // close the brts, reopen them, but leave the state on disk.
+    CLOSE_AND_REOPEN_LEAVE_ON_DISK   // close the fts, reopen them, but leave the state on disk.
 };
 
 static void maybe_reopen (enum memory_state ms, uint64_t limit) {

storage/tokudb/ft-index/ft/tests/le-cursor-provdel.cc

@@ -138,8 +138,8 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
     error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE, false);
     assert(error == 0);
 
-    FT_HANDLE brt = NULL;
+    FT_HANDLE ft = NULL;
-    error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_ft_cursor_keycompare);
+    error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_ft_cursor_keycompare);
     assert(error == 0);
 
     error = toku_txn_commit_txn(txn, true, NULL, NULL);
@@ -158,7 +158,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
         toku_fill_dbt(&key, &k, sizeof k);
         DBT val;
         toku_fill_dbt(&val, &v, sizeof v);
-        toku_ft_insert(brt, &key, &val, txn);
+        toku_ft_insert(ft, &key, &val, txn);
         assert(error == 0);
     }
 
@@ -166,7 +166,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
     assert(error == 0);
     toku_txn_close_txn(txn);
 
-    error = toku_close_ft_handle_nolsn(brt, NULL);
+    error = toku_close_ft_handle_nolsn(ft, NULL);
     assert(error == 0);
     
     CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct);
@@ -208,8 +208,8 @@ test_provdel(const char *logdir, const char *fname, int n) {
     error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE, false);
     assert(error == 0);
 
-    FT_HANDLE brt = NULL;
+    FT_HANDLE ft = NULL;
-    error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_ft_cursor_keycompare);
+    error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_ft_cursor_keycompare);
     assert(error == 0);
 
     error = toku_txn_commit_txn(txn, true, NULL, NULL);
@@ -225,7 +225,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
         int k = toku_htonl(i);
         DBT key;
         toku_fill_dbt(&key, &k, sizeof k);
-        toku_ft_delete(brt, &key, txn);
+        toku_ft_delete(ft, &key, txn);
         assert(error == 0);
     }
 
@@ -234,7 +234,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
     assert(error == 0);
 
     LE_CURSOR cursor = NULL;
-    error = toku_le_cursor_create(&cursor, brt, cursortxn);
+    error = toku_le_cursor_create(&cursor, ft, cursortxn);
     assert(error == 0);
 
     DBT key;
@@ -267,7 +267,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
     assert(error == 0);
     toku_txn_close_txn(txn);
 
-    error = toku_close_ft_handle_nolsn(brt, NULL);
+    error = toku_close_ft_handle_nolsn(ft, NULL);
     assert(error == 0);
     CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct);
     error = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);

storage/tokudb/ft-index/ft/tests/le-cursor-right.cc

@@ -142,8 +142,8 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
     error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE, false);
     assert(error == 0);
 
-    FT_HANDLE brt = NULL;
+    FT_HANDLE ft = NULL;
-    error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_keycompare);
+    error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_keycompare);
     assert(error == 0);
 
     error = toku_txn_commit_txn(txn, true, NULL, NULL);
@@ -162,14 +162,14 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
         toku_fill_dbt(&key, &k, sizeof k);
         DBT val;
         toku_fill_dbt(&val, &v, sizeof v);
-        toku_ft_insert(brt, &key, &val, txn);
+        toku_ft_insert(ft, &key, &val, txn);
     }
 
     error = toku_txn_commit_txn(txn, true, NULL, NULL);
     assert(error == 0);
     toku_txn_close_txn(txn);
 
-    error = toku_close_ft_handle_nolsn(brt, NULL);
+    error = toku_close_ft_handle_nolsn(ft, NULL);
     assert(error == 0);
 
     CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct);
@@ -198,13 +198,13 @@ test_pos_infinity(const char *fname, int n) {
     CACHETABLE ct = NULL;
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    FT_HANDLE brt = NULL;
+    FT_HANDLE ft = NULL;
-    error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
+    error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
     assert(error == 0);
 
     // position the cursor at -infinity
     LE_CURSOR cursor = NULL;
-    error = toku_le_cursor_create(&cursor, brt, NULL);
+    error = toku_le_cursor_create(&cursor, ft, NULL);
     assert(error == 0);
 
     for (int i = 0; i < 2*n; i++) {
@@ -217,7 +217,7 @@ test_pos_infinity(const char *fname, int n) {
         
     toku_le_cursor_close(cursor);
 
-    error = toku_close_ft_handle_nolsn(brt, 0);
+    error = toku_close_ft_handle_nolsn(ft, 0);
     assert(error == 0);
 
     toku_cachetable_close(&ct);
@@ -232,13 +232,13 @@ test_neg_infinity(const char *fname, int n) {
     CACHETABLE ct = NULL;
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    FT_HANDLE brt = NULL;
+    FT_HANDLE ft = NULL;
-    error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
+    error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
     assert(error == 0);
 
     // position the LE_CURSOR at +infinity
     LE_CURSOR cursor = NULL;
-    error = toku_le_cursor_create(&cursor, brt, NULL);
+    error = toku_le_cursor_create(&cursor, ft, NULL);
     assert(error == 0);
 
     DBT key;
@@ -271,7 +271,7 @@ test_neg_infinity(const char *fname, int n) {
 
     toku_le_cursor_close(cursor);
 
-    error = toku_close_ft_handle_nolsn(brt, 0);
+    error = toku_close_ft_handle_nolsn(ft, 0);
     assert(error == 0);
 
     toku_cachetable_close(&ct);
@@ -286,13 +286,13 @@ test_between(const char *fname, int n) {
     CACHETABLE ct = NULL;
     toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
 
-    FT_HANDLE brt = NULL;
+    FT_HANDLE ft = NULL;
-    error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
+    error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
     assert(error == 0);
 
     // position the LE_CURSOR at +infinity
     LE_CURSOR cursor = NULL;
-    error = toku_le_cursor_create(&cursor, brt, NULL);
+    error = toku_le_cursor_create(&cursor, ft, NULL);
     assert(error == 0);
 
     DBT key;
@@ -337,7 +337,7 @@ test_between(const char *fname, int n) {
 
     toku_le_cursor_close(cursor);
 
-    error = toku_close_ft_handle_nolsn(brt, 0);
+    error = toku_close_ft_handle_nolsn(ft, 0);
     assert(error == 0);
 
     toku_cachetable_close(&ct);