database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-09-02 09:41:40 +03:00
Code Activity

merge:

Browse Source 
git://github.com/Tokutek/ft-index.git
git://github.com/Tokutek/ft-engine.git
at the tag tokudb-7.1.6
This commit is contained in:
Sergei Golubchik
2014-05-05 22:59:44 +02:00
parent 2f7f2de5cb 5c88950f25 394202d207
commit aff4854dfd
416 changed files with 92641 additions and 84650 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
storage/tokudb/CMakeLists.txt
View File

@@ -40,17 +40,22 @@ IF(DEFINED TOKUDB_VERSION)
ADD_DEFINITIONS("-DTOKUDB_VERSION=\"${TOKUDB_VERSION}\"")
ENDIF()
IF(DEFINED TOKUDB_NOPATCH_CONFIG)
ADD_DEFINITIONS("-DTOKUDB_NOPATCH_CONFIG=${TOKUDB_NOPATCH_CONFIG}")
ENDIF()
IF(DEFINED TOKUDB_CHECK_JEMALLOC)
ADD_DEFINITIONS("-DTOKUDB_CHECK_JEMALLOC=${TOKUDB_CHECK_JEMALLOC}")
ENDIF()
ADD_SUBDIRECTORY(ft-index)
# TODO: clean up includes in ft-index
INCLUDE_DIRECTORIES(ft-index)
INCLUDE_DIRECTORIES(ft-index/include)
INCLUDE_DIRECTORIES(ft-index/portability)
INCLUDE_DIRECTORIES(ft-index/toku_include)
INCLUDE_DIRECTORIES(ft-index/util)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index/buildheader)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index/toku_include)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/ft-index/portability)
SET(TOKUDB_PLUGIN_DYNAMIC "ha_tokudb")
SET(TOKUDB_SOURCES ha_tokudb.cc)

15
storage/tokudb/README.md
View File

@@ -24,19 +24,20 @@ working MySQL or MariaDB with Tokutek patches, and with the TokuDB storage
engine, called `make.mysql.bash`. This script will download copies of the
needed source code from github and build everything.
To build MySQL with TokuDB 7.1.0:
To build MySQL 5.5.36 with TokuDB 7.1.5:
```sh
scripts/make.mysql.bash --mysqlbuild=mysql-5.5.30-tokudb-7.1.0-linux-x86_64
scripts/make.mysql.bash --mysqlbuild=mysql-5.5.36-tokudb-7.1.5-linux-x86_64
```
To build MariaDB with TokuDB 7.1.0:
To build MariaDB 5.5.36 with TokuDB 7.1.5:
```sh
scripts/make.mysql.bash --mysqlbuild=mariadb-5.5.30-tokudb-7.1.0-linux-x86_64
scripts/make.mysql.bash --mysqlbuild=mariadb-5.5.36-tokudb-7.1.5-linux-x86_64
```
Before you start, make sure you have a C++11-compatible compiler (GCC >=
4.7 is recommended), as well as CMake >=2.8.8, and the libraries and
header files for valgrind,zlib, and Berkeley DB.
header files for valgrind,zlib, and Berkeley DB. We are using the gcc 4.7
in devtoolset-1.1.
On CentOS, `yum install valgrind-devel zlib-devel libdb-devel`
@@ -76,5 +77,5 @@ TokuDB is available under the GPL version 2. See [COPYING][copying]
The TokuKV component of TokuDB is available under the GPL version 2, with
slight modifications. See [README-TOKUDB][license].
[copying]: http://github.com/Tokutek/ft-engine/blob/master/COPYING
[license]: http://github.com/Tokutek/ft-index/blob/master/README-TOKUDB
[copying]: http://github.com/Tokutek/tokudb-engine/blob/master/COPYING
[license]: http://github.com/Tokutek/tokudb-index/blob/master/README-TOKUDB

8
storage/tokudb/ft-index/CMakeLists.txt
View File

@@ -26,14 +26,12 @@ if (USE_VALGRIND AND NOT VALGRIND_INCLUDE_DIR MATCHES NOTFOUND)
)
endif()
include_directories(
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/toku_include
${CMAKE_CURRENT_SOURCE_DIR}/portability
${CMAKE_CURRENT_SOURCE_DIR} ## so you can include <ft/ft-ops.h> from inside src/
${CMAKE_CURRENT_BINARY_DIR} ## for logging code
)
## include where config.h will be generated
include_directories(${CMAKE_CURRENT_BINARY_DIR}/toku_include)
include_directories(${CMAKE_CURRENT_BINARY_DIR}/portability)
## build db.h and include where it will be generated
add_subdirectory(buildheader)
@@ -51,11 +49,9 @@ add_subdirectory(portability)
add_subdirectory(ft)
add_subdirectory(locktree)
add_subdirectory(src)
add_subdirectory(utils)
add_subdirectory(tools)
## subdirectories that just install things
add_subdirectory(include)
add_subdirectory(toku_include)
#add_subdirectory(examples)
INSTALL_DOCUMENTATION(README.md README-TOKUDB COMPONENT Server)

10
storage/tokudb/ft-index/buildheader/make_tdb.cc
View File

@@ -464,6 +464,7 @@ static void print_db_env_struct (void) {
"void (*set_loader_memory_size)(DB_ENV *env, uint64_t (*get_loader_memory_size_callback)(void))",
"uint64_t (*get_loader_memory_size)(DB_ENV *env)",
"void (*set_killed_callback)(DB_ENV *env, uint64_t default_killed_time_msec, uint64_t (*get_killed_time_callback)(uint64_t default_killed_time_msec), int (*killed_callback)(void))",
"void (*do_backtrace) (DB_ENV *env)",
NULL};
sort_and_dump_fields("db_env", true, extra);
@@ -575,7 +576,6 @@ static void print_db_txn_struct (void) {
STRUCT_SETUP(DB_TXN, parent, "DB_TXN *%s");
const char *extra[] = {
"int (*txn_stat)(DB_TXN *, struct txn_stat **)",
"struct toku_list open_txns",
"int (*commit_with_progress)(DB_TXN*, uint32_t, TXN_PROGRESS_POLL_FUNCTION, void*)",
"int (*abort_with_progress)(DB_TXN*, TXN_PROGRESS_POLL_FUNCTION, void*)",
"int (*xa_prepare) (DB_TXN*, TOKU_XA_XID *)",
@@ -608,6 +608,7 @@ static void print_dbc_struct (void) {
"int (*c_getf_set)(DBC *, uint32_t, DBT *, YDB_CALLBACK_FUNCTION, void *)",
"int (*c_getf_set_range)(DBC *, uint32_t, DBT *, YDB_CALLBACK_FUNCTION, void *)",
"int (*c_getf_set_range_reverse)(DBC *, uint32_t, DBT *, YDB_CALLBACK_FUNCTION, void *)",
"int (*c_getf_set_range_with_bound)(DBC *, uint32_t, DBT *k, DBT *k_bound, YDB_CALLBACK_FUNCTION, void *)",
"int (*c_set_bounds)(DBC*, const DBT*, const DBT*, bool pre_acquire, int out_of_range_error)",
"void (*c_set_check_interrupt_callback)(DBC*, bool (*)(void*), void *)",
"void (*c_remove_restriction)(DBC*)",
@@ -632,7 +633,6 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
//printf("#include <inttypes.h>\n");
printf("#if defined(__cplusplus) || defined(__cilkplusplus)\nextern \"C\" {\n#endif\n");
printf("#define TOKUDB 1\n");
printf("#define DB_VERSION_MAJOR %d\n", DB_VERSION_MAJOR);
printf("#define DB_VERSION_MINOR %d\n", DB_VERSION_MINOR);
printf("/* As of r40364 (post TokuDB 5.2.7), the patch version number is 100+ the BDB header patch version number.*/\n");
@@ -651,7 +651,6 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
" char data[DB_GID_SIZE];\n"
"} TOKU_XA_XID;\n");
//Typedef toku_off_t
printf("#ifndef TOKU_OFF_T_DEFINED\n"
"#define TOKU_OFF_T_DEFINED\n"
"typedef int64_t toku_off_t;\n"
@@ -670,7 +669,10 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
printf("typedef uint32_t db_recno_t;\n");
printf("typedef int(*YDB_CALLBACK_FUNCTION)(DBT const*, DBT const*, void*);\n");
printf("#include <tdb-internal.h>\n");
printf("struct simple_dbt {\n");
printf(" uint32_t len;\n");
printf(" void *data;\n");
printf("};\n");
//stat64
printf("typedef struct __toku_db_btree_stat64 {\n");

45
storage/tokudb/ft-index/cmake/merge_archives_unix.cmake.in
View File

@@ -26,15 +26,52 @@ SET(CMAKE_RANLIB "@CMAKE_RANLIB@")
SET(TEMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/merge_archives_${TARGET})
MAKE_DIRECTORY(${TEMP_DIR})
# Extract each archive to its own subdirectory(avoid object filename clashes)
# Extract each archive to its own subdirectory(avoid object filename
# clashes) Since the lib may contain objects with the same name, we first
# list the archive contents, then uniquify the object names as we extract
# them.
FOREACH(LIB ${STATIC_LIBS})
GET_FILENAME_COMPONENT(NAME_NO_EXT ${LIB} NAME_WE)
SET(TEMP_SUBDIR ${TEMP_DIR}/${NAME_NO_EXT})
MAKE_DIRECTORY(${TEMP_SUBDIR})
EXECUTE_PROCESS(
COMMAND ${CMAKE_AR} -t ${LIB}
OUTPUT_VARIABLE LIB_OBJS
)
STRING(REGEX REPLACE "\n" ";" LIB_OBJ_LIST "${LIB_OBJS}")
STRING(REGEX REPLACE ";$" "" LIB_OBJ_LIST "${LIB_OBJ_LIST}")
LIST(LENGTH LIB_OBJ_LIST LENGTH_WITH_DUPS)
SET(LIB_OBJ_LIST_NO_DUPS ${LIB_OBJ_LIST})
LIST(REMOVE_DUPLICATES LIB_OBJ_LIST_NO_DUPS)
LIST(LENGTH LIB_OBJ_LIST_NO_DUPS LENGTH_WITHOUT_DUPS)
IF(LENGTH_WITH_DUPS EQUAL LENGTH_WITHOUT_DUPS)
# Optimization for when lib doesn't actually have duplicate object
# names, we can just extract everything.
EXECUTE_PROCESS(
COMMAND ${CMAKE_AR} -x ${LIB}
WORKING_DIRECTORY ${TEMP_SUBDIR}
)
ELSE()
LIST(SORT LIB_OBJ_LIST)
SET(SAME_OBJ_COUNT 1)
SET(LAST_OBJ_NAME)
FOREACH(OBJ ${LIB_OBJ_LIST})
IF(OBJ STREQUAL LAST_OBJ_NAME)
GET_FILENAME_COMPONENT(OBJ_NO_EXT ${OBJ} NAME_WE)
FILE(RENAME "${TEMP_SUBDIR}/${OBJ}" "${TEMP_SUBDIR}/${OBJ_NO_EXT}.${SAME_OBJ_COUNT}.o")
MATH(EXPR SAME_OBJ_COUNT "${SAME_OBJ_COUNT}+1")
ELSE()
SET(SAME_OBJ_COUNT 1)
ENDIF()
SET(LAST_OBJ_NAME "${OBJ}")
EXECUTE_PROCESS(
COMMAND ${CMAKE_AR} -xN ${SAME_OBJ_COUNT} ${LIB} ${OBJ}
WORKING_DIRECTORY ${TEMP_SUBDIR}
)
ENDFOREACH()
ENDIF()
FILE(GLOB_RECURSE LIB_OBJECTS "${TEMP_SUBDIR}/*.o")
SET(OBJECTS ${OBJECTS} ${LIB_OBJECTS})
@@ -51,11 +88,7 @@ ENDFOREACH()
FILE(TO_NATIVE_PATH ${TARGET_LOCATION} ${TARGET_LOCATION})
# Now pack the objects into library with ar.
EXECUTE_PROCESS(
COMMAND ${CMAKE_AR} -r ${TARGET_LOCATION} ${ALL_OBJECTS}
WORKING_DIRECTORY ${TEMP_DIR}
)
EXECUTE_PROCESS(
COMMAND ${CMAKE_RANLIB} ${TARGET_LOCATION}
COMMAND ${CMAKE_AR} rcs ${TARGET_LOCATION} ${ALL_OBJECTS}
WORKING_DIRECTORY ${TEMP_DIR}
)

6
storage/tokudb/ft-index/cmake_modules/TokuBuildTagDatabases.cmake
View File

@@ -7,7 +7,6 @@ file(GLOB_RECURSE all_srcs
locktree/*.cc
portability/*.cc
src/*.cc
toku_include/*.cc
utils/*.cc
util/*.cc
db-benchmark-test/*.cc
@@ -24,13 +23,12 @@ file(GLOB_RECURSE all_hdrs
locktree/*.h
portability/*.h
src/*.h
toku_include/*.h
utils/*.h
util/*.h
db-benchmark-test/*.h
)
list(APPEND all_hdrs
${CMAKE_CURRENT_BINARY_DIR}/toku_include/toku_config.h
${CMAKE_CURRENT_BINARY_DIR}/portability/toku_config.h
${CMAKE_CURRENT_BINARY_DIR}/buildheader/db.h
${CMAKE_CURRENT_BINARY_DIR}/ft/log_header.h
)
@@ -79,7 +77,7 @@ if (USE_CSCOPE)
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/cscope.out"
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/cscope.in.out"
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/cscope.po.out"
COMMAND ${CSCOPE} -b -q -R -i"${CMAKE_CURRENT_BINARY_DIR}/cscope.files" -I"${CMAKE_CURRENT_SOURCE_DIR}" -I"${CMAKE_CURRENT_SOURCE_DIR}/include" -I"${CMAKE_CURRENT_SOURCE_DIR}/toku_include" -I"${CMAKE_CURRENT_SOURCE_DIR}/portability" -I"${CMAKE_CURRENT_SOURCE_DIR}/ft" -I"${CMAKE_CURRENT_SOURCE_DIR}/src" -I"${CMAKE_CURRENT_SOURCE_DIR}/locktree" -I"${CMAKE_CURRENT_SOURCE_DIR}/utils" -I"${CMAKE_CURRENT_SOURCE_DIR}/db-benchmark-test" -I"${CMAKE_CURRENT_BINARY_DIR}" -I"${CMAKE_CURRENT_BINARY_DIR}/toku_include" -I"${CMAKE_CURRENT_BINARY_DIR}/buildheader"
COMMAND ${CSCOPE} -b -q -R -i"${CMAKE_CURRENT_BINARY_DIR}/cscope.files" -I"${CMAKE_CURRENT_SOURCE_DIR}" -I"${CMAKE_CURRENT_SOURCE_DIR}/include" -I"${CMAKE_CURRENT_SOURCE_DIR}/portability" -I"${CMAKE_CURRENT_SOURCE_DIR}/portability" -I"${CMAKE_CURRENT_SOURCE_DIR}/ft" -I"${CMAKE_CURRENT_SOURCE_DIR}/src" -I"${CMAKE_CURRENT_SOURCE_DIR}/locktree" -I"${CMAKE_CURRENT_SOURCE_DIR}/utils" -I"${CMAKE_CURRENT_SOURCE_DIR}/db-benchmark-test" -I"${CMAKE_CURRENT_BINARY_DIR}" -I"${CMAKE_CURRENT_BINARY_DIR}/portability" -I"${CMAKE_CURRENT_BINARY_DIR}/buildheader"
DEPENDS ${all_srcs} ${all_hdrs} install_tdb_h generate_config_h generate_log_code
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
add_custom_target(build_cscope.out ALL DEPENDS

2
storage/tokudb/ft-index/cmake_modules/TokuMergeLibs.cmake
View File

@@ -50,7 +50,7 @@ MACRO(TOKU_MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE)
ENDFOREACH()
IF(OSLIBS)
#LIST(REMOVE_DUPLICATES OSLIBS)
TARGET_LINK_LIBRARIES(${TARGET} ${OSLIBS})
TARGET_LINK_LIBRARIES(${TARGET} LINK_PUBLIC ${OSLIBS})
ENDIF()
# Make the generated dummy source file depended on all static input

2
storage/tokudb/ft-index/cmake_modules/TokuSetupCTest.cmake
View File

@@ -94,8 +94,6 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
## set up full valgrind suppressions file (concatenate the suppressions files)
file(READ ft/valgrind.suppressions valgrind_suppressions)
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${valgrind_suppressions}")
file(READ src/tests/bdb.suppressions bdb_suppressions)
file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${bdb_suppressions}")
file(READ bash.suppressions bash_suppressions)
file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${bash_suppressions}")

3
storage/tokudb/ft-index/ft/CMakeLists.txt
View File

@@ -52,9 +52,7 @@ set(FT_SOURCES
logfilemgr
logger
log_upgrade
memarena
minicron
omt
pqueue
queue
quicklz
@@ -69,7 +67,6 @@ set(FT_SOURCES
txn_child_manager
txn_manager
ule
x1764
xids
ybt
"${CMAKE_CURRENT_BINARY_DIR}/log_code"

5
storage/tokudb/ft-index/ft/background_job_manager.cc
View File

@@ -89,11 +89,12 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2011-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include "background_job_manager.h"
#include "toku_config.h"
#include <portability/toku_config.h>
#include <memory.h>
#include <toku_pthread.h>
#include "background_job_manager.h"
struct background_job_manager_struct {
bool accepting_jobs;
uint32_t num_jobs;

6
storage/tokudb/ft-index/ft/block_table.cc
View File

@@ -427,7 +427,7 @@ toku_ft_unlock (FT ft) {
unlock_for_blocktable(bt);
}
// Also used only in brt-serialize-test.
// Also used only in ft-serialize-test.
void
toku_block_free(BLOCK_TABLE bt, uint64_t offset) {
lock_for_blocktable(bt);
@@ -580,7 +580,7 @@ void toku_serialize_translation_to_wbuf(BLOCK_TABLE bt, int fd, struct wbuf *w,
wbuf_DISKOFF(w, t->block_translation[i].u.diskoff);
wbuf_DISKOFF(w, t->block_translation[i].size);
}
uint32_t checksum = x1764_finish(&w->checksum);
uint32_t checksum = toku_x1764_finish(&w->checksum);
wbuf_int(w, checksum);
*address = t->block_translation[b.b].u.diskoff;
*size = size_translation;
@@ -915,7 +915,7 @@ translation_deserialize_from_buffer(struct translation *t, // destination int
t->type = TRANSLATION_CHECKPOINTED;
{
// check the checksum
uint32_t x1764 = x1764_memory(translation_buffer, size_on_disk - 4);
uint32_t x1764 = toku_x1764_memory(translation_buffer, size_on_disk - 4);
uint64_t offset = size_on_disk - 4;
//printf("%s:%d read from %ld (x1764 offset=%ld) size=%ld\n", __FILE__, __LINE__, block_translation_address_on_disk, offset, block_translation_size_on_disk);
uint32_t stored_x1764 = toku_dtoh32(*(int*)(translation_buffer + offset));

630
storage/tokudb/ft-index/ft/bndata.cc
View File

@@ -90,46 +90,197 @@ PATENT RIGHTS GRANT:
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <bndata.h>
#include <ft-ops.h>
static uint32_t klpair_size(KLPAIR klpair){
return sizeof(*klpair) + klpair->keylen + leafentry_memsize(get_le_from_klpair(klpair));
}
static uint32_t klpair_disksize(KLPAIR klpair){
return sizeof(*klpair) + klpair->keylen + leafentry_disksize(get_le_from_klpair(klpair));
using namespace toku;
uint32_t bn_data::klpair_disksize(const uint32_t klpair_len, const klpair_struct *klpair) const {
return sizeof(*klpair) + keylen_from_klpair_len(klpair_len) + leafentry_disksize(get_le_from_klpair(klpair));
}
void bn_data::init_zero() {
toku_mempool_zero(&m_buffer_mempool);
m_disksize_of_keys = 0;
}
void bn_data::initialize_empty() {
toku_mempool_zero(&m_buffer_mempool);
m_buffer.create_no_array();
init_zero();
m_buffer.create();
}
void bn_data::initialize_from_data(uint32_t num_entries, unsigned char *buf, uint32_t data_size) {
void bn_data::add_key(uint32_t keylen) {
m_disksize_of_keys += sizeof(keylen) + keylen;
}
void bn_data::add_keys(uint32_t n_keys, uint32_t combined_klpair_len) {
invariant(n_keys * sizeof(uint32_t) <= combined_klpair_len);
m_disksize_of_keys += combined_klpair_len;
}
void bn_data::remove_key(uint32_t keylen) {
m_disksize_of_keys -= sizeof(keylen) + keylen;
}
// Deserialize from format optimized for keys being inlined.
// Currently only supports fixed-length keys.
void bn_data::initialize_from_separate_keys_and_vals(uint32_t num_entries, struct rbuf *rb, uint32_t data_size, uint32_t version UU(),
uint32_t key_data_size, uint32_t val_data_size, bool all_keys_same_length,
uint32_t fixed_klpair_length) {
paranoid_invariant(version >= FT_LAYOUT_VERSION_26); // Support was added @26
uint32_t ndone_before = rb->ndone;
init_zero();
invariant(all_keys_same_length); // Until otherwise supported.
bytevec keys_src;
rbuf_literal_bytes(rb, &keys_src, key_data_size);
//Generate dmt
this->m_buffer.create_from_sorted_memory_of_fixed_size_elements(
keys_src, num_entries, key_data_size, fixed_klpair_length);
toku_mempool_construct(&this->m_buffer_mempool, val_data_size);
bytevec vals_src;
rbuf_literal_bytes(rb, &vals_src, val_data_size);
if (num_entries > 0) {
void *vals_dest = toku_mempool_malloc(&this->m_buffer_mempool, val_data_size, 1);
paranoid_invariant_notnull(vals_dest);
memcpy(vals_dest, vals_src, val_data_size);
}
add_keys(num_entries, num_entries * fixed_klpair_length);
toku_note_deserialized_basement_node(all_keys_same_length);
invariant(rb->ndone - ndone_before == data_size);
}
static int
wbufwriteleafentry(const void* key, const uint32_t keylen, const LEAFENTRY &le, const uint32_t UU(idx), struct wbuf * const wb) {
// need to pack the leafentry as it was in versions
// where the key was integrated into it (< 26)
uint32_t begin_spot UU() = wb->ndone;
uint32_t le_disk_size = leafentry_disksize(le);
wbuf_nocrc_uint8_t(wb, le->type);
wbuf_nocrc_uint32_t(wb, keylen);
if (le->type == LE_CLEAN) {
wbuf_nocrc_uint32_t(wb, le->u.clean.vallen);
wbuf_nocrc_literal_bytes(wb, key, keylen);
wbuf_nocrc_literal_bytes(wb, le->u.clean.val, le->u.clean.vallen);
}
else {
paranoid_invariant(le->type == LE_MVCC);
wbuf_nocrc_uint32_t(wb, le->u.mvcc.num_cxrs);
wbuf_nocrc_uint8_t(wb, le->u.mvcc.num_pxrs);
wbuf_nocrc_literal_bytes(wb, key, keylen);
wbuf_nocrc_literal_bytes(wb, le->u.mvcc.xrs, le_disk_size - (1 + 4 + 1));
}
uint32_t end_spot UU() = wb->ndone;
paranoid_invariant((end_spot - begin_spot) == keylen + sizeof(keylen) + le_disk_size);
return 0;
}
void bn_data::serialize_to_wbuf(struct wbuf *const wb) {
prepare_to_serialize();
serialize_header(wb);
if (m_buffer.value_length_is_fixed()) {
serialize_rest(wb);
} else {
//
// iterate over leafentries and place them into the buffer
//
iterate<struct wbuf, wbufwriteleafentry>(wb);
}
}
// If we have fixed-length keys, we prepare the dmt and mempool.
// The mempool is prepared by removing any fragmented space and ordering leafentries in the same order as their keys.
void bn_data::prepare_to_serialize(void) {
if (m_buffer.value_length_is_fixed()) {
m_buffer.prepare_for_serialize();
dmt_compress_kvspace(0, nullptr, true); // Gets it ready for easy serialization.
}
}
void bn_data::serialize_header(struct wbuf *wb) const {
bool fixed = m_buffer.value_length_is_fixed();
//key_data_size
wbuf_nocrc_uint(wb, m_disksize_of_keys);
//val_data_size
wbuf_nocrc_uint(wb, toku_mempool_get_used_size(&m_buffer_mempool));
//fixed_klpair_length
wbuf_nocrc_uint(wb, m_buffer.get_fixed_length());
// all_keys_same_length
wbuf_nocrc_uint8_t(wb, fixed);
// keys_vals_separate
wbuf_nocrc_uint8_t(wb, fixed);
}
void bn_data::serialize_rest(struct wbuf *wb) const {
//Write keys
invariant(m_buffer.value_length_is_fixed()); //Assumes prepare_to_serialize was called
m_buffer.serialize_values(m_disksize_of_keys, wb);
//Write leafentries
//Just ran dmt_compress_kvspace so there is no fragmentation and also leafentries are in sorted order.
paranoid_invariant(toku_mempool_get_frag_size(&m_buffer_mempool) == 0);
uint32_t val_data_size = toku_mempool_get_used_size(&m_buffer_mempool);
wbuf_nocrc_literal_bytes(wb, toku_mempool_get_base(&m_buffer_mempool), val_data_size);
}
// Deserialize from rbuf
void bn_data::deserialize_from_rbuf(uint32_t num_entries, struct rbuf *rb, uint32_t data_size, uint32_t version) {
uint32_t key_data_size = data_size; // overallocate if < version 26 (best guess that is guaranteed not too small)
uint32_t val_data_size = data_size; // overallocate if < version 26 (best guess that is guaranteed not too small)
bool all_keys_same_length = false;
bool keys_vals_separate = false;
uint32_t fixed_klpair_length = 0;
// In version 25 and older there is no header. Skip reading header for old version.
if (version >= FT_LAYOUT_VERSION_26) {
uint32_t ndone_before = rb->ndone;
key_data_size = rbuf_int(rb);
val_data_size = rbuf_int(rb);
fixed_klpair_length = rbuf_int(rb); // 0 if !all_keys_same_length
all_keys_same_length = rbuf_char(rb);
keys_vals_separate = rbuf_char(rb);
invariant(all_keys_same_length == keys_vals_separate); // Until we support otherwise
uint32_t header_size = rb->ndone - ndone_before;
data_size -= header_size;
invariant(header_size == HEADER_LENGTH);
if (keys_vals_separate) {
invariant(fixed_klpair_length >= sizeof(klpair_struct) || num_entries == 0);
initialize_from_separate_keys_and_vals(num_entries, rb, data_size, version,
key_data_size, val_data_size, all_keys_same_length,
fixed_klpair_length);
return;
}
}
// Version >= 26 and version 25 deserialization are now identical except that <= 25 might allocate too much memory.
bytevec bytes;
rbuf_literal_bytes(rb, &bytes, data_size);
const unsigned char *CAST_FROM_VOIDP(buf, bytes);
if (data_size == 0) {
invariant_zero(num_entries);
}
KLPAIR *XMALLOC_N(num_entries, array); // create array of pointers to leafentries
unsigned char *newmem = NULL;
// add same wiggle room that toku_mempool_construct would, 25% extra
uint32_t allocated_bytes = data_size + data_size/4;
CAST_FROM_VOIDP(newmem, toku_xmalloc(allocated_bytes));
unsigned char* curr_src_pos = buf;
init_zero();
klpair_dmt_t::builder dmt_builder;
dmt_builder.create(num_entries, key_data_size);
// TODO(leif): clean this up (#149)
unsigned char *newmem = nullptr;
// add 25% extra wiggle room
uint32_t allocated_bytes_vals = val_data_size + (val_data_size / 4);
CAST_FROM_VOIDP(newmem, toku_xmalloc(allocated_bytes_vals));
const unsigned char* curr_src_pos = buf;
unsigned char* curr_dest_pos = newmem;
for (uint32_t i = 0; i < num_entries; i++) {
KLPAIR curr_kl = (KLPAIR)curr_dest_pos;
array[i] = curr_kl;
uint8_t curr_type = curr_src_pos[0];
curr_src_pos++;
// first thing we do is lay out the key,
// to do so, we must extract it from the leafentry
// and write it in
uint32_t keylen = 0;
void* keyp = NULL;
const void* keyp = nullptr;
keylen = *(uint32_t *)curr_src_pos;
curr_src_pos += sizeof(uint32_t);
uint32_t clean_vallen = 0;
@@ -150,12 +301,10 @@ void bn_data::initialize_from_data(uint32_t num_entries, unsigned char *buf, uin
keyp = curr_src_pos;
curr_src_pos += keylen;
}
// now that we have the keylen and the key, we can copy it
// into the destination
*(uint32_t *)curr_dest_pos = keylen;
curr_dest_pos += sizeof(keylen);
memcpy(curr_dest_pos, keyp, keylen);
curr_dest_pos += keylen;
uint32_t le_offset = curr_dest_pos - newmem;
dmt_builder.append(klpair_dmtwriter(keylen, le_offset, keyp));
add_key(keylen);
// now curr_dest_pos is pointing to where the leafentry should be packed
curr_dest_pos[0] = curr_type;
curr_dest_pos++;
@@ -173,31 +322,44 @@ void bn_data::initialize_from_data(uint32_t num_entries, unsigned char *buf, uin
*(uint8_t *)curr_dest_pos = num_pxrs;
curr_dest_pos += sizeof(num_pxrs);
// now we need to pack the rest of the data
uint32_t num_rest_bytes = leafentry_rest_memsize(num_pxrs, num_cxrs, curr_src_pos);
uint32_t num_rest_bytes = leafentry_rest_memsize(num_pxrs, num_cxrs, const_cast<uint8_t*>(curr_src_pos));
memcpy(curr_dest_pos, curr_src_pos, num_rest_bytes);
curr_dest_pos += num_rest_bytes;
curr_src_pos += num_rest_bytes;
}
}
uint32_t num_bytes_read UU() = (uint32_t)(curr_src_pos - buf);
paranoid_invariant( num_bytes_read == data_size);
uint32_t num_bytes_written = curr_dest_pos - newmem;
paranoid_invariant( num_bytes_written == data_size);
toku_mempool_init(&m_buffer_mempool, newmem, (size_t)(num_bytes_written), allocated_bytes);
dmt_builder.build(&this->m_buffer);
toku_note_deserialized_basement_node(m_buffer.value_length_is_fixed());
// destroy old omt that was created by toku_create_empty_bn(), so we can create a new one
m_buffer.destroy();
m_buffer.create_steal_sorted_array(&array, num_entries, num_entries);
uint32_t num_bytes_read = (uint32_t)(curr_src_pos - buf);
invariant(num_bytes_read == data_size);
uint32_t num_bytes_written = curr_dest_pos - newmem + m_disksize_of_keys;
invariant(num_bytes_written == data_size);
toku_mempool_init(&m_buffer_mempool, newmem, (size_t)(curr_dest_pos - newmem), allocated_bytes_vals);
invariant(get_disk_size() == data_size);
// Versions older than 26 might have allocated too much memory. Try to shrink the mempool now that we
// know how much memory we need.
if (version < FT_LAYOUT_VERSION_26) {
// Unnecessary after version 26
// Reallocate smaller mempool to save memory
invariant_zero(toku_mempool_get_frag_size(&m_buffer_mempool));
toku_mempool_realloc_larger(&m_buffer_mempool, toku_mempool_get_used_size(&m_buffer_mempool));
}
}
uint64_t bn_data::get_memory_size() {
uint64_t retval = 0;
//TODO: Maybe ask for memory_size instead of mempool_footprint (either this todo or the next)
// include fragmentation overhead but do not include space in the
// mempool that has not yet been allocated for leaf entries
size_t poolsize = toku_mempool_footprint(&m_buffer_mempool);
invariant(poolsize >= get_disk_size());
retval += poolsize;
// This one includes not-yet-allocated for nodes (just like old constant-key omt)
//TODO: Maybe ask for mempool_footprint instead of memory_size.
retval += m_buffer.memory_size();
invariant(retval >= get_disk_size());
return retval;
}
@@ -207,167 +369,261 @@ void bn_data::delete_leafentry (
uint32_t old_le_size
)
{
remove_key(keylen);
m_buffer.delete_at(idx);
toku_mempool_mfree(&m_buffer_mempool, 0, old_le_size + keylen + sizeof(keylen)); // Must pass 0, since le is no good any more.
toku_mempool_mfree(&m_buffer_mempool, nullptr, old_le_size);
}
/* mempool support */
struct omt_compressor_state {
struct dmt_compressor_state {
struct mempool *new_kvspace;
KLPAIR *newvals;
class bn_data *bd;
};
static int move_it (const KLPAIR &klpair, const uint32_t idx, struct omt_compressor_state * const oc) {
uint32_t size = klpair_size(klpair);
KLPAIR CAST_FROM_VOIDP(newdata, toku_mempool_malloc(oc->new_kvspace, size, 1));
static int move_it (const uint32_t, klpair_struct *klpair, const uint32_t idx UU(), struct dmt_compressor_state * const oc) {
LEAFENTRY old_le = oc->bd->get_le_from_klpair(klpair);
uint32_t size = leafentry_memsize(old_le);
void* newdata = toku_mempool_malloc(oc->new_kvspace, size, 1);
paranoid_invariant_notnull(newdata); // we do this on a fresh mempool, so nothing bad should happen
memcpy(newdata, klpair, size);
oc->newvals[idx] = newdata;
memcpy(newdata, old_le, size);
klpair->le_offset = toku_mempool_get_offset_from_pointer_and_base(oc->new_kvspace, newdata);
return 0;
}
// Compress things, and grow the mempool if needed.
void bn_data::omt_compress_kvspace(size_t added_size, void **maybe_free) {
uint32_t total_size_needed = toku_mempool_get_used_space(&m_buffer_mempool) + added_size;
// set the new mempool size to be twice of the space we actually need.
// On top of the 25% that is padded within toku_mempool_construct (which we
// should consider getting rid of), that should be good enough.
// Compress things, and grow or shrink the mempool if needed.
// May (always if force_compress) have a side effect of putting contents of mempool in sorted order.
void bn_data::dmt_compress_kvspace(size_t added_size, void **maybe_free, bool force_compress) {
uint32_t total_size_needed = toku_mempool_get_used_size(&m_buffer_mempool) + added_size;
// If there is no fragmentation, e.g. in serial inserts, we can just increase the size
// of the mempool and move things over with a cheap memcpy. If force_compress is true,
// the caller needs the side effect that all contents are put in sorted order.
bool do_compress = toku_mempool_get_frag_size(&m_buffer_mempool) > 0 || force_compress;
void *old_mempool_base = toku_mempool_get_base(&m_buffer_mempool);
struct mempool new_kvspace;
toku_mempool_construct(&new_kvspace, 2*total_size_needed);
uint32_t numvals = omt_size();
KLPAIR *XMALLOC_N(numvals, newvals);
struct omt_compressor_state oc = { &new_kvspace, newvals };
m_buffer.iterate_on_range< decltype(oc), move_it >(0, omt_size(), &oc);
m_buffer.destroy();
m_buffer.create_steal_sorted_array(&newvals, numvals, numvals);
if (do_compress) {
size_t requested_size = force_compress ? total_size_needed : ((total_size_needed * 3) / 2);
toku_mempool_construct(&new_kvspace, requested_size);
struct dmt_compressor_state oc = { &new_kvspace, this };
m_buffer.iterate_ptr< decltype(oc), move_it >(&oc);
} else {
toku_mempool_construct(&new_kvspace, total_size_needed);
size_t old_offset_limit = toku_mempool_get_offset_limit(&m_buffer_mempool);
void *new_mempool_base = toku_mempool_malloc(&new_kvspace, old_offset_limit, 1);
memcpy(new_mempool_base, old_mempool_base, old_offset_limit);
}
if (maybe_free) {
*maybe_free = m_buffer_mempool.base;
*maybe_free = old_mempool_base;
} else {
toku_free(m_buffer_mempool.base);
toku_free(old_mempool_base);
}
m_buffer_mempool = new_kvspace;
}
// Effect: Allocate a new object of size SIZE in MP. If MP runs out of space, allocate new a new mempool space, and copy all the items
// from the OMT (which items refer to items in the old mempool) into the new mempool.
// If MAYBE_FREE is NULL then free the old mempool's space.
// If MAYBE_FREE is nullptr then free the old mempool's space.
// Otherwise, store the old mempool's space in maybe_free.
KLPAIR bn_data::mempool_malloc_from_omt(size_t size, void **maybe_free) {
LEAFENTRY bn_data::mempool_malloc_and_update_dmt(size_t size, void **maybe_free) {
void *v = toku_mempool_malloc(&m_buffer_mempool, size, 1);
if (v == NULL) {
omt_compress_kvspace(size, maybe_free);
if (v == nullptr) {
dmt_compress_kvspace(size, maybe_free, false);
v = toku_mempool_malloc(&m_buffer_mempool, size, 1);
paranoid_invariant_notnull(v);
}
return (KLPAIR)v;
return (LEAFENTRY)v;
}
//TODO: probably not free the "maybe_free" right away?
void bn_data::get_space_for_overwrite(
uint32_t idx,
const void* keyp,
uint32_t keylen,
const void* keyp UU(),
uint32_t keylen UU(),
uint32_t old_le_size,
uint32_t new_size,
LEAFENTRY* new_le_space
LEAFENTRY* new_le_space,
void **const maybe_free
)
{
void* maybe_free = nullptr;
uint32_t size_alloc = new_size + keylen + sizeof(keylen);
KLPAIR new_kl = mempool_malloc_from_omt(
size_alloc,
&maybe_free
);
uint32_t size_freed = old_le_size + keylen + sizeof(keylen);
toku_mempool_mfree(&m_buffer_mempool, nullptr, size_freed); // Must pass nullptr, since le is no good any more.
new_kl->keylen = keylen;
memcpy(new_kl->key_le, keyp, keylen);
m_buffer.set_at(new_kl, idx);
*new_le_space = get_le_from_klpair(new_kl);
// free at end, so that the keyp and keylen
// passed in is still valid
if (maybe_free) {
toku_free(maybe_free);
}
*maybe_free = nullptr;
LEAFENTRY new_le = mempool_malloc_and_update_dmt(new_size, maybe_free);
toku_mempool_mfree(&m_buffer_mempool, nullptr, old_le_size);
klpair_struct* klp = nullptr;
uint32_t klpair_len;
int r = m_buffer.fetch(idx, &klpair_len, &klp);
invariant_zero(r);
paranoid_invariant(klp!=nullptr);
// Key never changes.
paranoid_invariant(keylen_from_klpair_len(klpair_len) == keylen);
size_t new_le_offset = toku_mempool_get_offset_from_pointer_and_base(&this->m_buffer_mempool, new_le);
paranoid_invariant(new_le_offset <= UINT32_MAX - new_size); // Not using > 4GB
klp->le_offset = new_le_offset;
paranoid_invariant(new_le == get_le_from_klpair(klp));
*new_le_space = new_le;
}
//TODO: probably not free the "maybe_free" right away?
void bn_data::get_space_for_insert(
uint32_t idx,
const void* keyp,
uint32_t keylen,
size_t size,
LEAFENTRY* new_le_space
LEAFENTRY* new_le_space,
void **const maybe_free
)
{
void* maybe_free = nullptr;
uint32_t size_alloc = size + keylen + sizeof(keylen);
KLPAIR new_kl = mempool_malloc_from_omt(
size_alloc,
&maybe_free
);
new_kl->keylen = keylen;
memcpy(new_kl->key_le, keyp, keylen);
m_buffer.insert_at(new_kl, idx);
*new_le_space = get_le_from_klpair(new_kl);
// free at end, so that the keyp and keylen
// passed in is still valid (you never know if
// it was part of the old mempool, this is just
// safer).
if (maybe_free) {
toku_free(maybe_free);
}
add_key(keylen);
*maybe_free = nullptr;
LEAFENTRY new_le = mempool_malloc_and_update_dmt(size, maybe_free);
size_t new_le_offset = toku_mempool_get_offset_from_pointer_and_base(&this->m_buffer_mempool, new_le);
klpair_dmtwriter kl(keylen, new_le_offset, keyp);
m_buffer.insert_at(kl, idx);
*new_le_space = new_le;
}
void bn_data::move_leafentries_to(
BN_DATA dest_bd,
uint32_t lbi, //lower bound inclusive
uint32_t ube //upper bound exclusive
class split_klpairs_extra {
bn_data *const m_left_bn;
bn_data *const m_right_bn;
klpair_dmt_t::builder *const m_left_builder;
klpair_dmt_t::builder *const m_right_builder;
struct mempool *const m_left_dest_mp;
uint32_t m_split_at;
struct mempool *left_dest_mp(void) const { return m_left_dest_mp; }
struct mempool *right_dest_mp(void) const { return &m_right_bn->m_buffer_mempool; }
void copy_klpair(const uint32_t klpair_len, const klpair_struct &klpair,
klpair_dmt_t::builder *const builder,
struct mempool *const dest_mp,
bn_data *const bn) {
LEAFENTRY old_le = m_left_bn->get_le_from_klpair(&klpair);
size_t le_size = leafentry_memsize(old_le);
void *new_le = toku_mempool_malloc(dest_mp, le_size, 1);
paranoid_invariant_notnull(new_le);
memcpy(new_le, old_le, le_size);
size_t le_offset = toku_mempool_get_offset_from_pointer_and_base(dest_mp, new_le);
size_t keylen = keylen_from_klpair_len(klpair_len);
builder->append(klpair_dmtwriter(keylen, le_offset, klpair.key));
bn->add_key(keylen);
}
int move_leafentry(const uint32_t klpair_len, const klpair_struct &klpair, const uint32_t idx) {
m_left_bn->remove_key(keylen_from_klpair_len(klpair_len));
if (idx < m_split_at) {
copy_klpair(klpair_len, klpair, m_left_builder, left_dest_mp(), m_left_bn);
} else {
copy_klpair(klpair_len, klpair, m_right_builder, right_dest_mp(), m_right_bn);
}
return 0;
}
public:
split_klpairs_extra(bn_data *const left_bn, bn_data *const right_bn,
klpair_dmt_t::builder *const left_builder,
klpair_dmt_t::builder *const right_builder,
struct mempool *const left_new_mp,
uint32_t split_at)
: m_left_bn(left_bn),
m_right_bn(right_bn),
m_left_builder(left_builder),
m_right_builder(right_builder),
m_left_dest_mp(left_new_mp),
m_split_at(split_at) {}
static int cb(const uint32_t klpair_len, const klpair_struct &klpair, const uint32_t idx, split_klpairs_extra *const thisp) {
return thisp->move_leafentry(klpair_len, klpair, idx);
}
};
void bn_data::split_klpairs(
bn_data* right_bd,
uint32_t split_at //lower bound inclusive for right_bd
)
//Effect: move leafentries in the range [lbi, ube) from this to src_omt to newly created dest_omt
{
paranoid_invariant(lbi < ube);
paranoid_invariant(ube <= omt_size());
KLPAIR *XMALLOC_N(ube-lbi, newklpointers); // create new omt
// We use move_leafentries_to during a split, and the split algorithm should never call this
// if it's splitting on a boundary, so there must be some leafentries in the range to move.
paranoid_invariant(split_at < num_klpairs());
size_t mpsize = toku_mempool_get_used_space(&m_buffer_mempool); // overkill, but safe
struct mempool *dest_mp = &dest_bd->m_buffer_mempool;
struct mempool *src_mp = &m_buffer_mempool;
toku_mempool_construct(dest_mp, mpsize);
right_bd->init_zero();
uint32_t i = 0;
for (i = lbi; i < ube; i++) {
KLPAIR curr_kl;
m_buffer.fetch(i, &curr_kl);
size_t mpsize = toku_mempool_get_used_size(&m_buffer_mempool); // overkill, but safe
size_t kl_size = klpair_size(curr_kl);
KLPAIR new_kl = NULL;
CAST_FROM_VOIDP(new_kl, toku_mempool_malloc(dest_mp, kl_size, 1));
memcpy(new_kl, curr_kl, kl_size);
newklpointers[i-lbi] = new_kl;
toku_mempool_mfree(src_mp, curr_kl, kl_size);
}
struct mempool new_left_mp;
toku_mempool_construct(&new_left_mp, mpsize);
dest_bd->m_buffer.create_steal_sorted_array(&newklpointers, ube-lbi, ube-lbi);
// now remove the elements from src_omt
for (i=ube-1; i >= lbi; i--) {
m_buffer.delete_at(i);
}
struct mempool *right_mp = &right_bd->m_buffer_mempool;
toku_mempool_construct(right_mp, mpsize);
klpair_dmt_t::builder left_dmt_builder;
left_dmt_builder.create(split_at, m_disksize_of_keys); // overkill, but safe (builder will realloc at the end)
klpair_dmt_t::builder right_dmt_builder;
right_dmt_builder.create(num_klpairs() - split_at, m_disksize_of_keys); // overkill, but safe (builder will realloc at the end)
split_klpairs_extra extra(this, right_bd, &left_dmt_builder, &right_dmt_builder, &new_left_mp, split_at);
int r = m_buffer.iterate<split_klpairs_extra, split_klpairs_extra::cb>(&extra);
invariant_zero(r);
m_buffer.destroy();
toku_mempool_destroy(&m_buffer_mempool);
m_buffer_mempool = new_left_mp;
left_dmt_builder.build(&m_buffer);
right_dmt_builder.build(&right_bd->m_buffer);
// Potentially shrink memory pool for destination.
// We overallocated ("overkill") above
struct mempool *const left_mp = &m_buffer_mempool;
paranoid_invariant_zero(toku_mempool_get_frag_size(left_mp));
toku_mempool_realloc_larger(left_mp, toku_mempool_get_used_size(left_mp));
paranoid_invariant_zero(toku_mempool_get_frag_size(right_mp));
toku_mempool_realloc_larger(right_mp, toku_mempool_get_used_size(right_mp));
}
uint64_t bn_data::get_disk_size() {
return toku_mempool_get_used_space(&m_buffer_mempool);
return m_disksize_of_keys +
toku_mempool_get_used_size(&m_buffer_mempool);
}
struct verify_le_in_mempool_state {
size_t offset_limit;
class bn_data *bd;
};
static int verify_le_in_mempool (const uint32_t, klpair_struct *klpair, const uint32_t idx UU(), struct verify_le_in_mempool_state * const state) {
invariant(klpair->le_offset < state->offset_limit);
LEAFENTRY le = state->bd->get_le_from_klpair(klpair);
uint32_t size = leafentry_memsize(le);
size_t end_offset = klpair->le_offset+size;
invariant(end_offset <= state->offset_limit);
return 0;
}
//This is a debug-only (paranoid) verification.
//Verifies the dmt is valid, and all leafentries are entirely in the mempool's memory.
void bn_data::verify_mempool(void) {
// TODO: implement something
//Verify the dmt itself <- paranoid and slow
m_buffer.verify();
verify_le_in_mempool_state state = { .offset_limit = toku_mempool_get_offset_limit(&m_buffer_mempool), .bd = this };
//Verify every leafentry pointed to by the keys in the dmt are fully inside the mempool
m_buffer.iterate_ptr< decltype(state), verify_le_in_mempool >(&state);
}
uint32_t bn_data::omt_size(void) const {
uint32_t bn_data::num_klpairs(void) const {
return m_buffer.size();
}
@@ -375,40 +631,54 @@ void bn_data::destroy(void) {
// The buffer may have been freed already, in some cases.
m_buffer.destroy();
toku_mempool_destroy(&m_buffer_mempool);
m_disksize_of_keys = 0;
}
//TODO: Splitting key/val requires changing this
void bn_data::replace_contents_with_clone_of_sorted_array(
void bn_data::set_contents_as_clone_of_sorted_array(
uint32_t num_les,
const void** old_key_ptrs,
uint32_t* old_keylens,
LEAFENTRY* old_les,
size_t *le_sizes,
size_t mempool_size
size_t total_key_size,
size_t total_le_size
)
{
toku_mempool_construct(&m_buffer_mempool, mempool_size);
KLPAIR *XMALLOC_N(num_les, le_array);
for (uint32_t idx = 0; idx < num_les; idx++) {
KLPAIR new_kl = (KLPAIR)toku_mempool_malloc(
&m_buffer_mempool,
le_sizes[idx] + old_keylens[idx] + sizeof(uint32_t),
1); // point to new location
new_kl->keylen = old_keylens[idx];
memcpy(new_kl->key_le, old_key_ptrs[idx], new_kl->keylen);
memcpy(get_le_from_klpair(new_kl), old_les[idx], le_sizes[idx]);
CAST_FROM_VOIDP(le_array[idx], new_kl);
}
//TODO: Splitting key/val requires changing this; keys are stored in old omt.. cannot delete it yet?
//Enforce "just created" invariant.
paranoid_invariant_zero(m_disksize_of_keys);
paranoid_invariant_zero(num_klpairs());
paranoid_invariant_null(toku_mempool_get_base(&m_buffer_mempool));
paranoid_invariant_zero(toku_mempool_get_size(&m_buffer_mempool));
toku_mempool_construct(&m_buffer_mempool, total_le_size);
m_buffer.destroy();
m_buffer.create_steal_sorted_array(&le_array, num_les, num_les);
m_disksize_of_keys = 0;
klpair_dmt_t::builder dmt_builder;
dmt_builder.create(num_les, total_key_size);
for (uint32_t idx = 0; idx < num_les; idx++) {
void* new_le = toku_mempool_malloc(&m_buffer_mempool, le_sizes[idx], 1);
paranoid_invariant_notnull(new_le);
memcpy(new_le, old_les[idx], le_sizes[idx]);
size_t le_offset = toku_mempool_get_offset_from_pointer_and_base(&m_buffer_mempool, new_le);
dmt_builder.append(klpair_dmtwriter(old_keylens[idx], le_offset, old_key_ptrs[idx]));
add_key(old_keylens[idx]);
}
dmt_builder.build(&this->m_buffer);
}
LEAFENTRY bn_data::get_le_from_klpair(const klpair_struct *klpair) const {
void * ptr = toku_mempool_get_pointer_from_base_and_offset(&this->m_buffer_mempool, klpair->le_offset);
LEAFENTRY CAST_FROM_VOIDP(le, ptr);
return le;
}
// get info about a single leafentry by index
int bn_data::fetch_le(uint32_t idx, LEAFENTRY *le) {
KLPAIR klpair = NULL;
int r = m_buffer.fetch(idx, &klpair);
klpair_struct* klpair = nullptr;
int r = m_buffer.fetch(idx, nullptr, &klpair);
if (r == 0) {
*le = get_le_from_klpair(klpair);
}
@@ -416,59 +686,41 @@ int bn_data::fetch_le(uint32_t idx, LEAFENTRY *le) {
}
int bn_data::fetch_klpair(uint32_t idx, LEAFENTRY *le, uint32_t *len, void** key) {
KLPAIR klpair = NULL;
int r = m_buffer.fetch(idx, &klpair);
klpair_struct* klpair = nullptr;
uint32_t klpair_len;
int r = m_buffer.fetch(idx, &klpair_len, &klpair);
if (r == 0) {
*len = klpair->keylen;
*key = klpair->key_le;
*len = keylen_from_klpair_len(klpair_len);
*key = klpair->key;
*le = get_le_from_klpair(klpair);
}
return r;
}
int bn_data::fetch_klpair_disksize(uint32_t idx, size_t *size) {
KLPAIR klpair = NULL;
int r = m_buffer.fetch(idx, &klpair);
klpair_struct* klpair = nullptr;
uint32_t klpair_len;
int r = m_buffer.fetch(idx, &klpair_len, &klpair);
if (r == 0) {
*size = klpair_disksize(klpair);
*size = klpair_disksize(klpair_len, klpair);
}
return r;
}
int bn_data::fetch_le_key_and_len(uint32_t idx, uint32_t *len, void** key) {
KLPAIR klpair = NULL;
int r = m_buffer.fetch(idx, &klpair);
int bn_data::fetch_key_and_len(uint32_t idx, uint32_t *len, void** key) {
klpair_struct* klpair = nullptr;
uint32_t klpair_len;
int r = m_buffer.fetch(idx, &klpair_len, &klpair);
if (r == 0) {
*len = klpair->keylen;
*key = klpair->key_le;
*len = keylen_from_klpair_len(klpair_len);
*key = klpair->key;
}
return r;
}
struct mp_pair {
void* orig_base;
void* new_base;
klpair_omt_t* omt;
};
static int fix_mp_offset(const KLPAIR &klpair, const uint32_t idx, struct mp_pair * const p) {
char* old_value = (char *) klpair;
char *new_value = old_value - (char *)p->orig_base + (char *)p->new_base;
p->omt->set_at((KLPAIR)new_value, idx);
return 0;
}
void bn_data::clone(bn_data* orig_bn_data) {
toku_mempool_clone(&orig_bn_data->m_buffer_mempool, &m_buffer_mempool);
m_buffer.clone(orig_bn_data->m_buffer);
struct mp_pair p;
p.orig_base = toku_mempool_get_base(&orig_bn_data->m_buffer_mempool);
p.new_base = toku_mempool_get_base(&m_buffer_mempool);
p.omt = &m_buffer;
int r = m_buffer.iterate_on_range<decltype(p), fix_mp_offset>(0, omt_size(), &p);
invariant_zero(r);
this->m_disksize_of_keys = orig_bn_data->m_disksize_of_keys;
}

292
storage/tokudb/ft-index/ft/bndata.h
View File

@@ -91,166 +91,296 @@ PATENT RIGHTS GRANT:
#pragma once
#include <util/omt.h>
#include "leafentry.h"
#include <util/mempool.h>
#include "wbuf.h"
#include <util/dmt.h>
#include "leafentry.h"
#if 0 //for implementation
static int
UU() verify_in_mempool(OMTVALUE lev, uint32_t UU(idx), void *mpv)
{
LEAFENTRY CAST_FROM_VOIDP(le, lev);
struct mempool *CAST_FROM_VOIDP(mp, mpv);
int r = toku_mempool_inrange(mp, le, leafentry_memsize(le));
lazy_assert(r);
return 0;
}
toku_omt_iterate(bn->buffer, verify_in_mempool, &bn->buffer_mempool);
#endif
// Key/leafentry pair stored in a dmt. The key is inlined, the offset (in leafentry mempool) is stored for the leafentry.
struct klpair_struct {
uint32_t keylen;
uint8_t key_le[0]; // key, followed by le
uint32_t le_offset; //Offset of leafentry (in leafentry mempool)
uint8_t key[0]; // key, followed by le
};
typedef struct klpair_struct *KLPAIR;
static inline LEAFENTRY get_le_from_klpair(KLPAIR klpair){
uint32_t keylen = klpair->keylen;
LEAFENTRY le = (LEAFENTRY)(klpair->key_le + keylen);
return le;
static constexpr uint32_t keylen_from_klpair_len(const uint32_t klpair_len) {
return klpair_len - __builtin_offsetof(klpair_struct, key);
}
template<typename omtcmp_t,
int (*h)(const DBT &, const omtcmp_t &)>
static int wrappy_fun_find(const KLPAIR &klpair, const omtcmp_t &extra) {
//TODO: kill this function when we split, and/or use toku_fill_dbt
static_assert(__builtin_offsetof(klpair_struct, key) == 1*sizeof(uint32_t), "klpair alignment issues");
static_assert(__builtin_offsetof(klpair_struct, key) == sizeof(klpair_struct), "klpair size issues");
// A wrapper for the heaviside function provided to dmt->find*.
// Needed because the heaviside functions provided to bndata do not know about the internal types.
// Alternative to this wrapper is to expose accessor functions and rewrite all the external heaviside functions.
template<typename dmtcmp_t,
int (*h)(const DBT &, const dmtcmp_t &)>
static int klpair_find_wrapper(const uint32_t klpair_len, const klpair_struct &klpair, const dmtcmp_t &extra) {
DBT kdbt;
kdbt.data = klpair->key_le;
kdbt.size = klpair->keylen;
kdbt.data = const_cast<void*>(reinterpret_cast<const void*>(klpair.key));
kdbt.size = keylen_from_klpair_len(klpair_len);
return h(kdbt, extra);
}
template<typename inner_iterate_extra_t>
struct klpair_iterate_extra {
public:
inner_iterate_extra_t *inner;
const class bn_data * bd;
};
// A wrapper for the high-order function provided to dmt->iterate*
// Needed because the heaviside functions provided to bndata do not know about the internal types.
// Alternative to this wrapper is to expose accessor functions and rewrite all the external heaviside functions.
template<typename iterate_extra_t,
int (*h)(const void * key, const uint32_t keylen, const LEAFENTRY &, const uint32_t idx, iterate_extra_t *const)>
static int wrappy_fun_iterate(const KLPAIR &klpair, const uint32_t idx, iterate_extra_t *const extra) {
uint32_t keylen = klpair->keylen;
void* key = klpair->key_le;
LEAFENTRY le = get_le_from_klpair(klpair);
return h(key, keylen, le, idx, extra);
int (*f)(const void * key, const uint32_t keylen, const LEAFENTRY &, const uint32_t idx, iterate_extra_t *const)>
static int klpair_iterate_wrapper(const uint32_t klpair_len, const klpair_struct &klpair, const uint32_t idx, klpair_iterate_extra<iterate_extra_t> *const extra) {
const void* key = &klpair.key;
LEAFENTRY le = extra->bd->get_le_from_klpair(&klpair);
return f(key, keylen_from_klpair_len(klpair_len), le, idx, extra->inner);
}
typedef toku::omt<KLPAIR> klpair_omt_t;
namespace toku {
// dmt writer for klpair_struct
class klpair_dmtwriter {
public:
// Return the size needed for the klpair_struct that this dmtwriter represents
size_t get_size(void) const {
return sizeof(klpair_struct) + this->keylen;
}
// Write the klpair_struct this dmtwriter represents to a destination
void write_to(klpair_struct *const dest) const {
dest->le_offset = this->le_offset;
memcpy(dest->key, this->keyp, this->keylen);
}
klpair_dmtwriter(uint32_t _keylen, uint32_t _le_offset, const void* _keyp)
: keylen(_keylen), le_offset(_le_offset), keyp(_keyp) {}
klpair_dmtwriter(const uint32_t klpair_len, klpair_struct *const src)
: keylen(keylen_from_klpair_len(klpair_len)), le_offset(src->le_offset), keyp(src->key) {}
private:
const uint32_t keylen;
const uint32_t le_offset;
const void* keyp;
};
}
typedef toku::dmt<klpair_struct, klpair_struct*, toku::klpair_dmtwriter> klpair_dmt_t;
// This class stores the data associated with a basement node
class bn_data {
public:
// Initialize an empty bn_data _without_ a dmt backing.
// Externally only used for deserialization.
void init_zero(void);
// Initialize an empty bn_data _with_ a dmt
void initialize_empty(void);
void initialize_from_data(uint32_t num_entries, unsigned char *buf, uint32_t data_size);
// globals
// Deserialize a bn_data from rbuf.
// This is the entry point for deserialization.
void deserialize_from_rbuf(uint32_t num_entries, struct rbuf *rb, uint32_t data_size, uint32_t version);
// Retrieve the memory footprint of this basement node.
// May over or under count: see Tokutek/ft-index#136
// Also see dmt's implementation.
uint64_t get_memory_size(void);
// Get the serialized size of this basement node.
uint64_t get_disk_size(void);
// Perform (paranoid) verification that all leafentries are fully contained within the mempool
void verify_mempool(void);
// Interact with "omt"
uint32_t omt_size(void) const;
// size() of key dmt
uint32_t num_klpairs(void) const;
// iterate() on key dmt (and associated leafentries)
template<typename iterate_extra_t,
int (*f)(const void * key, const uint32_t keylen, const LEAFENTRY &, const uint32_t, iterate_extra_t *const)>
int omt_iterate(iterate_extra_t *const iterate_extra) const {
return omt_iterate_on_range<iterate_extra_t, f>(0, omt_size(), iterate_extra);
int iterate(iterate_extra_t *const iterate_extra) const {
return iterate_on_range<iterate_extra_t, f>(0, num_klpairs(), iterate_extra);
}
// iterate_on_range() on key dmt (and associated leafentries)
template<typename iterate_extra_t,
int (*f)(const void * key, const uint32_t keylen, const LEAFENTRY &, const uint32_t, iterate_extra_t *const)>
int omt_iterate_on_range(const uint32_t left, const uint32_t right, iterate_extra_t *const iterate_extra) const {
return m_buffer.iterate_on_range< iterate_extra_t, wrappy_fun_iterate<iterate_extra_t, f> >(left, right, iterate_extra);
int iterate_on_range(const uint32_t left, const uint32_t right, iterate_extra_t *const iterate_extra) const {
klpair_iterate_extra<iterate_extra_t> klpair_extra = { iterate_extra, this };
return m_buffer.iterate_on_range< klpair_iterate_extra<iterate_extra_t>, klpair_iterate_wrapper<iterate_extra_t, f> >(left, right, &klpair_extra);
}
template<typename omtcmp_t,
int (*h)(const DBT &, const omtcmp_t &)>
int find_zero(const omtcmp_t &extra, LEAFENTRY *const value, void** key, uint32_t* keylen, uint32_t *const idxp) const {
KLPAIR klpair = NULL;
int r = m_buffer.find_zero< omtcmp_t, wrappy_fun_find<omtcmp_t, h> >(extra, &klpair, idxp);
// find_zero() on key dmt
template<typename dmtcmp_t,
int (*h)(const DBT &, const dmtcmp_t &)>
int find_zero(const dmtcmp_t &extra, LEAFENTRY *const value, void** key, uint32_t* keylen, uint32_t *const idxp) const {
klpair_struct* klpair = nullptr;
uint32_t klpair_len;
int r = m_buffer.find_zero< dmtcmp_t, klpair_find_wrapper<dmtcmp_t, h> >(extra, &klpair_len, &klpair, idxp);
if (r == 0) {
if (value) {
*value = get_le_from_klpair(klpair);
}
if (key) {
paranoid_invariant(keylen != NULL);
*key = klpair->key_le;
*keylen = klpair->keylen;
paranoid_invariant_notnull(keylen);
*key = klpair->key;
*keylen = keylen_from_klpair_len(klpair_len);
}
else {
paranoid_invariant(keylen == NULL);
paranoid_invariant_null(keylen);
}
}
return r;
}
template<typename omtcmp_t,
int (*h)(const DBT &, const omtcmp_t &)>
int find(const omtcmp_t &extra, int direction, LEAFENTRY *const value, void** key, uint32_t* keylen, uint32_t *const idxp) const {
KLPAIR klpair = NULL;
int r = m_buffer.find< omtcmp_t, wrappy_fun_find<omtcmp_t, h> >(extra, direction, &klpair, idxp);
// find() on key dmt (and associated leafentries)
template<typename dmtcmp_t,
int (*h)(const DBT &, const dmtcmp_t &)>
int find(const dmtcmp_t &extra, int direction, LEAFENTRY *const value, void** key, uint32_t* keylen, uint32_t *const idxp) const {
klpair_struct* klpair = nullptr;
uint32_t klpair_len;
int r = m_buffer.find< dmtcmp_t, klpair_find_wrapper<dmtcmp_t, h> >(extra, direction, &klpair_len, &klpair, idxp);
if (r == 0) {
if (value) {
*value = get_le_from_klpair(klpair);
}
if (key) {
paranoid_invariant(keylen != NULL);
*key = klpair->key_le;
*keylen = klpair->keylen;
paranoid_invariant_notnull(keylen);
*key = klpair->key;
*keylen = keylen_from_klpair_len(klpair_len);
}
else {
paranoid_invariant(keylen == NULL);
paranoid_invariant_null(keylen);
}
}
return r;
}
// get info about a single leafentry by index
// Fetch leafentry by index
__attribute__((__nonnull__))
int fetch_le(uint32_t idx, LEAFENTRY *le);
// Fetch (leafentry, key, keylen) by index
__attribute__((__nonnull__))
int fetch_klpair(uint32_t idx, LEAFENTRY *le, uint32_t *len, void** key);
// Fetch (serialized size of leafentry, key, and keylen) by index
__attribute__((__nonnull__))
int fetch_klpair_disksize(uint32_t idx, size_t *size);
int fetch_le_key_and_len(uint32_t idx, uint32_t *len, void** key);
// Fetch (key, keylen) by index
__attribute__((__nonnull__))
int fetch_key_and_len(uint32_t idx, uint32_t *len, void** key);
// Interact with another bn_data
void move_leafentries_to(BN_DATA dest_bd,
uint32_t lbi, //lower bound inclusive
uint32_t ube //upper bound exclusive
);
// Move leafentries (and associated key/keylens) from this basement node to dest_bd
// Moves indexes [lbi-ube)
__attribute__((__nonnull__))
void split_klpairs(bn_data* dest_bd, uint32_t first_index_for_dest);
// Destroy this basement node and free memory.
void destroy(void);
// Replaces contents, into brand new mempool.
// Returns old mempool base, expects caller to free it.
void replace_contents_with_clone_of_sorted_array(
// Uses sorted array as input for this basement node.
// Expects this to be a basement node just initialized with initialize_empty()
void set_contents_as_clone_of_sorted_array(
uint32_t num_les,
const void** old_key_ptrs,
uint32_t* old_keylens,
LEAFENTRY* old_les,
size_t *le_sizes,
size_t mempool_size
size_t total_key_size,
size_t total_le_size
);
// Make this basement node a clone of orig_bn_data.
// orig_bn_data still owns all its memory (dmt, mempool)
// this basement node will have a new dmt, mempool containing same data.
void clone(bn_data* orig_bn_data);
// Delete klpair index idx with provided keylen and old leafentry with size old_le_size
void delete_leafentry (
uint32_t idx,
uint32_t keylen,
uint32_t old_le_size
);
void get_space_for_overwrite(uint32_t idx, const void* keyp, uint32_t keylen, uint32_t old_size, uint32_t new_size, LEAFENTRY* new_le_space);
void get_space_for_insert(uint32_t idx, const void* keyp, uint32_t keylen, size_t size, LEAFENTRY* new_le_space);
private:
// Private functions
KLPAIR mempool_malloc_from_omt(size_t size, void **maybe_free);
void omt_compress_kvspace(size_t added_size, void **maybe_free);
klpair_omt_t m_buffer; // pointers to individual leaf entries
// Allocates space in the mempool to store a new leafentry.
// This may require reorganizing the mempool and updating the dmt.
__attribute__((__nonnull__))
void get_space_for_overwrite(uint32_t idx, const void* keyp, uint32_t keylen, uint32_t old_size, uint32_t new_size, LEAFENTRY* new_le_space, void **const maybe_free);
// Allocates space in the mempool to store a new leafentry
// and inserts a new key into the dmt
// This may require reorganizing the mempool and updating the dmt.
__attribute__((__nonnull__))
void get_space_for_insert(uint32_t idx, const void* keyp, uint32_t keylen, size_t size, LEAFENTRY* new_le_space, void **const maybe_free);
// Gets a leafentry given a klpair from this basement node.
LEAFENTRY get_le_from_klpair(const klpair_struct *klpair) const;
void serialize_to_wbuf(struct wbuf *const wb);
// Prepares this basement node for serialization.
// Must be called before serializing this basement node.
// Between calling prepare_to_serialize and actually serializing, the basement node may not be modified
void prepare_to_serialize(void);
// Serialize the basement node header to a wbuf
// Requires prepare_to_serialize() to have been called first.
void serialize_header(struct wbuf *wb) const;
// Serialize all keys and leafentries to a wbuf
// Requires prepare_to_serialize() (and serialize_header()) has been called first.
// Currently only supported when all keys are fixed-length.
void serialize_rest(struct wbuf *wb) const;
static const uint32_t HEADER_LENGTH = 0
+ sizeof(uint32_t) // key_data_size
+ sizeof(uint32_t) // val_data_size
+ sizeof(uint32_t) // fixed_key_length
+ sizeof(uint8_t) // all_keys_same_length
+ sizeof(uint8_t) // keys_vals_separate
+ 0;
private:
// split_klpairs_extra should be a local class in split_klpairs, but
// the dmt template parameter for iterate needs linkage, so it has to be a
// separate class, but we want it to be able to call e.g. add_key
friend class split_klpairs_extra;
// Allocates space in the mempool.
// If there is insufficient space, the mempool is enlarged and leafentries may be shuffled to reduce fragmentation.
// If shuffling happens, the offsets stored in the dmt are updated.
LEAFENTRY mempool_malloc_and_update_dmt(size_t size, void **maybe_free);
// Change the size of the mempool to support what is already in it, plus added_size.
// possibly "compress" by shuffling leafentries around to reduce fragmentation to 0.
// If fragmentation is already 0 and force_compress is not true, shuffling may be skipped.
// If shuffling happens, leafentries will be stored in the mempool in sorted order.
void dmt_compress_kvspace(size_t added_size, void **maybe_free, bool force_compress);
// Note that a key was added (for maintaining disk-size of this basement node)
void add_key(uint32_t keylen);
// Note that multiple keys were added (for maintaining disk-size of this basement node)
void add_keys(uint32_t n_keys, uint32_t combined_klpair_len);
// Note that a key was removed (for maintaining disk-size of this basement node)
void remove_key(uint32_t keylen);
klpair_dmt_t m_buffer; // pointers to individual leaf entries
struct mempool m_buffer_mempool; // storage for all leaf entries
friend class bndata_bugfix_test;
// Get the serialized size of a klpair.
// As of Jan 14, 2014, serialized size of a klpair is independent of whether this basement node has fixed-length keys.
uint32_t klpair_disksize(const uint32_t klpair_len, const klpair_struct *klpair) const;
// The disk/memory size of all keys. (Note that the size of memory for the leafentries is maintained by m_buffer_mempool)
size_t m_disksize_of_keys;
// Deserialize this basement node from rbuf
// all keys will be first followed by all leafentries (both in sorted order)
void initialize_from_separate_keys_and_vals(uint32_t num_entries, struct rbuf *rb, uint32_t data_size, uint32_t version,
uint32_t key_data_size, uint32_t val_data_size, bool all_keys_same_length,
uint32_t fixed_klpair_length);
};

2
storage/tokudb/ft-index/ft/cachetable-internal.h
View File

@@ -446,6 +446,8 @@ public:
FILENUM m_next_filenum_to_use;
uint32_t m_next_hash_id_to_use;
toku_pthread_rwlock_t m_lock; // this field is publoc so we are still POD
toku::omt<CACHEFILE> m_active_filenum;
toku::omt<CACHEFILE> m_active_fileid;
private:
CACHEFILE find_cachefile_in_list_unlocked(CACHEFILE start, struct fileid* fileid);
};

156
storage/tokudb/ft-index/ft/cachetable.cc
View File

@@ -370,7 +370,7 @@ toku_cachetable_set_env_dir(CACHETABLE ct, const char *env_dir) {
// What cachefile goes with particular iname (iname relative to env)?
// The transaction that is adding the reference might not have a reference
// to the brt, therefore the cachefile might be closing.
// to the ft, therefore the cachefile might be closing.
// If closing, we want to return that it is not there, but must wait till after
// the close has finished.
// Once the close has finished, there must not be a cachefile with that name
@@ -380,7 +380,7 @@ int toku_cachefile_of_iname_in_env (CACHETABLE ct, const char *iname_in_env, CAC
}
// What cachefile goes with particular fd?
// This function can only be called if the brt is still open, so file must
// This function can only be called if the ft is still open, so file must
// still be open
int toku_cachefile_of_filenum (CACHETABLE ct, FILENUM filenum, CACHEFILE *cf) {
return ct->cf_list.cachefile_of_filenum(filenum, cf);
@@ -564,10 +564,10 @@ void toku_cachefile_close(CACHEFILE *cfp, bool oplsn_valid, LSN oplsn) {
// that do not persist across opens/closes
bjm_destroy(cf->bjm);
cf->bjm = NULL;
cf->filenum = FILENUM_NONE;
// remove the cf from the list of active cachefiles
ct->cf_list.remove_cf(cf);
cf->filenum = FILENUM_NONE;
// Unlink the file if the bit was set
if (cf->unlink_on_close) {
@@ -642,7 +642,7 @@ static void cachetable_free_pair(PAIR p) {
cachetable_evictions++;
PAIR_ATTR new_attr = p->attr;
// Note that flush_callback is called with write_me false, so the only purpose of this
// call is to tell the brt layer to evict the node (keep_me is false).
// call is to tell the ft layer to evict the node (keep_me is false).
// Also, because we have already removed the PAIR from the cachetable in
// cachetable_remove_pair, we cannot pass in p->cachefile and p->cachefile->fd
// for the first two parameters, as these may be invalid (#5171), so, we
@@ -1302,8 +1302,6 @@ void toku_cachetable_pf_pinned_pair(
pair_unlock(p);
}
// NOW A TEST ONLY FUNCTION!!!
int toku_cachetable_get_and_pin (
CACHEFILE cachefile,
CACHEKEY key,
@@ -1573,7 +1571,7 @@ exit:
return try_again;
}
int toku_cachetable_get_and_pin_with_dep_pairs_batched (
int toku_cachetable_get_and_pin_with_dep_pairs (
CACHEFILE cachefile,
CACHEKEY key,
uint32_t fullhash,
@@ -1766,43 +1764,6 @@ got_value:
return 0;
}
int toku_cachetable_get_and_pin_with_dep_pairs (
CACHEFILE cachefile,
CACHEKEY key,
uint32_t fullhash,
void**value,
long *sizep,
CACHETABLE_WRITE_CALLBACK write_callback,
CACHETABLE_FETCH_CALLBACK fetch_callback,
CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback,
CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback,
pair_lock_type lock_type,
void* read_extraargs, // parameter for fetch_callback, pf_req_callback, and pf_callback
uint32_t num_dependent_pairs, // number of dependent pairs that we may need to checkpoint
PAIR* dependent_pairs,
enum cachetable_dirty* dependent_dirty // array stating dirty/cleanness of dependent pairs
)
// See cachetable.h
{
int r = toku_cachetable_get_and_pin_with_dep_pairs_batched(
cachefile,
key,
fullhash,
value,
sizep,
write_callback,
fetch_callback,
pf_req_callback,
pf_callback,
lock_type,
read_extraargs,
num_dependent_pairs,
dependent_pairs,
dependent_dirty
);
return r;
}
// Lookup a key in the cachetable. If it is found and it is not being written, then
// acquire a read lock on the pair, update the LRU list, and return sucess.
//
@@ -2048,7 +2009,7 @@ maybe_pin_pair(
return retval;
}
int toku_cachetable_get_and_pin_nonblocking_batched(
int toku_cachetable_get_and_pin_nonblocking(
CACHEFILE cf,
CACHEKEY key,
uint32_t fullhash,
@@ -2200,40 +2161,6 @@ try_again:
abort();
}
int toku_cachetable_get_and_pin_nonblocking (
CACHEFILE cf,
CACHEKEY key,
uint32_t fullhash,
void**value,
long* sizep,
CACHETABLE_WRITE_CALLBACK write_callback,
CACHETABLE_FETCH_CALLBACK fetch_callback,
CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback,
CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback,
pair_lock_type lock_type,
void *read_extraargs,
UNLOCKERS unlockers
)
// See cachetable.h.
{
int r = 0;
r = toku_cachetable_get_and_pin_nonblocking_batched(
cf,
key,
fullhash,
value,
sizep,
write_callback,
fetch_callback,
pf_req_callback,
pf_callback,
lock_type,
read_extraargs,
unlockers
);
return r;
}
struct cachefile_prefetch_args {
PAIR p;
CACHETABLE_FETCH_CALLBACK fetch_callback;
@@ -4750,9 +4677,13 @@ void cachefile_list::init() {
m_next_filenum_to_use.fileid = 0;
m_next_hash_id_to_use = 0;
toku_pthread_rwlock_init(&m_lock, NULL);
m_active_filenum.create();
m_active_fileid.create();
}
void cachefile_list::destroy() {
m_active_filenum.destroy();
m_active_fileid.destroy();
toku_pthread_rwlock_destroy(&m_lock);
}
@@ -4804,6 +4735,21 @@ int cachefile_list::cachefile_of_filenum(FILENUM filenum, CACHEFILE *cf) {
return r;
}
static int cachefile_find_by_filenum(const CACHEFILE &a_cf, const FILENUM &b) {
const FILENUM a = a_cf->filenum;
if (a.fileid < b.fileid) {
return -1;
} else if (a.fileid == b.fileid) {
return 0;
} else {
return 1;
}
}
static int cachefile_find_by_fileid(const CACHEFILE &a_cf, const struct fileid &b) {
return toku_fileid_cmp(a_cf->fileid, b);
}
void cachefile_list::add_cf_unlocked(CACHEFILE cf) {
invariant(cf->next == NULL);
invariant(cf->prev == NULL);
@@ -4813,6 +4759,12 @@ void cachefile_list::add_cf_unlocked(CACHEFILE cf) {
m_active_head->prev = cf;
}
m_active_head = cf;
int r;
r = m_active_filenum.insert<FILENUM, cachefile_find_by_filenum>(cf, cf->filenum, nullptr);
assert_zero(r);
r = m_active_fileid.insert<struct fileid, cachefile_find_by_fileid>(cf, cf->fileid, nullptr);
assert_zero(r);
}
void cachefile_list::add_stale_cf(CACHEFILE cf) {
@@ -4847,6 +4799,19 @@ void cachefile_list::remove_cf(CACHEFILE cf) {
}
cf->prev = NULL;
cf->next = NULL;
uint32_t idx;
int r;
r = m_active_filenum.find_zero<FILENUM, cachefile_find_by_filenum>(cf->filenum, nullptr, &idx);
assert_zero(r);
r = m_active_filenum.delete_at(idx);
assert_zero(r);
r = m_active_fileid.find_zero<struct fileid, cachefile_find_by_fileid>(cf->fileid, nullptr, &idx);
assert_zero(r);
r = m_active_fileid.delete_at(idx);
assert_zero(r);
write_unlock();
}
@@ -4872,18 +4837,23 @@ void cachefile_list::remove_stale_cf_unlocked(CACHEFILE cf) {
}
FILENUM cachefile_list::reserve_filenum() {
CACHEFILE extant;
FILENUM filenum;
// taking a write lock because we are modifying next_filenum_to_use
write_lock();
try_again:
for (extant = m_active_head; extant; extant = extant->next) {
if (m_next_filenum_to_use.fileid==extant->filenum.fileid) {
while (1) {
int r = m_active_filenum.find_zero<FILENUM, cachefile_find_by_filenum>(m_next_filenum_to_use, nullptr, nullptr);
if (r == 0) {
m_next_filenum_to_use.fileid++;
goto try_again;
continue;
}
assert(r == DB_NOTFOUND);
break;
}
filenum = m_next_filenum_to_use;
FILENUM filenum = m_next_filenum_to_use;
#if TOKU_DEBUG_PARANOID
for (CACHEFILE extant = m_active_head; extant; extant = extant->next) {
assert(filenum.fileid != extant->filenum.fileid);
}
#endif
m_next_filenum_to_use.fileid++;
write_unlock();
return filenum;
@@ -4916,7 +4886,15 @@ exit:
}
CACHEFILE cachefile_list::find_cachefile_unlocked(struct fileid* fileid) {
return find_cachefile_in_list_unlocked(m_active_head, fileid);
CACHEFILE cf = nullptr;
int r = m_active_fileid.find_zero<struct fileid, cachefile_find_by_fileid>(*fileid, &cf, nullptr);
if (r == 0) {
assert(!cf->unlink_on_close);
}
#if TOKU_DEBUG_PARANOID
assert(cf == find_cachefile_in_list_unlocked(m_active_head, fileid));
#endif
return cf;
}
CACHEFILE cachefile_list::find_stale_cachefile_unlocked(struct fileid* fileid) {
@@ -4924,9 +4902,13 @@ CACHEFILE cachefile_list::find_stale_cachefile_unlocked(struct fileid* fileid) {
}
void cachefile_list::verify_unused_filenum(FILENUM filenum) {
int r = m_active_filenum.find_zero<FILENUM, cachefile_find_by_filenum>(filenum, nullptr, nullptr);
assert(r == DB_NOTFOUND);
#if TOKU_DEBUG_PARANOID
for (CACHEFILE extant = m_active_head; extant; extant = extant->next) {
invariant(extant->filenum.fileid != filenum.fileid);
}
#endif
}
// returns true if some eviction ran, false otherwise

45
storage/tokudb/ft-index/ft/cachetable.h
View File

@@ -344,28 +344,6 @@ void toku_cachetable_put(CACHEFILE cf, CACHEKEY key, uint32_t fullhash,
// then the required PAIRs are written to disk for checkpoint.
// KEY PROPERTY OF DEPENDENT PAIRS: They are already locked by the client
// Returns: 0 if the memory object is in memory, otherwise an error number.
// Rationale:
// begin_batched_pin and end_batched_pin take and release a read lock on the pair list.
// Normally, that would be done within this get_and_pin, but we want to pin multiple nodes with a single acquisition of the read lock.
int toku_cachetable_get_and_pin_with_dep_pairs_batched (
CACHEFILE cachefile,
CACHEKEY key,
uint32_t fullhash,
void**value,
long *sizep,
CACHETABLE_WRITE_CALLBACK write_callback,
CACHETABLE_FETCH_CALLBACK fetch_callback,
CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback,
CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback,
pair_lock_type lock_type,
void* read_extraargs, // parameter for fetch_callback, pf_req_callback, and pf_callback
uint32_t num_dependent_pairs, // number of dependent pairs that we may need to checkpoint
PAIR* dependent_pairs,
enum cachetable_dirty* dependent_dirty // array stating dirty/cleanness of dependent pairs
);
// Effect: call toku_cachetable_get_and_pin_with_dep_pairs_batched once,
// wrapped in begin_batched_pin and end_batched_pin calls.
int toku_cachetable_get_and_pin_with_dep_pairs (
CACHEFILE cachefile,
CACHEKEY key,
@@ -383,7 +361,6 @@ int toku_cachetable_get_and_pin_with_dep_pairs (
enum cachetable_dirty* dependent_dirty // array stating dirty/cleanness of dependent pairs
);
// Get and pin a memory object.
// Effects: If the memory object is in the cachetable acquire the PAIR lock on it.
// Otherwise, fetch it from storage by calling the fetch callback. If the fetch
@@ -422,10 +399,7 @@ struct unlockers {
// Effect: If the block is in the cachetable, then return it.
// Otherwise call the functions in unlockers, fetch the data (but don't pin it, since we'll just end up pinning it again later), and return TOKUDB_TRY_AGAIN.
// Rationale:
// begin_batched_pin and end_batched_pin take and release a read lock on the pair list.
// Normally, that would be done within this get_and_pin, but we want to pin multiple nodes with a single acquisition of the read lock.
int toku_cachetable_get_and_pin_nonblocking_batched (
int toku_cachetable_get_and_pin_nonblocking (
CACHEFILE cf,
CACHEKEY key,
uint32_t fullhash,
@@ -440,23 +414,6 @@ int toku_cachetable_get_and_pin_nonblocking_batched (
UNLOCKERS unlockers
);
// Effect: call toku_cachetable_get_and_pin_nonblocking_batched once,
// wrapped in begin_batched_pin and end_batched_pin calls.
int toku_cachetable_get_and_pin_nonblocking (
CACHEFILE cf,
CACHEKEY key,
uint32_t fullhash,
void**value,
long *sizep,
CACHETABLE_WRITE_CALLBACK write_callback,
CACHETABLE_FETCH_CALLBACK fetch_callback,
CACHETABLE_PARTIAL_FETCH_REQUIRED_CALLBACK pf_req_callback __attribute__((unused)),
CACHETABLE_PARTIAL_FETCH_CALLBACK pf_callback __attribute__((unused)),
pair_lock_type lock_type,
void *read_extraargs, // parameter for fetch_callback, pf_req_callback, and pf_callback
UNLOCKERS unlockers
);
int toku_cachetable_maybe_get_and_pin (CACHEFILE, CACHEKEY, uint32_t /*fullhash*/, pair_lock_type, void**);
// Effect: Maybe get and pin a memory object.
// This function is similar to the get_and_pin function except that it

4
storage/tokudb/ft-index/ft/checkpoint.cc
View File

@@ -158,6 +158,8 @@ status_init(void) {
STATUS_INIT(CP_TIME_LAST_CHECKPOINT_BEGIN, CHECKPOINT_LAST_BEGAN, UNIXTIME, "last checkpoint began ", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
STATUS_INIT(CP_TIME_LAST_CHECKPOINT_BEGIN_COMPLETE, CHECKPOINT_LAST_COMPLETE_BEGAN, UNIXTIME, "last complete checkpoint began ", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
STATUS_INIT(CP_TIME_LAST_CHECKPOINT_END, CHECKPOINT_LAST_COMPLETE_ENDED, UNIXTIME, "last complete checkpoint ended", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
STATUS_INIT(CP_TIME_CHECKPOINT_DURATION, CHECKPOINT_DURATION, UNIXTIME, "time spent during checkpoint (begin and end phases)", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
STATUS_INIT(CP_TIME_CHECKPOINT_DURATION_LAST, CHECKPOINT_DURATION_LAST, UNIXTIME, "time spent during last checkpoint (begin and end phases)", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
STATUS_INIT(CP_LAST_LSN, nullptr, UINT64, "last complete checkpoint LSN", TOKU_ENGINE_STATUS);
STATUS_INIT(CP_CHECKPOINT_COUNT, CHECKPOINT_TAKEN, UINT64, "checkpoints taken ", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
STATUS_INIT(CP_CHECKPOINT_COUNT_FAIL, CHECKPOINT_FAILED, UINT64, "checkpoints failed", TOKU_ENGINE_STATUS|TOKU_GLOBAL_STATUS);
@@ -379,6 +381,8 @@ toku_checkpoint(CHECKPOINTER cp, TOKULOGGER logger,
STATUS_VALUE(CP_LONG_BEGIN_TIME) += duration;
STATUS_VALUE(CP_LONG_BEGIN_COUNT) += 1;
}
STATUS_VALUE(CP_TIME_CHECKPOINT_DURATION) += ((time_t) STATUS_VALUE(CP_TIME_LAST_CHECKPOINT_END)) - ((time_t) STATUS_VALUE(CP_TIME_LAST_CHECKPOINT_BEGIN));
STATUS_VALUE(CP_TIME_CHECKPOINT_DURATION_LAST) = ((time_t) STATUS_VALUE(CP_TIME_LAST_CHECKPOINT_END)) - ((time_t) STATUS_VALUE(CP_TIME_LAST_CHECKPOINT_BEGIN));
STATUS_VALUE(CP_FOOTPRINT) = 0;
checkpoint_safe_checkpoint_unlock();

2
storage/tokudb/ft-index/ft/checkpoint.h
View File

@@ -179,6 +179,8 @@ typedef enum {
CP_TIME_LAST_CHECKPOINT_BEGIN,
CP_TIME_LAST_CHECKPOINT_BEGIN_COMPLETE,
CP_TIME_LAST_CHECKPOINT_END,
CP_TIME_CHECKPOINT_DURATION,
CP_TIME_CHECKPOINT_DURATION_LAST,
CP_LAST_LSN,
CP_CHECKPOINT_COUNT,
CP_CHECKPOINT_COUNT_FAIL,

3
storage/tokudb/ft-index/ft/checksum-benchmarks/Makefile
View File

@@ -1,3 +0,0 @@
CFLAGS = -O2 -Wall -W -Werror -g
LDFLAGS = -lz -lssl -g
adler32:

236
storage/tokudb/ft-index/ft/checksum-benchmarks/adler32.cc
View File

@@ -1,236 +0,0 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <toku_assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#include <zlib.h>
#include <openssl/md2.h>
#include <openssl/md4.h>
#include <openssl/md5.h>
const unsigned int prime = 2000000011;
unsigned int karprabin (unsigned char *datac, int N) {
assert(N%4==0);
unsigned int *data=(unsigned int*)datac;
N=N/4;
int i;
unsigned int result=0;
for (i=0; i<N; i++) {
result=(result*prime)+data[i];
}
return result;
}
// According to
// P. L'Ecuyer, "Tables of Linear Congruential Generators of
// Different Sizes and Good Lattice Structure", Mathematics of
// Computation 68:225, 249--260 (1999).
// m=2^{32}-5 a=1588635695 is good.
const unsigned int mkr = 4294967291U;
const unsigned int akr = 1588635695U;
// But this is slower
unsigned int karprabinP (unsigned char *datac, int N) {
assert(N%4==0);
unsigned int *data=(unsigned int*)datac;
N=N/4;
int i;
unsigned long long result=0;
for (i=0; i<N; i++) {
result=((result*akr)+data[i])%mkr;
}
return result;
}
float tdiff (struct timeval *start, struct timeval *end) {
return (end->tv_sec-start->tv_sec) +1e-6*(end->tv_usec - start->tv_usec);
}
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
struct timeval start, end;
const int N=2<<20;
unsigned char *data=malloc(N);
int i;
assert(data);
for (i=0; i<N; i++) data[i]=random();
// adler32
{
uLong a32 = adler32(0L, Z_NULL, 0);
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
a32 = adler32(a32, data, N);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("adler32=%lu, time=%9.6fs %9.6fns/b\n", a32, tm, 1e9*tm/N);
}
}
// crc32
{
uLong c32 = crc32(0L, Z_NULL, 0);
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
c32 = crc32(c32, data, N);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("crc32=%lu, time=%9.6fs %9.6fns/b\n", c32, tm, 1e9*tm/N);
}
}
// MD2
{
unsigned char buf[MD2_DIGEST_LENGTH];
int j;
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
MD2(data, N, buf);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("md2=");
for (j=0; j<MD2_DIGEST_LENGTH; j++) {
printf("%02x", buf[j]);
}
printf(" time=%9.6fs %9.6fns/b\n", tm, 1e9*tm/N);
}
}
// MD4
{
unsigned char buf[MD4_DIGEST_LENGTH];
int j;
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
MD4(data, N, buf);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("md4=");
for (j=0; j<MD4_DIGEST_LENGTH; j++) {
printf("%02x", buf[j]);
}
printf(" time=%9.6fs %9.6fns/b\n", tm, 1e9*tm/N);
}
}
// MD5
{
unsigned char buf[MD5_DIGEST_LENGTH];
int j;
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
MD5(data, N, buf);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("md5=");
for (j=0; j<MD5_DIGEST_LENGTH; j++) {
printf("%02x", buf[j]);
}
printf(" time=%9.6fs %9.6fns/b\n", tm, 1e9*tm/N);
}
}
// karp rabin
{
for (i=0; i<3; i++) {
gettimeofday(&start, 0);
unsigned int kr = karprabin(data, N);
gettimeofday(&end, 0);
float tm = tdiff(&start, &end);
printf("kr=%ud time=%9.6fs %9.6fns/b\n", kr, tm, 1e9*tm/N);
}
}
free(data);
return 0;
}

3
storage/tokudb/ft-index/ft/compression-ratio/Makefile
View File

@@ -1,3 +0,0 @@
CFLAGS = -O -Wall -W -Werror
LDFLAGS = -lz
cratio:

195
storage/tokudb/ft-index/ft/compression-ratio/cratio.cc
View File

@@ -1,195 +0,0 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* Measure the extent to which we can compress a file.
* Works on version 8. */
#define _XOPEN_SOURCE 500
#include <arpa/inet.h>
#include <toku_assert.h>
#include <fcntl.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <zlib.h>
#include <toku_portability.h>
toku_off_t fd_size (int fd) {
int64_t file_size;
int r = toku_os_get_file_size(fd, &file_size);
assert(r==0);
return file_size;
}
#define NSIZE (1<<20)
unsigned char fbuf[NSIZE];
unsigned char cbuf[NSIZE+500];
void
measure_header (int fd, toku_off_t off, // read header from this offset
toku_off_t *usize, // size uncompressed (but not including any padding)
toku_off_t *csize) // compressed size
{
int r;
r=pread(fd, fbuf, 12, off);
assert(r==12);
assert(memcmp(fbuf,"tokudata",8)==0);
int bsize = toku_dtoh32(*(uint32_t*)(fbuf+8));
//printf("Bsize=%d\n", bsize);
(*usize)+=bsize;
assert(bsize<=NSIZE);
r=pread(fd, fbuf, bsize, off);
assert(r==bsize);
uLongf destLen=sizeof(cbuf);
r=compress2(cbuf, &destLen,
fbuf+20, bsize-=20, // skip magic nodesize and version
1);
assert(r==Z_OK);
destLen+=16; // account for the size and magic and version
//printf("Csize=%ld\n", destLen);
(*csize)+=destLen;
}
void
measure_node (int fd, toku_off_t off, // read header from this offset
toku_off_t *usize, // size uncompressed (but not including any padding)
toku_off_t *csize) // compressed size
{
int r;
r=pread(fd, fbuf, 24, off);
assert(r==24);
//printf("fbuf[0..7]=%c%c%c%c%c%c%c%c\n", fbuf[0], fbuf[1], fbuf[2], fbuf[3], fbuf[4], fbuf[5], fbuf[6], fbuf[7]);
assert(memcmp(fbuf,"tokuleaf",8)==0 || memcmp(fbuf, "tokunode", 8)==0);
assert(8==toku_dtoh32(*(uint32_t*)(fbuf+8))); // check file version
int bsize = toku_dtoh32(*(uint32_t*)(fbuf+20));
//printf("Bsize=%d\n", bsize);
(*usize)+=bsize;
assert(bsize<=NSIZE);
r=pread(fd, fbuf, bsize, off);
assert(r==bsize);
uLongf destLen=sizeof(cbuf);
r=compress2(cbuf, &destLen,
fbuf+28, bsize-=28, // skip constant header stuff
1);
destLen += 24; // add in magic (8), version(4), lsn (8), and size (4). Actually lsn will be compressed, but ignore that for now.
assert(r==Z_OK);
//printf("Csize=%ld\n", destLen);
(*csize)+=destLen;
}
/* The single argument is the filename to measure. */
int main (int argc, const char *argv[]) {
assert(argc==2);
const char *fname=argv[1];
int fd = open(fname, O_RDONLY);
assert(fd>=0);
toku_off_t fsize = fd_size(fd);
printf("fsize (uncompressed with padding)=%lld\n", (long long)fsize);
toku_off_t usize=0, csize=0;
measure_header(fd, 0, &usize, &csize);
toku_off_t i;
for (i=NSIZE; i+24<fsize; i+=NSIZE) {
measure_node(fd, i, &usize, &csize);
}
printf("usize (uncompressed with no padding)=%10lld (ratio=%5.2f)\n", (long long)usize, (double)fsize/(double)usize);
printf("csize (compressed) =%10lld (ratio=%5.2f)\n", (long long)csize, (double)fsize/(double)csize);
close(fd);
return 0;
}

7
storage/tokudb/ft-index/ft/fifo.cc
View File

@@ -102,7 +102,6 @@ struct fifo {
int memory_used; // How many bytes are in use?
};
const int fifo_initial_size = 4096;
static void fifo_init(struct fifo *fifo) {
fifo->n_items_in_fifo = 0;
fifo->memory = 0;
@@ -118,12 +117,12 @@ static int fifo_entry_size(struct fifo_entry *entry) {
}
__attribute__((const,nonnull))
size_t toku_ft_msg_memsize_in_fifo(FT_MSG cmd) {
size_t toku_ft_msg_memsize_in_fifo(FT_MSG msg) {
// This must stay in sync with fifo_entry_size because that's what we
// really trust. But sometimes we only have an in-memory FT_MSG, not
// a serialized fifo_entry so we have to fake it.
return sizeof (struct fifo_entry) + cmd->u.id.key->size + cmd->u.id.val->size
+ xids_get_size(cmd->xids)
return sizeof (struct fifo_entry) + msg->u.id.key->size + msg->u.id.val->size
+ xids_get_size(msg->xids)
- sizeof(XIDS_S);
}

15
storage/tokudb/ft-index/ft/fifo.h
View File

@@ -97,10 +97,6 @@ PATENT RIGHTS GRANT:
// If the fifo_entry is unpacked, the compiler aligns the xids array and we waste a lot of space
#if TOKU_WINDOWS
#pragma pack(push, 1)
#endif
struct __attribute__((__packed__)) fifo_entry {
unsigned int keylen;
unsigned int vallen;
@@ -110,7 +106,7 @@ struct __attribute__((__packed__)) fifo_entry {
XIDS_S xids_s;
};
// get and set the brt message type for a fifo entry.
// get and set the ft message type for a fifo entry.
// it is internally stored as a single unsigned char.
static inline enum ft_msg_type
fifo_entry_get_msg_type(const struct fifo_entry * entry)
@@ -128,10 +124,6 @@ fifo_entry_set_msg_type(struct fifo_entry * entry,
entry->type = type;
}
#if TOKU_WINDOWS
#pragma pack(pop)
#endif
typedef struct fifo *FIFO;
int toku_fifo_create(FIFO *);
@@ -149,9 +141,6 @@ unsigned long toku_fifo_memory_size_in_use(FIFO fifo); // return how much memor
unsigned long toku_fifo_memory_footprint(FIFO fifo); // return how much memory the fifo occupies
//These two are problematic, since I don't want to malloc() the bytevecs, but dequeueing the fifo frees the memory.
//int toku_fifo_peek_deq (FIFO, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen, uint32_t *type, TXNID *xid);
//int toku_fifo_peek_deq_cmdstruct (FIFO, FT_MSG, DBT*, DBT*); // fill in the FT_MSG, using the two DBTs for the DBT part.
void toku_fifo_iterate(FIFO, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,ITEMLEN datalen, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, void*), void*);
#define FIFO_ITERATE(fifo,keyvar,keylenvar,datavar,datalenvar,typevar,msnvar,xidsvar,is_freshvar,body) ({ \
@@ -178,7 +167,7 @@ int toku_fifo_iterate_internal_has_more(FIFO fifo, int off);
int toku_fifo_iterate_internal_next(FIFO fifo, int off);
struct fifo_entry * toku_fifo_iterate_internal_get_entry(FIFO fifo, int off);
size_t toku_fifo_internal_entry_memsize(struct fifo_entry *e) __attribute__((const,nonnull));
size_t toku_ft_msg_memsize_in_fifo(FT_MSG cmd) __attribute__((const,nonnull));
size_t toku_ft_msg_memsize_in_fifo(FT_MSG msg) __attribute__((const,nonnull));
DBT *fill_dbt_for_fifo_entry(DBT *dbt, const struct fifo_entry *entry);
struct fifo_entry *toku_fifo_get_entry(FIFO fifo, int off);

105
storage/tokudb/ft-index/ft/ft-cachetable-wrappers.cc
View File

@@ -201,8 +201,8 @@ toku_create_new_ftnode (
// then a PL_WRITE_CHEAP lock is grabbed
//
int
toku_pin_ftnode_batched(
FT_HANDLE brt,
toku_pin_ftnode_for_query(
FT_HANDLE ft_handle,
BLOCKNUM blocknum,
uint32_t fullhash,
UNLOCKERS unlockers,
@@ -225,13 +225,13 @@ toku_pin_ftnode_batched(
paranoid_invariant(bfe->type == ftnode_fetch_subset);
}
int r = toku_cachetable_get_and_pin_nonblocking_batched(
brt->ft->cf,
int r = toku_cachetable_get_and_pin_nonblocking(
ft_handle->ft->cf,
blocknum,
fullhash,
&node_v,
NULL,
get_write_callbacks_for_node(brt->ft),
get_write_callbacks_for_node(ft_handle->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
@@ -245,7 +245,7 @@ toku_pin_ftnode_batched(
node = static_cast<FTNODE>(node_v);
if (apply_ancestor_messages && node->height == 0) {
needs_ancestors_messages = toku_ft_leaf_needs_ancestors_messages(
brt->ft,
ft_handle->ft,
node,
ancestors,
bounds,
@@ -255,14 +255,14 @@ toku_pin_ftnode_batched(
if (needs_ancestors_messages) {
toku::context apply_messages_ctx(CTX_MESSAGE_APPLICATION);
toku_unpin_ftnode_read_only(brt->ft, node);
int rr = toku_cachetable_get_and_pin_nonblocking_batched(
brt->ft->cf,
toku_unpin_ftnode_read_only(ft_handle->ft, node);
int rr = toku_cachetable_get_and_pin_nonblocking(
ft_handle->ft->cf,
blocknum,
fullhash,
&node_v,
NULL,
get_write_callbacks_for_node(brt->ft),
get_write_callbacks_for_node(ft_handle->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
@@ -276,7 +276,7 @@ toku_pin_ftnode_batched(
}
node = static_cast<FTNODE>(node_v);
toku_apply_ancestors_messages_to_node(
brt,
ft_handle,
node,
ancestors,
bounds,
@@ -317,47 +317,7 @@ exit:
}
void
toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p,
bool move_messages)
{
toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
h,
blocknum,
fullhash,
bfe,
lock_type,
num_dependent_nodes,
dependent_nodes,
node_p,
move_messages
);
}
void
toku_pin_ftnode_off_client_thread(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p)
{
toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
h, blocknum, fullhash, bfe, lock_type, num_dependent_nodes, dependent_nodes, node_p, true);
}
void
toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
toku_pin_ftnode_with_dep_nodes(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
@@ -376,7 +336,7 @@ toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
dependent_dirty_bits[i] = (enum cachetable_dirty) dependent_nodes[i]->dirty;
}
int r = toku_cachetable_get_and_pin_with_dep_pairs_batched(
int r = toku_cachetable_get_and_pin_with_dep_pairs(
h->cf,
blocknum,
fullhash,
@@ -392,27 +352,22 @@ toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
dependent_pairs,
dependent_dirty_bits
);
assert(r==0);
invariant_zero(r);
FTNODE node = (FTNODE) node_v;
if ((lock_type != PL_READ) && node->height > 0 && move_messages) {
if (lock_type != PL_READ && node->height > 0 && move_messages) {
toku_move_ftnode_messages_to_stale(h, node);
}
*node_p = node;
}
void
toku_pin_ftnode_off_client_thread_batched(
FT h,
void toku_pin_ftnode(FT ft,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p)
{
toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
h, blocknum, fullhash, bfe, lock_type, num_dependent_nodes, dependent_nodes, node_p, true);
FTNODE *node_p,
bool move_messages) {
toku_pin_ftnode_with_dep_nodes(ft, blocknum, fullhash, bfe, lock_type, 0, nullptr, node_p, move_messages);
}
int toku_maybe_pin_ftnode_clean(FT ft, BLOCKNUM blocknum, uint32_t fullhash, pair_lock_type lock_type, FTNODE *nodep) {
@@ -429,24 +384,12 @@ cleanup:
return r;
}
void
toku_unpin_ftnode_off_client_thread(FT ft, FTNODE node)
{
int r = toku_cachetable_unpin(
ft->cf,
void toku_unpin_ftnode(FT ft, FTNODE node) {
int r = toku_cachetable_unpin(ft->cf,
node->ct_pair,
(enum cachetable_dirty) node->dirty,
make_ftnode_pair_attr(node)
);
assert(r==0);
}
void
toku_unpin_ftnode(FT ft, FTNODE node)
{
// printf("%*sUnpin %ld\n", 8-node->height, "", node->thisnodename.b);
//VERIFY_NODE(brt,node);
toku_unpin_ftnode_off_client_thread(ft, node);
static_cast<enum cachetable_dirty>(node->dirty),
make_ftnode_pair_attr(node));
invariant_zero(r);
}
void

80
storage/tokudb/ft-index/ft/ft-cachetable-wrappers.h
View File

@@ -138,13 +138,10 @@ toku_create_new_ftnode (
int n_children
);
/**
* Batched version of toku_pin_ftnode, see cachetable batched API for more
* details.
*/
// This function returns a pinned ftnode to the caller.
int
toku_pin_ftnode_batched(
FT_HANDLE brt,
toku_pin_ftnode_for_query(
FT_HANDLE ft_h,
BLOCKNUM blocknum,
uint32_t fullhash,
UNLOCKERS unlockers,
@@ -156,27 +153,20 @@ toku_pin_ftnode_batched(
bool* msgs_applied
);
/**
* Unfortunately, this function is poorly named
* as over time, client threads have also started
* calling this function.
* This function returns a pinned ftnode to the caller.
* Unlike toku_pin_ftnode, this function blocks until the node is pinned.
*/
void
toku_pin_ftnode_off_client_thread(
// Pins an ftnode without dependent pairs
void toku_pin_ftnode(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p
FTNODE *node_p,
bool move_messages
);
void
toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
// Pins an ftnode with dependent pairs
// Unlike toku_pin_ftnode_for_query, this function blocks until the node is pinned.
void toku_pin_ftnode_with_dep_nodes(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
@@ -195,53 +185,9 @@ toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
int toku_maybe_pin_ftnode_clean(FT ft, BLOCKNUM blocknum, uint32_t fullhash, pair_lock_type lock_type, FTNODE *nodep);
/**
* Batched version of toku_pin_ftnode_off_client_thread, see cachetable
* batched API for more details.
* Effect: Unpin an ftnode.
*/
void
toku_pin_ftnode_off_client_thread_batched_and_maybe_move_messages(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p,
bool move_messages
);
/**
* Batched version of toku_pin_ftnode_off_client_thread, see cachetable
* batched API for more details.
*/
void
toku_pin_ftnode_off_client_thread_batched(
FT h,
BLOCKNUM blocknum,
uint32_t fullhash,
FTNODE_FETCH_EXTRA bfe,
pair_lock_type lock_type,
uint32_t num_dependent_nodes,
FTNODE* dependent_nodes,
FTNODE *node_p
);
/**
* Effect: Unpin a brt node. Used for
* nodes that were pinned off client thread.
*/
void
toku_unpin_ftnode_off_client_thread(FT h, FTNODE node);
/**
* Effect: Unpin a brt node.
* Used for nodes pinned on a client thread
*/
void
toku_unpin_ftnode(FT h, FTNODE node);
void
toku_unpin_ftnode_read_only(FT ft, FTNODE node);
void toku_unpin_ftnode(FT h, FTNODE node);
void toku_unpin_ftnode_read_only(FT ft, FTNODE node);
#endif

80
storage/tokudb/ft-index/ft/ft-flusher.cc
View File

@@ -104,7 +104,7 @@ PATENT RIGHTS GRANT:
*/
static FT_FLUSHER_STATUS_S ft_flusher_status;
#define STATUS_INIT(k,c,t,l,inc) TOKUDB_STATUS_INIT(ft_flusher_status, k, c, t, "brt flusher: " l, inc)
#define STATUS_INIT(k,c,t,l,inc) TOKUDB_STATUS_INIT(ft_flusher_status, k, c, t, "ft flusher: " l, inc)
#define STATUS_VALUE(x) ft_flusher_status.status[x].value.num
void toku_ft_flusher_status_init(void) {
@@ -334,7 +334,7 @@ default_merge_child(struct flusher_advice *fa,
// we are just going to unpin child and
// let ft_merge_child pin it again
//
toku_unpin_ftnode_off_client_thread(h, child);
toku_unpin_ftnode(h, child);
//
//
// it is responsibility of ft_merge_child to unlock parent
@@ -486,8 +486,8 @@ ct_maybe_merge_child(struct flusher_advice *fa,
default_pick_child_after_split,
&ctme);
toku_unpin_ftnode_off_client_thread(h, parent);
toku_unpin_ftnode_off_client_thread(h, child);
toku_unpin_ftnode(h, parent);
toku_unpin_ftnode(h, child);
FTNODE root_node = NULL;
{
@@ -496,7 +496,7 @@ ct_maybe_merge_child(struct flusher_advice *fa,
toku_calculate_root_offset_pointer(h, &root, &fullhash);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, h);
toku_pin_ftnode_off_client_thread(h, root, fullhash, &bfe, PL_WRITE_EXPENSIVE, 0, NULL, &root_node);
toku_pin_ftnode(h, root, fullhash, &bfe, PL_WRITE_EXPENSIVE, &root_node, true);
toku_assert_entire_node_in_memory(root_node);
}
@@ -689,16 +689,16 @@ ftleaf_get_split_loc(
switch (split_mode) {
case SPLIT_LEFT_HEAVY: {
*num_left_bns = node->n_children;
*num_left_les = BLB_DATA(node, *num_left_bns - 1)->omt_size();
*num_left_les = BLB_DATA(node, *num_left_bns - 1)->num_klpairs();
if (*num_left_les == 0) {
*num_left_bns = node->n_children - 1;
*num_left_les = BLB_DATA(node, *num_left_bns - 1)->omt_size();
*num_left_les = BLB_DATA(node, *num_left_bns - 1)->num_klpairs();
}
goto exit;
}
case SPLIT_RIGHT_HEAVY: {
*num_left_bns = 1;
*num_left_les = BLB_DATA(node, 0)->omt_size() ? 1 : 0;
*num_left_les = BLB_DATA(node, 0)->num_klpairs() ? 1 : 0;
goto exit;
}
case SPLIT_EVENLY: {
@@ -707,8 +707,8 @@ ftleaf_get_split_loc(
uint64_t sumlesizes = ftleaf_disk_size(node);
uint32_t size_so_far = 0;
for (int i = 0; i < node->n_children; i++) {
BN_DATA bd = BLB_DATA(node, i);
uint32_t n_leafentries = bd->omt_size();
bn_data* bd = BLB_DATA(node, i);
uint32_t n_leafentries = bd->num_klpairs();
for (uint32_t j=0; j < n_leafentries; j++) {
size_t size_this_le;
int rr = bd->fetch_klpair_disksize(j, &size_this_le);
@@ -725,7 +725,7 @@ ftleaf_get_split_loc(
(*num_left_les)--;
} else if (*num_left_bns > 1) {
(*num_left_bns)--;
*num_left_les = BLB_DATA(node, *num_left_bns - 1)->omt_size();
*num_left_les = BLB_DATA(node, *num_left_bns - 1)->num_klpairs();
} else {
// we are trying to split a leaf with only one
// leafentry in it
@@ -743,8 +743,6 @@ exit:
return;
}
// TODO: (Zardosht) possibly get rid of this function and use toku_omt_split_at in
// ftleaf_split
static void
move_leafentries(
BASEMENTNODE dest_bn,
@@ -754,7 +752,8 @@ move_leafentries(
)
//Effect: move leafentries in the range [lbi, upe) from src_omt to newly created dest_omt
{
src_bn->data_buffer.move_leafentries_to(&dest_bn->data_buffer, lbi, ube);
invariant(ube == src_bn->data_buffer.num_klpairs());
src_bn->data_buffer.split_klpairs(&dest_bn->data_buffer, lbi);
}
static void ftnode_finalize_split(FTNODE node, FTNODE B, MSN max_msn_applied_to_node) {
@@ -851,7 +850,7 @@ ftleaf_split(
ftleaf_get_split_loc(node, split_mode, &num_left_bns, &num_left_les);
{
// did we split right on the boundary between basement nodes?
const bool split_on_boundary = (num_left_les == 0) || (num_left_les == (int) BLB_DATA(node, num_left_bns - 1)->omt_size());
const bool split_on_boundary = (num_left_les == 0) || (num_left_les == (int) BLB_DATA(node, num_left_bns - 1)->num_klpairs());
// Now we know where we are going to break it
// the two nodes will have a total of n_children+1 basement nodes
// and n_children-1 pivots
@@ -912,7 +911,7 @@ ftleaf_split(
move_leafentries(BLB(B, curr_dest_bn_index),
BLB(node, curr_src_bn_index),
num_left_les, // first row to be moved to B
BLB_DATA(node, curr_src_bn_index)->omt_size() // number of rows in basement to be split
BLB_DATA(node, curr_src_bn_index)->num_klpairs() // number of rows in basement to be split
);
BLB_MAX_MSN_APPLIED(B, curr_dest_bn_index) = BLB_MAX_MSN_APPLIED(node, curr_src_bn_index);
curr_dest_bn_index++;
@@ -954,10 +953,10 @@ ftleaf_split(
toku_destroy_dbt(&node->childkeys[num_left_bns - 1]);
}
} else if (splitk) {
BN_DATA bd = BLB_DATA(node, num_left_bns - 1);
bn_data* bd = BLB_DATA(node, num_left_bns - 1);
uint32_t keylen;
void *key;
int rr = bd->fetch_le_key_and_len(bd->omt_size() - 1, &keylen, &key);
int rr = bd->fetch_key_and_len(bd->num_klpairs() - 1, &keylen, &key);
invariant_zero(rr);
toku_memdup_dbt(splitk, key, keylen);
}
@@ -1082,20 +1081,20 @@ ft_split_child(
// and possibly continue
// flushing one of the children
int picked_child = fa->pick_child_after_split(h, node, childnum, childnum + 1, fa->extra);
toku_unpin_ftnode_off_client_thread(h, node);
toku_unpin_ftnode(h, node);
if (picked_child == childnum ||
(picked_child < 0 && nodea->height > 0 && fa->should_recursively_flush(nodea, fa->extra))) {
toku_unpin_ftnode_off_client_thread(h, nodeb);
toku_unpin_ftnode(h, nodeb);
toku_ft_flush_some_child(h, nodea, fa);
}
else if (picked_child == childnum + 1 ||
(picked_child < 0 && nodeb->height > 0 && fa->should_recursively_flush(nodeb, fa->extra))) {
toku_unpin_ftnode_off_client_thread(h, nodea);
toku_unpin_ftnode(h, nodea);
toku_ft_flush_some_child(h, nodeb, fa);
}
else {
toku_unpin_ftnode_off_client_thread(h, nodea);
toku_unpin_ftnode_off_client_thread(h, nodeb);
toku_unpin_ftnode(h, nodea);
toku_unpin_ftnode(h, nodeb);
}
}
@@ -1168,11 +1167,11 @@ merge_leaf_nodes(FTNODE a, FTNODE b)
a->dirty = 1;
b->dirty = 1;
BN_DATA a_last_bd = BLB_DATA(a, a->n_children-1);
bn_data* a_last_bd = BLB_DATA(a, a->n_children-1);
// this bool states if the last basement node in a has any items or not
// If it does, then it stays in the merge. If it does not, the last basement node
// of a gets eliminated because we do not have a pivot to store for it (because it has no elements)
const bool a_has_tail = a_last_bd->omt_size() > 0;
const bool a_has_tail = a_last_bd->num_klpairs() > 0;
// move each basement node from b to a
// move the pivots, adding one of what used to be max(a)
@@ -1199,7 +1198,7 @@ merge_leaf_nodes(FTNODE a, FTNODE b)
if (a_has_tail) {
uint32_t keylen;
void *key;
int rr = a_last_bd->fetch_le_key_and_len(a_last_bd->omt_size() - 1, &keylen, &key);
int rr = a_last_bd->fetch_key_and_len(a_last_bd->num_klpairs() - 1, &keylen, &key);
invariant_zero(rr);
toku_memdup_dbt(&a->childkeys[a->n_children-1], key, keylen);
a->totalchildkeylens += keylen;
@@ -1343,7 +1342,7 @@ maybe_merge_pinned_nodes(
// For nonleaf nodes, we distribute the children evenly. That may leave one or both of the nodes overfull, but that's OK.
// If we distribute, we set *splitk to a malloced pivot key.
// Parameters:
// t The BRT.
// t The FT.
// parent The parent of the two nodes to be split.
// parent_splitk The pivot key between a and b. This is either free()'d or returned in *splitk.
// a The first node to merge.
@@ -1426,7 +1425,7 @@ ft_merge_child(
uint32_t childfullhash = compute_child_fullhash(h->cf, node, childnuma);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, h);
toku_pin_ftnode_off_client_thread(h, BP_BLOCKNUM(node, childnuma), childfullhash, &bfe, PL_WRITE_EXPENSIVE, 1, &node, &childa);
toku_pin_ftnode_with_dep_nodes(h, BP_BLOCKNUM(node, childnuma), childfullhash, &bfe, PL_WRITE_EXPENSIVE, 1, &node, &childa, true);
}
// for test
call_flusher_thread_callback(flt_flush_before_pin_second_node_for_merge);
@@ -1437,7 +1436,7 @@ ft_merge_child(
uint32_t childfullhash = compute_child_fullhash(h->cf, node, childnumb);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, h);
toku_pin_ftnode_off_client_thread(h, BP_BLOCKNUM(node, childnumb), childfullhash, &bfe, PL_WRITE_EXPENSIVE, 2, dep_nodes, &childb);
toku_pin_ftnode_with_dep_nodes(h, BP_BLOCKNUM(node, childnumb), childfullhash, &bfe, PL_WRITE_EXPENSIVE, 2, dep_nodes, &childb, true);
}
if (toku_bnc_n_entries(BNC(node,childnuma))>0) {
@@ -1525,7 +1524,7 @@ ft_merge_child(
// unlock the parent
paranoid_invariant(node->dirty);
toku_unpin_ftnode_off_client_thread(h, node);
toku_unpin_ftnode(h, node);
}
else {
// for test
@@ -1533,14 +1532,14 @@ ft_merge_child(
// unlock the parent
paranoid_invariant(node->dirty);
toku_unpin_ftnode_off_client_thread(h, node);
toku_unpin_ftnode_off_client_thread(h, childb);
toku_unpin_ftnode(h, node);
toku_unpin_ftnode(h, childb);
}
if (childa->height > 0 && fa->should_recursively_flush(childa, fa->extra)) {
toku_ft_flush_some_child(h, childa, fa);
}
else {
toku_unpin_ftnode_off_client_thread(h, childa);
toku_unpin_ftnode(h, childa);
}
}
@@ -1575,7 +1574,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
// Note that we don't read the entire node into memory yet.
// The idea is let's try to do the minimum work before releasing the parent lock
fill_bfe_for_min_read(&bfe, ft);
toku_pin_ftnode_off_client_thread(ft, targetchild, childfullhash, &bfe, PL_WRITE_EXPENSIVE, 1, &parent, &child);
toku_pin_ftnode_with_dep_nodes(ft, targetchild, childfullhash, &bfe, PL_WRITE_EXPENSIVE, 1, &parent, &child, true);
// for test
call_flusher_thread_callback(ft_flush_aflter_child_pin);
@@ -1591,7 +1590,6 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
bool may_child_be_reactive = may_node_be_reactive(ft, child);
paranoid_invariant(child->thisnodename.b!=0);
//VERIFY_NODE(brt, child);
// only do the following work if there is a flush to perform
if (toku_bnc_n_entries(BNC(parent, childnum)) > 0 || parent->height == 1) {
@@ -1614,7 +1612,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
// reactive, we can unpin the parent
//
if (!may_child_be_reactive) {
toku_unpin_ftnode_off_client_thread(ft, parent);
toku_unpin_ftnode(ft, parent);
parent = NULL;
}
@@ -1632,7 +1630,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
// for the root with a fresh one
enum reactivity child_re = get_node_reactivity(ft, child);
if (parent && child_re == RE_STABLE) {
toku_unpin_ftnode_off_client_thread(ft, parent);
toku_unpin_ftnode(ft, parent);
parent = NULL;
}
@@ -1671,7 +1669,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
)
{
if (parent) {
toku_unpin_ftnode_off_client_thread(ft, parent);
toku_unpin_ftnode(ft, parent);
parent = NULL;
}
//
@@ -1681,7 +1679,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa)
toku_ft_flush_some_child(ft, child, fa);
}
else {
toku_unpin_ftnode_off_client_thread(ft, child);
toku_unpin_ftnode(ft, child);
}
}
else if (child_re == RE_FISSIBLE) {
@@ -1837,7 +1835,7 @@ toku_ftnode_cleaner_callback(
ct_flusher_advice_init(&fa, &fste, h->h->nodesize);
toku_ft_flush_some_child(h, node, &fa);
} else {
toku_unpin_ftnode_off_client_thread(h, node);
toku_unpin_ftnode(h, node);
}
return 0;
}
@@ -1897,7 +1895,7 @@ static void flush_node_fun(void *fe_v)
toku_ft_flush_some_child(fe->h, fe->node, &fa);
}
else {
toku_unpin_ftnode_off_client_thread(fe->h,fe->node);
toku_unpin_ftnode(fe->h,fe->node);
}
}
else {

4
storage/tokudb/ft-index/ft/ft-flusher.h
View File

@@ -124,7 +124,7 @@ typedef enum {
FT_FLUSHER_SPLIT_NONLEAF, // number of nonleaf nodes split
FT_FLUSHER_MERGE_LEAF, // number of times leaf nodes are merged
FT_FLUSHER_MERGE_NONLEAF, // number of times nonleaf nodes are merged
FT_FLUSHER_BALANCE_LEAF, // number of times a leaf node is balanced inside brt
FT_FLUSHER_BALANCE_LEAF, // number of times a leaf node is balanced
FT_FLUSHER_STATUS_NUM_ROWS
} ft_flusher_status_entry;
@@ -230,7 +230,7 @@ void toku_ft_hot_get_status(FT_HOT_STATUS);
* we go until the end of the FT.
*/
int
toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
toku_ft_hot_optimize(FT_HANDLE ft_h, DBT* left, DBT* right,
int (*progress_callback)(void *extra, float progress),
void *progress_extra, uint64_t* loops_run);

25
storage/tokudb/ft-index/ft/ft-hot-flusher.cc
View File

@@ -298,7 +298,7 @@ hot_flusher_destroy(struct hot_flusher_extra *flusher)
// Entry point for Hot Optimize Table (HOT). Note, this function is
// not recursive. It iterates over root-to-leaf paths.
int
toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
toku_ft_hot_optimize(FT_HANDLE ft_handle, DBT* left, DBT* right,
int (*progress_callback)(void *extra, float progress),
void *progress_extra, uint64_t* loops_run)
{
@@ -316,7 +316,7 @@ toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
// start of HOT operation
(void) toku_sync_fetch_and_add(&STATUS_VALUE(FT_HOT_NUM_STARTED), 1);
toku_ft_note_hot_begin(brt);
toku_ft_note_hot_begin(ft_handle);
// Higher level logic prevents a dictionary from being deleted or
// truncated during a hot optimize operation. Doing so would violate
@@ -329,17 +329,16 @@ toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
{
// Get root node (the first parent of each successive HOT
// call.)
toku_calculate_root_offset_pointer(brt->ft, &root_key, &fullhash);
toku_calculate_root_offset_pointer(ft_handle->ft, &root_key, &fullhash);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, brt->ft);
toku_pin_ftnode_off_client_thread(brt->ft,
fill_bfe_for_full_read(&bfe, ft_handle->ft);
toku_pin_ftnode(ft_handle->ft,
(BLOCKNUM) root_key,
fullhash,
&bfe,
PL_WRITE_EXPENSIVE,
0,
NULL,
&root);
&root,
true);
toku_assert_entire_node_in_memory(root);
}
@@ -365,12 +364,12 @@ toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
// This should recurse to the bottom of the tree and then
// return.
if (root->height > 0) {
toku_ft_flush_some_child(brt->ft, root, &advice);
toku_ft_flush_some_child(ft_handle->ft, root, &advice);
} else {
// Since there are no children to flush, we should abort
// the HOT call.
flusher.rightmost_leaf_seen = 1;
toku_unpin_ftnode_off_client_thread(brt->ft, root);
toku_unpin_ftnode(ft_handle->ft, root);
}
// Set the highest pivot key seen here, since the parent may
@@ -386,8 +385,8 @@ toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
else if (right) {
// if we have flushed past the bounds set for us,
// set rightmost_leaf_seen so we exit
FAKE_DB(db, &brt->ft->cmp_descriptor);
int cmp = brt->ft->compare_fun(&db, &flusher.max_current_key, right);
FAKE_DB(db, &ft_handle->ft->cmp_descriptor);
int cmp = ft_handle->ft->compare_fun(&db, &flusher.max_current_key, right);
if (cmp > 0) {
flusher.rightmost_leaf_seen = 1;
}
@@ -417,7 +416,7 @@ toku_ft_hot_optimize(FT_HANDLE brt, DBT* left, DBT* right,
if (r == 0) { success = true; }
{
toku_ft_note_hot_complete(brt, success, msn_at_start_of_hot);
toku_ft_note_hot_complete(ft_handle, success, msn_at_start_of_hot);
}
if (success) {

55
storage/tokudb/ft-index/ft/ft-internal.h
View File

@@ -92,7 +92,7 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include "toku_config.h"
#include <portability/toku_config.h>
#include <toku_race_tools.h>
// Symbol TOKUDB_REVISION is not defined by fractal-tree makefiles, so
@@ -118,7 +118,7 @@ PATENT RIGHTS GRANT:
#include "bndata.h"
enum { KEY_VALUE_OVERHEAD = 8 }; /* Must store the two lengths. */
enum { FT_CMD_OVERHEAD = (2 + sizeof(MSN)) }; // the type plus freshness plus MSN
enum { FT_MSG_OVERHEAD = (2 + sizeof(MSN)) }; // the type plus freshness plus MSN
enum { FT_DEFAULT_FANOUT = 16 };
enum { FT_DEFAULT_NODE_SIZE = 4 * 1024 * 1024 };
enum { FT_DEFAULT_BASEMENT_NODE_SIZE = 128 * 1024 };
@@ -454,7 +454,7 @@ enum {
uint32_t compute_child_fullhash (CACHEFILE cf, FTNODE node, int childnum);
// The brt_header is not managed by the cachetable. Instead, it hangs off the cachefile as userdata.
// The ft_header is not managed by the cachetable. Instead, it hangs off the cachefile as userdata.
enum ft_type {FT_CURRENT=1, FT_CHECKPOINT_INPROGRESS};
@@ -470,7 +470,7 @@ struct ft_header {
// LSN of creation of "checkpoint-begin" record in log.
LSN checkpoint_lsn;
// see brt_layout_version.h. maybe don't need this if we assume
// see ft_layout_version.h. maybe don't need this if we assume
// it's always the current version after deserializing
const int layout_version;
// different (<) from layout_version if upgraded from a previous
@@ -504,7 +504,7 @@ struct ft_header {
enum toku_compression_method compression_method;
unsigned int fanout;
// Current Minimum MSN to be used when upgrading pre-MSN BRT's.
// Current Minimum MSN to be used when upgrading pre-MSN FT's.
// This is decremented from our currnt MIN_MSN so as not to clash
// with any existing 'normal' MSN's.
MSN highest_unused_msn_for_upgrade;
@@ -526,7 +526,7 @@ struct ft_header {
STAT64INFO_S on_disk_stats;
};
// brt_header is always the current version.
// ft_header is always the current version.
struct ft {
FT_HEADER h;
FT_HEADER checkpoint_header;
@@ -708,7 +708,7 @@ void toku_assert_entire_node_in_memory(FTNODE node);
// append a child node to a parent node
void toku_ft_nonleaf_append_child(FTNODE node, FTNODE child, const DBT *pivotkey);
// append a cmd to a nonleaf node child buffer
// append a message to a nonleaf node child buffer
void toku_ft_append_to_child_buffer(ft_compare_func compare_fun, DESCRIPTOR desc, FTNODE node, int childnum, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, const DBT *key, const DBT *val);
STAT64INFO_S toku_get_and_clear_basement_stats(FTNODE leafnode);
@@ -768,7 +768,7 @@ static inline CACHETABLE_WRITE_CALLBACK get_write_callbacks_for_node(FT h) {
static const FTNODE null_ftnode=0;
/* a brt cursor is represented as a kv pair in a tree */
/* an ft cursor is represented as a kv pair in a tree */
struct ft_cursor {
struct toku_list cursors_link;
FT_HANDLE ft_handle;
@@ -969,7 +969,7 @@ __attribute__((nonnull))
void toku_ft_bn_update_max_msn(FTNODE node, MSN max_msn_applied, int child_to_read);
__attribute__((const,nonnull))
size_t toku_ft_msg_memsize_in_fifo(FT_MSG cmd);
size_t toku_ft_msg_memsize_in_fifo(FT_MSG msg);
int
toku_ft_search_which_child(
@@ -1018,26 +1018,25 @@ int toku_ftnode_hot_next_child(FTNODE node,
/* Stuff for testing */
// toku_testsetup_initialize() must be called before any other test_setup_xxx() functions are called.
void toku_testsetup_initialize(void);
int toku_testsetup_leaf(FT_HANDLE brt, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens);
int toku_testsetup_nonleaf (FT_HANDLE brt, int height, BLOCKNUM *diskoff, int n_children, BLOCKNUM *children, char **keys, int *keylens);
int toku_testsetup_root(FT_HANDLE brt, BLOCKNUM);
int toku_testsetup_get_sersize(FT_HANDLE brt, BLOCKNUM); // Return the size on disk.
int toku_testsetup_insert_to_leaf (FT_HANDLE brt, BLOCKNUM, const char *key, int keylen, const char *val, int vallen);
int toku_testsetup_insert_to_nonleaf (FT_HANDLE brt, BLOCKNUM, enum ft_msg_type, const char *key, int keylen, const char *val, int vallen);
int toku_testsetup_leaf(FT_HANDLE ft_h, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens);
int toku_testsetup_nonleaf (FT_HANDLE ft_h, int height, BLOCKNUM *diskoff, int n_children, BLOCKNUM *children, char **keys, int *keylens);
int toku_testsetup_root(FT_HANDLE ft_h, BLOCKNUM);
int toku_testsetup_get_sersize(FT_HANDLE ft_h, BLOCKNUM); // Return the size on disk.
int toku_testsetup_insert_to_leaf (FT_HANDLE ft_h, BLOCKNUM, const char *key, int keylen, const char *val, int vallen);
int toku_testsetup_insert_to_nonleaf (FT_HANDLE ft_h, BLOCKNUM, enum ft_msg_type, const char *key, int keylen, const char *val, int vallen);
void toku_pin_node_with_min_bfe(FTNODE* node, BLOCKNUM b, FT_HANDLE t);
// toku_ft_root_put_cmd() accepts non-constant cmd because this is where we set the msn
void toku_ft_root_put_cmd(FT h, FT_MSG_S * cmd, txn_gc_info *gc_info);
void toku_ft_root_put_msg(FT h, FT_MSG msg, txn_gc_info *gc_info);
void
toku_get_node_for_verify(
BLOCKNUM blocknum,
FT_HANDLE brt,
FT_HANDLE ft_h,
FTNODE* nodep
);
int
toku_verify_ftnode (FT_HANDLE brt,
toku_verify_ftnode (FT_HANDLE ft_h,
MSN rootmsn, MSN parentmsn, bool messages_exist_above,
FTNODE node, int height,
const DBT *lesser_pivot, // Everything in the subtree should be > lesser_pivot. (lesser_pivot==NULL if there is no lesser pivot.)
@@ -1185,6 +1184,8 @@ typedef enum {
FT_PRO_NUM_STOP_LOCK_CHILD,
FT_PRO_NUM_STOP_CHILD_INMEM,
FT_PRO_NUM_DIDNT_WANT_PROMOTE,
FT_BASEMENT_DESERIALIZE_FIXED_KEYSIZE, // how many basement nodes were deserialized with a fixed keysize
FT_BASEMENT_DESERIALIZE_VARIABLE_KEYSIZE, // how many basement nodes were deserialized with a variable keysize
FT_STATUS_NUM_ROWS
} ft_status_entry;
@@ -1196,9 +1197,9 @@ typedef struct {
void toku_ft_get_status(FT_STATUS);
void
toku_ft_bn_apply_cmd_once (
toku_ft_bn_apply_msg_once(
BASEMENTNODE bn,
const FT_MSG cmd,
const FT_MSG msg,
uint32_t idx,
LEAFENTRY le,
txn_gc_info *gc_info,
@@ -1207,38 +1208,38 @@ toku_ft_bn_apply_cmd_once (
);
void
toku_ft_bn_apply_cmd (
toku_ft_bn_apply_msg(
ft_compare_func compare_fun,
ft_update_func update_fun,
DESCRIPTOR desc,
BASEMENTNODE bn,
FT_MSG cmd,
FT_MSG msg,
txn_gc_info *gc_info,
uint64_t *workdone,
STAT64INFO stats_to_update
);
void
toku_ft_leaf_apply_cmd (
toku_ft_leaf_apply_msg(
ft_compare_func compare_fun,
ft_update_func update_fun,
DESCRIPTOR desc,
FTNODE node,
int target_childnum,
FT_MSG cmd,
FT_MSG msg,
txn_gc_info *gc_info,
uint64_t *workdone,
STAT64INFO stats_to_update
);
void
toku_ft_node_put_cmd (
toku_ft_node_put_msg(
ft_compare_func compare_fun,
ft_update_func update_fun,
DESCRIPTOR desc,
FTNODE node,
int target_childnum,
FT_MSG cmd,
FT_MSG msg,
bool is_fresh,
txn_gc_info *gc_info,
size_t flow_deltas[],

4
storage/tokudb/ft-index/ft/ft-node-deserialize.cc
View File

@@ -193,7 +193,7 @@ check_node_info_checksum(struct rbuf *rb)
{
int r = 0;
// Verify checksum of header stored.
uint32_t checksum = x1764_memory(rb->buf, rb->ndone);
uint32_t checksum = toku_x1764_memory(rb->buf, rb->ndone);
uint32_t stored_checksum = rbuf_int(rb);
if (stored_checksum != checksum) {
@@ -229,7 +229,7 @@ check_legacy_end_checksum(struct rbuf *rb)
{
int r = 0;
uint32_t expected_xsum = rbuf_int(rb);
uint32_t actual_xsum = x1764_memory(rb->buf, rb->size - 4);
uint32_t actual_xsum = toku_x1764_memory(rb->buf, rb->size - 4);
if (expected_xsum != actual_xsum) {
r = TOKUDB_BAD_CHECKSUM;
}

899
storage/tokudb/ft-index/ft/ft-ops.cc
View File

File diff suppressed because it is too large Load Diff

74
storage/tokudb/ft-index/ft/ft-ops.h
View File

@@ -103,7 +103,7 @@ PATENT RIGHTS GRANT:
// A callback function is invoked with the key, and the data.
// The pointers (to the bytevecs) must not be modified. The data must be copied out before the callback function returns.
// Note: In the thread-safe version, the brt node remains locked while the callback function runs. So return soon, and don't call the BRT code from the callback function.
// Note: In the thread-safe version, the ftnode remains locked while the callback function runs. So return soon, and don't call the ft code from the callback function.
// If the callback function returns a nonzero value (an error code), then that error code is returned from the get function itself.
// The cursor object will have been updated (so that if result==0 the current value is the value being passed)
// (If r!=0 then the cursor won't have been updated.)
@@ -141,9 +141,9 @@ void toku_ft_handle_set_fanout(FT_HANDLE, unsigned int fanout);
void toku_ft_handle_get_fanout(FT_HANDLE, unsigned int *fanout);
void toku_ft_set_bt_compare(FT_HANDLE, ft_compare_func);
ft_compare_func toku_ft_get_bt_compare (FT_HANDLE brt);
ft_compare_func toku_ft_get_bt_compare (FT_HANDLE ft_h);
void toku_ft_set_redirect_callback(FT_HANDLE brt, on_redirect_callback redir_cb, void* extra);
void toku_ft_set_redirect_callback(FT_HANDLE ft_h, on_redirect_callback redir_cb, void* extra);
// How updates (update/insert/deletes) work:
// There are two flavers of upsertdels: Singleton and broadcast.
@@ -181,7 +181,7 @@ void toku_ft_set_redirect_callback(FT_HANDLE brt, on_redirect_callback redir_cb,
// Implementation note: Acquires a write lock on the entire database.
// This function works by sending an BROADCAST-UPDATE message containing
// the key and the extra.
void toku_ft_set_update(FT_HANDLE brt, ft_update_func update_fun);
void toku_ft_set_update(FT_HANDLE ft_h, ft_update_func update_fun);
int toku_ft_handle_open(FT_HANDLE, const char *fname_in_env,
int is_create, int only_create, CACHETABLE ct, TOKUTXN txn) __attribute__ ((warn_unused_result));
@@ -208,60 +208,60 @@ toku_ft_handle_open_with_dict_id(
DICTIONARY_ID use_dictionary_id
) __attribute__ ((warn_unused_result));
int toku_ft_lookup (FT_HANDLE brt, DBT *k, FT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_ft_lookup (FT_HANDLE ft_h, DBT *k, FT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
// Effect: Insert a key and data pair into a brt
void toku_ft_insert (FT_HANDLE brt, DBT *k, DBT *v, TOKUTXN txn);
// Effect: Insert a key and data pair into an ft
void toku_ft_insert (FT_HANDLE ft_h, DBT *k, DBT *v, TOKUTXN txn);
// Effect: Optimize the ft
void toku_ft_optimize (FT_HANDLE brt);
void toku_ft_optimize (FT_HANDLE ft_h);
// Effect: Insert a key and data pair into a brt if the oplsn is newer than the brt lsn. This function is called during recovery.
void toku_ft_maybe_insert (FT_HANDLE brt, DBT *k, DBT *v, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging, enum ft_msg_type type);
// Effect: Insert a key and data pair into an ft if the oplsn is newer than the ft's lsn. This function is called during recovery.
void toku_ft_maybe_insert (FT_HANDLE ft_h, DBT *k, DBT *v, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging, enum ft_msg_type type);
// Effect: Send an update message into a brt. This function is called
// Effect: Send an update message into an ft. This function is called
// during recovery.
void toku_ft_maybe_update(FT_HANDLE brt, const DBT *key, const DBT *update_function_extra, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging);
void toku_ft_maybe_update(FT_HANDLE ft_h, const DBT *key, const DBT *update_function_extra, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging);
// Effect: Send a broadcasting update message into a brt. This function
// Effect: Send a broadcasting update message into an ft. This function
// is called during recovery.
void toku_ft_maybe_update_broadcast(FT_HANDLE brt, const DBT *update_function_extra, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging, bool is_resetting_op);
void toku_ft_maybe_update_broadcast(FT_HANDLE ft_h, const DBT *update_function_extra, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging, bool is_resetting_op);
void toku_ft_load_recovery(TOKUTXN txn, FILENUM old_filenum, char const * new_iname, int do_fsync, int do_log, LSN *load_lsn);
void toku_ft_load(FT_HANDLE brt, TOKUTXN txn, char const * new_iname, int do_fsync, LSN *get_lsn);
void toku_ft_load(FT_HANDLE ft_h, TOKUTXN txn, char const * new_iname, int do_fsync, LSN *get_lsn);
void toku_ft_hot_index_recovery(TOKUTXN txn, FILENUMS filenums, int do_fsync, int do_log, LSN *hot_index_lsn);
void toku_ft_hot_index(FT_HANDLE brt, TOKUTXN txn, FILENUMS filenums, int do_fsync, LSN *lsn);
void toku_ft_hot_index(FT_HANDLE ft_h, TOKUTXN txn, FILENUMS filenums, int do_fsync, LSN *lsn);
void toku_ft_log_put_multiple (TOKUTXN txn, FT_HANDLE src_ft, FT_HANDLE *brts, uint32_t num_fts, const DBT *key, const DBT *val);
void toku_ft_log_put (TOKUTXN txn, FT_HANDLE brt, const DBT *key, const DBT *val);
void toku_ft_log_del_multiple (TOKUTXN txn, FT_HANDLE src_ft, FT_HANDLE *brts, uint32_t num_fts, const DBT *key, const DBT *val);
void toku_ft_log_del (TOKUTXN txn, FT_HANDLE brt, const DBT *key);
void toku_ft_log_put_multiple (TOKUTXN txn, FT_HANDLE src_ft, FT_HANDLE *fts, uint32_t num_fts, const DBT *key, const DBT *val);
void toku_ft_log_put (TOKUTXN txn, FT_HANDLE ft_h, const DBT *key, const DBT *val);
void toku_ft_log_del_multiple (TOKUTXN txn, FT_HANDLE src_ft, FT_HANDLE *fts, uint32_t num_fts, const DBT *key, const DBT *val);
void toku_ft_log_del (TOKUTXN txn, FT_HANDLE ft_h, const DBT *key);
// Effect: Delete a key from a brt
void toku_ft_delete (FT_HANDLE brt, DBT *k, TOKUTXN txn);
// Effect: Delete a key from an ft
void toku_ft_delete (FT_HANDLE ft_h, DBT *k, TOKUTXN txn);
// Effect: Delete a key from a brt if the oplsn is newer than the brt lsn. This function is called during recovery.
void toku_ft_maybe_delete (FT_HANDLE brt, DBT *k, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging);
// Effect: Delete a key from an ft if the oplsn is newer than the ft lsn. This function is called during recovery.
void toku_ft_maybe_delete (FT_HANDLE ft_h, DBT *k, TOKUTXN txn, bool oplsn_valid, LSN oplsn, bool do_logging);
TXNID toku_ft_get_oldest_referenced_xid_estimate(FT_HANDLE ft_h);
TXN_MANAGER toku_ft_get_txn_manager(FT_HANDLE ft_h);
void toku_ft_send_insert(FT_HANDLE brt, DBT *key, DBT *val, XIDS xids, enum ft_msg_type type, txn_gc_info *gc_info);
void toku_ft_send_delete(FT_HANDLE brt, DBT *key, XIDS xids, txn_gc_info *gc_info);
void toku_ft_send_commit_any(FT_HANDLE brt, DBT *key, XIDS xids, txn_gc_info *gc_info);
void toku_ft_send_insert(FT_HANDLE ft_h, DBT *key, DBT *val, XIDS xids, enum ft_msg_type type, txn_gc_info *gc_info);
void toku_ft_send_delete(FT_HANDLE ft_h, DBT *key, XIDS xids, txn_gc_info *gc_info);
void toku_ft_send_commit_any(FT_HANDLE ft_h, DBT *key, XIDS xids, txn_gc_info *gc_info);
int toku_close_ft_handle_nolsn (FT_HANDLE, char **error_string) __attribute__ ((warn_unused_result));
int toku_dump_ft (FILE *,FT_HANDLE brt) __attribute__ ((warn_unused_result));
int toku_dump_ft (FILE *,FT_HANDLE ft_h) __attribute__ ((warn_unused_result));
extern int toku_ft_debug_mode;
int toku_verify_ft (FT_HANDLE brt) __attribute__ ((warn_unused_result));
int toku_verify_ft_with_progress (FT_HANDLE brt, int (*progress_callback)(void *extra, float progress), void *extra, int verbose, int keep_going) __attribute__ ((warn_unused_result));
int toku_verify_ft (FT_HANDLE ft_h) __attribute__ ((warn_unused_result));
int toku_verify_ft_with_progress (FT_HANDLE ft_h, int (*progress_callback)(void *extra, float progress), void *extra, int verbose, int keep_going) __attribute__ ((warn_unused_result));
typedef struct ft_cursor *FT_CURSOR;
int toku_ft_cursor (FT_HANDLE, FT_CURSOR*, TOKUTXN, bool, bool) __attribute__ ((warn_unused_result));
void toku_ft_cursor_set_leaf_mode(FT_CURSOR);
// Sets a boolean on the brt cursor that prevents uncessary copying of
// Sets a boolean on the ft cursor that prevents uncessary copying of
// the cursor duing a one query.
void toku_ft_cursor_set_temporary(FT_CURSOR);
void toku_ft_cursor_remove_restriction(FT_CURSOR);
@@ -278,7 +278,7 @@ int toku_ft_cursor_next(FT_CURSOR cursor, FT_GET_CALLBACK_FUNCTION getf, void *g
int toku_ft_cursor_prev(FT_CURSOR cursor, FT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_ft_cursor_current(FT_CURSOR cursor, int op, FT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_ft_cursor_set(FT_CURSOR cursor, DBT *key, FT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_ft_cursor_set_range(FT_CURSOR cursor, DBT *key, FT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_ft_cursor_set_range(FT_CURSOR cursor, DBT *key, DBT *key_bound, FT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_ft_cursor_set_range_reverse(FT_CURSOR cursor, DBT *key, FT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_ft_cursor_get_both_range(FT_CURSOR cursor, DBT *key, DBT *val, FT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_ft_cursor_get_both_range_reverse(FT_CURSOR cursor, DBT *key, DBT *val, FT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
@@ -298,8 +298,8 @@ enum ft_flags {
TOKU_DB_VALCMP_BUILTIN_13 = (1<<3),
};
void toku_ft_keyrange(FT_HANDLE brt, DBT *key, uint64_t *less, uint64_t *equal, uint64_t *greater);
void toku_ft_keysrange(FT_HANDLE brt, DBT* key_left, DBT* key_right, uint64_t *less_p, uint64_t* equal_left_p, uint64_t* middle_p, uint64_t* equal_right_p, uint64_t* greater_p, bool* middle_3_exact_p);
void toku_ft_keyrange(FT_HANDLE ft_h, DBT *key, uint64_t *less, uint64_t *equal, uint64_t *greater);
void toku_ft_keysrange(FT_HANDLE ft_h, DBT* key_left, DBT* key_right, uint64_t *less_p, uint64_t* equal_left_p, uint64_t* middle_p, uint64_t* equal_right_p, uint64_t* greater_p, bool* middle_3_exact_p);
int toku_ft_get_key_after_bytes(FT_HANDLE ft_h, const DBT *start_key, uint64_t skip_len, void (*callback)(const DBT *end_key, uint64_t actually_skipped, void *extra), void *cb_extra);
@@ -341,9 +341,9 @@ void toku_maybe_preallocate_in_file (int fd, int64_t size, int64_t expected_size
// Effect: make the file bigger by either doubling it or growing by 16MiB whichever is less, until it is at least size
// Return 0 on success, otherwise an error number.
int toku_ft_get_fragmentation(FT_HANDLE brt, TOKU_DB_FRAGMENTATION report) __attribute__ ((warn_unused_result));
int toku_ft_get_fragmentation(FT_HANDLE ft_h, TOKU_DB_FRAGMENTATION report) __attribute__ ((warn_unused_result));
bool toku_ft_is_empty_fast (FT_HANDLE brt) __attribute__ ((warn_unused_result));
bool toku_ft_is_empty_fast (FT_HANDLE ft_h) __attribute__ ((warn_unused_result));
// Effect: Return true if there are no messages or leaf entries in the tree. If so, it's empty. If there are messages or leaf entries, we say it's not empty
// even though if we were to optimize the tree it might turn out that they are empty.
@@ -358,4 +358,6 @@ extern bool garbage_collection_debug;
void toku_ft_set_direct_io(bool direct_io_on);
void toku_ft_set_compress_buffers_before_eviction(bool compress_buffers);
void toku_note_deserialized_basement_node(bool fixed_key_size);
#endif

5
storage/tokudb/ft-index/ft/ft-search.h
View File

@@ -136,15 +136,18 @@ typedef struct ft_search {
// no guarantee that we will get everything pinned again. We ought to keep nodes pinned when we retry, except that on the
// way out with a DB_NOTFOUND we ought to unpin those nodes. See #3528.
DBT pivot_bound;
const DBT *k_bound;
} ft_search_t;
/* initialize the search compare object */
static inline ft_search_t *ft_search_init(ft_search_t *so, ft_search_compare_func_t compare, enum ft_search_direction_e direction, const DBT *k, void *context) {
static inline ft_search_t *ft_search_init(ft_search_t *so, ft_search_compare_func_t compare, enum ft_search_direction_e direction,
const DBT *k, const DBT *k_bound, void *context) {
so->compare = compare;
so->direction = direction;
so->k = k;
so->context = context;
toku_init_dbt(&so->pivot_bound);
so->k_bound = k_bound;
return so;
}

11
storage/tokudb/ft-index/ft/ft-serialize.cc
View File

@@ -132,7 +132,7 @@ toku_serialize_descriptor_contents_to_fd(int fd, const DESCRIPTOR desc, DISKOFF
toku_serialize_descriptor_contents_to_wbuf(&w, desc);
{
//Add checksum
uint32_t checksum = x1764_finish(&w.checksum);
uint32_t checksum = toku_x1764_finish(&w.checksum);
wbuf_int(&w, checksum);
}
lazy_assert(w.ndone==w.size);
@@ -185,7 +185,7 @@ deserialize_descriptor_from(int fd, BLOCK_TABLE bt, DESCRIPTOR desc, int layout_
}
{
// check the checksum
uint32_t x1764 = x1764_memory(dbuf, size-4);
uint32_t x1764 = toku_x1764_memory(dbuf, size-4);
//printf("%s:%d read from %ld (x1764 offset=%ld) size=%ld\n", __FILE__, __LINE__, block_translation_address_on_disk, offset, block_translation_size_on_disk);
uint32_t stored_x1764 = toku_dtoh32(*(int*)(dbuf + size-4));
if (x1764 != stored_x1764) {
@@ -210,7 +210,7 @@ exit:
int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
// Effect: Deserialize the ft header.
// We deserialize brt header only once and then share everything with all the brts.
// We deserialize ft_header only once and then share everything with all the FTs.
{
int r;
FT ft = NULL;
@@ -462,6 +462,7 @@ serialize_ft_min_size (uint32_t version) {
size_t size = 0;
switch(version) {
case FT_LAYOUT_VERSION_26:
case FT_LAYOUT_VERSION_25:
case FT_LAYOUT_VERSION_24:
case FT_LAYOUT_VERSION_23:
@@ -623,7 +624,7 @@ int deserialize_ft_from_fd_into_rbuf(int fd,
//Verify checksum (FT_LAYOUT_VERSION_13 or later, when checksum function changed)
uint32_t calculated_x1764;
calculated_x1764 = x1764_memory(rb->buf, rb->size-4);
calculated_x1764 = toku_x1764_memory(rb->buf, rb->size-4);
uint32_t stored_x1764;
stored_x1764 = toku_dtoh32(*(int*)(rb->buf+rb->size-4));
if (calculated_x1764 != stored_x1764) {
@@ -803,7 +804,7 @@ void toku_serialize_ft_to_wbuf (
wbuf_char(wbuf, (unsigned char) h->compression_method);
wbuf_MSN(wbuf, h->highest_unused_msn_for_upgrade);
wbuf_MSN(wbuf, h->max_msn_in_ft);
uint32_t checksum = x1764_finish(&wbuf->checksum);
uint32_t checksum = toku_x1764_finish(&wbuf->checksum);
wbuf_int(wbuf, checksum);
lazy_assert(wbuf->ndone == wbuf->size);
}

81
storage/tokudb/ft-index/ft/ft-test-helpers.cc
View File

@@ -96,7 +96,7 @@ PATENT RIGHTS GRANT:
#include "fttypes.h"
#include "ule.h"
// dummymsn needed to simulate msn because messages are injected at a lower level than toku_ft_root_put_cmd()
// dummymsn needed to simulate msn because messages are injected at a lower level than toku_ft_root_put_msg()
#define MIN_DUMMYMSN ((MSN) {(uint64_t)1 << 62})
static MSN dummymsn;
static int testsetup_initialized = 0;
@@ -119,10 +119,10 @@ next_dummymsn(void) {
bool ignore_if_was_already_open;
int toku_testsetup_leaf(FT_HANDLE brt, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens) {
int toku_testsetup_leaf(FT_HANDLE ft_handle, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens) {
FTNODE node;
assert(testsetup_initialized);
toku_create_new_ftnode(brt, &node, 0, n_children);
toku_create_new_ftnode(ft_handle, &node, 0, n_children);
int i;
for (i=0; i<n_children; i++) {
BP_STATE(node,i) = PT_AVAIL;
@@ -134,15 +134,15 @@ int toku_testsetup_leaf(FT_HANDLE brt, BLOCKNUM *blocknum, int n_children, char
}
*blocknum = node->thisnodename;
toku_unpin_ftnode(brt->ft, node);
toku_unpin_ftnode(ft_handle->ft, node);
return 0;
}
// Don't bother to clean up carefully if something goes wrong. (E.g., it's OK to have malloced stuff that hasn't been freed.)
int toku_testsetup_nonleaf (FT_HANDLE brt, int height, BLOCKNUM *blocknum, int n_children, BLOCKNUM *children, char **keys, int *keylens) {
int toku_testsetup_nonleaf (FT_HANDLE ft_handle, int height, BLOCKNUM *blocknum, int n_children, BLOCKNUM *children, char **keys, int *keylens) {
FTNODE node;
assert(testsetup_initialized);
toku_create_new_ftnode(brt, &node, height, n_children);
toku_create_new_ftnode(ft_handle, &node, height, n_children);
int i;
for (i=0; i<n_children; i++) {
BP_BLOCKNUM(node, i) = children[i];
@@ -153,28 +153,28 @@ int toku_testsetup_nonleaf (FT_HANDLE brt, int height, BLOCKNUM *blocknum, int n
node->totalchildkeylens += keylens[i];
}
*blocknum = node->thisnodename;
toku_unpin_ftnode(brt->ft, node);
toku_unpin_ftnode(ft_handle->ft, node);
return 0;
}
int toku_testsetup_root(FT_HANDLE brt, BLOCKNUM blocknum) {
int toku_testsetup_root(FT_HANDLE ft_handle, BLOCKNUM blocknum) {
assert(testsetup_initialized);
brt->ft->h->root_blocknum = blocknum;
ft_handle->ft->h->root_blocknum = blocknum;
return 0;
}
int toku_testsetup_get_sersize(FT_HANDLE brt, BLOCKNUM diskoff) // Return the size on disk
int toku_testsetup_get_sersize(FT_HANDLE ft_handle, BLOCKNUM diskoff) // Return the size on disk
{
assert(testsetup_initialized);
void *node_v;
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, brt->ft);
fill_bfe_for_full_read(&bfe, ft_handle->ft);
int r = toku_cachetable_get_and_pin(
brt->ft->cf, diskoff,
toku_cachetable_hash(brt->ft->cf, diskoff),
ft_handle->ft->cf, diskoff,
toku_cachetable_hash(ft_handle->ft->cf, diskoff),
&node_v,
NULL,
get_write_callbacks_for_node(brt->ft),
get_write_callbacks_for_node(ft_handle->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
@@ -184,25 +184,25 @@ int toku_testsetup_get_sersize(FT_HANDLE brt, BLOCKNUM diskoff) // Return the si
assert(r==0);
FTNODE CAST_FROM_VOIDP(node, node_v);
int size = toku_serialize_ftnode_size(node);
toku_unpin_ftnode(brt->ft, node);
toku_unpin_ftnode(ft_handle->ft, node);
return size;
}
int toku_testsetup_insert_to_leaf (FT_HANDLE brt, BLOCKNUM blocknum, const char *key, int keylen, const char *val, int vallen) {
int toku_testsetup_insert_to_leaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, const char *key, int keylen, const char *val, int vallen) {
void *node_v;
int r;
assert(testsetup_initialized);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, brt->ft);
fill_bfe_for_full_read(&bfe, ft_handle->ft);
r = toku_cachetable_get_and_pin(
brt->ft->cf,
ft_handle->ft->cf,
blocknum,
toku_cachetable_hash(brt->ft->cf, blocknum),
toku_cachetable_hash(ft_handle->ft->cf, blocknum),
&node_v,
NULL,
get_write_callbacks_for_node(brt->ft),
get_write_callbacks_for_node(ft_handle->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
@@ -216,19 +216,19 @@ int toku_testsetup_insert_to_leaf (FT_HANDLE brt, BLOCKNUM blocknum, const char
DBT keydbt,valdbt;
MSN msn = next_dummymsn();
FT_MSG_S cmd = { FT_INSERT, msn, xids_get_root_xids(),
FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(),
.u = { .id = { toku_fill_dbt(&keydbt, key, keylen),
toku_fill_dbt(&valdbt, val, vallen) } } };
static size_t zero_flow_deltas[] = { 0, 0 };
txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true);
toku_ft_node_put_cmd (
brt->ft->compare_fun,
brt->ft->update_fun,
&brt->ft->cmp_descriptor,
toku_ft_node_put_msg (
ft_handle->ft->compare_fun,
ft_handle->ft->update_fun,
&ft_handle->ft->cmp_descriptor,
node,
-1,
&cmd,
&msg,
true,
&gc_info,
zero_flow_deltas,
@@ -237,7 +237,7 @@ int toku_testsetup_insert_to_leaf (FT_HANDLE brt, BLOCKNUM blocknum, const char
toku_verify_or_set_counts(node);
toku_unpin_ftnode(brt->ft, node);
toku_unpin_ftnode(ft_handle->ft, node);
return 0;
}
@@ -254,33 +254,32 @@ toku_pin_node_with_min_bfe(FTNODE* node, BLOCKNUM b, FT_HANDLE t)
{
struct ftnode_fetch_extra bfe;
fill_bfe_for_min_read(&bfe, t->ft);
toku_pin_ftnode_off_client_thread(
toku_pin_ftnode(
t->ft,
b,
toku_cachetable_hash(t->ft->cf, b),
&bfe,
PL_WRITE_EXPENSIVE,
0,
NULL,
node
node,
true
);
}
int toku_testsetup_insert_to_nonleaf (FT_HANDLE brt, BLOCKNUM blocknum, enum ft_msg_type cmdtype, const char *key, int keylen, const char *val, int vallen) {
int toku_testsetup_insert_to_nonleaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, enum ft_msg_type msgtype, const char *key, int keylen, const char *val, int vallen) {
void *node_v;
int r;
assert(testsetup_initialized);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, brt->ft);
fill_bfe_for_full_read(&bfe, ft_handle->ft);
r = toku_cachetable_get_and_pin(
brt->ft->cf,
ft_handle->ft->cf,
blocknum,
toku_cachetable_hash(brt->ft->cf, blocknum),
toku_cachetable_hash(ft_handle->ft->cf, blocknum),
&node_v,
NULL,
get_write_callbacks_for_node(brt->ft),
get_write_callbacks_for_node(ft_handle->ft),
toku_ftnode_fetch_callback,
toku_ftnode_pf_req_callback,
toku_ftnode_pf_callback,
@@ -294,19 +293,19 @@ int toku_testsetup_insert_to_nonleaf (FT_HANDLE brt, BLOCKNUM blocknum, enum ft_
DBT k;
int childnum = toku_ftnode_which_child(node,
toku_fill_dbt(&k, key, keylen),
&brt->ft->cmp_descriptor, brt->ft->compare_fun);
&ft_handle->ft->cmp_descriptor, ft_handle->ft->compare_fun);
XIDS xids_0 = xids_get_root_xids();
MSN msn = next_dummymsn();
toku_bnc_insert_msg(BNC(node, childnum), key, keylen, val, vallen, cmdtype, msn, xids_0, true, NULL, testhelper_string_key_cmp);
toku_bnc_insert_msg(BNC(node, childnum), key, keylen, val, vallen, msgtype, msn, xids_0, true, NULL, testhelper_string_key_cmp);
// Hack to get the test working. The problem is that this test
// is directly queueing something in a FIFO instead of
// using brt APIs.
// using ft APIs.
node->max_msn_applied_to_node_on_disk = msn;
node->dirty = 1;
// Also hack max_msn_in_ft
brt->ft->h->max_msn_in_ft = msn;
ft_handle->ft->h->max_msn_in_ft = msn;
toku_unpin_ftnode(brt->ft, node);
toku_unpin_ftnode(ft_handle->ft, node);
return 0;
}

110
storage/tokudb/ft-index/ft/ft-verify.cc
View File

@@ -89,7 +89,7 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* Verify a BRT. */
/* Verify an FT. */
/* Check:
* The tree is of uniform depth (and the height is correct at every node)
* For each pivot key: the max of the stuff to the left is <= the pivot key < the min of the stuff to the right.
@@ -102,26 +102,26 @@ PATENT RIGHTS GRANT:
#include "ft.h"
static int
compare_pairs (FT_HANDLE brt, const DBT *a, const DBT *b) {
FAKE_DB(db, &brt->ft->cmp_descriptor);
int cmp = brt->ft->compare_fun(&db, a, b);
compare_pairs (FT_HANDLE ft_handle, const DBT *a, const DBT *b) {
FAKE_DB(db, &ft_handle->ft->cmp_descriptor);
int cmp = ft_handle->ft->compare_fun(&db, a, b);
return cmp;
}
static int
compare_pair_to_key (FT_HANDLE brt, const DBT *a, bytevec key, ITEMLEN keylen) {
compare_pair_to_key (FT_HANDLE ft_handle, const DBT *a, bytevec key, ITEMLEN keylen) {
DBT y;
FAKE_DB(db, &brt->ft->cmp_descriptor);
int cmp = brt->ft->compare_fun(&db, a, toku_fill_dbt(&y, key, keylen));
FAKE_DB(db, &ft_handle->ft->cmp_descriptor);
int cmp = ft_handle->ft->compare_fun(&db, a, toku_fill_dbt(&y, key, keylen));
return cmp;
}
static int
verify_msg_in_child_buffer(FT_HANDLE brt, enum ft_msg_type type, MSN msn, bytevec key, ITEMLEN keylen, bytevec UU(data), ITEMLEN UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot)
verify_msg_in_child_buffer(FT_HANDLE ft_handle, enum ft_msg_type type, MSN msn, bytevec key, ITEMLEN keylen, bytevec UU(data), ITEMLEN UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot)
__attribute__((warn_unused_result));
static int
verify_msg_in_child_buffer(FT_HANDLE brt, enum ft_msg_type type, MSN msn, bytevec key, ITEMLEN keylen, bytevec UU(data), ITEMLEN UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot) {
verify_msg_in_child_buffer(FT_HANDLE ft_handle, enum ft_msg_type type, MSN msn, bytevec key, ITEMLEN keylen, bytevec UU(data), ITEMLEN UU(datalen), XIDS UU(xids), const DBT *lesser_pivot, const DBT *greatereq_pivot) {
int result = 0;
if (msn.msn == ZERO_MSN.msn)
result = EINVAL;
@@ -135,12 +135,12 @@ verify_msg_in_child_buffer(FT_HANDLE brt, enum ft_msg_type type, MSN msn, byteve
case FT_COMMIT_ANY:
// verify key in bounds
if (lesser_pivot) {
int compare = compare_pair_to_key(brt, lesser_pivot, key, keylen);
int compare = compare_pair_to_key(ft_handle, lesser_pivot, key, keylen);
if (compare >= 0)
result = EINVAL;
}
if (result == 0 && greatereq_pivot) {
int compare = compare_pair_to_key(brt, greatereq_pivot, key, keylen);
int compare = compare_pair_to_key(ft_handle, greatereq_pivot, key, keylen);
if (compare < 0)
result = EINVAL;
}
@@ -152,7 +152,7 @@ verify_msg_in_child_buffer(FT_HANDLE brt, enum ft_msg_type type, MSN msn, byteve
static DBT
get_ith_key_dbt (BASEMENTNODE bn, int i) {
DBT kdbt;
int r = bn->data_buffer.fetch_le_key_and_len(i, &kdbt.size, &kdbt.data);
int r = bn->data_buffer.fetch_key_and_len(i, &kdbt.size, &kdbt.data);
invariant_zero(r); // this is a bad failure if it happens.
return kdbt;
}
@@ -243,7 +243,7 @@ int verify_marked_messages(const int32_t &offset, const uint32_t UU(idx), struct
template<typename verify_omt_t>
static int
verify_sorted_by_key_msn(FT_HANDLE brt, FIFO fifo, const verify_omt_t &mt) {
verify_sorted_by_key_msn(FT_HANDLE ft_handle, FIFO fifo, const verify_omt_t &mt) {
int result = 0;
size_t last_offset = 0;
for (uint32_t i = 0; i < mt.size(); i++) {
@@ -253,8 +253,8 @@ verify_sorted_by_key_msn(FT_HANDLE brt, FIFO fifo, const verify_omt_t &mt) {
if (i > 0) {
struct toku_fifo_entry_key_msn_cmp_extra extra;
ZERO_STRUCT(extra);
extra.desc = &brt->ft->cmp_descriptor;
extra.cmp = brt->ft->compare_fun;
extra.desc = &ft_handle->ft->cmp_descriptor;
extra.cmp = ft_handle->ft->compare_fun;
extra.fifo = fifo;
if (toku_fifo_entry_key_msn_cmp(extra, last_offset, offset) >= 0) {
result = TOKUDB_NEEDS_REPAIR;
@@ -268,11 +268,11 @@ verify_sorted_by_key_msn(FT_HANDLE brt, FIFO fifo, const verify_omt_t &mt) {
template<typename count_omt_t>
static int
count_eq_key_msn(FT_HANDLE brt, FIFO fifo, const count_omt_t &mt, const DBT *key, MSN msn) {
count_eq_key_msn(FT_HANDLE ft_handle, FIFO fifo, const count_omt_t &mt, const DBT *key, MSN msn) {
struct toku_fifo_entry_key_msn_heaviside_extra extra;
ZERO_STRUCT(extra);
extra.desc = &brt->ft->cmp_descriptor;
extra.cmp = brt->ft->compare_fun;
extra.desc = &ft_handle->ft->cmp_descriptor;
extra.cmp = ft_handle->ft->compare_fun;
extra.fifo = fifo;
extra.key = key;
extra.msn = msn;
@@ -290,28 +290,26 @@ count_eq_key_msn(FT_HANDLE brt, FIFO fifo, const count_omt_t &mt, const DBT *key
void
toku_get_node_for_verify(
BLOCKNUM blocknum,
FT_HANDLE brt,
FT_HANDLE ft_handle,
FTNODE* nodep
)
{
uint32_t fullhash = toku_cachetable_hash(brt->ft->cf, blocknum);
uint32_t fullhash = toku_cachetable_hash(ft_handle->ft->cf, blocknum);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, brt->ft);
toku_pin_ftnode_off_client_thread_and_maybe_move_messages(
brt->ft,
fill_bfe_for_full_read(&bfe, ft_handle->ft);
toku_pin_ftnode(
ft_handle->ft,
blocknum,
fullhash,
&bfe,
PL_WRITE_EXPENSIVE, // may_modify_node
0,
NULL,
nodep,
false
);
}
static int
toku_verify_ftnode_internal(FT_HANDLE brt,
toku_verify_ftnode_internal(FT_HANDLE ft_handle,
MSN rootmsn, MSN parentmsn, bool messages_exist_above,
FTNODE node, int height,
const DBT *lesser_pivot, // Everything in the subtree should be > lesser_pivot. (lesser_pivot==NULL if there is no lesser pivot.)
@@ -334,17 +332,17 @@ toku_verify_ftnode_internal(FT_HANDLE brt,
}
// Verify that all the pivot keys are in order.
for (int i = 0; i < node->n_children-2; i++) {
int compare = compare_pairs(brt, &node->childkeys[i], &node->childkeys[i+1]);
int compare = compare_pairs(ft_handle, &node->childkeys[i], &node->childkeys[i+1]);
VERIFY_ASSERTION(compare < 0, i, "Value is >= the next value");
}
// Verify that all the pivot keys are lesser_pivot < pivot <= greatereq_pivot
for (int i = 0; i < node->n_children-1; i++) {
if (lesser_pivot) {
int compare = compare_pairs(brt, lesser_pivot, &node->childkeys[i]);
int compare = compare_pairs(ft_handle, lesser_pivot, &node->childkeys[i]);
VERIFY_ASSERTION(compare < 0, i, "Pivot is >= the lower-bound pivot");
}
if (greatereq_pivot) {
int compare = compare_pairs(brt, greatereq_pivot, &node->childkeys[i]);
int compare = compare_pairs(ft_handle, greatereq_pivot, &node->childkeys[i]);
VERIFY_ASSERTION(compare >= 0, i, "Pivot is < the upper-bound pivot");
}
}
@@ -356,12 +354,12 @@ toku_verify_ftnode_internal(FT_HANDLE brt,
MSN last_msn = ZERO_MSN;
// Verify that messages in the buffers are in the right place.
NONLEAF_CHILDINFO bnc = BNC(node, i);
VERIFY_ASSERTION(verify_sorted_by_key_msn(brt, bnc->buffer, bnc->fresh_message_tree) == 0, i, "fresh_message_tree");
VERIFY_ASSERTION(verify_sorted_by_key_msn(brt, bnc->buffer, bnc->stale_message_tree) == 0, i, "stale_message_tree");
VERIFY_ASSERTION(verify_sorted_by_key_msn(ft_handle, bnc->buffer, bnc->fresh_message_tree) == 0, i, "fresh_message_tree");
VERIFY_ASSERTION(verify_sorted_by_key_msn(ft_handle, bnc->buffer, bnc->stale_message_tree) == 0, i, "stale_message_tree");
FIFO_ITERATE(bnc->buffer, key, keylen, data, datalen, itype, msn, xid, is_fresh,
({
enum ft_msg_type type = (enum ft_msg_type) itype;
int r = verify_msg_in_child_buffer(brt, type, msn, key, keylen, data, datalen, xid,
int r = verify_msg_in_child_buffer(ft_handle, type, msn, key, keylen, data, datalen, xid,
curr_less_pivot,
curr_geq_pivot);
VERIFY_ASSERTION(r==0, i, "A message in the buffer is out of place");
@@ -372,7 +370,7 @@ toku_verify_ftnode_internal(FT_HANDLE brt,
DBT keydbt;
toku_fill_dbt(&keydbt, key, keylen);
int total_count = 0;
count = count_eq_key_msn(brt, bnc->buffer, bnc->fresh_message_tree, toku_fill_dbt(&keydbt, key, keylen), msn);
count = count_eq_key_msn(ft_handle, bnc->buffer, bnc->fresh_message_tree, toku_fill_dbt(&keydbt, key, keylen), msn);
total_count += count;
if (is_fresh) {
VERIFY_ASSERTION(count == 1, i, "a fresh message was not found in the fresh message tree");
@@ -380,7 +378,7 @@ toku_verify_ftnode_internal(FT_HANDLE brt,
VERIFY_ASSERTION(count == 0, i, "a stale message was found in the fresh message tree");
}
VERIFY_ASSERTION(count <= 1, i, "a message was found multiple times in the fresh message tree");
count = count_eq_key_msn(brt, bnc->buffer, bnc->stale_message_tree, &keydbt, msn);
count = count_eq_key_msn(ft_handle, bnc->buffer, bnc->stale_message_tree, &keydbt, msn);
total_count += count;
if (is_fresh) {
@@ -424,20 +422,20 @@ toku_verify_ftnode_internal(FT_HANDLE brt,
}
else {
BASEMENTNODE bn = BLB(node, i);
for (uint32_t j = 0; j < bn->data_buffer.omt_size(); j++) {
for (uint32_t j = 0; j < bn->data_buffer.num_klpairs(); j++) {
VERIFY_ASSERTION((rootmsn.msn >= this_msn.msn), 0, "leaf may have latest msn, but cannot be greater than root msn");
DBT kdbt = get_ith_key_dbt(bn, j);
if (curr_less_pivot) {
int compare = compare_pairs(brt, curr_less_pivot, &kdbt);
int compare = compare_pairs(ft_handle, curr_less_pivot, &kdbt);
VERIFY_ASSERTION(compare < 0, j, "The leafentry is >= the lower-bound pivot");
}
if (curr_geq_pivot) {
int compare = compare_pairs(brt, curr_geq_pivot, &kdbt);
int compare = compare_pairs(ft_handle, curr_geq_pivot, &kdbt);
VERIFY_ASSERTION(compare >= 0, j, "The leafentry is < the upper-bound pivot");
}
if (0 < j) {
DBT prev_key_dbt = get_ith_key_dbt(bn, j-1);
int compare = compare_pairs(brt, &prev_key_dbt, &kdbt);
int compare = compare_pairs(ft_handle, &prev_key_dbt, &kdbt);
VERIFY_ASSERTION(compare < 0, j, "Adjacent leafentries are out of order");
}
}
@@ -451,7 +449,7 @@ done:
// input is a pinned node, on exit, node is unpinned
int
toku_verify_ftnode (FT_HANDLE brt,
toku_verify_ftnode (FT_HANDLE ft_handle,
MSN rootmsn, MSN parentmsn, bool messages_exist_above,
FTNODE node, int height,
const DBT *lesser_pivot, // Everything in the subtree should be > lesser_pivot. (lesser_pivot==NULL if there is no lesser pivot.)
@@ -471,15 +469,15 @@ toku_verify_ftnode (FT_HANDLE brt,
// Otherwise we'll just do the next call
result = toku_verify_ftnode_internal(
brt, rootmsn, parentmsn, messages_exist_above, node, height, lesser_pivot, greatereq_pivot,
ft_handle, rootmsn, parentmsn, messages_exist_above, node, height, lesser_pivot, greatereq_pivot,
verbose, keep_going_on_failure, false);
if (result != 0 && (!keep_going_on_failure || result != TOKUDB_NEEDS_REPAIR)) goto done;
}
if (node->height > 0) {
toku_move_ftnode_messages_to_stale(brt->ft, node);
toku_move_ftnode_messages_to_stale(ft_handle->ft, node);
}
result2 = toku_verify_ftnode_internal(
brt, rootmsn, parentmsn, messages_exist_above, node, height, lesser_pivot, greatereq_pivot,
ft_handle, rootmsn, parentmsn, messages_exist_above, node, height, lesser_pivot, greatereq_pivot,
verbose, keep_going_on_failure, true);
if (result == 0) {
result = result2;
@@ -490,8 +488,8 @@ toku_verify_ftnode (FT_HANDLE brt,
if (recurse && node->height > 0) {
for (int i = 0; i < node->n_children; i++) {
FTNODE child_node;
toku_get_node_for_verify(BP_BLOCKNUM(node, i), brt, &child_node);
int r = toku_verify_ftnode(brt, rootmsn, this_msn, messages_exist_above || toku_bnc_n_entries(BNC(node, i)) > 0,
toku_get_node_for_verify(BP_BLOCKNUM(node, i), ft_handle, &child_node);
int r = toku_verify_ftnode(ft_handle, rootmsn, this_msn, messages_exist_above || toku_bnc_n_entries(BNC(node, i)) > 0,
child_node, node->height-1,
(i==0) ? lesser_pivot : &node->childkeys[i-1],
(i==node->n_children-1) ? greatereq_pivot : &node->childkeys[i],
@@ -504,7 +502,7 @@ toku_verify_ftnode (FT_HANDLE brt,
}
}
done:
toku_unpin_ftnode(brt->ft, node);
toku_unpin_ftnode(ft_handle->ft, node);
if (result == 0 && progress_callback)
result = progress_callback(progress_extra, 0.0);
@@ -513,26 +511,26 @@ done:
}
int
toku_verify_ft_with_progress (FT_HANDLE brt, int (*progress_callback)(void *extra, float progress), void *progress_extra, int verbose, int keep_on_going) {
assert(brt->ft);
toku_verify_ft_with_progress (FT_HANDLE ft_handle, int (*progress_callback)(void *extra, float progress), void *progress_extra, int verbose, int keep_on_going) {
assert(ft_handle->ft);
FTNODE root_node = NULL;
{
uint32_t root_hash;
CACHEKEY root_key;
toku_calculate_root_offset_pointer(brt->ft, &root_key, &root_hash);
toku_get_node_for_verify(root_key, brt, &root_node);
toku_calculate_root_offset_pointer(ft_handle->ft, &root_key, &root_hash);
toku_get_node_for_verify(root_key, ft_handle, &root_node);
}
int r = toku_verify_ftnode(brt, brt->ft->h->max_msn_in_ft, brt->ft->h->max_msn_in_ft, false, root_node, -1, NULL, NULL, progress_callback, progress_extra, 1, verbose, keep_on_going);
int r = toku_verify_ftnode(ft_handle, ft_handle->ft->h->max_msn_in_ft, ft_handle->ft->h->max_msn_in_ft, false, root_node, -1, NULL, NULL, progress_callback, progress_extra, 1, verbose, keep_on_going);
if (r == 0) {
toku_ft_lock(brt->ft);
brt->ft->h->time_of_last_verification = time(NULL);
brt->ft->h->dirty = 1;
toku_ft_unlock(brt->ft);
toku_ft_lock(ft_handle->ft);
ft_handle->ft->h->time_of_last_verification = time(NULL);
ft_handle->ft->h->dirty = 1;
toku_ft_unlock(ft_handle->ft);
}
return r;
}
int
toku_verify_ft (FT_HANDLE brt) {
return toku_verify_ft_with_progress(brt, NULL, NULL, 0, 0);
toku_verify_ft (FT_HANDLE ft_handle) {
return toku_verify_ft_with_progress(ft_handle, NULL, NULL, 0, 0);
}

62
storage/tokudb/ft-index/ft/ft.cc
View File

@@ -465,8 +465,8 @@ void toku_ft_create(FT *ftp, FT_OPTIONS options, CACHEFILE cf, TOKUTXN txn) {
*ftp = ft;
}
// TODO: (Zardosht) get rid of brt parameter
int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_acceptable_lsn, FT *header)
// TODO: (Zardosht) get rid of ft parameter
int toku_read_ft_and_store_in_cachefile (FT_HANDLE ft_handle, CACHEFILE cf, LSN max_acceptable_lsn, FT *header)
// If the cachefile already has the header, then just get it.
// If the cachefile has not been initialized, then don't modify anything.
// max_acceptable_lsn is the latest acceptable checkpointed version of the file.
@@ -475,8 +475,8 @@ int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_ac
FT h;
if ((h = (FT) toku_cachefile_get_userdata(cf))!=0) {
*header = h;
assert(brt->options.update_fun == h->update_fun);
assert(brt->options.compare_fun == h->compare_fun);
assert(ft_handle->options.update_fun == h->update_fun);
assert(ft_handle->options.compare_fun == h->compare_fun);
return 0;
}
}
@@ -494,8 +494,8 @@ int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_ac
// GCC 4.8 seems to get confused by the gotos in the deserialize code and think h is maybe uninitialized.
invariant_notnull(h);
h->cf = cf;
h->compare_fun = brt->options.compare_fun;
h->update_fun = brt->options.update_fun;
h->compare_fun = ft_handle->options.compare_fun;
h->update_fun = ft_handle->options.update_fun;
toku_cachefile_set_userdata(cf,
(void*)h,
ft_log_fassociate_during_checkpoint,
@@ -557,13 +557,13 @@ FT_HANDLE toku_ft_get_only_existing_ft_handle(FT h) {
return ft_handle_ret;
}
// Purpose: set fields in brt_header to capture accountability info for start of HOT optimize.
// Purpose: set fields in ft_header to capture accountability info for start of HOT optimize.
// Note: HOT accountability variables in header are modified only while holding header lock.
// (Header lock is really needed for touching the dirty bit, but it's useful and
// convenient here for keeping the HOT variables threadsafe.)
void
toku_ft_note_hot_begin(FT_HANDLE brt) {
FT ft = brt->ft;
toku_ft_note_hot_begin(FT_HANDLE ft_handle) {
FT ft = ft_handle->ft;
time_t now = time(NULL);
// hold lock around setting and clearing of dirty bit
@@ -576,11 +576,11 @@ toku_ft_note_hot_begin(FT_HANDLE brt) {
}
// Purpose: set fields in brt_header to capture accountability info for end of HOT optimize.
// Purpose: set fields in ft_header to capture accountability info for end of HOT optimize.
// Note: See note for toku_ft_note_hot_begin().
void
toku_ft_note_hot_complete(FT_HANDLE brt, bool success, MSN msn_at_start_of_hot) {
FT ft = brt->ft;
toku_ft_note_hot_complete(FT_HANDLE ft_handle, bool success, MSN msn_at_start_of_hot) {
FT ft = ft_handle->ft;
time_t now = time(NULL);
toku_ft_lock(ft);
@@ -626,7 +626,7 @@ toku_ft_init(FT ft,
ft->h->checkpoint_lsn = checkpoint_lsn;
}
// Open a brt for use by redirect. The new brt must have the same dict_id as the old_ft passed in. (FILENUM is assigned by the ft_handle_open() function.)
// Open an ft for use by redirect. The new ft must have the same dict_id as the old_ft passed in. (FILENUM is assigned by the ft_handle_open() function.)
static int
ft_handle_open_for_redirect(FT_HANDLE *new_ftp, const char *fname_in_env, TOKUTXN txn, FT old_h) {
FT_HANDLE t;
@@ -664,9 +664,9 @@ dictionary_redirect_internal(const char *dst_fname_in_env, FT src_h, TOKUTXN txn
FT dst_h = NULL;
struct toku_list *list;
// open a dummy brt based off of
// open a dummy ft based off of
// dst_fname_in_env to get the header
// then we will change all the brt's to have
// then we will change all the ft's to have
// their headers point to dst_h instead of src_h
FT_HANDLE tmp_dst_ft = NULL;
r = ft_handle_open_for_redirect(&tmp_dst_ft, dst_fname_in_env, txn, src_h);
@@ -680,7 +680,7 @@ dictionary_redirect_internal(const char *dst_fname_in_env, FT src_h, TOKUTXN txn
assert(dst_filenum.fileid!=FILENUM_NONE.fileid);
assert(dst_filenum.fileid!=src_filenum.fileid); //Cannot be same file.
// for each live brt, brt->ft is currently src_h
// for each live ft_handle, ft_handle->ft is currently src_h
// we want to change it to dummy_dst
toku_ft_grab_reflock(src_h);
while (!toku_list_empty(&src_h->live_ft_handles)) {
@@ -720,7 +720,7 @@ toku_dictionary_redirect_abort(FT old_h, FT new_h, TOKUTXN txn) {
FILENUM new_filenum = toku_cachefile_filenum(new_h->cf);
assert(old_filenum.fileid!=new_filenum.fileid); //Cannot be same file.
//No living brts in old header.
//No living fts in old header.
toku_ft_grab_reflock(old_h);
assert(toku_list_empty(&old_h->live_ft_handles));
toku_ft_release_reflock(old_h);
@@ -738,13 +738,13 @@ toku_dictionary_redirect_abort(FT old_h, FT new_h, TOKUTXN txn) {
/****
* on redirect or abort:
* if redirect txn_note_doing_work(txn)
* if redirect connect src brt to txn (txn modified this brt)
* for each src brt
* open brt to dst file (create new brt struct)
* if redirect connect dst brt to txn
* redirect db to new brt
* redirect cursors to new brt
* close all src brts
* if redirect connect src ft to txn (txn modified this ft)
* for each src ft
* open ft to dst file (create new ft struct)
* if redirect connect dst ft to txn
* redirect db to new ft
* redirect cursors to new ft
* close all src fts
* if redirect make rollback log entry
*
* on commit:
@@ -756,21 +756,21 @@ int
toku_dictionary_redirect (const char *dst_fname_in_env, FT_HANDLE old_ft_h, TOKUTXN txn) {
// Input args:
// new file name for dictionary (relative to env)
// old_ft_h is a live brt of open handle ({DB, BRT} pair) that currently refers to old dictionary file.
// old_ft_h is a live ft of open handle ({DB, FT_HANDLE} pair) that currently refers to old dictionary file.
// (old_ft_h may be one of many handles to the dictionary.)
// txn that created the loader
// Requires:
// multi operation lock is held.
// The brt is open. (which implies there can be no zombies.)
// The ft is open. (which implies there can be no zombies.)
// The new file must be a valid dictionary.
// The block size and flags in the new file must match the existing BRT.
// The block size and flags in the new file must match the existing FT.
// The new file must already have its descriptor in it (and it must match the existing descriptor).
// Effect:
// Open new FTs (and related header and cachefile) to the new dictionary file with a new FILENUM.
// Redirect all DBs that point to brts that point to the old file to point to brts that point to the new file.
// Redirect all DBs that point to fts that point to the old file to point to fts that point to the new file.
// Copy the dictionary id (dict_id) from the header of the original file to the header of the new file.
// Create a rollback log entry.
// The original BRT, header, cachefile and file remain unchanged. They will be cleaned up on commmit.
// The original FT, header, cachefile and file remain unchanged. They will be cleaned up on commmit.
// If the txn aborts, then this operation will be undone
int r;
@@ -1077,8 +1077,8 @@ garbage_helper(BLOCKNUM blocknum, int64_t UU(size), int64_t UU(address), void *e
goto exit;
}
for (int i = 0; i < node->n_children; ++i) {
BN_DATA bd = BLB_DATA(node, i);
r = bd->omt_iterate<struct garbage_helper_extra, garbage_leafentry_helper>(info);
bn_data* bd = BLB_DATA(node, i);
r = bd->iterate<struct garbage_helper_extra, garbage_leafentry_helper>(info);
if (r != 0) {
goto exit;
}

6
storage/tokudb/ft-index/ft/ft.h
View File

@@ -113,7 +113,7 @@ void toku_ft_release_reflock(FT ft);
void toku_ft_create(FT *ftp, FT_OPTIONS options, CACHEFILE cf, TOKUTXN txn);
void toku_ft_free (FT h);
int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_acceptable_lsn, FT *header);
int toku_read_ft_and_store_in_cachefile (FT_HANDLE ft_h, CACHEFILE cf, LSN max_acceptable_lsn, FT *header);
void toku_ft_note_ft_handle_open(FT ft, FT_HANDLE live);
bool toku_ft_needed_unlocked(FT ft);
@@ -125,8 +125,8 @@ void toku_ft_evict_from_memory(FT ft, bool oplsn_valid, LSN oplsn);
FT_HANDLE toku_ft_get_only_existing_ft_handle(FT h);
void toku_ft_note_hot_begin(FT_HANDLE brt);
void toku_ft_note_hot_complete(FT_HANDLE brt, bool success, MSN msn_at_start_of_hot);
void toku_ft_note_hot_begin(FT_HANDLE ft_h);
void toku_ft_note_hot_complete(FT_HANDLE ft_h, bool success, MSN msn_at_start_of_hot);
void
toku_ft_init(

7
storage/tokudb/ft-index/ft/ft_layout_version.h
View File

@@ -99,7 +99,7 @@ enum ft_layout_version_e {
FT_LAYOUT_VERSION_7 = 7, // Diff from 6 to 7: Add exact-bit to leafentry_estimate #818, add magic to header #22, add per-subdatase flags #333
FT_LAYOUT_VERSION_8 = 8, // Diff from 7 to 8: Use murmur instead of crc32. We are going to make a simplification and stop supporting version 7 and before. Current As of Beta 1.0.6
FT_LAYOUT_VERSION_9 = 9, // Diff from 8 to 9: Variable-sized blocks and compression.
FT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates, translation table, dictionary descriptors, checksum in header, subdb support removed from brt layer
FT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates, translation table, dictionary descriptors, checksum in header, subdb support removed from ft layer
FT_LAYOUT_VERSION_11 = 11, // Diff from 10 to 11: Nested transaction leafentries (completely redesigned). FT_CMDs on disk now support XIDS (multiple txnids) instead of exactly one.
FT_LAYOUT_VERSION_12 = 12, // Diff from 11 to 12: Added FT_CMD 'FT_INSERT_NO_OVERWRITE', compressed block format, num old blocks
FT_LAYOUT_VERSION_13 = 13, // Diff from 12 to 13: Fixed loader pivot bug, added build_id to every node, timestamps to ft
@@ -107,8 +107,8 @@ enum ft_layout_version_e {
FT_LAYOUT_VERSION_15 = 15, // Diff from 14 to 15: basement nodes, last verification time
FT_LAYOUT_VERSION_16 = 16, // Dr. No: No subtree estimates, partition layout information represented more transparently.
// ALERT ALERT ALERT: version 16 never released to customers, internal and beta use only
FT_LAYOUT_VERSION_17 = 17, // Dr. No: Add STAT64INFO_S to brt_header
FT_LAYOUT_VERSION_18 = 18, // Dr. No: Add HOT info to brt_header
FT_LAYOUT_VERSION_17 = 17, // Dr. No: Add STAT64INFO_S to ft header
FT_LAYOUT_VERSION_18 = 18, // Dr. No: Add HOT info to ft header
FT_LAYOUT_VERSION_19 = 19, // Doofenshmirtz: Add compression method, highest_unused_msn_for_upgrade
FT_LAYOUT_VERSION_20 = 20, // Deadshot: Add compression method to log_fcreate,
// mgr_last_xid after begin checkpoint,
@@ -119,6 +119,7 @@ enum ft_layout_version_e {
FT_LAYOUT_VERSION_23 = 23, // Ming: Fix upgrade path #5902
FT_LAYOUT_VERSION_24 = 24, // Riddler: change logentries that log transactions to store TXNID_PAIRs instead of TXNIDs
FT_LAYOUT_VERSION_25 = 25, // SecretSquirrel: ROLLBACK_LOG_NODES (on disk and in memory) now just use blocknum (instead of blocknum + hash) to point to other log nodes. same for xstillopen log entry
FT_LAYOUT_VERSION_26 = 26, // Hojo: basements store key/vals separately on disk for fixed klpair length BNs
FT_NEXT_VERSION, // the version after the current version
FT_LAYOUT_VERSION = FT_NEXT_VERSION-1, // A hack so I don't have to change this line.
FT_LAYOUT_MIN_SUPPORTED_VERSION = FT_LAYOUT_VERSION_13, // Minimum version supported

139
storage/tokudb/ft-index/ft/ft_node-serialize.cc
View File

@@ -101,7 +101,7 @@ PATENT RIGHTS GRANT:
static FT_UPGRADE_STATUS_S ft_upgrade_status;
#define STATUS_INIT(k,c,t,l,inc) TOKUDB_STATUS_INIT(ft_upgrade_status, k, c, t, "brt upgrade: " l, inc)
#define STATUS_INIT(k,c,t,l,inc) TOKUDB_STATUS_INIT(ft_upgrade_status, k, c, t, "ft upgrade: " l, inc)
static void
status_init(void)
@@ -279,36 +279,11 @@ serialize_node_header(FTNODE node, FTNODE_DISK_DATA ndd, struct wbuf *wbuf) {
wbuf_nocrc_int(wbuf, BP_SIZE (ndd, i)); // and the size
}
// checksum the header
uint32_t end_to_end_checksum = x1764_memory(wbuf->buf, wbuf_get_woffset(wbuf));
uint32_t end_to_end_checksum = toku_x1764_memory(wbuf->buf, wbuf_get_woffset(wbuf));
wbuf_nocrc_int(wbuf, end_to_end_checksum);
invariant(wbuf->ndone == wbuf->size);
}
static int
wbufwriteleafentry(const void* key, const uint32_t keylen, const LEAFENTRY &le, const uint32_t UU(idx), struct wbuf * const wb) {
// need to pack the leafentry as it was in versions
// where the key was integrated into it
uint32_t begin_spot UU() = wb->ndone;
uint32_t le_disk_size = leafentry_disksize(le);
wbuf_nocrc_uint8_t(wb, le->type);
wbuf_nocrc_uint32_t(wb, keylen);
if (le->type == LE_CLEAN) {
wbuf_nocrc_uint32_t(wb, le->u.clean.vallen);
wbuf_nocrc_literal_bytes(wb, key, keylen);
wbuf_nocrc_literal_bytes(wb, le->u.clean.val, le->u.clean.vallen);
}
else {
paranoid_invariant(le->type == LE_MVCC);
wbuf_nocrc_uint32_t(wb, le->u.mvcc.num_cxrs);
wbuf_nocrc_uint8_t(wb, le->u.mvcc.num_pxrs);
wbuf_nocrc_literal_bytes(wb, key, keylen);
wbuf_nocrc_literal_bytes(wb, le->u.mvcc.xrs, le_disk_size - (1 + 4 + 1));
}
uint32_t end_spot UU() = wb->ndone;
paranoid_invariant((end_spot - begin_spot) == keylen + sizeof(keylen) + le_disk_size);
return 0;
}
static uint32_t
serialize_ftnode_partition_size (FTNODE node, int i)
{
@@ -320,14 +295,14 @@ serialize_ftnode_partition_size (FTNODE node, int i)
result += toku_bnc_nbytesinbuf(BNC(node, i));
}
else {
result += 4; // n_entries in buffer table
result += 4 + bn_data::HEADER_LENGTH; // n_entries in buffer table + basement header
result += BLB_NBYTESINDATA(node, i);
}
result += 4; // checksum
return result;
}
#define FTNODE_PARTITION_OMT_LEAVES 0xaa
#define FTNODE_PARTITION_DMT_LEAVES 0xaa
#define FTNODE_PARTITION_FIFO_MSG 0xbb
static void
@@ -340,7 +315,7 @@ serialize_nonleaf_childinfo(NONLEAF_CHILDINFO bnc, struct wbuf *wb)
FIFO_ITERATE(
bnc->buffer, key, keylen, data, datalen, type, msn, xids, is_fresh,
{
paranoid_invariant((int)type>=0 && type<256);
paranoid_invariant((int) type >= 0 && (int) type < 256);
wbuf_nocrc_char(wb, (unsigned char) type);
wbuf_nocrc_char(wb, (unsigned char) is_fresh);
wbuf_MSN(wb, msn);
@@ -374,18 +349,15 @@ serialize_ftnode_partition(FTNODE node, int i, struct sub_block *sb) {
serialize_nonleaf_childinfo(BNC(node, i), &wb);
}
else {
unsigned char ch = FTNODE_PARTITION_OMT_LEAVES;
BN_DATA bd = BLB_DATA(node, i);
unsigned char ch = FTNODE_PARTITION_DMT_LEAVES;
bn_data* bd = BLB_DATA(node, i);
wbuf_nocrc_char(&wb, ch);
wbuf_nocrc_uint(&wb, bd->omt_size());
wbuf_nocrc_uint(&wb, bd->num_klpairs());
//
// iterate over leafentries and place them into the buffer
//
bd->omt_iterate<struct wbuf, wbufwriteleafentry>(&wb);
bd->serialize_to_wbuf(&wb);
}
uint32_t end_to_end_checksum = x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
uint32_t end_to_end_checksum = toku_x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
wbuf_nocrc_int(&wb, end_to_end_checksum);
invariant(wb.ndone == wb.size);
invariant(sb->uncompressed_size==wb.ndone);
@@ -429,7 +401,7 @@ compress_ftnode_sub_block(struct sub_block *sb, enum toku_compression_method met
extra[1] = toku_htod32(sb->uncompressed_size);
// now checksum the entire thing
sb->compressed_size += 8; // now add the eight bytes that we saved for the sizes
sb->xsum = x1764_memory(sb->compressed_ptr,sb->compressed_size);
sb->xsum = toku_x1764_memory(sb->compressed_ptr,sb->compressed_size);
//
// This is the end result for Dr. No and forward. For ftnodes, sb->compressed_ptr contains
@@ -493,7 +465,7 @@ static void serialize_ftnode_info(FTNODE node,
}
}
uint32_t end_to_end_checksum = x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
uint32_t end_to_end_checksum = toku_x1764_memory(sb->uncompressed_ptr, wbuf_get_woffset(&wb));
wbuf_nocrc_int(&wb, end_to_end_checksum);
invariant(wb.ndone == wb.size);
invariant(sb->uncompressed_size==wb.ndone);
@@ -546,7 +518,7 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
// Count number of leaf entries in this leaf (num_le).
uint32_t num_le = 0;
for (uint32_t i = 0; i < num_orig_basements; i++) {
num_le += BLB_DATA(node, i)->omt_size();
num_le += BLB_DATA(node, i)->num_klpairs();
}
uint32_t num_alloc = num_le ? num_le : 1; // simplify logic below by always having at least one entry per array
@@ -571,10 +543,10 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
uint32_t curr_le = 0;
for (uint32_t i = 0; i < num_orig_basements; i++) {
BN_DATA bd = BLB_DATA(node, i);
bn_data* bd = BLB_DATA(node, i);
struct array_info ai {.offset = curr_le, .le_array = leafpointers, .key_sizes_array = key_sizes, .key_ptr_array = key_pointers };
bd->omt_iterate<array_info, array_item>(&ai);
curr_le += bd->omt_size();
bd->iterate<array_info, array_item>(&ai);
curr_le += bd->num_klpairs();
}
// Create an array that will store indexes of new pivots.
@@ -592,9 +564,14 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
// Create an array that will store the size of each basement.
// This is the sum of the leaf sizes of all the leaves in that basement.
// We don't know how many basements there will be, so we use num_le as the upper bound.
toku::scoped_malloc bn_sizes_buf(sizeof(size_t) * num_alloc);
size_t *bn_sizes = reinterpret_cast<size_t *>(bn_sizes_buf.get());
bn_sizes[0] = 0;
// Sum of all le sizes in a single basement
toku::scoped_calloc bn_le_sizes_buf(sizeof(size_t) * num_alloc);
size_t *bn_le_sizes = reinterpret_cast<size_t *>(bn_le_sizes_buf.get());
// Sum of all key sizes in a single basement
toku::scoped_calloc bn_key_sizes_buf(sizeof(size_t) * num_alloc);
size_t *bn_key_sizes = reinterpret_cast<size_t *>(bn_key_sizes_buf.get());
// TODO 4050: All these arrays should be combined into a single array of some bn_info struct (pivot, msize, num_les).
// Each entry is the number of leafentries in this basement. (Again, num_le is overkill upper baound.)
@@ -611,7 +588,7 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
for (uint32_t i = 0; i < num_le; i++) {
uint32_t curr_le_size = leafentry_disksize((LEAFENTRY) leafpointers[i]);
le_sizes[i] = curr_le_size;
if ((bn_size_so_far + curr_le_size > basementnodesize) && (num_le_in_curr_bn != 0)) {
if ((bn_size_so_far + curr_le_size + sizeof(uint32_t) + key_sizes[i] > basementnodesize) && (num_le_in_curr_bn != 0)) {
// cap off the current basement node to end with the element before i
new_pivots[curr_pivot] = i-1;
curr_pivot++;
@@ -620,8 +597,9 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
}
num_le_in_curr_bn++;
num_les_this_bn[curr_pivot] = num_le_in_curr_bn;
bn_le_sizes[curr_pivot] += curr_le_size;
bn_key_sizes[curr_pivot] += sizeof(uint32_t) + key_sizes[i]; // uint32_t le_offset
bn_size_so_far += curr_le_size + sizeof(uint32_t) + key_sizes[i];
bn_sizes[curr_pivot] = bn_size_so_far;
}
// curr_pivot is now the total number of pivot keys in the leaf node
int num_pivots = curr_pivot;
@@ -688,17 +666,15 @@ rebalance_ftnode_leaf(FTNODE node, unsigned int basementnodesize)
uint32_t num_les_to_copy = num_les_this_bn[i];
invariant(num_les_to_copy == num_in_bn);
// construct mempool for this basement
size_t size_this_bn = bn_sizes[i];
BN_DATA bd = BLB_DATA(node, i);
bd->replace_contents_with_clone_of_sorted_array(
bn_data* bd = BLB_DATA(node, i);
bd->set_contents_as_clone_of_sorted_array(
num_les_to_copy,
&key_pointers[baseindex_this_bn],
&key_sizes[baseindex_this_bn],
&leafpointers[baseindex_this_bn],
&le_sizes[baseindex_this_bn],
size_this_bn
bn_key_sizes[i], // Total key sizes
bn_le_sizes[i] // total le sizes
);
BP_STATE(node,i) = PT_AVAIL;
@@ -1272,7 +1248,7 @@ read_compressed_sub_block(struct rbuf *rb, struct sub_block *sb)
rbuf_literal_bytes(rb, cp, sb->compressed_size);
sb->xsum = rbuf_int(rb);
// let's check the checksum
uint32_t actual_xsum = x1764_memory((char *)sb->compressed_ptr-8, 8+sb->compressed_size);
uint32_t actual_xsum = toku_x1764_memory((char *)sb->compressed_ptr-8, 8+sb->compressed_size);
if (sb->xsum != actual_xsum) {
r = TOKUDB_BAD_CHECKSUM;
}
@@ -1317,7 +1293,7 @@ verify_ftnode_sub_block (struct sub_block *sb)
// first verify the checksum
uint32_t data_size = sb->uncompressed_size - 4; // checksum is 4 bytes at end
uint32_t stored_xsum = toku_dtoh32(*((uint32_t *)((char *)sb->uncompressed_ptr + data_size)));
uint32_t actual_xsum = x1764_memory(sb->uncompressed_ptr, data_size);
uint32_t actual_xsum = toku_x1764_memory(sb->uncompressed_ptr, data_size);
if (stored_xsum != actual_xsum) {
dump_bad_block((Bytef *) sb->uncompressed_ptr, sb->uncompressed_size);
r = TOKUDB_BAD_CHECKSUM;
@@ -1412,7 +1388,7 @@ setup_available_ftnode_partition(FTNODE node, int i) {
}
}
// Assign the child_to_read member of the bfe from the given brt node
// Assign the child_to_read member of the bfe from the given ftnode
// that has been brought into memory.
static void
update_bfe_using_ftnode(FTNODE node, struct ftnode_fetch_extra *bfe)
@@ -1447,7 +1423,7 @@ update_bfe_using_ftnode(FTNODE node, struct ftnode_fetch_extra *bfe)
}
// Using the search parameters in the bfe, this function will
// initialize all of the given brt node's partitions.
// initialize all of the given ftnode's partitions.
static void
setup_partitions_using_bfe(FTNODE node,
struct ftnode_fetch_extra *bfe,
@@ -1541,15 +1517,14 @@ deserialize_ftnode_partition(
BP_WORKDONE(node, childnum) = 0;
}
else {
assert(ch == FTNODE_PARTITION_OMT_LEAVES);
assert(ch == FTNODE_PARTITION_DMT_LEAVES);
BLB_SEQINSERT(node, childnum) = 0;
uint32_t num_entries = rbuf_int(&rb);
// we are now at the first byte of first leafentry
data_size -= rb.ndone; // remaining bytes of leafentry data
BASEMENTNODE bn = BLB(node, childnum);
bn->data_buffer.initialize_from_data(num_entries, &rb.buf[rb.ndone], data_size);
rb.ndone += data_size;
bn->data_buffer.deserialize_from_rbuf(num_entries, &rb, data_size, node->layout_version_read_from_disk);
}
assert(rb.ndone == rb.size);
exit:
@@ -1681,7 +1656,7 @@ deserialize_ftnode_header_from_rbuf_if_small_enough (FTNODE *ftnode,
}
uint32_t checksum;
checksum = x1764_memory(rb->buf, rb->ndone);
checksum = toku_x1764_memory(rb->buf, rb->ndone);
uint32_t stored_checksum;
stored_checksum = rbuf_int(rb);
if (stored_checksum != checksum) {
@@ -1707,7 +1682,7 @@ deserialize_ftnode_header_from_rbuf_if_small_enough (FTNODE *ftnode,
sb_node_info.xsum = rbuf_int(rb);
// let's check the checksum
uint32_t actual_xsum;
actual_xsum = x1764_memory((char *)sb_node_info.compressed_ptr-8, 8+sb_node_info.compressed_size);
actual_xsum = toku_x1764_memory((char *)sb_node_info.compressed_ptr-8, 8+sb_node_info.compressed_size);
if (sb_node_info.xsum != actual_xsum) {
r = TOKUDB_BAD_CHECKSUM;
goto cleanup;
@@ -1792,7 +1767,7 @@ cleanup:
// also creates MSN's for older messages created in older versions
// that did not generate MSN's for messages. These new MSN's are
// generated from the root downwards, counting backwards from MIN_MSN
// and persisted in the brt header.
// and persisted in the ft header.
static int
deserialize_and_upgrade_internal_node(FTNODE node,
struct rbuf *rb,
@@ -1978,7 +1953,7 @@ deserialize_and_upgrade_internal_node(FTNODE node,
// still have the pointer to the buffer).
if (version >= FT_FIRST_LAYOUT_VERSION_WITH_END_TO_END_CHECKSUM) {
uint32_t expected_xsum = toku_dtoh32(*(uint32_t*)(rb->buf+rb->size-4)); // 27. checksum
uint32_t actual_xsum = x1764_memory(rb->buf, rb->size-4);
uint32_t actual_xsum = toku_x1764_memory(rb->buf, rb->size-4);
if (expected_xsum != actual_xsum) {
fprintf(stderr, "%s:%d: Bad checksum: expected = %" PRIx32 ", actual= %" PRIx32 "\n",
__FUNCTION__,
@@ -2050,7 +2025,7 @@ deserialize_and_upgrade_leaf_node(FTNODE node,
setup_partitions_using_bfe(node, &temp_bfe, true);
// 11. Deserialize the partition maps, though they are not used in the
// newer versions of brt nodes.
// newer versions of ftnodes.
struct sub_block_map part_map[npartitions];
for (int i = 0; i < npartitions; ++i) {
sub_block_map_deserialize(&part_map[i], rb);
@@ -2086,13 +2061,18 @@ deserialize_and_upgrade_leaf_node(FTNODE node,
assert_zero(r);
// Copy the pointer value straight into the OMT
LEAFENTRY new_le_in_bn = nullptr;
void *maybe_free;
bn->data_buffer.get_space_for_insert(
i,
key,
keylen,
new_le_size,
&new_le_in_bn
&new_le_in_bn,
&maybe_free
);
if (maybe_free) {
toku_free(maybe_free);
}
memcpy(new_le_in_bn, new_le, new_le_size);
toku_free(new_le);
}
@@ -2101,8 +2081,7 @@ deserialize_and_upgrade_leaf_node(FTNODE node,
if (has_end_to_end_checksum) {
data_size -= sizeof(uint32_t);
}
bn->data_buffer.initialize_from_data(n_in_buf, &rb->buf[rb->ndone], data_size);
rb->ndone += data_size;
bn->data_buffer.deserialize_from_rbuf(n_in_buf, rb, data_size, node->layout_version_read_from_disk);
}
// Whatever this is must be less than the MSNs of every message above
@@ -2114,7 +2093,7 @@ deserialize_and_upgrade_leaf_node(FTNODE node,
// Checksum (end to end) is only on version 14
if (has_end_to_end_checksum) {
uint32_t expected_xsum = rbuf_int(rb); // 17. checksum
uint32_t actual_xsum = x1764_memory(rb->buf, rb->size - 4);
uint32_t actual_xsum = toku_x1764_memory(rb->buf, rb->size - 4);
if (expected_xsum != actual_xsum) {
fprintf(stderr, "%s:%d: Bad checksum: expected = %" PRIx32 ", actual= %" PRIx32 "\n",
__FUNCTION__,
@@ -2313,7 +2292,7 @@ deserialize_ftnode_from_rbuf(
}
// verify checksum of header stored
uint32_t checksum;
checksum = x1764_memory(rb->buf, rb->ndone);
checksum = toku_x1764_memory(rb->buf, rb->ndone);
uint32_t stored_checksum;
stored_checksum = rbuf_int(rb);
if (stored_checksum != checksum) {
@@ -2420,7 +2399,7 @@ cleanup:
// NOTE: Right now, callers higher in the stack will assert on
// failure, so this is OK for production. However, if we
// create tools that use this function to search for errors in
// the BRT, then we will leak memory.
// the FT, then we will leak memory.
if (node) {
toku_free(node);
}
@@ -2579,7 +2558,7 @@ deserialize_ftnode_from_fd(int fd,
return r;
}
// Read brt node from file into struct. Perform version upgrade if necessary.
// Read ftnode from file into struct. Perform version upgrade if necessary.
int
toku_deserialize_ftnode_from (int fd,
BLOCKNUM blocknum,
@@ -2628,7 +2607,7 @@ serialize_rollback_log_size(ROLLBACK_LOG_NODE log) {
+8 //blocknum
+8 //previous (blocknum)
+8 //resident_bytecount
+8 //memarena_size_needed_to_load
+8 //memarena size
+log->rollentry_resident_bytecount;
return size;
}
@@ -2649,7 +2628,7 @@ serialize_rollback_log_node_to_buf(ROLLBACK_LOG_NODE log, char *buf, size_t calc
wbuf_nocrc_BLOCKNUM(&wb, log->previous);
wbuf_nocrc_ulonglong(&wb, log->rollentry_resident_bytecount);
//Write down memarena size needed to restore
wbuf_nocrc_ulonglong(&wb, memarena_total_size_in_use(log->rollentry_arena));
wbuf_nocrc_ulonglong(&wb, toku_memarena_total_size_in_use(log->rollentry_arena));
{
//Store rollback logs
@@ -2705,7 +2684,7 @@ serialize_uncompressed_block_to_memory(char * uncompressed_buf,
// compute the header checksum and serialize it
uint32_t header_length = (char *)ptr - (char *)compressed_buf;
uint32_t xsum = x1764_memory(compressed_buf, header_length);
uint32_t xsum = toku_x1764_memory(compressed_buf, header_length);
*ptr = toku_htod32(xsum);
uint32_t padded_len = roundup_to_multiple(512, header_len + compressed_len);
@@ -2813,8 +2792,8 @@ deserialize_rollback_log_from_rbuf (BLOCKNUM blocknum, ROLLBACK_LOG_NODE *log_p,
result->rollentry_resident_bytecount = rbuf_ulonglong(rb);
size_t arena_initial_size = rbuf_ulonglong(rb);
result->rollentry_arena = memarena_create_presized(arena_initial_size);
if (0) { died1: memarena_close(&result->rollentry_arena); goto died0; }
result->rollentry_arena = toku_memarena_create_presized(arena_initial_size);
if (0) { died1: toku_memarena_destroy(&result->rollentry_arena); goto died0; }
//Load rollback entries
lazy_assert(rb->size > 4);
@@ -2877,7 +2856,7 @@ decompress_from_raw_block_into_rbuf(uint8_t *raw_block, size_t raw_block_size, s
{ // verify the header checksum
uint32_t header_length = node_header_overhead + sub_block_header_size(n_sub_blocks);
invariant(header_length <= raw_block_size);
uint32_t xsum = x1764_memory(raw_block, header_length);
uint32_t xsum = toku_x1764_memory(raw_block, header_length);
uint32_t stored_xsum = toku_dtoh32(*(uint32_t *)(raw_block + header_length));
if (xsum != stored_xsum) {
r = TOKUDB_BAD_CHECKSUM;

8
storage/tokudb/ft-index/ft/ftloader-internal.h
View File

@@ -256,7 +256,7 @@ struct ft_loader_s {
int progress_callback_result; // initially zero, if any call to the poll function callback returns nonzero, we save the result here (and don't call the poll callback function again).
LSN load_lsn; //LSN of the fsynced 'load' log entry. Write this LSN (as checkpoint_lsn) in brt headers made by this loader.
LSN load_lsn; //LSN of the fsynced 'load' log entry. Write this LSN (as checkpoint_lsn) in ft headers made by this loader.
TXNID load_root_xid; //(Root) transaction that performed the load.
QUEUE *fractal_queues; // an array of work queues, one for each secondary index.
@@ -279,7 +279,7 @@ uint64_t toku_ft_loader_get_n_rows(FTLOADER bl);
struct fractal_thread_args {
FTLOADER bl;
const DESCRIPTOR descriptor;
int fd; // write the brt into tfd.
int fd; // write the ft into fd.
int progress_allocation;
QUEUE q;
uint64_t total_disksize_estimate;
@@ -311,7 +311,7 @@ int toku_merge_some_files_using_dbufio (const bool to_q, FIDX dest_data, QUEUE q
int ft_loader_sort_and_write_rows (struct rowset *rows, struct merge_fileset *fs, FTLOADER bl, int which_db, DB *dest_db, ft_compare_func);
// This is probably only for testing.
int toku_loader_write_brt_from_q_in_C (FTLOADER bl,
int toku_loader_write_ft_from_q_in_C (FTLOADER bl,
const DESCRIPTOR descriptor,
int fd, // write to here
int progress_allocation,
@@ -338,7 +338,7 @@ int toku_ft_loader_internal_init (/* out */ FTLOADER *blp,
CACHETABLE cachetable,
generate_row_for_put_func g,
DB *src_db,
int N, FT_HANDLE brts[/*N*/], DB* dbs[/*N*/],
int N, FT_HANDLE ft_hs[/*N*/], DB* dbs[/*N*/],
const char *new_fnames_in_env[/*N*/],
ft_compare_func bt_compare_functions[/*N*/],
const char *temp_file_template,

36
storage/tokudb/ft-index/ft/ftloader.cc
View File

@@ -91,9 +91,7 @@ PATENT RIGHTS GRANT:
#include <toku_portability.h>
#if !TOKU_WINDOWS
#include <arpa/inet.h>
#endif
#include <stdio.h>
#include <memory.h>
@@ -101,7 +99,9 @@ PATENT RIGHTS GRANT:
#include <toku_assert.h>
#include <string.h>
#include <fcntl.h>
#include "x1764.h"
#include <util/x1764.h>
#include "ftloader-internal.h"
#include "ft-internal.h"
#include "sub_block.h"
@@ -535,7 +535,7 @@ int toku_ft_loader_internal_init (/* out */ FTLOADER *blp,
CACHETABLE cachetable,
generate_row_for_put_func g,
DB *src_db,
int N, FT_HANDLE brts[/*N*/], DB* dbs[/*N*/],
int N, FT_HANDLE fts[/*N*/], DB* dbs[/*N*/],
const char *new_fnames_in_env[/*N*/],
ft_compare_func bt_compare_functions[/*N*/],
const char *temp_file_template,
@@ -581,11 +581,11 @@ int toku_ft_loader_internal_init (/* out */ FTLOADER *blp,
#define SET_TO_MY_STRDUP(lval, s) do { char *v = toku_strdup(s); if (!v) { int r = get_error_errno(); toku_ft_loader_internal_destroy(bl, true); return r; } lval = v; } while (0)
MY_CALLOC_N(N, bl->root_xids_that_created);
for (int i=0; i<N; i++) if (brts[i]) bl->root_xids_that_created[i]=brts[i]->ft->h->root_xid_that_created;
for (int i=0; i<N; i++) if (fts[i]) bl->root_xids_that_created[i]=fts[i]->ft->h->root_xid_that_created;
MY_CALLOC_N(N, bl->dbs);
for (int i=0; i<N; i++) if (brts[i]) bl->dbs[i]=dbs[i];
for (int i=0; i<N; i++) if (fts[i]) bl->dbs[i]=dbs[i];
MY_CALLOC_N(N, bl->descriptors);
for (int i=0; i<N; i++) if (brts[i]) bl->descriptors[i]=&brts[i]->ft->descriptor;
for (int i=0; i<N; i++) if (fts[i]) bl->descriptors[i]=&fts[i]->ft->descriptor;
MY_CALLOC_N(N, bl->new_fnames_in_env);
for (int i=0; i<N; i++) SET_TO_MY_STRDUP(bl->new_fnames_in_env[i], new_fnames_in_env[i]);
MY_CALLOC_N(N, bl->extracted_datasizes); // the calloc_n zeroed everything, which is what we want
@@ -642,7 +642,7 @@ int toku_ft_loader_open (/* out */ FTLOADER *blp,
CACHETABLE cachetable,
generate_row_for_put_func g,
DB *src_db,
int N, FT_HANDLE brts[/*N*/], DB* dbs[/*N*/],
int N, FT_HANDLE fts[/*N*/], DB* dbs[/*N*/],
const char *new_fnames_in_env[/*N*/],
ft_compare_func bt_compare_functions[/*N*/],
const char *temp_file_template,
@@ -651,9 +651,9 @@ int toku_ft_loader_open (/* out */ FTLOADER *blp,
bool reserve_memory,
uint64_t reserve_memory_size,
bool compress_intermediates)
/* Effect: called by DB_ENV->create_loader to create a brt loader.
/* Effect: called by DB_ENV->create_loader to create an ft loader.
* Arguments:
* blp Return the brt loader here.
* blp Return the ft loader here.
* g The function for generating a row
* src_db The source database. Needed by g. May be NULL if that's ok with g.
* N The number of dbs to create.
@@ -666,7 +666,7 @@ int toku_ft_loader_open (/* out */ FTLOADER *blp,
int result = 0;
{
int r = toku_ft_loader_internal_init(blp, cachetable, g, src_db,
N, brts, dbs,
N, fts, dbs,
new_fnames_in_env,
bt_compare_functions,
temp_file_template,
@@ -1370,7 +1370,7 @@ static int process_primary_rows (FTLOADER bl, struct rowset *primary_rowset) {
}
int toku_ft_loader_put (FTLOADER bl, DBT *key, DBT *val)
/* Effect: Put a key-value pair into the brt loader. Called by DB_LOADER->put().
/* Effect: Put a key-value pair into the ft loader. Called by DB_LOADER->put().
* Return value: 0 on success, an error number otherwise.
*/
{
@@ -2621,7 +2621,7 @@ static int toku_loader_write_ft_from_q (FTLOADER bl,
char *XMALLOC_N(desc_size, buf);
wbuf_init(&wbuf, buf, desc_size);
toku_serialize_descriptor_contents_to_wbuf(&wbuf, descriptor);
uint32_t checksum = x1764_finish(&wbuf.checksum);
uint32_t checksum = toku_x1764_finish(&wbuf.checksum);
wbuf_int(&wbuf, checksum);
invariant(wbuf.ndone==desc_size);
r = toku_os_write(out.fd, wbuf.buf, wbuf.ndone);
@@ -2672,7 +2672,7 @@ static int toku_loader_write_ft_from_q (FTLOADER bl,
return result;
}
int toku_loader_write_brt_from_q_in_C (FTLOADER bl,
int toku_loader_write_ft_from_q_in_C (FTLOADER bl,
const DESCRIPTOR descriptor,
int fd, // write to here
int progress_allocation,
@@ -2917,17 +2917,17 @@ static void add_pair_to_leafnode (struct leaf_buf *lbuf, unsigned char *key, int
// #3588 TODO just make a clean ule and append it to the omt
// #3588 TODO can do the rebalancing here and avoid a lot of work later
FTNODE leafnode = lbuf->node;
uint32_t idx = BLB_DATA(leafnode, 0)->omt_size();
uint32_t idx = BLB_DATA(leafnode, 0)->num_klpairs();
DBT thekey = { .data = key, .size = (uint32_t) keylen };
DBT theval = { .data = val, .size = (uint32_t) vallen };
FT_MSG_S cmd = { .type = FT_INSERT,
FT_MSG_S msg = { .type = FT_INSERT,
.msn = ZERO_MSN,
.xids = lbuf->xids,
.u = { .id = { &thekey, &theval } } };
uint64_t workdone=0;
// there's no mvcc garbage in a bulk-loaded FT, so there's no need to pass useful gc info
txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true);
toku_ft_bn_apply_cmd_once(BLB(leafnode,0), &cmd, idx, NULL, &gc_info, &workdone, stats_to_update);
toku_ft_bn_apply_msg_once(BLB(leafnode,0), &msg, idx, NULL, &gc_info, &workdone, stats_to_update);
}
static int write_literal(struct dbout *out, void*data, size_t len) {
@@ -2992,7 +2992,7 @@ static int write_translation_table (struct dbout *out, long long *off_of_transla
putbuf_int64(&ttable, out->translation[i].off);
putbuf_int64(&ttable, out->translation[i].size);
}
unsigned int checksum = x1764_memory(ttable.buf, ttable.off);
unsigned int checksum = toku_x1764_memory(ttable.buf, ttable.off);
putbuf_int32(&ttable, checksum);
// pad it to 512 zeros
long long encoded_length = ttable.off;

2
storage/tokudb/ft-index/ft/ftloader.h
View File

@@ -105,7 +105,7 @@ int toku_ft_loader_open (FTLOADER *bl,
generate_row_for_put_func g,
DB *src_db,
int N,
FT_HANDLE brts[/*N*/], DB* dbs[/*N*/],
FT_HANDLE ft_hs[/*N*/], DB* dbs[/*N*/],
const char * new_fnames_in_env[/*N*/],
ft_compare_func bt_compare_functions[/*N*/],
const char *temp_file_template,

10
storage/tokudb/ft-index/ft/fttypes.h
View File

@@ -198,7 +198,7 @@ typedef struct {
static const STAT64INFO_S ZEROSTATS = {0,0};
/* At the brt layer, a FILENUM uniquely identifies an open file.
/* At the ft layer, a FILENUM uniquely identifies an open file.
* At the ydb layer, a DICTIONARY_ID uniquely identifies an open dictionary.
* With the introduction of the loader (ticket 2216), it is possible for the file that holds
* an open dictionary to change, so these are now separate and independent unique identifiers.
@@ -230,7 +230,7 @@ typedef struct cachetable *CACHETABLE;
typedef struct cachefile *CACHEFILE;
typedef struct ctpair *PAIR;
typedef class checkpointer *CHECKPOINTER;
typedef class bn_data *BN_DATA;
class bn_data;
/* tree command types */
enum ft_msg_type {
@@ -323,14 +323,14 @@ struct ft_msg {
XIDS xids;
union {
/* insert or delete */
struct ft_cmd_insert_delete {
struct ft_msg_insert_delete {
const DBT *key; // for insert, delete, upsertdel
const DBT *val; // for insert, delete, (and it is the "extra" for upsertdel, upsertdel_broadcast_all)
} id;
} u;
};
// Message sent into brt to implement command (insert, delete, etc.)
// This structure supports nested transactions, and obsoletes ft_msg.
// Message sent into the ft to implement insert, delete, update, etc
typedef struct ft_msg FT_MSG_S;
typedef struct ft_msg *FT_MSG;

1
storage/tokudb/ft-index/ft/ftverify.cc
View File

@@ -98,7 +98,6 @@ PATENT RIGHTS GRANT:
#include "ft-internal.h"
#include "ft_layout_version.h"
#include "block_table.h"
#include "x1764.h"
#include "rbuf.h"
#include "sub_block.h"

4
storage/tokudb/ft-index/ft/hash-benchmarks/Makefile
View File

@@ -1,4 +0,0 @@
LDFLAGS = -lz
CFLAGS = -O4 -Wall -W
hash-benchmark:

378
storage/tokudb/ft-index/ft/hash-benchmarks/hash-benchmark-manually-open.cc
View File

@@ -1,378 +0,0 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* Benchmark various hash functions. */
#include <sys/time.h>
#include <zlib.h>
#include <stdio.h>
#include <stdlib.h>
#include <toku_assert.h>
#define N 200000000
char *buf;
static double tdiff (struct timeval *a, struct timeval *b) {
return a->tv_sec - b->tv_sec + (1e-6)*(a->tv_usec - b->tv_usec);
}
#define measure_bandwidth(str, body) ({ \
int c; \
struct timeval start,end; \
gettimeofday(&start, 0); \
body; \
gettimeofday(&end, 0); \
double diff = tdiff(&end, &start); \
printf("%s=%08x %d bytes in %8.6fs for %8.3fMB/s\n", str, c, N, diff, N*(1e-6)/diff); \
})
int sum32 (int start, void *buf, int bytecount) {
int *ibuf = buf;
assert(bytecount%4==0);
while (bytecount>0) {
start+=*ibuf;
ibuf++;
bytecount-=4;
}
return start;
}
static const uint32_t m = 0x5bd1e995;
static const int r = 24;
static const uint32_t seed = 0x3dd3b51a;
#define USE_ZERO_CHECKSUM 0
static uint32_t MurmurHash2 ( const void * key, int len)
{
if (USE_ZERO_CHECKSUM) return 0;
// 'm' and 'r' are mixing constants generated offline.
// They're not really 'magic', they just happen to work well.
// Initialize the hash to a 'random' value
uint32_t h = seed;
// Mix 4 bytes at a time into the hash
const unsigned char * data = (const unsigned char *)key;
while(len >= 4)
{
uint32_t k = *(uint32_t *)data;
k *= m;
k ^= k >> r;
k *= m;
h *= m;
h ^= k;
data += 4;
len -= 4;
}
// Handle the last few bytes of the input array
switch(len)
{
case 3: h ^= data[2] << 16;
case 2: h ^= data[1] << 8;
case 1: h ^= data[0];
h *= m;
};
// Do a few final mixes of the hash to ensure the last few
// bytes are well-incorporated.
h ^= h >> 29;
h *= m;
h ^= h >> 31;
return h;
}
struct murmur {
int n_bytes_in_k; // How many bytes in k
uint32_t k; // These are the extra bytes. Bytes are shifted into the low-order bits.
uint32_t h; // The hash so far (up to the most recent 4-byte boundary)
};
void murmur_init (struct murmur *mm) {
mm->n_bytes_in_k=0;
mm->k =0;
mm->h = seed;
}
#define MIX() ({ k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; })
#define LD1() data[0]
#define LD2() ((data[0]<<8) | data[1])
#define LD3() ((data[0]<<16) | (data[1]<<8) | data[2])
#define ADD1_0() (mm->k = LD1())
#define ADD1() (mm->k = (k<<8) | LD1())
#define ADD2_0() (mm->k = LD2())
#define ADD2() (mm->k = (k<<16) | LD2())
#define ADD3_0() (mm->k = LD3())
#define ADD3() (mm->k = (k<<24) | LD3())
void murmur_add (struct murmur *mm, const void * key, unsigned int len) {
if (USE_ZERO_CHECKSUM) return;
if (len==0) return;
const int n_bytes_in_k = mm->n_bytes_in_k;
uint32_t k = mm->k;
const unsigned char *data = key;
uint32_t h = mm->h;
switch (n_bytes_in_k) {
case 0:
switch (len) {
case 1: ADD1_0(); mm->n_bytes_in_k = 1; mm->h=h; return;
case 2: ADD2_0(); mm->n_bytes_in_k = 2; mm->h=h; return;
case 3: ADD3_0(); mm->n_bytes_in_k = 3; mm->h=h; return;
default: break;
}
break;
case 1:
switch (len) {
case 1: ADD1(); mm->n_bytes_in_k = 2; mm->h=h; return;
case 2: ADD2(); mm->n_bytes_in_k = 3; mm->h=h; return;
case 3: ADD3(); mm->n_bytes_in_k = 0; MIX(); mm->h=h; return;
default: ADD3(); mm->n_bytes_in_k = 0; MIX(); len-=3; data+=3; break;
}
break;
case 2:
switch (len) {
case 1: ADD1(); mm->n_bytes_in_k = 3; mm->h=h; return;
case 2: ADD2(); mm->n_bytes_in_k = 0; MIX(); mm->h=h; return;
default: ADD2(); mm->n_bytes_in_k = 0; MIX(); len-=2; data+=2; break;
}
break;
case 3:
switch (len) {
case 1: ADD1(); mm->n_bytes_in_k = 0; MIX(); mm->h=h; return;
default: ADD1(); mm->n_bytes_in_k = 0; MIX(); len--; data++; break;
}
break;
default: assert(0);
}
// We've used up the partial bytes at the beginning of k.
assert(mm->n_bytes_in_k==0);
while (len >= 4) {
uint32_t k = toku_dtoh32(*(uint32_t *)data);
//printf(" oldh=%08x k=%08x", h, k);
k *= m;
k ^= k >> r;
k *= m;
h *= m;
h ^= k;
data += 4;
len -= 4;
//printf(" h=%08x\n", h);
}
mm->h=h;
//printf("%s:%d h=%08x\n", __FILE__, __LINE__, h);
{
uint32_t k=0;
switch (len) {
case 3: k = *data << 16; data++;
case 2: k |= *data << 8; data++;
case 1: k |= *data;
}
mm->k = k;
mm->n_bytes_in_k = len;
//printf("now extra=%08x (%d bytes) n_bytes=%d\n", mm->k, len, mm->n_bytes_in_k);
}
}
uint32_t murmur_finish (struct murmur *mm) {
if (USE_ZERO_CHECKSUM) return 0;
uint32_t h = mm->h;
if (mm->n_bytes_in_k>0) {
h ^= mm->k;
h *= m;
}
if (0) {
// The real murmur function does this extra mixing at the end. We don't need that for fingerprint.
h ^= h >> 29;
h *= m;
h ^= h >> 31;
}
return h;
}
struct sum84 {
uint32_t sum;
int i;
};
void sum84_init (struct sum84 *s) { s->sum=0; s->i=0; };
void sum84_add (struct sum84 *s, char *buf, int count) {
while (s->i%4!=0 && count>0) {
char v = *buf;
s->sum ^= v << (s->i%4)*8;
buf++; count--; s->i++;
}
while (count>4) {
s->sum ^= *(int*)buf;
buf+=4; count-=4;
}
while (count>0) {
char v = *buf;
s->sum ^= v << (s->i%4)*8;
buf++; count--; s->i++;
}
}
int sum84_finish (struct sum84 *s) {
return s->sum;
}
uint32_t xor8_add (uint32_t x, char *buf, int count) {
while (count>4) {
x ^= *(int*)buf;
buf+=4; count-=4;
}
while (count>0) {
char v = *buf;
x ^= v;
buf++; count--;
}
return x;
}
uint32_t xor8_finish (uint32_t x) {
return (x ^ (x>>8) ^ (x>>16) ^ (x>>24))&0xff;
}
uint64_t xor8_64_add (uint64_t x, char *buf, int count) {
while (count>8) {
x ^= *(uint64_t*)buf;
buf+=8; count-=8;
}
while (count>0) {
char v = *buf;
x ^= v;
buf++; count--;
}
return x;
}
uint32_t xor8_64_finish (uint64_t x) {
return (x ^ (x>>8) ^ (x>>16) ^ (x>>24) ^ (x>>32) ^ (x>>40) ^ (x>>48) ^ (x>>56))&0xff;
}
static void measure_bandwidths (void) {
measure_bandwidth("crc32 ", c=crc32(0, buf, N));
measure_bandwidth("sum32 ", c=sum32(0, buf, N));
measure_bandwidth("murmur ", c=MurmurHash2(buf, N));
measure_bandwidth("murmurf ", ({ struct murmur mm; murmur_init(&mm); murmur_add(&mm, buf, N); c=murmur_finish(&mm); }));
measure_bandwidth("sum84 ", ({ struct sum84 s; sum84_init(&s); sum84_add(&s, buf, N); c=sum84_finish(&s); }));
measure_bandwidth("xor32 ", ({ c=0; int j; for(j=0; j<N/4; j++) c^=*(int*)buf+j*4; }));
measure_bandwidth("xor8 ", c=xor8_finish(xor8_add(0, buf, N)));
measure_bandwidth("xor8_64 ", c=xor8_64_finish(xor8_64_add(0, buf, N)));
measure_bandwidth("crc32by1 ", ({ c=0; int j; for(j=0; j<N; j++) c=crc32(c, buf+j, 1); }));
measure_bandwidth("crc32by2 ", ({ c=0; int j; for(j=0; j<N; j+=2) c=crc32(c, buf+j, 2); }));
measure_bandwidth("sum8by1 ", ({ c=0; int j; for(j=0; j<N; j++) c+=buf[j]; }));
measure_bandwidth("murmurby1", ({ struct murmur mm; murmur_init(&mm); int j; for(j=0; j<N; j++) murmur_add(&mm, buf+j, 1); c=murmur_finish(&mm); }));
measure_bandwidth("murmurby2", ({ struct murmur mm; murmur_init(&mm); int j; for(j=0; j<N; j+=2) murmur_add(&mm, buf+j, 2); c=murmur_finish(&mm); }));
measure_bandwidth("sum84by1 ", ({ struct sum84 s; sum84_init(&s); int j; for(j=0; j<N; j++) sum84_add(&s, buf+j, 1); c=sum84_finish(&s); }));
measure_bandwidth("xor8by1 ", ({ int j; c=0; for(j=0; j<N; j++) c=xor8_add(c, buf+j, 1); c=xor8_finish(c); }));
measure_bandwidth("xor864by1", ({ int j; uint64_t x=0; for(j=0; j<N; j++) x=xor8_64_add(x, buf+j, 1); c=xor8_64_finish(x); }));
}
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
buf = malloc(N);
int i;
for (i=0; i<N; i++) buf[i]=random();
measure_bandwidths();
return 0;
}

1205
storage/tokudb/ft-index/ft/hash-benchmarks/hash-benchmark.cc
View File

File diff suppressed because it is too large Load Diff

4
storage/tokudb/ft-index/ft/le-cursor.h
View File

@@ -104,10 +104,10 @@ PATENT RIGHTS GRANT:
typedef struct le_cursor *LE_CURSOR;
// Create a leaf cursor for a tree (brt) within a transaction (txn)
// Create a leaf cursor for a tree (ft_h) within a transaction (txn)
// Success: returns 0, stores the LE_CURSOR in the le_cursor_result
// Failure: returns a non-zero error number
int toku_le_cursor_create(LE_CURSOR *le_cursor_result, FT_HANDLE brt, TOKUTXN txn);
int toku_le_cursor_create(LE_CURSOR *le_cursor_result, FT_HANDLE ft_h, TOKUTXN txn);
// Close and free the LE_CURSOR
void toku_le_cursor_close(LE_CURSOR le_cursor);

3
storage/tokudb/ft-index/ft/leafentry.h
View File

@@ -96,11 +96,10 @@ PATENT RIGHTS GRANT:
#include <toku_portability.h>
#include <util/mempool.h>
#include <util/omt.h>
#include "txn_manager.h"
#include "rbuf.h"
#include "x1764.h"
#include "omt.h"
/*
Memory format of packed leaf entry

4
storage/tokudb/ft-index/ft/locking-benchmarks/Makefile
View File

@@ -1,4 +0,0 @@
CPPFLAGS= -D_GNU_SOURCE -I../../toku_include
CFLAGS=-O2 -Wall -W -Werror
LDFLAGS=-lpthread
default: trylock trylock-rdtsc mfence-benchmark process_locks pthread-locks

217
storage/tokudb/ft-index/ft/locking-benchmarks/mfence-benchmark.cc
View File

@@ -1,217 +0,0 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* Time {m,l,s}fence vs.xchgl for a memory barrier. */
/* Timing numbers:
* Intel T2500 2GHZ
do1 9.0ns/loop
mfence: 29.0ns/loop (marginal cost= 20.0ns)
sfence: 17.3ns/loop (marginal cost= 8.3ns)
lfence: 23.6ns/loop (marginal cost= 14.6ns)
xchgl: 35.8ns/loop (marginal cost= 26.8ns)
* AMD Athlon 64 X2 Dual Core Processor 4200+
Timings are more crazy
do1 20.6ns/loop
mfence: 12.9ns/loop (marginal cost= -7.6ns)
sfence: 8.4ns/loop (marginal cost= -12.1ns)
lfence: 20.2ns/loop (marginal cost= -0.3ns)
xchgl: 16.6ns/loop (marginal cost= -3.9ns)
do1 13.0ns/loop
mfence: 25.6ns/loop (marginal cost= 12.6ns)
sfence: 21.0ns/loop (marginal cost= 8.1ns)
lfence: 12.9ns/loop (marginal cost= -0.1ns)
xchgl: 29.3ns/loop (marginal cost= 16.3ns)
*/
#include <sys/time.h>
#include <stdio.h>
#include <portability/toku_atomic.h>
enum { COUNT = 100000000 };
static inline void xchgl (void) {
{
/*
* According to the Intel Architecture Software Developer's
* Manual, Volume 3: System Programming Guide
* (http://www.intel.com/design/pro/manuals/243192.htm), page
* 7-6, "For the P6 family processors, locked operations
* serialize all outstanding load and store operations (that
* is, wait for them to complete)."
* Since xchg is locked by default, it is one way to do membar.
*/
int x=0, y;
asm volatile ("xchgl %0,%1" :"=r" (x) :"m" (y), "0" (x) :"memory");
}
}
static inline void mfence (void) {
asm volatile ("mfence":::"memory");
}
static inline void lfence (void) {
asm volatile ("lfence":::"memory");
}
static inline void sfence (void) {
asm volatile ("sfence":::"memory");
}
int lock_for_lock_and_unlock;
static inline void lock_and_unlock (void) {
(void)toku_sync_lock_test_and_set(&lock_for_lock_and_unlock, 1);
toku_sync_lock_release(&lock_for_lock_and_unlock);
}
double tdiff (struct timeval *start, struct timeval *end) {
return ((end->tv_sec-start->tv_sec + 1e-6*(end->tv_usec + start->tv_usec))/COUNT)*1e9;
}
double nop_cost;
void do1 (volatile int *x) {
int i;
struct timeval start, end;
gettimeofday(&start, 0);
for (i=0; i<COUNT; i++) {
x[0]++;
x[1]++;
x[2]++;
x[3]++;
}
gettimeofday(&end, 0);
printf("do1 %6.1fns/loop\n", nop_cost=tdiff(&start, &end));
}
#define doit(name) void do ##name (volatile int *x) { \
int i; \
struct timeval start, end; \
gettimeofday(&start, 0); \
for (i=0; i<COUNT; i++) { \
x[0]++; \
x[1]++; \
name(); \
x[2]++; \
x[3]++; \
} \
gettimeofday(&end, 0); \
double this_cost = tdiff(&start, &end); \
printf("%15s:%6.1fns/loop (marginal cost=%6.1fns)\n", #name, this_cost, this_cost-nop_cost); \
}
doit(mfence)
doit(lfence)
doit(sfence)
doit(xchgl)
doit(lock_and_unlock);
int main (int argc __attribute__((__unused__)),
char *argv[] __attribute__((__unused__))) {
int x[4];
int i;
for (i=0; i<4; i++) {
do1(x);
domfence(x);
dosfence(x);
dolfence(x);
doxchgl(x);
dolock_and_unlock(x);
}
return 0;
}

233
storage/tokudb/ft-index/ft/locking-benchmarks/process_locks.cc
View File

@@ -1,233 +0,0 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* Test pthread rwlocks in multiprocess environment. */
/* How expensive is
* - Obtaining a read-only lock for the first obtainer.
* - Obtaining it for the second one?
* - The third one? */
#include <toku_assert.h>
#include <fcntl.h>
#include <errno.h>
#include <pthread.h>
#include <stdio.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
float tdiff (struct timeval *start, struct timeval *end) {
return 1e6*(end->tv_sec-start->tv_sec) +(end->tv_usec - start->tv_usec);
}
#define FILE "process.data"
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
int r;
int fd;
void *p;
fd=open(FILE, O_CREAT|O_RDWR|O_TRUNC, 0666); assert(fd>=0);
int i;
for (i=0; i<4096; i++) {
r=write(fd, "\000", 1);
assert(r==1);
}
p=mmap(0, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
if (p==MAP_FAILED) {
printf("err=%d %s (EPERM=%d)\n", errno, strerror(errno), EPERM);
}
assert(p!=MAP_FAILED);
r=close(fd); assert(r==0);
pthread_rwlockattr_t attr;
pthread_rwlock_t *lock=p;
r=pthread_rwlockattr_init(&attr); assert(r==0);
r=pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); assert(r==0);
r=pthread_rwlock_init(lock, &attr); assert(r==0);
r=pthread_rwlock_init(lock+1, &attr); assert(r==0);
r=pthread_rwlock_wrlock(lock);
pid_t pid;
if ((pid=fork())==0) {
// I'm the child
r = munmap(p, 4096); assert(r==0);
fd = open(FILE, O_RDWR, 0666); assert(fd>=0);
p=mmap(0, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
assert(p!=MAP_FAILED);
r=close(fd); assert(r==0);
printf("A0\n");
r=pthread_rwlock_wrlock(lock);
printf("C\n");
sleep(1);
r=pthread_rwlock_unlock(lock);
printf("D\n");
r=pthread_rwlock_rdlock(lock);
printf("E0\n");
sleep(1);
} else {
printf("A1\n");
sleep(1);
printf("B\n");
r=pthread_rwlock_unlock(lock); // release the lock grabbed before the fork
assert(r==0);
sleep(1);
r=pthread_rwlock_rdlock(lock);
assert(r==0);
printf("E1\n");
sleep(1);
int status;
pid_t waited=wait(&status);
assert(waited==pid);
}
return 0;
#if 0
int j;
int i;
int r;
struct timeval start, end;
for (j=0; j<3; j++) {
for (i=0; i<K; i++) {
r=pthread_rwlock_init(&rwlocks[i], NULL);
assert(r==0);
}
gettimeofday(&start, 0);
for (i=0; i<K; i++) {
r = pthread_rwlock_tryrdlock(&rwlocks[i]);
assert(r==0);
}
gettimeofday(&end, 0);
printf("pthread_rwlock_tryrdlock took %9.3fus for %d ops: %9.3fus/lock (%9.3fMops/s)\n", tdiff(&start,&end), K, tdiff(&start,&end)/K, K/tdiff(&start,&end));
}
for (j=0; j<3; j++) {
for (i=0; i<K; i++) {
r=pthread_rwlock_init(&rwlocks[i], NULL);
assert(r==0);
}
gettimeofday(&start, 0);
for (i=0; i<K; i++) {
r = pthread_rwlock_rdlock(&rwlocks[i]);
assert(r==0);
}
gettimeofday(&end, 0);
printf("pthread_rwlock_rdlock took %9.3fus for %d ops: %9.3fus/lock (%9.3fMops/s)\n", tdiff(&start,&end), K, tdiff(&start,&end)/K, K/tdiff(&start,&end));
}
for (j=0; j<3; j++) {
for (i=0; i<K; i++) {
blocks[i].state=0;
blocks[i].mutex=0;
}
gettimeofday(&start, 0);
for (i=0; i<K; i++) {
brwl_rlock(&blocks[i]);
}
gettimeofday(&end, 0);
printf("brwl_rlock took %9.3fus for %d ops: %9.3fus/lock (%9.3fMops/s)\n", tdiff(&start,&end), K, tdiff(&start,&end)/K, K/tdiff(&start,&end));
}
return 0;
#endif
}

272
storage/tokudb/ft-index/ft/locking-benchmarks/pthread-locks.cc
View File

@@ -1,272 +0,0 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* How expensive is
* - Obtaining a read-only lock for the first obtainer.
* - Obtaining it for the second one?
* - The third one? */
#include <toku_assert.h>
#include <pthread.h>
#include <stdio.h>
#include <sys/time.h>
#include <pthread.h>
#include <portability/toku_atomic.h>
float tdiff (struct timeval *start, struct timeval *end) {
return 1e6*(end->tv_sec-start->tv_sec) +(end->tv_usec - start->tv_usec);
}
/* My own rwlock implementation. */
struct brwl {
int mutex;
int state; // 0 for unlocked, -1 for a writer, otherwise many readers
};
static inline int xchg(volatile int *ptr, int x)
{
__asm__("xchgl %0,%1" :"=r" (x) :"m" (*(ptr)), "0" (x) :"memory");
return x;
}
static inline void sfence (void) {
asm volatile ("sfence":::"memory");
}
static inline void brwl_rlock_fence (struct brwl *l) {
while (xchg(&l->mutex, 1)) ;
l->state++;
sfence();
l->mutex=0;
}
static inline void brwl_rlock_xchg (struct brwl *l) {
while (xchg(&l->mutex, 1)) ;
l->state++;
xchg(&l->mutex, 0);
}
// Something wrong with the compiler for longs
static inline long
fetch_and_add (volatile long *p, long incr)
{
long result = incr;
__asm__ __volatile__ ("lock; xaddl %0, %1" :
"+r" (result), "+m" (*p) : : "memory");
return result;
}
static inline int
fetch_and_add_i (volatile int *p, int incr)
{
int result = incr;
__asm__ __volatile__ ("lock; xadd %0, %1" :
"+r" (result), "+m" (*p) : : "memory");
return result;
}
static inline int
gcc_fetch_and_add_i (volatile int *p, int incr)
{
return toku_sync_fetch_and_add(p, incr);
}
static inline long
gcc_fetch_and_add_l (volatile long *p, long incr)
{
return toku_sync_fetch_and_add(p, incr);
}
// Something wrong with the compiler for longs
/* Returns nonzero if the comparison succeeded. */
static inline long
compare_and_swap_full(volatile long *addr,
long old, long new_val)
{
char result;
__asm__ __volatile__("lock; cmpxchgl %2, %0; setz %1"
: "+m"(*(addr)), "=q"(result)
: "r" (new_val), "a"(old) : "memory");
return (int) result;
}
/* Returns nonzero if the comparison succeeded. */
// Atomically compare *addr to old_val, and replace *addr by new_val
// if the first comparison succeeds. Returns nonzero if the comparison
// succeeded and *addr was updated.
static inline int
compare_and_swap_full_i(volatile int *addr,
int old, int new_val)
{
char result;
__asm__ __volatile__("lock; cmpxchg %2, %0; setz %1"
: "+m"(*(addr)), "=q"(result)
: "r" (new_val), "a"(old) : "memory");
return (int) result;
}
enum {K=100000};
pthread_rwlock_t rwlocks[K];
struct brwl blocks[K];
pthread_mutex_t mlocks[K];
long lvals[K];
int ivals[K];
#define TIME(s, i, init, body) ({ \
int j_tmp; \
printf("%-24s", s); \
for (j_tmp=0; j_tmp<3; j_tmp++) { \
struct timeval start,end; \
int i; \
for (i=0; i<K; i++) { \
init; \
} \
gettimeofday(&start, 0); \
for (i=0; i<K; i++) { \
body; \
} \
gettimeofday(&end, 0); \
printf(" %9.3fus", tdiff(&start,&end)/K); \
} \
printf("\n"); \
})
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
printf("sizeof (pthread_mutex_t) %lu\n", sizeof (pthread_mutex_t));
printf("sizeof (pthread_cond_t) %lu\n", sizeof (pthread_cond_t));
TIME("pthread_mutex_lock_errorcheck", i,
({ int r; pthread_mutexattr_t mattr;
r = pthread_mutexattr_init(&mattr); assert(r == 0);
r = pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_ERRORCHECK_NP); assert(r == 0);
r = pthread_mutex_init(&mlocks[i], &mattr); assert(r==0);
r = pthread_mutexattr_destroy(&mattr); assert(r == 0); }),
({ int r = pthread_mutex_lock(&mlocks[i]); assert(r==0); }));
TIME("pthread_mutex_lock", i,
({ int r = pthread_mutex_init(&mlocks[i], NULL); assert(r==0); }),
({ int r = pthread_mutex_lock(&mlocks[i]); assert(r==0); }));
TIME("pthread_mutex_unlock", i,
({ int r = pthread_mutex_init(&mlocks[i], NULL); assert(r==0); r = pthread_mutex_lock(&mlocks[i]); assert(r==0); }),
({ int r = pthread_mutex_unlock(&mlocks[i]); assert(r==0); }));
TIME("pthread_rwlock_tryrdlock", i,
({ int r = pthread_rwlock_init(&rwlocks[i], NULL); assert(r==0); }),
({ int r = pthread_rwlock_tryrdlock(&rwlocks[i]); assert(r==0); }));
TIME("pthread_rwlock_rdlock", i,
({ int r = pthread_rwlock_init(&rwlocks[i], NULL); assert(r==0); }),
({ int r = pthread_rwlock_rdlock(&rwlocks[i]); assert(r==0); }));
TIME("brwl_rlock_xchg", i,
(blocks[i].state=0, blocks[i].mutex=0),
brwl_rlock_xchg(&blocks[i]));
TIME("brwl_rlock_fence", i,
(blocks[i].state=0, blocks[i].mutex=0),
brwl_rlock_fence(&blocks[i]));
int fa=0;
TIME("fetchadd", i,
(void)0,
fetch_and_add_i(&fa, i));
// printf("fa=%d\n", fa);
fa=0;
TIME("gcc_fetchadd", i,
(void)0,
gcc_fetch_and_add_i(&fa, i));
// printf("fa=%d\n", fa);
long fal = 0;
TIME("gcc_fetchaddlong", i,
(void)0,
gcc_fetch_and_add_l(&fal, i));
// printf("fa=%d\n", fa);
TIME("compare_and_swap", i,
ivals[i]=0,
({ int r=compare_and_swap_full_i(&ivals[i], 0, 1); assert(r==1); }));
return 0;
}

247
storage/tokudb/ft-index/ft/locking-benchmarks/trylock-rdtsc.cc
View File

@@ -1,247 +0,0 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* Like trylock, except use rdstc */
#define _MULTI_THREADED
#include <pthread.h>
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/time.h>
#include <unistd.h>
#include <rdtsc.h>
#include <portability/toku_atomic.h>
float tdiff (struct timeval *start, struct timeval *end) {
return 1e6*(end->tv_sec-start->tv_sec) +(end->tv_usec - start->tv_usec);
}
unsigned long long rtdiff (unsigned long long a, unsigned long long b) {
return (b-a);
}
/* Simple function to check the return code and exit the program
if the function call failed
*/
static void compResults(char *string, int rc) {
if (rc) {
printf("Error on : %s, rc=%d",
string, rc);
exit(EXIT_FAILURE);
}
return;
}
pthread_rwlock_t rwlock = PTHREAD_RWLOCK_INITIALIZER;
void *rdlockThread(void *arg __attribute__((unused)))
{
int rc;
int count=0;
unsigned long long t_start, t_end;
printf("Entered thread, getting read lock with mp wait\n");
Retry:
t_start = rdtsc();
rc = pthread_rwlock_tryrdlock(&rwlock);
t_end = rdtsc();
printf("pthread_rwlock_tryrdlock took %llu clocks\n", rtdiff(t_start,t_end));
if (rc == EBUSY) {
if (count >= 10) {
printf("Retried too many times, failure!\n");
exit(EXIT_FAILURE);
}
++count;
printf("Could not get lock, do other work, then RETRY...\n");
sleep(1);
goto Retry;
}
compResults("pthread_rwlock_tryrdlock() 1\n", rc);
sleep(2);
printf("unlock the read lock\n");
t_start = rdtsc();
rc = pthread_rwlock_unlock(&rwlock);
t_end = rdtsc();
compResults("pthread_rwlock_unlock()\n", rc);
printf("Took %llu clocks\n", rtdiff(t_start, t_end));
printf("Secondary thread complete\n");
return NULL;
}
int main(int argc __attribute__((unused)), char **argv)
{
int rc=0;
pthread_t thread;
unsigned long long t_start, t_end;
printf("Enter Testcase - %s\n", argv[0]);
t_start = rdtsc();
t_end = rdtsc();
printf("nop Took %llu clocks\n", rtdiff(t_start, t_end));
{
int N=1000;
int i;
printf("Main, get and release the write lock %d times\n", N);
t_start = rdtsc();
for (i=0; i<N; i++) {
rc = pthread_rwlock_wrlock(&rwlock);
rc = pthread_rwlock_unlock(&rwlock);
}
t_end = rdtsc();
compResults("pthread_rwlock_wrlock()\n", rc);
printf("Took %5.2f clocks/op\n", ((double)(t_end-t_start))/N);
}
printf("Main, get the write lock\n");
t_start = rdtsc();
rc = pthread_rwlock_wrlock(&rwlock);
t_end = rdtsc();
compResults("pthread_rwlock_wrlock()\n", rc);
printf("Took %llu clocks\n", rtdiff(t_start, t_end));
printf("Main, create the try read lock thread\n");
rc = pthread_create(&thread, NULL, rdlockThread, NULL);
compResults("pthread_create\n", rc);
printf("Main, wait a bit holding the write lock\n");
sleep(5);
printf("Main, Now unlock the write lock\n");
t_start = rdtsc();
rc = pthread_rwlock_unlock(&rwlock);
t_end = rdtsc();
compResults("pthread_rwlock_unlock()\n", rc);
printf("Took %llu clocks\n", rtdiff(t_start, t_end));
printf("Main, wait for the thread to end\n");
rc = pthread_join(thread, NULL);
compResults("pthread_join\n", rc);
rc = pthread_rwlock_destroy(&rwlock);
compResults("pthread_rwlock_destroy()\n", rc);
printf("Main completed\n");
{
static int lock_for_lock_and_unlock;
t_start = rdtsc();
(void)toku_sync_lock_test_and_set(&lock_for_lock_and_unlock, 1);
t_end = rdtsc();
printf("sync_lock_test_and_set took %llu clocks\n", t_end-t_start);
t_start = rdtsc();
toku_sync_lock_release(&lock_for_lock_and_unlock);
t_end = rdtsc();
printf("sync_lock_release took %llu clocks\n", t_end-t_start);
}
{
t_start = rdtsc();
(void)toku_sync_synchronize();
t_end = rdtsc();
printf("sync_synchornize took %llu clocks\n", t_end-t_start);
}
t_start = rdtsc();
sleep(1);
t_end = rdtsc();
printf("sleep(1) took %llu clocks\n", t_end-t_start);
return 0;
}

213
storage/tokudb/ft-index/ft/locking-benchmarks/trylock.cc
View File

@@ -1,213 +0,0 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#define _MULTI_THREADED
#include <pthread.h>
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/time.h>
#include <unistd.h>
float tdiff (struct timeval *start, struct timeval *end) {
return 1e6*(end->tv_sec-start->tv_sec) +(end->tv_usec - start->tv_usec);
}
/* Simple function to check the return code and exit the program
if the function call failed
*/
static void compResults(char *string, int rc) {
if (rc) {
printf("Error on : %s, rc=%d",
string, rc);
exit(EXIT_FAILURE);
}
return;
}
pthread_rwlock_t rwlock = PTHREAD_RWLOCK_INITIALIZER;
void *rdlockThread(void *arg __attribute__((unused)))
{
int rc;
int count=0;
struct timeval start, end;
printf("Entered thread, getting read lock with mp wait\n");
Retry:
gettimeofday(&start, 0);
rc = pthread_rwlock_tryrdlock(&rwlock);
gettimeofday(&end, 0);
printf("pthread_rwlock_tryrdlock took %9.3fus\n", tdiff(&start,&end));
if (rc == EBUSY) {
if (count >= 10) {
printf("Retried too many times, failure!\n");
exit(EXIT_FAILURE);
}
++count;
printf("Could not get lock, do other work, then RETRY...\n");
sleep(1);
goto Retry;
}
compResults("pthread_rwlock_tryrdlock() 1\n", rc);
sleep(2);
printf("unlock the read lock\n");
gettimeofday(&start, 0);
rc = pthread_rwlock_unlock(&rwlock);
gettimeofday(&end, 0);
compResults("pthread_rwlock_unlock()\n", rc);
printf("%lu.%6lu to %lu.%6lu is %9.2f\n", start.tv_sec, start.tv_usec, end.tv_sec, end.tv_usec, tdiff(&start, &end));
printf("Secondary thread complete\n");
return NULL;
}
int main(int argc __attribute__((unused)), char **argv)
{
int rc=0;
pthread_t thread;
struct timeval start, end;
printf("Enter Testcase - %s\n", argv[0]);
gettimeofday(&start, 0);
gettimeofday(&end, 0);
printf("nop Took %9.2f\n", tdiff(&start, &end));
{
int N=1000;
int i;
printf("Main, get and release the write lock %d times\n", N);
gettimeofday(&start, 0);
for (i=0; i<N; i++) {
rc = pthread_rwlock_wrlock(&rwlock);
rc = pthread_rwlock_unlock(&rwlock);
}
gettimeofday(&end, 0);
compResults("pthread_rwlock_wrlock()\n", rc);
printf("Took %9.2fns/op\n", 1000*tdiff(&start, &end)/N);
}
printf("Main, get the write lock\n");
gettimeofday(&start, 0);
rc = pthread_rwlock_wrlock(&rwlock);
gettimeofday(&end, 0);
compResults("pthread_rwlock_wrlock()\n", rc);
printf("Took %9.2f\n", tdiff(&start, &end));
printf("Main, create the try read lock thread\n");
rc = pthread_create(&thread, NULL, rdlockThread, NULL);
compResults("pthread_create\n", rc);
printf("Main, wait a bit holding the write lock\n");
sleep(5);
printf("Main, Now unlock the write lock\n");
gettimeofday(&start, 0);
rc = pthread_rwlock_unlock(&rwlock);
gettimeofday(&end, 0);
compResults("pthread_rwlock_unlock()\n", rc);
printf("Took %9.2f\n", tdiff(&start, &end));
printf("Main, wait for the thread to end\n");
rc = pthread_join(thread, NULL);
compResults("pthread_join\n", rc);
rc = pthread_rwlock_destroy(&rwlock);
compResults("pthread_rwlock_destroy()\n", rc);
printf("Main completed\n");
return 0;
}

8
storage/tokudb/ft-index/ft/log-internal.h
View File

@@ -99,15 +99,17 @@ PATENT RIGHTS GRANT:
#include "ft-internal.h"
#include "log.h"
#include "toku_list.h"
#include "memarena.h"
#include "logfilemgr.h"
#include "txn.h"
#include "txn_manager.h"
#include <portability/toku_pthread.h>
#include <util/omt.h>
#include "rollback_log_node_cache.h"
#include "txn_child_manager.h"
#include <portability/toku_pthread.h>
#include <util/memarena.h>
#include <util/omt.h>
using namespace toku;
// Locking for the logger
// For most purposes we use the big ydb lock.

9
storage/tokudb/ft-index/ft/log.h
View File

@@ -94,19 +94,17 @@ PATENT RIGHTS GRANT:
#include <toku_portability.h>
#include <errno.h>
#include <db.h>
#include "fttypes.h"
#include "memory.h"
#include "x1764.h"
struct roll_entry;
#include "logger.h"
#include "rollback.h"
#include "recover.h"
#include "txn.h"
struct roll_entry;
static inline int toku_copy_BYTESTRING(BYTESTRING *target, BYTESTRING val) {
target->len = val.len;
target->data = (char *) toku_memdup(val.data, (size_t)val.len);
@@ -117,6 +115,7 @@ static inline int toku_copy_BYTESTRING(BYTESTRING *target, BYTESTRING val) {
}
static inline void toku_free_TXNID(TXNID txnid __attribute__((__unused__))) {}
static inline void toku_free_TXNID_PAIR(TXNID_PAIR txnid __attribute__((__unused__))) {}
static inline void toku_free_LSN(LSN lsn __attribute__((__unused__))) {}
static inline void toku_free_uint64_t(uint64_t u __attribute__((__unused__))) {}
static inline void toku_free_uint32_t(uint32_t u __attribute__((__unused__))) {}

3
storage/tokudb/ft-index/ft/logcursor.cc
View File

@@ -167,11 +167,8 @@ static int lc_open_logfile(TOKULOGCURSOR lc, int index) {
lc->cur_fp = fopen(lc->logfiles[index], "rb");
if ( lc->cur_fp == NULL )
return DB_NOTFOUND;
// debug printf("%s:%d %s %p %u\n", __FUNCTION__, __LINE__, lc->logfiles[index], lc->buffer, (unsigned) lc->buffer_size);
#if !TOKU_WINDOWS //Windows reads logs fastest if we use default settings (not use setvbuf to change buffering)
r = setvbuf(lc->cur_fp, (char *) lc->buffer, _IOFBF, lc->buffer_size);
assert(r == 0);
#endif
// position fp past header, ignore 0 length file (t:2384)
unsigned int version=0;
if ( lc_file_len(lc->logfiles[index]) >= 12 ) {

18
storage/tokudb/ft-index/ft/logformat.cc
View File

@@ -536,7 +536,7 @@ generate_log_writer (void) {
if (strcmp(field_type->name, "timestamp") == 0)
fprintf(cf, " if (timestamp == 0) timestamp = toku_get_timestamp();\n");
fprintf(cf, " wbuf_nocrc_%s(&wbuf, %s);\n", field_type->type, field_type->name));
fprintf(cf, " wbuf_nocrc_int(&wbuf, x1764_memory(wbuf.buf, wbuf.ndone));\n");
fprintf(cf, " wbuf_nocrc_int(&wbuf, toku_x1764_memory(wbuf.buf, wbuf.ndone));\n");
fprintf(cf, " wbuf_nocrc_int(&wbuf, buflen);\n");
fprintf(cf, " assert(wbuf.ndone==buflen);\n");
fprintf(cf, " logger->inbuf.n_in_buf += buflen;\n");
@@ -558,7 +558,7 @@ generate_log_reader (void) {
fprintf(cf, " uint32_t checksum_in_file, len_in_file;\n");
fprintf(cf, " r=toku_fread_uint32_t_nocrclen(infile, &checksum_in_file); actual_len+=4; if (r!=0) return r;\n");
fprintf(cf, " r=toku_fread_uint32_t_nocrclen(infile, &len_in_file); actual_len+=4; if (r!=0) return r;\n");
fprintf(cf, " if (checksum_in_file!=x1764_finish(checksum) || len_in_file!=actual_len || len1 != len_in_file) return DB_BADFORMAT;\n");
fprintf(cf, " if (checksum_in_file!=toku_x1764_finish(checksum) || len_in_file!=actual_len || len1 != len_in_file) return DB_BADFORMAT;\n");
fprintf(cf, " return 0;\n");
fprintf(cf, "}\n\n");
});
@@ -568,12 +568,12 @@ generate_log_reader (void) {
fprintf(cf, " uint32_t len1; int r;\n");
fprintf(cf, " uint32_t ignorelen=0;\n");
fprintf(cf, " struct x1764 checksum;\n");
fprintf(cf, " x1764_init(&checksum);\n");
fprintf(cf, " toku_x1764_init(&checksum);\n");
fprintf(cf, " r = toku_fread_uint32_t(infile, &len1, &checksum, &ignorelen); if (r!=0) return r;\n");
fprintf(cf, " int cmd=fgetc(infile);\n");
fprintf(cf, " if (cmd==EOF) return EOF;\n");
fprintf(cf, " char cmdchar = (char)cmd;\n");
fprintf(cf, " x1764_add(&checksum, &cmdchar, 1);\n");
fprintf(cf, " toku_x1764_add(&checksum, &cmdchar, 1);\n");
fprintf(cf, " le->cmd=(enum lt_cmd)cmd;\n");
fprintf(cf, " switch ((enum lt_cmd)cmd) {\n");
DO_LOGTYPES(lt, {
@@ -639,14 +639,14 @@ generate_logprint (void) {
fprintf(pf, " uint32_t len1, crc_in_file;\n");
fprintf(pf, " uint32_t ignorelen=0;\n");
fprintf(pf, " struct x1764 checksum;\n");
fprintf(pf, " x1764_init(&checksum);\n");
fprintf(pf, " toku_x1764_init(&checksum);\n");
fprintf(pf, " r=toku_fread_uint32_t(f, &len1, &checksum, &ignorelen);\n");
fprintf(pf, " if (r==EOF) return EOF;\n");
fprintf(pf, " cmd=fgetc(f);\n");
fprintf(pf, " if (cmd==EOF) return DB_BADFORMAT;\n");
fprintf(pf, " uint32_t len_in_file, len=1+4; // cmd + len1\n");
fprintf(pf, " char charcmd = (char)cmd;\n");
fprintf(pf, " x1764_add(&checksum, &charcmd, 1);\n");
fprintf(pf, " toku_x1764_add(&checksum, &charcmd, 1);\n");
fprintf(pf, " switch ((enum lt_cmd)cmd) {\n");
DO_LOGTYPES(lt, { if (strlen(lt->name)>maxnamelen) maxnamelen=strlen(lt->name); });
DO_LOGTYPES(lt, {
@@ -664,7 +664,7 @@ generate_logprint (void) {
fprintf(pf, "); if (r!=0) return r;\n");
});
fprintf(pf, " {\n");
fprintf(pf, " uint32_t actual_murmur = x1764_finish(&checksum);\n");
fprintf(pf, " uint32_t actual_murmur = toku_x1764_finish(&checksum);\n");
fprintf(pf, " r = toku_fread_uint32_t_nocrclen (f, &crc_in_file); len+=4; if (r!=0) return r;\n");
fprintf(pf, " fprintf(outf, \" crc=%%08x\", crc_in_file);\n");
fprintf(pf, " if (crc_in_file!=actual_murmur) fprintf(outf, \" checksum=%%08x\", actual_murmur);\n");
@@ -806,7 +806,7 @@ generate_rollbacks (void) {
DO_ROLLBACKS(lt, {
fprintf(cf, " case RT_%s:\n", lt->name);
fprintf(cf, " mem_needed = sizeof(item->u.%s) + __builtin_offsetof(struct roll_entry, u.%s);\n", lt->name, lt->name);
fprintf(cf, " CAST_FROM_VOIDP(item, malloc_in_memarena(ma, mem_needed));\n");
fprintf(cf, " CAST_FROM_VOIDP(item, toku_memarena_malloc(ma, mem_needed));\n");
fprintf(cf, " item->cmd = cmd;\n");
DO_FIELDS(field_type, lt, fprintf(cf, " rbuf_ma_%s(&rc, ma, &item->u.%s.%s);\n", field_type->type, lt->name, field_type->name));
fprintf(cf, " *itemp = item;\n");
@@ -858,7 +858,7 @@ int main (int argc, const char *const argv[]) {
fprintf2(cf, pf, "#include <ft/fttypes.h>\n");
fprintf2(cf, pf, "#include <ft/log-internal.h>\n");
fprintf(hf, "#include <ft/ft-internal.h>\n");
fprintf(hf, "#include <ft/memarena.h>\n");
fprintf(hf, "#include <util/memarena.h>\n");
generate_enum();
generate_log_struct();
generate_dispatch();

19
storage/tokudb/ft-index/ft/logger.cc
View File

@@ -98,7 +98,7 @@ PATENT RIGHTS GRANT:
#include "log-internal.h"
#include "txn_manager.h"
#include "rollback_log_node_cache.h"
#include "huge_page_detection.h"
#include <util/status.h>
static const int log_format_version=TOKU_LOG_VERSION;
@@ -164,11 +164,6 @@ static bool is_a_logfile (const char *name, long long *number_result) {
// TODO: can't fail
int toku_logger_create (TOKULOGGER *resultp) {
if (complain_and_return_true_if_huge_pages_are_enabled()) {
*resultp = NULL;
errno = TOKUDB_HUGE_PAGES_ENABLED;
return TOKUDB_HUGE_PAGES_ENABLED;
}
TOKULOGGER CALLOC(result);
if (result==0) return get_error_errno();
result->is_open=false;
@@ -187,7 +182,7 @@ int toku_logger_create (TOKULOGGER *resultp) {
result->last_completed_checkpoint_lsn = ZERO_LSN;
// next_log_file_number is uninitialized
// n_in_file is uninitialized
result->write_block_size = FT_DEFAULT_NODE_SIZE; // default logging size is the same as the default brt block size
result->write_block_size = FT_DEFAULT_NODE_SIZE; // default logging size is the same as the default ft block size
toku_logfilemgr_create(&result->logfilemgr);
*resultp=result;
ml_init(&result->input_lock);
@@ -234,7 +229,7 @@ toku_logger_open_with_last_xid(const char *directory, TOKULOGGER logger, TXNID l
if (logger->is_open) return EINVAL;
int r;
TXNID last_xid_if_clean_shutdown;
TXNID last_xid_if_clean_shutdown = TXNID_NONE;
r = toku_logfilemgr_init(logger->logfilemgr, directory, &last_xid_if_clean_shutdown);
if ( r!=0 )
return r;
@@ -285,7 +280,7 @@ toku_logger_open_rollback(TOKULOGGER logger, CACHETABLE cachetable, bool create)
assert(logger->is_open);
assert(!logger->rollback_cachefile);
FT_HANDLE t = NULL; // Note, there is no DB associated with this BRT.
FT_HANDLE t = NULL; // Note, there is no DB associated with this FT.
toku_ft_handle_create(&t);
int r = toku_ft_handle_open(t, toku_product_name_strings.rollback_cachefile, create, create, cachetable, NULL_TXN);
if (r == 0) {
@@ -313,7 +308,7 @@ void toku_logger_close_rollback(TOKULOGGER logger) {
CACHEFILE cf = logger->rollback_cachefile; // stored in logger at rollback cachefile open
if (cf) {
FT_HANDLE ft_to_close;
{ //Find "brt"
{ //Find "ft_to_close"
logger->rollback_cache.destroy();
FT CAST_FROM_VOIDP(ft, toku_cachefile_get_userdata(cf));
//Verify it is safe to close it.
@@ -938,7 +933,7 @@ int toku_fread_uint8_t (FILE *f, uint8_t *v, struct x1764 *mm, uint32_t *len) {
int vi=fgetc(f);
if (vi==EOF) return -1;
uint8_t vc=(uint8_t)vi;
x1764_add(mm, &vc, 1);
toku_x1764_add(mm, &vc, 1);
(*len)++;
*v = vc;
return 0;
@@ -1286,7 +1281,7 @@ static int peek_at_log (TOKULOGGER logger, char* filename, LSN *first_lsn) {
if (logger->write_log_files) printf("couldn't open: %s\n", strerror(er));
return er;
}
enum { SKIP = 12+1+4 }; // read the 12 byte header, the first cmd, and the first len
enum { SKIP = 12+1+4 }; // read the 12 byte header, the first message, and the first len
unsigned char header[SKIP+8];
int r = read(fd, header, SKIP+8);
if (r!=SKIP+8) return 0; // cannot determine that it's archivable, so we'll assume no. If a later-log is archivable is then this one will be too.

230
storage/tokudb/ft-index/ft/omt.cc
View File

@@ -1,230 +0,0 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <toku_portability.h>
#include <memory.h>
#include <string.h>
#include <db.h>
#include "omt.h"
int
toku_omt_create_steal_sorted_array(OMT *omtp, OMTVALUE **valuesp, uint32_t numvalues, uint32_t capacity) {
OMT XMALLOC(omt);
omt->create_steal_sorted_array(valuesp, numvalues, capacity);
*omtp = omt;
return 0;
}
//TODO: Put all omt API functions here.
int toku_omt_create (OMT *omtp) {
OMT XMALLOC(omt);
omt->create();
*omtp = omt;
return 0;
}
void toku_omt_destroy(OMT *omtp) {
OMT omt=*omtp;
omt->destroy();
toku_free(omt);
*omtp=NULL;
}
uint32_t toku_omt_size(OMT V) {
return V->size();
}
int toku_omt_create_from_sorted_array(OMT *omtp, OMTVALUE *values, uint32_t numvalues) {
OMT XMALLOC(omt);
omt->create_from_sorted_array(values, numvalues);
*omtp=omt;
return 0;
}
int toku_omt_insert_at(OMT omt, OMTVALUE value, uint32_t index) {
return omt->insert_at(value, index);
}
int toku_omt_set_at (OMT omt, OMTVALUE value, uint32_t index) {
return omt->set_at(value, index);
}
int toku_omt_delete_at(OMT omt, uint32_t index) {
return omt->delete_at(index);
}
int toku_omt_fetch(OMT omt, uint32_t i, OMTVALUE *v) {
return omt->fetch(i, v);
}
struct functor {
int (*f)(OMTVALUE, uint32_t, void *);
void *v;
};
static_assert(std::is_pod<functor>::value, "not POD");
int call_functor(const OMTVALUE &v, uint32_t idx, functor *const ftor);
int call_functor(const OMTVALUE &v, uint32_t idx, functor *const ftor) {
return ftor->f(const_cast<OMTVALUE>(v), idx, ftor->v);
}
int toku_omt_iterate(OMT omt, int (*f)(OMTVALUE, uint32_t, void*), void*v) {
struct functor ftor = { .f = f, .v = v };
return omt->iterate<functor, call_functor>(&ftor);
}
int toku_omt_iterate_on_range(OMT omt, uint32_t left, uint32_t right, int (*f)(OMTVALUE, uint32_t, void*), void*v) {
struct functor ftor = { .f = f, .v = v };
return omt->iterate_on_range<functor, call_functor>(left, right, &ftor);
}
struct heftor {
int (*h)(OMTVALUE, void *v);
void *v;
};
static_assert(std::is_pod<heftor>::value, "not POD");
int call_heftor(const OMTVALUE &v, const heftor &htor);
int call_heftor(const OMTVALUE &v, const heftor &htor) {
return htor.h(const_cast<OMTVALUE>(v), htor.v);
}
int toku_omt_insert(OMT omt, OMTVALUE value, int(*h)(OMTVALUE, void*v), void *v, uint32_t *index) {
struct heftor htor = { .h = h, .v = v };
return omt->insert<heftor, call_heftor>(value, htor, index);
}
int toku_omt_find_zero(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, OMTVALUE *value, uint32_t *index) {
struct heftor htor = { .h = h, .v = extra };
return V->find_zero<heftor, call_heftor>(htor, value, index);
}
int toku_omt_find(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, int direction, OMTVALUE *value, uint32_t *index) {
struct heftor htor = { .h = h, .v = extra };
return V->find<heftor, call_heftor>(htor, direction, value, index);
}
int toku_omt_split_at(OMT omt, OMT *newomtp, uint32_t index) {
OMT XMALLOC(newomt);
int r = omt->split_at(newomt, index);
if (r != 0) {
toku_free(newomt);
} else {
*newomtp = newomt;
}
return r;
}
int toku_omt_merge(OMT leftomt, OMT rightomt, OMT *newomtp) {
OMT XMALLOC(newomt);
newomt->merge(leftomt, rightomt);
toku_free(leftomt);
toku_free(rightomt);
*newomtp = newomt;
return 0;
}
int toku_omt_clone_noptr(OMT *dest, OMT src) {
OMT XMALLOC(omt);
omt->clone(*src);
*dest = omt;
return 0;
}
void toku_omt_clear(OMT omt) {
omt->clear();
}
size_t toku_omt_memory_size (OMT omt) {
return omt->memory_size();
}

416
storage/tokudb/ft-index/ft/omt.h
View File

@@ -1,416 +0,0 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#if !defined(TOKU_OMT_H)
#define TOKU_OMT_H
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
// Order Maintenance Tree (OMT)
//
// Maintains a collection of totally ordered values, where each value has an integer weight.
// The OMT is a mutable datatype.
//
// The Abstraction:
//
// An OMT is a vector of values, $V$, where $|V|$ is the length of the vector.
// The vector is numbered from $0$ to $|V|-1$.
// Each value has a weight. The weight of the $i$th element is denoted $w(V_i)$.
//
// We can create a new OMT, which is the empty vector.
//
// We can insert a new element $x$ into slot $i$, changing $V$ into $V'$ where
// $|V'|=1+|V|$ and
//
// V'_j = V_j if $j<i$
// x if $j=i$
// V_{j-1} if $j>i$.
//
// We can specify $i$ using a kind of function instead of as an integer.
// Let $b$ be a function mapping from values to nonzero integers, such that
// the signum of $b$ is monotically increasing.
// We can specify $i$ as the minimum integer such that $b(V_i)>0$.
//
// We look up a value using its index, or using a Heaviside function.
// For lookups, we allow $b$ to be zero for some values, and again the signum of $b$ must be monotonically increasing.
// When lookup up values, we can look up
// $V_i$ where $i$ is the minimum integer such that $b(V_i)=0$. (With a special return code if no such value exists.)
// (Rationale: Ordinarily we want $i$ to be unique. But for various reasons we want to allow multiple zeros, and we want the smallest $i$ in that case.)
// $V_i$ where $i$ is the minimum integer such that $b(V_i)>0$. (Or an indication that no such value exists.)
// $V_i$ where $i$ is the maximum integer such that $b(V_i)<0$. (Or an indication that no such value exists.)
//
// When looking up a value using a Heaviside function, we get the value and its index.
//
// We can also split an OMT into two OMTs, splitting the weight of the values evenly.
// Find a value $j$ such that the values to the left of $j$ have about the same total weight as the values to the right of $j$.
// The resulting two OMTs contain the values to the left of $j$ and the values to the right of $j$ respectively.
// All of the values from the original OMT go into one of the new OMTs.
// If the weights of the values don't split exactly evenly, then the implementation has the freedom to choose whether
// the new left OMT or the new right OMT is larger.
//
// Performance:
// Insertion and deletion should run with $O(\log |V|)$ time and $O(\log |V|)$ calls to the Heaviside function.
// The memory required is O(|V|).
//
// The programming API:
//typedef struct value *OMTVALUE; // A slight improvement over using void*.
#include <util/omt.h>
typedef void *OMTVALUE;
typedef toku::omt<OMTVALUE> *OMT;
int toku_omt_create (OMT *omtp);
// Effect: Create an empty OMT. Stores it in *omtp.
// Requires: omtp != NULL
// Returns:
// 0 success
// ENOMEM out of memory (and doesn't modify *omtp)
// Performance: constant time.
int toku_omt_create_from_sorted_array(OMT *omtp, OMTVALUE *values, uint32_t numvalues);
// Effect: Create a OMT containing values. The number of values is in numvalues.
// Stores the new OMT in *omtp.
// Requires: omtp != NULL
// Requires: values != NULL
// Requires: values is sorted
// Returns:
// 0 success
// ENOMEM out of memory (and doesn't modify *omtp)
// Performance: time=O(numvalues)
// Rational: Normally to insert N values takes O(N lg N) amortized time.
// If the N values are known in advance, are sorted, and
// the structure is empty, we can batch insert them much faster.
int toku_omt_create_steal_sorted_array(OMT *omtp, OMTVALUE **valuesp, uint32_t numvalues, uint32_t steal_capacity);
// Effect: Create an OMT containing values. The number of values is in numvalues.
// On success the OMT takes ownership of *valuesp array, and sets valuesp=NULL.
// Requires: omtp != NULL
// Requires: valuesp != NULL
// Requires: *valuesp is sorted
// Requires: *valuesp was allocated with toku_malloc
// Requires: Capacity of the *valuesp array is <= steal_capacity
// Requires: On success, *valuesp may not be accessed again by the caller.
// Returns:
// 0 success
// ENOMEM out of memory (and doesn't modify *omtp)
// EINVAL *valuesp == NULL or numvalues > capacity
// Performance: time=O(1)
// Rational: toku_omt_create_from_sorted_array takes O(numvalues) time.
// By taking ownership of the array, we save a malloc and memcpy,
// and possibly a free (if the caller is done with the array).
void toku_omt_destroy(OMT *omtp);
// Effect: Destroy an OMT, freeing all its memory.
// Does not free the OMTVALUEs stored in the OMT.
// Those values may be freed before or after calling toku_omt_destroy.
// Also sets *omtp=NULL.
// Requires: omtp != NULL
// Requires: *omtp != NULL
// Rationale: The usage is to do something like
// toku_omt_destroy(&s->omt);
// and now s->omt will have a NULL pointer instead of a dangling freed pointer.
// Rationale: Returns no values since free() cannot fail.
// Rationale: Does not free the OMTVALUEs to reduce complexity.
// Performance: time=O(toku_omt_size(*omtp))
uint32_t toku_omt_size(OMT V);
// Effect: return |V|.
// Requires: V != NULL
// Performance: time=O(1)
int toku_omt_iterate_on_range(OMT omt, uint32_t left, uint32_t right, int (*f)(OMTVALUE, uint32_t, void*), void*v);
// Effect: Iterate over the values of the omt, from left to right, calling f on each value.
// The second argument passed to f is the index of the value.
// The third argument passed to f is v.
// The indices run from 0 (inclusive) to toku_omt_size(omt) (exclusive).
// We will iterate only over [left,right)
//
// Requires: omt != NULL
// left <= right
// Requires: f != NULL
// Returns:
// If f ever returns nonzero, then the iteration stops, and the value returned by f is returned by toku_omt_iterate.
// If f always returns zero, then toku_omt_iterate returns 0.
// Requires: Don't modify omt while running. (E.g., f may not insert or delete values form omt.)
// Performance: time=O(i+\log N) where i is the number of times f is called, and N is the number of elements in omt.
// Rational: Although the functional iterator requires defining another function (as opposed to C++ style iterator), it is much easier to read.
int toku_omt_iterate(OMT omt, int (*f)(OMTVALUE, uint32_t, void*), void*v);
// Effect: Iterate over the values of the omt, from left to right, calling f on each value.
// The second argument passed to f is the index of the value.
// The third argument passed to f is v.
// The indices run from 0 (inclusive) to toku_omt_size(omt) (exclusive).
// Requires: omt != NULL
// Requires: f != NULL
// Returns:
// If f ever returns nonzero, then the iteration stops, and the value returned by f is returned by toku_omt_iterate.
// If f always returns zero, then toku_omt_iterate returns 0.
// Requires: Don't modify omt while running. (E.g., f may not insert or delete values form omt.)
// Performance: time=O(i+\log N) where i is the number of times f is called, and N is the number of elements in omt.
// Rational: Although the functional iterator requires defining another function (as opposed to C++ style iterator), it is much easier to read.
int toku_omt_insert_at(OMT omt, OMTVALUE value, uint32_t idx);
// Effect: Increases indexes of all items at slot >= index by 1.
// Insert value into the position at index.
//
// Returns:
// 0 success
// EINVAL if index>toku_omt_size(omt)
// ENOMEM
// On error, omt is unchanged.
// Performance: time=O(\log N) amortized time.
// Rationale: Some future implementation may be O(\log N) worst-case time, but O(\log N) amortized is good enough for now.
int toku_omt_set_at (OMT omt, OMTVALUE value, uint32_t idx);
// Effect: Replaces the item at index with value.
// Returns:
// 0 success
// EINVAL if index>=toku_omt_size(omt)
// On error, omt i sunchanged.
// Performance: time=O(\log N)
// Rationale: The BRT needs to be able to replace a value with another copy of the same value (allocated in a different location)
int toku_omt_insert(OMT omt, OMTVALUE value, int(*h)(OMTVALUE, void*v), void *v, uint32_t *idx);
// Effect: Insert value into the OMT.
// If there is some i such that $h(V_i, v)=0$ then returns DB_KEYEXIST.
// Otherwise, let i be the minimum value such that $h(V_i, v)>0$.
// If no such i exists, then let i be |V|
// Then this has the same effect as
// omt_insert_at(tree, value, i);
// If index!=NULL then i is stored in *index
// Requires: The signum of h must be monotonically increasing.
// Returns:
// 0 success
// DB_KEYEXIST the key is present (h was equal to zero for some value)
// ENOMEM
// On nonzero return, omt is unchanged.
// On nonzero non-DB_KEYEXIST return, *index is unchanged.
// Performance: time=O(\log N) amortized.
// Rationale: Some future implementation may be O(\log N) worst-case time, but O(\log N) amortized is good enough for now.
int toku_omt_delete_at(OMT omt, uint32_t idx);
// Effect: Delete the item in slot index.
// Decreases indexes of all items at slot >= index by 1.
// Returns
// 0 success
// EINVAL if index>=toku_omt_size(omt)
// On error, omt is unchanged.
// Rationale: To delete an item, first find its index using toku_omt_find, then delete it.
// Performance: time=O(\log N) amortized.
int toku_omt_fetch (OMT V, uint32_t i, OMTVALUE *v);
// Effect: Set *v=V_i
// If c!=NULL then set c's abstract offset to i.
// Requires: v != NULL
// Returns
// 0 success
// EINVAL if index>=toku_omt_size(omt)
// On nonzero return, *v is unchanged, and c (if nonnull) is either
// invalidated or unchanged.
// Performance: time=O(\log N)
// Implementation Notes: It is possible that c was previously valid and was
// associated with a different OMT. If c is changed by this
// function, the function must remove c's association with the old
// OMT, and associate it with the new OMT.
int toku_omt_find_zero(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, OMTVALUE *value, uint32_t *idx);
// Effect: Find the smallest i such that h(V_i, extra)>=0
// If there is such an i and h(V_i,extra)==0 then set *index=i and return 0.
// If there is such an i and h(V_i,extra)>0 then set *index=i and return DB_NOTFOUND.
// If there is no such i then set *index=toku_omt_size(V) and return DB_NOTFOUND.
// Requires: index!=NULL
int toku_omt_find(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, int direction, OMTVALUE *value, uint32_t *idx);
// Effect:
// If direction >0 then find the smallest i such that h(V_i,extra)>0.
// If direction <0 then find the largest i such that h(V_i,extra)<0.
// (Direction may not be equal to zero.)
// If value!=NULL then store V_i in *value
// If index!=NULL then store i in *index.
// Requires: The signum of h is monotically increasing.
// Returns
// 0 success
// DB_NOTFOUND no such value is found.
// On nonzero return, *value and *index are unchanged, and c (if nonnull) is either invalidated
// or unchanged.
// Performance: time=O(\log N)
// Rationale:
// Here's how to use the find function to find various things
// Cases for find:
// find first value: ( h(v)=+1, direction=+1 )
// find last value ( h(v)=-1, direction=-1 )
// find first X ( h(v)=(v< x) ? -1 : 1 direction=+1 )
// find last X ( h(v)=(v<=x) ? -1 : 1 direction=-1 )
// find X or successor to X ( same as find first X. )
//
// Rationale: To help understand heaviside functions and behavor of find:
// There are 7 kinds of heaviside functions.
// The signus of the h must be monotonically increasing.
// Given a function of the following form, A is the element
// returned for direction>0, B is the element returned
// for direction<0, C is the element returned for
// direction==0 (see find_zero) (with a return of 0), and D is the element
// returned for direction==0 (see find_zero) with a return of DB_NOTFOUND.
// If any of A, B, or C are not found, then asking for the
// associated direction will return DB_NOTFOUND.
// See find_zero for more information.
//
// Let the following represent the signus of the heaviside function.
//
// -...-
// A
// D
//
// +...+
// B
// D
//
// 0...0
// C
//
// -...-0...0
// AC
//
// 0...0+...+
// C B
//
// -...-+...+
// AB
// D
//
// -...-0...0+...+
// AC B
int toku_omt_split_at(OMT omt, OMT *newomt, uint32_t idx);
// Effect: Create a new OMT, storing it in *newomt.
// The values to the right of index (starting at index) are moved to *newomt.
// Requires: omt != NULL
// Requires: newomt != NULL
// Returns
// 0 success,
// EINVAL if index > toku_omt_size(omt)
// ENOMEM
// On nonzero return, omt and *newomt are unmodified.
// Performance: time=O(n)
// Rationale: We don't need a split-evenly operation. We need to split items so that their total sizes
// are even, and other similar splitting criteria. It's easy to split evenly by calling toku_omt_size(), and dividing by two.
int toku_omt_merge(OMT leftomt, OMT rightomt, OMT *newomt);
// Effect: Appends leftomt and rightomt to produce a new omt.
// Sets *newomt to the new omt.
// On success, leftomt and rightomt destroyed,.
// Returns 0 on success
// ENOMEM on out of memory.
// On error, nothing is modified.
// Performance: time=O(n) is acceptable, but one can imagine implementations that are O(\log n) worst-case.
int toku_omt_clone_noptr(OMT *dest, OMT src);
// Effect: Creates a copy of an omt.
// Sets *dest to the clone
// Each element is assumed to be stored directly in the omt, that is, the OMTVALUEs are not pointers, they are data. Thus no extra memory allocation is required.
// Returns 0 on success
// ENOMEM on out of memory.
// On error, nothing is modified.
// Performance: time between O(n) and O(n log n), depending how long it
// takes to traverse src.
void toku_omt_clear(OMT omt);
// Effect: Set the tree to be empty.
// Note: Will not reallocate or resize any memory, since returning void precludes calling malloc.
// Performance: time=O(1)
size_t toku_omt_memory_size (OMT omt);
// Effect: Return the size (in bytes) of the omt, as it resides in main memory. Don't include any of the OMTVALUES.
#endif /* #ifndef TOKU_OMT_H */

1
storage/tokudb/ft-index/ft/quicklz.cc
View File

@@ -311,6 +311,7 @@ static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui3
#endif
}
__attribute__((unused))
static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s)
{
#if QLZ_COMPRESSION_LEVEL == 1

6
storage/tokudb/ft-index/ft/rbuf.h
View File

@@ -93,12 +93,12 @@ PATENT RIGHTS GRANT:
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <toku_portability.h>
#include "memarena.h"
#include "toku_assert.h"
#include "fttypes.h"
#include "memory.h"
#include <toku_htonl.h>
#include <util/memarena.h>
struct rbuf {
unsigned char *buf;
@@ -250,7 +250,7 @@ static inline void rbuf_FILENUMS(struct rbuf *r, FILENUMS *filenums) {
// 2954
static inline void rbuf_ma_FILENUMS (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), FILENUMS *filenums) {
rbuf_ma_uint32_t(r, ma, &(filenums->num));
filenums->filenums = (FILENUM *) malloc_in_memarena(ma, filenums->num * sizeof(FILENUM) );
filenums->filenums = (FILENUM *) toku_memarena_malloc(ma, filenums->num * sizeof(FILENUM) );
assert(filenums->filenums != NULL);
for (uint32_t i=0; i < filenums->num; i++) {
rbuf_ma_FILENUM(r, ma, &(filenums->filenums[i]));
@@ -271,7 +271,7 @@ static inline void rbuf_ma_BYTESTRING (struct rbuf *r, MEMARENA ma, BYTESTRING
bs->len = rbuf_int(r);
uint32_t newndone = r->ndone + bs->len;
assert(newndone <= r->size);
bs->data = (char *) memarena_memdup(ma, &r->buf[r->ndone], (size_t)bs->len);
bs->data = (char *) toku_memarena_memdup(ma, &r->buf[r->ndone], (size_t)bs->len);
assert(bs->data);
r->ndone = newndone;
}

91
storage/tokudb/ft-index/ft/recover.cc
View File

@@ -98,6 +98,8 @@ PATENT RIGHTS GRANT:
#include "checkpoint.h"
#include "txn_manager.h"
#include <util/omt.h>
int tokudb_recovery_trace = 0; // turn on recovery tracing, default off.
//#define DO_VERIFY_COUNTS
@@ -154,9 +156,9 @@ struct file_map_tuple {
struct __toku_db fake_db;
};
static void file_map_tuple_init(struct file_map_tuple *tuple, FILENUM filenum, FT_HANDLE brt, char *iname) {
static void file_map_tuple_init(struct file_map_tuple *tuple, FILENUM filenum, FT_HANDLE ft_handle, char *iname) {
tuple->filenum = filenum;
tuple->ft_handle = brt;
tuple->ft_handle = ft_handle;
tuple->iname = iname;
// use a fake DB for comparisons, using the ft's cmp descriptor
memset(&tuple->fake_db, 0, sizeof(tuple->fake_db));
@@ -171,9 +173,9 @@ static void file_map_tuple_destroy(struct file_map_tuple *tuple) {
}
}
// Map filenum to brt
// Map filenum to ft_handle
struct file_map {
OMT filenums;
toku::omt<struct file_map_tuple *> *filenums;
};
// The recovery environment
@@ -199,31 +201,33 @@ typedef struct recover_env *RECOVER_ENV;
static void file_map_init(struct file_map *fmap) {
int r = toku_omt_create(&fmap->filenums);
assert(r == 0);
XMALLOC(fmap->filenums);
fmap->filenums->create();
}
static void file_map_destroy(struct file_map *fmap) {
toku_omt_destroy(&fmap->filenums);
fmap->filenums->destroy();
toku_free(fmap->filenums);
fmap->filenums = nullptr;
}
static uint32_t file_map_get_num_dictionaries(struct file_map *fmap) {
return toku_omt_size(fmap->filenums);
return fmap->filenums->size();
}
static void file_map_close_dictionaries(struct file_map *fmap, LSN oplsn) {
int r;
while (1) {
uint32_t n = toku_omt_size(fmap->filenums);
if (n == 0)
uint32_t n = fmap->filenums->size();
if (n == 0) {
break;
OMTVALUE v;
r = toku_omt_fetch(fmap->filenums, n-1, &v);
}
struct file_map_tuple *tuple;
r = fmap->filenums->fetch(n - 1, &tuple);
assert(r == 0);
r = toku_omt_delete_at(fmap->filenums, n-1);
r = fmap->filenums->delete_at(n - 1);
assert(r == 0);
struct file_map_tuple *CAST_FROM_VOIDP(tuple, v);
assert(tuple->ft_handle);
// Logging is on again, but we must pass the right LSN into close.
if (tuple->ft_handle) { // it's a DB, not a rollback file
@@ -234,27 +238,29 @@ static void file_map_close_dictionaries(struct file_map *fmap, LSN oplsn) {
}
}
static int file_map_h(OMTVALUE omtv, void *v) {
struct file_map_tuple *CAST_FROM_VOIDP(a, omtv);
FILENUM *CAST_FROM_VOIDP(b, v);
if (a->filenum.fileid < b->fileid) return -1;
if (a->filenum.fileid > b->fileid) return +1;
static int file_map_h(struct file_map_tuple *const &a, const FILENUM &b) {
if (a->filenum.fileid < b.fileid) {
return -1;
} else if (a->filenum.fileid > b.fileid) {
return 1;
} else {
return 0;
}
}
static int file_map_insert (struct file_map *fmap, FILENUM fnum, FT_HANDLE brt, char *iname) {
static int file_map_insert (struct file_map *fmap, FILENUM fnum, FT_HANDLE ft_handle, char *iname) {
struct file_map_tuple *XMALLOC(tuple);
file_map_tuple_init(tuple, fnum, brt, iname);
int r = toku_omt_insert(fmap->filenums, tuple, file_map_h, &fnum, NULL);
file_map_tuple_init(tuple, fnum, ft_handle, iname);
int r = fmap->filenums->insert<FILENUM, file_map_h>(tuple, fnum, nullptr);
return r;
}
static void file_map_remove(struct file_map *fmap, FILENUM fnum) {
OMTVALUE v; uint32_t idx;
int r = toku_omt_find_zero(fmap->filenums, file_map_h, &fnum, &v, &idx);
uint32_t idx;
struct file_map_tuple *tuple;
int r = fmap->filenums->find_zero<FILENUM, file_map_h>(fnum, &tuple, &idx);
if (r == 0) {
struct file_map_tuple *CAST_FROM_VOIDP(tuple, v);
r = toku_omt_delete_at(fmap->filenums, idx);
r = fmap->filenums->delete_at(idx);
file_map_tuple_destroy(tuple);
toku_free(tuple);
}
@@ -262,14 +268,15 @@ static void file_map_remove(struct file_map *fmap, FILENUM fnum) {
// Look up file info: given FILENUM, return file_map_tuple (or DB_NOTFOUND)
static int file_map_find(struct file_map *fmap, FILENUM fnum, struct file_map_tuple **file_map_tuple) {
OMTVALUE v; uint32_t idx;
int r = toku_omt_find_zero(fmap->filenums, file_map_h, &fnum, &v, &idx);
uint32_t idx;
struct file_map_tuple *tuple;
int r = fmap->filenums->find_zero<FILENUM, file_map_h>(fnum, &tuple, &idx);
if (r == 0) {
struct file_map_tuple *CAST_FROM_VOIDP(tuple, v);
assert(tuple->filenum.fileid == fnum.fileid);
*file_map_tuple = tuple;
} else {
assert(r == DB_NOTFOUND);
}
else assert(r==DB_NOTFOUND);
return r;
}
@@ -319,7 +326,7 @@ static int recover_env_init (RECOVER_ENV renv,
static void recover_env_cleanup (RECOVER_ENV renv) {
int r;
assert(toku_omt_size(renv->fmap.filenums)==0);
invariant_zero(renv->fmap.filenums->size());
file_map_destroy(&renv->fmap);
if (renv->destroy_logger_at_end) {
@@ -350,48 +357,48 @@ static const char *recover_state(RECOVER_ENV renv) {
static int internal_recover_fopen_or_fcreate (RECOVER_ENV renv, bool must_create, int UU(mode), BYTESTRING *bs_iname, FILENUM filenum, uint32_t treeflags,
TOKUTXN txn, uint32_t nodesize, uint32_t basementnodesize, enum toku_compression_method compression_method, LSN max_acceptable_lsn) {
int r = 0;
FT_HANDLE brt = NULL;
FT_HANDLE ft_handle = NULL;
char *iname = fixup_fname(bs_iname);
toku_ft_handle_create(&brt);
toku_ft_set_flags(brt, treeflags);
toku_ft_handle_create(&ft_handle);
toku_ft_set_flags(ft_handle, treeflags);
if (nodesize != 0) {
toku_ft_handle_set_nodesize(brt, nodesize);
toku_ft_handle_set_nodesize(ft_handle, nodesize);
}
if (basementnodesize != 0) {
toku_ft_handle_set_basementnodesize(brt, basementnodesize);
toku_ft_handle_set_basementnodesize(ft_handle, basementnodesize);
}
if (compression_method != TOKU_DEFAULT_COMPRESSION_METHOD) {
toku_ft_handle_set_compression_method(brt, compression_method);
toku_ft_handle_set_compression_method(ft_handle, compression_method);
}
// set the key compare functions
if (!(treeflags & TOKU_DB_KEYCMP_BUILTIN) && renv->bt_compare) {
toku_ft_set_bt_compare(brt, renv->bt_compare);
toku_ft_set_bt_compare(ft_handle, renv->bt_compare);
}
if (renv->update_function) {
toku_ft_set_update(brt, renv->update_function);
toku_ft_set_update(ft_handle, renv->update_function);
}
// TODO mode (FUTURE FEATURE)
//mode = mode;
r = toku_ft_handle_open_recovery(brt, iname, must_create, must_create, renv->ct, txn, filenum, max_acceptable_lsn);
r = toku_ft_handle_open_recovery(ft_handle, iname, must_create, must_create, renv->ct, txn, filenum, max_acceptable_lsn);
if (r != 0) {
//Note: If ft_handle_open fails, then close_ft will NOT write a header to disk.
//No need to provide lsn, so use the regular toku_ft_handle_close function
toku_ft_handle_close(brt);
toku_ft_handle_close(ft_handle);
toku_free(iname);
if (r == ENOENT) //Not an error to simply be missing.
r = 0;
return r;
}
file_map_insert(&renv->fmap, filenum, brt, iname);
file_map_insert(&renv->fmap, filenum, ft_handle, iname);
return 0;
}

4
storage/tokudb/ft-index/ft/recover.h
View File

@@ -96,10 +96,10 @@ PATENT RIGHTS GRANT:
#include <errno.h>
#include <db.h>
#include <util/x1764.h>
#include "fttypes.h"
#include "memory.h"
#include "x1764.h"
typedef void (*prepared_txn_callback_t)(DB_ENV*, TOKUTXN);
typedef void (*keep_cachetable_callback_t)(DB_ENV*, CACHETABLE);

4
storage/tokudb/ft-index/ft/roll.cc
View File

@@ -258,7 +258,7 @@ static int do_insertion (enum ft_msg_type type, FILENUM filenum, BYTESTRING key,
XIDS xids;
xids = toku_txn_get_xids(txn);
{
FT_MSG_S ftcmd = { type, ZERO_MSN, xids,
FT_MSG_S ftmsg = { type, ZERO_MSN, xids,
.u = { .id = { (key.len > 0)
? toku_fill_dbt(&key_dbt, key.data, key.len)
: toku_init_dbt(&key_dbt),
@@ -275,7 +275,7 @@ static int do_insertion (enum ft_msg_type type, FILENUM filenum, BYTESTRING key,
// no messages above us, we can implicitly promote uxrs based on this xid
oldest_referenced_xid_estimate,
!txn->for_recovery);
toku_ft_root_put_cmd(h, &ftcmd, &gc_info);
toku_ft_root_put_msg(h, &ftmsg, &gc_info);
if (reset_root_xid_that_created) {
TXNID new_root_xid_that_created = xids_get_outermost_xid(xids);
toku_reset_root_xid_that_created(h, new_root_xid_that_created);

6
storage/tokudb/ft-index/ft/rollback-apply.cc
View File

@@ -258,9 +258,9 @@ int toku_rollback_commit(TOKUTXN txn, LSN lsn) {
}
child_log->newest_logentry = child_log->oldest_logentry = 0;
// Put all the memarena data into the parent.
if (memarena_total_size_in_use(child_log->rollentry_arena) > 0) {
if (toku_memarena_total_size_in_use(child_log->rollentry_arena) > 0) {
// If there are no bytes to move, then just leave things alone, and let the memory be reclaimed on txn is closed.
memarena_move_buffers(parent_log->rollentry_arena, child_log->rollentry_arena);
toku_memarena_move_buffers(parent_log->rollentry_arena, child_log->rollentry_arena);
}
// each txn tries to give back at most one rollback log node
// to the cache. All other rollback log nodes for this child
@@ -281,7 +281,7 @@ int toku_rollback_commit(TOKUTXN txn, LSN lsn) {
toku_txn_unlock(txn->parent);
}
// Note the open brts, the omts must be merged
// Note the open FTs, the omts must be merged
r = txn->open_fts.iterate<struct tokutxn, note_ft_used_in_txns_parent>(txn);
assert(r==0);

6
storage/tokudb/ft-index/ft/rollback-ct-callbacks.cc
View File

@@ -89,15 +89,15 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include "rollback-ct-callbacks.h"
#include <toku_portability.h>
#include <memory.h>
#include "ft-internal.h"
#include "fttypes.h"
#include "memarena.h"
#include "rollback.h"
#include "rollback-ct-callbacks.h"
#include <util/memarena.h>
// Address used as a sentinel. Otherwise unused.
static struct serialized_rollback_log_node cloned_rollback;

10
storage/tokudb/ft-index/ft/rollback.cc
View File

@@ -121,7 +121,7 @@ toku_find_xid_by_xid (const TXNID &xid, const TXNID &xidfind) {
}
void *toku_malloc_in_rollback(ROLLBACK_LOG_NODE log, size_t size) {
return malloc_in_memarena(log->rollentry_arena, size);
return toku_memarena_malloc(log->rollentry_arena, size);
}
void *toku_memdup_in_rollback(ROLLBACK_LOG_NODE log, const void *v, size_t len) {
@@ -146,7 +146,7 @@ PAIR_ATTR
rollback_memory_size(ROLLBACK_LOG_NODE log) {
size_t size = sizeof(*log);
if (log->rollentry_arena) {
size += memarena_total_memory_size(log->rollentry_arena);
size += toku_memarena_total_footprint(log->rollentry_arena);
}
return make_rollback_pair_attr(size);
}
@@ -192,13 +192,13 @@ static void rollback_initialize_for_txn(
log->previous = previous;
log->oldest_logentry = NULL;
log->newest_logentry = NULL;
log->rollentry_arena = memarena_create();
log->rollentry_arena = toku_memarena_create();
log->rollentry_resident_bytecount = 0;
log->dirty = true;
}
void make_rollback_log_empty(ROLLBACK_LOG_NODE log) {
memarena_close(&log->rollentry_arena);
toku_memarena_destroy(&log->rollentry_arena);
rollback_empty_log_init(log);
}
@@ -267,7 +267,7 @@ int find_filenum (const FT &h, const FT &hfind) {
return 0;
}
//Notify a transaction that it has touched a brt.
//Notify a transaction that it has touched an ft.
void toku_txn_maybe_note_ft (TOKUTXN txn, FT ft) {
toku_txn_lock(txn);
FT ftv;

2
storage/tokudb/ft-index/ft/rollback_log_node_cache.cc
View File

@@ -90,7 +90,7 @@ PATENT RIGHTS GRANT:
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <memory.h>
#include <toku_include/toku_portability.h>
#include <portability/toku_portability.h>
#include "rollback_log_node_cache.h"

9
storage/tokudb/ft-index/ft/sub_block.cc
View File

@@ -89,16 +89,15 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include "sub_block.h"
#include "compress.h"
#include "sub_block.h"
#include "quicklz.h"
#include "x1764.h"
#include <memory.h>
#include <toku_assert.h>
#include <toku_portability.h>
#include <util/threadpool.h>
#include <util/x1764.h>
#include <stdio.h>
#include <string.h>
@@ -268,7 +267,7 @@ compress_sub_block(struct sub_block *sub_block, enum toku_compression_method met
method
);
// checksum it
sub_block->xsum = x1764_memory(sub_block->compressed_ptr, sub_block->compressed_size);
sub_block->xsum = toku_x1764_memory(sub_block->compressed_ptr, sub_block->compressed_size);
}
void *
@@ -365,7 +364,7 @@ decompress_sub_block(void *compress_ptr, uint32_t compress_size, void *uncompres
int result = 0;
// verify checksum
uint32_t xsum = x1764_memory(compress_ptr, compress_size);
uint32_t xsum = toku_x1764_memory(compress_ptr, compress_size);
if (xsum != expected_xsum) {
if (verbose_decompress_sub_block) fprintf(stderr, "%s:%d xsum %u expected %u\n", __FUNCTION__, __LINE__, xsum, expected_xsum);
result = EINVAL;

4
storage/tokudb/ft-index/ft/tests/cachetable-checkpoint-pending.cc
View File

@@ -108,13 +108,9 @@ static volatile int n_flush, n_write_me, n_keep_me, n_fetch;
static void
sleep_random (void)
{
#if TOKU_WINDOWS
usleep(random() % 1000); //Will turn out to be almost always 1ms.
#else
toku_timespec_t req = {.tv_sec = 0,
.tv_nsec = random()%1000000}; //Max just under 1ms
nanosleep(&req, NULL);
#endif
}
int expect_value = 42; // initially 42, later 43

3
storage/tokudb/ft-index/ft/tests/cachetable-checkpointer-class.cc
View File

@@ -165,6 +165,7 @@ void checkpointer_test::test_begin_checkpoint() {
}
ctbl.list.destroy();
m_cp.destroy();
cfl.destroy();
}
//------------------------------------------------------------------------------
@@ -257,6 +258,7 @@ void checkpointer_test::test_pending_bits() {
ctbl.list.destroy();
m_cp.destroy();
cfl.destroy();
}
//------------------------------------------------------------------------------
@@ -394,6 +396,7 @@ void checkpointer_test::test_end_checkpoint() {
}
m_cp.destroy();
ctbl.list.destroy();
cfl.destroy();
}

8
storage/tokudb/ft-index/ft/tests/cachetable-test.cc
View File

@@ -241,8 +241,6 @@ PAIR_ATTR *sizep __attribute__((__unused__)), int * dirtyp, void*extraargs) {
return 0;
}
#if !TOKU_WINDOWS
static void test_multi_filehandles (void) {
CACHETABLE t;
CACHEFILE f1,f2,f3;
@@ -291,8 +289,6 @@ static void test_multi_filehandles (void) {
toku_cachetable_close(&t);
}
#endif
static void test_dirty_flush(CACHEFILE f,
int UU(fd),
CACHEKEY key,
@@ -587,15 +583,11 @@ test_main (int argc, const char *argv[]) {
test_mutex_init();
// run tests
#if !TOKU_WINDOWS
test_multi_filehandles();
#endif
test_cachetable_create();
for (i=0; i<1; i++) {
test_nested_pin();
#if !TOKU_WINDOWS
test_multi_filehandles ();
#endif
test_dirty();
test_size_resize();
//test_size_flush();

467
storage/tokudb/ft-index/ft/tests/omt-test.cc → storage/tokudb/ft-index/ft/tests/dmt-test.cc
View File

@@ -90,10 +90,103 @@ PATENT RIGHTS GRANT:
#include "test.h"
#include "omt.h"
#include <util/omt.h>
#include <util/dmt.h>
typedef OMTVALUE TESTVALUE;
typedef void *DMTVALUE;
class dmtvalue_writer {
public:
size_t get_size(void) const {
return sizeof(DMTVALUE);
}
void write_to(DMTVALUE *const dest) const {
*dest = value;
}
dmtvalue_writer(DMTVALUE _value)
: value(_value) {
}
dmtvalue_writer(const uint32_t size UU(), DMTVALUE *const src)
: value(*src) {
paranoid_invariant(size == sizeof(DMTVALUE));
}
private:
const DMTVALUE value;
};
typedef toku::dmt<DMTVALUE, DMTVALUE, dmtvalue_writer> *DMT;
static int dmt_insert_at(DMT dmt, DMTVALUE value, uint32_t index) {
dmtvalue_writer functor(value);
return dmt->insert_at(functor, index);
}
static DMT dmt_create_from_sorted_array(DMTVALUE *values, uint32_t numvalues) {
DMT XMALLOC(dmt);
dmt->create();
for (uint32_t i = 0; i < numvalues; i++) {
dmt_insert_at(dmt, values[i], i);
}
return dmt;
}
struct heftor {
int (*h)(DMTVALUE, void *v);
void *v;
};
int call_heftor(const uint32_t size, const DMTVALUE &v, const heftor &htor);
int call_heftor(const uint32_t size, const DMTVALUE &v, const heftor &htor) {
invariant(size == sizeof(DMTVALUE));
return htor.h(const_cast<DMTVALUE>(v), htor.v);
}
static int dmt_insert(DMT dmt, DMTVALUE value, int(*h)(DMTVALUE, void*v), void *v, uint32_t *index) {
struct heftor htor = { .h = h, .v = v };
dmtvalue_writer functor(value);
return dmt->insert<heftor, call_heftor>(functor, htor, index);
}
static int dmt_find_zero(DMT V, int (*h)(DMTVALUE, void*extra), void*extra, DMTVALUE *value, uint32_t *index) {
struct heftor htor = { .h = h, .v = extra };
uint32_t ignore;
return V->find_zero<heftor, call_heftor>(htor, &ignore, value, index);
}
static int dmt_find(DMT V, int (*h)(DMTVALUE, void*extra), void*extra, int direction, DMTVALUE *value, uint32_t *index) {
struct heftor htor = { .h = h, .v = extra };
uint32_t ignore;
return V->find<heftor, call_heftor>(htor, direction, &ignore, value, index);
}
static int dmt_split_at(DMT dmt, DMT *newdmtp, uint32_t index) {
if (index > dmt->size()) { return EINVAL; }
DMT XMALLOC(newdmt);
newdmt->create();
int r;
for (uint32_t i = index; i < dmt->size(); i++) {
DMTVALUE v;
r = dmt->fetch(i, nullptr, &v);
invariant_zero(r);
r = dmt_insert_at(newdmt, v, i-index);
invariant_zero(r);
}
if (dmt->size() > 0) {
for (uint32_t i = dmt->size(); i > index; i--) {
r = dmt->delete_at(i - 1);
invariant_zero(r);
}
}
r = 0;
if (r != 0) {
toku_free(newdmt);
} else {
*newdmtp = newdmt;
}
return r;
}
static void
parse_args (int argc, const char *argv[]) {
@@ -133,26 +226,25 @@ enum close_when_done {
KEEP_WHEN_DONE
};
enum create_type {
STEAL_ARRAY,
BATCH_INSERT,
INSERT_AT,
INSERT_AT_ALMOST_RANDOM,
};
/* Globals */
OMT global_omt;
TESTVALUE* values = NULL;
struct value* nums = NULL;
DMT global_dmt;
DMTVALUE* values = nullptr;
struct value* nums = nullptr;
uint32_t length;
static void
cleanup_globals (void) {
assert(values);
toku_free(values);
values = NULL;
values = nullptr;
assert(nums);
toku_free(nums);
nums = NULL;
nums = nullptr;
}
const unsigned int random_seed = 0xFEADACBA;
@@ -178,7 +270,7 @@ init_identity_values (unsigned int seed, uint32_t num_elements) {
for (i = 0; i < length; i++) {
nums[i].number = i;
values[i] = (TESTVALUE)&nums[i];
values[i] = (DMTVALUE)&nums[i];
}
}
@@ -193,7 +285,7 @@ init_distinct_sorted_values (unsigned int seed, uint32_t num_elements) {
for (i = 0; i < length; i++) {
number += (uint32_t)(random() % 32) + 1;
nums[i].number = number;
values[i] = (TESTVALUE)&nums[i];
values[i] = (DMTVALUE)&nums[i];
}
}
@@ -229,25 +321,22 @@ static void
test_close (enum close_when_done do_close) {
if (do_close == KEEP_WHEN_DONE) return;
assert(do_close == CLOSE_WHEN_DONE);
toku_omt_destroy(&global_omt);
assert(global_omt==NULL);
global_dmt->destroy();
toku_free(global_dmt);
global_dmt = nullptr;
}
static void
test_create (enum close_when_done do_close) {
int r;
global_omt = NULL;
r = toku_omt_create(&global_omt);
CKERR(r);
assert(global_omt!=NULL);
XMALLOC(global_dmt);
global_dmt->create();
test_close(do_close);
}
static void
test_create_size (enum close_when_done do_close) {
test_create(KEEP_WHEN_DONE);
assert(toku_omt_size(global_omt) == 0);
assert(global_dmt->size() == 0);
test_close(do_close);
}
@@ -258,24 +347,24 @@ test_create_insert_at_almost_random (enum close_when_done do_close) {
uint32_t size = 0;
test_create(KEEP_WHEN_DONE);
r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+1);
r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+1);
CKERR2(r, EINVAL);
r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+2);
r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+2);
CKERR2(r, EINVAL);
for (i = 0; i < length/2; i++) {
assert(size==toku_omt_size(global_omt));
r = toku_omt_insert_at(global_omt, values[i], i);
assert(size==global_dmt->size());
r = dmt_insert_at(global_dmt, values[i], i);
CKERR(r);
assert(++size==toku_omt_size(global_omt));
r = toku_omt_insert_at(global_omt, values[length-1-i], i+1);
assert(++size==global_dmt->size());
r = dmt_insert_at(global_dmt, values[length-1-i], i+1);
CKERR(r);
assert(++size==toku_omt_size(global_omt));
assert(++size==global_dmt->size());
}
r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+1);
r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+1);
CKERR2(r, EINVAL);
r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+2);
r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+2);
CKERR2(r, EINVAL);
assert(size==toku_omt_size(global_omt));
assert(size==global_dmt->size());
test_close(do_close);
}
@@ -286,39 +375,30 @@ test_create_insert_at_sequential (enum close_when_done do_close) {
uint32_t size = 0;
test_create(KEEP_WHEN_DONE);
r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+1);
r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+1);
CKERR2(r, EINVAL);
r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+2);
r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+2);
CKERR2(r, EINVAL);
for (i = 0; i < length; i++) {
assert(size==toku_omt_size(global_omt));
r = toku_omt_insert_at(global_omt, values[i], i);
assert(size==global_dmt->size());
r = dmt_insert_at(global_dmt, values[i], i);
CKERR(r);
assert(++size==toku_omt_size(global_omt));
assert(++size==global_dmt->size());
}
r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+1);
r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+1);
CKERR2(r, EINVAL);
r = toku_omt_insert_at(global_omt, values[0], toku_omt_size(global_omt)+2);
r = dmt_insert_at(global_dmt, values[0], global_dmt->size()+2);
CKERR2(r, EINVAL);
assert(size==toku_omt_size(global_omt));
assert(size==global_dmt->size());
test_close(do_close);
}
static void
test_create_from_sorted_array (enum create_type create_choice, enum close_when_done do_close) {
int r;
global_omt = NULL;
global_dmt = nullptr;
if (create_choice == BATCH_INSERT) {
r = toku_omt_create_from_sorted_array(&global_omt, values, length);
CKERR(r);
}
else if (create_choice == STEAL_ARRAY) {
TESTVALUE* MALLOC_N(length, values_copy);
memcpy(values_copy, values, length*sizeof(*values));
r = toku_omt_create_steal_sorted_array(&global_omt, &values_copy, length, length);
CKERR(r);
assert(values_copy==NULL);
global_dmt = dmt_create_from_sorted_array(values, length);
}
else if (create_choice == INSERT_AT) {
test_create_insert_at_sequential(KEEP_WHEN_DONE);
@@ -326,33 +406,35 @@ test_create_from_sorted_array (enum create_type create_choice, enum close_when_d
else if (create_choice == INSERT_AT_ALMOST_RANDOM) {
test_create_insert_at_almost_random(KEEP_WHEN_DONE);
}
else assert(false);
else {
assert(false);
}
assert(global_omt!=NULL);
assert(global_dmt!=nullptr);
test_close(do_close);
}
static void
test_create_from_sorted_array_size (enum create_type create_choice, enum close_when_done do_close) {
test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
assert(toku_omt_size(global_omt)==length);
assert(global_dmt->size()==length);
test_close(do_close);
}
static void
test_fetch_verify (OMT omtree, TESTVALUE* val, uint32_t len ) {
test_fetch_verify (DMT dmtree, DMTVALUE* val, uint32_t len ) {
uint32_t i;
int r;
TESTVALUE v = (TESTVALUE)&i;
TESTVALUE oldv = v;
DMTVALUE v = (DMTVALUE)&i;
DMTVALUE oldv = v;
assert(len == toku_omt_size(omtree));
assert(len == dmtree->size());
for (i = 0; i < len; i++) {
assert(oldv!=val[i]);
v = NULL;
r = toku_omt_fetch(omtree, i, &v);
v = nullptr;
r = dmtree->fetch(i, nullptr, &v);
CKERR(r);
assert(v != NULL);
assert(v != nullptr);
assert(v != oldv);
assert(v == val[i]);
assert(V(v)->number == V(val[i])->number);
@@ -361,7 +443,7 @@ test_fetch_verify (OMT omtree, TESTVALUE* val, uint32_t len ) {
for (i = len; i < len*2; i++) {
v = oldv;
r = toku_omt_fetch(omtree, i, &v);
r = dmtree->fetch(i, nullptr, &v);
CKERR2(r, EINVAL);
assert(v == oldv);
}
@@ -371,30 +453,45 @@ test_fetch_verify (OMT omtree, TESTVALUE* val, uint32_t len ) {
static void
test_create_fetch_verify (enum create_type create_choice, enum close_when_done do_close) {
test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
test_fetch_verify(global_omt, values, length);
test_fetch_verify(global_dmt, values, length);
test_close(do_close);
}
static int iterate_helper_error_return = 1;
static int
iterate_helper (TESTVALUE v, uint32_t idx, void* extra) {
if (extra == NULL) return iterate_helper_error_return;
TESTVALUE* vals = (TESTVALUE *)extra;
assert(v != NULL);
iterate_helper (DMTVALUE v, uint32_t idx, void* extra) {
if (extra == nullptr) return iterate_helper_error_return;
DMTVALUE* vals = (DMTVALUE *)extra;
assert(v != nullptr);
assert(v == vals[idx]);
assert(V(v)->number == V(vals[idx])->number);
return 0;
}
struct functor {
int (*f)(DMTVALUE, uint32_t, void *);
void *v;
};
int call_functor(const uint32_t size, const DMTVALUE &v, uint32_t idx, functor *const ftor);
int call_functor(const uint32_t size, const DMTVALUE &v, uint32_t idx, functor *const ftor) {
invariant(size == sizeof(DMTVALUE));
return ftor->f(const_cast<DMTVALUE>(v), idx, ftor->v);
}
static int dmt_iterate(DMT dmt, int (*f)(DMTVALUE, uint32_t, void*), void*v) {
struct functor ftor = { .f = f, .v = v };
return dmt->iterate<functor, call_functor>(&ftor);
}
static void
test_iterate_verify (OMT omtree, TESTVALUE* vals, uint32_t len) {
test_iterate_verify (DMT dmtree, DMTVALUE* vals, uint32_t len) {
int r;
iterate_helper_error_return = 0;
r = toku_omt_iterate(omtree, iterate_helper, (void*)vals);
r = dmt_iterate(dmtree, iterate_helper, (void*)vals);
CKERR(r);
iterate_helper_error_return = 0xFEEDABBA;
r = toku_omt_iterate(omtree, iterate_helper, NULL);
r = dmt_iterate(dmtree, iterate_helper, nullptr);
if (!len) {
CKERR2(r, 0);
}
@@ -406,7 +503,7 @@ test_iterate_verify (OMT omtree, TESTVALUE* vals, uint32_t len) {
static void
test_create_iterate_verify (enum create_type create_choice, enum close_when_done do_close) {
test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
test_iterate_verify(global_omt, values, length);
test_iterate_verify(global_dmt, values, length);
test_close(do_close);
}
@@ -431,19 +528,26 @@ permute_array (uint32_t* arr, uint32_t len) {
}
}
static int
dmt_set_at (DMT dmt, DMTVALUE value, uint32_t index) {
int r = dmt->delete_at(index);
if (r!=0) return r;
return dmt_insert_at(dmt, value, index);
}
static void
test_create_set_at (enum create_type create_choice, enum close_when_done do_close) {
uint32_t i = 0;
struct value* old_nums = NULL;
struct value* old_nums = nullptr;
MALLOC_N(length, old_nums);
assert(nums);
uint32_t* perm = NULL;
uint32_t* perm = nullptr;
MALLOC_N(length, perm);
assert(perm);
TESTVALUE* old_values = NULL;
DMTVALUE* old_values = nullptr;
MALLOC_N(length, old_values);
assert(old_values);
@@ -459,22 +563,22 @@ test_create_set_at (enum create_type create_choice, enum close_when_done do_clos
}
test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
int r;
r = toku_omt_set_at (global_omt, values[0], length);
r = dmt_set_at (global_dmt, values[0], length);
CKERR2(r,EINVAL);
r = toku_omt_set_at (global_omt, values[0], length+1);
r = dmt_set_at (global_dmt, values[0], length+1);
CKERR2(r,EINVAL);
for (i = 0; i < length; i++) {
uint32_t choice = perm[i];
values[choice] = &nums[choice];
nums[choice].number = (uint32_t)random();
r = toku_omt_set_at (global_omt, values[choice], choice);
r = dmt_set_at (global_dmt, values[choice], choice);
CKERR(r);
test_iterate_verify(global_omt, values, length);
test_fetch_verify(global_omt, values, length);
test_iterate_verify(global_dmt, values, length);
test_fetch_verify(global_dmt, values, length);
}
r = toku_omt_set_at (global_omt, values[0], length);
r = dmt_set_at (global_dmt, values[0], length);
CKERR2(r,EINVAL);
r = toku_omt_set_at (global_omt, values[0], length+1);
r = dmt_set_at (global_dmt, values[0], length+1);
CKERR2(r,EINVAL);
toku_free(perm);
@@ -485,8 +589,8 @@ test_create_set_at (enum create_type create_choice, enum close_when_done do_clos
}
static int
insert_helper (TESTVALUE value, void* extra_insert) {
TESTVALUE to_insert = (OMTVALUE)extra_insert;
insert_helper (DMTVALUE value, void* extra_insert) {
DMTVALUE to_insert = (DMTVALUE)extra_insert;
assert(to_insert);
if (V(value)->number < V(to_insert)->number) return -1;
@@ -498,7 +602,7 @@ static void
test_create_insert (enum close_when_done do_close) {
uint32_t i = 0;
uint32_t* perm = NULL;
uint32_t* perm = nullptr;
MALLOC_N(length, perm);
assert(perm);
@@ -510,11 +614,11 @@ test_create_insert (enum close_when_done do_close) {
length = 0;
while (length < size) {
uint32_t choice = perm[length];
TESTVALUE to_insert = &nums[choice];
DMTVALUE to_insert = &nums[choice];
uint32_t idx = UINT32_MAX;
assert(length==toku_omt_size(global_omt));
r = toku_omt_insert(global_omt, to_insert, insert_helper, to_insert, &idx);
assert(length==global_dmt->size());
r = dmt_insert(global_dmt, to_insert, insert_helper, to_insert, &idx);
CKERR(r);
assert(idx <= length);
if (idx > 0) {
@@ -524,24 +628,24 @@ test_create_insert (enum close_when_done do_close) {
assert(V(to_insert)->number < V(values[idx])->number);
}
length++;
assert(length==toku_omt_size(global_omt));
assert(length==global_dmt->size());
/* Make room */
for (i = length-1; i > idx; i--) {
values[i] = values[i-1];
}
values[idx] = to_insert;
test_fetch_verify(global_omt, values, length);
test_iterate_verify(global_omt, values, length);
test_fetch_verify(global_dmt, values, length);
test_iterate_verify(global_dmt, values, length);
idx = UINT32_MAX;
r = toku_omt_insert(global_omt, to_insert, insert_helper, to_insert, &idx);
r = dmt_insert(global_dmt, to_insert, insert_helper, to_insert, &idx);
CKERR2(r, DB_KEYEXIST);
assert(idx < length);
assert(V(values[idx])->number == V(to_insert)->number);
assert(length==toku_omt_size(global_omt));
assert(length==global_dmt->size());
test_iterate_verify(global_omt, values, length);
test_fetch_verify(global_omt, values, length);
test_iterate_verify(global_dmt, values, length);
test_fetch_verify(global_dmt, values, length);
}
toku_free(perm);
@@ -555,91 +659,118 @@ test_create_delete_at (enum create_type create_choice, enum close_when_done do_c
int r = ENOSYS;
test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
assert(length == toku_omt_size(global_omt));
r = toku_omt_delete_at(global_omt, length);
assert(length == global_dmt->size());
r = global_dmt->delete_at(length);
CKERR2(r,EINVAL);
assert(length == toku_omt_size(global_omt));
r = toku_omt_delete_at(global_omt, length+1);
assert(length == global_dmt->size());
r = global_dmt->delete_at(length+1);
CKERR2(r,EINVAL);
while (length > 0) {
assert(length == toku_omt_size(global_omt));
assert(length == global_dmt->size());
uint32_t index_to_delete = random()%length;
r = toku_omt_delete_at(global_omt, index_to_delete);
r = global_dmt->delete_at(index_to_delete);
CKERR(r);
for (i = index_to_delete+1; i < length; i++) {
values[i-1] = values[i];
}
length--;
test_fetch_verify(global_omt, values, length);
test_iterate_verify(global_omt, values, length);
test_fetch_verify(global_dmt, values, length);
test_iterate_verify(global_dmt, values, length);
}
assert(length == 0);
assert(length == toku_omt_size(global_omt));
r = toku_omt_delete_at(global_omt, length);
assert(length == global_dmt->size());
r = global_dmt->delete_at(length);
CKERR2(r, EINVAL);
assert(length == toku_omt_size(global_omt));
r = toku_omt_delete_at(global_omt, length+1);
assert(length == global_dmt->size());
r = global_dmt->delete_at(length+1);
CKERR2(r, EINVAL);
test_close(do_close);
}
static int dmt_merge(DMT leftdmt, DMT rightdmt, DMT *newdmtp) {
DMT XMALLOC(newdmt);
newdmt->create();
int r;
for (uint32_t i = 0; i < leftdmt->size(); i++) {
DMTVALUE v;
r = leftdmt->fetch(i, nullptr, &v);
invariant_zero(r);
r = newdmt->insert_at(v, i);
invariant_zero(r);
}
uint32_t offset = leftdmt->size();
for (uint32_t i = 0; i < rightdmt->size(); i++) {
DMTVALUE v;
r = rightdmt->fetch(i, nullptr, &v);
invariant_zero(r);
r = newdmt->insert_at(v, i+offset);
invariant_zero(r);
}
leftdmt->destroy();
rightdmt->destroy();
toku_free(leftdmt);
toku_free(rightdmt);
*newdmtp = newdmt;
return 0;
}
static void
test_split_merge (enum create_type create_choice, enum close_when_done do_close) {
int r = ENOSYS;
uint32_t i = 0;
OMT left_split = NULL;
OMT right_split = NULL;
DMT left_split = nullptr;
DMT right_split = nullptr;
test_create_from_sorted_array(create_choice, KEEP_WHEN_DONE);
for (i = 0; i <= length; i++) {
r = toku_omt_split_at(global_omt, &right_split, length+1);
r = dmt_split_at(global_dmt, &right_split, length+1);
CKERR2(r,EINVAL);
r = toku_omt_split_at(global_omt, &right_split, length+2);
r = dmt_split_at(global_dmt, &right_split, length+2);
CKERR2(r,EINVAL);
//
// test successful split
//
r = toku_omt_split_at(global_omt, &right_split, i);
r = dmt_split_at(global_dmt, &right_split, i);
CKERR(r);
left_split = global_omt;
global_omt = NULL;
assert(toku_omt_size(left_split) == i);
assert(toku_omt_size(right_split) == length - i);
left_split = global_dmt;
global_dmt = nullptr;
assert(left_split->size() == i);
assert(right_split->size() == length - i);
test_fetch_verify(left_split, values, i);
test_iterate_verify(left_split, values, i);
test_fetch_verify(right_split, &values[i], length - i);
test_iterate_verify(right_split, &values[i], length - i);
//
// verify that new global_omt's cannot do bad splits
// verify that new global_dmt's cannot do bad splits
//
r = toku_omt_split_at(left_split, &global_omt, i+1);
r = dmt_split_at(left_split, &global_dmt, i+1);
CKERR2(r,EINVAL);
assert(toku_omt_size(left_split) == i);
assert(toku_omt_size(right_split) == length - i);
r = toku_omt_split_at(left_split, &global_omt, i+2);
assert(left_split->size() == i);
assert(right_split->size() == length - i);
r = dmt_split_at(left_split, &global_dmt, i+2);
CKERR2(r,EINVAL);
assert(toku_omt_size(left_split) == i);
assert(toku_omt_size(right_split) == length - i);
r = toku_omt_split_at(right_split, &global_omt, length - i + 1);
assert(left_split->size() == i);
assert(right_split->size() == length - i);
r = dmt_split_at(right_split, &global_dmt, length - i + 1);
CKERR2(r,EINVAL);
assert(toku_omt_size(left_split) == i);
assert(toku_omt_size(right_split) == length - i);
r = toku_omt_split_at(right_split, &global_omt, length - i + 1);
assert(left_split->size() == i);
assert(right_split->size() == length - i);
r = dmt_split_at(right_split, &global_dmt, length - i + 1);
CKERR2(r,EINVAL);
assert(toku_omt_size(left_split) == i);
assert(toku_omt_size(right_split) == length - i);
assert(left_split->size() == i);
assert(right_split->size() == length - i);
//
// test merge
//
r = toku_omt_merge(left_split,right_split,&global_omt);
r = dmt_merge(left_split,right_split,&global_dmt);
CKERR(r);
left_split = NULL;
right_split = NULL;
assert(toku_omt_size(global_omt) == length);
test_fetch_verify(global_omt, values, length);
test_iterate_verify(global_omt, values, length);
left_split = nullptr;
right_split = nullptr;
assert(global_dmt->size() == length);
test_fetch_verify(global_dmt, values, length);
test_iterate_verify(global_dmt, values, length);
}
test_close(do_close);
}
@@ -693,8 +824,8 @@ typedef struct {
static int
test_heaviside (OMTVALUE v_omt, void* x) {
TESTVALUE v = (OMTVALUE) v_omt;
test_heaviside (DMTVALUE v_dmt, void* x) {
DMTVALUE v = (DMTVALUE) v_dmt;
h_extra* extra = (h_extra*)x;
assert(v && x);
assert(extra->first_zero <= extra->first_pos);
@@ -712,24 +843,24 @@ heavy_extra (h_extra* extra, uint32_t first_zero, uint32_t first_pos) {
}
static void
test_find_dir (int dir, void* extra, int (*h)(OMTVALUE, void*),
test_find_dir (int dir, void* extra, int (*h)(DMTVALUE, void*),
int r_expect, bool idx_will_change, uint32_t idx_expect,
uint32_t number_expect, bool UU(cursor_valid)) {
uint32_t idx = UINT32_MAX;
uint32_t old_idx = idx;
TESTVALUE omt_val;
DMTVALUE dmt_val;
int r;
omt_val = NULL;
dmt_val = nullptr;
/* Verify we can pass NULL value. */
omt_val = NULL;
/* Verify we can pass nullptr value. */
dmt_val = nullptr;
idx = old_idx;
if (dir == 0) {
r = toku_omt_find_zero(global_omt, h, extra, NULL, &idx);
r = dmt_find_zero(global_dmt, h, extra, nullptr, &idx);
}
else {
r = toku_omt_find( global_omt, h, extra, dir, NULL, &idx);
r = dmt_find( global_dmt, h, extra, dir, nullptr, &idx);
}
CKERR2(r, r_expect);
if (idx_will_change) {
@@ -738,38 +869,38 @@ test_find_dir (int dir, void* extra, int (*h)(OMTVALUE, void*),
else {
assert(idx == old_idx);
}
assert(omt_val == NULL);
assert(dmt_val == nullptr);
/* Verify we can pass NULL idx. */
omt_val = NULL;
/* Verify we can pass nullptr idx. */
dmt_val = nullptr;
idx = old_idx;
if (dir == 0) {
r = toku_omt_find_zero(global_omt, h, extra, &omt_val, 0);
r = dmt_find_zero(global_dmt, h, extra, &dmt_val, 0);
}
else {
r = toku_omt_find( global_omt, h, extra, dir, &omt_val, 0);
r = dmt_find( global_dmt, h, extra, dir, &dmt_val, 0);
}
CKERR2(r, r_expect);
assert(idx == old_idx);
if (r == DB_NOTFOUND) {
assert(omt_val == NULL);
assert(dmt_val == nullptr);
}
else {
assert(V(omt_val)->number == number_expect);
assert(V(dmt_val)->number == number_expect);
}
/* Verify we can pass NULL both. */
omt_val = NULL;
/* Verify we can pass nullptr both. */
dmt_val = nullptr;
idx = old_idx;
if (dir == 0) {
r = toku_omt_find_zero(global_omt, h, extra, NULL, 0);
r = dmt_find_zero(global_dmt, h, extra, nullptr, 0);
}
else {
r = toku_omt_find( global_omt, h, extra, dir, NULL, 0);
r = dmt_find( global_dmt, h, extra, dir, nullptr, 0);
}
CKERR2(r, r_expect);
assert(idx == old_idx);
assert(omt_val == NULL);
assert(dmt_val == nullptr);
}
static void
@@ -857,33 +988,34 @@ runtests_create_choice (enum create_type create_choice) {
static void
test_clone(uint32_t nelts)
// Test that each clone operation gives the right data back. If nelts is
// zero, also tests that you still get a valid OMT back and that the way
// zero, also tests that you still get a valid DMT back and that the way
// to deallocate it still works.
{
OMT src = NULL, dest = NULL;
int r;
DMT src = nullptr, dest = nullptr;
int r = 0;
r = toku_omt_create(&src);
assert_zero(r);
XMALLOC(src);
src->create();
for (long i = 0; i < nelts; ++i) {
r = toku_omt_insert_at(src, (OMTVALUE) i, i);
r = dmt_insert_at(src, (DMTVALUE) i, i);
assert_zero(r);
}
r = toku_omt_clone_noptr(&dest, src);
assert_zero(r);
assert(dest != NULL);
assert(toku_omt_size(dest) == nelts);
XMALLOC(dest);
dest->clone(*src);
assert(dest->size() == nelts);
for (long i = 0; i < nelts; ++i) {
OMTVALUE v;
DMTVALUE v;
long l;
r = toku_omt_fetch(dest, i, &v);
r = dest->fetch(i, nullptr, &v);
assert_zero(r);
l = (long) v;
assert(l == i);
}
toku_omt_destroy(&dest);
toku_omt_destroy(&src);
dest->destroy();
toku_free(dest);
src->destroy();
toku_free(src);
}
int
@@ -893,7 +1025,6 @@ test_main(int argc, const char *argv[]) {
test_create( CLOSE_WHEN_DONE);
test_create_size( CLOSE_WHEN_DONE);
runtests_create_choice(BATCH_INSERT);
runtests_create_choice(STEAL_ARRAY);
runtests_create_choice(INSERT_AT);
runtests_create_choice(INSERT_AT_ALMOST_RANDOM);
test_clone(0);

373
storage/tokudb/ft-index/ft/tests/dmt-test2.cc Normal file
View File

@@ -0,0 +1,373 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#include "test.h"
#include <util/dmt.h>
static void
parse_args (int argc, const char *argv[]) {
const char *argv0=argv[0];
while (argc>1) {
int resultcode=0;
if (strcmp(argv[1], "-v")==0) {
verbose++;
} else if (strcmp(argv[1], "-q")==0) {
verbose = 0;
} else if (strcmp(argv[1], "-h")==0) {
do_usage:
fprintf(stderr, "Usage:\n%s [-v|-h]\n", argv0);
exit(resultcode);
} else {
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
/* End ".h like" stuff. */
struct value {
uint32_t number;
};
#define V(x) ((struct value *)(x))
const uint32_t MAXNUM = 1024;
const uint32_t MAXLEN = 32;
char data[MAXNUM][MAXLEN];
struct val_type {
char c[MAXLEN];
};
namespace toku {
class vwriter {
public:
size_t get_size(void) const {
size_t len = strlen(v.c);
invariant(len < sizeof(val_type));
return len + 1;
}
void write_to(val_type *const dest) const {
strcpy(dest->c, v.c);
}
vwriter(const char* c) {
invariant(strlen(c) < sizeof(val_type));
strcpy(v.c, c);
}
vwriter(const uint32_t klpair_len, val_type *const src) {
invariant(strlen(src->c) < sizeof(val_type));
strcpy(v.c, src->c);
invariant(klpair_len == get_size());
}
private:
val_type v;
};
}
/* Globals */
typedef toku::dmt<val_type, val_type*, toku::vwriter> vdmt;
const unsigned int random_seed = 0xFEADACBA;
///////////////
static void fail_one_verify(uint32_t len, uint32_t num, vdmt *v) {
val_type* fetched_data;
int count = 0;
v->verify();
for (uint32_t i = 0; i < num; i++) {
uint32_t fetched_len;
int r = v->fetch(i-count, &fetched_len, &fetched_data);
if (r != 0 || fetched_len != len || strcmp(fetched_data->c, data[i])) {
count++;
continue;
}
}
invariant(count == 1);
}
static void verify(uint32_t len, uint32_t num, vdmt *v) {
v->verify();
val_type* fetched_data;
for (uint32_t i = 0; i < num; i++) {
uint32_t fetched_len;
int r = v->fetch(i, &fetched_len, &fetched_data);
CKERR(r);
invariant(fetched_len == len);
invariant(!strcmp(fetched_data->c, data[i]));
}
}
static void test_builder_fixed(uint32_t len, uint32_t num) {
srandom(random_seed);
assert(len > 1);
assert(len <= MAXLEN);
assert(num <= MAXNUM);
for (uint32_t i = 0; i < num; i++) {
for (uint32_t j = 0; j < len-1; j++) {
data[i][j] = random() % 255 + 1; //This way it doesn't end up being 0 and thought of as NUL
}
data[i][len-1] = '\0'; //cap it
}
vdmt::builder builder;
builder.create(num, num * len);
for (uint32_t i = 0; i < num; i++) {
vwriter vfun(data[i]);
builder.append(vfun);
}
invariant(builder.value_length_is_fixed());
vdmt v;
builder.build(&v);
invariant(v.value_length_is_fixed());
invariant(v.get_fixed_length() == len || num == 0);
invariant(v.size() == num);
verify(len, num, &v);
for (uint32_t change = 0; change < num; change++) {
vdmt v2;
v2.clone(v);
v2.delete_at(change);
fail_one_verify(len, num, &v2);
vwriter vfun(data[change]);
v2.insert_at(vfun, change);
verify(len, num, &v2);
v2.destroy();
}
v.destroy();
}
static void test_builder_variable(uint32_t len, uint32_t len2, uint32_t num) {
srandom(random_seed);
assert(len > 1);
assert(len <= MAXLEN);
assert(num <= MAXNUM);
assert(num > 3);
uint32_t which2 = random() % num;
for (uint32_t i = 0; i < num; i++) {
uint32_t thislen = i == which2 ? len2 : len;
for (uint32_t j = 0; j < thislen-1; j++) {
data[i][j] = random() % 255 + 1; //This way it doesn't end up being 0 and thought of as NUL
}
data[i][thislen-1] = '\0'; //cap it
}
vdmt::builder builder;
builder.create(num, (num-1) * len + len2);
for (uint32_t i = 0; i < num; i++) {
vwriter vfun(data[i]);
builder.append(vfun);
}
invariant(!builder.value_length_is_fixed());
vdmt v;
builder.build(&v);
invariant(!v.value_length_is_fixed());
invariant(v.size() == num);
val_type* fetched_data;
for (uint32_t i = 0; i < num; i++) {
uint32_t fetched_len;
int r = v.fetch(i, &fetched_len, &fetched_data);
CKERR(r);
if (i == which2) {
invariant(fetched_len == len2);
invariant(!strcmp(fetched_data->c, data[i]));
} else {
invariant(fetched_len == len);
invariant(!strcmp(fetched_data->c, data[i]));
}
}
v.destroy();
}
static void test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(uint32_t len, uint32_t num) {
srandom(random_seed);
assert(len <= MAXLEN);
assert(num <= MAXNUM);
for (uint32_t i = 0; i < num; i++) {
for (uint32_t j = 0; j < len-1; j++) {
data[i][j] = random() % 255 + 1; //This way it doesn't end up being 0 and thought of as NUL
}
data[i][len-1] = '\0'; //cap it
}
char *flat = (char*)toku_xmalloc(len * num);
char *p = flat;
for (uint32_t i = 0; i < num; i++) {
memcpy(p, data[i], len);
p += len;
}
vdmt v;
v.create_from_sorted_memory_of_fixed_size_elements(flat, num, len*num, len);
invariant(v.value_length_is_fixed());
invariant(v.get_fixed_length() == len);
invariant(v.size() == num);
val_type* fetched_data;
for (uint32_t i = 0; i < num; i++) {
uint32_t fetched_len;
int r = v.fetch(i, &fetched_len, &fetched_data);
CKERR(r);
invariant(fetched_len == len);
invariant(!strcmp(fetched_data->c, data[i]));
}
char *serialized_flat = (char*)toku_xmalloc(len*num);
struct wbuf wb;
wbuf_nocrc_init(&wb, serialized_flat, len*num);
v.prepare_for_serialize();
v.serialize_values(len*num, &wb);
invariant(!memcmp(serialized_flat, flat, len*num));
if (num > 1) {
//Currently converting to dtree treats the entire thing as NOT fixed length.
//Optional additional perf here.
uint32_t which = (random() % (num-1)) + 1; // Not last, not first
invariant(which > 0 && which < num-1);
v.delete_at(which);
memmove(flat + which*len, flat+(which+1)*len, (num-which-1) * len);
v.prepare_for_serialize();
wbuf_nocrc_init(&wb, serialized_flat, len*(num-1));
v.serialize_values(len*(num-1), &wb);
invariant(!memcmp(serialized_flat, flat, len*(num-1)));
}
toku_free(flat);
toku_free(serialized_flat);
v.destroy();
}
int
test_main(int argc, const char *argv[]) {
parse_args(argc, argv);
// Do test with size divisible by 4 and not
test_builder_fixed(4, 0);
test_builder_fixed(5, 0);
test_builder_fixed(4, 1);
test_builder_fixed(5, 1);
test_builder_fixed(4, 100);
test_builder_fixed(5, 100);
// Do test with zero, one, or both sizes divisible
test_builder_variable(4, 8, 100);
test_builder_variable(4, 5, 100);
test_builder_variable(5, 8, 100);
test_builder_variable(5, 10, 100);
test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(4, 0);
test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(5, 0);
test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(4, 1);
test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(5, 1);
test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(4, 100);
test_create_from_sorted_memory_of_fixed_sized_elements_and_serialize(5, 100);
return 0;
}

76
storage/tokudb/ft-index/ft/tests/ft-bfe-query.cc
View File

@@ -101,9 +101,9 @@ int64_key_cmp (DB *db UU(), const DBT *a, const DBT *b) {
}
static void
test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
test_prefetch_read(int fd, FT_HANDLE UU(ft), FT ft_h) {
int r;
brt_h->compare_fun = int64_key_cmp;
ft_h->compare_fun = int64_key_cmp;
FT_CURSOR XMALLOC(cursor);
FTNODE dn = NULL;
PAIR_ATTR attr;
@@ -120,7 +120,7 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
// quick test to see that we have the right behavior when we set
// disable_prefetching to true
cursor->disable_prefetching = true;
fill_bfe_for_prefetch(&bfe, brt_h, cursor);
fill_bfe_for_prefetch(&bfe, ft_h, cursor);
FTNODE_DISK_DATA ndd = NULL;
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
assert(r==0);
@@ -139,14 +139,14 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
// now enable prefetching again
cursor->disable_prefetching = false;
fill_bfe_for_prefetch(&bfe, brt_h, cursor);
fill_bfe_for_prefetch(&bfe, ft_h, cursor);
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
assert(r==0);
assert(dn->n_children == 3);
assert(BP_STATE(dn,0) == PT_AVAIL);
assert(BP_STATE(dn,1) == PT_AVAIL);
assert(BP_STATE(dn,2) == PT_AVAIL);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_COMPRESSED);
assert(BP_STATE(dn,1) == PT_COMPRESSED);
assert(BP_STATE(dn,2) == PT_COMPRESSED);
@@ -161,14 +161,14 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
uint64_t left_key = 150;
toku_fill_dbt(&cursor->range_lock_left_key, &left_key, sizeof(uint64_t));
cursor->left_is_neg_infty = false;
fill_bfe_for_prefetch(&bfe, brt_h, cursor);
fill_bfe_for_prefetch(&bfe, ft_h, cursor);
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
assert(r==0);
assert(dn->n_children == 3);
assert(BP_STATE(dn,0) == PT_ON_DISK);
assert(BP_STATE(dn,1) == PT_AVAIL);
assert(BP_STATE(dn,2) == PT_AVAIL);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_ON_DISK);
assert(BP_STATE(dn,1) == PT_COMPRESSED);
assert(BP_STATE(dn,2) == PT_COMPRESSED);
@@ -183,14 +183,14 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
uint64_t right_key = 151;
toku_fill_dbt(&cursor->range_lock_right_key, &right_key, sizeof(uint64_t));
cursor->right_is_pos_infty = false;
fill_bfe_for_prefetch(&bfe, brt_h, cursor);
fill_bfe_for_prefetch(&bfe, ft_h, cursor);
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
assert(r==0);
assert(dn->n_children == 3);
assert(BP_STATE(dn,0) == PT_ON_DISK);
assert(BP_STATE(dn,1) == PT_AVAIL);
assert(BP_STATE(dn,2) == PT_ON_DISK);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_ON_DISK);
assert(BP_STATE(dn,1) == PT_COMPRESSED);
assert(BP_STATE(dn,2) == PT_ON_DISK);
@@ -204,14 +204,14 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
left_key = 100000;
right_key = 100000;
fill_bfe_for_prefetch(&bfe, brt_h, cursor);
fill_bfe_for_prefetch(&bfe, ft_h, cursor);
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
assert(r==0);
assert(dn->n_children == 3);
assert(BP_STATE(dn,0) == PT_ON_DISK);
assert(BP_STATE(dn,1) == PT_ON_DISK);
assert(BP_STATE(dn,2) == PT_AVAIL);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_ON_DISK);
assert(BP_STATE(dn,1) == PT_ON_DISK);
assert(BP_STATE(dn,2) == PT_COMPRESSED);
@@ -225,14 +225,14 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
left_key = 100;
right_key = 100;
fill_bfe_for_prefetch(&bfe, brt_h, cursor);
fill_bfe_for_prefetch(&bfe, ft_h, cursor);
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe);
assert(r==0);
assert(dn->n_children == 3);
assert(BP_STATE(dn,0) == PT_AVAIL);
assert(BP_STATE(dn,1) == PT_ON_DISK);
assert(BP_STATE(dn,2) == PT_ON_DISK);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_COMPRESSED);
assert(BP_STATE(dn,1) == PT_ON_DISK);
assert(BP_STATE(dn,2) == PT_ON_DISK);
@@ -248,9 +248,9 @@ test_prefetch_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
}
static void
test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
test_subset_read(int fd, FT_HANDLE UU(ft), FT ft_h) {
int r;
brt_h->compare_fun = int64_key_cmp;
ft_h->compare_fun = int64_key_cmp;
FT_CURSOR XMALLOC(cursor);
FTNODE dn = NULL;
FTNODE_DISK_DATA ndd = NULL;
@@ -271,7 +271,7 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
toku_fill_dbt(&right, &right_key, sizeof(right_key));
fill_bfe_for_subset_read(
&bfe,
brt_h,
ft_h,
NULL,
&left,
&right,
@@ -292,11 +292,11 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
assert(BP_STATE(dn,1) == PT_ON_DISK);
assert(BP_STATE(dn,2) == PT_AVAIL);
// need to call this twice because we had a subset read before, that touched the clock
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_ON_DISK);
assert(BP_STATE(dn,1) == PT_ON_DISK);
assert(BP_STATE(dn,2) == PT_AVAIL);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_ON_DISK);
assert(BP_STATE(dn,1) == PT_ON_DISK);
assert(BP_STATE(dn,2) == PT_COMPRESSED);
@@ -317,11 +317,11 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
assert(BP_STATE(dn,1) == PT_AVAIL);
assert(BP_STATE(dn,2) == PT_AVAIL);
// need to call this twice because we had a subset read before, that touched the clock
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_ON_DISK);
assert(BP_STATE(dn,1) == PT_COMPRESSED);
assert(BP_STATE(dn,2) == PT_AVAIL);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_ON_DISK);
assert(BP_STATE(dn,1) == PT_COMPRESSED);
assert(BP_STATE(dn,2) == PT_COMPRESSED);
@@ -341,11 +341,11 @@ test_subset_read(int fd, FT_HANDLE UU(brt), FT brt_h) {
assert(BP_STATE(dn,1) == PT_AVAIL);
assert(BP_STATE(dn,2) == PT_ON_DISK);
// need to call this twice because we had a subset read before, that touched the clock
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_AVAIL);
assert(BP_STATE(dn,1) == PT_COMPRESSED);
assert(BP_STATE(dn,2) == PT_ON_DISK);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(dn,0) == PT_COMPRESSED);
assert(BP_STATE(dn,1) == PT_COMPRESSED);
assert(BP_STATE(dn,2) == PT_ON_DISK);
@@ -412,9 +412,9 @@ test_prefetching(void) {
xids_destroy(&xids_123);
xids_destroy(&xids_234);
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -422,32 +422,32 @@ test_prefetching(void) {
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
toku_blocktable_create_new(&brt_h->blocktable);
ft->ft = ft_h;
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
FTNODE_DISK_DATA ndd = NULL;
r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, brt->ft, false);
r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false);
assert(r==0);
test_prefetch_read(fd, brt, brt_h);
test_subset_read(fd, brt, brt_h);
test_prefetch_read(fd, ft, ft_h);
test_subset_read(fd, ft, ft_h);
toku_free(sn.childkeys[0].data);
toku_free(sn.childkeys[1].data);
@@ -457,11 +457,11 @@ test_prefetching(void) {
toku_free(sn.bp);
toku_free(sn.childkeys);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
toku_free(ndd);
r = close(fd); assert(r != -1);

119
storage/tokudb/ft-index/ft/tests/ft-clock-test.cc
View File

@@ -115,13 +115,18 @@ le_add_to_bn(bn_data* bn, uint32_t idx, const char *key, int keylen, const char
{
LEAFENTRY r = NULL;
uint32_t size_needed = LE_CLEAN_MEMSIZE(vallen);
void *maybe_free = nullptr;
bn->get_space_for_insert(
idx,
key,
keylen,
size_needed,
&r
&r,
&maybe_free
);
if (maybe_free) {
toku_free(maybe_free);
}
resource_assert(r);
r->type = LE_CLEAN;
r->u.clean.vallen = vallen;
@@ -139,11 +144,11 @@ le_malloc(bn_data* bn, uint32_t idx, const char *key, const char *val)
static void
test1(int fd, FT brt_h, FTNODE *dn) {
test1(int fd, FT ft_h, FTNODE *dn) {
int r;
struct ftnode_fetch_extra bfe_all;
brt_h->compare_fun = string_key_cmp;
fill_bfe_for_full_read(&bfe_all, brt_h);
ft_h->compare_fun = string_key_cmp;
fill_bfe_for_full_read(&bfe_all, ft_h);
FTNODE_DISK_DATA ndd = NULL;
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, &ndd, &bfe_all);
bool is_leaf = ((*dn)->height == 0);
@@ -154,12 +159,12 @@ test1(int fd, FT brt_h, FTNODE *dn) {
// should sweep and NOT get rid of anything
PAIR_ATTR attr;
memset(&attr,0,sizeof(attr));
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
for (int i = 0; i < (*dn)->n_children; i++) {
assert(BP_STATE(*dn,i) == PT_AVAIL);
}
// should sweep and get compress all
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
for (int i = 0; i < (*dn)->n_children; i++) {
if (!is_leaf) {
assert(BP_STATE(*dn,i) == PT_COMPRESSED);
@@ -172,12 +177,12 @@ test1(int fd, FT brt_h, FTNODE *dn) {
bool req = toku_ftnode_pf_req_callback(*dn, &bfe_all);
assert(req);
toku_ftnode_pf_callback(*dn, ndd, &bfe_all, fd, &size);
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
for (int i = 0; i < (*dn)->n_children; i++) {
assert(BP_STATE(*dn,i) == PT_AVAIL);
}
// should sweep and get compress all
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
for (int i = 0; i < (*dn)->n_children; i++) {
if (!is_leaf) {
assert(BP_STATE(*dn,i) == PT_COMPRESSED);
@@ -190,15 +195,15 @@ test1(int fd, FT brt_h, FTNODE *dn) {
req = toku_ftnode_pf_req_callback(*dn, &bfe_all);
assert(req);
toku_ftnode_pf_callback(*dn, ndd, &bfe_all, fd, &size);
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
for (int i = 0; i < (*dn)->n_children; i++) {
assert(BP_STATE(*dn,i) == PT_AVAIL);
}
(*dn)->dirty = 1;
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
for (int i = 0; i < (*dn)->n_children; i++) {
assert(BP_STATE(*dn,i) == PT_AVAIL);
}
@@ -212,7 +217,7 @@ static int search_cmp(const struct ft_search& UU(so), const DBT* UU(key)) {
}
static void
test2(int fd, FT brt_h, FTNODE *dn) {
test2(int fd, FT ft_h, FTNODE *dn) {
struct ftnode_fetch_extra bfe_subset;
DBT left, right;
DB dummy_db;
@@ -221,17 +226,11 @@ test2(int fd, FT brt_h, FTNODE *dn) {
memset(&right, 0, sizeof(right));
ft_search_t search_t;
brt_h->compare_fun = string_key_cmp;
ft_h->compare_fun = string_key_cmp;
fill_bfe_for_subset_read(
&bfe_subset,
brt_h,
ft_search_init(
&search_t,
search_cmp,
FT_SEARCH_LEFT,
NULL,
NULL
),
ft_h,
ft_search_init(&search_t, search_cmp, FT_SEARCH_LEFT, nullptr, nullptr, nullptr),
&left,
&right,
true,
@@ -252,11 +251,11 @@ test2(int fd, FT brt_h, FTNODE *dn) {
assert(!BP_SHOULD_EVICT(*dn, 1));
PAIR_ATTR attr;
memset(&attr,0,sizeof(attr));
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(*dn, 0) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED);
assert(BP_STATE(*dn, 1) == PT_AVAIL);
assert(BP_SHOULD_EVICT(*dn, 1));
toku_ftnode_pe_callback(*dn, attr, brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
assert(BP_STATE(*dn, 1) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED);
bool req = toku_ftnode_pf_req_callback(*dn, &bfe_subset);
@@ -272,7 +271,7 @@ test2(int fd, FT brt_h, FTNODE *dn) {
}
static void
test3_leaf(int fd, FT brt_h, FTNODE *dn) {
test3_leaf(int fd, FT ft_h, FTNODE *dn) {
struct ftnode_fetch_extra bfe_min;
DBT left, right;
DB dummy_db;
@@ -280,10 +279,10 @@ test3_leaf(int fd, FT brt_h, FTNODE *dn) {
memset(&left, 0, sizeof(left));
memset(&right, 0, sizeof(right));
brt_h->compare_fun = string_key_cmp;
ft_h->compare_fun = string_key_cmp;
fill_bfe_for_min_read(
&bfe_min,
brt_h
ft_h
);
FTNODE_DISK_DATA ndd = NULL;
int r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, &ndd, &bfe_min);
@@ -347,9 +346,9 @@ test_serialize_nonleaf(void) {
xids_destroy(&xids_123);
xids_destroy(&xids_234);
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -357,33 +356,33 @@ test_serialize_nonleaf(void) {
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
ft->ft = ft_h;
toku_blocktable_create_new(&brt_h->blocktable);
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
FTNODE_DISK_DATA ndd = NULL;
r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, brt->ft, false);
r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false);
assert(r==0);
test1(fd, brt_h, &dn);
test2(fd, brt_h, &dn);
test1(fd, ft_h, &dn);
test2(fd, ft_h, &dn);
toku_free(hello_string);
destroy_nonleaf_childinfo(BNC(&sn, 0));
@@ -392,11 +391,11 @@ test_serialize_nonleaf(void) {
toku_free(sn.childkeys);
toku_free(ndd);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
r = close(fd); assert(r != -1);
}
@@ -431,9 +430,9 @@ test_serialize_leaf(void) {
le_malloc(BLB_DATA(&sn, 0), 1, "b", "bval");
le_malloc(BLB_DATA(&sn, 1), 0, "x", "xval");
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -441,33 +440,33 @@ test_serialize_leaf(void) {
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
ft->ft = ft_h;
toku_blocktable_create_new(&brt_h->blocktable);
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
FTNODE_DISK_DATA ndd = NULL;
r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, brt->ft, false);
r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false);
assert(r==0);
test1(fd, brt_h, &dn);
test3_leaf(fd, brt_h,&dn);
test1(fd, ft_h, &dn);
test3_leaf(fd, ft_h,&dn);
for (int i = 0; i < sn.n_children-1; ++i) {
toku_free(sn.childkeys[i].data);
@@ -478,11 +477,11 @@ test_serialize_leaf(void) {
toku_free(sn.bp);
toku_free(sn.childkeys);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
toku_free(ndd);
r = close(fd); assert(r != -1);
}

166
storage/tokudb/ft-index/ft/tests/ft-serialize-benchmark.cc
View File

@@ -105,13 +105,18 @@ le_add_to_bn(bn_data* bn, uint32_t idx, char *key, int keylen, char *val, int va
{
LEAFENTRY r = NULL;
uint32_t size_needed = LE_CLEAN_MEMSIZE(vallen);
void *maybe_free = nullptr;
bn->get_space_for_insert(
idx,
key,
keylen,
size_needed,
&r
&r,
&maybe_free
);
if (maybe_free) {
toku_free(maybe_free);
}
resource_assert(r);
r->type = LE_CLEAN;
r->u.clean.vallen = vallen;
@@ -127,7 +132,7 @@ long_key_cmp(DB *UU(e), const DBT *a, const DBT *b)
}
static void
test_serialize_leaf(int valsize, int nelts, double entropy) {
test_serialize_leaf(int valsize, int nelts, double entropy, int ser_runs, int deser_runs) {
// struct ft_handle source_ft;
struct ftnode *sn, *dn;
@@ -181,9 +186,9 @@ test_serialize_leaf(int valsize, int nelts, double entropy) {
}
}
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -191,70 +196,99 @@ test_serialize_leaf(int valsize, int nelts, double entropy) {
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
ft->ft = ft_h;
brt_h->compare_fun = long_key_cmp;
toku_blocktable_create_new(&brt_h->blocktable);
ft_h->compare_fun = long_key_cmp;
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
struct timeval total_start;
struct timeval total_end;
total_start.tv_sec = total_start.tv_usec = 0;
total_end.tv_sec = total_end.tv_usec = 0;
struct timeval t[2];
gettimeofday(&t[0], NULL);
FTNODE_DISK_DATA ndd = NULL;
r = toku_serialize_ftnode_to(fd, make_blocknum(20), sn, &ndd, true, brt->ft, false);
for (int i = 0; i < ser_runs; i++) {
gettimeofday(&t[0], NULL);
ndd = NULL;
sn->dirty = 1;
r = toku_serialize_ftnode_to(fd, make_blocknum(20), sn, &ndd, true, ft->ft, false);
assert(r==0);
gettimeofday(&t[1], NULL);
total_start.tv_sec += t[0].tv_sec;
total_start.tv_usec += t[0].tv_usec;
total_end.tv_sec += t[1].tv_sec;
total_end.tv_usec += t[1].tv_usec;
toku_free(ndd);
}
double dt;
dt = (t[1].tv_sec - t[0].tv_sec) + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC);
printf("serialize leaf: %0.05lf\n", dt);
dt = (total_end.tv_sec - total_start.tv_sec) + ((total_end.tv_usec - total_start.tv_usec) / USECS_PER_SEC);
dt *= 1000;
dt /= ser_runs;
printf("serialize leaf(ms): %0.05lf (average of %d runs)\n", dt, ser_runs);
//reset
total_start.tv_sec = total_start.tv_usec = 0;
total_end.tv_sec = total_end.tv_usec = 0;
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, brt_h);
for (int i = 0; i < deser_runs; i++) {
fill_bfe_for_full_read(&bfe, ft_h);
gettimeofday(&t[0], NULL);
FTNODE_DISK_DATA ndd2 = NULL;
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd2, &bfe);
assert(r==0);
gettimeofday(&t[1], NULL);
dt = (t[1].tv_sec - t[0].tv_sec) + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC);
printf("deserialize leaf: %0.05lf\n", dt);
printf("io time %lf decompress time %lf deserialize time %lf\n",
tokutime_to_seconds(bfe.io_time),
tokutime_to_seconds(bfe.decompress_time),
tokutime_to_seconds(bfe.deserialize_time)
);
total_start.tv_sec += t[0].tv_sec;
total_start.tv_usec += t[0].tv_usec;
total_end.tv_sec += t[1].tv_sec;
total_end.tv_usec += t[1].tv_usec;
toku_ftnode_free(&dn);
toku_free(ndd2);
}
dt = (total_end.tv_sec - total_start.tv_sec) + ((total_end.tv_usec - total_start.tv_usec) / USECS_PER_SEC);
dt *= 1000;
dt /= deser_runs;
printf("deserialize leaf(ms): %0.05lf (average of %d runs)\n", dt, deser_runs);
printf("io time(ms) %lf decompress time(ms) %lf deserialize time(ms) %lf (average of %d runs)\n",
tokutime_to_seconds(bfe.io_time)*1000,
tokutime_to_seconds(bfe.decompress_time)*1000,
tokutime_to_seconds(bfe.deserialize_time)*1000,
deser_runs
);
toku_ftnode_free(&sn);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_free(ndd);
toku_free(ndd2);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
r = close(fd); assert(r != -1);
}
static void
test_serialize_nonleaf(int valsize, int nelts, double entropy) {
test_serialize_nonleaf(int valsize, int nelts, double entropy, int ser_runs, int deser_runs) {
// struct ft_handle source_ft;
struct ftnode sn, *dn;
@@ -312,9 +346,9 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
xids_destroy(&xids_0);
xids_destroy(&xids_123);
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -322,25 +356,25 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
ft->ft = ft_h;
brt_h->compare_fun = long_key_cmp;
toku_blocktable_create_new(&brt_h->blocktable);
ft_h->compare_fun = long_key_cmp;
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
@@ -348,26 +382,29 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
struct timeval t[2];
gettimeofday(&t[0], NULL);
FTNODE_DISK_DATA ndd = NULL;
r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, brt->ft, false);
r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false);
assert(r==0);
gettimeofday(&t[1], NULL);
double dt;
dt = (t[1].tv_sec - t[0].tv_sec) + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC);
printf("serialize nonleaf: %0.05lf\n", dt);
dt *= 1000;
printf("serialize nonleaf(ms): %0.05lf (IGNORED RUNS=%d)\n", dt, ser_runs);
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, brt_h);
fill_bfe_for_full_read(&bfe, ft_h);
gettimeofday(&t[0], NULL);
FTNODE_DISK_DATA ndd2 = NULL;
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd2, &bfe);
assert(r==0);
gettimeofday(&t[1], NULL);
dt = (t[1].tv_sec - t[0].tv_sec) + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC);
printf("deserialize nonleaf: %0.05lf\n", dt);
printf("io time %lf decompress time %lf deserialize time %lf\n",
tokutime_to_seconds(bfe.io_time),
tokutime_to_seconds(bfe.decompress_time),
tokutime_to_seconds(bfe.deserialize_time)
dt *= 1000;
printf("deserialize nonleaf(ms): %0.05lf (IGNORED RUNS=%d)\n", dt, deser_runs);
printf("io time(ms) %lf decompress time(ms) %lf deserialize time(ms) %lf (IGNORED RUNS=%d)\n",
tokutime_to_seconds(bfe.io_time)*1000,
tokutime_to_seconds(bfe.decompress_time)*1000,
tokutime_to_seconds(bfe.deserialize_time)*1000,
deser_runs
);
toku_ftnode_free(&dn);
@@ -381,11 +418,11 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
toku_free(sn.bp);
toku_free(sn.childkeys);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
toku_free(ndd);
toku_free(ndd2);
@@ -394,19 +431,32 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
int
test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
long valsize, nelts;
const int DEFAULT_RUNS = 5;
long valsize, nelts, ser_runs = DEFAULT_RUNS, deser_runs = DEFAULT_RUNS;
double entropy = 0.3;
if (argc != 3) {
fprintf(stderr, "Usage: %s <valsize> <nelts>\n", argv[0]);
if (argc != 3 && argc != 5) {
fprintf(stderr, "Usage: %s <valsize> <nelts> [<serialize_runs> <deserialize_runs>]\n", argv[0]);
fprintf(stderr, "Default (and min) runs is %d\n", DEFAULT_RUNS);
return 2;
}
valsize = strtol(argv[1], NULL, 0);
nelts = strtol(argv[2], NULL, 0);
if (argc == 5) {
ser_runs = strtol(argv[3], NULL, 0);
deser_runs = strtol(argv[4], NULL, 0);
}
if (ser_runs <= 0) {
ser_runs = DEFAULT_RUNS;
}
if (deser_runs <= 0) {
deser_runs = DEFAULT_RUNS;
}
initialize_dummymsn();
test_serialize_leaf(valsize, nelts, entropy);
test_serialize_nonleaf(valsize, nelts, entropy);
test_serialize_leaf(valsize, nelts, entropy, ser_runs, deser_runs);
test_serialize_nonleaf(valsize, nelts, entropy, ser_runs, deser_runs);
return 0;
}

22
storage/tokudb/ft-index/ft/tests/ft-serialize-sub-block-test.cc
View File

@@ -92,9 +92,9 @@ PATENT RIGHTS GRANT:
#include "test.h"
// create a brt and put n rows into it
// write the brt to the file
// verify the rows in the brt
// create a ft and put n rows into it
// write the ft to the file
// verify the rows in the ft
static void test_sub_block(int n) {
if (verbose) printf("%s:%d %d\n", __FUNCTION__, __LINE__, n);
@@ -107,14 +107,14 @@ static void test_sub_block(int n) {
int error;
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
int i;
unlink(fname);
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
error = toku_open_ft_handle(fname, true, &brt, nodesize, basementnodesize, compression_method, ct, null_txn, toku_builtin_compare_fun);
error = toku_open_ft_handle(fname, true, &ft, nodesize, basementnodesize, compression_method, ct, null_txn, toku_builtin_compare_fun);
assert(error == 0);
// insert keys 0, 1, 2, .. (n-1)
@@ -124,20 +124,20 @@ static void test_sub_block(int n) {
DBT key, val;
toku_fill_dbt(&key, &k, sizeof k);
toku_fill_dbt(&val, &v, sizeof v);
toku_ft_insert(brt, &key, &val, 0);
toku_ft_insert(ft, &key, &val, 0);
assert(error == 0);
}
// write to the file
error = toku_close_ft_handle_nolsn(brt, 0);
error = toku_close_ft_handle_nolsn(ft, 0);
assert(error == 0);
// verify the brt by walking a cursor through the rows
error = toku_open_ft_handle(fname, false, &brt, nodesize, basementnodesize, compression_method, ct, null_txn, toku_builtin_compare_fun);
// verify the ft by walking a cursor through the rows
error = toku_open_ft_handle(fname, false, &ft, nodesize, basementnodesize, compression_method, ct, null_txn, toku_builtin_compare_fun);
assert(error == 0);
FT_CURSOR cursor;
error = toku_ft_cursor(brt, &cursor, NULL, false, false);
error = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(error == 0);
for (i=0; ; i++) {
@@ -155,7 +155,7 @@ static void test_sub_block(int n) {
toku_ft_cursor_close(cursor);
error = toku_close_ft_handle_nolsn(brt, 0);
error = toku_close_ft_handle_nolsn(ft, 0);
assert(error == 0);
toku_cachetable_close(&ct);

407
storage/tokudb/ft-index/ft/tests/ft-serialize-test.cc
View File

@@ -98,57 +98,53 @@ PATENT RIGHTS GRANT:
#endif
static size_t
calc_le_size(int keylen, int vallen) {
return LE_CLEAN_MEMSIZE(vallen) + keylen + sizeof(uint32_t);
}
static void
le_add_to_bn(bn_data* bn, uint32_t idx, const char *key, int keysize, const char *val, int valsize)
{
LEAFENTRY r = NULL;
uint32_t size_needed = LE_CLEAN_MEMSIZE(valsize);
void *maybe_free = nullptr;
bn->get_space_for_insert(
idx,
key,
keysize,
size_needed,
&r
&r,
&maybe_free
);
if (maybe_free) {
toku_free(maybe_free);
}
resource_assert(r);
r->type = LE_CLEAN;
r->u.clean.vallen = valsize;
memcpy(r->u.clean.val, val, valsize);
return size_needed + keysize + sizeof(uint32_t);
}
static KLPAIR
le_fastmalloc(struct mempool * mp, const char *key, int keylen, const char *val, int vallen)
{
KLPAIR kl;
size_t le_size = calc_le_size(keylen, vallen);
CAST_FROM_VOIDP(kl, toku_mempool_malloc(mp, le_size, 1));
resource_assert(kl);
kl->keylen = keylen;
memcpy(kl->key_le, key, keylen);
LEAFENTRY le = get_le_from_klpair(kl);
class test_key_le_pair {
public:
uint32_t keylen;
char* keyp;
LEAFENTRY le;
test_key_le_pair() : keylen(), keyp(), le() {}
void init(const char *_keyp, const char *_val) {
init(_keyp, strlen(_keyp) + 1, _val, strlen(_val) + 1);
}
void init(const char * _keyp, uint32_t _keylen, const char*_val, uint32_t _vallen) {
keylen = _keylen;
CAST_FROM_VOIDP(le, toku_malloc(LE_CLEAN_MEMSIZE(_vallen)));
le->type = LE_CLEAN;
le->u.clean.vallen = vallen;
memcpy(le->u.clean.val, val, vallen);
return kl;
}
le->u.clean.vallen = _vallen;
memcpy(le->u.clean.val, _val, _vallen);
static KLPAIR
le_malloc(struct mempool * mp, const char *key, const char *val)
{
int keylen = strlen(key) + 1;
int vallen = strlen(val) + 1;
return le_fastmalloc(mp, key, keylen, val, vallen);
CAST_FROM_VOIDP(keyp, toku_xmemdup(_keyp, keylen));
}
~test_key_le_pair() {
toku_free(le);
toku_free(keyp);
}
struct check_leafentries_struct {
int nelts;
LEAFENTRY *elts;
int i;
int (*cmp)(OMTVALUE, void *);
};
enum ftnode_verify_type {
@@ -166,18 +162,18 @@ string_key_cmp(DB *UU(e), const DBT *a, const DBT *b)
}
static void
setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_DATA* ndd) {
setup_dn(enum ftnode_verify_type bft, int fd, FT ft_h, FTNODE *dn, FTNODE_DISK_DATA* ndd) {
int r;
brt_h->compare_fun = string_key_cmp;
ft_h->compare_fun = string_key_cmp;
if (bft == read_all) {
struct ftnode_fetch_extra bfe;
fill_bfe_for_full_read(&bfe, brt_h);
fill_bfe_for_full_read(&bfe, ft_h);
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, ndd, &bfe);
assert(r==0);
}
else if (bft == read_compressed || bft == read_none) {
struct ftnode_fetch_extra bfe;
fill_bfe_for_min_read(&bfe, brt_h);
fill_bfe_for_min_read(&bfe, ft_h);
r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, ndd, &bfe);
assert(r==0);
// assert all bp's are compressed or on disk.
@@ -187,7 +183,7 @@ setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_
// if read_none, get rid of the compressed bp's
if (bft == read_none) {
if ((*dn)->height == 0) {
toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
// assert all bp's are on disk
for (int i = 0; i < (*dn)->n_children; i++) {
if ((*dn)->height == 0) {
@@ -204,7 +200,7 @@ setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_
// that it is available
// then run partial eviction to get it compressed
PAIR_ATTR attr;
fill_bfe_for_full_read(&bfe, brt_h);
fill_bfe_for_full_read(&bfe, ft_h);
assert(toku_ftnode_pf_req_callback(*dn, &bfe));
r = toku_ftnode_pf_callback(*dn, *ndd, &bfe, fd, &attr);
assert(r==0);
@@ -212,21 +208,21 @@ setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_
for (int i = 0; i < (*dn)->n_children; i++) {
assert(BP_STATE(*dn,i) == PT_AVAIL);
}
toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
for (int i = 0; i < (*dn)->n_children; i++) {
// assert all bp's are still available, because we touched the clock
assert(BP_STATE(*dn,i) == PT_AVAIL);
// now assert all should be evicted
assert(BP_SHOULD_EVICT(*dn, i));
}
toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), brt_h, def_pe_finalize_impl, nullptr);
toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr);
for (int i = 0; i < (*dn)->n_children; i++) {
assert(BP_STATE(*dn,i) == PT_COMPRESSED);
}
}
}
// now decompress them
fill_bfe_for_full_read(&bfe, brt_h);
fill_bfe_for_full_read(&bfe, ft_h);
assert(toku_ftnode_pf_req_callback(*dn, &bfe));
PAIR_ATTR attr;
r = toku_ftnode_pf_callback(*dn, *ndd, &bfe, fd, &attr);
@@ -243,20 +239,20 @@ setup_dn(enum ftnode_verify_type bft, int fd, FT brt_h, FTNODE *dn, FTNODE_DISK_
}
}
static void write_sn_to_disk(int fd, FT_HANDLE brt, FTNODE sn, FTNODE_DISK_DATA* src_ndd, bool do_clone) {
static void write_sn_to_disk(int fd, FT_HANDLE ft, FTNODE sn, FTNODE_DISK_DATA* src_ndd, bool do_clone) {
int r;
if (do_clone) {
void* cloned_node_v = NULL;
PAIR_ATTR attr;
long clone_size;
toku_ftnode_clone_callback(sn, &cloned_node_v, &clone_size, &attr, false, brt->ft);
toku_ftnode_clone_callback(sn, &cloned_node_v, &clone_size, &attr, false, ft->ft);
FTNODE CAST_FROM_VOIDP(cloned_node, cloned_node_v);
r = toku_serialize_ftnode_to(fd, make_blocknum(20), cloned_node, src_ndd, false, brt->ft, false);
r = toku_serialize_ftnode_to(fd, make_blocknum(20), cloned_node, src_ndd, false, ft->ft, false);
assert(r==0);
toku_ftnode_free(&cloned_node);
}
else {
r = toku_serialize_ftnode_to(fd, make_blocknum(20), sn, src_ndd, true, brt->ft, false);
r = toku_serialize_ftnode_to(fd, make_blocknum(20), sn, src_ndd, true, ft->ft, false);
assert(r==0);
}
}
@@ -290,16 +286,15 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) {
BP_STATE(&sn,1) = PT_AVAIL;
set_BLB(&sn, 0, toku_create_empty_bn());
set_BLB(&sn, 1, toku_create_empty_bn());
KLPAIR elts[3];
le_add_to_bn(BLB_DATA(&sn, 0), 0, "a", 2, "aval", 5);
le_add_to_bn(BLB_DATA(&sn, 0), 1, "b", 2, "bval", 5);
le_add_to_bn(BLB_DATA(&sn, 1), 0, "x", 2, "xval", 5);
BLB_MAX_MSN_APPLIED(&sn, 0) = ((MSN) { MIN_MSN.msn + 73 });
BLB_MAX_MSN_APPLIED(&sn, 1) = POSTSERIALIZE_MSN_ON_DISK;
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -307,33 +302,33 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) {
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
toku_blocktable_create_new(&brt_h->blocktable);
ft->ft = ft_h;
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
FTNODE_DISK_DATA src_ndd = NULL;
FTNODE_DISK_DATA dest_ndd = NULL;
write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
assert(dn->thisnodename.b==20);
@@ -346,11 +341,10 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) {
{
// Man, this is way too ugly. This entire test suite needs to be refactored.
// Create a dummy mempool and put the leaves there. Ugh.
struct mempool dummy_mp;
toku_mempool_construct(&dummy_mp, 1024);
elts[0] = le_malloc(&dummy_mp, "a", "aval");
elts[1] = le_malloc(&dummy_mp, "b", "bval");
elts[2] = le_malloc(&dummy_mp, "x", "xval");
test_key_le_pair elts[3];
elts[0].init("a", "aval");
elts[1].init("b", "bval");
elts[2].init("x", "xval");
const uint32_t npartitions = dn->n_children;
assert(dn->totalchildkeylens==(2*(npartitions-1)));
uint32_t last_i = 0;
@@ -361,22 +355,21 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) {
if (bn > 0) {
assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size);
}
for (uint32_t i = 0; i < BLB_DATA(dn, bn)->omt_size(); i++) {
for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) {
LEAFENTRY curr_le;
uint32_t curr_keylen;
void* curr_key;
BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key);
assert(leafentry_memsize(curr_le) == leafentry_memsize(get_le_from_klpair(elts[last_i])));
assert(memcmp(curr_le, get_le_from_klpair(elts[last_i]), leafentry_memsize(curr_le)) == 0);
assert(leafentry_memsize(curr_le) == leafentry_memsize(elts[last_i].le));
assert(memcmp(curr_le, elts[last_i].le, leafentry_memsize(curr_le)) == 0);
if (bn < npartitions-1) {
assert(strcmp((char*)dn->childkeys[bn].data, (char*)(elts[last_i]->key_le)) <= 0);
assert(strcmp((char*)dn->childkeys[bn].data, elts[last_i].keyp) <= 0);
}
// TODO for later, get a key comparison here as well
last_i++;
}
}
toku_mempool_destroy(&dummy_mp);
assert(last_i == 3);
}
toku_ftnode_free(&dn);
@@ -390,11 +383,11 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) {
toku_free(sn.bp);
toku_free(sn.childkeys);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
toku_free(src_ndd);
toku_free(dest_ndd);
@@ -436,14 +429,14 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone
if (i < nrows-1) {
uint32_t keylen;
void* curr_key;
BLB_DATA(&sn, i)->fetch_le_key_and_len(0, &keylen, &curr_key);
BLB_DATA(&sn, i)->fetch_key_and_len(0, &keylen, &curr_key);
toku_memdup_dbt(&sn.childkeys[i], curr_key, keylen);
}
}
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -451,32 +444,32 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
toku_blocktable_create_new(&brt_h->blocktable);
ft->ft = ft_h;
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
FTNODE_DISK_DATA src_ndd = NULL;
FTNODE_DISK_DATA dest_ndd = NULL;
write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
assert(dn->thisnodename.b==20);
@@ -485,18 +478,14 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone
{
// Man, this is way too ugly. This entire test suite needs to be refactored.
// Create a dummy mempool and put the leaves there. Ugh.
struct mempool dummy_mp;
size_t le_size = calc_le_size(keylens, vallens);
size_t mpsize = nrows * le_size;
toku_mempool_construct(&dummy_mp, mpsize);
KLPAIR les[nrows];
test_key_le_pair *les = new test_key_le_pair[nrows];
{
char key[keylens], val[vallens];
key[keylens-1] = '\0';
for (uint32_t i = 0; i < nrows; ++i) {
char c = 'a' + i;
memset(key, c, keylens-1);
les[i] = le_fastmalloc(&dummy_mp, (char *) &key, sizeof(key), (char *) &val, sizeof(val));
les[i].init((char *) &key, sizeof(key), (char *) &val, sizeof(val));
}
}
const uint32_t npartitions = dn->n_children;
@@ -508,23 +497,23 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone
if (bn > 0) {
assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size);
}
assert(BLB_DATA(dn, bn)->omt_size() > 0);
for (uint32_t i = 0; i < BLB_DATA(dn, bn)->omt_size(); i++) {
assert(BLB_DATA(dn, bn)->num_klpairs() > 0);
for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) {
LEAFENTRY curr_le;
uint32_t curr_keylen;
void* curr_key;
BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key);
assert(leafentry_memsize(curr_le) == leafentry_memsize(get_le_from_klpair(les[last_i])));
assert(memcmp(curr_le, get_le_from_klpair(les[last_i]), leafentry_memsize(curr_le)) == 0);
assert(leafentry_memsize(curr_le) == leafentry_memsize(les[last_i].le));
assert(memcmp(curr_le, les[last_i].le, leafentry_memsize(curr_le)) == 0);
if (bn < npartitions-1) {
assert(strcmp((char*)dn->childkeys[bn].data, (char*)(les[last_i]->key_le)) <= 0);
assert(strcmp((char*)dn->childkeys[bn].data, les[last_i].keyp) <= 0);
}
// TODO for later, get a key comparison here as well
last_i++;
}
}
toku_mempool_destroy(&dummy_mp);
assert(last_i == nrows);
delete[] les;
}
toku_ftnode_free(&dn);
@@ -537,11 +526,11 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone
}
toku_free(sn.bp);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
toku_free(src_ndd);
toku_free(dest_ndd);
@@ -552,7 +541,6 @@ static void
test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
int r;
struct ftnode sn, *dn;
const int keylens = sizeof(int), vallens = sizeof(int);
const uint32_t nrows = 196*1024;
int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0);
@@ -566,22 +554,23 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
sn.dirty = 1;
sn.oldest_referenced_xid_known = TXNID_NONE;
MALLOC_N(sn.n_children, sn.bp);
MALLOC_N(sn.n_children-1, sn.childkeys);
XMALLOC_N(sn.n_children, sn.bp);
XMALLOC_N(sn.n_children-1, sn.childkeys);
sn.totalchildkeylens = (sn.n_children-1)*sizeof(int);
for (int i = 0; i < sn.n_children; ++i) {
BP_STATE(&sn,i) = PT_AVAIL;
set_BLB(&sn, i, toku_create_empty_bn());
}
size_t total_size = 0;
for (uint32_t i = 0; i < nrows; ++i) {
uint32_t key = i;
uint32_t val = i;
le_add_to_bn(BLB_DATA(&sn, 0), i, (char *) &key, sizeof(key), (char *) &val, sizeof(val));
total_size += le_add_to_bn(BLB_DATA(&sn, 0), i, (char *) &key, sizeof(key), (char *) &val, sizeof(val));
}
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -589,33 +578,33 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
ft->ft = ft_h;
toku_blocktable_create_new(&brt_h->blocktable);
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
FTNODE_DISK_DATA src_ndd = NULL;
FTNODE_DISK_DATA dest_ndd = NULL;
write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
assert(dn->thisnodename.b==20);
@@ -624,15 +613,11 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
{
// Man, this is way too ugly. This entire test suite needs to be refactored.
// Create a dummy mempool and put the leaves there. Ugh.
struct mempool dummy_mp;
size_t le_size = calc_le_size(keylens, vallens);
size_t mpsize = nrows * le_size;
toku_mempool_construct(&dummy_mp, mpsize);
KLPAIR les[nrows];
test_key_le_pair *les = new test_key_le_pair[nrows];
{
int key = 0, val = 0;
for (uint32_t i = 0; i < nrows; ++i, key++, val++) {
les[i] = le_fastmalloc(&dummy_mp, (char *) &key, sizeof(key), (char *) &val, sizeof(val));
les[i].init((char *) &key, sizeof(key), (char *) &val, sizeof(val));
}
}
const uint32_t npartitions = dn->n_children;
@@ -644,17 +629,17 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
if (bn > 0) {
assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size);
}
assert(BLB_DATA(dn, bn)->omt_size() > 0);
for (uint32_t i = 0; i < BLB_DATA(dn, bn)->omt_size(); i++) {
assert(BLB_DATA(dn, bn)->num_klpairs() > 0);
for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) {
LEAFENTRY curr_le;
uint32_t curr_keylen;
void* curr_key;
BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key);
assert(leafentry_memsize(curr_le) == leafentry_memsize(get_le_from_klpair(les[last_i])));
assert(memcmp(curr_le, get_le_from_klpair(les[last_i]), leafentry_memsize(curr_le)) == 0);
assert(leafentry_memsize(curr_le) == leafentry_memsize(les[last_i].le));
assert(memcmp(curr_le, les[last_i].le, leafentry_memsize(curr_le)) == 0);
if (bn < npartitions-1) {
uint32_t *CAST_FROM_VOIDP(pivot, dn->childkeys[bn].data);
void* tmp = les[last_i]->key_le;
void* tmp = les[last_i].keyp;
uint32_t *CAST_FROM_VOIDP(item, tmp);
assert(*pivot >= *item);
}
@@ -664,8 +649,8 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
// don't check soft_copy_is_up_to_date or seqinsert
assert(BLB_DATA(dn, bn)->get_disk_size() < 128*1024); // BN_MAX_SIZE, apt to change
}
toku_mempool_destroy(&dummy_mp);
assert(last_i == nrows);
delete[] les;
}
toku_ftnode_free(&dn);
@@ -678,11 +663,11 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) {
toku_free(sn.bp);
toku_free(sn.childkeys);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
toku_free(src_ndd);
toku_free(dest_ndd);
@@ -727,9 +712,9 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
le_add_to_bn(BLB_DATA(&sn, 0), i,key, 8, val, val_size);
}
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -737,33 +722,33 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
ft->ft = ft_h;
toku_blocktable_create_new(&brt_h->blocktable);
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
FTNODE_DISK_DATA src_ndd = NULL;
FTNODE_DISK_DATA dest_ndd = NULL;
write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
assert(dn->thisnodename.b==20);
@@ -772,11 +757,7 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
{
// Man, this is way too ugly. This entire test suite needs to be refactored.
// Create a dummy mempool and put the leaves there. Ugh.
struct mempool dummy_mp;
size_t le_size = calc_le_size(key_size, val_size);
size_t mpsize = nrows * le_size;
toku_mempool_construct(&dummy_mp, mpsize);
KLPAIR les[nrows];
test_key_le_pair *les = new test_key_le_pair[nrows];
{
char key[key_size], val[val_size];
key[key_size-1] = '\0';
@@ -785,7 +766,7 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
char c = 'a' + i;
memset(key, c, key_size-1);
memset(val, c, val_size-1);
les[i] = le_fastmalloc(&dummy_mp, key, key_size, val, val_size);
les[i].init(key, key_size, val, val_size);
}
}
const uint32_t npartitions = dn->n_children;
@@ -798,24 +779,24 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
if (bn > 0) {
assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size);
}
assert(BLB_DATA(dn, bn)->omt_size() > 0);
for (uint32_t i = 0; i < BLB_DATA(dn, bn)->omt_size(); i++) {
assert(BLB_DATA(dn, bn)->num_klpairs() > 0);
for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) {
LEAFENTRY curr_le;
uint32_t curr_keylen;
void* curr_key;
BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key);
assert(leafentry_memsize(curr_le) == leafentry_memsize(get_le_from_klpair(les[last_i])));
assert(memcmp(curr_le, get_le_from_klpair(les[last_i]), leafentry_memsize(curr_le)) == 0);
assert(leafentry_memsize(curr_le) == leafentry_memsize(les[last_i].le));
assert(memcmp(curr_le, les[last_i].le, leafentry_memsize(curr_le)) == 0);
if (bn < npartitions-1) {
assert(strcmp((char*)dn->childkeys[bn].data, (char*)(les[last_i]->key_le)) <= 0);
assert(strcmp((char*)dn->childkeys[bn].data, (char*)(les[last_i].keyp)) <= 0);
}
// TODO for later, get a key comparison here as well
last_i++;
}
// don't check soft_copy_is_up_to_date or seqinsert
}
toku_mempool_destroy(&dummy_mp);
assert(last_i == 7);
delete[] les;
}
toku_ftnode_free(&dn);
@@ -828,11 +809,11 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone)
toku_free(sn.bp);
toku_free(sn.childkeys);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
toku_free(src_ndd);
toku_free(dest_ndd);
@@ -871,14 +852,13 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool
set_BLB(&sn, i, toku_create_empty_bn());
BLB_SEQINSERT(&sn, i) = 0;
}
KLPAIR elts[3];
le_add_to_bn(BLB_DATA(&sn, 1), 0, "a", 2, "aval", 5);
le_add_to_bn(BLB_DATA(&sn, 3), 0, "b", 2, "bval", 5);
le_add_to_bn(BLB_DATA(&sn, 5), 0, "x", 2, "xval", 5);
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -886,32 +866,32 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
ft->ft = ft_h;
toku_blocktable_create_new(&brt_h->blocktable);
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
FTNODE_DISK_DATA src_ndd = NULL;
FTNODE_DISK_DATA dest_ndd = NULL;
write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
assert(dn->thisnodename.b==20);
@@ -921,13 +901,13 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool
assert(dn->height == 0);
assert(dn->n_children>0);
{
test_key_le_pair elts[3];
// Man, this is way too ugly. This entire test suite needs to be refactored.
// Create a dummy mempool and put the leaves there. Ugh.
struct mempool dummy_mp;
toku_mempool_construct(&dummy_mp, 1024);
elts[0] = le_malloc(&dummy_mp, "a", "aval");
elts[1] = le_malloc(&dummy_mp, "b", "bval");
elts[2] = le_malloc(&dummy_mp, "x", "xval");
elts[0].init("a", "aval");
elts[1].init("b", "bval");
elts[2].init("x", "xval");
const uint32_t npartitions = dn->n_children;
assert(dn->totalchildkeylens==(2*(npartitions-1)));
uint32_t last_i = 0;
@@ -937,22 +917,21 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool
if (bn > 0) {
assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size);
}
for (uint32_t i = 0; i < BLB_DATA(dn, bn)->omt_size(); i++) {
for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) {
LEAFENTRY curr_le;
uint32_t curr_keylen;
void* curr_key;
BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key);
assert(leafentry_memsize(curr_le) == leafentry_memsize(get_le_from_klpair(elts[last_i])));
assert(memcmp(curr_le, get_le_from_klpair(elts[last_i]), leafentry_memsize(curr_le)) == 0);
assert(leafentry_memsize(curr_le) == leafentry_memsize(elts[last_i].le));
assert(memcmp(curr_le, elts[last_i].le, leafentry_memsize(curr_le)) == 0);
if (bn < npartitions-1) {
assert(strcmp((char*)dn->childkeys[bn].data, (char*)(elts[last_i]->key_le)) <= 0);
assert(strcmp((char*)dn->childkeys[bn].data, (char*)(elts[last_i].keyp)) <= 0);
}
// TODO for later, get a key comparison here as well
last_i++;
}
}
toku_mempool_destroy(&dummy_mp);
assert(last_i == 3);
}
toku_ftnode_free(&dn);
@@ -966,11 +945,11 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool
toku_free(sn.bp);
toku_free(sn.childkeys);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
toku_free(src_ndd);
toku_free(dest_ndd);
@@ -1005,9 +984,9 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b
set_BLB(&sn, i, toku_create_empty_bn());
}
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -1015,33 +994,33 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
ft->ft = ft_h;
toku_blocktable_create_new(&brt_h->blocktable);
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
FTNODE_DISK_DATA src_ndd = NULL;
FTNODE_DISK_DATA dest_ndd = NULL;
write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
assert(dn->thisnodename.b==20);
@@ -1059,7 +1038,7 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b
if (i > 0) {
assert(dest_ndd[i].start >= dest_ndd[i-1].start + dest_ndd[i-1].size);
}
assert(BLB_DATA(dn, i)->omt_size() == 0);
assert(BLB_DATA(dn, i)->num_klpairs() == 0);
}
}
toku_ftnode_free(&dn);
@@ -1073,11 +1052,11 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b
toku_free(sn.bp);
toku_free(sn.childkeys);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
toku_free(src_ndd);
toku_free(dest_ndd);
@@ -1131,9 +1110,9 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) {
xids_destroy(&xids_123);
xids_destroy(&xids_234);
FT_HANDLE XMALLOC(brt);
FT XCALLOC(brt_h);
toku_ft_init(brt_h,
FT_HANDLE XMALLOC(ft);
FT XCALLOC(ft_h);
toku_ft_init(ft_h,
make_blocknum(0),
ZERO_LSN,
TXNID_NONE,
@@ -1141,32 +1120,32 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) {
128*1024,
TOKU_DEFAULT_COMPRESSION_METHOD,
16);
brt->ft = brt_h;
ft->ft = ft_h;
toku_blocktable_create_new(&brt_h->blocktable);
toku_blocktable_create_new(&ft_h->blocktable);
{ int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); }
//Want to use block #20
BLOCKNUM b = make_blocknum(0);
while (b.b < 20) {
toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
toku_allocate_blocknum(ft_h->blocktable, &b, ft_h);
}
assert(b.b == 20);
{
DISKOFF offset;
DISKOFF size;
toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, fd, false);
toku_blocknum_realloc_on_disk(ft_h->blocktable, b, 100, &offset, ft_h, fd, false);
assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
toku_translate_blocknum_to_offset_size(ft_h->blocktable, b, &offset, &size);
assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
assert(size == 100);
}
FTNODE_DISK_DATA src_ndd = NULL;
FTNODE_DISK_DATA dest_ndd = NULL;
write_sn_to_disk(fd, brt, &sn, &src_ndd, do_clone);
write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone);
setup_dn(bft, fd, brt_h, &dn, &dest_ndd);
setup_dn(bft, fd, ft_h, &dn, &dest_ndd);
assert(dn->thisnodename.b==20);
@@ -1197,11 +1176,11 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) {
toku_free(sn.bp);
toku_free(sn.childkeys);
toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&brt_h->blocktable);
toku_free(brt_h->h);
toku_free(brt_h);
toku_free(brt);
toku_block_free(ft_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
toku_blocktable_destroy(&ft_h->blocktable);
toku_free(ft_h->h);
toku_free(ft_h);
toku_free(ft);
toku_free(src_ndd);
toku_free(dest_ndd);

10
storage/tokudb/ft-index/ft/tests/ft-test-cursor-2.cc
View File

@@ -113,14 +113,14 @@ static void test_multiple_ft_cursor_dbts(int n) {
int r;
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
FT_CURSOR cursors[n];
unlink(fname);
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
assert(r==0);
int i;
@@ -129,14 +129,14 @@ static void test_multiple_ft_cursor_dbts(int n) {
char key[10],val[10];
snprintf(key, sizeof key, "k%04d", i);
snprintf(val, sizeof val, "v%04d", i);
toku_ft_insert(brt,
toku_ft_insert(ft,
toku_fill_dbt(&kbt, key, 1+strlen(key)),
toku_fill_dbt(&vbt, val, 1+strlen(val)),
0);
}
for (i=0; i<n; i++) {
r = toku_ft_cursor(brt, &cursors[i], NULL, false, false);
r = toku_ft_cursor(ft, &cursors[i], NULL, false, false);
assert(r == 0);
}
@@ -166,7 +166,7 @@ static void test_multiple_ft_cursor_dbts(int n) {
toku_free(ptrs[i]);
}
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);

166
storage/tokudb/ft-index/ft/tests/ft-test-cursor.cc
View File

@@ -102,11 +102,11 @@ static int test_ft_cursor_keycompare(DB *desc __attribute__((unused)), const DBT
return toku_keycompare(a->data, a->size, b->data, b->size);
}
static void assert_cursor_notfound(FT_HANDLE brt, int position) {
static void assert_cursor_notfound(FT_HANDLE ft, int position) {
FT_CURSOR cursor=0;
int r;
r = toku_ft_cursor(brt, &cursor, NULL, false, false);
r = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(r==0);
struct check_pair pair = {0,0,0,0,0};
@@ -117,11 +117,11 @@ static void assert_cursor_notfound(FT_HANDLE brt, int position) {
toku_ft_cursor_close(cursor);
}
static void assert_cursor_value(FT_HANDLE brt, int position, long long value) {
static void assert_cursor_value(FT_HANDLE ft, int position, long long value) {
FT_CURSOR cursor=0;
int r;
r = toku_ft_cursor(brt, &cursor, NULL, false, false);
r = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(r==0);
if (test_cursor_debug && verbose) printf("key: ");
@@ -133,11 +133,11 @@ static void assert_cursor_value(FT_HANDLE brt, int position, long long value) {
toku_ft_cursor_close(cursor);
}
static void assert_cursor_first_last(FT_HANDLE brt, long long firstv, long long lastv) {
static void assert_cursor_first_last(FT_HANDLE ft, long long firstv, long long lastv) {
FT_CURSOR cursor=0;
int r;
r = toku_ft_cursor(brt, &cursor, NULL, false, false);
r = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(r==0);
if (test_cursor_debug && verbose) printf("first key: ");
@@ -162,7 +162,7 @@ static void assert_cursor_first_last(FT_HANDLE brt, long long firstv, long long
static void test_ft_cursor_first(int n) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
int r;
int i;
@@ -172,7 +172,7 @@ static void test_ft_cursor_first(int n) {
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
/* insert a bunch of kv pairs */
@@ -184,15 +184,15 @@ static void test_ft_cursor_first(int n) {
toku_fill_dbt(&kbt, key, strlen(key)+1);
v = i;
toku_fill_dbt(&vbt, &v, sizeof v);
toku_ft_insert(brt, &kbt, &vbt, 0);
toku_ft_insert(ft, &kbt, &vbt, 0);
}
if (n == 0)
assert_cursor_notfound(brt, DB_FIRST);
assert_cursor_notfound(ft, DB_FIRST);
else
assert_cursor_value(brt, DB_FIRST, 0);
assert_cursor_value(ft, DB_FIRST, 0);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
@@ -200,7 +200,7 @@ static void test_ft_cursor_first(int n) {
static void test_ft_cursor_last(int n) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
int r;
int i;
@@ -210,7 +210,7 @@ static void test_ft_cursor_last(int n) {
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
/* insert keys 0, 1, .. (n-1) */
@@ -222,16 +222,16 @@ static void test_ft_cursor_last(int n) {
toku_fill_dbt(&kbt, key, strlen(key)+1);
v = i;
toku_fill_dbt(&vbt, &v, sizeof v);
toku_ft_insert(brt, &kbt, &vbt, 0);
toku_ft_insert(ft, &kbt, &vbt, 0);
assert(r==0);
}
if (n == 0)
assert_cursor_notfound(brt, DB_LAST);
assert_cursor_notfound(ft, DB_LAST);
else
assert_cursor_value(brt, DB_LAST, n-1);
assert_cursor_value(ft, DB_LAST, n-1);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
@@ -239,7 +239,7 @@ static void test_ft_cursor_last(int n) {
static void test_ft_cursor_first_last(int n) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
int r;
int i;
@@ -249,7 +249,7 @@ static void test_ft_cursor_first_last(int n) {
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
/* insert a bunch of kv pairs */
@@ -262,16 +262,16 @@ static void test_ft_cursor_first_last(int n) {
v = i;
toku_fill_dbt(&vbt, &v, sizeof v);
toku_ft_insert(brt, &kbt, &vbt, 0);
toku_ft_insert(ft, &kbt, &vbt, 0);
}
if (n == 0) {
assert_cursor_notfound(brt, DB_FIRST);
assert_cursor_notfound(brt, DB_LAST);
assert_cursor_notfound(ft, DB_FIRST);
assert_cursor_notfound(ft, DB_LAST);
} else
assert_cursor_first_last(brt, 0, n-1);
assert_cursor_first_last(ft, 0, n-1);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
@@ -281,7 +281,7 @@ static void test_ft_cursor_first_last(int n) {
static void test_ft_cursor_rfirst(int n) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
int r;
int i;
@@ -291,7 +291,7 @@ static void test_ft_cursor_rfirst(int n) {
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
/* insert keys n-1, n-2, ... , 0 */
@@ -304,26 +304,26 @@ static void test_ft_cursor_rfirst(int n) {
toku_fill_dbt(&kbt, key, strlen(key)+1);
v = i;
toku_fill_dbt(&vbt, &v, sizeof v);
toku_ft_insert(brt, &kbt, &vbt, 0);
toku_ft_insert(ft, &kbt, &vbt, 0);
}
if (n == 0)
assert_cursor_notfound(brt, DB_FIRST);
assert_cursor_notfound(ft, DB_FIRST);
else
assert_cursor_value(brt, DB_FIRST, 0);
assert_cursor_value(ft, DB_FIRST, 0);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
}
static void assert_cursor_walk(FT_HANDLE brt, int n) {
static void assert_cursor_walk(FT_HANDLE ft, int n) {
FT_CURSOR cursor=0;
int i;
int r;
r = toku_ft_cursor(brt, &cursor, NULL, false, false);
r = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(r==0);
if (test_cursor_debug && verbose) printf("key: ");
@@ -345,7 +345,7 @@ static void assert_cursor_walk(FT_HANDLE brt, int n) {
static void test_ft_cursor_walk(int n) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
int r;
int i;
@@ -355,7 +355,7 @@ static void test_ft_cursor_walk(int n) {
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
/* insert a bunch of kv pairs */
@@ -367,25 +367,25 @@ static void test_ft_cursor_walk(int n) {
toku_fill_dbt(&kbt, key, strlen(key)+1);
v = i;
toku_fill_dbt(&vbt, &v, sizeof v);
toku_ft_insert(brt, &kbt, &vbt, 0);
toku_ft_insert(ft, &kbt, &vbt, 0);
}
/* walk the tree */
assert_cursor_walk(brt, n);
assert_cursor_walk(ft, n);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
}
static void assert_cursor_rwalk(FT_HANDLE brt, int n) {
static void assert_cursor_rwalk(FT_HANDLE ft, int n) {
FT_CURSOR cursor=0;
int i;
int r;
r = toku_ft_cursor(brt, &cursor, NULL, false, false);
r = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(r==0);
if (test_cursor_debug && verbose) printf("key: ");
@@ -407,7 +407,7 @@ static void assert_cursor_rwalk(FT_HANDLE brt, int n) {
static void test_ft_cursor_rwalk(int n) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
int r;
int i;
@@ -417,7 +417,7 @@ static void test_ft_cursor_rwalk(int n) {
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
/* insert a bunch of kv pairs */
@@ -429,13 +429,13 @@ static void test_ft_cursor_rwalk(int n) {
toku_fill_dbt(&kbt, &k, sizeof k);
v = i;
toku_fill_dbt(&vbt, &v, sizeof v);
toku_ft_insert(brt, &kbt, &vbt, 0);
toku_ft_insert(ft, &kbt, &vbt, 0);
}
/* walk the tree */
assert_cursor_rwalk(brt, n);
assert_cursor_rwalk(ft, n);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
@@ -462,13 +462,13 @@ ascending_key_string_checkf (ITEMLEN keylen, bytevec key, ITEMLEN UU(vallen), by
}
// The keys are strings (null terminated)
static void assert_cursor_walk_inorder(FT_HANDLE brt, int n) {
static void assert_cursor_walk_inorder(FT_HANDLE ft, int n) {
FT_CURSOR cursor=0;
int i;
int r;
char *prevkey = 0;
r = toku_ft_cursor(brt, &cursor, NULL, false, false);
r = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(r==0);
if (test_cursor_debug && verbose) printf("key: ");
@@ -488,7 +488,7 @@ static void assert_cursor_walk_inorder(FT_HANDLE brt, int n) {
static void test_ft_cursor_rand(int n) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
int r;
int i;
@@ -498,7 +498,7 @@ static void test_ft_cursor_rand(int n) {
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
/* insert a bunch of kv pairs */
@@ -513,22 +513,22 @@ static void test_ft_cursor_rand(int n) {
v = i;
toku_fill_dbt(&vbt, &v, sizeof v);
struct check_pair pair = {kbt.size, key, len_ignore, 0, 0};
r = toku_ft_lookup(brt, &kbt, lookup_checkf, &pair);
r = toku_ft_lookup(ft, &kbt, lookup_checkf, &pair);
if (r == 0) {
assert(pair.call_count==1);
if (verbose) printf("dup");
continue;
}
assert(pair.call_count==0);
toku_ft_insert(brt, &kbt, &vbt, 0);
toku_ft_insert(ft, &kbt, &vbt, 0);
break;
}
}
/* walk the tree */
assert_cursor_walk_inorder(brt, n);
assert_cursor_walk_inorder(ft, n);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
@@ -536,7 +536,7 @@ static void test_ft_cursor_rand(int n) {
static void test_ft_cursor_split(int n) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
FT_CURSOR cursor=0;
int r;
int keyseqnum;
@@ -548,7 +548,7 @@ static void test_ft_cursor_split(int n) {
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
/* insert a bunch of kv pairs */
@@ -560,10 +560,10 @@ static void test_ft_cursor_split(int n) {
toku_fill_dbt(&kbt, key, strlen(key)+1);
v = keyseqnum;
toku_fill_dbt(&vbt, &v, sizeof v);
toku_ft_insert(brt, &kbt, &vbt, 0);
toku_ft_insert(ft, &kbt, &vbt, 0);
}
r = toku_ft_cursor(brt, &cursor, NULL, false, false);
r = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(r==0);
if (test_cursor_debug && verbose) printf("key: ");
@@ -583,7 +583,7 @@ static void test_ft_cursor_split(int n) {
toku_fill_dbt(&kbt, key, strlen(key)+1);
v = keyseqnum;
toku_fill_dbt(&vbt, &v, sizeof v);
toku_ft_insert(brt, &kbt, &vbt, 0);
toku_ft_insert(ft, &kbt, &vbt, 0);
}
if (test_cursor_debug && verbose) printf("key: ");
@@ -601,7 +601,7 @@ static void test_ft_cursor_split(int n) {
toku_ft_cursor_close(cursor);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
@@ -612,19 +612,19 @@ static void test_multiple_ft_cursors(int n) {
int r;
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
FT_CURSOR cursors[n];
unlink(fname);
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
int i;
for (i=0; i<n; i++) {
r = toku_ft_cursor(brt, &cursors[i], NULL, false, false);
r = toku_ft_cursor(ft, &cursors[i], NULL, false, false);
assert(r == 0);
}
@@ -632,7 +632,7 @@ static void test_multiple_ft_cursors(int n) {
toku_ft_cursor_close(cursors[i]);
}
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
@@ -653,7 +653,7 @@ static void test_multiple_ft_cursor_walk(int n) {
int r;
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
const int cursor_gap = 1000;
const int ncursors = n/cursor_gap;
FT_CURSOR cursors[ncursors];
@@ -665,13 +665,13 @@ static void test_multiple_ft_cursor_walk(int n) {
int cachesize = 2 * h * ncursors * nodesize;
toku_cachetable_create(&ct, cachesize, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
int c;
/* create the cursors */
for (c=0; c<ncursors; c++) {
r = toku_ft_cursor(brt, &cursors[c], NULL, false, false);
r = toku_ft_cursor(ft, &cursors[c], NULL, false, false);
assert(r == 0);
}
@@ -686,7 +686,7 @@ static void test_multiple_ft_cursor_walk(int n) {
toku_fill_dbt(&key, &k, sizeof k);
toku_fill_dbt(&val, &v, sizeof v);
toku_ft_insert(brt, &key, &val, 0);
toku_ft_insert(ft, &key, &val, 0);
}
/* point cursor i / cursor_gap to the current last key i */
@@ -720,7 +720,7 @@ static void test_multiple_ft_cursor_walk(int n) {
toku_ft_cursor_close(cursors[i]);
}
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
@@ -731,14 +731,14 @@ static void test_ft_cursor_set(int n, int cursor_op) {
int r;
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
FT_CURSOR cursor=0;
unlink(fname);
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
int i;
@@ -750,10 +750,10 @@ static void test_ft_cursor_set(int n, int cursor_op) {
DBT key,val;
toku_fill_dbt(&key, &k, sizeof k);
toku_fill_dbt(&val, &v, sizeof v);
toku_ft_insert(brt, &key, &val, 0);
toku_ft_insert(ft, &key, &val, 0);
}
r = toku_ft_cursor(brt, &cursor, NULL, false, false);
r = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(r==0);
/* set cursor to random keys in set { 0, 10, 20, .. 10*(n-1) } */
@@ -788,7 +788,7 @@ static void test_ft_cursor_set(int n, int cursor_op) {
toku_ft_cursor_close(cursor);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
@@ -799,14 +799,14 @@ static void test_ft_cursor_set_range(int n) {
int r;
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
FT_CURSOR cursor=0;
unlink(fname);
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(r==0);
int i;
@@ -819,10 +819,10 @@ static void test_ft_cursor_set_range(int n) {
DBT key, val;
toku_fill_dbt(&key, &k, sizeof k);
toku_fill_dbt(&val, &v, sizeof v);
toku_ft_insert(brt, &key, &val, 0);
toku_ft_insert(ft, &key, &val, 0);
}
r = toku_ft_cursor(brt, &cursor, NULL, false, false);
r = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(r==0);
/* pick random keys v in 0 <= v < 10*n, the cursor should point
@@ -848,7 +848,7 @@ static void test_ft_cursor_set_range(int n) {
toku_ft_cursor_close(cursor);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
toku_cachetable_close(&ct);
@@ -859,17 +859,17 @@ static void test_ft_cursor_delete(int n) {
int error;
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
FT_CURSOR cursor=0;
unlink(fname);
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare);
assert(error == 0);
error = toku_ft_cursor(brt, &cursor, NULL, false, false);
error = toku_ft_cursor(ft, &cursor, NULL, false, false);
assert(error == 0);
DBT key, val;
@@ -882,7 +882,7 @@ static void test_ft_cursor_delete(int n) {
v = i;
toku_fill_dbt(&key, &k, sizeof k);
toku_fill_dbt(&val, &v, sizeof v);
toku_ft_insert(brt, &key, &val, 0);
toku_ft_insert(ft, &key, &val, 0);
}
/* walk the tree and delete under the cursor */
@@ -905,7 +905,7 @@ static void test_ft_cursor_delete(int n) {
toku_ft_cursor_close(cursor);
error = toku_close_ft_handle_nolsn(brt, 0);
error = toku_close_ft_handle_nolsn(ft, 0);
assert(error == 0);
toku_cachetable_close(&ct);

2
storage/tokudb/ft-index/ft/tests/ft-test-header.cc
View File

@@ -92,7 +92,7 @@ PATENT RIGHTS GRANT:
#include "test.h"
// The purpose of this test is to verify that certain information in the
// brt_header is properly serialized and deserialized.
// ft_header is properly serialized and deserialized.
static TOKUTXN const null_txn = 0;

118
storage/tokudb/ft-index/ft/tests/ft-test.cc
View File

@@ -214,7 +214,7 @@ static void test_multiple_ft_handles_one_db_one_file (void) {
/* Check to see if data can be read that was written. */
static void test_read_what_was_written (void) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
int r;
const int NVALS=10000;
@@ -224,34 +224,34 @@ static void test_read_what_was_written (void) {
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
toku_cachetable_close(&ct);
/* Now see if we can read an empty tree in. */
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 0, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
/* See if we can put something in it. */
{
DBT k,v;
toku_ft_insert(brt, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
toku_ft_insert(ft, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
}
r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
toku_cachetable_close(&ct);
/* Now see if we can read it in and get the value. */
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 0, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
ft_lookup_and_check_nodup(brt, "hello", "there");
ft_lookup_and_check_nodup(ft, "hello", "there");
assert(toku_verify_ft(brt)==0);
assert(toku_verify_ft(ft)==0);
/* Now put a bunch (NVALS) of things in. */
{
@@ -262,14 +262,14 @@ static void test_read_what_was_written (void) {
snprintf(key, 100, "key%d", i);
snprintf(val, 100, "val%d", i);
if (i<600) {
int verify_result=toku_verify_ft(brt);;
int verify_result=toku_verify_ft(ft);;
assert(verify_result==0);
}
toku_ft_insert(brt, toku_fill_dbt(&k, key, strlen(key)+1), toku_fill_dbt(&v, val, strlen(val)+1), null_txn);
toku_ft_insert(ft, toku_fill_dbt(&k, key, strlen(key)+1), toku_fill_dbt(&v, val, strlen(val)+1), null_txn);
if (i<600) {
int verify_result=toku_verify_ft(brt);
int verify_result=toku_verify_ft(ft);
if (verify_result) {
r = toku_dump_ft(stdout, brt);
r = toku_dump_ft(stdout, ft);
assert(r==0);
assert(0);
}
@@ -279,7 +279,7 @@ static void test_read_what_was_written (void) {
char expectedval[100];
snprintf(key, 100, "key%d", j);
snprintf(expectedval, 100, "val%d", j);
ft_lookup_and_check_nodup(brt, key, expectedval);
ft_lookup_and_check_nodup(ft, key, expectedval);
}
}
}
@@ -287,9 +287,9 @@ static void test_read_what_was_written (void) {
}
if (verbose) printf("Now read them out\n");
r = toku_verify_ft(brt);
r = toku_verify_ft(ft);
assert(r==0);
//dump_ft(brt);
//dump_ft(ft);
/* See if we can read them all out again. */
{
@@ -298,31 +298,31 @@ static void test_read_what_was_written (void) {
char key[100],expectedval[100];
snprintf(key, 100, "key%d", i);
snprintf(expectedval, 100, "val%d", i);
ft_lookup_and_check_nodup(brt, key, expectedval);
ft_lookup_and_check_nodup(ft, key, expectedval);
}
}
r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
if (verbose) printf("%s:%d About to close %p\n", __FILE__, __LINE__, ct);
toku_cachetable_close(&ct);
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 0, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
ft_lookup_and_check_nodup(brt, "hello", "there");
ft_lookup_and_check_nodup(ft, "hello", "there");
{
int i;
for (i=0; i<NVALS; i++) {
char key[100],expectedval[100];
snprintf(key, 100, "key%d", i);
snprintf(expectedval, 100, "val%d", i);
ft_lookup_and_check_nodup(brt, key, expectedval);
ft_lookup_and_check_nodup(ft, key, expectedval);
}
}
r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
toku_cachetable_close(&ct);
@@ -334,7 +334,7 @@ static void test_read_what_was_written (void) {
/* Test c_get(DB_LAST) on an empty tree */
static void test_cursor_last_empty(void) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
FT_CURSOR cursor=0;
int r;
if (verbose) printf("%s", __FUNCTION__);
@@ -343,9 +343,9 @@ static void test_cursor_last_empty(void) {
//printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
//printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
//printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
r = toku_ft_cursor(brt, &cursor, NULL, false, false); assert(r==0);
r = toku_ft_cursor(ft, &cursor, NULL, false, false); assert(r==0);
{
struct check_pair pair = {0,0,0,0,0};
r = toku_ft_cursor_get(cursor, NULL, lookup_checkf, &pair, DB_LAST);
@@ -359,7 +359,7 @@ static void test_cursor_last_empty(void) {
assert(r==DB_NOTFOUND);
}
toku_ft_cursor_close(cursor);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
//printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
toku_cachetable_close(&ct);
//printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
@@ -368,7 +368,7 @@ static void test_cursor_last_empty(void) {
static void test_cursor_next (void) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
FT_CURSOR cursor=0;
int r;
DBT kbt, vbt;
@@ -377,12 +377,12 @@ static void test_cursor_next (void) {
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
//printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0);
//printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
toku_ft_insert(brt, toku_fill_dbt(&kbt, "hello", 6), toku_fill_dbt(&vbt, "there", 6), null_txn);
toku_ft_insert(brt, toku_fill_dbt(&kbt, "byebye", 7), toku_fill_dbt(&vbt, "byenow", 7), null_txn);
toku_ft_insert(ft, toku_fill_dbt(&kbt, "hello", 6), toku_fill_dbt(&vbt, "there", 6), null_txn);
toku_ft_insert(ft, toku_fill_dbt(&kbt, "byebye", 7), toku_fill_dbt(&vbt, "byenow", 7), null_txn);
if (verbose) printf("%s:%d calling toku_ft_cursor(...)\n", __FILE__, __LINE__);
r = toku_ft_cursor(brt, &cursor, NULL, false, false); assert(r==0);
r = toku_ft_cursor(ft, &cursor, NULL, false, false); assert(r==0);
toku_init_dbt(&kbt);
//printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
toku_init_dbt(&vbt);
@@ -411,7 +411,7 @@ static void test_cursor_next (void) {
}
toku_ft_cursor_close(cursor);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
//printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
toku_cachetable_close(&ct);
//printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
@@ -436,7 +436,7 @@ static int wrong_compare_fun(DB* UU(desc), const DBT *a, const DBT *b) {
static void test_wrongendian_compare (int wrong_p, unsigned int N) {
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
int r;
unsigned int i;
@@ -455,7 +455,7 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
//printf("%s:%d WRONG=%d\n", __FILE__, __LINE__, wrong_p);
if (0) { // ???? Why is this commented out?
r = toku_open_ft_handle(fname, 1, &brt, 1<<20, 1<<17, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, wrong_p ? wrong_compare_fun : toku_builtin_compare_fun); assert(r==0);
r = toku_open_ft_handle(fname, 1, &ft, 1<<20, 1<<17, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, wrong_p ? wrong_compare_fun : toku_builtin_compare_fun); assert(r==0);
for (i=1; i<257; i+=255) {
unsigned char a[4],b[4];
b[3] = a[0] = (unsigned char)(i&255);
@@ -470,11 +470,11 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
printf("%s:%d insert: %02x%02x%02x%02x -> %02x%02x%02x%02x\n", __FILE__, __LINE__,
((char*)kbt.data)[0], ((char*)kbt.data)[1], ((char*)kbt.data)[2], ((char*)kbt.data)[3],
((char*)vbt.data)[0], ((char*)vbt.data)[1], ((char*)vbt.data)[2], ((char*)vbt.data)[3]);
toku_ft_insert(brt, &kbt, &vbt, null_txn);
toku_ft_insert(ft, &kbt, &vbt, null_txn);
}
{
FT_CURSOR cursor=0;
r = toku_ft_cursor(brt, &cursor, NULL, false, false); assert(r==0);
r = toku_ft_cursor(ft, &cursor, NULL, false, false); assert(r==0);
for (i=0; i<2; i++) {
unsigned char a[4],b[4];
@@ -489,13 +489,13 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
}
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
}
}
{
toku_cachetable_verify(ct);
r = toku_open_ft_handle(fname, 1, &brt, 1<<20, 1<<17, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, wrong_p ? wrong_compare_fun : toku_builtin_compare_fun); assert(r==0);
r = toku_open_ft_handle(fname, 1, &ft, 1<<20, 1<<17, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, wrong_p ? wrong_compare_fun : toku_builtin_compare_fun); assert(r==0);
toku_cachetable_verify(ct);
for (i=0; i<N; i++) {
@@ -511,11 +511,11 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
if (0) printf("%s:%d insert: %02x%02x%02x%02x -> %02x%02x%02x%02x\n", __FILE__, __LINE__,
((unsigned char*)kbt.data)[0], ((unsigned char*)kbt.data)[1], ((unsigned char*)kbt.data)[2], ((unsigned char*)kbt.data)[3],
((unsigned char*)vbt.data)[0], ((unsigned char*)vbt.data)[1], ((unsigned char*)vbt.data)[2], ((unsigned char*)vbt.data)[3]);
toku_ft_insert(brt, &kbt, &vbt, null_txn);
toku_ft_insert(ft, &kbt, &vbt, null_txn);
toku_cachetable_verify(ct);
}
FT_CURSOR cursor=0;
r = toku_ft_cursor(brt, &cursor, NULL, false, false); assert(r==0);
r = toku_ft_cursor(ft, &cursor, NULL, false, false); assert(r==0);
for (i=0; i<N; i++) {
unsigned char a[4],b[4];
@@ -530,7 +530,7 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
toku_cachetable_verify(ct);
}
toku_ft_cursor_close(cursor);
r = toku_close_ft_handle_nolsn(brt, 0);
r = toku_close_ft_handle_nolsn(ft, 0);
assert(r==0);
}
toku_cachetable_close(&ct);
@@ -806,8 +806,8 @@ static void test_ft_delete_cursor_first(int n) {
toku_cachetable_close(&ct);
}
/* test for bug: insert cmd in a nonleaf node, delete removes the
insert cmd, but lookup finds the insert cmd
/* test for bug: insert message in a nonleaf node, delete removes the
insert message, but lookup finds the insert message
build a 2 level tree, and expect the last insertion to be
buffered. then delete and lookup. */
@@ -881,22 +881,22 @@ static void test_ft_delete(void) {
static void test_new_ft_cursor_create_close (void) {
int r;
FT_HANDLE brt=0;
FT_HANDLE ft=0;
int n = 8;
FT_CURSOR cursors[n];
toku_ft_handle_create(&brt);
toku_ft_handle_create(&ft);
int i;
for (i=0; i<n; i++) {
r = toku_ft_cursor(brt, &cursors[i], NULL, false, false); assert(r == 0);
r = toku_ft_cursor(ft, &cursors[i], NULL, false, false); assert(r == 0);
}
for (i=0; i<n; i++) {
toku_ft_cursor_close(cursors[i]);
}
r = toku_close_ft_handle_nolsn(brt, 0); assert(r == 0);
r = toku_close_ft_handle_nolsn(ft, 0); assert(r == 0);
}
static void test_new_ft_cursor_first(int n) {
@@ -1177,14 +1177,14 @@ static void test_new_ft_cursor_set_range(int n) {
int r;
CACHETABLE ct;
FT_HANDLE brt=0;
FT_HANDLE ft=0;
FT_CURSOR cursor=0;
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
unlink(fname);
toku_ft_handle_create(&brt);
toku_ft_handle_set_nodesize(brt, 4096);
r = toku_ft_handle_open(brt, fname, 1, 1, ct, null_txn); assert(r==0);
toku_ft_handle_create(&ft);
toku_ft_handle_set_nodesize(ft, 4096);
r = toku_ft_handle_open(ft, fname, 1, 1, ct, null_txn); assert(r==0);
int i;
@@ -1194,10 +1194,10 @@ static void test_new_ft_cursor_set_range(int n) {
DBT key, val;
int k = toku_htonl(10*i);
int v = 10*i;
toku_ft_insert(brt, toku_fill_dbt(&key, &k, sizeof k), toku_fill_dbt(&val, &v, sizeof v), 0); assert(r == 0);
toku_ft_insert(ft, toku_fill_dbt(&key, &k, sizeof k), toku_fill_dbt(&val, &v, sizeof v), 0); assert(r == 0);
}
r = toku_ft_cursor(brt, &cursor, NULL, false, false); assert(r==0);
r = toku_ft_cursor(ft, &cursor, NULL, false, false); assert(r==0);
/* pick random keys v in 0 <= v < 10*n, the cursor should point
to the smallest key in the tree that is >= v */
@@ -1226,7 +1226,7 @@ static void test_new_ft_cursor_set_range(int n) {
toku_ft_cursor_close(cursor);
r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
toku_cachetable_close(&ct);
}
@@ -1236,14 +1236,14 @@ static void test_new_ft_cursor_set(int n, int cursor_op, DB *db) {
int r;
CACHETABLE ct;
FT_HANDLE brt;
FT_HANDLE ft;
FT_CURSOR cursor=0;
unlink(fname);
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
r = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare); assert(r==0);
r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_ft_cursor_keycompare); assert(r==0);
int i;
@@ -1252,10 +1252,10 @@ static void test_new_ft_cursor_set(int n, int cursor_op, DB *db) {
DBT key, val;
int k = toku_htonl(10*i);
int v = 10*i;
toku_ft_insert(brt, toku_fill_dbt(&key, &k, sizeof k), toku_fill_dbt(&val, &v, sizeof v), 0); assert(r == 0);
toku_ft_insert(ft, toku_fill_dbt(&key, &k, sizeof k), toku_fill_dbt(&val, &v, sizeof v), 0); assert(r == 0);
}
r = toku_ft_cursor(brt, &cursor, NULL, false, false); assert(r==0);
r = toku_ft_cursor(ft, &cursor, NULL, false, false); assert(r==0);
/* set cursor to random keys in set { 0, 10, 20, .. 10*(n-1) } */
for (i=0; i<n; i++) {
@@ -1287,7 +1287,7 @@ static void test_new_ft_cursor_set(int n, int cursor_op, DB *db) {
toku_ft_cursor_close(cursor);
r = toku_close_ft_handle_nolsn(brt, 0); assert(r==0);
r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0);
toku_cachetable_close(&ct);
}

6
storage/tokudb/ft-index/ft/tests/ftloader-test-bad-generate.cc
View File

@@ -158,19 +158,19 @@ static void test_extractor(int nrows, int nrowsets, bool expect_fail) {
// open the ft_loader. this runs the extractor.
const int N = 1;
FT_HANDLE brts[N];
FT_HANDLE fts[N];
DB* dbs[N];
const char *fnames[N];
ft_compare_func compares[N];
for (int i = 0; i < N; i++) {
brts[i] = NULL;
fts[i] = NULL;
dbs[i] = NULL;
fnames[i] = "";
compares[i] = compare_int;
}
FTLOADER loader;
r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, nullptr, true, 0, false);
r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, fts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, nullptr, true, 0, false);
assert(r == 0);
struct rowset *rowset[nrowsets];

6
storage/tokudb/ft-index/ft/tests/ftloader-test-extractor-errors.cc
View File

@@ -165,12 +165,12 @@ static void test_extractor(int nrows, int nrowsets, bool expect_fail, const char
// open the ft_loader. this runs the extractor.
const int N = 1;
FT_HANDLE brts[N];
FT_HANDLE fts[N];
DB* dbs[N];
const char *fnames[N];
ft_compare_func compares[N];
for (int i = 0; i < N; i++) {
brts[i] = NULL;
fts[i] = NULL;
dbs[i] = NULL;
fnames[i] = "";
compares[i] = compare_int;
@@ -180,7 +180,7 @@ static void test_extractor(int nrows, int nrowsets, bool expect_fail, const char
sprintf(temp, "%s/%s", testdir, "tempXXXXXX");
FTLOADER loader;
r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, nullptr, true, 0, false);
r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, fts, dbs, fnames, compares, "tempXXXXXX", ZERO_LSN, nullptr, true, 0, false);
assert(r == 0);
struct rowset *rowset[nrowsets];

8
storage/tokudb/ft-index/ft/tests/ftloader-test-extractor.cc
View File

@@ -89,7 +89,7 @@ PATENT RIGHTS GRANT:
#ident "Copyright (c) 2010-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
// The purpose of this test is to test the extractor component of the brt loader. We insert rowsets into the extractor queue and verify temp files
// The purpose of this test is to test the extractor component of the ft loader. We insert rowsets into the extractor queue and verify temp files
// after the extractor is finished.
#define DONT_DEPRECATE_MALLOC
@@ -387,12 +387,12 @@ static void test_extractor(int nrows, int nrowsets, const char *testdir) {
// open the ft_loader. this runs the extractor.
const int N = 1;
FT_HANDLE brts[N];
FT_HANDLE fts[N];
DB* dbs[N];
const char *fnames[N];
ft_compare_func compares[N];
for (int i = 0; i < N; i++) {
brts[i] = NULL;
fts[i] = NULL;
dbs[i] = NULL;
fnames[i] = "";
compares[i] = compare_int;
@@ -402,7 +402,7 @@ static void test_extractor(int nrows, int nrowsets, const char *testdir) {
sprintf(temp, "%s/%s", testdir, "tempXXXXXX");
FTLOADER loader;
r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, brts, dbs, fnames, compares, temp, ZERO_LSN, nullptr, true, 0, false);
r = toku_ft_loader_open(&loader, NULL, generate, NULL, N, fts, dbs, fnames, compares, temp, ZERO_LSN, nullptr, true, 0, false);
assert(r == 0);
struct rowset *rowset[nrowsets];

6
storage/tokudb/ft-index/ft/tests/ftloader-test-merge-files-dbufio.cc
View File

@@ -385,7 +385,7 @@ static void test (const char *directory, bool is_error) {
}
FTLOADER bl;
FT_HANDLE *XCALLOC_N(N_DEST_DBS, brts);
FT_HANDLE *XCALLOC_N(N_DEST_DBS, fts);
DB* *XCALLOC_N(N_DEST_DBS, dbs);
const char **XMALLOC_N(N_DEST_DBS, new_fnames_in_env);
for (int i=0; i<N_DEST_DBS; i++) {
@@ -407,7 +407,7 @@ static void test (const char *directory, bool is_error) {
ct,
(generate_row_for_put_func)NULL,
(DB*)NULL,
N_DEST_DBS, brts, dbs,
N_DEST_DBS, fts, dbs,
new_fnames_in_env,
bt_compare_functions,
"tempxxxxxx",
@@ -522,7 +522,7 @@ static void test (const char *directory, bool is_error) {
destroy_dbufio_fileset(bfs);
toku_free(fnames);
toku_free(fds);
toku_free(brts);
toku_free(fts);
toku_free(dbs);
toku_free(new_fnames_in_env);
toku_free(bt_compare_functions);

6
storage/tokudb/ft-index/ft/tests/ftloader-test-open.cc
View File

@@ -126,12 +126,12 @@ static void test_loader_open(int ndbs) {
FTLOADER loader;
// open the ft_loader. this runs the extractor.
FT_HANDLE brts[ndbs];
FT_HANDLE fts[ndbs];
DB* dbs[ndbs];
const char *fnames[ndbs];
ft_compare_func compares[ndbs];
for (int i = 0; i < ndbs; i++) {
brts[i] = NULL;
fts[i] = NULL;
dbs[i] = NULL;
fnames[i] = "";
compares[i] = my_compare;
@@ -143,7 +143,7 @@ static void test_loader_open(int ndbs) {
for (i = 0; ; i++) {
set_my_malloc_trigger(i+1);
r = toku_ft_loader_open(&loader, NULL, NULL, NULL, ndbs, brts, dbs, fnames, compares, "", ZERO_LSN, nullptr, true, 0, false);
r = toku_ft_loader_open(&loader, NULL, NULL, NULL, ndbs, fts, dbs, fnames, compares, "", ZERO_LSN, nullptr, true, 0, false);
if (r == 0)
break;
}

2
storage/tokudb/ft-index/ft/tests/ftloader-test-writer-errors.cc
View File

@@ -213,7 +213,7 @@ static int write_dbfile (char *tf_template, int n, char *output_name, bool expec
ft_loader_set_error_function(&bl.error_callback, NULL, NULL);
ft_loader_set_poll_function(&bl.poll_callback, loader_poll_callback, NULL);
result = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
result = toku_loader_write_ft_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
toku_set_func_malloc_only(NULL);
toku_set_func_realloc_only(NULL);

6
storage/tokudb/ft-index/ft/tests/ftloader-test-writer.cc
View File

@@ -137,9 +137,9 @@ static void verify_dbfile(int n, const char *name) {
toku_ft_set_bt_compare(t, compare_ints);
r = toku_ft_handle_open(t, name, 0, 0, ct, null_txn); assert(r==0);
if (verbose) traceit("Verifying brt internals");
if (verbose) traceit("Verifying ft internals");
r = toku_verify_ft(t);
if (verbose) traceit("Verified brt internals");
if (verbose) traceit("Verified ft internals");
FT_CURSOR cursor = NULL;
r = toku_ft_cursor(t, &cursor, NULL, false, false); assert(r == 0);
@@ -262,7 +262,7 @@ static void test_write_dbfile (char *tf_template, int n, char *output_name, TXNI
assert(fd>=0);
if (verbose) traceit("write to file");
r = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
r = toku_loader_write_ft_from_q_in_C(&bl, &desc, fd, 1000, q2, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
assert(r==0);
r = queue_destroy(q2);

2
storage/tokudb/ft-index/ft/tests/ftloader-test.cc
View File

@@ -425,7 +425,7 @@ static void test_merge_files (const char *tf_template, const char *output_name)
int fd = open(output_name, O_RDWR | O_CREAT | O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO);
assert(fd>=0);
r = toku_loader_write_brt_from_q_in_C(&bl, &desc, fd, 1000, q, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
r = toku_loader_write_ft_from_q_in_C(&bl, &desc, fd, 1000, q, size_est, 0, 0, 0, TOKU_DEFAULT_COMPRESSION_METHOD, 16);
assert(r==0);
destroy_merge_fileset(&fs);

26
storage/tokudb/ft-index/ft/tests/is_empty.cc
View File

@@ -98,7 +98,7 @@ PATENT RIGHTS GRANT:
#define FILENAME "test0.ft"
static void test_it (int N) {
FT_HANDLE brt;
FT_HANDLE ft;
int r;
toku_os_recursive_delete(TOKU_TEST_FILENAME);
r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU); CKERR(r);
@@ -119,18 +119,18 @@ static void test_it (int N) {
TOKUTXN txn;
r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT, false); CKERR(r);
r = toku_open_ft_handle(FILENAME, 1, &brt, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun); CKERR(r);
r = toku_open_ft_handle(FILENAME, 1, &ft, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun); CKERR(r);
r = toku_txn_commit_txn(txn, false, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn);
CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct);
r = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT); CKERR(r);
r = toku_close_ft_handle_nolsn(brt, NULL); CKERR(r);
r = toku_close_ft_handle_nolsn(ft, NULL); CKERR(r);
unsigned int rands[N];
for (int i=0; i<N; i++) {
r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT, false); CKERR(r);
r = toku_open_ft_handle(FILENAME, 0, &brt, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun); CKERR(r);
r = toku_open_ft_handle(FILENAME, 0, &ft, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun); CKERR(r);
r = toku_txn_commit_txn(txn, false, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn);
@@ -141,19 +141,19 @@ static void test_it (int N) {
snprintf(key, sizeof(key), "key%x.%x", rands[i], i);
memset(val, 'v', sizeof(val));
val[sizeof(val)-1]=0;
toku_ft_insert(brt, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), txn);
toku_ft_insert(ft, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), txn);
r = toku_txn_commit_txn(txn, false, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn);
r = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT); CKERR(r);
r = toku_close_ft_handle_nolsn(brt, NULL); CKERR(r);
r = toku_close_ft_handle_nolsn(ft, NULL); CKERR(r);
if (verbose) printf("i=%d\n", i);
}
for (int i=0; i<N; i++) {
r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT, false); CKERR(r);
r = toku_open_ft_handle(FILENAME, 0, &brt, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun); CKERR(r);
r = toku_open_ft_handle(FILENAME, 0, &ft, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun); CKERR(r);
r = toku_txn_commit_txn(txn, false, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn);
@@ -161,11 +161,11 @@ static void test_it (int N) {
char key[100];
DBT k;
snprintf(key, sizeof(key), "key%x.%x", rands[i], i);
toku_ft_delete(brt, toku_fill_dbt(&k, key, 1+strlen(key)), txn);
toku_ft_delete(ft, toku_fill_dbt(&k, key, 1+strlen(key)), txn);
if (0) {
bool is_empty;
is_empty = toku_ft_is_empty_fast(brt);
is_empty = toku_ft_is_empty_fast(ft);
assert(!is_empty);
}
@@ -173,23 +173,23 @@ static void test_it (int N) {
toku_txn_close_txn(txn);
r = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT); CKERR(r);
r = toku_close_ft_handle_nolsn(brt, NULL); CKERR(r);
r = toku_close_ft_handle_nolsn(ft, NULL); CKERR(r);
if (verbose) printf("d=%d\n", i);
}
r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT, false); CKERR(r);
r = toku_open_ft_handle(FILENAME, 0, &brt, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun); CKERR(r);
r = toku_open_ft_handle(FILENAME, 0, &ft, 1024, 256, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, toku_builtin_compare_fun); CKERR(r);
r = toku_txn_commit_txn(txn, false, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn);
if (0) {
bool is_empty;
is_empty = toku_ft_is_empty_fast(brt);
is_empty = toku_ft_is_empty_fast(ft);
assert(is_empty);
}
r = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT); CKERR(r);
r = toku_close_ft_handle_nolsn(brt, NULL); CKERR(r);
r = toku_close_ft_handle_nolsn(ft, NULL); CKERR(r);
r = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT); CKERR(r);
toku_logger_close_rollback(logger);

4
storage/tokudb/ft-index/ft/tests/keyrange.cc
View File

@@ -132,8 +132,8 @@ static void reload (uint64_t limit) {
enum memory_state {
LEAVE_IN_MEMORY, // leave the state in main memory
CLOSE_AND_RELOAD, // close the brts and reload them into main memory (that will cause >1 partitio in many leaves.)
CLOSE_AND_REOPEN_LEAVE_ON_DISK // close the brts, reopen them, but leave the state on disk.
CLOSE_AND_RELOAD, // close the fts and reload them into main memory (that will cause >1 partitio in many leaves.)
CLOSE_AND_REOPEN_LEAVE_ON_DISK // close the fts, reopen them, but leave the state on disk.
};
static void maybe_reopen (enum memory_state ms, uint64_t limit) {

18
storage/tokudb/ft-index/ft/tests/le-cursor-provdel.cc
View File

@@ -138,8 +138,8 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE, false);
assert(error == 0);
FT_HANDLE brt = NULL;
error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_ft_cursor_keycompare);
FT_HANDLE ft = NULL;
error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_ft_cursor_keycompare);
assert(error == 0);
error = toku_txn_commit_txn(txn, true, NULL, NULL);
@@ -158,7 +158,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
toku_fill_dbt(&key, &k, sizeof k);
DBT val;
toku_fill_dbt(&val, &v, sizeof v);
toku_ft_insert(brt, &key, &val, txn);
toku_ft_insert(ft, &key, &val, txn);
assert(error == 0);
}
@@ -166,7 +166,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
assert(error == 0);
toku_txn_close_txn(txn);
error = toku_close_ft_handle_nolsn(brt, NULL);
error = toku_close_ft_handle_nolsn(ft, NULL);
assert(error == 0);
CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct);
@@ -208,8 +208,8 @@ test_provdel(const char *logdir, const char *fname, int n) {
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE, false);
assert(error == 0);
FT_HANDLE brt = NULL;
error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_ft_cursor_keycompare);
FT_HANDLE ft = NULL;
error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_ft_cursor_keycompare);
assert(error == 0);
error = toku_txn_commit_txn(txn, true, NULL, NULL);
@@ -225,7 +225,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
int k = toku_htonl(i);
DBT key;
toku_fill_dbt(&key, &k, sizeof k);
toku_ft_delete(brt, &key, txn);
toku_ft_delete(ft, &key, txn);
assert(error == 0);
}
@@ -234,7 +234,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
assert(error == 0);
LE_CURSOR cursor = NULL;
error = toku_le_cursor_create(&cursor, brt, cursortxn);
error = toku_le_cursor_create(&cursor, ft, cursortxn);
assert(error == 0);
DBT key;
@@ -267,7 +267,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
assert(error == 0);
toku_txn_close_txn(txn);
error = toku_close_ft_handle_nolsn(brt, NULL);
error = toku_close_ft_handle_nolsn(ft, NULL);
assert(error == 0);
CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct);
error = toku_checkpoint(cp, logger, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);

32
storage/tokudb/ft-index/ft/tests/le-cursor-right.cc
View File

@@ -142,8 +142,8 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE, false);
assert(error == 0);
FT_HANDLE brt = NULL;
error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_keycompare);
FT_HANDLE ft = NULL;
error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, txn, test_keycompare);
assert(error == 0);
error = toku_txn_commit_txn(txn, true, NULL, NULL);
@@ -162,14 +162,14 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
toku_fill_dbt(&key, &k, sizeof k);
DBT val;
toku_fill_dbt(&val, &v, sizeof v);
toku_ft_insert(brt, &key, &val, txn);
toku_ft_insert(ft, &key, &val, txn);
}
error = toku_txn_commit_txn(txn, true, NULL, NULL);
assert(error == 0);
toku_txn_close_txn(txn);
error = toku_close_ft_handle_nolsn(brt, NULL);
error = toku_close_ft_handle_nolsn(ft, NULL);
assert(error == 0);
CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct);
@@ -198,13 +198,13 @@ test_pos_infinity(const char *fname, int n) {
CACHETABLE ct = NULL;
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
FT_HANDLE brt = NULL;
error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
FT_HANDLE ft = NULL;
error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
assert(error == 0);
// position the cursor at -infinity
LE_CURSOR cursor = NULL;
error = toku_le_cursor_create(&cursor, brt, NULL);
error = toku_le_cursor_create(&cursor, ft, NULL);
assert(error == 0);
for (int i = 0; i < 2*n; i++) {
@@ -217,7 +217,7 @@ test_pos_infinity(const char *fname, int n) {
toku_le_cursor_close(cursor);
error = toku_close_ft_handle_nolsn(brt, 0);
error = toku_close_ft_handle_nolsn(ft, 0);
assert(error == 0);
toku_cachetable_close(&ct);
@@ -232,13 +232,13 @@ test_neg_infinity(const char *fname, int n) {
CACHETABLE ct = NULL;
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
FT_HANDLE brt = NULL;
error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
FT_HANDLE ft = NULL;
error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
assert(error == 0);
// position the LE_CURSOR at +infinity
LE_CURSOR cursor = NULL;
error = toku_le_cursor_create(&cursor, brt, NULL);
error = toku_le_cursor_create(&cursor, ft, NULL);
assert(error == 0);
DBT key;
@@ -271,7 +271,7 @@ test_neg_infinity(const char *fname, int n) {
toku_le_cursor_close(cursor);
error = toku_close_ft_handle_nolsn(brt, 0);
error = toku_close_ft_handle_nolsn(ft, 0);
assert(error == 0);
toku_cachetable_close(&ct);
@@ -286,13 +286,13 @@ test_between(const char *fname, int n) {
CACHETABLE ct = NULL;
toku_cachetable_create(&ct, 0, ZERO_LSN, NULL_LOGGER);
FT_HANDLE brt = NULL;
error = toku_open_ft_handle(fname, 1, &brt, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
FT_HANDLE ft = NULL;
error = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, test_keycompare);
assert(error == 0);
// position the LE_CURSOR at +infinity
LE_CURSOR cursor = NULL;
error = toku_le_cursor_create(&cursor, brt, NULL);
error = toku_le_cursor_create(&cursor, ft, NULL);
assert(error == 0);
DBT key;
@@ -337,7 +337,7 @@ test_between(const char *fname, int n) {
toku_le_cursor_close(cursor);
error = toku_close_ft_handle_nolsn(brt, 0);
error = toku_close_ft_handle_nolsn(ft, 0);
assert(error == 0);
toku_cachetable_close(&ct);

Some files were not shown because too many files have changed in this diff Show More