database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-12-24 14:20:59 +03:00
Code Activity

the begginning

Browse Source
This commit is contained in:
david hill
2016-01-06 14:08:59 -06:00
parent 66a31debcb
commit f6afc42dd0
18251 changed files with 16460679 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
writeengine/dictionary/Makefile Executable file
View File

@@ -0,0 +1,87 @@
include ../../rules.mak
include ../build/we_rules.mak
#******************************************************************************************
# Copyright (C) 2009-2012 Calpont Corporation
# All rights reserved
#*****************************************************************************************/
.PHONY: install clean docs test coverage leakcheck
SRCS=we_dctnry.cpp
LINCLUDES=we_dctnry.h
OBJS=$(SRCS:.cpp=.o)
LOBJS=$(LOBJS_SHARED)
GOBJS=$(LOBJS:.o=-gcov.o)
object: $(OBJS)
mkdir -p $(LIBDIR)
cp *.o $(LIBDIR)
rm -f $(LIBDIR)/tdctnry.o
make install
install: bootstrap
bootstrap:
$(INSTALL) $(LINCLUDES) $(INSTALL_ROOT_INCLUDE)
clean:
rm -f $(LIBDIR)/tdctnry.o $(OBJS) tdctnry.o tdctnry core *~ *.tag *-gcov.* *.gcov tdctnry-gcov *.d *.swp *.dat
for file in $(SRCS); do \
bfile=`basename $$file .cpp`; \
rm -f $(LIBDIR)/$${bfile}.o ; \
done
rm -rf html
docs:
doxygen $(EXPORT_ROOT)/etc/Doxyfile
tdctnry: $(OBJS) tdctnry.o
$(LINK.cpp) -o $@ $^ $(LOBJS) $(TLIBS)
test:
xtest: clean tdctnry object
$(IPCS_CLEANUP)
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib ./tdctnry
qdctnry: $(OBJS) qdctnry.o
$(LINK.cpp) -o $@ $^ $(LOBJS) $(TLIBS)
qtest: qdctnry object
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib ./qdctnry
%-gcov.o: %.cpp
$(COMPILE.cpp) -o $@ $^
tdctnry-gcov: CXXFLAGS+=-fprofile-arcs -ftest-coverage
tdctnry-gcov: tdctnry-gcov.o $(subst .o,-gcov.o,$(OBJS))
$(LINK.cpp) -o $@ $^ $(GOBJS) $(GLIBS)
cp *-gcov.o $(LIBDIR)
coverage:
xcoverage: tdctnry-gcov
$(IPCS_CLEANUP)
rm -f *.gcda
LD_LIBRARY_PATH=$(EXPORT_ROOT)/lib:/usr/local/lib ./tdctnry-gcov
for file in $(SRCS); do \
bfile=`basename $$file .cpp`; \
gcov -o $${bfile}-gcov $$file >/dev/null; \
done
/usr/local/bin/genCoverage.pl $(SRCS)
leakcheck:
xleakcheck: tdctnry
$(IPCS_CLEANUP)
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib valgrind --tool=memcheck --leak-check=yes ./tdctnry
%.d: %.cpp
@set -e; rm -f $@; \
$(CC) -MM $(CPPFLAGS) $< > $@.$$$$; \
sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$$ > $@; \
rm -f $@.$$$$
ifndef BOOTSTRAP
-include $(SRCS:.cpp=.d) tdctnry.d
endif

32
writeengine/dictionary/Makefile.am Normal file
View File

@@ -0,0 +1,32 @@
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
# $Id: Makefile.am 864 2009-04-02 19:22:49Z rdempsey $
## Process this file with automake to produce Makefile.in
include_HEADERS = we_dctnry.h
test:
coverage:
leakcheck:
docs:
bootstrap: install-data-am

454
writeengine/dictionary/Makefile.in Normal file
View File

@@ -0,0 +1,454 @@
# Makefile.in generated by automake 1.9.6 from Makefile.am.
# @configure_input@
# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
# 2003, 2004, 2005 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
# with or without modifications, as long as this notice is preserved.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.
@SET_MAKE@
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
# $Id: Makefile.am 864 2009-04-02 19:22:49Z rdempsey $
srcdir = @srcdir@
top_srcdir = @top_srcdir@
VPATH = @srcdir@
pkgdatadir = $(datadir)/@PACKAGE@
pkglibdir = $(libdir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
top_builddir = ../..
am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
INSTALL = @INSTALL@
install_sh_DATA = $(install_sh) -c -m 644
install_sh_PROGRAM = $(install_sh) -c
install_sh_SCRIPT = $(install_sh) -c
INSTALL_HEADER = $(INSTALL_DATA)
transform = $(program_transform_name)
NORMAL_INSTALL = :
PRE_INSTALL = :
POST_INSTALL = :
NORMAL_UNINSTALL = :
PRE_UNINSTALL = :
POST_UNINSTALL = :
build_triplet = @build@
host_triplet = @host@
subdir = writeengine/dictionary
DIST_COMMON = $(include_HEADERS) $(srcdir)/Makefile.am \
$(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/m4/compilerflags.m4 \
$(top_srcdir)/m4/functions.m4 $(top_srcdir)/m4/install.m4 \
$(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
mkinstalldirs = $(install_sh) -d
CONFIG_HEADER = $(top_builddir)/config.h
CONFIG_CLEAN_FILES =
SOURCES =
DIST_SOURCES =
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
am__vpath_adj = case $$p in \
$(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
*) f=$$p;; \
esac;
am__strip_dir = `echo $$p | sed -e 's|^.*/||'`;
am__installdirs = "$(DESTDIR)$(includedir)"
includeHEADERS_INSTALL = $(INSTALL_HEADER)
HEADERS = $(include_HEADERS)
ETAGS = etags
CTAGS = ctags
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
ACLOCAL = @ACLOCAL@
ALLOCA = @ALLOCA@
AMDEP_FALSE = @AMDEP_FALSE@
AMDEP_TRUE = @AMDEP_TRUE@
AMTAR = @AMTAR@
AR = @AR@
AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
CC = @CC@
CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@
CPP = @CPP@
CPPFLAGS = @CPPFLAGS@
CXX = @CXX@
CXXCPP = @CXXCPP@
CXXDEPMODE = @CXXDEPMODE@
CXXFLAGS = @CXXFLAGS@
CYGPATH_W = @CYGPATH_W@
DEFS = @DEFS@
DEPDIR = @DEPDIR@
ECHO = @ECHO@
ECHO_C = @ECHO_C@
ECHO_N = @ECHO_N@
ECHO_T = @ECHO_T@
EGREP = @EGREP@
EXEEXT = @EXEEXT@
F77 = @F77@
FFLAGS = @FFLAGS@
INSTALL_DATA = @INSTALL_DATA@
INSTALL_PROGRAM = @INSTALL_PROGRAM@
INSTALL_SCRIPT = @INSTALL_SCRIPT@
INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
LDFLAGS = @LDFLAGS@
LEX = @LEX@
LEXLIB = @LEXLIB@
LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
LIBOBJS = @LIBOBJS@
LIBS = @LIBS@
LIBTOOL = @LIBTOOL@
LN_S = @LN_S@
LTLIBOBJS = @LTLIBOBJS@
MAKEINFO = @MAKEINFO@
OBJEXT = @OBJEXT@
PACKAGE = @PACKAGE@
PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_STRING = @PACKAGE_STRING@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
PATH_SEPARATOR = @PATH_SEPARATOR@
POW_LIB = @POW_LIB@
RANLIB = @RANLIB@
SED = @SED@
SET_MAKE = @SET_MAKE@
SHELL = @SHELL@
STRIP = @STRIP@
VERSION = @VERSION@
XML2_CONFIG = @XML2_CONFIG@
XML_CPPFLAGS = @XML_CPPFLAGS@
XML_LIBS = @XML_LIBS@
YACC = @YACC@
ac_ct_AR = @ac_ct_AR@
ac_ct_CC = @ac_ct_CC@
ac_ct_CXX = @ac_ct_CXX@
ac_ct_F77 = @ac_ct_F77@
ac_ct_RANLIB = @ac_ct_RANLIB@
ac_ct_STRIP = @ac_ct_STRIP@
am__fastdepCC_FALSE = @am__fastdepCC_FALSE@
am__fastdepCC_TRUE = @am__fastdepCC_TRUE@
am__fastdepCXX_FALSE = @am__fastdepCXX_FALSE@
am__fastdepCXX_TRUE = @am__fastdepCXX_TRUE@
am__include = @am__include@
am__leading_dot = @am__leading_dot@
am__quote = @am__quote@
am__tar = @am__tar@
am__untar = @am__untar@
bindir = @bindir@
build = @build@
build_alias = @build_alias@
build_cpu = @build_cpu@
build_os = @build_os@
build_vendor = @build_vendor@
datadir = @datadir@
etcdir = @etcdir@
exec_prefix = @exec_prefix@
host = @host@
host_alias = @host_alias@
host_cpu = @host_cpu@
host_os = @host_os@
host_vendor = @host_vendor@
idb_brm_libs = @idb_brm_libs@
idb_cflags = @idb_cflags@
idb_common_includes = @idb_common_includes@
idb_common_ldflags = @idb_common_ldflags@
idb_common_libs = @idb_common_libs@
idb_cppflags = @idb_cppflags@
idb_cxxflags = @idb_cxxflags@
idb_exec_libs = @idb_exec_libs@
idb_ldflags = @idb_ldflags@
idb_oam_libs = @idb_oam_libs@
idb_write_libs = @idb_write_libs@
idbinstall = @idbinstall@
includedir = @includedir@
infodir = @infodir@
install_sh = @install_sh@
libdir = @libdir@
libexecdir = @libexecdir@
localdir = @localdir@
localstatedir = @localstatedir@
mandir = @mandir@
march_flags = @march_flags@
mibdir = @mibdir@
mkdir_p = @mkdir_p@
mysqldir = @mysqldir@
netsnmp_libs = @netsnmp_libs@
netsnmpagntdir = @netsnmpagntdir@
netsnmpdir = @netsnmpdir@
netsnmplibrdir = @netsnmplibrdir@
netsnmpmachdir = @netsnmpmachdir@
netsnmpsysdir = @netsnmpsysdir@
oldincludedir = @oldincludedir@
postdir = @postdir@
prefix = @prefix@
program_transform_name = @program_transform_name@
sbindir = @sbindir@
sharedir = @sharedir@
sharedstatedir = @sharedstatedir@
sysconfdir = @sysconfdir@
target_alias = @target_alias@
toolsdir = @toolsdir@
include_HEADERS = we_dctnry.h
all: all-am
.SUFFIXES:
$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps)
@for dep in $?; do \
case '$(am__configure_deps)' in \
*$$dep*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh \
&& exit 0; \
exit 1;; \
esac; \
done; \
echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu writeengine/dictionary/Makefile'; \
cd $(top_srcdir) && \
$(AUTOMAKE) --gnu writeengine/dictionary/Makefile
.PRECIOUS: Makefile
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
@case '$?' in \
*config.status*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
*) \
echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
esac;
$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(top_srcdir)/configure: $(am__configure_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(ACLOCAL_M4): $(am__aclocal_m4_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
mostlyclean-libtool:
-rm -f *.lo
clean-libtool:
-rm -rf .libs _libs
distclean-libtool:
-rm -f libtool
uninstall-info-am:
install-includeHEADERS: $(include_HEADERS)
@$(NORMAL_INSTALL)
test -z "$(includedir)" || $(mkdir_p) "$(DESTDIR)$(includedir)"
@list='$(include_HEADERS)'; for p in $$list; do \
if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
f=$(am__strip_dir) \
echo " $(includeHEADERS_INSTALL) '$$d$$p' '$(DESTDIR)$(includedir)/$$f'"; \
$(includeHEADERS_INSTALL) "$$d$$p" "$(DESTDIR)$(includedir)/$$f"; \
done
uninstall-includeHEADERS:
@$(NORMAL_UNINSTALL)
@list='$(include_HEADERS)'; for p in $$list; do \
f=$(am__strip_dir) \
echo " rm -f '$(DESTDIR)$(includedir)/$$f'"; \
rm -f "$(DESTDIR)$(includedir)/$$f"; \
done
ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
mkid -fID $$unique
tags: TAGS
TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$(TAGS_FILES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
if test -z "$(ETAGS_ARGS)$$tags$$unique"; then :; else \
test -n "$$unique" || unique=$$empty_fix; \
$(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
$$tags $$unique; \
fi
ctags: CTAGS
CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$(TAGS_FILES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
test -z "$(CTAGS_ARGS)$$tags$$unique" \
|| $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
$$tags $$unique
GTAGS:
here=`$(am__cd) $(top_builddir) && pwd` \
&& cd $(top_srcdir) \
&& gtags -i $(GTAGS_ARGS) $$here
distclean-tags:
-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \
topsrcdirstrip=`echo "$(top_srcdir)" | sed 's|.|.|g'`; \
list='$(DISTFILES)'; for file in $$list; do \
case $$file in \
$(srcdir)/*) file=`echo "$$file" | sed "s|^$$srcdirstrip/||"`;; \
$(top_srcdir)/*) file=`echo "$$file" | sed "s|^$$topsrcdirstrip/|$(top_builddir)/|"`;; \
esac; \
if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
dir=`echo "$$file" | sed -e 's,/[^/]*$$,,'`; \
if test "$$dir" != "$$file" && test "$$dir" != "."; then \
dir="/$$dir"; \
$(mkdir_p) "$(distdir)$$dir"; \
else \
dir=''; \
fi; \
if test -d $$d/$$file; then \
if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
cp -pR $(srcdir)/$$file $(distdir)$$dir || exit 1; \
fi; \
cp -pR $$d/$$file $(distdir)$$dir || exit 1; \
else \
test -f $(distdir)/$$file \
|| cp -p $$d/$$file $(distdir)/$$file \
|| exit 1; \
fi; \
done
check-am: all-am
check: check-am
all-am: Makefile $(HEADERS)
installdirs:
for dir in "$(DESTDIR)$(includedir)"; do \
test -z "$$dir" || $(mkdir_p) "$$dir"; \
done
install: install-am
install-exec: install-exec-am
install-data: install-data-am
uninstall: uninstall-am
install-am: all-am
@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
installcheck: installcheck-am
install-strip:
$(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
`test -z '$(STRIP)' || \
echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install
mostlyclean-generic:
clean-generic:
distclean-generic:
-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
maintainer-clean-generic:
@echo "This command is intended for maintainers to use"
@echo "it deletes files that may require special tools to rebuild."
clean: clean-am
clean-am: clean-generic clean-libtool mostlyclean-am
distclean: distclean-am
-rm -f Makefile
distclean-am: clean-am distclean-generic distclean-libtool \
distclean-tags
dvi: dvi-am
dvi-am:
html: html-am
info: info-am
info-am:
install-data-am: install-includeHEADERS
install-exec-am:
install-info: install-info-am
install-man:
installcheck-am:
maintainer-clean: maintainer-clean-am
-rm -f Makefile
maintainer-clean-am: distclean-am maintainer-clean-generic
mostlyclean: mostlyclean-am
mostlyclean-am: mostlyclean-generic mostlyclean-libtool
pdf: pdf-am
pdf-am:
ps: ps-am
ps-am:
uninstall-am: uninstall-includeHEADERS uninstall-info-am
.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \
clean-libtool ctags distclean distclean-generic \
distclean-libtool distclean-tags distdir dvi dvi-am html \
html-am info info-am install install-am install-data \
install-data-am install-exec install-exec-am \
install-includeHEADERS install-info install-info-am \
install-man install-strip installcheck installcheck-am \
installdirs maintainer-clean maintainer-clean-generic \
mostlyclean mostlyclean-generic mostlyclean-libtool pdf pdf-am \
ps ps-am tags uninstall uninstall-am uninstall-includeHEADERS \
uninstall-info-am
test:
coverage:
leakcheck:
docs:
bootstrap: install-data-am
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
.NOEXPORT:

776
writeengine/dictionary/qdctnry.cpp Normal file
View File

@@ -0,0 +1,776 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <stdio.h>
#include <string>
#include <stdexcept>
using namespace std;
#include <boost/scoped_ptr.hpp>
using namespace boost;
#include <cppunit/extensions/HelperMacros.h>
#include "we_dctnry.h"
#include "we_dctnrystore.h"
using namespace WriteEngine;
class DctnryTest : public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE( DctnryTest );
// Dctnry basic testing
//CPPUNIT_TEST( testDctnryInsertLarge );
//CPPUNIT_TEST( testDctnryInsertDelete );
///CPPUNIT_TEST( testDctnryInsertStress);
//CPPUNIT_TEST( testGetBlockHdr);
//CPPUNIT_TEST( testGetFreeSpace);
CPPUNIT_TEST( testDctnryInsertDeleteStore);
CPPUNIT_TEST_SUITE_END();
private:
Dctnry m_Dctnry;
DctnryStore m_DctnryStore;
int m_oId;
public:
void setUp() {
}
void tearDown() {
}
DctnryTest()
{
m_oId =0;
}
void testDctnryInsertLarge() {
int oId=2000;
// int blockCount =10;
FILE* dFile =NULL;
int rc =0;
DataBlock curBlock;
// int largeSize=6144;
int largeSize=2724;
int smallSize=2032;
uint16_t freeSpace;
uint64_t nextPtr;
uint16_t offSet0;
uint16_t endHeader;
unsigned char dctnryHeader[14];
m_Dctnry.setUseSmallSize(false);
m_Dctnry.setDebugLevel( DEBUG_3 );
m_oId = oId;
memset(curBlock.data,0, sizeof (curBlock.data));
memset(dctnryHeader,0, sizeof(dctnryHeader));
printf("\nRunning testDctnryInsertLarge \n");
rc = m_Dctnry.deleteFile(oId);
rc = m_Dctnry.createDctnry(oId,3);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_Dctnry.openDctnry(oId);
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
rc = m_Dctnry.initDctnryHdr( dFile);
CPPUNIT_ASSERT( rc == NO_ERROR );
m_Dctnry.closeDctnry();
printf("After initDctnryHdr \n");
testGetBlockHdr();
//dFile= m_Dctnry.openFile( oId);
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
//12 for 6 bytes and 14 for 8 bytes
rc =m_Dctnry.readSubBlockEntry( dFile, &curBlock, 0, 0, 0, 14, &dctnryHeader);
CPPUNIT_ASSERT( rc == NO_ERROR );
memcpy(&freeSpace,dctnryHeader,2);
memcpy(&nextPtr,dctnryHeader+2,8); // 8 bytes
memcpy(&offSet0,dctnryHeader+10,2);
memcpy(&endHeader,dctnryHeader+12,2);
unsigned char sgnature_value[largeSize];
memset(sgnature_value,0, sizeof(sgnature_value));
int j=0;
for (int i=0; i<largeSize; i++)
{
// if (j>255)
// j=0;
sgnature_value[i]=119;
// j++;
}
//insert a signature value
Token token;
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token);
CPPUNIT_ASSERT( rc == NO_ERROR );
testGetBlockHdr();
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token);
CPPUNIT_ASSERT( rc == NO_ERROR );
testGetBlockHdr();
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token);
CPPUNIT_ASSERT( rc == NO_ERROR );
testGetBlockHdr();
token.fbo = 0;
token.op = 3;
rc =m_Dctnry.deleteDctnryValue( dFile, token);
testGetBlockHdr();
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token);
CPPUNIT_ASSERT( rc == NO_ERROR );
testGetBlockHdr();
return;
//add the same dictionary again. it should not be added
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token);
testGetBlockHdr();
//add the same dictionary again. it should not be added to the next block
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token);
testGetBlockHdr();
printf("token fbo = %i",token.fbo);
printf("token op = %i",token.op);
token.fbo = 0;
token.op = 2;
// token.op++;
rc =m_Dctnry.deleteDctnryValue( dFile, token);
testGetBlockHdr();
return;
//add a 2nd dictionary to fillup the whole file block
unsigned char sgnature_value_s[smallSize];
memset(sgnature_value_s,0, sizeof(sgnature_value_s));
for (int i=0; i<smallSize; i++)
{
// if (j>255)
// j=0;
sgnature_value_s[i]=118;
// j++;
}
//insert a signature value
rc = m_Dctnry.insertDctnry(dFile, smallSize,
sgnature_value_s, token);
CPPUNIT_ASSERT( rc == NO_ERROR );
testGetBlockHdr();
return;
//insert a signature value
Token token3;
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token3);
CPPUNIT_ASSERT( rc == NO_ERROR );
m_Dctnry.closeDctnry(dFile);
printf("After insert dictionary \n");
testGetBlockHdr();
rc = m_Dctnry.openDctnry(oId);
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
rc =m_Dctnry.deleteDctnryValue( dFile, token);
m_Dctnry.closeDctnry(dFile);
printf("After delete dictionary \n");
testGetBlockHdr();
CPPUNIT_ASSERT( rc == NO_ERROR );
j=255;
for (int i=0; i<smallSize; i++)
{
if (j==0)
j=255;
sgnature_value[i]=j;
j--;
}
//insert another signature value
rc = m_Dctnry.openDctnry(oId);
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
Token token2;
rc = m_Dctnry.insertDctnry(dFile, smallSize,
sgnature_value, token2);
CPPUNIT_ASSERT( rc == NO_ERROR );
m_Dctnry.closeDctnry(dFile);
printf("After insert dictionary \n");
testGetBlockHdr();
m_Dctnry.dropDctnry(oId);
return;
}//testDctnryInsert
void testDctnryInsertDelete() {
int oId=2001;
// int blockCount =10;
FILE* dFile =NULL;
int rc =0;
DataBlock curBlock;
int largeSize=9000;
int smallSize=1000;
unsigned char dctnryHeader[14];
uint16_t freeSpace;
uint64_t nextPtr;
uint16_t offSet0;
uint16_t endHeader;
m_Dctnry.setDebugLevel( DEBUG_3 );
m_Dctnry.setUseSmallSize(true);
m_oId =oId;
memset(curBlock.data,0, sizeof (curBlock.data));
memset(dctnryHeader,0, sizeof(dctnryHeader));
printf("\nRunning testDctnryInsertDelete \n");
rc = m_Dctnry.deleteFile(oId);
rc = m_Dctnry.createDctnry((FID)-1);
CPPUNIT_ASSERT( rc == ERR_OPEN_FILE );
rc = m_Dctnry.createDctnry(oId,10);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_Dctnry.openDctnry();
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
rc = m_Dctnry.initDctnryHdr( dFile);
CPPUNIT_ASSERT( rc == NO_ERROR );
//CPPUNIT_ASSERT(rc==NO_ERROR);
//m_Dctnry.closeFile(dFile);
m_Dctnry.closeDctnry();
printf("After initDctnryHdr");
testGetBlockHdr();
//testGetFreeSpace();
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
//12 for 6 bytes and 14 for 8 bytes
rc =m_Dctnry.readSubBlockEntry( dFile, &curBlock, 0, 0, 0, 14, &dctnryHeader);
CPPUNIT_ASSERT( rc == NO_ERROR );
memcpy(&freeSpace,dctnryHeader,2);
memcpy(&nextPtr,dctnryHeader+2,8); // 8 bytes
memcpy(&offSet0,dctnryHeader+10,2);
memcpy(&endHeader,dctnryHeader+12,2);
unsigned char sgnature_value[largeSize];
memset(sgnature_value,0, sizeof(sgnature_value));
int j=0;
for (int i=0; i<largeSize; i++)
{
if (j>255)
j=0;
sgnature_value[i]=j;
j++;
}
//insert a signature value
Token token;
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token);
CPPUNIT_ASSERT( rc ==ERR_DICT_SIZE_GT_8000);
largeSize = 6000;
Token token3;
for (int i=0; i<largeSize; i++)
{
if (j>255)
j=0;
sgnature_value[i]=j;
j++;
}
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token3);
CPPUNIT_ASSERT( rc == NO_ERROR );
CPPUNIT_ASSERT( token3.fbo == 0 );
CPPUNIT_ASSERT( token3.op == 1 );
m_Dctnry.closeDctnry(dFile);
printf("After insert token 3 into dictionary \n");
testGetBlockHdr();
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
j=255;
for (int i=0; i<smallSize; i++)
{
if (j==0)
j=255;
sgnature_value[i]=j;
j--;
}
//insert another signature value
Token token2;
rc = m_Dctnry.insertDctnry(dFile, smallSize,
sgnature_value, token2);
CPPUNIT_ASSERT( rc == NO_ERROR );
CPPUNIT_ASSERT( rc == NO_ERROR );
CPPUNIT_ASSERT( token2.fbo == 0 );
CPPUNIT_ASSERT( token2.op == 2 );
m_Dctnry.closeDctnry(dFile);
printf("After insert token 2 into dictionary \n");
testGetBlockHdr();
//delete
memset(&token,0, sizeof(token));
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
//token dose not exist in dictionary
rc =m_Dctnry.deleteDctnryValue( dFile, token);
CPPUNIT_ASSERT( rc == ERR_DICT_NO_OP_DELETE );
//rc =m_Dctnry.deleteDctnryValue( dFile, token3);
rc =m_Dctnry.deleteDctnryValue( dFile, token2);
CPPUNIT_ASSERT( rc == NO_ERROR );
m_Dctnry.closeDctnry();
printf("After delete dictionary token2 \n");
testGetBlockHdr();
//delete right after
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
//rc =m_Dctnry.deleteDctnryValue( dFile, token2);
rc =m_Dctnry.deleteDctnryValue( dFile, token3);
CPPUNIT_ASSERT( rc == NO_ERROR );
//m_Dctnry.closeFile( dFile );
m_Dctnry.closeDctnry();
printf("After delete dictionary token3\n");
testGetBlockHdr();
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
//rc =m_Dctnry.deleteDctnryValue( dFile, token2);
rc =m_Dctnry.deleteDctnryValue( dFile, token3);
CPPUNIT_ASSERT( rc == ERR_DICT_NO_OP_DELETE );
rc =m_Dctnry.deleteDctnryValue( dFile, token2);
CPPUNIT_ASSERT( rc == ERR_DICT_NO_OP_DELETE );
//CPPUNIT_ASSERT( rc == ERR_DICT_ZERO_LEN );
m_Dctnry.closeDctnry();
return;
}//testDctnryInsertDelete
void testDctnryInsertStress() {
int oId=2002;
int blockCount =2;
FILE* dFile =NULL;
int rc =0;
DataBlock curBlock;
int smallSize=1000;
uint16_t freeSpace;
Offset hdrOffsets[4039];
int opCount = 0;
m_Dctnry.setDebugLevel( DEBUG_3);
m_Dctnry.setUseSmallSize(true);
m_oId = oId;
memset(curBlock.data,0, sizeof (curBlock.data));
memset(hdrOffsets,0, sizeof(hdrOffsets));
printf("\nRunning testDctnryInsertStress \n");
rc = m_Dctnry.deleteFile(oId);
rc = m_Dctnry.createFile(oId,blockCount);
dFile= m_Dctnry.openFile( oId );
CPPUNIT_ASSERT( dFile != NULL );
rc = m_Dctnry.initDctnryHdr( dFile);
CPPUNIT_ASSERT( rc == NO_ERROR );
//m_Dctnry.closeFile(dFile);
m_Dctnry.closeDctnry();
printf("After initDctnryHdr");
testGetBlockHdr();
dFile= m_Dctnry.openFile( oId);
unsigned char sgnature_value[smallSize];
Token token[100];
int i,j,k;
j=255;
for (k=0; k<18; k++)
{
for (int i=0; i<smallSize; i++)
{
if (j==0)
j=255;
sgnature_value[i]=k;
j--;
}// endfor i
rc = m_Dctnry.insertDctnry(dFile, smallSize,
sgnature_value, token[k]);
if (rc != NO_ERROR)
{
printf ("k: %i Error Code is: %i \n",k, rc);
//return;
}
if (k >15)
{
CPPUNIT_ASSERT( rc != NO_ERROR );
printf("294. Error code is: %i \n", rc);
}
else
CPPUNIT_ASSERT( rc == NO_ERROR );
}//endfor k
Offset prevOffset, curOffset;
j=0;
unsigned char* value = NULL;
//This is to get the value out
for (i=0; i<blockCount; i++)
{
rc =m_Dctnry.readSubBlockEntry( dFile, &curBlock, i, 0, 0, 2, &freeSpace);
CPPUNIT_ASSERT( rc == NO_ERROR );
j=0;
prevOffset.hdrLoc =2+8; // 8 bytes
memcpy(&(prevOffset.offset),&(curBlock.data[prevOffset.hdrLoc]),2);
curOffset.hdrLoc = prevOffset.hdrLoc+2;
memcpy(&(curOffset.offset),&(curBlock.data[curOffset.hdrLoc]),2);
int op=1;
int size = prevOffset.offset - curOffset.offset;
value = (unsigned char*)malloc(sizeof(unsigned char)*size);
memcpy(value, &curBlock.data[curOffset.offset], size );
while (curOffset.offset!= 0xFFFF)
{
//printf("fbo: %i op: %i starting offset: %i ending offset %i size: %i \n",
// i, op, curOffset.offset, prevOffset.offset, size);
//printf("value : ");
for (k=0; k<size; k++)
{
//printf("%u",value[k]);
}
// printf("\n");
//start again
free(value);
value = NULL;
prevOffset.hdrLoc = curOffset.hdrLoc;
prevOffset.offset = curOffset.offset;
curOffset.hdrLoc+=2;
memcpy(&(curOffset.offset), &curBlock.data[curOffset.hdrLoc],2);
size = prevOffset.offset - curOffset.offset;
if (curOffset.offset!= 0xFFFF)
{
value = (unsigned char*)malloc(sizeof(unsigned char)*size);
memcpy(value, &curBlock.data[curOffset.offset], size );
}
op++;
}//end while
//Get Offset info
m_Dctnry.closeDctnry();
printf("After insertDctnry");
testGetBlockHdr();
dFile= m_Dctnry.openFile( oId );
CPPUNIT_ASSERT( dFile != NULL );
m_Dctnry.getBlockHdr(dFile, i, opCount, hdrOffsets);
int opCount2 =0;
for (k=0; k< opCount; k++)
{
Token token;
token.fbo = i;
token.op = k+1;
rc = m_Dctnry.findTokenValue(dFile, token,sgnature_value, size);
CPPUNIT_ASSERT( rc == NO_ERROR);
CPPUNIT_ASSERT( size == smallSize);
rc =m_Dctnry.deleteDctnryValue( dFile, token);
printf("After deleteDctnryValue fbo %i op %i\n", (int)i, k+1);
m_Dctnry.getBlockHdr(dFile, i, opCount2, hdrOffsets);
Offset startOffset, endOffset;
printf("Header Info for fbo: %i \n %i %llu ", i, (int)m_Dctnry.getFree(),
m_Dctnry.getNextPtr());
endOffset.hdrLoc = 10;
endOffset.offset = 8192;
printf("%i ",endOffset.offset);
for (int k1=0; k1< opCount2; k1++)
{
startOffset.hdrLoc = hdrOffsets[k1].hdrLoc;
startOffset.offset = hdrOffsets[k1].offset;
printf("%i ",startOffset.offset);
}//end for k1
rc =m_Dctnry.deleteDctnryValue( dFile, token);
printf("%x \n", 0xFFFF);
printf("k-> %i i-> %i error code -> %i \n ", k, i, rc);
if (k<opCount-1)
CPPUNIT_ASSERT( rc == ERR_DICT_ZERO_LEN);
else
CPPUNIT_ASSERT( rc == ERR_DICT_NO_OP_DELETE);
}//end for k
} //end for i
//m_Dctnry.closeFile( dFile );
char sigString[] = "Hello, I am a string; what are you? I am testing dictionary as a signature value, I don't know how long I am but we will find out in a minute";
int sigStringSize = strlen(sigString);
char resultString[sigStringSize+1];
Token stringToken;
rc = m_Dctnry.insertDctnry(dFile, sigStringSize,
(unsigned char*)sigString, stringToken);
rc = m_Dctnry.findTokenValue(dFile,stringToken, (unsigned char*)resultString,sigStringSize);
resultString[sigStringSize] ='\0';
printf("result String is %s --->size is %i\n",resultString, strlen(resultString) );
m_Dctnry.closeDctnry();
printf("After insert the result string, the header look like the following:\n");
testGetBlockHdr();
return;
}//testDctnryInsertStress
void testGetBlockHdr() {
int blockCount=2;
int oId =2002;
int i =0, k=0;
FILE* dFile;
int opCount;
Offset hdrOffsets[4040];
memset(hdrOffsets,0, sizeof(hdrOffsets));
m_Dctnry.setDebugLevel( DEBUG_3 );
printf("\nRunning testGetBlockHdr \n");
dFile= m_Dctnry.openFile( m_oId );
if (dFile==NULL)
dFile= m_Dctnry.openFile( oId );
CPPUNIT_ASSERT( dFile != NULL );
blockCount = m_Dctnry.getFileSize( dFile )/BYTE_PER_BLOCK ;
for (i=0; i<blockCount; i++)
{
m_Dctnry.getBlockHdr(dFile, i, opCount, hdrOffsets);
Offset startOffset, endOffset;
int sigSize;
printf("Header Info for fbo: %i \n %i %llu ", i, (int)m_Dctnry.getFree(),
m_Dctnry.getNextPtr());
//printf(" fbo %i total offset number %i \n", i, opCount);
endOffset.hdrLoc = 10;
endOffset.offset = 8192;
printf("%i ",endOffset.offset);
for (k=0; k< opCount; k++)
{
startOffset.hdrLoc = hdrOffsets[k].hdrLoc;
startOffset.offset = hdrOffsets[k].offset;
sigSize = endOffset.offset - startOffset.offset;
//printf(" OP %i signature size : %i \n from %i to %i \n", k+1,
// sigSize, startOffset.offset,endOffset.offset );
endOffset.hdrLoc = startOffset.hdrLoc ;
endOffset.offset = startOffset.offset;
printf("%i ",endOffset.offset);
}//end for k
printf("%x \n", 0xFFFF);
}//endfor i
m_Dctnry.closeDctnry();
}
void testGetFreeSpace() {
int oId =2002;
FILE* dFile;
m_Dctnry.setDebugLevel( DEBUG_1 );
printf("\nRunning testGetFreeSpace \n");
dFile= m_Dctnry.openFile( m_oId );
if (dFile==NULL)
dFile= m_Dctnry.openFile( oId );
CPPUNIT_ASSERT( dFile != NULL );
m_Dctnry.getFreeSpaceArray(dFile);
for (int i=0; i<m_Dctnry.getNumBlocks(); i++)
{
printf("fbo %i -->free space:%i \n", i, m_Dctnry.m_freeSpaceArray[i]);
}
m_Dctnry.closeDctnry();
}
void testDctnryInsertDeleteStore() {
int rc =0;
int smallSize=1000;
FID dctnryOID =2002;
FID treeOID = 101;
FID listOID = 102;
m_DctnryStore.setDebugLevel( DEBUG_3 );
printf("\nRunning testDctnryInsertDeleteStore \n");
rc = m_DctnryStore.dropDctnryStore( treeOID, dctnryOID, listOID);
//CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.dropDctnryStore( dctnryOID, treeOID, listOID);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.dropDctnryStore( treeOID, dctnryOID, listOID);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.dropDctnryStore();
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID );
CPPUNIT_ASSERT( rc ==NO_ERROR );
m_DctnryStore.closeDctnryStore();
CPPUNIT_ASSERT( rc ==NO_ERROR );
rc = m_DctnryStore.openDctnryStore();
CPPUNIT_ASSERT( rc ==NO_ERROR );
unsigned char sgnature_value[smallSize];
Token token[100];
memset(token, 0, sizeof(token));
int i,j,k;
for (int i=0; i<smallSize; i++)
{
// if (j>255)
// j=0;
sgnature_value[i]=119;
// j++;
}
//insert a signature value
for (k=1;k<12;k++) {
rc = m_DctnryStore.updateDctnryStore(sgnature_value, smallSize,token[k]);
printf("token fbo = %i",token[k].fbo);
printf("token op = %i",token[k].op);
}
testGetBlockHdr();
return;
rc = m_DctnryStore.updateDctnryStore(sgnature_value, smallSize,token[1]);
testGetBlockHdr();
//Recheck if the tokens are all inserted
for (k=1; k<99; k++)
{
for (i=0; i<smallSize; i++)
{
if (j==0)
j=255;
if (k>10)
sgnature_value[i]=k;
else
{
if (i<8)
sgnature_value[i] =1;
else
sgnature_value[i]=k;
}
j--;
}//endfor i ; second time
rc = m_DctnryStore.updateDctnryStore(sgnature_value,
smallSize,token[k]);
if (rc!= NO_ERROR)
{
printf("443. Attentione!!! ERROR CODE : %i \n", rc);
}
CPPUNIT_ASSERT( rc == NO_ERROR );
} //endof for k; second time
int tempSize = 7;
rc = m_DctnryStore.updateDctnryStore(sgnature_value,
tempSize,token[k]);
if (rc!= NO_ERROR)
{
printf("452. Predicted Error Code should be 1363: The result ERROR CODE : %i \n", rc);
}
CPPUNIT_ASSERT( rc ==1363 );
tempSize = 8;
rc = m_DctnryStore.updateDctnryStore(sgnature_value,
tempSize,token[k]);
if (rc!= NO_ERROR)
{
printf("461. Attention!!! ERROR CODE : %i \n", rc);
}
CPPUNIT_ASSERT( rc == NO_ERROR );
for (i=1; i<99; i++)
{
if( m_DctnryStore.isDebug( DEBUG_3 ))
{
printf("i : %i token.fbo %i token.op %i \n", i, (int)token[i].fbo, (int)token[i].op);
}
}
for (i=1; i<99; i++)
{
rc = m_DctnryStore.deleteDctnryToken(token[i]);
if (rc!= NO_ERROR)
{
printf("475 . Attention!!! ERROR CODE : %i \n", rc);
}
CPPUNIT_ASSERT( rc == NO_ERROR );
}
m_DctnryStore.closeDctnryStore();
//rc = m_DctnryStore.dropDctnryStore();
//CPPUNIT_ASSERT( rc == NO_ERROR );
return;
}//testDctnryInsertDeleteStore
};
CPPUNIT_TEST_SUITE_REGISTRATION( DctnryTest );
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main( int argc, char **argv)
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry &registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest( registry.makeTest() );
bool wasSuccessful = runner.run( "", false );
return (wasSuccessful ? 0 : 1);
}

640
writeengine/dictionary/tdctnry.cpp Normal file
View File

@@ -0,0 +1,640 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <stdio.h>
#include <string>
#include <stdexcept>
using namespace std;
#include <boost/scoped_ptr.hpp>
using namespace boost;
#include <cppunit/extensions/HelperMacros.h>
#include "we_dctnry.h"
#include "we_dctnrystore.h"
#include <cmath>
//using namespace dataconvert;
using namespace WriteEngine;
class DctnryTest : public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE( DctnryTest );
// Dctnry basic testing
CPPUNIT_TEST( testDctnryInsertDelete );
CPPUNIT_TEST( testDctnryInsertDeleteStore);
CPPUNIT_TEST( testDctnryInsertStressStore);
CPPUNIT_TEST(testDctnryDropCreate);
CPPUNIT_TEST(testDctnryOpen);
CPPUNIT_TEST( testDctnryMultipleInsert);
CPPUNIT_TEST_SUITE_END();
private:
Dctnry m_Dctnry;
DctnryStore m_DctnryStore;
int m_oId;
public:
void setUp() {
BRMWrapper::setUseVb(false);
}
void tearDown() {
}
DctnryTest()
{
m_oId =-1;
}
void testDctnryInsertDelete() {
int oId=2001;
FILE* dFile =NULL;
int rc =0;
DataBlock curBlock;
int largeSize=9000;
int smallSize=1000;
unsigned char dctnryHeader[DCTNRY_HEADER_SIZE];
uint16_t freeSpace;
uint64_t nextPtr;
uint16_t offSet0;
uint16_t endHeader;
m_Dctnry.setDebugLevel( DEBUG_3 );
m_Dctnry.setUseSmallSize(true);
m_oId =oId;
memset(curBlock.data,0, sizeof (curBlock.data));
memset(dctnryHeader,0, sizeof(dctnryHeader));
printf("\nRunning testDctnryInsertDelete \n");
rc = m_Dctnry.dropDctnry(oId);
rc = m_Dctnry.createDctnry(oId,10);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
//12 for 6 bytes and 14 for 8 bytes
uint64_t lbid =0;
BRMWrapper::getInstance()->getBrmInfo( oId, 0, lbid );
printf("passed brm above \n");
rc =m_Dctnry.readSubBlockEntry( dFile, &curBlock, lbid, 0, 0, DCTNRY_HEADER_SIZE, &dctnryHeader);
CPPUNIT_ASSERT( rc == NO_ERROR );
memcpy(&freeSpace,dctnryHeader,HDR_UNIT_SIZE);
memcpy(&nextPtr,dctnryHeader+HDR_UNIT_SIZE,NEXT_PTR_BYTES); // 8 bytes
memcpy(&offSet0,dctnryHeader+NEXT_PTR_BYTES+HDR_UNIT_SIZE, HDR_UNIT_SIZE);
memcpy(&endHeader,dctnryHeader+NEXT_PTR_BYTES+HDR_UNIT_SIZE+HDR_UNIT_SIZE,HDR_UNIT_SIZE);
unsigned char sgnature_value[largeSize];
memset(sgnature_value,0, sizeof(sgnature_value));
int j=0;
//j is a 1 byte character;
for (int i=0; i<largeSize; i++)
{
if (j>255)
j=0;
sgnature_value[i]=j;
j++;
}
//insert a signature value
Token token;
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token);
CPPUNIT_ASSERT( rc ==ERR_DICT_SIZE_GT_8000);
largeSize = 4000;
Token token3;
for (int i=0; i<largeSize; i++)
{
if (j>255)
j=0;
sgnature_value[i]=j;
j++;
}
rc = m_Dctnry.insertDctnry(dFile, largeSize,
sgnature_value, token3);
cout<<"rc=" << rc << endl;
CPPUNIT_ASSERT( rc == NO_ERROR );
int fboCnt;
BRMWrapper::getInstance()->getFboOffset( token3.fbo, fboCnt);
CPPUNIT_ASSERT( fboCnt == 0 );
CPPUNIT_ASSERT( token3.op == 1 );
m_Dctnry.closeDctnry(dFile);
printf("After insert token 3 into dictionary \n");
testGetBlockHdr();
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
j=255;
for (int i=0; i<smallSize; i++)
{
if (j==0)
j=255;
sgnature_value[i]=j;
j--;
}
//insert another signature value
Token token2;
rc = m_Dctnry.insertDctnry(dFile, smallSize,
sgnature_value, token2);
CPPUNIT_ASSERT( rc == NO_ERROR );
BRMWrapper::getInstance()->getFboOffset( token2.fbo, fboCnt);
CPPUNIT_ASSERT( fboCnt == 0 );
CPPUNIT_ASSERT( token2.op == 2 );
m_Dctnry.closeDctnry(dFile);
printf("After insert token 2 into dictionary \n");
testGetBlockHdr();
//delete
memset(&token,0, sizeof(token));
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
//token dose not exist in dictionary
rc =m_Dctnry.deleteDctnryValue( dFile, token);
CPPUNIT_ASSERT( rc == ERR_DICT_BAD_TOKEN_OP );
rc =m_Dctnry.deleteDctnryValue( dFile, token2);
CPPUNIT_ASSERT( rc == NO_ERROR );
m_Dctnry.closeDctnry();
printf("After delete dictionary token2 \n");
testGetBlockHdr();
//delete right after
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
unsigned char sigValue3[8000];
int sigSize3;
rc =m_Dctnry.findTokenValue (dFile, token3,
sigValue3,
sigSize3 );
CPPUNIT_ASSERT(sigSize3==4000);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc =m_Dctnry.deleteDctnryValue( dFile, token3);
CPPUNIT_ASSERT( rc == NO_ERROR );
m_Dctnry.closeDctnry();
printf("After delete dictionary token3\n");
testGetBlockHdr();
rc = m_Dctnry.openDctnry();
CPPUNIT_ASSERT( rc == NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
CPPUNIT_ASSERT( dFile != NULL );
rc =m_Dctnry.deleteDctnryValue( dFile, token3);
CPPUNIT_ASSERT( rc == ERR_DICT_BAD_TOKEN_OP );
rc =m_Dctnry.deleteDctnryValue( dFile, token2);
CPPUNIT_ASSERT( rc == ERR_DICT_BAD_TOKEN_OP );
m_Dctnry.closeDctnry();
printf("\nSuccessfully Running testDctnryInsertDelete \n");
return;
}//testDctnryInsertDelete
void testGetBlockHdr() {
int blockCount=2;
int i =0, k=0;
FILE* dFile;
int opCount;
int rc;
Offset hdrOffsets[1024];
memset(hdrOffsets,0, sizeof(hdrOffsets));
m_Dctnry.setDebugLevel( DEBUG_3 );
printf("\nRunning testGetBlockHdr \n");
rc = m_Dctnry.openDctnry( m_oId );
CPPUNIT_ASSERT( rc ==NO_ERROR );
dFile = m_Dctnry.getDctnryFile();
blockCount = m_Dctnry.getFileSize( dFile )/BYTE_PER_BLOCK ;
for (i=0; i<blockCount; i++)
{
int freeSpace;
freeSpace = (int)m_Dctnry.getFree();
if (freeSpace == 8178)
break;
printf("Header Info for fbo: %i %i %llu ", i, freeSpace,
m_Dctnry.getNextPtr());
m_Dctnry.getBlockHdr(dFile, i, opCount, hdrOffsets);
if (opCount ==-1)
{
printf("ERROR BAD DICITONARY FILE|n");
return;
}
Offset startOffset, endOffset;
int sigSize;
printf(" fbo %i total offset number %i \n", i, opCount);
endOffset.hdrLoc = 10;
endOffset.offset = 8192;
for (k=0; k< opCount; k++)
{
startOffset.hdrLoc = hdrOffsets[k].hdrLoc;
startOffset.offset = hdrOffsets[k].offset;
sigSize = endOffset.offset - startOffset.offset;
printf(" OP %i signature size : %i \n from %i to %i \n", k+1,
sigSize, startOffset.offset,endOffset.offset );
endOffset.hdrLoc = startOffset.hdrLoc ;
endOffset.offset = startOffset.offset;
printf("%i ",endOffset.offset);
}//end for k
//printf("%x \n", 0xFFFF);
}//endfor i
m_Dctnry.closeDctnry();
printf("\nSuccessfully Running testGetBlockHdr \n");
return;
}
void testDctnryDropCreate() {
int rc =0;
FID dctnryOID =700;
FID treeOID = 101;
FID listOID = 102;
m_DctnryStore.setDebugLevel( DEBUG_3 );
printf("\nRunning testDctnryDropCreate \n");
rc = m_DctnryStore.dropDctnryStore( dctnryOID, treeOID, listOID);
rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID );
CPPUNIT_ASSERT( rc ==NO_ERROR );
m_DctnryStore.closeDctnryStore();
return;
}
void testDctnryOpen() {
int rc =0;
FID dctnryOID =700;
FID treeOID = 101;
FID listOID = 102;
m_DctnryStore.setDebugLevel( DEBUG_3 );
printf("\nRunning testDctnryOpen \n");
rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID );
CPPUNIT_ASSERT( rc ==NO_ERROR );
m_DctnryStore.closeDctnryStore();
return;
}
void testDctnryInsertDeleteStore() {
int rc =0;
int smallSize=1000;
FID dctnryOID =100;
FID treeOID = 101;
FID listOID = 102;
Token token2[100];
m_DctnryStore.setDebugLevel( DEBUG_3 );
printf("\nRunning testDctnryInsertDeleteStore \n");
printf("dropDctnryStore\n");
rc = m_DctnryStore.dropDctnryStore( dctnryOID, treeOID, listOID);
rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID );
CPPUNIT_ASSERT( rc ==NO_ERROR );
unsigned char sgnature_value[smallSize];
Token token[500];
memset(token, 0, sizeof(token));
int i,j,k;
j=255;
for (k=1; k<499; k++)
{
for (i=0; i<smallSize; i++)
{
if (j==0)
j=255;
if (k>10)
sgnature_value[i]=k;
else
{
if (i<8)
sgnature_value[i] =1;
else
sgnature_value[i]=k;
}
j--;
}//endfor i
rc = m_DctnryStore.updateDctnryStore(sgnature_value,
smallSize,token[k]);
if (rc!= NO_ERROR)
printf("rc %i k %i \n", rc, k);
CPPUNIT_ASSERT( rc == NO_ERROR );
} //endof for k
printf("first time inserted %i values token\n", k);
//Recheck if the tokens are all inserted
memset(token2, 0, sizeof(token2));
for (k=1; k<99; k++)
{
for (i=0; i<smallSize; i++)
{
if (j==0)
j=255;
if (k>10)
sgnature_value[i]=k;
else
{
if (i<8)
sgnature_value[i] =1;
else
sgnature_value[i]=k;
}
j--;
}//endfor i ; second time
rc = m_DctnryStore.updateDctnryStore(sgnature_value,
smallSize,token2[k]);
if (rc!= NO_ERROR)
{
printf("443. Attentione!!! ERROR CODE : %i \n", rc);
}
CPPUNIT_ASSERT( rc == NO_ERROR );
} //endof for k; second time
printf("second time inserted %i values token\n", k);
int tempSize = 7;
rc = m_DctnryStore.updateDctnryStore(sgnature_value,
tempSize,token2[k]);
if (rc!= NO_ERROR)
{
printf("452. Predicted Error Code should be 1363: The result ERROR CODE : %i \n", rc);
}
CPPUNIT_ASSERT( rc ==NO_ERROR );
tempSize = 8;
rc = m_DctnryStore.updateDctnryStore(sgnature_value,
tempSize,token2[k]);
if (rc!= NO_ERROR)
{
printf("461. Attention!!! ERROR CODE : %i \n", rc);
}
CPPUNIT_ASSERT( rc == NO_ERROR );
for (i=1; i<99; i++)
{
printf("first time->i : %i token.fbo %i token.op %i \n", i, (int)token[i].fbo,
(int)token[i].op);
}
cout<< "finished printing the tokens" << endl;
cout<< "start deleting the tokens" << endl;
for (i=1; i<99; i++)
{
cout<< "start deleting the tokens i=" << i << endl;
rc = m_DctnryStore.deleteDctnryToken(token[i]);
if (rc!= NO_ERROR)
{
printf("475 . Attention!!! ERROR CODE : %i \n", rc);
}
CPPUNIT_ASSERT( rc == NO_ERROR );
}
cout<< "finish deleting the tokens" << endl;
m_DctnryStore.closeDctnryStore();
printf("\nSuccessfully Running testDctnryInsertDelete \n");
return;
}//testDctnryInsertDeleteStore
void testDctnryInsertStressStore() {
int rc =0;
FID dctnryOID =897;
FID treeOID = 101;
FID listOID = 102;
int count = 5000;
Token stringToken;
string msg;
string timeStr;
Token pToken;
int sigStringSize;
m_DctnryStore.setDebugLevel( DEBUG_3 );
m_DctnryStore.setUseHashMap(true);
printf("\nRunning testDctnryInsertStressStore \n");
rc = m_DctnryStore.dropDctnryStore( dctnryOID, treeOID, listOID);
cout << "m_DctnryStore.dropDctnryStore error code=" << rc << " dctnryOID=" << dctnryOID << " treeOID=" << treeOID << " listOID=" << listOID << endl;
rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID);
cout << "m_DctnryStore.createDctnryStore error code=" << rc << " dctnryOID=" << dctnryOID << " treeOID=" << treeOID << " listOID=" << listOID << endl;
cout << " I am here " << endl;
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID );
CPPUNIT_ASSERT( rc ==NO_ERROR );
char insertString[] = "Hello, I am a string; what are you? I am testing dictionary as a signature value, I don't know how long I am but we will find out in a minute";
char sigString[500];
memset(sigString,'\0', 500);
for (int i=0; i<count ; i++)
{
sprintf(sigString, "%d%s", i, insertString);
//sprintf(sigString, "%s%d", insertString, i);
sigStringSize = strlen(sigString);
m_DctnryStore.setAllTransId(10);
rc = m_DctnryStore.updateDctnryStore((unsigned char*)sigString,
sigStringSize,
stringToken);
cout << "stringToken token->fbo " << stringToken.fbo << " op->" << stringToken.op << endl;
}
for (int i=0; i<count ; i++)
{
sprintf(sigString, "%d%s", i, insertString);
sigStringSize = strlen(sigString);
m_DctnryStore.setAllTransId(10);
rc = m_DctnryStore.updateDctnryStore((unsigned char*)sigString,
sigStringSize,
pToken);
cout << "pToken token->fbo " << pToken.fbo << " op->" << pToken.op << endl;
}
m_DctnryStore.clearMap();
m_DctnryStore.closeDctnryStore();
printf("\nSuccessfully Running testDctnryInsertStressStore \n");
return;
}//testDctnryInsertStressStore
void testDctnryMultipleInsert() {
int rc =0;
FID dctnryOID =200;
int totalsize = 1;
int size = 18;//3,18
int letterSize=2;//27,2
int mapSize=1000;
//char base[]={'b', 'b','c','d','e','f','g','h','i','j','k','l',
// 'm','n','o','p','q','r','s','t','u','v','w','x','y','z',' '};
m_DctnryStore.setDebugLevel( DEBUG_3 );
printf("\nRunning testDctnryMultipleInsert \n");
rc = m_Dctnry.dropDctnry( dctnryOID);
rc = m_Dctnry.createDctnry( dctnryOID);
CPPUNIT_ASSERT( rc == NO_ERROR );
rc = m_Dctnry.openDctnry(dctnryOID);
CPPUNIT_ASSERT( rc ==NO_ERROR );
for (int i = 0; i<(size); i++)
{
totalsize = totalsize*letterSize;
}
cout<<"totalsize=" << totalsize << endl;
// There will not be a leak if the following throws an exception:
int totalRow = totalsize;
char buf[size+1];
ColPosPair** pos = new ColPosPair*[totalRow];
int loc =0;
//ColPosPair* pos[totalRow];
for (int i = 0; i<totalRow; i++)
{
pos[i]= new ColPosPair[1];
pos[i][0].start = loc;
pos[i][0].offset = size;
loc=loc+size+1;
}
int i=0;
/*
while (i<totalRow)
{
memset(buf[i], '\0', size+1);
for (int j=0; j< letterSize; j++)
{
for (int k=0; k < letterSize; k++)
{
for (int l=0; l < letterSize; l++)
{
buf[i][0]= base[j];
buf[i][1]= base[k];
buf[i][2]= base[l];
buf[i][3]='|';
char tmp[5];
memset(tmp,'\0',5);
strncpy(tmp, buf[i],4);
cout << "i=" << i << " buf[i]=" << tmp<< endl;
i++;
}
}
}
}
*/
int totalUnitSize[size];
for (int i=0; i< size; i++)
{
totalUnitSize[i]= (int)pow((double)letterSize, (double)(size-i-1));
//cout << "totalUnitSize[i]" << totalUnitSize[i] << endl;
}
i=0;
int tmpsize=100000;
char buf2[tmpsize*(size+1)];
while (i<tmpsize)
{
//memset(buf[i], '\0', size+1);
for (int m=0; m<size ; m++)
{
//cout << m<<"totalUnitSize[m]" << totalUnitSize[m] << endl;
int index;
int level=i/totalUnitSize[m];
index=(level%letterSize);
//cout <<"index=" << index <<endl;
//buf[i][m]= base[index] ;
buf[m]='c';
}
buf[size]='|';
memcpy(buf2+i*(size+1), buf, size+1);
/*
char tmp[size+1];
memset(tmp,'\0',size+1);
strncpy(tmp, buf[i],size);
cout << "i=" << i << " buf[i]=" << tmp<< endl;
*/
i++;
}
int rowStatus[tmpsize];
char tokenBuf[tmpsize*8];
mapSize=1000;
m_Dctnry.setHashMapSize(mapSize);
m_Dctnry.startfTimer();
rc = m_Dctnry.insertDctnry((const char*)buf2,
(ColPosPair**)pos,
(const int)tmpsize,
0,
(int*)rowStatus,
(char*)tokenBuf);
m_Dctnry.stopfTimer();
cout <<"total time in mlsec=" << m_Dctnry.getTotalfRunTime() << endl;
m_Dctnry.closeDctnry();
for (int i = totalRow; i > 0; --i)
{
//cout <<"i-1="<< i-1 << endl;
delete[] pos[i-1];
// delete [] buf[i-1];
}
delete[] pos;
//delete [] buf;
}
};
CPPUNIT_TEST_SUITE_REGISTRATION( DctnryTest );
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main( int argc, char **argv)
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry &registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest( registry.makeTest() );
bool wasSuccessful = runner.run( "", false );
return (wasSuccessful ? 0 : 1);
}

1332
writeengine/dictionary/we_dctnry.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

299
writeengine/dictionary/we_dctnry.h Normal file
View File

@@ -0,0 +1,299 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: we_dctnry.h 4726 2013-08-07 03:38:36Z bwilkinson $
/** @we_dctnry.h
* Defines the Dctnry class
* When a signature is given, the value will be stored in dictionary and
* a token will be issued. Given a token, the signature in the dictionary
* can be deleted
*/
#ifndef _WE_DCTNRY_H_
#define _WE_DCTNRY_H_
#include <cstdlib>
#include <cstddef>
#include <iostream>
#include <string>
#include "we_dbfileop.h"
#include "we_type.h"
#include "we_brm.h"
#include "bytestream.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
//---------------------------------------------------------------------------
// Structure used to store signatures in string cache
//---------------------------------------------------------------------------
typedef struct Signature
{
int size;
unsigned char* signature;
Token token;
} Signature;
/**
* @brief Class to interface with dictionary store files.
*/
class Dctnry : public DbFileOp
{
//--------------------------------------------------------------------------
// Public members
//--------------------------------------------------------------------------
public:
/**
* @brief Dctnry Constructor
*/
EXPORT Dctnry();
/**
* @brief Dctnry Destructor
*/
EXPORT virtual ~Dctnry();
/**
* @brief Close the dictionary file handle.
*/
EXPORT int closeDctnry(bool realClose = true);
/**
* @brief Close the dictionary file handle without flushing the current blk
* buffer or updating HWM to BRM.
*/
EXPORT int closeDctnryOnly();
/**
* @brief Create a dictionary extent
*
* If 'flag' is true, a new file is created with an abbreviated extent.
* If 'flag' is false, then function adds a full exent to an already open
* file, basically assuming that the file already has 1 or more extents.
*
* @param dctnryOID - dictionary file OID
* @param colWidth - dictionary string width (not the token width)
* @param dbRoot - DBRoot for store file
* @param partition - partition number for store file
* @param segment - column segment number for store file
* @param flag - indicates whether extent is added to new file (true)
* @param startLbid - starting LBID for the newly allocated extent
*/
EXPORT int createDctnry(const OID& dctnryOID,
int colWidth,
const uint16_t dbRoot,
const uint32_t partition,
const uint16_t segment,
BRM::LBID_t& startLbid,
bool flag=true);
/**
* @brief Drop dictionary store
*
* @param dctnryOID- OID of dictionary store file to be deleted
*/
EXPORT int dropDctnry( const OID& dctnryOID);
/**
* @brief Accessors
*/
const std::string& getFileName() const { return m_segFileName; }
HWM getHWM() const { return m_hwm; }
EXPORT bool getTokenFromArray(Signature& sig);
EXPORT uint64_t getCurLbid(){return m_curLbid;}
/**
* @brief Insert a signature value to a file block and return token/pointer.
* (for DDL/DML use)
*
* @param sgnature_size - size of signature to be inserted
* @param sgnature_value - signature to be inserted
* @param token - (output) token associated with inserted signature
*/
EXPORT int insertDctnry(const int& sgnature_size,
const unsigned char* sgnature_value, Token& token);
/**
* @brief Insert a signature value to a file block and return token/pointer
* (for Bulk use)
*
* @param buf - bulk buffer containing strings to be parsed
* @param pos - list of offsets into buf
* @param totalRow - total number of rows in buf
* @param col - the column to be parsed from buf
* @param tokenBuf - (output) list of tokens for the parsed strings
*/
EXPORT int insertDctnry(const char* buf,
ColPosPair ** pos,
const int totalRow, const int col,
char* tokenBuf,
long long& truncCount);
/**
* @brief Update dictionary store with tokenized strings (for DDL/DML use)
*
* @param sigValue - signature value
* @param sigSize - signature size
* @param token - (output) token that was added
*/
EXPORT int updateDctnry(unsigned char* sigValue,
int& sigSize,
Token& token);
/**
* @brief open dictionary store
*
* @param dctnryOID - dictionary file OID
* @param dbRoot - DBRoot for store file
* @param partition - partition number for store file
* @param segment - column segment number for store file
* @param useTmpSuffix - for Bulk HDFS usage: use or not use *.tmp file suffix
*/
EXPORT int openDctnry(const OID& dctnryOID, const uint16_t dbRoot,
const uint32_t partition, const uint16_t segment,
const bool useTmpSuffix);
/**
* @brief copy the dictionary header to buffer
*/
void copyDctnryHeader(void* buf);
/**
* @brief Set logger that can be used for logging (primarily by bulk load)
*/
void setLogger(Log* logger) { m_logger = logger; }
/**
* @brief Set dictionary column width for this column
*/
void setColWidth(int colWidth) { m_colWidth = colWidth; }
/**
* @brief Set dictionary default for this column
*/
void setDefault(const std::string& defVal) { m_defVal = defVal; }
void setImportDataMode( ImportDataMode importMode )
{ m_importDataMode = importMode; }
virtual int checkFixLastDictChunk() {return NO_ERROR;}
//------------------------------------------------------------------------------
// Protected members
//------------------------------------------------------------------------------
protected:
//
// Add the specified signature (string) to the string cache
//
void addToStringCache( const Signature& newSig );
//
// Clear the dictionary store.
//
void clear() { m_dFile = NULL; m_dctnryOID =(OID)INVALID_NUM; }
// Expand an abbreviated extent on disk.
int expandDctnryExtent();
// Free memory consumed by strings in the string cache
void freeStringCache();
//
// Functions to read data:
// getBlockOpCount - get the ordinal position (OP) count from the header
// getEndOp - read OP of the end of header for specified fbo
//
void getBlockOpCount(const DataBlock& fileBlock, int & op_count);
int getEndOp (IDBDataFile* dFile, int fbo, int &op);
//
// Initialization
//
int init();
//
// Support functions for inserting values into dictionary.
// insertDctnryHdr inserts the new value info into the header.
// insertSgnture inserts the new value into the block.
//
void insertDctnry2(Signature& sig);
void insertDctnryHdr( unsigned char* blockBuf, const int& size);
void insertSgnture(unsigned char* blockBuf,
const int& size, unsigned char*value);
//
// Preloads the strings from the specified DataBlock. Currently
// used to preload the first block, of a store file having only 1 block.
//
void preLoadStringCache( const DataBlock& fileBlock );
// methods to be overriden by compression classes
// (width argument in createDctnryFile() is string width, not token width)
virtual IDBDataFile* createDctnryFile(const char *name, int width,
const char *mode, int ioBuffSize);
virtual IDBDataFile* openDctnryFile(bool useTmpSuffix);
virtual void closeDctnryFile(bool doFlush, std::map<FID,FID> & oids);
virtual int numOfBlocksInFile();
Signature m_sigArray[MAX_STRING_CACHE_SIZE]; // string cache
int m_arraySize; // num strings in m_sigArray
// m_dctnryHeader used for hdr when readSubBlockEntry is used to read a blk
// m_dctnryHeader2 contains filled in template used to initialize new blocks
unsigned char m_dctnryHeader[DCTNRY_HEADER_SIZE]; // first 14 bytes of hdr
unsigned char m_dctnryHeader2[DCTNRY_HEADER_SIZE]; // first 14 bytes of hdr
uint64_t m_nextPtr; // next pointer
//relate to different Dictionary file
FID m_dctnryOID; // OID for the dctnry file
IDBDataFile* m_dFile; // dictionary file
uint32_t m_partition; // partition associated with OID
uint16_t m_segment; // segment associated with OID
uint16_t m_dbRoot; // DBRoot associated with OID
std::string m_segFileName; // current column segment file
int m_numBlocks; // num "raw" uncompressed blocks in file
int m_lastFbo;
HWM m_hwm;
//Need to be initialized for different Dictionary file
int m_newStartOffset; // start offset
uint16_t m_freeSpace; // free space (bytes) within current block
int m_curOp; // current ordinal pointer within m_curFbo
int m_curFbo; // current "raw" (uncompressed) FBO
BRM::LBID_t m_curLbid; // LBID associated with m_curFbo
DataBlock m_curBlock; // current "raw" (uncompressed) data block
Log* m_logger; // logger, mainly for bulk load
int m_colWidth; // width of this dictionary column
std::string m_defVal; // optional default string value
ImportDataMode m_importDataMode; // Import data in text or binary mode
};//end of class
} //end of namespace
#undef EXPORT
#endif // _WE_DCTNRY_H_

168
writeengine/dictionary/we_dctnrystore.cpp Normal file
View File

@@ -0,0 +1,168 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_dctnrystore.cpp 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
#include <string.h>
#ifndef _MSC_VER
#include <inttypes.h>
#endif
#define WRITEENGINEDCTSTORE_DLLEXPORT
#include "we_dctnrystore.h"
#undef WRITEENGINEDCTSTORE_DLLEXPORT
namespace WriteEngine
{
/***********************************************************
* Constructor
***********************************************************/
DctnryStore::DctnryStore()
:m_hashMapFlag(true), m_hashMapSize(MAX_HASHMAP_SIZE)
{
m_dctnry.setUseHashMap(m_hashMapFlag);
}
/***********************************************************
* Destructor
***********************************************************/
DctnryStore::~DctnryStore()
{
}
/***********************************************************
* Open a dictionary store file
***********************************************************/
const int DctnryStore::openDctnryStore(const OID& dctnryOID,
const OID& treeOID,
const OID& listOID,
const uint16_t dbRoot,
const uint32_t partition,
const uint16_t segment)
{
int rc2;
rc2 = m_dctnry.openDctnry(dctnryOID, dbRoot, partition, segment);
setUseHashMap(true);
return rc2;
}
/***********************************************************
* Create a dictionary store file
***********************************************************/
const int DctnryStore::createDctnryStore( const OID& dctnryOID,
const OID& treeOID, const OID& listOID, int colWidth, const uint16_t dbRoot,
const uint32_t partition, const uint16_t segment )
{
int rc2 ;
rc2 = m_dctnry.createDctnry(dctnryOID, colWidth, dbRoot, partition, segment);
return rc2;
}
/***********************************************************
* Drop a dictionary store file
***********************************************************/
const int DctnryStore::dropDctnryStore( const OID& dctnryOID,
const OID& treeOID,
const OID& listOID)
{
int rc2;
rc2 = m_dctnry.dropDctnry(dctnryOID);
return rc2;
}
/***********************************************************
* Add a signature value to the dictionary store.
* Function first checks to see if the signature is already
* in our string cache, and returns the corresponding token
* if it is found in the cache.
***********************************************************/
const int DctnryStore::updateDctnryStore(unsigned char* sigValue,
int& sigSize,
Token& token)
{
int rc = NO_ERROR;
Signature sig;
sig.signature = sigValue;
sig.size = sigSize;
//if String cache is enabled then look for string in cache
if (m_hashMapFlag)
{
if (m_dctnry.m_arraySize < (int)m_hashMapSize)
{
bool found = false;
found = m_dctnry.getTokenFromArray(sig);
if (found)
{
token = sig.token;
return NO_ERROR;
}
} //end if use hash map and size >0
}
//Insert into Dictionary
rc = m_dctnry.insertDctnry(sigSize, sigValue, token);
//Add the new signature and token into cache if the hashmap flag is on
// (We currently use an array instead of a hashmap.)
if ((m_hashMapFlag) && (m_dctnry.m_arraySize < (int)m_hashMapSize))
{
Signature sig;
sig.size = sigSize;
sig.signature = new unsigned char[sigSize];
memcpy (sig.signature, sigValue, sigSize);
sig.token = token;
m_dctnry.m_sigArray[m_dctnry.m_arraySize]=sig;
m_dctnry.m_arraySize++;
}
return rc;
}
/***********************************************************
* Delete signature from the dictionary store file
***********************************************************/
const int DctnryStore::deleteDctnryToken(Token& token)
{
int rc ;
int sigSize;
unsigned char* sigValue = NULL;
rc = m_dctnry.deleteDctnryValue( token, sigSize, &sigValue);
if (rc!=NO_ERROR)
{
return rc;
}
if (!sigValue)
{
rc = ERR_DICT_TOKEN_NOT_FOUND;
return rc;
}
free(sigValue);
return rc;
}
} //end of namespace

202
writeengine/dictionary/we_dctnrystore.h Normal file
View File

@@ -0,0 +1,202 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_dctnrystore.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
/**
* Wrapper around the Dctnry class used to maintain a dictionary store file.
* When signature(s) are specified to the update functions, the value(s) are
* stored in the dictionary, and token(s) are returned to denote the location
* of the signature(s). The caller is then responsible for storing the
* token(s) in the corresponding segment column file.
*
* References to tree and list OIDs are not currently pertinent, and may be
* removed at a later time.
*
* A deleteDctnryToken function is provided for possible future use to
* delete tokens. But this function is not currently used.
*/
#ifndef _WE_DctnryStore_H_
#define _WE_DctnryStore_H_
#include <string>
#include "we_dbfileop.h"
#include "we_type.h"
#include "we_dctnry.h"
#if defined(_MSC_VER) && defined(WRITEENGINEDCTSTORE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/**@brief Namespace WriteEngine */
namespace WriteEngine
{
/**@brief class DctnryStore
* A wrapper class for dictionary
*/
class DctnryStore : public DbFileOp
{
public:
/**
* @brief Constructor
*/
EXPORT DctnryStore();
/**
* @brief Destructor
*/
EXPORT ~DctnryStore();
/**
* @brief Close a dictionary store after it has been opened.
*/
EXPORT int closeDctnryStore(){return m_dctnry.closeDctnry();}
/**
* @brief Close a dictionary store without flushing the block buffer or
* updating BRM with HWM. Intended to be used for immediate file closure
* to shut down a job that has encountered an error, and intends to do
* a rollback.
*/
EXPORT int closeDctnryStoreOnly(){return m_dctnry.closeDctnryOnly();}
/**
* @brief create dictionary store
*
* @param dctnryOID - dictionary file OID
* @param treeOID - index tree OID (not used)
* @param listOID - list OID (not used)
* @param colWidth - column width
* @param dbRoot - DBRoot for store file
* @param partition - partition number for store file
* @param segment - column segment number for store file
*/
EXPORT const int createDctnryStore(
const OID& dctnryOID, const OID& treeOID,
const OID& listOID, int colWidth, const uint16_t dbRoot,
const uint32_t partition, const uint16_t segment );
/**
* @brief Delete a token from dictionary store, for maintanance use
*
* @param token - token to be deleted
*/
EXPORT const int deleteDctnryToken(Token& token);
/**
* @brief Drop dictionary store (for DDL/DML use)
*
* @param dctnryOID - dictionary file OID
* @param treeOID - index tree OID (not used)
* @param listOID - list OID (not used)
*/
EXPORT const int dropDctnryStore( const OID& dctnryOID, const OID& treeOID,
const OID& listOID);
/**
* @brief Open a dictionary store after creation
*
* @param dctnryOID - dictionary file OID
* @param treeOID - index tree OID (not used)
* @param listOID - list OID (not used)
* @param dbRoot - DBRoot for store file
* @param partition - partition number for store file
* @param segment - column segment number for store file
*/
EXPORT const int openDctnryStore(const OID& dctnryOID, const OID& treeOID,
const OID& listOID,
const uint16_t dbRoot,
const uint32_t partition,
const uint16_t segment);
/**
* @brief Update dictionary store with tokenized strings (for DDL/DML use)
*
* @param sigValue - signature value
* @param sigSize - signature size
* @param token - (output) token that was added
*/
EXPORT const int updateDctnryStore(unsigned char* sigValue,
int& sigSize, Token& token);
/**
* @brief Update dictionary store with tokenized strings (for Bulk use)
*
* @param buf - bulk buffer containing strings to be parsed
* @param pos - list of offsets into buf
* @param totalRow - total number of rows in buf
* @param col - the column to be parsed from buf
* @param colWidth - width of the dictionary column being parsed
* @param tokenBuf - (output) list of tokens for the parsed strings
*/
const int updateDctnryStore(const char* buf,
ColPosPair ** pos,
const int totalRow,
const int col,
const int colWidth,
char* tokenBuf)
{ return(m_dctnry.insertDctnry(
buf, pos, totalRow, col, colWidth, tokenBuf)); }
/**
* @brief TransId related function
*
* @param transId - Current transaction id (for DDL/DML use)
*/
void setAllTransId(const TxnID& transId){m_dctnry.setTransId(transId);}
/**
* @brief String cache related routines
*/
void clearMap() { m_dctnry.clearMap(); }
void createMap(){ m_dctnry.createMap(); }
void setUseHashMap(bool flag)
{ m_hashMapFlag = flag;
m_dctnry.setUseHashMap(flag); }
void setHashMapSize(int size)
{ if (size < MAX_HASHMAP_SIZE)
m_hashMapSize = size;
else
m_hashMapSize = MAX_HASHMAP_SIZE;
m_dctnry.setHashMapSize(m_hashMapSize); }
HWM getHWM() const { return m_dctnry.getHWM(); }
const std::string& getFileName() const { return m_dctnry.getFileName();}
/**
* @brief public instance
*/
Dctnry m_dctnry;
private:
// Used to configure string cache usage
bool m_hashMapFlag;
int m_hashMapSize;
};
} //end of namespace
#undef EXPORT
#endif // _WE_DctnryStore_H_