database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

the begginning

Browse Source
This commit is contained in:
david hill
2016-01-06 14:08:59 -06:00
parent 66a31debcb
commit f6afc42dd0
18251 changed files with 16460679 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
writeengine/shared/Makefile Executable file
View File

@ -0,0 +1,81 @@
include ../../rules.mak
include ../build/we_rules.mak
#******************************************************************************************
# $Id: Makefile 4589 2013-04-02 14:41:08Z rdempsey $
#
# Copyright (C) 2009-2013 Calpont Corporation
# All rights reserved
#*****************************************************************************************/
.PHONY: install clean docs test coverage leakcheck
SRCS=we_bulkrollbackmgr.cpp we_bulkrollbackfile.cpp we_bulkrollbackfilecompressed.cpp we_bulkrollbackfilecompressedhdfs.cpp we_fileop.cpp we_blockop.cpp we_dbfileop.cpp we_log.cpp we_simplesyslog.cpp we_convertor.cpp we_brm.cpp we_config.cpp we_cache.cpp we_stats.cpp we_define.cpp we_chunkmanager.cpp we_rbmetawriter.cpp we_dbrootextenttracker.cpp we_confirmhdfsdbfile.cpp
LINCLUDES=we_bulkrollbackmgr.h we_index.h we_define.h we_type.h we_fileop.h we_blockop.h we_dbfileop.h we_obj.h we_log.h we_simplesyslog.h we_convertor.h we_brm.h we_macro.h we_config.h we_cache.h we_stats.h we_typeext.h we_chunkmanager.h we_rbmetawriter.h we_dbrootextenttracker.h we_confirmhdfsdbfile.h
OBJS=$(SRCS:.cpp=.o)
object: $(OBJS)
mkdir -p $(LIBDIR)
cp *.o $(LIBDIR)
rm -f $(LIBDIR)/tshared.o
make install
install: bootstrap
bootstrap:
$(INSTALL) $(LINCLUDES) $(INSTALL_ROOT_INCLUDE)
clean:
rm -f $(LIBDIR)/tshared.o $(LIBDIR)/*gcov.o $(OBJS) tshared.o tshared core *~ *.tag *-gcov.* *.gcov tshared-gcov *.d *.swp *.dat *.txt *.log
for file in $(SRCS); do \
bfile=`basename $$file .cpp`; \
rm -f $(LIBDIR)/$${bfile}.o ; \
done
rm -rf html
docs:
doxygen $(EXPORT_ROOT)/etc/Doxyfile
tshared: $(OBJS) tshared.o
$(LINK.cpp) -o $@ $^ $(TLIBS)
test:
xtest: tshared object
$(IPCS_CLEANUP)
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib ./tshared
%-gcov.o: %.cpp
$(COMPILE.cpp) -o $@ $^
cp *-gcov.o $(LIBDIR)
tshared-gcov: CXXFLAGS+=-fprofile-arcs -ftest-coverage
tshared-gcov: tshared-gcov.o $(subst .o,-gcov.o,$(OBJS))
$(LINK.cpp) -o $@ $^ $(GLIBS)
coverage:
xcoverage: tshared-gcov
$(IPCS_CLEANUP)
rm -f *.gcda
LD_LIBRARY_PATH=$(EXPORT_ROOT)/lib:/usr/local/lib ./tshared-gcov
for file in $(SRCS); do \
bfile=`basename $$file .cpp`; \
gcov -o $${bfile}-gcov $$file >/dev/null; \
done
/usr/local/bin/genCoverage.pl $(SRCS)
leakcheck:
xleakcheck: $(LIBRARY) tshared
$(IPCS_CLEANUP)
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib valgrind --tool=memcheck --leak-check=yes ./tshared
%.d: %.cpp
@set -e; rm -f $@; \
$(CC) -MM $(CPPFLAGS) $< > $@.$$$$; \
sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$$ > $@; \
rm -f $@.$$$$
ifndef BOOTSTRAP
-include $(SRCS:.cpp=.d) tshared.d
endif

32
writeengine/shared/Makefile.am Normal file
View File

@ -0,0 +1,32 @@
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
# $Id: Makefile.am 3720 2012-04-04 18:18:49Z rdempsey $
## Process this file with automake to produce Makefile.in
include_HEADERS = we_index.h we_define.h we_type.h we_fileop.h we_blockop.h we_dbfileop.h we_obj.h we_log.h we_simplesyslog.h we_convertor.h we_brm.h we_macro.h we_config.h we_cache.h we_stats.h we_bulkrollbackmgr.h we_typeext.h we_chunkmanager.h we_bulkrollbackfilecompressed.h we_bulkrollbackfilecompressedhdfs.h we_bulkrollbackfile.h we_rbmetawriter.h we_dbrootextenttracker.h we_confirmhdfsdbfile.h
test:
coverage:
leakcheck:
docs:
bootstrap: install-data-am

454
writeengine/shared/Makefile.in Normal file
View File

@ -0,0 +1,454 @@
# Makefile.in generated by automake 1.9.6 from Makefile.am.
# @configure_input@
# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
# 2003, 2004, 2005 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
# with or without modifications, as long as this notice is preserved.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.
@SET_MAKE@
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
# $Id: Makefile.am 3720 2012-04-04 18:18:49Z rdempsey $
srcdir = @srcdir@
top_srcdir = @top_srcdir@
VPATH = @srcdir@
pkgdatadir = $(datadir)/@PACKAGE@
pkglibdir = $(libdir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
top_builddir = ../..
am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
INSTALL = @INSTALL@
install_sh_DATA = $(install_sh) -c -m 644
install_sh_PROGRAM = $(install_sh) -c
install_sh_SCRIPT = $(install_sh) -c
INSTALL_HEADER = $(INSTALL_DATA)
transform = $(program_transform_name)
NORMAL_INSTALL = :
PRE_INSTALL = :
POST_INSTALL = :
NORMAL_UNINSTALL = :
PRE_UNINSTALL = :
POST_UNINSTALL = :
build_triplet = @build@
host_triplet = @host@
subdir = writeengine/shared
DIST_COMMON = $(include_HEADERS) $(srcdir)/Makefile.am \
$(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/m4/compilerflags.m4 \
$(top_srcdir)/m4/functions.m4 $(top_srcdir)/m4/install.m4 \
$(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
mkinstalldirs = $(install_sh) -d
CONFIG_HEADER = $(top_builddir)/config.h
CONFIG_CLEAN_FILES =
SOURCES =
DIST_SOURCES =
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
am__vpath_adj = case $$p in \
$(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
*) f=$$p;; \
esac;
am__strip_dir = `echo $$p | sed -e 's|^.*/||'`;
am__installdirs = "$(DESTDIR)$(includedir)"
includeHEADERS_INSTALL = $(INSTALL_HEADER)
HEADERS = $(include_HEADERS)
ETAGS = etags
CTAGS = ctags
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
ACLOCAL = @ACLOCAL@
ALLOCA = @ALLOCA@
AMDEP_FALSE = @AMDEP_FALSE@
AMDEP_TRUE = @AMDEP_TRUE@
AMTAR = @AMTAR@
AR = @AR@
AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
CC = @CC@
CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@
CPP = @CPP@
CPPFLAGS = @CPPFLAGS@
CXX = @CXX@
CXXCPP = @CXXCPP@
CXXDEPMODE = @CXXDEPMODE@
CXXFLAGS = @CXXFLAGS@
CYGPATH_W = @CYGPATH_W@
DEFS = @DEFS@
DEPDIR = @DEPDIR@
ECHO = @ECHO@
ECHO_C = @ECHO_C@
ECHO_N = @ECHO_N@
ECHO_T = @ECHO_T@
EGREP = @EGREP@
EXEEXT = @EXEEXT@
F77 = @F77@
FFLAGS = @FFLAGS@
INSTALL_DATA = @INSTALL_DATA@
INSTALL_PROGRAM = @INSTALL_PROGRAM@
INSTALL_SCRIPT = @INSTALL_SCRIPT@
INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
LDFLAGS = @LDFLAGS@
LEX = @LEX@
LEXLIB = @LEXLIB@
LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
LIBOBJS = @LIBOBJS@
LIBS = @LIBS@
LIBTOOL = @LIBTOOL@
LN_S = @LN_S@
LTLIBOBJS = @LTLIBOBJS@
MAKEINFO = @MAKEINFO@
OBJEXT = @OBJEXT@
PACKAGE = @PACKAGE@
PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_STRING = @PACKAGE_STRING@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
PATH_SEPARATOR = @PATH_SEPARATOR@
POW_LIB = @POW_LIB@
RANLIB = @RANLIB@
SED = @SED@
SET_MAKE = @SET_MAKE@
SHELL = @SHELL@
STRIP = @STRIP@
VERSION = @VERSION@
XML2_CONFIG = @XML2_CONFIG@
XML_CPPFLAGS = @XML_CPPFLAGS@
XML_LIBS = @XML_LIBS@
YACC = @YACC@
ac_ct_AR = @ac_ct_AR@
ac_ct_CC = @ac_ct_CC@
ac_ct_CXX = @ac_ct_CXX@
ac_ct_F77 = @ac_ct_F77@
ac_ct_RANLIB = @ac_ct_RANLIB@
ac_ct_STRIP = @ac_ct_STRIP@
am__fastdepCC_FALSE = @am__fastdepCC_FALSE@
am__fastdepCC_TRUE = @am__fastdepCC_TRUE@
am__fastdepCXX_FALSE = @am__fastdepCXX_FALSE@
am__fastdepCXX_TRUE = @am__fastdepCXX_TRUE@
am__include = @am__include@
am__leading_dot = @am__leading_dot@
am__quote = @am__quote@
am__tar = @am__tar@
am__untar = @am__untar@
bindir = @bindir@
build = @build@
build_alias = @build_alias@
build_cpu = @build_cpu@
build_os = @build_os@
build_vendor = @build_vendor@
datadir = @datadir@
etcdir = @etcdir@
exec_prefix = @exec_prefix@
host = @host@
host_alias = @host_alias@
host_cpu = @host_cpu@
host_os = @host_os@
host_vendor = @host_vendor@
idb_brm_libs = @idb_brm_libs@
idb_cflags = @idb_cflags@
idb_common_includes = @idb_common_includes@
idb_common_ldflags = @idb_common_ldflags@
idb_common_libs = @idb_common_libs@
idb_cppflags = @idb_cppflags@
idb_cxxflags = @idb_cxxflags@
idb_exec_libs = @idb_exec_libs@
idb_ldflags = @idb_ldflags@
idb_oam_libs = @idb_oam_libs@
idb_write_libs = @idb_write_libs@
idbinstall = @idbinstall@
includedir = @includedir@
infodir = @infodir@
install_sh = @install_sh@
libdir = @libdir@
libexecdir = @libexecdir@
localdir = @localdir@
localstatedir = @localstatedir@
mandir = @mandir@
march_flags = @march_flags@
mibdir = @mibdir@
mkdir_p = @mkdir_p@
mysqldir = @mysqldir@
netsnmp_libs = @netsnmp_libs@
netsnmpagntdir = @netsnmpagntdir@
netsnmpdir = @netsnmpdir@
netsnmplibrdir = @netsnmplibrdir@
netsnmpmachdir = @netsnmpmachdir@
netsnmpsysdir = @netsnmpsysdir@
oldincludedir = @oldincludedir@
postdir = @postdir@
prefix = @prefix@
program_transform_name = @program_transform_name@
sbindir = @sbindir@
sharedir = @sharedir@
sharedstatedir = @sharedstatedir@
sysconfdir = @sysconfdir@
target_alias = @target_alias@
toolsdir = @toolsdir@
include_HEADERS = we_index.h we_define.h we_type.h we_fileop.h we_blockop.h we_dbfileop.h we_obj.h we_log.h we_simplesyslog.h we_convertor.h we_brm.h we_macro.h we_config.h we_cache.h we_stats.h we_bulkrollbackmgr.h we_typeext.h we_chunkmanager.h we_bulkrollbackfilecompressed.h we_bulkrollbackfilecompressedhdfs.h we_bulkrollbackfile.h we_rbmetawriter.h we_dbrootextenttracker.h we_confirmhdfsdbfile.h
all: all-am
.SUFFIXES:
$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps)
@for dep in $?; do \
case '$(am__configure_deps)' in \
*$$dep*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh \
&& exit 0; \
exit 1;; \
esac; \
done; \
echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu writeengine/shared/Makefile'; \
cd $(top_srcdir) && \
$(AUTOMAKE) --gnu writeengine/shared/Makefile
.PRECIOUS: Makefile
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
@case '$?' in \
*config.status*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
*) \
echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
esac;
$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(top_srcdir)/configure: $(am__configure_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(ACLOCAL_M4): $(am__aclocal_m4_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
mostlyclean-libtool:
-rm -f *.lo
clean-libtool:
-rm -rf .libs _libs
distclean-libtool:
-rm -f libtool
uninstall-info-am:
install-includeHEADERS: $(include_HEADERS)
@$(NORMAL_INSTALL)
test -z "$(includedir)" || $(mkdir_p) "$(DESTDIR)$(includedir)"
@list='$(include_HEADERS)'; for p in $$list; do \
if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
f=$(am__strip_dir) \
echo " $(includeHEADERS_INSTALL) '$$d$$p' '$(DESTDIR)$(includedir)/$$f'"; \
$(includeHEADERS_INSTALL) "$$d$$p" "$(DESTDIR)$(includedir)/$$f"; \
done
uninstall-includeHEADERS:
@$(NORMAL_UNINSTALL)
@list='$(include_HEADERS)'; for p in $$list; do \
f=$(am__strip_dir) \
echo " rm -f '$(DESTDIR)$(includedir)/$$f'"; \
rm -f "$(DESTDIR)$(includedir)/$$f"; \
done
ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
mkid -fID $$unique
tags: TAGS
TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$(TAGS_FILES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
if test -z "$(ETAGS_ARGS)$$tags$$unique"; then :; else \
test -n "$$unique" || unique=$$empty_fix; \
$(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
$$tags $$unique; \
fi
ctags: CTAGS
CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$(TAGS_FILES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
test -z "$(CTAGS_ARGS)$$tags$$unique" \
|| $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
$$tags $$unique
GTAGS:
here=`$(am__cd) $(top_builddir) && pwd` \
&& cd $(top_srcdir) \
&& gtags -i $(GTAGS_ARGS) $$here
distclean-tags:
-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \
topsrcdirstrip=`echo "$(top_srcdir)" | sed 's|.|.|g'`; \
list='$(DISTFILES)'; for file in $$list; do \
case $$file in \
$(srcdir)/*) file=`echo "$$file" | sed "s|^$$srcdirstrip/||"`;; \
$(top_srcdir)/*) file=`echo "$$file" | sed "s|^$$topsrcdirstrip/|$(top_builddir)/|"`;; \
esac; \
if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
dir=`echo "$$file" | sed -e 's,/[^/]*$$,,'`; \
if test "$$dir" != "$$file" && test "$$dir" != "."; then \
dir="/$$dir"; \
$(mkdir_p) "$(distdir)$$dir"; \
else \
dir=''; \
fi; \
if test -d $$d/$$file; then \
if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
cp -pR $(srcdir)/$$file $(distdir)$$dir || exit 1; \
fi; \
cp -pR $$d/$$file $(distdir)$$dir || exit 1; \
else \
test -f $(distdir)/$$file \
|| cp -p $$d/$$file $(distdir)/$$file \
|| exit 1; \
fi; \
done
check-am: all-am
check: check-am
all-am: Makefile $(HEADERS)
installdirs:
for dir in "$(DESTDIR)$(includedir)"; do \
test -z "$$dir" || $(mkdir_p) "$$dir"; \
done
install: install-am
install-exec: install-exec-am
install-data: install-data-am
uninstall: uninstall-am
install-am: all-am
@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
installcheck: installcheck-am
install-strip:
$(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
`test -z '$(STRIP)' || \
echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install
mostlyclean-generic:
clean-generic:
distclean-generic:
-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
maintainer-clean-generic:
@echo "This command is intended for maintainers to use"
@echo "it deletes files that may require special tools to rebuild."
clean: clean-am
clean-am: clean-generic clean-libtool mostlyclean-am
distclean: distclean-am
-rm -f Makefile
distclean-am: clean-am distclean-generic distclean-libtool \
distclean-tags
dvi: dvi-am
dvi-am:
html: html-am
info: info-am
info-am:
install-data-am: install-includeHEADERS
install-exec-am:
install-info: install-info-am
install-man:
installcheck-am:
maintainer-clean: maintainer-clean-am
-rm -f Makefile
maintainer-clean-am: distclean-am maintainer-clean-generic
mostlyclean: mostlyclean-am
mostlyclean-am: mostlyclean-generic mostlyclean-libtool
pdf: pdf-am
pdf-am:
ps: ps-am
ps-am:
uninstall-am: uninstall-includeHEADERS uninstall-info-am
.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \
clean-libtool ctags distclean distclean-generic \
distclean-libtool distclean-tags distdir dvi dvi-am html \
html-am info info-am install install-am install-data \
install-data-am install-exec install-exec-am \
install-includeHEADERS install-info install-info-am \
install-man install-strip installcheck installcheck-am \
installdirs maintainer-clean maintainer-clean-generic \
mostlyclean mostlyclean-generic mostlyclean-libtool pdf pdf-am \
ps ps-am tags uninstall uninstall-am uninstall-includeHEADERS \
uninstall-info-am
test:
coverage:
leakcheck:
docs:
bootstrap: install-data-am
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
.NOEXPORT:

96
writeengine/shared/tconfig.cpp Normal file
View File

@ -0,0 +1,96 @@
//------------------------------------------------------------------------------
// test driver used to test reloadable cache enhancement to we_config
// for bug 4486
// $Id$
//------------------------------------------------------------------------------
#include <iostream>
#include <vector>
#include <dbrm.h>
#include "we_config.h"
using namespace WriteEngine;
void test()
{
std::cout << "getDBRootByIdx(1): " <<
Config::getDBRootByIdx(1) << std::endl;
std::cout << "getDBRootByIdx(3): " <<
Config::getDBRootByIdx(3) << std::endl;
std::cout << "getDBRootByNum(1): " <<
Config::getDBRootByNum(1) << std::endl;
std::cout << "getDBRootByNum(3): " <<
Config::getDBRootByNum(3) << std::endl;
std::vector<unsigned short> dbRootIds;
Config::getRootIdList( dbRootIds );
std::cout << "getRootIdList: ";
for (unsigned k=0; k<dbRootIds.size(); k++) {
std::cout << dbRootIds[k] << ' '; }
std::cout << std::endl;
std::vector<std::string> dbRootPathList;
Config::getDBRootPathList( dbRootPathList );
std::cout << "getDBRootPathList: " << std::endl;
for (unsigned k=0; k<dbRootPathList.size(); k++)
std::cout << " " << k << ". " << dbRootPathList[k] << std::endl;
std::cout << "getBulkRoot(): " <<
Config::getBulkRoot() << std::endl;
std::cout << "DBRootCount(): " <<
Config::DBRootCount() << std::endl;
std::cout << "totalDBRootCount(): " <<
Config::totalDBRootCount() << std::endl;
std::cout << "getWaitPeriod(): " <<
Config::getWaitPeriod() << std::endl;
std::cout << "getFilePerColumnPartition(): " <<
Config::getFilesPerColumnPartition() << std::endl;
std::cout << "getExtentsPerSegmentFile(): " <<
Config::getExtentsPerSegmentFile() << std::endl;
std::cout << "getBulkProcessPriority(): " <<
Config::getBulkProcessPriority() << std::endl;
std::cout << "getBulkRollbackDir(): " <<
Config::getBulkRollbackDir() << std::endl;
std::cout << "getMaxFileSystemDiskUsage(): " <<
Config::getMaxFileSystemDiskUsage() << std::endl;
std::cout << "getNumCompressedPadBlks(): " <<
Config::getNumCompressedPadBlks() << std::endl;
std::cout << "getParentOAMModuleFlag(): " <<
Config::getParentOAMModuleFlag() << std::endl;
std::cout << "getLocalModuleType(): " <<
Config::getLocalModuleType() << std::endl;
std::cout << "getLocalModuleID(): " <<
Config::getLocalModuleID() << std::endl;
std::cout << "getVBRoot(): " <<
Config::getVBRoot() << std::endl;
}
int main()
{
Config::initConfigCache();
char resp;
int nTest = 0;
std::cout << std::endl;
while (1)
{
std::cout << "test" << nTest << "..." << std::endl;
test();
std::cout << "Pause..." << std::endl;
std::cin >> resp;
std::cout << std::endl;
if (resp == 'c')
{
std::cout << "Has local DBRootList changed: " <<
(bool)Config::hasLocalDBRootListChanged() << std::endl;
}
else if (resp == 'q')
{
break;
}
nTest++;
}
return 0;
}

1373
writeengine/shared/tshared.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

282
writeengine/shared/we_blockop.cpp Normal file
View File

@ -0,0 +1,282 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_blockop.cpp 4500 2013-01-31 20:25:42Z dhall $
*
*******************************************************************************/
/** @file */
#include <stdio.h>
#include <string.h>
#include "joblisttypes.h"
#include "we_blockop.h"
#include "we_config.h"
#include "we_brm.h"
#include "we_convertor.h"
using namespace execplan;
namespace WriteEngine
{
/**
* Constructor
*/
BlockOp::BlockOp()
{}
/**
* Default Destructor
*/
BlockOp::~BlockOp()
{}
/***********************************************************
* DESCRIPTION:
* Calculate the location of Row ID
* PARAMETERS:
* rowId - row id
* epb - entry per block
* width - width per column record
* RETURN:
* fbo - File block offset
* bio - Block internal offset
* true if success, false otherwise
***********************************************************/
bool BlockOp::calculateRowId(
RID rowId, const int epb, const int width, int& fbo, int& bio ) const
{
if( std::numeric_limits<WriteEngine::RID>::max() == rowId )
return false;
fbo = (int)( rowId/epb );
bio = ( rowId & ( epb - 1 )) * width;
return true;
}
/***********************************************************
* DESCRIPTION:
* Get the value that represents empty row
* PARAMETERS:
* colDataType - data type
* width - data width in byte
* RETURN:
* emptyVal - the value of empty row
***********************************************************/
uint64_t BlockOp::getEmptyRowValue(
const CalpontSystemCatalog::ColDataType colDataType, const int width ) const
{
uint64_t emptyVal = 0;
int offset = 0;
switch( colDataType ) {
case CalpontSystemCatalog::TINYINT : emptyVal = joblist::TINYINTEMPTYROW; break;
case CalpontSystemCatalog::SMALLINT: emptyVal = joblist::SMALLINTEMPTYROW; break;
case CalpontSystemCatalog::MEDINT :
case CalpontSystemCatalog::INT : emptyVal = joblist::INTEMPTYROW; break;
case CalpontSystemCatalog::BIGINT : emptyVal = joblist::BIGINTEMPTYROW; break;
case CalpontSystemCatalog::FLOAT :
case CalpontSystemCatalog::UFLOAT : emptyVal = joblist::FLOATEMPTYROW; break;
case CalpontSystemCatalog::DOUBLE :
case CalpontSystemCatalog::UDOUBLE : emptyVal = joblist::DOUBLEEMPTYROW; break;
case CalpontSystemCatalog::DECIMAL :
case CalpontSystemCatalog::UDECIMAL :
/* if( width <= 4 )
emptyVal = joblist::SMALLINTEMPTYROW;
else
if( width <= 9 )
emptyVal = 0x80000001;
else
if( width <= 18 )
emptyVal = 0x8000000000000001LL;
else
emptyVal = 0xFFFFFFFFFFFFFFFFLL;
*/
// @bug 194 use the correct logic in handling empty value for decimal
if (width <= 1)
emptyVal = joblist::TINYINTEMPTYROW;
else if( width <= 2 )
emptyVal = joblist::SMALLINTEMPTYROW;
else if( width <= 4 )
emptyVal = joblist::INTEMPTYROW;
else
emptyVal = joblist::BIGINTEMPTYROW;
break;
case CalpontSystemCatalog::UTINYINT : emptyVal = joblist::UTINYINTEMPTYROW; break;
case CalpontSystemCatalog::USMALLINT: emptyVal = joblist::USMALLINTEMPTYROW; break;
case CalpontSystemCatalog::UMEDINT :
case CalpontSystemCatalog::UINT : emptyVal = joblist::UINTEMPTYROW; break;
case CalpontSystemCatalog::UBIGINT : emptyVal = joblist::UBIGINTEMPTYROW; break;
case CalpontSystemCatalog::CHAR :
case CalpontSystemCatalog::VARCHAR :
case CalpontSystemCatalog::DATE :
case CalpontSystemCatalog::DATETIME :
default:
offset = ( colDataType == CalpontSystemCatalog::VARCHAR )? -1 : 0;
emptyVal = joblist::CHAR1EMPTYROW;
if( width == (2 + offset) )
emptyVal = joblist::CHAR2EMPTYROW;
else
if( width >= (3 + offset) && width <= ( 4 + offset ) )
emptyVal = joblist::CHAR4EMPTYROW;
else
if( width >= (5 + offset) )
emptyVal = joblist::CHAR8EMPTYROW;
break;
}
return emptyVal;
}
/***********************************************************
* DESCRIPTION:
* Get the correct width for a row
* PARAMETERS:
* colDataType - data type
* width - data width in byte
* RETURN:
* emptyVal - the value of empty row
***********************************************************/
int BlockOp::getCorrectRowWidth(
const CalpontSystemCatalog::ColDataType colDataType, const int width ) const
{
return Convertor::getCorrectRowWidth(colDataType, width);
}
/***********************************************************
* DESCRIPTION:
* Get row id
* PARAMETERS:
* fbo - file block offset
* bio - block internal offset
* bbo - byte internal offset
* RETURN:
* row id
***********************************************************/
RID BlockOp::getRowId(
const long fbo, const int width, const int rowPos
/*const int bio, const int bbo*/ ) const
{
// return fbo*BYTE_PER_BLOCK*ROW_PER_BYTE + bio*ROW_PER_BYTE + bbo;
return (BYTE_PER_BLOCK/width) * fbo + rowPos;
}
/***********************************************************
* DESCRIPTION:
* Get buffer value
* PARAMETERS:
* buf - buffer
* width - data width in byte
* RETURN:
* val - buffer value
***********************************************************/
void BlockOp::readBufValue(
const unsigned char* buf, void* val, const short width ) const
{
memcpy( val, buf, width );
}
/***********************************************************
* DESCRIPTION:
* Reset a buffer
* PARAMETERS:
* buf - buffer
* bufSize - buffer size
* RETURN:
* none
***********************************************************/
void BlockOp::resetBuf( unsigned char* buf, const int bufSize ) const
{
memset( buf, 0, bufSize );
}
/***********************************************************
* DESCRIPTION:
* Fill buffer with empty values
* PARAMETERS:
* buf - buffer
* bufSize - buffer size
* emptyVal - empty value
* width - type width
* RETURN:
* none
***********************************************************/
/* static */
void BlockOp::setEmptyBuf(
unsigned char* buf, const int bufSize, uint64_t emptyVal, const int width )
{
const int ARRAY_COUNT = 128;
const int NBYTES_IN_ARRAY = width * ARRAY_COUNT;
//unsigned char emptyValArray[NBYTES_IN_ARRAY];
unsigned char* emptyValArray = (unsigned char*)alloca(NBYTES_IN_ARRAY);
// Optimize buffer initialization by constructing and copying in an array
// instead of individual values. This reduces the number of calls to
// memcpy().
for (int j=0; j<ARRAY_COUNT; j++)
{
memcpy(emptyValArray+(j*width), &emptyVal, width);
}
int countFull128 = (bufSize/width) / ARRAY_COUNT;
int countRemain = (bufSize/width) % ARRAY_COUNT;
// Copy in the 128 element array into "buf" as many times as needed
if (countFull128 > 0)
{
for( int i = 0; i < countFull128; i++ )
memcpy( buf + (i * (NBYTES_IN_ARRAY)),
emptyValArray,
NBYTES_IN_ARRAY );
}
// Initialize the remainder of "buf" that is leftover
if (countRemain > 0)
{
memcpy( buf + (countFull128 * NBYTES_IN_ARRAY),
emptyValArray,
width*countRemain );
}
}
/***********************************************************
* DESCRIPTION:
* Set a value in a buffer
* PARAMETERS:
* buf - buffer
* val - buffer value
* width - data width in byte
* RETURN:
* none
***********************************************************/
void BlockOp::writeBufValue(
unsigned char* buf, void* val, const size_t width, const bool clear ) const
{
if( clear )
memset( buf, 0, width );
memcpy( buf, val, width );
}
} //end of namespace

130
writeengine/shared/we_blockop.h Normal file
View File

@ -0,0 +1,130 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_blockop.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
/** @file */
#ifndef _WE_BLOCKOP_H_
#define _WE_BLOCKOP_H_
#include <we_obj.h>
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
/** Class BlockOp */
class BlockOp : public WEObj
{
public:
/**
* @brief Constructor
*/
EXPORT BlockOp();
/**
* @brief Default Destructor
*/
EXPORT ~BlockOp();
/**
* @brief Calculate the location of Row ID
*/
EXPORT bool calculateRowId( RID rowId,
const int epb,
const int width,
int& fbo,
int& bio ) const;
/**
* @brief Calculate the location of Row ID
*/
void clearBlock( DataBlock* block )
{ memset(block->data, 0, sizeof(block->data));
block->no = -1;
block->dirty = false; }
/**
* @brief Get bit value after shift
*/
uint64_t getBitValue( uint64_t val,
int shiftBit,
uint64_t mask ) const
{ return ( val >> shiftBit ) & mask ; }
/**
* @brief Get correct row width
*/
EXPORT int getCorrectRowWidth( const execplan::CalpontSystemCatalog::ColDataType colDataType,
const int width ) const;
/**
* @brief Get an empty row value
*/
EXPORT uint64_t getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType,
const int width ) const;
/**
* @brief Calculate row id
*/
EXPORT RID getRowId( const long fbo,
const int width,
const int rowPos ) const;
/**
* @brief Get buffer value
*/
EXPORT void readBufValue( const unsigned char* buf,
void* val, const short width ) const;
/**
* @brief Reset a buffer
*/
EXPORT void resetBuf( unsigned char* buf,
const int bufSize ) const;
/**
* @brief Fill buffer with empty values
*/
EXPORT void static setEmptyBuf( unsigned char* buf,
const int bufSize,
uint64_t emptyVal, const int width );
/**
* @brief Set a value in a buffer
*/
EXPORT void writeBufValue( unsigned char* buf,
void* val,
const size_t width,
const bool clear = false ) const;
};
} //end of namespace
#undef EXPORT
#endif // _WE_BLOCKOP_H_

1648
writeengine/shared/we_brm.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

710
writeengine/shared/we_brm.h Normal file
View File

@ -0,0 +1,710 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_brm.h 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
/** @file */
#ifndef _WE_BRM_H_
#define _WE_BRM_H_
#include <iostream>
#include <vector>
#include <boost/thread.hpp>
#include <boost/thread/tss.hpp>
#include "brm.h"
#include "we_obj.h"
#include<sys/time.h>
#include "brmtypes.h"
#include "IDBDataFile.h"
#include "IDBPolicy.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
// forward reference
class DbFileOp;
/** Class BRMWrapper */
class BRMWrapper : public WEObj
{
public:
/**
* @brief Initialize an Auto Increment sequence for the specified OID
* @param colOID Column OID of interest
* @param startNextValue Starting next value for the AI sequence
* @param colWidth Width of the relevant column (in bytes)
* @param errMsg Applicable error message.
*/
EXPORT int startAutoIncrementSequence( OID colOID,
uint64_t startNextValue,
uint32_t colWidth,
execplan::CalpontSystemCatalog::ColDataType colDataType,
std::string& errMsg);
/**
* @brief Reserve a range of Auto Increment numbers for the specified OID
* @param colOID Column OID of interest
* @param count Requested range of auto increment numbers
* @param firstNum (out) First number of range that is reserved
* @param errMsg Applicable error message.
*/
EXPORT int getAutoIncrementRange( OID colOID,
uint64_t count,
uint64_t& firstNum,
std::string& errMsg);
/**
* @brief Inform BRM to add an extent to each of the requested OIDs at
* the specified DBRoot (and partition number if the DBRoot is empty).
* @param cols (in) List of column OIDs and column widths
* @param dbRoot (in) DBRoot for requested extents
* @param partition (in/out) Physical partition number in file path.
* If allocating OID's first extent for this DBRoot, then
* partition is input, else it is only for output.
* @param segmentNum (out) Segment number for new extents
* @param extents (out) List of lbids, numBlks, and fbo for new extents
*/
EXPORT int allocateStripeColExtents(
const std::vector<BRM::CreateStripeColumnExtentsArgIn>& cols,
uint16_t dbRoot,
uint32_t& partition,
uint16_t& segmentNum,
std::vector<BRM::CreateStripeColumnExtentsArgOut>& extents);
/**
* @brief Inform BRM to add extent to the exact segment file specified by
* OID, DBRoot, partition, and segment.
*/
EXPORT int allocateColExtentExactFile( const OID oid,
const uint32_t colWidth,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType,
BRM::LBID_t& startLbid,
int& allocSize,
uint32_t& startBlock);
/**
* @brief Inform BRM to add a dictionary store extent to the specified OID
*/
EXPORT int allocateDictStoreExtent( const OID oid,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
BRM::LBID_t& startLbid,
int& allocSize );
/**
* @brief Inform BRM to delete certain oid
*/
EXPORT int deleteOid( const OID oid );
/**
* @brief Inform BRM to delete list of oids
*/
EXPORT int deleteOIDsFromExtentMap (const std::vector<int32_t>& oids);
/**
* @brief Get BRM information based on a specfic OID, DBRoot, partition,
* and segment
*/
EXPORT int getBrmInfo( const OID oid,
const uint32_t partition,
const uint16_t segment,
const int fbo,
BRM::LBID_t& lbid );
/**
* @brief Get starting LBID from BRM for a specfic OID, DBRoot, partition,
* segment, and block offset.
*/
EXPORT int getStartLbid( const OID oid,
const uint32_t partition,
const uint16_t segment,
const int fbo,
BRM::LBID_t& startLbid );
/**
* @brief Get the real physical offset based on the LBID
*/
EXPORT int getFboOffset( const uint64_t lbid,
uint16_t& dbRoot,
uint32_t& partition,
uint16_t& segment,
int& fbo );
EXPORT int getFboOffset( const uint64_t lbid, int& oid,
uint16_t& dbRoot,
uint32_t& partition,
uint16_t& segment,
int& fbo );
/**
* @brief Get last "local" HWM, partition, and segment for an OID and DBRoot
*/
EXPORT int getLastHWM_DBroot( OID oid,
uint16_t dbRoot,
uint32_t& partition,
uint16_t& segment,
HWM& hwm,
int& status,
bool& bFound);
/**
* @brief Get HWM for a specific OID, partition, and segment
*/
int getLocalHWM( OID oid ,
uint32_t partition,
uint16_t segment,
HWM& hwm,
int& status);
/**
* @brief Get HWM info for a specific OID and PM
*/
EXPORT int getDbRootHWMInfo( const OID oid ,
BRM::EmDbRootHWMInfo_v& emDbRootHwmInfos);
/**
* @brief Get status or state of the extents in the specified segment file.
* bFound flag indicates whether an extent was found or not.
*/
int getExtentState( OID oid,
uint32_t partition,
uint16_t segment,
bool& bFound,
int& status);
/**
* @brief Get extentRows
*/
unsigned getExtentRows();
/**
* @brief Return the extents info for specified OID
*/
int getExtents( int oid,
std::vector<struct BRM::EMEntry>& entries,
bool sorted, bool notFoundErr,
bool incOutOfService );
/**
* @brief Return the extents info for specified OID and dbroot
*/
int getExtents_dbroot( int oid,
std::vector<struct BRM::EMEntry>& entries,
const uint16_t dbroot);
/**
* @brief Return the read/write status of DBRM (helps detect if DBRM is up)
*/
EXPORT int isReadWrite();
/**
* @brief Return the state of the system state shutdown pending
* flags
*/
EXPORT int isShutdownPending(bool& bRollback, bool& bForce);
/**
* @brief Return the state of the system state suspend pending
* flags
*/
EXPORT int isSuspendPending();
/**
* @brief Is InfiniDB system ready (completed startup)
*/
bool isSystemReady();
/**
* @brief Lookup LBID ranges for column specified OID
*/
int lookupLbidRanges(OID oid, BRM::LBIDRange_v& lbidRanges);
/**
* @brief Mark extent invalid for causal partioning
*/
int markExtentInvalid(const uint64_t lbid,
const execplan::CalpontSystemCatalog::ColDataType colDataType);
/**
* @brief Mark multiple extents invalid for causal partioning
*/
int markExtentsInvalid(std::vector<BRM::LBID_t>& lbids,
const std::vector<execplan::CalpontSystemCatalog::ColDataType>&
colDataTypes);
/**
* @brief set extents CP min/max info into extent map
*/
int setExtentsMaxMin(const BRM::CPInfoList_t& cpinfoList);
/**
* @brief Perform bulk rollback of any column extents that logically follow
* the specified HWM for the given column OID and DBRoot. The HWM for the
* last local extent is reset to the specified hwm as well. Any extents in
* subsequent partitions are deleted. If bDeleteAll is true, then all
* extents for the specified oid and dbroot are deleted.
*/
int rollbackColumnExtents_DBroot( const OID oid,
bool bDeleteAll,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
BRM::HWM_t hwm );
/**
* @brief Perform bulk rollback of the extents that follow the specified
* dictionary extents for the given column OID and DBRoot. The HWM for
* the last retained extents, are reset as well. Any trailing segment
* files for the same parition, that are not specified in the hwm list,
* are deleted. Any extents in subsequent partitions are deleted. If
* segNums and hwms vector are empty, then all extents for the specified
* oid and dbroot are deleted.
*/
int rollbackDictStoreExtents_DBroot( OID oid,
uint16_t dbRoot,
uint32_t partition,
const std::vector<uint16_t>& segNums,
const std::vector<BRM::HWM_t>& hwms );
/**
* @brief Perform delete column extents
*/
int deleteEmptyColExtents(const std::vector<BRM::ExtentInfo>& extentsInfo);
/**
* @brief Perform delete dictionary extents
*/
int deleteEmptyDictStoreExtents(
const std::vector<BRM::ExtentInfo>& extentsInfo );
/**
* @brief Set HWM for a specific OID, partition, and segment
*/
int setLocalHWM( OID oid,
uint32_t partition,
uint16_t segment,
const HWM hwm );
//Set hwm for all columns in a table
int bulkSetHWM( const std::vector<BRM::BulkSetHWMArg> & vec,
BRM::VER_t transID);
/**
* @brief Atomically apply a batch of HWM and CP updates within the scope
* of a single BRM lock. CP info is merged with current min/max range.
* @param hwmArgs Vector of HWM updates
* @param mergeCPDataArgs Vector of Casual Partition updates
*/
int bulkSetHWMAndCP( const std::vector<BRM::BulkSetHWMArg>& hwmArgs,
const std::vector<BRM::CPInfoMerge>& mergeCPDataArgs);
/**
* @brief Acquire a table lock for the specified table OID.
* If nonzero lockID is returned, then the table is already locked.
* @param tableOID Table to be locked.
* @param ownerName Requested (in) and current (out) owner for the lock.
* @param processID Requested (in) and current (out) pid for the lock.
* @param sessionID Requested (in) and current (out) session ID for the lock
* @param transID Requested (in) and current (out) transacton of the lock
* @param lockID Assigned or current lock for the specified table.
* @param errMsg Applicable error message.
*/
EXPORT int getTableLock ( OID tableOid,
std::string& ownerName,
uint32_t& processID,
int32_t& sessionID,
int32_t& transID,
uint64_t& lockID,
std::string& errMsg);
/**
* @brief Change the state of the specified table lock ID.
* @param lockID Lock for which the status is to be changed.
* @param lockState New state to be assigned to the specified lock.
* @param bChanged Indicates whether lock state was changed.
* @param errMsg Applicable error message.
*/
EXPORT int changeTableLockState ( uint64_t lockID,
BRM::LockState lockState,
bool& bChanged,
std::string& errMsg);
/**
* @brief Release the specified table lock ID.
* @param lockID Lock to be released.
* @param bReleased Indicates whether lock was released.
* @param errMsg Applicable error message.
*/
EXPORT int releaseTableLock( uint64_t lockID,
bool& bReleased,
std::string& errMsg);
/**
* @brief Get current table lock information for the specified lock ID.
* @param lockID Lock to be retrieved.
* @param lockInfo Current lock information for the specified lock.
* @param blockExists Indicates whether lock was found.
* @param errMsg Applicable error message.
*/
EXPORT int getTableLockInfo( uint64_t lockID,
BRM::TableLockInfo* lockInfo,
bool& bLockExists,
std::string& errMsg);
/**
* @brief Tell BRM to make a snapshot of it's current state to disk.
*/
int takeSnapshot();
/**
* @brief Save brm structures to file
*/
EXPORT int saveState();
//--------------------------------------------------------------------------
// Non-inline Versioning Functions Start Here
//--------------------------------------------------------------------------
/**
* @brief Commit the transaction
*/
EXPORT int commit( const BRM::VER_t transID );
/**
* @brief Copy blocks between write engine and version buffer
*/
EXPORT int copyVBBlock( IDBDataFile* pSourceFile,
IDBDataFile* pTargetFile,
const uint64_t sourceFbo,
const uint64_t targetFbo,
DbFileOp* fileOp,
const Column& column );
EXPORT int copyVBBlock( IDBDataFile* pSourceFile,
const OID sourceOid,
IDBDataFile* pTargetFile,
const OID targetOid,
const std::vector<uint32_t>& fboList,
const BRM::VBRange& freeList,
size_t& nBlocksProcessed,
DbFileOp* pFileOp,
const size_t fboCurrentOffset = 0 );
/**
* @brief Rollback the specified transaction
*/
EXPORT int rollBack( const BRM::VER_t transID, int sessionId );
/**
* @brief Rollback the specified transaction
*/
EXPORT int rollBackVersion( const BRM::VER_t transID, int sessionId );
/**
* @brief Rollback the specified transaction
*/
EXPORT int rollBackBlocks( const BRM::VER_t transID, int sessionId );
/**
* @brief Write specified LBID to version buffer
*/
EXPORT int writeVB( IDBDataFile* pFile,
const BRM::VER_t transID,
const OID oid,
const uint64_t lbid,
DbFileOp* pFileOp );
int writeVB( IDBDataFile* pFile,
const BRM::VER_t transID,
const OID weOid,
std::vector<uint32_t>& fboList,
std::vector<BRM::LBIDRange>& rangeList,
DbFileOp* pFileOp,
std::vector<BRM::VBRange>& freeList,
uint16_t dbRoot,
bool skipBeginVBCopy = false);
void writeVBEnd(const BRM::VER_t transID,
std::vector<BRM::LBIDRange>& rangeList);
BRM::DBRM* getDbrmObject();
void pruneLBIDList(BRM::VER_t transID,
std::vector<BRM::LBIDRange> *rangeList,
std::vector<uint32_t> *fboList) const;
//--------------------------------------------------------------------------
// Non-inline Versioning Functions End Here
//--------------------------------------------------------------------------
/**
* @brief static functions
*/
EXPORT static BRMWrapper* getInstance();
EXPORT static int getBrmRc(bool reset=true);
static bool getUseVb() { return m_useVb; }
static void setUseVb( const bool val ) { m_useVb = val; }
private:
//--------------------------------------------------------------------------
// Private methods
//--------------------------------------------------------------------------
BRMWrapper();
~BRMWrapper();
// disable copy constructor and assignment operator
BRMWrapper(const BRMWrapper&);
BRMWrapper& operator= ( const BRMWrapper& wrapper );
// Convert BRM return code to WE return code
int getRC( int brmRc, int errRc );
EXPORT void saveBrmRc( int brmRc );
IDBDataFile* openFile( const File& fileInfo,
const char* mode,
const bool bCache = false );
//--------------------------------------------------------------------------
// Private data members
//--------------------------------------------------------------------------
static BRMWrapper* volatile m_instance;
static boost::thread_specific_ptr<int> m_ThreadDataPtr;
static boost::mutex m_instanceCreateMutex;
#if defined(_MSC_VER) && !defined(WRITEENGINE_DLLEXPORT)
__declspec(dllimport)
#endif
EXPORT static bool m_useVb;
static OID m_curVBOid;
static IDBDataFile* m_curVBFile;
BRM::DBRM* blockRsltnMgrPtr;
};
//------------------------------------------------------------------------------
// Inline functions
//------------------------------------------------------------------------------
inline BRMWrapper::BRMWrapper()
{
blockRsltnMgrPtr = new BRM::DBRM();
}
inline BRMWrapper::~BRMWrapper()
{
if (blockRsltnMgrPtr)
delete blockRsltnMgrPtr;
blockRsltnMgrPtr = 0;
}
inline BRM::DBRM* BRMWrapper::getDbrmObject()
{
return blockRsltnMgrPtr;
}
inline int BRMWrapper::getRC( int brmRc, int errRc )
{
if (brmRc == BRM::ERR_OK)
return NO_ERROR;
saveBrmRc( brmRc );
return errRc;
}
inline int BRMWrapper::getLastHWM_DBroot( OID oid,
uint16_t dbRoot,
uint32_t& partition,
uint16_t& segment,
HWM& hwm,
int& status,
bool& bFound)
{
int rc = blockRsltnMgrPtr->getLastHWM_DBroot(
(BRM::OID_t)oid, dbRoot, partition, segment, hwm,
status, bFound);
return getRC( rc, ERR_BRM_GET_HWM );
}
inline int BRMWrapper::getLocalHWM( OID oid ,
uint32_t partition,
uint16_t segment,
HWM& hwm,
int& status)
{
int rc = blockRsltnMgrPtr->getLocalHWM(
(BRM::OID_t)oid, partition, segment, hwm, status);
return getRC( rc, ERR_BRM_GET_HWM );
}
inline int BRMWrapper::getExtentState( OID oid,
uint32_t partition,
uint16_t segment,
bool& bFound,
int& status)
{
int rc = blockRsltnMgrPtr->getExtentState(
(BRM::OID_t)oid, partition, segment, bFound, status);
return getRC( rc, ERR_BRM_GET_EXT_STATE );
}
inline unsigned BRMWrapper::getExtentRows()
{
return blockRsltnMgrPtr->getExtentRows( );
}
inline int BRMWrapper::getExtents( int oid,
std::vector<struct BRM::EMEntry>& entries,
bool sorted, bool notFoundErr,
bool incOutOfService )
{
int rc = blockRsltnMgrPtr->getExtents(
oid, entries, sorted, notFoundErr, incOutOfService);
return rc;
}
inline int BRMWrapper::getExtents_dbroot( int oid,
std::vector<struct BRM::EMEntry>& entries,
const uint16_t dbroot )
{
int rc = blockRsltnMgrPtr->getExtents_dbroot(
oid, entries, dbroot);
return rc;
}
inline bool BRMWrapper::isSystemReady()
{
return blockRsltnMgrPtr->getSystemReady() > 0 ? true : false;
}
inline int BRMWrapper::lookupLbidRanges( OID oid, BRM::LBIDRange_v& lbidRanges)
{
int rc = blockRsltnMgrPtr->lookup( oid, lbidRanges );
return getRC( rc, ERR_BRM_LOOKUP_LBID_RANGES );
}
inline int BRMWrapper::markExtentInvalid( const uint64_t lbid,
const execplan::CalpontSystemCatalog::ColDataType colDataType )
{
int rc = blockRsltnMgrPtr->markExtentInvalid( lbid, colDataType );
return getRC( rc, ERR_BRM_MARK_INVALID );
}
inline int BRMWrapper::markExtentsInvalid(std::vector<BRM::LBID_t>& lbids,
const std::vector<execplan::CalpontSystemCatalog::ColDataType>&
colDataTypes)
{
int rc = 0;
if (idbdatafile::IDBPolicy::useHdfs())
return rc;
rc = blockRsltnMgrPtr->markExtentsInvalid(lbids, colDataTypes);
return getRC( rc, ERR_BRM_MARK_INVALID );
}
inline int BRMWrapper::bulkSetHWMAndCP(
const std::vector<BRM::BulkSetHWMArg>& hwmArgs,
const std::vector<BRM::CPInfoMerge>& mergeCPDataArgs)
{
std::vector<BRM::CPInfo> setCPDataArgs; // not used
BRM::VER_t transID = 0; // n/a
int rc = blockRsltnMgrPtr->bulkSetHWMAndCP(
hwmArgs, setCPDataArgs, mergeCPDataArgs, transID );
return getRC( rc, ERR_BRM_BULK_UPDATE );
}
inline int BRMWrapper::setExtentsMaxMin(const BRM::CPInfoList_t& cpinfoList)
{
int rc = blockRsltnMgrPtr->setExtentsMaxMin( cpinfoList );
return getRC( rc, ERR_BRM_SET_EXTENTS_CP );
}
inline int BRMWrapper::rollbackColumnExtents_DBroot( const OID oid,
bool bDeleteAll,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
BRM::HWM_t hwm )
{
int rc = blockRsltnMgrPtr->rollbackColumnExtents_DBroot (
oid, bDeleteAll, dbRoot, partition, segment, hwm );
return getRC( rc, ERR_BRM_BULK_RB_COLUMN );
}
inline int BRMWrapper::rollbackDictStoreExtents_DBroot( OID oid,
uint16_t dbRoot,
uint32_t partition,
const std::vector<uint16_t>& segNums,
const std::vector<BRM::HWM_t>& hwms )
{
int rc = blockRsltnMgrPtr->rollbackDictStoreExtents_DBroot (
oid, dbRoot, partition, segNums, hwms );
return getRC( rc, ERR_BRM_BULK_RB_DCTNRY );
}
inline int BRMWrapper::deleteEmptyColExtents(
const std::vector<BRM::ExtentInfo>& extentsInfo )
{
int rc = blockRsltnMgrPtr->deleteEmptyColExtents ( extentsInfo );
return getRC( rc, ERR_BRM_DELETE_EXTENT_COLUMN );
}
inline int BRMWrapper::deleteEmptyDictStoreExtents(
const std::vector<BRM::ExtentInfo>& extentsInfo )
{
int rc = blockRsltnMgrPtr->deleteEmptyDictStoreExtents ( extentsInfo );
return getRC( rc, ERR_BRM_DELETE_EXTENT_DCTNRY );
}
inline int BRMWrapper::setLocalHWM( OID oid,
uint32_t partition,
uint16_t segment,
const HWM hwm )
{
int rc = blockRsltnMgrPtr->setLocalHWM(
(int)oid, partition, segment, hwm);
return getRC( rc, ERR_BRM_SET_HWM );
}
inline int BRMWrapper::bulkSetHWM( const std::vector<BRM::BulkSetHWMArg> & vec,
BRM::VER_t transID = 0)
{
int rc = blockRsltnMgrPtr->bulkSetHWM( vec, transID);
return getRC( rc, ERR_BRM_SET_HWM );
}
inline int BRMWrapper::takeSnapshot()
{
int rc = blockRsltnMgrPtr->takeSnapshot();
return getRC( rc, ERR_BRM_TAKE_SNAPSHOT );
}
} //end of namespace
#undef EXPORT
#endif // _WE_BRM_H_

485
writeengine/shared/we_bulkrollbackfile.cpp Normal file
View File

@ -0,0 +1,485 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: we_bulkrollbackfile.cpp 4737 2013-08-14 20:45:46Z bwilkinson $
*/
#include "we_bulkrollbackfile.h"
#include "we_bulkrollbackmgr.h"
#include <sstream>
#include "we_define.h"
#include "we_fileop.h"
#include "messageids.h"
#include "IDBDataFile.h"
using namespace idbdatafile;
using namespace execplan;
namespace WriteEngine
{
//------------------------------------------------------------------------------
// BulkRollbackFile constructor
//------------------------------------------------------------------------------
BulkRollbackFile::BulkRollbackFile(BulkRollbackMgr* mgr) : fMgr(mgr)
{
// Initialize empty dictionary header block used when reinitializing
// dictionary store extents.
const uint16_t freeSpace = BYTE_PER_BLOCK -
(HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE);
const uint64_t nextPtr = NOT_USED_PTR;
const uint16_t offSetZero = BYTE_PER_BLOCK;
const uint16_t endHeader = DCTNRY_END_HEADER;
memcpy(fDctnryHdr, &freeSpace, HDR_UNIT_SIZE);
memcpy(fDctnryHdr+ HDR_UNIT_SIZE, &nextPtr, NEXT_PTR_BYTES);
memcpy(fDctnryHdr+ HDR_UNIT_SIZE + NEXT_PTR_BYTES,
&offSetZero, HDR_UNIT_SIZE);
memcpy(fDctnryHdr+ HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE,
&endHeader, HDR_UNIT_SIZE);
}
//------------------------------------------------------------------------------
// BulkRollbackFile destructor
//------------------------------------------------------------------------------
BulkRollbackFile::~BulkRollbackFile()
{
}
//------------------------------------------------------------------------------
// Build the specified database segment file name.
//
// columnOID - OID of segment file to be found
// fileTypeFlag - true -> column file; false -> dictionary store file
// dbRoot - DBRoot of segment file to be found
// partNum - Partition number of segment file to be found
// segNum - Segment number of segment file to be found
// segFileName (out) - Name of segment file
//------------------------------------------------------------------------------
void BulkRollbackFile::buildSegmentFileName(
OID columnOID,
bool fileTypeFlag,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
std::string& segFileName )
{
char fileName[FILE_NAME_SIZE];
int rc = fDbFile.getFileName( columnOID, fileName,
dbRoot, partNum, segNum );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error constructing " <<
(fileTypeFlag ? "column" : "dictionary store") <<
" filename for deletion" <<
"; columnOID-" << columnOID <<
"; dbRoot-" << dbRoot <<
"; partNum-" << partNum <<
"; segNum-" << segNum <<
"; " << ec.errorString(rc);
throw WeException( oss.str(), rc );
}
segFileName = fileName;
}
//------------------------------------------------------------------------------
// Delete the specified database segment file.
//
// columnOID - OID of segment file to be deleted
// fileTypeFlag - true -> column file; false -> dictionary store file
// dbRoot - DBRoot of segment file to be deleted
// partNum - Partition number of segment file to be deleted
// segNum - Segment number of segment file to be deleted
// segFileName - Name of file to be deleted
//------------------------------------------------------------------------------
void BulkRollbackFile::deleteSegmentFile(
OID columnOID,
bool fileTypeFlag,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
const std::string& segFileName )
{
std::ostringstream msgText;
msgText << "Deleting " << (fileTypeFlag ? "column" : "dictionary store") <<
" file: dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText.str() );
// delete the db segment file if it exists
int rc = fDbFile.deleteFile( segFileName.c_str() );
if (rc != NO_ERROR)
{
if (rc != ERR_FILE_NOT_EXIST)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error deleting segment file"
"; columnOID-" << columnOID <<
"; dbRoot-" << dbRoot <<
"; partNum-" << partNum <<
"; segNum-" << segNum <<
"; " << ec.errorString(rc);
throw WeException( oss.str(), rc );
}
}
}
//------------------------------------------------------------------------------
// Truncate the specified database segment file to the given file offset.
//
// columnOID - OID of segment file to be truncated
// dbRoot - DBRoot of segment file to be truncated
// partNum - Partition number of segment file to be truncated
// segNum - Segment number of segment file to be truncated
// fileSizeBlocks - Number of blocks to be left in the file. Remainder of file
// is to be truncated.
//------------------------------------------------------------------------------
void BulkRollbackFile::truncateSegmentFile(
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long fileSizeBlocks )
{
long long fileSizeBytes = fileSizeBlocks * BYTE_PER_BLOCK;
std::ostringstream msgText;
msgText << "Truncating column file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; totBlks-" << fileSizeBlocks <<
"; fileSize(bytes)-" << fileSizeBytes;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText.str() );
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile);
if (pFile == 0)
{
std::ostringstream oss;
oss << "Error opening column segment file to rollback extents "
"from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), ERR_FILE_OPEN );
}
int rc = fDbFile.truncateFile( pFile, fileSizeBytes );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating column extents from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
fDbFile.closeFile( pFile );
}
//------------------------------------------------------------------------------
// Reinitialize a column segment extent (in the db file) to empty values,
// following the HWM. Remaining extents in the file are truncated.
//
// columnOID - OID of segment file to be reinitialized
// dbRoot - DBRoot of segment file to be reinitialized
// partNum - Partition number of segment file to be reinitialized
// segNum - Segment number of segment file to be reinitialized
// startOffsetBlk - File offset (after the HWM block), at which the file is
// to be reinitialized. Value is in blocks.
// nBlocks - Number of blocks to be reinitialized
// colType - Data type of the applicable column
// colWidth - Width in bytes, of the applicable column
// restoreHwmChk - Specifies whether HWM chunk is to be restored. n/a for
// uncompressed, but defined in this base class for the
// compressed derived class.
//------------------------------------------------------------------------------
void BulkRollbackFile::reInitTruncColumnExtent(
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool /*restoreHwmChk*/ )
{
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
std::ostringstream msgText;
msgText << "Reinit HWM column extent in db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; offset(bytes)-" << startOffset <<
"; freeBlks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText.str() );
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile);
if (pFile == 0)
{
std::ostringstream oss;
oss << "Error opening HWM column segment file to rollback extents "
"from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), ERR_FILE_OPEN );
}
// nBlocks is based on full extents, but if the database file only has an
// abbreviated extent, then we reset nBlocks to reflect the size of the file
// (Only the 1st extent in part0, seg0 employs an abbreviated extent.)
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
if ((partNum == 0) && (segNum == 0))
{
long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * colWidth;
if (startOffset <= nBytesInAbbrevExtent)
{
// This check would prevent us from truncating back to an
// abbreviated extent if the failed import expanded the initial
// extent; but when adding compression, decided to go ahead and
// truncate back to an abbreviated extent.
//long long fileSizeBytes;
//int rc = fDbFile.getFileSize2(pFile,fileSizeBytes);
//if (fileSizeBytes == nBytesInAbbrevExtent)
{
nBlocks = (nBytesInAbbrevExtent-startOffset) / BYTE_PER_BLOCK;
std::ostringstream msgText2;
msgText2 << "Reinit (abbrev) HWM column extent in db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; offset(bytes)-" << startOffset <<
"; freeBlks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText2.str() );
}
}
}
// Initialize the remainder of the extent after the HWM block
uint64_t emptyVal = fDbFile.getEmptyRowValue( colType, colWidth );
int rc = fDbFile.reInitPartialColumnExtent( pFile,
startOffset,
nBlocks,
emptyVal,
colWidth );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error rolling back HWM column extent from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
// Truncate the remainder of the file
rc = fDbFile.truncateFile( pFile, pFile->tell() );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating post-HWM column extents "
"from HWM segment DB file for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
fDbFile.closeFile( pFile );
}
//------------------------------------------------------------------------------
// Reinitialize a dictionary segment extent (in the db file) to empty blocks,
// following the HWM. Remaining extents in the file are truncated.
//
// dStoreOID - OID of segment store file to be reinitialized
// dbRoot - DBRoot of segment file to be reinitialized
// partNum - Partition number of segment file to be reinitialized
// segNum - Segment number of segment file to be reinitialized
// startOffsetBlk - Starting block (after the HWM block), at which the file is
// to be reinitialized. Value is in raw data blocks.
// nBlocks - Number of blocks to be reinitialized
//------------------------------------------------------------------------------
void BulkRollbackFile::reInitTruncDctnryExtent(
OID dStoreOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks )
{
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
std::ostringstream msgText;
msgText << "Reinit dictionary store extent in db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; offset(bytes)-" << startOffset <<
"; numblks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, dStoreOID, msgText.str() );
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(dStoreOID, dbRoot, partNum, segNum, segFile);
if (pFile == 0)
{
std::ostringstream oss;
oss << "Error opening dictionary store segment file to rollback extents"
" from DB for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), ERR_FILE_OPEN );
}
// nBlocks is based on full extents, but if the database file only has an
// abbreviated extent, then we reset nBlocks to reflect the size of the file
// (Unlike column files which only employ an abbreviated extent for the
// 1st extent in part0, seg0, all new store files start with abbrev extent)
const uint32_t PSEUDO_COL_WIDTH = 8; // simulated col width for dictionary
long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK *
PSEUDO_COL_WIDTH;
if (startOffset <= nBytesInAbbrevExtent)
{
// This check would prevent us from truncating back to an
// abbreviated extent if the failed import expanded the initial
// extent; but when adding compression, decided to go ahead and
// truncate back to an abbreviated extent.
//long long fileSizeBytes;
//int rc = fDbFile.getFileSize2(pFile,fileSizeBytes);
//if (fileSizeBytes == nBytesInAbbrevExtent)
{
nBlocks = (nBytesInAbbrevExtent-startOffset) / BYTE_PER_BLOCK;
std::ostringstream msgText2;
msgText2 << "Reinit (abbrev) dictionary store extent in db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; offset(bytes)-" << startOffset <<
"; numblks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, dStoreOID, msgText2.str() );
}
}
// Initialize the remainder of the extent after the HWM block
int rc = fDbFile.reInitPartialDctnryExtent( pFile,
startOffset,
nBlocks,
fDctnryHdr,
DCTNRY_HEADER_SIZE );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error rolling back HWM dictionary store extent from DB for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
// Truncate the remainder of the file
rc = fDbFile.truncateFile( pFile, pFile->tell() );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating post-HWM dictionary store extents "
"from DB file for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
fDbFile.closeFile( pFile );
}
//------------------------------------------------------------------------------
// For uncompressed data...
// Always return true, in order to always reInit the post-HWM blocks for the
// HWM extent to empty values.
// This function is defined as a stub, so that the derived compression
// class can override this functionality, and return true or false depending
// on whether the HWM chunk was modified and backed up to disk.
//------------------------------------------------------------------------------
bool BulkRollbackFile::doWeReInitExtent( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const
{
return true;
}
} //end of namespace

207
writeengine/shared/we_bulkrollbackfile.h Normal file
View File

@ -0,0 +1,207 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: we_bulkrollbackfile.h 4675 2013-06-13 15:20:37Z dcathey $
*/
/** @file
* Contains class to restore db files on behalf of BulkRollBackMgr.
* In some cases this means restoring an extent to it's previous state.
* In some cases, it may mean removing extent(s) that were added to a file.
* In other cases it may mean completely deleting a db segment file, if that
* file did not exist prior to an aborted bulk load.
*/
#ifndef WE_BULKROLLBACKFILE_H_
#define WE_BULKROLLBACKFILE_H_
#include <string>
#include "we_define.h"
#include "we_type.h"
#include "we_fileop.h"
namespace WriteEngine
{
class BulkRollbackMgr;
//------------------------------------------------------------------------------
/** @brief Class used by BulkRollbackMgr to restore db files.
*/
//------------------------------------------------------------------------------
class BulkRollbackFile
{
public:
/** @brief BulkRollbackFile constructor
* @param mgr The controlling BulkRollbackMgr object.
*/
BulkRollbackFile(BulkRollbackMgr* mgr);
/** @brief BulkRollbackFile destructor
*/
virtual ~BulkRollbackFile();
/** @brief Construct the relevant db filename.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the segment file to be deleted
* @param fileTypeFlag file type (true->column; false->dictionary)
* @param dbRoot DBRoot of the segment file to be deleted
* @param partNum Partition number of the segment file to be deleted
* @param segNum Segment number of the segment file to be deleted
* @param segFileName (out) Name of segment file
*/
void buildSegmentFileName(OID columnOID,
bool fileTypeFlag,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
std::string& segFileName);
/** @brief Delete a segment file.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the segment file to be deleted
* @param fileTypeFlag file type (true->column; false->dictionary)
* @param dbRoot DBRoot of the segment file to be deleted
* @param partNum Partition number of the segment file to be deleted
* @param segNum Segment number of the segment file to be deleted
* @param segFileName Name of segment file to be deleted
*/
void deleteSegmentFile(OID columnOID,
bool fileTypeFlag,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
const std::string& segFileName );
/** @brief Construct a directory path.
*
* @param oid (in) OID to use in constructing directory path
* @param dbRoot (in) DBRoot to use in constructing directory path
* @param partition (in) Partition number to use in constructing dir path
* @param dirName (out)Directory path constructed from input arguments
* @return returns NO_ERROR if success
*/
int buildDirName( OID oid,
uint16_t dbRoot,
uint32_t partition,
std::string& dirName);
/** @brief Do we reinit trailing blocks in the HWM extent for the specified
* segment file
*
* The base behavior of this function always returns true, to reinit
* any trailing blocks in the HWM extent (for uncompressed data) to
* empty values.
*
* @param columnOID OID of the segment file in question
* @param dbRoot DBRoot for the segment file in question
* @param partNum Partition number for the segment file in question
* @param segNum Segment number for the segment file in question
*/
virtual bool doWeReInitExtent( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const;
/** @brief Reinitialize the specified column segment file starting at
* startOffsetBlk, and truncate trailing extents.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
* @param colType Column type of the relevant segment file
* @param colWidth Width in bytes of column.
* @param restoreHwmChk Restore HWM chunk (n/a to uncompressed)
*/
virtual void reInitTruncColumnExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
execplan::CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool restoreHwmChk );
/** @brief Reinitialize the specified dictionary store segment file starting
* at startOffsetBlk, and truncate trailing extents.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
*/
virtual void reInitTruncDctnryExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks );
/** @brief Truncate the specified segment file to a specified num of bytes
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param fileSizeBlocks Number of blocks to retain in the file
*/
virtual void truncateSegmentFile( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long filesSizeBlocks );
protected:
BulkRollbackMgr* fMgr; // Bulk Rollback controller
FileOp fDbFile; // interface to DB file
unsigned char fDctnryHdr[DCTNRY_HEADER_SIZE]; // empty dctnry store blk
private:
// Disable unnecessary copy constructor and assignment operator
BulkRollbackFile(const BulkRollbackFile& rhs);
BulkRollbackFile& operator=(const BulkRollbackFile& rhs);
};
//------------------------------------------------------------------------------
// Inline functions
//------------------------------------------------------------------------------
inline int BulkRollbackFile::buildDirName( OID oid,
uint16_t dbRoot,
uint32_t partition,
std::string& dirName)
{
return fDbFile.getDirName( oid, dbRoot, partition, dirName );
}
} //end of namespace
#endif // WE_BULKROLLBACKFILE_H_

988
writeengine/shared/we_bulkrollbackfilecompressed.cpp Normal file
View File

@ -0,0 +1,988 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: we_bulkrollbackfilecompressed.cpp 4737 2013-08-14 20:45:46Z bwilkinson $
*/
#include "we_bulkrollbackfilecompressed.h"
#include <sstream>
#include <boost/scoped_array.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/convenience.hpp>
#include "we_define.h"
#include "we_fileop.h"
#include "we_bulkrollbackmgr.h"
#include "we_convertor.h"
#include "messageids.h"
#include "IDBDataFile.h"
#include "IDBPolicy.h"
using namespace idbdatafile;
using namespace compress;
using namespace execplan;
namespace
{
const char* DATA_DIR_SUFFIX = "_data";
}
namespace WriteEngine
{
//------------------------------------------------------------------------------
// BulkRollbackFileCompressed constructor
//------------------------------------------------------------------------------
BulkRollbackFileCompressed::BulkRollbackFileCompressed(BulkRollbackMgr* mgr) :
BulkRollbackFile(mgr)
{
}
//------------------------------------------------------------------------------
// BulkRollbackFileCompressed destructor
//------------------------------------------------------------------------------
BulkRollbackFileCompressed::~BulkRollbackFileCompressed()
{
}
//------------------------------------------------------------------------------
// Truncate the specified database segment file to the extent specified by
// the given file offset. Also updates the header(s) as well.
//
// columnOID - OID of segment file to be truncated
// dbRoot - DBRoot of segment file to be truncated
// partNum - Partition number of segment file to be truncated
// segNum - Segment number of segment file to be truncated
// fileSizeBlocks - Number of raw data blocks to be left in the file.
// Remainder of file is to be truncated.
//------------------------------------------------------------------------------
void BulkRollbackFileCompressed::truncateSegmentFile(
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long fileSizeBlocks )
{
std::ostringstream msgText1;
msgText1 << "Truncating compressed column file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; rawTotBlks-" << fileSizeBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText1.str() );
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile);
if (pFile == 0)
{
std::ostringstream oss;
oss << "Error opening compressed column segment file to rollback "
"extents from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), ERR_FILE_OPEN );
}
// Read and parse the header pointers
char hdrs[ IDBCompressInterface::HDR_BUF_LEN * 2 ];;
CompChunkPtrList chunkPtrs;
std::string errMsg;
int rc = loadColumnHdrPtrs(pFile, hdrs, chunkPtrs, errMsg);
if (rc != NO_ERROR)
{
std::ostringstream oss;
oss << "Error reading compressed column ptr headers from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << errMsg;
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
// Locate the chunk containing the last block we intend to keep
unsigned int blockOffset = fileSizeBlocks - 1;
unsigned int chunkIndex = 0;
unsigned int blkOffsetInChunk = 0;
fCompressor.locateBlock( blockOffset, chunkIndex, blkOffsetInChunk );
// Truncate the extra extents that are to be aborted
if (chunkIndex < chunkPtrs.size())
{
long long fileSizeBytes = chunkPtrs[chunkIndex].first +
chunkPtrs[chunkIndex].second;
std::ostringstream msgText2;
msgText2 << "Compressed column file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; truncated to " << fileSizeBytes << " bytes";
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText2.str() );
// Drop off any trailing pointers (that point beyond the last block)
fCompressor.setBlockCount( hdrs, fileSizeBlocks );
std::vector<uint64_t> ptrs;
for (unsigned i=0; i<=chunkIndex; i++)
{
ptrs.push_back( chunkPtrs[i].first );
}
ptrs.push_back( chunkPtrs[chunkIndex].first +
chunkPtrs[chunkIndex].second );
fCompressor.storePtrs( ptrs, hdrs );
rc = fDbFile.writeHeaders( pFile, hdrs );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error writing compressed column headers to DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
// Finally, we truncate the data base column segment file
rc = fDbFile.truncateFile( pFile, fileSizeBytes );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating compressed column extents from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
} // end of (chunkIndex < chunkPtrs.size())
fDbFile.closeFile( pFile );
}
//------------------------------------------------------------------------------
// Reinitialize a column segment extent (in the db file) to empty values,
// following the HWM. Remaining extents in the file are truncated.
// Also updates the header(s) as well.
//
// columnOID - OID of segment file to be reinitialized
// dbRoot - DBRoot of segment file to be reinitialized
// partNum - Partition number of segment file to be reinitialized
// segNum - Segment number of segment file to be reinitialized
// startOffsetBlk - File offset (after the HWM block), at which the file is
// to be reinitialized. Value is in raw data blocks.
// nBlocks - Number of blocks to be reinitialized
// colType - Data type of the applicable column
// colWidth - Width in bytes, of the applicable column
// restoreHwmChk - Specifies whether HWM chunk is to be restored.
//------------------------------------------------------------------------------
void BulkRollbackFileCompressed::reInitTruncColumnExtent(
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool restoreHwmChk )
{
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
std::ostringstream msgText1;
msgText1 << "Reinit HWM compressed column extent in db file" <<
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; rawOffset(bytes)-"<< startOffset <<
"; rawFreeBlks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText1.str() );
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile);
if (pFile == 0)
{
std::ostringstream oss;
oss << "Error opening compressed column segment file to rollback "
"extents from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), ERR_FILE_OPEN );
}
// Read and parse the header pointers
char hdrs[ IDBCompressInterface::HDR_BUF_LEN * 2 ];
CompChunkPtrList chunkPtrs;
std::string errMsg;
int rc = loadColumnHdrPtrs(pFile, hdrs, chunkPtrs, errMsg);
if (rc != NO_ERROR)
{
std::ostringstream oss;
oss << "Error reading compressed column ptr headers from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << errMsg;
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
// Locate the chunk containing the last block we intend to keep
unsigned int blockOffset = startOffsetBlk - 1;
unsigned int chunkIndex = 0;
unsigned int blkOffsetInChunk = 0;
fCompressor.locateBlock( blockOffset, chunkIndex, blkOffsetInChunk );
if (chunkIndex < chunkPtrs.size())
{
// Read backup copy of HWM chunk and restore it's contents
uint64_t restoredChunkLen = 0;
uint64_t restoredFileSize = 0;
if (restoreHwmChk)
{
rc = restoreHWMChunk(pFile, columnOID, partNum, segNum,
chunkPtrs[chunkIndex].first,
restoredChunkLen, restoredFileSize, errMsg);
if (rc != NO_ERROR)
{
std::ostringstream oss;
oss << "Error restoring HWM chunk for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; blkoff-" << blockOffset <<
"; " << errMsg;
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
}
else
{
restoredChunkLen = chunkPtrs[chunkIndex].second;
// leave truncated to last chunk if no extra blocks needed
if (nBlocks == 0)
restoredFileSize = chunkPtrs[chunkIndex].first +
chunkPtrs[chunkIndex].second;
else
restoredFileSize = (chunkPtrs[chunkIndex].first +
chunkPtrs[chunkIndex].second) +
(uint64_t)(nBlocks * BYTE_PER_BLOCK);
}
// nBlocks is based on full extents, but if database file only has an
// abbreviated extent, then we reset nBlocks to reflect the size of a
// file with a single abbreviated extent.
// (Only the 1st extent in part0, seg0 employs an abbreviated extent.)
bool bAbbreviatedExtent = false;
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
if ((partNum == 0) && (segNum == 0))
{
long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK *
colWidth;
if (startOffset <= nBytesInAbbrevExtent)
{
nBlocks = (nBytesInAbbrevExtent-startOffset) / BYTE_PER_BLOCK;
bAbbreviatedExtent = true;
}
}
long long fileSizeBytes = restoredFileSize;
std::ostringstream msgText2;
msgText2 << "HWM compressed column file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum;
if (bAbbreviatedExtent) // log adjusted nBlock count for abbrev extent
msgText2 << "; rawFreeBlks-" << nBlocks << " (abbrev)";
msgText2 << "; restoredChunk-" << restoredChunkLen << " bytes";
if (!restoreHwmChk)
msgText2 << " (no change)";
msgText2 << "; truncated to " << fileSizeBytes << " bytes";
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText2.str() );
// Initialize the remainder of the extent after the HWM chunk.
// Just doing an ftruncate() reinits the file to 0's, which may or may
// not actually reserve disk space if ftruncate is growing the file.
// So reinit the blocks by calling reInitPartialColumnExtent() to help
// avoid disk fragmentation. Be careful not to init > 1 extent, be-
// cause that is the limit on what that function was intended to do.
const unsigned BLKS_PER_EXTENT =
(BRMWrapper::getInstance()->getExtentRows() * colWidth) /
BYTE_PER_BLOCK;
long long nBlocksToInit = (fileSizeBytes -
(chunkPtrs[chunkIndex].first + restoredChunkLen)) / BYTE_PER_BLOCK;
if (nBlocksToInit > BLKS_PER_EXTENT)
nBlocksToInit = BLKS_PER_EXTENT; // don't init > 1 full extent
if (nBlocksToInit > 0)
{
uint64_t emptyVal = fDbFile.getEmptyRowValue( colType, colWidth );
rc = fDbFile.reInitPartialColumnExtent( pFile,
(chunkPtrs[chunkIndex].first + restoredChunkLen),
nBlocksToInit,
emptyVal,
colWidth );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error clearing HWM column extent from DB for"
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
}
// Drop off any trailing pointers (that point beyond the last block).
// Watch for the special case where we are restoring a db file as an
// empty file (chunkindex=0 and restoredChunkLen=0); in this case we
// just restore the first pointer (set to 8192).
fCompressor.setBlockCount( hdrs, (startOffsetBlk + nBlocks) );
std::vector<uint64_t> newPtrs;
if ((chunkIndex > 0) || (restoredChunkLen > 0))
{
for (unsigned int i=0; i<=chunkIndex; i++)
{
newPtrs.push_back( chunkPtrs[i].first );
}
}
newPtrs.push_back( chunkPtrs[chunkIndex].first + restoredChunkLen );
fCompressor.storePtrs( newPtrs, hdrs );
rc = fDbFile.writeHeaders( pFile, hdrs );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error writing compressed column headers to DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
// Finally, we truncate the data base column segment file
rc = fDbFile.truncateFile( pFile, fileSizeBytes );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating compressed column extents from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
} // end of (chunkIndex < chunkPtrs.size())
fDbFile.closeFile( pFile );
}
//------------------------------------------------------------------------------
// Load header pointer data for compressed column file.
//
// pFile - FILE ptr to be used in reading header data.
// hdrs - (out) Raw header data.
// chunkPtrs - (out) Chunk ptrs extracted from raw header data.
// errMsg - (out) Error message if applicable.
//------------------------------------------------------------------------------
int BulkRollbackFileCompressed::loadColumnHdrPtrs(
IDBDataFile* pFile,
char* hdrs,
CompChunkPtrList& chunkPtrs,
std::string& errMsg) const
{
// Read the header pointers
int rc = fDbFile.readHeaders( pFile, hdrs );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Header read error: " << ec.errorString(rc);
errMsg = oss.str();
return rc;
}
// Parse the header pointers
int rc1 = fCompressor.getPtrList( hdrs, chunkPtrs );
if (rc1 != 0)
{
rc = ERR_METADATABKUP_COMP_PARSE_HDRS;
WErrorCodes ec;
std::ostringstream oss;
oss << "Header parsing error (" << rc1 << "): " << ec.errorString(rc);
errMsg = oss.str();
return rc;
}
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Reinitialize a dictionary segment extent (in the db file) to empty blocks,
// following the HWM. Remaining extents in the file are truncated.
// Also updates the header(s) as well.
//
// dStoreOID - OID of segment store file to be reinitialized
// dbRoot - DBRoot of segment file to be reinitialized
// partNum - Partition number of segment file to be reinitialized
// segNum - Segment number of segment file to be reinitialized
// startOffsetBlk - Starting block (after the HWM block), at which the file is
// to be reinitialized. Value is in raw data blocks.
// nBlocks - Number of blocks to be reinitialized
//------------------------------------------------------------------------------
void BulkRollbackFileCompressed::reInitTruncDctnryExtent(
OID dStoreOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks )
{
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
std::ostringstream msgText1;
msgText1 << "Reinit HWM compressed dictionary store extent in db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; rawOffset(bytes)-" << startOffset <<
"; rawFreeBlks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, dStoreOID, msgText1.str() );
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(dStoreOID, dbRoot, partNum, segNum, segFile);
if (pFile == 0)
{
std::ostringstream oss;
oss << "Error opening compressed dictionary store segment file to "
"rollback extents from DB for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), ERR_FILE_OPEN );
}
char controlHdr[ IDBCompressInterface::HDR_BUF_LEN ];
CompChunkPtrList chunkPtrs;
uint64_t ptrHdrSize;
std::string errMsg;
int rc = loadDctnryHdrPtrs(pFile, controlHdr, chunkPtrs, ptrHdrSize,errMsg);
if (rc != NO_ERROR)
{
std::ostringstream oss;
oss << "Error reading compressed dctnry ptr headers from DB for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << errMsg;
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
// Locate the chunk containing the last block we intend to keep
unsigned int blockOffset = startOffsetBlk - 1;
unsigned int chunkIndex = 0;
unsigned int blkOffsetInChunk = 0;
fCompressor.locateBlock( blockOffset, chunkIndex, blkOffsetInChunk );
if (chunkIndex < chunkPtrs.size())
{
// Read backup copy of HWM chunk and restore it's contents
uint64_t restoredChunkLen = 0;
uint64_t restoredFileSize = 0;
rc = restoreHWMChunk(pFile, dStoreOID, partNum, segNum,
chunkPtrs[chunkIndex].first,
restoredChunkLen, restoredFileSize, errMsg);
if (rc == ERR_FILE_NOT_EXIST)
{
std::ostringstream msgText3;
msgText3 << "No restore needed to Compressed dictionary file" <<
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, dStoreOID, msgText3.str() );
fDbFile.closeFile( pFile );
return;
}
if (rc != NO_ERROR)
{
std::ostringstream oss;
oss << "Error restoring HWM chunk for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; blkoff-" << blockOffset <<
"; " << errMsg;
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
// nBlocks is based on full extents, but if database file only has an
// abbreviated extent, then we reset nBlocks to reflect the file size.
// (Unlike column files which only employ an abbreviated extent for the
// 1st extent in part0, seg0, all store files start with abbrev extent)
bool bAbbreviatedExtent = false;
const uint32_t PSEUDO_COL_WIDTH = 8; // simulated col width for dctnry
long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK *
PSEUDO_COL_WIDTH;
if (startOffset <= nBytesInAbbrevExtent)
{
nBlocks = (nBytesInAbbrevExtent-startOffset) / BYTE_PER_BLOCK;
bAbbreviatedExtent = true;
}
long long fileSizeBytes = restoredFileSize;
std::ostringstream msgText2;
msgText2 << "HWM compressed dictionary file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum;
if (bAbbreviatedExtent) // log adjusted nBlock count for abbrev extent
msgText2 << "; rawFreeBlks-" << nBlocks << " (abbrev)";
msgText2 << "; restoredChunk-" << restoredChunkLen << " bytes" <<
"; truncated to " << fileSizeBytes << " bytes";
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, dStoreOID, msgText2.str() );
// Initialize the remainder of the extent after the HWM chunk
// Just doing an ftruncate() reinits the file to 0's, which may or may
// not actually reserve disk space if ftruncate is growing the file.
// So reinit the blocks by calling reInitPartialDctnryExtent() to help
// avoid disk fragmentation. Be careful not to init > 1 extent, be-
// cause that is the limit on what that function was intended to do.
const unsigned BLKS_PER_EXTENT =
(BRMWrapper::getInstance()->getExtentRows() * PSEUDO_COL_WIDTH) /
BYTE_PER_BLOCK;
long long nBlocksToInit = (fileSizeBytes -
(chunkPtrs[chunkIndex].first + restoredChunkLen)) / BYTE_PER_BLOCK;
if (nBlocksToInit > BLKS_PER_EXTENT)
nBlocksToInit = BLKS_PER_EXTENT; // don't init > 1 full extent
if (nBlocksToInit > 0)
{
rc = fDbFile.reInitPartialDctnryExtent( pFile,
(chunkPtrs[chunkIndex].first + restoredChunkLen),
nBlocksToInit,
fDctnryHdr,
DCTNRY_HEADER_SIZE );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error clearing HWM dictionary store extent from DB for"
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
}
// Drop off any trailing pointers (that point beyond the last block).
// Watch for the special case where we are restoring a db file as an
// empty file (chunkindex=0 and restoredChunkLen=0); in this case we
// just restore the first pointer (set to 8192).
fCompressor.setBlockCount( controlHdr, (startOffsetBlk + nBlocks) );
std::vector<uint64_t> newPtrs;
if ((chunkIndex > 0) || (restoredChunkLen > 0))
{
for (unsigned int i=0; i<=chunkIndex; i++)
{
newPtrs.push_back( chunkPtrs[i].first );
}
}
newPtrs.push_back( chunkPtrs[chunkIndex].first + restoredChunkLen );
char* pointerHdr = new char[ptrHdrSize];
fCompressor.storePtrs( newPtrs, pointerHdr, ptrHdrSize );
rc = fDbFile.writeHeaders( pFile, controlHdr, pointerHdr, ptrHdrSize );
delete[] pointerHdr;
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error writing compressed dictionary headers to DB for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
// Finally, we truncate the data base dictionary store segment file
rc = fDbFile.truncateFile( pFile, fileSizeBytes );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating compressed dictionary store extents "
"from DB file for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
} // end of (chunkIndex < chunkPtrs.size())
fDbFile.closeFile( pFile );
}
//------------------------------------------------------------------------------
// Load header pointer data for compressed dictionary file.
//
// pFile - FILE ptr to be used in reading header data.
// controlHdr- (out) Raw data from control header.
// chunkPtrs - (out) Chunk ptrs extracted from raw header data.
// ptrHdrSize- (out) Size of pointer header.
// errMsg - (out) Error message if applicable.
//------------------------------------------------------------------------------
int BulkRollbackFileCompressed::loadDctnryHdrPtrs(
IDBDataFile* pFile,
char* controlHdr,
CompChunkPtrList& chunkPtrs,
uint64_t& ptrHdrSize,
std::string& errMsg) const
{
int rc = fDbFile.readFile(
pFile, (unsigned char*)controlHdr, IDBCompressInterface::HDR_BUF_LEN);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Control header read error: " << ec.errorString(rc);
errMsg = oss.str();
return rc;
}
int rc1 = fCompressor.verifyHdr( controlHdr );
if (rc1 != 0)
{
rc = ERR_METADATABKUP_COMP_VERIFY_HDRS;
WErrorCodes ec;
std::ostringstream oss;
oss << "Control header verify error (" << rc1 << "): " <<
ec.errorString(rc);
errMsg = oss.str();
return rc;
}
uint64_t hdrSize = fCompressor.getHdrSize(controlHdr);
ptrHdrSize = hdrSize - IDBCompressInterface::HDR_BUF_LEN;
char* pointerHdr = new char[ptrHdrSize];
rc = fDbFile.readFile(pFile, (unsigned char*)pointerHdr, ptrHdrSize);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Pointer header read error: " << ec.errorString(rc);
errMsg = oss.str();
delete[] pointerHdr;
return rc;
}
// Parse the header pointers
rc1 = fCompressor.getPtrList( pointerHdr, ptrHdrSize, chunkPtrs );
delete[] pointerHdr;
if (rc1 != 0)
{
rc = ERR_METADATABKUP_COMP_PARSE_HDRS;
WErrorCodes ec;
std::ostringstream oss;
oss << "Pointer header parsing error (" << rc1 << "): " <<
ec.errorString(rc);
errMsg = oss.str();
return rc;
}
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Restore the HWM chunk back to the contents saved in the backup file.
//
// pFile - FILE* to segment file being reinitialized
// columnOID - OID of segment file to be reinitialized
// partNum - Partition num of seg file to reinitialize
// segNum - Segment num of seg file to reinitialize
// fileOffsetByteForRestoredChunk - Offset to pFile where restored chunk is to
// be written
// restoredChunkLen (out) - Length of restored chunk (in bytes)
// restoredFileSize (out) - Size of file (in bytes) when backup was made
// errMsg (out) - Error msg if error returned
//------------------------------------------------------------------------------
int BulkRollbackFileCompressed::restoreHWMChunk(
IDBDataFile* pFile,
OID columnOID,
uint32_t partNum,
uint32_t segNum,
uint64_t fileOffsetByteForRestoredChunk,
uint64_t& restoredChunkLen,
uint64_t& restoredFileSize,
std::string& errMsg)
{
restoredChunkLen = 0;
restoredFileSize = 0;
// Open the backup HWM chunk file
std::ostringstream oss;
oss << "/" << columnOID << ".p" << partNum << ".s" << segNum;
std::string bulkRollbackSubPath( fMgr->getMetaFileName() );
bulkRollbackSubPath += DATA_DIR_SUFFIX;
bulkRollbackSubPath += oss.str();
if ( !IDBPolicy::exists( bulkRollbackSubPath.c_str() ) )
{
std::ostringstream oss;
oss << "Backup file does not exist: " << bulkRollbackSubPath;
errMsg = oss.str();
return ERR_FILE_NOT_EXIST;
}
IDBDataFile* backupFile = IDBDataFile::open(
IDBPolicy::getType( bulkRollbackSubPath.c_str(), IDBPolicy::WRITEENG ),
bulkRollbackSubPath.c_str(),
"rb",
0,
pFile->colWidth() );
if (!backupFile)
{
int errrc = errno;
std::string eMsg;
Convertor::mapErrnoToString(errrc, eMsg);
std::ostringstream oss;
oss << "Error opening backup file " <<
bulkRollbackSubPath << "; " << eMsg;
errMsg = oss.str();
return ERR_METADATABKUP_COMP_OPEN_BULK_BKUP;
}
// Read the chunk length and file size
uint64_t sizeHdr[2];
size_t bytesRead = readFillBuffer(backupFile, (char*)sizeHdr,
sizeof(uint64_t)*2);
if (bytesRead != sizeof(uint64_t)*2)
{
int errrc = errno;
std::string eMsg;
Convertor::mapErrnoToString(errrc, eMsg);
std::ostringstream oss;
oss << "Error reading chunk length from backup file " <<
bulkRollbackSubPath << "; " << eMsg;
errMsg = oss.str();
delete backupFile;
return ERR_METADATABKUP_COMP_READ_BULK_BKUP;
}
restoredChunkLen = sizeHdr[0];
restoredFileSize = sizeHdr[1];
// Position the destination offset in the DB file
int rc = fDbFile.setFileOffset(pFile, fileOffsetByteForRestoredChunk,
SEEK_SET);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error setting column file offset" <<
"; offset-" << fileOffsetByteForRestoredChunk <<
"; " << ec.errorString(rc);
errMsg = oss.str();
delete backupFile;
return rc;
}
// Copy backup version of chunk back to DB, unless chunk length is 0
// in which case we have nothing to copy.
if (restoredChunkLen > 0)
{
// Read the HWM chunk to be restored
unsigned char* chunk = new unsigned char[restoredChunkLen];
boost::scoped_array<unsigned char> scopedChunk( chunk );
bytesRead = readFillBuffer(backupFile, (char*)chunk, restoredChunkLen);
if (bytesRead != restoredChunkLen)
{
int errrc = errno;
std::string eMsg;
Convertor::mapErrnoToString(errrc, eMsg);
std::ostringstream oss;
oss << "Error reading chunk data from backup file " <<
bulkRollbackSubPath <<
"; size-" << restoredChunkLen <<
": " << eMsg;
errMsg = oss.str();
delete backupFile;
return ERR_METADATABKUP_COMP_READ_BULK_BKUP;
}
// Write/restore the HWM chunk to the applicable database file
rc = fDbFile.writeFile(pFile, chunk, restoredChunkLen);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error writing to column file" <<
"; offset-" << fileOffsetByteForRestoredChunk <<
"; bytes-" << restoredChunkLen <<
"; " << ec.errorString(rc);
errMsg = oss.str();
delete backupFile;
return rc;
}
}
delete backupFile;
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Return true/false depending on whether the applicable backup chunk file can
// be found to restore a backed up compressed chunk back into a db file. If
// the backup file is not found, we assume that it's because one was not created
// and thus not needed.
//------------------------------------------------------------------------------
bool BulkRollbackFileCompressed::doWeReInitExtent( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const
{
std::ostringstream oss;
oss << "/" << columnOID << ".p" << partNum << ".s" << segNum;
std::string bulkRollbackSubPath( fMgr->getMetaFileName() );
bulkRollbackSubPath += DATA_DIR_SUFFIX;
bulkRollbackSubPath += oss.str();
if ( !IDBPolicy::exists( bulkRollbackSubPath.c_str() ) )
{
return false;
}
return true;
}
//------------------------------------------------------------------------------
// Read requested number of bytes from the specified pFile into "buffer".
// Added this function as part of hdfs port, because IDBDataFile::read()
// may not return all the requested data in the first call to read().
//------------------------------------------------------------------------------
size_t BulkRollbackFileCompressed::readFillBuffer(
IDBDataFile* pFile,
char* buffer,
size_t bytesReq) const
{
char* pBuf = buffer;
ssize_t nBytes;
size_t bytesToRead = bytesReq;
size_t totalBytesRead = 0;
while (1)
{
nBytes = pFile->read(pBuf, bytesToRead);
if (nBytes > 0)
totalBytesRead += nBytes;
else
break;
if ((size_t)nBytes == bytesToRead)
break;
pBuf += nBytes;
bytesToRead = bytesToRead - (size_t)nBytes;
}
return totalBytesRead;
}
} //end of namespace

156
writeengine/shared/we_bulkrollbackfilecompressed.h Normal file
View File

@ -0,0 +1,156 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: we_bulkrollbackfilecompressed.h 4726 2013-08-07 03:38:36Z bwilkinson $
*/
/** @file
* Contains class to restore compressed db files on behalf of BulkRollBackMgr.
*/
#ifndef WE_BULKROLLBACKFILECOMPRESSED_H_
#define WE_BULKROLLBACKFILECOMPRESSED_H_
#include <cstdio>
#include <cstring>
#include "we_define.h"
#include "we_type.h"
#include "we_bulkrollbackfile.h"
#include "idbcompress.h"
namespace WriteEngine
{
class BulkRollbackMgr;
//------------------------------------------------------------------------------
/** @brief Class used by BulkRollbackMgr to restore compressed db files.
*/
//------------------------------------------------------------------------------
class BulkRollbackFileCompressed : public BulkRollbackFile
{
public:
/** @brief BulkRollbackFile constructor
* @param mgr The controlling BulkRollbackMgr object.
*/
BulkRollbackFileCompressed(BulkRollbackMgr* mgr);
/** @brief BulkRollbackFile destructor
*/
virtual ~BulkRollbackFileCompressed();
/** @brief Do we reinit trailing blocks in the HWM extent for the specified
* segment file
*
* @param columnOID OID of the segment file in question
* @param dbRoot DBRoot for the segment file in question
* @param partNum Partition number for the segment file in question
* @param segNum Segment number for the segment file in question
*/
virtual bool doWeReInitExtent( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const;
/** @brief Reinitialize the specified column segment file starting at
* startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
* @param colType Column type of the relevant segment file
* @param colWidth Width in bytes of column.
* @param restoreHwmChk Restore HWM chunk
*/
virtual void reInitTruncColumnExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
execplan::CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool restoreHwmChk );
/** @brief Reinitialize the specified dictionary store segment file starting
* at startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
*/
virtual void reInitTruncDctnryExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks );
/** @brief Truncate the specified segment file to a specified num of bytes
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param fileSizeBlocks Number of blocks to retain in the file
*/
virtual void truncateSegmentFile( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long filesSizeBlocks );
private:
// Disable unnecessary copy constructor and assignment operator
BulkRollbackFileCompressed(const BulkRollbackFileCompressed& rhs);
BulkRollbackFileCompressed& operator=(const BulkRollbackFileCompressed& rhs);
size_t readFillBuffer ( IDBDataFile* pFile,
char* buffer,
size_t bytesReq) const;
int restoreHWMChunk ( IDBDataFile* pFile,
OID columnOID,
uint32_t partNum,
uint32_t segNum,
uint64_t fileOffsetByteForRestoredChunk,
uint64_t& restoredChunkLen,
uint64_t& restoredFileSize,
std::string& errMsg );
int loadColumnHdrPtrs ( IDBDataFile* pFile,
char* hdrs,
compress::CompChunkPtrList& chunkPtrs,
std::string& errMsg) const;
int loadDctnryHdrPtrs ( IDBDataFile* pFile,
char* controlHdr,
compress::CompChunkPtrList& chunkPtrs,
uint64_t& ptrHdrSize,
std::string& errMsg ) const;
compress::IDBCompressInterface fCompressor;
};
} //end of namespace
#endif // WE_BULKROLLBACKFILECOMPRESSED_H_

284
writeengine/shared/we_bulkrollbackfilecompressedhdfs.cpp Normal file
View File

@ -0,0 +1,284 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "we_bulkrollbackfilecompressedhdfs.h"
#include <sstream>
#include <boost/scoped_array.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/convenience.hpp>
#include "we_define.h"
#include "we_fileop.h"
#include "we_bulkrollbackmgr.h"
#include "we_confirmhdfsdbfile.h"
#include "we_convertor.h"
#include "messageids.h"
#include "IDBDataFile.h"
#include "IDBPolicy.h"
using namespace idbdatafile;
using namespace compress;
using namespace execplan;
namespace
{
const char* DATA_DIR_SUFFIX = "_data";
const char* OLD_FILE_SUFFIX = ".old_bulk";
}
namespace WriteEngine
{
//------------------------------------------------------------------------------
// BulkRollbackFileCompressedHdfs constructor
//------------------------------------------------------------------------------
BulkRollbackFileCompressedHdfs::BulkRollbackFileCompressedHdfs(
BulkRollbackMgr* mgr) :
BulkRollbackFile(mgr)
{
}
//------------------------------------------------------------------------------
// BulkRollbackFileCompressedHdfs destructor
//------------------------------------------------------------------------------
BulkRollbackFileCompressedHdfs::~BulkRollbackFileCompressedHdfs()
{
}
//------------------------------------------------------------------------------
// Truncate the specified database segment file to the extent specified by
// the given file offset. Also updates the header(s) as well.
//
// columnOID - OID of segment file to be truncated
// dbRoot - DBRoot of segment file to be truncated
// partNum - Partition number of segment file to be truncated
// segNum - Segment number of segment file to be truncated
// fileSizeBlocks - Number of raw data blocks to be left in the file.
// Remainder of file is to be truncated.
//------------------------------------------------------------------------------
void BulkRollbackFileCompressedHdfs::truncateSegmentFile(
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long fileSizeBlocks )
{
std::ostringstream msgText;
msgText << "Truncating compressed HDFS column file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; rawTotBlks-" << fileSizeBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText.str() );
restoreFromBackup( "column", columnOID, dbRoot, partNum, segNum );
}
//------------------------------------------------------------------------------
// Reinitialize a column segment extent (in the db file) to empty values,
// following the HWM. Remaining extents in the file are truncated.
// Also updates the header(s) as well.
//
// columnOID - OID of segment file to be reinitialized
// dbRoot - DBRoot of segment file to be reinitialized
// partNum - Partition number of segment file to be reinitialized
// segNum - Segment number of segment file to be reinitialized
// startOffsetBlk - File offset (after the HWM block), at which the file is
// to be reinitialized. Value is in raw data blocks.
// nBlocks - Number of blocks to be reinitialized
// colType - Data type of the applicable column
// colWidth - Width in bytes, of the applicable column
// restoreHwmChk - Specifies whether HWM chunk is to be restored.
//------------------------------------------------------------------------------
void BulkRollbackFileCompressedHdfs::reInitTruncColumnExtent(
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool restoreHwmChk )
{
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
std::ostringstream msgText;
msgText << "Reinit HWM compressed column extent in HDFS db file" <<
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; rawOffset(bytes)-"<< startOffset <<
"; rawFreeBlks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText.str() );
restoreFromBackup( "column", columnOID, dbRoot, partNum, segNum );
}
//------------------------------------------------------------------------------
// Reinitialize a dictionary segment extent (in the db file) to empty blocks,
// following the HWM. Remaining extents in the file are truncated.
// Also updates the header(s) as well.
//
// dStoreOID - OID of segment store file to be reinitialized
// dbRoot - DBRoot of segment file to be reinitialized
// partNum - Partition number of segment file to be reinitialized
// segNum - Segment number of segment file to be reinitialized
// startOffsetBlk - Starting block (after the HWM block), at which the file is
// to be reinitialized. Value is in raw data blocks.
// nBlocks - Number of blocks to be reinitialized
//------------------------------------------------------------------------------
void BulkRollbackFileCompressedHdfs::reInitTruncDctnryExtent(
OID dStoreOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks )
{
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
std::ostringstream msgText;
msgText << "Reinit HWM compressed dictionary store extent in HDFS db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; rawOffset(bytes)-" << startOffset <<
"; rawFreeBlks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, dStoreOID, msgText.str() );
restoreFromBackup( "dictionary store", dStoreOID, dbRoot, partNum, segNum );
}
//------------------------------------------------------------------------------
// For HDFS system, just always return true.
// Let ConfirmHdfsDbFile later determine when/if/how to restore from any
// existing backup file.
//------------------------------------------------------------------------------
bool BulkRollbackFileCompressedHdfs::doWeReInitExtent( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const
{
return true;
}
//------------------------------------------------------------------------------
// Replace the currently specified db file with it's corresponding backup file.
// The backup file is a complete backup, not just a backup of a single chunk.
//
// The initial implementation for this function restored from a NNN.pNNN.sNNN
// file stored under the meta file directory.
// The latest implementation for this function restores from a FILENNN.cdf.tmp
// or FILENNN.cdf.orig file stored in the same OID directory as the FILENNN.cdf
// file.
// However, this function still looks for the first backup file (NNN.pNNN.sNNN)
// in case the user did not upgrade cleanly, and we have to restore using an
// old leftover backup file.
//------------------------------------------------------------------------------
void BulkRollbackFileCompressedHdfs::restoreFromBackup(const char* colType,
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum)
{
// Construct file name for db file to be restored
char dbFileName[FILE_NAME_SIZE];
int rc = fDbFile.getFileName( columnOID, dbFileName,
dbRoot, partNum, segNum );
if (rc != NO_ERROR)
{
std::ostringstream oss;
oss << "Error restoring " << colType <<
" HDFS file for OID " << columnOID <<
"; Can't construct file name for DBRoot" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), rc );
}
// Construct file name for backup copy of db file
std::ostringstream ossFile;
ossFile << "/" << columnOID << ".p" << partNum << ".s" << segNum;
std::string backupFileName( fMgr->getMetaFileName() );
backupFileName += DATA_DIR_SUFFIX;
backupFileName += ossFile.str();
std::string dbFileNameTmp = dbFileName;
dbFileNameTmp += OLD_FILE_SUFFIX;
// For backwards compatibility...
// Restore from backup file used in initial HDFS release, in case the user
// upgraded without going down cleanly. In that case we might need to
// rollback using an old backup file left from previous release.
if ( IDBPolicy::exists(backupFileName.c_str()) )
{
// Rename current db file to make room for restored file
rc = IDBPolicy::rename( dbFileName, dbFileNameTmp.c_str() );
if (rc != 0)
{
std::ostringstream oss;
oss << "Error restoring " << colType <<
" HDFS file for OID " << columnOID <<
"; Can't move old file for DBRoot" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), ERR_COMP_RENAME_FILE );
}
// Rename backup file to replace current db file
rc = IDBPolicy::rename( backupFileName.c_str(), dbFileName );
if (rc != 0)
{
std::ostringstream oss;
oss << "Error restoring " << colType <<
" HDFS file for OID " << columnOID <<
"; Can't rename backup file for DBRoot" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), ERR_METADATABKUP_COMP_RENAME );
}
// Delete db file we just replaced with backup
IDBPolicy::remove( dbFileNameTmp.c_str() );
}
else // Restore from HDFS temp swap backup file; This is the normal case
{
std::string errMsg;
ConfirmHdfsDbFile confirmHdfs;
rc = confirmHdfs.endDbFileChange( std::string("tmp"),
dbFileName,
false,
errMsg);
if (rc != 0)
{
std::ostringstream oss;
oss << "Error restoring " << colType <<
" HDFS file for OID " << columnOID <<
"; DBRoot" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << errMsg;
throw WeException( oss.str(), rc );
}
}
}
} //end of namespace

139
writeengine/shared/we_bulkrollbackfilecompressedhdfs.h Normal file
View File

@ -0,0 +1,139 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/** @file
* Class to restore compressed hdfs db files on behalf of BulkRollBackMgr.
*/
#ifndef WE_BULKROLLBACKFILECOMPRESSEDHDFS_H_
#define WE_BULKROLLBACKFILECOMPRESSEDHDFS_H_
#include <cstdio>
#include <cstring>
#include "we_define.h"
#include "we_type.h"
#include "we_bulkrollbackfile.h"
namespace WriteEngine
{
class BulkRollbackMgr;
//------------------------------------------------------------------------------
/** @brief Class used by BulkRollbackMgr to restore compressed hdfs db files.
*
* BulkRollbackFileCompressed is used by non-hdfs system to restore a backup
* of the hwm compressed chunk.
* BulkRollbackFileCompressedHdfs is used by hdfs system to restore a backup
* of the entire hwm compressed db file.
*/
//------------------------------------------------------------------------------
class BulkRollbackFileCompressedHdfs : public BulkRollbackFile
{
public:
/** @brief BulkRollbackFile constructor
* @param mgr The controlling BulkRollbackMgr object.
*/
BulkRollbackFileCompressedHdfs(BulkRollbackMgr* mgr);
/** @brief BulkRollbackFile destructor
*/
virtual ~BulkRollbackFileCompressedHdfs();
/** @brief Do we reinit trailing blocks in the HWM extent for the specified
* segment file
*
* @param columnOID OID of the segment file in question
* @param dbRoot DBRoot for the segment file in question
* @param partNum Partition number for the segment file in question
* @param segNum Segment number for the segment file in question
*/
virtual bool doWeReInitExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const;
/** @brief Reinitialize the specified column segment file starting at
* startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
* @param colType Column type of the relevant segment file
* @param colWidth Width in bytes of column.
* @param restoreHwmChk Restore HWM chunk
*/
virtual void reInitTruncColumnExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
execplan::CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool restoreHwmChk );
/** @brief Reinitialize the specified dictionary store segment file starting
* at startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
*/
virtual void reInitTruncDctnryExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks );
/** @brief Truncate the specified segment file to a specified num of bytes
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param fileSizeBlocks Number of blocks to retain in the file
*/
virtual void truncateSegmentFile(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long filesSizeBlocks );
private:
// Disable unnecessary copy constructor and assignment operator
BulkRollbackFileCompressedHdfs(const BulkRollbackFileCompressedHdfs& rhs);
BulkRollbackFileCompressedHdfs& operator=(
const BulkRollbackFileCompressedHdfs& rhs);
void restoreFromBackup(const char* colType,
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum );
};
} //end of namespace
#endif // WE_BULKROLLBACKFILECOMPRESSEDHDFS_H_

1967
writeengine/shared/we_bulkrollbackmgr.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

225
writeengine/shared/we_bulkrollbackmgr.h Normal file
View File

@ -0,0 +1,225 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: we_bulkrollbackmgr.h 4726 2013-08-07 03:38:36Z bwilkinson $
*/
/** @file
* Contains class to clear a database table lock, and rolls back extents
* based on HWM meta data saved by a bulk load.
*/
#ifndef WE_BULKROLLBACKMGR_H_
#define WE_BULKROLLBACKMGR_H_
#ifdef _MSC_VER
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#include <boost/thread/mutex.hpp>
#endif
#include <set>
#include <string>
#include <sstream>
#include <vector>
#include "we_type.h"
#include "messagelog.h"
#include "messageobj.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
namespace WriteEngine
{
class Log;
class BulkRollbackFile;
//------------------------------------------------------------------------------
/** @brief Class to clear a database table lock, and rolls back extents
* based on HWM meta data saved by a bulk load.
*/
//------------------------------------------------------------------------------
class BulkRollbackMgr
{
enum BulkRollbackVersion
{
BULK_RB_VERSION_OTHER,
BULK_RB_VERSION3 = 3,
BULK_RB_VERSION4 = 4
};
public:
/**
* @brief BulkRollbackMgr constructor
* @param tableOID table to be rolled back.
* @param lockID Table lock id of the table to be rolled back.
* Currently used for logging only.
* @param tableName name of table associated with tableOID.
* Currently used for logging only.
* @param applName application that is driving this bulk rollback.
* Currently used for logging only.
*/
EXPORT BulkRollbackMgr(OID tableOID,
uint64_t lockID,
const std::string& tableName,
const std::string& applName,
Log* logger=0);
/**
* @brief BulkRollbackMgr destructor
*/
EXPORT ~BulkRollbackMgr( ) { closeMetaDataFile ( ); }
/**
* @brief Clear table lock and rollback extents for fTableOID
* @param keepMetaFile controls whether the meta data file is deleted
* @return NO_ERROR upon success
*/
EXPORT int rollback ( bool keepMetaFile );
/**
* @brief Accessor to any error msg related to a bad return code.
* @return error message if rollback rejected or failed.
*/
const std::string& getErrorMsg( ) const { return fErrorMsg; }
/**
* @brief Accessor to the name of the meta file we are processing
*/
const std::string& getMetaFileName() const { return fMetaFileName; }
/**
* @brief Mutator to enable/disable debug logging to console.
*/
const void setDebugConsole ( bool debug ) { fDebugConsole = debug; }
/**
* @brief Log the specified message.
* @param logType type of message to be logged
* @param msgId message id to be used
* @param columnOID column OID
* @param text message text to be logged
*/
void logAMessage ( logging::LOG_TYPE logType, //log a message
logging::Message::MessageID msgId,
OID columnOID,
const std::string& text );
/**
* @brief Standalone utility that can be used to delete the bulk rollback
* meta data files. Caution: this function can throw an exception.
* @param tableOID Bulk rollback files for this table are to be deleted
*/
EXPORT static void deleteMetaFile( OID tableOID );
/*
* @brief Get list of segment file numbers found in dirName directory
* @param dirName Directory path to be searched
* @param bIncludeAlternateSegFileNames Include *.orig and *.tmp in search
* @param segList List of segment files found in dirName
* @param errMsg Error msg if return code is not NO_ERROR
*/
EXPORT static int getSegFileList( const std::string& dirName,
bool bIncludeAlternateSegFileNames,
std::vector<uint32_t>& segList,
std::string& errMsg );
private:
// Declare but don't define copy constructor and assignment operator
BulkRollbackMgr(const BulkRollbackMgr& rhs);
BulkRollbackMgr& operator=(const BulkRollbackMgr& rhs);
// Structure used to store info for the list of dictionary store
// segment files in the last partition.
struct RollbackData
{
uint32_t fDbRoot;
uint32_t fPartNum;
uint32_t fSegNum;
HWM fHwm;
bool fWithHwm;
};
void createFileDeletionEntry( OID columnOID,
bool fileTypeFlag,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
const std::string& segFileName );
void deleteColumn1Extents ( const char* inBuf ); // delete col extents
void deleteColumn1ExtentsV3(const char* inBuf );
void deleteColumn1ExtentsV4(const char* inBuf );
void deleteColumn2Extents ( const char* inBuf ); // delete col extents
void deleteColumn2ExtentsV3(const char* inBuf );
void deleteColumn2ExtentsV4(const char* inBuf );
void deleteDbFiles ( ); // delete DB files waiting to be deleted
void deleteDctnryExtents ( ); // delete dictionary store extents
void deleteDctnryExtentsV3( );
void deleteDctnryExtentsV4( );
void deleteExtents ( std::istringstream& metaDataStream );
// function that drives extent deletion
void readMetaDataRecDctnry(const char* inBuf );//read meta-data dct rec
void deleteSubDir ( const std::string& metaFileName ); // delete
// subdirectory used for backup chunks
EXPORT void closeMetaDataFile ( ); // close a metafile
void deleteMetaDataFiles ( ); // delete metafiles
int metaDataFileExists ( bool& exists ); // does meta-data file exists
BulkRollbackFile* makeFileRestorer(int compressionType);
bool openMetaDataFile ( uint16_t dbRoot, // open a metadata file
std::istringstream& metaDataStream );
void validateAllMetaFilesExist(const std::vector<uint16_t>& dbRoots) const;
// Data members
OID fTableOID; // table to be rolled back
uint64_t fLockID; // unique lock ID associated with table lock
std::string fTableName; // name of table associated with fTableOID
uint32_t fProcessId; // pid associated with current table lock
std::string fProcessName; // processName associated with fProcessId
IDBDataFile* fMetaFile; // current meta data file we are reading
std::string fMetaFileName;// name of current meta data file
std::vector<std::string> fMetaFileNames; // all relevant meta data files
std::string fErrorMsg;
unsigned char fDctnryHdr[DCTNRY_HEADER_SIZE]; // empty dctnry store blk
// Dictionary store extents for an OID are read in and managed as a
// group. The following data members are used to collect this info.
OID fPendingDctnryStoreOID;// Dctnry OID of pending dctnry extents
uint32_t fPendingDctnryStoreDbRoot; // DbRoot of pending dctnry extents
int fPendingDctnryStoreCompressionType; // Dctnry compression type
std::vector<RollbackData> fPendingDctnryExtents;
std::set<OID> fAllColDctOIDs; // List of all affected col and dctnry OIDS
// List of DB Files to be deleted. Files are deleted in reverse order.
std::vector<File> fPendingFilesToDelete;
logging::MessageLog fSysLogger; // Used for syslogging
bool fDebugConsole; // control debug logging to console
Log* fLog; // optional logger object
std::string fApplName; // application initiating the bulk rollback
int fVersion; // version of meta data file being read
};
} //end of namespace
#undef EXPORT
#endif // WE_BULKROLLBACKMGR_H_

418
writeengine/shared/we_cache.cpp Normal file
View File

@ -0,0 +1,418 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: we_cache.cpp 33 2006-10-30 13:45:13Z wzhou $
*
******************************************************************************************/
/** @file */
#include <we_cache.h>
using namespace std;
namespace WriteEngine
{
CacheControl* Cache::m_cacheParam = NULL;
FreeBufList* Cache::m_freeList = NULL;
CacheMap* Cache::m_lruList = NULL;
CacheMap* Cache::m_writeList = NULL;
#ifdef _MSC_VER
__declspec(dllexport)
#endif
bool Cache::m_useCache = false;
/***********************************************************
* DESCRIPTION:
* Clear all list and free memory
* PARAMETERS:
* none
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
void Cache::clear()
{
CacheMapIt it;
BlockBuffer* block;
size_t i;
// free list
if( m_freeList != NULL ) {
for( i = 0; i < m_freeList->size(); i++ ) {
block = m_freeList->at(i);
block->clear();
}
}
// LRU list
if( m_lruList != NULL ) {
for( it = m_lruList->begin(); it != m_lruList->end(); it++ ) {
block = it->second;
block->clear();
m_freeList->push_back( block );
}
m_lruList->clear();
}
// Write list
if( m_writeList != NULL ) {
for( it = m_writeList->begin(); it != m_writeList->end(); it++ ) {
block = it->second;
block->clear();
m_freeList->push_back( block );
}
m_writeList->clear();
}
}
/***********************************************************
* DESCRIPTION:
* Flush write cache
* PARAMETERS:
* none
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
const int Cache::flushCache()
{
bool bHasReadBlock = false;
BlockBuffer* curBuf;
// add lock here
if( m_lruList && m_lruList->size() > 0 ) {
bHasReadBlock = true;
for( CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++ ) {
curBuf = it->second;
curBuf->clear();
m_freeList->push_back( curBuf );
}
m_lruList->clear();
}
// must write to disk first
if( m_writeList && m_writeList->size() > 0 ) {
if( !bHasReadBlock )
for( CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++ ) {
curBuf = it->second;
curBuf->clear();
m_freeList->push_back( curBuf );
}
else
for( CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++ ) {
curBuf = it->second;
(*curBuf).block.dirty = false;
processCacheMap( m_lruList, curBuf, INSERT );
}
m_writeList->clear();
} // end of if( m_writeList->size()
// add unlock here
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
* Free memory
* PARAMETERS:
* none
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
void Cache::freeMemory()
{
CacheMapIt it;
BlockBuffer* block;
size_t i;
// free list
if( m_freeList != NULL ) {
for( i = 0; i < m_freeList->size(); i++ ) {
block = m_freeList->at(i);
block->freeMem();
delete block;
}
m_freeList->clear();
delete m_freeList;
m_freeList = NULL;
}
// LRU list
if( m_lruList != NULL ) {
for( it = m_lruList->begin(); it != m_lruList->end(); it++ ) {
block = it->second;
block->freeMem();
delete block;
}
m_lruList->clear();
delete m_lruList;
m_lruList = NULL;
}
// Write list
if( m_writeList != NULL ) {
for( it = m_writeList->begin(); it != m_writeList->end(); it++ ) {
block = it->second;
block->freeMem();
delete block;
}
m_writeList->clear();
delete m_writeList;
m_writeList = NULL;
}
// param
if( m_cacheParam != NULL ) {
delete m_cacheParam;
m_cacheParam = NULL;
}
}
/***********************************************************
* DESCRIPTION:
* get a list size
* PARAMETERS:
* listType - List type
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
const int Cache::getListSize( const CacheListType listType )
{
int size = 0;
if( !m_useCache )
return size;
switch( listType ) {
case FREE_LIST: size = m_freeList->size(); break;
case LRU_LIST: size = m_lruList->size(); break;
case WRITE_LIST:
default:
size = m_writeList->size(); break;
}
return size;
}
/***********************************************************
* DESCRIPTION:
* Init all parameters and list
* PARAMETERS:
* totalBlock - total blocks
* chkPoint - checkpoint interval
* pctFree - percentage free
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
void Cache::init( const int totalBlock, const int chkPoint, const int pctFree )
{
BlockBuffer* buffer;
if( m_cacheParam && m_freeList && m_lruList && m_writeList )
return;
m_cacheParam = new CacheControl();
m_cacheParam->totalBlock = totalBlock;
m_cacheParam->checkInterval = chkPoint;
m_cacheParam->pctFree = pctFree;
m_freeList = new FreeBufList();
m_lruList = new CacheMap();
m_writeList = new CacheMap();
for( int i = 0; i < m_cacheParam->totalBlock; i++ ) {
buffer = new BlockBuffer();
buffer->init();
m_freeList->push_back( buffer );
}
}
/***********************************************************
* DESCRIPTION:
* Insert a buffer to LRU list
* PARAMETERS:
* cb - Comm Block
* lbid - lbid value
* fbo - fbo
* buf - input buffer
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
const int Cache::insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const unsigned char* buf )
{
BlockBuffer* buffer;
vector<BlockBuffer*>::iterator it;
if( m_freeList->size() == 0 )
return ERR_FREE_LIST_EMPTY;
// make sure flush first if necessary
it = m_freeList->begin();
buffer = *it;
memcpy( (*buffer).block.data, buf, BYTE_PER_BLOCK );
(*buffer).listType = LRU_LIST;
(*buffer).block.lbid = lbid;
(*buffer).block.fbo = fbo;
(*buffer).block.dirty = false;
(*buffer).block.hitCount = 1;
(*buffer).cb.file.oid = cb.file.oid;
(*buffer).cb.file.pFile = cb.file.pFile;
RETURN_ON_ERROR( processCacheMap( m_lruList, buffer, INSERT ) );
m_freeList->erase( it );
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
* Load cache block
* PARAMETERS:
* key - Cache key
* buf - output buffer
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
const int Cache::loadCacheBlock( const CacheKey& key, unsigned char* buf )
{
BlockBuffer* buffer;
CacheMapIt iter;
iter = m_lruList->find( key );
if( iter != m_lruList->end() )
buffer = iter->second;
else {
iter = m_writeList->find( key );
if( iter != m_writeList->end() )
buffer = iter->second;
else
return ERR_CACHE_KEY_NOT_EXIST;
}
memcpy( buf, (*buffer).block.data, BYTE_PER_BLOCK );
(*buffer).block.hitCount++;
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
* Modify cache block
* PARAMETERS:
* key - Cache key
* buf - output buffer
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
const int Cache::modifyCacheBlock( const CacheKey& key, const unsigned char* buf )
{
BlockBuffer* buffer;
CacheMapIt iter;
iter = m_lruList->find( key );
if( iter != m_lruList->end() ) {
buffer = iter->second;
(*buffer).listType = WRITE_LIST;
(*buffer).block.dirty = true;
(*m_writeList)[key] = iter->second;
m_lruList->erase( iter );
}
else {
iter = m_writeList->find( key );
if( iter != m_writeList->end() )
buffer = iter->second;
else
return ERR_CACHE_KEY_NOT_EXIST;
}
memcpy( (*buffer).block.data, buf, BYTE_PER_BLOCK );
(*buffer).block.hitCount++;
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
* Print cache list
* PARAMETERS:
* none
* RETURN:
* none
***********************************************************/
void Cache::printCacheList()
{
BlockBuffer* buffer;
int i = 0;
if( !m_useCache )
return;
cout << "\nFree List has " << m_freeList->size() << " elements" << endl;
cout << "LRU List has " << m_lruList->size() << " elements" << endl;
for( CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++ ) {
buffer = it->second;
cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType
<< " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo
<< " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl;
}
i = 0;
cout << "Write List has " << m_writeList->size() << " elements" << endl;
for( CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++ ) {
buffer = it->second;
cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType
<< " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo
<< " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl;
}
}
/***********************************************************
* DESCRIPTION:
* Process a buffer in a cache map
* PARAMETERS:
* buffer - block buffer
* opType - insert or delete
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
const int Cache::processCacheMap( CacheMap* map, BlockBuffer* buffer, OpType opType )
{
RETURN_ON_NULL( buffer, ERR_NULL_BLOCK );
CacheMapIt iter;
CacheKey key = getCacheKey( buffer );
iter = map->find( key );
// only handle insert and delete
if( iter == map->end() ) {
if( opType == INSERT )
(*map)[key] = buffer;
else
return ERR_CACHE_KEY_NOT_EXIST;
}
else {
if( opType == INSERT )
return ERR_CACHE_KEY_EXIST;
else
map->erase( iter );
}
return NO_ERROR;
}
} //end of namespace

190
writeengine/shared/we_cache.h Normal file
View File

@ -0,0 +1,190 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: we_cache.h 33 2006-10-30 13:45:13Z wzhou $
*
******************************************************************************************/
/** @file */
#ifndef _WE_CACHE_H_
#define _WE_CACHE_H_
#include <iostream>
#ifdef _MSC_VER
#include <unordered_map>
#else
#if __GNUC__ == 4 && __GNUC_MINOR__ < 2
#include <ext/hash_map>
#else
#include <tr1/unordered_map>
#endif
#endif
#include <map>
#include <we_obj.h>
#include <we_convertor.h>
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
typedef std::vector<BlockBuffer*> FreeBufList; /** @brief Free buffer list */
//typedef std::string CacheKey; /** @brief Key definition */
typedef uint64_t CacheKey; /** @brief Key definition */
//typedef std::map<CacheKey, BlockBuffer*, std::greater<CacheKey> > CacheMap; /** @brief Cache map */
//typedef CacheMap::iterator CacheMapIt; /** @brief CacheMap iterator */
template<class T>struct hashCacheKey{ };
template<> struct hashCacheKey<CacheKey>
{
size_t
operator()(CacheKey __x) const
{ return __x; }
};
struct eqCacheKey
{
bool operator() (const CacheKey k1, const CacheKey k2 ) const
{
return k1 == k2;
}
};
//typedef hash_map<Signature, TokenStruc, signatureHash<Signature>, eqSig> DCTNRYHASHMAP;
#if __GNUC__ == 4 && __GNUC_MINOR__ < 2
typedef __gnu_cxx::hash_map<CacheKey, BlockBuffer*, hashCacheKey<CacheKey>, eqCacheKey> CacheMap;
#else
typedef std::tr1::unordered_map<CacheKey, BlockBuffer*, hashCacheKey<CacheKey>, eqCacheKey> CacheMap;
#endif
//typedef __gnu_cxx::hash_map<CacheKey, BlockBuffer*> CacheMap;
typedef CacheMap::iterator CacheMapIt;
//typedef CacheMap LRUBufList; /** @brief Least Recent Used Buffer list */
//typedef CacheMap WriteBufList; /** @brief Write buffer list */
/** Class Cache */
class Cache
{
public:
/**
* @brief Constructor
*/
Cache() {}
/**
* @brief Default Destructor
*/
~Cache() {}
/**
* @brief Check whether cache key exists
*/
static const bool cacheKeyExist( CacheMap* map, const OID oid, const uint64_t lbid ) { CacheKey key = getCacheKey( oid, lbid ); return map->find(key) == map->end() ? false: true; }
static const bool cacheKeyExist( CacheMap* map, BlockBuffer* buffer ) { return cacheKeyExist( map, (*buffer).cb.file.oid, (*buffer).block.lbid ); }
static const bool cacheKeyExist( const OID oid, const uint64_t lbid ) { return cacheKeyExist( m_lruList, oid, lbid ) || cacheKeyExist( m_writeList, oid, lbid ); }
/**
* @brief Clear the buffer
*/
EXPORT static void clear();
/**
* @brief Free the buffer memory
*/
EXPORT static void freeMemory();
/**
* @brief Flush the write cache
*/
EXPORT static const int flushCache();
/**
* @brief Get the cache key
*/
static CacheKey getCacheKey( const OID oid, const uint64_t lbid ) { CacheKey key = lbid; /*Convertor::int2Str( oid ) + "|" + Convertor::int2Str(lbid)*/; return key; }
static CacheKey getCacheKey( const BlockBuffer* buffer ) { return getCacheKey( (*buffer).cb.file.oid, (*buffer).block.lbid ); }
EXPORT static const int getListSize( const CacheListType listType );
/**
* @brief Init the buffers
*/
EXPORT static void init( const int totalBlock, const int chkPoint, const int pctFree );
static void init() { init( DEFAULT_CACHE_BLOCK, DEFAULT_CHK_INTERVAL, DEFAULT_CACHE_PCT_FREE ); }
/**
* @brief Insert into LRU list
*/
EXPORT static const int insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const unsigned char* buf );
static const int insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const DataBlock& block ) { return insertLRUList( cb, lbid, fbo, block.data ); }
/**
* @brief Insert into Write list
*/
// static const int insertWriteList( const CacheKey& key );
/**
* @brief Load cache block to a buffer
*/
static const int loadCacheBlock( const CacheKey& key, DataBlock& block ) { return loadCacheBlock( key, block.data ); }
EXPORT static const int loadCacheBlock( const CacheKey& key, unsigned char* buf );
/**
* @brief Modify a cache block
*/
static const int modifyCacheBlock( const CacheKey& key, const DataBlock& block ) { return modifyCacheBlock( key, block.data ); }
EXPORT static const int modifyCacheBlock( const CacheKey& key, const unsigned char* buf );
/**
* @brief Print
*/
EXPORT static void printCacheMapList( const CacheMap* map );
EXPORT static void printCacheList();
/**
* @brief Insert/Delete an element in cache map
*/
EXPORT static const int processCacheMap( CacheMap* map, BlockBuffer* buffer, OpType opType );
// accessory
static const int getTotalBlock() { return m_cacheParam->totalBlock; }
static const bool getUseCache() { return m_useCache; }
static void setUseCache( const bool flag ) { m_useCache = flag; }
static CacheControl* m_cacheParam; // Cache parameters
static FreeBufList* m_freeList; // free buffer list
static CacheMap* m_lruList; // LRU buffer list
static CacheMap* m_writeList; // Write buffer list
#if defined(_MSC_VER) && !defined(WRITEENGINE_DLLEXPORT)
__declspec(dllimport)
#endif
EXPORT static bool m_useCache; // Use cache flag
private:
};
} //end of namespace
#undef EXPORT
#endif // _WE_CACHE_H_

2499
writeengine/shared/we_chunkmanager.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

363
writeengine/shared/we_chunkmanager.h Normal file
View File

@ -0,0 +1,363 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: we_chunkmanager.h 4726 2013-08-07 03:38:36Z bwilkinson $
/** @file */
#ifndef CHUNK_MANAGER_H
#define CHUNK_MANAGER_H
#include <cstdio>
#include <map>
#include <list>
#include <string>
#include <boost/scoped_array.hpp>
#include "we_type.h"
#include "we_typeext.h"
#include "we_define.h"
#include "idbcompress.h"
#include "IDBFileSystem.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
#ifdef _MSC_VER
#define WE_COMP_DBG(x) {}
#else
//#define IDB_COMP_DEBUG
#ifdef IDB_COMP_DEBUG
#define WE_COMP_DBG(x) {x}
#else
#define WE_COMP_DBG(x) {}
#endif
#endif
namespace logging
{
// use Logger (not we_log) for now.
class Logger;
}
namespace WriteEngine
{
// forward reference
class FileOp;
const int UNCOMPRESSED_CHUNK_SIZE = compress::IDBCompressInterface::UNCOMPRESSED_INBUF_LEN;
const int COMPRESSED_FILE_HEADER_UNIT = compress::IDBCompressInterface::HDR_BUF_LEN;
// assume UNCOMPRESSED_CHUNK_SIZE > 0xBFFF (49151), 8 * 1024 bytes padding
const int COMPRESSED_CHUNK_SIZE = compress::IDBCompressInterface::maxCompressedSize(UNCOMPRESSED_CHUNK_SIZE) + 64+3 + 8*1024;
const int BLOCKS_IN_CHUNK = UNCOMPRESSED_CHUNK_SIZE / BYTE_PER_BLOCK;
const int MAXOFFSET_PER_CHUNK = 511*BYTE_PER_BLOCK;
// chunk information
typedef int64_t ChunkId;
struct ChunkData
{
ChunkId fChunkId;
unsigned int fLenUnCompressed;
char fBufUnCompressed[UNCOMPRESSED_CHUNK_SIZE];
bool fWriteToFile;
ChunkData(ChunkId id = 0) : fChunkId(id), fLenUnCompressed(0), fWriteToFile(false) {}
bool operator < (const ChunkData& rhs) const { return fChunkId < rhs.fChunkId; }
};
// compressed DB file header information
struct CompFileHeader
{
char fHeaderData[COMPRESSED_FILE_HEADER_UNIT * 2];
char *fControlData;
char *fPtrSection;
boost::scoped_array<char> fLongPtrSectData;
CompFileHeader() :
fControlData(fHeaderData), fPtrSection(fHeaderData+COMPRESSED_FILE_HEADER_UNIT) {}
};
// unique ID of a DB file
struct FileID
{
FID fFid;
uint32_t fDbRoot;
uint32_t fPartition;
uint32_t fSegment;
FileID(FID f, uint32_t r, uint32_t p, uint32_t s) :
fFid(f), fDbRoot(r), fPartition(p), fSegment(s) {}
bool operator < (const FileID& rhs) const
{ return (
(fFid < rhs.fFid) ||
(fFid == rhs.fFid && fDbRoot < rhs.fDbRoot) ||
(fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition < rhs.fPartition) ||
(fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition == rhs.fPartition && fSegment < rhs.fSegment)); }
bool operator == (const FileID& rhs) const
{ return (
fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition == rhs.fPartition && fSegment == rhs.fSegment); }
};
// compressed DB file information
class CompFileData
{
public:
CompFileData(const FileID& id, const FID& fid, const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth) :
fFileID(id), fFid(fid), fColDataType(colDataType), fColWidth(colWidth), fDctnryCol(false),
fFilePtr(NULL), fIoBSize(0) {}
ChunkData* findChunk(int64_t cid) const;
protected:
FileID fFileID;
FID fFid;
execplan::CalpontSystemCatalog::ColDataType fColDataType;
int fColWidth;
bool fDctnryCol;
IDBDataFile* fFilePtr;
std::string fFileName;
CompFileHeader fFileHeader;
std::list<ChunkData*> fChunkList;
boost::scoped_array<char> fIoBuffer;
size_t fIoBSize;
friend class ChunkManager;
};
class ChunkManager
{
public:
// @brief constructor
EXPORT ChunkManager();
// @brief destructor
EXPORT virtual ~ChunkManager();
// @brief Retrieve a file pointer in the chunk manager.
// for column file
IDBDataFile* getFilePtr(const Column& column,
uint16_t root,
uint32_t partition,
uint16_t segment,
std::string& filename,
const char* mode,
int size,
bool useTmpSuffix) const;
// @brief Retrieve a file pointer in the chunk manager.
// for dictionary file
IDBDataFile* getFilePtr(const FID& fid,
uint16_t root,
uint32_t partition,
uint16_t segment,
std::string& filename,
const char* mode,
int size,
bool useTmpSuffix) const;
// @brief Create a compressed dictionary file with an appropriate header.
IDBDataFile* createDctnryFile(const FID& fid,
int64_t width,
uint16_t root,
uint32_t partition,
uint16_t segment,
const char* filename,
const char* mode,
int size);
// @brief Read a block from pFile at offset fbo.
// The data may copied from memory if the chunk it belongs to is already available.
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo);
// @brief Save a block to a chunk in pFile.
// The block is not written to disk immediately, will be delayed until flush.
int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo);
// @brief Write all active chunks to disk, and reset all repository.
EXPORT int flushChunks(int rc, const std::map<FID, FID> & columOids);
// @brief Reset all repository without writing anything to disk.
void cleanUp(const std::map<FID, FID> & columOids);
// @brief Expand an initial column, not dictionary, extent to a full extent.
int expandAbbrevColumnExtent(IDBDataFile* pFile, uint64_t emptyVal, int width);
// @brief Update column extent
int updateColumnExtent(IDBDataFile* pFile, int addBlockCount);
// @brief Update dictionary extent
int updateDctnryExtent(IDBDataFile* pFile, int addBlockCount);
// @brief Read in n continuous blocks to read buffer.
// for backing up blocks to version buffer
int readBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, size_t n);
// @brief Restore the data block at offset fbo from version buffer
// for rollback
int restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo);
// @brief Retrieve the total block count of a DB file.
int getBlockCount(IDBDataFile* pFile);
// @brief Set FileOp pointer (for compression type, empty value, txnId, etc.)
void fileOp(FileOp* fileOp);
// @brief Control the number of active chunks being stored in memory
void setMaxActiveChunkNum(unsigned int maxActiveChunkNum)
{ fMaxActiveChunkNum = maxActiveChunkNum; }
// @brief Use this flag to avoid logging and backing up chunks, tmp files.
void setBulkFlag(bool isBulkLoad)
{ fIsBulkLoad = isBulkLoad; }
// @brief Use this flag to flush chunk when is full.
void setIsInsert(bool isInsert) { fIsInsert = isInsert; }
bool getIsInsert() { return fIsInsert; }
void setTransId(const TxnID& transId) { fTransId = transId; }
// @brief bug5504, Use non transactional DML for InfiniDB with HDFS
EXPORT int startTransaction(const TxnID& transId) const;
EXPORT int confirmTransaction(const TxnID& transId) const;
EXPORT int endTransaction(const TxnID& transId, bool success) const;
// @brief Use this flag to fix bad chunk.
void setFixFlag(bool isFix)
{ fIsFix = isFix; }
EXPORT int checkFixLastDictChunk(const FID& fid,
uint16_t root,
uint32_t partition,
uint16_t segment);
protected:
// @brief Retrieve pointer to a compressed DB file.
CompFileData* getFileData(const FID& fid,
uint16_t root,
uint32_t partition,
uint16_t segment,
std::string& filename,
const char* mode,
int size,
const execplan::CalpontSystemCatalog::ColDataType colDataType,
int colWidth,
bool useTmpSuffix,
bool dictnry = false) const;
// @brief Retrieve a chunk of pFile from disk.
int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData);
// @brief Compress a chunk and write it to file.
int writeChunkToFile(CompFileData* fileData, int64_t id);
int writeChunkToFile(CompFileData* fileData, ChunkData* chunkData);
// @brief Write the compressed data to file and log a recover entry.
int writeCompressedChunk(CompFileData* fileData, int64_t offset, int64_t size);
inline int writeCompressedChunk_(CompFileData* fileData, int64_t offset);
// @brief Write the file header to disk.
int writeHeader(CompFileData* fileData, int ln);
inline int writeHeader_(CompFileData* fileData, int ptrSecSize);
// @brief open a compressed DB file.
int openFile(CompFileData* fileData, const char* mode, int colWidth,
bool useTmpSuffix, int ln) const;
// @brief set offset in a compressed DB file from beginning.
int setFileOffset(IDBDataFile* pFile, const std::string& fileName, off64_t offset, int ln) const;
// @brief read from a compressed DB file.
int readFile(IDBDataFile* pFile, const std::string& fileName, void* buf, size_t size, int ln) const;
// @brief write to a compressed DB file.
int writeFile(IDBDataFile* pFile, const std::string& fileName, void* buf, size_t size, int ln) const;
// @brief Close a compressed DB file.
int closeFile(CompFileData* fileData);
// @brief Set empty values to a chunk.
void initializeColumnChunk(char* buf, CompFileData* fileData);
void initializeDctnryChunk(char* buf, int size);
// @brief Calculate the header size based on column width.
int calculateHeaderSize(int width);
// @brief Moving chunks as a result of expanding a chunk.
int reallocateChunks(CompFileData* fileData);
// @brief verify chunks in the file are OK
int verifyChunksAfterRealloc(CompFileData* fileData);
// @brief log a message to the syslog
void logMessage(int code, int level, int lineNum, int fromLine=-1) const;
void logMessage(const std::string& msg, int level) const;
// @brief Write a DML recovery log
int writeLog(TxnID txnId, std::string backUpFileType, std::string filename,
std::string &aDMLLogFileName, int64_t size=0, int64_t offset=0) const;
// @brief remove DML recovery logs
int removeBackups(TxnID txnId);
// @brief swap the src file to dest file
int swapTmpFile(const std::string& src, const std::string& dest);
// @brief construnct a DML log file name
int getDMLLogFileName(std::string& aDMLLogFileName, const TxnID& txnId) const;
mutable std::map<FileID, CompFileData*> fFileMap;
mutable std::map<IDBDataFile*, CompFileData*> fFilePtrMap;
std::list<std::pair<FileID, ChunkData*> > fActiveChunks;
unsigned int fMaxActiveChunkNum; // max active chunks per file
char* fBufCompressed;
unsigned int fLenCompressed;
unsigned int fMaxCompressedBufSize;
unsigned int fUserPaddings;
bool fIsBulkLoad;
bool fDropFdCache;
bool fIsInsert;
bool fIsHdfs;
FileOp* fFileOp;
compress::IDBCompressInterface fCompressor;
logging::Logger* fSysLogger;
TxnID fTransId;
int fLocalModuleId;
idbdatafile::IDBFileSystem& fFs;
bool fIsFix;
private:
};
}
#undef EXPORT
#endif // CHUNK_MANAGER_H

681
writeengine/shared/we_config.cpp Normal file
View File

@ -0,0 +1,681 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_config.cpp 4737 2013-08-14 20:45:46Z bwilkinson $
*
*******************************************************************************/
/** @file */
#include <string>
#include <boost/thread.hpp>
#include <vector>
#include <sstream>
#include <algorithm>
using namespace std;
#include "configcpp.h"
#include "liboamcpp.h"
#include "installdir.h"
#include "we_config.h"
using namespace config;
#include "IDBPolicy.h"
using namespace idbdatafile;
#include <boost/algorithm/string.hpp>
namespace WriteEngine
{
const int DEFAULT_WAIT_PERIOD = 10;
const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 4;
const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
const int DEFAULT_BULK_PROCESS_PRIORITY = -1;
const unsigned DEFAULT_MAX_FILESYSTEM_DISK_USAGE = 98; // allow 98% full
const unsigned DEFAULT_COMPRESSED_PADDING_BLKS = 1;
const int DEFAULT_LOCAL_MODULE_ID = 1;
const bool DEFAULT_PARENT_OAM = true;
const char* DEFAULT_LOCAL_MODULE_TYPE = "pm";
int Config::m_dbRootCount = 0;
Config::strvec_t Config::m_dbRootPath;
Config::intstrmap_t Config::m_dbRootPathMap;
Config::uint16vec_t Config::m_dbRootId;
string Config::m_bulkRoot;
unsigned long Config::fDBRootChangeCount = 0;
time_t Config::fCacheTime = 0;
boost::mutex Config::fCacheLock;
#ifdef SHARED_NOTHING_DEMO_2
boost::mutex Config::m_bulkRoot_lk;
#endif
int Config::m_WaitPeriod = DEFAULT_WAIT_PERIOD;
unsigned Config::m_FilesPerColumnPartition =
DEFAULT_FILES_PER_COLUMN_PARTITION;
unsigned Config::m_ExtentsPerSegmentFile =
DEFAULT_EXTENTS_PER_SEGMENT_FILE;
int Config::m_BulkProcessPriority = DEFAULT_BULK_PROCESS_PRIORITY;
string Config::m_BulkRollbackDir;
unsigned Config::m_MaxFileSystemDiskUsage =
DEFAULT_MAX_FILESYSTEM_DISK_USAGE;
unsigned Config::m_NumCompressedPadBlks =DEFAULT_COMPRESSED_PADDING_BLKS;
bool Config::m_ParentOAMModuleFlag = DEFAULT_PARENT_OAM;
string Config::m_LocalModuleType;
int Config::m_LocalModuleID = DEFAULT_LOCAL_MODULE_ID;
string Config::m_VersionBufferDir;
/*******************************************************************************
* DESCRIPTION:
* Loads config parms into local cache.
* Call can be made to initConfigCache() at the start
* of the program to initialize local cache, before calling accessors.
* PARAMETERS:
* none
******************************************************************************/
void Config::initConfigCache()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
}
/*******************************************************************************
* DESCRIPTION:
* Reload local cache using contents of Calpont.xml file.
* PARAMETERS:
* none
******************************************************************************/
void Config::checkReload( )
{
bool bFirstLoad = false;
if (fCacheTime == 0)
bFirstLoad = true;
config::Config* cf = config::Config::makeConfig();
// Immediately return if Calpont.xml timestamp has not changed
if (cf->getCurrentMTime() == fCacheTime)
return;
//std::cout << "RELOADING cache..." << std::endl;
//--------------------------------------------------------------------------
// Initialize bulk root directory
//--------------------------------------------------------------------------
m_bulkRoot = cf->getConfig("WriteEngine", "BulkRoot");
if ( m_bulkRoot.length() == 0 )
{
m_bulkRoot = startup::StartUp::installDir();
#ifndef _MSC_VER
m_bulkRoot += "/data";
#endif
m_bulkRoot += "/bulk";
}
// Get latest Calpont.xml timestamp after first access forced a reload
fCacheTime = cf ->getLastMTime();
//--------------------------------------------------------------------------
// Initialize time interval (in seconds) between retries
//--------------------------------------------------------------------------
m_WaitPeriod = DEFAULT_WAIT_PERIOD;
string waitPeriodStr = cf->getConfig("SystemConfig", "WaitPeriod");
if ( waitPeriodStr.length() != 0 )
m_WaitPeriod = static_cast<int>(config::Config::fromText(
waitPeriodStr));
//--------------------------------------------------------------------------
// Initialize files per column partition
//--------------------------------------------------------------------------
m_FilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION;
string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition");
if ( fpc.length() != 0 )
m_FilesPerColumnPartition = cf->uFromText(fpc);
//--------------------------------------------------------------------------
// Initialize extents per segment file
//--------------------------------------------------------------------------
m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
if ( epsf.length() != 0 )
m_ExtentsPerSegmentFile = cf->uFromText(epsf);
//--------------------------------------------------------------------------
// Initialize bulk load process priority
//--------------------------------------------------------------------------
m_BulkProcessPriority = DEFAULT_BULK_PROCESS_PRIORITY;
string prior = cf->getConfig("WriteEngine", "Priority");
if ( prior.length() != 0 )
{
int initialBPP = cf->fromText(prior);
// config file priority is 40..1 (highest..lowest)
// convert config file value to setpriority(2) value(-20..19, -1 is the
// default)
if (initialBPP > 0)
m_BulkProcessPriority = 20 - initialBPP;
else if (initialBPP < 0)
m_BulkProcessPriority = 19;
if (m_BulkProcessPriority < -20)
m_BulkProcessPriority = -20;
}
//--------------------------------------------------------------------------
// Initialize bulk rollback directory
// Note this uses m_bulkRoot, so this init section must be after the section
// that sets m_bulkRoot.
//--------------------------------------------------------------------------
m_BulkRollbackDir = cf->getConfig("WriteEngine", "BulkRollbackDir");
if (m_BulkRollbackDir.length() == 0)
{
m_BulkRollbackDir.assign( m_bulkRoot );
m_BulkRollbackDir += "/rollback";
}
//--------------------------------------------------------------------------
// Initialize max disk usage
//--------------------------------------------------------------------------
m_MaxFileSystemDiskUsage = DEFAULT_MAX_FILESYSTEM_DISK_USAGE;
string usg = cf->getConfig("WriteEngine", "MaxFileSystemDiskUsagePct");
if ( usg.length() != 0 )
m_MaxFileSystemDiskUsage = cf->uFromText(usg);
if (m_MaxFileSystemDiskUsage > 100)
m_MaxFileSystemDiskUsage = DEFAULT_MAX_FILESYSTEM_DISK_USAGE;
//--------------------------------------------------------------------------
// Number of compressed padding blocks
//--------------------------------------------------------------------------
m_NumCompressedPadBlks = DEFAULT_COMPRESSED_PADDING_BLKS;
string ncpb = cf->getConfig("WriteEngine", "CompressedPaddingBlocks");
if ( ncpb.length() != 0 )
m_NumCompressedPadBlks = cf->uFromText(ncpb);
#if 0 // common code, moved to IDBPolicy
//--------------------------------------------------------------------------
// IDBDataFile logging
//--------------------------------------------------------------------------
bool idblog = false;
string idblogstr = cf->getConfig("SystemConfig", "DataFileLog");
if ( idblogstr.length() != 0 )
{
boost::to_upper(idblogstr);
idblog = ( idblogstr == "ON" );
}
//--------------------------------------------------------------------------
// Optional File System Plugin - if a HDFS type plugin is loaded
// then the system will use HDFS for all IDB data files
//--------------------------------------------------------------------------
string fsplugin = cf->getConfig("SystemConfig", "DataFilePlugin");
if ( fsplugin.length() != 0 )
{
IDBPolicy::installPlugin(fsplugin);
}
//--------------------------------------------------------------------------
// HDFS file buffering
//--------------------------------------------------------------------------
// Maximum amount of memory to use for hdfs buffering.
bool bUseRdwrMemBuffer = true; // If true, use in-memory buffering, else use file buffering
int64_t hdfsRdwrBufferMaxSize = 0;
string strBufferMaxSize = cf->getConfig("SystemConfig", "hdfsRdwrBufferMaxSize");
if (strBufferMaxSize.length() == 0)
{
// Default is use membuf with no maximum size.
bUseRdwrMemBuffer = true;
}
else
{
hdfsRdwrBufferMaxSize = static_cast<int64_t>(cf->uFromText(strBufferMaxSize));
if ( hdfsRdwrBufferMaxSize == 0 )
{
// If we're given a size of 0, turn off membuffering.
bUseRdwrMemBuffer = false;
}
}
// Directory in which to place file buffer temporary files.
string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch");
if ( hdfsRdwrScratch.length() == 0 )
{
hdfsRdwrScratch = "/tmp/hdfsscratch";
}
IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize );
#endif
IDBPolicy::configIDBPolicy();
//--------------------------------------------------------------------------
// Initialize Parent OAM Module flag
// Initialize Module Type
// Initialize Local Module ID
//--------------------------------------------------------------------------
oam::Oam oam;
oam::oamModuleInfo_t t;
try {
t = oam.getModuleInfo();
m_ParentOAMModuleFlag = boost::get<4>(t);
m_LocalModuleType = boost::get<1>(t);
m_LocalModuleID = boost::get<2>(t);
}
catch (exception&) {
m_ParentOAMModuleFlag = DEFAULT_PARENT_OAM;
m_LocalModuleType.assign( DEFAULT_LOCAL_MODULE_TYPE );
m_LocalModuleID = DEFAULT_LOCAL_MODULE_ID;
}
//--------------------------------------------------------------------------
// Initialize Version Buffer
//--------------------------------------------------------------------------
m_VersionBufferDir = cf->getConfig("SystemConfig", "DBRMRoot");
if ( m_VersionBufferDir.length() == 0 )
{
#ifdef _MSC_VER
m_VersionBufferDir = startup::StartUp::installDir() + "\\version";
#else
m_VersionBufferDir =
startup::StartUp::installDir() + "/data1/systemFiles/dbrm/BRM_saves";
#endif
}
//--------------------------------------------------------------------------
// Initialize m_dbRootCount, m_dbRootPath, m_dbRootPathMap, m_dbRootId.
// Note this uses m_localModuleType and m_LocalModuleID, so this init
// section must be after the section(s) that set m_localModuleType and
// m_LocalModuleID.
//--------------------------------------------------------------------------
uint16vec_t dbRootIdPrevious( m_dbRootId ); // save current settings
strvec_t dbRootPathPrevious( m_dbRootPath ); // save current setttings
m_dbRootPath.clear();
m_dbRootPathMap.clear();
m_dbRootId.clear();
if (m_LocalModuleType == "pm")
{
oam::DBRootConfigList oamRootList;
try {
oam.getPmDbrootConfig( m_LocalModuleID, oamRootList );
std::sort( oamRootList.begin(), oamRootList.end() );
m_dbRootCount = oamRootList.size();
for (unsigned int idx=0; idx<oamRootList.size(); idx++)
{
ostringstream oss;
oss << "DBRoot" << oamRootList[idx];
std::string DbRootPath =
cf->getConfig("SystemConfig", oss.str());
m_dbRootPath.push_back( DbRootPath );
m_dbRootPathMap[ oamRootList[idx] ] = DbRootPath;
m_dbRootId.push_back( oamRootList[idx] );
}
}
catch (exception&) {
m_dbRootCount = 0;
}
}
else
{
m_dbRootCount = 0;
}
// Update counter used to track changes to local PM DBRoot list
if (!bFirstLoad)
{
if ((dbRootIdPrevious != m_dbRootId) ||
(dbRootPathPrevious != m_dbRootPath))
{
fDBRootChangeCount++;
}
}
// for (unsigned int n=0; n<m_dbRootPath.size(); n++)
// {
// std::cout << "dbrootpath: " << n << ". " << m_dbRootPath[n] <<std::endl;
// }
// for (unsigned int n=0; n<m_dbRootId.size(); n++)
// {
// std::cout << "dbrootId: " << n << ". " << m_dbRootId[n] << std::endl;
// }
// for (Config::intstrmap_t::iterator k=m_dbRootPathMap.begin();
// k!=m_dbRootPathMap.end(); ++k)
// {
// std::cout << "dbrootmap: " << k->first << "," << k->second << std::endl;
// }
}
/*******************************************************************************
* DESCRIPTION:
* Get db root count for local PM
* PARAMETERS:
* none
* RETURN:
* Number of DBRoot paths to be used for database files
******************************************************************************/
size_t Config::DBRootCount()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_dbRootCount;
}
/*******************************************************************************
* DESCRIPTION:
* Get db root
* PARAMETERS:
* idx - Index of the DBRootn entry to fetch (0 fetches DBRoot[0],etc.)
* RETURN:
* DBRoot path for specified index
******************************************************************************/
std::string Config::getDBRootByIdx(unsigned idx)
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
if (idx >= m_dbRootPath.size())
{
std::string emptyResult;
return emptyResult;
}
return m_dbRootPath[idx];
}
/*******************************************************************************
* DESCRIPTION:
* Get db root path list for the local PM
* PARAMETERS:
* dbRootPathList - return list of DBRoot paths
* RETURN:
* none
******************************************************************************/
void Config::getDBRootPathList( std::vector<std::string>& dbRootPathList )
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
dbRootPathList.clear();
dbRootPathList = m_dbRootPath;
}
/*******************************************************************************
* DESCRIPTION:
* Get db root
* PARAMETERS:
* num - DBRootN entry to fetch (1 fetches DBRoot1, etc.)
* RETURN:
* DBRoot path for specified index
******************************************************************************/
std::string Config::getDBRootByNum(unsigned num)
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
Config::intstrmap_t::const_iterator iter = m_dbRootPathMap.find( num );
if (iter == m_dbRootPathMap.end())
{
std::string emptyResult;
return emptyResult;
}
return iter->second;
}
/*******************************************************************************
* DESCRIPTION:
* Get list of applicable DBRoot ids for this job.
* PARAMETERS:
* N/A
* RETURN:
* The list of DBRoot ids
******************************************************************************/
void Config::getRootIdList( std::vector<uint16_t>& rootIds )
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
rootIds = m_dbRootId;
}
/*******************************************************************************
* DESCRIPTION:
* Get bulk root
* PARAMETERS:
* none
* RETURN:
* NO_ERROR if success, other otherwise
******************************************************************************/
std::string Config::getBulkRoot()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_bulkRoot;
}
#ifdef SHARED_NOTHING_DEMO_2
void Config::getSharedNothingRoot(char *ret)
{
string root;
boost::mutex::scoped_lock lk(m_bulkRoot_lk);
root = config::Config::makeConfig()->getConfig(
"WriteEngine", "SharedNothingRoot");
strncpy(ret, root.c_str(), FILE_NAME_SIZE);
}
#endif
/*******************************************************************************
* DESCRIPTION:
* Get wait period used between tries to get a transaction id.
* PARAMETERS:
* none
* RETURN:
* Wait Period in seconds
******************************************************************************/
int Config::getWaitPeriod()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_WaitPeriod;
}
/*******************************************************************************
* DESCRIPTION:
* Get number of segment files per column partition.
* PARAMETERS:
* none
* RETURN:
* Number of files
******************************************************************************/
unsigned Config::getFilesPerColumnPartition()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_FilesPerColumnPartition;
}
/*******************************************************************************
* DESCRIPTION:
* Get number of extents per column segment file
* PARAMETERS:
* none
* RETURN:
* Number of extents
******************************************************************************/
unsigned Config::getExtentsPerSegmentFile()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_ExtentsPerSegmentFile;
}
/*******************************************************************************
* DESCRIPTION:
* Get process priority for cpimport.bin process.
* Config file priority is in range 40..1 (highest..lowest)
* Return value is in range (-20..19)
*
* This range of values 40..1, and -20..19, matches the
* convention used by PrimProc and ExeMgr, so that we are
* consistent with those processes. Likewise we employ
* the same default priority of -1.
* PARAMETERS:
* none
* RETURN:
* cpimport.bin process priority
******************************************************************************/
int Config::getBulkProcessPriority()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_BulkProcessPriority;
}
/*******************************************************************************
* DESCRIPTION:
* Get bulk rollback directory path.
* PARAMETERS:
* none
******************************************************************************/
std::string Config::getBulkRollbackDir()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_BulkRollbackDir;
}
/*******************************************************************************
* DESCRIPTION:
* Get Max percentage of allowable file system disk usage for each DBRoot
* PARAMETERS:
* none
******************************************************************************/
unsigned Config::getMaxFileSystemDiskUsage()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_MaxFileSystemDiskUsage;
}
/*******************************************************************************
* DESCRIPTION:
* Get number of blocks to use in padding each compressed chunk (only
* applies to compressed columns).
* PARAMETERS:
* none
******************************************************************************/
unsigned Config::getNumCompressedPadBlks()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_NumCompressedPadBlks;
}
/*******************************************************************************
* DESCRIPTION:
* Get Parent OAM Module flag; are we running on active parent OAM node.
* PARAMETERS:
* none
******************************************************************************/
bool Config::getParentOAMModuleFlag()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_ParentOAMModuleFlag;
}
/*******************************************************************************
* DESCRIPTION:
* Get Module type (ex: "pm")
* PARAMETERS:
* none
******************************************************************************/
std::string Config::getLocalModuleType()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_LocalModuleType;
}
/*******************************************************************************
* DESCRIPTION:
* Get Module ID number (ex: 1)
* PARAMETERS:
* none
******************************************************************************/
uint16_t Config::getLocalModuleID()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_LocalModuleID;
}
/*******************************************************************************
* DESCRIPTION:
* Get version buffer root
* PARAMETERS:
* none
* RETURN:
* NO_ERROR if success, other otherwise
******************************************************************************/
std::string Config::getVBRoot()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
return m_VersionBufferDir;
}
/*******************************************************************************
* DESCRIPTION:
* Has the DBRoot list for the local PM changed since the last time this
* function was called.
* PARAMETERS:
* none
* RETURN:
* returns TRUE if local DBRoot list has changed since the last call.
******************************************************************************/
bool Config::hasLocalDBRootListChanged()
{
boost::mutex::scoped_lock lk(fCacheLock);
if (fDBRootChangeCount > 0)
{
fDBRootChangeCount = 0;
return true;
}
return false;
}
} //end of namespace

201
writeengine/shared/we_config.h Normal file
View File

@ -0,0 +1,201 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_config.h 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
/** @file */
#ifndef WE_CONFIG_H_
#define WE_CONFIG_H_
#include <string>
#include <boost/thread.hpp>
#include <vector>
#include <map>
#include "we_obj.h"
//#define SHARED_NOTHING_DEMO_2
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
/** Class Config */
class Config
{
public:
/**
* @brief Constructor
*/
Config() {}
/**
* @brief Default Destructor
*/
~Config() {}
/**
* @brief Get DB root (for local PM)
* @param idx Index of the DBRootn entry to fetch (0 fetches DBRoot[0],etc.)
*/
EXPORT static std::string getDBRootByIdx(unsigned idx);
/**
* @brief Get complete DBRoot path list for the local PM
* @param dbRootPathList vector of DBRoot paths
*/
EXPORT static void getDBRootPathList(
std::vector<std::string>& dbRootPathList );
/**
* @brief Get DB root (for local PM)
* @param num DBRootN entry to fetch (1 fetches DBRoot1, etc.)
*/
EXPORT static std::string getDBRootByNum(unsigned num);
/**
* @brief Get list of applicable DBRoot ids for this job.
*/
EXPORT static void getRootIdList( std::vector<uint16_t>& dbRootIds );
#ifdef SHARED_NOTHING_DEMO_2
EXPORT static void getSharedNothingRoot(char *); // pass in an char[FILE_NAME_SIZE]
#endif
/**
* @brief Bulkload DB root
*/
EXPORT static std::string getBulkRoot();
/**
* @brief DBRoot count for local PM
*/
EXPORT static size_t DBRootCount();
/**
* @brief Wait Period
*/
EXPORT static int getWaitPeriod();
/**
* @brief FilesPerColumnPartition
*/
EXPORT static unsigned getFilesPerColumnPartition();
/**
* @brief ExtentsPerSegmentFile
*/
EXPORT static unsigned getExtentsPerSegmentFile();
/**
* @brief Process Priority for cpimport.bin
* Return value is in range -20..19 (highest...lowest, 0=normal)
*/
EXPORT static int getBulkProcessPriority();
/**
* @brief Directory carrying Bulk Rollback meta data files
*/
EXPORT static std::string getBulkRollbackDir();
/**
* @brief Max percentage of allowable file system disk usage for each DBRoot
*/
EXPORT static unsigned getMaxFileSystemDiskUsage();
/**
* @brief Number of Blocks to pad each compressed chunk.
*/
EXPORT static unsigned getNumCompressedPadBlks();
/**
* @brief Parent OAM Module flag (is this the parent OAM node, ex: pm1)
*/
EXPORT static bool getParentOAMModuleFlag();
/**
* @brief Local Module Type (ex: "pm")
*/
EXPORT static std::string getLocalModuleType();
/**
* @brief Local Module ID (ex: 1 )
*/
EXPORT static uint16_t getLocalModuleID();
/**
* @brief Version Buffer root
*/
EXPORT static std::string getVBRoot();
/**
* @brief Cache the config parameters locally
* Initialize Config cache. Cache will be updated as needed.
*/
EXPORT static void initConfigCache();
/**
* @brief Has Local PM DBRoot info changed since last time this function
* was called. Can be used to monitor changes to DBRoot info.
*/
EXPORT static bool hasLocalDBRootListChanged();
private:
typedef std::vector<std::string> strvec_t;
typedef std::map<int,std::string> intstrmap_t;
typedef std::vector<uint16_t> uint16vec_t;
static void checkReload();
static int m_dbRootCount; // num DBRoots for local PM
static strvec_t m_dbRootPath; // root paths for open files
static intstrmap_t m_dbRootPathMap; // map of root id to root paths
static uint16vec_t m_dbRootId; // list of root ids
static std::string m_bulkRoot; // root path for bulk operation
static unsigned long fDBRootChangeCount; // track recent DBRoot changes
static time_t fCacheTime; // timestamp associated w/cache
static boost::mutex fCacheLock; // mutex for m_dbRoot sync
#ifdef SHARED_NOTHING_DEMO_2
static boost::mutex m_bulkRoot_lk; // mutex for m_bulkRoot sync
#endif
static int m_WaitPeriod; // secs to wait for transaction
static unsigned m_FilesPerColumnPartition;//# seg files per partition
static unsigned m_ExtentsPerSegmentFile; // # extents per segment file
static int m_BulkProcessPriority; // cpimport.bin proc priority
static std::string m_BulkRollbackDir; // bulk rollback meta data dir
static unsigned m_MaxFileSystemDiskUsage;// max file system % disk usage
static unsigned m_NumCompressedPadBlks; // num blks to pad comp chunks
static bool m_ParentOAMModuleFlag; // are we running on parent PM
static std::string m_LocalModuleType; // local node type (ex: "pm")
static int m_LocalModuleID; // local node id (ex: 1 )
static std::string m_VersionBufferDir; // Version buffer directory
};
} //end of namespace
#undef EXPORT
#endif // WE_CONFIG_H_

823
writeengine/shared/we_confirmhdfsdbfile.cpp Normal file
View File

@ -0,0 +1,823 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "we_confirmhdfsdbfile.h"
#include <cerrno>
#include <cstring>
#include <iostream>
#include <sstream>
#include <vector>
#include <boost/scoped_array.hpp>
#include <boost/scoped_ptr.hpp>
#include "we_define.h"
#include "we_config.h"
#include "we_fileop.h"
#include "we_rbmetawriter.h"
#include "IDBPolicy.h"
#include "IDBDataFile.h"
namespace
{
const int BUF_SIZE = 1024; // size of buffer used to read meta data records
}
namespace WriteEngine
{
//------------------------------------------------------------------------------
// Constructor
// This class should typically only be used on an HDFS system, so we could
// hardcode this to pass HDFS to getFsi(); but it comes in handy for testing,
// to be able to execute this class on a non-HDFS stack as well. So I rely
// on useHdfs() to tell me which FileSystem reference to get.
//------------------------------------------------------------------------------
ConfirmHdfsDbFile::ConfirmHdfsDbFile() :
fFs( (idbdatafile::IDBPolicy::useHdfs()) ?
idbdatafile::IDBFileSystem::getFs(idbdatafile::IDBDataFile::HDFS) :
idbdatafile::IDBFileSystem::getFs(idbdatafile::IDBDataFile::BUFFERED))
{
}
//------------------------------------------------------------------------------
// Destructor
//------------------------------------------------------------------------------
ConfirmHdfsDbFile::~ConfirmHdfsDbFile()
{
}
//------------------------------------------------------------------------------
// Backup a cdf file and replace it with the updated tmp file.
//------------------------------------------------------------------------------
int ConfirmHdfsDbFile::confirmDbFileChange(
const std::string& backUpFileType,
const std::string& filename,
std::string& errMsg) const
{
// return value
int rc = NO_ERROR;
// This rlc file should be renamed if success, just skip it
if (backUpFileType.compare("rlc") == 0)
{
return rc;
}
if (backUpFileType.compare("tmp") != 0 )
{
std::ostringstream oss;
oss << backUpFileType << " is a bad type to confirm DbFile change: " <<
filename;
errMsg = oss.str();
rc = ERR_HDFS_BACKUP;
return rc;
}
// add safety checks, just in case
std::string tmp(filename + ".tmp");
if (!fFs.exists(tmp.c_str())) // file already swapped
return rc;
if (fFs.size(tmp.c_str()) <= 0)
{
std::ostringstream oss;
oss << "tmp file " << tmp << " has bad size" << fFs.size(tmp.c_str());
errMsg = oss.str();
rc = ERR_COMP_RENAME_FILE;
return rc;
}
// remove the old orig if exists
std::string orig(filename + ".orig");
errno = 0;
if ((fFs.exists(orig.c_str())) &&
(fFs.remove(orig.c_str())) != 0)
{
int errNum = errno;
std::ostringstream oss;
oss << "remove old " << orig << " failed: " << strerror(errNum);
errMsg = oss.str();
rc = ERR_COMP_REMOVE_FILE;
return rc;
}
// backup the original
errno = 0;
if (fFs.rename(filename.c_str(), orig.c_str()) != 0)
{
int errNum = errno;
std::ostringstream oss;
oss << "rename " << filename << " to " << orig << " failed: " <<
strerror(errNum);
errMsg = oss.str();
rc = ERR_COMP_RENAME_FILE;
return rc;
}
// rename the new file
errno = 0;
if (fFs.rename(tmp.c_str(), filename.c_str()) != 0)
{
int errNum = errno;
std::ostringstream oss;
oss << "rename " << tmp << " to " << filename << " failed: " <<
strerror(errNum);
errMsg = oss.str();
rc = ERR_COMP_RENAME_FILE;
return rc;
}
return rc;
}
//------------------------------------------------------------------------------
// Finalize the changes to a db file.
// If success flag is true, then remove the orig
// otherwise, move the orig back to cdf
//------------------------------------------------------------------------------
int ConfirmHdfsDbFile::endDbFileChange(
const std::string& backUpFileType,
const std::string& filename,
bool success,
std::string& errMsg) const
{
// return value
int rc = NO_ERROR;
// This rlc file should be renamed if success, it is useless if failed.
if (backUpFileType.compare("rlc") == 0)
{
std::string rlc(filename + ".rlc");
if (fFs.exists(rlc.c_str()))
fFs.remove(rlc.c_str()); // TBD-okay to ignore failed removal?
return rc;
}
if (backUpFileType.compare("tmp") != 0)
{
std::ostringstream oss;
oss << backUpFileType << " is a bad type to finalize DbFile change: " <<
filename;
errMsg = oss.str();
rc = ERR_HDFS_BACKUP;
return rc;
}
std::string orig(filename + ".orig");
if (success)
{
// remove the orig file
errno = 0;
if ((fFs.exists(orig.c_str())) &&
(fFs.remove(orig.c_str())) != 0)
{
int errNum = errno;
std::ostringstream oss;
oss << "remove " << orig << " failed: " << strerror(errNum);
errMsg = oss.str();
rc = ERR_COMP_REMOVE_FILE;
return rc;
}
}
else
{
// restore the orig file
if (fFs.exists(orig.c_str()))
{
errno = 0;
// Try to remove file only if it exists
if ((fFs.exists(filename.c_str())) &&
(fFs.remove(filename.c_str()) != 0))
{
int errNum = errno;
std::ostringstream oss;
oss << "failed restore; remove " << filename << " failed: " <<
strerror(errNum);
errMsg = oss.str();
rc = ERR_COMP_REMOVE_FILE;
return rc;
}
errno = 0;
if (fFs.rename(orig.c_str(), filename.c_str()) != 0)
{
int errNum = errno;
std::ostringstream oss;
oss << "failed restore; rename " << orig << " failed: " <<
strerror(errNum);
errMsg = oss.str();
rc = ERR_COMP_RENAME_FILE;
return rc;
}
}
// remove the tmp file
std::string tmp(filename + ".tmp");
errno = 0;
if ((fFs.exists(tmp.c_str())) &&
(fFs.remove(tmp.c_str())) != 0)
{
int errNum = errno;
std::ostringstream oss;
oss << "failed restore; remove " << tmp << " failed: " <<
strerror(errNum);
errMsg = oss.str();
rc = ERR_COMP_REMOVE_FILE;
return rc;
}
// remove the chunk shifting helper
std::string rlc(filename + ".rlc");
errno = 0;
if ((fFs.exists(rlc.c_str())) &&
(fFs.remove(rlc.c_str())) != 0)
{
int errNum = errno;
std::ostringstream oss;
oss << "failed restore; remove " << rlc << " failed: " <<
strerror(errNum);
errMsg = oss.str();
rc = ERR_COMP_REMOVE_FILE;
return rc;
}
}
return rc;
}
//------------------------------------------------------------------------------
// Confirm the changes to the hwm DB files listed in the bulk rollback meta
// data file corresponding to the specified table OID.
//------------------------------------------------------------------------------
int ConfirmHdfsDbFile::confirmDbFileListFromMetaFile(
OID tableOID,
std::string& errMsg)
{
int rc = NO_ERROR;
try
{
std::vector<uint16_t> dbRoots;
Config::getRootIdList( dbRoots );
for (unsigned m=0; m<dbRoots.size(); m++)
{
std::istringstream metaDataStream;
openMetaDataFile ( tableOID,
dbRoots[m], metaDataStream );
confirmDbFiles( metaDataStream );
}
}
catch (WeException& ex)
{
std::ostringstream oss;
oss << "Error confirming changes to table " << tableOID <<
"; " << ex.what();
errMsg = oss.str();
rc = ex.errorCode();
}
catch (std::exception& ex)
{
std::ostringstream oss;
oss << "Error confirming changes to table " << tableOID <<
"; " << ex.what();
errMsg = oss.str();
rc = ERR_UNKNOWN;
}
return rc;
}
//------------------------------------------------------------------------------
// Confirm the changes to the hwm DB files listed in the bulk rollback meta
// data file stream stored in metaDataStream.
//------------------------------------------------------------------------------
void ConfirmHdfsDbFile::confirmDbFiles(std::istringstream& metaDataStream) const
{
char inBuf[ BUF_SIZE ];
// Loop through the records in the meta-data file
while (metaDataStream.getline( inBuf, BUF_SIZE ))
{
// Restore Files for current DBRoot
if (RBMetaWriter::verifyColumn1Rec(inBuf))
{
confirmColumnDbFile(inBuf);
}
else if (RBMetaWriter::verifyDStore1Rec(inBuf))
{
confirmDctnryStoreDbFile(inBuf);
}
}
}
//------------------------------------------------------------------------------
// Confirm the changes to the hwm column DB file described in the bulk
// rollback meta data file record stored in inBuf.
//------------------------------------------------------------------------------
void ConfirmHdfsDbFile::confirmColumnDbFile(const char* inBuf) const
{
char recType[100];
OID columnOID;
uint32_t dbRootHwm;
uint32_t partNumHwm;
uint32_t segNumHwm;
HWM lastLocalHwm;
int colTypeInt;
char colTypeName[100];
uint32_t colWidth;
int compressionType = 0; // optional parameter
// Read meta-data record
int numFields = sscanf(inBuf, "%s %u %u %u %u %u %d %s %u %d",
recType, &columnOID,
&dbRootHwm, &partNumHwm, &segNumHwm, &lastLocalHwm,
&colTypeInt, colTypeName, &colWidth, &compressionType );
if (numFields < 9) // compressionType is optional
{
std::ostringstream oss;
oss << "Invalid COLUM1 record in meta-data file " <<
fMetaFileName << "; record-<" << inBuf << ">";
throw WeException( oss.str(), ERR_INVALID_PARAM );
}
// Construct the DB file name
char dbFileName[FILE_NAME_SIZE];
FileOp dbFile(false);
int rc = dbFile.getFileName( columnOID,
dbFileName,
dbRootHwm,
partNumHwm,
segNumHwm );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error constructing column filename to confirm changes" <<
"; columnOID-" << columnOID <<
"; dbRoot-" << dbRootHwm <<
"; partNum-" << partNumHwm <<
"; segNum-" << segNumHwm <<
"; " << ec.errorString(rc);
throw WeException( oss.str(), rc );
}
// Confirm the changes to the DB file name
std::string errMsg;
rc = confirmDbFileChange( std::string("tmp"),
dbFileName,
errMsg );
if (rc != NO_ERROR)
{
throw WeException( errMsg, rc );
}
}
//------------------------------------------------------------------------------
// Confirm the changes to the hwm dctnry store DB file described in the bulk
// rollback meta data file record stored in inBuf.
//------------------------------------------------------------------------------
void ConfirmHdfsDbFile::confirmDctnryStoreDbFile(const char* inBuf) const
{
char recType[100];
OID dColumnOID;
OID dStoreOID;
uint32_t dbRootHwm;
uint32_t partNumHwm;
uint32_t segNumHwm;
HWM localHwm;
int compressionType = 0; // optional parameter
// Read meta-data record
int numFields = sscanf(inBuf, "%s %u %u %u %u %u %u %d",
recType, &dColumnOID, &dStoreOID,
&dbRootHwm, &partNumHwm, &segNumHwm, &localHwm, &compressionType );
if (numFields < 7) // compressionType optional
{
std::ostringstream oss;
oss << "Invalid DSTOR1 record in meta-data file " <<
fMetaFileName << "; record-<" << inBuf << ">";
throw WeException( oss.str(), ERR_INVALID_PARAM );
}
// Construct the DB file name
char dbFileName[FILE_NAME_SIZE];
FileOp dbFile(false);
int rc = dbFile.getFileName( dStoreOID,
dbFileName,
dbRootHwm,
partNumHwm,
segNumHwm );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss<<"Error constructing dictionary store filename to confirm changes"<<
"; columnOID-" << dStoreOID <<
"; dbRoot-" << dbRootHwm <<
"; partNum-" << partNumHwm <<
"; segNum-" << segNumHwm <<
"; " << ec.errorString(rc);
throw WeException( oss.str(), rc );
}
// Confirm the changes to the DB file name
std::string errMsg;
rc = confirmDbFileChange( std::string("tmp"),
dbFileName,
errMsg );
if (rc != NO_ERROR)
{
throw WeException( errMsg, rc );
}
}
//------------------------------------------------------------------------------
// End the changes to the hwm DB files listed in the bulk rollback meta
// data file corresponding to the specified table OID. Delete temp files.
//------------------------------------------------------------------------------
int ConfirmHdfsDbFile::endDbFileListFromMetaFile(
OID tableOID,
bool success,
std::string& errMsg)
{
int rc = NO_ERROR;
errMsg.clear();
std::vector<uint16_t> dbRoots;
Config::getRootIdList( dbRoots );
for (unsigned m=0; m<dbRoots.size(); m++)
{
std::istringstream metaDataStream;
try
{
std::istringstream metaDataStream;
openMetaDataFile ( tableOID,
dbRoots[m], metaDataStream );
endDbFiles( metaDataStream, success );
}
// We catch any errors, but not deleting a temp file is not fatal,
// so we capture the error msg and keep going if a problem occurs.
// We return a concatenated list of error msgs if multiple errors
// take place.
catch (WeException& ex)
{
if (errMsg.size() == 0)
{
std::ostringstream oss;
oss << "Error deleting temp files for table " << tableOID <<
"; " << ex.what();
errMsg = oss.str();
rc = ex.errorCode();
}
else
{
errMsg += "; ";
errMsg += ex.what();
}
}
catch (std::exception& ex)
{
if (errMsg.size() == 0)
{
std::ostringstream oss;
oss << "Error deleting temp files for table " << tableOID <<
"; " << ex.what();
errMsg = oss.str();
rc = ERR_UNKNOWN;
}
else
{
errMsg += "; ";
errMsg += ex.what();
}
}
}
return rc;
}
//------------------------------------------------------------------------------
// End the changes to the hwm DB files listed in the bulk rollback meta
// data file stream stored in metaDataStream. Delete temp files.
//------------------------------------------------------------------------------
void ConfirmHdfsDbFile::endDbFiles(
std::istringstream& metaDataStream,
bool success) const
{
char inBuf[ BUF_SIZE ];
std::string errMsg;
int rc = NO_ERROR;
// Loop through the records in the meta-data file
while (metaDataStream.getline( inBuf, BUF_SIZE ))
{
try
{
// Delete Temp Files for current DBRoot
if (RBMetaWriter::verifyColumn1Rec(inBuf))
{
endColumnDbFile(inBuf, success);
}
else if (RBMetaWriter::verifyDStore1Rec(inBuf))
{
endDctnryStoreDbFile(inBuf, success);
}
}
// We catch any errors, but not deleting a temp file is not fatal,
// so we capture the error msg and keep going if a problem occurs.
// We return a concatenated list of error msgs if multiple errors
// take place.
catch (WeException& ex)
{
if (errMsg.size() == 0)
{
rc = ex.errorCode();
}
else
{
errMsg += "; ";
}
errMsg += ex.what();
}
catch (std::exception& ex)
{
if (errMsg.size() == 0)
{
rc = ERR_UNKNOWN;
}
else
{
errMsg += "; ";
}
errMsg += ex.what();
}
}
// Throw exception with cumulative list of any error msgs
if (errMsg.size() > 0)
{
throw WeException( errMsg, rc );
}
}
//------------------------------------------------------------------------------
// End the changes to the hwm column DB file described in the bulk
// rollback meta data file record stored in inBuf. Delete the temp file.
//------------------------------------------------------------------------------
void ConfirmHdfsDbFile::endColumnDbFile(
const char* inBuf,
bool success) const
{
char recType[100];
OID columnOID;
uint32_t dbRootHwm;
uint32_t partNumHwm;
uint32_t segNumHwm;
HWM lastLocalHwm;
int colTypeInt;
char colTypeName[100];
uint32_t colWidth;
int compressionType = 0; // optional parameter
// Read meta-data record
int numFields = sscanf(inBuf, "%s %u %u %u %u %u %d %s %u %d",
recType, &columnOID,
&dbRootHwm, &partNumHwm, &segNumHwm, &lastLocalHwm,
&colTypeInt, colTypeName, &colWidth, &compressionType );
if (numFields < 9) // compressionType is optional
{
std::ostringstream oss;
oss << "Invalid COLUM1 record in meta-data file " <<
fMetaFileName << "; record-<" << inBuf << ">";
throw WeException( oss.str(), ERR_INVALID_PARAM );
}
// Construct the DB file name
char dbFileName[FILE_NAME_SIZE];
FileOp dbFile(false);
int rc = dbFile.getFileName( columnOID,
dbFileName,
dbRootHwm,
partNumHwm,
segNumHwm );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error constructing column filename to end changes" <<
"; columnOID-" << columnOID <<
"; dbRoot-" << dbRootHwm <<
"; partNum-" << partNumHwm <<
"; segNum-" << segNumHwm <<
"; " << ec.errorString(rc);
throw WeException( oss.str(), rc );
}
// Confirm the changes to the DB file name
std::string errMsg;
rc = endDbFileChange( std::string("tmp"),
dbFileName,
success,
errMsg );
if (rc != NO_ERROR)
{
throw WeException( errMsg, rc );
}
}
//------------------------------------------------------------------------------
// End the changes to the hwm dctnry store DB file described in the bulk
// rollback meta data file record stored in inBuf. Delete the temp file.
//------------------------------------------------------------------------------
void ConfirmHdfsDbFile::endDctnryStoreDbFile(
const char* inBuf,
bool success) const
{
char recType[100];
OID dColumnOID;
OID dStoreOID;
uint32_t dbRootHwm;
uint32_t partNumHwm;
uint32_t segNumHwm;
HWM localHwm;
int compressionType = 0; // optional parameter
// Read meta-data record
int numFields = sscanf(inBuf, "%s %u %u %u %u %u %u %d",
recType, &dColumnOID, &dStoreOID,
&dbRootHwm, &partNumHwm, &segNumHwm, &localHwm, &compressionType );
if (numFields < 7) // compressionType optional
{
std::ostringstream oss;
oss << "Invalid DSTOR1 record in meta-data file " <<
fMetaFileName << "; record-<" << inBuf << ">";
throw WeException( oss.str(), ERR_INVALID_PARAM );
}
// Construct the DB file name
char dbFileName[FILE_NAME_SIZE];
FileOp dbFile(false);
int rc = dbFile.getFileName( dStoreOID,
dbFileName,
dbRootHwm,
partNumHwm,
segNumHwm );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss<<"Error constructing dictionary store filename to end changes"<<
"; columnOID-" << dStoreOID <<
"; dbRoot-" << dbRootHwm <<
"; partNum-" << partNumHwm <<
"; segNum-" << segNumHwm <<
"; " << ec.errorString(rc);
throw WeException( oss.str(), rc );
}
// Confirm the changes to the DB file name
std::string errMsg;
rc = endDbFileChange( std::string("tmp"),
dbFileName,
success,
errMsg );
if (rc != NO_ERROR)
{
throw WeException( errMsg, rc );
}
}
//------------------------------------------------------------------------------
// Open and read the bulk rollback metadata file for the specified table OID
// and DBRoot. The contents of the metadata file is returned in the meta-
// DataStream argument.
//------------------------------------------------------------------------------
void ConfirmHdfsDbFile::openMetaDataFile(OID tableOID,
uint16_t dbRoot,
std::istringstream& metaDataStream)
{
std::string bulkRollbackPath( Config::getDBRootByNum( dbRoot ) );
// Construct file name and check for it's existence
std::ostringstream ossFileName;
ossFileName << '/' << DBROOT_BULK_ROLLBACK_SUBDIR << '/' << tableOID;
fMetaFileName = bulkRollbackPath;
fMetaFileName += ossFileName.str();
// Return if the meta-data file does not exist.
if ( !fFs.exists( fMetaFileName.c_str() ) )
{
std::ostringstream oss;
oss << "Bulk rollback meta-data file " <<
fMetaFileName << " does not exist.";
throw WeException( oss.str(), ERR_FILE_NOT_EXIST );
}
// Open the file
boost::scoped_ptr<IDBDataFile> metaFile;
errno = 0;
metaFile.reset(idbdatafile::IDBDataFile::open(
idbdatafile::IDBPolicy::getType(fMetaFileName.c_str(),
idbdatafile::IDBPolicy::WRITEENG),
fMetaFileName.c_str(), "rb", 0) );
if ( !metaFile )
{
int errRc = errno;
std::ostringstream oss;
oss << "Error opening bulk rollback meta-data file " <<
fMetaFileName << "; err-" <<
errRc << "; " << strerror( errRc );
throw WeException( oss.str(), ERR_FILE_OPEN );
}
// First record in the file must be a Version record.
char inBuf[ BUF_SIZE ];
ssize_t metaFileSize = fFs.size( fMetaFileName.c_str() );
boost::scoped_array<char> buf( new char[ metaFileSize ] );
// retry 10 times for partial reads, just in case
ssize_t readSofar = 0; // bytes read so far
ssize_t bytes = 0; // bytes read by one pread
char* p = buf.get();
for (int i = 0; i < 10 && readSofar < metaFileSize; i++)
{
errno = 0;
bytes = metaFile->pread( p+readSofar,
readSofar,
metaFileSize-readSofar);
if (bytes < 0)
break;
readSofar += bytes;
}
if ( readSofar != metaFileSize )
{
int errRc = errno;
std::ostringstream oss;
oss << "Error reading bulk rollback meta-data file "
<< fMetaFileName << "; read/expect:" << readSofar << "/"
<< metaFileSize
<< "; err-" << errRc << "; " << strerror( errRc );
throw WeException( oss.str(), ERR_FILE_READ );
}
// put the data in a string stream
metaDataStream.str( std::string( p, metaFileSize ) );
buf.reset();
// read data
metaDataStream.getline( inBuf, BUF_SIZE );
if (!RBMetaWriter::verifyVersion4(inBuf))
{
std::ostringstream oss;
oss << "Invalid version record in meta-data file " << fMetaFileName
<< "; record-<" << inBuf << ">";
throw WeException( oss.str(), ERR_INVALID_PARAM );
}
}
} // end of namespace

139
writeengine/shared/we_confirmhdfsdbfile.h Normal file
View File

@ -0,0 +1,139 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/** @file */
#ifndef CONFIRM_HDFS_DBFILE_H
#define CONFIRM_HDFS_DBFILE_H
#include <string>
#include "IDBFileSystem.h"
#include "we_type.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
namespace WriteEngine
{
/** @brief Encapsulates logic to confirm and finalize (or abort) changes
* to an HDFS db file. Class should only be used for HDFS db files.
*
* For the HDFS db files that are modified, the new version of the file
* may be written to a file with an alternative suffix (like .tmp file).
* The confirmDbFileChange() function confirms that the work is complete,
* and that the new temporary file (ex: *.tmp) can replace the original file.
* After all the columns in a table have been confirmed (through calls to
* confirmDbFileChange()), then endDbFileChange() should be called to finish
* committing or aborting all the work.
*/
class ConfirmHdfsDbFile
{
public:
EXPORT ConfirmHdfsDbFile( );
EXPORT ~ConfirmHdfsDbFile( );
/** @brief Confirm changes to the specified db file
* @param backUpFileType Backup file type to confirm. Types:
* "rlc" - reallocated chunk file
* "tmp" - updated db file
* @param filename Name of db file to be confirmed.
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int confirmDbFileChange( const std::string& backUpFileType,
const std::string& filename,
std::string& errMsg ) const;
/** @brief Finalize changes to the specified db file
*
* If success flag is true:
* The old version of the db file is deleted.
* If success flag is false:
* The old version is retained, and any temporary file with pending
* changes is deleted.
*
* @param backUpFileType Backup file type to finalize. Types:
* "rlc" - reallocated chunk file
* "tmp" - updated db file
* @param filename Name of db file to be finalized.
* @param success Final success/fail status of db file changes
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int endDbFileChange( const std::string& backUpFileType,
const std::string& filename,
bool success,
std::string& errMsg ) const;
/** @brief Confirm changes to the db files modified for tableOID
*
* The HWM db file for each DBRoot, as listed in the bulk rollback meta
* data file (for the specified tableOID), is confirmed.
*
* @param tableOID Table that has changes to be confirmed
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int confirmDbFileListFromMetaFile( OID tableOID,
std::string& errMsg );
/** @brief Finalize changes to the db files modified for tableOID
*
* The HWM db file for each DBRoot, as listed in the bulk rollback meta
* data file (for the specified tableOID), is finalized.
*
* If success flag is true:
* The old version of the db files are deleted.
* If success flag is false:
* The old versions are retained, and any temporary files with pending
* changes are deleted.
*
* @param tableOID Table that has changes to be confirmed
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int endDbFileListFromMetaFile( OID tableOID,
bool success,
std::string& errMsg );
private:
void openMetaDataFile( OID tableOID,
uint16_t dbRoot,
std::istringstream& metaDataStream );
void confirmDbFiles( std::istringstream& metaDataStream ) const;
void confirmColumnDbFile( const char* inBuf ) const;
void confirmDctnryStoreDbFile( const char* inBuf ) const;
void endDbFiles( std::istringstream& metaDataStream, bool success ) const;
void endColumnDbFile( const char* inBuf, bool success ) const;
void endDctnryStoreDbFile( const char* inBuf, bool success ) const;
idbdatafile::IDBFileSystem& fFs;
std::string fMetaFileName;
};
} // end of namespace
#undef EXPORT
#endif // CONFIRM_HDFS_DBFILE_H

824
writeengine/shared/we_convertor.cpp Normal file
View File

@ -0,0 +1,824 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_convertor.cpp 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
/** @file */
#include <unistd.h>
#include <limits>
#include <cstring>
#ifdef _MSC_VER
#include <cstdio>
#endif
#include "config.h"
#include "we_convertor.h"
using namespace std;
using namespace execplan;
namespace
{
const char DATE_TIME_FORMAT[] = "%04d-%02d-%02d %02d:%02d:%02d";
/*******************************************************************************
* DESCRIPTION:
* Takes an 8-bit value and converts it into a directory name.
* PARAMETERS:
* pBuffer(output) - a pointer to the output buffer
* blen (input) - the length of the output buffer (in bytes)
* val (input) - value to be used in the formatted name
* RETURN:
* the number of characters printed in the output buffer (not including
* the null terminator). -1 is returned if the input buffer pointer is NULL.
******************************************************************************/
int _doDir(char* pBuffer, int blen, unsigned int val)
{
int rc;
if (!pBuffer)
{
rc = -1;
} else {
rc = snprintf(pBuffer, blen, "%03u.dir", val);
pBuffer[blen-1] = (char)0;
}
return rc;
}
/*******************************************************************************
* DESCRIPTION:
* Takes an 8-bit value and converts it into a file name.
* PARAMETERS:
* pBuffer(output) - a pointer to the output buffer
* blen (input) - the length of the output buffer (in bytes)
* val (input) - value to be used in the formatted name
* RETURN:
* the number of characters printed in the output buffer (not including
* the null terminator). -1 is returned if the input buffer pointer is NULL.
******************************************************************************/
int _doFile(char* pBuffer, int blen, unsigned char val)
{
int rc;
if (!pBuffer)
{
rc = -1;
} else {
rc = snprintf(pBuffer, blen, "FILE%03d.cdf", val);
pBuffer[blen-1] = (char)0;
}
return rc;
}
}
namespace WriteEngine
{
struct Convertor::dmFilePathArgs_t
{
char* pDirA; // < OUT -- DirA's buffer
char* pDirB; // < OUT -- DirB's buffer
char* pDirC; // < OUT -- DirC's buffer
char* pDirD; // < OUT -- DirD's buffer
char* pDirE; // < OUT -- DirE's buffer
char* pFName; // < OUT -- Filename buffer
int ALen; // < IN -- Size in bytes of DirA's Buffer.
int BLen; // < IN -- Size in bytes of DirB's Buffer.
int CLen; // < IN -- Size in bytes of DirC's Buffer.
int DLen; // < IN -- Size in bytes of DirD's Buffer.
int ELen; // < IN -- Size in bytes of DirE's Buffer.
int FNLen; // < IN -- Size in bytes of Filename's Buffer.
int Arc; // < OUT -- result code for formatting DirA.
int Brc; // < OUT -- result code for formatting DirB.
int Crc; // < OUT -- result code for formatting DirC.
int Drc; // < OUT -- result code for formatting DirD.
int Erc; // < OUT -- result code for formatting DirE.
int FNrc; // < OUT -- result code for formatting Filename.
};
/*******************************************************************************
* DESCRIPTION:
* Get time string
* PARAMETERS:
* none
* RETURN:
* time string
******************************************************************************/
/* static */
const std::string Convertor::getTimeStr()
{
char buf[sizeof(DATE_TIME_FORMAT)+10] = {0};
time_t curTime = time(NULL);
struct tm pTime;
localtime_r(&curTime, &pTime);
string timeStr;
snprintf(buf, sizeof(buf), DATE_TIME_FORMAT, pTime.tm_year + 1900,
pTime.tm_mon + 1, pTime.tm_mday,
pTime.tm_hour, pTime.tm_min, pTime.tm_sec);
timeStr = buf;
return timeStr;
}
/*******************************************************************************
* DESCRIPTION:
* Convert int value to string
* PARAMETERS:
* val - value
* RETURN:
* string
******************************************************************************/
/* static */
const std::string Convertor::int2Str(int val)
{
char buf[12];
string myStr;
snprintf(buf, sizeof(buf), "%d", val);
myStr = buf;
return myStr;
}
/*******************************************************************************
* DESCRIPTION:
* Convert a numeric string to a decimal long long, given the specified
* scale. errno should be checked upon return from this function to see
* if it is set to ERANGE, meaning the value was out of range.
* PARAMETERS:
* field - string to be interpreted
* fieldLength - length of "field"
* scale - decimal scale value to be used to parse "field"
* RETURN:
* converted long long for specified "field"
******************************************************************************/
/* static */
long long Convertor::convertDecimalString(
const char* field,
int fieldLength,
int scale )
{
long long llVal = 0;
int nDigitsBeforeDecPt = 0;
int nDigitsAfterDecPt = 0;
long long roundUp = 0; //@bug 3405 round off decimal column values
// Determine the number of digits before and after the decimal point
char* posDecPt = (char*)memchr(field, '.', fieldLength);
if (posDecPt)
{
nDigitsBeforeDecPt = posDecPt - field;
nDigitsAfterDecPt = fieldLength - nDigitsBeforeDecPt - 1;
//@bug 3405 round off decimal column values
// We look at the scale+1 digit to see if we need to round up.
if (nDigitsAfterDecPt > scale)
{
char roundOffDigit = *(posDecPt + 1 + scale);
if ( (roundOffDigit > '4') &&
(roundOffDigit <='9') ) // round up
{
roundUp = 1;
// We can't just use the sign of llVal to determine whether to
// add +1 or -1, because if we read in -0.005 with scale 2, we
// end up parsing "-0.00", which yields 0; meaning we lose the
// sign. So better (though maybe slower) to look for any lead-
// ing negative sign in the input string.
for (int k=0; k<fieldLength; k++)
{
if (!isspace(field[k]))
{
if (field[k] == '-')
roundUp = -1;
break;
}
}
}
}
}
else
{
nDigitsBeforeDecPt = fieldLength;
nDigitsAfterDecPt = 0;
}
// Strip out the decimal point by stringing together
// the digits before and after the decimal point.
char* data = (char*)alloca(nDigitsBeforeDecPt + scale + 1);
memcpy(data, field, nDigitsBeforeDecPt);
if (nDigitsAfterDecPt)
{
if (scale > nDigitsAfterDecPt)
memcpy(data + nDigitsBeforeDecPt,
field + nDigitsBeforeDecPt + 1,
nDigitsAfterDecPt);
else // (scale <= nDigitsAfterDecPt)
memcpy(data + nDigitsBeforeDecPt,
field + nDigitsBeforeDecPt + 1,
scale);
}
// Add on any necessary zero padding at the end
if (scale > nDigitsAfterDecPt)
{
memset(data + nDigitsBeforeDecPt + nDigitsAfterDecPt,
'0',
scale - nDigitsAfterDecPt);
}
data[nDigitsBeforeDecPt + scale] = '\0';
// Convert our constructed decimal string back to a long long
//@bug 1814 Force strtoll to use base 10
errno = 0;
llVal = strtoll(data, 0, 10);
//@bug 3405 round off decimal values
if ((roundUp) && (errno == 0))
llVal += roundUp;
return llVal;
}
/*******************************************************************************
* DESCRIPTION:
* Convert an oid to a filename (with partition and segment number
* in the filepath.
* PARAMETERS:
* fid - fid
* fullFileName - file name
* dbDirName - components of fullFileName
* partition - partition number to be in used in filepath
* segment - segment number to be used in filename
* RETURN:
* NO_ERROR if success, other if fail
******************************************************************************/
/* static */
int Convertor::oid2FileName(FID fid,
char* fullFileName,
char dbDirName[][MAX_DB_DIR_NAME_SIZE],
uint32_t partition,
uint16_t segment)
{
dmFilePathArgs_t args;
int rc;
char aBuff[MAX_DB_DIR_NAME_SIZE];
char bBuff[MAX_DB_DIR_NAME_SIZE];
char cBuff[MAX_DB_DIR_NAME_SIZE];
char dBuff[MAX_DB_DIR_NAME_SIZE];
char eBuff[MAX_DB_DIR_NAME_SIZE];
char fnBuff[MAX_DB_DIR_NAME_SIZE];
args.pDirA = aBuff;
args.pDirB = bBuff;
args.pDirC = cBuff;
args.pDirD = dBuff;
args.pDirE = eBuff;
args.pFName = fnBuff;
args.ALen = sizeof(aBuff);
args.BLen = sizeof(bBuff);
args.CLen = sizeof(cBuff);
args.DLen = sizeof(dBuff);
args.ELen = sizeof(eBuff);
args.FNLen = sizeof(fnBuff);
args.Arc = 0;
args.Brc = 0;
args.Crc = 0;
args.Drc = 0;
args.Erc = 0;
args.FNrc = 0;
RETURN_ON_WE_ERROR(
(rc = dmOid2FPath(fid, partition, segment, &args)),
ERR_DM_CONVERT_OID);
sprintf(fullFileName, "%s/%s/%s/%s/%s/%s", args.pDirA,
args.pDirB, args.pDirC, args.pDirD, args.pDirE, args.pFName);
strcpy(dbDirName[0], args.pDirA);
strcpy(dbDirName[1], args.pDirB);
strcpy(dbDirName[2], args.pDirC);
strcpy(dbDirName[3], args.pDirD);
strcpy(dbDirName[4], args.pDirE);
strcpy(dbDirName[5], args.pFName);
// std::cout << "OID: " << fid <<
// " mapping to file: " << fullFileName <<std::endl;
return NO_ERROR;
}
/*******************************************************************************
* DESCRIPTION:
* Map specified errno to the associated error message string.
* PARAMETERS:
* errNum - errno to be converted
* errString-(output) error message string associated with errNum
* RETURN:
* none
******************************************************************************/
/* static */
void Convertor::mapErrnoToString(int errNum, std::string& errString)
{
char errnoMsgBuf[1024];
#if STRERROR_R_CHAR_P
char* errnoMsg = strerror_r(errNum, errnoMsgBuf, sizeof(errnoMsgBuf));
if (errnoMsg)
errString = errnoMsg;
else
errString.clear();
#else
int errnoMsg = strerror_r(errNum, errnoMsgBuf, sizeof(errnoMsgBuf));
if (errnoMsg == 0)
errString = errnoMsgBuf;
else
errString.clear();
#endif
}
/*******************************************************************************
* DESCRIPTION:
* Convert specified ColDataType to internal storage type (ColType).
* PARAMETERS:
* dataType - Interface data-type
* internalType - Internal data-type used for storing
* RETURN:
* none
******************************************************************************/
/* static */
void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType,
ColType& internalType, bool isToken)
{
if (isToken)
{
internalType = WriteEngine::WR_TOKEN;
return;
}
switch(dataType) {
// Map BIT and TINYINT to WR_BYTE
case CalpontSystemCatalog::BIT :
case CalpontSystemCatalog::TINYINT :
internalType = WriteEngine::WR_BYTE; break;
// Map SMALLINT to WR_SHORT
case CalpontSystemCatalog::SMALLINT :
internalType = WriteEngine::WR_SHORT; break;
// Map MEDINT, INT, and DATE to WR_INT
case CalpontSystemCatalog::MEDINT :
case CalpontSystemCatalog::INT :
case CalpontSystemCatalog::DATE :
internalType = WriteEngine::WR_INT; break;
// Map FLOAT and UFLOAT to WR_FLOAT
case CalpontSystemCatalog::FLOAT :
case CalpontSystemCatalog::UFLOAT:
internalType = WriteEngine::WR_FLOAT; break;
// Map BIGINT and DATETIME to WR_LONGLONG
case CalpontSystemCatalog::BIGINT :
case CalpontSystemCatalog::DATETIME :
internalType = WriteEngine::WR_LONGLONG; break;
// Map DOUBLE and UDOUBLE to WR_DOUBLE
case CalpontSystemCatalog::DOUBLE :
case CalpontSystemCatalog::UDOUBLE:
internalType = WriteEngine::WR_DOUBLE; break;
// Map BLOB to WR_BLOB
case CalpontSystemCatalog::BLOB :
internalType = WriteEngine::WR_BLOB; break;
// Map VARBINARY to WR_VARBINARY
case CalpontSystemCatalog::VARBINARY:
internalType = WriteEngine::WR_VARBINARY; break;
// Map DECIMAL to applicable WR_CHAR
// We can't map them to their proper int size, since in this version
// of convertColType(), we don't know what that is. Hopefully
// this is never used for DECIMAL.
case CalpontSystemCatalog::DECIMAL :
case CalpontSystemCatalog::UDECIMAL:
// Map CHAR, VARCHAR, and CLOB to WR_CHAR
case CalpontSystemCatalog::CHAR :
case CalpontSystemCatalog::VARCHAR :
case CalpontSystemCatalog::CLOB :
internalType = WriteEngine::WR_CHAR; break;
// Map UTINYINT to WR_UBYTE
case CalpontSystemCatalog::UTINYINT:
internalType = WriteEngine::WR_UBYTE; break;
// Map USMALLINT to WR_USHORT
case CalpontSystemCatalog::USMALLINT:
internalType = WriteEngine::WR_USHORT; break;
// Map UMEDINT and UINT to WR_UINT
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
internalType = WriteEngine::WR_UINT; break;
// Map UBIGINT to WR_ULONGLONG
case CalpontSystemCatalog::UBIGINT:
internalType = WriteEngine::WR_ULONGLONG; break;
default:
internalType = WriteEngine::WR_CHAR; break;
}
}
/*******************************************************************************
* DESCRIPTION:
* Convert specified internal storage type (ColType) to ColDataType. Since
* there is a one to many relationship, we choose the most general.
* PARAMETERS:
* internalType - Internal data-type used for storing
* dataType - Interface data-type
* RETURN:
* none
******************************************************************************/
/* static */
void Convertor::convertWEColType(ColType internalType,
CalpontSystemCatalog::ColDataType& dataType)
{
switch(internalType)
{
// Map BIT and TINYINT to WR_BYTE
case WriteEngine::WR_BYTE :
dataType = CalpontSystemCatalog::TINYINT; break;
// Map SMALLINT to WR_SHORT
case WriteEngine::WR_SHORT :
dataType = CalpontSystemCatalog::SMALLINT; break;
// Map MEDINT, INT, and DATE to WR_INT
case WriteEngine::WR_INT :
dataType = CalpontSystemCatalog::INT; break;
// Map FLOAT and UFLOAT to WR_FLOAT
case WriteEngine::WR_FLOAT:
dataType = CalpontSystemCatalog::FLOAT; break;
// Map BIGINT and DATETIME to WR_LONGLONG
case WriteEngine::WR_LONGLONG :
dataType = CalpontSystemCatalog::BIGINT; break;
// Map DOUBLE and UDOUBLE to WR_DOUBLE
case WriteEngine::WR_DOUBLE :
dataType = CalpontSystemCatalog::DOUBLE; break;
// Map BLOB to WR_BLOB
case WriteEngine::WR_BLOB :
dataType = CalpontSystemCatalog::BLOB; break;
// Map VARBINARY to WR_VARBINARY
case WriteEngine::WR_VARBINARY:
dataType = CalpontSystemCatalog::VARBINARY; break;
// Map CHAR, VARCHAR, and CLOB to WR_CHAR
case WriteEngine::WR_CHAR :
dataType = CalpontSystemCatalog::CHAR; break;
// Map UTINYINT to WR_UBYTE
case WriteEngine::WR_UBYTE:
dataType = CalpontSystemCatalog::UTINYINT; break;
// Map USMALLINT to WR_USHORT
case WriteEngine::WR_USHORT:
dataType = CalpontSystemCatalog::USMALLINT; break;
// Map UMEDINT and UINT to WR_UINT
case WriteEngine::WR_UINT:
dataType = CalpontSystemCatalog::UINT; break;
// Map UBIGINT to WR_ULONGLONG
case WriteEngine::WR_ULONGLONG:
dataType = CalpontSystemCatalog::UBIGINT; break;
default:
dataType = CalpontSystemCatalog::CHAR; break;
}
}
/*******************************************************************************
* DESCRIPTION:
* Convert curStruct from an interface-type struct to an internal-type
* struct. curStruct is handled as a ColStruct.
* PARAMETERS:
* curStruct - column struct to be initialized
* RETURN:
* none
******************************************************************************/
/* static */
void Convertor::convertColType(ColStruct* curStruct)
{
CalpontSystemCatalog::ColDataType dataType // This will be updated later,
= CalpontSystemCatalog::CHAR; // CHAR used only for initialization.
ColType* internalType = NULL;
bool bTokenFlag = false;
int* width = NULL;
dataType = curStruct->colDataType;
internalType = &(curStruct->colType);
bTokenFlag = curStruct->tokenFlag;
width = &(curStruct->colWidth);
switch(dataType) {
// Map BIT and TINYINT to WR_BYTE
case CalpontSystemCatalog::BIT :
case CalpontSystemCatalog::TINYINT :
*internalType = WriteEngine::WR_BYTE; break;
// Map SMALLINT to WR_SHORT
case CalpontSystemCatalog::SMALLINT :
*internalType = WriteEngine::WR_SHORT; break;
// Map MEDINT, INT, and DATE to WR_INT
case CalpontSystemCatalog::MEDINT :
case CalpontSystemCatalog::INT :
case CalpontSystemCatalog::DATE :
*internalType = WriteEngine::WR_INT; break;
// Map FLOAT and UFLOAT to WR_FLOAT
case CalpontSystemCatalog::FLOAT :
case CalpontSystemCatalog::UFLOAT :
*internalType = WriteEngine::WR_FLOAT; break;
// Map BIGINT and DATETIME to WR_LONGLONG
case CalpontSystemCatalog::BIGINT :
case CalpontSystemCatalog::DATETIME :
*internalType = WriteEngine::WR_LONGLONG; break;
// Map DOUBLE and UDOUBLE to WR_DOUBLE
case CalpontSystemCatalog::DOUBLE :
case CalpontSystemCatalog::UDOUBLE :
*internalType = WriteEngine::WR_DOUBLE; break;
// Map DECIMAL to applicable integer type
case CalpontSystemCatalog::DECIMAL :
case CalpontSystemCatalog::UDECIMAL :
{
switch (*width)
{
case 1 : *internalType = WriteEngine::WR_BYTE; break;
case 2 : *internalType = WriteEngine::WR_SHORT; break;
case 4 : *internalType = WriteEngine::WR_INT; break;
default: *internalType = WriteEngine::WR_LONGLONG; break;
}
break;
}
// Map BLOB to WR_BLOB
case CalpontSystemCatalog::BLOB :
*internalType = WriteEngine::WR_BLOB; break;
// Map VARBINARY to WR_VARBINARY
case CalpontSystemCatalog::VARBINARY:
*internalType = WriteEngine::WR_VARBINARY; break;
// Map CHAR, VARCHAR, and CLOB to WR_CHAR
case CalpontSystemCatalog::CHAR :
case CalpontSystemCatalog::VARCHAR :
case CalpontSystemCatalog::CLOB :
*internalType = WriteEngine::WR_CHAR; break;
// Map UTINYINT to WR_UBYTE
case CalpontSystemCatalog::UTINYINT:
*internalType = WriteEngine::WR_UBYTE; break;
// Map USMALLINT to WR_USHORT
case CalpontSystemCatalog::USMALLINT:
*internalType = WriteEngine::WR_USHORT; break;
// Map UMEDINT and UINT to WR_UINT
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
*internalType = WriteEngine::WR_UINT; break;
// Map UBIGINT to WR_ULONGLONG
case CalpontSystemCatalog::UBIGINT:
*internalType = WriteEngine::WR_ULONGLONG; break;
default:
*internalType = WriteEngine::WR_CHAR; break;
}
if (bTokenFlag) // token overwrite any other types
*internalType = WriteEngine::WR_TOKEN;
// check whether width is in sync with the requirement
*width = getCorrectRowWidth(dataType, *width);
// This is the patch for the decimal thing, override
// if (dataType == CalpontSystemCatalog::DECIMAL)
// {
// *internalType = *width <= 4 ?
// WriteEngine::WR_INT : WriteEngine::WR_LONGLONG;
// }
}
/*******************************************************************************
* DESCRIPTION:
* Get the correct width for a row
* PARAMETERS:
* dataType - data type
* width - data width in byte
* RETURN:
* emptyVal - the value of empty row
******************************************************************************/
/* static */
int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, int width)
{
int offset, newWidth = 4;
switch(dataType) {
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::UTINYINT:
newWidth = 1; break;
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::USMALLINT:
newWidth = 2; break;
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
newWidth = 4; break;
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::UBIGINT:
newWidth = 8; break;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
newWidth = 4; break;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
newWidth = 8; break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (width == 1)
newWidth = 1;
else if (width == 2)
newWidth = 2;
else if (width <= 4)
newWidth = 4;
else
newWidth = 8;
break;
case CalpontSystemCatalog::DATE:
newWidth = 4; break;
case CalpontSystemCatalog::DATETIME:
newWidth = 8; break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::VARBINARY: // treat same as varchar for now
default:
offset = (dataType == CalpontSystemCatalog::VARCHAR)? -1 : 0;
newWidth = 1;
if (width == (2 + offset))
newWidth = 2;
else if (width >= (3 + offset) && width <= (4 + offset))
newWidth = 4;
else if (width >= (5 + offset))
newWidth = 8;
break;
}
return newWidth;
}
/*******************************************************************************
* DESCRIPTION:
* Converts an OID into a group if directories and a filename.
*
* This function takes a 32-bit Object ID (OID). If DLen is 0, then the
* OID is converted into 3 hierarchical directory names and a filename.
* If DLen is >0 then the OID is converted into 5 hierarchical directory
* names and a filename (using the partition and segment as additional
* input into the filepath. The actual location
* of the file is <DBRoot>/<DirA>/<DirB>/<DirC>/<FName>, or
* <DBRoot>/<DirA>/<DirB>/<DirC>/<DirD>/<part#>/<segFName>. The <DBRoot>
* entry must be pre-pended by the calling application after calling
* this function. The value for <DBRoot> is stored in the Calpont.xml
* configuration file.
*
* PARAMETERS:
* oid INPUT -- The Object Id.
* partition INPUT -- partition to be included in filepath.
* segment INPUT -- segment to be included in filepath.
* dmFilePathArgs* INPUT/OUTPUT -- Points to a buffer structure
*
* RETURN:
* return 0 if everything went OK. -1 if an error occured. Two
* kinds of errors are possible:
*
* - a null pointer was passed in
* - truncation occured.
*
* If a null buffer pointer is passed in, a return code
* of -1 will be returned FOR THAT BUFFER.
*
* Truncation can occur if the buffer length specified in
* dmFilePathArgs is too small.
*
* If a buffer's return code is not zero, the appropriate
* return code in dmfilePathArgs can be examined. If a
* buffer's return code is be less than zero, the
* corresponding buffer pointer was NULL. If it is greater
* or equal to the buffer's length argument, length is too small
******************************************************************************/
/*static*/
int Convertor::dmOid2FPath(uint32_t oid, uint32_t partition, uint32_t segment,
dmFilePathArgs_t* pArgs)
{
pArgs->Arc = _doDir(
pArgs->pDirA,
pArgs->ALen,
(unsigned int)oid>>24);
pArgs->Brc = _doDir(
pArgs->pDirB,
pArgs->BLen,
(unsigned int)(oid&0x00ff0000)>>16);
pArgs->Crc = _doDir(
pArgs->pDirC,
pArgs->CLen,
(unsigned int)(oid&0x0000ff00)>>8);
// include partition and seg num in the file path if they are present
if (pArgs->DLen > 0)
{
pArgs->Drc = _doDir(
pArgs->pDirD,
pArgs->DLen,
(unsigned int)(oid&0x000000ff));
pArgs->Erc = _doDir(
pArgs->pDirE,
pArgs->ELen,
partition);
pArgs->FNrc = _doFile(
pArgs->pFName,
pArgs->FNLen,
segment);
if ( (pArgs->Drc < 0) ||
(pArgs->Erc < 0) )
return -1;
if ( (pArgs->Drc >= pArgs->ALen) ||
(pArgs->Erc >= pArgs->ALen) )
return -1;
}
else
{
pArgs->FNrc = _doFile(
pArgs->pFName,
pArgs->FNLen,
(unsigned int)(oid&0x000000ff));
}
if ( (pArgs->Arc < 0) ||
(pArgs->Brc < 0) ||
(pArgs->Crc < 0) ||
(pArgs->FNrc < 0) )
return -1;
if ( (pArgs->Arc >= pArgs->ALen) ||
(pArgs->Brc >= pArgs->BLen) ||
(pArgs->Crc >= pArgs->CLen) ||
(pArgs->FNrc >= pArgs->FNLen) )
return -1;
else
return 0;
}
} //end of namespace

148
writeengine/shared/we_convertor.h Normal file
View File

@ -0,0 +1,148 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_convertor.h 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
/** @file */
#ifndef _WE_CONVERTOR_H_
#define _WE_CONVERTOR_H_
#include <time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <string>
#include <iostream>
#include <fstream>
#include "we_obj.h"
#include "we_config.h"
#include "calpontsystemcatalog.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
/** Class Convertor */
class Convertor
{
public:
/**
* @brief Default Constructor
*/
Convertor(){}
/**
* @brief Destructor
*/
~Convertor(){}
/**
* @brief Get date/time string based on current date and time
*/
EXPORT static const std::string getTimeStr();
/**
* @brief Convert specified integer value to a string
*
* @param val Integer value to be converted to a string
*/
EXPORT static const std::string int2Str(int val);
/**
* @brief Convert an oid to a full file name (with partition and segment
* being included in the filename). This is used for all column and
* dictionary store db files. If dealing with a version buffer file,
* a partition and segment number of 0 should be used.
*/
EXPORT static int oid2FileName(FID fid, char* fullFileName,
char dbDirName[][MAX_DB_DIR_NAME_SIZE],
uint32_t partition, uint16_t segment);
/**
* @brief Convert specified errno to associated error msg string
*
* @param errNum System errno to be converted.
* @param errString Error msg string associated with the specified errno.
*/
EXPORT static void mapErrnoToString(int errNum, std::string& errString);
/**
* @brief Convert specified ColDataType to internal storage type (ColType)
*
* @param dataType Interface data-type
* @param internalType Internal data-type used for storing
*/
//BUG931
EXPORT static void convertColType(execplan::CalpontSystemCatalog::ColDataType dataType,
ColType& internalType, bool isToken=false);
/**
* @brief Convert specified internal storage type (ColType) to
* ColDataType
*
* @param internalType Internal data-type used for storing
* @param dataType Interface data-type
*/
EXPORT static void convertWEColType(ColType internalType,
execplan::CalpontSystemCatalog::ColDataType& dataType);
/**
* @brief Convert interface column type to a internal column type.
* curStruct is interpreted as a ColStruct.
*/
EXPORT static void convertColType(ColStruct* curStruct);
/*
* @brief Get the correct width for a row
*/
EXPORT static int getCorrectRowWidth( execplan::CalpontSystemCatalog::ColDataType dataType, int width );
/*
* @brief Convert a Decimal string to it's equivalent integer value.
* errno can be checked upon return to see if input value was
* out of range (ERANGE).
*
* field decimal string to be converted
* fieldLengh length of "field" in bytes
* scale decimal scale to be applied to value
*/
EXPORT static long long convertDecimalString ( const char* field,
int fieldLength,
int scale );
private:
struct dmFilePathArgs_t;
static int dmOid2FPath(uint32_t oid, uint32_t partition, uint32_t segment,
dmFilePathArgs_t* pArgs);
};
} //end of namespace
#undef EXPORT
#endif // _WE_CONVERTOR_H_

507
writeengine/shared/we_dbfileop.cpp Normal file
View File

@ -0,0 +1,507 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: we_dbfileop.cpp 4737 2013-08-14 20:45:46Z bwilkinson $
/** @file */
#include <unistd.h>
#include <stdio.h>
#include <cstring>
using namespace std;
#include "we_chunkmanager.h"
#include "we_dbfileop.h"
#include "we_stats.h"
#include "IDBDataFile.h"
using namespace idbdatafile;
using namespace BRM;
namespace WriteEngine
{
/**
* Constructor
*/
DbFileOp::DbFileOp() : m_chunkManager(NULL)
{}
/**
* Default Destructor
*/
DbFileOp::~DbFileOp()
{}
/***********************************************************
* DESCRIPTION:
* flush the cache
* PARAMETERS:
* none
* RETURN:
* NO_ERROR if success, otherwise if fail
***********************************************************/
int DbFileOp::flushCache()
{
BlockBuffer* curBuf;
if( !Cache::getUseCache() )
return NO_ERROR;
for( CacheMapIt it = Cache::m_writeList->begin();
it != Cache::m_writeList->end(); it++ ) {
curBuf = it->second;
RETURN_ON_ERROR( writeDBFile( (*curBuf).cb.file.pFile,
(*curBuf).block.data,
(*curBuf).block.lbid ) );
}
RETURN_ON_ERROR( Cache::flushCache() );
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
* get an entry within a sub block
* NOTE: the difference with readSubBlockEntry is that
* getSubBlockEntry only works for buffer while
* readSubBlockEntry works for file and block
* PARAMETERS:
* blockBuf - the block buffer
* sbid - sub block id
* entryNo - entry no within sub block
* width - width in bytes
* pStruct - sturcture pointer
* RETURN:
* none
***********************************************************/
void DbFileOp::getSubBlockEntry( unsigned char* blockBuf,
const int sbid, const int entryNo,
const int width, void* pStruct )
{
unsigned char* pBlock;
pBlock = blockBuf + BYTE_PER_SUBBLOCK * sbid + entryNo *MAX_COLUMN_BOUNDARY;
memcpy( pStruct, pBlock, width );
}
/***********************************************************
* DESCRIPTION:
* Read a block from a file at specified location
* PARAMETERS:
* pFile - file handle
* readBuf - read buffer
* fbo - file block offset
* RETURN:
* NO_ERROR if success
* other number if something wrong
***********************************************************/
int DbFileOp::readDBFile( IDBDataFile* pFile,
unsigned char* readBuf,
const uint64_t lbid,
const bool isFbo )
{
long long fboOffset = 0;
if( !isFbo ) {
RETURN_ON_ERROR( setFileOffsetBlock( pFile, lbid ) );
}
else {
fboOffset = (lbid)*(long)BYTE_PER_BLOCK;
RETURN_ON_ERROR( setFileOffset( pFile, fboOffset ) );
}
return readFile( pFile, readBuf, BYTE_PER_BLOCK );
}
int DbFileOp::readDBFile( IDBDataFile* pFile,
DataBlock* block,
const uint64_t lbid,
const bool isFbo )
{
block->dirty = false;
block->no = lbid;
Stats::incIoBlockRead();
return readDBFile( pFile, block->data, lbid, isFbo );
}
int DbFileOp::readDBFile( CommBlock& cb,
unsigned char* readBuf,
const uint64_t lbid )
{
CacheKey key;
if( Cache::getUseCache() )
{
if( Cache::cacheKeyExist( cb.file.oid, lbid ) ) {
key = Cache::getCacheKey( cb.file.oid, lbid );
RETURN_ON_ERROR( Cache::loadCacheBlock( key, readBuf ) );
return NO_ERROR;
}
}
RETURN_ON_ERROR( readDBFile( cb.file.pFile, readBuf, lbid ) );
if( Cache::getUseCache() )
{
int fbo = lbid;
uint16_t dbRoot;
uint32_t partition;
uint16_t segment;
RETURN_ON_ERROR( BRMWrapper::getInstance()->getFboOffset(
lbid, dbRoot, partition, segment, fbo ) );
if( Cache::getListSize( FREE_LIST ) == 0 ) {
if ( isDebug( DEBUG_1 ) ) {
printf( "\nBefore flushing cache " );
Cache::printCacheList();
}
// flush cache to give up more space
RETURN_ON_ERROR( flushCache() );
if ( isDebug( DEBUG_1 ) ) {
printf( "\nAfter flushing cache " );
Cache::printCacheList();
}
}
RETURN_ON_ERROR( Cache::insertLRUList( cb, lbid, fbo, readBuf ) );
}
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION: No change, old signature 10/17/06
* Read an entry within a sub block from a file
* NOTE: the difference with getSubBlockEntry is that
* getSubBlockEntry only works for buffer while
* readSubBlockEntry works for file and block
* PARAMETERS:
* pFile - file handler
* block - the block structure
* fbo - file block offset
* sbid - sub block id
* entryNo - entry no within sub block
* width - width in bytes
* pStruct - sturcture pointer
* RETURN:
* NO_ERROR if success
* other number if something wrong
***********************************************************/
const int DbFileOp::readSubBlockEntry( IDBDataFile* pFile, DataBlock* block,
const uint64_t lbid, const int sbid,
const int entryNo, const int width,
void* pStruct )
{
RETURN_ON_ERROR( readDBFile( pFile, block->data, lbid ) );
getSubBlockEntry( block->data, sbid, entryNo, width, pStruct );
return NO_ERROR;
}
const int DbFileOp::readSubBlockEntry( CommBlock& cb, DataBlock* block,
const uint64_t lbid, const int sbid,
const int entryNo, const int width,
void* pStruct )
{
RETURN_ON_ERROR( readDBFile( cb, block->data, lbid ) );
getSubBlockEntry( block->data, sbid, entryNo, width, pStruct );
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
* Set an entry within a sub block
* NOTE: the difference with writeSubBlockEntry is that
* setSubBlockEntry only works for buffer while
* writeSubBlockEntry works for file and block
* PARAMETERS:
* blockBuf - the block buffer
* sbid - sub block id
* entryNo - entry no within sub block
* width - width in bytes
* pStruct - sturcture pointer
* RETURN:
* none
***********************************************************/
void DbFileOp::setSubBlockEntry( unsigned char* blockBuf, const int sbid,
const int entryNo, const int width,
const void* pStruct )
{
unsigned char* pBlock;
pBlock = blockBuf + BYTE_PER_SUBBLOCK * sbid + entryNo *MAX_COLUMN_BOUNDARY;
memcpy( pBlock, pStruct, width );
}
/***********************************************************
* DESCRIPTION:
* Write a number of blocks to the file at specified location
* PARAMETERS:
* pFile - file handle
* writeBuf - write buffer
* fbo - file block offset
* numOfBlock - total number of file block offset
* RETURN:
* NO_ERROR if success
* other number if something wrong
***********************************************************/
int DbFileOp::writeDBFile( CommBlock& cb, const unsigned char* writeBuf,
const uint64_t lbid, const int numOfBlock )
{
CacheKey key;
int ret;
if( Cache::getUseCache() )
{
if( Cache::cacheKeyExist( cb.file.oid, lbid ) ) {
key = Cache::getCacheKey( cb.file.oid, lbid );
RETURN_ON_ERROR( Cache::modifyCacheBlock( key, writeBuf ) );
return NO_ERROR;
}
}
if (BRMWrapper::getUseVb())
{
RETURN_ON_ERROR( writeVB( cb.file.pFile, cb.file.oid, lbid ) );
}
ret = writeDBFile( cb.file.pFile, writeBuf, lbid, numOfBlock );
if (BRMWrapper::getUseVb())
{
LBIDRange_v ranges;
LBIDRange range;
range.start = lbid;
range.size = 1;
ranges.push_back(range);
BRMWrapper::getInstance()->writeVBEnd(getTransId(), ranges);
}
return ret;
}
int DbFileOp::writeDBFileNoVBCache(CommBlock & cb,
const unsigned char * writeBuf,
const int fbo,
const int numOfBlock)
{
return writeDBFileNoVBCache( cb.file.pFile, writeBuf, fbo, numOfBlock );
}
/***********************************************************
* DESCRIPTION:
* Core function for writing data w/o using VB cache
* (bulk load dictionary store inserts)
***********************************************************/
int DbFileOp::writeDBFileNoVBCache( IDBDataFile* pFile,
const unsigned char* writeBuf,
const int fbo,
const int numOfBlock )
{
#ifdef PROFILE
// This function is only used by bulk load for dictionary store files,
// so we log as such.
Stats::startParseEvent(WE_STATS_WRITE_DCT);
#endif
for( int i = 0; i < numOfBlock; i++ ) {
Stats::incIoBlockWrite();
RETURN_ON_ERROR( writeFile( pFile, writeBuf, BYTE_PER_BLOCK ) );
}
#ifdef PROFILE
Stats::stopParseEvent(WE_STATS_WRITE_DCT);
#endif
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
* Core function for writing data using VB cache
***********************************************************/
int DbFileOp::writeDBFile( IDBDataFile* pFile, const unsigned char* writeBuf,
const uint64_t lbid, const int numOfBlock )
{
RETURN_ON_ERROR( setFileOffsetBlock( pFile, lbid ) );
for( int i = 0; i < numOfBlock; i++ ) {
Stats::incIoBlockWrite();
RETURN_ON_ERROR( writeFile( pFile, writeBuf, BYTE_PER_BLOCK ) );
}
return NO_ERROR;
}
// just don't have a good solution to consolidate with above functions
// Note: This is used with absolute FBO, no lbid involved
int DbFileOp::writeDBFileFbo(IDBDataFile* pFile, const unsigned char* writeBuf,
const uint64_t fbo, const int numOfBlock )
{
long long fboOffset = 0;
fboOffset = (fbo)*(long)BYTE_PER_BLOCK;
RETURN_ON_ERROR( setFileOffset( pFile, fboOffset ) );
for( int i = 0; i < numOfBlock; i++ ) {
Stats::incIoBlockWrite();
RETURN_ON_ERROR( writeFile( pFile, writeBuf, BYTE_PER_BLOCK ) );
}
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
* Write an entry within a sub block to a file
* NOTE: the difference with getSubBlockEntry is that
* setSubBlockEntry only works for buffer while
* writeSubBlockEntry works for file and block
* PARAMETERS:
* pFile - file handler
* block - the block structure
* fbo - file block offset
* sbid - sub block id
* entryNo - entry no within sub block
* width - width in bytes
* pStruct - sturcture pointer
* RETURN:
* NO_ERROR if success
* other number if something wrong
***********************************************************/
const int DbFileOp::writeSubBlockEntry( IDBDataFile* pFile, DataBlock* block,
const uint64_t lbid, const int sbid,
const int entryNo, const int width,
void* pStruct )
{
setSubBlockEntry( block->data, sbid, entryNo, width, pStruct );
block->dirty = false;
return writeDBFile( pFile, block->data, lbid );
}
const int DbFileOp::writeSubBlockEntry( CommBlock& cb, DataBlock* block,
const uint64_t lbid, const int sbid,
const int entryNo, const int width,
void* pStruct )
{
setSubBlockEntry( block->data, sbid, entryNo, width, pStruct );
block->dirty = false;
return writeDBFile( cb, block->data, lbid );
}
/***********************************************************
* DESCRIPTION:
* Write to version buffer
* PARAMETERS:
* oid - file oid
* lbid - lbid
* RETURN:
* NO_ERROR if success
* other number if something wrong
***********************************************************/
const int DbFileOp::writeVB( IDBDataFile* pFile, const OID oid, const uint64_t lbid )
{
if( !BRMWrapper::getUseVb() )
return NO_ERROR;
int rc;
TxnID transId = getTransId();
if (transId !=((TxnID)INVALID_NUM))
{
rc= BRMWrapper::getInstance()->writeVB( pFile,
(const VER_t)transId,
oid, lbid, this );
//@Bug 4671. The error is already logged by worker node.
/* if (rc != NO_ERROR)
{
char msg[2048];
snprintf(msg, 2048,
"we_dbfileop->BRMWrapper::getInstance()->writeVB "
"transId %i oid %i lbid "
#if __LP64__
"%lu"
#else
"%llu"
#endif
" Error Code %i", transId, oid, lbid, rc);
puts(msg);
{
logging::MessageLog ml(logging::LoggingID(19));
logging::Message m;
logging::Message::Args args;
args.add(msg);
m.format(args);
ml.logCriticalMessage(m);
}
return rc;
} */
return rc;
}
return NO_ERROR;
}
int DbFileOp::readDbBlocks(IDBDataFile* pFile,
unsigned char* readBuf,
uint64_t fbo,
size_t n)
{
if (m_chunkManager) {
return m_chunkManager->readBlocks(pFile, readBuf, fbo, n);
}
if (setFileOffset(pFile, fbo*BYTE_PER_BLOCK, SEEK_SET) != NO_ERROR)
return -1;
return pFile->read(readBuf, BYTE_PER_BLOCK * n) / BYTE_PER_BLOCK;
}
int DbFileOp::restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo)
{
if (m_chunkManager)
return m_chunkManager->restoreBlock(pFile, writeBuf, fbo);
if (setFileOffset(pFile, fbo*BYTE_PER_BLOCK, SEEK_SET) != NO_ERROR)
return -1;
return pFile->write(writeBuf, BYTE_PER_BLOCK);
}
// @bug 5572 - HDFS usage: add *.tmp file backup flag
IDBDataFile* DbFileOp::getFilePtr(const Column& column, bool useTmpSuffix)
{
string filename;
return m_chunkManager->getFilePtr(column,
column.dataFile.fDbRoot,
column.dataFile.fPartition,
column.dataFile.fSegment,
filename,
"r+b",
column.colWidth,
useTmpSuffix);
}
} //end of namespace

235
writeengine/shared/we_dbfileop.h Normal file
View File

@ -0,0 +1,235 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: we_dbfileop.h 4726 2013-08-07 03:38:36Z bwilkinson $
/** @file */
#ifndef _WE_DBFILEOP_H_
#define _WE_DBFILEOP_H_
#include "we_type.h"
#include "we_fileop.h"
#include "we_blockop.h"
#include "we_cache.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
// forward reference
class ChunkManager;
/** Class DbFileOp */
class DbFileOp : public FileOp
{
public:
/**
* @brief Constructor
*/
EXPORT DbFileOp();
/**
* @brief Default Destructor
*/
EXPORT virtual ~DbFileOp();
EXPORT virtual int flushCache();
/**
* @brief Get an entry within a subblock
*/
EXPORT void getSubBlockEntry( unsigned char* blockBuf,
const int sbid,
const int entryNo,
const int width,
void* pStruct ) ;
/**
* @brief Get an entry within a subblock using block information
*/
void getSubBlockEntry( DataBlock* block,
const int sbid,
const int entryNo,
const int width,
void* pStruct )
{ getSubBlockEntry( block->data, sbid, entryNo, width, pStruct );}
/**
* @brief Read DB file to a buffer
*/
EXPORT virtual int readDBFile( IDBDataFile* pFile,
unsigned char* readBuf,
const uint64_t lbid,
const bool isFbo = false );
EXPORT int readDBFile( CommBlock& cb,
unsigned char* readBuf,
const uint64_t lbid );
EXPORT int readDBFile( IDBDataFile* pFile,
DataBlock* block,
const uint64_t lbid,
const bool isFbo = false );
int readDBFile( CommBlock& cb,
DataBlock* block,
const uint64_t lbid )
{ return readDBFile( cb, block->data, lbid );}
/**
* @brief Get an entry within a subblock and also populate block buffer
*
*/
EXPORT const int readSubBlockEntry(IDBDataFile* pFile,
DataBlock* block,
const uint64_t lbid,
const int sbid,
const int entryNo,
const int width,
void* pStruct ) ;
EXPORT const int readSubBlockEntry(CommBlock& cb,
DataBlock* block,
const uint64_t lbid,
const int sbid,
const int entryNo,
const int width,
void* pStruct );
/**
* @brief Set an entry within a subblock
*/
EXPORT void setSubBlockEntry( unsigned char* blockBuf,
const int sbid,
const int entryNo,
const int width,
const void* pStruct ) ;
/**
* @brief Set an entry within a subblock using block information
*/
void setSubBlockEntry( DataBlock* block,
const int sbid,
const int entryNo,
const int width,
const void* pStruct )
{ block->dirty = true;
setSubBlockEntry( block->data, sbid, entryNo, width, pStruct ); }
/**
* @brief Lbid Write a buffer to a DB file
*/
EXPORT virtual int writeDBFile( IDBDataFile* pFile,
const unsigned char* writeBuf,
const uint64_t lbid,
const int numOfBlock = 1 );
EXPORT int writeDBFile( CommBlock& cb,
const unsigned char* writeBuf,
const uint64_t lbid,
const int numOfBlock = 1 );
/**
* @brief Write designated block(s) w/o writing to Version Buffer or cache.
*/
EXPORT int writeDBFileNoVBCache(CommBlock & cb,
const unsigned char * writeBuf,
const int fbo,
const int numOfBlock = 1);
EXPORT virtual int writeDBFileNoVBCache(IDBDataFile *pFile,
const unsigned char * writeBuf,
const int fbo,
const int numOfBlock = 1);
int writeDBFile( IDBDataFile* pFile,
DataBlock* block,
const uint64_t lbid )
{ block->dirty=false; return writeDBFile( pFile, block->data, lbid ); }
int writeDBFile( CommBlock& cb,
DataBlock* block,
const uint64_t lbid )
{ return writeDBFile( cb, block->data, lbid ); }
EXPORT virtual int writeDBFileFbo( IDBDataFile* pFile,
const unsigned char* writeBuf,
const uint64_t fbo,
const int numOfBlock );
int writeDBFileNoVBCache(CommBlock & cb,
DataBlock * block,
const int fbo)
{ return writeDBFileNoVBCache(cb, block->data, fbo); }
/**
* @brief Write a sub block entry directly to a DB file
*/
EXPORT const int writeSubBlockEntry(IDBDataFile* pFile,
DataBlock* block,
const uint64_t lbid,
const int sbid,
const int entryNo,
const int width,
void* pStruct );
EXPORT const int writeSubBlockEntry(CommBlock& cb,
DataBlock* block,
const uint64_t lbid,
const int sbid,
const int entryNo,
const int width,
void* pStruct ) ;
/**
* @brief Write to version buffer
*/
EXPORT const int writeVB( IDBDataFile* pFile,
const OID oid,
const uint64_t lbid );
EXPORT virtual int readDbBlocks( IDBDataFile* pFile,
unsigned char* readBuf,
uint64_t fbo,
size_t n);
EXPORT virtual int restoreBlock( IDBDataFile* pFile,
const unsigned char* writeBuf,
uint64_t fbo);
EXPORT virtual IDBDataFile* getFilePtr(const Column& column,
bool useTmpSuffix);
virtual void chunkManager(ChunkManager* ptr) { m_chunkManager = ptr; }
virtual ChunkManager* chunkManager() { return m_chunkManager; }
protected:
ChunkManager* m_chunkManager;
private:
};
} //end of namespace
#undef EXPORT
#endif // _WE_DBFILEOP_H_

560
writeengine/shared/we_dbrootextenttracker.cpp Normal file
View File

@ -0,0 +1,560 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: we_dbrootextenttracker.cpp 4631 2013-05-02 15:21:09Z dcathey $
*/
#include "we_dbrootextenttracker.h"
#include <algorithm>
#include <sstream>
#include "we_brm.h"
#include "we_config.h"
#include "we_log.h"
namespace
{
const char* stateStrings[] = { "initState" ,
"PartialExtent" ,
"EmptyDbRoot" ,
"ExtentBoundary",
"OutOfService" };
}
namespace WriteEngine
{
//------------------------------------------------------------------------------
// DBRootExtentInfo constructor
//------------------------------------------------------------------------------
DBRootExtentInfo::DBRootExtentInfo(
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
BRM::LBID_t startLbid,
HWM localHwm,
uint64_t dbrootTotalBlocks,
DBRootExtentInfoState state) :
fPartition(partition),
fDbRoot(dbRoot),
fSegment(segment),
fStartLbid(startLbid),
fLocalHwm(localHwm),
fDBRootTotalBlocks(dbrootTotalBlocks),
fState(state)
{
}
//------------------------------------------------------------------------------
// LessThan operator used to sort DBRootExtentInfo objects by DBRoot.
//------------------------------------------------------------------------------
bool DBRootExtentInfo::operator<(
const DBRootExtentInfo& entry) const
{
if (fDbRoot < entry.fDbRoot)
return true;
return false;
}
//------------------------------------------------------------------------------
// DBRootExtentTracker constructor
//
// Mutex lock not needed in this function as it is only called from main thread
// before processing threads are spawned.
//------------------------------------------------------------------------------
DBRootExtentTracker::DBRootExtentTracker ( OID oid,
const std::vector<int>& colWidths,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoColVec,
unsigned int columnIdx,
Log* logger ) :
fOID(oid),
fLog(logger),
fCurrentDBRootIdx(-1),
fEmptyOrDisabledPM(false),
fEmptyPM(true),
fDisabledHWM(false)
{
const BRM::EmDbRootHWMInfo_v& emDbRootHWMInfo =
dbRootHWMInfoColVec[columnIdx];
int colWidth = colWidths[columnIdx];
fBlksPerExtent = (long long)BRMWrapper::getInstance()->getExtentRows() *
(long long)colWidth / (long long)BYTE_PER_BLOCK;
std::vector<bool> resetState;
for (unsigned int i=0; i<emDbRootHWMInfo.size(); i++)
{
resetState.push_back(false);
DBRootExtentInfoState state = determineState(
colWidths[columnIdx],
emDbRootHWMInfo[i].localHWM,
emDbRootHWMInfo[i].totalBlocks,
emDbRootHWMInfo[i].status);
// For a full extent...
// check to see if any of the column HWMs are partially full, in which
// case we consider all the columns for that DBRoot to be partially
// full. (This can happen if a table has columns with varying widths,
// as the HWM may be at the last extent block for a shorter column, and
// still have free blocks for wider columns.)
if (state == DBROOT_EXTENT_EXTENT_BOUNDARY)
{
for (unsigned int kCol=0; kCol<dbRootHWMInfoColVec.size(); kCol++)
{
const BRM::EmDbRootHWMInfo_v& emDbRootHWMInfo2 =
dbRootHWMInfoColVec[kCol];
DBRootExtentInfoState state2 = determineState(
colWidths[kCol],
emDbRootHWMInfo2[i].localHWM,
emDbRootHWMInfo2[i].totalBlocks,
emDbRootHWMInfo2[i].status);
if (state2 == DBROOT_EXTENT_PARTIAL_EXTENT)
{
state = DBROOT_EXTENT_PARTIAL_EXTENT;
resetState[ resetState.size()-1 ] = true;
break;
}
}
}
DBRootExtentInfo dbRootExtent(
emDbRootHWMInfo[i].dbRoot,
emDbRootHWMInfo[i].partitionNum,
emDbRootHWMInfo[i].segmentNum,
emDbRootHWMInfo[i].startLbid,
emDbRootHWMInfo[i].localHWM,
emDbRootHWMInfo[i].totalBlocks,
state);
fDBRootExtentList.push_back( dbRootExtent );
}
std::sort( fDBRootExtentList.begin(), fDBRootExtentList.end() );
if (fLog)
{
// Always log this info for now; may control with debug later
//if (fLog->isDebug(DEBUG_1))
{
std::ostringstream oss;
oss << "Starting DBRoot info for OID " << fOID;
for (unsigned int k=0; k<fDBRootExtentList.size(); k++)
{
oss << std::endl;
oss << " DBRoot-" << fDBRootExtentList[k].fDbRoot <<
", part/seg/hwm/LBID/totBlks/state: " <<
fDBRootExtentList[k].fPartition <<
"/" << fDBRootExtentList[k].fSegment <<
"/" << fDBRootExtentList[k].fLocalHwm <<
"/" << fDBRootExtentList[k].fStartLbid <<
"/" << fDBRootExtentList[k].fDBRootTotalBlocks <<
"/" << stateStrings[ fDBRootExtentList[k].fState ];
if (resetState[k])
oss << ".";
}
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
}
}
//------------------------------------------------------------------------------
// Determines the state of the HWM extent (for a DBRoot); considering the
// current BRM status, HWM, and total block count for the DBRoot.
//------------------------------------------------------------------------------
DBRootExtentInfoState DBRootExtentTracker::determineState(int colWidth,
HWM localHwm,
uint64_t dbRootTotalBlocks,
int16_t status)
{
DBRootExtentInfoState extentState;
if (status == BRM::EXTENTOUTOFSERVICE)
{
extentState = DBROOT_EXTENT_OUT_OF_SERVICE;
}
else
{
if (dbRootTotalBlocks == 0)
{
extentState = DBROOT_EXTENT_EMPTY_DBROOT;
}
else
{
extentState = DBROOT_EXTENT_PARTIAL_EXTENT;
// See if local hwm is on an extent bndry,in which case the extent
// is full and we won't be adding rows to the current HWM extent;
// we will instead need to allocate a new extent in order to begin
// adding any rows.
long long nRows= ((long long)(localHwm+1) *
(long long)BYTE_PER_BLOCK)/ (long long)colWidth;
long long nRem = nRows % BRMWrapper::getInstance()->getExtentRows();
if (nRem == 0)
{
extentState = DBROOT_EXTENT_EXTENT_BOUNDARY;
}
}
}
return extentState;
}
//------------------------------------------------------------------------------
// Select the first segment file to add rows to for the local PM.
// Function will first try to find the HWM extent with the fewest blocks.
// If all the HWM extents are full, then we select the DBRoot/segment file
// with the fewest total overall blocks for that DBRoot.
// Return value is 0 upon success, else non zero if no eligible entries found.
//
// Mutex lock not needed in this function as it is only called from main thread
// before processing threads are spawned.
//------------------------------------------------------------------------------
int DBRootExtentTracker::selectFirstSegFile(
DBRootExtentInfo& dbRootExtent, bool& bNoStartExtentOnThisPM,
bool& bEmptyPM,
std::string& errMsg )
{
int startExtentIdx = -1;
int fewestLocalBlocksIdx = -1; // track HWM extent with fewest blocks
int fewestTotalBlocksIdx = -1; // track DBRoot with fewest total blocks
bNoStartExtentOnThisPM = false;
unsigned int fewestTotalBlks = UINT_MAX;
unsigned int fewestLocalBlks = UINT_MAX;
uint16_t fewestTotalBlkSegNum = USHRT_MAX;
uint16_t fewestLocalBlkSegNum = USHRT_MAX;
// Find DBRoot having HWM extent with fewest blocks. If all HWM extents
// are full (remblks=0), then fall-back on selecting the DBRoot with fewest
// total blks.
//
// Selecting HWM extent with fewest blocks should be straight forward, be-
// cause all the DBRoots on a PM should end on an extent boundary except
// for the current last extent. But if the user has moved a DBRoot, then
// we can end up with 2 partially filled HWM extents on 2 DBRoots, on the
// same PM. That's why we loop through the DBRoots to see if we have more
// than 1 partially filled HWM extent.
for (unsigned int iroot=0;
iroot<fDBRootExtentList.size();
iroot++)
{
// Skip over DBRoots which have no extents
if (fDBRootExtentList[iroot].fState == DBROOT_EXTENT_EMPTY_DBROOT)
continue;
fEmptyPM = false;
// Find DBRoot and segment file with most incomplete extent.
// Break a tie by selecting the lowest segment number.
long long remBlks = (long long)(fDBRootExtentList[iroot].fLocalHwm + 1)%
fBlksPerExtent;
if (remBlks > 0)
{
if ( (remBlks < fewestLocalBlks) ||
((remBlks == fewestLocalBlks) &&
(fDBRootExtentList[iroot].fSegment < fewestLocalBlkSegNum)) )
{
fewestLocalBlocksIdx = iroot;
fewestLocalBlks = remBlks;
fewestLocalBlkSegNum = fDBRootExtentList[iroot].fSegment;
}
}
// Find DBRoot with fewest total of blocks.
// Break a tie by selecting the highest segment number.
if ( (fDBRootExtentList[iroot].fDBRootTotalBlocks < fewestTotalBlks) ||
((fDBRootExtentList[iroot].fDBRootTotalBlocks== fewestTotalBlks) &&
(fDBRootExtentList[iroot].fSegment > fewestTotalBlkSegNum)) )
{
fewestTotalBlocksIdx = iroot;
fewestTotalBlks = fDBRootExtentList[iroot].fDBRootTotalBlocks;
fewestTotalBlkSegNum = fDBRootExtentList[iroot].fSegment;
}
}
// Select HWM extent with fewest number of blocks;
// If chosen extent is disabled, then treat like an empty PM,
// meaning we have to allocate a new extent before adding any rows
if (fewestLocalBlocksIdx != -1)
{
startExtentIdx = fewestLocalBlocksIdx;
if (fDBRootExtentList[startExtentIdx].fState ==
DBROOT_EXTENT_OUT_OF_SERVICE)
{
fDisabledHWM = true;
}
}
// If the HWM on each DBRoot ends on an extent boundary, then
// select the DBRoot with the fewest total number of blocks;
// If chosen extent is disabled, then treat like an empty PM,
// meaning we have to allocate a new extent before adding any rows
else if (fewestTotalBlocksIdx != -1)
{
startExtentIdx = fewestTotalBlocksIdx;
if (fDBRootExtentList[startExtentIdx].fState ==
DBROOT_EXTENT_OUT_OF_SERVICE)
{
fDisabledHWM = true;
}
}
// PM with no extents (or all extents disabled), so select DBRoot/segment
// file from DBRoot list, where we will start inserting rows.
// Select lowest segment file number.
else
{
RETURN_ON_ERROR( selectFirstSegFileForEmptyPM( errMsg ) );
startExtentIdx = fCurrentDBRootIdx;
}
if ((fEmptyOrDisabledPM) || (fDisabledHWM))
bNoStartExtentOnThisPM = true;
bEmptyPM = fEmptyPM;
fCurrentDBRootIdx = startExtentIdx;
// Finish Initializing DBRootExtentList for empty DBRoots w/o any extents
initEmptyDBRoots( );
logFirstDBRootSelection( );
dbRootExtent = fDBRootExtentList[startExtentIdx];
fDBRootExtentList[startExtentIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY;
return NO_ERROR;
}
//------------------------------------------------------------------------------
// If we have encountered a PM with no extents (or all extent disabled), then
// this function can be called to determine the DBRoot to be used for the 1st
// extent for the applicable PM. First DBRoot for relevant PM is selected.
// At extent creation time, BRM will assign the segment number.
//
// Mutex lock not needed in this function as it is only called from main thread
// before processing threads are spawned.
//------------------------------------------------------------------------------
int DBRootExtentTracker::selectFirstSegFileForEmptyPM( std::string& errMsg )
{
fEmptyOrDisabledPM = true;
fCurrentDBRootIdx = 0; // Start with first DBRoot for this PM
// Always start empty PM with partition number 0. If the DBRoot has a HWM
// extent that is disabled, then BRM will override this partition number.
fDBRootExtentList[fCurrentDBRootIdx].fPartition = 0;
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Finish Initializing fDBRootExtentList for any empty DBRoots w/o any extents.
//------------------------------------------------------------------------------
void DBRootExtentTracker::initEmptyDBRoots( )
{
int startExtentIdx= fCurrentDBRootIdx;
bool bAnyChanges = false; // If fDBRootExtentList changes, log the contents
// Fill in starting partition for any DBRoots having no extents
for (unsigned int iroot=0;
iroot<fDBRootExtentList.size();
iroot++)
{
if ((fDBRootExtentList[iroot].fState == DBROOT_EXTENT_EMPTY_DBROOT) &&
((int)iroot != startExtentIdx)) // skip over selected dbroot
{
if (fDBRootExtentList[iroot].fPartition !=
fDBRootExtentList[startExtentIdx].fPartition)
{
bAnyChanges = true;
fDBRootExtentList[iroot].fPartition =
fDBRootExtentList[startExtentIdx].fPartition;
}
}
}
// Log fDBRootExtentList if modifications were made
if ((bAnyChanges) && (fLog))
{
// Always log this info for now; may control with debug later
//if (fLog->isDebug(DEBUG_1))
{
std::ostringstream oss;
oss << "Updated starting (empty) DBRoot info for OID " << fOID;
for (unsigned int k=0; k<fDBRootExtentList.size(); k++)
{
oss << std::endl;
oss << " DBRoot-" << fDBRootExtentList[k].fDbRoot <<
", part/seg/hwm/LBID/totBlks/state: " <<
fDBRootExtentList[k].fPartition <<
"/" << fDBRootExtentList[k].fSegment <<
"/" << fDBRootExtentList[k].fLocalHwm <<
"/" << fDBRootExtentList[k].fStartLbid <<
"/" << fDBRootExtentList[k].fDBRootTotalBlocks <<
"/" << stateStrings[ fDBRootExtentList[k].fState ];
}
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
}
}
//------------------------------------------------------------------------------
// Assign the DBRoot/segment file to be used for extent loading based on the
// setting in the specified reference tracker (set for a reference column).
//
// Mutex lock not needed in this function as it is only called from main thread
// before processing threads are spawned.
//------------------------------------------------------------------------------
void DBRootExtentTracker::assignFirstSegFile(
const DBRootExtentTracker& refTracker,
DBRootExtentInfo& dbRootExtent)
{
// Start with the same DBRoot index as the reference tracker; assumes that
// DBRoots for each column are listed in same order in fDBRootExtentList.
// That should be a safe assumption since DBRootExtentTracker constructor
// sorts the entries in fDBRootExtentList by fDbRoot.
int startExtentIdx = refTracker.fCurrentDBRootIdx;
fEmptyOrDisabledPM = refTracker.fEmptyOrDisabledPM;
fEmptyPM = refTracker.fEmptyPM;
fDisabledHWM = refTracker.fDisabledHWM;
// Always start empty PM with partition number 0. If the DBRoot has a HWM
// extent that is disabled, then BRM will override this partition number.
if (fEmptyOrDisabledPM)
{
fDBRootExtentList[startExtentIdx].fPartition = 0;
}
fCurrentDBRootIdx = startExtentIdx;
// Finish Initializing DBRootExtentList for empty DBRoots w/o any extents
initEmptyDBRoots( );
logFirstDBRootSelection( );
dbRootExtent = fDBRootExtentList[startExtentIdx];
fDBRootExtentList[startExtentIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY;
}
//------------------------------------------------------------------------------
// Log information about the first DBRoot/segment file that is selected.
//------------------------------------------------------------------------------
void DBRootExtentTracker::logFirstDBRootSelection( ) const
{
if (fLog)
{
int extentIdx = fCurrentDBRootIdx;
if (fEmptyOrDisabledPM)
{
std::ostringstream oss;
oss << "No active extents; will add partition to start adding "
"rows for oid-" << fOID <<
"; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
else if (fDisabledHWM)
{
std::ostringstream oss;
oss << "HWM extent disabled; will add partition to start adding "
"rows for oid-" << fOID <<
"; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
else
{
std::ostringstream oss;
oss<<"Selecting existing segFile to begin adding rows: oid-"<<fOID<<
"; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot <<
", part/seg/hwm/LBID/totBlks/state: " <<
fDBRootExtentList[extentIdx].fPartition <<
"/" << fDBRootExtentList[extentIdx].fSegment <<
"/" << fDBRootExtentList[extentIdx].fLocalHwm <<
"/" << fDBRootExtentList[extentIdx].fStartLbid <<
"/" << fDBRootExtentList[extentIdx].fDBRootTotalBlocks<<
"/" << stateStrings[ fDBRootExtentList[extentIdx].fState ];
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
}
}
//------------------------------------------------------------------------------
// Iterate/return next DBRoot to be used for the next extent
//
// If it is the "very" 1st extent for the specified DBRoot, then the applicable
// partition to be used for the first extent, is also returned.
// If a partial extent is to be filled in, then the current HWM
// and starting LBID for the relevant extent are returned.
// If a disabled extent is encountered, the return flag is true, so that
// we can allocate a new extent at the next physical partition. BRM should
// take care of figuring out where to allocate this next extent.
//
// Returns true if new extent needs to be allocated, else false indicates that
// the extent is partially full.
//------------------------------------------------------------------------------
bool DBRootExtentTracker::nextSegFile(
uint16_t& dbRoot,
uint32_t& partition,
uint16_t& segment,
HWM& localHwm,
BRM::LBID_t& startLbid)
{
boost::mutex::scoped_lock lock(fDBRootExtTrkMutex);
fCurrentDBRootIdx++;
if ((unsigned int)fCurrentDBRootIdx >= fDBRootExtentList.size())
fCurrentDBRootIdx = 0;
dbRoot = fDBRootExtentList[fCurrentDBRootIdx].fDbRoot;
segment = fDBRootExtentList[fCurrentDBRootIdx].fSegment;
partition = fDBRootExtentList[fCurrentDBRootIdx].fPartition;
localHwm = fDBRootExtentList[fCurrentDBRootIdx].fLocalHwm;
startLbid = fDBRootExtentList[fCurrentDBRootIdx].fStartLbid;
//std::cout << "NextSegFile: Current idx: " << fCurrentDBRootIdx <<
//"; new dbroot: " << dbRoot <<
//"; segment: " << segment <<
//"; partition: " << partition <<
//"; localHwm: " << localHwm <<
//"; startLbid: " << startLbid <<
//"; state: " << stateStrings[fDBRootExtentList[fCurrentDBRootIdx].fState]
// << std::endl;
bool bAllocExtentFlag = true;
if (fDBRootExtentList[fCurrentDBRootIdx].fState ==
DBROOT_EXTENT_PARTIAL_EXTENT)
bAllocExtentFlag = false;
// After we have taken care of the "first" extent for each DBRoot, we can
// zero out everything. The only thing we need to continue rotating thru
// the DBRoots, is the DBRoot number itself.
fDBRootExtentList[fCurrentDBRootIdx].fSegment = 0;
fDBRootExtentList[fCurrentDBRootIdx].fPartition = 0;
fDBRootExtentList[fCurrentDBRootIdx].fLocalHwm = 0;
fDBRootExtentList[fCurrentDBRootIdx].fStartLbid = 0;
fDBRootExtentList[fCurrentDBRootIdx].fDBRootTotalBlocks = 0;
fDBRootExtentList[fCurrentDBRootIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY;
return bAllocExtentFlag;
}
const std::vector<DBRootExtentInfo>& DBRootExtentTracker::getDBRootExtentList()
{
boost::mutex::scoped_lock lock(fDBRootExtTrkMutex);
return fDBRootExtentList;
}
} // end of namespace

240
writeengine/shared/we_dbrootextenttracker.h Normal file
View File

@ -0,0 +1,240 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: we_dbrootextenttracker.h 4631 2013-05-02 15:21:09Z dcathey $
*/
/** @file we_dbrootextenttracker.h
* Contains classes to track the order of placement (rotation) of extents as
* they are filled in and/or added to the DBRoots for the local PM.
*
* DBRootExtentTracker did select the next DBRoot and segment number when-
* ever either selectFirstSegFile() or nextSegFile() were called. The logic
* for selecting a "new" segment file number previously in nextSegFile() has
* been moved to the DBRM extent allocation function. The segment number
* argument returned by nextSegFile() is now only applicable if the return
* value is false, indicating that a partially filled extent has been en-
* countered.
*/
#ifndef WE_DBROOTEXTENTTRACKER_H_
#define WE_DBROOTEXTENTTRACKER_H_
#include <boost/thread/mutex.hpp>
#include <vector>
#include "we_type.h"
#include "brmtypes.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
namespace WriteEngine
{
class Log;
//
// PARTIAL_EXTENT - Extent is partially filled
// EMPTY_DBROOT - DRoot is empty (has no extents)
// EXTENT_BOUNDARY- Encountered extent boundary, add next extent
// OUT_OF_SERVICE - Extent is disabled or out-of-service
//
// Changes to this enum should be reflected in stateStrings array in
// we_dbrootextenttracker.cpp.
//
enum DBRootExtentInfoState
{
DBROOT_EXTENT_PARTIAL_EXTENT = 1,
DBROOT_EXTENT_EMPTY_DBROOT = 2,
DBROOT_EXTENT_EXTENT_BOUNDARY = 3,
DBROOT_EXTENT_OUT_OF_SERVICE = 4
};
//------------------------------------------------------------------------------
/** @brief Tracks the current DBRoot/extent we are loading.
*/
//------------------------------------------------------------------------------
struct DBRootExtentInfo
{
uint32_t fPartition;
uint16_t fDbRoot;
uint16_t fSegment;
BRM::LBID_t fStartLbid;
HWM fLocalHwm;
uint64_t fDBRootTotalBlocks;
DBRootExtentInfoState fState;
DBRootExtentInfo() :
fPartition(0),
fDbRoot(0),
fSegment(0),
fStartLbid(0),
fLocalHwm(0),
fDBRootTotalBlocks(0),
fState(DBROOT_EXTENT_PARTIAL_EXTENT) { }
DBRootExtentInfo(
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
BRM::LBID_t startLbid,
HWM localHwm,
uint64_t dbrootTotalBlocks,
DBRootExtentInfoState state);
bool operator<(const DBRootExtentInfo& entry) const;
};
//------------------------------------------------------------------------------
/** @brief Class to track the order of placement (rotation) of extents as
* they are filled in and/or added to the DBRoots for the relevant PM.
*/
//------------------------------------------------------------------------------
class DBRootExtentTracker
{
public:
/** @brief DBRootExtentTracker constructor
* @param oid Column OID of interest.
* @param colWidths Widths (in bytes) of all the columns in the table.
* @param dbRootHWMInfoColVec Column HWM, DBRoots, etc for this table.
* @param columnIdx Index (into colWidths and dbRootHWMInfoColVec)
* referencing the column that applies to this ExtentTracker.
* @param logger Logger to be used for logging messages.
*/
EXPORT DBRootExtentTracker ( OID oid,
const std::vector<int>& colWidths,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoColVec,
unsigned int columnIdx,
Log* logger );
/** @brief Select the first DBRoot/segment file to add rows to, for this PM.
* @param dbRootExtent Dbroot/segment file selected for first set of rows.
* @param bNoStartExtentOnThisPM Is starting HWM extent missing or disabled.
* If HWM extent is missing or disabled, the app will have to allo-
* cate a new extent (at the DBRoot returned in dbRootExtent)) in
* order to add any rows.
* @param bEmptyPM Is this PM void of any available extents
* @return Returns NO_ERROR if success, else returns error code.
*/
EXPORT int selectFirstSegFile ( DBRootExtentInfo& dbRootExtent,
bool& bNoStartExtentOnThisPM,
bool& bEmptyPM,
std::string& errMsg );
/** @brief Set up this Tracker to select the same first DBRoot/segment file
* as the reference DBRootExtentTracker that is specified from a ref column.
*
* Application code should call selectFirstSegFile for a reference column,
* and assignFirstSegFile for all other columns in the same table.
* @param refTracker Tracker object used to assign first DBRoot/segment.
* @param dbRootExtent Dbroot/segment file selected for first set of rows.
*/
EXPORT void assignFirstSegFile( const DBRootExtentTracker& refTracker,
DBRootExtentInfo& dbRootExtent );
/** @brief Iterate/return next DBRoot to be used for the next extent.
*
* Case 1)
* If it is the "very" first extent for the specified DBRoot, then the
* applicable partition to be used for the first extent is also returned.
*
* Case 2)
* If the user moves a DBRoot to a different PM, then the next cpimport.bin
* job on the recepient PM may encounter 2 partially filled in extents.
* This differs from the norm, where we only have 1 partially filled extent
* at any given time, on a PM. When a DBRoot is moved, we may finish an ex-
* tent on 1 DBRoot, and instead of advancing to start a new extent, we ro-
* tate to the recently moved DBRoot, and have to first fill in a partilly
* filled in extent instead of adding a new extent. Case 2 is intended to
* cover this use case.
* In this case, in the middle of an import, if the next extent to receive
* rows is a partially filled in extent, then the DBRoot, partition, and
* segment number for the partial extent are returned. In addition, the
* current HWM and starting LBID for the relevant extent are returned.
*
* Case 3)
* If we are just finishing one extent and adding the next extent, then
* only the DBRoot argument is relevant, telling us where to create the
* next extent. Return value will be true. This case also applies to
* the instance where the HWM extent for the next DBRoot is disabled.
*
* @param dbRoot DBRoot for the next extent
* @param partition If first extent on dbRoot (or partial extent), then
* this is the partition #
* @param segment If partially full extent, then this is the segment #
* @param localHwm If partially full extent, then this is current HWM.
* @param startLbid If partially full extent, then this is starting LBID of
* the current HWM extent.
*
* @return Returns true if new extent needs to be allocated, returns false
* if extent is partially full, and has room for more rows.
*/
EXPORT bool nextSegFile( uint16_t& dbRoot,
uint32_t& partition,
uint16_t& segment,
HWM& localHwm,
BRM::LBID_t& startLbid );
/** @brief get the DBRootExtentInfo list
*/
const std::vector<DBRootExtentInfo>& getDBRootExtentList();
/** @brief get the CurrentDBRootIdx
*/
inline const int getCurrentDBRootIdx()
{
boost::mutex::scoped_lock lock(fDBRootExtTrkMutex);
return fCurrentDBRootIdx;
}
private:
DBRootExtentInfoState determineState(int colWidth,
HWM localHwm,
uint64_t dbRootTotalBlocks,
int16_t status);
// Select First DBRoot/segment file on a PM having no extents for fOID
int selectFirstSegFileForEmptyPM ( std::string& errMsg );
void initEmptyDBRoots(); // init ExtentList for empty DBRoots
void logFirstDBRootSelection() const;
OID fOID; // applicable colunn OID
long long fBlksPerExtent; // blocks per extent for fOID
Log* fLog; // logger
boost::mutex fDBRootExtTrkMutex; // mutex to access fDBRootExtentList
int fCurrentDBRootIdx; // Index into fDBRootExtentList,
// DBRoot where current extent is
// being added
std::vector<DBRootExtentInfo> fDBRootExtentList; // List of current pending
// DBRoot/extents for each DBRoot
// assigned to the local PM.
bool fEmptyOrDisabledPM; // true if PM has no extents or all
// extents are disabled
bool fEmptyPM; // true if PM has no available or
// disabled extents
bool fDisabledHWM; // Did job start with disabled HWM
};
} //end of namespace
#undef EXPORT
#endif // WE_DBROOTEXTENTTRACKER_H_

298
writeengine/shared/we_define.cpp Normal file
View File

@ -0,0 +1,298 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_define.cpp 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
/** @file" **/
#include <string>
#include "we_define.h"
#include "idberrorinfo.h"
#include "errorids.h"
#include "brmtypes.h"
#include "we_brm.h"
/** Namespace WriteEngine **/
namespace WriteEngine
{
WErrorCodes::WErrorCodes() : fErrorCodes()
{
fErrorCodes[ERR_UNKNOWN] = " a Generic (unknown) error";
fErrorCodes[ERR_INVALID_PARAM] = " due to Invalid parameters";
fErrorCodes[ERR_STRUCT_EMPTY] = " because the Structure is empty";
fErrorCodes[ERR_VALUE_OUTOFRANGE] = " because a Value is out of range";
fErrorCodes[ERR_PARSING] = " a Value is out of range";
fErrorCodes[ERR_NO_MEM] = " out of memory";
fErrorCodes[ERR_DML_LOG_NAME] = " construct DML log filename failed";
fErrorCodes[ERR_OPEN_DML_LOG] = " open DML log file failed";
fErrorCodes[ERR_HDFS_BACKUP] = " DML backup error in HDFS";
// File level error
fErrorCodes[ERR_FILE_CREATE] = " The column file could not be created; it may already exist or be inaccessible.";
fErrorCodes[ERR_FILE_OPEN] = " opening a column file. The file was not found or was inaccessible.";
fErrorCodes[ERR_FILE_DELETE] = " it can not delete the file, because it does not exist or was inaccessible. ";
fErrorCodes[ERR_FILE_EXIST] = " The File already exists. ";
fErrorCodes[ERR_FILE_NOT_EXIST] = " The File does not exist. " ;
fErrorCodes[ERR_FILE_NULL] = " The FILE pointer is null." ;
fErrorCodes[ERR_FILE_WRITE] = " Error writing to a database file. ";
fErrorCodes[ERR_FILE_READ] = " Error reading from a database file. ";
fErrorCodes[ERR_FILE_SEEK] = " Error in positioning file handle. ";
fErrorCodes[ERR_FILE_READ_IMPORT] = " Error reading import source file. ";
fErrorCodes[ERR_DIR_CREATE] = " Error in creating a directory. ";
fErrorCodes[ERR_FILE_NEW_EXTENT_FBO] = " New extent FBO too high for current file. ";
fErrorCodes[ERR_FILE_FBO_NEG] = " Specified file FBO is negative. ";
fErrorCodes[ERR_FILE_TRUNCATE] = " Error truncating db file. ";
fErrorCodes[ERR_FILE_DISK_SPACE] = "Not able to add extent; adding extent "
"would exceed max file system disk usage. ";
fErrorCodes[ERR_FILE_STAT] = " Error getting stats on db file. ";
fErrorCodes[ERR_VB_FILE_NOT_EXIST] = " Version buffer file does not exists.";
fErrorCodes[ERR_FILE_FLUSH] = " Error flushing db file. ";
fErrorCodes[ERR_FILE_GLOBBING] = " Error globbing a file name. ";
// XML level error
fErrorCodes[ERR_XML_FILE] = " An xml file error, usually because the file does not exist";
fErrorCodes[ERR_XML_ROOT_ELEM] = " An xml file Root element error ";
fErrorCodes[ERR_XML_EMPTY] = " An Empty XML file ";
fErrorCodes[ERR_XML_PARSE] = " An XML Parsing error";
// table lock level error
fErrorCodes[ERR_TBLLOCK_LOCK_NOT_FOUND]= "Table is not locked.";
fErrorCodes[ERR_TBLLOCK_GET_LOCK] = "Error getting table lock.";
fErrorCodes[ERR_TBLLOCK_GET_LOCK_LOCKED]="Table locked by another user.";
fErrorCodes[ERR_TBLLOCK_RELEASE_LOCK] = "Error releasing table lock.";
fErrorCodes[ERR_TBLLOCK_CHANGE_STATE] = "Error changing table lock state.";
fErrorCodes[ERR_TBLLOCK_GET_INFO] = "Error getting table lock info.";
fErrorCodes[ERR_TBLLOCK_LOCKID_CONFLICT]="Table LockID for different table than expected.";
// DDL/DML Interface level error
fErrorCodes[ERR_STRUCT_VALUE_NOT_MATCH] = " the number of structs does not match with the number of value sets";
fErrorCodes[ERR_ROWID_VALUE_NOT_MATCH] = " the number of rowids does not match with the number of values";
fErrorCodes[ERR_TBL_SYSCAT_ERROR] = "Error occured when querying systemcatalog.";
// index error
fErrorCodes[ERR_IDX_TREE_MOVE_ENTRY] = " an error in moving part of an index tree to a new subblock";
fErrorCodes[ERR_IDX_TREE_INVALID_TYPE] = " an Invalid index tree entry type";
fErrorCodes[ERR_IDX_TREE_BITTEST_VAL] = " a Wrong bit test value in the index tree entry";
fErrorCodes[ERR_IDX_TREE_INVALID_LEVEL] = " an Invalid testbit index tree level";
fErrorCodes[ERR_IDX_TREE_INVALID_GRP] = "an Invalid index tree group type ";
fErrorCodes[ERR_IDX_TREE_LISTPTR_CHANGE] = " an index tree List pointer change";
//index list error
fErrorCodes[ERR_IDX_LIST_INVALID_ADDHDR] = " a Create indexlist header error";
fErrorCodes[ERR_IDX_LIST_INVALID_UPDATE] = " an pdate Index List error ";
fErrorCodes[ERR_IDX_LIST_INVALID_DELETE] = " a Delete rowid in indexlist err";
fErrorCodes[ERR_IDX_LIST_INVALID_KEY] = " an Invalid index listbppseeder.cpp Key passed";
fErrorCodes[ERR_IDX_LIST_GET_RID_ARRARY] = " an index list RID array";
fErrorCodes[ERR_IDX_LIST_WRONG_KEY ] = " a not matched Key passed to an index list";
fErrorCodes[ERR_IDX_LIST_HDR_EMPTY] = " an empty index list header";
fErrorCodes[ERR_IDX_LIST_GET_SEGMT] = " in an index list Get Segment";
fErrorCodes[ERR_IDX_LIST_WRONG_LBID_WRITE] = " an index list incorrect LBID write";
fErrorCodes[ERR_IDX_LIST_UPDATE_SUB] = " in an index list update sub";
fErrorCodes[ERR_IDX_LIST_UPDATE_NARRAY] = " in an index list update narray";
fErrorCodes[ERR_IDX_LIST_LAST_FBO_NEG] = " the last index list FBO neg";
fErrorCodes[ERR_IDX_LIST_INIT_NEW_BLKS] = " in an index list initialize new blocks";
fErrorCodes[ERR_IDX_LIST_INIT_LINK_BLKS] = " in an index list initialize link blocks";
fErrorCodes[ERR_IDX_LIST_UPDATE_COUNT] = " in an index list update count";
fErrorCodes[ERR_IDX_LIST_SET_NEXT_LBID] = " in an index list set next LBID";
fErrorCodes[ERR_IDX_LIST_INVALID_LBID] = "an index list invalid LBID";
fErrorCodes[ERR_IDX_LIST_INVALID_BLK_READ] = " in an index list invalid LBID read";
fErrorCodes[ERR_IDX_LIST_UPDATE_HDR_COUNT] = " in an index list update header count";
fErrorCodes[ERR_IDX_LIST_WRONG_BLK] = " an index list wrong block";
fErrorCodes[ERR_IDX_LIST_WRONG_TYPE] = " an index list wrong type";
fErrorCodes[ERR_IDX_LIST_GET_COUNT] = " in an index list get count";
fErrorCodes[ERR_IDX_LIST_GET_NEXT] = " in an index list get next";
fErrorCodes[ERR_IDX_LIST_GET_PARENT] = " in an index list get parent";
fErrorCodes[ERR_IDX_LIST_GET_SUB_BLK] = " in an index list get sub block";
fErrorCodes[ERR_IDX_LIST_INVALID_UP_HDR] = " an invalid Update Index List header ";
fErrorCodes[ERR_IDX_LIST_INVALID_ADD_LIST] = " an invalid add Index List";
fErrorCodes[ERR_IDX_LIST_INVALID_UP] = " an invalid Update Index List";
//freemgr error
fErrorCodes[ERR_FM_ASSIGN_ERR] = " in an assignment";
fErrorCodes[ERR_FM_RELEASE_ERR] = " in a release";
fErrorCodes[ERR_FM_BAD_FBO] = " an invalid File Block Offset";
fErrorCodes[ERR_FM_BAD_TYPE] = "an invalid type that must be pointer or list";
fErrorCodes[ERR_FM_NO_SPACE] = " that No blocks are available";
fErrorCodes[ERR_FM_EXTEND] = " while extending a file";
// Dictionary error
fErrorCodes[ERR_DICT_NO_SPACE_INSERT] = " no space for a dictionary insert";
fErrorCodes[ERR_DICT_SIZE_GT_8000] = " the dictionary size was >8000";
fErrorCodes[ERR_DICT_NO_OP_DELETE] = " in the dictionary no op delete";
fErrorCodes[ERR_DICT_NO_OFFSET_DELETE] = " a dictionary bad Delete offset";
fErrorCodes[ERR_DICT_INVALID_HDR] = " a dictionary bad Delete Hdr";
fErrorCodes[ERR_DICT_ZERO_LEN] = " a dictionary zero len";
fErrorCodes[ERR_DICT_TOKEN_NOT_FOUND] = " a dictionary token not found";
fErrorCodes[ERR_DICT_FILE_NOT_FOUND] = " a dictionary file not found";
fErrorCodes[ERR_DICT_BAD_TOKEN_LBID] = " a dictionary token lbid is bad";
fErrorCodes[ERR_DICT_BAD_TOKEN_OP] = " a dictionary token op is bad";
// Bulk error
fErrorCodes[ERR_BULK_MAX_ERR_NUM] = " the Maximum number of error rows reached";
fErrorCodes[ERR_BULK_DATA_COL_NUM] = " the total number of data column not match with column definitions";
fErrorCodes[ERR_BULK_SEND_MSG_ERR] = " in a bulk load send msg";
fErrorCodes[ERR_BULK_MISSING_EXTENT_ENTRY] = " missing Extent Entry when trying to save LBID info for CP";
fErrorCodes[ERR_BULK_MISSING_EXTENT_ROW] = " missing Extent Row when trying to save LBID info for CP";
fErrorCodes[ERR_BULK_ROW_FILL_BUFFER] = " Single row fills read buffer; try larger read buffer.";
fErrorCodes[ERR_BULK_DBROOT_CHANGE] = " Local PM DBRoot settings changed during bulk load.";
fErrorCodes[ERR_BULK_ROLLBACK_MISS_ROOT] = " Mode3 automatic rollback not performed. DBRoot missing.";
fErrorCodes[ERR_BULK_ROLLBACK_SEG_LIST] = " Error building segment file list in a directory.";
fErrorCodes[ERR_BULK_BINARY_PARTIAL_REC] = " Binary import did not end on fixed length record boundary.";
fErrorCodes[ERR_BULK_BINARY_IGNORE_FLD] = " <IgnoreField> tag not supported for binary imports.";
// BRM error
fErrorCodes[ERR_BRM_LOOKUP_LBID] = " a BRM Lookup LBID error.";
fErrorCodes[ERR_BRM_LOOKUP_FBO] = " a BRM Lookup FBO error.";
fErrorCodes[ERR_BRM_ALLOC_EXTEND] = " a BRM Allocate extent error.";
fErrorCodes[ERR_BRM_COMMIT] = " a BRM Commit error.";
fErrorCodes[ERR_BRM_ROLLBACK] = " a BRM Rollback error.";
fErrorCodes[ERR_BRM_GET_UNCOMM_LBID] = " a BRM get uncommitted lbid list error.";
fErrorCodes[ERR_BRM_DEL_OID] = " a BRM Delete oid error.";
fErrorCodes[ERR_BRM_BEGIN_COPY] = " a BRM Begin copy error.";
fErrorCodes[ERR_BRM_END_COPY] = " a BRM End copy error.";
fErrorCodes[ERR_BRM_GET_HWM] = " a BRM get hwm error.";
fErrorCodes[ERR_BRM_SET_HWM] = " a BRM Set hwm error.";
fErrorCodes[ERR_BRM_WR_VB_ENTRY] = " a BRM VB entry error.";
fErrorCodes[ERR_BRM_VB_COPY_READ] = " a BRM VB copy read error.";
fErrorCodes[ERR_BRM_VB_COPY_SEEK_DB] = " a BRM VB copy seek error against DB file.";
fErrorCodes[ERR_BRM_VB_COPY_SEEK_VB] = " a BRM VB copy seek error against VB file.";
fErrorCodes[ERR_BRM_VB_COPY_WRITE] = " a BRM VB copy write.";
fErrorCodes[ERR_BRM_DEAD_LOCK] = " a BRM DEAD lock error.";
fErrorCodes[ERR_BRM_MARK_INVALID] = " a BRM Mark extent invalid error from casual paritioning.";
fErrorCodes[ERR_BRM_SAVE_STATE] = " a BRM Save state error.";
fErrorCodes[ERR_BRM_GET_START_EXTENT] = " a BRM get start Extent error.";
fErrorCodes[ERR_BRM_VB_OVERFLOW] = "BRM block version buffer overflow error.";
fErrorCodes[ERR_BRM_READ_ONLY] = "BRM is in read-only state.";
fErrorCodes[ERR_BRM_GET_READ_WRITE] = "BRM error getting read-write state.";
fErrorCodes[ERR_BRM_BULK_RB_COLUMN] = "BRM error performing bulk rollback of column extents.";
fErrorCodes[ERR_BRM_BULK_RB_DCTNRY] = "BRM error performing bulk rollback of dictionary store extents.";
fErrorCodes[ERR_BRM_DELETE_EXTENT_COLUMN] = "BRM error deleting column extents.";
fErrorCodes[ERR_BRM_DELETE_EXTENT_DCTNRY] = "BRM error deleting dictionary extents.";
fErrorCodes[ERR_BRM_TAKE_SNAPSHOT] = "BRM error requesting snapshot of BRM state.";
fErrorCodes[ERR_BRM_LOOKUP_START_LBID] = "BRM start LBID lookup error.";
fErrorCodes[ERR_BRM_BULK_UPDATE] = "BRM error executing bulk update of HWM and CP.";
fErrorCodes[ERR_BRM_GET_EXT_STATE] = "BRM error getting segment file extent state.";
fErrorCodes[ERR_EXTENTMAP_LOOKUP] = " a extent map Lookup error.";
fErrorCodes[ERR_BRM_LOOKUP_VERSION] = " a vssLookup version info error.";
fErrorCodes[ERR_BRM_LOOKUP_LBID_RANGES] = " BRM error getting LBID ranges.";
fErrorCodes[ERR_BRM_HWMS_NOT_EQUAL] = " HWMs for same width columns not equal. ";
fErrorCodes[ERR_BRM_HWMS_OUT_OF_SYNC] = " HWMs for different width columns not in sync. ";
fErrorCodes[ERR_BRM_DBROOT_HWMS] = " BRM error getting HWMs for DBRoots. ";
fErrorCodes[ERR_BRM_NETWORK] = " Network error in DBRM call. ";
fErrorCodes[ERR_BRM_READONLY] = " DBRM is read only. ";
fErrorCodes[ERR_INVALID_VBOID] = " The VB oid is invalid ";
fErrorCodes[ERR_BRM_SET_EXTENTS_CP] = " BRM error setting extents min/max ";
fErrorCodes[ERR_BRM_SHUTDOWN] = " The system is being shutdown ";
fErrorCodes[ERR_BRM_GET_SHUTDOWN] = " BRM error get the system shutdown flag ";
fErrorCodes[ERR_BRM_SUSPEND] = " The system is in write suspended mode";
fErrorCodes[ERR_BRM_GET_SUSPEND] = " BRM error get the system suspend flag ";
fErrorCodes[ERR_BRM_BAD_STRIPE_CNT] = " Incorrect number of column extents allocated in stripe";
// DM error
fErrorCodes[ERR_DM_CONVERT_OID] = " a DM Conversion error";
// Cache error
fErrorCodes[ERR_CACHE_KEY_EXIST ] = " a Cache key exists";
fErrorCodes[ERR_CACHE_KEY_NOT_EXIST] = " a Cache key does not exist";
fErrorCodes[ERR_NULL_BLOCK] = " a Block is NULL";
fErrorCodes[ERR_FREE_LIST_EMPTY] = " a Free list is empty";
// Compression error
fErrorCodes[ERR_COMP_COMPRESS] = " Error in compressing data. ";
fErrorCodes[ERR_COMP_UNCOMPRESS] = " Error in uncompressing data. ";
fErrorCodes[ERR_COMP_PARSE_HDRS] = " Error parsing compression headers. ";
fErrorCodes[ERR_COMP_VERIFY_HDRS] = " Error verifying compression headers. ";
fErrorCodes[ERR_COMP_PAD_DATA] = " Error in padding compressed data. ";
fErrorCodes[ERR_COMP_READ_BLOCK] = " Error in reading a data block. ";
fErrorCodes[ERR_COMP_SAVE_BLOCK] = " Error in saving a data block. ";
fErrorCodes[ERR_COMP_WRONG_PTR] = " Invalid pointer in compression headers. ";
fErrorCodes[ERR_COMP_FILE_NOT_FOUND] = " Error searching for a compressed file. ";
fErrorCodes[ERR_COMP_CHUNK_NOT_FOUND] = " Error searching for a compressed chunk. ";
fErrorCodes[ERR_COMP_UNAVAIL_TYPE] = " Unavailable compressino type. ";
fErrorCodes[ERR_COMP_REMOVE_FILE] = " Failed to remove a file. ";
fErrorCodes[ERR_COMP_RENAME_FILE] = " Failed to rename a file. ";
fErrorCodes[ERR_COMP_OPEN_FILE] = " Failed to open a compressed data file. ";
fErrorCodes[ERR_COMP_SET_OFFSET] = " Failed to set offset in a compressed data file. ";
fErrorCodes[ERR_COMP_READ_FILE] = " Failed to read from a compressed data file. ";
fErrorCodes[ERR_COMP_WRITE_FILE] = " Failed to write to a compresssed data file. ";
fErrorCodes[ERR_COMP_CLOSE_FILE] = " Failed to close a compressed data file. ";
fErrorCodes[ERR_COMP_TRUNCATE_ZERO] = " Attempting to truncate compressed file to 0 bytes. ";
// Auto-increment error
fErrorCodes[ERR_AUTOINC_GEN_EXCEED_MAX] = " Generated auto-increment value "
"exceeds maximum value for the column type.";
fErrorCodes[ERR_AUTOINC_USER_OUT_OF_RANGE] = " User specified auto-"
"increment value is out of range for the column type.";
fErrorCodes[ERR_AUTOINC_TABLE_NAME] = " Invalid schema/tablename for auto increment. ";
fErrorCodes[ERR_AUTOINC_INIT1] = " Unable to initialize auto-increment value. ";
fErrorCodes[ERR_AUTOINC_INIT2] = " Unable to initialize auto-increment value. Unknown exception. ";
fErrorCodes[ERR_AUTOINC_RID] = " Failed to get row information from calpontsystemcatalog.";
fErrorCodes[ERR_AUTOINC_START_SEQ] = " Unable to setup AI sequence in BRM.";
fErrorCodes[ERR_AUTOINC_GET_RANGE] = " Unable to reserve AI range from BRM.";
fErrorCodes[ERR_AUTOINC_GET_LOCK] = " Unable to lock AI column in BRM.";
fErrorCodes[ERR_AUTOINC_REL_LOCK] = " Unable to release AI column in BRM.";
fErrorCodes[ERR_AUTOINC_UPDATE] = " Unable to update nextValue in system catalog.";
// Block cache flush error
fErrorCodes[ERR_BLKCACHE_FLUSH_LIST] = " Failed to flush list of blocks from PrimProc cache. ";
// Backup bulk meta data file error
fErrorCodes[ERR_METADATABKUP_FILE_RENAME] = " Unable to rename temporary bulk meta data file. ";
fErrorCodes[ERR_METADATABKUP_COMP_PARSE_HDRS] = " Error parsing compression headers in bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_VERIFY_HDRS] = " Error verifying compression headers in bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND]= " Error searching for compressed chunk in db file being backed up. ";
fErrorCodes[ERR_METADATABKUP_COMP_OPEN_BULK_BKUP] = " Error opening compressed chunk in bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_WRITE_BULK_BKUP]= " Error writing compressed chunk to bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_READ_BULK_BKUP] = " Error reading compressed chunk from bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_RENAME] = " Unable to rename compressed chunk bulk backup file. ";
}
std::string WErrorCodes::errorString(int code)
{
// Look for error message overrides from system-wide error messages
switch (code)
{
case ERR_FILE_DISK_SPACE:
{
logging::Message::Args args;
std::string msgArg; // empty str arg; no extra info in this context
args.add( msgArg );
return logging::IDBErrorInfo::instance()->errorMsg(
logging::ERR_EXTENT_DISK_SPACE, args);
break;
}
}
int brmRc = BRMWrapper::getBrmRc();
if (brmRc == BRM::ERR_OK)
return (fErrorCodes[code]);
std::string errMsg( fErrorCodes[code] );
std::string brmMsg;
errMsg += " [BRM error status: ";
BRM::errString(brmRc, brmMsg);
errMsg += brmMsg;
errMsg += "]";
return errMsg;
}
} //end of namespace

393
writeengine/shared/we_define.h Normal file
View File

@ -0,0 +1,393 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: we_define.h 4726 2013-08-07 03:38:36Z bwilkinson $
/** @file */
#undef NO_ERROR
#ifndef _WE_DEFINE_H_
#define _WE_DEFINE_H_
#include <string>
#include <map>
#include <stdint.h>
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
const short MAX_COLUMN_BOUNDARY = 8; // Max bytes for one column
const int MAX_SIGNATURE_SIZE = 8000; // Max len of dict sig val
const int MAX_FIELD_SIZE = 1000; // Max len non-dict fld val
const int MAX_DB_DIR_LEVEL = 6; // Max lvl of db dir struct
const int MAX_DB_DIR_NAME_SIZE = 20; // Max len of db dir size
const short ROW_PER_BYTE = 8; // Rows/byte in bitmap file
const int BYTE_PER_BLOCK = 8192; // Num bytes per data block
const int BYTE_PER_SUBBLOCK = 256; // Num bytes per sub block
const int ENTRY_PER_SUBBLOCK = 32; // Num entries per sub block
const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024;
// Num rows reserved to disk for 'initial' extent
const int FILE_NAME_SIZE = 200; // Max size of file name
const long long MAX_ALLOW_ERROR_COUNT = 100000; //Max allowable error count
//--------------------------------------------------------------------------
// Dictionary related constants
//--------------------------------------------------------------------------
const uint16_t DCTNRY_END_HEADER = 0xffff ; // end of header
const uint64_t NOT_USED_PTR = 0x0 ; // not continuous ptr
const int HDR_UNIT_SIZE = 2; // hdr unit size
const int NEXT_PTR_BYTES = 8; // const ptr size
const int MAX_OP_COUNT = 1024; // op max size
const int DCTNRY_HEADER_SIZE = 14; // header total size
const int MAX_STRING_CACHE_SIZE = 1000;
// End of Dictionary related constants
const int COLPOSPAIR_NULL_TOKEN_OFFSET= -1; // offset value denoting a null token
const uint32_t BULK_SYSCAT_SESSION_ID = 0; // SessionID for syscat queries
const char COL_TYPE_DICT = 'D'; // Dictionary type
const uint64_t INVALID_LBID = 0xFFFFFFFFFULL; // 2**36 - 1
const unsigned int SUBSYSTEM_ID_DDLPROC = 15;
const unsigned int SUBSYSTEM_ID_DMLPROC = 20;
const unsigned int SUBSYSTEM_ID_WE = 19;
const unsigned int SUBSYSTEM_ID_WE_SRV = 32;
const unsigned int SUBSYSTEM_ID_WE_SPLIT= 33;
const unsigned int SUBSYSTEM_ID_WE_BULK = 34;
//--------------------------------------------------------------------------
// Default definitions
//--------------------------------------------------------------------------
const int DEFAULT_CACHE_BLOCK = 256; // Max num of cache blocks
const int DEFAULT_CHK_INTERVAL = 3; // Checkpoint in seconds
const int DEFAULT_CACHE_PCT_FREE = 25; // Min % of free cache
const int DEFAULT_BUFSIZ = 1*1024*1024; // setvbuf buffer size
const int DEFAULT_COLSIZ = 8; // col size for hdfs rdwr buf
const int BLK_INIT = 0;
const int BLK_READ = 1;
const int BLK_WRITE = 2;
//--------------------------------------------------------------------------
// Return code definitions
//--------------------------------------------------------------------------
const int NO_ERROR = 0; // No error
const int NOT_FOUND = -1; // Not found
const int INVALID_NUM = -1; // Invalid number
//--------------------------------------------------------------------------
// Error code definition
//--------------------------------------------------------------------------
const int ERR_CODEBASE = 1000; // Generic error codes
const int ERR_FILEBASE = 1050; // File-related error codes
const int ERR_XMLBASE = 1150; // XML job file error codes
const int ERR_TBLLOCKBASE = 1200; // Table-lock error codes
const int ERR_WRAPPERBASE = 1250; // DDL/DML API related errors
const int ERR_INDEXBASE = 1300; // Index-related error codes
const int ERR_FMGRBASE = 1350; // Freemgr errors
const int ERR_DCTNRYBASE = 1400; // Dictionary errors
const int ERR_BULKBASE = 1450; // Bulk specific errors
const int ERR_BRMBASE = 1500; // BRM errors
const int ERR_DMBASE = 1550; // Disk manager errors
const int ERR_CACHEBASE = 1600; // Cche management errors
const int ERR_COMPBASE = 1650; // Compression errors
const int ERR_AUTOINCBASE = 1700; // Auto-increment errors
const int ERR_BLKCACHEBASE = 1750; // Block cache flush errors
const int ERR_METABKUPBASE = 1800; // Backup bulk meta file errors
//--------------------------------------------------------------------------
// Generic error
//--------------------------------------------------------------------------
const int ERR_UNKNOWN = ERR_CODEBASE + 1; // Generic error
const int ERR_INVALID_PARAM = ERR_CODEBASE + 2; // Invalid parms
const int ERR_STRUCT_EMPTY = ERR_CODEBASE + 3; // Struct is empty
const int ERR_VALUE_OUTOFRANGE = ERR_CODEBASE + 4; // Val out of range
const int ERR_PARSING = ERR_CODEBASE + 5; // Parsing error
const int ERR_NO_MEM = ERR_CODEBASE + 6; // Mem alloc error
const int ERR_DML_LOG_NAME = ERR_CODEBASE + 7; // DML log filename error
const int ERR_OPEN_DML_LOG = ERR_CODEBASE + 8; // Open DML log file error
const int ERR_HDFS_BACKUP = ERR_CODEBASE + 9; // HDFS backup error
//--------------------------------------------------------------------------
// File level error
//--------------------------------------------------------------------------
const int ERR_FILE_CREATE = ERR_FILEBASE + 1; // File creation error, mostly because file has already existed
const int ERR_FILE_OPEN = ERR_FILEBASE + 2; // Can not open the file, mostly because file not found
const int ERR_FILE_DELETE = ERR_FILEBASE + 3; // Can not delete the file, common reason is file not exist
const int ERR_FILE_EXIST = ERR_FILEBASE + 4; // File alreay exists
const int ERR_FILE_NOT_EXIST = ERR_FILEBASE + 5; // File not exists
const int ERR_FILE_NULL = ERR_FILEBASE + 6; // File is empty
const int ERR_FILE_WRITE = ERR_FILEBASE + 7; // Error writing to a DB file
const int ERR_FILE_READ = ERR_FILEBASE + 8; // Error reading from a DB file
const int ERR_FILE_SEEK = ERR_FILEBASE + 9; // Error in positioning file handle
const int ERR_FILE_READ_IMPORT = ERR_FILEBASE + 10;// Error reading import source file
const int ERR_DIR_CREATE = ERR_FILEBASE + 11;// Error in creating directory
const int ERR_FILE_NEW_EXTENT_FBO = ERR_FILEBASE + 12;// New extent fbo too large
const int ERR_FILE_FBO_NEG = ERR_FILEBASE + 13;// File FBO is negative
const int ERR_FILE_TRUNCATE = ERR_FILEBASE + 14;// Error truncating file
const int ERR_FILE_DISK_SPACE = ERR_FILEBASE + 15;// Out of space on file system
const int ERR_FILE_STAT = ERR_FILEBASE + 16;// Error getting stats on file
const int ERR_VB_FILE_NOT_EXIST = ERR_FILEBASE + 17;// Version buffer file not exists
const int ERR_FILE_FLUSH = ERR_FILEBASE + 18;// Error flushing file
const int ERR_FILE_GLOBBING = ERR_FILEBASE + 19;// Error globbing a file name
//--------------------------------------------------------------------------
// XML level error
//--------------------------------------------------------------------------
const int ERR_XML_FILE = ERR_XMLBASE + 1; // File error, probably because file does not exist
const int ERR_XML_ROOT_ELEM = ERR_XMLBASE + 2; // Root element err
const int ERR_XML_EMPTY = ERR_XMLBASE + 3; // Empty XML file
const int ERR_XML_PARSE = ERR_XMLBASE + 4; // Parsing error
//--------------------------------------------------------------------------
// table lock level error
//--------------------------------------------------------------------------
const int ERR_TBLLOCK_LOCK_NOT_FOUND = ERR_TBLLOCKBASE + 1; // table has no lock
const int ERR_TBLLOCK_GET_LOCK = ERR_TBLLOCKBASE + 2; // error acquiring a table lock
const int ERR_TBLLOCK_GET_LOCK_LOCKED= ERR_TBLLOCKBASE + 3; // table currently locked
const int ERR_TBLLOCK_RELEASE_LOCK = ERR_TBLLOCKBASE + 4; // error releasing a table lock
const int ERR_TBLLOCK_CHANGE_STATE = ERR_TBLLOCKBASE + 5; // error changing state of lock
const int ERR_TBLLOCK_GET_INFO = ERR_TBLLOCKBASE + 6; // error getting info about a lock
const int ERR_TBLLOCK_LOCKID_CONFLICT= ERR_TBLLOCKBASE + 7; // lockID for different table than expected
//--------------------------------------------------------------------------
// DDL/DML Interface level error
//--------------------------------------------------------------------------
const int ERR_STRUCT_VALUE_NOT_MATCH = ERR_WRAPPERBASE + 1; // The number of struct not match with the number of value set
const int ERR_ROWID_VALUE_NOT_MATCH = ERR_WRAPPERBASE + 2; // The number of rowid not match with the number of values
const int ERR_TBL_SYSCAT_ERROR = ERR_WRAPPERBASE + 3; /** @brief Syscatalog query error */
//--------------------------------------------------------------------------
// index error
//--------------------------------------------------------------------------
const int ERR_IDX_TREE_MOVE_ENTRY = ERR_INDEXBASE + 1; // The error in move part of tree to a new subblock
const int ERR_IDX_TREE_INVALID_TYPE = ERR_INDEXBASE + 2; // Invalid tree entry type
const int ERR_IDX_TREE_BITTEST_VAL = ERR_INDEXBASE + 3; // Wrong bit test value in the entry
const int ERR_IDX_TREE_INVALID_LEVEL = ERR_INDEXBASE + 4; // Invalid testbit treel level
const int ERR_IDX_TREE_INVALID_GRP = ERR_INDEXBASE + 5; // Invalid group type
const int ERR_IDX_TREE_LISTPTR_CHANGE = ERR_INDEXBASE + 6; // List pointer change
//index list error
const int ERR_IDX_LIST_INVALID_ADDHDR = ERR_INDEXBASE + 7; // Create indexlist header error
const int ERR_IDX_LIST_INVALID_UPDATE = ERR_INDEXBASE + 8; // Update Index List error
const int ERR_IDX_LIST_INVALID_DELETE = ERR_INDEXBASE + 9; // Delete rowid in indexlist err*/
const int ERR_IDX_LIST_INVALID_KEY = ERR_INDEXBASE + 10;// Invalid Key passed
const int ERR_IDX_LIST_GET_RID_ARRARY = ERR_INDEXBASE + 11;// RID array
const int ERR_IDX_LIST_WRONG_KEY = ERR_INDEXBASE + 12;// not matched Key passed
const int ERR_IDX_LIST_HDR_EMPTY = ERR_INDEXBASE + 13;// Delete rowid in indexlist err
const int ERR_IDX_LIST_GET_SEGMT = ERR_INDEXBASE + 14;// Get Segment
const int ERR_IDX_LIST_WRONG_LBID_WRITE=ERR_INDEXBASE + 15;
const int ERR_IDX_LIST_UPDATE_SUB = ERR_INDEXBASE + 16;
const int ERR_IDX_LIST_UPDATE_NARRAY = ERR_INDEXBASE + 17;
const int ERR_IDX_LIST_LAST_FBO_NEG = ERR_INDEXBASE + 18;
const int ERR_IDX_LIST_INIT_NEW_BLKS = ERR_INDEXBASE + 19;
const int ERR_IDX_LIST_INIT_LINK_BLKS = ERR_INDEXBASE + 20;
const int ERR_IDX_LIST_UPDATE_COUNT = ERR_INDEXBASE + 21;
const int ERR_IDX_LIST_SET_NEXT_LBID = ERR_INDEXBASE + 22;
const int ERR_IDX_LIST_INVALID_LBID = ERR_INDEXBASE + 23;
const int ERR_IDX_LIST_INVALID_BLK_READ=ERR_INDEXBASE + 24;
const int ERR_IDX_LIST_UPDATE_HDR_COUNT=ERR_INDEXBASE + 25;
const int ERR_IDX_LIST_WRONG_BLK = ERR_INDEXBASE + 26;
const int ERR_IDX_LIST_WRONG_TYPE = ERR_INDEXBASE + 27;
const int ERR_IDX_LIST_GET_COUNT = ERR_INDEXBASE + 28;
const int ERR_IDX_LIST_GET_NEXT = ERR_INDEXBASE + 29;
const int ERR_IDX_LIST_GET_PARENT = ERR_INDEXBASE + 30;
const int ERR_IDX_LIST_GET_SUB_BLK = ERR_INDEXBASE + 31;
const int ERR_IDX_LIST_INVALID_UP_HDR = ERR_INDEXBASE + 32;// Update Index List error
const int ERR_IDX_LIST_INVALID_ADD_LIST=ERR_INDEXBASE + 33;// Update Index List error
const int ERR_IDX_LIST_INVALID_UP = ERR_INDEXBASE + 34;// Update Index List error
//--------------------------------------------------------------------------
// freemgr error
//--------------------------------------------------------------------------
const int ERR_FM_ASSIGN_ERR = ERR_FMGRBASE + 1; // General assignment error
const int ERR_FM_RELEASE_ERR = ERR_FMGRBASE + 2; // General release error
const int ERR_FM_BAD_FBO = ERR_FMGRBASE + 3; // File Block Offset err
const int ERR_FM_BAD_TYPE = ERR_FMGRBASE + 4; // type must be pointer or list
const int ERR_FM_NO_SPACE = ERR_FMGRBASE + 5; // No blocks available
const int ERR_FM_EXTEND = ERR_FMGRBASE + 6; // Error extending file
//--------------------------------------------------------------------------
// Dictionary error
//--------------------------------------------------------------------------
const int ERR_DICT_NO_SPACE_INSERT= ERR_DCTNRYBASE+ 1; // ins no space
const int ERR_DICT_SIZE_GT_8000 = ERR_DCTNRYBASE+ 2; // ins size >8000
const int ERR_DICT_NO_OP_DELETE = ERR_DCTNRYBASE+ 3; // del no op
const int ERR_DICT_NO_OFFSET_DELETE=ERR_DCTNRYBASE+ 4; // del bad offset
const int ERR_DICT_INVALID_HDR = ERR_DCTNRYBASE+ 5; // Delete Hdr
const int ERR_DICT_ZERO_LEN = ERR_DCTNRYBASE+ 6; // Delete zero len
const int ERR_DICT_TOKEN_NOT_FOUND= ERR_DCTNRYBASE+ 7; // token not found
const int ERR_DICT_FILE_NOT_FOUND = ERR_DCTNRYBASE+ 8; // dict file not found
const int ERR_DICT_BAD_TOKEN_LBID = ERR_DCTNRYBASE+ 9; // bad token lbid
const int ERR_DICT_BAD_TOKEN_OP = ERR_DCTNRYBASE+ 10;// token op is bad
//--------------------------------------------------------------------------
// Bulk error
//--------------------------------------------------------------------------
const int ERR_BULK_MAX_ERR_NUM = ERR_BULKBASE + 1; // Maximum number of error rows reached
const int ERR_BULK_DATA_COL_NUM = ERR_BULKBASE + 2; // The total number of data column not match with column definitions
const int ERR_BULK_SEND_MSG_ERR = ERR_BULKBASE + 3; // send msg to primproc to flush cache
const int ERR_BULK_MISSING_EXTENT_ENTRY=ERR_BULKBASE + 4; // Missing Extent Entry when trying to save LBID info
const int ERR_BULK_MISSING_EXTENT_ROW = ERR_BULKBASE + 5; // Missing Extent Row when trying to save LBID info
const int ERR_BULK_ROW_FILL_BUFFER = ERR_BULKBASE + 6; // Single row fills read buffer
const int ERR_BULK_DBROOT_CHANGE = ERR_BULKBASE + 7; // Local DBRoot settings changed during an import
const int ERR_BULK_ROLLBACK_MISS_ROOT = ERR_BULKBASE + 8; // Mode3 automatic rollback skipped with missing DBRoot
const int ERR_BULK_ROLLBACK_SEG_LIST = ERR_BULKBASE + 9; // Error building segment file list in a directory
const int ERR_BULK_BINARY_PARTIAL_REC = ERR_BULKBASE + 10;// Binary input did not end on fixed length record boundary
const int ERR_BULK_BINARY_IGNORE_FLD = ERR_BULKBASE + 11;// <IgnoreField> tag not supported for binary import
//--------------------------------------------------------------------------
// BRM error
//--------------------------------------------------------------------------
const int ERR_BRM_LOOKUP_LBID = ERR_BRMBASE + 1; // Lookup LBID error
const int ERR_BRM_LOOKUP_FBO = ERR_BRMBASE + 2; // Lookup FBO error
const int ERR_BRM_ALLOC_EXTEND = ERR_BRMBASE + 3; // Allocate extent error
const int ERR_BRM_COMMIT = ERR_BRMBASE + 4; // Commit error
const int ERR_BRM_ROLLBACK = ERR_BRMBASE + 5; // Rollback error
const int ERR_BRM_GET_UNCOMM_LBID = ERR_BRMBASE + 6; // Get uncommitted lbid list error
const int ERR_BRM_DEL_OID = ERR_BRMBASE + 7; // Delete oid error
const int ERR_BRM_BEGIN_COPY = ERR_BRMBASE + 8; // Begin copy error
const int ERR_BRM_END_COPY = ERR_BRMBASE + 9; // End copy error
const int ERR_BRM_GET_HWM = ERR_BRMBASE + 10;// Get hwm error
const int ERR_BRM_SET_HWM = ERR_BRMBASE + 11;// Set hwm error
const int ERR_BRM_WR_VB_ENTRY = ERR_BRMBASE + 12;// Write VB entry error
const int ERR_BRM_VB_COPY_READ = ERR_BRMBASE + 13;// VB copy read error
const int ERR_BRM_VB_COPY_SEEK_DB = ERR_BRMBASE + 14;// VB copy seek error to DB file
const int ERR_BRM_VB_COPY_SEEK_VB = ERR_BRMBASE + 15;// VB copy seek error to VB file
const int ERR_BRM_VB_COPY_WRITE = ERR_BRMBASE + 16;// VB copy write
const int ERR_BRM_DEAD_LOCK = ERR_BRMBASE + 17;// DEAD lock error
const int ERR_BRM_MARK_INVALID = ERR_BRMBASE + 18;// Mark extent invalid error from casual paritioning
const int ERR_BRM_SAVE_STATE = ERR_BRMBASE + 19;// Save state error
const int ERR_BRM_GET_START_EXTENT= ERR_BRMBASE + 20;// Get starting Extent error
const int ERR_BRM_VB_OVERFLOW = ERR_BRMBASE + 21;// Version buffer overflow
const int ERR_BRM_READ_ONLY = ERR_BRMBASE + 22;// BRM is in READ-ONLY state
const int ERR_BRM_GET_READ_WRITE = ERR_BRMBASE + 23;// error getting BRM READ/WRITE state
const int ERR_BRM_BULK_RB_COLUMN = ERR_BRMBASE + 24;// error during column bulk rollback
const int ERR_BRM_BULK_RB_DCTNRY = ERR_BRMBASE + 25;// error during dctnry bulk rollback
const int ERR_BRM_DELETE_EXTENT_COLUMN= ERR_BRMBASE + 26;// error during delete column extents
const int ERR_BRM_DELETE_EXTENT_DCTNRY= ERR_BRMBASE + 27;// error during delete dictionary extents
const int ERR_BRM_TAKE_SNAPSHOT = ERR_BRMBASE + 28;// Taking snapshot of BRM state
const int ERR_BRM_LOOKUP_START_LBID=ERR_BRMBASE + 29;// Lookup starting LBID error
const int ERR_BRM_BULK_UPDATE = ERR_BRMBASE + 30;// Error with bulk update of HWM and CP
const int ERR_BRM_GET_EXT_STATE = ERR_BRMBASE + 31;// Error getting extent state
const int ERR_EXTENTMAP_LOOKUP = ERR_BRMBASE + 32;// Lookup extent map error
const int ERR_BRM_LOOKUP_VERSION = ERR_BRMBASE + 33;// Lookup version error
const int ERR_BRM_LOOKUP_LBID_RANGES = ERR_BRMBASE + 34;// Lookup LBID Ranges error
const int ERR_BRM_HWMS_NOT_EQUAL = ERR_BRMBASE + 35;// HWMs of same col width not equal
const int ERR_BRM_HWMS_OUT_OF_SYNC= ERR_BRMBASE + 36;// HWMs for dif col width not in sync
const int ERR_BRM_DBROOT_HWMS = ERR_BRMBASE + 37;// Error getting HWMs for each DBRoot
const int ERR_BRM_NETWORK = ERR_BRMBASE + 38;// Network error when calling BRM functions
const int ERR_BRM_READONLY = ERR_BRMBASE + 39;// DBRM is readonly
const int ERR_INVALID_VBOID = ERR_BRMBASE + 40;// returned if the given vboid is invalid
const int ERR_BRM_SET_EXTENTS_CP = ERR_BRMBASE + 41;// Error setting extents min/max
const int ERR_BRM_SHUTDOWN = ERR_BRMBASE + 42;// BRM is set to shutdown
const int ERR_BRM_GET_SHUTDOWN = ERR_BRMBASE + 43;// error getting BRM Shutdown flag
const int ERR_BRM_SUSPEND = ERR_BRMBASE + 44;// BRM is set to Suspend writes
const int ERR_BRM_GET_SUSPEND = ERR_BRMBASE + 45;// error getting BRM Suspend flag
const int ERR_BRM_BAD_STRIPE_CNT = ERR_BRMBASE + 46;// Incorrect num of cols allocated in stripe
//--------------------------------------------------------------------------
// DM error
//--------------------------------------------------------------------------
const int ERR_DM_CONVERT_OID = ERR_DMBASE + 1; // Conversion error
//--------------------------------------------------------------------------
// Cache error
//--------------------------------------------------------------------------
const int ERR_CACHE_KEY_EXIST = ERR_CACHEBASE + 1; // Cache key exist
const int ERR_CACHE_KEY_NOT_EXIST = ERR_CACHEBASE + 2; // Cache key not exist
const int ERR_NULL_BLOCK = ERR_CACHEBASE + 3; // Block is NULL
const int ERR_FREE_LIST_EMPTY = ERR_CACHEBASE + 4; // Empty Free list
//--------------------------------------------------------------------------
// Compression error
//--------------------------------------------------------------------------
const int ERR_COMP_COMPRESS = ERR_COMPBASE + 1; // Error compressing data
const int ERR_COMP_UNCOMPRESS = ERR_COMPBASE + 2; // Error uncompressing data
const int ERR_COMP_PARSE_HDRS = ERR_COMPBASE + 3; // Error parsing compression headers
const int ERR_COMP_VERIFY_HDRS = ERR_COMPBASE + 4; // Error verifying compression headers
const int ERR_COMP_PAD_DATA = ERR_COMPBASE + 5; // Pad compressed data failed
const int ERR_COMP_READ_BLOCK = ERR_COMPBASE + 6; // Failed to read a block
const int ERR_COMP_SAVE_BLOCK = ERR_COMPBASE + 7; // Failed to save a block
const int ERR_COMP_WRONG_PTR = ERR_COMPBASE + 8; // Pointer in header is wrong
const int ERR_COMP_FILE_NOT_FOUND = ERR_COMPBASE + 9; // File not found in map
const int ERR_COMP_CHUNK_NOT_FOUND= ERR_COMPBASE + 10;// Chunk not found in map
const int ERR_COMP_UNAVAIL_TYPE = ERR_COMPBASE + 11;// Unavailable compression type
const int ERR_COMP_REMOVE_FILE = ERR_COMPBASE + 12;// Failed to remove a file
const int ERR_COMP_RENAME_FILE = ERR_COMPBASE + 13;// Failed to rename a file
const int ERR_COMP_OPEN_FILE = ERR_COMPBASE + 14;// Failed to open a compressed data file
const int ERR_COMP_SET_OFFSET = ERR_COMPBASE + 15;// Failed to set offset in a compressed data file
const int ERR_COMP_READ_FILE = ERR_COMPBASE + 16;// Failed to read from a compressed data file
const int ERR_COMP_WRITE_FILE = ERR_COMPBASE + 17;// Failed to write to a compresssed data file
const int ERR_COMP_CLOSE_FILE = ERR_COMPBASE + 18;// Failed to close a compressed data file
const int ERR_COMP_TRUNCATE_ZERO = ERR_COMPBASE + 19;// Invalid attempt to truncate file to 0 bytes
//--------------------------------------------------------------------------
// Auto-increment error
//--------------------------------------------------------------------------
const int ERR_AUTOINC_GEN_EXCEED_MAX = ERR_AUTOINCBASE + 1; // Generated autoinc value exceeds max auto increment value/
const int ERR_AUTOINC_USER_OUT_OF_RANGE=ERR_AUTOINCBASE + 2; // User specified autoinc value is out of range
const int ERR_AUTOINC_TABLE_NAME = ERR_AUTOINCBASE + 3; // Invalid schema/tablename for auto increment
const int ERR_AUTOINC_INIT1 = ERR_AUTOINCBASE + 4; // Error initializing auto increment (known exception)
const int ERR_AUTOINC_INIT2 = ERR_AUTOINCBASE + 5; // Error initializing auto increment (unknown exception)
const int ERR_AUTOINC_RID = ERR_AUTOINCBASE + 6; // Error initializing auto increment (unknown exception)
const int ERR_AUTOINC_START_SEQ = ERR_AUTOINCBASE + 7; // Error setting up an auto-increment sequence
const int ERR_AUTOINC_GET_RANGE = ERR_AUTOINCBASE + 8; // Error reserving an auto-increment range
const int ERR_AUTOINC_GET_LOCK = ERR_AUTOINCBASE + 9; // Error getting a lock to update auto-inc next value
const int ERR_AUTOINC_REL_LOCK = ERR_AUTOINCBASE +10; // Error releasing lock to update auto-inc next value
const int ERR_AUTOINC_UPDATE = ERR_AUTOINCBASE +11; // Error updating nextValue in system catalog
//--------------------------------------------------------------------------
// Block cache flush error
//--------------------------------------------------------------------------
const int ERR_BLKCACHE_FLUSH_LIST = ERR_BLKCACHEBASE + 1; // Error flushing list of blocks to PrimProc
//--------------------------------------------------------------------------
// Bulk backup metadata file and corresponding HWM compressed chunk files
//--------------------------------------------------------------------------
const int ERR_METADATABKUP_FILE_RENAME = ERR_METABKUPBASE + 1; // Error renaming meta file */
const int ERR_METADATABKUP_COMP_PARSE_HDRS = ERR_METABKUPBASE + 2; // Error parsing compression headers */
const int ERR_METADATABKUP_COMP_VERIFY_HDRS = ERR_METABKUPBASE + 3; // Error verifying compression headers */
const int ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND= ERR_METABKUPBASE + 4; // Chunk not found in file */
const int ERR_METADATABKUP_COMP_OPEN_BULK_BKUP = ERR_METABKUPBASE + 5; // Error opening backup chunk file */
const int ERR_METADATABKUP_COMP_WRITE_BULK_BKUP= ERR_METABKUPBASE + 6; // Error writing to backup chunk file */
const int ERR_METADATABKUP_COMP_READ_BULK_BKUP = ERR_METABKUPBASE + 7; // Error reading from backup chunk file */
const int ERR_METADATABKUP_COMP_RENAME = ERR_METABKUPBASE + 8; // Error renaming chunk file */
//------------------------------------------------------------------------------
// Class used to convert an error code to a corresponding error message string
//------------------------------------------------------------------------------
struct WErrorCodes
{
EXPORT WErrorCodes();
EXPORT std::string errorString(int code);
private:
typedef std::map<int, std::string> CodeMap;
CodeMap fErrorCodes;
};
} //end of namespace
#undef EXPORT
#endif // _WE_DEFINE_H_

2639
writeengine/shared/we_fileop.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

582
writeengine/shared/we_fileop.h Normal file
View File

@ -0,0 +1,582 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: we_fileop.h 4737 2013-08-14 20:45:46Z bwilkinson $
/** @file */
#ifndef _WE_FILEOP_H_
#define _WE_FILEOP_H_
#include <sys/types.h>
#include <sys/stat.h>
#include <string>
#include <vector>
#include <map>
#include <boost/thread.hpp>
#ifdef _MSC_VER
#include <direct.h>
#define S_IRWXU 0
#define S_IRWXG 0
#ifndef S_IROTH
#define S_IROTH 0
#endif
#define S_IXOTH 0
#endif
#include "we_blockop.h"
#include "we_brm.h"
#include "we_config.h"
#include "we_stats.h"
#include "idbcompress.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
#include "brmtypes.h"
/** Namespace WriteEngine */
namespace WriteEngine
{
/** Class FileOp */
class FileOp : public BlockOp
{
public:
/**
* @brief Constructor
*/
EXPORT explicit FileOp(bool doAlloc=true);
/**
* @brief Destructor
*/
EXPORT virtual ~FileOp();
/**
* @brief Close a file
*/
EXPORT void closeFile( IDBDataFile* pFile ) const;
/**
* @brief Create a directory
*/
EXPORT int createDir( const char* dirName, mode_t mode ) const;
int createDir( const char* dirName ) const;
/**
* @brief Create a file with a fixed file size and file id
*/
EXPORT int createFile( FID fid,
int & allocSize,
uint16_t dbRoot, uint32_t partition,
execplan::CalpontSystemCatalog::ColDataType colDataType,
uint64_t emptyVal = 0, int width = 1 ) ;
/**
* @brief Delete a file
*/
EXPORT int deleteFile( const char* fileName ) const;
/**
* @brief Delete the db files corresponding to the specified file id
*/
EXPORT int deleteFile( FID fid ) const;
/**
* @brief Delete the db files corresponding to the specified file id
*/
EXPORT int deleteFiles( const std::vector<int32_t>& fids ) const;
/**
* @brief Delete db files corresponding to specified file id and partition
*/
EXPORT int deletePartitions( const std::vector<OID>& fids,
const std::vector<BRM::PartitionInfo>& partitions )
const;
/**
* @brief Delete a specific database segment file.
*/
EXPORT int deleteFile( FID fid, uint16_t dbRoot,
uint32_t partition,
uint16_t segment ) const;
/**
* @brief Check whether a file exists or not
*/
EXPORT bool exists( const char* fileName ) const;
/**
* @brief @brief Check whether file exists or not by using file id, DBRoot,
* partition, and segment number.
*/
EXPORT bool exists( FID fid, uint16_t dbRoot,
uint32_t partition, uint16_t segment ) const;
/**
* @brief Check whether a column exists or not by using file id. Since this
* is not enough to fully qualify a db filename, all it can do is to verify
* that the OID directory exists on one or more of the DBRoots.
*/
EXPORT bool existsOIDDir( FID fid ) const;
/**
* @brief Expand current abbreviated extent for this column to a full extent
*
* @param pFile FILE ptr of segment file we are updating.
* @param dbRoot DBRoot of the file being updated.
* @param emptyVal Empty value used in initializing extents for this column
* @param width Width of this column (in bytes)
*/
EXPORT virtual int expandAbbrevColumnExtent(
IDBDataFile* pFile,
uint16_t dbRoot,
uint64_t emptyVal,
int width );
/**
* @brief Add an extent to the specified Column OID and DBRoot.
* The extent must already exist in the extentmap prior to calling this fctn.
*
* The partition, segment, and HWM of the column file where the
* extent is added is returned. If needed, the applicable column segment
* file will be created. This extendFile1 function should supplant other
* extendFile functions with Multiple-File-per-OID enhancement, "but" we
* may want to rethink when we do Shared-Nothing. When this function
* returns, the file position will be located at the end of the file.
* For shared-everything DBRoot was an output argument, as BRM selected the
* the DBRoot. For shared-nothing DBRoot is an input argument, as the
* application code must track/control the DBRoot selection.
* If this is the very first file for the specified DBRoot, then the
* partition and segment number must be specified, else the selected
* partition and segment numbers are returned.
*
* @param oid OID of the column to be extended
* @param emptyVal Empty value to be used for oid
* @param width Width of the column
* @param hwm The fbo of the column segment file where the new extent begins
* @param startLbid The starting LBID for the new extent
* @param allocSize Number of blocks allocated to the extent.
* @param dbRoot The DBRoot of the file with the new extent.
* @param partition The partnum of the file with the new extent.
* @param segment The segnum of the file with the new extent.
* @param segFile (out) Name of the segment file where extent was added.
* @param pFile (out) FILE ptr to the file where the extent is added.
* @param newFile (out) Indicates if a new file was created for the extent
* @param hdrs (in/out) Contents of headers, if file is compressed.
* @return returns NO_ERROR if success.
*/
EXPORT int extendFile(OID oid, uint64_t emptyVal,
int width,
HWM hwm,
BRM::LBID_t startLbid,
int allocSize,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
std::string& segFile,
IDBDataFile*& pFile,
bool& newFile,
char* hdrs);
/**
* @brief For alter table add column; add an extent to a specific file
*
* @param oid OID of the column to be extended
* @param emptyVal Empty value to be used for oid
* @param width Width of the column
* @param allocSize (out) Number of blocks allocated to the extent.
* @param dbRoot The DBRoot of the file with the new extent.
* @param partition The partnum of the file with the new extent.
* @param segment The segnum of the file with the new extent.
* @param segFile (out) Name of the segment file where extent was added.
* @param startLbid (out) The starting LBID for the new extent
* @param newFile (out) Indicates if a new file was created for the extent
* @param hdrs (in/out) Contents of headers, if file is compressed.
*/
EXPORT int addExtentExactFile(OID oid, uint64_t emptyVal,
int width,
int& allocSize,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType,
std::string& segFile,
BRM::LBID_t& startLbid,
bool& newFile,
char* hdrs);
/**
* @brief Pad the specified compressed extent with empty chunks
* @param oid OID of relevant column
* @param width Width in bytes of this column
* @param emptyVal Empty value to be employed in filling the chunks
* @param dbRoot DBRoot of the extent to be filled
* @param partition Partition of the extent to be filled
* @param segment Segment file number of the extent to be filled
* @param hwm New HWM blk setting for the segment file after extent is padded
* @param segFile (out) Name of updated segment file
* @param errTask (out) Task that failed if error occurs
* @return returns NO_ERROR if success.
*/
EXPORT int fillCompColumnExtentEmptyChunks(OID oid,
int colWidth,
uint64_t emptyVal,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM hwm,
std::string& segFile,
std::string& errTask);
/**
* @brief Write the specified header info to compressed column file pFile.
*
* @param pFile Column file to be written to
* @param hdr Header info to be written
*/
EXPORT int writeHeaders(IDBDataFile* pFile, const char* hdr) const;
/**
* @brief Write the specified header info to compressed column or
* dictionary file pFile.
*
* @param pFile Column file to be written to
* @param controlHdr Control header info to be written
* @param pointerHdr Pointer header info to be written
* @param ptrHdrSize Size (in bytes) of pointerHdr
*/
EXPORT int writeHeaders(IDBDataFile* pFile,
const char* controlHdr,
const char* pointerHdr,
uint64_t ptrHdrSize) const;
/**
* @brief Get the Version Buffer filename for the specified fid (OID).
*
* This version of getFileName automatically uses 0 for the partition and
* segment numbers. The applicable DBRoot is assigned based on the OID.
*
* @param fid (in) OID of the Version Buffer DB file of interest
* @param fileName (out) the name of the pertinent file that was found
*
* @return returns NO_ERROR if success; ERR_FILE_NOT_EXIST if file not found
*/
int getVBFileName( FID fid, char* fileName ) const;
/**
* @brief Get the filename for the specified fid (OID). DBRoot, partition,
* and segment number.
*
* @param fid (in) OID of the DB file of interest
* @param fileName (out) the name of the pertinent file that was found
* @param dbRoot (in) DBRoot of the file of interest. If 0, then all the
* DBRoots will be searched.
* @param partition (in) partition number of the file of interest
* @param segment (in) segment number of the file of interest
*/
int getFileName( FID fid, char* fileName,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment ) const;
/**
* @brief Construct directory path for the specified fid (OID), DBRoot, and
* partition number. Directory does not have to exist, nor is it created.
*/
int getDirName( FID fid, uint16_t dbRoot,
uint32_t partition,
std::string& dirName) const;
/**
* @brief Get the file size
*/
EXPORT int getFileSize( IDBDataFile* pFile, long long& fileSize ) const;
EXPORT int getFileSize( FID fid, uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
long long& fileSize ) const;
/**
* @brief Initialize an extent in a dictionary store file
* @param pFile (in) IDBDataFile* of dictionary store file to be written to
* @param dbRoot (in) - DBRoot of pFile
* @param nBlocks (in) - number of blocks to be written for an extent
* @param blockHdrInit(in) - data used to initialize each block header
* @param blockHdrInitSize(in) - number of bytes in blockHdrInit
* @param bExpandExtent (in) - Expand existing extent, or initialize new one
*/
EXPORT int initDctnryExtent( IDBDataFile* pFile,
uint16_t dbRoot,
int nBlocks,
unsigned char* blockHdrInit,
int blockHdrInitSize,
bool bExpandExtent );
/**
* @brief Check whether it is an directory
*/
EXPORT bool isDir( const char* dirName ) const;
/**
* @brief See if there is room in the file system for specific number of blks
* @param fileName Name of file to extend (does not have to be full name)
* @param nBlocks Number of 8192-byte blocks to be added
* @return returns TRUE if file system has room for 'nBlocks', else FALSE
*/
EXPORT bool isDiskSpaceAvail(const std::string& fileName,
int nBlocks) const;
/**
* @brief Convert an oid to a full file name
*/
EXPORT int oid2FileName( FID fid, char* fullFileName,
bool bCreateDir, uint16_t dbRoot,
uint32_t partition, uint16_t segment ) const;
EXPORT int oid2DirName( FID fid, char* oidDirName ) const;
/**
* @brief Open a file using a filename.
* @param fileName Name of the file to open.
* @param mode Mode to use in opening the file (ex: "r+b").
* @param ioBuffSize Buffer size to be employed by setvbuf().
* @return returns the IDBDataFile* of the opened file.
*/
EXPORT IDBDataFile* openFile( const char* fileName,
const char* mode = "r+b",
int ioColSize = DEFAULT_COLSIZ,
bool useTmpSuffix = false) const;
/**
* @brief Open a file using an OID, dbroot, partition, and segment number.
* @param fid OID of the file to be opened.
* @param dbRoot DBRoot of the file to be opened.
* @param partition Partition number of the file to be opened.
* @param segment Segment number of the file to be opened.
* @param mode Mode to use in opening the file (default of "r+b" will open
* an existing binary file as read/write.
* @param ioBuffSize Buffer size to be employed by setvbuf().
* @return returns the IDBDataFile* of the opened file.
*/
EXPORT IDBDataFile* openFile( FID fid,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
std::string& segFile,
const char* mode = "r+b",
int ioColSize = DEFAULT_COLSIZ,
bool useTmpSuffix = false) const;
/**
* @brief Read to a buffer from a file at current location
*/
EXPORT int readFile( IDBDataFile* pFile, unsigned char* readBuf,
int readSize ) const;
/**
* @brief Reads in 2 compression header blocks from a column segment file.
* IDBDataFile* points to start of data when function returns.
* @param pFile (in) IDBDataFile* of column segment file to be read.
* @param hdrs (out) Contents of headers that are read.
*/
EXPORT int readHeaders( IDBDataFile* pFile, char* hdrs ) const;
EXPORT int readHeaders( IDBDataFile* pFile, char* hdr1, char* hdr2 )const;
/**
* @brief Reinitialize a partial extent in a column segment file
* @param pFile (in) IDBDataFile* of column segment file to be written to
* @param startOffset (in) - file offset where blocks are to be written
* @param nBlocks (in) - number of blocks to be written to the extent
* @param emptyVal(in) - empty value to be used for column data values
* width (in) - width of the applicable column
*/
EXPORT int reInitPartialColumnExtent( IDBDataFile* pFile,
long long startOffset,
int nBlocks,
uint64_t emptyVal,
int width );
/**
* @brief Reinitialize an extent in a dictionary store file
* @param pFile (in) IDBDataFile* of dictionary store file to be written to
* @param startOffset (in) - file offset where blocks are to be written
* @param nBlocks (in) - number of blocks to be written to the extent
* @param blockHdrInit(in) - data used to initialize each block header
* @param blockHdrInitSize(in) - number of bytes in blockHdrInit
*/
EXPORT int reInitPartialDctnryExtent( IDBDataFile* pFile,
long long startOffset,
int nBlocks,
unsigned char* blockHdrInit,
int blockHdrInitSize );
/**
* @brief Set the file to specified location based on the offset
*/
EXPORT int setFileOffset( IDBDataFile* pFile,
long long offset,
int origin = SEEK_SET ) const;
EXPORT int setFileOffsetBlock( IDBDataFile* pFile,
uint64_t lbid,
int origin = SEEK_SET ) const;
/**
* @brief Truncate the file to the specified file size
*/
EXPORT int truncateFile( IDBDataFile* pFile,
long long fileSize ) const;
/**
* @brief Write a buffer to a file at current location
*/
EXPORT int writeFile( IDBDataFile* pFile,
const unsigned char* buf, int bufSize ) const;
/**
* @brief set the flag to use the instance to access the brm wrapper class
*/
EXPORT virtual void setTransId( const TxnID& transId);
EXPORT virtual void setBulkFlag(bool isBulkLoad);
EXPORT virtual void setFixFlag(bool isFix);
TxnID getTransId() const;
void compressionType(int t);
int compressionType() const;
EXPORT virtual int flushFile(int rc, std::map<FID,FID> & oids);
protected:
EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks);
EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks);
int m_compressionType; // compresssion type
private:
//not copyable
FileOp(const FileOp& rhs);
FileOp& operator=(const FileOp& rhs);
int createFile( const char* fileName, int fileSize,
uint64_t emptyVal, int width,
uint16_t dbRoot );
int expandAbbrevColumnChunk( IDBDataFile* pFile,
uint64_t emptyVal,
int colWidth,
const compress::CompChunkPtr& chunkInPtr,
compress::CompChunkPtr& chunkOutPt);
int initAbbrevCompColumnExtent( IDBDataFile* pFile,
uint16_t dbRoot,
int nBlocks,
uint64_t emptyVal,
int width);
// Initialize an extent in a column segment file
// pFile (in) IDBDataFile* of column segment file to be written to
// dbRoot (in) - DBRoot of pFile
// nBlocks (in) - number of blocks to be written for an extent
// emptyVal(in) - empty value to be used for column data values
// width (in) - width of the applicable column
// bNewFile (in) - Adding extent to new file
// bExpandExtent (in) - Expand existing extent, or initialize new one
// bAbbrevExtent (in) - If adding new extent, is it abbreviated
int initColumnExtent( IDBDataFile* pFile,
uint16_t dbRoot,
int nBlocks,
uint64_t emptyVal,
int width,
bool bNewFile,
bool bExpandExtent,
bool bAbbrevExtent );
static void initDbRootExtentMutexes();
static void removeDbRootExtentMutexes();
int writeInitialCompColumnChunk( IDBDataFile* pFile,
int nBlocksAllocated,
int nRows,
uint64_t emptyVal,
int width,
char* hdrs);
TxnID m_transId;
bool m_isBulk;
bool m_isFix;
// protect creation of m_DbRootAddExtentMutexes
static boost::mutex m_createDbRootMutexes;
// Mutexes used to serialize extent creation within each DBRoot
static std::map<int,boost::mutex*> m_DbRootAddExtentMutexes;
// protect race condition in creating directories
static boost::mutex m_mkdirMutex;
char* m_buffer; // buffer used with setvbuf()
};
//------------------------------------------------------------------------------
// Inline functions
//------------------------------------------------------------------------------
inline void FileOp::compressionType(int t)
{
m_compressionType = t;
}
inline int FileOp::compressionType() const
{
return m_compressionType;
}
inline int FileOp::createDir( const char* dirName ) const
{
return createDir( dirName, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH );
}
inline int FileOp::getVBFileName( FID fid, char* fileName ) const
{
uint16_t dbRoot = 0;
uint32_t partition = 0;
uint16_t segment = 0;
return oid2FileName( fid, fileName, true, dbRoot, partition, segment );
}
inline int FileOp::getFileName( FID fid, char* fileName,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment ) const
{
return oid2FileName( fid, fileName, false, dbRoot, partition, segment );
}
inline TxnID FileOp::getTransId() const
{
return m_transId;
}
} //end of namespace
#undef EXPORT
#endif // _WE_FILEOP_H_

351
writeengine/shared/we_index.h Normal file
View File

@ -0,0 +1,351 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: we_index.h 4450 2013-01-21 14:13:24Z rdempsey $
*
******************************************************************************************/
/** @file */
#ifndef _WE_INDEX_H_
#define _WE_INDEX_H_
#include <bitset>
#include "we_type.h"
/** Namespace WriteEngine */
namespace WriteEngine
{
/*****************************************************
* index definition
******************************************************/
const int IDX_BITTEST_SIZE = 10; /** @brief The bit size of bit test */
const int IDX_GROUP_SIZE = 3; /** @brief The bit size of group */
const int IDX_INSTRU_SIZE = 4; /** @brief The bit size of instruction */
const int IDX_PTR_SIZE = 46; /** @brief The bit size of address pointer */
const int IDX_TYPE_SIZE = 3; /** @brief The bit size of type */
const int IDX_BITMAP_SUBBLOCK_NO = 1; /** @brief Subblock 1 of root block is for bitmap pointer*/
const int IDX_MAX_TREE_LEVEL = 128; /** @brief The maximum depth of a tree */
const int IDX_MAX_MULTI_COL_BIT = 256; /** @brief The maximum bits of a multi-column tree (256 bit)*/
const int IDX_MAX_MULTI_COL_IDX_LEVEL = 52; /** @brief The maximum depth of a multi-column tree */
const int IDX_MAX_MULTI_COL_IDX_NUM = 64; /** @brief The maximum number of columns for a multi-column index */
const int MAX_IDX_RID = 1024; /** @brief Maximum index rids for one shot */
const int IDX_DEFAULT_READ_ROW = 10000; /** @brief Default number of rows for one read for index */
// todo: need to move a higher level share file for dictionary
const int RID_SIZE = 46;
// const int OID_SIZE = 24; /** @brief The bit size of object id */
const int FBO_SIZE = 36; /** @brief The bit size of file block offset */
const int SBID_SIZE = 5; /** @brief The bit size of sub block id */
const int ENTRY_SIZE = 5; /** @brief The bit size of entry location with a sub block */
const int LIST_SIZE_TYPE = 0;
const int LIST_RID_TYPE = 3;
const int LIST_NOT_USED_TYPE = 7;
const int LIST_HDR_SIZE = 32;
const int LIST_SUBBLOCK_TYPE = 4 ;
const int LIST_BLOCK_TYPE = 5 ;
const int LIST_LLP_TYPE = 6 ;
const int SUBBLOCK_TOTAL_BYTES = 256;
const int LIST_SUB_LLP_POS = 31;
const int LIST_LAST_LBID_POS = 30;
const int LIST_BLOCK_LLP_POS = 1023;
const int MAX_BLOCK_ENTRY = 1024;
const int MAX_SUB_RID_CNT = 30;
const int MAX_BLK_RID_CNT = 1023;
const int MAX_BLK_NARRAY_RID_CNT = 1018;
const int LBID_SBID_ENTRY = 46;
const int RID_COUNT_SIZE = 10;
const int CUR_BLK_POS_WIDTH = 2;
const int LLP_STATUS_WIDTH = 2;
const int LIST_ENTRY_WIDTH = 8;
const int LIST_BLK_LLP_ENTRY_WIDTH= 48;
const int BEGIN_LIST_BLK_LLP_POS = 1018;
const int NEXT_BLK_PTR_OFFSET = 5;
const int PARENT_PTR_OFFSET = 4;
const int TOTAL_NUM_ARRAY_PTR = 4;
const int ARRAY_LLP_EXIST = 1;
const int LLP_NOT_FULL = 0;
const int LLP_FULL = 1;
const int TOTAL_CUR_LEVEL = 10;
const int CUR_LEVEL_POS_WIDTH = 20;
const uint64_t INVALID_KEY = -1LL; /** @brief Invalid number */
/*****************************************************
* mask definition
******************************************************/
const int BIT_MASK_ARRAY[] = { 0x0,
0x01, /** @brief 1 bit mask */
0x03, /** @brief 2 bit mask */
0x07, /** @brief 3 bit mask */
0x0F, /** @brief 4 bit mask */
0x1F, /** @brief 5 bit mask */
0x3F /** @brief 6 bit mask */
};
/************************************************************************
* Type enumerations
************************************************************************/
enum IdxTreeEntryType { /** @brief Index tree entry types */
EMPTY_ENTRY = 0, /** @brief Empty entry */
UNIQUE_VAL = 7, /** @brief Unique value */
EMPTY_LIST = 1, /** @brief Empty list pointer entry */
EMPTY_PTR = 2, /** @brief Empty pointer entry */
BIT_TEST = 3, /** @brief Bit test entry */
LEAF_LIST = 4, /** @brief Leaf list pointer */
BITMAP_PTR = 5, /** @brief Bitmap pointer */
// SORT_LIST = 5, /** @brief Sorted list pointer */
MULTI_COL = 6 /** @brief Multi-column index pointer */
};
enum IdxTreeGroupType { /** @brief Index tree group types */
ENTRY_1 = 0, /** @brief 1 entry per group */
ENTRY_2 = 1, /** @brief 2 entry per group */
ENTRY_4 = 2, /** @brief 4 entry per group */
ENTRY_8 = 3, /** @brief 8 entry per group */
ENTRY_16 = 4, /** @brief 16 entry per group */
ENTRY_32 = 5, /** @brief 32 entry per group */
ENTRY_BLK = 6 /** @brief 1k entry per group */
};
enum IdxBitCompareType { /** @brief Index bit compare types */
BIT_5 = 0, /** @brief 5-bit compare */
BIT_10 = 1 /** @brief 10-bit compare */
};
enum IdxFreeMgrType { /** @brief Index free manager types */
TREE = 0, /** @brief Index tree type */
LIST = 1 /** @brief Index list type */
};
/************************************************************************
* @brief index defintions
************************************************************************/
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 12; /** @brief spare bits */
uint64_t group : IDX_GROUP_SIZE; /** @brief entry group type */
// The following is related to ptr
uint64_t fbo : FBO_SIZE; /** @brief file block offset */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxStartSubBlockEntry; /** @brief Index start block entry structure */
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 2; /** @brief spare bits */
uint64_t group : IDX_GROUP_SIZE; /** @brief entry group type */
// The following is related to ptr
uint64_t spare2 : 10; /** @brief spare bits */
uint64_t fbo : FBO_SIZE; /** @brief file block offset */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxEmptyListEntry; /** @brief Index empty list entry structure */
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
// The following is related to ptr
uint64_t fbo : FBO_SIZE; /** @brief file block offset */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxBitmapPointerEntry; /** @brief Index bitmap pointer entry structure */
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t bitTest : IDX_BITTEST_SIZE; /** @brief index bittest */
uint64_t group : IDX_GROUP_SIZE; /** @brief entry group type */
uint64_t bitCompare : 1;
uint64_t spare : 1; /** @brief spare bits */
// The following is related to ptr
uint64_t fbo : FBO_SIZE; /** @brief file block offset */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxBitTestEntry; /** @brief Index bit test entry structure */
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
// The following is related to ptr
uint64_t fbo : FBO_SIZE; /** @brief file block offset */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxTreePointerEntry; /** @brief Index tree pointer entry structure */
/************************************************************************
* @brief index list node defintions
************************************************************************/
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type 3 */
uint64_t spare : 15; /** @brief spare bits */
RID rid : RID_SIZE; /** @brief row id */
} IdxRidListEntry; /** @brief Index rid list entry structure */
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 5;
uint64_t count : RID_COUNT_SIZE; /** the count of rids on the current blk */
uint64_t llp : LBID_SBID_ENTRY; /** @brief size */
} IdxRidListPtr;
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 5;
uint64_t count : RID_COUNT_SIZE; /** the count of rids on the current blk */
uint64_t lbid : FBO_SIZE; /** @brief size */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxRidLastListPtr;
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 13;
uint64_t llpStat : LLP_STATUS_WIDTH; /** llp status */
uint64_t childLbid : FBO_SIZE; /** @brief file block offset */
uint64_t spare2 : 10;
} IdxRidChildListPtr;
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type 0 or 6 */
uint64_t spare : 5;
uint64_t count : RID_COUNT_SIZE; /** the count of rids on the current blk */
uint64_t nextLbid : FBO_SIZE; /** @brief file block offset */
uint64_t curLevel : TOTAL_CUR_LEVEL;
} IdxRidNextListPtr;
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type 6*/
uint64_t spare : 3; /** @brief spare bits */
uint64_t curLevelPos : CUR_LEVEL_POS_WIDTH;
uint64_t curBlkPos : CUR_BLK_POS_WIDTH; /** the position of current blk */
uint64_t parentLbid : FBO_SIZE; /** @brief file block offset */
} IdxRidParentListPtr;
typedef struct {
IdxRidChildListPtr childIdxRidListPtr[4];
IdxRidParentListPtr parentIdxListPtr;
IdxRidNextListPtr nextIdxListPtr;
} IdxRidListArrayPtr;
/************************************************************************
* @brief index list header defintions
************************************************************************/
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
uint64_t size : RID_SIZE; /** @brief size */
} IdxRidListHdrSize;
typedef struct {
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
uint64_t llp : RID_SIZE; /** @brief size */
} IdxRidListHdrPtr;
typedef struct {
IdxRidListHdrSize idxRidListSize;
uint64_t key;
IdxRidListEntry firstIdxRidListEntry;
IdxRidListHdrPtr nextIdxRidListPtr;
} IdxRidListHdr;
typedef struct {
uint64_t part1 : 15; /** @brief entry type */
uint64_t part2 : 15; /** @brief spare bits */
uint64_t spare : 34; /** @brief size */
} IdxRidListOffSet;
/************************************************************************
* @brief index tree node defintions
************************************************************************/
typedef struct {
IdxBitTestEntry next; /** @brief next in the node */
IdxBitTestEntry current; /** @brief current addr */
uint16_t level; /** @brief tree level */
uint16_t allocCount; /** @brief allocated entry cound from free mgr */
uint16_t useCount; /** @brief actual use entry count */
uint16_t offset; /** @brief entry offset */
bool used; /** @brief used flag */
} IdxTreeNode; /** @brief Index tree node */
typedef struct {
IdxTreeNode node[IDX_MAX_TREE_LEVEL]; /** @brief node array */
uint16_t maxLevel; /** @brief max level */
RID rid; /** @brief current row id */
uint64_t key; /** @brief current key */
uint16_t width; /** @brief current width */
} IdxTree; /** @brief Index tree */
struct IdxTreeCacheNode {
RID rid; /** @brief RID */
uint64_t key; /** @brief Key */
IdxEmptyListEntry entry; /** @brief List pointer */
bool used; /** @brief Used flag */
IdxTreeCacheNode() { used = false; }
};
struct IdxMultiColKey {
std::bitset<IDX_MAX_MULTI_COL_BIT> bitSet; /** @brief BitArray for all bits */
std::bitset<IDX_MAX_MULTI_COL_BIT> curBitset;/** @brief Current working column */
std::bitset<IDX_MAX_MULTI_COL_BIT> curMask; /** @brief Current bitset mask */
unsigned char keyBuf[IDX_MAX_MULTI_COL_BIT/8]; /** @brief Key buffer */
int curLevel; /** @brief Current index level */
int maxLevel; /** @brief Maximum index level */
int totalBit; /** @brief Total bits */
int testbitArray[IDX_MAX_MULTI_COL_IDX_LEVEL]; /** @brief Test bit array */
void clear() { bitSet.reset(); curBitset.reset(); curMask.reset();
curLevel = maxLevel = 0; totalBit = 0;
memset( testbitArray, 0, IDX_MAX_MULTI_COL_IDX_LEVEL); memset( keyBuf, 0, IDX_MAX_MULTI_COL_BIT/8 );
curMask = 0x1F; curMask = curMask << (IDX_MAX_MULTI_COL_BIT - 5);
}
IdxMultiColKey() { clear(); }
};
struct IdxMultiRid {
RID* ridArray; /** @brief RID array */
int totalRid; /** @brief Total number of row id */
IdxMultiRid() { totalRid = 0; ridArray = NULL; }
void setMultiRid( RID* rids, const int size ) {
totalRid = size;
ridArray = rids;
/* ridArray = new RID[size];
memcpy( ridArray, rids, size * sizeof( RID ) ); */
}
void clearMultiRid() { /*if( ridArray != NULL ) delete [] ridArray; ridArray = NULL;*/ } // we don't want to get into this mem business
};
struct IdxLoadParam {
File sourceFile; /** @brief Source file contatin values */
OID indexTreeOid; /** @brief Target index tree oid */
OID indexListOid; /** @brief Target index list oid */
execplan::CalpontSystemCatalog::ColDataType indexColDataType; /** @brief Target index column type */
int indexWidth; /** @brief Target index width */
int maxLoadRow; /** @brief Max rows for one load */
void setIdxLoadParam( const OID treeOid, const OID listOid, const execplan::CalpontSystemCatalog::ColDataType colDataType, const int width, const int maxRow )
{ indexTreeOid = treeOid; indexListOid = listOid; indexColDataType = colDataType;
indexWidth = width; maxLoadRow = maxRow; }
bool isValid() { return indexTreeOid && indexListOid && indexWidth && maxLoadRow; }
IdxLoadParam() { indexTreeOid = indexListOid = indexWidth = maxLoadRow = 0; }
};
} //end of namespace
#endif // _WE_INDEX_H_

244
writeengine/shared/we_log.cpp Normal file
View File

@ -0,0 +1,244 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_log.cpp 4504 2013-02-02 00:07:43Z bpaul $
*
*******************************************************************************/
#include "we_log.h"
#include "messageids.h"
#include "we_define.h"
#include "we_simplesyslog.h"
#include "we_convertor.h"
namespace WriteEngine
{
WriteEngine::WErrorCodes ec; // referenced as extern by chunkmanager
//------------------------------------------------------------------------------
// Constructor
//------------------------------------------------------------------------------
Log::Log() : m_bConsoleOutput( true ),
m_logFileName( "" ),
m_errlogFileName( "" )
{
m_pid = ::getpid();
}
//------------------------------------------------------------------------------
// Destructor
//------------------------------------------------------------------------------
Log::~Log()
{
m_logFile.close();
m_errLogFile.close();
}
//------------------------------------------------------------------------------
// DESCRIPTION:
// Format a message to be logged
// PARAMETERS:
// msg - message
// level - message level
// oss - formatted msg
// code - return status code to include in the message
// RETURN:
// none
//------------------------------------------------------------------------------
void Log::formatMsg( const std::string& msg,
MsgLevel level,
std::ostringstream& oss,
int code ) const
{
// Constructing and logging the entire message as one string, should
// help avoid any thread contention that could cause logging output
// to be interweaved between threads.
oss << Convertor::getTimeStr();
// Include thread id in log message based on debug level
if (isDebug( DEBUG_2 ))
{
oss << " (" << m_pid << ":" <<
#ifdef _MSC_VER
GetCurrentThreadId()
#else
pthread_self()
#endif
<< ") " <<
MSG_LEVEL_STR[level] << " : " << msg ;
}
else
{
oss << " (" << m_pid << ") " << MSG_LEVEL_STR[level] << " : " << msg ;
}
if( code > 0 )
oss << " [" << code << "]";
}
//------------------------------------------------------------------------------
// DESCRIPTION:
// Log a message to the log file and to the console.
// We used to log error and critical error msgs to error log only.
// But changed to log to m_logFile too, because it's easy to
// forget to look at the err log. And seeing the error log msg
// in the correct timeline context in the m_logFile helps to see
// what else was going on when the problem occurred.
// PARAMETERS:
// msg - message
// code - return status code to include in the message
// level - message level
// RETURN:
// none
//------------------------------------------------------------------------------
void Log::logMsg( const char* msg,
int code,
MsgLevel level )
{
std::ostringstream oss;
formatMsg( msg, level, oss, code );
// log error and critical msgs to syslog
if( level == MSGLVL_ERROR || level == MSGLVL_CRITICAL )
{
{ //log to log file and error log file within scope of mutex lock.
//logSyslog uses SimpleSyslog which has it's own lock.
boost::mutex::scoped_lock lk(m_WriteLockMutex);
m_errLogFile << oss.str() << std::endl;
m_logFile << oss.str() << std::endl;
std::cerr << oss.str() << std::endl;
}
logSyslog( std::string(msg), code );
}
else
{
std::ostringstream oss2;
// Format msg again without including the status code.
// Only log INFO2 msgs to console if m_bConsoleOutput is TRUE;
// All other msg levels always go to console.
if( (level != MSGLVL_INFO2) || (m_bConsoleOutput) )
formatMsg ( msg, level, oss2 );
boost::mutex::scoped_lock lk(m_WriteLockMutex);
m_logFile << oss.str() << std::endl;
if( (level != MSGLVL_INFO2) || (m_bConsoleOutput) )
std::cout << oss2.str() << std::endl;
}
}
//------------------------------------------------------------------------------
// DESCRIPTION:
// Set log file name
// PARAMETERS:
// logfile - log file name
// errlogfile - error log file name
// RETURN:
// none
//------------------------------------------------------------------------------
void Log::setLogFileName( const char* logfile,
const char* errlogfile,
bool consoleFlag )
{
m_logFileName = logfile;
m_errlogFileName = errlogfile;
m_bConsoleOutput = consoleFlag;
#ifdef _MSC_VER
// cpimport.bin calls BulkLoad::loadJobInfo() before calling
// BulkLoad::processJob(). loadJobInfo() attempts to write to this log
// before it's opened (by processJob()). This doesn't seem to bother Linux
// but puts Windows in a bad state. Once this logic is fixed, this hack can
// go away.
// This code probably wouldn't hurt if run on Linux, but I'll leave this
// here as a reminder to fix the logic for all platforms.
m_logFile.close();
m_logFile.clear();
m_errLogFile.close();
m_errLogFile.clear();
#endif
m_logFile.open( m_logFileName.c_str(),
std::ofstream::out | std::ofstream::app );
m_errLogFile.open(m_errlogFileName.c_str(),
std::ofstream::out | std::ofstream::app);
}
//------------------------------------------------------------------------------
// DESCRIPTION:
// Log specified (error or critical) message to syslog error log.
// PARAMETERS:
// msg - message
// statusCode - WriteEngine return status code to include in the message
// RETURN:
// none
//------------------------------------------------------------------------------
void Log::logSyslog( const std::string& msg,
int statusCode )
{
logging::Message::MessageID msgId = logging::M0087;
switch (statusCode)
{
case ERR_FILE_DISK_SPACE:
{
msgId = logging::M0076;
break;
}
case ERR_UNKNOWN:
{
msgId = logging::M0017;
break;
}
default:
{
msgId = logging::M0087;
break;
}
}
logging::Message::Args errMsgArgs;
errMsgArgs.add( msg );
SimpleSysLog::instance()->logMsg(
errMsgArgs,
logging::LOG_TYPE_ERROR,
msgId);
}
//------------------------------------------------------------------------------
// DESCRIPTION:
// BUG 5022
// Close the log files with out calling d'tor. That way we can use the
// object again for logging while importing another table or so
// PARAMETERS:
// none
// RETURN:
// none
//------------------------------------------------------------------------------
void Log::closeLog()
{
m_logFile.close();
m_errLogFile.close();
}
} //end of namespace

120
writeengine/shared/we_log.h Normal file
View File

@ -0,0 +1,120 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_log.h 4504 2013-02-02 00:07:43Z bpaul $
*
*******************************************************************************/
/** @file */
#ifndef _WE_LOG_H_
#define _WE_LOG_H_
#include <time.h>
#include <sys/types.h>
#include <unistd.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <we_obj.h>
#include <boost/thread.hpp>
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
const std::string MSG_LEVEL_STR[] = {
"INFO",
"INFO",
"WARN",
"ERR ",
"CRIT" };
/** @brief Class is used to format and write log messages to cpimport.bin log
* file. When applicable, messages are also logged to syslog logs as well.
*/
class Log : public WEObj
{
public:
/**
* @brief Constructor
*/
EXPORT Log();
/**
* @brief Destructor
*/
EXPORT ~Log();
/**
* @brief Log a cpimport.bin logfile message; logs errors to syslog as well
*/
EXPORT void logMsg( const char* msg, int code, MsgLevel level );
EXPORT void logMsg( const char* msg, MsgLevel level )
{ logMsg( msg, 0, level ); }
EXPORT void logMsg( const std::string& msg, MsgLevel level )
{ logMsg( msg.c_str(), level ); }
EXPORT void logMsg( const std::string& msg, int code, MsgLevel level )
{ logMsg( msg.c_str(), code, level ); }
/**
* @brief Set log file name
*/
EXPORT void setLogFileName( const char* logfile,
const char* errlogfile,
bool consoleFlag = true );
// BUG 5022
/**
* @brief Set log files close other than calling d'tor
*/
EXPORT void closeLog();
private:
void logSyslog ( const std::string& msg,
int statusCode);
void formatMsg( const std::string& msg,
MsgLevel level,
std::ostringstream& oss,
int code = 0 ) const;
bool m_bConsoleOutput; // flag allowing INFO2 msg
// to display to console
std::string m_logFileName; // log file name
std::string m_errlogFileName; // error log file name
pid_t m_pid; // current pid
std::ofstream m_logFile; // log file stream
std::ofstream m_errLogFile; // error log file stream
boost::mutex m_WriteLockMutex; // logging mutex
};
} //end of namespace
#undef EXPORT
#endif // _WE_LOG_H_

53
writeengine/shared/we_macro.h Normal file
View File

@ -0,0 +1,53 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: we_macro.h 33 2006-10-18 14:37:12Z wzhou $
*
******************************************************************************************/
/** @file */
#ifndef _WE_MACRO_H_
#define _WE_MACRO_H_
#include <we_type.h>
/** Namespace WriteEngine */
namespace WriteEngine
{
#define RETURN_ON_ERROR( statement ) \
{ int rcVal = (statement); \
if( rcVal != NO_ERROR ) \
return rcVal; }
#define RETURN_ON_NULL( obj, rc ) \
if( obj == NULL ) \
return rc;
#define RETURN_ON_WE_ERROR( oldRc, newRc ) \
if( oldRc != NO_ERROR ) \
return newRc;
#define RETURN_RC( oldRc, newRc ) \
if( oldRc != NO_ERROR ) \
return newRc; \
else \
return NO_ERROR;
} //end of namespace
#endif // _WE_MACRO_H_

81
writeengine/shared/we_obj.h Normal file
View File

@ -0,0 +1,81 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: we_obj.h 33 2006-08-17 10:31:20Z wzhou $
*
******************************************************************************************/
/** @file */
#ifndef _WE_OBJ_H_
#define _WE_OBJ_H_
//#include <we_type.h>
#include <we_macro.h>
/** Namespace WriteEngine */
namespace WriteEngine
{
class Log;
/** Class WEObj */
class WEObj
{
public:
/**
* @brief Constructor
*/
WEObj() : m_debugLevel( DEBUG_0 ), m_log( 0 ) {}
/**
* @brief Default Destructor
*/
~WEObj() {}
/**
* @brief Is it required to debug
*/
const bool isDebug( const DebugLevel level ) const { return level <= m_debugLevel; }
/**
* @brief Get debug level
*/
const DebugLevel getDebugLevel() const { return m_debugLevel; }
/**
* @brief Get Logger object
*/
Log* getLogger() const { return m_log; }
/**
* @brief Set debug level
*/
void setDebugLevel( const DebugLevel level ) { m_debugLevel = level; }
/**
* @brief Set debug logger and debug level
*/
void setLogger( Log* logger ) { m_log = logger; }
private:
DebugLevel m_debugLevel; // internal use debug level
Log* m_log; // logger object for debug output
};
} //end of namespace
#endif // _WE_OBJ_H_

1413
writeengine/shared/we_rbmetawriter.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

378
writeengine/shared/we_rbmetawriter.h Normal file
View File

@ -0,0 +1,378 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: we_rbmetawriter.h 4450 2013-01-21 14:13:24Z rdempsey $
*/
/** @file we_rbmetawriter.h
* Contains class to write HWM-related information used to rollback a
* cpimport.bin job that abnormally terminated, leaving the db in an
* inconsistent state.
* Class was moved from bulk directory for resuse by DML as part of Shared
* Nothing.
*/
#ifndef WE_RBMETAWRITER_H_
#define WE_RBMETAWRITER_H_
#include <string>
#include <sstream>
#include <set>
#include <map>
#include <vector>
#include <boost/thread/mutex.hpp>
#include "we_type.h"
#include "brmtypes.h"
#include "we_fileop.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
#define DBROOT_BULK_ROLLBACK_SUBDIR "bulkRollback"
namespace WriteEngine
{
class Log;
//------------------------------------------------------------------------------
/** @brief Class used to store Dictionary store file information used in backing
* up HWM chunks "as needed".
*/
//------------------------------------------------------------------------------
struct RBChunkInfo
{
OID fOid; // dctnry store OID containing relevant chunk
uint16_t fDbRoot; // dbroot, partition, segment of file
uint32_t fPartition; // containing relevant HWM chunk
uint16_t fSegment; //
HWM fHwm; // HWM block of interest
RBChunkInfo(OID oid, uint16_t dbRoot, uint32_t partition,
uint16_t segment, HWM hwm ) :
fOid(oid), fDbRoot(dbRoot), fPartition(partition),
fSegment(segment), fHwm(hwm) { }
};
class RBChunkInfoCompare
{
public:
bool operator()(const RBChunkInfo& lhs, const RBChunkInfo& rhs) const;
};
typedef std::set< RBChunkInfo, RBChunkInfoCompare > RBChunkSet;
//------------------------------------------------------------------------------
/** @brief Class to write HWM-related information to support bulk rollbacks.
*
* Should cpimport.bin terminate abnormally, leaving the db in an inconsistent
* state, then the information written by this class can be used to perform
* a bulk rollback, to restore the db to its previous state, prior to the
* execution of the import job.
*
* For SharedEveryThing:
* Note that column segment files, carry a logical concatenation of extents,
* so only the HWM of the last logical extent needs to be written to the meta
* data file, as the information about where the data stops (and needs to
* be rolled back) for the other segment files can be derived from knowing
* the last Local HWM.
*
* For Shared-Nothing:
* Column segment files only carry a logical concatenation of extents, within
* a DBRoot. So the HWM of the last logical extent in each DBRoot (for a PM)
* must be written to the meta data file. The information about where the data
* stops (and needs to be rolled back) for the other segment files within a
* DBRoot can be derived from knowing the last Local HWM per DBRoot.
* One meta data control file is created for each DBRoot, and in fact, the
* meta file is created under the corresponding DBRoot directory.
*
* In the case of dictionary store segment files, each store file is
* independent of the other segment store files for the same OID. So for
* dictionary store segment files, the HWM must be written for each segment
* file in the last partition for each DBRoot.
*
* API Usage:
* Public functions would typically be called in the following order:
* 1. Create and initialize Bulk Rollback Metadata object:
* RBMetaWriter()
* init()
* 2. Create primary meta data file (see note below);
* openMetaFile()
* writeColumnMetaData()
* writeDictionaryStoreMetaData()
* writeDictionaryStoreMetaNoDataMarker()
* closeMetaFile()
* 3. Backup necessary HWM chunks to backup chunk files:
* a. backupColumnHWMChunk()
* b. backupDctnryHWMChunk()
* 4. Delete meta data file and HWM chunk files at end of successful job:
* deleteFile()
* 5. Delete Bulk Rollback Metadata object:
* ~RBMetaWriter()
*
* Steps 2 and 3a have been replaced with a single call to saveBulkRollback-
* MetaData(). This is a helper function that takes care of all the calls in
* steps 2 and 3a. If an error should occur in saveBulkRollbackMetaData(),
* closeMetaFile() will automatically be called to close the meta data file.
*
* backupDctnryHWMChunk() must be thread-safe as step 3b can be executed in
* parallel by several threads for different dictionary columns.
*/
//------------------------------------------------------------------------------
class RBMetaWriter
{
public:
/** @brief RBMetaWriter constructor
* @param appDesc Description of application that is using RBMetaWriter
* @param logger Logger to be used for logging messages.
*/
EXPORT RBMetaWriter ( const std::string& appDesc,
Log* logger );
/** @brief RBMetaWriter destructor
*/
EXPORT ~RBMetaWriter ( ) { closeMetaFile ( ); }
/** @brief Initialize this RBMetaWriter object
* Warning: This function may throw a WeException.
*
* @param tableOID OID of the table whose state is to be saved.
* @param tableName Name of the table associated with tableOID.
*/
EXPORT void init ( OID tableOID,
const std::string& tableName );
/** @brief Make a backup copy of the specified HWM dictionary store chunk.
* This operation only applies to compressed columns. Backup may not be
* necessary. Return value indicates whether the specified file needs to
* be backed up or not.
* Warning: This function may throw a WeException.
*
* This function is thread-safe since concurrent calls could be made by
* different threads, each for a different dictionary column.
*
* @param dctnryOID column OID to be saved
* @param dbRoot current dbRoot of last local HWM for columnOID
* @param partition current partition of last local HWM for columnOID
* @param segment current segment of last local HWM for columnOID
* @return Indicates whether it is necessary to perform backup
*/
EXPORT bool backupDctnryHWMChunk (
OID dctnryOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment );
/** @brief Delete the rollback meta files associated with this table
* Warning: This function may throw a WeException.
*/
EXPORT void deleteFile ( );
/** @brief Helper function that creates the primary meta data file.
* Warning: This function may throw a WeException.
*
* See class description for more details.
* @param columns Vector of column information. The dataFile member in
* each columns entry should be filled in with HWM information about
* the start extent where rows will be added. This information is
* needed so that the HWM chunk in that extent can be backed up.
* @param dctnryStoreOids Vector of dictionary store OID associated with
* columns vector. dctnryStoreOid[n] should be 0 if columns[n] is
* not a dictionary column.
* @param dbRootHwmInfoVecCol Vector of EmDbRootHWMInfo_v objects obtained
* from multiple calls to DBRM::getDbRootHWMInfo(). There is one
* EmDbRootHWMInfo_v entry per column. Each
* EmDbRootHWMInfo_v object carries a vector of DBRoot, HWM, etc
* objects representing the current HWM extents for a column's
* DBRoots on the local PM.
*/
EXPORT void saveBulkRollbackMetaData(
const std::vector<Column>& columns,
const std::vector<OID>& dctnryStoreOids,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoVecCol );
/** @brief Verify that specified version record is for Version 3 */
static bool verifyVersion3(const char* versionRec);
/** @brief Verify that specified version record is for Version 4 */
static bool verifyVersion4(const char* versionRec);
/** @brief Verify that specified record type is a Column1 record */
static bool verifyColumn1Rec(const char* recType);
/** @brief Verify that specified record type is a Column2 record */
static bool verifyColumn2Rec(const char* recType);
/** @brief Verify that specified record type is a DStore1 record */
static bool verifyDStore1Rec(const char* recType);
/** @brief Verify that specified record type is a DStore2 record */
static bool verifyDStore2Rec(const char* recType);
private:
// disable copy constructor and assignment operator
RBMetaWriter(const RBMetaWriter&);
RBMetaWriter& operator=(const RBMetaWriter&);
// Make a backup copy of the specified HWM column chunk.
// This operation only applies to compressed columns.
// Warning: This function may throw a WeException.
// columnOID column OID to be saved
// dbRoot current dbRoot of last local HWM for columnOID
// partition current partition of last local HWM for columnOID
// segment current segment of last local HWM for columnOID
// lastLocalHwm current last local for column OID
void backupColumnHWMChunk (
OID columnOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM lastLocalHwm );
// This function must be thread-safe since it is called directly by
// backupDctnryHWMChunk(). Employed by non-hdfs.
void backupHWMChunk (
bool bColumnFile,
OID columnOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM lastLocalHwm );
// This function must be thread-safe since it is called directly by
// backupDctnryHWMFile(). Employed by hdfs.
void backupHWMFile (
bool bColumnFile,
OID columnOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM lastLocalHwm );
// Close the current meta data file.
EXPORT void closeMetaFile ( );
void createSubDir( const std::string& metaFileName );
void deleteSubDir( const std::string& metaFileName );
int getSubDirPath(const uint16_t dbRoot,
std::string& subDirPath ) const;
// Open a meta data file to save HWM bulk rollback info for tableOID
// Warning: This function may throw a WeException.
// dbRoot is the DBRoot of interest for the applicable table.
std::string openMetaFile ( uint16_t dbRoot );
// Rename temporary metadata control file(s) to the permanent name.
// Filenames are taken from fMetaFileNames.
// Warning: This function may throw a WeException.
void renameMetaFile( );
// Save column meta data to the currently open file.
// This is the Shared-Nothing version of this function.
// Warning: This function may throw a WeException.
// metaFileName name of metafile to be written
// columnOID column OID to be saved
// dbRoot current dbRoot of last local HWM for columnOID
// partition current partition of last local HWM for columnOID
// segment current segment of last local HWM for columnOID
// lastLocalHwm current last local for column OID
// colType type of columnOID
// colTypeName type name of columnOID
// colWidth width (in bytes) of columnOID
// compressionType compression type
void writeColumnMetaData (
const std::string& metaFileName,
bool withHWM,
OID columnOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM lastLocalHwm,
execplan::CalpontSystemCatalog::ColDataType colType,
const std::string& colTypeName,
int colWidth,
int compressionType );
// Save dictionary store meta data to the currently open file.
// This is the Shared-Nothing version of this function.
// dictionaryStoreOID dictionary store OID to be saved
// dbRoot dbRoot of store file
// partition partition of store file
// segment segment of store file
// localHwm current local HWM for specified partition and seg file
// compressionType compression type
void writeDictionaryStoreMetaData (
OID columnOID,
OID dictionaryStoreOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM localHwm,
int compressionType );
// For first extent stripe in a partition, this function is used to
// to log a marker to denote a trailing segment file that does not exist.
// This is the Shared-Nothing version of this function.
// dictionaryStoreOID dictionary store OID to be saved
// dbRoot dbRoot of store file
// partition partition of store file
// segment segment of store file
// compressionType compression type
void writeDictionaryStoreMetaNoDataMarker (
OID columnOID,
OID dictionaryStoreOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
int compressionType );
// This function must be thread-safe since it is called indirectly by
// backupDctnryHWMChunk() (through backupHWMChunk()).
int writeHWMChunk (
bool bColumnFile,
OID columnOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
const unsigned char* compressedOutBuf,
uint64_t chunkSize,
uint64_t fileSize,
HWM chunkHwm,
std::string& errMsg) const;
void printDctnryChunkList(const RBChunkInfo& rbChk, const char* action);
IDBDataFile* fMetaDataFile; // current meta data file to write
std::ostringstream fMetaDataStream; // adapter for IDBDataFile
std::map<uint16_t, std::string> fMetaFileNames;//map of dbroots to metafiles
std::string fAppDesc; // description of application user
Log* fLog; // import log file
bool fCreatedSubDir; // has subdir path been created
RBChunkSet fRBChunkDctnrySet; // Dctnry HWM chunk info
boost::mutex fRBChunkDctnryMutex;//Mutex lock for RBChunkSet
OID fTableOID; // OID of relevant table
std::string fTableName; // Name of relevant table
};
} //end of namespace
#undef EXPORT
#endif // WE_RBMETAWRITER_H_

106
writeengine/shared/we_simplesyslog.cpp Normal file
View File

@ -0,0 +1,106 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_simplesyslog.cpp 4607 2013-04-11 21:38:09Z rdempsey $
*
*******************************************************************************/
#include "we_simplesyslog.h"
#include "we_define.h"
#include "messagelog.h"
using namespace std;
namespace WriteEngine
{
/*static*/ SimpleSysLog* SimpleSysLog::fSysLogger = 0;
//------------------------------------------------------------------------------
// Singleton instance accessor (Warning, not thread-safe)
//------------------------------------------------------------------------------
/* static */
SimpleSysLog* SimpleSysLog::instance()
{
if ( !fSysLogger )
fSysLogger = new SimpleSysLog();
return fSysLogger;
}
//------------------------------------------------------------------------------
// SimpleSysLog constructor.
//------------------------------------------------------------------------------
SimpleSysLog::SimpleSysLog() : fLoggingID( SUBSYSTEM_ID_WE )
{
}
//------------------------------------------------------------------------------
// Reset LoggingID (in order to set the subsystem id)
//------------------------------------------------------------------------------
void SimpleSysLog::setLoggingID( const logging::LoggingID& loggingID )
{
fLoggingID = loggingID;
}
//------------------------------------------------------------------------------
// Log arguments (msgArgs) for specified msgId to the requested log (logType).
//------------------------------------------------------------------------------
void SimpleSysLog::logMsg( const logging::Message::Args& msgArgs,
logging::LOG_TYPE logType,
logging::Message::MessageID msgId )
{
logging::MessageLog ml( fLoggingID );
logging::Message m(msgId);
m.format(msgArgs);
boost::mutex::scoped_lock lk(fWriteLockMutex);
switch (logType)
{
case logging::LOG_TYPE_DEBUG:
{
ml.logDebugMessage(m);
break;
}
case logging::LOG_TYPE_INFO:
default:
{
ml.logInfoMessage(m);
break;
}
case logging::LOG_TYPE_WARNING:
{
ml.logWarningMessage(m);
break;
}
case logging::LOG_TYPE_ERROR:
{
ml.logErrorMessage(m);
break;
}
case logging::LOG_TYPE_CRITICAL:
{
ml.logCriticalMessage(m);
break;
}
}
}
} //end of namespace

82
writeengine/shared/we_simplesyslog.h Normal file
View File

@ -0,0 +1,82 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_simplesyslog.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
/** @file */
#ifndef _WE_SIMPLESYSLOG_H_
#define _WE_SIMPLESYSLOG_H_
#include <boost/thread.hpp>
#include "messagelog.h"
#include "messageobj.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
/**
* @brief SimpleSysLog class is a simple logger that only logs to syslog.
*
* Note that the instance() and setLoggingID() funtions are not thread-safe.
* They should be called once, upfront by the main thread to perform setup.
*/
class SimpleSysLog
{
public:
/**
* @brief Singleton accessor.
*/
EXPORT static SimpleSysLog* instance();
/**
* @brief Modify the LoggingID to be used. Mainly used to control the
* subsystem ID.
*/
EXPORT void setLoggingID( const logging::LoggingID& loggingID );
/**
* @brief Function that logs a syslog msg.
*/
EXPORT void logMsg( const logging::Message::Args& msgArgs,
logging::LOG_TYPE logType,
logging::Message::MessageID msgId );
private:
SimpleSysLog( );
SimpleSysLog( const SimpleSysLog& );
SimpleSysLog& operator= ( const SimpleSysLog& );
static SimpleSysLog* fSysLogger;
logging::LoggingID fLoggingID;
boost::mutex fWriteLockMutex; // logging mutex
};
#undef EXPORT
} //end of namespace
#endif // _WE_SIMPLESYSLOG_H_

230
writeengine/shared/we_stats.cpp Normal file
View File

@ -0,0 +1,230 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_stats.cpp 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
/** @file */
#include <we_stats.h>
using namespace std;
namespace WriteEngine
{
#ifdef PROFILE
/* static */ bool Stats::fProfiling = false;
/* static */ boost::mutex Stats::fRegisterReaderMutex;
/* static */ boost::mutex Stats::fRegisterParseMutex;
/* static */ std::vector<pthread_t> Stats::fReadProfThreads;
/* static */ std::vector<pthread_t> Stats::fParseProfThreads;
/* static */ std::vector<logging::StopWatch> Stats::fReadStopWatch;
/* static */ std::vector<logging::StopWatch> Stats::fParseStopWatch;
#endif
struct IoStats Stats::m_ioStats = { 0, 0 };
bool Stats::m_bUseStats = false;
/***********************************************************
* DESCRIPTION:
* Increase the counter for block read
* PARAMETERS:
* blockNum - the number of blocks
* RETURN:
* none
***********************************************************/
void Stats::incIoBlockRead( const int blockNum )
{
if( !m_bUseStats )
return;
m_ioStats.blockRead += blockNum;
}
/***********************************************************
* DESCRIPTION:
* Increase the counter for block write
* PARAMETERS:
* blockNum - the number of blocks
* RETURN:
* none
***********************************************************/
void Stats::incIoBlockWrite( const int blockNum )
{
if( !m_bUseStats )
return;
m_ioStats.blockWrite += blockNum;
}
#ifdef PROFILE
//-------------------------------------------------------------------------------
// Functions that follow are used for profiling using the StopWatch class
//-------------------------------------------------------------------------------
/***********************************************************
* DESCRIPTION:
* Enable/Initialize the profiling functions
* PARAMETERS:
* nReadThreads - number of read threads to be profiled
* nParseThreads - number of parse threads to be profiled
* RETURN:
* none
***********************************************************/
void Stats::enableProfiling(int nReadThreads, int nParseThreads)
{
fProfiling = true;
// @bug 2625: pre-reserve space for our vectors; else we could have a race
// condition whereby one parsing thread is adding itself to the vectors
// and thus "growing" the vector (in registerParseProfThread), at the
// same time that another parsing thread is reading the vector in parse-
// Event(). By pre-reserving the space, the vectors won't be growing,
// thus eliminating the problem with this race condition.
fReadProfThreads.reserve ( nReadThreads );
fReadStopWatch.reserve ( nReadThreads );
fParseProfThreads.reserve( nParseThreads );
fParseStopWatch.reserve ( nParseThreads );
}
/***********************************************************
* DESCRIPTION:
* Register the current thread as a Read thread to be profiled
* PARAMETERS:
* none
* RETURN:
* none
***********************************************************/
void Stats::registerReadProfThread( )
{
boost::mutex::scoped_lock lk(fRegisterReaderMutex);
fReadProfThreads.push_back( pthread_self() );
logging::StopWatch readStopWatch;
fReadStopWatch.push_back ( readStopWatch );
}
/***********************************************************
* DESCRIPTION:
* Register the current thread as a Parse thread to be profiled
* PARAMETERS:
* none
* RETURN:
* none
***********************************************************/
void Stats::registerParseProfThread( )
{
boost::mutex::scoped_lock lk(fRegisterParseMutex);
fParseProfThreads.push_back( pthread_self() );
logging::StopWatch parseStopWatch;
fParseStopWatch.push_back ( parseStopWatch );
}
/***********************************************************
* DESCRIPTION:
* Track the specified Read event in the current Read thread.
* PARAMETERS:
* eventString - string that identifies the event.
* start - boolean indicating whether the is the start or the
* end of the event. TRUE=>start FALSE=>end
* RETURN:
* none
***********************************************************/
void Stats::readEvent ( const std::string& eventString, bool start )
{
if (fProfiling)
{
pthread_t thread = pthread_self();
for (unsigned i=0; i<fReadProfThreads.size(); i++)
{
if (fReadProfThreads[i] == thread)
{
if (start)
fReadStopWatch[i].start( eventString );
else
fReadStopWatch[i].stop ( eventString );
break;
}
}
}
}
/***********************************************************
* DESCRIPTION:
* Track the specified Parse event in the current Parse thread.
* PARAMETERS:
* eventString - string that identifies the event.
* start - boolean indicating whether the is the start or the
* end of the event. TRUE=>start FALSE=>end
* RETURN:
* none
***********************************************************/
void Stats::parseEvent ( const std::string& eventString, bool start )
{
if (fProfiling)
{
pthread_t thread = pthread_self();
for (unsigned i=0; i<fParseProfThreads.size(); i++)
{
if (fParseProfThreads[i] == thread)
{
if (start)
fParseStopWatch[i].start( eventString );
else
fParseStopWatch[i].stop ( eventString );
break;
}
}
}
}
/***********************************************************
* DESCRIPTION:
* Print profiling results.
* PARAMETERS:
* none
* RETURN:
* none
***********************************************************/
void Stats::printProfilingResults ( )
{
if (fProfiling)
{
std::cout << endl;
for (unsigned j=0; j<fReadStopWatch.size(); j++)
{
std::cout << "Execution Stats for Read Thread " << j << " (" <<
fReadProfThreads[j] << ")" << std::endl <<
"-------------------------------" << std::endl;
fReadStopWatch[j].finish();
std::cout << std::endl;
}
for (unsigned j=0; j<fParseStopWatch.size(); j++)
{
std::cout << "Execution Stats for Parse Thread "<< j << " (" <<
fParseProfThreads[j] << ")" << std::endl <<
"--------------------------------" << std::endl;
fParseStopWatch[j].finish();
std::cout << std::endl;
}
}
}
#endif
} //end of namespace

163
writeengine/shared/we_stats.h Normal file
View File

@ -0,0 +1,163 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_stats.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
/** @file */
#ifndef _WE_STATS_H_
#define _WE_STATS_H_
#include <we_obj.h>
#ifdef PROFILE
#include <boost/thread/mutex.hpp>
#include "stopwatch.h"
#endif
/** Namespace WriteEngine */
namespace WriteEngine
{
struct IoStats
{
long blockRead;
long blockWrite;
};
/** Class Stats */
class Stats
{
public:
/**
* @brief Constructor
*/
Stats() {}
/**
* @brief Default Destructor
*/
~Stats() {}
/**
* @brief I/O
*/
static long getIoBlockRead() { return m_ioStats.blockRead; }
static long getIoBlockWrite() { return m_ioStats.blockWrite; }
static void incIoBlockRead( const int blockNum = 1 );
static void incIoBlockWrite( const int blockNum = 1 );
static bool getUseStats() { return m_bUseStats; }
static void setUseStats( const bool flag ) { m_bUseStats = flag; }
static IoStats m_ioStats; // IO
#ifdef PROFILE
// Prefined event labels
#define WE_STATS_ALLOC_DCT_EXTENT "AllocDctExtent"
#define WE_STATS_COMPACT_VARBINARY "CompactingVarBinary"
#define WE_STATS_COMPLETING_PARSE "CompletingParse"
#define WE_STATS_COMPLETING_READ "CompletingRead"
#define WE_STATS_COMPRESS_COL_INIT_ABBREV_EXT "CmpColInitAbbrevExtent"
#define WE_STATS_COMPRESS_COL_INIT_BUF "CmpColInitBuf"
#define WE_STATS_COMPRESS_COL_COMPRESS "CmpColCompress"
#define WE_STATS_COMPRESS_COL_FINISH_EXTENT "CmpColFinishExtent"
#define WE_STATS_COMPRESS_DCT_INIT_BUF "CmpDctInitBuf"
#define WE_STATS_COMPRESS_DCT_COMPRESS "CmpDctCompress"
#define WE_STATS_COMPRESS_DCT_SEEKO_CHUNK "CmpDctSeekOutputChunk"
#define WE_STATS_COMPRESS_DCT_WRITE_CHUNK "CmpDctWriteChunk"
#define WE_STATS_COMPRESS_DCT_SEEKO_HDR "CmpDctSeekOutputHdr"
#define WE_STATS_COMPRESS_DCT_WRITE_HDR "CmpDctWriteHdr"
#define WE_STATS_COMPRESS_DCT_BACKUP_CHUNK "CmpDctBackupChunk"
#define WE_STATS_CREATE_COL_EXTENT "CreateColExtent"
#define WE_STATS_CREATE_DCT_EXTENT "CreateDctExtent"
#define WE_STATS_EXPAND_COL_EXTENT "ExpandColExtent"
#define WE_STATS_EXPAND_DCT_EXTENT "ExpandDctExtent"
#define WE_STATS_FLUSH_PRIMPROC_BLOCKS "FlushPrimProcBlocks"
#define WE_STATS_INIT_COL_EXTENT "InitColExtent"
#define WE_STATS_INIT_DCT_EXTENT "InitDctExtent"
#define WE_STATS_OPEN_DCT_FILE "OpenDctFile"
#define WE_STATS_PARSE_COL "ParseCol"
#define WE_STATS_PARSE_DCT "ParseDct"
#define WE_STATS_PARSE_DCT_SEEK_EXTENT_BLK "ParseDctSeekExtentBlk"
#define WE_STATS_READ_INTO_BUF "ReadIntoBuf"
#define WE_STATS_RESIZE_OUT_BUF "ResizeOutBuf"
#define WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH "WaitForIntermediateFlush"
#define WE_STATS_WAIT_FOR_READ_BUF "WaitForReadBuf"
#define WE_STATS_WAIT_TO_COMPLETE_PARSE "WaitCompleteParse"
#define WE_STATS_WAIT_TO_COMPLETE_READ "WaitCompleteRead"
#define WE_STATS_WAIT_TO_CREATE_COL_EXTENT "WaitCreateColExtent"
#define WE_STATS_WAIT_TO_CREATE_DCT_EXTENT "WaitCreateDctExtent"
#define WE_STATS_WAIT_TO_EXPAND_COL_EXTENT "WaitExpandColExtent"
#define WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT "WaitExpandDctExtent"
#define WE_STATS_WAIT_TO_PARSE_DCT "WaitParseDct"
#define WE_STATS_WAIT_TO_RELEASE_OUT_BUF "WaitReleaseOutBuf"
#define WE_STATS_WAIT_TO_RESERVE_OUT_BUF "WaitReserveOutBuf"
#define WE_STATS_WAIT_TO_RESIZE_OUT_BUF "WaitResizeOutBuf"
#define WE_STATS_WAIT_TO_SELECT_COL "WaitSelectCol"
#define WE_STATS_WAIT_TO_SELECT_TBL "WaitSelectTbl"
#define WE_STATS_WRITE_COL "WriteCol"
#define WE_STATS_WRITE_DCT "WriteDct"
// Functions used to support performance profiling
static void enableProfiling(int nReadThreads, int nParseThreads);
static void registerReadProfThread ( );
static void registerParseProfThread( );
static void startReadEvent ( const std::string& eventString )
{ readEvent ( eventString, true ); }
static void stopReadEvent ( const std::string& eventString )
{ readEvent ( eventString, false ); }
static void startParseEvent( const std::string& eventString )
{ parseEvent( eventString, true ); }
static void stopParseEvent ( const std::string& eventString )
{ parseEvent( eventString, false ); }
static void printProfilingResults( );
#endif
private:
static bool m_bUseStats; // Use statistic flag
#ifdef PROFILE
// Data members and functions used to support performance profiling
static bool fProfiling; // Is profiling enabled
// Protect concurrent addition of Readers
static boost::mutex fRegisterReaderMutex;
// Protect concurrent addition of Parsers
static boost::mutex fRegisterParseMutex;
// Read threads to be profiled
static std::vector<pthread_t> fReadProfThreads;
// Parse threads to be profiled
static std::vector<pthread_t> fParseProfThreads;
// Track/profile Read events
static std::vector<logging::StopWatch> fReadStopWatch;
// Track/profile Parse events
static std::vector<logging::StopWatch> fParseStopWatch;
static void readEvent ( const std::string& eventString, bool start );
static void parseEvent ( const std::string& eventString, bool start );
#endif
};
} //end of namespace
#endif // _WE_STATIS_H_

473
writeengine/shared/we_type.h Normal file
View File

@ -0,0 +1,473 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: we_type.h 4737 2013-08-14 20:45:46Z bwilkinson $
/** @file */
#ifndef _WE_TYPE_H_
#define _WE_TYPE_H_
#undef EXPORT
#undef DELETE
#undef NO_ERROR
#include <sys/types.h>
#include <string>
#include <vector>
#include <boost/any.hpp>
#include <cstring>
#include <stdexcept>
#include "we_define.h"
#include "we_typeext.h"
#include "calpontsystemcatalog.h"
#include "IDBDataFile.h"
#include "IDBPolicy.h"
#undef EXPORT
#undef DELETE
#undef NO_ERROR
/** Namespace WriteEngine */
namespace WriteEngine
{
typedef idbdatafile::IDBDataFile IDBDataFile;
/************************************************************************
* Type definitions
************************************************************************/
typedef uint32_t OID; /** @brief Object ID */
typedef uint32_t FID; /** @brief File ID */
typedef uint64_t RID; /** @brief Row ID */
typedef uint32_t TxnID; /** @brief Transaction ID (New)*/
typedef uint32_t HWM; /** @brief high water mark */
/************************************************************************
* Type enumerations
************************************************************************/
enum DebugLevel { /** @brief Debug level type */
DEBUG_0 = 0, /** @brief No debug info */
DEBUG_1 = 1, /** @brief Summary level debug */
DEBUG_2 = 2, /** @brief Moderate debug */
DEBUG_3 = 3, /** @brief Detail debug */
};
// INFO2 only goes to log file unless '-i' cmd line arg is specified,
// in which case the msg will also get logged to the console.
// All other messages always get logged to the log file and the console.
enum MsgLevel { /** @brief Message level */
MSGLVL_INFO1 = 0, /** @brief Basic Information level*/
MSGLVL_INFO2 = 1, /** @brief More Information level */
MSGLVL_WARNING = 2, /** @brief Warning level */
MSGLVL_ERROR = 3, /** @brief Error level */
MSGLVL_CRITICAL = 4, /** @brief Critical level */
};
enum OpType { /** @brief Operation type */
NOOP = 0, /** @brief No oper */
INSERT = 1, /** @brief Insert */
UPDATE = 2, /** @brief Update */
DELETE = 4, /** @brief Delete */
QUERY = 8, /** @brief Query */
};
enum ColType { /** @brief Column type enumeration*/
// WR_BIT = 1, /** @brief Bit */
WR_BYTE = 2, /** @brief Byte */
WR_SHORT = 3, /** @brief Short */
WR_INT = 4, /** @brief Int */
// WR_LONG = 5, /** @brief Long */
WR_LONGLONG = 6, /** @brief Long long*/
WR_FLOAT = 7, /** @brief Float */
WR_DOUBLE = 8, /** @brief Double */
WR_CHAR = 9, /** @brief Char */
WR_TOKEN = 10, /** @brief Token */
WR_BLOB = 11, /** @brief BLOB */
WR_VARBINARY = 12, /** @brief VARBINARY */
WR_UBYTE = 13, /** @brief Unsigned Byte */
WR_USHORT = 14, /** @brief Unsigned Short */
WR_UINT = 15, /** @brief Unsigned Int */
WR_ULONGLONG = 16, /** @brief Unsigned Long long*/
};
// Describes relation of field to column for a bulk load
enum BulkFldColRel { BULK_FLDCOL_COLUMN_FIELD, // map input field to db col
BULK_FLDCOL_COLUMN_DEFAULT,// import def val to db col
BULK_FLDCOL_IGNORE_FIELD };// ignore fld in import file
// Bulk Load Mode (ex: local vs remote, single src vs multiple src files)
enum BulkModeType { BULK_MODE_REMOTE_SINGLE_SRC = 1,
BULK_MODE_REMOTE_MULTIPLE_SRC = 2,
BULK_MODE_LOCAL = 3 };
// Import Mode 0-text Import (default)
// 1-Binary Import with NULL values
// 2-Binary Import with saturated NULL values
enum ImportDataMode { IMPORT_DATA_TEXT = 0,
IMPORT_DATA_BIN_ACCEPT_NULL = 1,
IMPORT_DATA_BIN_SAT_NULL = 2 };
/**
* the set of Calpont column data type names; MUST match ColDataType in
* calpontsystemcatalog.h.
*/
const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE][20] = {
"bit",
"tinyint",
"char",
"smallint",
"decimal",
"medint",
"integer",
"float",
"date",
"bigint",
"double",
"datetime",
"varchar",
"varbinary",
"clob",
"blob",
"unsigned-tinyint",
"unsigned-smallint",
"unsigned-decimal",
"unsigned-med int",
"unsigned-int",
"unsigned-float",
"unsigned-bigint",
"unsigned-double"
};
enum FuncType { FUNC_WRITE_ENGINE, FUNC_INDEX, FUNC_DICTIONARY };
enum CacheListType { FREE_LIST, LRU_LIST, WRITE_LIST }; /** @brief List type */
/************************************************************************
* struct data block structure
************************************************************************/
struct DataBlock /** @brief Data block structure */
{
long no; /** @brief block number */
uint64_t lbid; /** @brief lbid */
bool dirty; /** @brief block dirty flag */
int state; /** @brief initialized 0, read 1 , modified 2 */
unsigned char data[BYTE_PER_BLOCK];/** @brief data buffer */
DataBlock() { dirty = false; /** @brief constructor */
memset( data, 0, BYTE_PER_BLOCK ); }
};
struct DataSubBlock /** @brief Data subblock structure*/
{
long no; /** @brief sub block number */
bool dirty; /** @brief block dirty flag */
unsigned char data[BYTE_PER_SUBBLOCK]; /** @brief data buffer */
DataSubBlock() { dirty = false; memset( data, 0, BYTE_PER_SUBBLOCK ); } /** @brief constructor */
};
/************************************************************************
* @brief file structure. Default copy constructor, assignment oper, etc
* are in play here, as they are not overridden. Beware that if copies
* of a File object are made, only one user should be closing the pFile.
* oid and fid replicate one another. oid mostly used by index, cache,
* and dictionary. fid mostly used by colop and bulk.
************************************************************************/
struct File /** @brief File structure */
{
OID oid; /** @brief Oid */
FID fid; /** @brief File id */
HWM hwm; /** @brief High water mark */
IDBDataFile* pFile; /** @brief File handle */
uint32_t fPartition; /** @brief Partition for pFile*/
uint16_t fSegment; /** @brief Segment for pFile */
uint16_t fDbRoot; /** @brief DbRoot for pFile */
std::string fSegFileName; /** @brief Current seg file path */
File() { clear(); } /** @brief constructor */
void clear() { pFile = NULL; oid = fid = hwm = 0;
fPartition = fSegment = fDbRoot = 0;
fSegFileName.clear(); }
};
/************************************************************************
* @brief Internal communication block structure
************************************************************************/
struct CommBlock /** @brief Communication Block */
{
File file; /** @brief File structure */
void clear() { file.clear(); }
};
/************************************************************************
* @brief column structure used to pass data in/out of we_colop functions
************************************************************************/
struct Column /** @brief Column structure */
{
int colNo; /** @brief column number */
int colWidth; /** @brief column width */
ColType colType; /** @brief column type (internal use)*/
execplan::CalpontSystemCatalog::ColDataType colDataType; /** @brief column data type (from interface)*/
File dataFile; /** @brief column data file */
int compressionType; /** @brief column compression type*/
Column() : colNo(0), colWidth(0), colType(WR_INT),
colDataType(execplan::CalpontSystemCatalog::INT),
compressionType(idbdatafile::IDBPolicy::useHdfs()?2:0) { }
};
/************************************************************************
* @brief dictionary related structures (Token struct is defined in
* we_typeext.h to facilitate its use in dbcon and utils/dataconvert).
************************************************************************/
typedef struct offset_ /** @brief Offset structure */
{
int hdrLoc; /** @brief offset postion in hdr */
uint16_t offset; /** @brief offset in block */
} Offset;
/************************************************************************
* @brief interfaces with DDL/DML
************************************************************************/
typedef struct colTuple_struct /** @brief Column Tuple definition*/
{
boost::any data; /** @brief column value */
} ColTuple;
typedef std::vector<ColTuple> ColTupleList; /** @brief column value list */
struct ColStruct /** @brief Column Interface Struct*/
{
OID dataOid; /** @brief column data file object id */
int colWidth; /** @brief column width */
bool tokenFlag; /** @brief column token flag, must be set to true if it is a token column */
execplan::CalpontSystemCatalog::ColDataType colDataType; /** @brief column data type (for interface)*/
ColType colType; /** @brief column type (internal use for write engine)*/
uint32_t fColPartition; /** @brief Partition for column file */
uint16_t fColSegment; /** @brief Segment for column file*/
uint16_t fColDbRoot; /** @brief DBRoot for column file */
int fCompressionType; /** @brief Compression tpye for column file */
ColStruct() : dataOid(0), colWidth(0), /** @brief constructor */
tokenFlag(false), colDataType(execplan::CalpontSystemCatalog::INT), colType(WR_INT),
fColPartition(0), fColSegment(0), fColDbRoot(0),
fCompressionType(idbdatafile::IDBPolicy::useHdfs()?2:0) { }
};
typedef std::vector<ColStruct> ColStructList; /** @brief column struct list */
typedef std::vector<ColTupleList> ColValueList; /** @brief column value list */
typedef std::vector<RID> RIDList; /** @brief RID list */
typedef std::vector<std::string> dictStr;
typedef std::vector<dictStr> DictStrList;
// dictionary
struct DctnryStruct /** @brief Dctnry Interface Struct*/
{
OID dctnryOid; /** @brief dictionary signature file */
OID columnOid; /** @brief corresponding column file */
int colWidth; /** @brief string width for the dictionary column*/
uint32_t fColPartition; /** @brief Partition for column file */
uint16_t fColSegment; /** @brief Segment for column file */
uint16_t fColDbRoot; /** @brief DBRoot for column file */
int fCompressionType; /** @brief Compression tpye for column file */
DctnryStruct() : dctnryOid(0), columnOid(0), /** @brief constructor */
colWidth(0),
fColPartition(0), fColSegment(0),
fColDbRoot(0), fCompressionType(idbdatafile::IDBPolicy::useHdfs()?2:0) { }
};
struct DctnryTuple /** @brief Dictionary Tuple struct*/
{
unsigned char sigValue[MAX_SIGNATURE_SIZE]; /** @brief dictionary signature value*/
int sigSize; /** @brief dictionary signature size */
Token token; /** @brief dictionary token */
bool isNull;
DctnryTuple() { }
~DctnryTuple() { }
};
typedef std::vector<DctnryTuple> DctColTupleList;
typedef std::vector<DctnryStruct> DctnryStructList; /** @brief column struct list */
typedef std::vector<DctColTupleList> DctnryValueList; /** @brief column value list */
/************************************************************************
* @brief Used by Bulk Load to describe a column
************************************************************************/
struct JobColumn /** @brief Job Column Structure */
{
std::string colName; /** @brief column name */
OID mapOid; /** @brief column OID */
execplan::CalpontSystemCatalog::ColDataType dataType; /** @brief column data type */
ColType weType; /** @brief write engine data type */
std::string typeName; /** @brief data type name */
uint64_t emptyVal; /** @brief default empty value */
int width; /** @brief column width; for a dictionary column, this is "eventually" the token width */
int definedWidth; /** @brief column width as defined in the table, used for non-dictionary strings */
int dctnryWidth; /** @brief dictionary width */
int precision; /** @brief precision of decimal */
int scale; /** @brief scale of decimal */
bool fNotNull; /** @brief not null flag */
BulkFldColRel fFldColRelation; /** @brief type of field/col relation*/
char colType; /** @brief column type, blank is regular, D is dictionary */
int compressionType; /** @brief compression type */
bool autoIncFlag; /** @brief auto increment flag */
DctnryStruct dctnry; /** @brief dictionary structure */
int64_t fMinIntSat; /** @brief For integer type, the min saturation value */
uint64_t fMaxIntSat; /** @brief For integer type, the max saturation value */
double fMinDblSat; /** @brief for float/double, the min saturation value */
double fMaxDblSat; /** @brief for float/double, the max saturation value */
bool fWithDefault; /** @brief With default */
long long fDefaultInt; /** @brief Integer column default */
unsigned long long fDefaultUInt; /** @brief UnsignedInt col default*/
double fDefaultDbl; /** @brief Dbl/Flt column default */
std::string fDefaultChr; /** @brief Char column default */
JobColumn() : mapOid(0), dataType(execplan::CalpontSystemCatalog::INT), weType(WR_INT),
typeName("integer"), emptyVal(0),
width(0), definedWidth(0), dctnryWidth(0),
precision(0), scale(0), fNotNull(false),
fFldColRelation(BULK_FLDCOL_COLUMN_FIELD), colType(' '),
compressionType(0),autoIncFlag(false),
fMinIntSat(0), fMaxIntSat(0),
fMinDblSat(0), fMaxDblSat(0), fWithDefault(false),
fDefaultInt(0), fDefaultUInt(0), fDefaultDbl(0.0)
{ }
};
typedef std::vector<JobColumn> JobColList; /** @brief column value list */
struct JobFieldRef // references field/column in JobTable
{
BulkFldColRel fFldColType; // type of field or column
unsigned fArrayIndex; // index into colList or fIgnoredFields
// in JobTable based on fFldColType.
JobFieldRef( ) : fFldColType(BULK_FLDCOL_COLUMN_FIELD), fArrayIndex(0) { }
JobFieldRef( BulkFldColRel fldColType, unsigned idx ) :
fFldColType( fldColType ), fArrayIndex( idx ) { }
};
typedef std::vector<JobFieldRef> JobFieldRefList;
struct JobTable /** @brief Job Table Structure */
{
std::string tblName; /** @brief table name */
OID mapOid; /** @brief table OID */
std::string loadFileName; /** @brief table load file name */
uint64_t maxErrNum; /** @brief max number of error rows before abort */
JobColList colList; /** @brief list of columns to be loaded; followed by default columns to be loaded */
JobColList fIgnoredFields; /** @brief list of fields in input file to be ignored */
JobFieldRefList fFldRefs; /** @brief Combined list of refs to entries in colList and fIgnoredFields */
JobTable() : mapOid(0), maxErrNum(0) { }
};
typedef std::vector<JobTable> JobTableList;/** @brief table list */
struct Job /** @brief Job Structure */
{
int id; /** @brief job id */
std::string schema; /** @brief database name */
std::string name; /** @brief job name */
std::string desc; /** @brief job description */
std::string userName; /** @brief user name */
JobTableList jobTableList; /** @brief job table list */
std::string createDate; /** @brief job create date */
std::string createTime; /** @brief job create time */
char fDelimiter;
char fEnclosedByChar;
char fEscapeChar;
int numberOfReadBuffers;
unsigned readBufferSize;
unsigned writeBufferSize;
Job() : id(0), fDelimiter('|'),
fEnclosedByChar('\0'), fEscapeChar('\0'),
numberOfReadBuffers(0), readBufferSize(0), writeBufferSize(0) { }
};
/************************************************************************
* @brief Cache memory
************************************************************************/
struct CacheBlock /** @brief Cache block structure */
{
uint64_t fbo; /** @brief file fbo */
uint64_t lbid; /** @brief lbid */
bool dirty; /** @brief dirty flag */
int hitCount; /** @brief hit count */
unsigned char* data; /** @brief block buffer */
CacheBlock() { data = NULL; clear(); }/** @brief constructor */
void clear() { fbo = lbid = hitCount = 0;
dirty = false;
if( data ) memset( data, 0, BYTE_PER_BLOCK); } /** @brief clear, NOTE: buf must be free by caller first */
void init() { data = (unsigned char*)malloc(BYTE_PER_BLOCK); }
void freeMem() { if( data ) free( data ); }
};
struct BlockBuffer /** @brief Block buffer */
{
CommBlock cb; /** @brief Communication block structure */
CacheBlock block; /** @brief Cache block strucutre */
CacheListType listType; /** @brief List number, 0 - free, 1 - LRU, 2 - write */
BlockBuffer() { clear(); } /** @brief constructor */
void init() { block.init(); }
void freeMem() { block.freeMem(); }
void clear() { cb.clear(); block.clear(); listType = FREE_LIST; }
};
struct CacheControl /** @brief Cache control structure */
{
int totalBlock; /** @brief The toal number of allocated blocks */
int pctFree; /** @brief The percentage of free blocks when some blocks must be aged out */
int checkInterval; /** @brief A check point interval in seconds */
CacheControl() { totalBlock = pctFree = checkInterval; } /** @brief constructor */
};
/************************************************************************
* @brief Bulk parse meta data describing data in a read buffer.
* An offset of COLPOSPAIR_NULL_TOKEN_OFFSET represents a null token.
************************************************************************/
struct ColPosPair /** @brief Column position pair structure */
{
int start; /** @brief start position */
int offset; /** @brief length of token*/
};
/************************************************************************
* @brief SecondaryShutdown used to terminate a thread when it sees that the
* JobStatus flag has been set to EXIT_FAILURE (by another thread).
************************************************************************/
class SecondaryShutdownException : public std::runtime_error
{
public:
SecondaryShutdownException(const std::string& msg) :
std::runtime_error(msg) { }
};
/************************************************************************
* @brief Generic exception class used to store exception string and error
* code for a writeengine error.
************************************************************************/
class WeException : public std::runtime_error
{
public:
WeException(const std::string& msg, int err=0) :
std::runtime_error(msg), fErrorCode(err) { }
void errorCode(int code) { fErrorCode = code; }
int errorCode() const { return fErrorCode; }
private:
int fErrorCode;
};
} //end of namespace
#endif // _WE_TYPE_H_

60
writeengine/shared/we_typeext.h Normal file
View File

@ -0,0 +1,60 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id$
/** @file
*
* File contains basic typedefs shared externally with other parts of the
* system. These types are placed here rather than in we_type.h in order
* to decouple we_type.h and we_define.h from external components, so that
* the other libraries won't have to recompile every time we change something
* in we_type.h and/or we_define.h.
*/
#ifndef _WE_TYPEEXT_H_
#define _WE_TYPEEXT_H_
#include <stdint.h>
#include <sys/types.h>
/** Namespace WriteEngine */
namespace WriteEngine
{
/************************************************************************
* Type definitions
************************************************************************/
typedef uint64_t RID; // Row ID
/************************************************************************
* Dictionary related structure
************************************************************************/
struct Token {
uint64_t op : 10; // ordinal position within a block
uint64_t fbo : 36; // file block number
uint64_t spare : 18; // spare
Token() // constructor, set to null value
{
op = 0x3FE;
fbo = 0xFFFFFFFFFLL;
spare = 0x3FFFF;
}
};
} //end of namespace
#endif // _WE_TYPEEXT_H_