database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-12-24 14:20:59 +03:00
Code Activity

the begginning

Browse Source
This commit is contained in:
david hill
2016-01-06 14:08:59 -06:00
parent 66a31debcb
commit f6afc42dd0
18251 changed files with 16460679 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
utils/batchloader/Makefile Executable file
View File

@@ -0,0 +1,91 @@
#******************************************************************************************
# $Id: Makefile 3022 2012-03-22 12:58:08Z chao $
#
# Copyright (C) 2009-2012 Calpont Corporation
# All rights reserved
#*****************************************************************************************/
include ../../rules.mak
.PHONY: install clean test docs coverage leakcheck
PROGRAM=libbatchloader.so
VERSION=1.0.0
LIBRARY=$(PROGRAM).$(VERSION)
CPPFLAGS=-I$(EXPORT_ROOT)/include -I/usr/include/libxml2
CXXFLAGS+=$(DEBUG_FLAGS) -Wall -fpic
CLIBS=-L$(EXPORT_ROOT)/lib -lxml2 -lcppunit -lexecplan -liboamcpp -ldl
TLIBS=-L. -lbatchloader $(CLIBS)
GLIBS=$(CLIBS)
LLIBS=-L$(CALPONT_LIBRARY_PATH) -lmessageqcpp
SRCS=batchloader.cpp
LINCLUDES=batchloader.h
OBJS=$(SRCS:.cpp=.o)
$(LIBRARY): $(OBJS)
$(LINK.cpp) -shared -o $(LIBRARY) $(OBJS)
rm -f $(PROGRAM)
ln -s $(LIBRARY) $(PROGRAM)
install: bootstrap $(LIBRARY)
umask 002; \
mkdir -p $(INSTALL_ROOT_LIB) $(INSTALL_ROOT_INCLUDE); \
pushd $(INSTALL_ROOT_LIB) >/dev/null; \
rm -f $(LIBRARY) $(PROGRAM); \
ln -s $(LIBRARY) $(PROGRAM); \
popd >/dev/null; \
$(INSTALL) $(LIBRARY) $(INSTALL_ROOT_LIB)
bootstrap:
$(INSTALL) $(LINCLUDES) $(INSTALL_ROOT_INCLUDE)
clean:
rm -f $(OBJS) tdriver.o $(PROGRAM) $(LIBRARY) tdriver core *~ *.tag *-gcov.* *.gcov tdriver-gcov *.d
rm -rf html
docs:
doxygen $(EXPORT_ROOT)/etc/Doxyfile
tdriver: tdriver.o
$(LINK.cpp) -o $@ $^ $(TLIBS)
test:
xtest: $(LIBRARY) tdriver
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib ./tdriver
%-gcov.o: %.cpp
$(COMPILE.cpp) -o $@ $^
tdriver-gcov: CXXFLAGS+=-fprofile-arcs -ftest-coverage
tdriver-gcov: tdriver-gcov.o $(subst .o,-gcov.o,$(OBJS))
$(LINK.cpp) -o $@ $^ $(GLIBS)
coverage:
xcoverage: tdriver-gcov
rm -f *.gcda
LD_LIBRARY_PATH=$(EXPORT_ROOT)/lib:/usr/local/lib ./tdriver-gcov
for file in $(SRCS); do \
bfile=`basename $$file .cpp`; \
gcov -o $${bfile}-gcov $$file >/dev/null; \
done
/usr/local/bin/genCoverage.pl $(SRCS)
leakcheck:
xleakcheck: $(LIBRARY) tdriver
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib valgrind --tool=memcheck --leak-check=yes ./tdriver
%.d: %.cpp
@set -e; rm -f $@; \
$(CC) -MM $(CPPFLAGS) $< > $@.$$$$; \
sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$$ > $@; \
rm -f $@.$$$$
ifndef BOOTSTRAP
-include $(SRCS:.cpp=.d) tdriver.d
endif

38
utils/batchloader/Makefile.am Normal file
View File

@@ -0,0 +1,38 @@
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
# $Id$
## Process this file with automake to produce Makefile.in
AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags)
AM_CFLAGS = $(idb_cflags)
AM_CXXFLAGS = $(idb_cxxflags)
AM_LDFLAGS = -version-info 1:0:0 $(idb_ldflags)
lib_LTLIBRARIES = libbatchloader.la
libbatchloader_la_SOURCES = batchloader.cpp
include_HEADERS = batchloader.h
test:
coverage:
leakcheck:
docs:
bootstrap: install-data-am

543
utils/batchloader/Makefile.in Normal file
View File

@@ -0,0 +1,543 @@
# Makefile.in generated by automake 1.9.6 from Makefile.am.
# @configure_input@
# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
# 2003, 2004, 2005 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
# with or without modifications, as long as this notice is preserved.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.
@SET_MAKE@
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
# $Id$
srcdir = @srcdir@
top_srcdir = @top_srcdir@
VPATH = @srcdir@
pkgdatadir = $(datadir)/@PACKAGE@
pkglibdir = $(libdir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
top_builddir = ../..
am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
INSTALL = @INSTALL@
install_sh_DATA = $(install_sh) -c -m 644
install_sh_PROGRAM = $(install_sh) -c
install_sh_SCRIPT = $(install_sh) -c
INSTALL_HEADER = $(INSTALL_DATA)
transform = $(program_transform_name)
NORMAL_INSTALL = :
PRE_INSTALL = :
POST_INSTALL = :
NORMAL_UNINSTALL = :
PRE_UNINSTALL = :
POST_UNINSTALL = :
build_triplet = @build@
host_triplet = @host@
subdir = utils/batchloader
DIST_COMMON = $(include_HEADERS) $(srcdir)/Makefile.am \
$(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/m4/compilerflags.m4 \
$(top_srcdir)/m4/functions.m4 $(top_srcdir)/m4/install.m4 \
$(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
mkinstalldirs = $(install_sh) -d
CONFIG_HEADER = $(top_builddir)/config.h
CONFIG_CLEAN_FILES =
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
am__vpath_adj = case $$p in \
$(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
*) f=$$p;; \
esac;
am__strip_dir = `echo $$p | sed -e 's|^.*/||'`;
am__installdirs = "$(DESTDIR)$(libdir)" "$(DESTDIR)$(includedir)"
libLTLIBRARIES_INSTALL = $(INSTALL)
LTLIBRARIES = $(lib_LTLIBRARIES)
libbatchloader_la_LIBADD =
am_libbatchloader_la_OBJECTS = batchloader.lo
libbatchloader_la_OBJECTS = $(am_libbatchloader_la_OBJECTS)
DEFAULT_INCLUDES = -I. -I$(srcdir) -I$(top_builddir)
depcomp = $(SHELL) $(top_srcdir)/depcomp
am__depfiles_maybe = depfiles
CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
LTCXXCOMPILE = $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) \
$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
$(AM_CXXFLAGS) $(CXXFLAGS)
CXXLD = $(CXX)
CXXLINK = $(LIBTOOL) --tag=CXX --mode=link $(CXXLD) $(AM_CXXFLAGS) \
$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
SOURCES = $(libbatchloader_la_SOURCES)
DIST_SOURCES = $(libbatchloader_la_SOURCES)
includeHEADERS_INSTALL = $(INSTALL_HEADER)
HEADERS = $(include_HEADERS)
ETAGS = etags
CTAGS = ctags
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
ACLOCAL = @ACLOCAL@
ALLOCA = @ALLOCA@
AMDEP_FALSE = @AMDEP_FALSE@
AMDEP_TRUE = @AMDEP_TRUE@
AMTAR = @AMTAR@
AR = @AR@
AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
CC = @CC@
CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@
CPP = @CPP@
CPPFLAGS = @CPPFLAGS@
CXX = @CXX@
CXXCPP = @CXXCPP@
CXXDEPMODE = @CXXDEPMODE@
CXXFLAGS = @CXXFLAGS@
CYGPATH_W = @CYGPATH_W@
DEFS = @DEFS@
DEPDIR = @DEPDIR@
ECHO = @ECHO@
ECHO_C = @ECHO_C@
ECHO_N = @ECHO_N@
ECHO_T = @ECHO_T@
EGREP = @EGREP@
EXEEXT = @EXEEXT@
F77 = @F77@
FFLAGS = @FFLAGS@
INSTALL_DATA = @INSTALL_DATA@
INSTALL_PROGRAM = @INSTALL_PROGRAM@
INSTALL_SCRIPT = @INSTALL_SCRIPT@
INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
LDFLAGS = @LDFLAGS@
LEX = @LEX@
LEXLIB = @LEXLIB@
LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
LIBOBJS = @LIBOBJS@
LIBS = @LIBS@
LIBTOOL = @LIBTOOL@
LN_S = @LN_S@
LTLIBOBJS = @LTLIBOBJS@
MAKEINFO = @MAKEINFO@
OBJEXT = @OBJEXT@
PACKAGE = @PACKAGE@
PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_STRING = @PACKAGE_STRING@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
PATH_SEPARATOR = @PATH_SEPARATOR@
POW_LIB = @POW_LIB@
RANLIB = @RANLIB@
SED = @SED@
SET_MAKE = @SET_MAKE@
SHELL = @SHELL@
STRIP = @STRIP@
VERSION = @VERSION@
XML2_CONFIG = @XML2_CONFIG@
XML_CPPFLAGS = @XML_CPPFLAGS@
XML_LIBS = @XML_LIBS@
YACC = @YACC@
ac_ct_AR = @ac_ct_AR@
ac_ct_CC = @ac_ct_CC@
ac_ct_CXX = @ac_ct_CXX@
ac_ct_F77 = @ac_ct_F77@
ac_ct_RANLIB = @ac_ct_RANLIB@
ac_ct_STRIP = @ac_ct_STRIP@
am__fastdepCC_FALSE = @am__fastdepCC_FALSE@
am__fastdepCC_TRUE = @am__fastdepCC_TRUE@
am__fastdepCXX_FALSE = @am__fastdepCXX_FALSE@
am__fastdepCXX_TRUE = @am__fastdepCXX_TRUE@
am__include = @am__include@
am__leading_dot = @am__leading_dot@
am__quote = @am__quote@
am__tar = @am__tar@
am__untar = @am__untar@
bindir = @bindir@
build = @build@
build_alias = @build_alias@
build_cpu = @build_cpu@
build_os = @build_os@
build_vendor = @build_vendor@
datadir = @datadir@
etcdir = @etcdir@
exec_prefix = @exec_prefix@
host = @host@
host_alias = @host_alias@
host_cpu = @host_cpu@
host_os = @host_os@
host_vendor = @host_vendor@
idb_brm_libs = @idb_brm_libs@
idb_cflags = @idb_cflags@
idb_common_includes = @idb_common_includes@
idb_common_ldflags = @idb_common_ldflags@
idb_common_libs = @idb_common_libs@
idb_cppflags = @idb_cppflags@
idb_cxxflags = @idb_cxxflags@
idb_exec_libs = @idb_exec_libs@
idb_ldflags = @idb_ldflags@
idb_oam_libs = @idb_oam_libs@
idb_write_libs = @idb_write_libs@
idbinstall = @idbinstall@
includedir = @includedir@
infodir = @infodir@
install_sh = @install_sh@
libdir = @libdir@
libexecdir = @libexecdir@
localdir = @localdir@
localstatedir = @localstatedir@
mandir = @mandir@
march_flags = @march_flags@
mibdir = @mibdir@
mkdir_p = @mkdir_p@
mysqldir = @mysqldir@
netsnmp_libs = @netsnmp_libs@
netsnmpagntdir = @netsnmpagntdir@
netsnmpdir = @netsnmpdir@
netsnmplibrdir = @netsnmplibrdir@
netsnmpmachdir = @netsnmpmachdir@
netsnmpsysdir = @netsnmpsysdir@
oldincludedir = @oldincludedir@
postdir = @postdir@
prefix = @prefix@
program_transform_name = @program_transform_name@
sbindir = @sbindir@
sharedir = @sharedir@
sharedstatedir = @sharedstatedir@
sysconfdir = @sysconfdir@
target_alias = @target_alias@
toolsdir = @toolsdir@
AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags)
AM_CFLAGS = $(idb_cflags)
AM_CXXFLAGS = $(idb_cxxflags)
AM_LDFLAGS = -version-info 1:0:0 $(idb_ldflags)
lib_LTLIBRARIES = libbatchloader.la
libbatchloader_la_SOURCES = batchloader.cpp
include_HEADERS = batchloader.h
all: all-am
.SUFFIXES:
.SUFFIXES: .cpp .lo .o .obj
$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps)
@for dep in $?; do \
case '$(am__configure_deps)' in \
*$$dep*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh \
&& exit 0; \
exit 1;; \
esac; \
done; \
echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu utils/batchloader/Makefile'; \
cd $(top_srcdir) && \
$(AUTOMAKE) --gnu utils/batchloader/Makefile
.PRECIOUS: Makefile
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
@case '$?' in \
*config.status*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
*) \
echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
esac;
$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(top_srcdir)/configure: $(am__configure_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(ACLOCAL_M4): $(am__aclocal_m4_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
install-libLTLIBRARIES: $(lib_LTLIBRARIES)
@$(NORMAL_INSTALL)
test -z "$(libdir)" || $(mkdir_p) "$(DESTDIR)$(libdir)"
@list='$(lib_LTLIBRARIES)'; for p in $$list; do \
if test -f $$p; then \
f=$(am__strip_dir) \
echo " $(LIBTOOL) --mode=install $(libLTLIBRARIES_INSTALL) $(INSTALL_STRIP_FLAG) '$$p' '$(DESTDIR)$(libdir)/$$f'"; \
$(LIBTOOL) --mode=install $(libLTLIBRARIES_INSTALL) $(INSTALL_STRIP_FLAG) "$$p" "$(DESTDIR)$(libdir)/$$f"; \
else :; fi; \
done
uninstall-libLTLIBRARIES:
@$(NORMAL_UNINSTALL)
@set -x; list='$(lib_LTLIBRARIES)'; for p in $$list; do \
p=$(am__strip_dir) \
echo " $(LIBTOOL) --mode=uninstall rm -f '$(DESTDIR)$(libdir)/$$p'"; \
$(LIBTOOL) --mode=uninstall rm -f "$(DESTDIR)$(libdir)/$$p"; \
done
clean-libLTLIBRARIES:
-test -z "$(lib_LTLIBRARIES)" || rm -f $(lib_LTLIBRARIES)
@list='$(lib_LTLIBRARIES)'; for p in $$list; do \
dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
test "$$dir" != "$$p" || dir=.; \
echo "rm -f \"$${dir}/so_locations\""; \
rm -f "$${dir}/so_locations"; \
done
libbatchloader.la: $(libbatchloader_la_OBJECTS) $(libbatchloader_la_DEPENDENCIES)
$(CXXLINK) -rpath $(libdir) $(libbatchloader_la_LDFLAGS) $(libbatchloader_la_OBJECTS) $(libbatchloader_la_LIBADD) $(LIBS)
mostlyclean-compile:
-rm -f *.$(OBJEXT)
distclean-compile:
-rm -f *.tab.c
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/batchloader.Plo@am__quote@
.cpp.o:
@am__fastdepCXX_TRUE@ if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $<
.cpp.obj:
@am__fastdepCXX_TRUE@ if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ `$(CYGPATH_W) '$<'`; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
.cpp.lo:
@am__fastdepCXX_TRUE@ if $(LTCXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Plo"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $<
mostlyclean-libtool:
-rm -f *.lo
clean-libtool:
-rm -rf .libs _libs
distclean-libtool:
-rm -f libtool
uninstall-info-am:
install-includeHEADERS: $(include_HEADERS)
@$(NORMAL_INSTALL)
test -z "$(includedir)" || $(mkdir_p) "$(DESTDIR)$(includedir)"
@list='$(include_HEADERS)'; for p in $$list; do \
if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
f=$(am__strip_dir) \
echo " $(includeHEADERS_INSTALL) '$$d$$p' '$(DESTDIR)$(includedir)/$$f'"; \
$(includeHEADERS_INSTALL) "$$d$$p" "$(DESTDIR)$(includedir)/$$f"; \
done
uninstall-includeHEADERS:
@$(NORMAL_UNINSTALL)
@list='$(include_HEADERS)'; for p in $$list; do \
f=$(am__strip_dir) \
echo " rm -f '$(DESTDIR)$(includedir)/$$f'"; \
rm -f "$(DESTDIR)$(includedir)/$$f"; \
done
ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
mkid -fID $$unique
tags: TAGS
TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$(TAGS_FILES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
if test -z "$(ETAGS_ARGS)$$tags$$unique"; then :; else \
test -n "$$unique" || unique=$$empty_fix; \
$(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
$$tags $$unique; \
fi
ctags: CTAGS
CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$(TAGS_FILES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
test -z "$(CTAGS_ARGS)$$tags$$unique" \
|| $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
$$tags $$unique
GTAGS:
here=`$(am__cd) $(top_builddir) && pwd` \
&& cd $(top_srcdir) \
&& gtags -i $(GTAGS_ARGS) $$here
distclean-tags:
-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \
topsrcdirstrip=`echo "$(top_srcdir)" | sed 's|.|.|g'`; \
list='$(DISTFILES)'; for file in $$list; do \
case $$file in \
$(srcdir)/*) file=`echo "$$file" | sed "s|^$$srcdirstrip/||"`;; \
$(top_srcdir)/*) file=`echo "$$file" | sed "s|^$$topsrcdirstrip/|$(top_builddir)/|"`;; \
esac; \
if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
dir=`echo "$$file" | sed -e 's,/[^/]*$$,,'`; \
if test "$$dir" != "$$file" && test "$$dir" != "."; then \
dir="/$$dir"; \
$(mkdir_p) "$(distdir)$$dir"; \
else \
dir=''; \
fi; \
if test -d $$d/$$file; then \
if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
cp -pR $(srcdir)/$$file $(distdir)$$dir || exit 1; \
fi; \
cp -pR $$d/$$file $(distdir)$$dir || exit 1; \
else \
test -f $(distdir)/$$file \
|| cp -p $$d/$$file $(distdir)/$$file \
|| exit 1; \
fi; \
done
check-am: all-am
check: check-am
all-am: Makefile $(LTLIBRARIES) $(HEADERS)
installdirs:
for dir in "$(DESTDIR)$(libdir)" "$(DESTDIR)$(includedir)"; do \
test -z "$$dir" || $(mkdir_p) "$$dir"; \
done
install: install-am
install-exec: install-exec-am
install-data: install-data-am
uninstall: uninstall-am
install-am: all-am
@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
installcheck: installcheck-am
install-strip:
$(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
`test -z '$(STRIP)' || \
echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install
mostlyclean-generic:
clean-generic:
distclean-generic:
-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
maintainer-clean-generic:
@echo "This command is intended for maintainers to use"
@echo "it deletes files that may require special tools to rebuild."
clean: clean-am
clean-am: clean-generic clean-libLTLIBRARIES clean-libtool \
mostlyclean-am
distclean: distclean-am
-rm -rf ./$(DEPDIR)
-rm -f Makefile
distclean-am: clean-am distclean-compile distclean-generic \
distclean-libtool distclean-tags
dvi: dvi-am
dvi-am:
html: html-am
info: info-am
info-am:
install-data-am: install-includeHEADERS
install-exec-am: install-libLTLIBRARIES
install-info: install-info-am
install-man:
installcheck-am:
maintainer-clean: maintainer-clean-am
-rm -rf ./$(DEPDIR)
-rm -f Makefile
maintainer-clean-am: distclean-am maintainer-clean-generic
mostlyclean: mostlyclean-am
mostlyclean-am: mostlyclean-compile mostlyclean-generic \
mostlyclean-libtool
pdf: pdf-am
pdf-am:
ps: ps-am
ps-am:
uninstall-am: uninstall-includeHEADERS uninstall-info-am \
uninstall-libLTLIBRARIES
.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \
clean-libLTLIBRARIES clean-libtool ctags distclean \
distclean-compile distclean-generic distclean-libtool \
distclean-tags distdir dvi dvi-am html html-am info info-am \
install install-am install-data install-data-am install-exec \
install-exec-am install-includeHEADERS install-info \
install-info-am install-libLTLIBRARIES install-man \
install-strip installcheck installcheck-am installdirs \
maintainer-clean maintainer-clean-generic mostlyclean \
mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
pdf pdf-am ps ps-am tags uninstall uninstall-am \
uninstall-includeHEADERS uninstall-info-am \
uninstall-libLTLIBRARIES
test:
coverage:
leakcheck:
docs:
bootstrap: install-data-am
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
.NOEXPORT:

559
utils/batchloader/batchloader.cpp Normal file
View File

@@ -0,0 +1,559 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: we_dbrootextenttracker.cpp 3672 2012-03-26 12:31:27Z rdempsey $
*/
#include <unistd.h>
#include <algorithm>
#include <sstream>
#include <boost/scoped_ptr.hpp>
#include "dbrm.h"
#include "batchloader.h"
using namespace std;
using namespace boost;
using namespace BRM;
using namespace oam;
using namespace execplan;
namespace batchloader
{
//------------------------------------------------------------------------------
// ChooseStartLoadPM constructor
//
// Mutex lock not needed in this function as it is only called from main thread
//------------------------------------------------------------------------------
BatchLoader::BatchLoader ( uint32_t tableOid,
execplan::CalpontSystemCatalog::SCN sessionId,
std::vector<uint32_t>& PMs )
{
fFirstPm=0;
fNextIdx=0;
fPMs = PMs;
fSessionId = sessionId;
fTableOid = tableOid;
OamCache * oamcache = OamCache::makeOamCache();
oam::OamCache::PMDbrootsMap_t systemPmDbrootMap = oamcache->getPMToDbrootsMap();
std::map<int, OamCache::dbRoots>::iterator iter = systemPmDbrootMap->begin();
//cout << "fPMs size is " << fPMs.size() << endl;
fPmDbrootMap.reset(new OamCache::PMDbrootsMap_t::element_type());
fDbrootPMmap.reset(new map<int, int>());
for (uint32_t i=0; i < fPMs.size(); i++)
{
iter = systemPmDbrootMap->find(fPMs[i]);
if (iter != systemPmDbrootMap->end())
{
fDbRoots.insert(fDbRoots.end(), (iter->second).begin(), (iter->second).end());
(*fPmDbrootMap)[fPMs[i]] = iter->second;
}
}
//Build dbroot to PM map
for (iter = fPmDbrootMap->begin(); iter != fPmDbrootMap->end(); iter++)
{
for ( uint32_t i = 0; i < iter->second.size(); i++)
{
(*fDbrootPMmap)[iter->second[i]] = iter->first;
}
}
}
//------------------------------------------------------------------------------
// Select the first PM to send the first batch of rows.
/*Look up extent map to decide which dbroot to start
1. If newly created table, starts from the pm where the abbreviated extent created.
2. If the abbreviated extent has hwm > 0, and other PMs don't have any extent, start from next PM on the list.
2.5If some DBRoots have extents and some don't the DBRoot with the fewest extents or blocks is chosen,
"unless" the partition 0, segment 0 extent is one of the HWM extents. In that case the partition 0,
segment 0 extent still takes precedence.
3. If all PMs have extents, count the number of extents under each dbroot, find the dbroot which has the least extents to start
4. If all dbroots have same number of extents, starts from the dbroot which has least number of blocks
*/
//------------------------------------------------------------------------------
void BatchLoader::selectFirstPM ( uint32_t& PMId)
{
boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionId);
//cout << "calling tableName for oid " << fTableOid << endl;
CalpontSystemCatalog::TableName aTableName = systemCatalogPtr->tableName(fTableOid);
CalpontSystemCatalog::RIDList ridList;
ridList = systemCatalogPtr->columnRIDs(aTableName, true); //exception will be handled in caller program.
scoped_ptr<DBRM> dbrmp(new DBRM());
//Build distVec, start from the PM where the table is created. If not in the PM list, 0 will be used.
uint16_t createdDbroot = 0;
int rc = 0;
std::vector<BRM::EmDbRootHWMInfo_v> allInfo (fPMs.size());
for (unsigned i = 0; i < fPMs.size(); i++)
{
rc = dbrmp->getDbRootHWMInfo((ridList[0].objnum), fPMs[i], allInfo[i]);
if ( rc !=0 ) //@Bug 4760
break;
}
if ( rc != 0 ) {
ostringstream oss;
oss << "There is no extent information for table " << aTableName.table;
throw std::runtime_error(oss.str());
}
uint32_t numDbroot = fDbRoots.size();
if (numDbroot == 0)
{
throw std::runtime_error("There are no dbroots found during selectFirstPM");
}
uint64_t* rootExtents = (uint64_t*)alloca((numDbroot + 1) * sizeof(uint64_t)); //array of number extents for each dbroot
for (unsigned i = 0; i < fDbRoots.size(); i++)
{
uint64_t numExtents;
dbrmp->getExtentCount_dbroot((ridList[0].objnum), fDbRoots[i],
false, numExtents);
rootExtents[fDbRoots[i]] = numExtents;
}
bool startDBRootSet = false;
uint64_t* rootBlocks = (uint64_t*)alloca((numDbroot + 1) * sizeof(uint64_t)); //array of number of blocks for the last partition for each dbroot
//cout << "allInfo size is " << allInfo.size() << endl;
//--------------------------------------------------------------------------
// Load rootBlocks to carry total blocks for each DBRoot
// Set startDBRootSet if the partition 0 segment 0 extent is empty
// Set createdDbroot for the DBRoot that carries the partition 0
// segment 0 extent (used to set the default PMId)
//--------------------------------------------------------------------------
for (unsigned i=0; i < allInfo.size(); i++) //All PMs
{
BRM::EmDbRootHWMInfo_v emDbRootHWMInfos = allInfo[i]; //one pm
for (unsigned j=0; j < emDbRootHWMInfos.size(); j++)
{
if (emDbRootHWMInfos[j].totalBlocks == 0) {
//cout << "totalBlocks is 0" << endl;
continue; }
//------------------------------------------------------------------
// Ignore partition 0, segment 0 HWM extent if it is disabled
//------------------------------------------------------------------
if ((emDbRootHWMInfos[j].partitionNum == 0) &&
(emDbRootHWMInfos[j].segmentNum ==0) &&
(emDbRootHWMInfos[j].status != BRM::EXTENTOUTOFSERVICE))
{
if (emDbRootHWMInfos[j].localHWM == 0)
{
//newly created table
//cout << " This is newly created table. PM id is " << PMId;
startDBRootSet = true;
createdDbroot = emDbRootHWMInfos[j].dbRoot;
break;
}
else
{
createdDbroot = emDbRootHWMInfos[j].dbRoot;
//cout << " and createdDbroot is " << createdDbroot << endl;
rootBlocks[emDbRootHWMInfos[j].dbRoot] = emDbRootHWMInfos[j].totalBlocks;
}
}
else
{
rootBlocks[emDbRootHWMInfos[j].dbRoot] = emDbRootHWMInfos[j].totalBlocks;
}
}
if (startDBRootSet)
break;
}
//--------------------------------------------------------------------------
// Set the default PMId to the PM with the partition 0 segment 0 extent
//--------------------------------------------------------------------------
PMId = 0;
if ( createdDbroot != 0)
{
std::map<int, int>::iterator iter = fDbrootPMmap->begin();
iter = fDbrootPMmap->find(createdDbroot);
if (iter != fDbrootPMmap->end())
PMId = iter->second;
}
// This will build the batch distribution sequence
//cout << "Building BatchDistSeqVector with PMId " << PMId << endl;
buildBatchDistSeqVector(PMId);
//cout << "startDBRootSet = " << startDBRootSet << endl;
bool allEqual = true;
bool allOtherDbrootEmpty = true;
//--------------------------------------------------------------------------
// startDBRootSet == false
// We don't have an empty partition 0, segment 0 extent to load;
// so evaluate more selection criteria.
//--------------------------------------------------------------------------
if (!startDBRootSet)
{
std::vector<PMRootInfo> rootsExtentsBlocks;
std::map<int, OamCache::dbRoots>::iterator iter;
//----------------------------------------------------------------------
// Load rootsExtentsBlocks to carry the number of extents and blocks
// for each DBRoot.
//----------------------------------------------------------------------
for (unsigned j=0; j < fPmDistSeq.size(); j++)
{
PMRootInfo aEntry;
aEntry.PMId = fPmDistSeq[j];
iter = fPmDbrootMap->find(aEntry.PMId);
for (unsigned k=0; k < (iter->second).size(); k++)
{
RootExtentsBlocks aRootInfo;
aRootInfo.DBRoot = (iter->second)[k];
aRootInfo.numExtents = rootExtents[aRootInfo.DBRoot];
aRootInfo.numBlocks = rootBlocks[aRootInfo.DBRoot];
//cout << "aRootInfo DBRoot:numExtents:numBlocks = " << aRootInfo.DBRoot<<":"<<aRootInfo.numExtents<<":"<<aRootInfo.numBlocks<<endl;
aEntry.rootInfo.push_back(aRootInfo);
}
rootsExtentsBlocks.push_back(aEntry);
}
//cout << "rootsExtentsBlocks size is " << rootsExtentsBlocks.size() << " and allOtherDbrootEmpty is " << allOtherDbrootEmpty<< endl;
//----------------------------------------------------------------------
// See if all other DBRoots other than the createdDbroot have 0 extents
//----------------------------------------------------------------------
for (unsigned i=1; i < rootsExtentsBlocks.size(); i++)
{
if (!allOtherDbrootEmpty)
break;
//cout << "createdDbroot is " << createdDbroot << endl;
if (i != createdDbroot)
{
for ( unsigned j=0; j < rootsExtentsBlocks[i].rootInfo.size(); j++)
{
if (rootsExtentsBlocks[i].rootInfo[j].numExtents != 0)
{
//cout << "setting allOtherDbrootEmpty to false and i:j = " << i<<":"<<j<<endl;
//cout << "numExtents = " << rootsExtentsBlocks[i].rootInfo[j].numExtents << endl;
allOtherDbrootEmpty = false;
break;
}
}
}
}
//cout << "allOtherDbrootEmpty is " << allOtherDbrootEmpty << endl;
//----------------------------------------------------------------------
// allOtherDbrootEmpty == true
// No DBRoots (other than the DBRoot having the partition 0,
// segment 0 extent) have extents
//----------------------------------------------------------------------
if (allOtherDbrootEmpty)
{
//find the next PM id on the list
startDBRootSet = true;
buildBatchDistSeqVector();
allEqual = false;
}
//----------------------------------------------------------------------
// allOtherDbrootEmpty == false
// Some DBRoots (other than the DBRoot having the partition 0,
// segment 0 extent) have extents. More evaluation necessary.
//----------------------------------------------------------------------
else //find the dbroot which has the least extents to start
{
//cout << "finding least dbroot to start." << endl;
//------------------------------------------------------------------
// Select PM with DBRoot having the fewest extents.
//------------------------------------------------------------------
uint32_t tmpLeastExtents = rootsExtentsBlocks[0].rootInfo[0].numExtents;
PMId = rootsExtentsBlocks[0].PMId; //@Bug 4809.
for ( unsigned j=1; j < rootsExtentsBlocks[0].rootInfo.size(); j++)
{
if (tmpLeastExtents > rootsExtentsBlocks[0].rootInfo[j].numExtents)
tmpLeastExtents = rootsExtentsBlocks[0].rootInfo[j].numExtents;
}
for (unsigned i=1; i < rootsExtentsBlocks.size(); i++)
{
uint32_t leastExtents = rootsExtentsBlocks[i].rootInfo[0].numExtents;
for ( unsigned j=0; j < rootsExtentsBlocks[i].rootInfo.size(); j++)
{
if (leastExtents > rootsExtentsBlocks[i].rootInfo[j].numExtents)
leastExtents = rootsExtentsBlocks[i].rootInfo[j].numExtents;
}
if (leastExtents < tmpLeastExtents)
{
tmpLeastExtents = leastExtents;
PMId = rootsExtentsBlocks[i].PMId;
allEqual = false;
}
else if (leastExtents > tmpLeastExtents)
allEqual = false;
}
//cout << "allEqual is " << allEqual << endl;
//------------------------------------------------------------------
// All DBRoots have the same number of extents.
// Select PM with DBRoot having the fewest number of blocks.
//------------------------------------------------------------------
if (allEqual) //Find the dbroot which has least number of blocks
{
//cout << "All PMs have equal # of least extents" << endl;
uint32_t tmpBloks = rootsExtentsBlocks[0].rootInfo[0].numBlocks;
PMId = rootsExtentsBlocks[0].PMId;
//cout << "tmpBloks:PMId = " << tmpBloks <<":"<<PMId<<endl;
for ( unsigned j=1; j < rootsExtentsBlocks[0].rootInfo.size(); j++)
{
if (tmpBloks > rootsExtentsBlocks[0].rootInfo[j].numBlocks)
tmpBloks = rootsExtentsBlocks[0].rootInfo[j].numBlocks;
}
for (unsigned i=1; i < rootsExtentsBlocks.size(); i++)
{
uint32_t leastBlocks = rootsExtentsBlocks[i].rootInfo[0].numBlocks;
//cout << "leastBlocks = " << leastBlocks << endl;
for ( unsigned j=0; j < rootsExtentsBlocks[i].rootInfo.size(); j++)
{
if (leastBlocks > rootsExtentsBlocks[i].rootInfo[j].numBlocks)
leastBlocks = rootsExtentsBlocks[i].rootInfo[j].numBlocks;
}
if (leastBlocks < tmpBloks)
{
tmpBloks = leastBlocks;
//cout << "tmpBloks changed to " << tmpBloks << endl;
PMId = rootsExtentsBlocks[i].PMId;
//cout << "setting allEqual to false now" << endl;
allEqual = false;
}
}
}
}
}
//--------------------------------------------------------------------------
// startDBRootSet == true
// We select the empty partition 0, segment 0 extent to load
//--------------------------------------------------------------------------
else
{
allEqual = false;
}
fFirstPm = PMId;
if (!allOtherDbrootEmpty || (PMId == 0))
{
prepareForSecondPM();
//cout << "prepareForSecondPM is called. " << endl;
}
if ((allEqual && (PMId == 0)) || allOtherDbrootEmpty)
{
PMId = selectNextPM();
fFirstPm = PMId;
//cout << "PMId is now " << PMId << endl;
}
}
//------------------------------------------------------------------------------
/*
* Build the sequence of distribution as a vector.
*/
//------------------------------------------------------------------------------
void BatchLoader::buildBatchDistSeqVector()
{
fPmDistSeq.clear();
BlIntVec aDbCntVec(fPMs.size());
std::map<int, OamCache::dbRoots>::iterator iter = fPmDbrootMap->begin();
for (uint32_t i=0; i < fPMs.size(); i++)
{
iter = fPmDbrootMap->find(fPMs[i]);
if ((iter != fPmDbrootMap->end()) && ((iter->second).begin()!=(iter->second).end()))
{
try
{
aDbCntVec[i] = (iter->second).size();
//cout << "PM - "<<fPMs[i] << " Size = " << aDbCntVec[i] << endl;
}
catch(std::exception& exp)
{
throw runtime_error(exp.what());
}
}
else
aDbCntVec[i] = 0;
}
int aTotDbRoots = 0;
for (uint32_t i=0; i<aDbCntVec.size(); i++) aTotDbRoots+=aDbCntVec[i];
int aIdx=0;
while(aIdx < aTotDbRoots)
{
uint32_t aMax=0;
uint32_t aPmId=0;
uint32_t aRefIdx=0;
for (uint32_t i=0; i<aDbCntVec.size(); i++)
{
if(aDbCntVec[i] > aMax)
{
aMax = aDbCntVec[i];
aPmId = fPMs[i];
aRefIdx = i;
}
}
if(aMax>0)
{
fPmDistSeq.push_back(aPmId);
aDbCntVec[aRefIdx]--;
}
aIdx++;
}
}
//------------------------------------------------------------------------------
/*
* Build the sequence of distribution as a vector.
*/
//------------------------------------------------------------------------------
void BatchLoader::buildBatchDistSeqVector(uint32_t StartPm)
{
fPmDistSeq.clear();
BlIntVec aDbCntVec(fPMs.size());
BlIntVec aPms;
if((fPMs.size()==0)&&(StartPm!=0))
throw runtime_error("ERROR : PM list empty while Start != 0");
//for (uint32_t i=0; i<fPMs.size(); i++)
// cout <<"fPM list "<<i <<" = " << fPMs[i] << endl;
//cout << "StartPm = "<< StartPm << endl;
if (StartPm == 0)
aPms = fPMs;
else
{
aPms.push_back(StartPm);
uint32_t aLast = fPMs.back();
uint32_t aFirst = fPMs.front();
// Add all the PMs with index more than "StartPm"
for(uint32_t i=0; i<fPMs.size(); i++)
{
if((fPMs[i]>StartPm)&&(fPMs[i]<=aLast)) aPms.push_back(fPMs[i]);
}
// Add all the PMs with index less than "StartPm"
for(uint32_t i=0; i<fPMs.size(); i++)
{
if((fPMs[i]<StartPm)&&(fPMs[i]>=aFirst)) aPms.push_back(fPMs[i]);
}
}
std::map<int, OamCache::dbRoots>::iterator iter = fPmDbrootMap->begin();
for (uint32_t i=0; i < aPms.size(); i++)
{
iter = fPmDbrootMap->find(aPms[i]);
if ((iter != fPmDbrootMap->end()) && ((iter->second).begin()!=(iter->second).end()))
{
aDbCntVec[i] = (iter->second).size();
//cout << "PM - "<<aPms[i] << " Size = " << aDbCntVec[i] << endl;
}
else
aDbCntVec[i] = 0;
}
int aTotDbRoots = 0;
for (uint32_t i=0; i<aDbCntVec.size(); i++) aTotDbRoots+=aDbCntVec[i];
//cout << "DbCntVec Size = " << aDbCntVec.size() << " TotDbRoots = "<<aTotDbRoots << endl;
int aIdx=0;
while(aIdx < aTotDbRoots)
{
uint32_t aMax=0;
uint32_t aPmId=0;
uint32_t aRefIdx=0;
for (uint32_t i=0; i<aDbCntVec.size(); i++)
{
if(aDbCntVec[i] > aMax)
{
aMax = aDbCntVec[i];
aPmId = aPms[i];
aRefIdx = i;
}
}
if(aMax>0)
{
fPmDistSeq.push_back(aPmId);
aDbCntVec[aRefIdx]--;
}
aIdx++;
}
//cout <<"PM Distribution vector size "<< fPmDistSeq.size() << endl;
//for (uint32_t i=0; i<fPmDistSeq.size(); i++)
// cout <<"PM Distribution vector "<<i <<" = " << fPmDistSeq[i] << endl;
}
//------------------------------------------------------------------------------
uint32_t BatchLoader::selectNextPM()
{
if(0 == fPmDistSeq.size()) //Dist sequence not ready. First time the function is called
{
uint32_t PMId = 0;
//cout << "selectNextPM: size is 0. " << endl;
selectFirstPM(PMId);
return PMId;
}
if(fNextIdx >= fPmDistSeq.size()) fNextIdx=0; //reset it
return fPmDistSeq[fNextIdx++];
}
//------------------------------------------------------------------------------
void BatchLoader::reverseSequence()
{
if(0 == fNextIdx) fNextIdx = fPmDistSeq.size()-1;
else fNextIdx--;
}
//------------------------------------------------------------------------------
void BatchLoader::prepareForSecondPM()
{
// We loop thru by calling the selectNextPM()
// When we get the aSecPm == aFirstPM, we will break so that when
// we call the same function again when we come back get the next PM
if((fFirstPm != 0)&&(fPMs.size()>1))
{
unsigned int aSecPm = selectNextPM();
while(fFirstPm!=aSecPm ) aSecPm = selectNextPM();
}
}
} // end of namespace

133
utils/batchloader/batchloader.h Normal file
View File

@@ -0,0 +1,133 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: we_dbrootextenttracker.h 3672 2012-03-26 12:31:27Z rdempsey $
*/
/** @file we_dbrootextenttracker.h
* Contains classes to track the order of placement (rotation) of extents as
* they are filled in and/or added to the DBRoots for the local PM.
*/
#ifndef BATCHLOADER_H_
#define BATCHLOADER_H_
#include <vector>
#include "oamcache.h"
#include "brmtypes.h"
#include "calpontsystemcatalog.h"
namespace batchloader
{
//------------------------------------------------------------------------------
/** @brief Class to find the PM id to send a batch of row
*/
//------------------------------------------------------------------------------
class BatchLoader
{
public:
/** @brief BatchLoader constructor
* @param tableOid table OID of interest.
* @param sessionId cpimport use 0 as session id.
* @param PMs vector Collection of PM ids.
*/
BatchLoader ( uint32_t tableOid, execplan::CalpontSystemCatalog::SCN sessionId, std::vector<uint32_t>& PMs);
/**
* @brief select the Next PM where batch data to be distributed.
* return the PM where next batch to be send.
* if an error occurs, 0 will be returned
*/
uint32_t selectNextPM();
/**
* @brief Move to previous Sequence in the array.
* This can be used when we find that we cannot use the PM right now
* and want to use it later. for example the queue is full of the
* current PM and we want to get the same PM Id when we call selectNextPM()
* next time also.
*/
void reverseSequence();
/*
* @brief After calling selectFirstPM(), if we need to keep continuing to
* the next PM in the list, we need to call this. If we just want to start
* distributing from dbroot 1 onwards, no need to call this function.
*/
void prepareForSecondPM();
struct RootExtentsBlocks
{
/** @brief the dbroot
*/
uint32_t DBRoot;
/** @brief the number of extents
*/
uint64_t numExtents;
/** @brief the number of blocks in the last partition
*/
uint64_t numBlocks;
};
struct PMRootInfo
{
/** @brief the module id
*/
uint32_t PMId;
/** @brief the dbroot info
*/
std::vector<RootExtentsBlocks> rootInfo;
};
private:
/** @brief Select the first PM to send the first batch of rows.
* @param startFromNextPM - if true, don't use the PMId. Instead use the next one in the sorted dmList.
* @param PMId - The PM id to send the first batch of rows if startFromNextPM is false.
* @return Returns 0 if success, else returns error code.
*/
void selectFirstPM ( uint32_t& PMId);
/** @brief build the batch distribution sequence in a vector
* return void
*/
void buildBatchDistSeqVector();
/** @brief build the batch distribution sequence in a vector
* return void
*/
void buildBatchDistSeqVector(uint32_t StartPm);
typedef std::vector<uint32_t> BlIntVec;
BlIntVec fPMs;
BlIntVec fDbRoots;
BlIntVec fPmDistSeq;
uint32_t fNextIdx;
uint32_t fFirstPm;
execplan::CalpontSystemCatalog::SCN fSessionId;
uint32_t fTableOid;
oam::OamCache::PMDbrootsMap_t fPmDbrootMap;
oam::OamCache::dbRootPMMap_t fDbrootPMmap;
};
} //end of namespace
#undef EXPORT
#endif // WE_DBROOTEXTENTTRACKER_H_