database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

the begginning

Browse Source
This commit is contained in:
david hill
2016-01-06 14:08:59 -06:00
parent 66a31debcb
commit f6afc42dd0
18251 changed files with 16460679 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
writeengine/splitter/Makefile Executable file
View File

@ -0,0 +1,105 @@
include ../../rules.mak
include ../build/we_rules.mak
#******************************************************************************************
# Copyright (C) 2009-2012 Calpont Corporation
# All rights reserved
#
# $Id: Makefile 3153 2011-10-12 20:41:55Z bpaul $
#
#*****************************************************************************************/
# The name of the executable
PROGRAM=cpimport
VERSION=1.0.0
# List all the source files here
SRCS=we_splitterapp.cpp \
we_cmdargs.cpp \
we_sdhandler.cpp \
we_respreadthread.cpp \
we_filereadthread.cpp \
we_splclient.cpp \
we_brmupdater.cpp \
we_tablelockgrabber.cpp \
we_xmlgetter.cpp
# Preprocessor flags
CPPFLAGS=-I$(EXPORT_ROOT)/include -I/usr/include/libxml2
# Compiler flags
CXXFLAGS+=$(DEBUG_FLAGS) -Wall
# Linker flags
LDFLAGS+=-L$(CALPONT_LIBRARY_PATH) $(IDB_COMMON_LIBS) $(IDB_WRITE_LIBS) -lthreadpool $(IDB_SNMP_LIBS) -Wl,--rpath -Wl,/usr/local/Calpont/lib -lbatchloader
.PHONY: install clean docs test coverage leakcheck
TLIBS+=-L$(CALPONT_LIBRARY_PATH) $(IDB_COMMON_LIBS) $(IDB_WRITE_LIBS)
GLIBS=-L$(CALPONT_LIBRARY_PATH) $(IDB_COMMON_LIBS) $(IDB_WRITE_LIBS) $(IDB_SNMP_LIBS) -lthreadpool -ldl -lbatchloader
LINCLUDES=
OBJS=$(SRCS:.cpp=.o)
objects: $(OBJS)
GOBJS=$(LOBJS:.o=-gcov.o)
$(PROGRAM): $(OBJS)
$(LINK.cpp) -o $@ $^
install: bootstrap
install_splitter: $(PROGRAM)
mkdir -p $(INSTALL_ROOT_BIN)
$(INSTALL) $(PROGRAM) $(INSTALL_ROOT_BIN)
bootstrap:
# $(INSTALL) $(LINCLUDES) $(INSTALL_ROOT_INCLUDE)
clean:
rm -f $(PROGRAM) $(OBJS) core *~ *.tag *-gcov.* *.gcov *.d *.swp *.dat
rm -rf html
docs:
doxygen $(EXPORT_ROOT)/etc/Doxyfile
test:
xtest: $(PROGRAM)
LD_LIBRARY_PATH=$(LD_LIBRARY_PATH) ./$(PROGRAM)
%-gcov.o: %.cpp
$(COMPILE.cpp) -o $@ $^
$(PROGRAM)-gcov: CXXFLAGS+=-fprofile-arcs -ftest-coverage
$(PROGRAM)-gcov: $(PROGRAM)-gcov.o $(subst .o,-gcov.o,$(OBJS))
$(LINK.cpp) -o $@ $^ $(GOBJS) $(GLIBS)
cp *-gcov.o $(LIBDIR)
coverage:
xcoverage: $(PROGRAM)-gcov
$(IPCS_CLEANUP)
rm -f *.gcda
LD_LIBRARY_PATH=$(LD_LIBRARY_PATH) ./$(PROGRAM)-gcov
for file in $(SRCS); do \
bfile=`basename $$file .cpp`; \
gcov -o $${bfile}-gcov $$file >/dev/null; \
done
# /usr/local/bin/genCoverage.pl $(SRCS)
leakcheck:
xleakcheck: $(PROGRAM)
$(IPCS_CLEANUP)
LD_LIBRARY_PATH=$(LD_LIBRARY_PATH) valgrind --error-limit=no --tool=memcheck --leak-check=yes ./$(PROGRAM)
%.d: %.cpp
@set -e; rm -f $@; \
$(CC) -MM $(CPPFLAGS) $< > $@.$$$$; \
sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$$ > $@; \
rm -f $@.$$$$
ifndef BOOTSTRAP
-include $(SRCS:.cpp=.d)
endif

48
writeengine/splitter/Makefile.am Normal file
View File

@ -0,0 +1,48 @@
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
# $Id: Makefile.am 696 2011-03-07 21:54:16Z rdempsey $
## Process this file with automake to produce Makefile.in
AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags)
AM_CFLAGS = $(idb_cflags)
AM_CXXFLAGS = $(idb_cxxflags)
AM_LDFLAGS = $(idb_ldflags)
bin_PROGRAMS = cpimport
cpimport_SOURCES = we_splitterapp.cpp \
we_cmdargs.cpp \
we_sdhandler.cpp \
we_respreadthread.cpp \
we_filereadthread.cpp \
we_splclient.cpp \
we_brmupdater.cpp \
we_tablelockgrabber.cpp \
we_xmlgetter.cpp
cpimport_LDFLAGS = $(idb_common_ldflags) $(idb_exec_libs) $(idb_write_libs) -lbatchloader -lthreadpool $(AM_LDFLAGS)
install_splitter: install
test:
coverage:
leakcheck:
docs:
bootstrap: install-data-am

537
writeengine/splitter/Makefile.in Normal file
View File

@ -0,0 +1,537 @@
# Makefile.in generated by automake 1.9.6 from Makefile.am.
# @configure_input@
# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
# 2003, 2004, 2005 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
# with or without modifications, as long as this notice is preserved.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.
@SET_MAKE@
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
# $Id: Makefile.am 696 2011-03-07 21:54:16Z rdempsey $
srcdir = @srcdir@
top_srcdir = @top_srcdir@
VPATH = @srcdir@
pkgdatadir = $(datadir)/@PACKAGE@
pkglibdir = $(libdir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
top_builddir = ../..
am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
INSTALL = @INSTALL@
install_sh_DATA = $(install_sh) -c -m 644
install_sh_PROGRAM = $(install_sh) -c
install_sh_SCRIPT = $(install_sh) -c
INSTALL_HEADER = $(INSTALL_DATA)
transform = $(program_transform_name)
NORMAL_INSTALL = :
PRE_INSTALL = :
POST_INSTALL = :
NORMAL_UNINSTALL = :
PRE_UNINSTALL = :
POST_UNINSTALL = :
build_triplet = @build@
host_triplet = @host@
bin_PROGRAMS = cpimport$(EXEEXT)
subdir = writeengine/splitter
DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/m4/compilerflags.m4 \
$(top_srcdir)/m4/functions.m4 $(top_srcdir)/m4/install.m4 \
$(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
mkinstalldirs = $(install_sh) -d
CONFIG_HEADER = $(top_builddir)/config.h
CONFIG_CLEAN_FILES =
am__installdirs = "$(DESTDIR)$(bindir)"
binPROGRAMS_INSTALL = $(INSTALL_PROGRAM)
PROGRAMS = $(bin_PROGRAMS)
am_cpimport_OBJECTS = we_splitterapp.$(OBJEXT) we_cmdargs.$(OBJEXT) \
we_sdhandler.$(OBJEXT) we_respreadthread.$(OBJEXT) \
we_filereadthread.$(OBJEXT) we_splclient.$(OBJEXT) \
we_brmupdater.$(OBJEXT) we_tablelockgrabber.$(OBJEXT) \
we_xmlgetter.$(OBJEXT)
cpimport_OBJECTS = $(am_cpimport_OBJECTS)
cpimport_LDADD = $(LDADD)
DEFAULT_INCLUDES = -I. -I$(srcdir) -I$(top_builddir)
depcomp = $(SHELL) $(top_srcdir)/depcomp
am__depfiles_maybe = depfiles
CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
LTCXXCOMPILE = $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) \
$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
$(AM_CXXFLAGS) $(CXXFLAGS)
CXXLD = $(CXX)
CXXLINK = $(LIBTOOL) --tag=CXX --mode=link $(CXXLD) $(AM_CXXFLAGS) \
$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
SOURCES = $(cpimport_SOURCES)
DIST_SOURCES = $(cpimport_SOURCES)
ETAGS = etags
CTAGS = ctags
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
ACLOCAL = @ACLOCAL@
ALLOCA = @ALLOCA@
AMDEP_FALSE = @AMDEP_FALSE@
AMDEP_TRUE = @AMDEP_TRUE@
AMTAR = @AMTAR@
AR = @AR@
AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
CC = @CC@
CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@
CPP = @CPP@
CPPFLAGS = @CPPFLAGS@
CXX = @CXX@
CXXCPP = @CXXCPP@
CXXDEPMODE = @CXXDEPMODE@
CXXFLAGS = @CXXFLAGS@
CYGPATH_W = @CYGPATH_W@
DEFS = @DEFS@
DEPDIR = @DEPDIR@
ECHO = @ECHO@
ECHO_C = @ECHO_C@
ECHO_N = @ECHO_N@
ECHO_T = @ECHO_T@
EGREP = @EGREP@
EXEEXT = @EXEEXT@
F77 = @F77@
FFLAGS = @FFLAGS@
INSTALL_DATA = @INSTALL_DATA@
INSTALL_PROGRAM = @INSTALL_PROGRAM@
INSTALL_SCRIPT = @INSTALL_SCRIPT@
INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
LDFLAGS = @LDFLAGS@
LEX = @LEX@
LEXLIB = @LEXLIB@
LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
LIBOBJS = @LIBOBJS@
LIBS = @LIBS@
LIBTOOL = @LIBTOOL@
LN_S = @LN_S@
LTLIBOBJS = @LTLIBOBJS@
MAKEINFO = @MAKEINFO@
OBJEXT = @OBJEXT@
PACKAGE = @PACKAGE@
PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_STRING = @PACKAGE_STRING@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
PATH_SEPARATOR = @PATH_SEPARATOR@
POW_LIB = @POW_LIB@
RANLIB = @RANLIB@
SED = @SED@
SET_MAKE = @SET_MAKE@
SHELL = @SHELL@
STRIP = @STRIP@
VERSION = @VERSION@
XML2_CONFIG = @XML2_CONFIG@
XML_CPPFLAGS = @XML_CPPFLAGS@
XML_LIBS = @XML_LIBS@
YACC = @YACC@
ac_ct_AR = @ac_ct_AR@
ac_ct_CC = @ac_ct_CC@
ac_ct_CXX = @ac_ct_CXX@
ac_ct_F77 = @ac_ct_F77@
ac_ct_RANLIB = @ac_ct_RANLIB@
ac_ct_STRIP = @ac_ct_STRIP@
am__fastdepCC_FALSE = @am__fastdepCC_FALSE@
am__fastdepCC_TRUE = @am__fastdepCC_TRUE@
am__fastdepCXX_FALSE = @am__fastdepCXX_FALSE@
am__fastdepCXX_TRUE = @am__fastdepCXX_TRUE@
am__include = @am__include@
am__leading_dot = @am__leading_dot@
am__quote = @am__quote@
am__tar = @am__tar@
am__untar = @am__untar@
bindir = @bindir@
build = @build@
build_alias = @build_alias@
build_cpu = @build_cpu@
build_os = @build_os@
build_vendor = @build_vendor@
datadir = @datadir@
etcdir = @etcdir@
exec_prefix = @exec_prefix@
host = @host@
host_alias = @host_alias@
host_cpu = @host_cpu@
host_os = @host_os@
host_vendor = @host_vendor@
idb_brm_libs = @idb_brm_libs@
idb_cflags = @idb_cflags@
idb_common_includes = @idb_common_includes@
idb_common_ldflags = @idb_common_ldflags@
idb_common_libs = @idb_common_libs@
idb_cppflags = @idb_cppflags@
idb_cxxflags = @idb_cxxflags@
idb_exec_libs = @idb_exec_libs@
idb_ldflags = @idb_ldflags@
idb_oam_libs = @idb_oam_libs@
idb_write_libs = @idb_write_libs@
idbinstall = @idbinstall@
includedir = @includedir@
infodir = @infodir@
install_sh = @install_sh@
libdir = @libdir@
libexecdir = @libexecdir@
localdir = @localdir@
localstatedir = @localstatedir@
mandir = @mandir@
march_flags = @march_flags@
mibdir = @mibdir@
mkdir_p = @mkdir_p@
mysqldir = @mysqldir@
netsnmp_libs = @netsnmp_libs@
netsnmpagntdir = @netsnmpagntdir@
netsnmpdir = @netsnmpdir@
netsnmplibrdir = @netsnmplibrdir@
netsnmpmachdir = @netsnmpmachdir@
netsnmpsysdir = @netsnmpsysdir@
oldincludedir = @oldincludedir@
postdir = @postdir@
prefix = @prefix@
program_transform_name = @program_transform_name@
sbindir = @sbindir@
sharedir = @sharedir@
sharedstatedir = @sharedstatedir@
sysconfdir = @sysconfdir@
target_alias = @target_alias@
toolsdir = @toolsdir@
AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags)
AM_CFLAGS = $(idb_cflags)
AM_CXXFLAGS = $(idb_cxxflags)
AM_LDFLAGS = $(idb_ldflags)
cpimport_SOURCES = we_splitterapp.cpp \
we_cmdargs.cpp \
we_sdhandler.cpp \
we_respreadthread.cpp \
we_filereadthread.cpp \
we_splclient.cpp \
we_brmupdater.cpp \
we_tablelockgrabber.cpp \
we_xmlgetter.cpp
cpimport_LDFLAGS = $(idb_common_ldflags) $(idb_exec_libs) $(idb_write_libs) -lbatchloader -lthreadpool $(AM_LDFLAGS)
all: all-am
.SUFFIXES:
.SUFFIXES: .cpp .lo .o .obj
$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps)
@for dep in $?; do \
case '$(am__configure_deps)' in \
*$$dep*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh \
&& exit 0; \
exit 1;; \
esac; \
done; \
echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu writeengine/splitter/Makefile'; \
cd $(top_srcdir) && \
$(AUTOMAKE) --gnu writeengine/splitter/Makefile
.PRECIOUS: Makefile
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
@case '$?' in \
*config.status*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
*) \
echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
esac;
$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(top_srcdir)/configure: $(am__configure_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(ACLOCAL_M4): $(am__aclocal_m4_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
install-binPROGRAMS: $(bin_PROGRAMS)
@$(NORMAL_INSTALL)
test -z "$(bindir)" || $(mkdir_p) "$(DESTDIR)$(bindir)"
@list='$(bin_PROGRAMS)'; for p in $$list; do \
p1=`echo $$p|sed 's/$(EXEEXT)$$//'`; \
if test -f $$p \
|| test -f $$p1 \
; then \
f=`echo "$$p1" | sed 's,^.*/,,;$(transform);s/$$/$(EXEEXT)/'`; \
echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) --mode=install $(binPROGRAMS_INSTALL) '$$p' '$(DESTDIR)$(bindir)/$$f'"; \
$(INSTALL_PROGRAM_ENV) $(LIBTOOL) --mode=install $(binPROGRAMS_INSTALL) "$$p" "$(DESTDIR)$(bindir)/$$f" || exit 1; \
else :; fi; \
done
uninstall-binPROGRAMS:
@$(NORMAL_UNINSTALL)
@list='$(bin_PROGRAMS)'; for p in $$list; do \
f=`echo "$$p" | sed 's,^.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/'`; \
echo " rm -f '$(DESTDIR)$(bindir)/$$f'"; \
rm -f "$(DESTDIR)$(bindir)/$$f"; \
done
clean-binPROGRAMS:
@list='$(bin_PROGRAMS)'; for p in $$list; do \
f=`echo $$p|sed 's/$(EXEEXT)$$//'`; \
echo " rm -f $$p $$f"; \
rm -f $$p $$f ; \
done
cpimport$(EXEEXT): $(cpimport_OBJECTS) $(cpimport_DEPENDENCIES)
@rm -f cpimport$(EXEEXT)
$(CXXLINK) $(cpimport_LDFLAGS) $(cpimport_OBJECTS) $(cpimport_LDADD) $(LIBS)
mostlyclean-compile:
-rm -f *.$(OBJEXT)
distclean-compile:
-rm -f *.tab.c
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_brmupdater.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_cmdargs.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_filereadthread.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_respreadthread.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_sdhandler.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_splclient.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_splitterapp.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_tablelockgrabber.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_xmlgetter.Po@am__quote@
.cpp.o:
@am__fastdepCXX_TRUE@ if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $<
.cpp.obj:
@am__fastdepCXX_TRUE@ if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ `$(CYGPATH_W) '$<'`; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
.cpp.lo:
@am__fastdepCXX_TRUE@ if $(LTCXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Plo"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $<
mostlyclean-libtool:
-rm -f *.lo
clean-libtool:
-rm -rf .libs _libs
distclean-libtool:
-rm -f libtool
uninstall-info-am:
ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
mkid -fID $$unique
tags: TAGS
TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$(TAGS_FILES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
if test -z "$(ETAGS_ARGS)$$tags$$unique"; then :; else \
test -n "$$unique" || unique=$$empty_fix; \
$(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
$$tags $$unique; \
fi
ctags: CTAGS
CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$(TAGS_FILES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
test -z "$(CTAGS_ARGS)$$tags$$unique" \
|| $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
$$tags $$unique
GTAGS:
here=`$(am__cd) $(top_builddir) && pwd` \
&& cd $(top_srcdir) \
&& gtags -i $(GTAGS_ARGS) $$here
distclean-tags:
-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \
topsrcdirstrip=`echo "$(top_srcdir)" | sed 's|.|.|g'`; \
list='$(DISTFILES)'; for file in $$list; do \
case $$file in \
$(srcdir)/*) file=`echo "$$file" | sed "s|^$$srcdirstrip/||"`;; \
$(top_srcdir)/*) file=`echo "$$file" | sed "s|^$$topsrcdirstrip/|$(top_builddir)/|"`;; \
esac; \
if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
dir=`echo "$$file" | sed -e 's,/[^/]*$$,,'`; \
if test "$$dir" != "$$file" && test "$$dir" != "."; then \
dir="/$$dir"; \
$(mkdir_p) "$(distdir)$$dir"; \
else \
dir=''; \
fi; \
if test -d $$d/$$file; then \
if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
cp -pR $(srcdir)/$$file $(distdir)$$dir || exit 1; \
fi; \
cp -pR $$d/$$file $(distdir)$$dir || exit 1; \
else \
test -f $(distdir)/$$file \
|| cp -p $$d/$$file $(distdir)/$$file \
|| exit 1; \
fi; \
done
check-am: all-am
check: check-am
all-am: Makefile $(PROGRAMS)
installdirs:
for dir in "$(DESTDIR)$(bindir)"; do \
test -z "$$dir" || $(mkdir_p) "$$dir"; \
done
install: install-am
install-exec: install-exec-am
install-data: install-data-am
uninstall: uninstall-am
install-am: all-am
@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
installcheck: installcheck-am
install-strip:
$(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
`test -z '$(STRIP)' || \
echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install
mostlyclean-generic:
clean-generic:
distclean-generic:
-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
maintainer-clean-generic:
@echo "This command is intended for maintainers to use"
@echo "it deletes files that may require special tools to rebuild."
clean: clean-am
clean-am: clean-binPROGRAMS clean-generic clean-libtool mostlyclean-am
distclean: distclean-am
-rm -rf ./$(DEPDIR)
-rm -f Makefile
distclean-am: clean-am distclean-compile distclean-generic \
distclean-libtool distclean-tags
dvi: dvi-am
dvi-am:
html: html-am
info: info-am
info-am:
install-data-am:
install-exec-am: install-binPROGRAMS
install-info: install-info-am
install-man:
installcheck-am:
maintainer-clean: maintainer-clean-am
-rm -rf ./$(DEPDIR)
-rm -f Makefile
maintainer-clean-am: distclean-am maintainer-clean-generic
mostlyclean: mostlyclean-am
mostlyclean-am: mostlyclean-compile mostlyclean-generic \
mostlyclean-libtool
pdf: pdf-am
pdf-am:
ps: ps-am
ps-am:
uninstall-am: uninstall-binPROGRAMS uninstall-info-am
.PHONY: CTAGS GTAGS all all-am check check-am clean clean-binPROGRAMS \
clean-generic clean-libtool ctags distclean distclean-compile \
distclean-generic distclean-libtool distclean-tags distdir dvi \
dvi-am html html-am info info-am install install-am \
install-binPROGRAMS install-data install-data-am install-exec \
install-exec-am install-info install-info-am install-man \
install-strip installcheck installcheck-am installdirs \
maintainer-clean maintainer-clean-generic mostlyclean \
mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
pdf pdf-am ps ps-am tags uninstall uninstall-am \
uninstall-binPROGRAMS uninstall-info-am
install_splitter: install
test:
coverage:
leakcheck:
docs:
bootstrap: install-data-am
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
.NOEXPORT:

14
writeengine/splitter/resource.h Normal file
View File

@ -0,0 +1,14 @@
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by cpimport.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif

101
writeengine/splitter/splitter.rc Normal file
View File

@ -0,0 +1,101 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "FileDescription", "InfiniDB Bulk Loader"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "splitter"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "cpimport.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

637
writeengine/splitter/we_brmupdater.cpp Normal file
View File

@ -0,0 +1,637 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_brmupdater.cpp
*
* Created on: Dec 13, 2011
* Author: bpaul
*/
#include <iostream>
#include <sstream>
using namespace std;
#include "we_define.h"
#include <boost/lexical_cast.hpp>
using namespace boost;
//#include "we_simplesyslog.h"
#include "we_sdhandler.h"
#include "brm.h"
#include "brmtypes.h"
using namespace BRM;
#include "we_brmupdater.h"
namespace WriteEngine
{
//-----------------------------------------------------------------------------
bool WEBrmUpdater::updateCasualPartitionAndHighWaterMarkInBRM()
{
try
{
bool aGood = prepareCasualPartitionInfo();
if(!aGood)
cout << "prepareCasualPartitionInfo Failed" << endl;
}
catch(std::exception& ex)
{
std::string aStr = "Exception in prepareCasualPartitionInfo(); Error Ignored";
logging::Message::Args errMsgArgs;
errMsgArgs.add(aStr);
errMsgArgs.add(ex.what());
fRef.sysLog(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000);
cout << aStr << endl;
}
try
{
bool aSuccess = prepareHighWaterMarkInfo();
if(!aSuccess)
throw(std::runtime_error("prepareHWMInfo Failed"));
}
catch(std::exception& ex)
{
std::string aStr = "prepareHWMInfo() failed... Bailing out!!";
logging::Message::Args errMsgArgs;
errMsgArgs.add(aStr);
aStr = "Need to rollback bulk upload";
errMsgArgs.add(aStr);
errMsgArgs.add(ex.what());
fRef.sysLog(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000);
cout << aStr << endl;
return false;
}
// If we are here, we packaged informations properly
// Lets connect to BRM
if(!createBrmConnection())
{
cout << "Brm Connection FAILED" << endl;
return false;
}
int aRc = updateCPAndHWMInBRM();
if(aRc != 0)
{
cout << "Updating High Water Mark Failed" << endl;
}
else
{
if(fRef.getDebugLvl())
cout << "Updating High Water Mark Successful!!" << endl << endl;
}
/*
int cpRc = updateCasualPartitionInBRM();
if(cpRc != 0)
cout << "Updating Casual Partition Failed" << endl;
else
{
if(fRef.getDebugLvl())
cout << "Updating Casual Partition Successful" << endl;
}
int hwmRc = updateHighWaterMarkInBRM();
if(hwmRc != 0)
cout << "Updating High Water Mark Failed" << endl;
else
{
if(fRef.getDebugLvl())
cout << "Updating High Water Mark Successful!!" << endl << endl;
}
*/
releaseBrmConnection();
//return(!hwmRc)? true: false;
return(!aRc)? true: false;
}
//------------------------------------------------------------------------------
// Update Casual Partition information in BRM
//------------------------------------------------------------------------------
int WEBrmUpdater::updateCasualPartitionInBRM()
{
int rc = 0;
if (fCPInfo.size() > 0)
{
//std::ostringstream oss;
//oss << "Committing " << fCPInfo.size() << " CP updates for table " <<
// fRef.getTableName() << " to BRM";
//cout << endl << oss.str() << endl;
//TODO - NOTE. later make this Objection creation once for both CP & HWM
if(fpBrm)
{
rc = fpBrm->mergeExtentsMaxMin(fCPInfo);
if(rc != BRM::ERR_OK)
{
std::string errStr;
BRM::errString(rc, errStr);
cout << "BRM ERROR is ***** " << errStr << endl;
std::ostringstream oss;
oss << "Error updating BRM with CP data for table " <<
fRef.getTableName() <<" Error: "<< errStr << endl;
cout << endl << oss.str() << endl;
}
}
}
return rc;
}
//------------------------------------------------------------------------------
// Send HWM update information to BRM
//------------------------------------------------------------------------------
int WEBrmUpdater::updateHighWaterMarkInBRM()
{
int rc = 0;
if (fHWMInfo.size() > 0)
{
//std::ostringstream oss;
//oss << "Committing " << fHWMInfo.size() << " HWM update(s) for table "<<
// fRef.getTableName() << " to BRM";
//cout << endl << oss.str() << endl;
if(fpBrm)
{
rc = fpBrm->bulkSetHWM(fHWMInfo, 0);
if(rc != BRM::ERR_OK)
{
std::string errStr;
BRM::errString(rc, errStr);
cout << "BRM ERROR is ***** " << errStr << endl;
std::ostringstream oss;
oss << "Error updating BRM with HWM data for table "<<
fRef.getTableName() << "error: " << errStr << endl;
cout << endl << oss.str() << endl;
return rc;
}
}
}
return rc;
}
//-----------------------------------------------------------------------------
int WEBrmUpdater::updateCPAndHWMInBRM()
{
int rc = 0;
//BUG 4232. some imports may not contain CP but HWM
if ((fCPInfo.size() > 0) || (fHWMInfo.size() > 0))
{
//TODO - NOTE. later make this Objection creation once for both CP & HWM
if(fpBrm)
{
/*
rc = bulkSetHWMAndCP(const std::vector<BulkSetHWMArg> &,
const std::vector<CPInfo> & setCPDataArgs,
const std::vector<CPInfoMerge> & mergeCPDataArgs,
VER_t transID = 0) DBRM_THROW;
*/
rc = fpBrm->bulkSetHWMAndCP(fHWMInfo, fCPInfoData, fCPInfo, 0);
//rc = fpBrm->mergeExtentsMaxMin(fCPInfo);
//rc = fpBrm->bulkSetHWM(fHWMInfo, 0);
if(rc != BRM::ERR_OK)
{
std::string errStr;
BRM::errString(rc, errStr);
cout << "BRM ERROR is ***** " << errStr << endl;
std::ostringstream oss;
oss << "Error updating BRM with HWM data for table "<<
fRef.getTableName() << "error: " << errStr << endl;
cout << endl << oss.str() << endl;
cout << "ERROR: HWM and CP set failed!!" << endl;
//throw runtime_error("ERROR: bulSetHWMAndCp Failed!!");
return rc;
}
}
else
return rc;
}
return rc;
}
//------------------------------------------------------------------------------
bool WEBrmUpdater::prepareCasualPartitionInfo()
{
//cout << "Started prepareCasualPartitionInfo()!!" << endl;
//CP: 275456 6000000 4776193 -1 0 1
WESDHandler::StrVec::iterator aIt = fRef.fBrmRptVec.begin();
while (aIt != fRef.fBrmRptVec.end())
{
std::string aEntry = *aIt;
if ((!aEntry.empty()) && (aEntry.at(0) == 'C'))
{
BRM::CPInfoMerge cpInfoMerge;
const int BUFLEN=128;
char aBuff[BUFLEN];
strncpy(aBuff, aEntry.c_str(),BUFLEN);
aBuff[BUFLEN-1]=0;
char*pTok = strtok(aBuff, " ");
if (!pTok) // ignore the Msg Body
{
//cout << "CP Entry : " << aEntry << endl;
throw(runtime_error("Bad Body in CP entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
cpInfoMerge.startLbid = boost::lexical_cast<uint64_t>(pTok);
else
{
//cout << "CP Entry : " << aEntry << endl;
throw(runtime_error("Bad startLbid in CP entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
cpInfoMerge.max = boost::lexical_cast<int64_t>(pTok);
else
{
//cout << "CP Entry : " << aEntry << endl;
throw(runtime_error("Bad MAX in CP entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
cpInfoMerge.min = boost::lexical_cast<int64_t>(pTok);
else
{
//cout << "CP Entry : " << aEntry << endl;
throw(runtime_error("Bad MIN in CP entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
cpInfoMerge.seqNum = atoi(pTok);
else
{
//cout << "CP Entry : " << aEntry << endl;
throw(runtime_error("Bad seqNUM in CP entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
cpInfoMerge.type = (execplan::CalpontSystemCatalog::ColDataType)atoi(pTok);
else
{
//cout << "CP Entry : " << aEntry << endl;
throw(runtime_error("Bad type in CP entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
cpInfoMerge.newExtent = (atoi(pTok) != 0);
else
{
//cout << "CP Entry : " << aEntry << endl;
throw(runtime_error("Bad newExtent in CP entry string"));
}
fCPInfo.push_back(cpInfoMerge);
}
++aIt;
}
if(fRef.getDebugLvl())
cout << "Finished prepareCasualPartitionInfo()!!" << endl;
return true;
}
//-----------------------------------------------------------------------------
bool WEBrmUpdater::prepareHighWaterMarkInfo()
{
//HWM: 3056 0 0 8191
WESDHandler::StrVec::iterator aIt = fRef.fBrmRptVec.begin();
while(aIt != fRef.fBrmRptVec.end())
{
std::string aEntry = *aIt;
if ((!aEntry.empty()) && (aEntry.at(0) == 'H'))
{
BRM::BulkSetHWMArg hwmArg;
const int BUFLEN=128;
char aBuff[BUFLEN];
strncpy(aBuff, aEntry.c_str(),BUFLEN);
aBuff[BUFLEN-1]=0;
char*pTok = strtok(aBuff, " ");
if(!pTok) // ignore the Msg Body
{
//cout << "HWM Entry : " << aEntry << endl;
throw (runtime_error("Bad Body in HWM entry string"));
}
pTok = strtok(NULL, " ");
if(pTok)
hwmArg.oid = atoi(pTok);
else
{
//cout << "HWM Entry : " << aEntry << endl;
throw (runtime_error("Bad OID in HWM entry string"));
}
pTok = strtok(NULL, " ");
if(pTok)
hwmArg.partNum = atoi(pTok);
else
{
//cout << "HWM Entry : " << aEntry << endl;
throw (runtime_error("Bad partNum in HWM entry string"));
}
pTok = strtok(NULL, " ");
if(pTok)
hwmArg.segNum = atoi(pTok);
else
{
//cout << "HWM Entry : " << aEntry << endl;
throw (runtime_error("Bad partNum in HWM entry string"));
}
pTok = strtok(NULL, " ");
if(pTok)
hwmArg.hwm = atoi(pTok);
else
{
//cout << "HWM Entry : " << aEntry << endl;
throw (runtime_error("Bad partNum in HWM entry string"));
}
fHWMInfo.push_back( hwmArg );
}
++aIt;
}
if(fRef.getDebugLvl())
cout << "prepareHighWaterMarkInfo() finished" << endl;
return true;
}
//------------------------------------------------------------------------------
//#ROWS: numRowsRead numRowsInserted
bool WEBrmUpdater::prepareRowsInsertedInfo(std::string Entry,
int64_t& TotRows, int64_t& InsRows)
{
bool aFound=false;
//ROWS: 3 1
if ((!Entry.empty()) && (Entry.at(0) == 'R'))
{
aFound = true;
const int BUFLEN=128;
char aBuff[BUFLEN];
strncpy(aBuff, Entry.c_str(),BUFLEN);
aBuff[BUFLEN-1]=0;
char*pTok = strtok(aBuff, " ");
if (!pTok) // ignore the Msg Body
{
//cout << "ROWS Entry : " << aEntry << endl;
throw(runtime_error("Bad Body in entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
TotRows = strtol(pTok, NULL, 10);
else {
//cout << "HWM Entry : " << aEntry << endl;
throw(runtime_error("Bad Tot ROWS entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
InsRows = strtol(pTok, NULL, 10);
else {
//cout << "HWM Entry : " << aEntry << endl;
throw(runtime_error("Bad inserted ROWS in entry string"));
}
}
return aFound;
}
//------------------------------------------------------------------------------
//#DATA: columnNumber columnType columnName numOutOfRangeValues
bool WEBrmUpdater::prepareColumnOutOfRangeInfo(std::string Entry,
int& ColNum,
CalpontSystemCatalog::ColDataType& ColType,
std::string& ColName,
int& OorValues)
{
bool aFound=false;
boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr =
CalpontSystemCatalog::makeCalpontSystemCatalog();
//DATA: 3 1
if ((!Entry.empty()) && (Entry.at(0) == 'D'))
{
aFound = true;
const int BUFLEN=128;
char aBuff[BUFLEN];
strncpy(aBuff, Entry.c_str(),BUFLEN);
aBuff[BUFLEN-1]=0;
char* pTok = strtok(aBuff, " ");
if (!pTok) // ignore the Msg Body
{
//cout << "ROWS Entry : " << aEntry << endl;
throw(runtime_error("Bad OOR entry info"));
}
pTok = strtok(NULL, " ");
if (pTok)
{
ColNum = atoi(pTok);
}
else
{
//cout << "HWM Entry : " << aEntry << endl;
throw(runtime_error("Bad Oor Column Number entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
{
ColType = (CalpontSystemCatalog::ColDataType)atoi(pTok);
}
else
{
//cout << "HWM Entry : " << aEntry << endl;
throw(runtime_error("Bad Oor Column Type entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
{
uint64_t columnOid = strtol(pTok, NULL, 10);
CalpontSystemCatalog::TableColName colname = systemCatalogPtr->colName(columnOid);
ColName = colname.schema + "." + colname.table + "." + colname.column;
}
else
{
//cout << "HWM Entry : " << aEntry << endl;
throw(runtime_error("Bad Column Name entry string"));
}
pTok = strtok(NULL, " ");
if (pTok)
{
OorValues = atoi(pTok);
}
else
{
//cout << "HWM Entry : " << aEntry << endl;
throw(runtime_error("Bad OorValues entry string"));
}
}
return aFound;
}
//------------------------------------------------------------------------------
//#ERR: error message file
bool WEBrmUpdater::prepareErrorFileInfo(std::string Entry, std::string& ErrFileName)
{
bool aFound=false;
if ((!Entry.empty()) && (Entry.at(0) == 'E'))
{
aFound = true;
const int BUFLEN=128;
char aBuff[BUFLEN];
strncpy(aBuff, Entry.c_str(),BUFLEN);
aBuff[BUFLEN-1]=0;
char*pTok = strtok(aBuff, " ");
if (!pTok) // ignore the Msg Body
{
//cout << "ROWS Entry : " << aEntry << endl;
throw(runtime_error("Bad ERR File entry info"));
}
pTok = strtok(NULL, " ");
if (pTok)
{
ErrFileName = pTok;
//int aLen = ErrFileName.length();
//if(aLen>0) ErrFileName.insert(aLen-1, 1, 0);
}
else
{
//cout << "HWM Entry : " << aEntry << endl;
throw(runtime_error("Bad Error Filename entry string"));
}
}
return aFound;
}
//------------------------------------------------------------------------------
//#BAD: bad data file, with rejected rows
bool WEBrmUpdater::prepareBadDataFileInfo(std::string Entry, std::string& BadFileName)
{
bool aFound=false;
if ((!Entry.empty()) && (Entry.at(0) == 'B'))
{
aFound = true;
const int BUFLEN=128;
char aBuff[BUFLEN];
strncpy(aBuff, Entry.c_str(),BUFLEN);
aBuff[BUFLEN-1]=0;
char*pTok = strtok(aBuff, " ");
if (!pTok) // ignore the Msg Body
{
//cout << "ROWS Entry : " << aEntry << endl;
throw(runtime_error("Bad BAD Filename entry "));
}
pTok = strtok(NULL, " ");
if (pTok)
{
BadFileName = pTok;
//int aLen = BadFileName.length();
//if(aLen>0) BadFileName.insert(aLen-1, 1, 0);
}
else
{
//cout << "HWM Entry : " << aEntry << endl;
throw(runtime_error("Bad BAD Filename entry string"));
}
}
return aFound;
}
//------------------------------------------------------------------------------
} /* namespace WriteEngine */

90
writeengine/splitter/we_brmupdater.h Normal file
View File

@ -0,0 +1,90 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_brmupdater.h
*
* Created on: Dec 13, 2011
* Author: bpaul
*/
#ifndef WE_BRMUPDATER_H_
#define WE_BRMUPDATER_H_
namespace WriteEngine
{
class WESDHandler; // forward deceleration
class WEBrmUpdater
{
public:
WEBrmUpdater(WESDHandler& Ref): fRef(Ref), fpBrm(0) {}
virtual ~WEBrmUpdater() {}
public:
bool updateCasualPartitionAndHighWaterMarkInBRM();
int updateCPAndHWMInBRM();
int updateCasualPartitionInBRM();
int updateHighWaterMarkInBRM();
bool prepareCasualPartitionInfo();
bool prepareHighWaterMarkInfo();
bool createBrmConnection()
{
fpBrm = new BRM::DBRM();
return (fpBrm)?true:false;
}
void releaseBrmConnection()
{
delete fpBrm;
fpBrm = 0;
}
public:
static bool prepareRowsInsertedInfo(std::string Entry, int64_t& TotRows,
int64_t& InsRows);
static bool prepareColumnOutOfRangeInfo(std::string Entry, int& ColNum,
CalpontSystemCatalog::ColDataType& ColType,
std::string& ColName, int& OorValues);
static bool prepareErrorFileInfo(std::string Entry, std::string& ErrFileName);
static bool prepareBadDataFileInfo(std::string Entry, std::string& BadFileName);
private:
WESDHandler& fRef;
BRM::DBRM* fpBrm;
//BRM::CPInfoMergeList_t fCPInfo;
std::vector<BRM::CPInfoMerge> fCPInfo;
std::vector<BRM::BulkSetHWMArg> fHWMInfo;
std::vector<BRM::CPInfo> fCPInfoData;
};
} /* namespace WriteEngine */
#endif /* WE_BRMUPDATER_H_ */

1701
writeengine/splitter/we_cmdargs.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

181
writeengine/splitter/we_cmdargs.h Normal file
View File

@ -0,0 +1,181 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
#ifndef WE_CMDARGS_H_
#define WE_CMDARGS_H_
#include <set>
#include <boost/uuid/uuid.hpp>
#include "we_xmlgetter.h"
#include "we_type.h"
namespace WriteEngine
{
class WECmdArgs
{
public:
WECmdArgs(int argc, char** argv);
virtual ~WECmdArgs(){}
public:
void appTestFunction();
void parseCmdLineArgs(int argc, char** argv);
std::string getCpImportCmdLine();
void setSchemaAndTableFromJobFile(std::string& JobName);
void setEnclByAndEscCharFromJobFile(std::string& JobName);
void usage();
void usageMode3();
bool checkForCornerCases();
void checkForBulkLogDir(const std::string& BulkRoot);
void addJobFilesToVector(std::string& JobName);
void splitConfigFilePerTable(std::string& ConfigName, int tblCount);
void write2ConfigFiles(std::vector<std::ofstream*>& Files,
char*pBuff, int FileIdx);
void updateWithJobFile(int Idx);
public:
std::string getJobFileName();
std::string getBrmRptFileName();
std::string getTmpFileDir();
std::string getBulkRootDir();
unsigned int getBatchQuantity();
void checkJobIdCase();
std::string getFileNameFromPath(const std::string& Path) const;
std::string getModuleID();
std::string replaceCharInStr(const std::string& Str,char C,char R);
std::string getJobId() const { return fJobId; }
std::string getOrigJobId() const { return fOrigJobId; }
std::string getLocFile() const { return fLocFile; }
int getMode() const { return fMode; }
int getArgMode() const { return fArgMode; }
bool isHelpMode() { return fHelp; }
int getDebugLvl() { return fDebugLvl; }
char getEnclChar() { return fEnclosedChar; }
char getEscChar() { return fEscChar; }
char getDelimChar() { return fColDelim; }
ImportDataMode getImportDataMode() const { return fImportDataMode; }
bool getConsoleLog() { return fConsoleLog; }
bool isCpimportInvokeMode(){return (fBlockMode3)? false : fCpiInvoke;}
bool isQuiteMode() const { return fQuiteMode; }
void setJobId(std::string fJobId) { this->fJobId = fJobId; }
void setOrigJobId(std::string fOrigJobId){ this->fOrigJobId = fJobId; }
void setLocFile(std::string fLocFile) { this->fLocFile = fLocFile; }
void setMode(int fMode) { this->fMode = fMode; }
void setArgMode(int ArgMode) { this->fArgMode = ArgMode; }
void setPmFile(std::string fPmFile) { this->fPmFile = fPmFile; }
void setQuiteMode(bool fQuiteMode) { this->fQuiteMode = fQuiteMode; }
void setVerbose(int fVerbose) { this->fVerbose = fVerbose; }
void setJobFileName(std::string& JobFileName){fJobFile = JobFileName;}
void setBrmRptFileName(std::string& BrmFileName)
{ this->fBrmRptFile = BrmFileName; }
void setCpiInvoke(bool CpiInvoke=true) { this->fCpiInvoke = CpiInvoke; }
void setBlockMode3(bool Block) { this->fBlockMode3 = Block; }
void setTruncationAsError(bool bTruncationAsError)
{ fbTruncationAsError = bTruncationAsError; }
bool isJobLogOnly() const { return fJobLogOnly; }
void setJobUUID(const boost::uuids::uuid& jobUUID) { fUUID = jobUUID; }
bool getConsoleOutput( ) {return fConsoleOutput; }
private: // variables for SplitterApp
typedef std::vector<std::string> VecArgs;
VecArgs fVecArgs;
typedef std::vector<unsigned int> VecInts;
VecInts fPmVec;
VecArgs fVecJobFiles; //JobFiles splitter from master JobFile
int fMultiTableCount; //MultiTable count
VecArgs fColFldsFromJobFile;//List of columns from any job file, that
// represent fields in the import data
public:
bool getPmStatus(int Id);
bool str2PmList(std::string& PmList, VecInts& V);
int getPmVecSize(){ return fPmVec.size(); }
void add2PmVec(int PmId){ fPmVec.push_back(PmId); }
WECmdArgs::VecInts& getPmVec() { return fPmVec; }
std::string getPmFile() const { return fPmFile; }
int getVerbose() const { return fVerbose; }
std::string getTableName() const { return fTable; }
std::string getSchemaName() const { return fSchema; }
bool getTruncationAsError() const {return fbTruncationAsError; }
int getMultiTableCount() const { return fMultiTableCount; }
void setMultiTableCount(int Count) { fMultiTableCount = Count; }
std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342
void getColumnList( std::set<std::string>& columnList ) const;
private: // variables for SplitterApp
std::string fJobId; // JobID
std::string fOrigJobId; // Original JobID, in case we have to split it
bool fJobLogOnly; // Job number is only for log filename only
bool fHelp; // Help mode
int fMode; // splitter Mode
int fArgMode; // Argument mode, dep. on this fMode is decided.
bool fQuiteMode; // in quite mode or not
bool fConsoleLog; // Log everything to console - w.r.t cpimport
int fVerbose; // how many v's
std::string fPmFile; // FileName at PM
std::string fPmFilePath;// Path of input file in PM
std::string fLocFile; // Local file name
std::string fBrmRptFile;// BRM report file
std::string fJobPath; // Path to Job File
std::string fTmpFileDir;// Temp file directory.
std::string fBulkRoot; // Bulk Root path
std::string fJobFile; // Job File Name
unsigned int fBatchQty; // No. of batch Qty.
int fNoOfReadThrds; // No. of read buffers
//std::string fConfig; // config filename
int fDebugLvl; // Debug level
int fMaxErrors; // Max allowable errors
int fReadBufSize; // Read buffer size
int fIOReadBufSize; // I/O read buffer size
int fSetBufSize; // Buff size w/setvbuf
char fColDelim; // column delimiter
char fEnclosedChar; // enclosed by char
char fEscChar; // esc char
int fNoOfWriteThrds; // No. of write threads
bool fNullStrMode; // set null string mode - treat null as null
ImportDataMode fImportDataMode; // Importing text or binary data
std::string fPrgmName; // argv[0]
std::string fSchema; // Schema name - positional parmater
std::string fTable; // Table name - table name parameter
bool fCpiInvoke; // invoke cpimport in mode 3
bool fBlockMode3; // Do not allow Mode 3
bool fbTruncationAsError; // Treat string truncation as error
boost::uuids::uuid fUUID;
bool fConsoleOutput; // If false, no output to console.
};
//----------------------------------------------------------------------
}
#endif /* WE_CMDARGS_H_ */

544
writeengine/splitter/we_filereadthread.cpp Normal file
View File

@ -0,0 +1,544 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_filereadthread.cpp
*
* Created on: Oct 25, 2011
* Author: bpaul
*/
#include "we_messages.h"
#include "we_sdhandler.h"
#include <boost/thread/condition.hpp>
#include <boost/scoped_array.hpp>
#include <boost/thread.hpp>
using namespace boost;
#include "messagequeue.h"
#include "bytestream.h"
using namespace messageqcpp;
#include <fstream>
#include <istream>
#include <list>
#ifdef _MSC_VER
#include <io.h>
#endif
using namespace std;
#include "we_filereadthread.h"
namespace WriteEngine
{
void WEReadThreadRunner::operator ()()
{
try
{
fRef.feedData();
}
catch(std::exception& ex)
{
throw runtime_error(ex.what());
}
}
//------------------------------------------------------------------------------
//
//------------------------------------------------------------------------------
WEFileReadThread::WEFileReadThread(WESDHandler& aSdh):fSdh(aSdh),
fpThread(0),
fFileMutex(),
fContinue(true),
fInFileName(),
fInFile(std::cin.rdbuf()), //@BUG 4326
fTgtPmId(0),
fBatchQty(0),
fEnclEsc(false),
fEncl('\0'),
fEsc('\\'),
fDelim('|')
{
//TODO batch qty to get from config
fBatchQty = 10000;
}
//WEFileReadThread::WEFileReadThread(const WEFileReadThread& rhs):fSdh(rhs.fSdh)
//{
// // TODO copy constructor
//}
WEFileReadThread::~WEFileReadThread()
{
//if(fInFile.is_open()) fInFile.close(); //@BUG 4326
if(fIfFile.is_open()) fIfFile.close();
setTgtPmId(0);
if (fpThread)
{
delete fpThread;
}
fpThread=0;
//cout << "WEFileReadThread destructor called" << endl;
}
//------------------------------------------------------------------------------
void WEFileReadThread::reset()
{
//if(fInFile.is_open()) fInFile.close(); //@BUG 4326
if(fIfFile.is_open()) fIfFile.close();
setTgtPmId(0);
if (fpThread)
{
delete fpThread;
}
fpThread=0;
//cout << "WEFileReadThread destructor called" << endl;
this->setContinue(true);
}
//------------------------------------------------------------------------------
void WEFileReadThread::setup(std::string FileName)
{
if(fSdh.getDebugLvl())
cout << "WEFileReadThread::setup : *** Input files = " << FileName <<endl;
reset();
try
{
char aEncl = fSdh.getEnclChar();
char aEsc = fSdh.getEscChar();
char aDelim = fSdh.getDelimChar();
if(aEncl) fEncl = aEncl;
if(aEsc) fEsc = aEsc;
if(aDelim) fDelim = aDelim;
if(aEncl != 0) fEnclEsc = true;
//BUG 4342 - Need to support "list of infiles"
//chkForListOfFiles(FileName); - List prepared in sdhandler.
string aStrName = getNextInputDataFile();
if(fSdh.getDebugLvl()>2)
cout << "Next InFileName = " << aStrName << endl;
setInFileName(aStrName);
//setInFileName(FileName);
openInFile();
//set the target PM
fpThread = new boost::thread(WEReadThreadRunner(*this));
}
catch(std::exception& ex)
{
//cout << ex.what() << endl;
//throw runtime_error("Exception occured in WEFileReadThread\n");
throw runtime_error(ex.what());
}
if(fpThread)
{
// Need to send a all clear??
}
}
//------------------------------------------------------------------------------
bool WEFileReadThread::chkForListOfFiles(std::string& FileName)
{
//cout << "Inside chkForListOfFiles("<< FileName << ")" << endl;
std::string aFileName = FileName;
istringstream iss(aFileName);
size_t start = 0, end = 0;
const char* sep = " ,|";
end = aFileName.find_first_of(sep);
do
{
if(end != string::npos)
{
std::string aFile = aFileName.substr(start, end-start);
if(fSdh.getDebugLvl()>2)
cout << "File: " << aFileName.substr(start, end-start) << endl;
start = end + 1;
fInfileList.push_back(aFile);
}
else
{
std::string aFile = aFileName.substr(start, end-start);
if(fSdh.getDebugLvl()>1)
cout << "Next Input File " << aFileName.substr(start, end-start) << endl;
fInfileList.push_back(aFile);
break;
}
end = aFileName.find_first_of(sep, start);
}
while(start != end);
//cout << "Going out chkForListOfFiles("<< FileName << ")" << endl;
return false;
}
//------------------------------------------------------------------------------
std::string WEFileReadThread::getNextInputDataFile()
{
std::string aNextFile;
if(fInfileList.size()>0)
{
aNextFile = fInfileList.front();
fInfileList.pop_front();
}
//cout << "Next Input DataFile = " << aNextFile << endl;
return aNextFile;
}
//------------------------------------------------------------------------------
void WEFileReadThread::add2InputDataFileList(std::string& FileName)
{
fInfileList.push_front(FileName);
}
//------------------------------------------------------------------------------
void WEFileReadThread::shutdown()
{
this->setContinue(false);
mutex::scoped_lock aLock(fFileMutex); //wait till readDataFile() finish
//if(fInFile.is_open()) fInFile.close(); //@BUG 4326
if(fIfFile.is_open()) fIfFile.close();
}
//------------------------------------------------------------------------------
void WEFileReadThread::feedData()
{
unsigned int aRowCnt = 0;
const unsigned int c10mSec = 10000;
while(isContinue())
{
unsigned int TgtPmId = getTgtPmId();
if(TgtPmId == 0)
{
setTgtPmId(fSdh.getNextPm2Feed());
TgtPmId = getTgtPmId();
}
if((TgtPmId>0)&&(fInFile.good()))
{
try
{
messageqcpp::SBS aSbs(new messageqcpp::ByteStream);
if (fSdh.getImportDataMode() == IMPORT_DATA_TEXT)
aRowCnt = readDataFile(aSbs);
else
aRowCnt = readBinaryDataFile(aSbs,
fSdh.getTableRecLen() );
//cout << "Length " << aSbs->length() <<endl; - for debug
fSdh.updateRowTx(aRowCnt, TgtPmId);
mutex::scoped_lock aLock(fSdh.fSendMutex);
fSdh.send2Pm(aSbs, TgtPmId);
aLock.unlock();
setTgtPmId(0); //reset PmId. Send the data to next least data
}
catch(std::exception& ex)
{
throw runtime_error(ex.what());
}
}
else
{
usleep(c10mSec);
setTgtPmId(0);
}
// Finish reading file and thread can go away once data sent
if(fInFile.eof())
{
if(fInfileList.size() != 0)
{
if(fIfFile.is_open()) fIfFile.close();
string aStrName = getNextInputDataFile();
setInFileName(aStrName);
openInFile();
}
else
{
//if there is no more files to be read send EOD
//cout << "Sending EOD message to PM" << endl;
fSdh.sendEODMsg();
setContinue(false);
}
}
}
}
//------------------------------------------------------------------------------
// Read input data as ASCII text
//------------------------------------------------------------------------------
unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs)
{
mutex::scoped_lock aLock(fFileMutex);
// For now we are going to send KEEPALIVES
//*Sbs << (ByteStream::byte)(WE_CLT_SRV_KEEPALIVE);
if((fInFile.good()) && (!fInFile.eof()))
{
//cout << "Inside WEFileReadThread::readDataFile" << endl;
//char aBuff[1024*1024]; // TODO May have to change it later
//char*pStart = aBuff;
unsigned int aIdx=0;
unsigned int aLen=0;
*Sbs << (ByteStream::byte)(WE_CLT_SRV_DATA);
while((!fInFile.eof())&&(aIdx< getBatchQty()))
{
if(fEnclEsc)
{
//pStart = aBuff;
aLen = getNextRow(fInFile, fBuff, sizeof(fBuff)-1);
}
else
{
fInFile.getline(fBuff, sizeof(fBuff)-1);
aLen=fInFile.gcount();
}
////aLen chars incl \n, Therefore aLen-1; '<<' oper won't go past it
//cout << "Data Length " << aLen <<endl;
if((aLen < (sizeof(fBuff)-2)) && (aLen>0))
{
fBuff[aLen-1] = '\n';
fBuff[aLen]=0;
if(fSdh.getDebugLvl()>3) cout << "Data Read " << fBuff <<endl;
(*Sbs).append(reinterpret_cast<ByteStream::byte*>(fBuff), aLen);
aIdx++;
if(fSdh.getDebugLvl()>2) cout << "File data line = " << aIdx <<endl;
}
else if(aLen>=sizeof(fBuff)-2) //Didn't hit delim; BIG ROW
{
cout <<"Bad Row data " << endl;
cout << fBuff << endl;
throw runtime_error("Data Row too BIG to handle!!");
}
//for debug
//if(fSdh.getDebugLvl()>3) cout << aIdx << endl;
}// while
return aIdx;
}// if
return 0;
}
//------------------------------------------------------------------------------
// Read input data as binary data
//------------------------------------------------------------------------------
unsigned int WEFileReadThread::readBinaryDataFile(messageqcpp::SBS& Sbs,
unsigned int recLen)
{
mutex::scoped_lock aLock(fFileMutex);
if((fInFile.good()) && (!fInFile.eof()))
{
unsigned int aIdx=0;
unsigned int aLen=0;
*Sbs << (ByteStream::byte)(WE_CLT_SRV_DATA);
while( (!fInFile.eof()) && (aIdx<getBatchQty()) )
{
fInFile.read(fBuff, recLen);
aLen = fInFile.gcount();
if (aLen > 0)
{
(*Sbs).append(reinterpret_cast<ByteStream::byte*>(fBuff), aLen);
aIdx++;
if(fSdh.getDebugLvl()>2)
cout << "Binary input data line = " << aIdx << endl;
if (aLen != recLen)
{
cout << "Binary input data does not end on record boundary;"
" Last record is " << aLen << " bytes long." <<
" Expected record length is: " << recLen << endl;
}
}
} // while
return aIdx;
} // if
return 0;
}
//------------------------------------------------------------------------------
void WEFileReadThread::openInFile()
{
try
{
if(fSdh.getDebugLvl()) cout << "Input FileName: " << fInFileName <<endl;
if(fInFileName == "/dev/stdin")
{
char aDefCon[16], aGreenCol[16];
snprintf(aDefCon, sizeof(aDefCon), "\033[0m");
snprintf(aGreenCol, sizeof(aGreenCol), "\033[0;32m");
if(fSdh.getDebugLvl()) // BUG 4195
cout << aGreenCol
<< "trying to read from STDIN... "
<< aDefCon << endl;
}
cout.flush();
//@BUG 4326
if(fInFileName != "/dev/stdin")
{
if (!fIfFile.is_open())
{
if (fSdh.getImportDataMode() == IMPORT_DATA_TEXT)
fIfFile.open(fInFileName.c_str());
else // @bug 5193: binary import
fIfFile.open(fInFileName.c_str(),
std::ios_base::in | std::ios_base::binary);
}
if (!fIfFile.good())
throw runtime_error("Could not open Input file "+fInFileName);
fInFile.rdbuf(fIfFile.rdbuf()); //@BUG 4326
}
#ifdef _MSC_VER
else // @bug 5193: binary import
{
if (fSdh.getImportDataMode() != IMPORT_DATA_TEXT)
{
if (_setmode(_fileno(stdin), _O_BINARY) == -1)
{
throw runtime_error("Could not change stdin to binary");
}
}
}
#endif
//@BUG 4326 -below three lines commented out
// if (!fInFile.is_open()) fInFile.open(fInFileName.c_str());
// if (!fInFile.good())
// throw runtime_error("Could not open Input file "+fInFileName);
}
catch(std::exception& ex)
{
cout << "Error in Opening input data file "<< fInFileName << endl;
throw runtime_error(ex.what()); //BUG 4201 FIX
}
}
//------------------------------------------------------------------------------
int WEFileReadThread::getNextRow(istream& ifs, char* pBuf, int MaxLen)
{
//const char ENCL ='\"'; //TODO for time being
//const char ESC = '\0'; //TODO for time being
const char ENCL = fEncl;
const char ESC = fEsc;
bool aTrailEsc = false;
char *pEnd = pBuf;
char aCh = ifs.get();
while (ifs.good())
{
if(aCh == ENCL)
{
// we got the first enclosedBy char.
*pEnd++ = aCh;
aCh = ifs.get();
//cout << "aCh 1 = " << aCh << endl;
while(aCh != ENCL) //Loop thru till we hit another one
{
if(aCh == ESC) //check spl cond ESC inside ENCL of '\n' here
{
*pEnd++ = aCh;
aCh = ifs.get();
*pEnd++ = aCh;
aCh = ifs.get(); // get the next char for while loop
//cout << "aCh 2 = " << aCh << endl;
}// case ESC
else
{
*pEnd++ = aCh;
aCh = ifs.get();
//cout << "aCh 3 = " << aCh << endl;
}
}
*pEnd++ = aCh; // ENCL char got
aTrailEsc = true; //@BUG 4641
}//case ENCL
else if(aCh == ESC)
{
*pEnd++ = aCh;
aCh = ifs.get();
*pEnd++ = aCh;
//cout << "aCh 4 = " << aCh << endl;
}// case ESC
else
{
*pEnd++ = aCh;
//cout << "aCh 5 = " << aCh << endl;
}
//cout << "pBuf1 " << pBuf << endl;
if((aCh == '\n')||((pEnd-pBuf)==MaxLen)) break; // we got a full row
aCh = ifs.get();
// BUG 4641 To avoid seg fault when a wrong/no ESC char provided.
while(aTrailEsc)
{
// BUG 4903 EOF, to handle files ending w/ EOF and w/o '\n'
if((aCh == '\n')||(aCh == EOF) ||(aCh == fDelim))
{
aTrailEsc = false;
break;
}
else
{
*pEnd++ = aCh;
aCh = ifs.get();
}
}
}// end of while loop
return pEnd - pBuf;
}
//------------------------------------------------------------------------------
} /* namespace WriteEngine */

127
writeengine/splitter/we_filereadthread.h Normal file
View File

@ -0,0 +1,127 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_filereadthread.h
*
* Created on: Oct 25, 2011
* Author: bpaul
*/
#ifndef WE_FILEREADTHREAD_H_
#define WE_FILEREADTHREAD_H_
namespace WriteEngine
{
class WESDHandler;
class WEFileReadThread;
class WEReadThreadRunner
{
public:
WEReadThreadRunner(WEFileReadThread& Owner):fRef(Owner)
{ // ctor
}
~WEReadThreadRunner()
{
}
void operator()(); // Thread function
private:
WEFileReadThread& fRef;
};
//------------------------------------------------------------------------------
class WEFileReadThread
{
public:
WEFileReadThread(WESDHandler& aSdh);
virtual ~WEFileReadThread();
void reset();
void setup(std::string FileName);
void shutdown();
void feedData();
unsigned int readDataFile(messageqcpp::SBS& Sbs);
unsigned int readBinaryDataFile(messageqcpp::SBS& Sbs, unsigned int recLen);
void openInFile();
int getNextRow(std::istream& ifs, char*pBuf, int MaxLen);
boost::thread* getFpThread() const { return fpThread; }
bool isContinue() const { return fContinue; }
void setContinue(bool fContinue) { this->fContinue = fContinue; }
std::string getInFileName() const { return fInFileName; }
unsigned int getTgtPmId() const { return fTgtPmId; }
unsigned int getBatchQty() const { return fBatchQty; }
void setFpThread(boost::thread* fpThread) { this->fpThread = fpThread; }
void setInFileName(std::string fInFileName)
{
if((0==fInFileName.compare("STDIN"))||(0==fInFileName.compare("stdin")))
this->fInFileName = "/dev/stdin";
else
this->fInFileName = fInFileName;
}
//@BUG 4326
const std::istream& getInFile() const { return fInFile; }
void setBatchQty(unsigned int BatchQty) { fBatchQty = BatchQty; }
bool chkForListOfFiles(std::string& FileName);
std::string getNextInputDataFile();
void add2InputDataFileList(std::string& FileName);
private:
enum { MAXBUFFSIZE=1024*1024 };
// don't allow anyone else to set
void setTgtPmId(unsigned int fTgtPmId) { this->fTgtPmId = fTgtPmId; }
WESDHandler & fSdh;
boost::thread *fpThread;
boost::mutex fFileMutex;
bool fContinue;
std::string fInFileName;
std::istream fInFile; //@BUG 4326
std::ifstream fIfFile; //@BUG 4326
typedef std::list<std::string> strList;
strList fInfileList;
unsigned int fTgtPmId;
unsigned int fBatchQty;
bool fEnclEsc; // Encl/Esc char is set
char fEncl; // Encl char
char fEsc; // Esc char
char fDelim; // Column Delimit char
char fBuff[MAXBUFFSIZE]; // main data buffer
};
} /* namespace WriteEngine */
#endif /* WE_FILEREADTHREAD_H_ */

63
writeengine/splitter/we_respreadthread.cpp Normal file
View File

@ -0,0 +1,63 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_respreadthread.cpp
*
* Created on: Oct 18, 2011
* Author: bpaul
*/
#include <boost/thread/condition.hpp>
#include <boost/scoped_array.hpp>
#include <boost/thread.hpp>
using namespace boost;
#include "we_sdhandler.h"
#include "we_respreadthread.h"
namespace WriteEngine
{
WERespReadThread::WERespReadThread(WESDHandler& aSdh):
fSdh(aSdh)
{
// ctor
}
WERespReadThread::WERespReadThread(const WERespReadThread& rhs):
fSdh(rhs.fSdh)
{
// copy ctor
}
WERespReadThread::~WERespReadThread()
{
// dtor
}
void WERespReadThread::operator()()
{
//call datahandler checkForAllPmMsgs()
fSdh.checkForRespMsgs();
}
} /* namespace WriteEngine */

51
writeengine/splitter/we_respreadthread.h Normal file
View File

@ -0,0 +1,51 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_respreadthread.h
*
* Created on: Oct 18, 2011
* Author: bpaul
*/
#ifndef WE_RESPREADTHREAD_H_
#define WE_RESPREADTHREAD_H_
namespace WriteEngine
{
class WERespReadThread
{
public:
WERespReadThread(WESDHandler& aSdh);
WERespReadThread(const WERespReadThread& rhs);
virtual ~WERespReadThread();
void operator()();
private:
WESDHandler& fSdh;
};
} /* namespace WriteEngine */
#endif /* WE_RESPREADTHREAD_H_ */

2589
writeengine/splitter/we_sdhandler.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

322
writeengine/splitter/we_sdhandler.h Normal file
View File

@ -0,0 +1,322 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_spltrdatahandler.h
*
* Created on: Oct 17, 2011
* Author: bpaul
*/
#ifndef WE_SPLITTERDATAHANDLER_H_
#define WE_SPLITTERDATAHANDLER_H_
#include "liboamcpp.h"
#include "resourcemanager.h"
#include "threadsafequeue.h"
#include "dbrm.h"
#include "batchloader.h"
#include "we_log.h"
#include "we_type.h"
#include "we_filereadthread.h"
#include "we_splclient.h"
namespace WriteEngine
{
class WESplitterApp; //forward declaration
class WESplClient;
class WEFileReadThread;
//------------------------------------------------------------------------------
// a stl list to keep Next PM to send data
//------------------------------------------------------------------------------
class WEPmList
{
public:
WEPmList():fPmList(),fListMutex(){}
virtual ~WEPmList(){ fPmList.clear(); }
void addPm2List(int PmId);
void addPriorityPm2List(int PmId);
int getNextPm();
void clearPmList();
bool check4Pm(int PmId);
private:
typedef std::list<int> WePmList; // List to add in front
WePmList fPmList;
boost::mutex fListMutex; //mutex controls add/remove
};
//------------------------------------------------------------------------------
class WESDHandler
{
public:
WESDHandler(WESplitterApp& Ref);
WESDHandler(const WESDHandler& rhs);
virtual ~WESDHandler();
void setup();
void shutdown();
void reset();
void send2Pm(messageqcpp::SBS& Sbs, unsigned int PmId=0);
void send2Pm(messageqcpp::ByteStream & Bs, unsigned int PmId=0);
void sendEODMsg();
void checkForRespMsgs();
void add2RespQueue(const messageqcpp::SBS & Sbs);
void exportJobFile(std::string &JobId, std::string &JobFileName);
int leastDataSendPm();
bool check4AllBrmReports();
bool updateCPAndHWMInBRM();
void cancelOutstandingCpimports();
void doRollback();
void doCleanup(bool deleteHdfsTempDbFiles);
void getErrorLog(int PmId, const std::string& ErrFileName);
void getBadLog(int PmId, const std::string& BadFileName);
int check4RollbackRslts();
bool check4AllRollbackStatus();
int check4CleanupRslts();
bool check4AllCleanupStatus();
bool check4AllCpiStarts();
bool releaseTableLocks();
void check4CpiInvokeMode();
bool check4PmArguments();
bool check4InputFile(std::string InFileName);
bool check4CriticalErrMsgs(std::string& Entry);
void onStartCpiResponse(int PmId);
void onDataRqstResponse(int PmId);
void onAckResponse(int PmId);
void onNakResponse(int PmId);
void onEodResponse(int Pmid);
void onPmErrorResponse(int PmId);
void onKeepAliveMessage(int PmId);
void onCpimportPass(int PmId);
void onCpimportFail(int PmId, bool SigHandle=false);
void onImpFileError(int PmId);
void onBrmReport(int PmId, messageqcpp::SBS& Sbs);
void onErrorFile(int PmId, messageqcpp::SBS& Sbs);
void onBadFile(int PmId, messageqcpp::SBS& Sbs);
void onRollbackResult(int PmId, messageqcpp::SBS& Sbs);
void onCleanupResult(int PmId, messageqcpp::SBS& Sbs);
void onDBRootCount(int PmId, messageqcpp::SBS& Sbs);
void onHandlingSignal();
void onHandlingSigHup();
void onDisconnectFailure();
int getNextPm2Feed();
int getNextDbrPm2Send();
int getTableOID(std::string Schema, std::string Table);
std::string getTime2Str() const;
bool checkAllCpiPassStatus();
bool checkAllCpiFailStatus();
bool checkForRollbackAndCleanup();
bool checkForCpiFailStatus();
void checkForConnections();
void sendHeartbeats();
std::string getTableName() const;
std::string getSchemaName() const;
char getEnclChar();
char getEscChar();
char getDelimChar();
bool getConsoleLog();
ImportDataMode getImportDataMode() const;
void sysLog(const logging::Message::Args& msgArgs,
logging::LOG_TYPE logType, logging::Message::MessageID msgId);
boost::thread* getFpRespThread() const { return fpRespThread; }
unsigned int getQId() const { return fQId; }
void setFpRespThread(boost::thread *pRespThread)
{ fpRespThread = pRespThread; }
void setQId(unsigned int QId) { fQId = QId; }
bool isContinue() const { return fContinue; }
void setContinue(bool Continue) { fContinue = Continue; }
int getPmCount() const { return fPmCount; }
void setPmCount(int PmCount) { fPmCount = PmCount; }
int getNextPm2Send() { return fDataFeedList.getNextPm(); }
bool check4Ack(unsigned int PmId) { return fDataFeedList.check4Pm(PmId); }
int getTableOID() { return fTableOId; }
void setDebugLvl(int DebugLvl) { fDebugLvl = DebugLvl; }
int getDebugLvl() { return fDebugLvl; }
unsigned int getTableRecLen() const { return fFixedBinaryRecLen; }
void updateRowTx(unsigned int RowCnt, int CIdx)
{ fWeSplClients[CIdx]->updateRowTx(RowCnt); }
void resetRowTx(){ for (int aCnt = 1; aCnt <= fPmCount; aCnt++)
{if (fWeSplClients[aCnt] != 0) {fWeSplClients[aCnt]->resetRowTx(); } } }
void setRowsUploadInfo(int PmId, int64_t RowsRead, int64_t RowsInserted)
{ fWeSplClients[PmId]->setRowsUploadInfo(RowsRead, RowsInserted); }
void add2ColOutOfRangeInfo(int PmId, int ColNum,
CalpontSystemCatalog::ColDataType ColType,
std::string& ColName, int NoOfOors)
{ fWeSplClients[PmId]->add2ColOutOfRangeInfo(ColNum, ColType, ColName, NoOfOors); }
void setErrorFileName(int PmId, const std::string& ErrFileName)
{ fWeSplClients[PmId]->setErrInfoFile(ErrFileName); }
void setBadFileName(int PmId, const std::string& BadFileName)
{ fWeSplClients[PmId]->setBadDataFile(BadFileName); }
void setDisconnectFailure(bool Flag);
bool getDisconnectFailure(){ return fDisconnectFailure; }
public: // for multi-table support
WESplitterApp& fRef;
Log fLog; // logger
private:
unsigned int fQId;
joblist::ResourceManager fRm;
oam::Oam fOam;
oam::ModuleTypeConfig fModuleTypeConfig;
int fDebugLvl;
int fPmCount;
int64_t fTableLock;
int32_t fTableOId;
uint32_t fFixedBinaryRecLen;
boost::mutex fRespMutex;
boost::condition fRespCond;
boost::mutex fSendMutex;
// It could be a queue too. Stores all the responses from PMs
typedef std::list<messageqcpp::SBS> WESRespList;
WESRespList fRespList;
// Other member variables
boost::thread *fpRespThread;
WEPmList fDataFeedList;
WEFileReadThread fFileReadThread;
bool fDisconnectFailure; //Failure due to disconnect from PM
bool fForcedFailure;
bool fAllCpiStarted;
bool fFirstDataSent;
unsigned int fFirstPmToSend;
bool fSelectOtherPm; // Don't send first data to First PM
bool fContinue;
// set of PM specific vector entries
typedef std::vector<WESplClient*> WESplClients;
WESplClients fWeSplClients;
enum { MAX_PMS = 512, MAX_QSIZE=10, MAX_WES_QSIZE=100};
typedef std::vector<std::string> StrVec;
StrVec fBrmRptVec;
BRM::DBRM fDbrm;
batchloader::BatchLoader* fpBatchLoader;
unsigned int calcTableRecLen(const std::string& schema,
const std::string table);
class WEImportRslt
{
public:
WEImportRslt():fRowsPro(0),fRowsIns(0),fStartTime(),fEndTime(),fTotTime(0){}
~WEImportRslt(){}
public:
void reset(){fRowsPro=0; fRowsIns=0; fTotTime=0; fColOorVec.clear();}
void updateRowsProcessed(int64_t Rows){ fRowsPro+=Rows; }
void updateRowsInserted(int64_t Rows){ fRowsIns+=Rows; }
void updateColOutOfRangeInfo(int aColNum, CalpontSystemCatalog::ColDataType aColType,
std::string aColName, int aNoOfOors)
{
WEColOorVec::iterator aIt = fColOorVec.begin();
while(aIt != fColOorVec.end())
{
if ((*aIt).fColNum == aColNum)
{
(*aIt).fNoOfOORs += aNoOfOors;
break;
}
aIt++;
}
if (aIt == fColOorVec.end())
{
// First time for aColNum to have out of range count
WEColOORInfo aColOorInfo;
aColOorInfo.fColNum = aColNum;
aColOorInfo.fColType = aColType;
aColOorInfo.fColName = aColName;
aColOorInfo.fNoOfOORs = aNoOfOors;
fColOorVec.push_back(aColOorInfo);
}
#if 0
try
{
fColOorVec.at(aColNum).fNoOfOORs += aNoOfOors;
}
catch (out_of_range& e)
{
// First time for aColNum to have out of range count
WEColOORInfo aColOorInfo;
aColOorInfo.fColNum = aColNum;
aColOorInfo.fColName = aColName;
aColOorInfo.fNoOfOORs = aNoOfOors;
fColOorVec[aColNum] = aColOorInfo;
}
#endif
}
void startTimer(){ gettimeofday( &fStartTime, 0 ); }
void stopTimer(){ gettimeofday( &fEndTime, 0 ); }
float getTotalRunTime()
{
//fTotTime = (fEndTime>0)?(fEndTime-fStartTime):0;
fTotTime = (fEndTime.tv_sec + (fEndTime.tv_usec / 1000000.0)) -
(fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0));
return fTotTime;
}
public:
int64_t fRowsPro; //Rows processed
int64_t fRowsIns; //Rows inserted
timeval fStartTime; //StartTime
timeval fEndTime; //EndTime
float fTotTime; //TotalTime
// A vector containing a list of rows and counts of Out of Range values
WEColOorVec fColOorVec;
};
WEImportRslt fImportRslt;
friend class WESplClient;
friend class WEBrmUpdater;
friend class WESplitterApp;
friend class WEFileReadThread;
friend class WETableLockGrabber;
};
//------------------------------------------------------------------------------
} /* namespace WriteEngine */
#endif /* WE_SPLITTERDATAHANDLER_H_ */

462
writeengine/splitter/we_splclient.cpp Normal file
View File

@ -0,0 +1,462 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_splclient.cpp
*
* Created on: Oct 20, 2011
* Author: bpaul
*/
#include <cstdio>
#include <iostream>
#include <string>
using namespace std;
#include "errorids.h"
#include "exceptclasses.h"
#include "messagelog.h"
#include "messageobj.h"
#include "loggingid.h"
using namespace logging;
#include <boost/thread/mutex.hpp>
using namespace boost;
#include "messagequeue.h"
#include "bytestream.h"
using namespace messageqcpp;
#include "liboamcpp.h"
using namespace oam;
#include "snmpmanager.h"
using namespace snmpmanager;
#include "we_sdhandler.h"
#include "we_splclient.h"
namespace WriteEngine
{
//------------------------------------------------------------------------------
//BP 10/24/2011 14:25
//------------------------------------------------------------------------------
void WESplClientRunner::operator()()
{
fOwner.sendAndRecv();
}
//------------------------------------------------------------------------------
//BP 10/24/2011 14:25
//------------------------------------------------------------------------------
WESplClient::WESplClient(WESDHandler& Sdh, int PmId):
fContinue(true),
fConnected(false),
fPmId(PmId),
fDbrCnt(0),
fDbrVar(0),
fDataRqstCnt(0),
fRdSecTo(0),
fRowTx(0),
fBytesTx(0),
fBytesRcv(0),
fLastInTime(0),
fStartTime(time(0)),
fSend(true),
fCpiStarted(false),
fCpiPassed(false),
fCpiFailed(false),
fBrmRptRcvd(false),
fRollbackRslt(0),
fCleanupRslt(0),
fServer(),
fClnt(),
fpThread(0),
fOwner(Sdh)
{
// TODO ctor
}
//------------------------------------------------------------------------------
WESplClient::~WESplClient()
{
delete fpThread;
fpThread=0;
}
//------------------------------------------------------------------------------
void WESplClient::setup()
{
// do the setup stuffs here
if(fOwner.getDebugLvl())
cout << "setting connection to moduleid " << getPmId() << endl;
char buff[32];
snprintf(buff, sizeof(buff), "pm%u_WriteEngineServer", getPmId());
fServer = buff;
fClnt.reset(new MessageQueueClient(fServer));
if(fOwner.getDebugLvl())
cout << "WEServer : " << fServer << " " << fClnt->addr2String() <<endl;
try
{
if (fClnt->connect())
{
onConnect();
startClientThread();
}
else
{
throw runtime_error("Connection refused");
}
} catch (std::exception& ex)
{
cerr << "Could not connect to " << fServer << ": " << ex.what() << endl;
throw runtime_error("Problem in connecting to PM");
} catch (...)
{
cerr << "Could not connect to " << fServer << endl;
throw runtime_error("Problem in connecting to PM");
}
}
//------------------------------------------------------------------------------
void WESplClient::startClientThread()
{
this->fpThread = new boost::thread(WESplClientRunner(*this));
}
//------------------------------------------------------------------------------
void WESplClient::sendAndRecv()
{
while(fContinue)
{
try
{
// Send messages if out queue has something
send();
// Recv messages if there is something in socket or timeout
recv();
}
catch (runtime_error&)
{
//setCpiFailed(true); - done in onDisconnect() BUG
setConnected(false);
if(fOwner.getDebugLvl())
cout <<"Disconnect from PM - " << getPmId() << endl;
onDisconnect();
}
}
if(this->fCpiFailed)
{
// NOTE : commented out to avoid sending rollback twice.
//fOwner.onCpimportFail(this->getPmId());
char aDefCon[16], aRedCol[16];
snprintf(aDefCon, sizeof(aDefCon), "\033[0m");
snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m");
if(fOwner.getDebugLvl())
cout << aRedCol << "Bulk load FAILED on PM "
<< getPmId()<< aDefCon << endl;
}
else if(this->fCpiPassed)
{
//if(fOwner.getDebugLvl())
//BUG 4195
char aDefCon[16], aGreenCol[16];
snprintf(aDefCon, sizeof(aDefCon), "\033[0m");
snprintf(aGreenCol, sizeof(aGreenCol), "\033[0;32m");
if(fOwner.getDebugLvl())
cout << aGreenCol << "Bulk load Finished Successfully on PM "
<< getPmId()<< aDefCon << endl;
}
else if(!this->fCpiStarted)
{
if(fOwner.getDebugLvl())
cout << "Cpimport Failed to Start!!!"<< this->getPmId() << endl;
}
}
//------------------------------------------------------------------------------
void WESplClient::send()
{
if ((!fSendQueue.empty())&&(getDataRqstCount()>0))
{
if(fOwner.getDebugLvl()>2)
cout << "DataRqstCnt [" << getPmId() << "] = "
<< getDataRqstCount() << endl;
mutex::scoped_lock aLock(fSentQMutex);
messageqcpp::SBS aSbs = fSendQueue.front();
fSendQueue.pop();
aLock.unlock();
int aLen = (*aSbs).length();
if (aLen > 0)
{
mutex::scoped_lock aLock(fWriteMutex);
setBytesTx(getBytesTx() + aLen);
try
{
if(isConnected())
fClnt->write(aSbs);
}
catch(...)
{
}
aLock.unlock();
}
decDataRqstCount();
//decDbRootVar();
}
//setSendFlag(fOwner.check4Ack(fPmId));
}
//------------------------------------------------------------------------------
void WESplClient::recv()
{
messageqcpp::SBS aSbs;
struct timespec rm_ts;
rm_ts.tv_sec = fRdSecTo; //0 when data sending otherwise 1- second
rm_ts.tv_nsec = 20000000; // 20 milliSec
bool isTimeOut = false;
int aLen = 0;
try
{
if(isConnected())
aSbs = fClnt->read(&rm_ts, &isTimeOut);
}
catch (std::exception& ex)
{
setConnected(false);
cout << ex.what() <<endl;
cout << "fClnt read error on " << getPmId() << endl;
throw runtime_error("fClnt read error");
}
// - aSbs->length()>0 add to the sdh.fWesMsgQueue
try
{
if(aSbs)
aLen = aSbs->length();
}
catch(...)
{
aLen = 0;
}
if(aLen > 0)
{
setLastInTime(time(0)); //added back for BUG 4535 / BUG 4195
setBytesRcv( getBytesRcv()+ aLen);
fOwner.add2RespQueue(aSbs);
}
else if ((aLen <= 0) && (!isTimeOut)) //disconnect
{
cout <<"Disconnect from PM - " << getPmId() << " IP " << endl;
onDisconnect();
}
}
//------------------------------------------------------------------------------
void WESplClient::write(const messageqcpp::ByteStream& Msg)
{
setBytesTx(getBytesTx() + Msg.length());
try
{
if(Msg.length()>0)
fClnt->write(Msg);
}
catch(...)
{
//ignore it
}
}
//------------------------------------------------------------------------------
void WESplClient::read(messageqcpp::SBS& Sbs)
{
// read from the WEServerMsgQueue
// if Key is needed give that constant here
}
//------------------------------------------------------------------------------
//TODO - We may need to make it much more efficient by incorporating file read
void WESplClient::add2SendQueue(const messageqcpp::SBS& Sbs)
{
this->fSendQueue.push(Sbs);
}
void WESplClient::clearSendQueue()
{
mutex::scoped_lock aLock(fSentQMutex);
while(!fSendQueue.empty())
fSendQueue.pop();
aLock.unlock();
}
int WESplClient::getSendQSize()
{
int aQSize=0;
mutex::scoped_lock aLock(fSentQMutex);
aQSize = fSendQueue.size();
aLock.unlock();
return aQSize;
}
//------------------------------------------------------------------------------
void WESplClient::printStats()
{
if(fOwner.getDebugLvl())
{
cout <<"\tPMid \t"<<getPmId()<<endl;
cout <<"\tTx Rows \t"<<getRowTx()<<endl;
//if(fOwner.getDebugLvl())
cout <<"\tTx Bytes \t"<<getBytesTx()<<endl;
//if(fOwner.getDebugLvl())
cout <<"\tRcv Bytes \t"<<getBytesRcv()<<endl;
cout <<"\tInserted/Read Rows "<<fRowsUploadInfo.fRowsInserted<<"/"
<<fRowsUploadInfo.fRowsRead<< endl;
if(fColOorVec.size()>0)
cout <<"\tCol Id\tColName\t\t\tout-of-range count" <<endl;
WEColOorVec::iterator aIt = fColOorVec.begin();
while(aIt != fColOorVec.end())
{
cout <<"\t"<<(*aIt).fColNum <<"\t"<<(*aIt).fColName <<"\t\t" << (*aIt).fNoOfOORs <<endl;
aIt++;
}
if(!fBadDataFile.empty())cout<<"\tBad Data Filename "<<fBadDataFile<<endl;
if(!fErrInfoFile.empty())cout<<"\tError Filename "<<fErrInfoFile<<endl;
cout <<"\t("<<getLastInTime()-getStartTime()<<"sec)"<<endl;
cout <<"\t"<<endl;
}
}
//------------------------------------------------------------------------------
void WESplClient::onConnect()
{
//TODO
// update all the flags on Connect.
// alert data can be send now
// do not allow to connect back again.
// when reconnect happens, reset the variables
setRollbackRslt(0);
setCleanupRslt(0);
setCpiPassed(false);
setCpiFailed(false);
setContinue(true);
setConnected(true);
ByteStream bsWrite;
bsWrite << (ByteStream::byte) WE_CLT_SRV_KEEPALIVE;
try
{
this->write(bsWrite); // send the keep init keep alive
}
catch(...)
{
}
// need to send Alarm
fIpAddress = fClnt->addr2String();
}
//------------------------------------------------------------------------------
void WESplClient::onDisconnect()
{
//TODO
// - set fContinue to false - set the thread free
setContinue(false);
setConnected(false);
setRollbackRslt(-1);
setCleanupRslt(-1);
if((!fCpiPassed)&&(!fCpiFailed)) //a hard disconnection
{
fOwner.onCpimportFail(fPmId);
fOwner.setDisconnectFailure(true);
}
// update all the flags of disconnect.
// alert on roll back
// do not allow to connect back again.
try
{
// send alarm
SNMPManager alarmMgr;
//std::string alarmItem = sin_addr2String(fClnt->serv_addr().sin_addr);
std::string alarmItem = fClnt->addr2String();
alarmItem.append(" WriteEngineServer");
alarmMgr.sendAlarmReport(alarmItem.c_str(), oam::CONN_FAILURE, SET);
}
catch(...)
{
// just ignore it for time being.
}
}
//------------------------------------------------------------------------------
void WESplClient::setRowsUploadInfo(int64_t RowsRead, int64_t RowsInserted)
{
fRowsUploadInfo.fRowsRead = RowsRead;
fRowsUploadInfo.fRowsInserted = RowsInserted;
}
//------------------------------------------------------------------------------
void WESplClient::add2ColOutOfRangeInfo(int ColNum,
CalpontSystemCatalog::ColDataType ColType,
std::string& ColName, int NoOfOors)
{
WEColOORInfo aColOorInfo;
aColOorInfo.fColNum = ColNum;
aColOorInfo.fColType = ColType;
aColOorInfo.fColName = ColName;
aColOorInfo.fNoOfOORs = NoOfOors;
fColOorVec.push_back(aColOorInfo);
}
//------------------------------------------------------------------------------
void WESplClient::setBadDataFile(const std::string& BadDataFile)
{
fBadDataFile = BadDataFile;
}
//------------------------------------------------------------------------------
void WESplClient::setErrInfoFile(const std::string& ErrInfoFile)
{
fErrInfoFile = ErrInfoFile;
}
//------------------------------------------------------------------------------
} /* namespace WriteEngine */

293
writeengine/splitter/we_splclient.h Normal file
View File

@ -0,0 +1,293 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_splclient.h
*
* Created on: Oct 20, 2011
* Author: bpaul
*/
#ifndef WE_SPLCLIENT_H_
#define WE_SPLCLIENT_H_
#include "threadsafequeue.h"
#include "resourcemanager.h"
#include "we_messages.h"
#include "calpontsystemcatalog.h"
using namespace execplan;
namespace WriteEngine
{
class WESplClient; //forward decleration
// Structure for holding the Out of Range data from the BRMReport
// This class is also used by we_sdhandler to hold the agregate info.
class WEColOORInfo // Column Out-Of-Range Info
{
public:
WEColOORInfo():fColNum(0),fColType(CalpontSystemCatalog::INT), fNoOfOORs(0){}
~WEColOORInfo(){}
public:
int fColNum;
CalpontSystemCatalog::ColDataType fColType;
std::string fColName;
int fNoOfOORs;
};
typedef std::vector<WEColOORInfo> WEColOorVec;
//------------------------------------------------------------------------------
class WESdHandlerException: public std::exception
{
public:
std::string fWhat;
WESdHandlerException(std::string& What) throw() { fWhat = What; }
virtual ~WESdHandlerException() throw() {}
virtual const char* what() const throw()
{
return fWhat.c_str();
}
};
//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
class WESplClientRunner
{
public:
WESplClientRunner(WESplClient& Sc): fOwner(Sc){ /* ctor */ }
virtual ~WESplClientRunner(){/* dtor */}
void operator()();
public:
WESplClient& fOwner;
};
//------------------------------------------------------------------------------
class WESplClient
{
public:
WESplClient(WESDHandler& Sdh, int PmId);
virtual ~WESplClient();
void setup();
void startClientThread();
void sendAndRecv();
void send();
void recv();
void write(const messageqcpp::ByteStream& Msg);
void read(messageqcpp::SBS& Sbs);
void add2SendQueue(const messageqcpp::SBS& Sbs);
void clearSendQueue();
int getSendQSize();
void printStats();
void onConnect();
void onDisconnect();
unsigned int getRowTx() const { return fRowTx; }
uint32_t getBytesRcv() const { return fBytesRcv; }
uint32_t getBytesTx()
{
boost::mutex::scoped_lock aLock(fTxMutex);
return fBytesTx;
}
boost::thread* getFpThread() const { return fpThread; }
time_t getLastInTime()
{
boost::mutex::scoped_lock aLock(fLastInMutex);
return(fLastInTime>0)?fLastInTime:fStartTime; //BUG 4309
}
time_t getStartTime() const { return fStartTime; }
time_t getElapsedTime() { return (getLastInTime() - getStartTime()); }
bool isCpiStarted() const { return fCpiStarted; }
bool isCpiPassed() const { return fCpiPassed; }
bool isCpiFailed() const { return fCpiFailed; }
bool isBrmRptRcvd() const { return fBrmRptRcvd; }
int getRollbackRslt() const { return fRollbackRslt; }
int getCleanupRslt() const { return fCleanupRslt; }
bool getSendFlag() const { return fSend; }
unsigned int getPmId() const { return fPmId; }
unsigned int getDbRootCnt() const { return fDbrCnt; }
unsigned int getDbRootVar()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
return fDbrVar;
}
int getDataRqstCount()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
return fDataRqstCnt;
}
long getRdSecTo() const { return fRdSecTo; }
bool isConnected() const { return fConnected; }
bool isContinue() const { return fContinue; }
const std::string& getServer() const { return fServer; }
const std::string& getIpAddress() const { return fIpAddress; }
void setBytesRcv(uint32_t BytesRcv) { fBytesRcv = BytesRcv; }
void setBytesTx(uint32_t BytesTx)
{
boost::mutex::scoped_lock aLock(fTxMutex);
BytesTx = BytesTx;
aLock.unlock();
}
void updateBytesTx(uint32_t fBytes)
{
boost::mutex::scoped_lock aLock(fTxMutex);
fBytesTx += fBytes;
aLock.unlock();
}
void setConnected(bool Connected) { fConnected = Connected; }
void setContinue(bool Continue) { fContinue = Continue; }
void setFpThread(boost::thread* pThread) { fpThread = pThread; }
void setLastInTime(time_t LastInTime) { fLastInTime = LastInTime; }
void setStartTime(time_t StartTime)
{
boost::mutex::scoped_lock aLock(fLastInMutex);
fStartTime = StartTime;
aLock.lock();
}
void setSendFlag(bool Send) { fSend = Send; }
void setCpiStarted(bool Start) { fCpiStarted = Start; }
void setCpiPassed(bool Pass)
{
setLastInTime(time(0));
fCpiPassed = Pass;
}
void setCpiFailed(bool Fail)
{
setLastInTime(time(0));
fCpiFailed = Fail;
fRowsUploadInfo.fRowsRead = 0;
fRowsUploadInfo.fRowsInserted = 0;
}
void setBrmRptRcvd(bool Rcvd) { fBrmRptRcvd = Rcvd; }
void setRollbackRslt(int Rslt) { fRollbackRslt = Rslt; }
void setCleanupRslt(int Rslt) { fCleanupRslt = Rslt; }
void setPmId(unsigned int PmId) { fPmId = PmId; }
void setDbRootCnt(unsigned int DbrCnt) { fDbrCnt = DbrCnt; }
void resetDbRootVar()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
fDbrVar = fDbrCnt;
aLock.unlock();
}
void decDbRootVar()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
if(fDbrVar>0) --fDbrVar;
aLock.unlock();
}
void setRdSecTo(long RdSecTo)
{
fRdSecTo = RdSecTo;
}
void setDataRqstCount(int DataRqstCnt)
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
fDataRqstCnt = DataRqstCnt;
aLock.unlock();
}
void decDataRqstCount()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
if(fDataRqstCnt>0) --fDataRqstCnt;
aLock.unlock();
}
void incDataRqstCount()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
++fDataRqstCnt;
aLock.unlock();
}
void setServer(const std::string& Server) { fServer = Server; }
void setIpAddress(const std::string& IpAddr) { fIpAddress = IpAddr; }
void updateRowTx(unsigned int aCnt) { fRowTx += aCnt; }
void resetRowTx() { fRowTx = 0; }
private:
bool fContinue;
bool fConnected;
unsigned int fPmId;
unsigned int fDbrCnt;
unsigned int fDbrVar; // Var to keep track next PM to send.
int fDataRqstCnt; // Data request count
long fRdSecTo; // read timeout sec
unsigned int fRowTx; // No. Of Rows Transmitted
uint32_t fBytesTx;
uint32_t fBytesRcv;
time_t fLastInTime;
time_t fStartTime;
bool fSend;
bool fCpiStarted;
bool fCpiPassed;
bool fCpiFailed;
bool fBrmRptRcvd;
int fRollbackRslt;
int fCleanupRslt;
boost::mutex fTxMutex; //mutex for TxBytes
boost::mutex fDataRqstMutex;
boost::mutex fWriteMutex;
boost::mutex fSentQMutex;
boost::mutex fLastInMutex;
typedef std::queue<messageqcpp::SBS> WESendQueue;
WESendQueue fSendQueue;
std::string fServer;
std::string fIpAddress;
boost::shared_ptr<messageqcpp::MessageQueueClient> fClnt;
boost::thread *fpThread;
WESDHandler& fOwner;
class WERowsUploadInfo
{
public:
WERowsUploadInfo():fRowsRead(0),fRowsInserted(0){}
~WERowsUploadInfo(){}
public:
int64_t fRowsRead;
int64_t fRowsInserted;
};
WERowsUploadInfo fRowsUploadInfo;
WEColOorVec fColOorVec;
std::string fBadDataFile;
std::string fErrInfoFile;
void setRowsUploadInfo(int64_t RowsRead, int64_t RowsInserted);
void add2ColOutOfRangeInfo(int ColNum,
CalpontSystemCatalog::ColDataType ColType,
std::string& ColName, int NoOfOors);
void setBadDataFile(const std::string& BadDataFile);
void setErrInfoFile(const std::string& ErrInfoFile);
friend class WESDHandler;
};
//------------------------------------------------------------------------------
} /* namespace WriteEngine */
#endif /* WE_SPLCLIENT_H_ */

615
writeengine/splitter/we_splitterapp.cpp Normal file
View File

@ -0,0 +1,615 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_splitterapp.cpp
*
* Created on: Oct 7, 2011
* Author: bpaul
*/
#include <unistd.h>
#include <cstdlib>
#include <csignal>
#include <string>
using namespace std;
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
#include "batchloader.h"
using namespace batchloader;
#include "we_messages.h"
#include "we_splitterapp.h"
#include "installdir.h"
static int SPLTR_EXIT_STATUS=0;
namespace WriteEngine
{
bool WESplitterApp::fContinue = true;
bool WESplitterApp::fSignaled = false;
bool WESplitterApp::fSigHup = false;
SimpleSysLog* WESplitterApp::fpSysLog = 0;
//WESplitterApp::WESplitterApp(WECmdArgs& CmdArgs) :
// fCmdArgs(CmdArgs), fDh(*this), fpSysLog(0)
WESplitterApp::WESplitterApp(WECmdArgs& CmdArgs) :
fCmdArgs(CmdArgs), fDh(*this)
{
fpSysLog = SimpleSysLog::instance();
fpSysLog->setLoggingID(logging::LoggingID(SUBSYSTEM_ID_WE_SPLIT));
setupSignalHandlers();
std::string err;
fDh.setDebugLvl(fCmdArgs.getDebugLvl());
fDh.check4CpiInvokeMode();
fCmdArgs.checkForCornerCases();
if (fCmdArgs.isCpimportInvokeMode())
{
try
{
invokeCpimport();
}
catch(std::exception& ex)
{
cout << "Invoking Mode 3" << endl;
cout << ex.what() << endl;
SPLTR_EXIT_STATUS=1;
exit(SPLTR_EXIT_STATUS);
}
exit(SPLTR_EXIT_STATUS);
}
else
{
if(fCmdArgs.isHelpMode()) fCmdArgs.usage();
if(fCmdArgs.getMultiTableCount() <= 1)
{
try
{
fDh.setup();
}
catch (std::exception& ex)
{
//err = string("Error in constructing WESplitterApp") + ex.what();
err = ex.what(); //cleaning up for BUG 4298
logging::Message::Args errMsgArgs;
errMsgArgs.add(err);
fpSysLog->logMsg(errMsgArgs,logging::LOG_TYPE_ERROR,logging::M0000);
if (!fCmdArgs.getConsoleOutput())
{
ofstream dmlFile;
ostringstream oss;
oss << "/tmp/" <<fDh.getTableOID() << ".txt";
dmlFile.open(oss.str().c_str());
if (dmlFile.is_open())
{
dmlFile << err;
dmlFile << endl;
dmlFile.close();
}
SPLTR_EXIT_STATUS=2;
}
else
SPLTR_EXIT_STATUS=1;
//cout << err << endl;
fDh.fLog.logMsg( err, MSGLVL_ERROR );
fContinue = false;
//throw runtime_error(err); BUG 4298
}
}
}
}
WESplitterApp::~WESplitterApp()
{
//fDh.shutdown();
usleep(1000); //1 millisec just checking
std::string aStr = "Calling WESplitterApp Destructor\n";
if(fDh.getDebugLvl()) cout << aStr << endl;
}
//------------------------------------------------------------------------------
// Initialize signal handling
//------------------------------------------------------------------------------
void WESplitterApp::setupSignalHandlers()
{
#ifdef _MSC_VER
//FIXME
#else
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = WESplitterApp::onSigInterrupt;
sigaction(SIGINT, &sa, 0);
sa.sa_handler = WESplitterApp::onSigTerminate;
sigaction(SIGTERM, &sa, 0);
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, 0);
sa.sa_handler = WESplitterApp::onSigHup;
sigaction(SIGPIPE, &sa, 0);
sa.sa_handler = WESplitterApp::onSigInterrupt;
sigaction(SIGUSR1, &sa, 0);
/*
signal(SIGPIPE, SIG_IGN);
signal(SIGINT, WESplitterApp::onSigInterrupt);
signal(SIGTERM, WESplitterApp::onSigTerminate);
signal(SIGHUP, WESplitterApp::onSigHup); */
#endif
}
//------------------------------------------------------------------------------
// handles on signal Terminate
//------------------------------------------------------------------------------
void WESplitterApp::onSigTerminate(int aInt)
{
cout << "onSigTerminate received signal " << aInt << endl;
if(15 == aInt)
{
fSignaled = true;
}
fContinue = false; //force to call destructor
if(aInt == 1) SPLTR_EXIT_STATUS = 1;
}
//------------------------------------------------------------------------------
// handles on signal Interrupt
//------------------------------------------------------------------------------
void WESplitterApp::onSigInterrupt(int aInt)
{
//cout << "onSigInterrupt received signal " << aInt << endl;
if((2 == aInt) || (10 == aInt))
{
fSignaled = true;
//cout << "ctrl-c received" << endl;
}
fContinue = false; //force to call destructor
if(aInt == 1) SPLTR_EXIT_STATUS = 1;
}
//------------------------------------------------------------------------------
// handles on signal HUP send by OAM
//------------------------------------------------------------------------------
void WESplitterApp::onSigHup(int aInt)
{
fSigHup = true;
fContinue = false;
std::string aStr = "Interrupt received...Program Exiting...";
cout << aStr << endl;
if(aInt == 1) SPLTR_EXIT_STATUS = 1;
logging::Message::Args errMsgArgs;
errMsgArgs.add(aStr);
fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000);
exit(SPLTR_EXIT_STATUS); //BUG 4534 - exit w/o writing to log
}
//------------------------------------------------------------------------------
// Process messages on Main thread
//------------------------------------------------------------------------------
void WESplitterApp::processMessages()
{
boost::uuids::uuid u=boost::uuids::random_generator()();
fCmdArgs.setJobUUID(u);
messageqcpp::ByteStream aBs;
unsigned int aRollCount = 0;
if(fDh.getDebugLvl())
cout << "Inside WESplitterApp::processMessages() "
<< "Mode = " << fCmdArgs.getMode() << endl;
//TODO - handle all the messages here
if (fCmdArgs.getMode() == 2)
{
try
{
aBs << (ByteStream::byte) WE_CLT_SRV_MODE;
aBs << (ByteStream::quadbyte) fCmdArgs.getMode();
fDh.send2Pm(aBs);
std::string aJobId = fCmdArgs.getJobId();
if((aJobId.length()>0)&&(!fCmdArgs.isJobLogOnly())) // Export jobFile NOW
{
std::string aJobFileName = fCmdArgs.getJobFileName();
fDh.exportJobFile(aJobId, aJobFileName );
}
aBs.restart();
std::string aCpImpCmd = fCmdArgs.getCpImportCmdLine();
fDh.fLog.logMsg( aCpImpCmd, MSGLVL_INFO2 );
if(fDh.getDebugLvl())
cout << "CPImport cmd line - " << aCpImpCmd << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_CMDLINEARGS;
aBs << aCpImpCmd;
fDh.send2Pm(aBs);
aBs.restart();
std::string aBrmRpt = fCmdArgs.getBrmRptFileName();
if(fDh.getDebugLvl())
cout << "BrmReport FileName - " << aBrmRpt << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_BRMRPT;
aBs << aBrmRpt;
fDh.send2Pm(aBs);
}
catch (std::exception& exp)
{
//cout << exp.what() << endl;
SPLTR_EXIT_STATUS=1;
//exit(SPLTR_EXIT_STATUS);
throw runtime_error(exp.what());
}
}
else if (fCmdArgs.getMode() == 1)
{
try
{
// In this mode we ignore almost all cmd lines args which
// are usually send to cpimport
aBs << (ByteStream::byte) WE_CLT_SRV_MODE;
aBs << (ByteStream::quadbyte) fCmdArgs.getMode();
fDh.send2Pm(aBs);
std::string aJobId = fCmdArgs.getJobId();
if(fDh.getDebugLvl()) cout<<"ProcessMsgs aJobId "<<aJobId<<endl;
if((aJobId.length()>0)&&(!fCmdArgs.isJobLogOnly())) // Export jobFile NOW
{
std::string aJobFileName = fCmdArgs.getJobFileName();
if(fDh.getDebugLvl()) cout<<"ProcessMsgs Calling exportJobFile "<<endl;
fDh.exportJobFile(aJobId, aJobFileName );
if(fDh.getDebugLvl()) cout<<"ProcessMsgs Calling exportJobFile "<<endl;
}
aBs.restart();
std::string aCpImpCmd = fCmdArgs.getCpImportCmdLine();
fDh.fLog.logMsg( aCpImpCmd, MSGLVL_INFO2 );
if(fDh.getDebugLvl())
cout << "CPImport cmd line - " << aCpImpCmd << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_CMDLINEARGS;
aBs << aCpImpCmd;
fDh.send2Pm(aBs);
aBs.restart();
std::string aBrmRpt = fCmdArgs.getBrmRptFileName();
if(fDh.getDebugLvl())
cout << "BrmReport FileName - " << aBrmRpt << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_BRMRPT;
aBs << aBrmRpt;
fDh.send2Pm(aBs);
} catch (std::exception& exp)
{
//cout << exp.what() << endl;
SPLTR_EXIT_STATUS=1;
//exit(SPLTR_EXIT_STATUS);
throw runtime_error(exp.what());
}
}
else if (fCmdArgs.getMode() == 0)
{
try
{
// In this mode we ignore almost all cmd lines args which
// are usually send to cpimport
aBs << (ByteStream::byte) WE_CLT_SRV_MODE;
aBs << (ByteStream::quadbyte) fCmdArgs.getMode();
fDh.send2Pm(aBs);
aBs.restart();
std::string aCpImpFileName = fCmdArgs.getPmFile();
if(aCpImpFileName.length()==0)
{
fCmdArgs.setPmFile(fCmdArgs.getLocFile());
aCpImpFileName = fCmdArgs.getPmFile();
if((aCpImpFileName.length()==0)||(aCpImpFileName == "STDIN"))
{
throw (runtime_error("PM Remote filename not specified"));
}
}
if(fDh.getDebugLvl())
cout << "CPImport FileName - " << aCpImpFileName << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_IMPFILENAME;
aBs << aCpImpFileName;
fDh.send2Pm(aBs);
}
catch (std::exception& exp)
{
//cout << exp.what() << endl;
SPLTR_EXIT_STATUS=1;
//exit(SPLTR_EXIT_STATUS);
throw runtime_error(exp.what());
}
}
int aNoSec=2;
bool bRollback = false;
bool bForce = false;
int iShutdown;
// TODO - this is for just time being....
// we need to process message of main thread here..
// here we need to cont check the status of different things
while (fContinue)
{
++aRollCount;
usleep(1000000);
// Check to see if someone has ordered a shutdown with rollback or force.
iShutdown = fDh.fDbrm.getSystemShutdownPending(bRollback, bForce);
if (iShutdown >= 0)
{
if (bRollback)
{
if (iShutdown > 0) // Means a shutdown, stop or restart
{
cout << "System stop has been ordered. Rollback" << endl;
}
else
{
cout << "Database writes have been suspended. Rollback" << endl;
}
fSignaled = true;
fContinue = false;
}
else
if (bForce)
{
//BUG 5012 - added to avoid rollback
fContinue = false;
ostringstream oss;
oss << "Table "<<fCmdArgs.getSchemaName()<<".";
oss << fCmdArgs.getTableName() << ": (OID-";
oss << fDh.getTableOID() << ") was NOT successfully loaded.";
cout << oss.str() << endl;
logging::Message::Args errMsgArgs;
//BUG 4152
errMsgArgs.add(fCmdArgs.getSchemaName());
errMsgArgs.add(fCmdArgs.getTableName());
errMsgArgs.add(fDh.getTableOID());
std::string aStr = "Immediate system stop has been ordered, rollback deferred";
cout << aStr << endl;
SPLTR_EXIT_STATUS = 1;
errMsgArgs.add(aStr);
fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0096);
exit(SPLTR_EXIT_STATUS);
//BUG 5012 - commented out to avoid rollback
//cout << "Immediate system stop has been ordered. No rollback" << endl;
//fSignaled = true;
//fContinue = false;
}
}
// Send out a heartbeat to the WriteEnginServers every 10 seconds
if ((0 == (aRollCount % aNoSec)) && (!fSignaled)) // Debugging - every 10 seconds
{
if(aNoSec<10) aNoSec++; //progressively go up to 10Sec interval
aBs.restart();
aBs << (ByteStream::byte) WE_CLT_SRV_KEEPALIVE;
mutex::scoped_lock aLock(fDh.fSendMutex);
fDh.send2Pm(aBs);
aLock.unlock();
//fDh.sendHeartbeats();
//fDh.checkForConnections(); - decided to recv SIGHUP from OAM instead of this
}
}
fDh.shutdown();
} // processMessages
void WESplitterApp::invokeCpimport()
{
boost::uuids::uuid u=boost::uuids::random_generator()();
fCmdArgs.setJobUUID(u);
//BUG 4361 - check cpimport.bin is available or not
std::string aCpiBinFile = getCalpontHome() + "/cpimport.bin"; //BUG 4361
if (access(aCpiBinFile.c_str(), X_OK) != 0)
throw runtime_error("Error: Missing File " + aCpiBinFile);
fCmdArgs.setMode(3);
std::string aCmdLineStr = fCmdArgs.getCpImportCmdLine();
updateCmdLineWithPath(aCmdLineStr);
if(fDh.getDebugLvl())
cout << "CPI CmdLineArgs : " << aCmdLineStr << endl;
std::vector<char*> Cmds;
std::istringstream ss(aCmdLineStr);
std::string arg;
std::vector<std::string> v2;
while(ss >> arg)
{
//we need something that works on Windows as well as linux
char* ptr = 0;
v2.push_back(arg);
//we're going to exec() below, so don't worry about freeing
ptr = strdup(v2.back().c_str());
Cmds.push_back(ptr);
}
Cmds.push_back(0); //null terminate
int aRet = execv(Cmds[0], &Cmds[0]); //NOTE - works with full Path
if(fDh.getDebugLvl())
cout << "Return status of cpimport is " << aRet <<endl;
}
//-----------------------------------------------------------------------------
/**
*
* @brief Include the absolute path to prgm name, which is
* @brief the first element in the vector
* @param V vector which contains each element of argv
*
**/
std::string WESplitterApp::getCalpontHome()
{
string calpontDir = config::Config::makeConfig()->getConfig(
"SystemConfig", "CalpontHome");
if(0 == calpontDir.length())
{
calpontDir = startup::StartUp::installDir() + "/bin";
}
else
{
calpontDir += "/bin";
}
return calpontDir;
}
//-----------------------------------------------------------------------------
/**
*
* @brief Include the absolute path to prgm name, which is
* @brief the first element in the vector
* @param V vector which contains each element of argv
*
**/
std::string WESplitterApp::getPrgmPath(std::string& PrgmName)
{
std::string cpimportPath = getCalpontHome();
cpimportPath += "/";
cpimportPath += PrgmName;
return cpimportPath;
}
//-----------------------------------------------------------------------------
/**
*
* @brief Include the absolute path to prgm name, which is
* @brief the first element in the vector
* @param V vector which contains each element of argv
*
**/
void WESplitterApp::updateCmdLineWithPath(string& CmdLine)
{
std::istringstream iSs(CmdLine);
std::ostringstream oSs;
std::string aArg;
int aCount=0;
while(iSs >> aArg)
{
if(0 == aCount)
{
string aPrgmPath = getPrgmPath(aArg);
oSs << aPrgmPath;
}
else
{
oSs << " ";
oSs << aArg;
}
++aCount;
}
CmdLine = oSs.str();
}
//-----------------------------------------------------------------------------
void WESplitterApp::updateWithJobFile(int aIdx)
{
fCmdArgs.updateWithJobFile(aIdx);
}
//-----------------------------------------------------------------------------
} /* namespace WriteEngine */
//------------------------------------------------------------------------------
// main function
//------------------------------------------------------------------------------
int main(int argc, char** argv)
{
std::string err;
setuid(0); //@BUG 4343 set effective userid to root.
std::cin.sync_with_stdio(false);
try
{
WriteEngine::WECmdArgs aWeCmdArgs(argc, argv);
WriteEngine::WESplitterApp aWESplitterApp(aWeCmdArgs);
int aTblCnt = aWESplitterApp.fCmdArgs.getMultiTableCount();
if(aTblCnt>1)
{
for(int idx=0; idx<aTblCnt; idx++)
{
aWESplitterApp.fDh.reset();
aWESplitterApp.fContinue = true;
aWESplitterApp.updateWithJobFile(idx);
try
{
aWESplitterApp.fDh.setup();
}
catch (std::exception& ex)
{
//err = string("Error in constructing WESplitterApp") + ex.what();
err = ex.what(); //cleaning up for BUG 4298
logging::Message::Args errMsgArgs;
errMsgArgs.add(err);
aWESplitterApp.fpSysLog->logMsg(errMsgArgs,logging::LOG_TYPE_ERROR,logging::M0000);
SPLTR_EXIT_STATUS=1;
aWESplitterApp.fDh.fLog.logMsg( err, MSGLVL_ERROR );
aWESplitterApp.fContinue = false;
//throw runtime_error(err); BUG 4298
}
aWESplitterApp.processMessages();
if(SPLTR_EXIT_STATUS == 1) break;
}
}
else
{
aWESplitterApp.processMessages();
}
} catch (std::exception& exp)
{
cerr << exp.what() << endl;
SPLTR_EXIT_STATUS = 1;
exit(SPLTR_EXIT_STATUS); // exit with an error
}
return SPLTR_EXIT_STATUS;
}
// vim:ts=4 sw=4:

95
writeengine/splitter/we_splitterapp.h Normal file
View File

@ -0,0 +1,95 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_splitterapp.h
*
* Created on: Oct 7, 2011
* Author: bpaul
*/
#ifndef WE_SPLITTERAPP_H_
#define WE_SPLITTERAPP_H_
#include <boost/thread/condition.hpp>
#include <boost/scoped_array.hpp>
#include <boost/thread.hpp>
using namespace boost;
#include "bytestream.h"
using namespace messageqcpp;
#include "we_cmdargs.h"
#include "we_sdhandler.h"
#include "we_simplesyslog.h"
using namespace WriteEngine;
namespace WriteEngine
{
class WESplitterApp
{
public:
WESplitterApp(WECmdArgs& CmdArgs);
virtual ~WESplitterApp();
void processMessages();
int getMode(){ return fCmdArgs.getMode(); }
bool getPmStatus(int Id){ return fCmdArgs.getPmStatus(Id); }
std::string getLocFile() { return fCmdArgs.getLocFile(); }
std::string getPmFile() { return fCmdArgs.getPmFile(); }
void updateWithJobFile(int aIdx);
// setup the signal handlers for the main app
void setupSignalHandlers();
static void onSigTerminate(int aInt);
static void onSigInterrupt(int aInt);
static void onSigHup(int aInt);
void invokeCpimport();
std::string getCalpontHome();
std::string getPrgmPath(std::string& PrgmName);
void updateCmdLineWithPath(string& CmdLine);
private:
public: // for multi table support
WECmdArgs& fCmdArgs;
WESDHandler fDh;
static bool fContinue;
public:
static bool fSignaled;
static bool fSigHup;
public:
static SimpleSysLog* fpSysLog;
public:
friend class WESDHandler;
};
} /* namespace WriteEngine */
#endif /* WE_SPLITTERAPP_H_ */

124
writeengine/splitter/we_tablelockgrabber.cpp Normal file
View File

@ -0,0 +1,124 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_tablelockgrabber.cpp
*
* Created on: Dec 19, 2011
* Author: bpaul
*/
#include "brm.h"
#include "brmtypes.h"
using namespace BRM;
#include <iostream>
using namespace std;
#include "we_sdhandler.h"
#include "we_tablelockgrabber.h"
namespace WriteEngine
{
uint64_t WETableLockGrabber::grabTableLock(std::vector<unsigned int> &PmList,
uint32_t tableOID)
{
uint64_t aLockId;
std::string aProcName = "cpimport";
uint32_t aProcId = getpid();
int32_t aSessId = -1;
int32_t aTxnId = -1;
try
{
aLockId = fRef.fDbrm.getTableLock(PmList, tableOID, &aProcName,
&aProcId, &aSessId, &aTxnId, BRM::LOADING);
}
catch (std::exception &e)
{
cout << "ERROR: Failed to get Table Lock " << e.what() << endl;
throw runtime_error(e.what());
}
if (aLockId == 0)
{
ostringstream oss;
oss << " Table currently locked by process-" <<
aProcName << "; pid-" << aProcId <<
"; session-" << aSessId <<
"; txn-" << aTxnId;
throw runtime_error(oss.str());
}
//cout << "lock ID = " << aLockId << endl;
//if (aLockId == 0)
// cout << " existing owner name = " << aProcName << " pid = " <<
// aProcId << " session = " << aSessId << endl;
return aLockId;
}
bool WETableLockGrabber::releaseTableLock(uint64_t LockId)
{
bool aRet;
//cout << "releasing lock " << LockId << endl;
try
{
aRet = fRef.fDbrm.releaseTableLock(LockId);
}
catch (std::exception &e)
{
cout << "caught an exception: " << e.what() << endl;
throw runtime_error(e.what());
}
return aRet;
}
bool WETableLockGrabber::changeTableLockState(uint64_t LockId)
{
bool aRet;
//cout << "changing state of lock " << LockId << endl;
try
{
aRet = fRef.fDbrm.changeState(LockId, BRM::CLEANUP);
}
catch (std::exception &e)
{
cout << "caught an exception: " << e.what() << endl;
throw runtime_error(e.what());
}
return aRet;
}
} /* namespace WriteEngine */

63
writeengine/splitter/we_tablelockgrabber.h Normal file
View File

@ -0,0 +1,63 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_tableLockgrabber.h
*
* Created on: Dec 19, 2011
* Author: bpaul
*/
#ifndef WE_TABLELOCKGRABBER_H_
#define WE_TABLELOCKGRABBER_H_
namespace WriteEngine
{
class WESDHandler; // forward deceleration
class WETableLockGrabber
{
public:
WETableLockGrabber(WESDHandler& Ref): fRef(Ref) { }
virtual ~WETableLockGrabber() { }
public:
uint64_t grabTableLock(std::vector<unsigned int> &PmList,
uint32_t tableOID);
bool releaseTableLock(uint64_t LockId);
bool changeTableLockState(uint64_t LockId);
private:
WESDHandler& fRef;
};
} /* namespace WriteEngine */
#endif /* WE_TABLELOCKGRABBER_H_ */

343
writeengine/splitter/we_xmlgetter.cpp Normal file
View File

@ -0,0 +1,343 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_xmlgetter.cpp
*
* Created on: Feb 7, 2012
* Author: bpaul
*/
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <stdexcept>
#include <iostream>
#include <string>
#include <vector>
using namespace std;
#include "we_xmlgetter.h"
using namespace std;
namespace WriteEngine
{
//------------------------------------------------------------------------------
// WEXmlgetter constructor
//------------------------------------------------------------------------------
WEXmlgetter::WEXmlgetter(std::string& ConfigName):
fConfigName(ConfigName),
fDoc( NULL ),
fpRoot( NULL )
{
// xmlNodePtr curPtr;
fDoc = xmlParseFile( ConfigName.c_str() );
if(fDoc == NULL )
throw runtime_error("WEXmlgetter::getConfig(): no XML document!");
fpRoot = xmlDocGetRootElement( fDoc );
if( fpRoot == NULL )
{
xmlFreeDoc( fDoc );
fDoc = NULL;
throw runtime_error("WEXmlgetter::getConfig(): no XML Root Tag!");
}
}
//------------------------------------------------------------------------------
// WEXmlgetter destructor
//------------------------------------------------------------------------------
WEXmlgetter::~WEXmlgetter()
{
xmlFreeDoc( fDoc );
fDoc = NULL;
}
//------------------------------------------------------------------------------
// Get/return the property or attribute value (strVal) for the specified xml tag
// (pNode) and property/attribute (pTag)
//------------------------------------------------------------------------------
bool WEXmlgetter::getNodeAttribute(const xmlNode* pNode,
const char* pTag, std::string& strVal) const
{
xmlChar* pTmp = NULL;
bool bFound = false;
pTmp = xmlGetProp( const_cast<xmlNode*>(pNode), (xmlChar*) pTag );
if( pTmp ) {
bFound = true;
strVal = (char*)pTmp;
xmlFree( pTmp );
}
else {
strVal.clear();
} // end if
return bFound;
}
//------------------------------------------------------------------------------
// Get/return the node content (strVal) for the specified xml tag (pNode)
//------------------------------------------------------------------------------
bool WEXmlgetter::getNodeContent( const xmlNode* pNode,
std::string& strVal) const
{
xmlChar* pTmp = NULL;
bool bFound = false;
if( pNode->children != NULL ) {
pTmp = xmlNodeGetContent( pNode->children );
if( pTmp ) {
bFound = true;
strVal = (char*)pTmp;
xmlFree( pTmp );
}
else {
strVal.clear();
}
}
else {
strVal.clear();
}
return bFound;
}
//------------------------------------------------------------------------------
// Get/returns node content for the "first" child node under each section/name.
// Example:
// <section>
// <name>
// <subname1>
// </subname1>
// </name>
// <name>
// <subname1>
// </subname1>
// </name>
// </section>
//
// Looks like xml2 is currently returning the text node as the first child
// node under a node. So in the example above, this function is currently
// always returning the text node content inside each <name> rather than
// any <subname1> node that might be within each <name> tag.
//------------------------------------------------------------------------------
void WEXmlgetter::getConfig(const string& section,
const string& name, vector<string>& values) const
{
string res;
if (section.length() == 0)
throw invalid_argument("Config::getConfig: section must have a length");
xmlNode* pPtr = fpRoot->xmlChildrenNode;
while (pPtr != NULL)
{
//cout << "pPtr->name: " <<
// (const xmlChar*)pPtr->name << std::endl;
if ((!xmlStrcmp(pPtr->name, (const xmlChar *)section.c_str())))
{
xmlNodePtr pPtr2 = pPtr->xmlChildrenNode;
while (pPtr2 != NULL)
{
//cout << " pPtr2->name: " <<
// (const xmlChar*)pPtr2->name << std::endl;
if ((!xmlStrcmp(pPtr2->name, (const xmlChar*)name.c_str())))
{
xmlNodePtr pPtr3 = pPtr2->xmlChildrenNode;
values.push_back((const char*)pPtr3->content);
//cout << " pPtr3->name: " <<
// (const xmlChar*)pPtr3->name <<
// "; content: " << (const xmlChar*)pPtr3->content <<
// "; len: " << strlen((char*)pPtr3->content) << std::endl;
}
pPtr2 = pPtr2->next;
}
}
pPtr = pPtr->next;
}
}
//------------------------------------------------------------------------------
// Returns node content for the last node in the node tree defined by
// "sections". So if sections[] were:
// sections[0] = "house"
// sections[1] = "room"
// Then this function would return the node content for the first <room>
// tag found under the first <house> tag.
// Function assumes that the desired node has no children nodes other than
// the text content node.
//------------------------------------------------------------------------------
std::string WEXmlgetter::getValue(const vector<string>& sections) const
{
std::string aRet;
const xmlNode* pPtr = fpRoot;
int aSize = sections.size();
int aIdx = 0;
//cout << aSize << endl;
while(aIdx < aSize)
{
//cout << aIdx <<" "<< sections[aIdx] << endl;
pPtr = getNode(pPtr, sections[aIdx]);
if((pPtr == NULL) || (aIdx == aSize-1)) break;
else
{
//cout << "getValue Name " << (const char*)pPtr->name << endl;
pPtr = pPtr->xmlChildrenNode;
aIdx++;
}
}
if(pPtr != NULL)
{
//aRet = (const char*)pPtr->content;
std::string aBuff;
if(getNodeContent(pPtr, aBuff)) aRet = aBuff;
}
return aRet;
}
//------------------------------------------------------------------------------
// Iterate through the sibling nodes starting with pParent, looking for
// a node with the specified name (section). The xmlNode (if found) is
// returned.
//------------------------------------------------------------------------------
const xmlNode* WEXmlgetter::getNode(const xmlNode* pParent,
const string& section)const
{
if(pParent == NULL) return NULL;
const xmlNode* pPtr = pParent;
while(pPtr != NULL )
{
//cout << "getNode Name " << (const char*)pPtr->name << endl;
if(!xmlStrcmp(pPtr->name, (const xmlChar *)section.c_str()))
return pPtr;
else
pPtr = pPtr->next;
}
return pPtr;
}
//------------------------------------------------------------------------------
// Iterate down through the node tree represented by the sections vector.
// In the last child of this tree, we look for the specified attribute tag,
// and return its value.
//------------------------------------------------------------------------------
std::string WEXmlgetter::getAttribute(const vector<string>& sections,
const string& Tag) const
{
std::string aRet;
const xmlNode* pPtr = fpRoot;
int aSize = sections.size();
if(aSize==0)
throw invalid_argument("WEXmlgetter::getAttribute(): section must be valid");
int aIdx = 0;
//cout << aSize << endl;
while(aIdx < aSize)
{
//cout << aIdx <<" "<< sections[aIdx] << endl;
pPtr = getNode(pPtr, sections[aIdx]);
if((pPtr == NULL) || (aIdx == aSize-1)) break;
else
{
//cout << "getValue Name " << (const char*)pPtr->name << endl;
pPtr = pPtr->xmlChildrenNode;
aIdx++;
}
}
if(pPtr != NULL)
{
std::string aBuff;
//cout << "attrTagNode Name " << (const char*)pPtr->name << endl;
if (getNodeAttribute(pPtr, Tag.c_str(), aBuff))
aRet = aBuff;
//aRet = (const char*)pPtr->content;
//cout << "Attribute("<<Tag<<") = "<< aRet<< endl;
}
return aRet;
}
//------------------------------------------------------------------------------
// Iterate down through the node tree represented by the sections vector.
// At the end of the branch, there may be several sibling nodes matching
// the node search vector.
// For each of the matching children nodes found, we look for the specified
// attribute tag, and return its value. Hence a vector of attribute values
// is returned.
//------------------------------------------------------------------------------
void WEXmlgetter::getAttributeListForAllChildren(
const vector<string>& sections,
const string& attributeTag,
vector<string>& attributeValues)
{
const xmlNode* pPtr = fpRoot;
int aSize = sections.size();
if(aSize==0)
{
throw invalid_argument("WEXmlgetter::getAttributeListForAllChildren():"
" No XML nodes specified in section search list");
}
// Step down the branch that has the nodes of interest
int aIdx = 0;
while (aIdx < aSize)
{
pPtr = getNode(pPtr, sections[aIdx]);
if ((pPtr == NULL) || (aIdx == aSize-1))
{
break;
}
else
{
pPtr = pPtr->xmlChildrenNode;
aIdx++;
}
}
// Look for all the "matching" nodes at the end of the branch, and
// get the requested attribute value for each matching node.
if (pPtr != NULL)
{
while(pPtr != NULL )
{
std::string attrib;
if (getNodeAttribute(pPtr, attributeTag.c_str(), attrib))
{
attributeValues.push_back(attrib);
}
pPtr = pPtr->next;
}
}
}
} /* namespace WriteEngine */

71
writeengine/splitter/we_xmlgetter.h Normal file
View File

@ -0,0 +1,71 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_xmlgetter.h
*
* Created on: Feb 7, 2012
* Author: bpaul
*/
#ifndef WE_XMLGETTER_H_
#define WE_XMLGETTER_H_
#include <libxml/parser.h>
namespace WriteEngine
{
class WEXmlgetter
{
public:
WEXmlgetter(std::string& ConfigName);
virtual ~WEXmlgetter();
public:
//..Public methods
std::string getValue(const vector<string>& section) const;
std::string getAttribute(const std::vector<string>& sections,
const std::string& Tag) const;
void getConfig(const std::string& section,
const std::string& name, std::vector<std::string>& values ) const;
void getAttributeListForAllChildren(
const vector<string>& sections,
const string& attributeTag,
vector<string>& attributeValues);
private:
//..Private methods
const xmlNode* getNode(const xmlNode* pParent,
const std::string& section)const;
bool getNodeAttribute(const xmlNode* pNode,
const char* pTag, std::string& strVal ) const;
bool getNodeContent( const xmlNode* pNode, std::string& strVal) const;
//..Private data members
std::string fConfigName; // xml filename
xmlDocPtr fDoc; // xml document pointer
xmlNode* fpRoot; // root element
};
} /* namespace WriteEngine */
#endif /* WE_XMLGETTER_H_ */