the begginning

writeengine/server/Makefile

@@ -0,0 +1,98 @@
+include ../../rules.mak
+include ../build/we_rules.mak
+#******************************************************************************************
+# Copyright (C) 2009-2012 Calpont Corporation
+# All rights reserved
+#
+# $Id: Makefile 4624 2013-04-29 16:19:59Z rdempsey $
+#
+#*****************************************************************************************/
+# The name of the executable
+PROGRAM=WriteEngineServer
+VERSION=1.0.0
+
+.PHONY: install clean docs test coverage leakcheck install_server
+TLIBS+=-L$(CALPONT_LIBRARY_PATH) $(IDB_COMMON_LIBS) $(IDB_WRITE_LIBS)
+LDFLAGS+=-L$(CALPONT_LIBRARY_PATH) $(IDB_COMMON_LIBS) $(IDB_WRITE_LIBS) $(IDB_SNMP_LIBS) -lthreadpool -lwriteengineredistribute -Wl,--rpath -Wl,/usr/local/Calpont/lib
+GLIBS=$(LDFLAGS)
+
+SRCS= \
+	we_brmrprtparser.cpp \
+	we_cleartablelockcmd.cpp \
+	we_cpifeederthread.cpp \
+	we_dataloader.cpp \
+	we_ddlcommandproc.cpp \
+	we_dmlcommandproc.cpp \
+	we_getfilesizes.cpp \
+	we_observer.cpp \
+	we_readthread.cpp \
+	we_server.cpp
+
+LINCLUDES= we_messages.h
+
+OBJS=$(SRCS:.cpp=.o)
+
+GOBJS=$(LOBJS:.o=-gcov.o)
+
+objects: $(OBJS)
+
+install_server: bootstrap WriteEngineServer
+	mkdir -p $(INSTALL_ROOT_BIN)
+	$(INSTALL) $(PROGRAM) $(INSTALL_ROOT_BIN)
+
+WriteEngineServer: $(OBJS)
+	$(LINK.cpp) -o $@ $^
+
+install: bootstrap
+
+bootstrap:
+	$(INSTALL) $(LINCLUDES) $(INSTALL_ROOT_INCLUDE)
+
+clean:
+	rm -f $(PROGRAM) $(OBJS) core *~ *.tag *-gcov.* *.gcov *.d *.swp *.dat
+	rm -rf html
+
+docs:
+	doxygen $(EXPORT_ROOT)/etc/Doxyfile
+
+test:
+
+xtest: tbulk
+	$(IPCS_CLEANUP)
+	LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib ./tbulk
+
+%-gcov.o: %.cpp
+	$(COMPILE.cpp) -o $@ $^
+
+tbulk-gcov: CXXFLAGS+=-fprofile-arcs -ftest-coverage
+tbulk-gcov: tbulk-gcov.o $(subst .o,-gcov.o,$(OBJS))
+	$(LINK.cpp) -o $@ $^ $(GOBJS) $(GLIBS)
+	cp *-gcov.o $(LIBDIR)
+
+coverage:
+
+xcoverage: tbulk-gcov
+	$(IPCS_CLEANUP)
+	rm -f *.gcda
+	LD_LIBRARY_PATH=$(EXPORT_ROOT)/lib:/usr/local/lib ./tbulk-gcov
+	for file in $(SRCS); do \
+		bfile=`basename $$file .cpp`; \
+		gcov -o $${bfile}-gcov $$file >/dev/null; \
+	done
+	/usr/local/bin/genCoverage.pl $(SRCS)
+
+leakcheck:
+
+xleakcheck:  tbulk
+	$(IPCS_CLEANUP)
+	LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib valgrind --error-limit=no --tool=memcheck --leak-check=yes ./tbulk
+
+%.d: %.cpp
+	@set -e; rm -f $@; \
+	$(CC) -MM $(CPPFLAGS) $< > $@.$$$$; \
+	sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$$ > $@; \
+	rm -f $@.$$$$
+
+ifndef BOOTSTRAP
+-include $(SRCS:.cpp=.d)
+endif

writeengine/server/Makefile.am

@@ -0,0 +1,50 @@
+# Copyright (C) 2014 InfiniDB, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2 of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301, USA.
+
+# $Id: Makefile.am 696 2011-03-07 21:54:16Z rdempsey $
+## Process this file with automake to produce Makefile.in
+
+AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags)
+AM_CFLAGS = $(idb_cflags)
+AM_CXXFLAGS = $(idb_cxxflags)
+AM_LDFLAGS = $(idb_ldflags)
+bin_PROGRAMS = WriteEngineServer
+WriteEngineServer_SOURCES = we_server.cpp \
+we_readthread.cpp \
+we_ddlcommandproc.cpp \
+we_observer.cpp \
+we_dataloader.cpp \
+we_brmrprtparser.cpp \
+we_dmlcommandproc.cpp \
+we_cleartablelockcmd.cpp \
+we_cpifeederthread.cpp \
+we_getfilesizes.cpp
+include_HEADERS = we_messages.h
+WriteEngineServer_LDFLAGS = $(idb_common_ldflags) $(idb_exec_libs) -lthreadpool -lwriteengineredistribute $(idb_write_libs) $(AM_LDFLAGS)
+
+install_server: install
+
+test:
+
+coverage:
+
+leakcheck:
+
+docs:
+
+bootstrap: install-data-am
+

writeengine/server/Makefile.in

@@ -0,0 +1,570 @@
+# Makefile.in generated by automake 1.9.6 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
+# 2003, 2004, 2005  Free Software Foundation, Inc.
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+# Copyright (C) 2014 InfiniDB, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2 of
+# the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301, USA.
+
+# $Id: Makefile.am 696 2011-03-07 21:54:16Z rdempsey $
+
+
+srcdir = @srcdir@
+top_srcdir = @top_srcdir@
+VPATH = @srcdir@
+pkgdatadir = $(datadir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+top_builddir = ../..
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+INSTALL = @INSTALL@
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+bin_PROGRAMS = WriteEngineServer$(EXEEXT)
+subdir = writeengine/server
+DIST_COMMON = $(include_HEADERS) $(srcdir)/Makefile.am \
+	$(srcdir)/Makefile.in
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/compilerflags.m4 \
+	$(top_srcdir)/m4/functions.m4 $(top_srcdir)/m4/install.m4 \
+	$(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(includedir)"
+binPROGRAMS_INSTALL = $(INSTALL_PROGRAM)
+PROGRAMS = $(bin_PROGRAMS)
+am_WriteEngineServer_OBJECTS = we_server.$(OBJEXT) \
+	we_readthread.$(OBJEXT) we_ddlcommandproc.$(OBJEXT) \
+	we_observer.$(OBJEXT) we_dataloader.$(OBJEXT) \
+	we_brmrprtparser.$(OBJEXT) we_dmlcommandproc.$(OBJEXT) \
+	we_cleartablelockcmd.$(OBJEXT) we_cpifeederthread.$(OBJEXT) \
+	we_getfilesizes.$(OBJEXT)
+WriteEngineServer_OBJECTS = $(am_WriteEngineServer_OBJECTS)
+WriteEngineServer_LDADD = $(LDADD)
+DEFAULT_INCLUDES = -I. -I$(srcdir) -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__depfiles_maybe = depfiles
+CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
+LTCXXCOMPILE = $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CXXFLAGS) $(CXXFLAGS)
+CXXLD = $(CXX)
+CXXLINK = $(LIBTOOL) --tag=CXX --mode=link $(CXXLD) $(AM_CXXFLAGS) \
+	$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
+SOURCES = $(WriteEngineServer_SOURCES)
+DIST_SOURCES = $(WriteEngineServer_SOURCES)
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = `echo $$p | sed -e 's|^.*/||'`;
+includeHEADERS_INSTALL = $(INSTALL_HEADER)
+HEADERS = $(include_HEADERS)
+ETAGS = etags
+CTAGS = ctags
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ALLOCA = @ALLOCA@
+AMDEP_FALSE = @AMDEP_FALSE@
+AMDEP_TRUE = @AMDEP_TRUE@
+AMTAR = @AMTAR@
+AR = @AR@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CFLAGS = @CFLAGS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+ECHO = @ECHO@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+F77 = @F77@
+FFLAGS = @FFLAGS@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LEX = @LEX@
+LEXLIB = @LEXLIB@
+LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBTOOL = @LIBTOOL@
+LN_S = @LN_S@
+LTLIBOBJS = @LTLIBOBJS@
+MAKEINFO = @MAKEINFO@
+OBJEXT = @OBJEXT@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+POW_LIB = @POW_LIB@
+RANLIB = @RANLIB@
+SED = @SED@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+STRIP = @STRIP@
+VERSION = @VERSION@
+XML2_CONFIG = @XML2_CONFIG@
+XML_CPPFLAGS = @XML_CPPFLAGS@
+XML_LIBS = @XML_LIBS@
+YACC = @YACC@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_F77 = @ac_ct_F77@
+ac_ct_RANLIB = @ac_ct_RANLIB@
+ac_ct_STRIP = @ac_ct_STRIP@
+am__fastdepCC_FALSE = @am__fastdepCC_FALSE@
+am__fastdepCC_TRUE = @am__fastdepCC_TRUE@
+am__fastdepCXX_FALSE = @am__fastdepCXX_FALSE@
+am__fastdepCXX_TRUE = @am__fastdepCXX_TRUE@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+datadir = @datadir@
+etcdir = @etcdir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+idb_brm_libs = @idb_brm_libs@
+idb_cflags = @idb_cflags@
+idb_common_includes = @idb_common_includes@
+idb_common_ldflags = @idb_common_ldflags@
+idb_common_libs = @idb_common_libs@
+idb_cppflags = @idb_cppflags@
+idb_cxxflags = @idb_cxxflags@
+idb_exec_libs = @idb_exec_libs@
+idb_ldflags = @idb_ldflags@
+idb_oam_libs = @idb_oam_libs@
+idb_write_libs = @idb_write_libs@
+idbinstall = @idbinstall@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localdir = @localdir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+march_flags = @march_flags@
+mibdir = @mibdir@
+mkdir_p = @mkdir_p@
+mysqldir = @mysqldir@
+netsnmp_libs = @netsnmp_libs@
+netsnmpagntdir = @netsnmpagntdir@
+netsnmpdir = @netsnmpdir@
+netsnmplibrdir = @netsnmplibrdir@
+netsnmpmachdir = @netsnmpmachdir@
+netsnmpsysdir = @netsnmpsysdir@
+oldincludedir = @oldincludedir@
+postdir = @postdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+sbindir = @sbindir@
+sharedir = @sharedir@
+sharedstatedir = @sharedstatedir@
+sysconfdir = @sysconfdir@
+target_alias = @target_alias@
+toolsdir = @toolsdir@
+AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags)
+AM_CFLAGS = $(idb_cflags)
+AM_CXXFLAGS = $(idb_cxxflags)
+AM_LDFLAGS = $(idb_ldflags)
+WriteEngineServer_SOURCES = we_server.cpp \
+we_readthread.cpp \
+we_ddlcommandproc.cpp \
+we_observer.cpp \
+we_dataloader.cpp \
+we_brmrprtparser.cpp \
+we_dmlcommandproc.cpp \
+we_cleartablelockcmd.cpp \
+we_cpifeederthread.cpp \
+we_getfilesizes.cpp
+
+include_HEADERS = we_messages.h
+WriteEngineServer_LDFLAGS = $(idb_common_ldflags) $(idb_exec_libs) -lthreadpool -lwriteengineredistribute $(idb_write_libs) $(AM_LDFLAGS)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .cpp .lo .o .obj
+$(srcdir)/Makefile.in:  $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh \
+		&& exit 0; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu  writeengine/server/Makefile'; \
+	cd $(top_srcdir) && \
+	  $(AUTOMAKE) --gnu  writeengine/server/Makefile
+.PRECIOUS: Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure:  $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4):  $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+install-binPROGRAMS: $(bin_PROGRAMS)
+	@$(NORMAL_INSTALL)
+	test -z "$(bindir)" || $(mkdir_p) "$(DESTDIR)$(bindir)"
+	@list='$(bin_PROGRAMS)'; for p in $$list; do \
+	  p1=`echo $$p|sed 's/$(EXEEXT)$$//'`; \
+	  if test -f $$p \
+	     || test -f $$p1 \
+	  ; then \
+	    f=`echo "$$p1" | sed 's,^.*/,,;$(transform);s/$$/$(EXEEXT)/'`; \
+	   echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) --mode=install $(binPROGRAMS_INSTALL) '$$p' '$(DESTDIR)$(bindir)/$$f'"; \
+	   $(INSTALL_PROGRAM_ENV) $(LIBTOOL) --mode=install $(binPROGRAMS_INSTALL) "$$p" "$(DESTDIR)$(bindir)/$$f" || exit 1; \
+	  else :; fi; \
+	done
+
+uninstall-binPROGRAMS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(bin_PROGRAMS)'; for p in $$list; do \
+	  f=`echo "$$p" | sed 's,^.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/'`; \
+	  echo " rm -f '$(DESTDIR)$(bindir)/$$f'"; \
+	  rm -f "$(DESTDIR)$(bindir)/$$f"; \
+	done
+
+clean-binPROGRAMS:
+	@list='$(bin_PROGRAMS)'; for p in $$list; do \
+	  f=`echo $$p|sed 's/$(EXEEXT)$$//'`; \
+	  echo " rm -f $$p $$f"; \
+	  rm -f $$p $$f ; \
+	done
+WriteEngineServer$(EXEEXT): $(WriteEngineServer_OBJECTS) $(WriteEngineServer_DEPENDENCIES) 
+	@rm -f WriteEngineServer$(EXEEXT)
+	$(CXXLINK) $(WriteEngineServer_LDFLAGS) $(WriteEngineServer_OBJECTS) $(WriteEngineServer_LDADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_brmrprtparser.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_cleartablelockcmd.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_cpifeederthread.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_dataloader.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_ddlcommandproc.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_dmlcommandproc.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_getfilesizes.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_observer.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_readthread.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/we_server.Po@am__quote@
+
+.cpp.o:
+@am__fastdepCXX_TRUE@	if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \
+@am__fastdepCXX_TRUE@	then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(CXXCOMPILE) -c -o $@ $<
+
+.cpp.obj:
+@am__fastdepCXX_TRUE@	if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ `$(CYGPATH_W) '$<'`; \
+@am__fastdepCXX_TRUE@	then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.cpp.lo:
+@am__fastdepCXX_TRUE@	if $(LTCXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \
+@am__fastdepCXX_TRUE@	then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Plo"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(LTCXXCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+
+distclean-libtool:
+	-rm -f libtool
+uninstall-info-am:
+install-includeHEADERS: $(include_HEADERS)
+	@$(NORMAL_INSTALL)
+	test -z "$(includedir)" || $(mkdir_p) "$(DESTDIR)$(includedir)"
+	@list='$(include_HEADERS)'; for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  f=$(am__strip_dir) \
+	  echo " $(includeHEADERS_INSTALL) '$$d$$p' '$(DESTDIR)$(includedir)/$$f'"; \
+	  $(includeHEADERS_INSTALL) "$$d$$p" "$(DESTDIR)$(includedir)/$$f"; \
+	done
+
+uninstall-includeHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(include_HEADERS)'; for p in $$list; do \
+	  f=$(am__strip_dir) \
+	  echo " rm -f '$(DESTDIR)$(includedir)/$$f'"; \
+	  rm -f "$(DESTDIR)$(includedir)/$$f"; \
+	done
+
+ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
+	list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
+	unique=`for i in $$list; do \
+	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+	  done | \
+	  $(AWK) '    { files[$$0] = 1; } \
+	       END { for (i in files) print i; }'`; \
+	mkid -fID $$unique
+tags: TAGS
+
+TAGS:  $(HEADERS) $(SOURCES)  $(TAGS_DEPENDENCIES) \
+		$(TAGS_FILES) $(LISP)
+	tags=; \
+	here=`pwd`; \
+	list='$(SOURCES) $(HEADERS)  $(LISP) $(TAGS_FILES)'; \
+	unique=`for i in $$list; do \
+	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+	  done | \
+	  $(AWK) '    { files[$$0] = 1; } \
+	       END { for (i in files) print i; }'`; \
+	if test -z "$(ETAGS_ARGS)$$tags$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	    $$tags $$unique; \
+	fi
+ctags: CTAGS
+CTAGS:  $(HEADERS) $(SOURCES)  $(TAGS_DEPENDENCIES) \
+		$(TAGS_FILES) $(LISP)
+	tags=; \
+	here=`pwd`; \
+	list='$(SOURCES) $(HEADERS)  $(LISP) $(TAGS_FILES)'; \
+	unique=`for i in $$list; do \
+	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+	  done | \
+	  $(AWK) '    { files[$$0] = 1; } \
+	       END { for (i in files) print i; }'`; \
+	test -z "$(CTAGS_ARGS)$$tags$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$tags $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && cd $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) $$here
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's|.|.|g'`; \
+	list='$(DISTFILES)'; for file in $$list; do \
+	  case $$file in \
+	    $(srcdir)/*) file=`echo "$$file" | sed "s|^$$srcdirstrip/||"`;; \
+	    $(top_srcdir)/*) file=`echo "$$file" | sed "s|^$$topsrcdirstrip/|$(top_builddir)/|"`;; \
+	  esac; \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  dir=`echo "$$file" | sed -e 's,/[^/]*$$,,'`; \
+	  if test "$$dir" != "$$file" && test "$$dir" != "."; then \
+	    dir="/$$dir"; \
+	    $(mkdir_p) "$(distdir)$$dir"; \
+	  else \
+	    dir=''; \
+	  fi; \
+	  if test -d $$d/$$file; then \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -pR $(srcdir)/$$file $(distdir)$$dir || exit 1; \
+	    fi; \
+	    cp -pR $$d/$$file $(distdir)$$dir || exit 1; \
+	  else \
+	    test -f $(distdir)/$$file \
+	    || cp -p $$d/$$file $(distdir)/$$file \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(PROGRAMS) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(bindir)" "$(DESTDIR)$(includedir)"; do \
+	  test -z "$$dir" || $(mkdir_p) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	$(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	  install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	  `test -z '$(STRIP)' || \
+	    echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-binPROGRAMS clean-generic clean-libtool mostlyclean-am
+
+distclean: distclean-am
+	-rm -rf ./$(DEPDIR)
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-libtool distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+info: info-am
+
+info-am:
+
+install-data-am: install-includeHEADERS
+
+install-exec-am: install-binPROGRAMS
+
+install-info: install-info-am
+
+install-man:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+	-rm -rf ./$(DEPDIR)
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-binPROGRAMS uninstall-includeHEADERS \
+	uninstall-info-am
+
+.PHONY: CTAGS GTAGS all all-am check check-am clean clean-binPROGRAMS \
+	clean-generic clean-libtool ctags distclean distclean-compile \
+	distclean-generic distclean-libtool distclean-tags distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-binPROGRAMS install-data install-data-am install-exec \
+	install-exec-am install-includeHEADERS install-info \
+	install-info-am install-man install-strip installcheck \
+	installcheck-am installdirs maintainer-clean \
+	maintainer-clean-generic mostlyclean mostlyclean-compile \
+	mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
+	tags uninstall uninstall-am uninstall-binPROGRAMS \
+	uninstall-includeHEADERS uninstall-info-am
+
+
+install_server: install
+
+test:
+
+coverage:
+
+leakcheck:
+
+docs:
+
+bootstrap: install-data-am
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:

writeengine/server/WriteEngineServer.rc

@@ -0,0 +1,102 @@
+// Microsoft Visual C++ generated resource script.
+//
+#include "resource.h"
+
+#define APSTUDIO_READONLY_SYMBOLS
+/////////////////////////////////////////////////////////////////////////////
+//
+// Generated from the TEXTINCLUDE 2 resource.
+//
+#include "afxres.h"
+
+/////////////////////////////////////////////////////////////////////////////
+#undef APSTUDIO_READONLY_SYMBOLS
+
+/////////////////////////////////////////////////////////////////////////////
+// English (U.S.) resources
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+#ifdef _WIN32
+LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
+#pragma code_page(1252)
+#endif //_WIN32
+
+#ifdef APSTUDIO_INVOKED
+/////////////////////////////////////////////////////////////////////////////
+//
+// TEXTINCLUDE
+//
+
+1 TEXTINCLUDE 
+BEGIN
+    "resource.h\0"
+END
+
+2 TEXTINCLUDE 
+BEGIN
+    "#include ""afxres.h""\r\n"
+    "\0"
+END
+
+3 TEXTINCLUDE 
+BEGIN
+    "\r\n"
+    "\0"
+END
+
+#endif    // APSTUDIO_INVOKED
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION 4,6,0,0
+ PRODUCTVERSION 4,6,0,0
+ FILEFLAGSMASK 0x17L
+#ifdef _DEBUG
+ FILEFLAGS 0x1L
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS 0x4L
+ FILETYPE 0x1L
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+            VALUE "CompanyName", "InfiniDB, Inc."
+            VALUE "FileDescription", "InfiniDB Write Engine Server"
+            VALUE "FileVersion", "4.6.0-0"
+            VALUE "InternalName", "WriteEngineServer"
+            VALUE "LegalCopyright", "Copyright (C) 2014"
+            VALUE "OriginalFilename", "WriteEngineServer.exe"
+            VALUE "ProductName", "InfiniDB"
+            VALUE "ProductVersion", "4.6.0.0 Beta"
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1200
+    END
+END
+
+#endif    // English (U.S.) resources
+/////////////////////////////////////////////////////////////////////////////
+
+
+
+#ifndef APSTUDIO_INVOKED
+/////////////////////////////////////////////////////////////////////////////
+//
+// Generated from the TEXTINCLUDE 3 resource.
+//
+
+
+/////////////////////////////////////////////////////////////////////////////
+#endif    // not APSTUDIO_INVOKED
+

writeengine/server/resource.h

@@ -0,0 +1,14 @@
+//{{NO_DEPENDENCIES}}
+// Microsoft Visual C++ generated include file.
+// Used by cpimport.rc
+
+// Next default values for new objects
+// 
+#ifdef APSTUDIO_INVOKED
+#ifndef APSTUDIO_READONLY_SYMBOLS
+#define _APS_NEXT_RESOURCE_VALUE        101
+#define _APS_NEXT_COMMAND_VALUE         40001
+#define _APS_NEXT_CONTROL_VALUE         1001
+#define _APS_NEXT_SYMED_VALUE           101
+#endif
+#endif

writeengine/server/we_brmrprtparser.cpp

@@ -0,0 +1,176 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id$
+*
+*******************************************************************************/
+
+/*
+ * we_brmrprtparser.cpp
+ *
+ *  Created on: Dec 13, 2011
+ *      Author: bpaul
+ */
+
+#include <fstream>
+#include <iostream>
+using namespace std;
+
+#include "we_brmrprtparser.h"
+
+namespace WriteEngine
+{
+
+BrmReportParser::BrmReportParser():fRptFile()
+{
+
+}
+
+BrmReportParser::~BrmReportParser()
+{
+
+}
+
+//------------------------------------------------------------------------------
+// Serialize specified RptFileName into the ByteStream Bs.
+// Serialization is done one record at a time.  File should be an ASCII file
+// with newlines markers, because getline() is used.
+// Function is limited to records that are no longer than 255 bytes.
+// If a record is longer than 255 bytes, the function misbehaves.
+//------------------------------------------------------------------------------
+bool BrmReportParser::serialize(std::string RptFileName,
+									messageqcpp::ByteStream& Bs)
+{
+	try
+	{
+	fRptFile.open(RptFileName.c_str(), ifstream::in);
+	}
+	catch(std::exception& ex)
+	{
+		cout <<"Failed to open BRMRptFile "<< RptFileName <<endl;
+		cout << ex.what() << endl;
+		throw runtime_error(ex.what());
+	}
+	if(fRptFile.good())
+	{
+		char aBuff[10240];
+		unsigned int aLen=0;
+		while(fRptFile.good() && !fRptFile.eof())
+		{
+			fRptFile.getline(aBuff, sizeof(aBuff)-1);
+			aLen=fRptFile.gcount();
+			if((aLen != (sizeof(aBuff)-1)) && (aLen>0))
+			{
+				//aBuff[aLen-1] = '\n';
+				//aBuff[aLen]=0;
+				//cout << "Data Read " << aBuff <<endl;
+				if(aBuff[0] != '#')	// do not serialize comments
+				{
+					std::string strData = aBuff;
+					Bs << strData;
+				}
+			}
+		}// while
+		fRptFile.close();
+		cout << "Closed File : "<< RptFileName <<" "<< errno << endl;
+	}
+	else
+	{
+		cout << "Failed to open : "<< RptFileName <<" "<< errno << endl;
+		return false;
+	}
+	return true;
+}
+
+//------------------------------------------------------------------------------
+// Serialize specified RptFileName into the ByteStream Bs.
+// Serialization is done in 8192 byte blocks instead of by line or by record,
+// so that this function will work for both character and binary files.
+//------------------------------------------------------------------------------
+bool BrmReportParser::serializeBlocks(std::string RptFileName,
+	messageqcpp::ByteStream& Bs)
+{
+	try
+	{
+		fRptFile.open(RptFileName.c_str(), ifstream::in);
+	}
+	catch (std::exception& ex)
+	{
+		cout <<"Failed to open Report File "<< RptFileName << endl;
+		cout << ex.what() << endl;
+		return false;
+	}
+
+	if (fRptFile.good())
+	{
+		char aBuff[8192];
+		unsigned int aLen=0;
+		std::string strBuff;
+
+		while ( fRptFile.good() )
+		{
+			fRptFile.read(aBuff, sizeof(aBuff));
+			aLen = fRptFile.gcount();
+
+			if (aLen > 0)
+			{
+				strBuff.assign( aBuff, aLen );
+				Bs << strBuff;
+			}
+		}
+
+		fRptFile.close();
+		cout << "Closed Report File : "<< RptFileName << endl;
+	}
+	else
+	{
+        std::ostringstream oss;
+		oss << "Failed to open Report File " << RptFileName << endl;
+		cout << oss.str() << endl;
+		return false;
+	}
+
+	return true;
+}
+
+void BrmReportParser::unserialize(messageqcpp::ByteStream& Bs)
+{
+	//TODO to be changed. left it here to understand how to implement
+	/*
+	ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN);
+	ReturnedColumn::unserialize(b); // parent class unserialize
+	b >> (uint32_t&) fOid;
+	b >> fData;
+	b >> reinterpret_cast<ByteStream::doublebyte&>(fReturnAll);
+	b >> (uint32_t&) fSequence;
+	*/
+
+	std::string aStrLine;
+	while(Bs.length()>0)
+	{
+		Bs >> aStrLine;
+		cout << aStrLine;
+	}
+
+}
+
+
+
+
+
+} /* namespace WriteEngine */

writeengine/server/we_brmrprtparser.h

@@ -0,0 +1,76 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id$
+*
+*******************************************************************************/
+/*
+ * we_brmrprtparser.h
+ *
+ *  Created on: Dec 13, 2011
+ *      Author: bpaul
+ */
+
+#ifndef WE_BRMRPRTPARSER_H_
+#define WE_BRMRPRTPARSER_H_
+
+#include <fstream>
+#include <iostream>
+using namespace std;
+
+#include "bytestream.h"
+#include "messagequeue.h"
+using namespace messageqcpp;
+
+/*
+ *
+  #CP:   startLBID max min seqnum type newExtent
+  #HWM:  oid partition segment hwm
+  #ROWS: numRowsRead numRowsInserted
+  #DATA: columnNumber numOutOfRangeValues
+  #ERR:  error message file
+  #BAD:  bad data file, with rejected rows
+  CP: 234496 6 6 -1 0 0
+  HWM: 3091 0 0 1
+  ERR: /home/dcathey/t1.tbl.Job_3090_27983.err
+  BAD: /home/dcathey/t1.tbl.Job_3090_27983.bad
+  ROWS: 3 1
+ *
+ */
+
+namespace WriteEngine
+{
+
+class BrmReportParser
+{
+public:
+	BrmReportParser();
+	virtual ~BrmReportParser();
+
+public:
+	bool serialize(std::string RptFileName, messageqcpp::ByteStream& Bs);
+	bool serializeBlocks(std::string RptFileName, messageqcpp::ByteStream& Bs);
+	void unserialize(messageqcpp::ByteStream& Bs);
+
+private:
+	ifstream fRptFile;
+
+};
+
+} /* namespace WriteEngine */
+#endif /* WE_BRMRPRTPARSER_H_ */

writeengine/server/we_cleartablelockcmd.cpp

@@ -0,0 +1,152 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*****************************************************************************
+ * $Id: we_cleartablelockcmd.cpp 4450 2013-01-21 14:13:24Z rdempsey $
+ *
+ ****************************************************************************/
+#include "we_cleartablelockcmd.h"
+
+#include <iostream>
+#include <stdexcept>
+
+#include "we_bulkrollbackmgr.h"
+#include "we_confirmhdfsdbfile.h"
+
+#include "IDBPolicy.h"
+
+namespace WriteEngine
+{
+
+//------------------------------------------------------------------------------
+// Process a bulk rollback request based on input from the specified input
+// bytestream object.
+//------------------------------------------------------------------------------
+int WE_ClearTableLockCmd::processRollback(
+	messageqcpp::ByteStream& bs,
+	std::string& errMsg)
+{
+	uint8_t rc = 0;
+	errMsg.clear();
+
+	try {
+		uint32_t    tableOID;
+		uint64_t    tableLockID;
+		std::string tableName;
+		std::string appName;
+
+		// May want to eventually comment out this logging to stdout,
+		// but it shouldn't hurt to keep in here.
+		std::cout << "ClearTableLockCmd::processRollback for " << fUserDesc;
+		bs >> tableLockID;
+		std::cout << ": tableLock-"<< tableLockID;
+
+		bs >> tableOID;
+		std::cout << "; tableOID-" << tableOID;
+
+		bs >> tableName;
+		std::cout << "; table-"    << tableName;
+
+		bs >> appName;
+		std::cout << "; app-"      << appName << std::endl;
+
+		int we_rc = fWEWrapper.bulkRollback(
+			tableOID,
+			tableLockID,
+			tableName,
+			appName,
+			false, // no extra debug logging to the console
+			errMsg );
+
+		if (we_rc != NO_ERROR)
+			rc = 2;
+	}
+	catch(std::exception& ex)
+	{
+		std::cout << "ClearTableLockCmd::Rollback exception-" << ex.what() <<
+			std::endl;
+		errMsg = ex.what();
+		rc     = 1;
+	}
+
+	return rc;
+}
+
+//------------------------------------------------------------------------------
+// Process a bulk rollback cleanup request based on input from the specified
+// input bytestream object.
+// Optionally, an additional flag is appended to the message to indicate
+// whether HDFS db file *.orig db files are to be cleaned up as well.
+// We only need this for the "successful" case, as the bulk rollback takes
+// care of db file cleanup in the "unsuccessful" case.
+//------------------------------------------------------------------------------
+int WE_ClearTableLockCmd::processCleanup(
+	messageqcpp::ByteStream& bs,
+	std::string& errMsg)
+{
+	uint8_t rc = 0;
+	errMsg.clear();
+
+	try {
+		uint32_t tableOID;
+
+		// May want to eventually comment out this logging to stdout,
+		// but it shouldn't hurt to keep in here.
+		std::cout << "ClearTableLockCmd::processCleanup for " << fUserDesc;
+		bs >> tableOID;
+		std::cout << ": tableOID-" << tableOID << std::endl;
+
+		// On an HDFS system, this is where we delete any DB files
+		// (ex: *.orig) that we no longer need.
+		if ((idbdatafile::IDBPolicy::useHdfs()) &&
+			(bs.length() >= sizeof(messageqcpp::ByteStream::byte)))
+		{
+			messageqcpp::ByteStream::byte deleteHdfsTempDbFiles;
+			bs >> deleteHdfsTempDbFiles;
+			if (deleteHdfsTempDbFiles)
+			{
+				std::string endDbErrMsg;
+				ConfirmHdfsDbFile confirmHdfs;
+
+                // We always pass "true", as this only applies to the "success-
+                // ful" case.  See comments that precede this function.
+				int endRc = confirmHdfs.endDbFileListFromMetaFile(
+					tableOID, true, endDbErrMsg);
+
+				// In this case, a deletion error is not fatal.
+				// so we don't propagate a bad return code.  We
+				// may want to add syslog msg (TBD)
+				if (endRc != NO_ERROR)
+					std::cout << "Orig db file deletion error: " <<
+					endDbErrMsg << std::endl;
+			}
+		}
+
+		BulkRollbackMgr::deleteMetaFile( tableOID );
+	}
+	catch(std::exception& ex)
+	{
+		std::cout << "ClearTableLockCmd::Cleanup exception-" << ex.what() <<
+			std::endl;
+		errMsg = ex.what();
+		rc     = 1;
+	}
+
+	return rc;
+}
+
+}

writeengine/server/we_cleartablelockcmd.h

@@ -0,0 +1,63 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id: we_cleartablelockcmd.h 4450 2013-01-21 14:13:24Z rdempsey $
+*
+*******************************************************************************/
+#ifndef WE_CLEARTABLELOCKCMD_H__
+#define WE_CLEARTABLELOCKCMD_H__
+
+#include <string>
+
+#include "bytestream.h"
+
+#include "writeengine.h"
+
+namespace WriteEngine {
+
+/** @brief Process messages from a cleartablelock or filesplitter client
+ */
+class WE_ClearTableLockCmd
+{
+public:
+	/** @brief WE_ClearTableLockCmd constructor
+	 */
+	WE_ClearTableLockCmd(const char* userDesc) : fUserDesc(userDesc) { }
+
+	/** @brief Process bulk rollback request
+	 *  @param ibs Input byte stream
+	 *  @param errMsg Return error message
+	 */
+	int processRollback(messageqcpp::ByteStream& ibs,
+						std::string& errMsg);
+
+	/** @brief Process bulk rollback cleanup request
+	 *  @param ibs Input byte stream
+	 *  @param errMsg Return error message
+	 */
+	int processCleanup (messageqcpp::ByteStream& ibs,
+						std::string& errMsg);
+
+private:
+	WriteEngineWrapper fWEWrapper; // WriteEngineWrapper object
+	std::string fUserDesc;
+};
+
+}
+
+#endif

writeengine/server/we_cpifeederthread.cpp

@@ -0,0 +1,182 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id$
+*
+*******************************************************************************/
+
+/*
+ * we_cpifeederthread.cpp
+ *
+ *  Created on: Mar 29, 2012
+ *      Author: bpaul
+ */
+
+
+#include <iostream>
+#include <string>
+#include <queue>
+using namespace std;
+
+#include <boost/thread/condition.hpp>
+#include <boost/scoped_array.hpp>
+#include <boost/thread.hpp>
+using namespace boost;
+
+#include "messagequeue.h"
+#include "bytestream.h"
+using namespace messageqcpp;
+
+#include "we_dataloader.h"
+
+#include "we_cpifeederthread.h"
+
+namespace WriteEngine
+{
+
+//------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
+
+void WECpiFeederRunner::operator()()
+{
+	fOwner.feedData2Cpi();
+	cout << "Finished running Feeder Thread!!" << endl;
+}
+
+
+//------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
+WECpiFeederThread::WECpiFeederThread(WEDataLoader& Ref):
+		fOwner(Ref),
+		fpThread(0),
+		fContinue(true),
+		fStopped(true)
+{
+	cout << "Inside WECpiFeederThread constructor"<< endl;
+}
+
+//------------------------------------------------------------------------------
+
+WECpiFeederThread::~WECpiFeederThread()
+{
+	if (fpThread)
+	{
+		delete fpThread;
+	}
+	fpThread=0;
+}
+//------------------------------------------------------------------------------
+void WECpiFeederThread::startFeederThread()
+{
+	fStopped = false;
+	cout << "Starting Feeder Thread!!" << endl;
+    fpThread = new boost::thread(WECpiFeederRunner(*this));
+}
+//------------------------------------------------------------------------------
+
+void WECpiFeederThread::add2MsgQueue(ByteStream& Ibs)
+{
+
+	//TODO creating copy is NOT good; later read from socket using a SBS
+	messageqcpp::SBS aSbs(new messageqcpp::ByteStream(Ibs));
+	Ibs.reset();	//forcefully clearing it
+	mutex::scoped_lock aLock(fMsgQMutex);
+	//cout << "pushing to the MsgQueue" << endl;
+	fMsgQueue.push(aSbs);
+	fFeederCond.notify_one();	// as per preference of Damon
+	aLock.unlock();
+
+}
+
+//------------------------------------------------------------------------------
+
+void WECpiFeederThread::feedData2Cpi()
+{
+	while(isContinue())
+	{
+
+		mutex::scoped_lock aLock(fMsgQMutex);
+		if(fMsgQueue.empty())
+		{
+			bool aTimedOut = fFeederCond.timed_wait(aLock, boost::posix_time::milliseconds(3000));
+			if(!isContinue()) { aLock.unlock(); break; }
+			// to handle spurious wake ups and timeout wake ups
+			if((fMsgQueue.empty())||(!aTimedOut)) {	aLock.unlock();	continue; }
+		}
+
+		messageqcpp::SBS aSbs = fMsgQueue.front();
+		fMsgQueue.pop();
+
+		aLock.unlock();
+
+		try
+		{
+			fOwner.pushData2Cpimport((*aSbs));
+			//cout << "Finished PUSHING data " << endl;
+		}
+		catch(runtime_error&)
+		{
+			//cout << "Caught exception : " << e.what() << endl;
+			//break;
+		}
+
+		aSbs.reset();	//forcefully clearing it
+		// We start sending data request from here ONLY
+		if(getQueueSize() == WEDataLoader::MAX_QSIZE) fOwner.sendDataRequest();
+	}
+
+	cout << "CpiFeedThread Stopped!! " << endl;
+	fStopped = true;
+
+}
+
+//------------------------------------------------------------------------------
+
+bool WECpiFeederThread::isMsgQueueEmpty()
+{
+	bool aRet = false;
+	mutex::scoped_lock aLock(fMsgQMutex);
+	aRet = fMsgQueue.empty();
+	aLock.unlock();
+	return aRet;
+}
+
+//------------------------------------------------------------------------------
+
+void WECpiFeederThread::stopThread()
+{
+	mutex::scoped_lock aCondLock(fContMutex);
+	fContinue = false;
+	aCondLock.unlock();
+	fFeederCond.notify_all();
+	cout << "Notified all" << endl;
+}
+
+//------------------------------------------------------------------------------
+
+bool WECpiFeederThread::isContinue()
+{
+	mutex::scoped_lock aCondLock(fContMutex);
+	return fContinue;
+}
+
+//------------------------------------------------------------------------------
+
+
+
+} /* namespace WriteEngine */

writeengine/server/we_cpifeederthread.h

@@ -0,0 +1,90 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id$
+*
+*******************************************************************************/
+
+/*
+ * we_cpifeederthread.h
+ *
+ *  Created on: Mar 29, 2012
+ *      Author: bpaul
+ */
+
+#ifndef WE_CPIFEEDERTHREAD_H_
+#define WE_CPIFEEDERTHREAD_H_
+
+#include <queue>
+
+namespace WriteEngine
+{
+
+class WEDataLoader;
+class WECpiFeederThread;
+
+class WECpiFeederRunner
+{
+public:
+	WECpiFeederRunner(WECpiFeederThread& Ref): fOwner(Ref){ /* ctor */ }
+	virtual ~WECpiFeederRunner(){/* dtor */}
+	void operator()();
+
+public:
+	WECpiFeederThread& fOwner;
+};
+
+
+class WECpiFeederThread
+{
+public:
+	WECpiFeederThread(WEDataLoader& Ref);
+	virtual ~WECpiFeederThread();
+
+public:
+	void startFeederThread();
+	void add2MsgQueue(messageqcpp::ByteStream& Ibs);
+	void feedData2Cpi();
+	void stopThread();
+	bool isMsgQueueEmpty();
+	//bool isPushing() { return fPushing; }
+	bool isStopped() { return fStopped; }
+	int getQueueSize() { return fMsgQueue.size(); }
+	bool isContinue();
+private:
+
+    WEDataLoader& fOwner;
+
+	boost::condition fFeederCond;
+	boost::mutex fMsgQMutex;
+    typedef std::queue<messageqcpp::SBS> WEMsgQueue;
+    WEMsgQueue fMsgQueue;
+
+    boost::thread *fpThread;
+	bool fContinue;
+	boost::mutex fContMutex;
+	//bool fPushing;
+	bool fStopped;
+
+
+    friend class WEDataLoader;
+};
+
+} /* namespace WriteEngine */
+
+#endif /* WE_CPIFEEDERTHREAD_H_ */

writeengine/server/we_dataloader.h

@@ -0,0 +1,175 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id$
+*
+*******************************************************************************/
+
+/*
+ * 	we_dataloader.h
+ *
+ *  Created on: Oct 4, 2011
+ *      Author: Boby Paul : bpaul@calpont.com
+ *
+ */
+
+
+
+#ifndef WE_DATALOADER_H_
+#define WE_DATALOADER_H_
+
+#include "rwlock_local.h"
+#include "resourcemanager.h"
+
+#include "we_simplesyslog.h"
+
+#include "we_observer.h"
+//#include "we_readthread.h"
+
+#include "we_cpifeederthread.h"
+
+namespace WriteEngine {
+
+class SplitterReadThread;
+
+// This class will go with the read thread & client socket
+// It will manage communication between WEClient & cpimport
+
+class WEDataLoader : public Observer
+{
+public:
+	explicit WEDataLoader(SplitterReadThread& pSrt );
+	virtual ~WEDataLoader();
+
+	virtual bool update(Subject* pSub);
+
+public:
+	bool setupCpimport();		// fork the cpimport
+	void teardownCpimport(bool useStoredWaitPidStatus); // @bug 4267
+	void pushData2Cpimport(ByteStream& Ibs);	// push data to cpimport from the queue
+	void closeWritePipe();
+	void str2Argv(std::string CmdLine, std::vector<char*>& V);
+	std::string getCalpontHome();
+	std::string getPrgmPath(std::string& PrgmName);
+	void updateCmdLineWithPath(string& CmdLine);
+
+    
+public:
+	void onReceiveKeepAlive(ByteStream& Ibs);
+	void onReceiveData(ByteStream& Ibs);
+	void onReceiveEod(ByteStream& Ibs);		//end of data
+	void onReceiveMode(ByteStream& Ibs);
+	//void onReceiveCmd(messageqcpp::SBS bs);// {(ByteStream& Ibs);
+	void onReceiveCmd(ByteStream& bs);// {(ByteStream& Ibs);
+	void onReceiveAck(ByteStream& Ibs);
+	void onReceiveNak(ByteStream& Ibs);
+	void onReceiveError(ByteStream& Ibs);
+   
+	void onReceiveJobId(ByteStream& Ibs);
+	void onReceiveJobData(ByteStream& Ibs);
+    void onReceiveImportFileName(ByteStream& Ibs);
+    void onReceiveCmdLineArgs(ByteStream& Ibs);
+    void onReceiveStartCpimport();
+    void onReceiveBrmRptFileName(ByteStream& Ibs);
+    void onReceiveCleanup(ByteStream& Ibs);
+    void onReceiveRollback(ByteStream& Ibs);
+
+    void onReceiveErrFileRqst(ByteStream& Ibs);
+    void onReceiveBadFileRqst(ByteStream& Ibs);
+
+    void onCpimportSuccess();
+    void onCpimportFailure();
+
+    void sendDataRequest();
+    void sendCpimportFailureNotice();
+
+	void serialize(messageqcpp::ByteStream& b) const;
+	void unserialize(messageqcpp::ByteStream& b);
+
+
+    // setup the signal handlers for the main app
+    void setupSignalHandlers();
+    static void onSigChild(int aInt);
+
+public:
+	void setMode(int Mode){ fMode = Mode; }
+    void updateTxBytes(unsigned int Tx){ fTxBytes += Tx; }
+    void updateRxBytes(unsigned int Rx){ fRxBytes += Rx; }
+    void setChPid(pid_t pid){ fCh_pid = pid; }
+    void setPid(pid_t pid){ fThis_pid = pid; }
+    void setPPid(pid_t pid){ fP_pid = pid; }
+    void setCmdLineStr(std::string& Str){ fCmdLineStr = Str; }
+    void setObjId(int ObjId){ fObjId = ObjId; }
+
+    unsigned int getTxBytes(){ return fTxBytes; }
+    unsigned int getRxBytes(){ return fRxBytes; }
+
+    int getObjId(){ return fObjId; }
+    int getMode(){ return fMode; }
+    pid_t getChPid(){ return fCh_pid; }
+    pid_t getPid(){ return fThis_pid; }
+    pid_t getPPid(){ return fP_pid; }
+    std::string getCmdLineStr(){ return fCmdLineStr; }
+
+private:
+	SplitterReadThread& fRef;
+
+	int fMode;
+	ofstream fDataDumpFile;
+	ofstream fJobFile;
+    unsigned int fTxBytes;
+    unsigned int fRxBytes;
+	char fPmId;
+	int fObjId;					// Object Identifier for logging
+
+	// CpImport related Member variables
+	int fFIFO[2];			//I/O Pipes
+	pid_t fCh_pid;
+	pid_t fThis_pid;
+	pid_t fP_pid;
+	bool fCpIStarted;
+	std::string fCmdLineStr;
+	std::string fBrmRptFileName;
+
+	//CPI Feeder Thread
+	WECpiFeederThread* fpCfThread;
+
+	boost::mutex fClntMsgMutex;		//mutex in sending messages to client.
+
+    //static bool fTearDownCpimport; // @bug 4267
+    bool fTearDownCpimport; // @bug 4267
+	pid_t fWaitPidRc;       // @bug 4267
+	int   fWaitPidStatus;   // @bug 4267
+
+	bool fForceKill;
+	bool fPipeErr;			// Err Flag to restrict err msgs logging.
+
+private:
+	// more enums follow
+	enum CmdId { BULKFILENAME };
+public:
+	enum { MIN_QSIZE=25, MAX_QSIZE=250};
+
+public:
+    SimpleSysLog* fpSysLog;
+
+};
+
+}
+
+#endif /* WE_DATALOADER_H_ */

writeengine/server/we_ddlcommandproc.h

@@ -0,0 +1,120 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id: we_ddlcommandproc.h 3043 2011-08-29 22:03:03Z chao $
+*
+*******************************************************************************/
+#ifndef WE_DDLCOMMANDPROC_H__
+#define WE_DDLCOMMANDPROC_H__
+
+#include <unistd.h>
+#include "bytestream.h"
+#include "we_messages.h"
+#include "dbrm.h"
+#include "we_message_handlers.h"
+#include "liboamcpp.h"
+#include <boost/date_time/gregorian/gregorian.hpp>
+#include "dataconvert.h"
+#include "writeengine.h"
+
+#if defined(_MSC_VER) && defined(xxxDDLPKGPROC_DLLEXPORT)
+#define EXPORT __declspec(dllexport)
+#else
+#define EXPORT
+#endif
+
+namespace WriteEngine
+{
+
+class WE_DDLCommandProc
+{
+	public:
+		enum LogFileType { DROPTABLE_LOG, DROPPART_LOG, TRUNCATE_LOG};
+		EXPORT WE_DDLCommandProc();
+		EXPORT WE_DDLCommandProc(const WE_DDLCommandProc& rhs);
+		EXPORT ~WE_DDLCommandProc();
+		/** @brief Update SYSCOLUMN nextval column for the columnoid with nextVal.
+		*
+		* Update SYSCOLUMN nextval column for the columnoid with nexValue.
+		* @param columnOid (in) The column OID
+		* @param nextVal (in) The partition number
+		* @return 0 on success, non-0 on error.
+		*/		
+		EXPORT uint8_t updateSyscolumnNextval(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t writeSystable(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t writeSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t writeCreateSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t createtablefiles(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t deleteSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t deleteSyscolumnRow(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t deleteSystable(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t deleteSystables(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t dropFiles(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t updateSyscolumnAuto(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t updateSyscolumnNextvalCol(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t updateSyscolumnTablename(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t updateSystableAuto(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t updateSystableTablename(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t updateSystablesTablename(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t updateSyscolumnColumnposCol(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t fillNewColumn(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t updateSyscolumnRenameColumn(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t updateSyscolumnSetDefault(messageqcpp::ByteStream& bs, std::string & err);
+	//	EXPORT uint8_t updateSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t writeTruncateLog(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t writeDropPartitionLog(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t writeDropTableLog(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t deleteDDLLog(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t fetchDDLLog(messageqcpp::ByteStream& bs, std::string & err);
+		void purgeFDCache();
+		/** @brief drop a set of partitions
+		 *
+		 * Do many VSS lookups under one lock.
+		 * @param bs (in) bytestream carry command info
+		 * @param err (out) error message when error occurs
+		 * @return 0 on success, otherwise error.
+		 */
+		EXPORT uint8_t dropPartitions(messageqcpp::ByteStream& bs, std::string& err);
+		inline void convertRidToColumn (uint64_t& rid, uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, const int32_t oid )
+		{
+			fDbrm.getSysCatDBRoot(oid, dbRoot);
+			partition = rid / ( filesPerColumnPartition * extentsPerSegmentFile * extentRows );
+		
+			segment =( ( ( rid % ( filesPerColumnPartition * extentsPerSegmentFile * extentRows )) / extentRows ) ) % filesPerColumnPartition;
+		
+			//Calculate the relative rid for this segment file
+			uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
+			assert ( relRidInPartition <= (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows );
+			uint32_t numExtentsInThisPart = relRidInPartition / extentRows;
+			unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition;
+			uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows;
+			rid = relRidInThisExtent +  numExtentsInThisSegPart * extentRows;
+		}
+
+	private:	
+		WriteEngineWrapper fWEWrapper;
+		BRM::DBRM fDbrm;
+		unsigned  extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
+	
+};
+}
+#undef EXPORT
+#endif

writeengine/server/we_ddlcommon.h

@@ -0,0 +1,521 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id$
+*
+*******************************************************************************/
+
+
+#ifndef WE_DDLCOMMON_H
+#define WE_DDLCOMMON_H
+
+#include <string>
+#include <stdexcept>
+#include <sstream>
+#include <iostream>
+#include <stdint.h>
+
+#include <boost/any.hpp>
+#include <boost/tuple/tuple.hpp>
+
+#include "calpontsystemcatalog.h"
+#include "objectidmanager.h"
+#include "sessionmanager.h"
+#include "ddlpkg.h"
+#include "messageobj.h"
+#include "we_type.h"
+#include "we_define.h"
+#include "writeengine.h"
+#include "columnresult.h"
+#include "brmtypes.h"
+#include "joblist.h"
+
+#if defined(_MSC_VER) && defined(xxxDDLPKGPROC_DLLEXPORT)
+#define EXPORT __declspec(dllexport)
+#else
+#define EXPORT
+#endif
+
+using namespace std;
+using namespace execplan;
+
+#include <boost/algorithm/string/case_conv.hpp>
+using namespace boost::algorithm;
+
+template <class T>
+bool from_string(T& t,
+                 const std::string& s,
+                 std::ios_base& (*f)(std::ios_base&))
+{
+    std::istringstream iss(s);
+    return !(iss >> f >> t).fail();
+}
+
+namespace WriteEngine
+{
+	struct DDLColumn
+    {
+        execplan::CalpontSystemCatalog::OID oid;
+        execplan::CalpontSystemCatalog::ColType colType;
+        execplan:: CalpontSystemCatalog::TableColName tableColName;
+    };
+	
+	typedef std::vector<DDLColumn> ColumnList;
+	
+	struct DictOID
+    {
+        int dictOID;
+        int listOID;
+        int treeOID;
+		int colWidth;
+		int compressionType;
+    };
+	
+	struct extentInfo
+	{
+		uint16_t dbRoot;
+		uint32_t partition;
+		uint16_t segment;
+		bool operator==(const extentInfo& rhs) const { return (dbRoot == rhs.dbRoot && partition == rhs.partition && segment == rhs.segment); }
+		bool operator!=(const extentInfo& rhs) const { return !(*this == rhs); }
+	};
+inline void  getColumnsForTable(uint32_t sessionID, std::string schema,std::string table,
+		ColumnList& colList)
+{
+
+	CalpontSystemCatalog::TableName tableName;
+	tableName.schema = schema;
+	tableName.table = table;
+	std::string err;
+	try
+	{
+		boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID);
+		systemCatalogPtr->identity(CalpontSystemCatalog::EC);
+
+		const CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName);
+
+		CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin();
+		while (rid_iterator != ridList.end())
+		{
+			CalpontSystemCatalog::ROPair roPair = *rid_iterator;
+
+			DDLColumn column;
+			column.oid = roPair.objnum;
+			column.colType = systemCatalogPtr->colType(column.oid);
+			column.tableColName = systemCatalogPtr->colName(column.oid);
+
+			colList.push_back(column);
+
+			++rid_iterator;
+		}
+
+	}
+	catch (exception& ex)
+	{
+
+		err = "DDLPackageProcessor::getColumnsForTable: while reading columns for table " +  schema + '.' + table + ": " + ex.what();
+		throw std::runtime_error(err);
+	}
+	catch (...)
+	{
+		err = "DDLPackageProcessor::getColumnsForTable: caught unkown exception!" ;
+		throw std::runtime_error(err);
+	}
+
+}
+
+inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColType& colType)
+{
+	boost::any value;
+	switch(colType.colDataType)
+	{
+		case execplan::CalpontSystemCatalog::BIT:
+			break;
+
+		case execplan::CalpontSystemCatalog::TINYINT:
+			{
+				char tinyintvalue = joblist::TINYINTNULL;
+				value = tinyintvalue;
+
+			}
+			break;
+
+		case execplan::CalpontSystemCatalog::UTINYINT:
+			{
+				uint8_t tinyintvalue = joblist::UTINYINTNULL;
+				value = tinyintvalue;
+
+			}
+			break;
+
+		case execplan::CalpontSystemCatalog::SMALLINT:
+			{
+				short smallintvalue = joblist::SMALLINTNULL;
+				value = smallintvalue;
+			}
+			break;
+
+		case execplan::CalpontSystemCatalog::USMALLINT:
+			{
+				uint16_t smallintvalue = joblist::USMALLINTNULL;
+				value = smallintvalue;
+			}
+			break;
+
+		case execplan::CalpontSystemCatalog::MEDINT:
+		case execplan::CalpontSystemCatalog::INT:
+			{
+				int intvalue = joblist::INTNULL;
+				value = intvalue;
+			}
+			break;
+
+		case execplan::CalpontSystemCatalog::UMEDINT:
+		case execplan::CalpontSystemCatalog::UINT:
+			{
+				uint32_t intvalue = joblist::UINTNULL;
+				value = intvalue;
+			}
+			break;
+
+		case execplan::CalpontSystemCatalog::BIGINT:
+			{
+				long long bigint = joblist::BIGINTNULL;
+				value = bigint;
+			}
+			break;
+
+        case execplan::CalpontSystemCatalog::UBIGINT:
+            {
+                uint64_t bigint = joblist::UBIGINTNULL;
+                value = bigint;
+            }
+            break;
+
+        case execplan::CalpontSystemCatalog::DECIMAL:
+		case execplan::CalpontSystemCatalog::UDECIMAL:
+			{
+				if (colType.colWidth <= execplan::CalpontSystemCatalog::FOUR_BYTE)
+				{
+					short smallintvalue = joblist::SMALLINTNULL;
+					value = smallintvalue;
+				}
+				else if (colType.colWidth <= 9)
+				{
+					int intvalue = joblist::INTNULL;
+					value = intvalue;
+				}
+				else if (colType.colWidth <= 18)
+				{
+					long long eightbyte = joblist::BIGINTNULL;
+					value = eightbyte;
+				}
+				else
+				{
+					WriteEngine::Token nullToken;
+					value = nullToken;
+				}
+			}
+			break;
+		case execplan::CalpontSystemCatalog::FLOAT:
+		case execplan::CalpontSystemCatalog::UFLOAT:
+			{
+				uint32_t jlfloatnull = joblist::FLOATNULL;
+				float* fp = reinterpret_cast<float*>(&jlfloatnull);
+				value = *fp;
+			}
+			break;
+
+		case execplan::CalpontSystemCatalog::DOUBLE:
+		case execplan::CalpontSystemCatalog::UDOUBLE:
+			{
+				uint64_t jldoublenull = joblist::DOUBLENULL;
+				double* dp = reinterpret_cast<double*>(&jldoublenull);
+				value = *dp;
+			}
+			break;
+
+		case execplan::CalpontSystemCatalog::DATE:
+			{
+				int d = joblist::DATENULL;
+				value = d;
+			}
+			break;
+
+		case execplan::CalpontSystemCatalog::DATETIME:
+			{
+				long long d = joblist::DATETIMENULL;
+				value = d;
+			}
+			break;
+
+		case execplan::CalpontSystemCatalog::CHAR:
+			{
+				std::string charnull;
+				if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE)
+				{
+					//charnull = joblist::CHAR1NULL;
+					charnull = "\376";
+					value = charnull;
+				}
+				else if (colType.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE)
+				{
+					//charnull = joblist::CHAR2NULL;
+					charnull = "\377\376";
+					value = charnull;
+				}
+				else if (colType.colWidth <= execplan::CalpontSystemCatalog::FOUR_BYTE)
+				{
+					//charnull = joblist::CHAR4NULL;
+					charnull = "\377\377\377\376";
+					value = charnull;
+				}
+				else
+				{
+					WriteEngine::Token nullToken;
+					value = nullToken;
+				}
+
+			}
+			break;
+		case execplan::CalpontSystemCatalog::VARCHAR:
+			{
+				std::string charnull;
+				if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE)
+				{
+					//charnull = joblist::CHAR2NULL;
+					charnull = "\377\376";
+					value = charnull;
+				}
+				else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE)
+				{
+					//charnull = joblist::CHAR4NULL;
+					charnull = "\377\377\377\376";
+					value = charnull;
+				}
+				else
+				{
+					WriteEngine::Token nullToken;
+					value = nullToken;
+				}
+
+			}
+			break;
+		case execplan::CalpontSystemCatalog::VARBINARY:
+			{
+				std::string charnull;
+				if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE)
+				{
+					//charnull = joblist::CHAR2NULL;
+					charnull = "\377\376";
+					value = charnull;
+				}
+				else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE)
+				{
+					//charnull = joblist::CHAR4NULL;
+					charnull = "\377\377\377\376";
+					value = charnull;
+				}
+				else
+				{
+					WriteEngine::Token nullToken;
+					value = nullToken;
+				}
+
+			}
+			break;
+
+
+		default:
+			throw std::runtime_error("getNullValueForType: unkown column data type");
+			break;
+
+	}
+
+	return value;
+}
+
+inline int convertDataType(int dataType)
+{
+	int calpontDataType;
+
+	switch (dataType)
+	{
+		case ddlpackage::DDL_CHAR:
+			calpontDataType = CalpontSystemCatalog::CHAR;
+			break;
+
+		case ddlpackage::DDL_VARCHAR:
+			calpontDataType = CalpontSystemCatalog::VARCHAR;
+			break;
+
+		case ddlpackage::DDL_VARBINARY:
+			calpontDataType = CalpontSystemCatalog::VARBINARY;
+			break;
+
+		case ddlpackage::DDL_BIT:
+			calpontDataType = CalpontSystemCatalog::BIT;
+			break;
+
+		case ddlpackage::DDL_REAL:
+		case ddlpackage::DDL_DECIMAL:
+		case ddlpackage::DDL_NUMERIC:
+		case ddlpackage::DDL_NUMBER:
+			calpontDataType = CalpontSystemCatalog::DECIMAL;
+			break;
+
+		case ddlpackage::DDL_FLOAT:
+			calpontDataType = CalpontSystemCatalog::FLOAT;
+			break;
+
+		case ddlpackage::DDL_DOUBLE:
+			calpontDataType = CalpontSystemCatalog::DOUBLE;
+			break;
+
+		case ddlpackage::DDL_INT:
+		case ddlpackage::DDL_INTEGER:
+			calpontDataType = CalpontSystemCatalog::INT;
+			break;
+
+		case ddlpackage::DDL_BIGINT:
+			calpontDataType = CalpontSystemCatalog::BIGINT;
+			break;
+
+		case ddlpackage::DDL_MEDINT:
+			calpontDataType = CalpontSystemCatalog::MEDINT;
+			break;
+
+		case ddlpackage::DDL_SMALLINT:
+			calpontDataType = CalpontSystemCatalog::SMALLINT;
+			break;
+
+		case ddlpackage::DDL_TINYINT:
+			calpontDataType = CalpontSystemCatalog::TINYINT;
+			break;
+
+		case ddlpackage::DDL_DATE:
+			calpontDataType = CalpontSystemCatalog::DATE;
+			break;
+
+		case ddlpackage::DDL_DATETIME:
+			calpontDataType = CalpontSystemCatalog::DATETIME;
+			break;
+
+		case ddlpackage::DDL_CLOB:
+			calpontDataType = CalpontSystemCatalog::CLOB;
+			break;
+
+		case ddlpackage::DDL_BLOB:
+			calpontDataType = CalpontSystemCatalog::BLOB;
+			break;
+
+        case ddlpackage::DDL_UNSIGNED_TINYINT:
+            calpontDataType = CalpontSystemCatalog::UTINYINT;
+            break;
+
+        case ddlpackage::DDL_UNSIGNED_SMALLINT:
+            calpontDataType = CalpontSystemCatalog::USMALLINT;
+            break;
+
+        case ddlpackage::DDL_UNSIGNED_INT:
+            calpontDataType = CalpontSystemCatalog::UINT;
+            break;
+
+        case ddlpackage::DDL_UNSIGNED_BIGINT:
+            calpontDataType = CalpontSystemCatalog::UBIGINT;
+            break;
+
+        case ddlpackage::DDL_UNSIGNED_DECIMAL:
+        case ddlpackage::DDL_UNSIGNED_NUMERIC:
+            calpontDataType = CalpontSystemCatalog::UDECIMAL;
+            break;
+
+        case ddlpackage::DDL_UNSIGNED_FLOAT:
+            calpontDataType = CalpontSystemCatalog::UFLOAT;
+            break;
+
+        case ddlpackage::DDL_UNSIGNED_DOUBLE:
+            calpontDataType = CalpontSystemCatalog::UDOUBLE;
+            break;
+
+        default:
+			throw runtime_error("Unsupported datatype!");
+
+	}
+
+	return calpontDataType;
+}
+
+inline void findColumnData(uint32_t sessionID, execplan::CalpontSystemCatalog::TableName& systableName,
+		const std::string& colName, DDLColumn& sysCol)
+{
+	ColumnList columns;
+	ColumnList::const_iterator column_iterator;
+	std::string err;
+	try
+	{
+		getColumnsForTable(sessionID, systableName.schema,systableName.table, columns);
+		column_iterator = columns.begin();
+		while (column_iterator != columns.end())
+		{
+			sysCol = *column_iterator;
+			boost::to_lower(sysCol.tableColName.column);
+
+			if (colName == sysCol.tableColName.column)
+			{
+				break;
+			}
+			++column_iterator;
+		}
+	}
+	catch (exception& ex)
+	{
+		err = ex.what();
+		throw std::runtime_error(err);
+	}
+	catch (...)
+	{
+		err = "findColumnData:Unknown exception caught";
+		throw std::runtime_error(err);
+	}
+}
+
+inline void convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition,
+				unsigned& segment, unsigned filesPerColumnPartition,
+				unsigned  extentsPerSegmentFile, unsigned extentRows,
+				unsigned startDBRoot, unsigned dbrootCnt)
+{
+	partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows);
+
+	segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) % filesPerColumnPartition;
+
+	dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1;
+
+	//Calculate the relative rid for this segment file
+	uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
+	idbassert(relRidInPartition <= (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
+	uint32_t numExtentsInThisPart = relRidInPartition / extentRows;
+	unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition;
+	uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows;
+	rid = relRidInThisExtent +  numExtentsInThisSegPart * extentRows;
+}
+
+}
+#undef EXPORT
+#endif
+

writeengine/server/we_dmlcommandproc.h

@@ -0,0 +1,136 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id: we_ddlcommandproc.h 3043 2011-08-29 22:03:03Z chao $
+*
+*******************************************************************************/
+#ifndef WE_DMLCOMMANDPROC_H__
+#define WE_DMLCOMMANDPROC_H__
+
+#include <unistd.h>
+#include <boost/scoped_ptr.hpp>
+#include "bytestream.h"
+#include "we_messages.h"
+#include "dbrm.h"
+#include "we_message_handlers.h"
+#include "calpontdmlpackage.h"
+#include "updatedmlpackage.h"
+#include "calpontsystemcatalog.h"
+#include "insertdmlpackage.h"
+#include "liboamcpp.h"
+#include <boost/date_time/gregorian/gregorian.hpp>
+#include "dataconvert.h"
+#include "writeengine.h"
+#include "we_convertor.h"
+#include "we_dbrootextenttracker.h"
+#include "we_rbmetawriter.h"
+#include "rowgroup.h"
+#include "we_log.h"
+
+#if defined(_MSC_VER) && defined(xxxDDLPKGPROC_DLLEXPORT)
+#define EXPORT __declspec(dllexport)
+#else
+#define EXPORT
+#endif
+
+namespace WriteEngine
+{
+
+class WE_DMLCommandProc
+{
+	public:
+	typedef std::vector<std::string> ColValues;
+	
+		EXPORT WE_DMLCommandProc();
+		EXPORT WE_DMLCommandProc(const WE_DMLCommandProc& rhs);
+		EXPORT ~WE_DMLCommandProc();
+		inline void isFirstBatchPm (bool firstBatch)
+		{
+			fIsFirstBatchPm = firstBatch;
+		}
+		
+		inline bool isFirstBatchPm ()
+		{
+			return fIsFirstBatchPm;
+		}
+		
+		//Convert rid from logical block rid to file relative rid
+		inline void convertToRelativeRid (uint64_t& rid, const uint8_t extentNum, const uint16_t blockNum)
+		{
+			rid = rid + extentNum*extentRows + blockNum * 8192;
+		}
+	
+		EXPORT uint8_t processSingleInsert(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t processBatchInsert(messageqcpp::ByteStream& bs, std::string & err, ByteStream::quadbyte & PMId);
+		EXPORT uint8_t commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t rollbackBatchAutoOff(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t processBatchInsertHwm(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t processUpdate(messageqcpp::ByteStream& bs, std::string & err, ByteStream::quadbyte & PMId, uint64_t& blocksChanged);
+		EXPORT uint8_t processUpdate1(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t processFlushFiles(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t processDelete(messageqcpp::ByteStream& bs, std::string & err, ByteStream::quadbyte & PMId, uint64_t& blocksChanged);
+		EXPORT uint8_t processRemoveMeta(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t processBulkRollback(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t processBulkRollbackCleanup(messageqcpp::ByteStream& bs, std::string & err);
+		EXPORT uint8_t updateSyscolumnNextval(ByteStream& bs, std::string & err);
+		EXPORT uint8_t processPurgeFDCache(ByteStream& bs, std::string & err);
+		EXPORT uint8_t processEndTransaction(ByteStream& bs, std::string & err);
+		EXPORT uint8_t processFixRows(ByteStream& bs, std::string & err, ByteStream::quadbyte & PMId);
+		int validateColumnHWMs(
+				execplan::CalpontSystemCatalog::RIDList& ridList,
+				boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr,
+				const std::vector<DBRootExtentInfo>& segFileInfo,
+				const char* stage );
+	private:	
+		WriteEngineWrapper fWEWrapper;
+		boost::scoped_ptr<RBMetaWriter> fRBMetaWriter;
+		std::vector<boost::shared_ptr<DBRootExtentTracker> >   dbRootExtTrackerVec;
+		inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType colType )
+		{
+			if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) 
+				|| ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) 
+				|| ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18))
+				|| ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 18))
+				|| (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)) 
+			{
+				return true;
+			}
+			else
+				return false;
+		}
+		uint8_t processBatchInsertHwmFlushChunks(uint32_t tableOID, int txnID,
+			const std::vector<BRM::FileInfo>& files,
+			const std::vector<BRM::OID_t>& oidsToFlush,
+			std::string& err);
+
+		bool fIsFirstBatchPm;
+		std::map<uint32_t,rowgroup::RowGroup *> rowGroups;
+		std::map<uint32_t, dmlpackage::UpdateDMLPackage> cpackages;
+		BRM::DBRM fDbrm;
+		unsigned  extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
+		Log fLog;
+};
+
+}
+#undef EXPORT
+#endif

writeengine/server/we_getfilesizes.cpp

@@ -0,0 +1,359 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*****************************************************************************
+ * $Id: we_getfilesizes.cpp 4450 2013-01-21 14:13:24Z rdempsey $
+ *
+ ****************************************************************************/
+#include "we_getfilesizes.h"
+
+#include <iostream>
+#include <stdexcept>
+using namespace std;
+
+#include "calpontsystemcatalog.h"
+using namespace execplan;
+
+#include "threadpool.h"
+using namespace threadpool;
+
+#include "bytestream.h"
+using namespace messageqcpp;
+#include "we_fileop.h"
+
+#include "idbcompress.h"
+using namespace compress;
+
+#include "IDBFileSystem.h"
+#include "IDBPolicy.h"
+using namespace idbdatafile;
+
+namespace WriteEngine
+{
+struct FileInfo                             
+{
+        uint32_t      partition;          /** @brief Partition for a file*/
+        uint16_t      segment;            /** @brief Segment for a file */
+        uint16_t      dbRoot;             /** @brief DbRoot for a file */
+        std::string    segFileName;        /** @brief seg file path */
+		double  	   fileSize;		   /** @brief seg file size in giga bytes */
+		void serialize(messageqcpp::ByteStream& bs)
+		{
+			bs << partition;
+			bs << segment;
+			bs << dbRoot;
+			bs << segFileName;
+			bs << (*(uint64_t*)(&fileSize));
+		}
+};
+typedef std::vector<FileInfo> Files;
+typedef std::map<uint32_t, Files> columnMap;
+typedef std::map<int, columnMap*> allColumnMap;
+allColumnMap wholeMap;
+boost::mutex columnMapLock;	
+ActiveThreadCounter *activeThreadCounter;
+
+size_t readFillBuffer(
+	idbdatafile::IDBDataFile* pFile,
+	char*   buffer,
+	size_t  bytesReq)
+{
+	char*   pBuf = buffer;
+	ssize_t nBytes;
+	size_t  bytesToRead = bytesReq;
+	size_t  totalBytesRead = 0;
+
+	while (1)
+	{
+		nBytes = pFile->read(pBuf, bytesToRead);
+		if (nBytes > 0)
+			totalBytesRead += nBytes;
+		else
+			break;
+
+		if ((size_t)nBytes == bytesToRead)
+			break;
+
+		pBuf        += nBytes;
+		bytesToRead  =  bytesToRead - (size_t)nBytes;
+	}
+
+	return totalBytesRead;
+}
+
+off64_t getCompressedDataSize(string& fileName)
+{
+		off64_t dataSize = 0;
+		IDBDataFile* pFile = 0;
+		size_t nBytes;
+		// Some IDBPolicy functions can throw exceptions, caller will catch it
+		IDBPolicy::configIDBPolicy();
+		bool bHdfsFile = IDBPolicy::useHdfs();
+		if (bHdfsFile)
+			pFile = IDBDataFile::open(IDBDataFile::HDFS,
+				fileName.c_str(), "r", 0);
+		else
+			pFile = IDBDataFile::open(IDBDataFile::BUFFERED,
+				fileName.c_str(), "r", 0);
+		if (!pFile)
+		{
+			std::ostringstream oss;
+			oss << "Cannot open file " << fileName << " for read.";
+			throw std::runtime_error(oss.str());
+		}
+		
+		IDBCompressInterface decompressor;
+		//--------------------------------------------------------------------------
+		// Read headers and extract compression pointers
+		//--------------------------------------------------------------------------
+		char hdr1[IDBCompressInterface::HDR_BUF_LEN];
+		nBytes = readFillBuffer( pFile,hdr1,IDBCompressInterface::HDR_BUF_LEN);
+		if ( nBytes != IDBCompressInterface::HDR_BUF_LEN )
+		{
+			std::ostringstream oss;
+			oss << "Error reading first header from file " << fileName;
+			throw std::runtime_error(oss.str());
+		}
+
+		int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - IDBCompressInterface::HDR_BUF_LEN;
+		char* hdr2 = new char[ptrSecSize];
+		nBytes = readFillBuffer( pFile,hdr2,ptrSecSize);
+		if ( (int64_t)nBytes != ptrSecSize )
+		{
+			std::ostringstream oss;
+			oss << "Error reading second header from file " << fileName;
+			throw std::runtime_error(oss.str());
+		}
+
+		CompChunkPtrList chunkPtrs;
+		int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs);
+		delete[] hdr2;
+		if (rc != 0)
+		{
+			std::ostringstream oss;
+			oss << "Error decompressing second header from file " << fileName;
+			throw std::runtime_error(oss.str());
+		}
+
+		unsigned k = chunkPtrs.size();
+		// last header's offset + length will be the data bytes
+		dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second;
+		delete pFile;
+		return dataSize;
+}
+struct ColumnThread
+{
+    ColumnThread(uint32_t oid, int32_t compressionType, bool reportRealUse, int key) 
+    : fOid(oid), fCompressionType(compressionType), fReportRealUse(reportRealUse), fKey(key) 
+    {}
+    void operator()()
+    {
+		Config config;
+		config.initConfigCache();
+		std::vector<uint16_t> rootList;
+		config.getRootIdList( rootList );
+		FileOp fileOp;
+		Files aFiles;
+
+		// This function relies on IDBPolicy being initialized by
+		// IDBPolicy::init().  This is done when WriteEngineServer main() calls
+		// IDBPolicy::configIDBPolicy();
+		IDBDataFile::Types fileType;
+		bool bUsingHdfs = IDBPolicy::useHdfs();
+		if (bUsingHdfs)
+			fileType = IDBDataFile::HDFS;
+		else
+			fileType = IDBDataFile::UNBUFFERED;
+		IDBFileSystem& fs = IDBFileSystem::getFs( fileType );
+		for (uint32_t i=0; i < rootList.size(); i++)
+		{
+			std::vector<struct BRM::EMEntry> entries;
+			(void)BRMWrapper::getInstance()->getExtents_dbroot(fOid, entries, rootList[i]);
+			std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
+			while ( iter != entries.end() ) //organize extents into files
+			{	
+				//Find the size of this file
+				//string fileName;
+				char fileName[200];
+				(void)fileOp.getFileName( fOid, fileName, rootList[i], entries[0].partitionNum, entries[0].segmentNum);
+				string aFile(fileName); //convert between char* and string
+				off64_t fileSize = 0;
+				if (fReportRealUse && (fCompressionType > 0))
+				{
+					try {
+						fileSize = getCompressedDataSize(aFile);
+					}
+					catch (std::exception& ex)
+					{
+						cerr << ex.what();
+					}
+				}
+				else
+					fileSize = fs.size( fileName );
+					
+				if (fileSize > 0) // File exists, add to list
+				{
+					FileInfo aFileInfo;
+					aFileInfo.partition = entries[0].partitionNum;
+					aFileInfo.segment = entries[0].segmentNum;
+					aFileInfo.dbRoot = rootList[i];
+					aFileInfo.segFileName = aFile;
+					aFileInfo.fileSize = (double)fileSize / (1024 * 1024 * 1024);
+					aFiles.push_back(aFileInfo);
+					//cout.precision(15);
+					//cout << "The file " << aFileInfo.segFileName << " has size " << fixed << aFileInfo.fileSize << "GB" << endl;
+				}
+				
+				//erase the entries from this dbroot.
+				std::vector<struct BRM::EMEntry> entriesTrimed;
+				for (uint32_t m=0; m<entries.size(); m++)
+				{
+					if ((entries[0].partitionNum != entries[m].partitionNum) || (entries[0].segmentNum != entries[m].segmentNum))
+					entriesTrimed.push_back(entries[m]);
+				}
+				entriesTrimed.swap(entries);
+				iter = entries.begin();
+			}
+		}
+		
+		boost::mutex::scoped_lock lk(columnMapLock);
+		//cout << "Current size of columnsMap is " << columnsMap.size() << endl;
+		allColumnMap::iterator colMapiter = wholeMap.find(fKey);
+		if (colMapiter != wholeMap.end())
+		{
+			(colMapiter->second)->insert(make_pair(fOid,aFiles));	
+			activeThreadCounter->decr();
+		//cout << "Added to columnsMap aFiles with size " << aFiles.size() << " for oid " << fOid << endl;		
+		}
+	}
+	uint32_t fOid;
+	int32_t fCompressionType;
+	bool fReportRealUse;
+	int fKey;
+};
+//------------------------------------------------------------------------------
+// Process a table size based on input from the
+// bytestream object.
+//------------------------------------------------------------------------------
+int WE_GetFileSizes::processTable(
+	messageqcpp::ByteStream& bs,
+	std::string& errMsg, int key)
+{
+	uint8_t rc = 0;
+	errMsg.clear();
+
+	try {
+		std::string aTableName;
+		std::string schemaName;
+		bool reportRealUse = false;
+		ByteStream::byte tmp8;
+		bs >> schemaName;
+		//cout << "schema: "<< schemaName << endl;
+
+		bs >> aTableName;
+		//cout << "tableName: " << aTableName << endl;
+		bs >> tmp8;
+		reportRealUse = (tmp8 != 0);
+		
+		//get column oids
+		boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(0);
+		CalpontSystemCatalog::TableName tableName;
+		tableName.schema = schemaName;
+		tableName.table = aTableName;
+		CalpontSystemCatalog::RIDList columnList = systemCatalogPtr->columnRIDs(tableName);
+		CalpontSystemCatalog::ColType colType;
+		CalpontSystemCatalog::DictOIDList dictOidList = systemCatalogPtr->dictOIDs(tableName);
+		int serverThreads = 20;
+		int serverQueueSize = serverThreads * 100;
+		threadpool::ThreadPool tp(serverThreads,serverQueueSize);
+		int totalSize = columnList.size() + dictOidList.size();
+		activeThreadCounter = new ActiveThreadCounter(totalSize);
+		
+		columnMap *columnsMap = new columnMap();
+		{	
+			boost::mutex::scoped_lock lk(columnMapLock);
+			wholeMap[key] = columnsMap;
+		}
+		for (uint32_t i=0; i < columnList.size(); i++)
+		{
+			colType = systemCatalogPtr->colType(columnList[i].objnum);
+			tp.invoke(ColumnThread(columnList[i].objnum, colType.compressionType, reportRealUse, key));	
+			if (colType.ddn.dictOID > 0)
+				tp.invoke(ColumnThread(colType.ddn.dictOID, colType.compressionType, reportRealUse, key));	
+		}
+	/*	for (uint32_t i=0; i < dictOidList.size(); i++)
+		{
+			tp.invoke(ColumnThread(dictOidList[i].dictOID));		
+		} */
+		//check whether all threads finish
+		int sleepTime = 100; // sleep 100 milliseconds between checks
+		struct timespec rm_ts;
+
+		rm_ts.tv_sec = sleepTime/1000;
+		rm_ts.tv_nsec = sleepTime%1000 *1000000;
+		uint32_t currentActiveThreads = 10;
+		while (currentActiveThreads > 0)
+		{
+#ifdef _MSC_VER
+			Sleep(sleepTime);
+#else
+			struct timespec abs_ts;
+			do
+			{
+				abs_ts.tv_sec = rm_ts.tv_sec;
+				abs_ts.tv_nsec = rm_ts.tv_nsec;
+			}
+			while(nanosleep(&abs_ts,&rm_ts) < 0);
+#endif
+			currentActiveThreads = activeThreadCounter->cur();
+		}
+	}
+	catch(std::exception& ex)
+	{
+		//cout << "WE_GetFileSizes got exception-" << ex.what() <<
+		//	std::endl;
+		errMsg = ex.what();
+		rc     = 1;
+	}
+	//Build the message to send to the caller
+	bs.reset();
+	boost::mutex::scoped_lock lk(columnMapLock);
+	allColumnMap::iterator colMapiter = wholeMap.find(key);
+	if (colMapiter != wholeMap.end())
+	{
+		columnMap::iterator iter = colMapiter->second->begin();
+		uint64_t size;
+		Files::iterator it;
+		while ( iter != colMapiter->second->end())
+		{
+			bs << iter->first;
+			//cout << "processTable::coloid = " << iter->first << endl;
+
+			size = iter->second.size();
+			bs << size;
+			for (it = iter->second.begin(); it != iter->second.end(); it++)
+				it->serialize(bs);
+			//cout << "length now is " << bs.length() << endl;
+			iter++;
+		}
+		wholeMap.erase(colMapiter);
+	}
+	return rc;
+}
+
+
+}

writeengine/server/we_getfilesizes.h

@@ -0,0 +1,78 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id: we_getfilesizes.h 4450 2013-04-15 14:13:24Z chao $
+*
+*******************************************************************************/
+#ifndef WE_GETFILESIZES_H__
+#define WE_GETFILESIZES_H__
+
+#include <unistd.h>
+#include <string>
+
+#include "bytestream.h"
+
+#include "atomicops.h"
+
+namespace WriteEngine {
+
+/** @brief Get all file sizes for the given table
+ */
+class WE_GetFileSizes
+{
+public:
+	
+	static int processTable(messageqcpp::ByteStream& bs, std::string& errMsg, int key);
+
+};
+
+class ActiveThreadCounter
+{
+public:
+	ActiveThreadCounter(int size) : factiveThreadCount(size){}
+	virtual ~ActiveThreadCounter() {}
+
+	void decr()
+	{
+		int atc;
+		for (;;) {
+			atomicops::atomicMb();
+			atc = factiveThreadCount;
+			if (atc <= 0)		//hopefully atc will never be < 0!
+				return;
+			if (atomicops::atomicCAS(&factiveThreadCount, atc, (atc - 1)))
+				return;
+			atomicops::atomicYield();
+		}
+	}
+
+	uint32_t cur() 
+	{ 
+		return factiveThreadCount; 
+	}
+
+private:
+	ActiveThreadCounter(const ActiveThreadCounter& rhs);
+	ActiveThreadCounter& operator=(const ActiveThreadCounter& rhs);
+
+	volatile int32_t factiveThreadCount;
+};
+
+}
+
+#endif

writeengine/server/we_message_handlers.h

@@ -0,0 +1,36 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id: we_message_handlers.h 4450 2013-01-21 14:13:24Z rdempsey $
+*
+*******************************************************************************/
+#ifndef WE_MESSAGE_HANDLERS_H__
+#define WE_MESSAGE_HANDLERS_H__
+
+#include "bytestream.h"
+
+namespace WriteEngine
+{
+
+extern messageqcpp::ByteStream::byte doMsg1(messageqcpp::ByteStream& bs, std::string err);
+extern messageqcpp::ByteStream::byte doMsg2(messageqcpp::ByteStream& bs, std::string err);
+extern messageqcpp::ByteStream::byte doUpdateSyscolumnNextval(messageqcpp::ByteStream& bs, std::string err);
+}
+
+#endif
+

writeengine/server/we_messages.h

@@ -0,0 +1,118 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id: we_messages.h 4609 2013-04-19 15:32:02Z chao $
+*
+*******************************************************************************/
+#ifndef WE_MESSAGES_H__
+#define WE_MESSAGES_H__
+
+namespace WriteEngine
+{
+
+enum ServerMessages
+{
+	WE_SVR_LOOPBACK,
+	WE_SVR_DDL_KEEPALIVE,
+	WE_SVR_DML_KEEPALIVE,
+	WE_UPDATE_NEXTVAL,
+	WE_SVR_WRITE_SYSTABLE,
+	WE_SVR_WRITE_SYSCOLUMN,
+	WE_SVR_WRITE_CREATETABLEFILES,
+	WE_SVR_COMMIT_VERSION,
+	WE_SVR_ROLLBACK_BLOCKS,
+	WE_SVR_ROLLBACK_VERSION,
+	WE_SVR_DELETE_SYSTABLE,
+	WE_SVR_DELETE_SYSTABLES,
+	WE_SVR_DELETE_SYSCOLUMN,
+	WE_SVR_DELETE_SYSCOLUMN_ROW,
+	WE_SVR_WRITE_DROPFILES,
+	WE_SVR_UPDATE_SYSTABLE_AUTO,
+	WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL,
+	WE_SVR_UPDATE_SYSCOLUMN_TABLENAME,
+	WE_SVR_UPDATE_SYSCOLUMN_AUTO,
+	WE_SVR_UPDATE_SYSCOLUMN_COLPOS,
+	WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN,
+	WE_SVR_UPDATE_SYSTABLE_TABLENAME,
+	WE_SVR_UPDATE_SYSTABLES_TABLENAME,
+	WE_SVR_DROP_PARTITIONS,
+	WE_SVR_SINGLE_INSERT,
+	WE_SVR_BATCH_KEEPALIVE,
+	WE_SVR_BATCH_INSERT,
+	WE_SVR_BATCH_INSERT_END,
+	WE_SVR_COMMIT_BATCH_AUTO_ON,
+	WE_SVR_ROLLBACK_BATCH_AUTO_ON,
+	WE_SVR_COMMIT_BATCH_AUTO_OFF,
+	WE_SVR_ROLLBACK_BATCH_AUTO_OFF,
+	WE_SVR_BATCH_AUTOON_REMOVE_META, 
+	WE_SVR_UPDATE, //35
+	WE_SVR_FLUSH_FILES,
+	WE_SVR_DELETE,
+	WE_SVR_DML_BULKROLLBACK,
+	WE_SVR_DML_BULKROLLBACK_CLEANUP,
+	WE_SVR_FILL_COLUMN,
+	WE_SVR_WRITE_TRUNCATE,
+	WE_SVR_WRITE_DROPPARTITION,
+	WE_SVR_WRITE_DROPTABLE,
+	WE_SVR_DELETE_DDLLOG,
+	WE_SVR_FETCH_DDL_LOGS,
+	WE_SVR_REMOVE_TABLEDATA,
+	WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL,
+	WE_SVR_REDISTRIBUTE,
+	WE_SVR_CLOSE_CONNECTION,
+	WE_SVR_GET_FILESIZES,
+	WE_SVR_PURGEFD,
+	WE_END_TRANSACTION,
+	WE_SRV_FIX_ROWS,
+	WE_SVR_WRITE_CREATE_SYSCOLUMN,
+
+	WE_CLT_SRV_DATA=100,
+	WE_CLT_SRV_EOD,
+	WE_CLT_SRV_CMD,
+	WE_CLT_SRV_ACK,
+	WE_CLT_SRV_NAK,
+	WE_CLT_SRV_PM_ERROR,
+	WE_CLT_SRV_KEEPALIVE,
+	WE_CLT_SRV_IMPFILENAME,
+	WE_CLT_SRV_IMPFILEERROR,
+	WE_CLT_SRV_CMDLINEARGS,
+	WE_CLT_SRV_STARTCPI,
+	WE_CLT_SRV_CPIPASS,
+	WE_CLT_SRV_CPIFAIL,
+	WE_CLT_SRV_ROLLBACK,
+	WE_CLT_SRV_CLEANUP,
+	WE_CLT_SRV_DATARQST,
+	WE_CLT_SRV_MODE,
+	WE_CLT_SRV_DBRCNT,
+	WE_CLT_SRV_BRMRPT,
+	WE_CLT_SRV_JOBID,
+	WE_CLT_SRV_JOBDATA,
+	WE_CLT_SRV_ERRLOG,
+	WE_CLT_SRV_BADLOG,
+	WE_CLT_SRV_CLEAR_TABLE_LOCK,
+	WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP
+
+};
+
+
+
+
+}
+
+#endif
+

writeengine/server/we_msg1.cpp

@@ -0,0 +1,40 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id$
+*
+*******************************************************************************/
+
+#include <unistd.h>
+
+#include "bytestream.h"
+using namespace messageqcpp;
+
+#include "we_messages.h"
+#include "we_message_handlers.h"
+
+namespace WriteEngine
+{
+
+ByteStream::byte doMsg1(ByteStream& bs, std::string err)
+{
+	return 0;
+}
+
+}
+

writeengine/server/we_msg2.cpp

@@ -0,0 +1,40 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id$
+*
+*******************************************************************************/
+
+#include <unistd.h>
+
+#include "bytestream.h"
+using namespace messageqcpp;
+
+#include "we_messages.h"
+#include "we_message_handlers.h"
+
+namespace WriteEngine
+{
+
+ByteStream::byte doMsg2(ByteStream& bs, std::string err)
+{
+	return 0;
+}
+
+}
+

writeengine/server/we_observer.cpp

@@ -0,0 +1,94 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id$
+*
+*******************************************************************************/
+
+/*
+ * Observer.cpp
+ *
+ *  Created on: Oct 5, 2011
+ *      Author: bpaul@calpont.com
+ */
+
+#include <boost/thread/mutex.hpp>
+
+#include "we_observer.h"
+
+
+namespace WriteEngine {
+
+//-----------------------------------------------------------------------------
+
+//	ctor
+Observer::Observer() {
+
+}
+
+//-----------------------------------------------------------------------------
+//dtor
+Observer::~Observer() {
+	//
+}
+
+//-----------------------------------------------------------------------------
+//ctor
+Subject::Subject(){
+
+}
+//-----------------------------------------------------------------------------
+//dtor
+Subject::~Subject(){
+
+}
+
+//-----------------------------------------------------------------------------
+
+void Subject::attach(Observer* Obs) {
+	boost::mutex::scoped_lock aLstLock;
+	fObs.push_back(Obs);
+}
+
+//-----------------------------------------------------------------------------
+
+void Subject::detach(Observer* Obs) {
+	boost::mutex::scoped_lock aLstLock;
+	Observers::iterator aIt = fObs.begin();
+	while (aIt != fObs.end()) {
+		if ((*aIt) == Obs) {
+			fObs.erase(aIt);
+			break;
+		}
+	}
+}
+
+//-----------------------------------------------------------------------------
+
+void Subject::notify() {
+	boost::mutex::scoped_lock aLstLock;
+	Observers::iterator aIt = fObs.begin();
+	while (aIt != fObs.end()) {
+		(*aIt)->update(this);
+	}
+}
+
+//-----------------------------------------------------------------------------
+
+}// namespace WriteEngine
+

writeengine/server/we_observer.h

@@ -0,0 +1,68 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id$
+*
+*******************************************************************************/
+
+/*
+ * Observer.h
+ *
+ *  Created on: Oct 5, 2011
+ *      Author: bpaul@calpont.com
+ */
+
+#ifndef OBSERVER_H_
+#define OBSERVER_H_
+
+#include <list>
+using namespace std;
+
+
+namespace WriteEngine
+{
+
+class Subject; 	// forward deceleration
+
+class Observer {
+public:
+	virtual ~Observer();
+	virtual bool update(Subject* pSub)=0;
+
+protected:
+	Observer();
+};
+
+class Subject {
+public:
+	Subject();
+	virtual ~Subject();
+
+	virtual void attach(Observer* Obs);
+	virtual void detach(Observer* Obs);
+	virtual void notify();
+
+
+private:
+	typedef std::list<Observer*> Observers;
+	Observers fObs;
+};
+
+}
+
+#endif /* OBSERVER_H_ */

writeengine/server/we_readthread.cpp

@@ -0,0 +1,864 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id: we_readthread.cpp 4609 2013-04-19 15:32:02Z chao $
+*
+*******************************************************************************/
+
+#include <unistd.h>
+
+#include "messagequeue.h"
+#include "bytestream.h"
+using namespace messageqcpp;
+
+#include "threadpool.h"
+using namespace threadpool;
+
+#include "we_dataloader.h"
+#include "we_readthread.h"
+#include "we_messages.h"
+#include "we_message_handlers.h"
+#include "we_ddlcommandproc.h"
+#include "we_dmlcommandproc.h"
+#include "we_redistribute.h"
+#include "we_config.h"
+#include "stopwatch.h"
+using namespace logging;
+using namespace WriteEngine;
+//StopWatch timer;
+namespace WriteEngine
+{
+
+ReadThread::ReadThread(const IOSocket& ios): fIos(ios) 
+{
+
+}
+
+ReadThread::~ReadThread(){
+
+}
+
+void ReadThread::operator ()()
+{
+    // We should never come here
+}
+
+//-----------------------------------------------------------------------------
+//ctor
+DmlReadThread::DmlReadThread(const messageqcpp::IOSocket& ios,
+                             ByteStream& Ibs ): ReadThread(ios), fWeDMLprocessor(new WE_DMLCommandProc), fWeDDLprocessor(new WE_DDLCommandProc)
+{
+    fIbs = Ibs;
+}
+//dtor
+DmlReadThread::~DmlReadThread()
+{
+    //cout << "in DmlReadThread destructor" << endl;
+}
+
+void DmlReadThread::operator()()
+{
+    // DCH Since fIbs is a class member, there's no reason to make a copy here.
+    // Why waste the CPU? Note that Splitter thread doesn't make a copy.
+    // The only reason I can think of to make such a copy is to guarantee a
+    // strong exception, but that doesn't appear to be in play here.
+    ByteStream ibs = fIbs;
+    ByteStream obs;
+    ByteStream::byte msgId;
+    ByteStream::octbyte uniqueID;
+    ByteStream::quadbyte PMId;
+    ByteStream::byte rc = 0;
+    std::string errMsg;
+    //cout << "DmlReadThread created ..." << endl;
+    // queryStats.blocksChanged for delete/update
+    uint64_t blocksChanged = 0;
+	
+    while (ibs.length()>0)
+    {
+        try
+        {
+            errMsg.clear();
+
+            //do work here...
+            ibs >> msgId;
+			if (msgId != WE_SVR_CLOSE_CONNECTION)
+				ibs >> uniqueID;
+            //cout << "DmlReadThread " << pthread_self () << " received message id " << (uint32_t)msgId << " and bytestream length " << ibs.length() << endl;
+            switch (msgId)
+            {
+            case WE_SVR_SINGLE_INSERT:
+                {
+                    rc = fWeDMLprocessor->processSingleInsert(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_COMMIT_VERSION:
+                {
+                    rc = fWeDMLprocessor->commitVersion(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_ROLLBACK_BLOCKS:
+                {
+                    rc = fWeDMLprocessor->rollbackBlocks(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_ROLLBACK_VERSION:
+                {
+                    rc = fWeDMLprocessor->rollbackVersion(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_COMMIT_BATCH_AUTO_ON:
+                {
+                    rc = fWeDMLprocessor->commitBatchAutoOn(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_ROLLBACK_BATCH_AUTO_ON:
+                {
+                    rc = fWeDMLprocessor->rollbackBatchAutoOn(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_COMMIT_BATCH_AUTO_OFF:
+                {
+                    rc = fWeDMLprocessor->commitBatchAutoOn(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_ROLLBACK_BATCH_AUTO_OFF:
+                {
+                    rc = fWeDMLprocessor->rollbackBatchAutoOff(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_BATCH_INSERT:
+                {
+					//timer.start("processBatchInsert");
+                    rc = fWeDMLprocessor->processBatchInsert(ibs, errMsg, PMId);
+					//timer.stop("processBatchInsert");
+					//cout << "fWeDMLprocessor " << fWeDMLprocessor << " is processing batchinsert ..." << endl;
+                    break;
+                }
+            case WE_SVR_BATCH_INSERT_END:
+                {
+                    rc = fWeDMLprocessor->processBatchInsertHwm(ibs, errMsg);
+					//timer.finish();
+                    break;
+                }
+            case WE_SVR_UPDATE:
+                {
+                    rc = fWeDMLprocessor->processUpdate(ibs, errMsg, PMId, blocksChanged);
+                    break;
+                }
+            case WE_SVR_FLUSH_FILES:
+                {
+                    rc = fWeDMLprocessor->processFlushFiles(ibs, errMsg);
+                    break;  
+                }
+            case WE_SVR_DELETE:
+                {
+                    rc = fWeDMLprocessor->processDelete(ibs, errMsg, PMId, blocksChanged);
+                    break;
+                }
+            case WE_SVR_BATCH_AUTOON_REMOVE_META:
+                {
+                    rc = fWeDMLprocessor->processRemoveMeta(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_DML_BULKROLLBACK:
+                {
+                    rc = fWeDMLprocessor->processBulkRollback(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_DML_BULKROLLBACK_CLEANUP:
+                {
+                    rc = fWeDMLprocessor->processBulkRollbackCleanup(ibs, errMsg);
+                    break;
+                }
+            case WE_UPDATE_NEXTVAL:
+                {
+                    rc = fWeDMLprocessor->updateSyscolumnNextval(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_WRITE_SYSTABLE:
+                {
+                    rc = fWeDDLprocessor->writeSystable(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_WRITE_SYSCOLUMN:
+                {
+                    rc = fWeDDLprocessor->writeSyscolumn(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_WRITE_CREATE_SYSCOLUMN:
+                {
+                    rc = fWeDDLprocessor->writeCreateSyscolumn(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_WRITE_CREATETABLEFILES:
+                {
+                    rc = fWeDDLprocessor->createtablefiles(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_DELETE_SYSCOLUMN:
+                {
+                    rc = fWeDDLprocessor->deleteSyscolumn(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_DELETE_SYSCOLUMN_ROW:
+                {
+                    rc = fWeDDLprocessor->deleteSyscolumnRow(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_DELETE_SYSTABLE:
+                {
+                    rc = fWeDDLprocessor->deleteSystable(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_DELETE_SYSTABLES:
+                {
+                    rc = fWeDDLprocessor->deleteSystables(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_WRITE_DROPFILES:
+                {
+                    rc = fWeDDLprocessor->dropFiles(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_UPDATE_SYSCOLUMN_AUTO:
+                {
+                    rc = fWeDDLprocessor->updateSyscolumnAuto(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL:
+                {
+                    rc = fWeDDLprocessor->updateSyscolumnNextvalCol(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL:
+                {
+                    rc = fWeDDLprocessor->updateSyscolumnSetDefault(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_UPDATE_SYSCOLUMN_TABLENAME:
+                {
+                    rc = fWeDDLprocessor->updateSyscolumnTablename(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN:
+                {
+                    rc = fWeDDLprocessor->updateSyscolumnRenameColumn(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_UPDATE_SYSCOLUMN_COLPOS:
+                {
+                    rc = fWeDDLprocessor->updateSyscolumnColumnposCol(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_UPDATE_SYSTABLE_AUTO:
+                {
+                    rc = fWeDDLprocessor->updateSystableAuto(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_UPDATE_SYSTABLE_TABLENAME:
+                {
+                    rc = fWeDDLprocessor->updateSystableTablename(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_UPDATE_SYSTABLES_TABLENAME:
+                {
+                    rc = fWeDDLprocessor->updateSystablesTablename(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_FILL_COLUMN:
+                {
+                    rc = fWeDDLprocessor->fillNewColumn(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_DROP_PARTITIONS:
+                {
+                    rc = fWeDDLprocessor->dropPartitions(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_WRITE_TRUNCATE:
+                {
+                    rc = fWeDDLprocessor->writeTruncateLog(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_WRITE_DROPPARTITION:
+                {
+                    rc = fWeDDLprocessor->writeDropPartitionLog(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_WRITE_DROPTABLE:
+                {
+                    rc = fWeDDLprocessor->writeDropTableLog(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_DELETE_DDLLOG:
+                {
+                    rc = fWeDDLprocessor->deleteDDLLog(ibs, errMsg);
+                    break;
+                }
+            case WE_SVR_FETCH_DDL_LOGS:
+                {
+                    rc = fWeDDLprocessor->fetchDDLLog(ibs, errMsg);
+                    break;
+                }
+			case WE_SVR_PURGEFD:
+				{
+					rc = fWeDMLprocessor->processPurgeFDCache(ibs, errMsg);
+					break;
+				}
+			case WE_END_TRANSACTION:
+				{
+					rc = fWeDMLprocessor->processEndTransaction(ibs, errMsg);
+					break;
+				}
+			case WE_SRV_FIX_ROWS:
+				{
+					rc = fWeDMLprocessor->processFixRows(ibs, errMsg, PMId);
+					break;
+				}
+			case WE_SVR_CLOSE_CONNECTION:
+				{
+					break;
+				}
+            default:
+                break;
+            }
+        }
+        catch (std::exception& ex)
+        {
+            logging::LoggingID logid(19, 0, 0);
+            logging::Message::Args args;
+            logging::Message msg(1);
+            args.add("we_readthread caught exception ");
+            args.add(ex.what());
+            msg.format(args);
+            logging::Logger logger(logid.fSubsysID);
+            logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid);
+            rc = 1;
+            errMsg = msg.msg();
+        }
+        catch (...)
+        {
+            logging::LoggingID logid(19, 0, 0);
+            logging::Message::Args args;
+            logging::Message msg(1);
+            args.add("we_readthread caught ... exception ");
+            msg.format(args);
+            logging::Logger logger(logid.fSubsysID);
+            logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid);
+            rc = 1;
+            errMsg = msg.msg();
+        }  
+		
+		if (msgId != WE_SVR_CLOSE_CONNECTION) 
+		{
+			//send response
+			obs.restart();
+			obs << uniqueID;
+			obs << rc;
+			obs << errMsg;
+		}
+
+        if ((msgId == WE_SVR_COMMIT_BATCH_AUTO_ON) || (msgId ==WE_SVR_BATCH_INSERT_END) || (msgId == WE_SVR_FETCH_DDL_LOGS))
+        {
+            obs += ibs;
+            //cout << " sending back hwm info with ibs length " << endl;
+        }
+        else if ((msgId == WE_SVR_BATCH_INSERT) || (msgId == WE_SVR_UPDATE) || (msgId == WE_SVR_DELETE))
+        {
+            obs << PMId;
+        }
+        else if ((msgId == WE_SVR_DML_BULKROLLBACK) ||
+                 (msgId == WE_SVR_DML_BULKROLLBACK_CLEANUP))
+        {
+            obs << Config::getLocalModuleID();
+        }
+
+        if (msgId == WE_SVR_UPDATE || msgId == WE_SVR_DELETE)
+            obs << blocksChanged; // send stats back to DMLProc
+        blocksChanged = 0; // reset
+		if (msgId == WE_SVR_CLOSE_CONNECTION)
+		{
+			//cout << "received request. closing connection ..." << endl;
+			break;
+		}
+		else
+		{ 
+			try
+			{
+				fIos.write(obs);
+            //cout << "dmlthread sent back response for msgid " << (uint32_t)msgId << " with uniqueID:rc= "
+            //<< (uint32_t)uniqueID<<":"<< (uint32_t)rc<<" and error message is " << errMsg <<endl;
+            //get next message
+				ibs = fIos.read();
+			}
+			catch (...)
+			{
+				break;
+			}
+		}
+	}
+	//cout << "closing connection for thread " << pthread_self () << endl;
+    fIos.close();
+}
+
+
+//-----------------------------------------------------------------------------
+//ctor
+SplitterReadThread::SplitterReadThread(const messageqcpp::IOSocket& ios,
+                                       ByteStream& Ibs):ReadThread(ios), fWeDataLoader(*this)
+{
+    fIbs = Ibs;
+}
+
+//-----------------------------------------------------------------------------
+//copy ctor
+SplitterReadThread::SplitterReadThread(const SplitterReadThread& rhs): 
+ReadThread(rhs.fIos), fWeDataLoader(*this)
+{
+    fIbs = rhs.fIbs;
+
+}
+//-----------------------------------------------------------------------------
+// dtor
+SplitterReadThread::~SplitterReadThread()
+{
+}
+//-----------------------------------------------------------------------------
+/**
+* @brief Thread Function which process incoming messages
+*
+*/
+void SplitterReadThread::operator()()
+{
+    ByteStream::byte msgId;
+    while (fIbs.length()>0)
+    {
+        fWeDataLoader.updateRxBytes(fIbs.length());
+
+        //do work here...
+        fIbs >> msgId;
+
+        //cout << (int)msgId << endl;
+
+        switch (msgId)
+        {
+        case WE_CLT_SRV_KEEPALIVE:
+            {
+                fWeDataLoader.onReceiveKeepAlive(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_DATA:
+            {
+                fWeDataLoader.onReceiveData(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_EOD:
+            {
+                fWeDataLoader.onReceiveEod(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_MODE:
+            {
+                fWeDataLoader.onReceiveMode(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_IMPFILENAME:
+            {
+                fWeDataLoader.onReceiveImportFileName(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_CMDLINEARGS:
+            {
+                fWeDataLoader.onReceiveCmdLineArgs(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_CMD:
+            {
+                fWeDataLoader.onReceiveCmd(fIbs);   //fig out share_ptr on BS& is better
+                break;
+            }
+        case WE_CLT_SRV_ACK:
+            {
+                fWeDataLoader.onReceiveAck(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_NAK:
+            {
+                fWeDataLoader.onReceiveNak(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_PM_ERROR:
+            {
+                fWeDataLoader.onReceiveError(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_STARTCPI:
+            {
+                fWeDataLoader.onReceiveStartCpimport();
+                break;
+            }
+        case WE_CLT_SRV_BRMRPT:
+            {
+                fWeDataLoader.onReceiveBrmRptFileName(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_CLEANUP:
+            {
+                fWeDataLoader.onReceiveCleanup(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_ROLLBACK:
+            {
+                fWeDataLoader.onReceiveRollback(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_JOBID:
+            {
+                fWeDataLoader.onReceiveJobId(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_JOBDATA:
+            {
+                fWeDataLoader.onReceiveJobData(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_ERRLOG:
+            {
+                fWeDataLoader.onReceiveErrFileRqst(fIbs);
+                break;
+            }
+        case WE_CLT_SRV_BADLOG:
+            {
+                fWeDataLoader.onReceiveBadFileRqst(fIbs);
+                break;
+            }
+        default:
+            break;
+
+        }
+
+        fIbs.restart();
+        try
+        {   //get next message
+            fIbs = fIos.read();
+        }
+        catch (...)
+        {
+            fIbs.restart();     //setting length=0, get out of loop
+            cout << "Broken Pipe" << endl;
+
+            logging::LoggingID logid(19, 0, 0);
+            logging::Message::Args args;
+            logging::Message msg(1);
+            args.add("SplitterReadThread::operator: Broken Pipe ");
+            msg.format(args);
+            logging::Logger logger(logid.fSubsysID);
+            logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid);
+        }
+    }
+
+    fIos.close();
+
+}
+
+//------------------------------------------------------------------------------
+// ClearTableLockReadThread constructor.
+//------------------------------------------------------------------------------
+ClearTableLockReadThread::ClearTableLockReadThread(
+                                                  const messageqcpp::IOSocket& ios,
+                                                  ByteStream& Ibs ): ReadThread(ios),
+fWeClearTableLockCmd(new WE_ClearTableLockCmd("ClearTableLockTool"))
+{
+    fIbs = Ibs;
+}
+
+//------------------------------------------------------------------------------
+// ClearTableLockReadThread destructor.
+//------------------------------------------------------------------------------
+ClearTableLockReadThread::~ClearTableLockReadThread()
+{
+}
+
+//------------------------------------------------------------------------------
+// Thread entry point to ClearTableLockReadThread object used to receive msgs
+// from a cleartablelock tool client.
+//------------------------------------------------------------------------------
+void ClearTableLockReadThread::operator()()
+{
+    ByteStream::byte msgId;
+    ByteStream obs;
+    ByteStream::byte rc = 0;
+    std::string errMsg;
+
+    // Read msgid from ByteStream and forward to applicable processing function
+    while (fIbs.length() > 0)
+    {
+        fIbs >> msgId;
+        switch (msgId)
+        {
+        case WE_CLT_SRV_CLEAR_TABLE_LOCK:
+            {
+                rc = fWeClearTableLockCmd->processRollback(fIbs, errMsg);
+                break;
+            }
+        case WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP:
+            {
+                rc = fWeClearTableLockCmd->processCleanup(fIbs, errMsg);
+                break;
+            }
+        default:
+            {
+                break;
+            }
+        }
+
+        // Send response
+        obs.restart();
+        obs << rc;
+        obs << errMsg;
+
+        try
+        {
+            fIos.write(obs);
+
+            // Get next message
+            fIbs = fIos.read();
+        }
+        catch (...)
+        {
+            logging::LoggingID logid(19, 0, 0);
+            logging::Message::Args args;
+            logging::Message msg(1);
+            args.add("ClearTableLockReadThread::operator: Broken Pipe ");
+            msg.format(args);
+            logging::Logger logger(logid.fSubsysID);
+            logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid);
+            break;
+        }
+    }
+
+    fIos.close();
+}
+
+
+//------------------------------------------------------------------------------
+// RedistributeReadThread constructor.
+//------------------------------------------------------------------------------
+RedistributeReadThread::RedistributeReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs)
+: ReadThread(ios)
+{
+    fIbs = Ibs;
+}
+
+//------------------------------------------------------------------------------
+// RedistributeReadThread destructor.
+//------------------------------------------------------------------------------
+RedistributeReadThread::~RedistributeReadThread()
+{
+}
+
+//------------------------------------------------------------------------------
+// Thread entry point to RedistributeReadThread object used to receive msgs
+// from a cleartablelock tool client.
+//------------------------------------------------------------------------------
+void RedistributeReadThread::operator()()
+{
+    try
+    {
+        redistribute::Redistribute::handleRedistributeMessage(fIbs, fIos);
+    }
+    catch (...)
+    {
+        logging::LoggingID logid(19, 0, 0);
+        logging::Message::Args args;
+        logging::Message msg(1);
+        args.add("RedistributeReadThread::operator exception handled ");
+        msg.format(args);
+        logging::Logger logger(logid.fSubsysID);
+        logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid);
+    }
+
+    fIos.close();
+}
+//------------------------------------------------------------------------------
+// GetFileSizeThread constructor.
+//------------------------------------------------------------------------------
+GetFileSizeThread::GetFileSizeThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs, BRM::DBRM& dbrm)
+: ReadThread(ios), fWeGetFileSizes(new WE_GetFileSizes())
+{
+    fIbs = Ibs;
+    key = dbrm.getUnique32();
+}
+
+//------------------------------------------------------------------------------
+// GetFileSizeThread destructor.
+//------------------------------------------------------------------------------
+GetFileSizeThread::~GetFileSizeThread()
+{
+}
+
+//------------------------------------------------------------------------------
+// Thread entry point to GetFileSizeThread object used to receive msgs
+//------------------------------------------------------------------------------
+void GetFileSizeThread::operator()()
+{
+    ByteStream::byte msgId;
+    ByteStream obs;
+    ByteStream::byte rc = 0;
+    std::string errMsg;
+
+    // Read msgid from ByteStream and forward to applicable processing function
+    while (fIbs.length() > 0)
+    {
+        fIbs >> msgId;
+        switch (msgId)
+        {
+        case WE_SVR_GET_FILESIZES:
+            {
+                rc = fWeGetFileSizes->processTable(fIbs, errMsg, key);
+                break;
+            }
+        default:
+            {
+                break;
+            }
+        }
+
+        // Send response
+        obs.restart();
+        obs << rc;
+		obs << errMsg;
+        obs += fIbs;
+
+        try
+        {
+            fIos.write(obs);
+
+            // Get next message
+            fIbs = fIos.read();
+        }
+        catch (...)
+        {
+            logging::LoggingID logid(19, 0, 0);
+            logging::Message::Args args;
+            logging::Message msg(1);
+            args.add("GetFileSizeThread::operator: Broken Pipe ");
+            msg.format(args);
+            logging::Logger logger(logid.fSubsysID);
+            logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid);
+            break;
+        }
+    }
+
+    fIos.close();
+}
+
+//-----------------------------------------------------------------------------
+
+void ReadThreadFactory::CreateReadThread(ThreadPool& Tp, IOSocket& Ios, BRM::DBRM& dbrm)
+{
+    struct timespec rm_ts;
+    int sleepTime = 20000; // wait for 20 seconds
+    rm_ts.tv_sec = sleepTime/1000;
+    rm_ts.tv_nsec = sleepTime%1000 *1000000;
+    bool isTimeOut = false;
+
+    ByteStream::byte msgId;
+    ByteStream aBs;
+    try
+    {
+        aBs = Ios.read(&rm_ts, &isTimeOut);
+    }
+    catch (std::exception& ex)
+    {
+        cout<< "Handled : " << ex.what() <<endl;
+        logging::LoggingID logid(19, 0, 0);
+        logging::Message::Args args;
+        logging::Message msg(1);
+        args.add("ReadThreadFactory::CreateReadThread: read() error");
+        args.add(ex.what());
+        msg.format(args);
+        logging::Logger logger(logid.fSubsysID);
+        logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid);
+    }
+    if ((aBs.length()<=0)||(isTimeOut))
+    {
+        Ios.close();
+        return;
+    }
+
+    aBs.peek(msgId);
+
+    switch (msgId)
+    {
+    case WE_SVR_DDL_KEEPALIVE:
+    case WE_SVR_DML_KEEPALIVE:
+        {
+            DmlReadThread dmlReadThread(Ios, aBs);
+			boost::thread t(dmlReadThread);
+			//cout << "starting DML thread id " << t.get_id() << endl;
+        }
+        break;
+    case WE_CLT_SRV_KEEPALIVE:
+    case WE_CLT_SRV_MODE:
+    case WE_CLT_SRV_DATA:
+    case WE_CLT_SRV_CMD:
+    case WE_CLT_SRV_ACK:
+    case WE_CLT_SRV_NAK:
+    case WE_CLT_SRV_PM_ERROR:
+    case WE_CLT_SRV_CMDLINEARGS:
+        {
+            //SplitterReadThread aSpReadThread(Ios, aBs);
+            //fOwner.attach(reinterpret_cast<Observer*>(&(aSpReadThread.fWeDataLoader)));
+            //Tp.invoke(aSpReadThread);
+            Tp.invoke(SplitterReadThread(Ios,aBs));
+        }
+        break;
+
+    case WE_CLT_SRV_CLEAR_TABLE_LOCK:
+    case WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP:
+        {
+            ClearTableLockReadThread clearTableLockThread(Ios, aBs);
+            Tp.invoke( clearTableLockThread );
+        }
+        break;
+
+    case WE_SVR_REDISTRIBUTE:
+        {
+            RedistributeReadThread RedistributeReadThread(Ios, aBs);
+            Tp.invoke(RedistributeReadThread);
+        }
+        break;
+	case WE_SVR_GET_FILESIZES:
+        {
+            GetFileSizeThread getFileSizeThread(Ios, aBs, dbrm);
+            Tp.invoke(getFileSizeThread);
+        }
+        break;
+    default:
+        {
+            Ios.close();    // don't know who is this
+        }
+        break;
+
+    }
+
+}
+
+}
+

writeengine/server/we_readthread.h

@@ -0,0 +1,161 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id: we_readthread.h 4609 2013-04-19 15:32:02Z chao $
+*
+*******************************************************************************/
+#ifndef WE_SRV_READTHREAD_H__
+#define WE_SRV_READTHREAD_H__
+
+#include <unistd.h>
+#include <boost/shared_ptr.hpp>
+#include "messagequeue.h"
+#include "threadpool.h"
+#include "we_ddlcommandproc.h"
+using namespace threadpool;
+
+#include "we_ddlcommandproc.h"
+#include "we_dmlcommandproc.h"
+#include "we_cleartablelockcmd.h"
+#include "we_dataloader.h"
+#include "we_getfilesizes.h"
+#include "dbrm.h"
+
+namespace WriteEngine
+{
+
+
+class ReadThread
+{
+public:
+	explicit ReadThread(const messageqcpp::IOSocket& ios);
+	virtual ~ReadThread();
+
+	virtual void operator()();
+
+
+//protected:
+public:
+	messageqcpp::IOSocket fIos;
+    messageqcpp::ByteStream fIbs;
+
+private:
+	//defaults okay
+	//ReadThread(const ReadThread& rhs);
+	//ReadThread& operator=(const ReadThread& rhs);
+
+};
+
+
+class DmlReadThread: public ReadThread
+{
+public:
+	explicit DmlReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs );
+	virtual ~DmlReadThread();
+
+	virtual void operator()();
+
+
+private:
+	boost::shared_ptr<WE_DMLCommandProc> fWeDMLprocessor;
+	boost::shared_ptr<WE_DDLCommandProc> fWeDDLprocessor;
+
+};
+
+class SplitterReadThread: public ReadThread
+{
+public:
+	SplitterReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs);
+    SplitterReadThread(const SplitterReadThread& rhs);
+	virtual ~SplitterReadThread();
+
+	virtual void operator()();
+
+
+private:
+    //WEDataLoader* fpWeDataLoader;
+    WEDataLoader fWeDataLoader;
+
+friend class ReadThreadFactory; 
+//friend class WEDataLoader;
+};
+
+
+//------------------------------------------------------------------------------
+// Thread object that receives a cleartablelock tool command.
+//------------------------------------------------------------------------------
+class ClearTableLockReadThread : public ReadThread
+{
+public:
+	ClearTableLockReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
+	virtual ~ClearTableLockReadThread();
+
+	virtual void operator()();
+private:
+	boost::shared_ptr<WE_ClearTableLockCmd> fWeClearTableLockCmd;
+};
+
+
+//------------------------------------------------------------------------------
+// Thread object that receives a redistributeDB tool command.
+//------------------------------------------------------------------------------
+class RedistributeReadThread : public ReadThread
+{
+public:
+	RedistributeReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
+	virtual ~RedistributeReadThread();
+
+	virtual void operator()();
+
+private:
+	
+};
+
+//------------------------------------------------------------------------------
+// Thread object that get file sizes for a table.
+//------------------------------------------------------------------------------
+class GetFileSizeThread : public ReadThread
+{
+public:
+	GetFileSizeThread(const messageqcpp::IOSocket& ios, ByteStream& ibs, BRM::DBRM &dbrm);
+	virtual ~GetFileSizeThread();
+
+	virtual void operator()();
+
+private:
+	boost::shared_ptr<WE_GetFileSizes> fWeGetFileSizes;
+	int key;
+};
+
+class ReadThreadFactory
+{
+public:
+	ReadThreadFactory(){}
+	virtual ~ReadThreadFactory(){}
+
+public:
+	static void CreateReadThread(ThreadPool& Tp, IOSocket& ios, BRM::DBRM &dbrm);
+
+
+};
+
+
+}
+
+#endif
+

writeengine/server/we_server.cpp

@@ -0,0 +1,243 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*******************************************************************************
+* $Id: we_server.cpp 4700 2013-07-08 16:43:49Z bpaul $
+*
+*******************************************************************************/
+
+#include <unistd.h>
+#include <iostream>
+#include <string>
+#include <sstream>
+#ifndef _MSC_VER
+#include <signal.h>
+#include <stdexcept>
+#include "logger.h"
+#endif
+#ifndef _MSC_VER
+#include <sys/resource.h>
+#endif
+using namespace std;
+
+#include "messagequeue.h"
+using namespace messageqcpp;
+
+#include "threadpool.h"
+using namespace threadpool;
+
+#include "we_readthread.h"
+using namespace WriteEngine;
+
+#include "liboamcpp.h"
+using namespace oam;
+
+#include "distributedenginecomm.h"
+#include "IDBPolicy.h"
+#include "utils_utf8.h"
+#include "dbrm.h"
+
+namespace
+{
+	void added_a_pm(int)
+	{
+		logging::LoggingID logid(21, 0, 0);
+		logging::Message::Args args1;
+		logging::Message msg(1);
+		args1.add("we_server caught SIGHUP. Resetting connections");
+		msg.format( args1 );
+		logging::Logger logger(logid.fSubsysID);
+		logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid);
+		joblist::DistributedEngineComm::reset();
+	}
+}
+
+int setupResources()
+{
+#ifndef _MSC_VER
+        struct rlimit rlim;
+
+        if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
+                return -1;
+        }
+        rlim.rlim_cur = rlim.rlim_max = 65536;
+        if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) {
+                return -2;
+        }
+
+        if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
+                return -3;
+        }
+
+        if (rlim.rlim_cur != 65536) {
+                return -4;
+        }
+#endif
+        return 0;
+}
+
+int main(int argc, char** argv)
+{
+	// get and set locale language
+    string systemLang = "C";
+	systemLang = funcexp::utf8::idb_setlocale();
+
+    printf ("Locale is : %s\n", systemLang.c_str() );
+
+	//set BUSY_INIT state
+	{
+		// Is there a reason to have a seperate Oam instance for this?
+		Oam oam;
+		try
+		{
+			oam.processInitComplete("WriteEngineServer", oam::BUSY_INIT);
+		}
+		catch (...)
+		{
+		}
+	}
+	//BUG 2991
+	setlocale(LC_NUMERIC, "C");
+#ifndef _MSC_VER
+	struct sigaction sa;
+	memset(&sa, 0, sizeof(sa));
+    sa.sa_handler = added_a_pm;
+	sigaction(SIGHUP, &sa, 0);
+	sa.sa_handler = SIG_IGN;
+	sigaction(SIGPIPE, &sa, 0);
+#endif
+
+	// Init WriteEngine Wrapper (including Config Calpont.xml cache)
+	WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_WE_SRV );
+
+#ifdef _MSC_VER
+    // In windows, initializing the wrapper (A dll) does not set the static variables
+    // in the main program
+    idbdatafile::IDBPolicy::configIDBPolicy();
+#endif
+	Config weConfig;
+	setupResources();
+	
+	ostringstream serverParms;
+	serverParms << "pm" << weConfig.getLocalModuleID() << "_WriteEngineServer";
+
+	// Create MessageQueueServer, with one retry in case the call to bind the
+	// known port fails with "Address already in use".
+	boost::scoped_ptr<MessageQueueServer> mqs;
+	bool tellUser = true;
+	for (;;)
+	{
+		try {
+			mqs.reset(new MessageQueueServer(serverParms.str()));
+			break;
+		}
+		// @bug4393 Error Handling for MessageQueueServer constructor exception
+		catch (runtime_error& re) {
+			string what = re.what();
+			if (what.find("Address already in use") != string::npos)
+			{
+				if (tellUser)
+				{
+					cerr << "Address already in use, retrying..." << endl;
+					tellUser = false;
+				}
+				sleep(5);
+			}
+			else
+			{
+				Oam oam;
+				try // Get out of BUSYINIT state; else OAM will not retry
+				{
+					oam.processInitComplete("WriteEngineServer");
+				}
+				catch (...)
+				{
+				}
+
+				// If/when a common logging class or function is added to the
+				// WriteEngineServer, we should use that.  In the mean time,
+				// I will log this errmsg with inline calls to the logging.
+				logging::Message::Args args;
+				logging::Message message;
+				string errMsg("WriteEngineServer failed to initiate: ");
+				errMsg += what;
+				args.add( errMsg );
+				message.format(args);
+				logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
+				logging::MessageLog ml(lid);
+				ml.logCriticalMessage( message );
+
+				return 2;
+			}
+		}
+	}
+
+	IOSocket ios;
+	size_t mt = 20;
+	size_t qs = mt * 100;
+	ThreadPool tp(mt, qs);
+
+	//set ACTIVE state
+	{
+		Oam oam;
+		try
+		{
+			oam.processInitComplete("WriteEngineServer", ACTIVE);
+		}
+		catch (...)
+		{
+		}
+	}
+	cout << "WriteEngineServer is ready" << endl;
+	BRM::DBRM dbrm;
+	for (;;)
+	{
+		try // BUG 4834 -
+		{
+			ios = mqs->accept();
+			//tp.invoke(ReadThread(ios));
+			ReadThreadFactory::CreateReadThread(tp,ios, dbrm);
+			{
+/*				logging::Message::Args args;
+				logging::Message message;
+				string aMsg("WriteEngineServer : New incoming connection");
+				args.add(aMsg);
+				message.format(args);
+				logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
+				logging::MessageLog ml(lid);
+				ml.logInfoMessage( message ); */
+			}
+		}
+		catch(std::exception& ex) // BUG 4834 - log the exception
+		{
+			logging::Message::Args args;
+			logging::Message message;
+			string errMsg("WriteEngineServer : Exception caught on accept(): ");
+			errMsg += ex.what();
+			args.add( errMsg );
+			message.format(args);
+			logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
+			logging::MessageLog ml(lid);
+			ml.logCriticalMessage( message );
+			break;
+		}
+	}
+
+	//It is an error to reach here...
+	return 1;
+}
+