database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

the begginning

Browse Source
This commit is contained in:
david hill
2016-01-06 14:08:59 -06:00
parent 66a31debcb
commit f6afc42dd0
18251 changed files with 16460679 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
utils/threadpool/Makefile Executable file
View File

@ -0,0 +1,106 @@
include ../../rules.mak
#******************************************************************************************
# $Id: Makefile 3193 2012-07-23 18:47:31Z pleblanc $
#
# Copyright (C) 2009-2012 Calpont Corporation
# All rights reserved
#*****************************************************************************************/
.PHONY: install clean test docs coverage leakcheck
PROGRAM=libthreadpool.so
VERSION=1.0.0
LIBRARY=$(PROGRAM).$(VERSION)
CPPFLAGS=-I$(EXPORT_ROOT)/include -I/usr/include/libxml2
CXXFLAGS+=$(DEBUG_FLAGS) -Wall -fpic
CLIBS=-L$(EXPORT_ROOT)/lib -lconfigcpp -lxml2 -lcppunit -lloggingcpp -ldl
XLIBS=-L. -lthreadpool -L$(EXPORT_ROOT)/lib -lconfigcpp -lxml2 -lloggingcpp -ldl
TLIBS=-L. -lthreadpool $(CLIBS) -lmessageqcpp
GLIBS=$(CLIBS) -lmessageqcpp
LLIBS=-L$(CALPONT_LIBRARY_PATH) -lloggingcpp
SRCS=weightedthreadpool.cpp threadpool.cpp prioritythreadpool.cpp
LINCLUDES=weightedthreadpool.h threadpool.h prioritythreadpool.h
OBJS=$(SRCS:.cpp=.o)
$(LIBRARY): $(OBJS)
$(LINK.cpp) -shared -o $(LIBRARY) $(OBJS)
rm -f $(PROGRAM)
ln -s $(LIBRARY) $(PROGRAM)
install: bootstrap $(LIBRARY)
umask 002; \
mkdir -p $(INSTALL_ROOT_LIB) $(INSTALL_ROOT_INCLUDE); \
pushd $(INSTALL_ROOT_LIB) >/dev/null; \
rm -f $(LIBRARY) $(PROGRAM); \
ln -s $(LIBRARY) $(PROGRAM); \
popd >/dev/null; \
$(INSTALL) $(LIBRARY) $(INSTALL_ROOT_LIB)
bootstrap:
$(INSTALL) $(LINCLUDES) $(INSTALL_ROOT_INCLUDE)
clean:
rm -f $(OBJS) tdriver.o $(PROGRAM) $(LIBRARY) tdriver core *~ *.tag *-gcov.* *.gcov tdriver-gcov *.d
rm -rf html
docs:
doxygen $(EXPORT_ROOT)/etc/Doxyfile
tp: tp.o
$(LINK.cpp) -o $@ $^ $(XLIBS)
wtp: wtp.o
$(LINK.cpp) -o $@ $^ $(XLIBS)
tdriver: tdriver.o
$(LINK.cpp) -o $@ $^ $(TLIBS)
test: $(LIBRARY) tdriver
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib ./tdriver
%-gcov.o: %.cpp
$(COMPILE.cpp) -o $@ $^
tdriver-gcov: CXXFLAGS+=-fprofile-arcs -ftest-coverage
tdriver-gcov: tdriver-gcov.o $(subst .o,-gcov.o,$(OBJS))
$(LINK.cpp) -o $@ $^ $(GLIBS)
coverage: tdriver-gcov
rm -f *.gcda
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib ./tdriver-gcov
for file in $(SRCS); do \
bfile=`basename $$file .cpp`; \
gcov -o $${bfile}-gcov $$file >/dev/null; \
done
/usr/local/bin/genCoverage.pl $(SRCS)
PGLIBS=$(GLIBS)
%-prof.o: %.cpp
$(COMPILE.cpp) -o $@ $^
tdriver-prof: CXXFLAGS+=-pg
tdriver-prof: tdriver-prof.o $(subst .o,-prof.o,$(OBJS))
$(LINK.cpp) -o $@ $^ $(PGLIBS)
profile: tdriver-prof
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib ./tdriver-prof
gprof ./tdriver-prof
leakcheck: $(LIBRARY) tdriver
LD_LIBRARY_PATH=.:$(EXPORT_ROOT)/lib:/usr/local/lib valgrind --tool=memcheck --leak-check=yes ./tdriver --leakcheck
%.d: %.cpp
@set -e; rm -f $@; \
$(CC) -MM $(CPPFLAGS) $< > $@.$$$$; \
sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$$ > $@; \
rm -f $@.$$$$
ifndef BOOTSTRAP
-include $(SRCS:.cpp=.d) tdriver.d
endif

38
utils/threadpool/Makefile.am Normal file
View File

@ -0,0 +1,38 @@
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
# $Id: Makefile.am 3194 2012-07-24 13:15:08Z rdempsey $
## Process this file with automake to produce Makefile.in
AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags)
AM_CFLAGS = $(idb_cflags)
AM_CXXFLAGS = $(idb_cxxflags)
AM_LDFLAGS = -version-info 1:0:0 $(idb_ldflags)
lib_LTLIBRARIES = libthreadpool.la
libthreadpool_la_SOURCES = weightedthreadpool.cpp threadpool.cpp prioritythreadpool.cpp
include_HEADERS = weightedthreadpool.h threadpool.h prioritythreadpool.h
test:
coverage:
leakcheck:
docs:
bootstrap: install-data-am

546
utils/threadpool/Makefile.in Normal file
View File

@ -0,0 +1,546 @@
# Makefile.in generated by automake 1.9.6 from Makefile.am.
# @configure_input@
# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
# 2003, 2004, 2005 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
# with or without modifications, as long as this notice is preserved.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.
@SET_MAKE@
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
# $Id: Makefile.am 3194 2012-07-24 13:15:08Z rdempsey $
srcdir = @srcdir@
top_srcdir = @top_srcdir@
VPATH = @srcdir@
pkgdatadir = $(datadir)/@PACKAGE@
pkglibdir = $(libdir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
top_builddir = ../..
am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
INSTALL = @INSTALL@
install_sh_DATA = $(install_sh) -c -m 644
install_sh_PROGRAM = $(install_sh) -c
install_sh_SCRIPT = $(install_sh) -c
INSTALL_HEADER = $(INSTALL_DATA)
transform = $(program_transform_name)
NORMAL_INSTALL = :
PRE_INSTALL = :
POST_INSTALL = :
NORMAL_UNINSTALL = :
PRE_UNINSTALL = :
POST_UNINSTALL = :
build_triplet = @build@
host_triplet = @host@
subdir = utils/threadpool
DIST_COMMON = $(include_HEADERS) $(srcdir)/Makefile.am \
$(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/m4/compilerflags.m4 \
$(top_srcdir)/m4/functions.m4 $(top_srcdir)/m4/install.m4 \
$(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
mkinstalldirs = $(install_sh) -d
CONFIG_HEADER = $(top_builddir)/config.h
CONFIG_CLEAN_FILES =
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
am__vpath_adj = case $$p in \
$(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
*) f=$$p;; \
esac;
am__strip_dir = `echo $$p | sed -e 's|^.*/||'`;
am__installdirs = "$(DESTDIR)$(libdir)" "$(DESTDIR)$(includedir)"
libLTLIBRARIES_INSTALL = $(INSTALL)
LTLIBRARIES = $(lib_LTLIBRARIES)
libthreadpool_la_LIBADD =
am_libthreadpool_la_OBJECTS = weightedthreadpool.lo threadpool.lo \
prioritythreadpool.lo
libthreadpool_la_OBJECTS = $(am_libthreadpool_la_OBJECTS)
DEFAULT_INCLUDES = -I. -I$(srcdir) -I$(top_builddir)
depcomp = $(SHELL) $(top_srcdir)/depcomp
am__depfiles_maybe = depfiles
CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
LTCXXCOMPILE = $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) \
$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
$(AM_CXXFLAGS) $(CXXFLAGS)
CXXLD = $(CXX)
CXXLINK = $(LIBTOOL) --tag=CXX --mode=link $(CXXLD) $(AM_CXXFLAGS) \
$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
SOURCES = $(libthreadpool_la_SOURCES)
DIST_SOURCES = $(libthreadpool_la_SOURCES)
includeHEADERS_INSTALL = $(INSTALL_HEADER)
HEADERS = $(include_HEADERS)
ETAGS = etags
CTAGS = ctags
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
ACLOCAL = @ACLOCAL@
ALLOCA = @ALLOCA@
AMDEP_FALSE = @AMDEP_FALSE@
AMDEP_TRUE = @AMDEP_TRUE@
AMTAR = @AMTAR@
AR = @AR@
AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
CC = @CC@
CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@
CPP = @CPP@
CPPFLAGS = @CPPFLAGS@
CXX = @CXX@
CXXCPP = @CXXCPP@
CXXDEPMODE = @CXXDEPMODE@
CXXFLAGS = @CXXFLAGS@
CYGPATH_W = @CYGPATH_W@
DEFS = @DEFS@
DEPDIR = @DEPDIR@
ECHO = @ECHO@
ECHO_C = @ECHO_C@
ECHO_N = @ECHO_N@
ECHO_T = @ECHO_T@
EGREP = @EGREP@
EXEEXT = @EXEEXT@
F77 = @F77@
FFLAGS = @FFLAGS@
INSTALL_DATA = @INSTALL_DATA@
INSTALL_PROGRAM = @INSTALL_PROGRAM@
INSTALL_SCRIPT = @INSTALL_SCRIPT@
INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
LDFLAGS = @LDFLAGS@
LEX = @LEX@
LEXLIB = @LEXLIB@
LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
LIBOBJS = @LIBOBJS@
LIBS = @LIBS@
LIBTOOL = @LIBTOOL@
LN_S = @LN_S@
LTLIBOBJS = @LTLIBOBJS@
MAKEINFO = @MAKEINFO@
OBJEXT = @OBJEXT@
PACKAGE = @PACKAGE@
PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_STRING = @PACKAGE_STRING@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
PATH_SEPARATOR = @PATH_SEPARATOR@
POW_LIB = @POW_LIB@
RANLIB = @RANLIB@
SED = @SED@
SET_MAKE = @SET_MAKE@
SHELL = @SHELL@
STRIP = @STRIP@
VERSION = @VERSION@
XML2_CONFIG = @XML2_CONFIG@
XML_CPPFLAGS = @XML_CPPFLAGS@
XML_LIBS = @XML_LIBS@
YACC = @YACC@
ac_ct_AR = @ac_ct_AR@
ac_ct_CC = @ac_ct_CC@
ac_ct_CXX = @ac_ct_CXX@
ac_ct_F77 = @ac_ct_F77@
ac_ct_RANLIB = @ac_ct_RANLIB@
ac_ct_STRIP = @ac_ct_STRIP@
am__fastdepCC_FALSE = @am__fastdepCC_FALSE@
am__fastdepCC_TRUE = @am__fastdepCC_TRUE@
am__fastdepCXX_FALSE = @am__fastdepCXX_FALSE@
am__fastdepCXX_TRUE = @am__fastdepCXX_TRUE@
am__include = @am__include@
am__leading_dot = @am__leading_dot@
am__quote = @am__quote@
am__tar = @am__tar@
am__untar = @am__untar@
bindir = @bindir@
build = @build@
build_alias = @build_alias@
build_cpu = @build_cpu@
build_os = @build_os@
build_vendor = @build_vendor@
datadir = @datadir@
etcdir = @etcdir@
exec_prefix = @exec_prefix@
host = @host@
host_alias = @host_alias@
host_cpu = @host_cpu@
host_os = @host_os@
host_vendor = @host_vendor@
idb_brm_libs = @idb_brm_libs@
idb_cflags = @idb_cflags@
idb_common_includes = @idb_common_includes@
idb_common_ldflags = @idb_common_ldflags@
idb_common_libs = @idb_common_libs@
idb_cppflags = @idb_cppflags@
idb_cxxflags = @idb_cxxflags@
idb_exec_libs = @idb_exec_libs@
idb_ldflags = @idb_ldflags@
idb_oam_libs = @idb_oam_libs@
idb_write_libs = @idb_write_libs@
idbinstall = @idbinstall@
includedir = @includedir@
infodir = @infodir@
install_sh = @install_sh@
libdir = @libdir@
libexecdir = @libexecdir@
localdir = @localdir@
localstatedir = @localstatedir@
mandir = @mandir@
march_flags = @march_flags@
mibdir = @mibdir@
mkdir_p = @mkdir_p@
mysqldir = @mysqldir@
netsnmp_libs = @netsnmp_libs@
netsnmpagntdir = @netsnmpagntdir@
netsnmpdir = @netsnmpdir@
netsnmplibrdir = @netsnmplibrdir@
netsnmpmachdir = @netsnmpmachdir@
netsnmpsysdir = @netsnmpsysdir@
oldincludedir = @oldincludedir@
postdir = @postdir@
prefix = @prefix@
program_transform_name = @program_transform_name@
sbindir = @sbindir@
sharedir = @sharedir@
sharedstatedir = @sharedstatedir@
sysconfdir = @sysconfdir@
target_alias = @target_alias@
toolsdir = @toolsdir@
AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags)
AM_CFLAGS = $(idb_cflags)
AM_CXXFLAGS = $(idb_cxxflags)
AM_LDFLAGS = -version-info 1:0:0 $(idb_ldflags)
lib_LTLIBRARIES = libthreadpool.la
libthreadpool_la_SOURCES = weightedthreadpool.cpp threadpool.cpp prioritythreadpool.cpp
include_HEADERS = weightedthreadpool.h threadpool.h prioritythreadpool.h
all: all-am
.SUFFIXES:
.SUFFIXES: .cpp .lo .o .obj
$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps)
@for dep in $?; do \
case '$(am__configure_deps)' in \
*$$dep*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh \
&& exit 0; \
exit 1;; \
esac; \
done; \
echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu utils/threadpool/Makefile'; \
cd $(top_srcdir) && \
$(AUTOMAKE) --gnu utils/threadpool/Makefile
.PRECIOUS: Makefile
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
@case '$?' in \
*config.status*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
*) \
echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
esac;
$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(top_srcdir)/configure: $(am__configure_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(ACLOCAL_M4): $(am__aclocal_m4_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
install-libLTLIBRARIES: $(lib_LTLIBRARIES)
@$(NORMAL_INSTALL)
test -z "$(libdir)" || $(mkdir_p) "$(DESTDIR)$(libdir)"
@list='$(lib_LTLIBRARIES)'; for p in $$list; do \
if test -f $$p; then \
f=$(am__strip_dir) \
echo " $(LIBTOOL) --mode=install $(libLTLIBRARIES_INSTALL) $(INSTALL_STRIP_FLAG) '$$p' '$(DESTDIR)$(libdir)/$$f'"; \
$(LIBTOOL) --mode=install $(libLTLIBRARIES_INSTALL) $(INSTALL_STRIP_FLAG) "$$p" "$(DESTDIR)$(libdir)/$$f"; \
else :; fi; \
done
uninstall-libLTLIBRARIES:
@$(NORMAL_UNINSTALL)
@set -x; list='$(lib_LTLIBRARIES)'; for p in $$list; do \
p=$(am__strip_dir) \
echo " $(LIBTOOL) --mode=uninstall rm -f '$(DESTDIR)$(libdir)/$$p'"; \
$(LIBTOOL) --mode=uninstall rm -f "$(DESTDIR)$(libdir)/$$p"; \
done
clean-libLTLIBRARIES:
-test -z "$(lib_LTLIBRARIES)" || rm -f $(lib_LTLIBRARIES)
@list='$(lib_LTLIBRARIES)'; for p in $$list; do \
dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
test "$$dir" != "$$p" || dir=.; \
echo "rm -f \"$${dir}/so_locations\""; \
rm -f "$${dir}/so_locations"; \
done
libthreadpool.la: $(libthreadpool_la_OBJECTS) $(libthreadpool_la_DEPENDENCIES)
$(CXXLINK) -rpath $(libdir) $(libthreadpool_la_LDFLAGS) $(libthreadpool_la_OBJECTS) $(libthreadpool_la_LIBADD) $(LIBS)
mostlyclean-compile:
-rm -f *.$(OBJEXT)
distclean-compile:
-rm -f *.tab.c
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/prioritythreadpool.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/threadpool.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/weightedthreadpool.Plo@am__quote@
.cpp.o:
@am__fastdepCXX_TRUE@ if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $<
.cpp.obj:
@am__fastdepCXX_TRUE@ if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ `$(CYGPATH_W) '$<'`; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
.cpp.lo:
@am__fastdepCXX_TRUE@ if $(LTCXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Plo"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $<
mostlyclean-libtool:
-rm -f *.lo
clean-libtool:
-rm -rf .libs _libs
distclean-libtool:
-rm -f libtool
uninstall-info-am:
install-includeHEADERS: $(include_HEADERS)
@$(NORMAL_INSTALL)
test -z "$(includedir)" || $(mkdir_p) "$(DESTDIR)$(includedir)"
@list='$(include_HEADERS)'; for p in $$list; do \
if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
f=$(am__strip_dir) \
echo " $(includeHEADERS_INSTALL) '$$d$$p' '$(DESTDIR)$(includedir)/$$f'"; \
$(includeHEADERS_INSTALL) "$$d$$p" "$(DESTDIR)$(includedir)/$$f"; \
done
uninstall-includeHEADERS:
@$(NORMAL_UNINSTALL)
@list='$(include_HEADERS)'; for p in $$list; do \
f=$(am__strip_dir) \
echo " rm -f '$(DESTDIR)$(includedir)/$$f'"; \
rm -f "$(DESTDIR)$(includedir)/$$f"; \
done
ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
mkid -fID $$unique
tags: TAGS
TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$(TAGS_FILES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
if test -z "$(ETAGS_ARGS)$$tags$$unique"; then :; else \
test -n "$$unique" || unique=$$empty_fix; \
$(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
$$tags $$unique; \
fi
ctags: CTAGS
CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$(TAGS_FILES) $(LISP)
tags=; \
here=`pwd`; \
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
unique=`for i in $$list; do \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | \
$(AWK) ' { files[$$0] = 1; } \
END { for (i in files) print i; }'`; \
test -z "$(CTAGS_ARGS)$$tags$$unique" \
|| $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
$$tags $$unique
GTAGS:
here=`$(am__cd) $(top_builddir) && pwd` \
&& cd $(top_srcdir) \
&& gtags -i $(GTAGS_ARGS) $$here
distclean-tags:
-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \
topsrcdirstrip=`echo "$(top_srcdir)" | sed 's|.|.|g'`; \
list='$(DISTFILES)'; for file in $$list; do \
case $$file in \
$(srcdir)/*) file=`echo "$$file" | sed "s|^$$srcdirstrip/||"`;; \
$(top_srcdir)/*) file=`echo "$$file" | sed "s|^$$topsrcdirstrip/|$(top_builddir)/|"`;; \
esac; \
if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
dir=`echo "$$file" | sed -e 's,/[^/]*$$,,'`; \
if test "$$dir" != "$$file" && test "$$dir" != "."; then \
dir="/$$dir"; \
$(mkdir_p) "$(distdir)$$dir"; \
else \
dir=''; \
fi; \
if test -d $$d/$$file; then \
if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
cp -pR $(srcdir)/$$file $(distdir)$$dir || exit 1; \
fi; \
cp -pR $$d/$$file $(distdir)$$dir || exit 1; \
else \
test -f $(distdir)/$$file \
|| cp -p $$d/$$file $(distdir)/$$file \
|| exit 1; \
fi; \
done
check-am: all-am
check: check-am
all-am: Makefile $(LTLIBRARIES) $(HEADERS)
installdirs:
for dir in "$(DESTDIR)$(libdir)" "$(DESTDIR)$(includedir)"; do \
test -z "$$dir" || $(mkdir_p) "$$dir"; \
done
install: install-am
install-exec: install-exec-am
install-data: install-data-am
uninstall: uninstall-am
install-am: all-am
@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
installcheck: installcheck-am
install-strip:
$(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
`test -z '$(STRIP)' || \
echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install
mostlyclean-generic:
clean-generic:
distclean-generic:
-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
maintainer-clean-generic:
@echo "This command is intended for maintainers to use"
@echo "it deletes files that may require special tools to rebuild."
clean: clean-am
clean-am: clean-generic clean-libLTLIBRARIES clean-libtool \
mostlyclean-am
distclean: distclean-am
-rm -rf ./$(DEPDIR)
-rm -f Makefile
distclean-am: clean-am distclean-compile distclean-generic \
distclean-libtool distclean-tags
dvi: dvi-am
dvi-am:
html: html-am
info: info-am
info-am:
install-data-am: install-includeHEADERS
install-exec-am: install-libLTLIBRARIES
install-info: install-info-am
install-man:
installcheck-am:
maintainer-clean: maintainer-clean-am
-rm -rf ./$(DEPDIR)
-rm -f Makefile
maintainer-clean-am: distclean-am maintainer-clean-generic
mostlyclean: mostlyclean-am
mostlyclean-am: mostlyclean-compile mostlyclean-generic \
mostlyclean-libtool
pdf: pdf-am
pdf-am:
ps: ps-am
ps-am:
uninstall-am: uninstall-includeHEADERS uninstall-info-am \
uninstall-libLTLIBRARIES
.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \
clean-libLTLIBRARIES clean-libtool ctags distclean \
distclean-compile distclean-generic distclean-libtool \
distclean-tags distdir dvi dvi-am html html-am info info-am \
install install-am install-data install-data-am install-exec \
install-exec-am install-includeHEADERS install-info \
install-info-am install-libLTLIBRARIES install-man \
install-strip installcheck installcheck-am installdirs \
maintainer-clean maintainer-clean-generic mostlyclean \
mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
pdf pdf-am ps ps-am tags uninstall uninstall-am \
uninstall-includeHEADERS uninstall-info-am \
uninstall-libLTLIBRARIES
test:
coverage:
leakcheck:
docs:
bootstrap: install-data-am
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
.NOEXPORT:

181
utils/threadpool/prioritythreadpool.cpp Normal file
View File

@ -0,0 +1,181 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***********************************************************************
* $Id: threadpool.cpp 553 2008-02-27 17:51:16Z rdempsey $
*
*
***********************************************************************/
#include <stdexcept>
#include <unistd.h>
#include <exception>
using namespace std;
#include "messageobj.h"
#include "messagelog.h"
using namespace logging;
#include "prioritythreadpool.h"
using namespace boost;
namespace threadpool
{
PriorityThreadPool::PriorityThreadPool(uint targetWeightPerRun, uint highThreads,
uint midThreads, uint lowThreads, uint ID) :
_stop(false), weightPerRun(targetWeightPerRun), id(ID)
{
for (uint32_t i = 0; i < highThreads; i++)
threads.create_thread(ThreadHelper(this, HIGH));
for (uint32_t i = 0; i < midThreads; i++)
threads.create_thread(ThreadHelper(this, MEDIUM));
for (uint32_t i = 0; i < lowThreads; i++)
threads.create_thread(ThreadHelper(this, LOW));
cout << "started " << highThreads << " high, " << midThreads << " med, " << lowThreads
<< " low.\n";
threadCounts[HIGH] = highThreads;
threadCounts[MEDIUM] = midThreads;
threadCounts[LOW] = lowThreads;
}
PriorityThreadPool::~PriorityThreadPool()
{
stop();
}
void PriorityThreadPool::addJob(const Job &job, bool useLock)
{
mutex::scoped_lock lk(mutex, defer_lock_t());
if (useLock)
lk.lock();
if (job.priority > 66)
jobQueues[HIGH].push_back(job);
else if (job.priority > 33)
jobQueues[MEDIUM].push_back(job);
else
jobQueues[LOW].push_back(job);
if (useLock)
newJob.notify_one();
}
void PriorityThreadPool::removeJobs(uint32_t id)
{
list<Job>::iterator it;
mutex::scoped_lock lk(mutex);
for (uint32_t i = 0; i < _COUNT; i++)
for (it = jobQueues[i].begin(); it != jobQueues[i].end();)
if (it->id == id)
it = jobQueues[i].erase(it);
else
++it;
}
PriorityThreadPool::Priority PriorityThreadPool::pickAQueue(Priority preference)
{
if (!jobQueues[preference].empty())
return preference;
else if (!jobQueues[HIGH].empty())
return HIGH;
else if (!jobQueues[MEDIUM].empty())
return MEDIUM;
else
return LOW;
}
void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw()
{
Priority queue;
uint32_t weight, i;
vector<Job> runList;
vector<bool> reschedule;
uint32_t rescheduleCount;
uint32_t queueSize;
while (!_stop) {
mutex::scoped_lock lk(mutex);
queue = pickAQueue(preferredQueue);
if (jobQueues[queue].empty()) {
newJob.wait(lk);
continue;
}
queueSize = jobQueues[queue].size();
weight = 0;
// 3 conditions stop this thread from grabbing all jobs in the queue
//
// 1: The weight limit has been exceeded
// 2: The queue is empty
// 3: It has grabbed more than half of the jobs available &
// should leave some to the other threads
while ((weight < weightPerRun) && (!jobQueues[queue].empty())
&& (runList.size() <= queueSize/2)) {
runList.push_back(jobQueues[queue].front());
jobQueues[queue].pop_front();
weight += runList.back().weight;
}
lk.unlock();
reschedule.resize(runList.size());
rescheduleCount = 0;
for (i = 0; i < runList.size() && !_stop; i++) {
try {
reschedule[i] = false;
reschedule[i] = (*(runList[i].functor))();
if (reschedule[i])
rescheduleCount++;
}
catch (std::exception &e) {
cerr << e.what() << endl;
}
}
// no real work was done, prevent intensive busy waiting
if (rescheduleCount == runList.size())
usleep(1000);
if (rescheduleCount > 0) {
lk.lock();
for (i = 0; i < runList.size(); i++)
if (reschedule[i])
addJob(runList[i], false);
if (rescheduleCount > 1)
newJob.notify_all();
else
newJob.notify_one();
lk.unlock();
}
runList.clear();
}
}
void PriorityThreadPool::stop()
{
_stop = true;
threads.join_all();
}
} // namespace threadpool
// vim:ts=4 sw=4:

121
utils/threadpool/prioritythreadpool.h Normal file
View File

@ -0,0 +1,121 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***********************************************************************
* $Id: $
*
*
***********************************************************************/
/** @file */
#ifndef PRIORITYTHREADPOOL_H
#define PRIORITYTHREADPOOL_H
#include <string>
#include <iostream>
#include <cstdlib>
#include <sstream>
#include <stdexcept>
#include <boost/thread/thread.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/condition.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/function.hpp>
#include "winport.h"
namespace threadpool
{
class PriorityThreadPool
{
public:
class Functor {
public:
virtual ~Functor() { };
// as of 12/3/13, all implementors return 0 and -1. -1 will cause
// this thread pool to reschedule the job, 0 will throw it away on return.
virtual int operator()() = 0;
};
//typedef boost::function0<int> Functor;
struct Job {
Job() : weight(1), priority(0), id(0) { }
boost::shared_ptr<Functor> functor;
uint32_t weight;
uint32_t priority;
uint32_t id;
};
enum Priority {
LOW,
MEDIUM,
HIGH,
_COUNT
};
/*********************************************
* ctor/dtor
*
*********************************************/
/** @brief ctor
*/
PriorityThreadPool(uint targetWeightPerRun, uint highThreads, uint midThreads,
uint lowThreads, uint id = 0);
virtual ~PriorityThreadPool();
void removeJobs(uint32_t id);
void addJob(const Job &job, bool useLock = true);
void stop();
/** @brief for use in debugging
*/
void dump();
protected:
private:
struct ThreadHelper {
ThreadHelper(PriorityThreadPool *impl, Priority queue) : ptp(impl), preferredQueue(queue) { }
void operator()() { ptp->threadFcn(preferredQueue); }
PriorityThreadPool *ptp;
Priority preferredQueue;
};
explicit PriorityThreadPool();
explicit PriorityThreadPool(const PriorityThreadPool &);
PriorityThreadPool & operator=(const PriorityThreadPool &);
Priority pickAQueue(Priority preference);
void threadFcn(const Priority preferredQueue) throw();
std::list<Job> jobQueues[3]; // higher indexes = higher priority
uint32_t threadCounts[3];
boost::mutex mutex;
boost::condition newJob;
boost::thread_group threads;
bool _stop;
uint32_t weightPerRun;
volatile uint id; // prevent it from being optimized out
};
} // namespace threadpool
#endif //PRIORITYTHREADPOOL_H

127
utils/threadpool/tdriver.cpp Normal file
View File

@ -0,0 +1,127 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: tdriver.cpp 3495 2013-01-21 14:09:51Z rdempsey $
#include <string>
#include <stdexcept>
#include <iostream>
#include <fstream>
using namespace std;
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <boost/thread.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/scoped_array.hpp>
#include <boost/shared_ptr.hpp>
#include <cppunit/extensions/HelperMacros.h>
#include "threadpool.h"
int thecount = 0;
boost::mutex mutex;
class ThreadPoolTestSuite : public CppUnit::TestFixture
{
CPPUNIT_TEST_SUITE( ThreadPoolTestSuite );
CPPUNIT_TEST( test_1 );
CPPUNIT_TEST_SUITE_END();
private:
// Functor class
struct foo
{
void operator ()()
{
for (int i = 0; i < 500; i++)
{
// simulate some work
fData++;
}
boost::mutex::scoped_lock lock(mutex);
std::cout << "count = " << ++thecount << ' ' << fData << std::endl;
}
foo(int i):
fData(i)
{}
foo(const foo& copy)
: fData(copy.fData)
{}
int fData;
};
public:
void setUp()
{}
void tearDown()
{}
void test_1()
{
threadpool::ThreadPool pool( 5, 10 );
for (int y = 0; y < 10; y++)
{
foo bar(y);
for (int i = 0; i < 10; ++i)
{
pool.invoke(bar);
}
// Wait until all of the queued up and in-progress work has finished
pool.wait();
pool.dump();
}
}
};
CPPUNIT_TEST_SUITE_REGISTRATION( ThreadPoolTestSuite );
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main( int argc, char **argv)
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry &registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest( registry.makeTest() );
bool wasSuccessful = runner.run( "", false );
return (wasSuccessful ? 0 : 1);
}

320
utils/threadpool/threadpool.cpp Normal file
View File

@ -0,0 +1,320 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***********************************************************************
* $Id: threadpool.cpp 3495 2013-01-21 14:09:51Z rdempsey $
*
*
***********************************************************************/
#include <stdexcept>
using namespace std;
#include "messageobj.h"
#include "messagelog.h"
using namespace logging;
#define THREADPOOL_DLLEXPORT
#include "threadpool.h"
#undef THREADPOOL_DLLEXPORT
namespace threadpool
{
ThreadPool::ThreadPool()
:fMaxThreads( 0 ), fQueueSize( 0 )
{
init();
}
ThreadPool::ThreadPool( size_t maxThreads, size_t queueSize )
:fMaxThreads( maxThreads ), fQueueSize( queueSize )
{
init();
if (fQueueSize == 0)
fQueueSize = fMaxThreads*2;
}
ThreadPool::~ThreadPool() throw()
{
// delete fThreadCreated;
try
{
stop();
}
catch(...)
{}
}
void ThreadPool::init()
{
fThreadCount = 0;
fGeneralErrors = 0;
fFunctorErrors = 0;
waitingFunctorsSize = 0;
issued = 0;
fStop = false;
// fThreadCreated = new NoOp();
fNextFunctor = fWaitingFunctors.end();
}
void ThreadPool::setQueueSize(size_t queueSize)
{
boost::mutex::scoped_lock lock1(fMutex);
fQueueSize = queueSize;
}
void ThreadPool::setMaxThreads(size_t maxThreads)
{
boost::mutex::scoped_lock lock1(fMutex);
fMaxThreads = maxThreads;
}
void ThreadPool::setThreadCreatedListener(const Functor_T &f)
{
// fThreadCreated = f;
}
void ThreadPool::stop()
{
boost::mutex::scoped_lock lock1(fMutex);
fStop = true;
lock1.unlock();
fNeedThread.notify_all();
fThreads.join_all();
}
void ThreadPool::wait()
{
boost::mutex::scoped_lock lock1(fMutex);
while (waitingFunctorsSize > 0)
{
fThreadAvailable.wait(lock1);
//cerr << "woke!" << endl;
}
}
void ThreadPool::invoke(const Functor_T &threadfunc)
{
boost::mutex::scoped_lock lock1(fMutex);
for(;;)
{
try
{
if ( waitingFunctorsSize < fThreadCount)
{
// Don't create a thread unless it's needed. There
// is a thread available to service this request.
addFunctor(threadfunc);
lock1.unlock();
break;
}
bool bAdded = false;
if ( waitingFunctorsSize < fQueueSize)
{
// Don't create a thread unless you have to
addFunctor(threadfunc);
bAdded = true;
}
if ( fThreadCount < fMaxThreads)
{
++fThreadCount;
lock1.unlock();
fThreads.create_thread(beginThreadFunc(*this));
if (bAdded)
break;
// If the mutex is unlocked before creating the thread
// this allows fThreadAvailable to be triggered
// before the wait below runs. So run the loop again.
lock1.lock();
continue;
}
if (bAdded)
{
lock1.unlock();
break;
}
fThreadAvailable.wait(lock1);
}
catch(...)
{
++fGeneralErrors;
throw;
}
}
fNeedThread.notify_one();
}
void ThreadPool::beginThread() throw()
{
try
{
// fThreadCreated();
boost::mutex::scoped_lock lock1(fMutex);
for(;;)
{
if (fStop)
break;
if (fNextFunctor == fWaitingFunctors.end())
{
// Wait until someone needs a thread
fNeedThread.wait(lock1);
}
else
{
/* Need to tune these magic #s */
vector<Container_T::iterator> todoList;
int i, num;
Container_T::const_iterator iter;
/* Use this to control how many jobs are issued to a single thread */
num = (waitingFunctorsSize - issued >= 1 ? 1 : 0);
for (i = 0; i < num; i++)
todoList.push_back(fNextFunctor++);
issued += num;
// cerr << "got " << num << " jobs." << endl;
// cerr << "got " << num << " jobs. waitingFunctorsSize=" <<
// waitingFunctorsSize << " issued=" << issued << " fThreadCount=" <<
// fThreadCount << endl;
lock1.unlock();
for (i = 0; i < num; i++) {
try {
(*todoList[i])();
}
catch(exception &e) {
++fFunctorErrors;
cerr << e.what() << endl;
}
}
lock1.lock();
issued -= num;
waitingFunctorsSize -= num;
for (i = 0; i < num; i++)
fWaitingFunctors.erase(todoList[i]);
/*
if (waitingFunctorsSize != fWaitingFunctors.size())
cerr << "size mismatch! fake size=" << waitingFunctorsSize <<
" real size=" << fWaitingFunctors.size() << endl;
*/
fThreadAvailable.notify_all();
}
}
}
catch (exception& ex)
{
++fGeneralErrors;
// Log the exception and exit this thread
try
{
logging::Message::Args args;
logging::Message message(5);
args.add("beginThread: Caught exception: ");
args.add(ex.what());
message.format( args );
logging::LoggingID lid(22);
logging::MessageLog ml(lid);
ml.logErrorMessage( message );
}
catch(...)
{
}
}
catch(...)
{
++fGeneralErrors;
// Log the exception and exit this thread
try
{
logging::Message::Args args;
logging::Message message(6);
args.add("beginThread: Caught unknown exception!");
message.format( args );
logging::LoggingID lid(22);
logging::MessageLog ml(lid);
ml.logErrorMessage( message );
}
catch(...)
{
}
}
}
void ThreadPool::addFunctor(const Functor_T &func)
{
bool bAtEnd = false;
if (fNextFunctor == fWaitingFunctors.end())
bAtEnd = true;
fWaitingFunctors.push_back(func);
waitingFunctorsSize++;
if (bAtEnd)
{
--fNextFunctor;
}
}
void ThreadPool::dump()
{
std::cout << "General Errors: " << fGeneralErrors << std::endl;
std::cout << "Functor Errors: " << fFunctorErrors << std::endl;
std::cout << "Waiting functors: " << fWaitingFunctors.size() << std::endl;
}
} // namespace threadpool

216
utils/threadpool/threadpool.h Normal file
View File

@ -0,0 +1,216 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***********************************************************************
*
* Work dervied from Devguy.com's Open Source C++ thread pool implementation
* released under public domain:
* http://web.archive.org/liveweb/http://dgpctk.cvs.sourceforge.net/viewvc/dgpctk/dgc%2B%2B/include/dg/thread/threadpool.h?revision=1.22&content-type=text%2Fplain
*
* http://web.archive.org/web/20100104101109/http://devguy.com/bb/viewtopic.php?t=460
*
***********************************************************************/
/** @file */
#ifndef THREADPOOL_H
#define THREADPOOL_H
#include <string>
#include <iostream>
#include <cstdlib>
#include <sstream>
#include <stdexcept>
#include <stdint.h>
#include <boost/thread/thread.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/condition.hpp>
#include <boost/bind.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/function.hpp>
#if defined(_MSC_VER) && defined(xxxTHREADPOOL_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif
namespace threadpool
{
/** @brief ThreadPool is a component for working with pools of threads and asynchronously
* executing tasks. It is responsible for creating threads and tracking which threads are "busy"
* and which are idle. Idle threads are utilized as "work" is added to the system.
*/
class ThreadPool
{
public:
typedef boost::function0<void> Functor_T;
/*********************************************
* ctor/dtor
*
*********************************************/
/** @brief ctor
*/
EXPORT ThreadPool();
/** @brief ctor
*
* @param maxThreads the maximum number of threads in this pool. This is the maximum number
* of simultaneuous operations that can go on.
* @param queueSize the maximum number of work tasks in the queue. This is the maximum
* number of jobs that can queue up in the work list before invoke() blocks.
*/
EXPORT explicit ThreadPool( size_t maxThreads, size_t queueSize );
/** @brief dtor
*/
EXPORT ~ThreadPool() throw();
/*********************************************
* accessors/mutators
*
*********************************************/
/** @brief set the work queue size
*
* @param queueSize the size of the work queue
*/
EXPORT void setQueueSize( size_t queueSize );
/** @brief fet the work queue size
*/
inline size_t getQueueSize() const { return fQueueSize; }
/** @brief set the maximum number of threads to be used to process
* the work queue
*
* @param maxThreads the maximum number of threads
*/
EXPORT void setMaxThreads( size_t maxThreads );
/** @brief get the maximum number of threads
*/
inline size_t getMaxThreads() const { return fMaxThreads; }
/** @brief register a functor to be called when a new thread
* is created
*/
EXPORT void setThreadCreatedListener(const Functor_T &f) ;
/** @brief queue size accessor
*
*/
inline uint32_t getWaiting() const { return waitingFunctorsSize; }
/*********************************************
* operations
*
*********************************************/
/** @brief invoke a functor in a separate thread managed by the pool
*
* If all maxThreads are busy, threadfunc will be added to a work list and
* will run when a thread comes free. If all threads are busy and there are
* queueSize tasks already waiting, invoke() will block until a slot in the
* queue comes free.
*/
EXPORT void invoke(const Functor_T &threadfunc);
/** @brief stop the threads
*/
EXPORT void stop();
/** @brief wait on all the threads to complete
*/
EXPORT void wait();
/** @brief for use in debugging
*/
EXPORT void dump();
protected:
private:
/** @brief initialize data memebers
*/
void init();
/** @brief add a functor to the list
*/
void addFunctor(const Functor_T &func);
/** @brief thread entry point
*/
void beginThread() throw();
ThreadPool(const ThreadPool&);
ThreadPool& operator = (const ThreadPool&);
friend struct beginThreadFunc;
struct beginThreadFunc
{
beginThreadFunc(ThreadPool& impl)
: fImpl(impl)
{}
void operator() ()
{
fImpl.beginThread();
}
ThreadPool &fImpl;
};
struct NoOp
{
void operator () () const
{}}
;
size_t fThreadCount;
size_t fMaxThreads;
size_t fQueueSize;
typedef std::list<Functor_T> Container_T;
Container_T fWaitingFunctors;
Container_T::iterator fNextFunctor;
// Functor_T * fThreadCreated;
uint32_t issued;
boost::mutex fMutex;
boost::condition fThreadAvailable; // triggered when a thread is available
boost::condition fNeedThread; // triggered when a thread is needed
boost::thread_group fThreads;
bool fStop;
long fGeneralErrors;
long fFunctorErrors;
uint32_t waitingFunctorsSize;
};
} // namespace threadpool
#undef EXPORT
#endif //THREADPOOL_H

391
utils/threadpool/weightedthreadpool.cpp Normal file
View File

@ -0,0 +1,391 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***********************************************************************
* $Id: threadpool.cpp 553 2008-02-27 17:51:16Z rdempsey $
*
*
***********************************************************************/
#include <stdexcept>
#include <unistd.h>
using namespace std;
#include "messageobj.h"
#include "messagelog.h"
using namespace logging;
#include "weightedthreadpool.h"
namespace threadpool
{
WeightedThreadPool::WeightedThreadPool()
:fMaxThreadWeight(0), fMaxThreads( 0 ), fQueueSize( 0 )
{
init();
}
WeightedThreadPool::WeightedThreadPool( size_t maxThreadWeight, size_t maxThreads, size_t queueSize )
:fMaxThreadWeight(maxThreadWeight), fMaxThreads( maxThreads ), fQueueSize( queueSize )
{
init();
if (fQueueSize == 0)
fQueueSize = fMaxThreads*2;
}
WeightedThreadPool::~WeightedThreadPool() throw()
{
// delete fThreadCreated;
try
{
stop();
}
catch(...)
{}
}
void WeightedThreadPool::init()
{
fThreadCount = 0;
fGeneralErrors = 0;
fFunctorErrors = 0;
fWaitingFunctorsSize = 0;
fWaitingFunctorsWeight=0;
issued = 0;
fStop = false;
// fThreadCreated = new NoOp();
fNextFunctor = fWaitingFunctors.end();
}
void WeightedThreadPool::setQueueSize(size_t queueSize)
{
boost::mutex::scoped_lock lock1(fMutex);
fQueueSize = queueSize;
}
void WeightedThreadPool::setMaxThreads(size_t maxThreads)
{
boost::mutex::scoped_lock lock1(fMutex);
fMaxThreads = maxThreads;
}
void WeightedThreadPool::setMaxThreadWeight(size_t maxWeight)
{
boost::mutex::scoped_lock lock1(fMutex);
fMaxThreadWeight = maxWeight;
}
void WeightedThreadPool::setThreadCreatedListener(const Functor_T &f)
{
// fThreadCreated = f;
}
void WeightedThreadPool::stop()
{
boost::mutex::scoped_lock lock1(fMutex);
fStop = true;
lock1.unlock();
fNeedThread.notify_all();
fThreads.join_all();
}
void WeightedThreadPool::wait()
{
boost::mutex::scoped_lock lock1(fMutex);
while (fWaitingFunctorsSize > 0)
{
//cout << "waiting ..." << endl;
fThreadAvailable.wait(lock1);
//cerr << "woke!" << endl;
}
}
void WeightedThreadPool::removeJobs(uint32_t id)
{
boost::mutex::scoped_lock lock1(fMutex);
Container_T::iterator it;
it = fNextFunctor;
while (it != fWaitingFunctors.end()) {
if (it->id == id) {
fWaitingFunctorsWeight -= it->functorWeight;
fWaitingFunctorsSize--;
if (it == fNextFunctor) {
fWaitingFunctors.erase(fNextFunctor++);
it = fNextFunctor;
}
else
fWaitingFunctors.erase(it++);
}
else
++it;
}
}
void WeightedThreadPool::invoke(const Functor_T &threadfunc, uint32_t functor_weight,
uint32_t id)
{
boost::mutex::scoped_lock lock1(fMutex);
for(;;)
{
try
{
if ( fWaitingFunctorsSize < fThreadCount)
{
// Don't create a thread unless it's needed. There
// is a thread available to service this request.
addFunctor(threadfunc, functor_weight, id);
lock1.unlock();
break;
}
bool bAdded = false;
if ( fWaitingFunctorsSize < fQueueSize)
{
// Don't create a thread unless you have to
addFunctor(threadfunc, functor_weight, id);
bAdded = true;
}
// add a thread is necessary
if ( fThreadCount < fMaxThreads)
{
++fThreadCount;
//cout << "\t++invoke() tcnt=" << fThreadCount << endl;
lock1.unlock();
fThreads.create_thread(beginThreadFunc(*this));
if (bAdded)
break;
// If the mutex is unlocked before creating the thread
// this allows fThreadAvailable to be triggered
// before the wait below runs. So run the loop again.
lock1.lock();
continue;
}
//else
// cout << "invoke() no thread created c=" << fThreadCount << " m=" << fMaxThreads << endl;
if (bAdded)
{
lock1.unlock();
break;
}
fThreadAvailable.wait(lock1);
}
catch(...)
{
++fGeneralErrors;
throw;
}
}
fNeedThread.notify_one();
}
void WeightedThreadPool::beginThread() throw()
{
vector<bool> reschedule;
try
{
// fThreadCreated();
boost::mutex::scoped_lock lock1(fMutex);
for(;;)
{
if (fStop)
break;
if (fNextFunctor == fWaitingFunctors.end())
{
// Wait until someone needs a thread
fNeedThread.wait(lock1);
}
else
{
vector<Container_T::iterator> todoList;
int i, num = (fWaitingFunctorsSize - issued);
Container_T::const_iterator iter;
uint32_t weight=0;
for (i = 0; i < num && weight < fMaxThreadWeight; i++) {
weight += (*fNextFunctor).functorWeight;
todoList.push_back(fNextFunctor++);
}
issued+=i;
num=i;
lock1.unlock();
//cerr << "beginThread() " << num
// << " jobs - fWaitingFunctorsSize=" << fWaitingFunctorsSize
// << " fWaitingFunctorsWeight=" << fWaitingFunctorsWeight
// << " weight=" << weight
// << " issued=" << issued << " todo=" << todoList.size()
// << " fThreadCount=" << fThreadCount << endl;
i = 0;
reschedule.resize(num);
bool allWereRescheduled = true, someWereRescheduled = false;
while (i < num) {
try {
for (; i < num; i++) {
reschedule[i] = false; // in case of exception in the next line
reschedule[i] = ((*todoList[i]).functor)();
allWereRescheduled &= reschedule[i];
someWereRescheduled |= reschedule[i];
}
}
catch(exception &e) {
i++;
++fFunctorErrors;
cerr << e.what() << endl;
}
}
// no real work was done, prevent intensive busy waiting
if (allWereRescheduled)
usleep(1000);
//cout << "running " << i << "/" << num << " functor" <<endl;
lock1.lock();
if (someWereRescheduled) {
for (i = 0; i < num; i++)
if (reschedule[i])
addFunctor((*todoList[i]).functor, (*todoList[i]).functorWeight,
(*todoList[i]).id);
if (num > 1)
fNeedThread.notify_all();
else
fNeedThread.notify_one();
}
issued -= num;
for (i = 0; i < num; i++) {
fWaitingFunctorsWeight-=(*todoList[i]).functorWeight;
fWaitingFunctors.erase(todoList[i]);
}
fWaitingFunctorsSize -= num;
//if (fWaitingFunctorsSize != fWaitingFunctors.size()) ;
// cerr << "num=" << num << " cleaned=" << i << " size="
// << fWaitingFunctorsSize << " list size="
// << fWaitingFunctors.size()
// << " w="<<fWaitingFunctorsWeight << endl;
fThreadAvailable.notify_all();
}
}
}
catch (exception& ex)
{
++fGeneralErrors;
// Log the exception and exit this thread
try
{
logging::Message::Args args;
logging::Message message(5);
args.add("beginThread: Caught exception: ");
args.add(ex.what());
message.format( args );
logging::LoggingID lid(22);
logging::MessageLog ml(lid);
ml.logErrorMessage( message );
}
catch(...)
{
}
}
catch(...)
{
++fGeneralErrors;
// Log the exception and exit this thread
try
{
logging::Message::Args args;
logging::Message message(6);
args.add("beginThread: Caught unknown exception!");
message.format( args );
logging::LoggingID lid(22);
logging::MessageLog ml(lid);
ml.logErrorMessage( message );
}
catch(...)
{
}
}
}
void WeightedThreadPool::addFunctor(const Functor_T &func, uint32_t functor_weight,
uint32_t id)
{
bool bAtEnd=false;
if (fNextFunctor == fWaitingFunctors.end())
bAtEnd = true;
//cout << "addFunctor() w=" << fWaitingFunctorsWeight
// << " s=" << fWaitingFunctorsSize << " i=" << id << endl;
FunctorListItem fl = {func, functor_weight, id};
fWaitingFunctors.push_back(fl);
fWaitingFunctorsSize++;
fWaitingFunctorsWeight+=functor_weight;
if (bAtEnd)
{
--fNextFunctor;
}
}
void WeightedThreadPool::dump()
{
std::cout << "General Errors: " << fGeneralErrors << std::endl;
std::cout << "Functor Errors: " << fFunctorErrors << std::endl;
std::cout << "Waiting functors: " << fWaitingFunctors.size() << std::endl;
std::cout << "Waiting functors weight : " << fWaitingFunctorsWeight << std::endl;
}
} // namespace threadpool

226
utils/threadpool/weightedthreadpool.h Normal file
View File

@ -0,0 +1,226 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***********************************************************************
* $Id: $
*
*
***********************************************************************/
/** @file */
#ifndef WEIGHTEDTHREADPOOL_H
#define WEIGHTEDTHREADPOOL_H
#include <string>
#include <iostream>
#include <cstdlib>
#include <sstream>
#include <stdexcept>
#include <boost/thread/thread.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/condition.hpp>
#include <boost/bind.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/function.hpp>
namespace threadpool
{
/** @brief ThreadPool is a component for working with pools of threads and asynchronously
* executing tasks. It is responsible for creating threads and tracking which threads are "busy"
* and which are idle. Idle threads are utilized as "work" is added to the system.
*/
class WeightedThreadPool
{
public:
typedef boost::function0<int> Functor_T;
/*********************************************
* ctor/dtor
*
*********************************************/
/** @brief ctor
*/
WeightedThreadPool();
/** @brief ctor
*
* @param maxThreads the maximum number of threads in this pool. This is the maximum number
* of simultaneuous operations that can go on.
* @param queueSize the maximum number of work tasks in the queue. This is the maximum
* number of jobs that can queue up in the work list before invoke() blocks.
*/
explicit WeightedThreadPool( size_t maxThreadWeight, size_t maxThreads, size_t queueSize );
/** @brief dtor
*/
~WeightedThreadPool() throw();
/*********************************************
* accessors/mutators
*
*********************************************/
/** @brief set the work queue size
*
* @param queueSize the size of the work queue
*/
void setQueueSize( size_t queueSize );
/** @brief fet the work queue size
*/
inline size_t getQueueSize() const { return fQueueSize; }
/** @brief set the maximum number of threads to be used to process
* the work queue
*
* @param maxThreads the maximum number of threads
*/
void setMaxThreads( size_t maxThreads );
/** @brief get the maximum number of threads
*/
inline size_t getMaxThreads() const { return fMaxThreads; }
/** @brief set the maximum processing weight of a thread to be
* submitted for execution from the existing jobs
* scheduled in the work queue
*
* @param maxWeight for execution
*/
void setMaxThreadWeight( size_t maxWeight );
/** @brief get the maximum number of threads
*/
inline uint32_t getMaxThreadWeight() const { return fMaxThreadWeight; }
/** @brief register a functor to be called when a new thread
* is created
*/
void setThreadCreatedListener(const Functor_T &f) ;
/** @brief queue size accessor
*
*/
inline uint32_t getWaiting() const { return fWaitingFunctorsSize; }
inline uint32_t getWeight() const { return fWaitingFunctorsWeight; }
void removeJobs(uint32_t id);
/*********************************************
* operations
*
*********************************************/
/** @brief invoke a functor in a separate thread managed by the pool
*
* If all maxThreads are busy, threadfunc will be added to a work list and
* will run when a thread comes free. If all threads are busy and there are
* queueSize tasks already waiting, invoke() will block until a slot in the
* queue comes free.
*/
void invoke(const Functor_T &threadfunc, uint32_t functor_weight, uint32_t id);
/** @brief stop the threads
*/
void stop();
/** @brief wait on all the threads to complete
*/
void wait();
/** @brief for use in debugging
*/
void dump();
protected:
private:
/** @brief initialize data memebers
*/
void init();
/** @brief add a functor to the list
*/
void addFunctor(const Functor_T &func, uint32_t functor_weight, uint32_t id);
/** @brief thread entry point
*/
void beginThread() throw();
WeightedThreadPool(const WeightedThreadPool&);
WeightedThreadPool& operator = (const WeightedThreadPool&);
friend struct beginThreadFunc;
struct beginThreadFunc
{
beginThreadFunc(WeightedThreadPool& impl)
: fImpl(impl)
{}
void operator() ()
{
fImpl.beginThread();
}
WeightedThreadPool &fImpl;
};
struct NoOp
{
void operator () () const
{}
};
size_t fThreadCount;
size_t fMaxThreadWeight;
size_t fMaxThreads;
size_t fQueueSize;
//typedef std::list<Functor_T> Container_T;
struct FunctorListItemStruct {
Functor_T functor;
uint32_t functorWeight;
uint32_t id;
};
typedef FunctorListItemStruct FunctorListItem;
typedef std::list<FunctorListItem> Container_T;
Container_T fWaitingFunctors;
Container_T::iterator fNextFunctor;
uint32_t issued;
boost::mutex fMutex;
boost::condition fThreadAvailable; // triggered when a thread is available
boost::condition fNeedThread; // triggered when a thread is needed
boost::thread_group fThreads;
bool fStop;
long fGeneralErrors;
long fFunctorErrors;
uint16_t fWaitingFunctorsSize;
uint16_t fWaitingFunctorsWeight;
};
} // namespace threadpool
#endif //WEIGHTEDTHREADPOOL_H

85
utils/threadpool/wtp.cpp Normal file
View File

@ -0,0 +1,85 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <string>
#include <stdexcept>
#include <iostream>
#include <fstream>
using namespace std;
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <boost/thread.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/scoped_array.hpp>
#include <boost/shared_ptr.hpp>
#include "weightedthreadpool.h"
int thecount = 0;
boost::mutex mutex;
// Functor class
struct foo
{
void operator ()()
{
for (int i = 0; i < 1024*1024*10; i++)
// simulate some work
fData++;
//boost::mutex::scoped_lock lock(mutex);
//std::cout << "foo count = " << ++thecount << " " << fData << std::endl;
}
foo(int i):
fData(i)
{}
foo(const foo& copy)
: fData(copy.fData)
{}
int fData;
};
int main( int argc, char **argv)
{
threadpool::WeightedThreadPool pool( 100, 10, 5 );
for (int y = 0; y < 10; y++)
{
foo bar(y);
for (int i = 0; i < 10; ++i)
{
pool.invoke(bar, 25);
}
boost::mutex::scoped_lock lock(mutex);
std::cout << "count = " << ++thecount << std::endl;
// Wait until all of the queued up and in-progress work has finished
pool.wait();
pool.dump();
}
}