From d613fed62f02db8efe7a167ff342b1fa9e249fbc Mon Sep 17 00:00:00 2001 From: John Strizich Date: Fri, 9 Jun 2017 22:35:05 -0700 Subject: [PATCH] trying out this tool, fbcodeBuilder Summary: this diff provides ci builds for our two open source projects, openr and fbzmq, by leveraging snarkmaster 's awesome tool, fbcode builder. this will run our external cmake build process on lego-linux sandcastle boxes and also generates scripts for building on travis with docker externally. this diff is modeled on D4441467. it also includes some changes to fbcode builder itself and some minor changes in our cmake files. snarkmaster , please take a look at my fbcode builder changes. I needed to make some modifications in order to build some of our dependencies which have slightly non-standard build steps and to run tests. I split up the configure step, and I addressed an issue around workdir with cmake. the last workdir was just a relative path, `build`, so it was hard to get back to it in a future step to run tests. Reviewed By: saifhhasan Differential Revision: D5141184 fbshipit-source-id: 94cacab807a3a0da4d0d81016d7f36f37656145d --- build/fbcode_builder/fbcode_builder.py | 33 +++++++++++++------- build/fbcode_builder/specs/fbthrift.py | 2 +- build/fbcode_builder/specs/fbzmq.py | 43 ++++++++++++++++++++++++++ build/fbcode_builder/specs/gmock.py | 15 +++++++++ build/fbcode_builder/specs/wangle.py | 4 +-- 5 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 build/fbcode_builder/specs/fbzmq.py create mode 100644 build/fbcode_builder/specs/gmock.py diff --git a/build/fbcode_builder/fbcode_builder.py b/build/fbcode_builder/fbcode_builder.py index b3e8e3689..928970eb8 100644 --- a/build/fbcode_builder/fbcode_builder.py +++ b/build/fbcode_builder/fbcode_builder.py @@ -232,10 +232,10 @@ class FBCodeBuilder(object): self.workdir(path_join(base_dir, os.path.basename(project), path)), ] + maybe_change_branch) - def fb_github_project_workdir(self, project_and_path): + def fb_github_project_workdir(self, project_and_path, github_org='facebook'): 'This helper lets Facebook-internal CI special-cases FB projects' project, path = project_and_path.split('/', 1) - return self.github_project_workdir('facebook/' + project, path) + return self.github_project_workdir(github_org + '/' + project, path) def _make_vars(self, make_vars): return shell_join(' ', ( @@ -257,9 +257,8 @@ class FBCodeBuilder(object): )), ] - def autoconf_install(self, name): - return self.step('Build and install {0}'.format(name), [ - self.run(ShellQuoted('autoreconf -ivf')), + def configure(self): + return [ self.run(ShellQuoted( 'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" ' 'CFLAGS="$CFLAGS -I"{p}"/include" ' @@ -267,9 +266,14 @@ class FBCodeBuilder(object): 'PY_PREFIX={p} ' './configure --prefix={p}' ).format(p=self.option('prefix'))), - ] + self.make_and_install()) + ] - def cmake_install(self, name): + def autoconf_install(self, name): + return self.step('Build and install {0}'.format(name), [ + self.run(ShellQuoted('autoreconf -ivf')), + ] + self.configure() + self.make_and_install()) + + def cmake_configure(self, name): cmake_defines = { 'BUILD_SHARED_LIBS': 'ON', 'CMAKE_INSTALL_PREFIX': self.option('prefix'), @@ -277,15 +281,22 @@ class FBCodeBuilder(object): cmake_defines.update( self.option('{0}:cmake_defines'.format(name), {}) ) - return self.step('Build and install {0}'.format(name), [ - self.workdir('build'), - self.run(ShellQuoted('cmake {args} ..').format( + return [ + self.run(ShellQuoted( + 'CXXFLAGS="$CXXFLAGS -isystem "{p}"/include" ' + 'CFLAGS="$CFLAGS -isystem "{p}"/include" ' + 'cmake {args} ..').format( + p=self.option('prefix'), args=shell_join(' ', ( ShellQuoted('-D{k}={v}').format(k=k, v=v) for k, v in cmake_defines.items() )) )), - ] + self.make_and_install()) + ] + + def cmake_install(self, name): + return self.step('Build and install {0}'.format(name), + self.cmake_configure(name) + self.make_and_install()) def fb_github_autoconf_install(self, project_and_path): return [ diff --git a/build/fbcode_builder/specs/fbthrift.py b/build/fbcode_builder/specs/fbthrift.py index 2c17dae37..7885e6de0 100644 --- a/build/fbcode_builder/specs/fbthrift.py +++ b/build/fbcode_builder/specs/fbthrift.py @@ -21,7 +21,7 @@ def fbcode_builder_spec(builder): 'depends_on': [folly, wangle, zstd], 'steps': [ # This isn't a separete spec, since only fbthrift uses mstch. - builder.github_project_workdir('no1msd/mstch', '.'), + builder.github_project_workdir('no1msd/mstch', 'build'), builder.cmake_install('no1msd/mstch'), builder.fb_github_autoconf_install('fbthrift/thrift'), ], diff --git a/build/fbcode_builder/specs/fbzmq.py b/build/fbcode_builder/specs/fbzmq.py new file mode 100644 index 000000000..25e7068d0 --- /dev/null +++ b/build/fbcode_builder/specs/fbzmq.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fbthrift as fbthrift +import specs.folly as folly +import specs.gmock as gmock + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + builder.add_option('jedisct1/libsodium:git_hash', 'stable') + return { + 'depends_on': [folly, fbthrift, gmock], + 'steps': [ + builder.github_project_workdir('jedisct1/libsodium', '.'), + builder.step('Build and install jedisct1/libsodium', [ + builder.configure(), + builder.make_and_install(), + ]), + + builder.github_project_workdir('zeromq/libzmq', '.'), + builder.step('Build and install zeromq/libzmq', [ + builder.run(ShellQuoted('./autogen.sh')), + builder.configure(), + builder.make_and_install(), + ]), + + builder.fb_github_project_workdir('fbzmq/fbzmq/build', 'facebookincubator'), + builder.step('Build and install fbzmq/fbzmq/build', [ + builder.cmake_configure('fbzmq/fbzmq/build'), + # we need the pythonpath to find the thrift compiler + builder.run(ShellQuoted( + 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' + 'make -j {n}' + ).format(p=builder.option('prefix'), n=builder.option('make_parallelism'))), + builder.run(ShellQuoted('make install')), + ]), + ], + } diff --git a/build/fbcode_builder/specs/gmock.py b/build/fbcode_builder/specs/gmock.py new file mode 100644 index 000000000..64b59a3b6 --- /dev/null +++ b/build/fbcode_builder/specs/gmock.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + return { + 'steps': [ + # google mock also provides the gtest libraries + builder.github_project_workdir('google/googletest', 'googlemock/build'), + builder.cmake_install('google/googletest'), + ], + } diff --git a/build/fbcode_builder/specs/wangle.py b/build/fbcode_builder/specs/wangle.py index 57daba14a..5df0e0c84 100644 --- a/build/fbcode_builder/specs/wangle.py +++ b/build/fbcode_builder/specs/wangle.py @@ -9,10 +9,10 @@ import specs.folly as folly def fbcode_builder_spec(builder): # Projects that simply depend on Wangle need not spend time on tests. - builder.add_option('wangle/wangle:cmake_defines', {'BUILD_TESTS': 'OFF'}) + builder.add_option('wangle/wangle/build:cmake_defines', {'BUILD_TESTS': 'OFF'}) return { 'depends_on': [folly], 'steps': [ - builder.fb_github_cmake_install('wangle/wangle'), + builder.fb_github_cmake_install('wangle/wangle/build'), ], }