lib/proxygen
1
0
Fork 0
mirror of https://github.com/facebook/proxygen.git synced 2025-08-05 19:55:47 +03:00
Code Activity

Opt in opensource/fbcode_builder to pyfmt

Browse Source 
Reviewed By: zertosh

Differential Revision: D29612107

fbshipit-source-id: ac450058134e23a3831db35d2e49c80eb8cde36a
This commit is contained in:
Zsolt Dollenstein
2021-07-09 06:17:52 -07:00
committed by Facebook GitHub Bot
parent f3b62e2316
commit 40b6e08b08
24 changed files with 612 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
build/fbcode_builder/CMake/fb_py_test_main.py
View File

@@ -485,7 +485,7 @@ class Loader(object):
return loader.suiteClass(suites)
_COVERAGE_INI = '''\
_COVERAGE_INI = """\
[report]
exclude_lines =
pragma: no cover
@@ -495,7 +495,7 @@ exclude_lines =
pragma:.*no${PY_IMPL}${PY_MAJOR}
pragma:.*nopy${PY_MAJOR}
pragma:.*nopy${PY_MAJOR}${PY_MINOR}
'''
"""
class MainProgram(object):
@@ -734,7 +734,7 @@ class MainProgram(object):
if not self.options.collect_coverage:
return
with tempfile.NamedTemporaryFile('w', delete=False) as coverage_ini:
with tempfile.NamedTemporaryFile("w", delete=False) as coverage_ini:
coverage_ini.write(_COVERAGE_INI)
self._coverage_ini_path = coverage_ini.name

95
build/fbcode_builder/docker_builder.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
"""
Extends FBCodeBuilder to produce Docker context directories.
@@ -15,26 +16,23 @@ caching, you will want to:
that change the least often, and
- Put the steps that you are debugging towards the very end.
'''
"""
import logging
import os
import shutil
import tempfile
from fbcode_builder import FBCodeBuilder
from shell_quoting import (
raw_shell, shell_comment, shell_join, ShellQuoted, path_join
)
from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted, path_join
from utils import recursively_flatten_list, run_command
class DockerFBCodeBuilder(FBCodeBuilder):
def _user(self):
return self.option('user', 'root')
return self.option("user", "root")
def _change_user(self):
return ShellQuoted('USER {u}').format(u=self._user())
return ShellQuoted("USER {u}").format(u=self._user())
def setup(self):
# Please add RPM-based OSes here as appropriate.
@@ -63,17 +61,18 @@ class DockerFBCodeBuilder(FBCodeBuilder):
# it is present when the resulting container is run add to PATH
actions = []
if self.option("PYTHON_VENV", "OFF") == "ON":
actions = ShellQuoted('ENV PATH={p}:$PATH').format(
p=path_join(self.option('prefix'), "venv", "bin"))
return(actions)
actions = ShellQuoted("ENV PATH={p}:$PATH").format(
p=path_join(self.option("prefix"), "venv", "bin")
)
return actions
def step(self, name, actions):
assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
b = ShellQuoted('')
return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]
assert "\n" not in name, "Name {0} would span > 1 line".format(name)
b = ShellQuoted("")
return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b]
def run(self, shell_cmd):
return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd)
return ShellQuoted("RUN {cmd}").format(cmd=shell_cmd)
def set_env(self, key, value):
return ShellQuoted("ENV {key}={val}").format(key=key, val=value)
@@ -84,12 +83,12 @@ class DockerFBCodeBuilder(FBCodeBuilder):
# by root:root -- the explicit `mkdir` works around the bug:
# USER nobody
# WORKDIR build
ShellQuoted('USER root'),
ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format(
ShellQuoted("USER root"),
ShellQuoted("RUN mkdir -p {d} && chown {u} {d}").format(
d=dir, u=self._user()
),
self._change_user(),
ShellQuoted('WORKDIR {dir}').format(dir=dir),
ShellQuoted("WORKDIR {dir}").format(dir=dir),
]
def comment(self, comment):
@@ -99,60 +98,58 @@ class DockerFBCodeBuilder(FBCodeBuilder):
def copy_local_repo(self, repo_dir, dest_name):
fd, archive_path = tempfile.mkstemp(
prefix='local_repo_{0}_'.format(dest_name),
suffix='.tgz',
dir=os.path.abspath(self.option('docker_context_dir')),
prefix="local_repo_{0}_".format(dest_name),
suffix=".tgz",
dir=os.path.abspath(self.option("docker_context_dir")),
)
os.close(fd)
run_command('tar', 'czf', archive_path, '.', cwd=repo_dir)
run_command("tar", "czf", archive_path, ".", cwd=repo_dir)
return [
ShellQuoted('ADD {archive} {dest_name}').format(
ShellQuoted("ADD {archive} {dest_name}").format(
archive=os.path.basename(archive_path), dest_name=dest_name
),
# Docker permissions make very little sense... see also workdir()
ShellQuoted('USER root'),
ShellQuoted('RUN chown -R {u} {d}').format(
d=dest_name, u=self._user()
),
ShellQuoted("USER root"),
ShellQuoted("RUN chown -R {u} {d}").format(d=dest_name, u=self._user()),
self._change_user(),
]
def _render_impl(self, steps):
return raw_shell(shell_join('\n', recursively_flatten_list(steps)))
return raw_shell(shell_join("\n", recursively_flatten_list(steps)))
def debian_ccache_setup_steps(self):
source_ccache_tgz = self.option('ccache_tgz', '')
source_ccache_tgz = self.option("ccache_tgz", "")
if not source_ccache_tgz:
logging.info('Docker ccache not enabled')
logging.info("Docker ccache not enabled")
return []
dest_ccache_tgz = os.path.join(
self.option('docker_context_dir'), 'ccache.tgz'
)
dest_ccache_tgz = os.path.join(self.option("docker_context_dir"), "ccache.tgz")
try:
try:
os.link(source_ccache_tgz, dest_ccache_tgz)
except OSError:
logging.exception(
'Hard-linking {s} to {d} failed, falling back to copy'
.format(s=source_ccache_tgz, d=dest_ccache_tgz)
"Hard-linking {s} to {d} failed, falling back to copy".format(
s=source_ccache_tgz, d=dest_ccache_tgz
)
)
shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
except Exception:
logging.exception(
'Failed to copy or link {s} to {d}, aborting'
.format(s=source_ccache_tgz, d=dest_ccache_tgz)
"Failed to copy or link {s} to {d}, aborting".format(
s=source_ccache_tgz, d=dest_ccache_tgz
)
)
raise
return [
# Separate layer so that in development we avoid re-downloads.
self.run(ShellQuoted('apt-get install -yq ccache')),
ShellQuoted('ADD ccache.tgz /'),
self.run(ShellQuoted("apt-get install -yq ccache")),
ShellQuoted("ADD ccache.tgz /"),
ShellQuoted(
# Set CCACHE_DIR before the `ccache` invocations below.
'ENV CCACHE_DIR=/ccache '
"ENV CCACHE_DIR=/ccache "
# No clang support for now, so it's easiest to hardcode gcc.
'CC="ccache gcc" CXX="ccache g++" '
# Always log for ease of debugging. For real FB projects,
@@ -166,9 +163,10 @@ class DockerFBCodeBuilder(FBCodeBuilder):
#
# apt-get install sharutils
# bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
'CCACHE_LOGFILE=/tmp/ccache.log'
"CCACHE_LOGFILE=/tmp/ccache.log"
),
self.run(ShellQuoted(
self.run(
ShellQuoted(
# Future: Skipping this part made this Docker step instant,
# saving ~1min of build time. It's unclear if it is the
# chown or the du, but probably the chown -- since a large
@@ -176,16 +174,17 @@ class DockerFBCodeBuilder(FBCodeBuilder):
#
# ccache.tgz may be empty, or may have the wrong
# permissions.
'mkdir -p /ccache && time chown -R nobody /ccache && '
'time du -sh /ccache && '
"mkdir -p /ccache && time chown -R nobody /ccache && "
"time du -sh /ccache && "
# Reset stats so `docker_build_with_ccache.sh` can print
# useful values at the end of the run.
'echo === Prev run stats === && ccache -s && ccache -z && '
"echo === Prev run stats === && ccache -s && ccache -z && "
# Record the current time to let travis_build.sh figure out
# the number of bytes in the cache that are actually used --
# this is crucial for tuning the maximum cache size.
'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && '
"date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && "
# The build running as `nobody` should be able to write here
'chown nobody /tmp/ccache.log'
)),
"chown nobody /tmp/ccache.log"
)
),
]

367
build/fbcode_builder/fbcode_builder.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
"""
This is a small DSL to describe builds of Facebook's open-source projects
that are published to Github from a single internal repo, including projects
@@ -57,7 +58,7 @@ Ideas for the future -- these may not be very good :)
* do `make` and `cmake`
- If we get non-Debian OSes, part of ccache setup should be factored out.
'''
"""
import os
import re
@@ -66,22 +67,21 @@ from shell_quoting import path_join, shell_join, ShellQuoted
def _read_project_github_hashes():
base_dir = 'deps/github_hashes/' # trailing slash used in regex below
base_dir = "deps/github_hashes/" # trailing slash used in regex below
for dirname, _, files in os.walk(base_dir):
for filename in files:
path = os.path.join(dirname, filename)
with open(path) as f:
m_proj = re.match('^' + base_dir + '(.*)-rev\.txt$', path)
m_proj = re.match("^" + base_dir + "(.*)-rev\.txt$", path)
if m_proj is None:
raise RuntimeError('Not a hash file? {0}'.format(path))
m_hash = re.match('^Subproject commit ([0-9a-f]+)\n$', f.read())
raise RuntimeError("Not a hash file? {0}".format(path))
m_hash = re.match("^Subproject commit ([0-9a-f]+)\n$", f.read())
if m_hash is None:
raise RuntimeError('No hash in {0}'.format(path))
raise RuntimeError("No hash in {0}".format(path))
yield m_proj.group(1), m_hash.group(1)
class FBCodeBuilder(object):
def __init__(self, **kwargs):
self._options_do_not_access = kwargs # Use .option() instead.
# This raises upon detecting options that are specified but unused,
@@ -90,22 +90,22 @@ class FBCodeBuilder(object):
# Mark 'projects_dir' used even if the build installs no github
# projects. This is needed because driver programs like
# `shell_builder.py` unconditionally set this for all builds.
self._github_dir = self.option('projects_dir')
self._github_dir = self.option("projects_dir")
self._github_hashes = dict(_read_project_github_hashes())
def __repr__(self):
return '{0}({1})'.format(
return "{0}({1})".format(
self.__class__.__name__,
', '.join(
'{0}={1}'.format(k, repr(v))
", ".join(
"{0}={1}".format(k, repr(v))
for k, v in self._options_do_not_access.items()
)
),
)
def option(self, name, default=None):
value = self._options_do_not_access.get(name, default)
if value is None:
raise RuntimeError('Option {0} is required'.format(name))
raise RuntimeError("Option {0} is required".format(name))
self.options_used.add(name)
return value
@@ -114,7 +114,7 @@ class FBCodeBuilder(object):
def add_option(self, name, value):
if name in self._options_do_not_access:
raise RuntimeError('Option {0} already set'.format(name))
raise RuntimeError("Option {0} already set".format(name))
self._options_do_not_access[name] = value
#
@@ -122,12 +122,12 @@ class FBCodeBuilder(object):
#
def render(self, steps):
'''
"""
Converts nested actions to your builder's expected output format.
Typically takes the output of build().
'''
"""
res = self._render_impl(steps) # Implementation-dependent
# Now that the output is rendered, we expect all options to have
# been used.
@@ -135,41 +135,46 @@ class FBCodeBuilder(object):
unused_options -= self.options_used
if unused_options:
raise RuntimeError(
'Unused options: {0} -- please check if you made a typo '
'in any of them. Those that are truly not useful should '
'be not be set so that this typo detection can be useful.'
.format(unused_options)
"Unused options: {0} -- please check if you made a typo "
"in any of them. Those that are truly not useful should "
"be not be set so that this typo detection can be useful.".format(
unused_options
)
)
return res
def build(self, steps):
if not steps:
raise RuntimeError('Please ensure that the config you are passing '
'contains steps')
raise RuntimeError(
"Please ensure that the config you are passing " "contains steps"
)
return [self.setup(), self.diagnostics()] + steps
def setup(self):
'Your builder may want to install packages here.'
"Your builder may want to install packages here."
raise NotImplementedError
def diagnostics(self):
'Log some system diagnostics before/after setup for ease of debugging'
"Log some system diagnostics before/after setup for ease of debugging"
# The builder's repr is not used in a command to avoid pointlessly
# invalidating Docker's build cache.
return self.step('Diagnostics', [
self.comment('Builder {0}'.format(repr(self))),
self.run(ShellQuoted('hostname')),
self.run(ShellQuoted('cat /etc/issue || echo no /etc/issue')),
self.run(ShellQuoted('g++ --version || echo g++ not installed')),
self.run(ShellQuoted('cmake --version || echo cmake not installed')),
])
return self.step(
"Diagnostics",
[
self.comment("Builder {0}".format(repr(self))),
self.run(ShellQuoted("hostname")),
self.run(ShellQuoted("cat /etc/issue || echo no /etc/issue")),
self.run(ShellQuoted("g++ --version || echo g++ not installed")),
self.run(ShellQuoted("cmake --version || echo cmake not installed")),
],
)
def step(self, name, actions):
'A labeled collection of actions or other steps'
"A labeled collection of actions or other steps"
raise NotImplementedError
def run(self, shell_cmd):
'Run this bash command'
"Run this bash command"
raise NotImplementedError
def set_env(self, key, value):
@@ -177,54 +182,54 @@ class FBCodeBuilder(object):
raise NotImplementedError
def workdir(self, dir):
'Create this directory if it does not exist, and change into it'
"Create this directory if it does not exist, and change into it"
raise NotImplementedError
def copy_local_repo(self, dir, dest_name):
'''
"""
Copy the local repo at `dir` into this step's `workdir()`, analog of:
cp -r /path/to/folly folly
'''
"""
raise NotImplementedError
def python_deps(self):
return [
'wheel',
'cython==0.28.6',
"wheel",
"cython==0.28.6",
]
def debian_deps(self):
return [
'autoconf-archive',
'bison',
'build-essential',
'cmake',
'curl',
'flex',
'git',
'gperf',
'joe',
'libboost-all-dev',
'libcap-dev',
'libdouble-conversion-dev',
'libevent-dev',
'libgflags-dev',
'libgoogle-glog-dev',
'libkrb5-dev',
'libpcre3-dev',
'libpthread-stubs0-dev',
'libnuma-dev',
'libsasl2-dev',
'libsnappy-dev',
'libsqlite3-dev',
'libssl-dev',
'libtool',
'netcat-openbsd',
'pkg-config',
'sudo',
'unzip',
'wget',
'python3-venv',
"autoconf-archive",
"bison",
"build-essential",
"cmake",
"curl",
"flex",
"git",
"gperf",
"joe",
"libboost-all-dev",
"libcap-dev",
"libdouble-conversion-dev",
"libevent-dev",
"libgflags-dev",
"libgoogle-glog-dev",
"libkrb5-dev",
"libpcre3-dev",
"libpthread-stubs0-dev",
"libnuma-dev",
"libsasl2-dev",
"libsnappy-dev",
"libsqlite3-dev",
"libssl-dev",
"libtool",
"netcat-openbsd",
"pkg-config",
"sudo",
"unzip",
"wget",
"python3-venv",
]
#
@@ -234,51 +239,72 @@ class FBCodeBuilder(object):
def install_debian_deps(self):
actions = [
self.run(
ShellQuoted('apt-get update && apt-get install -yq {deps}').format(
deps=shell_join(' ', (
ShellQuoted(dep) for dep in self.debian_deps())))
ShellQuoted("apt-get update && apt-get install -yq {deps}").format(
deps=shell_join(
" ", (ShellQuoted(dep) for dep in self.debian_deps())
)
)
),
]
gcc_version = self.option('gcc_version')
gcc_version = self.option("gcc_version")
# Make the selected GCC the default before building anything
actions.extend([
self.run(ShellQuoted('apt-get install -yq {c} {cpp}').format(
c=ShellQuoted('gcc-{v}').format(v=gcc_version),
cpp=ShellQuoted('g++-{v}').format(v=gcc_version),
)),
self.run(ShellQuoted(
'update-alternatives --install /usr/bin/gcc gcc {c} 40 '
'--slave /usr/bin/g++ g++ {cpp}'
actions.extend(
[
self.run(
ShellQuoted("apt-get install -yq {c} {cpp}").format(
c=ShellQuoted("gcc-{v}").format(v=gcc_version),
cpp=ShellQuoted("g++-{v}").format(v=gcc_version),
)
),
self.run(
ShellQuoted(
"update-alternatives --install /usr/bin/gcc gcc {c} 40 "
"--slave /usr/bin/g++ g++ {cpp}"
).format(
c=ShellQuoted('/usr/bin/gcc-{v}').format(v=gcc_version),
cpp=ShellQuoted('/usr/bin/g++-{v}').format(v=gcc_version),
)),
self.run(ShellQuoted('update-alternatives --config gcc')),
])
c=ShellQuoted("/usr/bin/gcc-{v}").format(v=gcc_version),
cpp=ShellQuoted("/usr/bin/g++-{v}").format(v=gcc_version),
)
),
self.run(ShellQuoted("update-alternatives --config gcc")),
]
)
actions.extend(self.debian_ccache_setup_steps())
return self.step('Install packages for Debian-based OS', actions)
return self.step("Install packages for Debian-based OS", actions)
def create_python_venv(self):
actions = []
if self.option("PYTHON_VENV", "OFF") == "ON":
actions.append(self.run(ShellQuoted("python3 -m venv {p}").format(
p=path_join(self.option('prefix'), "venv"))))
return(actions)
actions.append(
self.run(
ShellQuoted("python3 -m venv {p}").format(
p=path_join(self.option("prefix"), "venv")
)
)
)
return actions
def python_venv(self):
actions = []
if self.option("PYTHON_VENV", "OFF") == "ON":
actions.append(ShellQuoted("source {p}").format(
p=path_join(self.option('prefix'), "venv", "bin", "activate")))
actions.append(
ShellQuoted("source {p}").format(
p=path_join(self.option("prefix"), "venv", "bin", "activate")
)
)
actions.append(self.run(
actions.append(
self.run(
ShellQuoted("python3 -m pip install {deps}").format(
deps=shell_join(' ', (ShellQuoted(dep) for dep in
self.python_deps())))))
return(actions)
deps=shell_join(
" ", (ShellQuoted(dep) for dep in self.python_deps())
)
)
)
)
return actions
def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True):
choices = set(["stable", "beta", "nightly"])
@@ -333,117 +359,148 @@ class FBCodeBuilder(object):
# Only check out a non-default branch if requested. This especially
# makes sense when building from a local repo.
git_hash = self.option(
'{0}:git_hash'.format(project),
"{0}:git_hash".format(project),
# Any repo that has a hash in deps/github_hashes defaults to
# that, with the goal of making builds maximally consistent.
self._github_hashes.get(project, '')
self._github_hashes.get(project, ""),
)
maybe_change_branch = (
[
self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash)),
]
if git_hash
else []
)
maybe_change_branch = [
self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
] if git_hash else []
local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
return self.step('Check out {0}, workdir {1}'.format(project, path), [
local_repo_dir = self.option("{0}:local_repo_dir".format(project), "")
return self.step(
"Check out {0}, workdir {1}".format(project, path),
[
self.workdir(self._github_dir),
self.run(
ShellQuoted('git clone {opts} https://github.com/{p}').format(
ShellQuoted("git clone {opts} https://github.com/{p}").format(
p=project,
opts=ShellQuoted(self.option('{}:git_clone_opts'.format(project), '')))
) if not local_repo_dir else self.copy_local_repo(
local_repo_dir, os.path.basename(project)
opts=ShellQuoted(
self.option("{}:git_clone_opts".format(project), "")
),
)
)
if not local_repo_dir
else self.copy_local_repo(local_repo_dir, os.path.basename(project)),
self.workdir(
path_join(self._github_dir, os.path.basename(project), path),
),
] + maybe_change_branch)
]
+ maybe_change_branch,
)
def fb_github_project_workdir(self, project_and_path, github_org='facebook'):
'This helper lets Facebook-internal CI special-cases FB projects'
project, path = project_and_path.split('/', 1)
return self.github_project_workdir(github_org + '/' + project, path)
def fb_github_project_workdir(self, project_and_path, github_org="facebook"):
"This helper lets Facebook-internal CI special-cases FB projects"
project, path = project_and_path.split("/", 1)
return self.github_project_workdir(github_org + "/" + project, path)
def _make_vars(self, make_vars):
return shell_join(' ', (
ShellQuoted('{k}={v}').format(k=k, v=v)
return shell_join(
" ",
(
ShellQuoted("{k}={v}").format(k=k, v=v)
for k, v in ({} if make_vars is None else make_vars).items()
))
),
)
def parallel_make(self, make_vars=None):
return self.run(ShellQuoted('make -j {n} VERBOSE=1 {vars}').format(
n=self.option('make_parallelism'),
return self.run(
ShellQuoted("make -j {n} VERBOSE=1 {vars}").format(
n=self.option("make_parallelism"),
vars=self._make_vars(make_vars),
))
)
)
def make_and_install(self, make_vars=None):
return [
self.parallel_make(make_vars),
self.run(ShellQuoted('make install VERBOSE=1 {vars}').format(
self.run(
ShellQuoted("make install VERBOSE=1 {vars}").format(
vars=self._make_vars(make_vars),
)),
)
),
]
def configure(self, name=None):
autoconf_options = {}
if name is not None:
autoconf_options.update(
self.option('{0}:autoconf_options'.format(name), {})
self.option("{0}:autoconf_options".format(name), {})
)
return [
self.run(ShellQuoted(
self.run(
ShellQuoted(
'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
'CFLAGS="$CFLAGS -I"{p}"/include" '
'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
'PY_PREFIX={p} '
'./configure --prefix={p} {args}'
"PY_PREFIX={p} "
"./configure --prefix={p} {args}"
).format(
p=self.option('prefix'),
args=shell_join(' ', (
ShellQuoted('{k}={v}').format(k=k, v=v)
p=self.option("prefix"),
args=shell_join(
" ",
(
ShellQuoted("{k}={v}").format(k=k, v=v)
for k, v in autoconf_options.items()
)),
)),
),
),
)
),
]
def autoconf_install(self, name):
return self.step('Build and install {0}'.format(name), [
self.run(ShellQuoted('autoreconf -ivf')),
] + self.configure() + self.make_and_install())
return self.step(
"Build and install {0}".format(name),
[
self.run(ShellQuoted("autoreconf -ivf")),
]
+ self.configure()
+ self.make_and_install(),
)
def cmake_configure(self, name, cmake_path='..'):
def cmake_configure(self, name, cmake_path=".."):
cmake_defines = {
'BUILD_SHARED_LIBS': 'ON',
'CMAKE_INSTALL_PREFIX': self.option('prefix'),
"BUILD_SHARED_LIBS": "ON",
"CMAKE_INSTALL_PREFIX": self.option("prefix"),
}
# Hacks to add thriftpy3 support
if 'BUILD_THRIFT_PY3' in os.environ and 'folly' in name:
cmake_defines['PYTHON_EXTENSIONS'] = 'True'
if "BUILD_THRIFT_PY3" in os.environ and "folly" in name:
cmake_defines["PYTHON_EXTENSIONS"] = "True"
if 'BUILD_THRIFT_PY3' in os.environ and 'fbthrift' in name:
cmake_defines['thriftpy3'] = 'ON'
if "BUILD_THRIFT_PY3" in os.environ and "fbthrift" in name:
cmake_defines["thriftpy3"] = "ON"
cmake_defines.update(
self.option('{0}:cmake_defines'.format(name), {})
)
cmake_defines.update(self.option("{0}:cmake_defines".format(name), {}))
return [
self.run(ShellQuoted(
self.run(
ShellQuoted(
'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" '
'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" '
'cmake {args} {cmake_path}'
"cmake {args} {cmake_path}"
).format(
p=self.option('prefix'),
args=shell_join(' ', (
ShellQuoted('-D{k}={v}').format(k=k, v=v)
p=self.option("prefix"),
args=shell_join(
" ",
(
ShellQuoted("-D{k}={v}").format(k=k, v=v)
for k, v in cmake_defines.items()
)),
),
),
cmake_path=cmake_path,
)),
)
),
]
def cmake_install(self, name, cmake_path='..'):
def cmake_install(self, name, cmake_path=".."):
return self.step(
'Build and install {0}'.format(name),
self.cmake_configure(name, cmake_path) + self.make_and_install()
"Build and install {0}".format(name),
self.cmake_configure(name, cmake_path) + self.make_and_install(),
)
def cargo_build(self, name):
@@ -458,13 +515,15 @@ class FBCodeBuilder(object):
],
)
def fb_github_autoconf_install(self, project_and_path, github_org='facebook'):
def fb_github_autoconf_install(self, project_and_path, github_org="facebook"):
return [
self.fb_github_project_workdir(project_and_path, github_org),
self.autoconf_install(project_and_path),
]
def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org='facebook'):
def fb_github_cmake_install(
self, project_and_path, cmake_path="..", github_org="facebook"
):
return [
self.fb_github_project_workdir(project_and_path, github_org),
self.cmake_install(project_and_path, cmake_path),

11
build/fbcode_builder/fbcode_builder_config.py
View File

@@ -4,12 +4,13 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'Demo config, so that `make_docker_context.py --help` works in this directory.'
"Demo config, so that `make_docker_context.py --help` works in this directory."
config = {
'fbcode_builder_spec': lambda _builder: {
'depends_on': [],
'steps': [],
"fbcode_builder_spec": lambda _builder: {
"depends_on": [],
"steps": [],
},
'github_project': 'demo/project',
"github_project": "demo/project",
}

151
build/fbcode_builder/make_docker_context.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
"""
Reads `fbcode_builder_config.py` from the current directory, and prepares a
Docker context directory to build this project. Prints to stdout the path
to the context directory.
@@ -14,7 +15,7 @@ Try `.../make_docker_context.py --help` from a project's `build/` directory.
By default, the Docker context directory will be in /tmp. It will always
contain a Dockerfile, and might also contain copies of your local repos, and
other data needed for the build container.
'''
"""
import os
import tempfile
@@ -27,7 +28,7 @@ from parse_args import parse_args_to_fbcode_builder_opts
def make_docker_context(
get_steps_fn, github_project, opts=None, default_context_dir=None
):
'''
"""
Returns a path to the Docker context directory. See parse_args.py.
Helper for making a command-line utility that writes your project's
@@ -38,83 +39,97 @@ def make_docker_context(
lambda builder: [builder.step(...), ...],
'facebook/your_project',
))
'''
"""
if opts is None:
opts = {}
valid_versions = (
('ubuntu:16.04', '5'),
('ubuntu:18.04', '7'),
("ubuntu:16.04", "5"),
("ubuntu:18.04", "7"),
)
def add_args(parser):
parser.add_argument(
'--docker-context-dir', metavar='DIR',
"--docker-context-dir",
metavar="DIR",
default=default_context_dir,
help='Write the Dockerfile and its context into this directory. '
'If empty, make a temporary directory. Default: %(default)s.',
help="Write the Dockerfile and its context into this directory. "
"If empty, make a temporary directory. Default: %(default)s.",
)
parser.add_argument(
'--user', metavar='NAME', default=opts.get('user', 'nobody'),
help='Build and install as this user. Default: %(default)s.',
"--user",
metavar="NAME",
default=opts.get("user", "nobody"),
help="Build and install as this user. Default: %(default)s.",
)
parser.add_argument(
'--prefix', metavar='DIR',
default=opts.get('prefix', '/home/install'),
help='Install all libraries in this prefix. Default: %(default)s.',
"--prefix",
metavar="DIR",
default=opts.get("prefix", "/home/install"),
help="Install all libraries in this prefix. Default: %(default)s.",
)
parser.add_argument(
'--projects-dir', metavar='DIR',
default=opts.get('projects_dir', '/home'),
help='Place project code directories here. Default: %(default)s.',
"--projects-dir",
metavar="DIR",
default=opts.get("projects_dir", "/home"),
help="Place project code directories here. Default: %(default)s.",
)
parser.add_argument(
'--os-image', metavar='IMG', choices=zip(*valid_versions)[0],
default=opts.get('os_image', valid_versions[0][0]),
help='Docker OS image -- be sure to use only ones you trust (See '
'README.docker). Choices: %(choices)s. Default: %(default)s.',
"--os-image",
metavar="IMG",
choices=zip(*valid_versions)[0],
default=opts.get("os_image", valid_versions[0][0]),
help="Docker OS image -- be sure to use only ones you trust (See "
"README.docker). Choices: %(choices)s. Default: %(default)s.",
)
parser.add_argument(
'--gcc-version', metavar='VER',
"--gcc-version",
metavar="VER",
choices=set(zip(*valid_versions)[1]),
default=opts.get('gcc_version', valid_versions[0][1]),
help='Choices: %(choices)s. Default: %(default)s.',
default=opts.get("gcc_version", valid_versions[0][1]),
help="Choices: %(choices)s. Default: %(default)s.",
)
parser.add_argument(
'--make-parallelism', metavar='NUM', type=int,
default=opts.get('make_parallelism', 1),
help='Use `make -j` on multi-CPU systems with lots of RAM. '
'Default: %(default)s.',
"--make-parallelism",
metavar="NUM",
type=int,
default=opts.get("make_parallelism", 1),
help="Use `make -j` on multi-CPU systems with lots of RAM. "
"Default: %(default)s.",
)
parser.add_argument(
'--local-repo-dir', metavar='DIR',
help='If set, build {0} from a local directory instead of Github.'
.format(github_project),
"--local-repo-dir",
metavar="DIR",
help="If set, build {0} from a local directory instead of Github.".format(
github_project
),
)
parser.add_argument(
'--ccache-tgz', metavar='PATH',
help='If set, enable ccache for the build. To initialize the '
'cache, first try to hardlink, then to copy --cache-tgz '
'as ccache.tgz into the --docker-context-dir.'
"--ccache-tgz",
metavar="PATH",
help="If set, enable ccache for the build. To initialize the "
"cache, first try to hardlink, then to copy --cache-tgz "
"as ccache.tgz into the --docker-context-dir.",
)
opts = parse_args_to_fbcode_builder_opts(
add_args,
# These have add_argument() calls, others are set via --option.
(
'docker_context_dir',
'user',
'prefix',
'projects_dir',
'os_image',
'gcc_version',
'make_parallelism',
'local_repo_dir',
'ccache_tgz',
"docker_context_dir",
"user",
"prefix",
"projects_dir",
"os_image",
"gcc_version",
"make_parallelism",
"local_repo_dir",
"ccache_tgz",
),
opts,
help=textwrap.dedent('''
help=textwrap.dedent(
"""
Reads `fbcode_builder_config.py` from the current directory, and
prepares a Docker context directory to build {github_project} and
@@ -130,47 +145,55 @@ def make_docker_context(
Usage:
(cd $(./make_docker_context.py) && docker build . 2>&1 | tee log)
'''.format(github_project=github_project)),
""".format(
github_project=github_project
)
),
)
# This allows travis_docker_build.sh not to know the main Github project.
local_repo_dir = opts.pop('local_repo_dir', None)
local_repo_dir = opts.pop("local_repo_dir", None)
if local_repo_dir is not None:
opts['{0}:local_repo_dir'.format(github_project)] = local_repo_dir
opts["{0}:local_repo_dir".format(github_project)] = local_repo_dir
if (opts.get('os_image'), opts.get('gcc_version')) not in valid_versions:
if (opts.get("os_image"), opts.get("gcc_version")) not in valid_versions:
raise Exception(
'Due to 4/5 ABI changes (std::string), we can only use {0}'.format(
' / '.join('GCC {1} on {0}'.format(*p) for p in valid_versions)
"Due to 4/5 ABI changes (std::string), we can only use {0}".format(
" / ".join("GCC {1} on {0}".format(*p) for p in valid_versions)
)
)
if opts.get('docker_context_dir') is None:
opts['docker_context_dir'] = tempfile.mkdtemp(prefix='docker-context-')
elif not os.path.exists(opts.get('docker_context_dir')):
os.makedirs(opts.get('docker_context_dir'))
if opts.get("docker_context_dir") is None:
opts["docker_context_dir"] = tempfile.mkdtemp(prefix="docker-context-")
elif not os.path.exists(opts.get("docker_context_dir")):
os.makedirs(opts.get("docker_context_dir"))
builder = DockerFBCodeBuilder(**opts)
context_dir = builder.option('docker_context_dir') # Mark option "in-use"
context_dir = builder.option("docker_context_dir") # Mark option "in-use"
# The renderer may also populate some files into the context_dir.
dockerfile = builder.render(get_steps_fn(builder))
with os.fdopen(os.open(
os.path.join(context_dir, 'Dockerfile'),
with os.fdopen(
os.open(
os.path.join(context_dir, "Dockerfile"),
os.O_RDWR | os.O_CREAT | os.O_EXCL, # Do not overwrite existing files
0o644,
), 'w') as f:
),
"w",
) as f:
f.write(dockerfile)
return context_dir
if __name__ == '__main__':
if __name__ == "__main__":
from utils import read_fbcode_builder_config, build_fbcode_builder_config
# Load a spec from the current directory
config = read_fbcode_builder_config('fbcode_builder_config.py')
print(make_docker_context(
config = read_fbcode_builder_config("fbcode_builder_config.py")
print(
make_docker_context(
build_fbcode_builder_config(config),
config['github_project'],
))
config["github_project"],
)
)

43
build/fbcode_builder/parse_args.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'Argument parsing logic shared by all fbcode_builder CLI tools.'
"Argument parsing logic shared by all fbcode_builder CLI tools."
import argparse
import logging
@@ -13,7 +14,7 @@ from shell_quoting import raw_shell, ShellQuoted
def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
'''
"""
Provides some standard arguments: --debug, --option, --shell-quoted-option
@@ -26,46 +27,52 @@ def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
`help` is printed in response to the `--help` argument.
'''
"""
top_level_opts = set(top_level_opts)
parser = argparse.ArgumentParser(
description=help,
formatter_class=argparse.RawDescriptionHelpFormatter
description=help, formatter_class=argparse.RawDescriptionHelpFormatter
)
add_args_fn(parser)
parser.add_argument(
'--option', nargs=2, metavar=('KEY', 'VALUE'), action='append',
"--option",
nargs=2,
metavar=("KEY", "VALUE"),
action="append",
default=[
(k, v) for k, v in opts.items()
(k, v)
for k, v in opts.items()
if k not in top_level_opts and not isinstance(v, ShellQuoted)
],
help='Set project-specific options. These are assumed to be raw '
'strings, to be shell-escaped as needed. Default: %(default)s.',
help="Set project-specific options. These are assumed to be raw "
"strings, to be shell-escaped as needed. Default: %(default)s.",
)
parser.add_argument(
'--shell-quoted-option', nargs=2, metavar=('KEY', 'VALUE'),
action='append',
"--shell-quoted-option",
nargs=2,
metavar=("KEY", "VALUE"),
action="append",
default=[
(k, raw_shell(v)) for k, v in opts.items()
(k, raw_shell(v))
for k, v in opts.items()
if k not in top_level_opts and isinstance(v, ShellQuoted)
],
help='Set project-specific options. These are assumed to be shell-'
'quoted, and may be used in commands as-is. Default: %(default)s.',
help="Set project-specific options. These are assumed to be shell-"
"quoted, and may be used in commands as-is. Default: %(default)s.",
)
parser.add_argument('--debug', action='store_true', help='Log more')
parser.add_argument("--debug", action="store_true", help="Log more")
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.debug else logging.INFO,
format='%(levelname)s: %(message)s'
format="%(levelname)s: %(message)s",
)
# Map command-line args back into opts.
logging.debug('opts before command-line arguments: {0}'.format(opts))
logging.debug("opts before command-line arguments: {0}".format(opts))
new_opts = {}
for key in top_level_opts:
@@ -78,6 +85,6 @@ def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
for key, val in args.shell_quoted_option:
new_opts[key] = ShellQuoted(val)
logging.debug('opts after command-line arguments: {0}'.format(new_opts))
logging.debug("opts after command-line arguments: {0}".format(new_opts))
return new_opts

74
build/fbcode_builder/shell_builder.py
View File

@@ -5,7 +5,7 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
"""
shell_builder.py allows running the fbcode_builder logic
on the host rather than in a container.
@@ -17,50 +17,50 @@ any failing step will cause the script to exit with failure.
cd build
python fbcode_builder/shell_builder.py > ~/run.sh
bash ~/run.sh
'''
"""
import os
import distutils.spawn
import os
from fbcode_builder import FBCodeBuilder
from shell_quoting import (
raw_shell, shell_comment, shell_join, ShellQuoted
)
from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted
from utils import recursively_flatten_list
class ShellFBCodeBuilder(FBCodeBuilder):
def _render_impl(self, steps):
return raw_shell(shell_join('\n', recursively_flatten_list(steps)))
return raw_shell(shell_join("\n", recursively_flatten_list(steps)))
def set_env(self, key, value):
return ShellQuoted("export {key}={val}").format(key=key, val=value)
def workdir(self, dir):
return [
ShellQuoted('mkdir -p {d} && cd {d}').format(
d=dir
),
ShellQuoted("mkdir -p {d} && cd {d}").format(d=dir),
]
def run(self, shell_cmd):
return ShellQuoted('{cmd}').format(cmd=shell_cmd)
return ShellQuoted("{cmd}").format(cmd=shell_cmd)
def step(self, name, actions):
assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
b = ShellQuoted('')
return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]
assert "\n" not in name, "Name {0} would span > 1 line".format(name)
b = ShellQuoted("")
return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b]
def setup(self):
steps = [
ShellQuoted('set -exo pipefail'),
] + self.create_python_venv() + self.python_venv()
if self.has_option('ccache_dir'):
ccache_dir = self.option('ccache_dir')
steps = (
[
ShellQuoted("set -exo pipefail"),
]
+ self.create_python_venv()
+ self.python_venv()
)
if self.has_option("ccache_dir"):
ccache_dir = self.option("ccache_dir")
steps += [
ShellQuoted(
# Set CCACHE_DIR before the `ccache` invocations below.
'export CCACHE_DIR={ccache_dir} '
"export CCACHE_DIR={ccache_dir} "
'CC="ccache ${{CC:-gcc}}" CXX="ccache ${{CXX:-g++}}"'
).format(ccache_dir=ccache_dir)
]
@@ -71,44 +71,44 @@ class ShellFBCodeBuilder(FBCodeBuilder):
def copy_local_repo(self, dir, dest_name):
return [
ShellQuoted('cp -r {dir} {dest_name}').format(
dir=dir,
dest_name=dest_name
),
ShellQuoted("cp -r {dir} {dest_name}").format(dir=dir, dest_name=dest_name),
]
def find_project_root():
here = os.path.dirname(os.path.realpath(__file__))
maybe_root = os.path.dirname(os.path.dirname(here))
if os.path.isdir(os.path.join(maybe_root, '.git')):
if os.path.isdir(os.path.join(maybe_root, ".git")):
return maybe_root
raise RuntimeError(
"I expected shell_builder.py to be in the "
"build/fbcode_builder subdir of a git repo")
"build/fbcode_builder subdir of a git repo"
)
def persistent_temp_dir(repo_root):
escaped = repo_root.replace('/', 'sZs').replace('\\', 'sZs').replace(':', '')
return os.path.join(os.path.expandvars("$HOME"), '.fbcode_builder-' + escaped)
escaped = repo_root.replace("/", "sZs").replace("\\", "sZs").replace(":", "")
return os.path.join(os.path.expandvars("$HOME"), ".fbcode_builder-" + escaped)
if __name__ == '__main__':
if __name__ == "__main__":
from utils import read_fbcode_builder_config, build_fbcode_builder_config
repo_root = find_project_root()
temp = persistent_temp_dir(repo_root)
config = read_fbcode_builder_config('fbcode_builder_config.py')
config = read_fbcode_builder_config("fbcode_builder_config.py")
builder = ShellFBCodeBuilder(projects_dir=temp)
if distutils.spawn.find_executable('ccache'):
builder.add_option('ccache_dir',
os.environ.get('CCACHE_DIR', os.path.join(temp, '.ccache')))
builder.add_option('prefix', os.path.join(temp, 'installed'))
builder.add_option('make_parallelism', 4)
if distutils.spawn.find_executable("ccache"):
builder.add_option(
'{project}:local_repo_dir'.format(project=config['github_project']),
repo_root)
"ccache_dir", os.environ.get("CCACHE_DIR", os.path.join(temp, ".ccache"))
)
builder.add_option("prefix", os.path.join(temp, "installed"))
builder.add_option("make_parallelism", 4)
builder.add_option(
"{project}:local_repo_dir".format(project=config["github_project"]), repo_root
)
make_steps = build_fbcode_builder_config(config)
steps = make_steps(builder)
print(builder.render(steps))

64
build/fbcode_builder/shell_quoting.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
"""
Almost every FBCodeBuilder string is ultimately passed to a shell. Escaping
too little or too much tends to be the most common error. The utilities in
@@ -16,15 +17,14 @@ this file give a systematic way of avoiding such bugs:
- Use `path_join` to join path components.
- Use `shell_join` to join already-quoted command arguments or shell lines.
'''
"""
import os
from collections import namedtuple
class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))):
'''
class ShellQuoted(namedtuple("ShellQuoted", ("do_not_use_raw_str",))):
"""
Wrap a string with this to make it transparent to shell_quote(). It
will almost always suffice to use ShellQuoted.format(), path_join(),
@@ -32,27 +32,25 @@ class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))):
If you really must, use raw_shell() to access the raw string.
'''
"""
def __new__(cls, s):
'No need to nest ShellQuoted.'
"No need to nest ShellQuoted."
return super(ShellQuoted, cls).__new__(
cls, s.do_not_use_raw_str if isinstance(s, ShellQuoted) else s
)
def __str__(self):
raise RuntimeError(
'One does not simply convert {0} to a string -- use path_join() '
'or ShellQuoted.format() instead'.format(repr(self))
"One does not simply convert {0} to a string -- use path_join() "
"or ShellQuoted.format() instead".format(repr(self))
)
def __repr__(self):
return '{0}({1})'.format(
self.__class__.__name__, repr(self.do_not_use_raw_str)
)
return "{0}({1})".format(self.__class__.__name__, repr(self.do_not_use_raw_str))
def format(self, **kwargs):
'''
"""
Use instead of str.format() when the arguments are either
`ShellQuoted()` or raw strings needing to be `shell_quote()`d.
@@ -60,40 +58,46 @@ class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))):
Positional args are deliberately not supported since they are more
error-prone.
'''
return ShellQuoted(self.do_not_use_raw_str.format(**dict(
"""
return ShellQuoted(
self.do_not_use_raw_str.format(
**dict(
(k, shell_quote(v).do_not_use_raw_str) for k, v in kwargs.items()
)))
)
)
)
def shell_quote(s):
'Quotes a string if it is not already quoted'
return s if isinstance(s, ShellQuoted) \
"Quotes a string if it is not already quoted"
return (
s
if isinstance(s, ShellQuoted)
else ShellQuoted("'" + str(s).replace("'", "'\\''") + "'")
)
def raw_shell(s):
'Not a member of ShellQuoted so we get a useful error for raw strings'
"Not a member of ShellQuoted so we get a useful error for raw strings"
if isinstance(s, ShellQuoted):
return s.do_not_use_raw_str
raise RuntimeError('{0} should have been ShellQuoted'.format(s))
raise RuntimeError("{0} should have been ShellQuoted".format(s))
def shell_join(delim, it):
'Joins an iterable of ShellQuoted with a delimiter between each two'
"Joins an iterable of ShellQuoted with a delimiter between each two"
return ShellQuoted(delim.join(raw_shell(s) for s in it))
def path_join(*args):
'Joins ShellQuoted and raw pieces of paths to make a shell-quoted path'
return ShellQuoted(os.path.join(*[
raw_shell(shell_quote(s)) for s in args
]))
"Joins ShellQuoted and raw pieces of paths to make a shell-quoted path"
return ShellQuoted(os.path.join(*[raw_shell(shell_quote(s)) for s in args]))
def shell_comment(c):
'Do not shell-escape raw strings in comments, but do handle line breaks.'
return ShellQuoted('# {c}').format(c=ShellQuoted(
(raw_shell(c) if isinstance(c, ShellQuoted) else c)
.replace('\n', '\n# ')
))
"Do not shell-escape raw strings in comments, but do handle line breaks."
return ShellQuoted("# {c}").format(
c=ShellQuoted(
(raw_shell(c) if isinstance(c, ShellQuoted) else c).replace("\n", "\n# ")
)
)

6
build/fbcode_builder/specs/fbthrift.py
View File

@@ -15,8 +15,8 @@ import specs.zstd as zstd
def fbcode_builder_spec(builder):
return {
'depends_on': [fmt, folly, fizz, sodium, wangle, zstd],
'steps': [
builder.fb_github_cmake_install('fbthrift/thrift'),
"depends_on": [fmt, folly, fizz, sodium, wangle, zstd],
"steps": [
builder.fb_github_cmake_install("fbthrift/thrift"),
],
}

43
build/fbcode_builder/specs/fbzmq.py
View File

@@ -10,31 +10,40 @@ import specs.fmt as fmt
import specs.folly as folly
import specs.gmock as gmock
import specs.sodium as sodium
from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder):
builder.add_option('zeromq/libzmq:git_hash', 'v4.2.2')
builder.add_option("zeromq/libzmq:git_hash", "v4.2.2")
return {
'depends_on': [fmt, folly, fbthrift, gmock, sodium],
'steps': [
builder.github_project_workdir('zeromq/libzmq', '.'),
builder.step('Build and install zeromq/libzmq', [
builder.run(ShellQuoted('./autogen.sh')),
"depends_on": [fmt, folly, fbthrift, gmock, sodium],
"steps": [
builder.github_project_workdir("zeromq/libzmq", "."),
builder.step(
"Build and install zeromq/libzmq",
[
builder.run(ShellQuoted("./autogen.sh")),
builder.configure(),
builder.make_and_install(),
]),
builder.fb_github_project_workdir('fbzmq/_build', 'facebook'),
builder.step('Build and install fbzmq/', [
builder.cmake_configure('fbzmq/_build'),
],
),
builder.fb_github_project_workdir("fbzmq/_build", "facebook"),
builder.step(
"Build and install fbzmq/",
[
builder.cmake_configure("fbzmq/_build"),
# we need the pythonpath to find the thrift compiler
builder.run(ShellQuoted(
builder.run(
ShellQuoted(
'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
'make -j {n}'
).format(p=builder.option('prefix'), n=builder.option('make_parallelism'))),
builder.run(ShellQuoted('make install')),
]),
"make -j {n}"
).format(
p=builder.option("prefix"),
n=builder.option("make_parallelism"),
)
),
builder.run(ShellQuoted("make install")),
],
),
],
}

2
build/fbcode_builder/specs/fizz.py
View File

@@ -5,9 +5,9 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.gmock as gmock
import specs.fmt as fmt
import specs.folly as folly
import specs.gmock as gmock
import specs.sodium as sodium

14
build/fbcode_builder/specs/fmt.py
View File

@@ -7,20 +7,20 @@ from __future__ import unicode_literals
def fbcode_builder_spec(builder):
builder.add_option('fmtlib/fmt:git_hash', '6.2.1')
builder.add_option("fmtlib/fmt:git_hash", "6.2.1")
builder.add_option(
'fmtlib/fmt:cmake_defines',
"fmtlib/fmt:cmake_defines",
{
# Avoids a bizarred failure to run tests in Bistro:
# test_crontab_selector: error while loading shared libraries:
# libfmt.so.6: cannot open shared object file:
# No such file or directory
'BUILD_SHARED_LIBS': 'OFF',
}
"BUILD_SHARED_LIBS": "OFF",
},
)
return {
'steps': [
builder.github_project_workdir('fmtlib/fmt', 'build'),
builder.cmake_install('fmtlib/fmt'),
"steps": [
builder.github_project_workdir("fmtlib/fmt", "build"),
builder.cmake_install("fmtlib/fmt"),
],
}

6
build/fbcode_builder/specs/folly.py
View File

@@ -11,13 +11,13 @@ import specs.fmt as fmt
def fbcode_builder_spec(builder):
return {
"depends_on": [fmt],
'steps': [
"steps": [
# on macOS the filesystem is typically case insensitive.
# We need to ensure that the CWD is not the folly source
# dir when we build, otherwise the system will decide
# that `folly/String.h` is the file it wants when including
# `string.h` and the build will fail.
builder.fb_github_project_workdir('folly/_build'),
builder.cmake_install('facebook/folly'),
builder.fb_github_project_workdir("folly/_build"),
builder.cmake_install("facebook/folly"),
],
}

16
build/fbcode_builder/specs/gmock.py
View File

@@ -7,18 +7,18 @@ from __future__ import unicode_literals
def fbcode_builder_spec(builder):
builder.add_option('google/googletest:git_hash', 'release-1.8.1')
builder.add_option("google/googletest:git_hash", "release-1.8.1")
builder.add_option(
'google/googletest:cmake_defines',
"google/googletest:cmake_defines",
{
'BUILD_GTEST': 'ON',
"BUILD_GTEST": "ON",
# Avoid problems with MACOSX_RPATH
'BUILD_SHARED_LIBS': 'OFF',
}
"BUILD_SHARED_LIBS": "OFF",
},
)
return {
'steps': [
builder.github_project_workdir('google/googletest', 'build'),
builder.cmake_install('google/googletest'),
"steps": [
builder.github_project_workdir("google/googletest", "build"),
builder.cmake_install("google/googletest"),
],
}

4
build/fbcode_builder/specs/mvfst.py
View File

@@ -5,9 +5,9 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.gmock as gmock
import specs.folly as folly
import specs.fizz as fizz
import specs.folly as folly
import specs.gmock as gmock
def fbcode_builder_spec(builder):

4
build/fbcode_builder/specs/proxygen.py
View File

@@ -5,10 +5,10 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.gmock as gmock
import specs.fizz as fizz
import specs.fmt as fmt
import specs.folly as folly
import specs.fizz as fizz
import specs.gmock as gmock
import specs.mvfst as mvfst
import specs.sodium as sodium
import specs.wangle as wangle

4
build/fbcode_builder/specs/proxygen_quic.py
View File

@@ -5,10 +5,10 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.gmock as gmock
import specs.fizz as fizz
import specs.fmt as fmt
import specs.folly as folly
import specs.fizz as fizz
import specs.gmock as gmock
import specs.mvfst as mvfst
import specs.sodium as sodium
import specs.wangle as wangle

6
build/fbcode_builder/specs/re2.py
View File

@@ -8,8 +8,8 @@ from __future__ import unicode_literals
def fbcode_builder_spec(builder):
return {
'steps': [
builder.github_project_workdir('google/re2', 'build'),
builder.cmake_install('google/re2'),
"steps": [
builder.github_project_workdir("google/re2", "build"),
builder.cmake_install("google/re2"),
],
}

7
build/fbcode_builder/specs/rocksdb.py
View File

@@ -7,10 +7,13 @@ from __future__ import unicode_literals
def fbcode_builder_spec(builder):
builder.add_option("rocksdb/_build:cmake_defines", {
builder.add_option(
"rocksdb/_build:cmake_defines",
{
"USE_RTTI": "1",
"PORTABLE": "ON",
})
},
)
return {
"steps": [
builder.fb_github_cmake_install("rocksdb/_build"),

15
build/fbcode_builder/specs/sodium.py
View File

@@ -9,14 +9,17 @@ from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder):
builder.add_option('jedisct1/libsodium:git_hash', 'stable')
builder.add_option("jedisct1/libsodium:git_hash", "stable")
return {
'steps': [
builder.github_project_workdir('jedisct1/libsodium', '.'),
builder.step('Build and install jedisct1/libsodium', [
builder.run(ShellQuoted('./autogen.sh')),
"steps": [
builder.github_project_workdir("jedisct1/libsodium", "."),
builder.step(
"Build and install jedisct1/libsodium",
[
builder.run(ShellQuoted("./autogen.sh")),
builder.configure(),
builder.make_and_install(),
]),
],
),
],
}

4
build/fbcode_builder/specs/wangle.py
View File

@@ -5,10 +5,10 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.gmock as gmock
import specs.fizz as fizz
import specs.fmt as fmt
import specs.folly as folly
import specs.fizz as fizz
import specs.gmock as gmock
import specs.sodium as sodium

23
build/fbcode_builder/specs/zstd.py
View File

@@ -11,16 +11,21 @@ from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder):
# This API should change rarely, so build the latest tag instead of master.
builder.add_option(
'facebook/zstd:git_hash',
ShellQuoted('$(git describe --abbrev=0 --tags origin/master)')
"facebook/zstd:git_hash",
ShellQuoted("$(git describe --abbrev=0 --tags origin/master)"),
)
return {
'steps': [
builder.github_project_workdir('facebook/zstd', '.'),
builder.step('Build and install zstd', [
builder.make_and_install(make_vars={
'PREFIX': builder.option('prefix'),
})
]),
"steps": [
builder.github_project_workdir("facebook/zstd", "."),
builder.step(
"Build and install zstd",
[
builder.make_and_install(
make_vars={
"PREFIX": builder.option("prefix"),
}
)
],
),
],
}

43
build/fbcode_builder/utils.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'Miscellaneous utility functions.'
"Miscellaneous utility functions."
import itertools
import logging
@@ -12,21 +13,19 @@ import os
import shutil
import subprocess
import sys
from contextlib import contextmanager
def recursively_flatten_list(l):
return itertools.chain.from_iterable(
(recursively_flatten_list(i) if type(i) is list else (i,))
for i in l
(recursively_flatten_list(i) if type(i) is list else (i,)) for i in l
)
def run_command(*cmd, **kwargs):
'The stdout of most fbcode_builder utilities is meant to be parsed.'
logging.debug('Running: {0} with {1}'.format(cmd, kwargs))
kwargs['stdout'] = sys.stderr
"The stdout of most fbcode_builder utilities is meant to be parsed."
logging.debug("Running: {0} with {1}".format(cmd, kwargs))
kwargs["stdout"] = sys.stderr
subprocess.check_call(cmd, **kwargs)
@@ -40,13 +39,13 @@ def make_temp_dir(d):
def _inner_read_config(path):
'''
"""
Helper to read a named config file.
The grossness with the global is a workaround for this python bug:
https://bugs.python.org/issue21591
The bug prevents us from defining either a local function or a lambda
in the scope of read_fbcode_builder_config below.
'''
"""
global _project_dir
full_path = os.path.join(_project_dir, path)
return read_fbcode_builder_config(full_path)
@@ -60,37 +59,37 @@ def read_fbcode_builder_config(filename):
global _project_dir
_project_dir = os.path.dirname(filename)
scope = {'read_fbcode_builder_config': _inner_read_config}
scope = {"read_fbcode_builder_config": _inner_read_config}
with open(filename) as config_file:
code = compile(config_file.read(), filename, mode='exec')
code = compile(config_file.read(), filename, mode="exec")
exec(code, scope)
return scope['config']
return scope["config"]
def steps_for_spec(builder, spec, processed_modules=None):
'''
"""
Sets `builder` configuration, and returns all the builder steps
necessary to build `spec` and its dependencies.
Traverses the dependencies in depth-first order, honoring the sequencing
in each 'depends_on' list.
'''
"""
if processed_modules is None:
processed_modules = set()
steps = []
for module in spec.get('depends_on', []):
for module in spec.get("depends_on", []):
if module not in processed_modules:
processed_modules.add(module)
steps.extend(steps_for_spec(
builder,
module.fbcode_builder_spec(builder),
processed_modules
))
steps.extend(spec.get('steps', []))
steps.extend(
steps_for_spec(
builder, module.fbcode_builder_spec(builder), processed_modules
)
)
steps.extend(spec.get("steps", []))
return steps
def build_fbcode_builder_config(config):
return lambda builder: builder.build(
steps_for_spec(builder, config['fbcode_builder_spec'](builder))
steps_for_spec(builder, config["fbcode_builder_spec"](builder))
)