lib/proxygen
1
0
Fork 0
mirror of https://github.com/facebook/proxygen.git synced 2025-08-05 19:55:47 +03:00
Code Activity

Opt in opensource/fbcode_builder to pyfmt

Browse Source 
Reviewed By: zertosh

Differential Revision: D29612107

fbshipit-source-id: ac450058134e23a3831db35d2e49c80eb8cde36a
This commit is contained in:
Zsolt Dollenstein
2021-07-09 06:17:52 -07:00
committed by Facebook GitHub Bot
parent f3b62e2316
commit 40b6e08b08
24 changed files with 612 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
build/fbcode_builder/CMake/fb_py_test_main.py
View File

@@ -485,7 +485,7 @@ class Loader(object):
return loader.suiteClass(suites) return loader.suiteClass(suites)
_COVERAGE_INI = '''\ _COVERAGE_INI = """\
[report] [report]
exclude_lines = exclude_lines =
pragma: no cover pragma: no cover
@@ -495,7 +495,7 @@ exclude_lines =
pragma:.*no${PY_IMPL}${PY_MAJOR} pragma:.*no${PY_IMPL}${PY_MAJOR}
pragma:.*nopy${PY_MAJOR} pragma:.*nopy${PY_MAJOR}
pragma:.*nopy${PY_MAJOR}${PY_MINOR} pragma:.*nopy${PY_MAJOR}${PY_MINOR}
''' """
class MainProgram(object): class MainProgram(object):
@@ -734,7 +734,7 @@ class MainProgram(object):
if not self.options.collect_coverage: if not self.options.collect_coverage:
return return
with tempfile.NamedTemporaryFile('w', delete=False) as coverage_ini: with tempfile.NamedTemporaryFile("w", delete=False) as coverage_ini:
coverage_ini.write(_COVERAGE_INI) coverage_ini.write(_COVERAGE_INI)
self._coverage_ini_path = coverage_ini.name self._coverage_ini_path = coverage_ini.name

6
build/fbcode_builder/CMake/make_fbpy_archive.py
View File

@@ -124,7 +124,7 @@ def populate_install_tree(inst_dir, path_map):
def build_zipapp(args, path_map): def build_zipapp(args, path_map):
""" Create a self executing python binary using Python 3's built-in """Create a self executing python binary using Python 3's built-in
zipapp module. zipapp module.
This type of Python binary is relatively simple, as zipapp is part of the This type of Python binary is relatively simple, as zipapp is part of the
@@ -165,7 +165,7 @@ if __name__ == "__main__":
def build_install_dir(args, path_map): def build_install_dir(args, path_map):
""" Create a directory that contains all of the sources, with a __main__ """Create a directory that contains all of the sources, with a __main__
module to run the program. module to run the program.
""" """
# Populate a temporary directory first, then rename to the destination # Populate a temporary directory first, then rename to the destination
@@ -188,7 +188,7 @@ def ensure_directory(path):
def install_library(args, path_map): def install_library(args, path_map):
""" Create an installation directory a python library. """ """Create an installation directory a python library."""
out_dir = args.output out_dir = args.output
out_manifest = args.output + ".manifest" out_manifest = args.output + ".manifest"

121
build/fbcode_builder/docker_builder.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
'''
"""
Extends FBCodeBuilder to produce Docker context directories. Extends FBCodeBuilder to produce Docker context directories.
@@ -15,26 +16,23 @@ caching, you will want to:
that change the least often, and that change the least often, and
- Put the steps that you are debugging towards the very end. - Put the steps that you are debugging towards the very end.
''' """
import logging import logging
import os import os
import shutil import shutil
import tempfile import tempfile
from fbcode_builder import FBCodeBuilder from fbcode_builder import FBCodeBuilder
from shell_quoting import ( from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted, path_join
raw_shell, shell_comment, shell_join, ShellQuoted, path_join
)
from utils import recursively_flatten_list, run_command from utils import recursively_flatten_list, run_command
class DockerFBCodeBuilder(FBCodeBuilder): class DockerFBCodeBuilder(FBCodeBuilder):
def _user(self): def _user(self):
return self.option('user', 'root') return self.option("user", "root")
def _change_user(self): def _change_user(self):
return ShellQuoted('USER {u}').format(u=self._user()) return ShellQuoted("USER {u}").format(u=self._user())
def setup(self): def setup(self):
# Please add RPM-based OSes here as appropriate. # Please add RPM-based OSes here as appropriate.
@@ -63,17 +61,18 @@ class DockerFBCodeBuilder(FBCodeBuilder):
# it is present when the resulting container is run add to PATH # it is present when the resulting container is run add to PATH
actions = [] actions = []
if self.option("PYTHON_VENV", "OFF") == "ON": if self.option("PYTHON_VENV", "OFF") == "ON":
actions = ShellQuoted('ENV PATH={p}:$PATH').format( actions = ShellQuoted("ENV PATH={p}:$PATH").format(
p=path_join(self.option('prefix'), "venv", "bin")) p=path_join(self.option("prefix"), "venv", "bin")
return(actions) )
return actions
def step(self, name, actions): def step(self, name, actions):
assert '\n' not in name, 'Name {0} would span > 1 line'.format(name) assert "\n" not in name, "Name {0} would span > 1 line".format(name)
b = ShellQuoted('') b = ShellQuoted("")
return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b] return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b]
def run(self, shell_cmd): def run(self, shell_cmd):
return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd) return ShellQuoted("RUN {cmd}").format(cmd=shell_cmd)
def set_env(self, key, value): def set_env(self, key, value):
return ShellQuoted("ENV {key}={val}").format(key=key, val=value) return ShellQuoted("ENV {key}={val}").format(key=key, val=value)
@@ -84,12 +83,12 @@ class DockerFBCodeBuilder(FBCodeBuilder):
# by root:root -- the explicit `mkdir` works around the bug: # by root:root -- the explicit `mkdir` works around the bug:
# USER nobody # USER nobody
# WORKDIR build # WORKDIR build
ShellQuoted('USER root'), ShellQuoted("USER root"),
ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format( ShellQuoted("RUN mkdir -p {d} && chown {u} {d}").format(
d=dir, u=self._user() d=dir, u=self._user()
), ),
self._change_user(), self._change_user(),
ShellQuoted('WORKDIR {dir}').format(dir=dir), ShellQuoted("WORKDIR {dir}").format(dir=dir),
] ]
def comment(self, comment): def comment(self, comment):
@@ -99,60 +98,58 @@ class DockerFBCodeBuilder(FBCodeBuilder):
def copy_local_repo(self, repo_dir, dest_name): def copy_local_repo(self, repo_dir, dest_name):
fd, archive_path = tempfile.mkstemp( fd, archive_path = tempfile.mkstemp(
prefix='local_repo_{0}_'.format(dest_name), prefix="local_repo_{0}_".format(dest_name),
suffix='.tgz', suffix=".tgz",
dir=os.path.abspath(self.option('docker_context_dir')), dir=os.path.abspath(self.option("docker_context_dir")),
) )
os.close(fd) os.close(fd)
run_command('tar', 'czf', archive_path, '.', cwd=repo_dir) run_command("tar", "czf", archive_path, ".", cwd=repo_dir)
return [ return [
ShellQuoted('ADD {archive} {dest_name}').format( ShellQuoted("ADD {archive} {dest_name}").format(
archive=os.path.basename(archive_path), dest_name=dest_name archive=os.path.basename(archive_path), dest_name=dest_name
), ),
# Docker permissions make very little sense... see also workdir() # Docker permissions make very little sense... see also workdir()
ShellQuoted('USER root'), ShellQuoted("USER root"),
ShellQuoted('RUN chown -R {u} {d}').format( ShellQuoted("RUN chown -R {u} {d}").format(d=dest_name, u=self._user()),
d=dest_name, u=self._user()
),
self._change_user(), self._change_user(),
] ]
def _render_impl(self, steps): def _render_impl(self, steps):
return raw_shell(shell_join('\n', recursively_flatten_list(steps))) return raw_shell(shell_join("\n", recursively_flatten_list(steps)))
def debian_ccache_setup_steps(self): def debian_ccache_setup_steps(self):
source_ccache_tgz = self.option('ccache_tgz', '') source_ccache_tgz = self.option("ccache_tgz", "")
if not source_ccache_tgz: if not source_ccache_tgz:
logging.info('Docker ccache not enabled') logging.info("Docker ccache not enabled")
return [] return []
dest_ccache_tgz = os.path.join( dest_ccache_tgz = os.path.join(self.option("docker_context_dir"), "ccache.tgz")
self.option('docker_context_dir'), 'ccache.tgz'
)
try: try:
try: try:
os.link(source_ccache_tgz, dest_ccache_tgz) os.link(source_ccache_tgz, dest_ccache_tgz)
except OSError: except OSError:
logging.exception( logging.exception(
'Hard-linking {s} to {d} failed, falling back to copy' "Hard-linking {s} to {d} failed, falling back to copy".format(
.format(s=source_ccache_tgz, d=dest_ccache_tgz) s=source_ccache_tgz, d=dest_ccache_tgz
)
) )
shutil.copyfile(source_ccache_tgz, dest_ccache_tgz) shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
except Exception: except Exception:
logging.exception( logging.exception(
'Failed to copy or link {s} to {d}, aborting' "Failed to copy or link {s} to {d}, aborting".format(
.format(s=source_ccache_tgz, d=dest_ccache_tgz) s=source_ccache_tgz, d=dest_ccache_tgz
)
) )
raise raise
return [ return [
# Separate layer so that in development we avoid re-downloads. # Separate layer so that in development we avoid re-downloads.
self.run(ShellQuoted('apt-get install -yq ccache')), self.run(ShellQuoted("apt-get install -yq ccache")),
ShellQuoted('ADD ccache.tgz /'), ShellQuoted("ADD ccache.tgz /"),
ShellQuoted( ShellQuoted(
# Set CCACHE_DIR before the `ccache` invocations below. # Set CCACHE_DIR before the `ccache` invocations below.
'ENV CCACHE_DIR=/ccache ' "ENV CCACHE_DIR=/ccache "
# No clang support for now, so it's easiest to hardcode gcc. # No clang support for now, so it's easiest to hardcode gcc.
'CC="ccache gcc" CXX="ccache g++" ' 'CC="ccache gcc" CXX="ccache g++" '
# Always log for ease of debugging. For real FB projects, # Always log for ease of debugging. For real FB projects,
@@ -166,26 +163,28 @@ class DockerFBCodeBuilder(FBCodeBuilder):
# #
# apt-get install sharutils # apt-get install sharutils
# bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2 # bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
'CCACHE_LOGFILE=/tmp/ccache.log' "CCACHE_LOGFILE=/tmp/ccache.log"
),
self.run(
ShellQuoted(
# Future: Skipping this part made this Docker step instant,
# saving ~1min of build time. It's unclear if it is the
# chown or the du, but probably the chown -- since a large
# part of the cost is incurred at image save time.
#
# ccache.tgz may be empty, or may have the wrong
# permissions.
"mkdir -p /ccache && time chown -R nobody /ccache && "
"time du -sh /ccache && "
# Reset stats so `docker_build_with_ccache.sh` can print
# useful values at the end of the run.
"echo === Prev run stats === && ccache -s && ccache -z && "
# Record the current time to let travis_build.sh figure out
# the number of bytes in the cache that are actually used --
# this is crucial for tuning the maximum cache size.
"date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && "
# The build running as `nobody` should be able to write here
"chown nobody /tmp/ccache.log"
)
), ),
self.run(ShellQuoted(
# Future: Skipping this part made this Docker step instant,
# saving ~1min of build time. It's unclear if it is the
# chown or the du, but probably the chown -- since a large
# part of the cost is incurred at image save time.
#
# ccache.tgz may be empty, or may have the wrong
# permissions.
'mkdir -p /ccache && time chown -R nobody /ccache && '
'time du -sh /ccache && '
# Reset stats so `docker_build_with_ccache.sh` can print
# useful values at the end of the run.
'echo === Prev run stats === && ccache -s && ccache -z && '
# Record the current time to let travis_build.sh figure out
# the number of bytes in the cache that are actually used --
# this is crucial for tuning the maximum cache size.
'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && '
# The build running as `nobody` should be able to write here
'chown nobody /tmp/ccache.log'
)),
] ]

411
build/fbcode_builder/fbcode_builder.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
'''
"""
This is a small DSL to describe builds of Facebook's open-source projects This is a small DSL to describe builds of Facebook's open-source projects
that are published to Github from a single internal repo, including projects that are published to Github from a single internal repo, including projects
@@ -57,7 +58,7 @@ Ideas for the future -- these may not be very good :)
* do `make` and `cmake` * do `make` and `cmake`
- If we get non-Debian OSes, part of ccache setup should be factored out. - If we get non-Debian OSes, part of ccache setup should be factored out.
''' """
import os import os
import re import re
@@ -66,22 +67,21 @@ from shell_quoting import path_join, shell_join, ShellQuoted
def _read_project_github_hashes(): def _read_project_github_hashes():
base_dir = 'deps/github_hashes/' # trailing slash used in regex below base_dir = "deps/github_hashes/" # trailing slash used in regex below
for dirname, _, files in os.walk(base_dir): for dirname, _, files in os.walk(base_dir):
for filename in files: for filename in files:
path = os.path.join(dirname, filename) path = os.path.join(dirname, filename)
with open(path) as f: with open(path) as f:
m_proj = re.match('^' + base_dir + '(.*)-rev\.txt$', path) m_proj = re.match("^" + base_dir + "(.*)-rev\.txt$", path)
if m_proj is None: if m_proj is None:
raise RuntimeError('Not a hash file? {0}'.format(path)) raise RuntimeError("Not a hash file? {0}".format(path))
m_hash = re.match('^Subproject commit ([0-9a-f]+)\n$', f.read()) m_hash = re.match("^Subproject commit ([0-9a-f]+)\n$", f.read())
if m_hash is None: if m_hash is None:
raise RuntimeError('No hash in {0}'.format(path)) raise RuntimeError("No hash in {0}".format(path))
yield m_proj.group(1), m_hash.group(1) yield m_proj.group(1), m_hash.group(1)
class FBCodeBuilder(object): class FBCodeBuilder(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
self._options_do_not_access = kwargs # Use .option() instead. self._options_do_not_access = kwargs # Use .option() instead.
# This raises upon detecting options that are specified but unused, # This raises upon detecting options that are specified but unused,
@@ -90,22 +90,22 @@ class FBCodeBuilder(object):
# Mark 'projects_dir' used even if the build installs no github # Mark 'projects_dir' used even if the build installs no github
# projects. This is needed because driver programs like # projects. This is needed because driver programs like
# `shell_builder.py` unconditionally set this for all builds. # `shell_builder.py` unconditionally set this for all builds.
self._github_dir = self.option('projects_dir') self._github_dir = self.option("projects_dir")
self._github_hashes = dict(_read_project_github_hashes()) self._github_hashes = dict(_read_project_github_hashes())
def __repr__(self): def __repr__(self):
return '{0}({1})'.format( return "{0}({1})".format(
self.__class__.__name__, self.__class__.__name__,
', '.join( ", ".join(
'{0}={1}'.format(k, repr(v)) "{0}={1}".format(k, repr(v))
for k, v in self._options_do_not_access.items() for k, v in self._options_do_not_access.items()
) ),
) )
def option(self, name, default=None): def option(self, name, default=None):
value = self._options_do_not_access.get(name, default) value = self._options_do_not_access.get(name, default)
if value is None: if value is None:
raise RuntimeError('Option {0} is required'.format(name)) raise RuntimeError("Option {0} is required".format(name))
self.options_used.add(name) self.options_used.add(name)
return value return value
@@ -114,7 +114,7 @@ class FBCodeBuilder(object):
def add_option(self, name, value): def add_option(self, name, value):
if name in self._options_do_not_access: if name in self._options_do_not_access:
raise RuntimeError('Option {0} already set'.format(name)) raise RuntimeError("Option {0} already set".format(name))
self._options_do_not_access[name] = value self._options_do_not_access[name] = value
# #
@@ -122,12 +122,12 @@ class FBCodeBuilder(object):
# #
def render(self, steps): def render(self, steps):
''' """
Converts nested actions to your builder's expected output format. Converts nested actions to your builder's expected output format.
Typically takes the output of build(). Typically takes the output of build().
''' """
res = self._render_impl(steps) # Implementation-dependent res = self._render_impl(steps) # Implementation-dependent
# Now that the output is rendered, we expect all options to have # Now that the output is rendered, we expect all options to have
# been used. # been used.
@@ -135,41 +135,46 @@ class FBCodeBuilder(object):
unused_options -= self.options_used unused_options -= self.options_used
if unused_options: if unused_options:
raise RuntimeError( raise RuntimeError(
'Unused options: {0} -- please check if you made a typo ' "Unused options: {0} -- please check if you made a typo "
'in any of them. Those that are truly not useful should ' "in any of them. Those that are truly not useful should "
'be not be set so that this typo detection can be useful.' "be not be set so that this typo detection can be useful.".format(
.format(unused_options) unused_options
)
) )
return res return res
def build(self, steps): def build(self, steps):
if not steps: if not steps:
raise RuntimeError('Please ensure that the config you are passing ' raise RuntimeError(
'contains steps') "Please ensure that the config you are passing " "contains steps"
)
return [self.setup(), self.diagnostics()] + steps return [self.setup(), self.diagnostics()] + steps
def setup(self): def setup(self):
'Your builder may want to install packages here.' "Your builder may want to install packages here."
raise NotImplementedError raise NotImplementedError
def diagnostics(self): def diagnostics(self):
'Log some system diagnostics before/after setup for ease of debugging' "Log some system diagnostics before/after setup for ease of debugging"
# The builder's repr is not used in a command to avoid pointlessly # The builder's repr is not used in a command to avoid pointlessly
# invalidating Docker's build cache. # invalidating Docker's build cache.
return self.step('Diagnostics', [ return self.step(
self.comment('Builder {0}'.format(repr(self))), "Diagnostics",
self.run(ShellQuoted('hostname')), [
self.run(ShellQuoted('cat /etc/issue || echo no /etc/issue')), self.comment("Builder {0}".format(repr(self))),
self.run(ShellQuoted('g++ --version || echo g++ not installed')), self.run(ShellQuoted("hostname")),
self.run(ShellQuoted('cmake --version || echo cmake not installed')), self.run(ShellQuoted("cat /etc/issue || echo no /etc/issue")),
]) self.run(ShellQuoted("g++ --version || echo g++ not installed")),
self.run(ShellQuoted("cmake --version || echo cmake not installed")),
],
)
def step(self, name, actions): def step(self, name, actions):
'A labeled collection of actions or other steps' "A labeled collection of actions or other steps"
raise NotImplementedError raise NotImplementedError
def run(self, shell_cmd): def run(self, shell_cmd):
'Run this bash command' "Run this bash command"
raise NotImplementedError raise NotImplementedError
def set_env(self, key, value): def set_env(self, key, value):
@@ -177,54 +182,54 @@ class FBCodeBuilder(object):
raise NotImplementedError raise NotImplementedError
def workdir(self, dir): def workdir(self, dir):
'Create this directory if it does not exist, and change into it' "Create this directory if it does not exist, and change into it"
raise NotImplementedError raise NotImplementedError
def copy_local_repo(self, dir, dest_name): def copy_local_repo(self, dir, dest_name):
''' """
Copy the local repo at `dir` into this step's `workdir()`, analog of: Copy the local repo at `dir` into this step's `workdir()`, analog of:
cp -r /path/to/folly folly cp -r /path/to/folly folly
''' """
raise NotImplementedError raise NotImplementedError
def python_deps(self): def python_deps(self):
return [ return [
'wheel', "wheel",
'cython==0.28.6', "cython==0.28.6",
] ]
def debian_deps(self): def debian_deps(self):
return [ return [
'autoconf-archive', "autoconf-archive",
'bison', "bison",
'build-essential', "build-essential",
'cmake', "cmake",
'curl', "curl",
'flex', "flex",
'git', "git",
'gperf', "gperf",
'joe', "joe",
'libboost-all-dev', "libboost-all-dev",
'libcap-dev', "libcap-dev",
'libdouble-conversion-dev', "libdouble-conversion-dev",
'libevent-dev', "libevent-dev",
'libgflags-dev', "libgflags-dev",
'libgoogle-glog-dev', "libgoogle-glog-dev",
'libkrb5-dev', "libkrb5-dev",
'libpcre3-dev', "libpcre3-dev",
'libpthread-stubs0-dev', "libpthread-stubs0-dev",
'libnuma-dev', "libnuma-dev",
'libsasl2-dev', "libsasl2-dev",
'libsnappy-dev', "libsnappy-dev",
'libsqlite3-dev', "libsqlite3-dev",
'libssl-dev', "libssl-dev",
'libtool', "libtool",
'netcat-openbsd', "netcat-openbsd",
'pkg-config', "pkg-config",
'sudo', "sudo",
'unzip', "unzip",
'wget', "wget",
'python3-venv', "python3-venv",
] ]
# #
@@ -234,51 +239,72 @@ class FBCodeBuilder(object):
def install_debian_deps(self): def install_debian_deps(self):
actions = [ actions = [
self.run( self.run(
ShellQuoted('apt-get update && apt-get install -yq {deps}').format( ShellQuoted("apt-get update && apt-get install -yq {deps}").format(
deps=shell_join(' ', ( deps=shell_join(
ShellQuoted(dep) for dep in self.debian_deps()))) " ", (ShellQuoted(dep) for dep in self.debian_deps())
)
)
), ),
] ]
gcc_version = self.option('gcc_version') gcc_version = self.option("gcc_version")
# Make the selected GCC the default before building anything # Make the selected GCC the default before building anything
actions.extend([ actions.extend(
self.run(ShellQuoted('apt-get install -yq {c} {cpp}').format( [
c=ShellQuoted('gcc-{v}').format(v=gcc_version), self.run(
cpp=ShellQuoted('g++-{v}').format(v=gcc_version), ShellQuoted("apt-get install -yq {c} {cpp}").format(
)), c=ShellQuoted("gcc-{v}").format(v=gcc_version),
self.run(ShellQuoted( cpp=ShellQuoted("g++-{v}").format(v=gcc_version),
'update-alternatives --install /usr/bin/gcc gcc {c} 40 ' )
'--slave /usr/bin/g++ g++ {cpp}' ),
).format( self.run(
c=ShellQuoted('/usr/bin/gcc-{v}').format(v=gcc_version), ShellQuoted(
cpp=ShellQuoted('/usr/bin/g++-{v}').format(v=gcc_version), "update-alternatives --install /usr/bin/gcc gcc {c} 40 "
)), "--slave /usr/bin/g++ g++ {cpp}"
self.run(ShellQuoted('update-alternatives --config gcc')), ).format(
]) c=ShellQuoted("/usr/bin/gcc-{v}").format(v=gcc_version),
cpp=ShellQuoted("/usr/bin/g++-{v}").format(v=gcc_version),
)
),
self.run(ShellQuoted("update-alternatives --config gcc")),
]
)
actions.extend(self.debian_ccache_setup_steps()) actions.extend(self.debian_ccache_setup_steps())
return self.step('Install packages for Debian-based OS', actions) return self.step("Install packages for Debian-based OS", actions)
def create_python_venv(self): def create_python_venv(self):
actions = [] actions = []
if self.option("PYTHON_VENV", "OFF") == "ON": if self.option("PYTHON_VENV", "OFF") == "ON":
actions.append(self.run(ShellQuoted("python3 -m venv {p}").format( actions.append(
p=path_join(self.option('prefix'), "venv")))) self.run(
return(actions) ShellQuoted("python3 -m venv {p}").format(
p=path_join(self.option("prefix"), "venv")
)
)
)
return actions
def python_venv(self): def python_venv(self):
actions = [] actions = []
if self.option("PYTHON_VENV", "OFF") == "ON": if self.option("PYTHON_VENV", "OFF") == "ON":
actions.append(ShellQuoted("source {p}").format( actions.append(
p=path_join(self.option('prefix'), "venv", "bin", "activate"))) ShellQuoted("source {p}").format(
p=path_join(self.option("prefix"), "venv", "bin", "activate")
)
)
actions.append(self.run( actions.append(
ShellQuoted("python3 -m pip install {deps}").format( self.run(
deps=shell_join(' ', (ShellQuoted(dep) for dep in ShellQuoted("python3 -m pip install {deps}").format(
self.python_deps()))))) deps=shell_join(
return(actions) " ", (ShellQuoted(dep) for dep in self.python_deps())
)
)
)
)
return actions
def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True): def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True):
choices = set(["stable", "beta", "nightly"]) choices = set(["stable", "beta", "nightly"])
@@ -333,117 +359,148 @@ class FBCodeBuilder(object):
# Only check out a non-default branch if requested. This especially # Only check out a non-default branch if requested. This especially
# makes sense when building from a local repo. # makes sense when building from a local repo.
git_hash = self.option( git_hash = self.option(
'{0}:git_hash'.format(project), "{0}:git_hash".format(project),
# Any repo that has a hash in deps/github_hashes defaults to # Any repo that has a hash in deps/github_hashes defaults to
# that, with the goal of making builds maximally consistent. # that, with the goal of making builds maximally consistent.
self._github_hashes.get(project, '') self._github_hashes.get(project, ""),
)
maybe_change_branch = (
[
self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash)),
]
if git_hash
else []
) )
maybe_change_branch = [
self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
] if git_hash else []
local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '') local_repo_dir = self.option("{0}:local_repo_dir".format(project), "")
return self.step('Check out {0}, workdir {1}'.format(project, path), [ return self.step(
self.workdir(self._github_dir), "Check out {0}, workdir {1}".format(project, path),
self.run( [
ShellQuoted('git clone {opts} https://github.com/{p}').format( self.workdir(self._github_dir),
p=project, self.run(
opts=ShellQuoted(self.option('{}:git_clone_opts'.format(project), ''))) ShellQuoted("git clone {opts} https://github.com/{p}").format(
) if not local_repo_dir else self.copy_local_repo( p=project,
local_repo_dir, os.path.basename(project) opts=ShellQuoted(
), self.option("{}:git_clone_opts".format(project), "")
self.workdir( ),
path_join(self._github_dir, os.path.basename(project), path), )
), )
] + maybe_change_branch) if not local_repo_dir
else self.copy_local_repo(local_repo_dir, os.path.basename(project)),
self.workdir(
path_join(self._github_dir, os.path.basename(project), path),
),
]
+ maybe_change_branch,
)
def fb_github_project_workdir(self, project_and_path, github_org='facebook'): def fb_github_project_workdir(self, project_and_path, github_org="facebook"):
'This helper lets Facebook-internal CI special-cases FB projects' "This helper lets Facebook-internal CI special-cases FB projects"
project, path = project_and_path.split('/', 1) project, path = project_and_path.split("/", 1)
return self.github_project_workdir(github_org + '/' + project, path) return self.github_project_workdir(github_org + "/" + project, path)
def _make_vars(self, make_vars): def _make_vars(self, make_vars):
return shell_join(' ', ( return shell_join(
ShellQuoted('{k}={v}').format(k=k, v=v) " ",
(
ShellQuoted("{k}={v}").format(k=k, v=v)
for k, v in ({} if make_vars is None else make_vars).items() for k, v in ({} if make_vars is None else make_vars).items()
)) ),
)
def parallel_make(self, make_vars=None): def parallel_make(self, make_vars=None):
return self.run(ShellQuoted('make -j {n} VERBOSE=1 {vars}').format( return self.run(
n=self.option('make_parallelism'), ShellQuoted("make -j {n} VERBOSE=1 {vars}").format(
vars=self._make_vars(make_vars), n=self.option("make_parallelism"),
)) vars=self._make_vars(make_vars),
)
)
def make_and_install(self, make_vars=None): def make_and_install(self, make_vars=None):
return [ return [
self.parallel_make(make_vars), self.parallel_make(make_vars),
self.run(ShellQuoted('make install VERBOSE=1 {vars}').format( self.run(
vars=self._make_vars(make_vars), ShellQuoted("make install VERBOSE=1 {vars}").format(
)), vars=self._make_vars(make_vars),
)
),
] ]
def configure(self, name=None): def configure(self, name=None):
autoconf_options = {} autoconf_options = {}
if name is not None: if name is not None:
autoconf_options.update( autoconf_options.update(
self.option('{0}:autoconf_options'.format(name), {}) self.option("{0}:autoconf_options".format(name), {})
) )
return [ return [
self.run(ShellQuoted( self.run(
'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" ' ShellQuoted(
'CFLAGS="$CFLAGS -I"{p}"/include" ' 'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
'CPPFLAGS="$CPPFLAGS -I"{p}"/include" ' 'CFLAGS="$CFLAGS -I"{p}"/include" '
'PY_PREFIX={p} ' 'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
'./configure --prefix={p} {args}' "PY_PREFIX={p} "
).format( "./configure --prefix={p} {args}"
p=self.option('prefix'), ).format(
args=shell_join(' ', ( p=self.option("prefix"),
ShellQuoted('{k}={v}').format(k=k, v=v) args=shell_join(
for k, v in autoconf_options.items() " ",
)), (
)), ShellQuoted("{k}={v}").format(k=k, v=v)
for k, v in autoconf_options.items()
),
),
)
),
] ]
def autoconf_install(self, name): def autoconf_install(self, name):
return self.step('Build and install {0}'.format(name), [ return self.step(
self.run(ShellQuoted('autoreconf -ivf')), "Build and install {0}".format(name),
] + self.configure() + self.make_and_install()) [
self.run(ShellQuoted("autoreconf -ivf")),
]
+ self.configure()
+ self.make_and_install(),
)
def cmake_configure(self, name, cmake_path='..'): def cmake_configure(self, name, cmake_path=".."):
cmake_defines = { cmake_defines = {
'BUILD_SHARED_LIBS': 'ON', "BUILD_SHARED_LIBS": "ON",
'CMAKE_INSTALL_PREFIX': self.option('prefix'), "CMAKE_INSTALL_PREFIX": self.option("prefix"),
} }
# Hacks to add thriftpy3 support # Hacks to add thriftpy3 support
if 'BUILD_THRIFT_PY3' in os.environ and 'folly' in name: if "BUILD_THRIFT_PY3" in os.environ and "folly" in name:
cmake_defines['PYTHON_EXTENSIONS'] = 'True' cmake_defines["PYTHON_EXTENSIONS"] = "True"
if 'BUILD_THRIFT_PY3' in os.environ and 'fbthrift' in name: if "BUILD_THRIFT_PY3" in os.environ and "fbthrift" in name:
cmake_defines['thriftpy3'] = 'ON' cmake_defines["thriftpy3"] = "ON"
cmake_defines.update( cmake_defines.update(self.option("{0}:cmake_defines".format(name), {}))
self.option('{0}:cmake_defines'.format(name), {})
)
return [ return [
self.run(ShellQuoted( self.run(
'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" ' ShellQuoted(
'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" ' 'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" '
'cmake {args} {cmake_path}' 'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" '
).format( "cmake {args} {cmake_path}"
p=self.option('prefix'), ).format(
args=shell_join(' ', ( p=self.option("prefix"),
ShellQuoted('-D{k}={v}').format(k=k, v=v) args=shell_join(
for k, v in cmake_defines.items() " ",
)), (
cmake_path=cmake_path, ShellQuoted("-D{k}={v}").format(k=k, v=v)
)), for k, v in cmake_defines.items()
),
),
cmake_path=cmake_path,
)
),
] ]
def cmake_install(self, name, cmake_path='..'): def cmake_install(self, name, cmake_path=".."):
return self.step( return self.step(
'Build and install {0}'.format(name), "Build and install {0}".format(name),
self.cmake_configure(name, cmake_path) + self.make_and_install() self.cmake_configure(name, cmake_path) + self.make_and_install(),
) )
def cargo_build(self, name): def cargo_build(self, name):
@@ -458,13 +515,15 @@ class FBCodeBuilder(object):
], ],
) )
def fb_github_autoconf_install(self, project_and_path, github_org='facebook'): def fb_github_autoconf_install(self, project_and_path, github_org="facebook"):
return [ return [
self.fb_github_project_workdir(project_and_path, github_org), self.fb_github_project_workdir(project_and_path, github_org),
self.autoconf_install(project_and_path), self.autoconf_install(project_and_path),
] ]
def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org='facebook'): def fb_github_cmake_install(
self, project_and_path, cmake_path="..", github_org="facebook"
):
return [ return [
self.fb_github_project_workdir(project_and_path, github_org), self.fb_github_project_workdir(project_and_path, github_org),
self.cmake_install(project_and_path, cmake_path), self.cmake_install(project_and_path, cmake_path),

11
build/fbcode_builder/fbcode_builder_config.py
View File

@@ -4,12 +4,13 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
'Demo config, so that `make_docker_context.py --help` works in this directory.'
"Demo config, so that `make_docker_context.py --help` works in this directory."
config = { config = {
'fbcode_builder_spec': lambda _builder: { "fbcode_builder_spec": lambda _builder: {
'depends_on': [], "depends_on": [],
'steps': [], "steps": [],
}, },
'github_project': 'demo/project', "github_project": "demo/project",
} }

157
build/fbcode_builder/make_docker_context.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
'''
"""
Reads `fbcode_builder_config.py` from the current directory, and prepares a Reads `fbcode_builder_config.py` from the current directory, and prepares a
Docker context directory to build this project. Prints to stdout the path Docker context directory to build this project. Prints to stdout the path
to the context directory. to the context directory.
@@ -14,7 +15,7 @@ Try `.../make_docker_context.py --help` from a project's `build/` directory.
By default, the Docker context directory will be in /tmp. It will always By default, the Docker context directory will be in /tmp. It will always
contain a Dockerfile, and might also contain copies of your local repos, and contain a Dockerfile, and might also contain copies of your local repos, and
other data needed for the build container. other data needed for the build container.
''' """
import os import os
import tempfile import tempfile
@@ -27,7 +28,7 @@ from parse_args import parse_args_to_fbcode_builder_opts
def make_docker_context( def make_docker_context(
get_steps_fn, github_project, opts=None, default_context_dir=None get_steps_fn, github_project, opts=None, default_context_dir=None
): ):
''' """
Returns a path to the Docker context directory. See parse_args.py. Returns a path to the Docker context directory. See parse_args.py.
Helper for making a command-line utility that writes your project's Helper for making a command-line utility that writes your project's
@@ -38,83 +39,97 @@ def make_docker_context(
lambda builder: [builder.step(...), ...], lambda builder: [builder.step(...), ...],
'facebook/your_project', 'facebook/your_project',
)) ))
''' """
if opts is None: if opts is None:
opts = {} opts = {}
valid_versions = ( valid_versions = (
('ubuntu:16.04', '5'), ("ubuntu:16.04", "5"),
('ubuntu:18.04', '7'), ("ubuntu:18.04", "7"),
) )
def add_args(parser): def add_args(parser):
parser.add_argument( parser.add_argument(
'--docker-context-dir', metavar='DIR', "--docker-context-dir",
metavar="DIR",
default=default_context_dir, default=default_context_dir,
help='Write the Dockerfile and its context into this directory. ' help="Write the Dockerfile and its context into this directory. "
'If empty, make a temporary directory. Default: %(default)s.', "If empty, make a temporary directory. Default: %(default)s.",
) )
parser.add_argument( parser.add_argument(
'--user', metavar='NAME', default=opts.get('user', 'nobody'), "--user",
help='Build and install as this user. Default: %(default)s.', metavar="NAME",
default=opts.get("user", "nobody"),
help="Build and install as this user. Default: %(default)s.",
) )
parser.add_argument( parser.add_argument(
'--prefix', metavar='DIR', "--prefix",
default=opts.get('prefix', '/home/install'), metavar="DIR",
help='Install all libraries in this prefix. Default: %(default)s.', default=opts.get("prefix", "/home/install"),
help="Install all libraries in this prefix. Default: %(default)s.",
) )
parser.add_argument( parser.add_argument(
'--projects-dir', metavar='DIR', "--projects-dir",
default=opts.get('projects_dir', '/home'), metavar="DIR",
help='Place project code directories here. Default: %(default)s.', default=opts.get("projects_dir", "/home"),
help="Place project code directories here. Default: %(default)s.",
) )
parser.add_argument( parser.add_argument(
'--os-image', metavar='IMG', choices=zip(*valid_versions)[0], "--os-image",
default=opts.get('os_image', valid_versions[0][0]), metavar="IMG",
help='Docker OS image -- be sure to use only ones you trust (See ' choices=zip(*valid_versions)[0],
'README.docker). Choices: %(choices)s. Default: %(default)s.', default=opts.get("os_image", valid_versions[0][0]),
help="Docker OS image -- be sure to use only ones you trust (See "
"README.docker). Choices: %(choices)s. Default: %(default)s.",
) )
parser.add_argument( parser.add_argument(
'--gcc-version', metavar='VER', "--gcc-version",
metavar="VER",
choices=set(zip(*valid_versions)[1]), choices=set(zip(*valid_versions)[1]),
default=opts.get('gcc_version', valid_versions[0][1]), default=opts.get("gcc_version", valid_versions[0][1]),
help='Choices: %(choices)s. Default: %(default)s.', help="Choices: %(choices)s. Default: %(default)s.",
) )
parser.add_argument( parser.add_argument(
'--make-parallelism', metavar='NUM', type=int, "--make-parallelism",
default=opts.get('make_parallelism', 1), metavar="NUM",
help='Use `make -j` on multi-CPU systems with lots of RAM. ' type=int,
'Default: %(default)s.', default=opts.get("make_parallelism", 1),
help="Use `make -j` on multi-CPU systems with lots of RAM. "
"Default: %(default)s.",
) )
parser.add_argument( parser.add_argument(
'--local-repo-dir', metavar='DIR', "--local-repo-dir",
help='If set, build {0} from a local directory instead of Github.' metavar="DIR",
.format(github_project), help="If set, build {0} from a local directory instead of Github.".format(
github_project
),
) )
parser.add_argument( parser.add_argument(
'--ccache-tgz', metavar='PATH', "--ccache-tgz",
help='If set, enable ccache for the build. To initialize the ' metavar="PATH",
'cache, first try to hardlink, then to copy --cache-tgz ' help="If set, enable ccache for the build. To initialize the "
'as ccache.tgz into the --docker-context-dir.' "cache, first try to hardlink, then to copy --cache-tgz "
"as ccache.tgz into the --docker-context-dir.",
) )
opts = parse_args_to_fbcode_builder_opts( opts = parse_args_to_fbcode_builder_opts(
add_args, add_args,
# These have add_argument() calls, others are set via --option. # These have add_argument() calls, others are set via --option.
( (
'docker_context_dir', "docker_context_dir",
'user', "user",
'prefix', "prefix",
'projects_dir', "projects_dir",
'os_image', "os_image",
'gcc_version', "gcc_version",
'make_parallelism', "make_parallelism",
'local_repo_dir', "local_repo_dir",
'ccache_tgz', "ccache_tgz",
), ),
opts, opts,
help=textwrap.dedent(''' help=textwrap.dedent(
"""
Reads `fbcode_builder_config.py` from the current directory, and Reads `fbcode_builder_config.py` from the current directory, and
prepares a Docker context directory to build {github_project} and prepares a Docker context directory to build {github_project} and
@@ -130,47 +145,55 @@ def make_docker_context(
Usage: Usage:
(cd $(./make_docker_context.py) && docker build . 2>&1 | tee log) (cd $(./make_docker_context.py) && docker build . 2>&1 | tee log)
'''.format(github_project=github_project)), """.format(
github_project=github_project
)
),
) )
# This allows travis_docker_build.sh not to know the main Github project. # This allows travis_docker_build.sh not to know the main Github project.
local_repo_dir = opts.pop('local_repo_dir', None) local_repo_dir = opts.pop("local_repo_dir", None)
if local_repo_dir is not None: if local_repo_dir is not None:
opts['{0}:local_repo_dir'.format(github_project)] = local_repo_dir opts["{0}:local_repo_dir".format(github_project)] = local_repo_dir
if (opts.get('os_image'), opts.get('gcc_version')) not in valid_versions: if (opts.get("os_image"), opts.get("gcc_version")) not in valid_versions:
raise Exception( raise Exception(
'Due to 4/5 ABI changes (std::string), we can only use {0}'.format( "Due to 4/5 ABI changes (std::string), we can only use {0}".format(
' / '.join('GCC {1} on {0}'.format(*p) for p in valid_versions) " / ".join("GCC {1} on {0}".format(*p) for p in valid_versions)
) )
) )
if opts.get('docker_context_dir') is None: if opts.get("docker_context_dir") is None:
opts['docker_context_dir'] = tempfile.mkdtemp(prefix='docker-context-') opts["docker_context_dir"] = tempfile.mkdtemp(prefix="docker-context-")
elif not os.path.exists(opts.get('docker_context_dir')): elif not os.path.exists(opts.get("docker_context_dir")):
os.makedirs(opts.get('docker_context_dir')) os.makedirs(opts.get("docker_context_dir"))
builder = DockerFBCodeBuilder(**opts) builder = DockerFBCodeBuilder(**opts)
context_dir = builder.option('docker_context_dir') # Mark option "in-use" context_dir = builder.option("docker_context_dir") # Mark option "in-use"
# The renderer may also populate some files into the context_dir. # The renderer may also populate some files into the context_dir.
dockerfile = builder.render(get_steps_fn(builder)) dockerfile = builder.render(get_steps_fn(builder))
with os.fdopen(os.open( with os.fdopen(
os.path.join(context_dir, 'Dockerfile'), os.open(
os.O_RDWR | os.O_CREAT | os.O_EXCL, # Do not overwrite existing files os.path.join(context_dir, "Dockerfile"),
0o644, os.O_RDWR | os.O_CREAT | os.O_EXCL, # Do not overwrite existing files
), 'w') as f: 0o644,
),
"w",
) as f:
f.write(dockerfile) f.write(dockerfile)
return context_dir return context_dir
if __name__ == '__main__': if __name__ == "__main__":
from utils import read_fbcode_builder_config, build_fbcode_builder_config from utils import read_fbcode_builder_config, build_fbcode_builder_config
# Load a spec from the current directory # Load a spec from the current directory
config = read_fbcode_builder_config('fbcode_builder_config.py') config = read_fbcode_builder_config("fbcode_builder_config.py")
print(make_docker_context( print(
build_fbcode_builder_config(config), make_docker_context(
config['github_project'], build_fbcode_builder_config(config),
)) config["github_project"],
)
)

47
build/fbcode_builder/parse_args.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
'Argument parsing logic shared by all fbcode_builder CLI tools.'
"Argument parsing logic shared by all fbcode_builder CLI tools."
import argparse import argparse
import logging import logging
@@ -13,7 +14,7 @@ from shell_quoting import raw_shell, ShellQuoted
def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help): def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
''' """
Provides some standard arguments: --debug, --option, --shell-quoted-option Provides some standard arguments: --debug, --option, --shell-quoted-option
@@ -26,46 +27,52 @@ def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
`help` is printed in response to the `--help` argument. `help` is printed in response to the `--help` argument.
''' """
top_level_opts = set(top_level_opts) top_level_opts = set(top_level_opts)
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description=help, description=help, formatter_class=argparse.RawDescriptionHelpFormatter
formatter_class=argparse.RawDescriptionHelpFormatter
) )
add_args_fn(parser) add_args_fn(parser)
parser.add_argument( parser.add_argument(
'--option', nargs=2, metavar=('KEY', 'VALUE'), action='append', "--option",
nargs=2,
metavar=("KEY", "VALUE"),
action="append",
default=[ default=[
(k, v) for k, v in opts.items() (k, v)
if k not in top_level_opts and not isinstance(v, ShellQuoted) for k, v in opts.items()
if k not in top_level_opts and not isinstance(v, ShellQuoted)
], ],
help='Set project-specific options. These are assumed to be raw ' help="Set project-specific options. These are assumed to be raw "
'strings, to be shell-escaped as needed. Default: %(default)s.', "strings, to be shell-escaped as needed. Default: %(default)s.",
) )
parser.add_argument( parser.add_argument(
'--shell-quoted-option', nargs=2, metavar=('KEY', 'VALUE'), "--shell-quoted-option",
action='append', nargs=2,
metavar=("KEY", "VALUE"),
action="append",
default=[ default=[
(k, raw_shell(v)) for k, v in opts.items() (k, raw_shell(v))
if k not in top_level_opts and isinstance(v, ShellQuoted) for k, v in opts.items()
if k not in top_level_opts and isinstance(v, ShellQuoted)
], ],
help='Set project-specific options. These are assumed to be shell-' help="Set project-specific options. These are assumed to be shell-"
'quoted, and may be used in commands as-is. Default: %(default)s.', "quoted, and may be used in commands as-is. Default: %(default)s.",
) )
parser.add_argument('--debug', action='store_true', help='Log more') parser.add_argument("--debug", action="store_true", help="Log more")
args = parser.parse_args() args = parser.parse_args()
logging.basicConfig( logging.basicConfig(
level=logging.DEBUG if args.debug else logging.INFO, level=logging.DEBUG if args.debug else logging.INFO,
format='%(levelname)s: %(message)s' format="%(levelname)s: %(message)s",
) )
# Map command-line args back into opts. # Map command-line args back into opts.
logging.debug('opts before command-line arguments: {0}'.format(opts)) logging.debug("opts before command-line arguments: {0}".format(opts))
new_opts = {} new_opts = {}
for key in top_level_opts: for key in top_level_opts:
@@ -78,6 +85,6 @@ def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
for key, val in args.shell_quoted_option: for key, val in args.shell_quoted_option:
new_opts[key] = ShellQuoted(val) new_opts[key] = ShellQuoted(val)
logging.debug('opts after command-line arguments: {0}'.format(new_opts)) logging.debug("opts after command-line arguments: {0}".format(new_opts))
return new_opts return new_opts

74
build/fbcode_builder/shell_builder.py
View File

@@ -5,7 +5,7 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
''' """
shell_builder.py allows running the fbcode_builder logic shell_builder.py allows running the fbcode_builder logic
on the host rather than in a container. on the host rather than in a container.
@@ -17,50 +17,50 @@ any failing step will cause the script to exit with failure.
cd build cd build
python fbcode_builder/shell_builder.py > ~/run.sh python fbcode_builder/shell_builder.py > ~/run.sh
bash ~/run.sh bash ~/run.sh
''' """
import os
import distutils.spawn import distutils.spawn
import os
from fbcode_builder import FBCodeBuilder from fbcode_builder import FBCodeBuilder
from shell_quoting import ( from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted
raw_shell, shell_comment, shell_join, ShellQuoted
)
from utils import recursively_flatten_list from utils import recursively_flatten_list
class ShellFBCodeBuilder(FBCodeBuilder): class ShellFBCodeBuilder(FBCodeBuilder):
def _render_impl(self, steps): def _render_impl(self, steps):
return raw_shell(shell_join('\n', recursively_flatten_list(steps))) return raw_shell(shell_join("\n", recursively_flatten_list(steps)))
def set_env(self, key, value): def set_env(self, key, value):
return ShellQuoted("export {key}={val}").format(key=key, val=value) return ShellQuoted("export {key}={val}").format(key=key, val=value)
def workdir(self, dir): def workdir(self, dir):
return [ return [
ShellQuoted('mkdir -p {d} && cd {d}').format( ShellQuoted("mkdir -p {d} && cd {d}").format(d=dir),
d=dir
),
] ]
def run(self, shell_cmd): def run(self, shell_cmd):
return ShellQuoted('{cmd}').format(cmd=shell_cmd) return ShellQuoted("{cmd}").format(cmd=shell_cmd)
def step(self, name, actions): def step(self, name, actions):
assert '\n' not in name, 'Name {0} would span > 1 line'.format(name) assert "\n" not in name, "Name {0} would span > 1 line".format(name)
b = ShellQuoted('') b = ShellQuoted("")
return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b] return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b]
def setup(self): def setup(self):
steps = [ steps = (
ShellQuoted('set -exo pipefail'), [
] + self.create_python_venv() + self.python_venv() ShellQuoted("set -exo pipefail"),
if self.has_option('ccache_dir'): ]
ccache_dir = self.option('ccache_dir') + self.create_python_venv()
+ self.python_venv()
)
if self.has_option("ccache_dir"):
ccache_dir = self.option("ccache_dir")
steps += [ steps += [
ShellQuoted( ShellQuoted(
# Set CCACHE_DIR before the `ccache` invocations below. # Set CCACHE_DIR before the `ccache` invocations below.
'export CCACHE_DIR={ccache_dir} ' "export CCACHE_DIR={ccache_dir} "
'CC="ccache ${{CC:-gcc}}" CXX="ccache ${{CXX:-g++}}"' 'CC="ccache ${{CC:-gcc}}" CXX="ccache ${{CXX:-g++}}"'
).format(ccache_dir=ccache_dir) ).format(ccache_dir=ccache_dir)
] ]
@@ -71,44 +71,44 @@ class ShellFBCodeBuilder(FBCodeBuilder):
def copy_local_repo(self, dir, dest_name): def copy_local_repo(self, dir, dest_name):
return [ return [
ShellQuoted('cp -r {dir} {dest_name}').format( ShellQuoted("cp -r {dir} {dest_name}").format(dir=dir, dest_name=dest_name),
dir=dir,
dest_name=dest_name
),
] ]
def find_project_root(): def find_project_root():
here = os.path.dirname(os.path.realpath(__file__)) here = os.path.dirname(os.path.realpath(__file__))
maybe_root = os.path.dirname(os.path.dirname(here)) maybe_root = os.path.dirname(os.path.dirname(here))
if os.path.isdir(os.path.join(maybe_root, '.git')): if os.path.isdir(os.path.join(maybe_root, ".git")):
return maybe_root return maybe_root
raise RuntimeError( raise RuntimeError(
"I expected shell_builder.py to be in the " "I expected shell_builder.py to be in the "
"build/fbcode_builder subdir of a git repo") "build/fbcode_builder subdir of a git repo"
)
def persistent_temp_dir(repo_root): def persistent_temp_dir(repo_root):
escaped = repo_root.replace('/', 'sZs').replace('\\', 'sZs').replace(':', '') escaped = repo_root.replace("/", "sZs").replace("\\", "sZs").replace(":", "")
return os.path.join(os.path.expandvars("$HOME"), '.fbcode_builder-' + escaped) return os.path.join(os.path.expandvars("$HOME"), ".fbcode_builder-" + escaped)
if __name__ == '__main__': if __name__ == "__main__":
from utils import read_fbcode_builder_config, build_fbcode_builder_config from utils import read_fbcode_builder_config, build_fbcode_builder_config
repo_root = find_project_root() repo_root = find_project_root()
temp = persistent_temp_dir(repo_root) temp = persistent_temp_dir(repo_root)
config = read_fbcode_builder_config('fbcode_builder_config.py') config = read_fbcode_builder_config("fbcode_builder_config.py")
builder = ShellFBCodeBuilder(projects_dir=temp) builder = ShellFBCodeBuilder(projects_dir=temp)
if distutils.spawn.find_executable('ccache'): if distutils.spawn.find_executable("ccache"):
builder.add_option('ccache_dir', builder.add_option(
os.environ.get('CCACHE_DIR', os.path.join(temp, '.ccache'))) "ccache_dir", os.environ.get("CCACHE_DIR", os.path.join(temp, ".ccache"))
builder.add_option('prefix', os.path.join(temp, 'installed')) )
builder.add_option('make_parallelism', 4) builder.add_option("prefix", os.path.join(temp, "installed"))
builder.add_option("make_parallelism", 4)
builder.add_option( builder.add_option(
'{project}:local_repo_dir'.format(project=config['github_project']), "{project}:local_repo_dir".format(project=config["github_project"]), repo_root
repo_root) )
make_steps = build_fbcode_builder_config(config) make_steps = build_fbcode_builder_config(config)
steps = make_steps(builder) steps = make_steps(builder)
print(builder.render(steps)) print(builder.render(steps))

66
build/fbcode_builder/shell_quoting.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
'''
"""
Almost every FBCodeBuilder string is ultimately passed to a shell. Escaping Almost every FBCodeBuilder string is ultimately passed to a shell. Escaping
too little or too much tends to be the most common error. The utilities in too little or too much tends to be the most common error. The utilities in
@@ -16,15 +17,14 @@ this file give a systematic way of avoiding such bugs:
- Use `path_join` to join path components. - Use `path_join` to join path components.
- Use `shell_join` to join already-quoted command arguments or shell lines. - Use `shell_join` to join already-quoted command arguments or shell lines.
''' """
import os import os
from collections import namedtuple from collections import namedtuple
class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))): class ShellQuoted(namedtuple("ShellQuoted", ("do_not_use_raw_str",))):
''' """
Wrap a string with this to make it transparent to shell_quote(). It Wrap a string with this to make it transparent to shell_quote(). It
will almost always suffice to use ShellQuoted.format(), path_join(), will almost always suffice to use ShellQuoted.format(), path_join(),
@@ -32,27 +32,25 @@ class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))):
If you really must, use raw_shell() to access the raw string. If you really must, use raw_shell() to access the raw string.
''' """
def __new__(cls, s): def __new__(cls, s):
'No need to nest ShellQuoted.' "No need to nest ShellQuoted."
return super(ShellQuoted, cls).__new__( return super(ShellQuoted, cls).__new__(
cls, s.do_not_use_raw_str if isinstance(s, ShellQuoted) else s cls, s.do_not_use_raw_str if isinstance(s, ShellQuoted) else s
) )
def __str__(self): def __str__(self):
raise RuntimeError( raise RuntimeError(
'One does not simply convert {0} to a string -- use path_join() ' "One does not simply convert {0} to a string -- use path_join() "
'or ShellQuoted.format() instead'.format(repr(self)) "or ShellQuoted.format() instead".format(repr(self))
) )
def __repr__(self): def __repr__(self):
return '{0}({1})'.format( return "{0}({1})".format(self.__class__.__name__, repr(self.do_not_use_raw_str))
self.__class__.__name__, repr(self.do_not_use_raw_str)
)
def format(self, **kwargs): def format(self, **kwargs):
''' """
Use instead of str.format() when the arguments are either Use instead of str.format() when the arguments are either
`ShellQuoted()` or raw strings needing to be `shell_quote()`d. `ShellQuoted()` or raw strings needing to be `shell_quote()`d.
@@ -60,40 +58,46 @@ class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))):
Positional args are deliberately not supported since they are more Positional args are deliberately not supported since they are more
error-prone. error-prone.
''' """
return ShellQuoted(self.do_not_use_raw_str.format(**dict( return ShellQuoted(
(k, shell_quote(v).do_not_use_raw_str) for k, v in kwargs.items() self.do_not_use_raw_str.format(
))) **dict(
(k, shell_quote(v).do_not_use_raw_str) for k, v in kwargs.items()
)
)
)
def shell_quote(s): def shell_quote(s):
'Quotes a string if it is not already quoted' "Quotes a string if it is not already quoted"
return s if isinstance(s, ShellQuoted) \ return (
s
if isinstance(s, ShellQuoted)
else ShellQuoted("'" + str(s).replace("'", "'\\''") + "'") else ShellQuoted("'" + str(s).replace("'", "'\\''") + "'")
)
def raw_shell(s): def raw_shell(s):
'Not a member of ShellQuoted so we get a useful error for raw strings' "Not a member of ShellQuoted so we get a useful error for raw strings"
if isinstance(s, ShellQuoted): if isinstance(s, ShellQuoted):
return s.do_not_use_raw_str return s.do_not_use_raw_str
raise RuntimeError('{0} should have been ShellQuoted'.format(s)) raise RuntimeError("{0} should have been ShellQuoted".format(s))
def shell_join(delim, it): def shell_join(delim, it):
'Joins an iterable of ShellQuoted with a delimiter between each two' "Joins an iterable of ShellQuoted with a delimiter between each two"
return ShellQuoted(delim.join(raw_shell(s) for s in it)) return ShellQuoted(delim.join(raw_shell(s) for s in it))
def path_join(*args): def path_join(*args):
'Joins ShellQuoted and raw pieces of paths to make a shell-quoted path' "Joins ShellQuoted and raw pieces of paths to make a shell-quoted path"
return ShellQuoted(os.path.join(*[ return ShellQuoted(os.path.join(*[raw_shell(shell_quote(s)) for s in args]))
raw_shell(shell_quote(s)) for s in args
]))
def shell_comment(c): def shell_comment(c):
'Do not shell-escape raw strings in comments, but do handle line breaks.' "Do not shell-escape raw strings in comments, but do handle line breaks."
return ShellQuoted('# {c}').format(c=ShellQuoted( return ShellQuoted("# {c}").format(
(raw_shell(c) if isinstance(c, ShellQuoted) else c) c=ShellQuoted(
.replace('\n', '\n# ') (raw_shell(c) if isinstance(c, ShellQuoted) else c).replace("\n", "\n# ")
)) )
)

6
build/fbcode_builder/specs/fbthrift.py
View File

@@ -15,8 +15,8 @@ import specs.zstd as zstd
def fbcode_builder_spec(builder): def fbcode_builder_spec(builder):
return { return {
'depends_on': [fmt, folly, fizz, sodium, wangle, zstd], "depends_on": [fmt, folly, fizz, sodium, wangle, zstd],
'steps': [ "steps": [
builder.fb_github_cmake_install('fbthrift/thrift'), builder.fb_github_cmake_install("fbthrift/thrift"),
], ],
} }

51
build/fbcode_builder/specs/fbzmq.py
View File

@@ -10,31 +10,40 @@ import specs.fmt as fmt
import specs.folly as folly import specs.folly as folly
import specs.gmock as gmock import specs.gmock as gmock
import specs.sodium as sodium import specs.sodium as sodium
from shell_quoting import ShellQuoted from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder): def fbcode_builder_spec(builder):
builder.add_option('zeromq/libzmq:git_hash', 'v4.2.2') builder.add_option("zeromq/libzmq:git_hash", "v4.2.2")
return { return {
'depends_on': [fmt, folly, fbthrift, gmock, sodium], "depends_on": [fmt, folly, fbthrift, gmock, sodium],
'steps': [ "steps": [
builder.github_project_workdir('zeromq/libzmq', '.'), builder.github_project_workdir("zeromq/libzmq", "."),
builder.step('Build and install zeromq/libzmq', [ builder.step(
builder.run(ShellQuoted('./autogen.sh')), "Build and install zeromq/libzmq",
builder.configure(), [
builder.make_and_install(), builder.run(ShellQuoted("./autogen.sh")),
]), builder.configure(),
builder.make_and_install(),
builder.fb_github_project_workdir('fbzmq/_build', 'facebook'), ],
builder.step('Build and install fbzmq/', [ ),
builder.cmake_configure('fbzmq/_build'), builder.fb_github_project_workdir("fbzmq/_build", "facebook"),
# we need the pythonpath to find the thrift compiler builder.step(
builder.run(ShellQuoted( "Build and install fbzmq/",
'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' [
'make -j {n}' builder.cmake_configure("fbzmq/_build"),
).format(p=builder.option('prefix'), n=builder.option('make_parallelism'))), # we need the pythonpath to find the thrift compiler
builder.run(ShellQuoted('make install')), builder.run(
]), ShellQuoted(
'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
"make -j {n}"
).format(
p=builder.option("prefix"),
n=builder.option("make_parallelism"),
)
),
builder.run(ShellQuoted("make install")),
],
),
], ],
} }

2
build/fbcode_builder/specs/fizz.py
View File

@@ -5,9 +5,9 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
import specs.gmock as gmock
import specs.fmt as fmt import specs.fmt as fmt
import specs.folly as folly import specs.folly as folly
import specs.gmock as gmock
import specs.sodium as sodium import specs.sodium as sodium

14
build/fbcode_builder/specs/fmt.py
View File

@@ -7,20 +7,20 @@ from __future__ import unicode_literals
def fbcode_builder_spec(builder): def fbcode_builder_spec(builder):
builder.add_option('fmtlib/fmt:git_hash', '6.2.1') builder.add_option("fmtlib/fmt:git_hash", "6.2.1")
builder.add_option( builder.add_option(
'fmtlib/fmt:cmake_defines', "fmtlib/fmt:cmake_defines",
{ {
# Avoids a bizarred failure to run tests in Bistro: # Avoids a bizarred failure to run tests in Bistro:
# test_crontab_selector: error while loading shared libraries: # test_crontab_selector: error while loading shared libraries:
# libfmt.so.6: cannot open shared object file: # libfmt.so.6: cannot open shared object file:
# No such file or directory # No such file or directory
'BUILD_SHARED_LIBS': 'OFF', "BUILD_SHARED_LIBS": "OFF",
} },
) )
return { return {
'steps': [ "steps": [
builder.github_project_workdir('fmtlib/fmt', 'build'), builder.github_project_workdir("fmtlib/fmt", "build"),
builder.cmake_install('fmtlib/fmt'), builder.cmake_install("fmtlib/fmt"),
], ],
} }

6
build/fbcode_builder/specs/folly.py
View File

@@ -11,13 +11,13 @@ import specs.fmt as fmt
def fbcode_builder_spec(builder): def fbcode_builder_spec(builder):
return { return {
"depends_on": [fmt], "depends_on": [fmt],
'steps': [ "steps": [
# on macOS the filesystem is typically case insensitive. # on macOS the filesystem is typically case insensitive.
# We need to ensure that the CWD is not the folly source # We need to ensure that the CWD is not the folly source
# dir when we build, otherwise the system will decide # dir when we build, otherwise the system will decide
# that `folly/String.h` is the file it wants when including # that `folly/String.h` is the file it wants when including
# `string.h` and the build will fail. # `string.h` and the build will fail.
builder.fb_github_project_workdir('folly/_build'), builder.fb_github_project_workdir("folly/_build"),
builder.cmake_install('facebook/folly'), builder.cmake_install("facebook/folly"),
], ],
} }

16
build/fbcode_builder/specs/gmock.py
View File

@@ -7,18 +7,18 @@ from __future__ import unicode_literals
def fbcode_builder_spec(builder): def fbcode_builder_spec(builder):
builder.add_option('google/googletest:git_hash', 'release-1.8.1') builder.add_option("google/googletest:git_hash", "release-1.8.1")
builder.add_option( builder.add_option(
'google/googletest:cmake_defines', "google/googletest:cmake_defines",
{ {
'BUILD_GTEST': 'ON', "BUILD_GTEST": "ON",
# Avoid problems with MACOSX_RPATH # Avoid problems with MACOSX_RPATH
'BUILD_SHARED_LIBS': 'OFF', "BUILD_SHARED_LIBS": "OFF",
} },
) )
return { return {
'steps': [ "steps": [
builder.github_project_workdir('google/googletest', 'build'), builder.github_project_workdir("google/googletest", "build"),
builder.cmake_install('google/googletest'), builder.cmake_install("google/googletest"),
], ],
} }

4
build/fbcode_builder/specs/mvfst.py
View File

@@ -5,9 +5,9 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
import specs.gmock as gmock
import specs.folly as folly
import specs.fizz as fizz import specs.fizz as fizz
import specs.folly as folly
import specs.gmock as gmock
def fbcode_builder_spec(builder): def fbcode_builder_spec(builder):

4
build/fbcode_builder/specs/proxygen.py
View File

@@ -5,10 +5,10 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
import specs.gmock as gmock import specs.fizz as fizz
import specs.fmt as fmt import specs.fmt as fmt
import specs.folly as folly import specs.folly as folly
import specs.fizz as fizz import specs.gmock as gmock
import specs.mvfst as mvfst import specs.mvfst as mvfst
import specs.sodium as sodium import specs.sodium as sodium
import specs.wangle as wangle import specs.wangle as wangle

4
build/fbcode_builder/specs/proxygen_quic.py
View File

@@ -5,10 +5,10 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
import specs.gmock as gmock import specs.fizz as fizz
import specs.fmt as fmt import specs.fmt as fmt
import specs.folly as folly import specs.folly as folly
import specs.fizz as fizz import specs.gmock as gmock
import specs.mvfst as mvfst import specs.mvfst as mvfst
import specs.sodium as sodium import specs.sodium as sodium
import specs.wangle as wangle import specs.wangle as wangle

6
build/fbcode_builder/specs/re2.py
View File

@@ -8,8 +8,8 @@ from __future__ import unicode_literals
def fbcode_builder_spec(builder): def fbcode_builder_spec(builder):
return { return {
'steps': [ "steps": [
builder.github_project_workdir('google/re2', 'build'), builder.github_project_workdir("google/re2", "build"),
builder.cmake_install('google/re2'), builder.cmake_install("google/re2"),
], ],
} }

11
build/fbcode_builder/specs/rocksdb.py
View File

@@ -7,10 +7,13 @@ from __future__ import unicode_literals
def fbcode_builder_spec(builder): def fbcode_builder_spec(builder):
builder.add_option("rocksdb/_build:cmake_defines", { builder.add_option(
"USE_RTTI": "1", "rocksdb/_build:cmake_defines",
"PORTABLE": "ON", {
}) "USE_RTTI": "1",
"PORTABLE": "ON",
},
)
return { return {
"steps": [ "steps": [
builder.fb_github_cmake_install("rocksdb/_build"), builder.fb_github_cmake_install("rocksdb/_build"),

19
build/fbcode_builder/specs/sodium.py
View File

@@ -9,14 +9,17 @@ from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder): def fbcode_builder_spec(builder):
builder.add_option('jedisct1/libsodium:git_hash', 'stable') builder.add_option("jedisct1/libsodium:git_hash", "stable")
return { return {
'steps': [ "steps": [
builder.github_project_workdir('jedisct1/libsodium', '.'), builder.github_project_workdir("jedisct1/libsodium", "."),
builder.step('Build and install jedisct1/libsodium', [ builder.step(
builder.run(ShellQuoted('./autogen.sh')), "Build and install jedisct1/libsodium",
builder.configure(), [
builder.make_and_install(), builder.run(ShellQuoted("./autogen.sh")),
]), builder.configure(),
builder.make_and_install(),
],
),
], ],
} }

4
build/fbcode_builder/specs/wangle.py
View File

@@ -5,10 +5,10 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
import specs.gmock as gmock import specs.fizz as fizz
import specs.fmt as fmt import specs.fmt as fmt
import specs.folly as folly import specs.folly as folly
import specs.fizz as fizz import specs.gmock as gmock
import specs.sodium as sodium import specs.sodium as sodium

23
build/fbcode_builder/specs/zstd.py
View File

@@ -11,16 +11,21 @@ from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder): def fbcode_builder_spec(builder):
# This API should change rarely, so build the latest tag instead of master. # This API should change rarely, so build the latest tag instead of master.
builder.add_option( builder.add_option(
'facebook/zstd:git_hash', "facebook/zstd:git_hash",
ShellQuoted('$(git describe --abbrev=0 --tags origin/master)') ShellQuoted("$(git describe --abbrev=0 --tags origin/master)"),
) )
return { return {
'steps': [ "steps": [
builder.github_project_workdir('facebook/zstd', '.'), builder.github_project_workdir("facebook/zstd", "."),
builder.step('Build and install zstd', [ builder.step(
builder.make_and_install(make_vars={ "Build and install zstd",
'PREFIX': builder.option('prefix'), [
}) builder.make_and_install(
]), make_vars={
"PREFIX": builder.option("prefix"),
}
)
],
),
], ],
} }

43
build/fbcode_builder/utils.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
'Miscellaneous utility functions.'
"Miscellaneous utility functions."
import itertools import itertools
import logging import logging
@@ -12,21 +13,19 @@ import os
import shutil import shutil
import subprocess import subprocess
import sys import sys
from contextlib import contextmanager from contextlib import contextmanager
def recursively_flatten_list(l): def recursively_flatten_list(l):
return itertools.chain.from_iterable( return itertools.chain.from_iterable(
(recursively_flatten_list(i) if type(i) is list else (i,)) (recursively_flatten_list(i) if type(i) is list else (i,)) for i in l
for i in l
) )
def run_command(*cmd, **kwargs): def run_command(*cmd, **kwargs):
'The stdout of most fbcode_builder utilities is meant to be parsed.' "The stdout of most fbcode_builder utilities is meant to be parsed."
logging.debug('Running: {0} with {1}'.format(cmd, kwargs)) logging.debug("Running: {0} with {1}".format(cmd, kwargs))
kwargs['stdout'] = sys.stderr kwargs["stdout"] = sys.stderr
subprocess.check_call(cmd, **kwargs) subprocess.check_call(cmd, **kwargs)
@@ -40,13 +39,13 @@ def make_temp_dir(d):
def _inner_read_config(path): def _inner_read_config(path):
''' """
Helper to read a named config file. Helper to read a named config file.
The grossness with the global is a workaround for this python bug: The grossness with the global is a workaround for this python bug:
https://bugs.python.org/issue21591 https://bugs.python.org/issue21591
The bug prevents us from defining either a local function or a lambda The bug prevents us from defining either a local function or a lambda
in the scope of read_fbcode_builder_config below. in the scope of read_fbcode_builder_config below.
''' """
global _project_dir global _project_dir
full_path = os.path.join(_project_dir, path) full_path = os.path.join(_project_dir, path)
return read_fbcode_builder_config(full_path) return read_fbcode_builder_config(full_path)
@@ -60,37 +59,37 @@ def read_fbcode_builder_config(filename):
global _project_dir global _project_dir
_project_dir = os.path.dirname(filename) _project_dir = os.path.dirname(filename)
scope = {'read_fbcode_builder_config': _inner_read_config} scope = {"read_fbcode_builder_config": _inner_read_config}
with open(filename) as config_file: with open(filename) as config_file:
code = compile(config_file.read(), filename, mode='exec') code = compile(config_file.read(), filename, mode="exec")
exec(code, scope) exec(code, scope)
return scope['config'] return scope["config"]
def steps_for_spec(builder, spec, processed_modules=None): def steps_for_spec(builder, spec, processed_modules=None):
''' """
Sets `builder` configuration, and returns all the builder steps Sets `builder` configuration, and returns all the builder steps
necessary to build `spec` and its dependencies. necessary to build `spec` and its dependencies.
Traverses the dependencies in depth-first order, honoring the sequencing Traverses the dependencies in depth-first order, honoring the sequencing
in each 'depends_on' list. in each 'depends_on' list.
''' """
if processed_modules is None: if processed_modules is None:
processed_modules = set() processed_modules = set()
steps = [] steps = []
for module in spec.get('depends_on', []): for module in spec.get("depends_on", []):
if module not in processed_modules: if module not in processed_modules:
processed_modules.add(module) processed_modules.add(module)
steps.extend(steps_for_spec( steps.extend(
builder, steps_for_spec(
module.fbcode_builder_spec(builder), builder, module.fbcode_builder_spec(builder), processed_modules
processed_modules )
)) )
steps.extend(spec.get('steps', [])) steps.extend(spec.get("steps", []))
return steps return steps
def build_fbcode_builder_config(config): def build_fbcode_builder_config(config):
return lambda builder: builder.build( return lambda builder: builder.build(
steps_for_spec(builder, config['fbcode_builder_spec'](builder)) steps_for_spec(builder, config["fbcode_builder_spec"](builder))
) )