diff --git a/build/fbcode_builder/CMake/fb_py_test_main.py b/build/fbcode_builder/CMake/fb_py_test_main.py index a5b96c408..1f3563aff 100644 --- a/build/fbcode_builder/CMake/fb_py_test_main.py +++ b/build/fbcode_builder/CMake/fb_py_test_main.py @@ -485,7 +485,7 @@ class Loader(object): return loader.suiteClass(suites) -_COVERAGE_INI = '''\ +_COVERAGE_INI = """\ [report] exclude_lines = pragma: no cover @@ -495,7 +495,7 @@ exclude_lines = pragma:.*no${PY_IMPL}${PY_MAJOR} pragma:.*nopy${PY_MAJOR} pragma:.*nopy${PY_MAJOR}${PY_MINOR} -''' +""" class MainProgram(object): @@ -734,7 +734,7 @@ class MainProgram(object): if not self.options.collect_coverage: return - with tempfile.NamedTemporaryFile('w', delete=False) as coverage_ini: + with tempfile.NamedTemporaryFile("w", delete=False) as coverage_ini: coverage_ini.write(_COVERAGE_INI) self._coverage_ini_path = coverage_ini.name diff --git a/build/fbcode_builder/CMake/make_fbpy_archive.py b/build/fbcode_builder/CMake/make_fbpy_archive.py index 4e91447af..3724feb21 100755 --- a/build/fbcode_builder/CMake/make_fbpy_archive.py +++ b/build/fbcode_builder/CMake/make_fbpy_archive.py @@ -124,7 +124,7 @@ def populate_install_tree(inst_dir, path_map): def build_zipapp(args, path_map): - """ Create a self executing python binary using Python 3's built-in + """Create a self executing python binary using Python 3's built-in zipapp module. This type of Python binary is relatively simple, as zipapp is part of the @@ -165,7 +165,7 @@ if __name__ == "__main__": def build_install_dir(args, path_map): - """ Create a directory that contains all of the sources, with a __main__ + """Create a directory that contains all of the sources, with a __main__ module to run the program. """ # Populate a temporary directory first, then rename to the destination @@ -188,7 +188,7 @@ def ensure_directory(path): def install_library(args, path_map): - """ Create an installation directory a python library. """ + """Create an installation directory a python library.""" out_dir = args.output out_manifest = args.output + ".manifest" diff --git a/build/fbcode_builder/docker_builder.py b/build/fbcode_builder/docker_builder.py index f551ec739..83df7137c 100644 --- a/build/fbcode_builder/docker_builder.py +++ b/build/fbcode_builder/docker_builder.py @@ -4,7 +4,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals -''' + +""" Extends FBCodeBuilder to produce Docker context directories. @@ -15,26 +16,23 @@ caching, you will want to: that change the least often, and - Put the steps that you are debugging towards the very end. -''' +""" import logging import os import shutil import tempfile from fbcode_builder import FBCodeBuilder -from shell_quoting import ( - raw_shell, shell_comment, shell_join, ShellQuoted, path_join -) +from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted, path_join from utils import recursively_flatten_list, run_command class DockerFBCodeBuilder(FBCodeBuilder): - def _user(self): - return self.option('user', 'root') + return self.option("user", "root") def _change_user(self): - return ShellQuoted('USER {u}').format(u=self._user()) + return ShellQuoted("USER {u}").format(u=self._user()) def setup(self): # Please add RPM-based OSes here as appropriate. @@ -63,17 +61,18 @@ class DockerFBCodeBuilder(FBCodeBuilder): # it is present when the resulting container is run add to PATH actions = [] if self.option("PYTHON_VENV", "OFF") == "ON": - actions = ShellQuoted('ENV PATH={p}:$PATH').format( - p=path_join(self.option('prefix'), "venv", "bin")) - return(actions) + actions = ShellQuoted("ENV PATH={p}:$PATH").format( + p=path_join(self.option("prefix"), "venv", "bin") + ) + return actions def step(self, name, actions): - assert '\n' not in name, 'Name {0} would span > 1 line'.format(name) - b = ShellQuoted('') - return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b] + assert "\n" not in name, "Name {0} would span > 1 line".format(name) + b = ShellQuoted("") + return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b] def run(self, shell_cmd): - return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd) + return ShellQuoted("RUN {cmd}").format(cmd=shell_cmd) def set_env(self, key, value): return ShellQuoted("ENV {key}={val}").format(key=key, val=value) @@ -84,12 +83,12 @@ class DockerFBCodeBuilder(FBCodeBuilder): # by root:root -- the explicit `mkdir` works around the bug: # USER nobody # WORKDIR build - ShellQuoted('USER root'), - ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format( + ShellQuoted("USER root"), + ShellQuoted("RUN mkdir -p {d} && chown {u} {d}").format( d=dir, u=self._user() ), self._change_user(), - ShellQuoted('WORKDIR {dir}').format(dir=dir), + ShellQuoted("WORKDIR {dir}").format(dir=dir), ] def comment(self, comment): @@ -99,60 +98,58 @@ class DockerFBCodeBuilder(FBCodeBuilder): def copy_local_repo(self, repo_dir, dest_name): fd, archive_path = tempfile.mkstemp( - prefix='local_repo_{0}_'.format(dest_name), - suffix='.tgz', - dir=os.path.abspath(self.option('docker_context_dir')), + prefix="local_repo_{0}_".format(dest_name), + suffix=".tgz", + dir=os.path.abspath(self.option("docker_context_dir")), ) os.close(fd) - run_command('tar', 'czf', archive_path, '.', cwd=repo_dir) + run_command("tar", "czf", archive_path, ".", cwd=repo_dir) return [ - ShellQuoted('ADD {archive} {dest_name}').format( + ShellQuoted("ADD {archive} {dest_name}").format( archive=os.path.basename(archive_path), dest_name=dest_name ), # Docker permissions make very little sense... see also workdir() - ShellQuoted('USER root'), - ShellQuoted('RUN chown -R {u} {d}').format( - d=dest_name, u=self._user() - ), + ShellQuoted("USER root"), + ShellQuoted("RUN chown -R {u} {d}").format(d=dest_name, u=self._user()), self._change_user(), ] def _render_impl(self, steps): - return raw_shell(shell_join('\n', recursively_flatten_list(steps))) + return raw_shell(shell_join("\n", recursively_flatten_list(steps))) def debian_ccache_setup_steps(self): - source_ccache_tgz = self.option('ccache_tgz', '') + source_ccache_tgz = self.option("ccache_tgz", "") if not source_ccache_tgz: - logging.info('Docker ccache not enabled') + logging.info("Docker ccache not enabled") return [] - dest_ccache_tgz = os.path.join( - self.option('docker_context_dir'), 'ccache.tgz' - ) + dest_ccache_tgz = os.path.join(self.option("docker_context_dir"), "ccache.tgz") try: try: os.link(source_ccache_tgz, dest_ccache_tgz) except OSError: logging.exception( - 'Hard-linking {s} to {d} failed, falling back to copy' - .format(s=source_ccache_tgz, d=dest_ccache_tgz) + "Hard-linking {s} to {d} failed, falling back to copy".format( + s=source_ccache_tgz, d=dest_ccache_tgz + ) ) shutil.copyfile(source_ccache_tgz, dest_ccache_tgz) except Exception: logging.exception( - 'Failed to copy or link {s} to {d}, aborting' - .format(s=source_ccache_tgz, d=dest_ccache_tgz) + "Failed to copy or link {s} to {d}, aborting".format( + s=source_ccache_tgz, d=dest_ccache_tgz + ) ) raise return [ # Separate layer so that in development we avoid re-downloads. - self.run(ShellQuoted('apt-get install -yq ccache')), - ShellQuoted('ADD ccache.tgz /'), + self.run(ShellQuoted("apt-get install -yq ccache")), + ShellQuoted("ADD ccache.tgz /"), ShellQuoted( # Set CCACHE_DIR before the `ccache` invocations below. - 'ENV CCACHE_DIR=/ccache ' + "ENV CCACHE_DIR=/ccache " # No clang support for now, so it's easiest to hardcode gcc. 'CC="ccache gcc" CXX="ccache g++" ' # Always log for ease of debugging. For real FB projects, @@ -166,26 +163,28 @@ class DockerFBCodeBuilder(FBCodeBuilder): # # apt-get install sharutils # bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2 - 'CCACHE_LOGFILE=/tmp/ccache.log' + "CCACHE_LOGFILE=/tmp/ccache.log" + ), + self.run( + ShellQuoted( + # Future: Skipping this part made this Docker step instant, + # saving ~1min of build time. It's unclear if it is the + # chown or the du, but probably the chown -- since a large + # part of the cost is incurred at image save time. + # + # ccache.tgz may be empty, or may have the wrong + # permissions. + "mkdir -p /ccache && time chown -R nobody /ccache && " + "time du -sh /ccache && " + # Reset stats so `docker_build_with_ccache.sh` can print + # useful values at the end of the run. + "echo === Prev run stats === && ccache -s && ccache -z && " + # Record the current time to let travis_build.sh figure out + # the number of bytes in the cache that are actually used -- + # this is crucial for tuning the maximum cache size. + "date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && " + # The build running as `nobody` should be able to write here + "chown nobody /tmp/ccache.log" + ) ), - self.run(ShellQuoted( - # Future: Skipping this part made this Docker step instant, - # saving ~1min of build time. It's unclear if it is the - # chown or the du, but probably the chown -- since a large - # part of the cost is incurred at image save time. - # - # ccache.tgz may be empty, or may have the wrong - # permissions. - 'mkdir -p /ccache && time chown -R nobody /ccache && ' - 'time du -sh /ccache && ' - # Reset stats so `docker_build_with_ccache.sh` can print - # useful values at the end of the run. - 'echo === Prev run stats === && ccache -s && ccache -z && ' - # Record the current time to let travis_build.sh figure out - # the number of bytes in the cache that are actually used -- - # this is crucial for tuning the maximum cache size. - 'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && ' - # The build running as `nobody` should be able to write here - 'chown nobody /tmp/ccache.log' - )), ] diff --git a/build/fbcode_builder/fbcode_builder.py b/build/fbcode_builder/fbcode_builder.py index 5727d71be..742099321 100644 --- a/build/fbcode_builder/fbcode_builder.py +++ b/build/fbcode_builder/fbcode_builder.py @@ -4,7 +4,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals -''' + +""" This is a small DSL to describe builds of Facebook's open-source projects that are published to Github from a single internal repo, including projects @@ -57,7 +58,7 @@ Ideas for the future -- these may not be very good :) * do `make` and `cmake` - If we get non-Debian OSes, part of ccache setup should be factored out. -''' +""" import os import re @@ -66,22 +67,21 @@ from shell_quoting import path_join, shell_join, ShellQuoted def _read_project_github_hashes(): - base_dir = 'deps/github_hashes/' # trailing slash used in regex below + base_dir = "deps/github_hashes/" # trailing slash used in regex below for dirname, _, files in os.walk(base_dir): for filename in files: path = os.path.join(dirname, filename) with open(path) as f: - m_proj = re.match('^' + base_dir + '(.*)-rev\.txt$', path) + m_proj = re.match("^" + base_dir + "(.*)-rev\.txt$", path) if m_proj is None: - raise RuntimeError('Not a hash file? {0}'.format(path)) - m_hash = re.match('^Subproject commit ([0-9a-f]+)\n$', f.read()) + raise RuntimeError("Not a hash file? {0}".format(path)) + m_hash = re.match("^Subproject commit ([0-9a-f]+)\n$", f.read()) if m_hash is None: - raise RuntimeError('No hash in {0}'.format(path)) + raise RuntimeError("No hash in {0}".format(path)) yield m_proj.group(1), m_hash.group(1) class FBCodeBuilder(object): - def __init__(self, **kwargs): self._options_do_not_access = kwargs # Use .option() instead. # This raises upon detecting options that are specified but unused, @@ -90,22 +90,22 @@ class FBCodeBuilder(object): # Mark 'projects_dir' used even if the build installs no github # projects. This is needed because driver programs like # `shell_builder.py` unconditionally set this for all builds. - self._github_dir = self.option('projects_dir') + self._github_dir = self.option("projects_dir") self._github_hashes = dict(_read_project_github_hashes()) def __repr__(self): - return '{0}({1})'.format( + return "{0}({1})".format( self.__class__.__name__, - ', '.join( - '{0}={1}'.format(k, repr(v)) - for k, v in self._options_do_not_access.items() - ) + ", ".join( + "{0}={1}".format(k, repr(v)) + for k, v in self._options_do_not_access.items() + ), ) def option(self, name, default=None): value = self._options_do_not_access.get(name, default) if value is None: - raise RuntimeError('Option {0} is required'.format(name)) + raise RuntimeError("Option {0} is required".format(name)) self.options_used.add(name) return value @@ -114,7 +114,7 @@ class FBCodeBuilder(object): def add_option(self, name, value): if name in self._options_do_not_access: - raise RuntimeError('Option {0} already set'.format(name)) + raise RuntimeError("Option {0} already set".format(name)) self._options_do_not_access[name] = value # @@ -122,12 +122,12 @@ class FBCodeBuilder(object): # def render(self, steps): - ''' + """ Converts nested actions to your builder's expected output format. Typically takes the output of build(). - ''' + """ res = self._render_impl(steps) # Implementation-dependent # Now that the output is rendered, we expect all options to have # been used. @@ -135,41 +135,46 @@ class FBCodeBuilder(object): unused_options -= self.options_used if unused_options: raise RuntimeError( - 'Unused options: {0} -- please check if you made a typo ' - 'in any of them. Those that are truly not useful should ' - 'be not be set so that this typo detection can be useful.' - .format(unused_options) + "Unused options: {0} -- please check if you made a typo " + "in any of them. Those that are truly not useful should " + "be not be set so that this typo detection can be useful.".format( + unused_options + ) ) return res def build(self, steps): if not steps: - raise RuntimeError('Please ensure that the config you are passing ' - 'contains steps') + raise RuntimeError( + "Please ensure that the config you are passing " "contains steps" + ) return [self.setup(), self.diagnostics()] + steps def setup(self): - 'Your builder may want to install packages here.' + "Your builder may want to install packages here." raise NotImplementedError def diagnostics(self): - 'Log some system diagnostics before/after setup for ease of debugging' + "Log some system diagnostics before/after setup for ease of debugging" # The builder's repr is not used in a command to avoid pointlessly # invalidating Docker's build cache. - return self.step('Diagnostics', [ - self.comment('Builder {0}'.format(repr(self))), - self.run(ShellQuoted('hostname')), - self.run(ShellQuoted('cat /etc/issue || echo no /etc/issue')), - self.run(ShellQuoted('g++ --version || echo g++ not installed')), - self.run(ShellQuoted('cmake --version || echo cmake not installed')), - ]) + return self.step( + "Diagnostics", + [ + self.comment("Builder {0}".format(repr(self))), + self.run(ShellQuoted("hostname")), + self.run(ShellQuoted("cat /etc/issue || echo no /etc/issue")), + self.run(ShellQuoted("g++ --version || echo g++ not installed")), + self.run(ShellQuoted("cmake --version || echo cmake not installed")), + ], + ) def step(self, name, actions): - 'A labeled collection of actions or other steps' + "A labeled collection of actions or other steps" raise NotImplementedError def run(self, shell_cmd): - 'Run this bash command' + "Run this bash command" raise NotImplementedError def set_env(self, key, value): @@ -177,54 +182,54 @@ class FBCodeBuilder(object): raise NotImplementedError def workdir(self, dir): - 'Create this directory if it does not exist, and change into it' + "Create this directory if it does not exist, and change into it" raise NotImplementedError def copy_local_repo(self, dir, dest_name): - ''' + """ Copy the local repo at `dir` into this step's `workdir()`, analog of: cp -r /path/to/folly folly - ''' + """ raise NotImplementedError def python_deps(self): return [ - 'wheel', - 'cython==0.28.6', + "wheel", + "cython==0.28.6", ] def debian_deps(self): return [ - 'autoconf-archive', - 'bison', - 'build-essential', - 'cmake', - 'curl', - 'flex', - 'git', - 'gperf', - 'joe', - 'libboost-all-dev', - 'libcap-dev', - 'libdouble-conversion-dev', - 'libevent-dev', - 'libgflags-dev', - 'libgoogle-glog-dev', - 'libkrb5-dev', - 'libpcre3-dev', - 'libpthread-stubs0-dev', - 'libnuma-dev', - 'libsasl2-dev', - 'libsnappy-dev', - 'libsqlite3-dev', - 'libssl-dev', - 'libtool', - 'netcat-openbsd', - 'pkg-config', - 'sudo', - 'unzip', - 'wget', - 'python3-venv', + "autoconf-archive", + "bison", + "build-essential", + "cmake", + "curl", + "flex", + "git", + "gperf", + "joe", + "libboost-all-dev", + "libcap-dev", + "libdouble-conversion-dev", + "libevent-dev", + "libgflags-dev", + "libgoogle-glog-dev", + "libkrb5-dev", + "libpcre3-dev", + "libpthread-stubs0-dev", + "libnuma-dev", + "libsasl2-dev", + "libsnappy-dev", + "libsqlite3-dev", + "libssl-dev", + "libtool", + "netcat-openbsd", + "pkg-config", + "sudo", + "unzip", + "wget", + "python3-venv", ] # @@ -234,51 +239,72 @@ class FBCodeBuilder(object): def install_debian_deps(self): actions = [ self.run( - ShellQuoted('apt-get update && apt-get install -yq {deps}').format( - deps=shell_join(' ', ( - ShellQuoted(dep) for dep in self.debian_deps()))) + ShellQuoted("apt-get update && apt-get install -yq {deps}").format( + deps=shell_join( + " ", (ShellQuoted(dep) for dep in self.debian_deps()) + ) + ) ), ] - gcc_version = self.option('gcc_version') + gcc_version = self.option("gcc_version") # Make the selected GCC the default before building anything - actions.extend([ - self.run(ShellQuoted('apt-get install -yq {c} {cpp}').format( - c=ShellQuoted('gcc-{v}').format(v=gcc_version), - cpp=ShellQuoted('g++-{v}').format(v=gcc_version), - )), - self.run(ShellQuoted( - 'update-alternatives --install /usr/bin/gcc gcc {c} 40 ' - '--slave /usr/bin/g++ g++ {cpp}' - ).format( - c=ShellQuoted('/usr/bin/gcc-{v}').format(v=gcc_version), - cpp=ShellQuoted('/usr/bin/g++-{v}').format(v=gcc_version), - )), - self.run(ShellQuoted('update-alternatives --config gcc')), - ]) + actions.extend( + [ + self.run( + ShellQuoted("apt-get install -yq {c} {cpp}").format( + c=ShellQuoted("gcc-{v}").format(v=gcc_version), + cpp=ShellQuoted("g++-{v}").format(v=gcc_version), + ) + ), + self.run( + ShellQuoted( + "update-alternatives --install /usr/bin/gcc gcc {c} 40 " + "--slave /usr/bin/g++ g++ {cpp}" + ).format( + c=ShellQuoted("/usr/bin/gcc-{v}").format(v=gcc_version), + cpp=ShellQuoted("/usr/bin/g++-{v}").format(v=gcc_version), + ) + ), + self.run(ShellQuoted("update-alternatives --config gcc")), + ] + ) actions.extend(self.debian_ccache_setup_steps()) - return self.step('Install packages for Debian-based OS', actions) + return self.step("Install packages for Debian-based OS", actions) def create_python_venv(self): actions = [] if self.option("PYTHON_VENV", "OFF") == "ON": - actions.append(self.run(ShellQuoted("python3 -m venv {p}").format( - p=path_join(self.option('prefix'), "venv")))) - return(actions) + actions.append( + self.run( + ShellQuoted("python3 -m venv {p}").format( + p=path_join(self.option("prefix"), "venv") + ) + ) + ) + return actions def python_venv(self): actions = [] if self.option("PYTHON_VENV", "OFF") == "ON": - actions.append(ShellQuoted("source {p}").format( - p=path_join(self.option('prefix'), "venv", "bin", "activate"))) + actions.append( + ShellQuoted("source {p}").format( + p=path_join(self.option("prefix"), "venv", "bin", "activate") + ) + ) - actions.append(self.run( - ShellQuoted("python3 -m pip install {deps}").format( - deps=shell_join(' ', (ShellQuoted(dep) for dep in - self.python_deps()))))) - return(actions) + actions.append( + self.run( + ShellQuoted("python3 -m pip install {deps}").format( + deps=shell_join( + " ", (ShellQuoted(dep) for dep in self.python_deps()) + ) + ) + ) + ) + return actions def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True): choices = set(["stable", "beta", "nightly"]) @@ -333,117 +359,148 @@ class FBCodeBuilder(object): # Only check out a non-default branch if requested. This especially # makes sense when building from a local repo. git_hash = self.option( - '{0}:git_hash'.format(project), + "{0}:git_hash".format(project), # Any repo that has a hash in deps/github_hashes defaults to # that, with the goal of making builds maximally consistent. - self._github_hashes.get(project, '') + self._github_hashes.get(project, ""), + ) + maybe_change_branch = ( + [ + self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash)), + ] + if git_hash + else [] ) - maybe_change_branch = [ - self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)), - ] if git_hash else [] - local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '') - return self.step('Check out {0}, workdir {1}'.format(project, path), [ - self.workdir(self._github_dir), - self.run( - ShellQuoted('git clone {opts} https://github.com/{p}').format( - p=project, - opts=ShellQuoted(self.option('{}:git_clone_opts'.format(project), ''))) - ) if not local_repo_dir else self.copy_local_repo( - local_repo_dir, os.path.basename(project) - ), - self.workdir( - path_join(self._github_dir, os.path.basename(project), path), - ), - ] + maybe_change_branch) + local_repo_dir = self.option("{0}:local_repo_dir".format(project), "") + return self.step( + "Check out {0}, workdir {1}".format(project, path), + [ + self.workdir(self._github_dir), + self.run( + ShellQuoted("git clone {opts} https://github.com/{p}").format( + p=project, + opts=ShellQuoted( + self.option("{}:git_clone_opts".format(project), "") + ), + ) + ) + if not local_repo_dir + else self.copy_local_repo(local_repo_dir, os.path.basename(project)), + self.workdir( + path_join(self._github_dir, os.path.basename(project), path), + ), + ] + + maybe_change_branch, + ) - def fb_github_project_workdir(self, project_and_path, github_org='facebook'): - 'This helper lets Facebook-internal CI special-cases FB projects' - project, path = project_and_path.split('/', 1) - return self.github_project_workdir(github_org + '/' + project, path) + def fb_github_project_workdir(self, project_and_path, github_org="facebook"): + "This helper lets Facebook-internal CI special-cases FB projects" + project, path = project_and_path.split("/", 1) + return self.github_project_workdir(github_org + "/" + project, path) def _make_vars(self, make_vars): - return shell_join(' ', ( - ShellQuoted('{k}={v}').format(k=k, v=v) + return shell_join( + " ", + ( + ShellQuoted("{k}={v}").format(k=k, v=v) for k, v in ({} if make_vars is None else make_vars).items() - )) + ), + ) def parallel_make(self, make_vars=None): - return self.run(ShellQuoted('make -j {n} VERBOSE=1 {vars}').format( - n=self.option('make_parallelism'), - vars=self._make_vars(make_vars), - )) + return self.run( + ShellQuoted("make -j {n} VERBOSE=1 {vars}").format( + n=self.option("make_parallelism"), + vars=self._make_vars(make_vars), + ) + ) def make_and_install(self, make_vars=None): return [ self.parallel_make(make_vars), - self.run(ShellQuoted('make install VERBOSE=1 {vars}').format( - vars=self._make_vars(make_vars), - )), + self.run( + ShellQuoted("make install VERBOSE=1 {vars}").format( + vars=self._make_vars(make_vars), + ) + ), ] def configure(self, name=None): autoconf_options = {} if name is not None: autoconf_options.update( - self.option('{0}:autoconf_options'.format(name), {}) + self.option("{0}:autoconf_options".format(name), {}) ) return [ - self.run(ShellQuoted( - 'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" ' - 'CFLAGS="$CFLAGS -I"{p}"/include" ' - 'CPPFLAGS="$CPPFLAGS -I"{p}"/include" ' - 'PY_PREFIX={p} ' - './configure --prefix={p} {args}' - ).format( - p=self.option('prefix'), - args=shell_join(' ', ( - ShellQuoted('{k}={v}').format(k=k, v=v) - for k, v in autoconf_options.items() - )), - )), + self.run( + ShellQuoted( + 'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" ' + 'CFLAGS="$CFLAGS -I"{p}"/include" ' + 'CPPFLAGS="$CPPFLAGS -I"{p}"/include" ' + "PY_PREFIX={p} " + "./configure --prefix={p} {args}" + ).format( + p=self.option("prefix"), + args=shell_join( + " ", + ( + ShellQuoted("{k}={v}").format(k=k, v=v) + for k, v in autoconf_options.items() + ), + ), + ) + ), ] def autoconf_install(self, name): - return self.step('Build and install {0}'.format(name), [ - self.run(ShellQuoted('autoreconf -ivf')), - ] + self.configure() + self.make_and_install()) + return self.step( + "Build and install {0}".format(name), + [ + self.run(ShellQuoted("autoreconf -ivf")), + ] + + self.configure() + + self.make_and_install(), + ) - def cmake_configure(self, name, cmake_path='..'): + def cmake_configure(self, name, cmake_path=".."): cmake_defines = { - 'BUILD_SHARED_LIBS': 'ON', - 'CMAKE_INSTALL_PREFIX': self.option('prefix'), + "BUILD_SHARED_LIBS": "ON", + "CMAKE_INSTALL_PREFIX": self.option("prefix"), } # Hacks to add thriftpy3 support - if 'BUILD_THRIFT_PY3' in os.environ and 'folly' in name: - cmake_defines['PYTHON_EXTENSIONS'] = 'True' + if "BUILD_THRIFT_PY3" in os.environ and "folly" in name: + cmake_defines["PYTHON_EXTENSIONS"] = "True" - if 'BUILD_THRIFT_PY3' in os.environ and 'fbthrift' in name: - cmake_defines['thriftpy3'] = 'ON' + if "BUILD_THRIFT_PY3" in os.environ and "fbthrift" in name: + cmake_defines["thriftpy3"] = "ON" - cmake_defines.update( - self.option('{0}:cmake_defines'.format(name), {}) - ) + cmake_defines.update(self.option("{0}:cmake_defines".format(name), {})) return [ - self.run(ShellQuoted( - 'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" ' - 'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" ' - 'cmake {args} {cmake_path}' - ).format( - p=self.option('prefix'), - args=shell_join(' ', ( - ShellQuoted('-D{k}={v}').format(k=k, v=v) - for k, v in cmake_defines.items() - )), - cmake_path=cmake_path, - )), + self.run( + ShellQuoted( + 'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" ' + 'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" ' + "cmake {args} {cmake_path}" + ).format( + p=self.option("prefix"), + args=shell_join( + " ", + ( + ShellQuoted("-D{k}={v}").format(k=k, v=v) + for k, v in cmake_defines.items() + ), + ), + cmake_path=cmake_path, + ) + ), ] - def cmake_install(self, name, cmake_path='..'): + def cmake_install(self, name, cmake_path=".."): return self.step( - 'Build and install {0}'.format(name), - self.cmake_configure(name, cmake_path) + self.make_and_install() + "Build and install {0}".format(name), + self.cmake_configure(name, cmake_path) + self.make_and_install(), ) def cargo_build(self, name): @@ -458,13 +515,15 @@ class FBCodeBuilder(object): ], ) - def fb_github_autoconf_install(self, project_and_path, github_org='facebook'): + def fb_github_autoconf_install(self, project_and_path, github_org="facebook"): return [ self.fb_github_project_workdir(project_and_path, github_org), self.autoconf_install(project_and_path), ] - def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org='facebook'): + def fb_github_cmake_install( + self, project_and_path, cmake_path="..", github_org="facebook" + ): return [ self.fb_github_project_workdir(project_and_path, github_org), self.cmake_install(project_and_path, cmake_path), diff --git a/build/fbcode_builder/fbcode_builder_config.py b/build/fbcode_builder/fbcode_builder_config.py index c8f868a51..5ba6e607a 100644 --- a/build/fbcode_builder/fbcode_builder_config.py +++ b/build/fbcode_builder/fbcode_builder_config.py @@ -4,12 +4,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals -'Demo config, so that `make_docker_context.py --help` works in this directory.' + +"Demo config, so that `make_docker_context.py --help` works in this directory." config = { - 'fbcode_builder_spec': lambda _builder: { - 'depends_on': [], - 'steps': [], + "fbcode_builder_spec": lambda _builder: { + "depends_on": [], + "steps": [], }, - 'github_project': 'demo/project', + "github_project": "demo/project", } diff --git a/build/fbcode_builder/make_docker_context.py b/build/fbcode_builder/make_docker_context.py index 11986ca65..d4b0f0a89 100755 --- a/build/fbcode_builder/make_docker_context.py +++ b/build/fbcode_builder/make_docker_context.py @@ -4,7 +4,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals -''' + +""" Reads `fbcode_builder_config.py` from the current directory, and prepares a Docker context directory to build this project. Prints to stdout the path to the context directory. @@ -14,7 +15,7 @@ Try `.../make_docker_context.py --help` from a project's `build/` directory. By default, the Docker context directory will be in /tmp. It will always contain a Dockerfile, and might also contain copies of your local repos, and other data needed for the build container. -''' +""" import os import tempfile @@ -27,7 +28,7 @@ from parse_args import parse_args_to_fbcode_builder_opts def make_docker_context( get_steps_fn, github_project, opts=None, default_context_dir=None ): - ''' + """ Returns a path to the Docker context directory. See parse_args.py. Helper for making a command-line utility that writes your project's @@ -38,83 +39,97 @@ def make_docker_context( lambda builder: [builder.step(...), ...], 'facebook/your_project', )) - ''' + """ if opts is None: opts = {} valid_versions = ( - ('ubuntu:16.04', '5'), - ('ubuntu:18.04', '7'), + ("ubuntu:16.04", "5"), + ("ubuntu:18.04", "7"), ) def add_args(parser): parser.add_argument( - '--docker-context-dir', metavar='DIR', + "--docker-context-dir", + metavar="DIR", default=default_context_dir, - help='Write the Dockerfile and its context into this directory. ' - 'If empty, make a temporary directory. Default: %(default)s.', + help="Write the Dockerfile and its context into this directory. " + "If empty, make a temporary directory. Default: %(default)s.", ) parser.add_argument( - '--user', metavar='NAME', default=opts.get('user', 'nobody'), - help='Build and install as this user. Default: %(default)s.', + "--user", + metavar="NAME", + default=opts.get("user", "nobody"), + help="Build and install as this user. Default: %(default)s.", ) parser.add_argument( - '--prefix', metavar='DIR', - default=opts.get('prefix', '/home/install'), - help='Install all libraries in this prefix. Default: %(default)s.', + "--prefix", + metavar="DIR", + default=opts.get("prefix", "/home/install"), + help="Install all libraries in this prefix. Default: %(default)s.", ) parser.add_argument( - '--projects-dir', metavar='DIR', - default=opts.get('projects_dir', '/home'), - help='Place project code directories here. Default: %(default)s.', + "--projects-dir", + metavar="DIR", + default=opts.get("projects_dir", "/home"), + help="Place project code directories here. Default: %(default)s.", ) parser.add_argument( - '--os-image', metavar='IMG', choices=zip(*valid_versions)[0], - default=opts.get('os_image', valid_versions[0][0]), - help='Docker OS image -- be sure to use only ones you trust (See ' - 'README.docker). Choices: %(choices)s. Default: %(default)s.', + "--os-image", + metavar="IMG", + choices=zip(*valid_versions)[0], + default=opts.get("os_image", valid_versions[0][0]), + help="Docker OS image -- be sure to use only ones you trust (See " + "README.docker). Choices: %(choices)s. Default: %(default)s.", ) parser.add_argument( - '--gcc-version', metavar='VER', + "--gcc-version", + metavar="VER", choices=set(zip(*valid_versions)[1]), - default=opts.get('gcc_version', valid_versions[0][1]), - help='Choices: %(choices)s. Default: %(default)s.', + default=opts.get("gcc_version", valid_versions[0][1]), + help="Choices: %(choices)s. Default: %(default)s.", ) parser.add_argument( - '--make-parallelism', metavar='NUM', type=int, - default=opts.get('make_parallelism', 1), - help='Use `make -j` on multi-CPU systems with lots of RAM. ' - 'Default: %(default)s.', + "--make-parallelism", + metavar="NUM", + type=int, + default=opts.get("make_parallelism", 1), + help="Use `make -j` on multi-CPU systems with lots of RAM. " + "Default: %(default)s.", ) parser.add_argument( - '--local-repo-dir', metavar='DIR', - help='If set, build {0} from a local directory instead of Github.' - .format(github_project), + "--local-repo-dir", + metavar="DIR", + help="If set, build {0} from a local directory instead of Github.".format( + github_project + ), ) parser.add_argument( - '--ccache-tgz', metavar='PATH', - help='If set, enable ccache for the build. To initialize the ' - 'cache, first try to hardlink, then to copy --cache-tgz ' - 'as ccache.tgz into the --docker-context-dir.' + "--ccache-tgz", + metavar="PATH", + help="If set, enable ccache for the build. To initialize the " + "cache, first try to hardlink, then to copy --cache-tgz " + "as ccache.tgz into the --docker-context-dir.", ) opts = parse_args_to_fbcode_builder_opts( add_args, # These have add_argument() calls, others are set via --option. ( - 'docker_context_dir', - 'user', - 'prefix', - 'projects_dir', - 'os_image', - 'gcc_version', - 'make_parallelism', - 'local_repo_dir', - 'ccache_tgz', + "docker_context_dir", + "user", + "prefix", + "projects_dir", + "os_image", + "gcc_version", + "make_parallelism", + "local_repo_dir", + "ccache_tgz", ), opts, - help=textwrap.dedent(''' + help=textwrap.dedent( + """ Reads `fbcode_builder_config.py` from the current directory, and prepares a Docker context directory to build {github_project} and @@ -130,47 +145,55 @@ def make_docker_context( Usage: (cd $(./make_docker_context.py) && docker build . 2>&1 | tee log) - '''.format(github_project=github_project)), + """.format( + github_project=github_project + ) + ), ) # This allows travis_docker_build.sh not to know the main Github project. - local_repo_dir = opts.pop('local_repo_dir', None) + local_repo_dir = opts.pop("local_repo_dir", None) if local_repo_dir is not None: - opts['{0}:local_repo_dir'.format(github_project)] = local_repo_dir + opts["{0}:local_repo_dir".format(github_project)] = local_repo_dir - if (opts.get('os_image'), opts.get('gcc_version')) not in valid_versions: + if (opts.get("os_image"), opts.get("gcc_version")) not in valid_versions: raise Exception( - 'Due to 4/5 ABI changes (std::string), we can only use {0}'.format( - ' / '.join('GCC {1} on {0}'.format(*p) for p in valid_versions) + "Due to 4/5 ABI changes (std::string), we can only use {0}".format( + " / ".join("GCC {1} on {0}".format(*p) for p in valid_versions) ) ) - if opts.get('docker_context_dir') is None: - opts['docker_context_dir'] = tempfile.mkdtemp(prefix='docker-context-') - elif not os.path.exists(opts.get('docker_context_dir')): - os.makedirs(opts.get('docker_context_dir')) + if opts.get("docker_context_dir") is None: + opts["docker_context_dir"] = tempfile.mkdtemp(prefix="docker-context-") + elif not os.path.exists(opts.get("docker_context_dir")): + os.makedirs(opts.get("docker_context_dir")) builder = DockerFBCodeBuilder(**opts) - context_dir = builder.option('docker_context_dir') # Mark option "in-use" + context_dir = builder.option("docker_context_dir") # Mark option "in-use" # The renderer may also populate some files into the context_dir. dockerfile = builder.render(get_steps_fn(builder)) - with os.fdopen(os.open( - os.path.join(context_dir, 'Dockerfile'), - os.O_RDWR | os.O_CREAT | os.O_EXCL, # Do not overwrite existing files - 0o644, - ), 'w') as f: + with os.fdopen( + os.open( + os.path.join(context_dir, "Dockerfile"), + os.O_RDWR | os.O_CREAT | os.O_EXCL, # Do not overwrite existing files + 0o644, + ), + "w", + ) as f: f.write(dockerfile) return context_dir -if __name__ == '__main__': +if __name__ == "__main__": from utils import read_fbcode_builder_config, build_fbcode_builder_config # Load a spec from the current directory - config = read_fbcode_builder_config('fbcode_builder_config.py') - print(make_docker_context( - build_fbcode_builder_config(config), - config['github_project'], - )) + config = read_fbcode_builder_config("fbcode_builder_config.py") + print( + make_docker_context( + build_fbcode_builder_config(config), + config["github_project"], + ) + ) diff --git a/build/fbcode_builder/parse_args.py b/build/fbcode_builder/parse_args.py index def9e504d..8d5e35330 100644 --- a/build/fbcode_builder/parse_args.py +++ b/build/fbcode_builder/parse_args.py @@ -4,7 +4,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals -'Argument parsing logic shared by all fbcode_builder CLI tools.' + +"Argument parsing logic shared by all fbcode_builder CLI tools." import argparse import logging @@ -13,7 +14,7 @@ from shell_quoting import raw_shell, ShellQuoted def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help): - ''' + """ Provides some standard arguments: --debug, --option, --shell-quoted-option @@ -26,46 +27,52 @@ def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help): `help` is printed in response to the `--help` argument. - ''' + """ top_level_opts = set(top_level_opts) parser = argparse.ArgumentParser( - description=help, - formatter_class=argparse.RawDescriptionHelpFormatter + description=help, formatter_class=argparse.RawDescriptionHelpFormatter ) add_args_fn(parser) parser.add_argument( - '--option', nargs=2, metavar=('KEY', 'VALUE'), action='append', + "--option", + nargs=2, + metavar=("KEY", "VALUE"), + action="append", default=[ - (k, v) for k, v in opts.items() - if k not in top_level_opts and not isinstance(v, ShellQuoted) + (k, v) + for k, v in opts.items() + if k not in top_level_opts and not isinstance(v, ShellQuoted) ], - help='Set project-specific options. These are assumed to be raw ' - 'strings, to be shell-escaped as needed. Default: %(default)s.', + help="Set project-specific options. These are assumed to be raw " + "strings, to be shell-escaped as needed. Default: %(default)s.", ) parser.add_argument( - '--shell-quoted-option', nargs=2, metavar=('KEY', 'VALUE'), - action='append', + "--shell-quoted-option", + nargs=2, + metavar=("KEY", "VALUE"), + action="append", default=[ - (k, raw_shell(v)) for k, v in opts.items() - if k not in top_level_opts and isinstance(v, ShellQuoted) + (k, raw_shell(v)) + for k, v in opts.items() + if k not in top_level_opts and isinstance(v, ShellQuoted) ], - help='Set project-specific options. These are assumed to be shell-' - 'quoted, and may be used in commands as-is. Default: %(default)s.', + help="Set project-specific options. These are assumed to be shell-" + "quoted, and may be used in commands as-is. Default: %(default)s.", ) - parser.add_argument('--debug', action='store_true', help='Log more') + parser.add_argument("--debug", action="store_true", help="Log more") args = parser.parse_args() logging.basicConfig( level=logging.DEBUG if args.debug else logging.INFO, - format='%(levelname)s: %(message)s' + format="%(levelname)s: %(message)s", ) # Map command-line args back into opts. - logging.debug('opts before command-line arguments: {0}'.format(opts)) + logging.debug("opts before command-line arguments: {0}".format(opts)) new_opts = {} for key in top_level_opts: @@ -78,6 +85,6 @@ def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help): for key, val in args.shell_quoted_option: new_opts[key] = ShellQuoted(val) - logging.debug('opts after command-line arguments: {0}'.format(new_opts)) + logging.debug("opts after command-line arguments: {0}".format(new_opts)) return new_opts diff --git a/build/fbcode_builder/shell_builder.py b/build/fbcode_builder/shell_builder.py index cb848490b..e0d5429ad 100644 --- a/build/fbcode_builder/shell_builder.py +++ b/build/fbcode_builder/shell_builder.py @@ -5,7 +5,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -''' +""" shell_builder.py allows running the fbcode_builder logic on the host rather than in a container. @@ -17,50 +17,50 @@ any failing step will cause the script to exit with failure. cd build python fbcode_builder/shell_builder.py > ~/run.sh bash ~/run.sh -''' +""" -import os import distutils.spawn +import os from fbcode_builder import FBCodeBuilder -from shell_quoting import ( - raw_shell, shell_comment, shell_join, ShellQuoted -) +from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted from utils import recursively_flatten_list class ShellFBCodeBuilder(FBCodeBuilder): def _render_impl(self, steps): - return raw_shell(shell_join('\n', recursively_flatten_list(steps))) + return raw_shell(shell_join("\n", recursively_flatten_list(steps))) def set_env(self, key, value): return ShellQuoted("export {key}={val}").format(key=key, val=value) def workdir(self, dir): return [ - ShellQuoted('mkdir -p {d} && cd {d}').format( - d=dir - ), + ShellQuoted("mkdir -p {d} && cd {d}").format(d=dir), ] def run(self, shell_cmd): - return ShellQuoted('{cmd}').format(cmd=shell_cmd) + return ShellQuoted("{cmd}").format(cmd=shell_cmd) def step(self, name, actions): - assert '\n' not in name, 'Name {0} would span > 1 line'.format(name) - b = ShellQuoted('') - return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b] + assert "\n" not in name, "Name {0} would span > 1 line".format(name) + b = ShellQuoted("") + return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b] def setup(self): - steps = [ - ShellQuoted('set -exo pipefail'), - ] + self.create_python_venv() + self.python_venv() - if self.has_option('ccache_dir'): - ccache_dir = self.option('ccache_dir') + steps = ( + [ + ShellQuoted("set -exo pipefail"), + ] + + self.create_python_venv() + + self.python_venv() + ) + if self.has_option("ccache_dir"): + ccache_dir = self.option("ccache_dir") steps += [ ShellQuoted( # Set CCACHE_DIR before the `ccache` invocations below. - 'export CCACHE_DIR={ccache_dir} ' + "export CCACHE_DIR={ccache_dir} " 'CC="ccache ${{CC:-gcc}}" CXX="ccache ${{CXX:-g++}}"' ).format(ccache_dir=ccache_dir) ] @@ -71,44 +71,44 @@ class ShellFBCodeBuilder(FBCodeBuilder): def copy_local_repo(self, dir, dest_name): return [ - ShellQuoted('cp -r {dir} {dest_name}').format( - dir=dir, - dest_name=dest_name - ), + ShellQuoted("cp -r {dir} {dest_name}").format(dir=dir, dest_name=dest_name), ] def find_project_root(): here = os.path.dirname(os.path.realpath(__file__)) maybe_root = os.path.dirname(os.path.dirname(here)) - if os.path.isdir(os.path.join(maybe_root, '.git')): + if os.path.isdir(os.path.join(maybe_root, ".git")): return maybe_root raise RuntimeError( "I expected shell_builder.py to be in the " - "build/fbcode_builder subdir of a git repo") + "build/fbcode_builder subdir of a git repo" + ) def persistent_temp_dir(repo_root): - escaped = repo_root.replace('/', 'sZs').replace('\\', 'sZs').replace(':', '') - return os.path.join(os.path.expandvars("$HOME"), '.fbcode_builder-' + escaped) + escaped = repo_root.replace("/", "sZs").replace("\\", "sZs").replace(":", "") + return os.path.join(os.path.expandvars("$HOME"), ".fbcode_builder-" + escaped) -if __name__ == '__main__': +if __name__ == "__main__": from utils import read_fbcode_builder_config, build_fbcode_builder_config + repo_root = find_project_root() temp = persistent_temp_dir(repo_root) - config = read_fbcode_builder_config('fbcode_builder_config.py') + config = read_fbcode_builder_config("fbcode_builder_config.py") builder = ShellFBCodeBuilder(projects_dir=temp) - if distutils.spawn.find_executable('ccache'): - builder.add_option('ccache_dir', - os.environ.get('CCACHE_DIR', os.path.join(temp, '.ccache'))) - builder.add_option('prefix', os.path.join(temp, 'installed')) - builder.add_option('make_parallelism', 4) + if distutils.spawn.find_executable("ccache"): + builder.add_option( + "ccache_dir", os.environ.get("CCACHE_DIR", os.path.join(temp, ".ccache")) + ) + builder.add_option("prefix", os.path.join(temp, "installed")) + builder.add_option("make_parallelism", 4) builder.add_option( - '{project}:local_repo_dir'.format(project=config['github_project']), - repo_root) + "{project}:local_repo_dir".format(project=config["github_project"]), repo_root + ) make_steps = build_fbcode_builder_config(config) steps = make_steps(builder) print(builder.render(steps)) diff --git a/build/fbcode_builder/shell_quoting.py b/build/fbcode_builder/shell_quoting.py index f3b968a6d..7429226bd 100644 --- a/build/fbcode_builder/shell_quoting.py +++ b/build/fbcode_builder/shell_quoting.py @@ -4,7 +4,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals -''' + +""" Almost every FBCodeBuilder string is ultimately passed to a shell. Escaping too little or too much tends to be the most common error. The utilities in @@ -16,15 +17,14 @@ this file give a systematic way of avoiding such bugs: - Use `path_join` to join path components. - Use `shell_join` to join already-quoted command arguments or shell lines. -''' +""" import os - from collections import namedtuple -class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))): - ''' +class ShellQuoted(namedtuple("ShellQuoted", ("do_not_use_raw_str",))): + """ Wrap a string with this to make it transparent to shell_quote(). It will almost always suffice to use ShellQuoted.format(), path_join(), @@ -32,27 +32,25 @@ class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))): If you really must, use raw_shell() to access the raw string. - ''' + """ def __new__(cls, s): - 'No need to nest ShellQuoted.' + "No need to nest ShellQuoted." return super(ShellQuoted, cls).__new__( cls, s.do_not_use_raw_str if isinstance(s, ShellQuoted) else s ) def __str__(self): raise RuntimeError( - 'One does not simply convert {0} to a string -- use path_join() ' - 'or ShellQuoted.format() instead'.format(repr(self)) + "One does not simply convert {0} to a string -- use path_join() " + "or ShellQuoted.format() instead".format(repr(self)) ) def __repr__(self): - return '{0}({1})'.format( - self.__class__.__name__, repr(self.do_not_use_raw_str) - ) + return "{0}({1})".format(self.__class__.__name__, repr(self.do_not_use_raw_str)) def format(self, **kwargs): - ''' + """ Use instead of str.format() when the arguments are either `ShellQuoted()` or raw strings needing to be `shell_quote()`d. @@ -60,40 +58,46 @@ class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))): Positional args are deliberately not supported since they are more error-prone. - ''' - return ShellQuoted(self.do_not_use_raw_str.format(**dict( - (k, shell_quote(v).do_not_use_raw_str) for k, v in kwargs.items() - ))) + """ + return ShellQuoted( + self.do_not_use_raw_str.format( + **dict( + (k, shell_quote(v).do_not_use_raw_str) for k, v in kwargs.items() + ) + ) + ) def shell_quote(s): - 'Quotes a string if it is not already quoted' - return s if isinstance(s, ShellQuoted) \ + "Quotes a string if it is not already quoted" + return ( + s + if isinstance(s, ShellQuoted) else ShellQuoted("'" + str(s).replace("'", "'\\''") + "'") + ) def raw_shell(s): - 'Not a member of ShellQuoted so we get a useful error for raw strings' + "Not a member of ShellQuoted so we get a useful error for raw strings" if isinstance(s, ShellQuoted): return s.do_not_use_raw_str - raise RuntimeError('{0} should have been ShellQuoted'.format(s)) + raise RuntimeError("{0} should have been ShellQuoted".format(s)) def shell_join(delim, it): - 'Joins an iterable of ShellQuoted with a delimiter between each two' + "Joins an iterable of ShellQuoted with a delimiter between each two" return ShellQuoted(delim.join(raw_shell(s) for s in it)) def path_join(*args): - 'Joins ShellQuoted and raw pieces of paths to make a shell-quoted path' - return ShellQuoted(os.path.join(*[ - raw_shell(shell_quote(s)) for s in args - ])) + "Joins ShellQuoted and raw pieces of paths to make a shell-quoted path" + return ShellQuoted(os.path.join(*[raw_shell(shell_quote(s)) for s in args])) def shell_comment(c): - 'Do not shell-escape raw strings in comments, but do handle line breaks.' - return ShellQuoted('# {c}').format(c=ShellQuoted( - (raw_shell(c) if isinstance(c, ShellQuoted) else c) - .replace('\n', '\n# ') - )) + "Do not shell-escape raw strings in comments, but do handle line breaks." + return ShellQuoted("# {c}").format( + c=ShellQuoted( + (raw_shell(c) if isinstance(c, ShellQuoted) else c).replace("\n", "\n# ") + ) + ) diff --git a/build/fbcode_builder/specs/fbthrift.py b/build/fbcode_builder/specs/fbthrift.py index 5a81d9980..f0c7e7ac7 100644 --- a/build/fbcode_builder/specs/fbthrift.py +++ b/build/fbcode_builder/specs/fbthrift.py @@ -15,8 +15,8 @@ import specs.zstd as zstd def fbcode_builder_spec(builder): return { - 'depends_on': [fmt, folly, fizz, sodium, wangle, zstd], - 'steps': [ - builder.fb_github_cmake_install('fbthrift/thrift'), + "depends_on": [fmt, folly, fizz, sodium, wangle, zstd], + "steps": [ + builder.fb_github_cmake_install("fbthrift/thrift"), ], } diff --git a/build/fbcode_builder/specs/fbzmq.py b/build/fbcode_builder/specs/fbzmq.py index 1f8f2ba40..78c8bc9dd 100644 --- a/build/fbcode_builder/specs/fbzmq.py +++ b/build/fbcode_builder/specs/fbzmq.py @@ -10,31 +10,40 @@ import specs.fmt as fmt import specs.folly as folly import specs.gmock as gmock import specs.sodium as sodium - from shell_quoting import ShellQuoted def fbcode_builder_spec(builder): - builder.add_option('zeromq/libzmq:git_hash', 'v4.2.2') + builder.add_option("zeromq/libzmq:git_hash", "v4.2.2") return { - 'depends_on': [fmt, folly, fbthrift, gmock, sodium], - 'steps': [ - builder.github_project_workdir('zeromq/libzmq', '.'), - builder.step('Build and install zeromq/libzmq', [ - builder.run(ShellQuoted('./autogen.sh')), - builder.configure(), - builder.make_and_install(), - ]), - - builder.fb_github_project_workdir('fbzmq/_build', 'facebook'), - builder.step('Build and install fbzmq/', [ - builder.cmake_configure('fbzmq/_build'), - # we need the pythonpath to find the thrift compiler - builder.run(ShellQuoted( - 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' - 'make -j {n}' - ).format(p=builder.option('prefix'), n=builder.option('make_parallelism'))), - builder.run(ShellQuoted('make install')), - ]), + "depends_on": [fmt, folly, fbthrift, gmock, sodium], + "steps": [ + builder.github_project_workdir("zeromq/libzmq", "."), + builder.step( + "Build and install zeromq/libzmq", + [ + builder.run(ShellQuoted("./autogen.sh")), + builder.configure(), + builder.make_and_install(), + ], + ), + builder.fb_github_project_workdir("fbzmq/_build", "facebook"), + builder.step( + "Build and install fbzmq/", + [ + builder.cmake_configure("fbzmq/_build"), + # we need the pythonpath to find the thrift compiler + builder.run( + ShellQuoted( + 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' + "make -j {n}" + ).format( + p=builder.option("prefix"), + n=builder.option("make_parallelism"), + ) + ), + builder.run(ShellQuoted("make install")), + ], + ), ], } diff --git a/build/fbcode_builder/specs/fizz.py b/build/fbcode_builder/specs/fizz.py index 5d8e0eff2..f951b156f 100644 --- a/build/fbcode_builder/specs/fizz.py +++ b/build/fbcode_builder/specs/fizz.py @@ -5,9 +5,9 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -import specs.gmock as gmock import specs.fmt as fmt import specs.folly as folly +import specs.gmock as gmock import specs.sodium as sodium diff --git a/build/fbcode_builder/specs/fmt.py b/build/fbcode_builder/specs/fmt.py index 9e3a33467..395316799 100644 --- a/build/fbcode_builder/specs/fmt.py +++ b/build/fbcode_builder/specs/fmt.py @@ -7,20 +7,20 @@ from __future__ import unicode_literals def fbcode_builder_spec(builder): - builder.add_option('fmtlib/fmt:git_hash', '6.2.1') + builder.add_option("fmtlib/fmt:git_hash", "6.2.1") builder.add_option( - 'fmtlib/fmt:cmake_defines', + "fmtlib/fmt:cmake_defines", { # Avoids a bizarred failure to run tests in Bistro: # test_crontab_selector: error while loading shared libraries: # libfmt.so.6: cannot open shared object file: # No such file or directory - 'BUILD_SHARED_LIBS': 'OFF', - } + "BUILD_SHARED_LIBS": "OFF", + }, ) return { - 'steps': [ - builder.github_project_workdir('fmtlib/fmt', 'build'), - builder.cmake_install('fmtlib/fmt'), + "steps": [ + builder.github_project_workdir("fmtlib/fmt", "build"), + builder.cmake_install("fmtlib/fmt"), ], } diff --git a/build/fbcode_builder/specs/folly.py b/build/fbcode_builder/specs/folly.py index 09e1531c4..e89d5e955 100644 --- a/build/fbcode_builder/specs/folly.py +++ b/build/fbcode_builder/specs/folly.py @@ -11,13 +11,13 @@ import specs.fmt as fmt def fbcode_builder_spec(builder): return { "depends_on": [fmt], - 'steps': [ + "steps": [ # on macOS the filesystem is typically case insensitive. # We need to ensure that the CWD is not the folly source # dir when we build, otherwise the system will decide # that `folly/String.h` is the file it wants when including # `string.h` and the build will fail. - builder.fb_github_project_workdir('folly/_build'), - builder.cmake_install('facebook/folly'), + builder.fb_github_project_workdir("folly/_build"), + builder.cmake_install("facebook/folly"), ], } diff --git a/build/fbcode_builder/specs/gmock.py b/build/fbcode_builder/specs/gmock.py index 8b0562f7e..774137301 100644 --- a/build/fbcode_builder/specs/gmock.py +++ b/build/fbcode_builder/specs/gmock.py @@ -7,18 +7,18 @@ from __future__ import unicode_literals def fbcode_builder_spec(builder): - builder.add_option('google/googletest:git_hash', 'release-1.8.1') + builder.add_option("google/googletest:git_hash", "release-1.8.1") builder.add_option( - 'google/googletest:cmake_defines', + "google/googletest:cmake_defines", { - 'BUILD_GTEST': 'ON', + "BUILD_GTEST": "ON", # Avoid problems with MACOSX_RPATH - 'BUILD_SHARED_LIBS': 'OFF', - } + "BUILD_SHARED_LIBS": "OFF", + }, ) return { - 'steps': [ - builder.github_project_workdir('google/googletest', 'build'), - builder.cmake_install('google/googletest'), + "steps": [ + builder.github_project_workdir("google/googletest", "build"), + builder.cmake_install("google/googletest"), ], } diff --git a/build/fbcode_builder/specs/mvfst.py b/build/fbcode_builder/specs/mvfst.py index e9213ce43..ce8b003d9 100644 --- a/build/fbcode_builder/specs/mvfst.py +++ b/build/fbcode_builder/specs/mvfst.py @@ -5,9 +5,9 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -import specs.gmock as gmock -import specs.folly as folly import specs.fizz as fizz +import specs.folly as folly +import specs.gmock as gmock def fbcode_builder_spec(builder): diff --git a/build/fbcode_builder/specs/proxygen.py b/build/fbcode_builder/specs/proxygen.py index 28f160f65..6a584d710 100644 --- a/build/fbcode_builder/specs/proxygen.py +++ b/build/fbcode_builder/specs/proxygen.py @@ -5,10 +5,10 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -import specs.gmock as gmock +import specs.fizz as fizz import specs.fmt as fmt import specs.folly as folly -import specs.fizz as fizz +import specs.gmock as gmock import specs.mvfst as mvfst import specs.sodium as sodium import specs.wangle as wangle diff --git a/build/fbcode_builder/specs/proxygen_quic.py b/build/fbcode_builder/specs/proxygen_quic.py index b095c06e1..b4959fb89 100644 --- a/build/fbcode_builder/specs/proxygen_quic.py +++ b/build/fbcode_builder/specs/proxygen_quic.py @@ -5,10 +5,10 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -import specs.gmock as gmock +import specs.fizz as fizz import specs.fmt as fmt import specs.folly as folly -import specs.fizz as fizz +import specs.gmock as gmock import specs.mvfst as mvfst import specs.sodium as sodium import specs.wangle as wangle diff --git a/build/fbcode_builder/specs/re2.py b/build/fbcode_builder/specs/re2.py index b6b81ab94..cf4e08a0b 100644 --- a/build/fbcode_builder/specs/re2.py +++ b/build/fbcode_builder/specs/re2.py @@ -8,8 +8,8 @@ from __future__ import unicode_literals def fbcode_builder_spec(builder): return { - 'steps': [ - builder.github_project_workdir('google/re2', 'build'), - builder.cmake_install('google/re2'), + "steps": [ + builder.github_project_workdir("google/re2", "build"), + builder.cmake_install("google/re2"), ], } diff --git a/build/fbcode_builder/specs/rocksdb.py b/build/fbcode_builder/specs/rocksdb.py index c7d7c6ac2..9ebfe4739 100644 --- a/build/fbcode_builder/specs/rocksdb.py +++ b/build/fbcode_builder/specs/rocksdb.py @@ -7,10 +7,13 @@ from __future__ import unicode_literals def fbcode_builder_spec(builder): - builder.add_option("rocksdb/_build:cmake_defines", { - "USE_RTTI": "1", - "PORTABLE": "ON", - }) + builder.add_option( + "rocksdb/_build:cmake_defines", + { + "USE_RTTI": "1", + "PORTABLE": "ON", + }, + ) return { "steps": [ builder.fb_github_cmake_install("rocksdb/_build"), diff --git a/build/fbcode_builder/specs/sodium.py b/build/fbcode_builder/specs/sodium.py index 52bb0006e..8be9833cf 100644 --- a/build/fbcode_builder/specs/sodium.py +++ b/build/fbcode_builder/specs/sodium.py @@ -9,14 +9,17 @@ from shell_quoting import ShellQuoted def fbcode_builder_spec(builder): - builder.add_option('jedisct1/libsodium:git_hash', 'stable') + builder.add_option("jedisct1/libsodium:git_hash", "stable") return { - 'steps': [ - builder.github_project_workdir('jedisct1/libsodium', '.'), - builder.step('Build and install jedisct1/libsodium', [ - builder.run(ShellQuoted('./autogen.sh')), - builder.configure(), - builder.make_and_install(), - ]), + "steps": [ + builder.github_project_workdir("jedisct1/libsodium", "."), + builder.step( + "Build and install jedisct1/libsodium", + [ + builder.run(ShellQuoted("./autogen.sh")), + builder.configure(), + builder.make_and_install(), + ], + ), ], } diff --git a/build/fbcode_builder/specs/wangle.py b/build/fbcode_builder/specs/wangle.py index db1c5fc1d..62b5b3c86 100644 --- a/build/fbcode_builder/specs/wangle.py +++ b/build/fbcode_builder/specs/wangle.py @@ -5,10 +5,10 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -import specs.gmock as gmock +import specs.fizz as fizz import specs.fmt as fmt import specs.folly as folly -import specs.fizz as fizz +import specs.gmock as gmock import specs.sodium as sodium diff --git a/build/fbcode_builder/specs/zstd.py b/build/fbcode_builder/specs/zstd.py index d24385dd7..14d9a1249 100644 --- a/build/fbcode_builder/specs/zstd.py +++ b/build/fbcode_builder/specs/zstd.py @@ -11,16 +11,21 @@ from shell_quoting import ShellQuoted def fbcode_builder_spec(builder): # This API should change rarely, so build the latest tag instead of master. builder.add_option( - 'facebook/zstd:git_hash', - ShellQuoted('$(git describe --abbrev=0 --tags origin/master)') + "facebook/zstd:git_hash", + ShellQuoted("$(git describe --abbrev=0 --tags origin/master)"), ) return { - 'steps': [ - builder.github_project_workdir('facebook/zstd', '.'), - builder.step('Build and install zstd', [ - builder.make_and_install(make_vars={ - 'PREFIX': builder.option('prefix'), - }) - ]), + "steps": [ + builder.github_project_workdir("facebook/zstd", "."), + builder.step( + "Build and install zstd", + [ + builder.make_and_install( + make_vars={ + "PREFIX": builder.option("prefix"), + } + ) + ], + ), ], } diff --git a/build/fbcode_builder/utils.py b/build/fbcode_builder/utils.py index bdf7b01d5..02459a200 100644 --- a/build/fbcode_builder/utils.py +++ b/build/fbcode_builder/utils.py @@ -4,7 +4,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals -'Miscellaneous utility functions.' + +"Miscellaneous utility functions." import itertools import logging @@ -12,21 +13,19 @@ import os import shutil import subprocess import sys - from contextlib import contextmanager def recursively_flatten_list(l): return itertools.chain.from_iterable( - (recursively_flatten_list(i) if type(i) is list else (i,)) - for i in l + (recursively_flatten_list(i) if type(i) is list else (i,)) for i in l ) def run_command(*cmd, **kwargs): - 'The stdout of most fbcode_builder utilities is meant to be parsed.' - logging.debug('Running: {0} with {1}'.format(cmd, kwargs)) - kwargs['stdout'] = sys.stderr + "The stdout of most fbcode_builder utilities is meant to be parsed." + logging.debug("Running: {0} with {1}".format(cmd, kwargs)) + kwargs["stdout"] = sys.stderr subprocess.check_call(cmd, **kwargs) @@ -40,13 +39,13 @@ def make_temp_dir(d): def _inner_read_config(path): - ''' + """ Helper to read a named config file. The grossness with the global is a workaround for this python bug: https://bugs.python.org/issue21591 The bug prevents us from defining either a local function or a lambda in the scope of read_fbcode_builder_config below. - ''' + """ global _project_dir full_path = os.path.join(_project_dir, path) return read_fbcode_builder_config(full_path) @@ -60,37 +59,37 @@ def read_fbcode_builder_config(filename): global _project_dir _project_dir = os.path.dirname(filename) - scope = {'read_fbcode_builder_config': _inner_read_config} + scope = {"read_fbcode_builder_config": _inner_read_config} with open(filename) as config_file: - code = compile(config_file.read(), filename, mode='exec') + code = compile(config_file.read(), filename, mode="exec") exec(code, scope) - return scope['config'] + return scope["config"] def steps_for_spec(builder, spec, processed_modules=None): - ''' + """ Sets `builder` configuration, and returns all the builder steps necessary to build `spec` and its dependencies. Traverses the dependencies in depth-first order, honoring the sequencing in each 'depends_on' list. - ''' + """ if processed_modules is None: processed_modules = set() steps = [] - for module in spec.get('depends_on', []): + for module in spec.get("depends_on", []): if module not in processed_modules: processed_modules.add(module) - steps.extend(steps_for_spec( - builder, - module.fbcode_builder_spec(builder), - processed_modules - )) - steps.extend(spec.get('steps', [])) + steps.extend( + steps_for_spec( + builder, module.fbcode_builder_spec(builder), processed_modules + ) + ) + steps.extend(spec.get("steps", [])) return steps def build_fbcode_builder_config(config): return lambda builder: builder.build( - steps_for_spec(builder, config['fbcode_builder_spec'](builder)) + steps_for_spec(builder, config["fbcode_builder_spec"](builder)) )