lib/proxygen
1
0
Fork 0
mirror of https://github.com/facebook/proxygen.git synced 2025-08-10 05:22:59 +03:00
Code Activity

Opt in opensource/fbcode_builder to pyfmt

Browse Source 
Reviewed By: zertosh

Differential Revision: D29612107

fbshipit-source-id: ac450058134e23a3831db35d2e49c80eb8cde36a
This commit is contained in:
Zsolt Dollenstein
2021-07-09 06:17:52 -07:00
committed by Facebook GitHub Bot
parent f3b62e2316
commit 40b6e08b08
24 changed files with 612 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

411
build/fbcode_builder/fbcode_builder.py
View File

@@ -4,7 +4,8 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
"""
This is a small DSL to describe builds of Facebook's open-source projects
that are published to Github from a single internal repo, including projects
@@ -57,7 +58,7 @@ Ideas for the future -- these may not be very good :)
* do `make` and `cmake`
- If we get non-Debian OSes, part of ccache setup should be factored out.
'''
"""
import os
import re
@@ -66,22 +67,21 @@ from shell_quoting import path_join, shell_join, ShellQuoted
def _read_project_github_hashes():
base_dir = 'deps/github_hashes/' # trailing slash used in regex below
base_dir = "deps/github_hashes/" # trailing slash used in regex below
for dirname, _, files in os.walk(base_dir):
for filename in files:
path = os.path.join(dirname, filename)
with open(path) as f:
m_proj = re.match('^' + base_dir + '(.*)-rev\.txt$', path)
m_proj = re.match("^" + base_dir + "(.*)-rev\.txt$", path)
if m_proj is None:
raise RuntimeError('Not a hash file? {0}'.format(path))
m_hash = re.match('^Subproject commit ([0-9a-f]+)\n$', f.read())
raise RuntimeError("Not a hash file? {0}".format(path))
m_hash = re.match("^Subproject commit ([0-9a-f]+)\n$", f.read())
if m_hash is None:
raise RuntimeError('No hash in {0}'.format(path))
raise RuntimeError("No hash in {0}".format(path))
yield m_proj.group(1), m_hash.group(1)
class FBCodeBuilder(object):
def __init__(self, **kwargs):
self._options_do_not_access = kwargs # Use .option() instead.
# This raises upon detecting options that are specified but unused,
@@ -90,22 +90,22 @@ class FBCodeBuilder(object):
# Mark 'projects_dir' used even if the build installs no github
# projects. This is needed because driver programs like
# `shell_builder.py` unconditionally set this for all builds.
self._github_dir = self.option('projects_dir')
self._github_dir = self.option("projects_dir")
self._github_hashes = dict(_read_project_github_hashes())
def __repr__(self):
return '{0}({1})'.format(
return "{0}({1})".format(
self.__class__.__name__,
', '.join(
'{0}={1}'.format(k, repr(v))
for k, v in self._options_do_not_access.items()
)
", ".join(
"{0}={1}".format(k, repr(v))
for k, v in self._options_do_not_access.items()
),
)
def option(self, name, default=None):
value = self._options_do_not_access.get(name, default)
if value is None:
raise RuntimeError('Option {0} is required'.format(name))
raise RuntimeError("Option {0} is required".format(name))
self.options_used.add(name)
return value
@@ -114,7 +114,7 @@ class FBCodeBuilder(object):
def add_option(self, name, value):
if name in self._options_do_not_access:
raise RuntimeError('Option {0} already set'.format(name))
raise RuntimeError("Option {0} already set".format(name))
self._options_do_not_access[name] = value
#
@@ -122,12 +122,12 @@ class FBCodeBuilder(object):
#
def render(self, steps):
'''
"""
Converts nested actions to your builder's expected output format.
Typically takes the output of build().
'''
"""
res = self._render_impl(steps) # Implementation-dependent
# Now that the output is rendered, we expect all options to have
# been used.
@@ -135,41 +135,46 @@ class FBCodeBuilder(object):
unused_options -= self.options_used
if unused_options:
raise RuntimeError(
'Unused options: {0} -- please check if you made a typo '
'in any of them. Those that are truly not useful should '
'be not be set so that this typo detection can be useful.'
.format(unused_options)
"Unused options: {0} -- please check if you made a typo "
"in any of them. Those that are truly not useful should "
"be not be set so that this typo detection can be useful.".format(
unused_options
)
)
return res
def build(self, steps):
if not steps:
raise RuntimeError('Please ensure that the config you are passing '
'contains steps')
raise RuntimeError(
"Please ensure that the config you are passing " "contains steps"
)
return [self.setup(), self.diagnostics()] + steps
def setup(self):
'Your builder may want to install packages here.'
"Your builder may want to install packages here."
raise NotImplementedError
def diagnostics(self):
'Log some system diagnostics before/after setup for ease of debugging'
"Log some system diagnostics before/after setup for ease of debugging"
# The builder's repr is not used in a command to avoid pointlessly
# invalidating Docker's build cache.
return self.step('Diagnostics', [
self.comment('Builder {0}'.format(repr(self))),
self.run(ShellQuoted('hostname')),
self.run(ShellQuoted('cat /etc/issue || echo no /etc/issue')),
self.run(ShellQuoted('g++ --version || echo g++ not installed')),
self.run(ShellQuoted('cmake --version || echo cmake not installed')),
])
return self.step(
"Diagnostics",
[
self.comment("Builder {0}".format(repr(self))),
self.run(ShellQuoted("hostname")),
self.run(ShellQuoted("cat /etc/issue || echo no /etc/issue")),
self.run(ShellQuoted("g++ --version || echo g++ not installed")),
self.run(ShellQuoted("cmake --version || echo cmake not installed")),
],
)
def step(self, name, actions):
'A labeled collection of actions or other steps'
"A labeled collection of actions or other steps"
raise NotImplementedError
def run(self, shell_cmd):
'Run this bash command'
"Run this bash command"
raise NotImplementedError
def set_env(self, key, value):
@@ -177,54 +182,54 @@ class FBCodeBuilder(object):
raise NotImplementedError
def workdir(self, dir):
'Create this directory if it does not exist, and change into it'
"Create this directory if it does not exist, and change into it"
raise NotImplementedError
def copy_local_repo(self, dir, dest_name):
'''
"""
Copy the local repo at `dir` into this step's `workdir()`, analog of:
cp -r /path/to/folly folly
'''
"""
raise NotImplementedError
def python_deps(self):
return [
'wheel',
'cython==0.28.6',
"wheel",
"cython==0.28.6",
]
def debian_deps(self):
return [
'autoconf-archive',
'bison',
'build-essential',
'cmake',
'curl',
'flex',
'git',
'gperf',
'joe',
'libboost-all-dev',
'libcap-dev',
'libdouble-conversion-dev',
'libevent-dev',
'libgflags-dev',
'libgoogle-glog-dev',
'libkrb5-dev',
'libpcre3-dev',
'libpthread-stubs0-dev',
'libnuma-dev',
'libsasl2-dev',
'libsnappy-dev',
'libsqlite3-dev',
'libssl-dev',
'libtool',
'netcat-openbsd',
'pkg-config',
'sudo',
'unzip',
'wget',
'python3-venv',
"autoconf-archive",
"bison",
"build-essential",
"cmake",
"curl",
"flex",
"git",
"gperf",
"joe",
"libboost-all-dev",
"libcap-dev",
"libdouble-conversion-dev",
"libevent-dev",
"libgflags-dev",
"libgoogle-glog-dev",
"libkrb5-dev",
"libpcre3-dev",
"libpthread-stubs0-dev",
"libnuma-dev",
"libsasl2-dev",
"libsnappy-dev",
"libsqlite3-dev",
"libssl-dev",
"libtool",
"netcat-openbsd",
"pkg-config",
"sudo",
"unzip",
"wget",
"python3-venv",
]
#
@@ -234,51 +239,72 @@ class FBCodeBuilder(object):
def install_debian_deps(self):
actions = [
self.run(
ShellQuoted('apt-get update && apt-get install -yq {deps}').format(
deps=shell_join(' ', (
ShellQuoted(dep) for dep in self.debian_deps())))
ShellQuoted("apt-get update && apt-get install -yq {deps}").format(
deps=shell_join(
" ", (ShellQuoted(dep) for dep in self.debian_deps())
)
)
),
]
gcc_version = self.option('gcc_version')
gcc_version = self.option("gcc_version")
# Make the selected GCC the default before building anything
actions.extend([
self.run(ShellQuoted('apt-get install -yq {c} {cpp}').format(
c=ShellQuoted('gcc-{v}').format(v=gcc_version),
cpp=ShellQuoted('g++-{v}').format(v=gcc_version),
)),
self.run(ShellQuoted(
'update-alternatives --install /usr/bin/gcc gcc {c} 40 '
'--slave /usr/bin/g++ g++ {cpp}'
).format(
c=ShellQuoted('/usr/bin/gcc-{v}').format(v=gcc_version),
cpp=ShellQuoted('/usr/bin/g++-{v}').format(v=gcc_version),
)),
self.run(ShellQuoted('update-alternatives --config gcc')),
])
actions.extend(
[
self.run(
ShellQuoted("apt-get install -yq {c} {cpp}").format(
c=ShellQuoted("gcc-{v}").format(v=gcc_version),
cpp=ShellQuoted("g++-{v}").format(v=gcc_version),
)
),
self.run(
ShellQuoted(
"update-alternatives --install /usr/bin/gcc gcc {c} 40 "
"--slave /usr/bin/g++ g++ {cpp}"
).format(
c=ShellQuoted("/usr/bin/gcc-{v}").format(v=gcc_version),
cpp=ShellQuoted("/usr/bin/g++-{v}").format(v=gcc_version),
)
),
self.run(ShellQuoted("update-alternatives --config gcc")),
]
)
actions.extend(self.debian_ccache_setup_steps())
return self.step('Install packages for Debian-based OS', actions)
return self.step("Install packages for Debian-based OS", actions)
def create_python_venv(self):
actions = []
if self.option("PYTHON_VENV", "OFF") == "ON":
actions.append(self.run(ShellQuoted("python3 -m venv {p}").format(
p=path_join(self.option('prefix'), "venv"))))
return(actions)
actions.append(
self.run(
ShellQuoted("python3 -m venv {p}").format(
p=path_join(self.option("prefix"), "venv")
)
)
)
return actions
def python_venv(self):
actions = []
if self.option("PYTHON_VENV", "OFF") == "ON":
actions.append(ShellQuoted("source {p}").format(
p=path_join(self.option('prefix'), "venv", "bin", "activate")))
actions.append(
ShellQuoted("source {p}").format(
p=path_join(self.option("prefix"), "venv", "bin", "activate")
)
)
actions.append(self.run(
ShellQuoted("python3 -m pip install {deps}").format(
deps=shell_join(' ', (ShellQuoted(dep) for dep in
self.python_deps())))))
return(actions)
actions.append(
self.run(
ShellQuoted("python3 -m pip install {deps}").format(
deps=shell_join(
" ", (ShellQuoted(dep) for dep in self.python_deps())
)
)
)
)
return actions
def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True):
choices = set(["stable", "beta", "nightly"])
@@ -333,117 +359,148 @@ class FBCodeBuilder(object):
# Only check out a non-default branch if requested. This especially
# makes sense when building from a local repo.
git_hash = self.option(
'{0}:git_hash'.format(project),
"{0}:git_hash".format(project),
# Any repo that has a hash in deps/github_hashes defaults to
# that, with the goal of making builds maximally consistent.
self._github_hashes.get(project, '')
self._github_hashes.get(project, ""),
)
maybe_change_branch = (
[
self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash)),
]
if git_hash
else []
)
maybe_change_branch = [
self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
] if git_hash else []
local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
return self.step('Check out {0}, workdir {1}'.format(project, path), [
self.workdir(self._github_dir),
self.run(
ShellQuoted('git clone {opts} https://github.com/{p}').format(
p=project,
opts=ShellQuoted(self.option('{}:git_clone_opts'.format(project), '')))
) if not local_repo_dir else self.copy_local_repo(
local_repo_dir, os.path.basename(project)
),
self.workdir(
path_join(self._github_dir, os.path.basename(project), path),
),
] + maybe_change_branch)
local_repo_dir = self.option("{0}:local_repo_dir".format(project), "")
return self.step(
"Check out {0}, workdir {1}".format(project, path),
[
self.workdir(self._github_dir),
self.run(
ShellQuoted("git clone {opts} https://github.com/{p}").format(
p=project,
opts=ShellQuoted(
self.option("{}:git_clone_opts".format(project), "")
),
)
)
if not local_repo_dir
else self.copy_local_repo(local_repo_dir, os.path.basename(project)),
self.workdir(
path_join(self._github_dir, os.path.basename(project), path),
),
]
+ maybe_change_branch,
)
def fb_github_project_workdir(self, project_and_path, github_org='facebook'):
'This helper lets Facebook-internal CI special-cases FB projects'
project, path = project_and_path.split('/', 1)
return self.github_project_workdir(github_org + '/' + project, path)
def fb_github_project_workdir(self, project_and_path, github_org="facebook"):
"This helper lets Facebook-internal CI special-cases FB projects"
project, path = project_and_path.split("/", 1)
return self.github_project_workdir(github_org + "/" + project, path)
def _make_vars(self, make_vars):
return shell_join(' ', (
ShellQuoted('{k}={v}').format(k=k, v=v)
return shell_join(
" ",
(
ShellQuoted("{k}={v}").format(k=k, v=v)
for k, v in ({} if make_vars is None else make_vars).items()
))
),
)
def parallel_make(self, make_vars=None):
return self.run(ShellQuoted('make -j {n} VERBOSE=1 {vars}').format(
n=self.option('make_parallelism'),
vars=self._make_vars(make_vars),
))
return self.run(
ShellQuoted("make -j {n} VERBOSE=1 {vars}").format(
n=self.option("make_parallelism"),
vars=self._make_vars(make_vars),
)
)
def make_and_install(self, make_vars=None):
return [
self.parallel_make(make_vars),
self.run(ShellQuoted('make install VERBOSE=1 {vars}').format(
vars=self._make_vars(make_vars),
)),
self.run(
ShellQuoted("make install VERBOSE=1 {vars}").format(
vars=self._make_vars(make_vars),
)
),
]
def configure(self, name=None):
autoconf_options = {}
if name is not None:
autoconf_options.update(
self.option('{0}:autoconf_options'.format(name), {})
self.option("{0}:autoconf_options".format(name), {})
)
return [
self.run(ShellQuoted(
'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
'CFLAGS="$CFLAGS -I"{p}"/include" '
'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
'PY_PREFIX={p} '
'./configure --prefix={p} {args}'
).format(
p=self.option('prefix'),
args=shell_join(' ', (
ShellQuoted('{k}={v}').format(k=k, v=v)
for k, v in autoconf_options.items()
)),
)),
self.run(
ShellQuoted(
'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
'CFLAGS="$CFLAGS -I"{p}"/include" '
'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
"PY_PREFIX={p} "
"./configure --prefix={p} {args}"
).format(
p=self.option("prefix"),
args=shell_join(
" ",
(
ShellQuoted("{k}={v}").format(k=k, v=v)
for k, v in autoconf_options.items()
),
),
)
),
]
def autoconf_install(self, name):
return self.step('Build and install {0}'.format(name), [
self.run(ShellQuoted('autoreconf -ivf')),
] + self.configure() + self.make_and_install())
return self.step(
"Build and install {0}".format(name),
[
self.run(ShellQuoted("autoreconf -ivf")),
]
+ self.configure()
+ self.make_and_install(),
)
def cmake_configure(self, name, cmake_path='..'):
def cmake_configure(self, name, cmake_path=".."):
cmake_defines = {
'BUILD_SHARED_LIBS': 'ON',
'CMAKE_INSTALL_PREFIX': self.option('prefix'),
"BUILD_SHARED_LIBS": "ON",
"CMAKE_INSTALL_PREFIX": self.option("prefix"),
}
# Hacks to add thriftpy3 support
if 'BUILD_THRIFT_PY3' in os.environ and 'folly' in name:
cmake_defines['PYTHON_EXTENSIONS'] = 'True'
if "BUILD_THRIFT_PY3" in os.environ and "folly" in name:
cmake_defines["PYTHON_EXTENSIONS"] = "True"
if 'BUILD_THRIFT_PY3' in os.environ and 'fbthrift' in name:
cmake_defines['thriftpy3'] = 'ON'
if "BUILD_THRIFT_PY3" in os.environ and "fbthrift" in name:
cmake_defines["thriftpy3"] = "ON"
cmake_defines.update(
self.option('{0}:cmake_defines'.format(name), {})
)
cmake_defines.update(self.option("{0}:cmake_defines".format(name), {}))
return [
self.run(ShellQuoted(
'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" '
'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" '
'cmake {args} {cmake_path}'
).format(
p=self.option('prefix'),
args=shell_join(' ', (
ShellQuoted('-D{k}={v}').format(k=k, v=v)
for k, v in cmake_defines.items()
)),
cmake_path=cmake_path,
)),
self.run(
ShellQuoted(
'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" '
'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" '
"cmake {args} {cmake_path}"
).format(
p=self.option("prefix"),
args=shell_join(
" ",
(
ShellQuoted("-D{k}={v}").format(k=k, v=v)
for k, v in cmake_defines.items()
),
),
cmake_path=cmake_path,
)
),
]
def cmake_install(self, name, cmake_path='..'):
def cmake_install(self, name, cmake_path=".."):
return self.step(
'Build and install {0}'.format(name),
self.cmake_configure(name, cmake_path) + self.make_and_install()
"Build and install {0}".format(name),
self.cmake_configure(name, cmake_path) + self.make_and_install(),
)
def cargo_build(self, name):
@@ -458,13 +515,15 @@ class FBCodeBuilder(object):
],
)
def fb_github_autoconf_install(self, project_and_path, github_org='facebook'):
def fb_github_autoconf_install(self, project_and_path, github_org="facebook"):
return [
self.fb_github_project_workdir(project_and_path, github_org),
self.autoconf_install(project_and_path),
]
def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org='facebook'):
def fb_github_cmake_install(
self, project_and_path, cmake_path="..", github_org="facebook"
):
return [
self.fb_github_project_workdir(project_and_path, github_org),
self.cmake_install(project_and_path, cmake_path),