mirror of
https://github.com/facebook/proxygen.git
synced 2025-08-05 19:55:47 +03:00
getdeps: Add support for brew packages and define package dependencies
Summary: In order to speed up build times on a mac, add homebrew support to getdeps. Homebrew packages can be declared in a manifest using the `homebrew` header. Futher, ahornby has added manifest entries for homebrew packages which are included in this diff and also included a change to use the correct version of openssl. Without this openssl change, homebrew cmake configure finds an old openssl 1.0.2 install. This diff provides a 2x speed up for building folly: Timings for clean getdeps folly build on mid-2018 2.9Ghz i9 6 core intel macbook pro with 32GB RAM: With new homebrew system deps: ``` rm -rf /Users/ahornby/.scratch/UsersZahornbyZfbsource/fbcode_builder_getdeps/ time ./opensource/fbcode_builder/getdeps.py build --allow-system-packages folly real 17m39.329s user 76m10.317s sys 5m50.163s ``` Without: ``` rm -rf /Users/ahornby/.scratch/UsersZahornbyZfbsource/fbcode_builder_getdeps/ time ./opensource/fbcode_builder/getdeps.py build folly real 32m10.344s user 105m53.448s sys 15m57.858s ``` Reviewed By: ahornby Differential Revision: D33842632 fbshipit-source-id: ac785d4a8dcfa31b77292bddd9e747022ac36e3b
This commit is contained in:
Harvey Hunt
committed by
Facebook GitHub Bot
Facebook GitHub Bot
parent
1cd8738897
commit
a75c04f38c
@@ -415,6 +415,11 @@ class InstallSysDepsCmd(ProjectCmdBase):
|
||||
packages = sorted(set(all_packages["deb"]))
|
||||
if packages:
|
||||
cmd_args = ["apt", "install", "-y"] + packages
|
||||
elif manager == "homebrew":
|
||||
packages = sorted(set(all_packages["homebrew"]))
|
||||
if packages:
|
||||
cmd_args = ["brew", "install"] + packages
|
||||
|
||||
else:
|
||||
host_tuple = loader.build_opts.host_type.as_tuple_string()
|
||||
print(
|
||||
|
||||
@@ -14,7 +14,7 @@ from typing import Optional, Mapping
|
||||
|
||||
from .copytree import containing_repo_type
|
||||
from .envfuncs import Env, add_path_entry
|
||||
from .fetcher import get_fbsource_repo_data
|
||||
from .fetcher import get_fbsource_repo_data, homebrew_package_prefix
|
||||
from .manifest import ContextGenerator
|
||||
from .platform import HostType, is_windows, get_available_ram
|
||||
|
||||
@@ -206,6 +206,12 @@ class BuildOptions(object):
|
||||
sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"])
|
||||
env["SDKROOT"] = sdkroot.decode().strip()
|
||||
|
||||
# MacOS includes a version of bison so homebrew won't automatically add
|
||||
# its own version to PATH. Find where the homebrew bison is and prepend
|
||||
# it to PATH.
|
||||
if self.is_darwin() and self.host_type.get_package_manager() == "homebrew":
|
||||
add_homebrew_package_to_path(env, "bison")
|
||||
|
||||
if self.fbsource_dir:
|
||||
env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join(
|
||||
self.fbsource_dir, "xplat/third-party/yarn/offline-mirror"
|
||||
@@ -300,6 +306,14 @@ class BuildOptions(object):
|
||||
if os.path.isfile(cert_file):
|
||||
env["SSL_CERT_FILE"] = cert_file
|
||||
|
||||
# Try extra hard to find openssl, needed with homebrew on macOS
|
||||
if (
|
||||
self.is_darwin()
|
||||
and "OPENSSL_DIR" not in env
|
||||
and "OPENSSL_ROOT_DIR" in os.environ
|
||||
):
|
||||
env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"]
|
||||
|
||||
return env
|
||||
|
||||
|
||||
@@ -491,3 +505,9 @@ def setup_build_options(args, host_type=None):
|
||||
install_dir=args.install_prefix,
|
||||
**build_args
|
||||
)
|
||||
|
||||
|
||||
def add_homebrew_package_to_path(env, package):
|
||||
prefix = homebrew_package_prefix(package)
|
||||
if prefix and os.path.exists(os.path.join(prefix, "bin")):
|
||||
add_path_entry(env, "PATH", os.path.join(prefix, "bin"), append=False)
|
||||
|
||||
@@ -158,6 +158,7 @@ class SystemPackageFetcher(object):
|
||||
def __init__(self, build_options, packages):
|
||||
self.manager = build_options.host_type.get_package_manager()
|
||||
self.packages = packages.get(self.manager)
|
||||
self.host_type = build_options.host_type
|
||||
if self.packages:
|
||||
self.installed = None
|
||||
else:
|
||||
@@ -172,6 +173,8 @@ class SystemPackageFetcher(object):
|
||||
cmd = ["rpm", "-q"] + sorted(self.packages)
|
||||
elif self.manager == "deb":
|
||||
cmd = ["dpkg", "-s"] + sorted(self.packages)
|
||||
elif self.manager == "homebrew":
|
||||
cmd = ["brew", "ls", "--versions"] + sorted(self.packages)
|
||||
|
||||
if cmd:
|
||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
@@ -181,9 +184,23 @@ class SystemPackageFetcher(object):
|
||||
else:
|
||||
# Need all packages to be present to consider us installed
|
||||
self.installed = False
|
||||
|
||||
else:
|
||||
self.installed = False
|
||||
|
||||
# Hack to make openssl discovery with homebrew work. If openssl was
|
||||
# built with autoconf we could use autoconf.envcmd.OPENSSL_ROOT_DIR
|
||||
# from the manifest, but it isn't, so handle the special case here.
|
||||
if (
|
||||
self.installed
|
||||
and self.host_type.is_darwin()
|
||||
and self.manager == "homebrew"
|
||||
and "openssl@1.1" in self.packages
|
||||
):
|
||||
candidate = homebrew_package_prefix("openssl@1.1")
|
||||
if os.path.exists(candidate):
|
||||
os.environ["OPENSSL_ROOT_DIR"] = candidate
|
||||
|
||||
return bool(self.installed)
|
||||
|
||||
def update(self):
|
||||
@@ -789,3 +806,14 @@ class ArchiveFetcher(Fetcher):
|
||||
|
||||
def get_src_dir(self):
|
||||
return self.src_dir
|
||||
|
||||
|
||||
def homebrew_package_prefix(package):
|
||||
cmd = ["brew", "--prefix", package]
|
||||
try:
|
||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
|
||||
if proc.returncode == 0:
|
||||
return proc.stdout.decode("utf-8").rstrip()
|
||||
|
||||
@@ -81,6 +81,7 @@ SCHEMA = {
|
||||
"autoconf.envcmd.LDFLAGS": {"optional_section": True},
|
||||
"rpms": {"optional_section": True},
|
||||
"debs": {"optional_section": True},
|
||||
"homebrew": {"optional_section": True},
|
||||
"preinstalled.env": {"optional_section": True},
|
||||
"bootstrap.args": {"optional_section": True},
|
||||
"b2.args": {"optional_section": True},
|
||||
@@ -112,6 +113,7 @@ ALLOWED_EXPR_SECTIONS = [
|
||||
"debs",
|
||||
"shipit.pathmap",
|
||||
"shipit.strip",
|
||||
"homebrew",
|
||||
]
|
||||
|
||||
|
||||
@@ -358,6 +360,7 @@ class ManifestParser(object):
|
||||
return {
|
||||
"rpm": self.get_section_as_args("rpms", ctx),
|
||||
"deb": self.get_section_as_args("debs", ctx),
|
||||
"homebrew": self.get_section_as_args("homebrew", ctx),
|
||||
}
|
||||
|
||||
def _is_satisfied_by_preinstalled_environment(self, ctx):
|
||||
|
||||
@@ -193,8 +193,10 @@ class HostType(object):
|
||||
)
|
||||
|
||||
def get_package_manager(self):
|
||||
if not self.is_linux():
|
||||
if not self.is_linux() and not self.is_darwin():
|
||||
return None
|
||||
if self.is_darwin():
|
||||
return "homebrew"
|
||||
if self.distro in ("fedora", "centos", "centos_stream"):
|
||||
return "rpm"
|
||||
if self.distro.startswith(("debian", "ubuntu")):
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
[manifest]
|
||||
name = autoconf
|
||||
|
||||
[rpms]
|
||||
[debs]
|
||||
autoconf
|
||||
|
||||
[debs]
|
||||
[homebrew]
|
||||
autoconf
|
||||
|
||||
[rpms]
|
||||
autoconf
|
||||
|
||||
[download]
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
[manifest]
|
||||
name = automake
|
||||
|
||||
[rpms]
|
||||
[homebrew]
|
||||
automake
|
||||
|
||||
[debs]
|
||||
automake
|
||||
|
||||
[rpms]
|
||||
automake
|
||||
|
||||
[download]
|
||||
url = http://ftp.gnu.org/gnu/automake/automake-1.16.1.tar.gz
|
||||
sha256 = 608a97523f97db32f1f5d5615c98ca69326ced2054c9f82e65bade7fc4c9dea8
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
[manifest]
|
||||
name = bison
|
||||
|
||||
[rpms]
|
||||
[debs]
|
||||
bison
|
||||
|
||||
[debs]
|
||||
[homebrew]
|
||||
bison
|
||||
|
||||
[rpms]
|
||||
bison
|
||||
|
||||
[download.not(os=windows)]
|
||||
|
||||
@@ -15,6 +15,9 @@ BOOST_ROOT_1_78_0
|
||||
[debs]
|
||||
libboost-all-dev
|
||||
|
||||
[homebrew]
|
||||
boost
|
||||
|
||||
[rpms.all(distro=centos_stream,distro_vers=8)]
|
||||
boost169
|
||||
boost169-math
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
[manifest]
|
||||
name = cmake
|
||||
|
||||
[homebrew]
|
||||
cmake
|
||||
|
||||
[rpms]
|
||||
cmake
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
[manifest]
|
||||
name = cpptoml
|
||||
|
||||
[homebrew]
|
||||
cpptoml
|
||||
|
||||
[download]
|
||||
url = https://github.com/chadaustin/cpptoml/archive/refs/tags/v0.1.2.tar.gz
|
||||
sha256 = beda37e94f9746874436c8090c045fd80ae6f8a51f7c668c932a2b110a4fc277
|
||||
|
||||
@@ -5,6 +5,9 @@ name = double-conversion
|
||||
url = https://github.com/google/double-conversion/archive/v3.1.4.tar.gz
|
||||
sha256 = 95004b65e43fefc6100f337a25da27bb99b9ef8d4071a36a33b5e83eb1f82021
|
||||
|
||||
[homebrew]
|
||||
double-conversion
|
||||
|
||||
[rpms]
|
||||
double-conversion
|
||||
double-conversion-devel
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
[manifest]
|
||||
name = flex
|
||||
|
||||
[rpms]
|
||||
[debs]
|
||||
flex
|
||||
|
||||
[debs]
|
||||
[homebrew]
|
||||
flex
|
||||
|
||||
[rpms]
|
||||
flex
|
||||
|
||||
[download.not(os=windows)]
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
[manifest]
|
||||
name = libevent
|
||||
|
||||
[rpms]
|
||||
libevent-devel
|
||||
|
||||
[debs]
|
||||
libevent-dev
|
||||
|
||||
[homebrew]
|
||||
libevent
|
||||
|
||||
[rpms]
|
||||
libevent-devel
|
||||
|
||||
# Note that the CMakeLists.txt file is present only in
|
||||
# git repo and not in the release tarball, so take care
|
||||
# to use the github generated source tarball rather than
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
[manifest]
|
||||
name = libffi
|
||||
|
||||
[debs]
|
||||
libffi-dev
|
||||
|
||||
[homebrew]
|
||||
libffi
|
||||
|
||||
[rpms]
|
||||
libffi-devel
|
||||
libffi
|
||||
|
||||
[debs]
|
||||
libffi-dev
|
||||
|
||||
[download]
|
||||
url = https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz
|
||||
sha256 = 540fb721619a6aba3bdeef7d940d8e9e0e6d2c193595bc243241b77ff9e93620
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
[manifest]
|
||||
name = libgit2
|
||||
|
||||
[homebrew]
|
||||
libgit2
|
||||
|
||||
[rpms]
|
||||
libgit2-devel
|
||||
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
[manifest]
|
||||
name = libsodium
|
||||
|
||||
[debs]
|
||||
libsodium-dev
|
||||
|
||||
[homebrew]
|
||||
libsodium
|
||||
|
||||
[rpms]
|
||||
libsodium-devel
|
||||
libsodium-static
|
||||
|
||||
[debs]
|
||||
libsodium-dev
|
||||
|
||||
[download.not(os=windows)]
|
||||
url = https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz
|
||||
sha256 = 0cc3dae33e642cc187b5ceb467e0ad0e1b51dcba577de1190e9ffa17766ac2b1
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
[manifest]
|
||||
name = libtool
|
||||
|
||||
[homebrew]
|
||||
libtool
|
||||
|
||||
[rpms]
|
||||
libtool
|
||||
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
[manifest]
|
||||
name = libusb
|
||||
|
||||
[debs]
|
||||
libusb-1.0-0-dev
|
||||
|
||||
[homebrew]
|
||||
libusb
|
||||
|
||||
[rpms]
|
||||
libusb-devel
|
||||
libusb
|
||||
|
||||
[debs]
|
||||
libusb-1.0-0-dev
|
||||
|
||||
[download]
|
||||
url = https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.tar.bz2
|
||||
sha256 = 75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
[manifest]
|
||||
name = libzmq
|
||||
|
||||
[debs]
|
||||
libzmq3-dev
|
||||
|
||||
[homebrew]
|
||||
zeromq
|
||||
|
||||
[rpms]
|
||||
zeromq-devel
|
||||
zeromq
|
||||
|
||||
[debs]
|
||||
libzmq3-dev
|
||||
|
||||
[download]
|
||||
url = https://github.com/zeromq/libzmq/releases/download/v4.3.1/zeromq-4.3.1.tar.gz
|
||||
sha256 = bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
[manifest]
|
||||
name = lz4
|
||||
|
||||
[homebrew]
|
||||
lz4
|
||||
|
||||
[rpms]
|
||||
lz4-devel
|
||||
lz4-static
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
[manifest]
|
||||
name = lzo
|
||||
|
||||
[rpms]
|
||||
lzo-devel
|
||||
|
||||
[debs]
|
||||
liblzo2-dev
|
||||
|
||||
[homebrew]
|
||||
lzo
|
||||
|
||||
[rpms]
|
||||
lzo-devel
|
||||
|
||||
[download]
|
||||
url = http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz
|
||||
sha256 = c0f892943208266f9b6543b3ae308fab6284c5c90e627931446fb49b4221a072
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
[manifest]
|
||||
name = ninja
|
||||
|
||||
[rpms]
|
||||
[debs]
|
||||
ninja-build
|
||||
|
||||
[debs]
|
||||
[homebrew]
|
||||
ninja
|
||||
|
||||
[rpms]
|
||||
ninja-build
|
||||
|
||||
[download.os=windows]
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
[manifest]
|
||||
name = openssl
|
||||
|
||||
[debs]
|
||||
libssl-dev
|
||||
|
||||
[homebrew]
|
||||
openssl@1.1
|
||||
|
||||
[rpms]
|
||||
openssl
|
||||
openssl-devel
|
||||
openssl-libs
|
||||
|
||||
[debs]
|
||||
libssl-dev
|
||||
|
||||
[download]
|
||||
url = https://www.openssl.org/source/openssl-1.1.1l.tar.gz
|
||||
sha256 = 0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
[manifest]
|
||||
name = pcre
|
||||
|
||||
[homebrew]
|
||||
pcre
|
||||
|
||||
[rpms]
|
||||
pcre-devel
|
||||
pcre-static
|
||||
@@ -15,4 +18,3 @@ sha256 = 0b8e7465dc5e98c757cc3650a20a7843ee4c3edf50aaf60bb33fd879690d2c73
|
||||
[build]
|
||||
builder = cmake
|
||||
subdir = pcre-8.43
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
[manifest]
|
||||
name = python
|
||||
|
||||
[homebrew]
|
||||
python@3.8
|
||||
|
||||
[rpms]
|
||||
python3
|
||||
python3-devel
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
[manifest]
|
||||
name = re2
|
||||
|
||||
[homebrew]
|
||||
re2
|
||||
|
||||
[debs]
|
||||
libre2-dev
|
||||
|
||||
[rpms]
|
||||
re2
|
||||
re2-devel
|
||||
|
||||
[debs]
|
||||
libre2-dev
|
||||
|
||||
[download]
|
||||
url = https://github.com/google/re2/archive/2019-06-01.tar.gz
|
||||
sha256 = 02b7d73126bd18e9fbfe5d6375a8bb13fadaf8e99e48cbb062e4500fc18e8e2e
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
[manifest]
|
||||
name = sqlite3
|
||||
|
||||
[debs]
|
||||
libsqlite3-dev
|
||||
|
||||
[homebrew]
|
||||
sqlite
|
||||
|
||||
[rpms]
|
||||
sqlite-devel
|
||||
sqlite-libs
|
||||
|
||||
[debs]
|
||||
libsqlite3-dev
|
||||
|
||||
[download]
|
||||
url = https://sqlite.org/2019/sqlite-amalgamation-3280000.zip
|
||||
sha256 = d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
[manifest]
|
||||
name = tree
|
||||
|
||||
[rpms]
|
||||
[debs]
|
||||
tree
|
||||
|
||||
[debs]
|
||||
[homebrew]
|
||||
tree
|
||||
|
||||
[rpms]
|
||||
tree
|
||||
|
||||
[download.os=linux]
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
[manifest]
|
||||
name = zlib
|
||||
|
||||
[debs]
|
||||
zlib1g-dev
|
||||
|
||||
[homebrew]
|
||||
zlib
|
||||
|
||||
[rpms]
|
||||
zlib-devel
|
||||
zlib-static
|
||||
|
||||
[debs]
|
||||
zlib1g-dev
|
||||
|
||||
[download]
|
||||
url = http://www.zlib.net/zlib-1.2.11.tar.gz
|
||||
sha256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
[manifest]
|
||||
name = zstd
|
||||
|
||||
[homebrew]
|
||||
zstd
|
||||
|
||||
[rpms]
|
||||
libzstd-devel
|
||||
libzstd
|
||||
|
||||
Reference in New Issue
Block a user