lib/proxygen
1
0
Fork 0
mirror of https://github.com/facebook/proxygen.git synced 2025-08-08 18:02:05 +03:00
Code Activity
Files
a01757bb45ebf111576409d6cc8fb05a1a64ae6f
proxygen/build/fbcode_builder/getdeps/builder.py
Xavier Deguillard a01757bb45 getdeps: add filtering of tests 
Summary:
For large projects, with lots of tests, running all the tests can take a lot
of time, but for quick development iteration, only a subset of the tests may
be needed to run.

On non-Windows platforms, this can be easily achieved by manually executing
the individual tests binaries and use the builtin filtering mechanism of that
test binary to achieve the goal. On Windows, this can quickly become
impossible as DLLs might not be available, and the right PATH would need to
be manually specified by hand to execute the tests binaries[0].

To solve this, let's simply provide a unified way of running specific tests
by passing in a regexp. Both testpilot and CTest do support regex to execute
specific tests. My understanding is that cargo doesn't yet allows regex, but
will in the future.

[0]: And a missing DLLs would produce no output when executed from
     PowerShell, which makes this very confusing.

Reviewed By: wez

Differential Revision: D21484774

fbshipit-source-id: ee32e950e25bb2a498a2b364a447955a917b0590
2020-05-11 11:50:15 -07:00

1185 lines
43 KiB
Python
Raw Blame History

#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
from __future__ import absolute_import, division, print_function, unicode_literals
import json
import os
import shutil
import stat
import subprocess
import sys
from .dyndeps import create_dyn_dep_munger
from .envfuncs import Env, add_path_entry, path_search
from .fetcher import copy_if_different
from .runcmd import run_cmd
class BuilderBase(object):
def __init__(
self,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
env=None,
final_install_prefix=None,
):
self.env = Env()
if env:
self.env.update(env)
subdir = manifest.get("build", "subdir", ctx=ctx)
if subdir:
src_dir = os.path.join(src_dir, subdir)
self.ctx = ctx
self.src_dir = src_dir
self.build_dir = build_dir or src_dir
self.inst_dir = inst_dir
self.build_opts = build_opts
self.manifest = manifest
self.final_install_prefix = final_install_prefix
def _get_cmd_prefix(self):
if self.build_opts.is_windows():
vcvarsall = self.build_opts.get_vcvars_path()
if vcvarsall is not None:
# Since it sets rather a large number of variables we mildly abuse
# the cmd quoting rules to assemble a command that calls the script
# to prep the environment and then triggers the actual command that
# we wanted to run.
return [vcvarsall, "amd64", "&&"]
return []
def _run_cmd(self, cmd, cwd=None, env=None, use_cmd_prefix=True):
if env:
e = self.env.copy()
e.update(env)
env = e
else:
env = self.env
if use_cmd_prefix:
cmd_prefix = self._get_cmd_prefix()
if cmd_prefix:
cmd = cmd_prefix + cmd
log_file = os.path.join(self.build_dir, "getdeps_build.log")
run_cmd(cmd=cmd, env=env, cwd=cwd or self.build_dir, log_file=log_file)
def build(self, install_dirs, reconfigure):
print("Building %s..." % self.manifest.name)
if self.build_dir is not None:
if not os.path.isdir(self.build_dir):
os.makedirs(self.build_dir)
reconfigure = True
self._build(install_dirs=install_dirs, reconfigure=reconfigure)
# On Windows, emit a wrapper script that can be used to run build artifacts
# directly from the build directory, without installing them. On Windows $PATH
# needs to be updated to include all of the directories containing the runtime
# library dependencies in order to run the binaries.
if self.build_opts.is_windows():
script_path = self.get_dev_run_script_path()
dep_munger = create_dyn_dep_munger(self.build_opts, install_dirs)
dep_dirs = self.get_dev_run_extra_path_dirs(install_dirs, dep_munger)
dep_munger.emit_dev_run_script(script_path, dep_dirs)
def run_tests(self, install_dirs, schedule_type, owner, test_filter):
""" Execute any tests that we know how to run. If they fail,
raise an exception. """
pass
def _build(self, install_dirs, reconfigure):
""" Perform the build.
install_dirs contains the list of installation directories for
the dependencies of this project.
reconfigure will be set to true if the fetcher determined
that the sources have changed in such a way that the build
system needs to regenerate its rules. """
pass
def _compute_env(self, install_dirs):
# CMAKE_PREFIX_PATH is only respected when passed through the
# environment, so we construct an appropriate path to pass down
return self.build_opts.compute_env_for_install_dirs(install_dirs, env=self.env)
def get_dev_run_script_path(self):
assert self.build_opts.is_windows()
return os.path.join(self.build_dir, "run.ps1")
def get_dev_run_extra_path_dirs(self, install_dirs, dep_munger=None):
assert self.build_opts.is_windows()
if dep_munger is None:
dep_munger = create_dyn_dep_munger(self.build_opts, install_dirs)
return dep_munger.compute_dependency_paths(self.build_dir)
class MakeBuilder(BuilderBase):
def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args):
super(MakeBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
)
self.args = args or []
def _build(self, install_dirs, reconfigure):
env = self._compute_env(install_dirs)
# Need to ensure that PREFIX is set prior to install because
# libbpf uses it when generating its pkg-config file
cmd = (
["make", "-j%s" % self.build_opts.num_jobs]
+ self.args
+ ["PREFIX=" + self.inst_dir]
)
self._run_cmd(cmd, env=env)
install_cmd = ["make", "install"] + self.args + ["PREFIX=" + self.inst_dir]
self._run_cmd(install_cmd, env=env)
class AutoconfBuilder(BuilderBase):
def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args):
super(AutoconfBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
)
self.args = args or []
def _build(self, install_dirs, reconfigure):
configure_path = os.path.join(self.src_dir, "configure")
autogen_path = os.path.join(self.src_dir, "autogen.sh")
env = self._compute_env(install_dirs)
if not os.path.exists(configure_path):
print("%s doesn't exist, so reconfiguring" % configure_path)
# This libtoolize call is a bit gross; the issue is that
# `autoreconf` as invoked by libsodium's `autogen.sh` doesn't
# seem to realize that it should invoke libtoolize and then
# error out when the configure script references a libtool
# related symbol.
self._run_cmd(["libtoolize"], cwd=self.src_dir, env=env)
# We generally prefer to call the `autogen.sh` script provided
# by the project on the basis that it may know more than plain
# autoreconf does.
if os.path.exists(autogen_path):
self._run_cmd(["bash", autogen_path], cwd=self.src_dir, env=env)
else:
self._run_cmd(["autoreconf", "-ivf"], cwd=self.src_dir, env=env)
configure_cmd = [configure_path, "--prefix=" + self.inst_dir] + self.args
self._run_cmd(configure_cmd, env=env)
self._run_cmd(["make", "-j%s" % self.build_opts.num_jobs], env=env)
self._run_cmd(["make", "install"], env=env)
class Iproute2Builder(BuilderBase):
# ./configure --prefix does not work for iproute2.
# Thus, explicitly copy sources from src_dir to build_dir, bulid,
# and then install to inst_dir using DESTDIR
# lastly, also copy include from build_dir to inst_dir
def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir):
super(Iproute2Builder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
)
def _patch(self):
# FBOSS build currently depends on an old version of iproute2 (commit
# 7ca63aef7d1b0c808da0040c6b366ef7a61f38c1). This is missing a commit
# (ae717baf15fb4d30749ada3948d9445892bac239) needed to build iproute2
# successfully. Apply it viz.: include stdint.h
# Reference: https://fburl.com/ilx9g5xm
with open(self.build_dir + "/tc/tc_core.c", "r") as f:
data = f.read()
with open(self.build_dir + "/tc/tc_core.c", "w") as f:
f.write("#include <stdint.h>\n")
f.write(data)
def _build(self, install_dirs, reconfigure):
configure_path = os.path.join(self.src_dir, "configure")
env = self.env.copy()
self._run_cmd([configure_path], env=env)
shutil.rmtree(self.build_dir)
shutil.copytree(self.src_dir, self.build_dir)
self._patch()
self._run_cmd(["make", "-j%s" % self.build_opts.num_jobs], env=env)
install_cmd = ["make", "install", "DESTDIR=" + self.inst_dir]
for d in ["include", "lib"]:
if not os.path.isdir(os.path.join(self.inst_dir, d)):
shutil.copytree(
os.path.join(self.build_dir, d), os.path.join(self.inst_dir, d)
)
self._run_cmd(install_cmd, env=env)
class CMakeBuilder(BuilderBase):
MANUAL_BUILD_SCRIPT = """\
#!{sys.executable}
from __future__ import absolute_import, division, print_function, unicode_literals
import argparse
import subprocess
import sys
CMAKE = {cmake!r}
CTEST = {ctest!r}
SRC_DIR = {src_dir!r}
BUILD_DIR = {build_dir!r}
INSTALL_DIR = {install_dir!r}
CMD_PREFIX = {cmd_prefix!r}
CMAKE_ENV = {env_str}
CMAKE_DEFINE_ARGS = {define_args_str}
def get_jobs_argument(num_jobs_arg: int) -> str:
if num_jobs_arg > 0:
return "-j" + str(num_jobs_arg)
import multiprocessing
num_jobs = multiprocessing.cpu_count()
if sys.platform == "win32":
num_jobs //= 2
return "-j" + str(num_jobs)
def main():
ap = argparse.ArgumentParser()
ap.add_argument(
"cmake_args",
nargs=argparse.REMAINDER,
help='Any extra arguments after an "--" argument will be passed '
"directly to CMake."
)
ap.add_argument(
"--mode",
choices=["configure", "build", "install", "test"],
default="configure",
help="The mode to run: configure, build, or install. "
"Defaults to configure",
)
ap.add_argument(
"--build",
action="store_const",
const="build",
dest="mode",
help="An alias for --mode=build",
)
ap.add_argument(
"-j",
"--num-jobs",
action="store",
type=int,
default=0,
help="Run the build or tests with the specified number of parallel jobs",
)
ap.add_argument(
"--install",
action="store_const",
const="install",
dest="mode",
help="An alias for --mode=install",
)
ap.add_argument(
"--test",
action="store_const",
const="test",
dest="mode",
help="An alias for --mode=test",
)
args = ap.parse_args()
# Strip off a leading "--" from the additional CMake arguments
if args.cmake_args and args.cmake_args[0] == "--":
args.cmake_args = args.cmake_args[1:]
env = CMAKE_ENV
if args.mode == "configure":
full_cmd = CMD_PREFIX + [CMAKE, SRC_DIR] + CMAKE_DEFINE_ARGS + args.cmake_args
elif args.mode in ("build", "install"):
target = "all" if args.mode == "build" else "install"
full_cmd = CMD_PREFIX + [
CMAKE,
"--build",
BUILD_DIR,
"--target",
target,
"--config",
"Release",
get_jobs_argument(args.num_jobs),
] + args.cmake_args
elif args.mode == "test":
full_cmd = CMD_PREFIX + [
{dev_run_script}CTEST,
"--output-on-failure",
get_jobs_argument(args.num_jobs),
] + args.cmake_args
else:
ap.error("unknown invocation mode: %s" % (args.mode,))
cmd_str = " ".join(full_cmd)
print("Running: %r" % (cmd_str,))
proc = subprocess.run(full_cmd, env=env, cwd=BUILD_DIR)
sys.exit(proc.returncode)
if __name__ == "__main__":
main()
"""
def __init__(
self,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
defines,
final_install_prefix=None,
):
super(CMakeBuilder, self).__init__(
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
final_install_prefix=final_install_prefix,
)
self.defines = defines or {}
def _invalidate_cache(self):
for name in [
"CMakeCache.txt",
"CMakeFiles/CMakeError.log",
"CMakeFiles/CMakeOutput.log",
]:
name = os.path.join(self.build_dir, name)
if os.path.isdir(name):
shutil.rmtree(name)
elif os.path.exists(name):
os.unlink(name)
def _needs_reconfigure(self):
for name in ["CMakeCache.txt", "build.ninja"]:
name = os.path.join(self.build_dir, name)
if not os.path.exists(name):
return True
return False
def _write_build_script(self, **kwargs):
env_lines = [" {!r}: {!r},".format(k, v) for k, v in kwargs["env"].items()]
kwargs["env_str"] = "\n".join(["{"] + env_lines + ["}"])
if self.build_opts.is_windows():
kwargs["dev_run_script"] = '"powershell.exe", {!r}, '.format(
self.get_dev_run_script_path()
)
else:
kwargs["dev_run_script"] = ""
define_arg_lines = ["["]
for arg in kwargs["define_args"]:
# Replace the CMAKE_INSTALL_PREFIX argument to use the INSTALL_DIR
# variable that we define in the MANUAL_BUILD_SCRIPT code.
if arg.startswith("-DCMAKE_INSTALL_PREFIX="):
value = " {!r}.format(INSTALL_DIR),".format(
"-DCMAKE_INSTALL_PREFIX={}"
)
else:
value = " {!r},".format(arg)
define_arg_lines.append(value)
define_arg_lines.append("]")
kwargs["define_args_str"] = "\n".join(define_arg_lines)
# In order to make it easier for developers to manually run builds for
# CMake-based projects, write out some build scripts that can be used to invoke
# CMake manually.
build_script_path = os.path.join(self.build_dir, "run_cmake.py")
script_contents = self.MANUAL_BUILD_SCRIPT.format(**kwargs)
with open(build_script_path, "w") as f:
f.write(script_contents)
os.chmod(build_script_path, 0o755)
def _compute_cmake_define_args(self, env):
defines = {
"CMAKE_INSTALL_PREFIX": self.final_install_prefix or self.inst_dir,
"BUILD_SHARED_LIBS": "OFF",
# Some of the deps (rsocket) default to UBSAN enabled if left
# unspecified. Some of the deps fail to compile in release mode
# due to warning->error promotion. RelWithDebInfo is the happy
# medium.
"CMAKE_BUILD_TYPE": "RelWithDebInfo",
}
if "SANDCASTLE" not in os.environ:
# We sometimes see intermittent ccache related breakages on some
# of the FB internal CI hosts, so we prefer to disable ccache
# when running in that environment.
ccache = path_search(env, "ccache")
if ccache:
defines["CMAKE_CXX_COMPILER_LAUNCHER"] = ccache
else:
# rocksdb does its own probing for ccache.
# Ensure that it is disabled on sandcastle
env["CCACHE_DISABLE"] = "1"
# Some sandcastle hosts have broken ccache related dirs, and
# even though we've asked for it to be disabled ccache is
# still invoked by rocksdb's cmake.
# Redirect its config directory to somewhere that is guaranteed
# fresh to us, and that won't have any ccache data inside.
env["CCACHE_DIR"] = f"{self.build_opts.scratch_dir}/ccache"
if "GITHUB_ACTIONS" in os.environ and self.build_opts.is_windows():
# GitHub actions: the host has both gcc and msvc installed, and
# the default behavior of cmake is to prefer gcc.
# Instruct cmake that we want it to use cl.exe; this is important
# because Boost prefers cl.exe and the mismatch results in cmake
# with gcc not being able to find boost built with cl.exe.
defines["CMAKE_C_COMPILER"] = "cl.exe"
defines["CMAKE_CXX_COMPILER"] = "cl.exe"
if self.build_opts.is_darwin():
# Try to persuade cmake to set the rpath to match the lib
# dirs of the dependencies. This isn't automatic, and to
# make things more interesting, cmake uses `;` as the path
# separator, so translate the runtime path to something
# that cmake will parse
defines["CMAKE_INSTALL_RPATH"] = ";".join(
env.get("DYLD_LIBRARY_PATH", "").split(":")
)
# Tell cmake that we want to set the rpath in the tree
# at build time. Without this the rpath is only set
# at the moment that the binaries are installed. That
# default is problematic for example when using the
# gtest integration in cmake which runs the built test
# executables during the build to discover the set of
# tests.
defines["CMAKE_BUILD_WITH_INSTALL_RPATH"] = "ON"
defines.update(self.defines)
define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()]
# if self.build_opts.is_windows():
# define_args += ["-G", "Visual Studio 15 2017 Win64"]
define_args += ["-G", "Ninja"]
return define_args
def _build(self, install_dirs, reconfigure):
reconfigure = reconfigure or self._needs_reconfigure()
env = self._compute_env(install_dirs)
if not self.build_opts.is_windows() and self.final_install_prefix:
env["DESTDIR"] = self.inst_dir
# Resolve the cmake that we installed
cmake = path_search(env, "cmake")
if cmake is None:
raise Exception("Failed to find CMake")
if reconfigure:
define_args = self._compute_cmake_define_args(env)
self._write_build_script(
cmd_prefix=self._get_cmd_prefix(),
cmake=cmake,
ctest=path_search(env, "ctest"),
env=env,
define_args=define_args,
src_dir=self.src_dir,
build_dir=self.build_dir,
install_dir=self.inst_dir,
sys=sys,
)
self._invalidate_cache()
self._run_cmd([cmake, self.src_dir] + define_args, env=env)
self._run_cmd(
[
cmake,
"--build",
self.build_dir,
"--target",
"install",
"--config",
"Release",
"-j",
str(self.build_opts.num_jobs),
],
env=env,
)
def run_tests(self, install_dirs, schedule_type, owner, test_filter):
env = self._compute_env(install_dirs)
ctest = path_search(env, "ctest")
cmake = path_search(env, "cmake")
# On Windows, we also need to update $PATH to include the directories that
# contain runtime library dependencies. This is not needed on other platforms
# since CMake will emit RPATH properly in the binary so they can find these
# dependencies.
if self.build_opts.is_windows():
path_entries = self.get_dev_run_extra_path_dirs(install_dirs)
path = env.get("PATH")
if path:
path_entries.insert(0, path)
env["PATH"] = ";".join(path_entries)
# Don't use the cmd_prefix when running tests. This is vcvarsall.bat on
# Windows. vcvarsall.bat is only needed for the build, not tests. It
# unfortunately fails if invoked with a long PATH environment variable when
# running the tests.
use_cmd_prefix = False
def get_property(test, propname, defval=None):
""" extracts a named property from a cmake test info json blob.
The properties look like:
[{"name": "WORKING_DIRECTORY"},
{"value": "something"}]
We assume that it is invalid for the same named property to be
listed more than once.
"""
props = test.get("properties", [])
for p in props:
if p.get("name", None) == propname:
return p.get("value", defval)
return defval
def list_tests():
output = subprocess.check_output(
[ctest, "--show-only=json-v1"], env=env, cwd=self.build_dir
)
try:
data = json.loads(output.decode("utf-8"))
except ValueError as exc:
raise Exception(
"Failed to decode cmake test info using %s: %s. Output was: %r"
% (ctest, str(exc), output)
)
tests = []
machine_suffix = self.build_opts.host_type.as_tuple_string()
for test in data["tests"]:
working_dir = get_property(test, "WORKING_DIRECTORY")
labels = []
if get_property(test, "DISABLED"):
labels.append("disabled")
command = test["command"]
if working_dir:
command = [cmake, "-E", "chdir", working_dir] + command
tests.append(
{
"type": "custom",
"target": "%s-%s-getdeps-%s"
% (self.manifest.name, test["name"], machine_suffix),
"command": command,
"labels": labels,
}
)
return tests
testpilot = path_search(env, "testpilot")
if testpilot:
buck_test_info = list_tests()
buck_test_info_name = os.path.join(self.build_dir, ".buck-test-info.json")
with open(buck_test_info_name, "w") as f:
json.dump(buck_test_info, f)
env.set("http_proxy", "")
env.set("https_proxy", "")
machine_suffix = self.build_opts.host_type.as_tuple_string()
runs = []
testpilot_args = [
testpilot,
# Need to force the repo type otherwise testpilot on windows
# can be confused (presumably sparse profile related)
"--force-repo",
"fbcode",
"--force-repo-root",
self.build_opts.fbsource_dir,
"--buck-test-info",
buck_test_info_name,
"--retry=3",
"-j=%s" % str(self.build_opts.num_jobs),
"--test-config",
"platform=%s" % machine_suffix,
"buildsystem=getdeps",
"--print-long-results",
]
if owner:
testpilot_args += ["--contacts", owner]
if schedule_type == "continuous":
runs.append(
[
"--tag-new-tests",
"--collection",
"oss-continuous",
"--purpose",
"continuous",
]
)
elif schedule_type == "testwarden":
# One run to assess new tests
runs.append(
[
"--tag-new-tests",
"--collection",
"oss-new-test-stress",
"--stress-runs",
"10",
"--purpose",
"stress-run-new-test",
]
)
# And another for existing tests
runs.append(
[
"--tag-new-tests",
"--collection",
"oss-existing-test-stress",
"--stress-runs",
"10",
"--purpose",
"stress-run",
]
)
else:
runs.append(["--collection", "oss-diff", "--purpose", "diff"])
if test_filter:
testpilot_args += [test_filter]
for run in runs:
self._run_cmd(
testpilot_args + run,
cwd=self.build_opts.fbcode_builder_dir,
env=env,
use_cmd_prefix=use_cmd_prefix,
)
else:
args = [ctest, "--output-on-failure", "-j", str(self.build_opts.num_jobs)]
if test_filter:
args += ["-R", test_filter]
self._run_cmd(args, env=env, use_cmd_prefix=use_cmd_prefix)
class NinjaBootstrap(BuilderBase):
def __init__(self, build_opts, ctx, manifest, build_dir, src_dir, inst_dir):
super(NinjaBootstrap, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
)
def _build(self, install_dirs, reconfigure):
self._run_cmd([sys.executable, "configure.py", "--bootstrap"], cwd=self.src_dir)
src_ninja = os.path.join(self.src_dir, "ninja")
dest_ninja = os.path.join(self.inst_dir, "bin/ninja")
bin_dir = os.path.dirname(dest_ninja)
if not os.path.exists(bin_dir):
os.makedirs(bin_dir)
shutil.copyfile(src_ninja, dest_ninja)
shutil.copymode(src_ninja, dest_ninja)
class OpenSSLBuilder(BuilderBase):
def __init__(self, build_opts, ctx, manifest, build_dir, src_dir, inst_dir):
super(OpenSSLBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
)
def _build(self, install_dirs, reconfigure):
configure = os.path.join(self.src_dir, "Configure")
# prefer to resolve the perl that we installed from
# our manifest on windows, but fall back to the system
# path on eg: darwin
env = self.env.copy()
for d in install_dirs:
bindir = os.path.join(d, "bin")
add_path_entry(env, "PATH", bindir, append=False)
perl = path_search(env, "perl", "perl")
if self.build_opts.is_windows():
make = "nmake.exe"
args = ["VC-WIN64A-masm", "-utf-8"]
elif self.build_opts.is_darwin():
make = "make"
args = ["darwin64-x86_64-cc"]
else:
raise Exception("don't know how to build openssl for %r" % self.ctx)
self._run_cmd(
[
perl,
configure,
"--prefix=%s" % self.inst_dir,
"--openssldir=%s" % self.inst_dir,
]
+ args
+ [
"enable-static-engine",
"enable-capieng",
"no-makedepend",
"no-unit-test",
"no-tests",
]
)
self._run_cmd([make, "install_sw", "install_ssldirs"])
class Boost(BuilderBase):
def __init__(
self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, b2_args
):
children = os.listdir(src_dir)
assert len(children) == 1, "expected a single directory entry: %r" % (children,)
boost_src = children[0]
assert boost_src.startswith("boost")
src_dir = os.path.join(src_dir, children[0])
super(Boost, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
)
self.b2_args = b2_args
def _build(self, install_dirs, reconfigure):
linkage = ["static"]
if self.build_opts.is_windows():
linkage.append("shared")
for link in linkage:
args = []
if self.build_opts.is_windows():
bootstrap = os.path.join(self.src_dir, "bootstrap.bat")
self._run_cmd([bootstrap], cwd=self.src_dir)
args += ["address-model=64"]
else:
bootstrap = os.path.join(self.src_dir, "bootstrap.sh")
self._run_cmd(
[bootstrap, "--prefix=%s" % self.inst_dir], cwd=self.src_dir
)
b2 = os.path.join(self.src_dir, "b2")
self._run_cmd(
[
b2,
"-j%s" % self.build_opts.num_jobs,
"--prefix=%s" % self.inst_dir,
"--builddir=%s" % self.build_dir,
]
+ args
+ self.b2_args
+ [
"link=%s" % link,
"runtime-link=shared",
"variant=release",
"threading=multi",
"debug-symbols=on",
"visibility=global",
"-d2",
"install",
],
cwd=self.src_dir,
)
class NopBuilder(BuilderBase):
def __init__(self, build_opts, ctx, manifest, src_dir, inst_dir):
super(NopBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, None, inst_dir
)
def build(self, install_dirs, reconfigure):
print("Installing %s -> %s" % (self.src_dir, self.inst_dir))
parent = os.path.dirname(self.inst_dir)
if not os.path.exists(parent):
os.makedirs(parent)
install_files = self.manifest.get_section_as_ordered_pairs(
"install.files", self.ctx
)
if install_files:
for src_name, dest_name in self.manifest.get_section_as_ordered_pairs(
"install.files", self.ctx
):
full_dest = os.path.join(self.inst_dir, dest_name)
full_src = os.path.join(self.src_dir, src_name)
dest_parent = os.path.dirname(full_dest)
if not os.path.exists(dest_parent):
os.makedirs(dest_parent)
if os.path.isdir(full_src):
if not os.path.exists(full_dest):
shutil.copytree(full_src, full_dest)
else:
shutil.copyfile(full_src, full_dest)
shutil.copymode(full_src, full_dest)
# This is a bit gross, but the mac ninja.zip doesn't
# give ninja execute permissions, so force them on
# for things that look like they live in a bin dir
if os.path.dirname(dest_name) == "bin":
st = os.lstat(full_dest)
os.chmod(full_dest, st.st_mode | stat.S_IXUSR)
else:
if not os.path.exists(self.inst_dir):
shutil.copytree(self.src_dir, self.inst_dir)
class OpenNSABuilder(NopBuilder):
# OpenNSA libraries are stored with git LFS. As a result, fetcher fetches
# LFS pointers and not the contents. Use git-lfs to pull the real contents
# before copying to install dir using NoopBuilder.
# In future, if more builders require git-lfs, we would consider installing
# git-lfs as part of the sandcastle infra as against repeating similar
# logic for each builder that requires git-lfs.
def __init__(self, build_opts, ctx, manifest, src_dir, inst_dir):
super(OpenNSABuilder, self).__init__(
build_opts, ctx, manifest, src_dir, inst_dir
)
def build(self, install_dirs, reconfigure):
env = self._compute_env(install_dirs)
self._run_cmd(["git", "lfs", "install", "--local"], cwd=self.src_dir, env=env)
self._run_cmd(["git", "lfs", "pull"], cwd=self.src_dir, env=env)
super(OpenNSABuilder, self).build(install_dirs, reconfigure)
class SqliteBuilder(BuilderBase):
def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir):
super(SqliteBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
)
def _build(self, install_dirs, reconfigure):
for f in ["sqlite3.c", "sqlite3.h", "sqlite3ext.h"]:
src = os.path.join(self.src_dir, f)
dest = os.path.join(self.build_dir, f)
copy_if_different(src, dest)
cmake_lists = """
cmake_minimum_required(VERSION 3.1.3 FATAL_ERROR)
project(sqlite3 C)
add_library(sqlite3 STATIC sqlite3.c)
# These options are taken from the defaults in Makefile.msc in
# the sqlite distribution
target_compile_definitions(sqlite3 PRIVATE
-DSQLITE_ENABLE_COLUMN_METADATA=1
-DSQLITE_ENABLE_FTS3=1
-DSQLITE_ENABLE_RTREE=1
-DSQLITE_ENABLE_GEOPOLY=1
-DSQLITE_ENABLE_JSON1=1
-DSQLITE_ENABLE_STMTVTAB=1
-DSQLITE_ENABLE_DBPAGE_VTAB=1
-DSQLITE_ENABLE_DBSTAT_VTAB=1
-DSQLITE_INTROSPECTION_PRAGMAS=1
-DSQLITE_ENABLE_DESERIALIZE=1
)
install(TARGETS sqlite3)
install(FILES sqlite3.h sqlite3ext.h DESTINATION include)
"""
with open(os.path.join(self.build_dir, "CMakeLists.txt"), "w") as f:
f.write(cmake_lists)
defines = {
"CMAKE_INSTALL_PREFIX": self.inst_dir,
"BUILD_SHARED_LIBS": "OFF",
"CMAKE_BUILD_TYPE": "RelWithDebInfo",
}
define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()]
define_args += ["-G", "Ninja"]
env = self._compute_env(install_dirs)
# Resolve the cmake that we installed
cmake = path_search(env, "cmake")
self._run_cmd([cmake, self.build_dir] + define_args, env=env)
self._run_cmd(
[
cmake,
"--build",
self.build_dir,
"--target",
"install",
"--config",
"Release",
"-j",
str(self.build_opts.num_jobs),
],
env=env,
)
class CargoBuilder(BuilderBase):
def __init__(
self,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
build_doc,
workspace_dir,
loader,
):
super(CargoBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
)
self.build_doc = build_doc
self.ws_dir = workspace_dir
self.loader = loader
def run_cargo(self, install_dirs, operation, args=None):
args = args or []
env = self._compute_env(install_dirs)
# Enable using nightly features with stable compiler
env["RUSTC_BOOTSTRAP"] = "1"
cmd = [
"cargo",
operation,
"--workspace",
"-j%s" % self.build_opts.num_jobs,
] + args
self._run_cmd(cmd, cwd=self.workspace_dir(), env=env)
def build_source_dir(self):
return os.path.join(self.build_dir, "source")
def workspace_dir(self):
return os.path.join(self.build_source_dir(), self.ws_dir)
def recreate_dir(self, src, dst):
if os.path.isdir(dst):
shutil.rmtree(dst)
shutil.copytree(src, dst)
def _build(self, install_dirs, reconfigure):
build_source_dir = self.build_source_dir()
self.recreate_dir(self.src_dir, build_source_dir)
dot_cargo_dir = os.path.join(build_source_dir, ".cargo")
if not os.path.isdir(dot_cargo_dir):
os.mkdir(dot_cargo_dir)
with open(os.path.join(dot_cargo_dir, "config"), "w+") as f:
f.write(
"""\
[build]
target-dir = '''{}'''
[net]
git-fetch-with-cli = true
""".format(
self.build_dir.replace("\\", "\\\\")
)
)
self._patchup_workspace()
try:
from getdeps.facebook.rust import vendored_crates
vendored_crates(self.build_opts, build_source_dir)
except ImportError:
# This FB internal module isn't shippped to github,
# so just rely on cargo downloading crates on it's own
pass
self.run_cargo(install_dirs, "build")
self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source"))
def run_tests(self, install_dirs, schedule_type, owner, test_filter):
if test_filter:
args = ["--", test_filter]
else:
args = None
self.run_cargo(install_dirs, "test", args)
if self.build_doc:
self.run_cargo(install_dirs, "doc", ["--no-deps"])
def _patchup_workspace(self):
"""
This method makes some assumptions about the state of the project and
its cargo dependendies:
1. Crates from cargo dependencies can be extracted from Cargo.toml files
using _extract_crates function. It is using a heuristic so check its
code to understand how it is done.
2. The extracted cargo dependencies crates can be found in the
dependency's install dir using _resolve_crate_to_path function
which again is using a heuristic.
Notice that many things might go wrong here. E.g. if someone depends
on another getdeps crate by writing in their Cargo.toml file:
my-rename-of-crate = { package = "crate", git = "..." }
they can count themselves lucky because the code will raise an
Exception. There migh be more cases where the code will silently pass
producing bad results.
"""
workspace_dir = self.workspace_dir()
config = self._resolve_config()
if config:
with open(os.path.join(workspace_dir, "Cargo.toml"), "a") as f:
# A fake manifest has to be crated to change the virtual
# manifest into a non-virtual. The virtual manifests are limited
# in many ways and the inability to define patches on them is
# one. Check https://github.com/rust-lang/cargo/issues/4934 to
# see if it is resolved.
f.write(
"""
[package]
name = "fake_manifest_of_{}"
version = "0.0.0"
[lib]
path = "/dev/null"
""".format(
self.manifest.name
)
)
f.write(config)
def _resolve_config(self):
"""
Returns a configuration to be put inside root Cargo.toml file which
patches the dependencies git code with local getdeps versions.
See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section
"""
dep_to_git = self._resolve_dep_to_git()
dep_to_crates = CargoBuilder._resolve_dep_to_crates(
self.build_source_dir(), dep_to_git
)
config = []
for name in sorted(dep_to_git.keys()):
git_conf = dep_to_git[name]
crates = sorted(dep_to_crates.get(name, []))
if not crates:
continue # nothing to patch, move along
crates_patches = [
'{} = {{ path = "{}" }}'.format(
crate,
CargoBuilder._resolve_crate_to_path(crate, git_conf).replace(
"\\", "\\\\"
),
)
for crate in crates
]
config.append(
'[patch."{0}"]\n'.format(git_conf["repo_url"])
+ "\n".join(crates_patches)
)
return "\n".join(config)
def _resolve_dep_to_git(self):
"""
For each direct dependency of the currently build manifest check if it
is also cargo-builded and if yes then extract it's git configs and
install dir
"""
dependencies = self.manifest.get_section_as_dict("dependencies", ctx=self.ctx)
if not dependencies:
return []
dep_to_git = {}
for dep in dependencies.keys():
dep_manifest = self.loader.load_manifest(dep)
dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx)
if dep_builder not in ["cargo", "nop"] or dep == "rust":
# This is a direct dependency, but it is not build with cargo
# and it is not simply copying files with nop, so ignore it.
# The "rust" dependency is an exception since it contains the
# toolchain.
continue
git_conf = dep_manifest.get_section_as_dict("git", ctx=self.ctx)
if "repo_url" not in git_conf:
raise Exception(
"A cargo dependency requires git.repo_url to be defined."
)
source_dir = self.loader.get_project_install_dir(dep_manifest)
if dep_builder == "cargo":
source_dir = os.path.join(source_dir, "source")
git_conf["source_dir"] = source_dir
dep_to_git[dep] = git_conf
return dep_to_git
@staticmethod
def _resolve_dep_to_crates(build_source_dir, dep_to_git):
"""
This function traverse the build_source_dir in search of Cargo.toml
files, extracts the crate names from them using _extract_crates
function and returns a merged result containing crate names per
dependency name from all Cargo.toml files in the project.
"""
if not dep_to_git:
return {} # no deps, so don't waste time traversing files
dep_to_crates = {}
for root, _, files in os.walk(build_source_dir):
for f in files:
if f == "Cargo.toml":
more_dep_to_crates = CargoBuilder._extract_crates(
os.path.join(root, f), dep_to_git
)
for name, crates in more_dep_to_crates.items():
dep_to_crates.setdefault(name, set()).update(crates)
return dep_to_crates
@staticmethod
def _extract_crates(cargo_toml_file, dep_to_git):
"""
This functions reads content of provided cargo toml file and extracts
crate names per each dependency. The extraction is done by a heuristic
so it might be incorrect.
"""
deps_to_crates = {}
with open(cargo_toml_file, "r") as f:
for line in f.readlines():
if line.startswith("#") or "git = " not in line:
continue # filter out commented lines and ones without git deps
for name, conf in dep_to_git.items():
if 'git = "{}"'.format(conf["repo_url"]) in line:
crate_name, _, _ = line.partition("=")
deps_to_crates.setdefault(name, set()).add(crate_name.strip())
return deps_to_crates
@staticmethod
def _resolve_crate_to_path(crate, git_conf):
"""
Tries to find <crate> in git_conf["inst_dir"] by searching a [package]
keyword followed by name = "<crate>".
"""
source_dir = git_conf["source_dir"]
search_pattern = '[package]\nname = "{}"'.format(crate)
for root, _, files in os.walk(source_dir):
for fname in files:
if fname == "Cargo.toml":
with open(os.path.join(root, fname), "r") as f:
if search_pattern in f.read():
return root
raise Exception("Failed to found crate {} in path {}".format(crate, source_dir))
View Git Blame Copy Permalink