lib/proxygen
1
0
Fork 0
mirror of https://github.com/facebook/proxygen.git synced 2025-08-05 19:55:47 +03:00
Code Activity

improve crate vendoring (#110)

Browse Source 
Summary:
Pull Request resolved: https://github.com/facebookexperimental/eden/pull/110

Pull Request resolved: https://github.com/facebookexperimental/rust-shed/pull/27

Make it so that changes to rust-shed or other common rust source are used locally vendored, so they don't need to be pushed to github before they are visible in a build.

There was already some support for cargo vendoring in getdeps, but it was limited to dependencies between manifests built with cargo builder.  This wasn't enough to build something like eden (cmake is main entry point, with later calls cargo) or eden_scm (make is main entry point, with later calls to cargo), so this diff adds a cargo prepare step for getdeps other primary build systems.

The cargo vendoring is done by using a cargo config file to point to the source files used by getdeps.  It has two modes:

1. per crate, existing mode which is already automatic for cargo to cargo manifest dependencies.  To use it for a non cargo build manifest, add crate.pathmap
2. per git url, existing mode which was only use for crates.io third-party crates, now can be enabled by setting cargo.cargo_config_file

Reviewed By: yancouto

Differential Revision: D33895469

fbshipit-source-id: 7b13c0b679532492a336ce217de875c25fe1be90
This commit is contained in:
Alex Hornby
2022-02-16 01:07:57 -08:00
committed by Facebook GitHub Bot
parent 234713f961
commit fa26d2a99b
10 changed files with 292 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
build/fbcode_builder/CMake/RustStaticLibrary.cmake
View File

@@ -8,7 +8,14 @@ set(
)
set_property(CACHE USE_CARGO_VENDOR PROPERTY STRINGS AUTO ON OFF)
set(
GENERATE_CARGO_VENDOR_CONFIG AUTO CACHE STRING
"Whether to generate Rust cargo vendor config or use existing"
)
set_property(CACHE GENERATE_CARGO_VENDOR_CONFIG PROPERTY STRINGS AUTO ON OFF)
set(RUST_VENDORED_CRATES_DIR "$ENV{RUST_VENDORED_CRATES_DIR}")
if("${USE_CARGO_VENDOR}" STREQUAL "AUTO")
if(EXISTS "${RUST_VENDORED_CRATES_DIR}")
set(USE_CARGO_VENDOR ON)
@@ -17,7 +24,11 @@ if("${USE_CARGO_VENDOR}" STREQUAL "AUTO")
endif()
endif()
if(USE_CARGO_VENDOR)
if("${GENERATE_CARGO_VENDOR_CONFIG}" STREQUAL "AUTO")
set(GENERATE_CARGO_VENDOR_CONFIG "${USE_CARGO_VENDOR}")
endif()
if(GENERATE_CARGO_VENDOR_CONFIG)
if(NOT EXISTS "${RUST_VENDORED_CRATES_DIR}")
message(
FATAL "vendored rust crates not present: "

12
build/fbcode_builder/getdeps.py
View File

@@ -599,6 +599,18 @@ class BuildCmd(ProjectCmdBase):
if os.path.exists(built_marker):
os.unlink(built_marker)
src_dir = fetcher.get_src_dir()
# Prepare builders write out config before the main builder runs
prepare_builders = m.create_prepare_builders(
loader.build_opts,
ctx,
src_dir,
build_dir,
inst_dir,
loader,
)
for preparer in prepare_builders:
preparer.prepare(install_dirs, reconfigure=reconfigure)
builder = m.create_builder(
loader.build_opts,
src_dir,

20
build/fbcode_builder/getdeps/builder.py
View File

@@ -86,14 +86,22 @@ class BuilderBase(object):
allow_fail=allow_fail,
)
def build(self, install_dirs, reconfigure: bool) -> None:
print("Building %s..." % self.manifest.name)
def _reconfigure(self, reconfigure):
if self.build_dir is not None:
if not os.path.isdir(self.build_dir):
os.makedirs(self.build_dir)
reconfigure = True
return reconfigure
def prepare(self, install_dirs, reconfigure):
print("Preparing %s..." % self.manifest.name)
reconfigure = self._reconfigure(reconfigure)
self._prepare(install_dirs=install_dirs, reconfigure=reconfigure)
def build(self, install_dirs, reconfigure) -> None:
print("Building %s..." % self.manifest.name)
reconfigure = self._reconfigure(reconfigure)
self._prepare(install_dirs=install_dirs, reconfigure=reconfigure)
self._build(install_dirs=install_dirs, reconfigure=reconfigure)
# On Windows, emit a wrapper script that can be used to run build artifacts
@@ -135,6 +143,12 @@ class BuilderBase(object):
raise an exception."""
pass
def _prepare(self, install_dirs, reconfigure):
"""Prepare the build. Useful when need to generate config,
but builder is not the primary build system.
e.g. cargo when called from cmake"""
pass
def _build(self, install_dirs, reconfigure) -> None:
"""Perform the build.
install_dirs contains the list of installation directories for

248
build/fbcode_builder/getdeps/cargo.py
View File

@@ -5,6 +5,7 @@
# LICENSE file in the root directory of this source tree.
import os
import re
import shutil
from .builder import BuilderBase
@@ -23,6 +24,7 @@ class CargoBuilder(BuilderBase):
workspace_dir,
manifests_to_build,
loader,
cargo_config_file,
) -> None:
super(CargoBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
@@ -31,6 +33,7 @@ class CargoBuilder(BuilderBase):
self.ws_dir = workspace_dir
self.manifests_to_build = manifests_to_build and manifests_to_build.split(",")
self.loader = loader
self.cargo_config_file_subdir = cargo_config_file
def run_cargo(self, install_dirs, operation, args=None) -> None:
args = args or []
@@ -60,17 +63,24 @@ class CargoBuilder(BuilderBase):
shutil.rmtree(dst)
shutil.copytree(src, dst)
def _build(self, install_dirs, reconfigure) -> None:
build_source_dir = self.build_source_dir()
self.recreate_dir(self.src_dir, build_source_dir)
def cargo_config_file(self):
build_source_dir = self.build_dir
if self.cargo_config_file_subdir:
return os.path.join(build_source_dir, self.cargo_config_file_subdir)
else:
return os.path.join(build_source_dir, ".cargo", "config")
dot_cargo_dir = os.path.join(build_source_dir, ".cargo")
if not os.path.isdir(dot_cargo_dir):
os.mkdir(dot_cargo_dir)
def _create_cargo_config(self):
cargo_config_file = self.cargo_config_file()
cargo_config_dir = os.path.dirname(cargo_config_file)
if not os.path.isdir(cargo_config_dir):
os.mkdir(cargo_config_dir)
with open(os.path.join(dot_cargo_dir, "config"), "w+") as f:
print(f"Writing cargo config for {self.manifest.name} to {cargo_config_file}")
with open(cargo_config_file, "w+") as f:
f.write(
"""\
# Generated by getdeps.py
[build]
target-dir = '''{}'''
@@ -85,18 +95,45 @@ incremental = false
)
)
if self.ws_dir is not None:
self._patchup_workspace()
# Point to vendored sources from getdeps manifests
dep_to_git = self._resolve_dep_to_git()
for _dep, git_conf in dep_to_git.items():
if "cargo_vendored_sources" in git_conf:
with open(cargo_config_file, "a") as f:
vendored_dir = git_conf["cargo_vendored_sources"].replace(
"\\", "\\\\"
)
f.write(
f"""
[source."{git_conf["repo_url"]}"]
directory = "{vendored_dir}"
"""
)
# Point to vendored crates.io if possible
try:
from .facebook.rust import vendored_crates
vendored_crates(self.build_opts, build_source_dir)
vendored_crates(self.build_opts, cargo_config_file)
except ImportError:
# This FB internal module isn't shippped to github,
# so just rely on cargo downloading crates on it's own
pass
return dep_to_git
def _prepare(self, install_dirs, reconfigure):
build_source_dir = self.build_source_dir()
self.recreate_dir(self.src_dir, build_source_dir)
dep_to_git = self._create_cargo_config()
if self.ws_dir is not None:
self._patchup_workspace(dep_to_git)
def _build(self, install_dirs, reconfigure) -> None:
# _prepare has been run already. Actually do the build
build_source_dir = self.build_source_dir()
if self.manifests_to_build is None:
self.run_cargo(
install_dirs,
@@ -138,7 +175,7 @@ incremental = false
if self.build_doc:
self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs)
def _patchup_workspace(self) -> None:
def _patchup_workspace(self, dep_to_git) -> None:
"""
This method makes some assumptions about the state of the project and
its cargo dependendies:
@@ -159,9 +196,11 @@ incremental = false
producing bad results.
"""
workspace_dir = self.workspace_dir()
config = self._resolve_config()
config = self._resolve_config(dep_to_git)
if config:
with open(os.path.join(workspace_dir, "Cargo.toml"), "r+") as f:
patch_cargo = os.path.join(workspace_dir, "Cargo.toml")
print(f"writing patch to {patch_cargo}")
with open(patch_cargo, "r+") as f:
manifest_content = f.read()
if "[package]" not in manifest_content:
# A fake manifest has to be crated to change the virtual
@@ -169,52 +208,63 @@ incremental = false
# in many ways and the inability to define patches on them is
# one. Check https://github.com/rust-lang/cargo/issues/4934 to
# see if it is resolved.
null_file = "/dev/null"
if self.build_opts.is_windows():
null_file = "nul"
f.write(
"""
[package]
name = "fake_manifest_of_{}"
version = "0.0.0"
[lib]
path = "/dev/null"
""".format(
self.manifest.name
)
f"""
[package]
name = "fake_manifest_of_{self.manifest.name}"
version = "0.0.0"
[lib]
path = "{null_file}"
"""
)
else:
f.write("\n")
f.write(config)
def _resolve_config(self) -> str:
def _resolve_config(self, dep_to_git) -> str:
"""
Returns a configuration to be put inside root Cargo.toml file which
patches the dependencies git code with local getdeps versions.
See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section
"""
dep_to_git = self._resolve_dep_to_git()
dep_to_crates = CargoBuilder._resolve_dep_to_crates(
self.build_source_dir(), dep_to_git
)
dep_to_crates = self._resolve_dep_to_crates(self.build_source_dir(), dep_to_git)
config = []
for name in sorted(dep_to_git.keys()):
git_conf = dep_to_git[name]
crates = sorted(dep_to_crates.get(name, []))
if not crates:
git_url_to_crates_and_paths = {}
for dep_name in sorted(dep_to_git.keys()):
git_conf = dep_to_git[dep_name]
req_crates = sorted(dep_to_crates.get(dep_name, []))
if not req_crates:
continue # nothing to patch, move along
git_url = git_conf.get("repo_url", None)
crate_source_map = git_conf["crate_source_map"]
if git_url and crate_source_map:
crates_to_patch_path = git_url_to_crates_and_paths.get(git_url, {})
for c in req_crates:
if c in crate_source_map and c not in crates_to_patch_path:
crates_to_patch_path[c] = crate_source_map[c]
print(
f"{self.manifest.name}: Patching crate {c} via virtual manifest in {self.workspace_dir()}"
)
if crates_to_patch_path:
git_url_to_crates_and_paths[git_url] = crates_to_patch_path
for git_url, crates_to_patch_path in git_url_to_crates_and_paths.items():
crates_patches = [
'{} = {{ path = "{}" }}'.format(
crate,
CargoBuilder._resolve_crate_to_path(crate, git_conf).replace(
"\\", "\\\\"
),
crates_to_patch_path[crate].replace("\\", "\\\\"),
)
for crate in crates
for crate in sorted(crates_to_patch_path.keys())
]
config.append(f'\n[patch."{git_url}"]\n' + "\n".join(crates_patches))
config.append(
'[patch."{0}"]\n'.format(git_conf["repo_url"])
+ "\n".join(crates_patches)
)
return "\n".join(config)
def _resolve_dep_to_git(self):
@@ -231,27 +281,74 @@ incremental = false
for dep in dependencies:
dep_manifest = self.loader.load_manifest(dep)
dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx)
if dep_builder not in ["cargo", "nop"] or dep == "rust":
# This is a direct dependency, but it is not build with cargo
# and it is not simply copying files with nop, so ignore it.
dep_cargo_conf = dep_manifest.get_section_as_dict("cargo", self.ctx)
dep_crate_map = dep_manifest.get_section_as_dict("crate.pathmap", self.ctx)
if (
not (dep_crate_map or dep_cargo_conf)
and dep_builder not in ["cargo"]
or dep == "rust"
):
# This dependency has no cargo rust content so ignore it.
# The "rust" dependency is an exception since it contains the
# toolchain.
continue
git_conf = dep_manifest.get_section_as_dict("git", self.ctx)
if "repo_url" not in git_conf:
if dep != "rust" and "repo_url" not in git_conf:
raise Exception(
"A cargo dependency requires git.repo_url to be defined."
f"{dep}: A cargo dependency requires git.repo_url to be defined."
)
source_dir = self.loader.get_project_install_dir(dep_manifest)
if dep_builder == "cargo":
source_dir = os.path.join(source_dir, "source")
git_conf["source_dir"] = source_dir
dep_source_dir = self.loader.get_project_install_dir(dep_manifest)
dep_source_dir = os.path.join(dep_source_dir, "source")
else:
fetcher = self.loader.create_fetcher(dep_manifest)
dep_source_dir = fetcher.get_src_dir()
crate_source_map = {}
if dep_crate_map:
for (crate, subpath) in dep_crate_map.items():
if crate not in crate_source_map:
if self.build_opts.is_windows():
subpath = subpath.replace("/", "\\")
crate_path = os.path.join(dep_source_dir, subpath)
print(
f"{self.manifest.name}: Mapped crate {crate} to dep {dep} dir {crate_path}"
)
crate_source_map[crate] = crate_path
elif dep_cargo_conf:
# We don't know what crates are defined buy the dep, look for them
search_pattern = re.compile('\\[package\\]\nname = "(.*)"')
for crate_root, _, files in os.walk(dep_source_dir):
if "Cargo.toml" in files:
with open(os.path.join(crate_root, "Cargo.toml"), "r") as f:
content = f.read()
match = search_pattern.search(content)
if match:
crate = match.group(1)
if crate:
print(
f"{self.manifest.name}: Discovered crate {crate} in dep {dep} dir {crate_root}"
)
crate_source_map[crate] = crate_root
git_conf["crate_source_map"] = crate_source_map
if not dep_crate_map and dep_cargo_conf:
dep_cargo_dir = self.loader.get_project_build_dir(dep_manifest)
dep_cargo_dir = os.path.join(dep_cargo_dir, "source")
dep_ws_dir = dep_cargo_conf.get("workspace_dir", None)
if dep_ws_dir:
dep_cargo_dir = os.path.join(dep_cargo_dir, dep_ws_dir)
git_conf["cargo_vendored_sources"] = dep_cargo_dir
dep_to_git[dep] = git_conf
return dep_to_git
@staticmethod
def _resolve_dep_to_crates(build_source_dir, dep_to_git):
def _resolve_dep_to_crates(self, build_source_dir, dep_to_git):
"""
This function traverse the build_source_dir in search of Cargo.toml
files, extracts the crate names from them using _extract_crates
@@ -262,18 +359,33 @@ incremental = false
return {} # no deps, so don't waste time traversing files
dep_to_crates = {}
# First populate explicit crate paths from depedencies
for name, git_conf in dep_to_git.items():
crates = git_conf["crate_source_map"].keys()
if crates:
dep_to_crates.setdefault(name, set()).update(crates)
# Now find from Cargo.tomls
for root, _, files in os.walk(build_source_dir):
for f in files:
if f == "Cargo.toml":
more_dep_to_crates = CargoBuilder._extract_crates(
more_dep_to_crates = CargoBuilder._extract_crates_used(
os.path.join(root, f), dep_to_git
)
for name, crates in more_dep_to_crates.items():
dep_to_crates.setdefault(name, set()).update(crates)
for dep_name, crates in more_dep_to_crates.items():
existing_crates = dep_to_crates.get(dep_name, set())
for c in crates:
if c not in existing_crates:
print(
f"Patch {self.manifest.name} uses {dep_name} crate {crates}"
)
existing_crates.insert(c)
dep_to_crates.setdefault(name, set()).update(existing_crates)
return dep_to_crates
@staticmethod
def _extract_crates(cargo_toml_file, dep_to_git):
def _extract_crates_used(cargo_toml_file, dep_to_git):
"""
This functions reads content of provided cargo toml file and extracts
crate names per each dependency. The extraction is done by a heuristic
@@ -284,7 +396,8 @@ incremental = false
for line in f.readlines():
if line.startswith("#") or "git = " not in line:
continue # filter out commented lines and ones without git deps
for name, conf in dep_to_git.items():
for dep_name, conf in dep_to_git.items():
# Only redirect deps that point to git URLS
if 'git = "{}"'.format(conf["repo_url"]) in line:
pkg_template = ' package = "'
if pkg_template in line:
@@ -293,23 +406,26 @@ incremental = false
].partition('"')
else:
crate_name, _, _ = line.partition("=")
deps_to_crates.setdefault(name, set()).add(crate_name.strip())
deps_to_crates.setdefault(dep_name, set()).add(
crate_name.strip()
)
return deps_to_crates
@staticmethod
def _resolve_crate_to_path(crate, git_conf):
def _resolve_crate_to_path(self, crate, crate_source_map):
"""
Tries to find <crate> in git_conf["inst_dir"] by searching a [package]
Tries to find <crate> in source_dir by searching a [package]
keyword followed by name = "<crate>".
"""
source_dir = git_conf["source_dir"]
search_pattern = '[package]\nname = "{}"'.format(crate)
for root, _, files in os.walk(source_dir):
for fname in files:
if fname == "Cargo.toml":
with open(os.path.join(root, fname), "r") as f:
if search_pattern in f.read():
return root
for (_crate, crate_source_dir) in crate_source_map.items():
for crate_root, _, files in os.walk(crate_source_dir):
if "Cargo.toml" in files:
with open(os.path.join(crate_root, "Cargo.toml"), "r") as f:
content = f.read()
if search_pattern in content:
return crate_root
raise Exception("Failed to found crate {} in path {}".format(crate, source_dir))
raise Exception(
f"{self.manifest.name}: Failed to find dep crate {crate} in paths {crate_source_map}"
)

64
build/fbcode_builder/getdeps/manifest.py
View File

@@ -75,6 +75,8 @@ SCHEMA = {
"build_doc": OPTIONAL,
"workspace_dir": OPTIONAL,
"manifests_to_build": OPTIONAL,
# Where to write cargo config (defaults to build_dir/.cargo/config)
"cargo_config_file": OPTIONAL,
},
},
"github.actions": {
@@ -83,6 +85,7 @@ SCHEMA = {
"run_tests": OPTIONAL,
},
},
"crate.pathmap": {"optional_section": True},
"cmake.defines": {"optional_section": True},
"autoconf.args": {"optional_section": True},
"autoconf.envcmd.LDFLAGS": {"optional_section": True},
@@ -447,6 +450,12 @@ class ManifestParser(object):
"project %s has no fetcher configuration matching %s" % (self.name, ctx)
)
def get_builder_name(self, ctx):
builder = self.get("build", "builder", ctx=ctx)
if not builder:
raise Exception("project %s has no builder for %r" % (self.name, ctx))
return builder
def create_builder( # noqa:C901
self,
build_options,
@@ -458,9 +467,7 @@ class ManifestParser(object):
final_install_prefix=None,
extra_cmake_defines=None,
):
builder = self.get("build", "builder", ctx=ctx)
if not builder:
raise Exception("project %s has no builder for %r" % (self.name, ctx))
builder = self.get_builder_name(ctx)
build_in_src_dir = self.get("build", "build_in_src_dir", "false", ctx=ctx)
if build_in_src_dir == "true":
# Some scripts don't work when they are configured and build in
@@ -574,20 +581,8 @@ class ManifestParser(object):
)
if builder == "cargo":
build_doc = self.get("cargo", "build_doc", False, ctx)
workspace_dir = self.get("cargo", "workspace_dir", None, ctx)
manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx)
return CargoBuilder(
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
build_doc,
workspace_dir,
manifests_to_build,
loader,
return self.create_cargo_builder(
build_options, ctx, src_dir, build_dir, inst_dir, loader
)
if builder == "OpenNSA":
@@ -595,6 +590,41 @@ class ManifestParser(object):
raise KeyError("project %s has no known builder" % (self.name))
def create_prepare_builders(
self, build_options, ctx, src_dir, build_dir, inst_dir, loader
):
"""Create builders that have a prepare step run, e.g. to write config files"""
prepare_builders = []
builder = self.get_builder_name(ctx)
cargo = self.get_section_as_dict("cargo", ctx)
if not builder == "cargo" and cargo:
cargo_builder = self.create_cargo_builder(
build_options, ctx, src_dir, build_dir, inst_dir, loader
)
prepare_builders.append(cargo_builder)
return prepare_builders
def create_cargo_builder(
self, build_options, ctx, src_dir, build_dir, inst_dir, loader
):
build_doc = self.get("cargo", "build_doc", False, ctx)
workspace_dir = self.get("cargo", "workspace_dir", None, ctx)
manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx)
cargo_config_file = self.get("cargo", "cargo_config_file", None, ctx)
return CargoBuilder(
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
build_doc,
workspace_dir,
manifests_to_build,
loader,
cargo_config_file,
)
class ManifestContext(object):
"""ProjectContext contains a dictionary of values to use when evaluating boolean

6
build/fbcode_builder/manifests/eden
View File

@@ -10,6 +10,9 @@ repo_url = https://github.com/facebookexperimental/eden.git
[github.actions]
run_tests = off
[cargo]
cargo_config_file = _cargo_home/config
[build]
builder = cmake
@@ -26,6 +29,7 @@ lz4
pexpect
python-toml
python-filelock
rust-shed
[dependencies.fbsource=on]
rust
@@ -100,6 +104,8 @@ INSTALL_PYTHON_LIB=ON
ENABLE_GIT=OFF
[cmake.defines.fbsource=on]
# OFF as getdeps will write the cargo config file
GENERATE_CARGO_VENDOR_CONFIG=OFF
USE_CARGO_VENDOR=ON
[cmake.defines.fb=on]

4
build/fbcode_builder/manifests/eden_scm
View File

@@ -53,9 +53,11 @@ fbcode/tools/lfs = tools/lfs
[dependencies]
fb303
fbthrift
python
rust-shed
[dependencies.not(os=windows)]
python
# We use the system openssl on linux
[dependencies.not(os=linux)]
openssl

6
build/fbcode_builder/manifests/fb303
View File

@@ -7,6 +7,12 @@ shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/fb303.git
[cargo]
cargo_config_file = source/fb303/thrift/.cargo/config
[crate.pathmap]
fb303_core = fb303/thrift
[build]
builder = cmake

6
build/fbcode_builder/manifests/fbthrift
View File

@@ -7,6 +7,12 @@ shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/fbthrift.git
[cargo]
cargo_config_file = source/thrift/lib/rust/.cargo/config
[crate.pathmap]
fbthrift = thrift/lib/rust
[build]
builder = cmake
job_weight_mib = 2048

1
build/fbcode_builder/manifests/rust-shed
View File

@@ -25,6 +25,7 @@ tools/rust/ossconfigs = .
[dependencies]
fbthrift
fb303
# We use the system openssl on linux
[dependencies.not(os=linux)]