From b1f77bce82f61f989c63c93b216b3c7acd3c7e3c Mon Sep 17 00:00:00 2001 From: Lukas Piatkowski Date: Thu, 2 Jul 2020 17:52:20 -0700 Subject: [PATCH] eden/scm: provide getdeps.py way of building eden/scm on GitHub Summary: In order to do what the title says, this diff does: 1. Add the `eden/oss/.../third-party/rust/.../Cargo.toml` files. As mentioned in the previous diff, those are required by GitHub so that the third party dependencies that are local in fbsource are properly defined with a "git" dependency in order for Cargo to "link" crates properly. 2. Changes to `eden/scm/Makefile` to add build/install commands for getdeps to invoke. Those command knowing that they are called from withing getdeps context they link the dependencies brought by getdeps into their proper places that match their folder layout in fbsource. Those Makefile commands also pass a GETDEPS_BUILD env to the setup.py invocations so that it knows it is being called withing a getdeps build. 3. Changes to `eden/scm/setup.py` that add "thriftasset" that makes use of the getdeps.py provided "thrift" binary to build .py files out of thrift files. 4. Changes to `distutils_rust` to use the vendored crates dir provided by getdeps. 5. Changes to `getdeps/builder.py` and `getdeps/manifest.py` that enable more fine-grained configuratior of how Makefile builds are invoked. 6. Changes to `getdeps/buildopts.py` and `getdeps/manifest.py` to disable overriding PATH and pkgconfig env, so that "eden/scm" builds in getdeps using system libraries rather than getdeps-provided ones (NOTE: I've tried to use getdeps provided libraries, but the trickiest bit was that Rust links with Python, which is currently not providable by getdeps, so if you try to build everything the system provided Python libraries will collide with getdeps provided ones) 7. Added `opensource/fbcode_builder/manifests/eden_scm` for the getdeps build. Reviewed By: quark-zju Differential Revision: D22336485 fbshipit-source-id: 244d10c9e06ee83de61e97e62a1f2a2184d2312f --- build/fbcode_builder/getdeps/builder.py | 23 +++++++--- build/fbcode_builder/getdeps/buildopts.py | 53 +++++++++++++---------- build/fbcode_builder/getdeps/manifest.py | 26 ++++++++--- build/fbcode_builder/manifests/eden | 1 + build/fbcode_builder/manifests/eden_scm | 51 ++++++++++++++++++++++ build/fbcode_builder/manifests/libbpf | 6 ++- build/fbcode_builder/manifests/mononoke | 1 + 7 files changed, 125 insertions(+), 36 deletions(-) create mode 100644 build/fbcode_builder/manifests/eden_scm diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 5e7b20f2d..11f908c41 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -111,7 +111,9 @@ class BuilderBase(object): def _compute_env(self, install_dirs): # CMAKE_PREFIX_PATH is only respected when passed through the # environment, so we construct an appropriate path to pass down - return self.build_opts.compute_env_for_install_dirs(install_dirs, env=self.env) + return self.build_opts.compute_env_for_install_dirs( + install_dirs, env=self.env, manifest=self.manifest + ) def get_dev_run_script_path(self): assert self.build_opts.is_windows() @@ -125,11 +127,22 @@ class BuilderBase(object): class MakeBuilder(BuilderBase): - def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args): + def __init__( + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + build_args, + install_args, + ): super(MakeBuilder, self).__init__( build_opts, ctx, manifest, src_dir, build_dir, inst_dir ) - self.args = args or [] + self.build_args = build_args or [] + self.install_args = install_args or [] def _build(self, install_dirs, reconfigure): env = self._compute_env(install_dirs) @@ -138,12 +151,12 @@ class MakeBuilder(BuilderBase): # libbpf uses it when generating its pkg-config file cmd = ( ["make", "-j%s" % self.build_opts.num_jobs] - + self.args + + self.build_args + ["PREFIX=" + self.inst_dir] ) self._run_cmd(cmd, env=env) - install_cmd = ["make", "install"] + self.args + ["PREFIX=" + self.inst_dir] + install_cmd = ["make"] + self.install_args + ["PREFIX=" + self.inst_dir] self._run_cmd(install_cmd, env=env) diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index c9705dbc7..0d6a1b307 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -178,7 +178,7 @@ class BuildOptions(object): } ) - def compute_env_for_install_dirs(self, install_dirs, env=None): + def compute_env_for_install_dirs(self, install_dirs, env=None, manifest=None): if env is not None: env = env.copy() else: @@ -214,30 +214,35 @@ class BuildOptions(object): lib_path = None for d in install_dirs: - add_path_entry(env, "CMAKE_PREFIX_PATH", d) - - pkgconfig = os.path.join(d, "lib/pkgconfig") - if os.path.exists(pkgconfig): - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) - - pkgconfig = os.path.join(d, "lib64/pkgconfig") - if os.path.exists(pkgconfig): - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) - - # Allow resolving shared objects built earlier (eg: zstd - # doesn't include the full path to the dylib in its linkage - # so we need to give it an assist) - if lib_path: - for lib in ["lib", "lib64"]: - libdir = os.path.join(d, lib) - if os.path.exists(libdir): - add_path_entry(env, lib_path, libdir) - - # Allow resolving binaries (eg: cmake, ninja) and dlls - # built by earlier steps bindir = os.path.join(d, "bin") - if os.path.exists(bindir): - add_path_entry(env, "PATH", bindir, append=False) + + if not ( + manifest and manifest.get("build", "disable_env_override_pkgconfig") + ): + pkgconfig = os.path.join(d, "lib/pkgconfig") + if os.path.exists(pkgconfig): + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) + + pkgconfig = os.path.join(d, "lib64/pkgconfig") + if os.path.exists(pkgconfig): + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) + + if not (manifest and manifest.get("build", "disable_env_override_path")): + add_path_entry(env, "CMAKE_PREFIX_PATH", d) + + # Allow resolving shared objects built earlier (eg: zstd + # doesn't include the full path to the dylib in its linkage + # so we need to give it an assist) + if lib_path: + for lib in ["lib", "lib64"]: + libdir = os.path.join(d, lib) + if os.path.exists(libdir): + add_path_entry(env, lib_path, libdir) + + # Allow resolving binaries (eg: cmake, ninja) and dlls + # built by earlier steps + if os.path.exists(bindir): + add_path_entry(env, "PATH", bindir, append=False) # If rustc is present in the `bin` directory, set RUSTC to prevent # cargo uses the rustc installed in the system. diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index b6c894cc5..02b1d96b2 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -67,6 +67,8 @@ SCHEMA = { "builder": REQUIRED, "subdir": OPTIONAL, "build_in_src_dir": OPTIONAL, + "disable_env_override_pkgconfig": OPTIONAL, + "disable_env_override_path": OPTIONAL, }, }, "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}}, @@ -80,7 +82,8 @@ SCHEMA = { "debs": {"optional_section": True}, "preinstalled.env": {"optional_section": True}, "b2.args": {"optional_section": True}, - "make.args": {"optional_section": True}, + "make.build_args": {"optional_section": True}, + "make.install_args": {"optional_section": True}, "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}}, "shipit.pathmap": {"optional_section": True}, "shipit.strip": {"optional_section": True}, @@ -94,7 +97,8 @@ ALLOWED_EXPR_SECTIONS = [ "build", "cmake.defines", "dependencies", - "make.args", + "make.build_args", + "make.install_args", "b2.args", "download", "git", @@ -234,8 +238,8 @@ class ManifestParser(object): return defval def get_section_as_args(self, section, ctx=None): - """ Intended for use with the make.args and autoconf.args - sections, this method collects the entries and returns an + """ Intended for use with the make.[build_args/install_args] and + autoconf.args sections, this method collects the entries and returns an array of strings. If the manifest contains conditional sections, ctx is used to evaluate the condition and merge in the values. @@ -421,8 +425,18 @@ class ManifestParser(object): print("build_dir is %s" % build_dir) # just to quiet lint if builder == "make": - args = self.get_section_as_args("make.args", ctx) - return MakeBuilder(build_options, ctx, self, src_dir, None, inst_dir, args) + build_args = self.get_section_as_args("make.build_args", ctx) + install_args = self.get_section_as_args("make.install_args", ctx) + return MakeBuilder( + build_options, + ctx, + self, + src_dir, + None, + inst_dir, + build_args, + install_args, + ) if builder == "autoconf": args = self.get_section_as_args("autoconf.args", ctx) diff --git a/build/fbcode_builder/manifests/eden b/build/fbcode_builder/manifests/eden index 4f757b137..bac7e0ec1 100644 --- a/build/fbcode_builder/manifests/eden +++ b/build/fbcode_builder/manifests/eden @@ -52,6 +52,7 @@ fbcode/tools/lfs = tools/lfs ^fbcode/eden/hg/.*$ ^fbcode/eden/mononoke/(?!lfs_protocol) ^fbcode/eden/scm/build/.*$ +^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$ ^fbcode/eden/.*/\.cargo/.*$ /Cargo\.lock$ \.pyc$ diff --git a/build/fbcode_builder/manifests/eden_scm b/build/fbcode_builder/manifests/eden_scm new file mode 100644 index 000000000..a6e687cc7 --- /dev/null +++ b/build/fbcode_builder/manifests/eden_scm @@ -0,0 +1,51 @@ +[manifest] +name = eden_scm +fbsource_path = fbcode/eden +shipit_project = eden +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookexperimental/eden.git + +[build.not(os=windows)] +builder = make +subdir = eden/scm +disable_env_override_pkgconfig = 1 +disable_env_override_path = 1 + +[build.os=windows] +# For now the biggest blocker is missing "make" on windows, but there are bound +# to be more +builder = nop + +[make.build_args] +getdepsbuild + +[make.install_args] +install-getdeps + +[shipit.pathmap] +fbcode/eden/oss = . +fbcode/eden = eden +fbcode/tools/lfs = tools/lfs +fbcode/fboss/common = common + +[shipit.strip] +^fbcode/eden/fs/eden-config\.h$ +^fbcode/eden/fs/py/eden/config\.py$ +^fbcode/eden/hg/.*$ +^fbcode/eden/mononoke/(?!lfs_protocol) +^fbcode/eden/scm/build/.*$ +^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$ +^fbcode/eden/.*/\.cargo/.*$ +^.*/fb/.*$ +/Cargo\.lock$ +\.pyc$ + +[dependencies] +fb303 +fbthrift +rust-shed + +[dependencies.fb=on] +rust diff --git a/build/fbcode_builder/manifests/libbpf b/build/fbcode_builder/manifests/libbpf index 087fc51bc..e6e75485f 100644 --- a/build/fbcode_builder/manifests/libbpf +++ b/build/fbcode_builder/manifests/libbpf @@ -13,7 +13,11 @@ builder = nop builder = make subdir = libbpf-0.0.4/src -[make.args] +[make.build_args] +BUILD_STATIC_ONLY=y + +[make.install_args] +install BUILD_STATIC_ONLY=y [dependencies] diff --git a/build/fbcode_builder/manifests/mononoke b/build/fbcode_builder/manifests/mononoke index 6e6e4cfa1..8ee8e5210 100644 --- a/build/fbcode_builder/manifests/mononoke +++ b/build/fbcode_builder/manifests/mononoke @@ -30,6 +30,7 @@ tools/rust/ossconfigs = . [shipit.strip] # strip all code unrelated to mononoke to prevent triggering unnecessary checks ^fbcode/eden/(?!mononoke|scm/lib/xdiff.*)/.*$ +^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$ ^fbcode/eden/mononoke/Cargo\.toml$ ^fbcode/eden/mononoke/(?!public_autocargo).+/Cargo\.toml$ ^fbcode/configerator/structs/scm/mononoke/(?!public_autocargo).+/Cargo\.toml$