From 94da94532598fc1408e5257bb8da0c0192b3fac5 Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Wed, 3 Jul 2019 16:18:27 -0700 Subject: [PATCH] getdeps: dynamic dependency munging Summary: This diff adds a `fixup-dyn-deps` subcommand that is intended to aid in packaging on multiple platforms. Its purpose is to copy a set of executable object files from the getdeps installation directories and place them into an installation staging area that will then be used to create some kind of package (rpm, tarball etc.). The dynamic dependencies of the executables are determined and also copied into the destination area, and the important part: the execute is rewritten such that it will load the deps out of an alternate installation prefix. The implementation of this command draws on similar scripts in use for the watchman and eden packaging on windows and macos. This diff adds linux support using the `patchelf` utility. Reviewed By: pkaush Differential Revision: D16101902 fbshipit-source-id: 5885125971947139407841e08c0cf9f35fdf5895 --- build/fbcode_builder/getdeps.py | 59 +++++++ build/fbcode_builder/getdeps/dyndeps.py | 202 ++++++++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 build/fbcode_builder/getdeps/dyndeps.py diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 30aac76bf..48c7609e8 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -16,6 +16,7 @@ import subprocess import sys from getdeps.buildopts import setup_build_options +from getdeps.dyndeps import create_dyn_dep_munger from getdeps.errors import TransientFailure from getdeps.load import load_project, manifests_in_dependency_order from getdeps.manifest import ManifestParser @@ -307,6 +308,64 @@ class BuildCmd(SubCmd): ) +@cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes") +class FixupDeps(SubCmd): + def run(self, args): + opts = setup_build_options(args) + + manifest = load_project(opts, args.project) + + ctx = context_from_host_tuple(facebook_internal=args.facebook_internal) + projects = manifests_in_dependency_order(opts, manifest, ctx) + manifests_by_name = {m.name: m for m in projects} + + # Accumulate the install directories so that the build steps + # can find their dep installation + install_dirs = [] + + for m in projects: + ctx = dict(ctx) + if args.enable_tests and m.name == manifest.name: + ctx["test"] = "on" + else: + ctx["test"] = "off" + fetcher = m.create_fetcher(opts, ctx) + + dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx) + inst_dir = dirs["inst_dir"] + + install_dirs.append(inst_dir) + + if m == manifest: + dep_munger = create_dyn_dep_munger(opts, install_dirs) + dep_munger.process_deps(args.destdir, args.final_install_prefix) + + def setup_parser(self, parser): + parser.add_argument( + "project", + help=( + "name of the project or path to a manifest " + "file describing the project" + ), + ) + parser.add_argument("destdir", help=("Where to copy the fixed up executables")) + parser.add_argument( + "--final-install-prefix", help=("specify the final installation prefix") + ) + parser.add_argument( + "--enable-tests", + action="store_true", + default=False, + help=( + "For the named project, build tests so that the test command " + "is able to execute tests" + ), + ) + parser.add_argument( + "--schedule-type", help="Indicates how the build was activated" + ) + + @cmd("test", "test a given project") class TestCmd(SubCmd): def run(self, args): diff --git a/build/fbcode_builder/getdeps/dyndeps.py b/build/fbcode_builder/getdeps/dyndeps.py new file mode 100644 index 000000000..9cd113eb1 --- /dev/null +++ b/build/fbcode_builder/getdeps/dyndeps.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python +# Copyright (c) 2019-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import re +import shutil +import subprocess +import sys +from struct import unpack + +from .envfuncs import path_search + + +def copyfile(src, dest): + shutil.copyfile(src, dest) + shutil.copymode(src, dest) + + +class DepBase(object): + def __init__(self, buildopts, install_dirs): + self.buildopts = buildopts + self.env = buildopts.compute_env_for_install_dirs(install_dirs) + self.install_dirs = install_dirs + self.processed_deps = set() + + def list_dynamic_deps(self, objfile): + raise RuntimeError("list_dynamic_deps not implemented") + + def interesting_dep(self, d): + return True + + def process_deps(self, destdir, final_install_prefix=None): + if final_install_prefix is None: + final_install_prefix = destdir + + if self.buildopts.is_windows(): + self.munged_lib_dir = os.path.join(final_install_prefix, "bin") + else: + self.munged_lib_dir = os.path.join(final_install_prefix, "lib") + + if not os.path.isdir(self.munged_lib_dir): + os.makedirs(self.munged_lib_dir) + + # Look only at the things that got installed in the leaf package, + # which will be the last entry in the install dirs list + inst_dir = self.install_dirs[-1] + print("Process deps under %s" % inst_dir, file=sys.stderr) + + for dir in ["bin", "lib", "lib64"]: + src_dir = os.path.join(inst_dir, dir) + if not os.path.isdir(src_dir): + continue + dest_dir = os.path.join(final_install_prefix, dir) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + + for objfile in self.list_objs_in_dir(src_dir): + print("Consider %s/%s" % (dir, objfile)) + dest_obj = os.path.join(dest_dir, objfile) + copyfile(os.path.join(src_dir, objfile), dest_obj) + self.munge_in_place(dest_obj) + + def munge_in_place(self, objfile): + print("Munging %s" % objfile) + for d in self.list_dynamic_deps(objfile): + if not self.interesting_dep(d): + continue + + # Resolve this dep: does it exist in any of our installation + # directories? If so, then it is a candidate for processing + dep = self.resolve_loader_path(d) + print("dep: %s -> %s" % (d, dep)) + if dep: + dest_dep = os.path.join(self.munged_lib_dir, os.path.basename(dep)) + if dep not in self.processed_deps: + self.processed_deps.add(dep) + copyfile(dep, dest_dep) + self.munge_in_place(dest_dep) + + self.rewrite_dep(objfile, d, dep, dest_dep) + + def rewrite_dep(self, objfile, depname, old_dep, new_dep): + raise RuntimeError("rewrite_dep not implemented") + + def resolve_loader_path(self, dep): + if os.path.isabs(dep): + return dep + d = os.path.basename(dep) + for inst_dir in self.install_dirs: + for libdir in ["lib", "lib64"]: + candidate = os.path.join(inst_dir, libdir, d) + if os.path.exists(candidate): + return candidate + return None + + def list_objs_in_dir(self, dir): + objs = [] + for d in os.listdir(dir): + if self.is_objfile(os.path.join(dir, d)): + objs.append(os.path.normcase(d)) + + return objs + + def is_objfile(self, objfile): + return True + + +class ElfDeps(DepBase): + def __init__(self, buildopts, install_dirs): + super(ElfDeps, self).__init__(buildopts, install_dirs) + self.patchelf = path_search(self.env, "patchelf") + + def list_dynamic_deps(self, objfile): + out = ( + subprocess.check_output( + [self.patchelf, "--print-needed", objfile], env=dict(self.env.items()) + ) + .decode("utf-8") + .strip() + ) + lines = out.split("\n") + return lines + + def rewrite_dep(self, objfile, depname, old_dep, new_dep): + subprocess.check_call( + [self.patchelf, "--replace-needed", depname, new_dep, objfile] + ) + + def is_objfile(self, objfile): + if not os.path.isfile(objfile): + return False + with open(objfile, "rb") as f: + # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header + magic = f.read(4) + return magic == b"\x7fELF" + + +# MACH-O magic number +MACH_MAGIC = 0xFEEDFACF + + +class MachDeps(DepBase): + def interesting_dep(self, d): + if d.startswith("/usr/lib/") or d.startswith("/System/"): + return False + return True + + def is_objfile(self, objfile): + if not os.path.isfile(objfile): + return False + with open(objfile, "rb") as f: + # mach stores the magic number in native endianness, + # so unpack as native here and compare + magic = unpack("I", f.read(4))[0] + return magic == MACH_MAGIC + + def list_dynamic_deps(self, objfile): + if not self.interesting_dep(objfile): + return + out = ( + subprocess.check_output( + ["otool", "-L", objfile], env=dict(self.env.items()) + ) + .decode("utf-8") + .strip() + ) + lines = out.split("\n") + deps = [] + for line in lines: + m = re.match("\t(\\S+)\\s", line) + if m: + if os.path.basename(m.group(1)) != os.path.basename(objfile): + deps.append(os.path.normcase(m.group(1))) + return deps + + def rewrite_dep(self, objfile, depname, old_dep, new_dep): + if objfile.endswith(".dylib"): + # Erase the original location from the id of the shared + # object. It doesn't appear to hurt to retain it, but + # it does look weird, so let's rewrite it to be sure. + subprocess.check_call( + ["install_name_tool", "-id", os.path.basename(objfile), objfile] + ) + subprocess.check_call( + ["install_name_tool", "-change", depname, new_dep, objfile] + ) + + +def create_dyn_dep_munger(buildopts, install_dirs): + if buildopts.is_linux(): + return ElfDeps(buildopts, install_dirs) + if buildopts.is_darwin(): + return MachDeps(buildopts, install_dirs) + if buildopts.is_windows(): + return DepBase(buildopts, install_dirs)