lib/proxygen
1
0
Fork 0
mirror of https://github.com/facebook/proxygen.git synced 2025-08-07 07:02:53 +03:00
Code Activity

getdeps: dynamic dependency munging

Browse Source 
Summary:
This diff adds a `fixup-dyn-deps` subcommand that is intended to
aid in packaging on multiple platforms.

Its purpose is to copy a set of executable object files from the getdeps
installation directories and place them into an installation staging
area that will then be used to create some kind of package (rpm, tarball etc.).

The dynamic dependencies of the executables are determined and also copied
into the destination area, and the important part: the execute is rewritten
such that it will load the deps out of an alternate installation prefix.

The implementation of this command draws on similar scripts in use for
the watchman and eden packaging on windows and macos.  This diff adds
linux support using the `patchelf` utility.

Reviewed By: pkaush

Differential Revision: D16101902

fbshipit-source-id: 5885125971947139407841e08c0cf9f35fdf5895
This commit is contained in:
Wez Furlong
2019-07-03 16:18:27 -07:00
committed by Facebook Github Bot
parent ecc5dd6d71
commit 94da945325
2 changed files with 261 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
build/fbcode_builder/getdeps.py
View File

@@ -16,6 +16,7 @@ import subprocess
import sys
from getdeps.buildopts import setup_build_options
from getdeps.dyndeps import create_dyn_dep_munger
from getdeps.errors import TransientFailure
from getdeps.load import load_project, manifests_in_dependency_order
from getdeps.manifest import ManifestParser
@@ -307,6 +308,64 @@ class BuildCmd(SubCmd):
)
@cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes")
class FixupDeps(SubCmd):
def run(self, args):
opts = setup_build_options(args)
manifest = load_project(opts, args.project)
ctx = context_from_host_tuple(facebook_internal=args.facebook_internal)
projects = manifests_in_dependency_order(opts, manifest, ctx)
manifests_by_name = {m.name: m for m in projects}
# Accumulate the install directories so that the build steps
# can find their dep installation
install_dirs = []
for m in projects:
ctx = dict(ctx)
if args.enable_tests and m.name == manifest.name:
ctx["test"] = "on"
else:
ctx["test"] = "off"
fetcher = m.create_fetcher(opts, ctx)
dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx)
inst_dir = dirs["inst_dir"]
install_dirs.append(inst_dir)
if m == manifest:
dep_munger = create_dyn_dep_munger(opts, install_dirs)
dep_munger.process_deps(args.destdir, args.final_install_prefix)
def setup_parser(self, parser):
parser.add_argument(
"project",
help=(
"name of the project or path to a manifest "
"file describing the project"
),
)
parser.add_argument("destdir", help=("Where to copy the fixed up executables"))
parser.add_argument(
"--final-install-prefix", help=("specify the final installation prefix")
)
parser.add_argument(
"--enable-tests",
action="store_true",
default=False,
help=(
"For the named project, build tests so that the test command "
"is able to execute tests"
),
)
parser.add_argument(
"--schedule-type", help="Indicates how the build was activated"
)
@cmd("test", "test a given project")
class TestCmd(SubCmd):
def run(self, args):

202
build/fbcode_builder/getdeps/dyndeps.py Normal file
View File

@@ -0,0 +1,202 @@
#!/usr/bin/env python
# Copyright (c) 2019-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
from __future__ import absolute_import, division, print_function, unicode_literals
import os
import re
import shutil
import subprocess
import sys
from struct import unpack
from .envfuncs import path_search
def copyfile(src, dest):
shutil.copyfile(src, dest)
shutil.copymode(src, dest)
class DepBase(object):
def __init__(self, buildopts, install_dirs):
self.buildopts = buildopts
self.env = buildopts.compute_env_for_install_dirs(install_dirs)
self.install_dirs = install_dirs
self.processed_deps = set()
def list_dynamic_deps(self, objfile):
raise RuntimeError("list_dynamic_deps not implemented")
def interesting_dep(self, d):
return True
def process_deps(self, destdir, final_install_prefix=None):
if final_install_prefix is None:
final_install_prefix = destdir
if self.buildopts.is_windows():
self.munged_lib_dir = os.path.join(final_install_prefix, "bin")
else:
self.munged_lib_dir = os.path.join(final_install_prefix, "lib")
if not os.path.isdir(self.munged_lib_dir):
os.makedirs(self.munged_lib_dir)
# Look only at the things that got installed in the leaf package,
# which will be the last entry in the install dirs list
inst_dir = self.install_dirs[-1]
print("Process deps under %s" % inst_dir, file=sys.stderr)
for dir in ["bin", "lib", "lib64"]:
src_dir = os.path.join(inst_dir, dir)
if not os.path.isdir(src_dir):
continue
dest_dir = os.path.join(final_install_prefix, dir)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
for objfile in self.list_objs_in_dir(src_dir):
print("Consider %s/%s" % (dir, objfile))
dest_obj = os.path.join(dest_dir, objfile)
copyfile(os.path.join(src_dir, objfile), dest_obj)
self.munge_in_place(dest_obj)
def munge_in_place(self, objfile):
print("Munging %s" % objfile)
for d in self.list_dynamic_deps(objfile):
if not self.interesting_dep(d):
continue
# Resolve this dep: does it exist in any of our installation
# directories? If so, then it is a candidate for processing
dep = self.resolve_loader_path(d)
print("dep: %s -> %s" % (d, dep))
if dep:
dest_dep = os.path.join(self.munged_lib_dir, os.path.basename(dep))
if dep not in self.processed_deps:
self.processed_deps.add(dep)
copyfile(dep, dest_dep)
self.munge_in_place(dest_dep)
self.rewrite_dep(objfile, d, dep, dest_dep)
def rewrite_dep(self, objfile, depname, old_dep, new_dep):
raise RuntimeError("rewrite_dep not implemented")
def resolve_loader_path(self, dep):
if os.path.isabs(dep):
return dep
d = os.path.basename(dep)
for inst_dir in self.install_dirs:
for libdir in ["lib", "lib64"]:
candidate = os.path.join(inst_dir, libdir, d)
if os.path.exists(candidate):
return candidate
return None
def list_objs_in_dir(self, dir):
objs = []
for d in os.listdir(dir):
if self.is_objfile(os.path.join(dir, d)):
objs.append(os.path.normcase(d))
return objs
def is_objfile(self, objfile):
return True
class ElfDeps(DepBase):
def __init__(self, buildopts, install_dirs):
super(ElfDeps, self).__init__(buildopts, install_dirs)
self.patchelf = path_search(self.env, "patchelf")
def list_dynamic_deps(self, objfile):
out = (
subprocess.check_output(
[self.patchelf, "--print-needed", objfile], env=dict(self.env.items())
)
.decode("utf-8")
.strip()
)
lines = out.split("\n")
return lines
def rewrite_dep(self, objfile, depname, old_dep, new_dep):
subprocess.check_call(
[self.patchelf, "--replace-needed", depname, new_dep, objfile]
)
def is_objfile(self, objfile):
if not os.path.isfile(objfile):
return False
with open(objfile, "rb") as f:
# https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header
magic = f.read(4)
return magic == b"\x7fELF"
# MACH-O magic number
MACH_MAGIC = 0xFEEDFACF
class MachDeps(DepBase):
def interesting_dep(self, d):
if d.startswith("/usr/lib/") or d.startswith("/System/"):
return False
return True
def is_objfile(self, objfile):
if not os.path.isfile(objfile):
return False
with open(objfile, "rb") as f:
# mach stores the magic number in native endianness,
# so unpack as native here and compare
magic = unpack("I", f.read(4))[0]
return magic == MACH_MAGIC
def list_dynamic_deps(self, objfile):
if not self.interesting_dep(objfile):
return
out = (
subprocess.check_output(
["otool", "-L", objfile], env=dict(self.env.items())
)
.decode("utf-8")
.strip()
)
lines = out.split("\n")
deps = []
for line in lines:
m = re.match("\t(\\S+)\\s", line)
if m:
if os.path.basename(m.group(1)) != os.path.basename(objfile):
deps.append(os.path.normcase(m.group(1)))
return deps
def rewrite_dep(self, objfile, depname, old_dep, new_dep):
if objfile.endswith(".dylib"):
# Erase the original location from the id of the shared
# object. It doesn't appear to hurt to retain it, but
# it does look weird, so let's rewrite it to be sure.
subprocess.check_call(
["install_name_tool", "-id", os.path.basename(objfile), objfile]
)
subprocess.check_call(
["install_name_tool", "-change", depname, new_dep, objfile]
)
def create_dyn_dep_munger(buildopts, install_dirs):
if buildopts.is_linux():
return ElfDeps(buildopts, install_dirs)
if buildopts.is_darwin():
return MachDeps(buildopts, install_dirs)
if buildopts.is_windows():
return DepBase(buildopts, install_dirs)