From 98b76fab5016bd1050e7ad0069a4eab54b605e80 Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Wed, 29 Apr 2020 19:07:20 -0700 Subject: [PATCH] getdeps: GH actions: strip artifacts before capturing them (#809) Summary: On Linux the debug info sections in projects downstream from folly and thrift are huge due to a lot of C++ template usage. We build in RelWithDebInfo mode as that is most useful when debugging locally and because the build times are long, but most casual consumers of the binaries via GH actions really don't care about debug info, and this should save ~180MB of size in (for example) the watchman executables. Pull Request resolved: https://github.com/facebook/watchman/pull/809 Refs: https://github.com/facebook/watchman/issues/804 Test Plan: Review the artifact size on the linux build in this PR (see https://github.com/facebook/watchman/actions/runs/91688952) and confirm that it is a bit smaller than 180MB; now under 3 MB. Reviewed By: simpkins Differential Revision: D21318302 Pulled By: wez fbshipit-source-id: f78bc5e735dd78771e0604abae64c0a23cf9158d --- .github/workflows/getdeps_linux.yml | 2 +- build/fbcode_builder/getdeps.py | 21 +++++++++++++++-- build/fbcode_builder/getdeps/dyndeps.py | 30 +++++++++++++++++-------- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/.github/workflows/getdeps_linux.yml b/.github/workflows/getdeps_linux.yml index a990a3307..dcbe29653 100644 --- a/.github/workflows/getdeps_linux.yml +++ b/.github/workflows/getdeps_linux.yml @@ -100,7 +100,7 @@ jobs: - name: Build proxygen run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local - name: Copy artifacts - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. proxygen _artifacts/linux --project-install-prefix proxygen:/usr/local --final-install-prefix /usr/local + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --strip --src-dir=. proxygen _artifacts/linux --project-install-prefix proxygen:/usr/local --final-install-prefix /usr/local - uses: actions/upload-artifact@master with: name: proxygen diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 467b981ac..1d3bb1f8e 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -572,7 +572,9 @@ class FixupDeps(ProjectCmdBase): install_dirs.append(inst_dir) if m == manifest: - dep_munger = create_dyn_dep_munger(loader.build_opts, install_dirs) + dep_munger = create_dyn_dep_munger( + loader.build_opts, install_dirs, args.strip + ) dep_munger.process_deps(args.destdir, args.final_install_prefix) def setup_project_cmd_parser(self, parser): @@ -580,6 +582,12 @@ class FixupDeps(ProjectCmdBase): parser.add_argument( "--final-install-prefix", help=("specify the final installation prefix") ) + parser.add_argument( + "--strip", + action="store_true", + default=False, + help="Strip debug info while processing executables", + ) @cmd("test", "test a given project") @@ -757,8 +765,17 @@ jobs: ) out.write(" - name: Copy artifacts\n") + if build_opts.is_linux(): + # Strip debug info from the binaries, but only on linux. + # While the `strip` utility is also available on macOS, + # attempting to strip there results in an error. + # The `strip` utility is not available on Windows. + strip = " --strip" + else: + strip = "" + out.write( - f" run: {getdeps} fixup-dyn-deps " + f" run: {getdeps} fixup-dyn-deps{strip} " f"--src-dir=. {manifest.name} _artifacts/{job_name} {project_prefix} " f"--final-install-prefix /usr/local\n" ) diff --git a/build/fbcode_builder/getdeps/dyndeps.py b/build/fbcode_builder/getdeps/dyndeps.py index 2775d9fae..216f26c46 100644 --- a/build/fbcode_builder/getdeps/dyndeps.py +++ b/build/fbcode_builder/getdeps/dyndeps.py @@ -27,10 +27,11 @@ def copyfile(src, dest): class DepBase(object): - def __init__(self, buildopts, install_dirs): + def __init__(self, buildopts, install_dirs, strip): self.buildopts = buildopts self.env = buildopts.compute_env_for_install_dirs(install_dirs) self.install_dirs = install_dirs + self.strip = strip self.processed_deps = set() def list_dynamic_deps(self, objfile): @@ -111,6 +112,9 @@ class DepBase(object): self.rewrite_dep(objfile, d, dep, dest_dep, final_lib_dir) + if self.strip: + self.strip_debug_info(objfile) + def rewrite_dep(self, objfile, depname, old_dep, new_dep, final_lib_dir): raise RuntimeError("rewrite_dep not implemented") @@ -143,10 +147,15 @@ class DepBase(object): def is_objfile(self, objfile): return True + def strip_debug_info(self, objfile): + """override this to define how to remove debug information + from an object file""" + pass + class WinDeps(DepBase): - def __init__(self, buildopts, install_dirs): - super(WinDeps, self).__init__(buildopts, install_dirs) + def __init__(self, buildopts, install_dirs, strip): + super(WinDeps, self).__init__(buildopts, install_dirs, strip) self.dumpbin = self.find_dumpbin() def find_dumpbin(self): @@ -308,8 +317,8 @@ try {{ class ElfDeps(DepBase): - def __init__(self, buildopts, install_dirs): - super(ElfDeps, self).__init__(buildopts, install_dirs) + def __init__(self, buildopts, install_dirs, strip): + super(ElfDeps, self).__init__(buildopts, install_dirs, strip) # We need patchelf to rewrite deps, so ensure that it is built... subprocess.check_call([sys.executable, sys.argv[0], "build", "patchelf"]) @@ -350,6 +359,9 @@ class ElfDeps(DepBase): magic = f.read(4) return magic == b"\x7fELF" + def strip_debug_info(self, objfile): + subprocess.check_call(["strip", objfile]) + # MACH-O magic number MACH_MAGIC = 0xFEEDFACF @@ -409,10 +421,10 @@ class MachDeps(DepBase): ) -def create_dyn_dep_munger(buildopts, install_dirs): +def create_dyn_dep_munger(buildopts, install_dirs, strip=False): if buildopts.is_linux(): - return ElfDeps(buildopts, install_dirs) + return ElfDeps(buildopts, install_dirs, strip) if buildopts.is_darwin(): - return MachDeps(buildopts, install_dirs) + return MachDeps(buildopts, install_dirs, strip) if buildopts.is_windows(): - return WinDeps(buildopts, install_dirs) + return WinDeps(buildopts, install_dirs, strip)