From ce9e15c734ca8d8e3b2ab6333e9fb4ea2da16beb Mon Sep 17 00:00:00 2001 From: Adam Simpkins Date: Thu, 19 Sep 2019 11:25:33 -0700 Subject: [PATCH] add a builder that can re-package python wheel files Summary: Add a new builder that can extract Python wheel files, and re-package them for consumption by our add_fb_python_library() and add_fb_python_executable() CMake functions. This is useful for dependencies on packages from PyPI. At the moment this code only handles architecture-independent pure-Python packages. It shouldn't be too hard to extend this to handle more complex wheels, but for now I only need to use it for some pure-Python wheels and so I haven't tested with more complex wheel files. This also includes two new manifests for python-six and python-toml that take use this new builder. Reviewed By: wez Differential Revision: D17401216 fbshipit-source-id: d6f74565887c3f004e1c06503dc9ec81599dd697 --- build/fbcode_builder/getdeps/builder.py | 1 - build/fbcode_builder/getdeps/load.py | 2 +- build/fbcode_builder/getdeps/manifest.py | 6 + .../getdeps/py_wheel_builder.py | 271 ++++++++++++++++++ build/fbcode_builder/manifests/python-six | 9 + build/fbcode_builder/manifests/python-toml | 9 + 6 files changed, 296 insertions(+), 2 deletions(-) create mode 100644 build/fbcode_builder/getdeps/py_wheel_builder.py create mode 100644 build/fbcode_builder/manifests/python-six create mode 100644 build/fbcode_builder/manifests/python-toml diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 206d7ee9d..9007bc5ef 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright (c) 2019-present, Facebook, Inc. # All rights reserved. # diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py index 2b5757d99..648136379 100644 --- a/build/fbcode_builder/getdeps/load.py +++ b/build/fbcode_builder/getdeps/load.py @@ -168,7 +168,7 @@ class ManifestLoader(object): ctx = self.ctx_gen.get_context(m.name) dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys()) builder = m.get("build", "builder", ctx=ctx) - if builder == "cmake": + if builder in ("cmake", "python-wheel"): dep_list.append("cmake") elif builder == "autoconf" and m.name not in ( "autoconf", diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 443c75885..6a5a8d779 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -28,6 +28,7 @@ from .fetcher import ( ShipitTransformerFetcher, SimpleShipitTransformerFetcher, ) +from .py_wheel_builder import PythonWheelBuilder try: @@ -383,6 +384,11 @@ class ManifestParser(object): build_options, ctx, self, src_dir, build_dir, inst_dir, defines ) + if builder == "python-wheel": + return PythonWheelBuilder( + build_options, ctx, self, src_dir, build_dir, inst_dir + ) + if builder == "sqlite": return SqliteBuilder(build_options, ctx, self, src_dir, build_dir, inst_dir) diff --git a/build/fbcode_builder/getdeps/py_wheel_builder.py b/build/fbcode_builder/getdeps/py_wheel_builder.py new file mode 100644 index 000000000..4287b79b5 --- /dev/null +++ b/build/fbcode_builder/getdeps/py_wheel_builder.py @@ -0,0 +1,271 @@ +# Copyright (c) 2019-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import codecs +import collections +import email +import os +import re +import stat + +from .builder import BuilderBase, CMakeBuilder + + +WheelNameInfo = collections.namedtuple( + "WheelNameInfo", ("distribution", "version", "build", "python", "abi", "platform") +) + +CMAKE_HEADER = """ +cmake_minimum_required(VERSION 3.8) + +project("{manifest_name}" LANGUAGES C) + +set(CMAKE_MODULE_PATH + "{cmake_dir}" + ${{CMAKE_MODULE_PATH}} +) +include(FBPythonBinary) + +set(CMAKE_INSTALL_DIR lib/cmake/{manifest_name} CACHE STRING + "The subdirectory where CMake package config files should be installed") +""" + +CMAKE_FOOTER = """ +install_fb_python_library({lib_name} EXPORT all) +install( + EXPORT all + FILE {manifest_name}-targets.cmake + NAMESPACE {namespace}:: + DESTINATION ${{CMAKE_INSTALL_DIR}} +) + +include(CMakePackageConfigHelpers) +configure_package_config_file( + ${{CMAKE_BINARY_DIR}}/{manifest_name}-config.cmake.in + {manifest_name}-config.cmake + INSTALL_DESTINATION ${{CMAKE_INSTALL_DIR}} + PATH_VARS + CMAKE_INSTALL_DIR +) +install( + FILES ${{CMAKE_CURRENT_BINARY_DIR}}/{manifest_name}-config.cmake + DESTINATION ${{CMAKE_INSTALL_DIR}} +) +""" + +CMAKE_CONFIG_FILE = """ +@PACKAGE_INIT@ + +set_and_check({upper_name}_CMAKE_DIR "@PACKAGE_CMAKE_INSTALL_DIR@") + +if (NOT TARGET {namespace}::{lib_name}) + include("${{{upper_name}_CMAKE_DIR}}/{manifest_name}-targets.cmake") +endif() + +set({upper_name}_LIBRARIES {namespace}::{lib_name}) + +if (NOT {manifest_name}_FIND_QUIETLY) + message(STATUS "Found {manifest_name}: ${{PACKAGE_PREFIX_DIR}}") +endif() +""" + + +# Note: for now we are manually manipulating the wheel packet contents. +# The wheel format is documented here: +# https://www.python.org/dev/peps/pep-0491/#file-format +# +# We currently aren't particularly smart about correctly handling the full wheel +# functionality, but this is good enough to handle simple pure-python wheels, +# which is the main thing we care about right now. +# +# We could potentially use pip to install the wheel to a temporary location and +# then copy its "installed" files, but this has its own set of complications. +# This would require pip to already be installed and available, and we would +# need to correctly find the right version of pip or pip3 to use. +# If we did ever want to go down that path, we would probably want to use +# something like the following pip3 command: +# pip3 --isolated install --no-cache-dir --no-index --system \ +# --target +class PythonWheelBuilder(BuilderBase): + """This Builder can take Python wheel archives and install them as python libraries + that can be used by add_fb_python_library()/add_fb_python_executable() CMake rules. + """ + + def _build(self, install_dirs, reconfigure): + # type: (List[str], bool) -> None + + # When we are invoked, self.src_dir contains the unpacked wheel contents. + # + # Since a wheel file is just a zip file, the Fetcher code recognizes it as such + # and goes ahead and unpacks it. (We could disable that Fetcher behavior in the + # future if we ever wanted to, say if we wanted to call pip here.) + wheel_name = self._parse_wheel_name() + name_version_prefix = "-".join((wheel_name.distribution, wheel_name.version)) + dist_info_name = name_version_prefix + ".dist-info" + data_dir_name = name_version_prefix + ".data" + self.dist_info_dir = os.path.join(self.src_dir, dist_info_name) + wheel_metadata = self._read_wheel_metadata(wheel_name) + + # Check that we can understand the wheel version. + # We don't really care about wheel_metadata["Root-Is-Purelib"] since + # we are generating our own standalone python archives rather than installing + # into site-packages. + version = wheel_metadata["Wheel-Version"] + if not version.startswith("1."): + raise Exception("unsupported wheel version %s" % (version,)) + + getdeps_cmake_dir = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "CMake" + ) + self.template_format_dict = { + # Note that CMake files always uses forward slash separators in path names, + # even on Windows. Therefore replace path separators here. + "cmake_dir": _to_cmake_path(getdeps_cmake_dir), + "lib_name": self.manifest.name, + "manifest_name": self.manifest.name, + "namespace": self.manifest.name, + "upper_name": self.manifest.name.upper().replace("-", "_"), + } + + # Find sources from the root directory + path_mapping = {} + for entry in os.listdir(self.src_dir): + if entry in (dist_info_name, data_dir_name): + continue + self._add_sources(path_mapping, os.path.join(self.src_dir, entry), entry) + + # Files under the .data directory also need to be installed in the correct + # locations + if os.path.exists(data_dir_name): + # TODO: process the subdirectories of data_dir_name + # This isn't implemented yet since for now we have only needed dependencies + # on some simple pure Python wheels, so I haven't tested against wheels with + # additional files in the .data directory. + raise Exception( + "handling of the subdirectories inside %s is not implemented yet" + % data_dir_name + ) + + # Emit CMake files + self._write_cmakelists(path_mapping) + self._write_cmake_config_template() + + # Run the build + self._run_cmake_build(install_dirs, reconfigure) + + def _run_cmake_build(self, install_dirs, reconfigure): + # type: (List[str], bool) -> None + + cmake_builder = CMakeBuilder( + build_opts=self.build_opts, + ctx=self.ctx, + manifest=self.manifest, + # Note that we intentionally supply src_dir=build_dir, + # since we wrote out our generated CMakeLists.txt in the build directory + src_dir=self.build_dir, + build_dir=self.build_dir, + inst_dir=self.inst_dir, + defines={}, + ) + cmake_builder.build(install_dirs=install_dirs, reconfigure=reconfigure) + + def _write_cmakelists(self, path_mapping): + # type: (List[str]) -> None + + cmake_path = os.path.join(self.build_dir, "CMakeLists.txt") + with open(cmake_path, "w") as f: + f.write(CMAKE_HEADER.format(**self.template_format_dict)) + + f.write( + "add_fb_python_library({lib_name}\n".format(**self.template_format_dict) + ) + f.write(' BASE_DIR "%s"\n' % _to_cmake_path(self.src_dir)) + f.write(" SOURCES\n") + for src_path, install_path in path_mapping.items(): + f.write( + ' "%s=%s"\n' + % (_to_cmake_path(src_path), _to_cmake_path(install_path)) + ) + f.write(")\n") + + f.write(CMAKE_FOOTER.format(**self.template_format_dict)) + + def _write_cmake_config_template(self): + config_path_name = self.manifest.name + "-config.cmake.in" + output_path = os.path.join(self.build_dir, config_path_name) + + with open(output_path, "w") as f: + f.write(CMAKE_CONFIG_FILE.format(**self.template_format_dict)) + + def _add_sources(self, path_mapping, src_path, install_path): + # type: (List[str], str, str) -> None + + s = os.lstat(src_path) + if not stat.S_ISDIR(s.st_mode): + path_mapping[src_path] = install_path + return + + for entry in os.listdir(src_path): + self._add_sources( + path_mapping, + os.path.join(src_path, entry), + os.path.join(install_path, entry), + ) + + def _parse_wheel_name(self): + # type: () -> WheelNameInfo + + # The ArchiveFetcher prepends "manifest_name-", so strip that off first. + wheel_name = os.path.basename(self.src_dir) + prefix = self.manifest.name + "-" + if not wheel_name.startswith(prefix): + raise Exception( + "expected wheel source directory to be of the form %s-NAME.whl" + % (prefix,) + ) + wheel_name = wheel_name[len(prefix) :] + + wheel_name_re = re.compile( + r"(?P[^-]+)" + r"-(?P\d+[^-]*)" + r"(-(?P\d+[^-]*))?" + r"-(?P\w+\d+(\.\w+\d+)*)" + r"-(?P\w+)" + r"-(?P\w+(\.\w+)*)" + r"\.whl" + ) + match = wheel_name_re.match(wheel_name) + if not match: + raise Exception( + "bad python wheel name %s: expected to have the form " + "DISTRIBUTION-VERSION-[-BUILD]-PYTAG-ABI-PLATFORM" + ) + + return WheelNameInfo( + distribution=match.group("distribution"), + version=match.group("version"), + build=match.group("build"), + python=match.group("python"), + abi=match.group("abi"), + platform=match.group("platform"), + ) + + def _read_wheel_metadata(self, wheel_name): + metadata_path = os.path.join(self.dist_info_dir, "WHEEL") + with codecs.open(metadata_path, "r", encoding="utf-8") as f: + return email.message_from_file(f) + + +def _to_cmake_path(path): + # CMake always uses forward slashes to separate paths in CMakeLists.txt files, + # even on Windows. It treats backslashes as character escapes, so using + # backslashes in the path will cause problems. Therefore replace all path + # separators with forward slashes to make sure the paths are correct on Windows. + # e.g. "C:\foo\bar.txt" becomes "C:/foo/bar.txt" + return path.replace(os.path.sep, "/") diff --git a/build/fbcode_builder/manifests/python-six b/build/fbcode_builder/manifests/python-six new file mode 100644 index 000000000..a712188dc --- /dev/null +++ b/build/fbcode_builder/manifests/python-six @@ -0,0 +1,9 @@ +[manifest] +name = python-six + +[download] +url = https://files.pythonhosted.org/packages/73/fb/00a976f728d0d1fecfe898238ce23f502a721c0ac0ecfedb80e0d88c64e9/six-1.12.0-py2.py3-none-any.whl +sha256 = 3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c + +[build] +builder = python-wheel diff --git a/build/fbcode_builder/manifests/python-toml b/build/fbcode_builder/manifests/python-toml new file mode 100644 index 000000000..b49a3b8fb --- /dev/null +++ b/build/fbcode_builder/manifests/python-toml @@ -0,0 +1,9 @@ +[manifest] +name = python-toml + +[download] +url = https://files.pythonhosted.org/packages/a2/12/ced7105d2de62fa7c8fb5fce92cc4ce66b57c95fb875e9318dba7f8c5db0/toml-0.10.0-py2.py3-none-any.whl +sha256 = 235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e + +[build] +builder = python-wheel