diff --git a/build/fbcode_builder/CMake/FBCMakeParseArgs.cmake b/build/fbcode_builder/CMake/FBCMakeParseArgs.cmake new file mode 100644 index 000000000..933180189 --- /dev/null +++ b/build/fbcode_builder/CMake/FBCMakeParseArgs.cmake @@ -0,0 +1,141 @@ +# +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Helper function for parsing arguments to a CMake function. +# +# This function is very similar to CMake's built-in cmake_parse_arguments() +# function, with some improvements: +# - This function correctly handles empty arguments. (cmake_parse_arguments() +# ignores empty arguments.) +# - If a multi-value argument is specified more than once, the subsequent +# arguments are appended to the original list rather than replacing it. e.g. +# if "SOURCES" is a multi-value argument, and the argument list contains +# "SOURCES a b c SOURCES x y z" then the resulting value for SOURCES will be +# "a;b;c;x;y;z" rather than "x;y;z" +# - This function errors out by default on unrecognized arguments. You can +# pass in an extra "ALLOW_UNPARSED_ARGS" argument to make it behave like +# cmake_parse_arguments(), and return the unparsed arguments in a +# _UNPARSED_ARGUMENTS variable instead. +# +# It does look like cmake_parse_arguments() handled empty arguments correctly +# from CMake 3.0 through 3.3, but it seems like this was probably broken when +# it was turned into a built-in function in CMake 3.4. Here is discussion and +# patches that fixed this behavior prior to CMake 3.0: +# https://cmake.org/pipermail/cmake-developers/2013-November/020607.html +# +# The one downside to this function over the built-in cmake_parse_arguments() +# is that I don't think we can achieve the PARSE_ARGV behavior in a non-builtin +# function, so we can't properly handle arguments that contain ";". CMake will +# treat the ";" characters as list element separators, and treat it as multiple +# separate arguments. +# +function(fb_cmake_parse_args PREFIX OPTIONS ONE_VALUE_ARGS MULTI_VALUE_ARGS ARGS) + foreach(option IN LISTS ARGN) + if ("${option}" STREQUAL "ALLOW_UNPARSED_ARGS") + set(ALLOW_UNPARSED_ARGS TRUE) + else() + message( + FATAL_ERROR + "unknown optional argument for fb_cmake_parse_args(): ${option}" + ) + endif() + endforeach() + + # Define all options as FALSE in the parent scope to start with + foreach(var_name IN LISTS OPTIONS) + set("${PREFIX}_${var_name}" "FALSE" PARENT_SCOPE) + endforeach() + + # TODO: We aren't extremely strict about error checking for one-value + # arguments here. e.g., we don't complain if a one-value argument is + # followed by another option/one-value/multi-value name rather than an + # argument. We also don't complain if a one-value argument is the last + # argument and isn't followed by a value. + + list(APPEND all_args ${ONE_VALUE_ARGS}) + list(APPEND all_args ${MULTI_VALUE_ARGS}) + set(current_variable) + set(unparsed_args) + foreach(arg IN LISTS ARGS) + list(FIND OPTIONS "${arg}" opt_index) + if("${opt_index}" EQUAL -1) + list(FIND all_args "${arg}" arg_index) + if("${arg_index}" EQUAL -1) + # This argument does not match an argument name, + # must be an argument value + if("${current_variable}" STREQUAL "") + list(APPEND unparsed_args "${arg}") + else() + # Ugh, CMake lists have a pretty fundamental flaw: they cannot + # distinguish between an empty list and a list with a single empty + # element. We track our own SEEN_VALUES_arg setting to help + # distinguish this and behave properly here. + if ("${SEEN_${current_variable}}" AND "${${current_variable}}" STREQUAL "") + set("${current_variable}" ";${arg}") + else() + list(APPEND "${current_variable}" "${arg}") + endif() + set("SEEN_${current_variable}" TRUE) + endif() + else() + # We found a single- or multi-value argument name + set(current_variable "VALUES_${arg}") + set("SEEN_${arg}" TRUE) + endif() + else() + # We found an option variable + set("${PREFIX}_${arg}" "TRUE" PARENT_SCOPE) + set(current_variable) + endif() + endforeach() + + foreach(arg_name IN LISTS ONE_VALUE_ARGS) + if(NOT "${SEEN_${arg_name}}") + unset("${PREFIX}_${arg_name}" PARENT_SCOPE) + elseif(NOT "${SEEN_VALUES_${arg_name}}") + # If the argument was seen but a value wasn't specified, error out. + # We require exactly one value to be specified. + message( + FATAL_ERROR "argument ${arg_name} was specified without a value" + ) + else() + list(LENGTH "VALUES_${arg_name}" num_args) + if("${num_args}" EQUAL 0) + # We know an argument was specified and that we called list(APPEND). + # If CMake thinks the list is empty that means there is really a single + # empty element in the list. + set("${PREFIX}_${arg_name}" "" PARENT_SCOPE) + elseif("${num_args}" EQUAL 1) + list(GET "VALUES_${arg_name}" 0 arg_value) + set("${PREFIX}_${arg_name}" "${arg_value}" PARENT_SCOPE) + else() + message( + FATAL_ERROR "too many arguments specified for ${arg_name}: " + "${VALUES_${arg_name}}" + ) + endif() + endif() + endforeach() + + foreach(arg_name IN LISTS MULTI_VALUE_ARGS) + # If this argument name was never seen, then unset the parent scope + if (NOT "${SEEN_${arg_name}}") + unset("${PREFIX}_${arg_name}" PARENT_SCOPE) + else() + # TODO: Our caller still won't be able to distinguish between an empty + # list and a list with a single empty element. We can tell which is + # which, but CMake lists don't make it easy to show this to our caller. + set("${PREFIX}_${arg_name}" "${VALUES_${arg_name}}" PARENT_SCOPE) + endif() + endforeach() + + # By default we fatal out on unparsed arguments, but return them to the + # caller if ALLOW_UNPARSED_ARGS was specified. + if (DEFINED unparsed_args) + if ("${ALLOW_UNPARSED_ARGS}") + set("${PREFIX}_UNPARSED_ARGUMENTS" "${unparsed_args}" PARENT_SCOPE) + else() + message(FATAL_ERROR "unrecognized arguments: ${unparsed_args}") + endif() + endif() +endfunction() diff --git a/build/fbcode_builder/CMake/FBPythonBinary.cmake b/build/fbcode_builder/CMake/FBPythonBinary.cmake new file mode 100644 index 000000000..ebeb11c10 --- /dev/null +++ b/build/fbcode_builder/CMake/FBPythonBinary.cmake @@ -0,0 +1,407 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +include(FBCMakeParseArgs) + +# +# This file contains helper functions for building self-executing Python +# binaries. +# +# This is somewhat different than typical python installation with +# distutils/pip/virtualenv/etc. We primarily want to build a standalone +# executable, isolated from other Python packages on the system. We don't want +# to install files into the standard library python paths. This is more +# similar to PEX (https://github.com/pantsbuild/pex) and XAR +# (https://github.com/facebookincubator/xar). (In the future it would be nice +# to update this code to also support directly generating XAR files if XAR is +# available.) +# +# We also want to be able to easily define "libraries" of python files that can +# be shared and re-used between these standalone python executables, and can be +# shared across projects in different repositories. This means that we do need +# a way to "install" libraries so that they are visible to CMake builds in +# other repositories, without actually installing them in the standard python +# library paths. +# + +# Find our helper program. +# We typically install this in the same directory as this .cmake file. +find_program( + MAKE_PYTHON_ARCHIVE "make_fbpy_archive.py" + PATHS ${CMAKE_MODULE_PATH} +) +if (NOT MAKE_PYTHON_ARCHIVE) + message( + FATAL_ERROR "unable to find make_fbpy_archive.py helper program (it " + "should be located in the same directory as FBPythonRules.cmake)" + ) +endif() + +# An option to control the default installation location for +# install_fb_python_library(). This is relative to ${CMAKE_INSTALL_PREFIX} +set( + FBPY_LIB_INSTALL_DIR "lib/fb-py-libs" CACHE STRING + "The subdirectory where FB python libraries should be installed" +) + +# +# Build a self-executing python binary. +# +# This accepts the same arguments as add_fb_python_library(). +# In addition, a MAIN_MODULE argument is required. This argument specifies +# which module should be started as the __main__ module when the executable is +# run. +# +function(add_fb_python_executable EXE_NAME) + # Parse the arguments + set(one_value_args BASE_DIR NAMESPACE MAIN_MODULE TYPE) + set(multi_value_args SOURCES DEPENDS) + fb_cmake_parse_args( + ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" + ) + fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR) + + # Use add_fb_python_library() to perform most of our source handling + add_fb_python_library( + "${EXE_NAME}.main_lib" + BASE_DIR "${ARG_BASE_DIR}" + NAMESPACE "${ARG_NAMESPACE}" + SOURCES ${ARG_SOURCES} + DEPENDS ${ARG_DEPENDS} + ) + + set( + manifest_files + "$" + ) + set( + source_files + "$" + ) + + # The command to build the executable archive. + # + # If we are using CMake 3.8+ we can use COMMAND_EXPAND_LISTS. + # CMP0067 isn't really the policy we care about, but seems like the best way + # to check if we are running 3.8+. + if (POLICY CMP0067) + set(extra_cmd_params COMMAND_EXPAND_LISTS) + set(make_py_args "${manifest_files}") + else() + set(extra_cmd_params) + set(make_py_args --manifest-separator "::" "$") + endif() + + set(output_file "${EXE_NAME}") + if(DEFINED ARG_TYPE) + list(APPEND make_py_args "--type" "${ARG_TYPE}") + if ("${ARG_TYPE}" STREQUAL "dir") + # CMake doesn't really seem to like having a directory specified as an + # output; specify the __main__.py file as the output instead. + set(output_file "${EXE_NAME}/__main__.py") + list(APPEND + extra_cmd_params + COMMAND "${CMAKE_COMMAND}" -E remove_directory "${EXE_NAME}" + ) + endif() + endif() + + add_custom_command( + OUTPUT "${output_file}" + ${extra_cmd_params} + COMMAND + "${MAKE_PYTHON_ARCHIVE}" -o "${EXE_NAME}" --main "${ARG_MAIN_MODULE}" + ${make_py_args} + DEPENDS + ${source_files} + "${EXE_NAME}.main_lib.py_sources_built" + "${MAKE_PYTHON_ARCHIVE}" + ) + + # Add an "ALL" target that depends on force ${EXE_NAME}, + # so that ${EXE_NAME} will be included in the default list of build targets. + add_custom_target("${EXE_NAME}.GEN_PY_EXE" ALL DEPENDS "${output_file}") +endfunction() + +# +# Define a python library. +# +# If you want to install a python library generated from this rule note that +# you need to use install_fb_python_library() rather than CMake's built-in +# install() function. This will make it available for other downstream +# projects to use in their add_fb_python_executable() and +# add_fb_python_library() calls. (You do still need to use `install(EXPORT)` +# later to install the CMake exports.) +# +# Parameters: +# - BASE_DIR : +# The base directory path to strip off from each source path. All source +# files must be inside this directory. If not specified it defaults to +# ${CMAKE_CURRENT_SOURCE_DIR}. +# - NAMESPACE : +# The destination namespace where these files should be installed in python +# binaries. If not specified, this defaults to the current relative path of +# ${CMAKE_CURRENT_SOURCE_DIR} inside ${CMAKE_SOURCE_DIR}. e.g., a python +# library defined in the directory repo_root/foo/bar will use a default +# namespace of "foo.bar" +# - SOURCES <...>: +# The python source files. +# - DEPENDS <...>: +# Other python libraries that this one depends on. +# - INSTALL_DIR : +# The directory where this library should be installed. +# install_fb_python_library() must still be called later to perform the +# installation. If a relative path is given it will be treated relative to +# ${CMAKE_INSTALL_PREFIX} +# +# CMake is unfortunately pretty crappy at being able to define custom build +# rules & behaviors. It doesn't support transitive property propagation +# between custom targets; only the built-in add_executable() and add_library() +# targets support transitive properties. +# +# We hack around this janky CMake behavior by (ab)using interface libraries to +# propagate some of the data we want between targets, without actually +# generating a C library. +# +# add_fb_python_library(SOMELIB) generates the following things: +# - An INTERFACE library rule named SOMELIB.py_lib which tracks some +# information about transitive dependencies: +# - the transitive set of source files in the INTERFACE_SOURCES property +# - the transitive set of manifest files that this library depends on in +# the INTERFACE_INCLUDE_DIRECTORIES property. +# - A custom command that generates a SOMELIB.manifest file. +# This file contains the mapping of source files to desired destination +# locations in executables that depend on this library. This manifest file +# will then be read at build-time in order to build executables. +# +function(add_fb_python_library LIB_NAME) + # Parse the arguments + # We use fb_cmake_parse_args() rather than cmake_parse_arguments() since + # cmake_parse_arguments() does not handle empty arguments, and it is common + # for callers to want to specify an empty NAMESPACE parameter. + set(one_value_args BASE_DIR NAMESPACE INSTALL_DIR) + set(multi_value_args SOURCES DEPENDS) + fb_cmake_parse_args( + ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" + ) + fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR) + + string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}") + if (NOT "${namespace_dir}" STREQUAL "") + set(namespace_dir "${namespace_dir}/") + endif() + + if(NOT DEFINED ARG_INSTALL_DIR) + set(install_dir "${FBPY_LIB_INSTALL_DIR}/") + elseif("${ARG_INSTALL_DIR}" STREQUAL "") + set(install_dir "") + else() + set(install_dir "${ARG_INSTALL_DIR}/") + endif() + + # message(STATUS "fb py library ${LIB_NAME}: " + # "NS=${namespace_dir} BASE=${ARG_BASE_DIR}") + + # TODO: In the future it would be nice to support pre-compiling the source + # files. We could emit a rule to compile each source file and emit a + # .pyc/.pyo file here, and then have the manifest reference the pyc/pyo + # files. + + # Define a library target to help pass around information about the library, + # and propagate dependency information. + # + # CMake make a lot of assumptions that libraries are C++ libraries. To help + # avoid confusion we name our target "${LIB_NAME}.py_lib" rather than just + # "${LIB_NAME}". This helps avoid confusion if callers try to use + # "${LIB_NAME}" on their own as a target name. (e.g., attempting to install + # it directly with install(TARGETS) won't work. Callers must use + # install_fb_python_library() instead.) + add_library("${LIB_NAME}.py_lib" INTERFACE) + + # Emit the manifest file. + # + # We write the manifest file to a temporary path first, then copy it with + # configure_file(COPYONLY). This is necessary to get CMake to understand + # that "${manifest_path}" is generated by the CMake configure phase, + # and allow using it as a dependency for add_custom_command(). + # (https://gitlab.kitware.com/cmake/cmake/issues/16367) + set(manifest_path "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}.manifest") + set(tmp_manifest "${manifest_path}.tmp") + file(WRITE "${tmp_manifest}" "FBPY_MANIFEST 1\n") + set(abs_sources) + foreach(src_path IN LISTS ARG_SOURCES) + get_filename_component(abs_source "${src_path}" ABSOLUTE) + list(APPEND abs_sources "${abs_source}") + file(RELATIVE_PATH rel_src "${ARG_BASE_DIR}" "${abs_source}") + target_sources( + "${LIB_NAME}.py_lib" INTERFACE + "$" + "$" + ) + if("${rel_src}" MATCHES "^../") + message( + FATAL_ERROR "${LIB_NAME}: source file \"${abs_source}\" is not inside " + "the base directory ${ARG_BASE_DIR}" + ) + endif() + file( + APPEND "${tmp_manifest}" + "${abs_source} :: ${namespace_dir}${rel_src}\n" + ) + endforeach() + configure_file("${tmp_manifest}" "${manifest_path}" COPYONLY) + + target_include_directories( + "${LIB_NAME}.py_lib" INTERFACE + "$" + "$" + ) + + # Add a target that depends on all of the source files. + # This is needed in case some of the source files are generated. This will + # ensure that these source files are brought up-to-date before we build + # any python binaries that depend on this library. + add_custom_target("${LIB_NAME}.py_sources_built" DEPENDS ${ARG_SOURCES}) + add_dependencies("${LIB_NAME}.py_lib" "${LIB_NAME}.py_sources_built") + + # Hook up library dependencies, and also make the *.py_sources_built target + # depend on the sources for all of our dependencies also being up-to-date. + foreach(dep IN LISTS ARG_DEPENDS) + target_link_libraries("${LIB_NAME}.py_lib" INTERFACE "${dep}.py_lib") + + # Mark that our .py_sources_built target depends on each our our dependent + # libraries. This serves two functions: + # - This causes CMake to generate an error message if one of the + # dependencies is never defined. The target_link_libraries() call above + # won't complain if one of the dependencies doesn't exist (since it is + # intended to allow passing in file names for plain library files rather + # than just targets). + # - It ensures that sources for our depencencies are built before any + # executable that depends on us. Note that we depend on "${dep}.py_lib" + # rather than "${dep}.py_sources_built" for this purpose because the + # ".py_sources_built" target won't be available for imported targets. + add_dependencies("${LIB_NAME}.py_sources_built" "${dep}.py_lib") + endforeach() + + # Add a custom command to help with library installation, in case + # install_fb_python_library() is called later for this library. + # add_custom_command() only works with file dependencies defined in the same + # CMakeLists.txt file, so we want to make sure this is defined here, rather + # then where install_fb_python_library() is called. + # This command won't be run by default, but will only be run if it is needed + # by a subsequent install_fb_python_library() call. + # + # This command copies the library contents into the build directory. + # It would be nicer if we could skip this intermediate copy, and just run + # make_fbpy_archive.py at install time to copy them directly to the desired + # installation directory. Unfortunately this is difficult to do, and seems + # to interfere with some of the CMake code that wants to generate a manifest + # of installed files. + set(build_install_dir "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}.lib_install") + add_custom_command( + OUTPUT + "${build_install_dir}/${LIB_NAME}.manifest" + COMMAND "${CMAKE_COMMAND}" -E remove_directory "${build_install_dir}" + COMMAND + "${MAKE_PYTHON_ARCHIVE}" --type lib-install + --install-dir "${LIB_NAME}" + -o "${build_install_dir}/${LIB_NAME}" "${manifest_path}" + DEPENDS + "${ARG_SOURCES}" + "${manifest_path}" + "${MAKE_PYTHON_ARCHIVE}" + ) + add_custom_target( + "${LIB_NAME}.py_lib_install" + DEPENDS "${build_install_dir}/${LIB_NAME}.manifest" + ) + + # Set some properties to pass through the install paths to + # install_fb_python_library() + # + # Passing through ${build_install_dir} allows install_fb_python_library() + # to work even if used from a different CMakeLists.txt file than where + # add_fb_python_library() was called (i.e. such that + # ${CMAKE_CURRENT_BINARY_DIR} is different between the two calls). + set(abs_install_dir "${install_dir}") + if(NOT IS_ABSOLUTE "${abs_install_dir}") + set(abs_install_dir "${CMAKE_INSTALL_PREFIX}/${abs_install_dir}") + endif() + set_target_properties( + "${LIB_NAME}.py_lib_install" + PROPERTIES + INSTALL_DIR "${abs_install_dir}" + BUILD_INSTALL_DIR "${build_install_dir}" + ) +endfunction() + +# +# Install a python library. +# +# - EXPORT : +# Associate the installed target files with the given export-name. +# +# Note that unlike the built-in CMake install() function we do not accept a +# DESTINATION parameter. Instead, use the INSTALL_DIR parameter to +# add_fb_python_library() to set the installation location. +# +function(install_fb_python_library LIB_NAME) + set(one_value_args EXPORT) + fb_cmake_parse_args(ARG "" "${one_value_args}" "" "${ARGN}") + + # Export our "${LIB_NAME}.py_lib" target so that it will be available to + # downstream projects in our installed CMake config files. + if(DEFINED ARG_EXPORT) + install(TARGETS "${LIB_NAME}.py_lib" EXPORT "${ARG_EXPORT}") + endif() + + # add_fb_python_library() emits a .py_lib_install target that will prepare + # the installation directory. However, it isn't part of the "ALL" target and + # therefore isn't built by default. + # + # Make sure the ALL target depends on it now. We have to do this by + # introducing yet another custom target. + # Add it as a dependency to the ALL target now. + add_custom_target("${LIB_NAME}.py_lib_install_all" ALL) + add_dependencies( + "${LIB_NAME}.py_lib_install_all" "${LIB_NAME}.py_lib_install" + ) + + # Copy the intermediate install directory generated at build time into + # the desired install location. + set(dest_dir "$") + set( + build_install_dir + "$" + ) + install( + DIRECTORY "${build_install_dir}/${LIB_NAME}" + DESTINATION "${dest_dir}" + ) + install( + FILES "${build_install_dir}/${LIB_NAME}.manifest" + DESTINATION "${dest_dir}" + ) +endfunction() + +# Helper macro to process the BASE_DIR and NAMESPACE arguments for +# add_fb_python_executable() and add_fb_python_executable() +macro(fb_py_process_default_args NAMESPACE_VAR BASE_DIR_VAR) + # If the namespace was not specified, default to the relative path to the + # current directory (starting from the repository root). + if(NOT DEFINED "${NAMESPACE_VAR}") + file( + RELATIVE_PATH "${NAMESPACE_VAR}" + "${CMAKE_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" + ) + endif() + + if(NOT DEFINED "${BASE_DIR_VAR}") + # If the base directory was not specified, default to the current directory + set("${BASE_DIR_VAR}" "${CMAKE_CURRENT_SOURCE_DIR}") + else() + # If the base directory was specified, always convert it to an + # absolute path. + get_filename_component("${BASE_DIR_VAR}" "${${BASE_DIR_VAR}}" ABSOLUTE) + endif() +endmacro() diff --git a/build/fbcode_builder/CMake/make_fbpy_archive.py b/build/fbcode_builder/CMake/make_fbpy_archive.py new file mode 100755 index 000000000..3229110ee --- /dev/null +++ b/build/fbcode_builder/CMake/make_fbpy_archive.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +# +# Copyright (c) Facebook, Inc. and its affiliates. +# +import argparse +import collections +import errno +import os +import shutil +import sys +import tempfile +import zipapp + +MANIFEST_SEPARATOR = " :: " +MANIFEST_HEADER_V1 = "FBPY_MANIFEST 1\n" + + +class UsageError(Exception): + def __init__(self, message): + self.message = message + + def __str__(self): + return self.message + + +class BadManifestError(UsageError): + def __init__(self, path, line_num, message): + full_msg = "%s:%s: %s" % (path, line_num, message) + super().__init__(full_msg) + self.path = path + self.line_num = line_num + self.raw_message = message + + +PathInfo = collections.namedtuple( + "PathInfo", ("src", "dest", "manifest_path", "manifest_line") +) + + +def parse_manifest(manifest, path_map): + bad_prefix = ".." + os.path.sep + manifest_dir = os.path.dirname(manifest) + with open(manifest, "r") as f: + line_num = 1 + line = f.readline() + if line != MANIFEST_HEADER_V1: + raise BadManifestError( + manifest, line_num, "Unexpected manifest file header" + ) + + for line in f: + line_num += 1 + if line.startswith("#"): + continue + line = line.rstrip("\n") + parts = line.split(MANIFEST_SEPARATOR) + if len(parts) != 2: + msg = "line must be of the form SRC %s DEST" % MANIFEST_SEPARATOR + raise BadManifestError(manifest, line_num, msg) + src, dest = parts + dest = os.path.normpath(dest) + if dest.startswith(bad_prefix): + msg = "destination path starts with %s: %s" % (bad_prefix, dest) + raise BadManifestError(manifest, line_num, msg) + + if not os.path.isabs(src): + src = os.path.normpath(os.path.join(manifest_dir, src)) + + if dest in path_map: + prev_info = path_map[dest] + msg = ( + "multiple source paths specified for destination " + "path %s. Previous source was %s from %s:%s" + % ( + dest, + prev_info.src, + prev_info.manifest_path, + prev_info.manifest_line, + ) + ) + raise BadManifestError(manifest, line_num, msg) + + info = PathInfo( + src=src, + dest=dest, + manifest_path=manifest, + manifest_line=line_num, + ) + path_map[dest] = info + + +def populate_install_tree(inst_dir, path_map): + os.mkdir(inst_dir) + dest_dirs = {"": False} + + def make_dest_dir(path): + if path in dest_dirs: + return + parent = os.path.dirname(path) + make_dest_dir(parent) + abs_path = os.path.join(inst_dir, path) + os.mkdir(abs_path) + dest_dirs[path] = False + + def install_file(info): + dir_name, base_name = os.path.split(info.dest) + make_dest_dir(dir_name) + if base_name == "__init__.py": + dest_dirs[dir_name] = True + abs_dest = os.path.join(inst_dir, info.dest) + shutil.copy2(info.src, abs_dest) + + # Copy all of the destination files + for info in path_map.values(): + install_file(info) + + # Create __init__ files in any directories that don't have them. + for dir_path, has_init in dest_dirs.items(): + if has_init: + continue + init_path = os.path.join(inst_dir, dir_path, "__init__.py") + with open(init_path, "w"): + pass + + +def build_zipapp(args, path_map): + """ Create a self executing python binary using Python 3's built-in + zipapp module. + + This type of Python binary is relatively simple, as zipapp is part of the + standard library, but it does not support native language extensions + (.so/.dll files). + """ + dest_dir = os.path.dirname(args.output) + with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir: + inst_dir = os.path.join(tmpdir, "tree") + populate_install_tree(inst_dir, path_map) + + tmp_output = os.path.join(tmpdir, "output.exe") + zipapp.create_archive( + inst_dir, target=tmp_output, interpreter=args.python, main=args.main + ) + os.rename(tmp_output, args.output) + + +def create_main_module(args, inst_dir, path_map): + if not args.main: + assert "__main__.py" in path_map + return + + dest_path = os.path.join(inst_dir, "__main__.py") + main_module, main_fn = args.main.split(":") + main_contents = """\ +#!{python} + +if __name__ == "__main__": + import {main_module} + {main_module}.{main_fn}() +""".format( + python=args.python, main_module=main_module, main_fn=main_fn + ) + with open(dest_path, "w") as f: + f.write(main_contents) + os.chmod(dest_path, 0o755) + + +def build_install_dir(args, path_map): + """ Create a directory that contains all of the sources, with a __main__ + module to run the program. + """ + # Populate a temporary directory first, then rename to the destination + # location. This ensures that we don't ever leave a halfway-built + # directory behind at the output path if something goes wrong. + dest_dir = os.path.dirname(args.output) + with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir: + inst_dir = os.path.join(tmpdir, "tree") + populate_install_tree(inst_dir, path_map) + create_main_module(args, inst_dir, path_map) + os.rename(inst_dir, args.output) + + +def ensure_directory(path): + try: + os.makedirs(path) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + + +def install_library(args, path_map): + """ Create an installation directory a python library. """ + out_dir = args.output + out_manifest = args.output + ".manifest" + + install_dir = args.install_dir + if not install_dir: + install_dir = out_dir + + os.makedirs(out_dir) + with open(out_manifest, "w") as manifest: + manifest.write(MANIFEST_HEADER_V1) + for info in path_map.values(): + abs_dest = os.path.join(out_dir, info.dest) + ensure_directory(os.path.dirname(abs_dest)) + print("copy %r --> %r" % (info.src, abs_dest)) + shutil.copy2(info.src, abs_dest) + installed_dest = os.path.join(install_dir, info.dest) + manifest.write("%s%s%s\n" % (installed_dest, MANIFEST_SEPARATOR, info.dest)) + + +def parse_manifests(args): + # Process args.manifest_separator to help support older versions of CMake + if args.manifest_separator: + manifests = [] + for manifest_arg in args.manifests: + split_arg = manifest_arg.split(args.manifest_separator) + manifests.extend(split_arg) + args.manifests = manifests + + path_map = {} + for manifest in args.manifests: + parse_manifest(manifest, path_map) + + return path_map + + +def check_main_module(args, path_map): + # Translate an empty string in the --main argument to None, + # just to allow the CMake logic to be slightly simpler and pass in an + # empty string when it really wants the default __main__.py module to be + # used. + if args.main == "": + args.main = None + + if args.type == "lib-install": + if args.main is not None: + raise UsageError("cannot specify a --main argument with --type=lib-install") + return + + main_info = path_map.get("__main__.py") + if args.main: + if main_info is not None: + msg = ( + "specified an explicit main module with --main, " + "but the file listing already includes __main__.py" + ) + raise BadManifestError( + main_info.manifest_path, main_info.manifest_line, msg + ) + parts = args.main.split(":") + if len(parts) != 2: + raise UsageError( + "argument to --main must be of the form MODULE:CALLABLE " + "(received %s)" % (args.main,) + ) + else: + if main_info is None: + raise UsageError( + "no main module specified with --main, " + "and no __main__.py module present" + ) + + +BUILD_TYPES = { + "zipapp": build_zipapp, + "dir": build_install_dir, + "lib-install": install_library, +} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("-o", "--output", required=True, help="The output file path") + ap.add_argument( + "--install-dir", + help="When used with --type=lib-install, this parameter specifies the " + "final location where the library where be installed. This can be " + "used to generate the library in one directory first, when you plan " + "to move or copy it to another final location later.", + ) + ap.add_argument( + "--manifest-separator", + help="Split manifest arguments around this separator. This is used " + "to support older versions of CMake that cannot supply the manifests " + "as separate arguments.", + ) + ap.add_argument( + "--main", + help="The main module to run, specified as :. " + "This must be specified if and only if the archive does not contain " + "a __main__.py file.", + ) + ap.add_argument( + "--python", + help="Explicitly specify the python interpreter to use for the " "executable.", + ) + ap.add_argument( + "--type", choices=BUILD_TYPES.keys(), help="The type of output to build." + ) + ap.add_argument( + "manifests", + nargs="+", + help="The manifest files specifying how to construct the archive", + ) + args = ap.parse_args() + + if args.python is None: + args.python = sys.executable + + if args.type is None: + # In the future we might want different default output types + # for different platforms. + args.type = "zipapp" + build_fn = BUILD_TYPES[args.type] + + try: + path_map = parse_manifests(args) + check_main_module(args, path_map) + except UsageError as ex: + print("error: %s" % (ex,), file=sys.stderr) + sys.exit(1) + + build_fn(args, path_map) + + +if __name__ == "__main__": + main()