mirror of
https://github.com/facebook/proxygen.git
synced 2025-07-30 23:03:05 +03:00
26616c4c75
Summary: X-link: https://github.com/facebookincubator/fizz/pull/163 The github [windows-2019 actions image was retired by github](https://github.com/actions/runner-images/issues/12045), so all jobs on it fail Update to windows-2022 to get them running again windows-2022 has [different tools and versions than windows-2019](https://github.com/actions/runner-images/issues/3949). Notably it moves from Visual Studio 2019 (aka msvc 16.x) to Visual Studio 2022 (aka msvc 17.x), hence the update to buildopts.py discovery In the course of regenerating the github actions I also fixed a couple of issues that stopped regeneration matching repo contents * a few workflows were using workflow_dispatch, added support * there were a trailing and double spaces for project_prefix, fixed (use ignore whitespace to remove this from review!) Reviewed By: bigfootjon, yfeldblum Differential Revision: D78019509 fbshipit-source-id: f8b0e9438bfc6b481b4207ad82bc1002e496a2d9
1604 lines
59 KiB
Python
Executable File
1604 lines
59 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
#
|
|
# This source code is licensed under the MIT license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
|
|
# We don't import cache.create_cache directly as the facebook
|
|
# specific import below may monkey patch it, and we want to
|
|
# observe the patched version of this function!
|
|
import getdeps.cache as cache_module
|
|
from getdeps.buildopts import setup_build_options
|
|
from getdeps.dyndeps import create_dyn_dep_munger
|
|
from getdeps.errors import TransientFailure
|
|
from getdeps.fetcher import (
|
|
file_name_is_cmake_file,
|
|
list_files_under_dir_newer_than_timestamp,
|
|
SystemPackageFetcher,
|
|
)
|
|
from getdeps.load import ManifestLoader
|
|
from getdeps.manifest import ManifestParser
|
|
from getdeps.platform import HostType
|
|
from getdeps.runcmd import check_cmd
|
|
from getdeps.subcmd import add_subcommands, cmd, SubCmd
|
|
|
|
try:
|
|
import getdeps.facebook # noqa: F401
|
|
except ImportError:
|
|
# we don't ship the facebook specific subdir,
|
|
# so allow that to fail silently
|
|
pass
|
|
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps"))
|
|
|
|
|
|
class UsageError(Exception):
|
|
pass
|
|
|
|
|
|
@cmd("validate-manifest", "parse a manifest and validate that it is correct")
|
|
class ValidateManifest(SubCmd):
|
|
def run(self, args):
|
|
try:
|
|
ManifestParser(file_name=args.file_name)
|
|
print("OK", file=sys.stderr)
|
|
return 0
|
|
except Exception as exc:
|
|
print("ERROR: %s" % str(exc), file=sys.stderr)
|
|
return 1
|
|
|
|
def setup_parser(self, parser):
|
|
parser.add_argument("file_name", help="path to the manifest file")
|
|
|
|
|
|
@cmd("show-host-type", "outputs the host type tuple for the host machine")
|
|
class ShowHostType(SubCmd):
|
|
def run(self, args):
|
|
host = HostType()
|
|
print("%s" % host.as_tuple_string())
|
|
return 0
|
|
|
|
|
|
class ProjectCmdBase(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
|
|
if args.current_project is not None:
|
|
opts.repo_project = args.current_project
|
|
if args.project is None:
|
|
if opts.repo_project is None:
|
|
raise UsageError(
|
|
"no project name specified, and no .projectid file found"
|
|
)
|
|
if opts.repo_project == "fbsource":
|
|
# The fbsource repository is a little special. There is no project
|
|
# manifest file for it. A specific project must always be explicitly
|
|
# specified when building from fbsource.
|
|
raise UsageError(
|
|
"no project name specified (required when building in fbsource)"
|
|
)
|
|
args.project = opts.repo_project
|
|
|
|
ctx_gen = opts.get_context_generator()
|
|
if args.test_dependencies:
|
|
ctx_gen.set_value_for_all_projects("test", "on")
|
|
if args.enable_tests:
|
|
ctx_gen.set_value_for_project(args.project, "test", "on")
|
|
else:
|
|
ctx_gen.set_value_for_project(args.project, "test", "off")
|
|
|
|
if opts.shared_libs:
|
|
ctx_gen.set_value_for_all_projects("shared_libs", "on")
|
|
|
|
loader = ManifestLoader(opts, ctx_gen)
|
|
self.process_project_dir_arguments(args, loader)
|
|
|
|
manifest = loader.load_manifest(args.project)
|
|
|
|
self.run_project_cmd(args, loader, manifest)
|
|
|
|
def process_project_dir_arguments(self, args, loader):
|
|
def parse_project_arg(arg, arg_type):
|
|
parts = arg.split(":")
|
|
if len(parts) == 2:
|
|
project, path = parts
|
|
elif len(parts) == 1:
|
|
project = args.project
|
|
path = parts[0]
|
|
# On Windows path contains colon, e.g. C:\open
|
|
elif os.name == "nt" and len(parts) == 3:
|
|
project = parts[0]
|
|
path = parts[1] + ":" + parts[2]
|
|
else:
|
|
raise UsageError(
|
|
"invalid %s argument; too many ':' characters: %s" % (arg_type, arg)
|
|
)
|
|
|
|
return project, os.path.abspath(path)
|
|
|
|
# If we are currently running from a project repository,
|
|
# use the current repository for the project sources.
|
|
build_opts = loader.build_opts
|
|
if build_opts.repo_project is not None and build_opts.repo_root is not None:
|
|
loader.set_project_src_dir(build_opts.repo_project, build_opts.repo_root)
|
|
|
|
for arg in args.src_dir:
|
|
project, path = parse_project_arg(arg, "--src-dir")
|
|
loader.set_project_src_dir(project, path)
|
|
|
|
for arg in args.build_dir:
|
|
project, path = parse_project_arg(arg, "--build-dir")
|
|
loader.set_project_build_dir(project, path)
|
|
|
|
for arg in args.install_dir:
|
|
project, path = parse_project_arg(arg, "--install-dir")
|
|
loader.set_project_install_dir(project, path)
|
|
|
|
for arg in args.project_install_prefix:
|
|
project, path = parse_project_arg(arg, "--install-prefix")
|
|
loader.set_project_install_prefix(project, path)
|
|
|
|
def setup_parser(self, parser):
|
|
parser.add_argument(
|
|
"project",
|
|
nargs="?",
|
|
help=(
|
|
"name of the project or path to a manifest "
|
|
"file describing the project"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--no-tests",
|
|
action="store_false",
|
|
dest="enable_tests",
|
|
default=True,
|
|
help="Disable building tests for this project.",
|
|
)
|
|
parser.add_argument(
|
|
"--test-dependencies",
|
|
action="store_true",
|
|
help="Enable building tests for dependencies as well.",
|
|
)
|
|
parser.add_argument(
|
|
"--current-project",
|
|
help="Specify the name of the fbcode_builder manifest file for the "
|
|
"current repository. If not specified, the code will attempt to find "
|
|
"this in a .projectid file in the repository root.",
|
|
)
|
|
parser.add_argument(
|
|
"--src-dir",
|
|
default=[],
|
|
action="append",
|
|
help="Specify a local directory to use for the project source, "
|
|
"rather than fetching it.",
|
|
)
|
|
parser.add_argument(
|
|
"--build-dir",
|
|
default=[],
|
|
action="append",
|
|
help="Explicitly specify the build directory to use for the "
|
|
"project, instead of the default location in the scratch path. "
|
|
"This only affects the project specified, and not its dependencies.",
|
|
)
|
|
parser.add_argument(
|
|
"--install-dir",
|
|
default=[],
|
|
action="append",
|
|
help="Explicitly specify the install directory to use for the "
|
|
"project, instead of the default location in the scratch path. "
|
|
"This only affects the project specified, and not its dependencies.",
|
|
)
|
|
parser.add_argument(
|
|
"--project-install-prefix",
|
|
default=[],
|
|
action="append",
|
|
help="Specify the final deployment installation path for a project",
|
|
)
|
|
|
|
self.setup_project_cmd_parser(parser)
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
pass
|
|
|
|
def create_builder(self, loader, manifest):
|
|
fetcher = loader.create_fetcher(manifest)
|
|
src_dir = fetcher.get_src_dir()
|
|
ctx = loader.ctx_gen.get_context(manifest.name)
|
|
build_dir = loader.get_project_build_dir(manifest)
|
|
inst_dir = loader.get_project_install_dir(manifest)
|
|
return manifest.create_builder(
|
|
loader.build_opts,
|
|
src_dir,
|
|
build_dir,
|
|
inst_dir,
|
|
ctx,
|
|
loader,
|
|
loader.dependencies_of(manifest),
|
|
)
|
|
|
|
def check_built(self, loader, manifest):
|
|
built_marker = os.path.join(
|
|
loader.get_project_install_dir(manifest), ".built-by-getdeps"
|
|
)
|
|
return os.path.exists(built_marker)
|
|
|
|
|
|
class CachedProject(object):
|
|
"""A helper that allows calling the cache logic for a project
|
|
from both the build and the fetch code"""
|
|
|
|
def __init__(self, cache, loader, m):
|
|
self.m = m
|
|
self.inst_dir = loader.get_project_install_dir(m)
|
|
self.project_hash = loader.get_project_hash(m)
|
|
self.ctx = loader.ctx_gen.get_context(m.name)
|
|
self.loader = loader
|
|
self.cache = cache
|
|
|
|
self.cache_key = "-".join(
|
|
(
|
|
m.name,
|
|
self.ctx.get("os"),
|
|
self.ctx.get("distro") or "none",
|
|
self.ctx.get("distro_vers") or "none",
|
|
self.project_hash,
|
|
)
|
|
)
|
|
self.cache_file_name = self.cache_key + "-buildcache.tgz"
|
|
|
|
def is_cacheable(self):
|
|
"""We only cache third party projects"""
|
|
return self.cache and self.m.shipit_project is None
|
|
|
|
def was_cached(self):
|
|
cached_marker = os.path.join(self.inst_dir, ".getdeps-cached-build")
|
|
return os.path.exists(cached_marker)
|
|
|
|
def download(self):
|
|
if self.is_cacheable() and not os.path.exists(self.inst_dir):
|
|
print("check cache for %s" % self.cache_file_name)
|
|
dl_dir = os.path.join(self.loader.build_opts.scratch_dir, "downloads")
|
|
if not os.path.exists(dl_dir):
|
|
os.makedirs(dl_dir)
|
|
try:
|
|
target_file_name = os.path.join(dl_dir, self.cache_file_name)
|
|
if self.cache.download_to_file(self.cache_file_name, target_file_name):
|
|
tf = tarfile.open(target_file_name, "r")
|
|
print(
|
|
"Extracting %s -> %s..." % (self.cache_file_name, self.inst_dir)
|
|
)
|
|
tf.extractall(self.inst_dir)
|
|
|
|
cached_marker = os.path.join(self.inst_dir, ".getdeps-cached-build")
|
|
with open(cached_marker, "w") as f:
|
|
f.write("\n")
|
|
|
|
return True
|
|
except Exception as exc:
|
|
print("%s" % str(exc))
|
|
|
|
return False
|
|
|
|
def upload(self):
|
|
if self.is_cacheable():
|
|
# We can prepare an archive and stick it in LFS
|
|
tempdir = tempfile.mkdtemp()
|
|
tarfilename = os.path.join(tempdir, self.cache_file_name)
|
|
print("Archiving for cache: %s..." % tarfilename)
|
|
tf = tarfile.open(tarfilename, "w:gz")
|
|
tf.add(self.inst_dir, arcname=".")
|
|
tf.close()
|
|
try:
|
|
self.cache.upload_from_file(self.cache_file_name, tarfilename)
|
|
except Exception as exc:
|
|
print(
|
|
"Failed to upload to cache (%s), continue anyway" % str(exc),
|
|
file=sys.stderr,
|
|
)
|
|
shutil.rmtree(tempdir)
|
|
|
|
|
|
@cmd("fetch", "fetch the code for a given project")
|
|
class FetchCmd(ProjectCmdBase):
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="fetch the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
parser.add_argument(
|
|
"--host-type",
|
|
help=(
|
|
"When recursively fetching, fetch deps for "
|
|
"this host type rather than the current system"
|
|
),
|
|
)
|
|
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.recursive:
|
|
projects = loader.manifests_in_dependency_order()
|
|
else:
|
|
projects = [manifest]
|
|
|
|
cache = cache_module.create_cache()
|
|
for m in projects:
|
|
fetcher = loader.create_fetcher(m)
|
|
if isinstance(fetcher, SystemPackageFetcher):
|
|
# We are guaranteed that if the fetcher is set to
|
|
# SystemPackageFetcher then this item is completely
|
|
# satisfied by the appropriate system packages
|
|
continue
|
|
cached_project = CachedProject(cache, loader, m)
|
|
if cached_project.download():
|
|
continue
|
|
|
|
inst_dir = loader.get_project_install_dir(m)
|
|
built_marker = os.path.join(inst_dir, ".built-by-getdeps")
|
|
if os.path.exists(built_marker):
|
|
with open(built_marker, "r") as f:
|
|
built_hash = f.read().strip()
|
|
|
|
project_hash = loader.get_project_hash(m)
|
|
if built_hash == project_hash:
|
|
continue
|
|
|
|
# We need to fetch the sources
|
|
fetcher.update()
|
|
|
|
|
|
@cmd("install-system-deps", "Install system packages to satisfy the deps for a project")
|
|
class InstallSysDepsCmd(ProjectCmdBase):
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="install the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
action="store_true",
|
|
default=False,
|
|
help="Don't install, just print the commands specs we would run",
|
|
)
|
|
parser.add_argument(
|
|
"--os-type",
|
|
help="Filter to just this OS type to run",
|
|
choices=["linux", "darwin", "windows", "pacman-package"],
|
|
action="store",
|
|
dest="ostype",
|
|
default=None,
|
|
)
|
|
parser.add_argument(
|
|
"--distro",
|
|
help="Filter to just this distro to run",
|
|
choices=["ubuntu", "centos_stream"],
|
|
action="store",
|
|
dest="distro",
|
|
default=None,
|
|
)
|
|
parser.add_argument(
|
|
"--distro-version",
|
|
help="Filter to just this distro version",
|
|
action="store",
|
|
dest="distrovers",
|
|
default=None,
|
|
)
|
|
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.recursive:
|
|
projects = loader.manifests_in_dependency_order()
|
|
else:
|
|
projects = [manifest]
|
|
|
|
rebuild_ctx_gen = False
|
|
if args.ostype:
|
|
loader.build_opts.host_type.ostype = args.ostype
|
|
loader.build_opts.host_type.distro = None
|
|
loader.build_opts.host_type.distrovers = None
|
|
rebuild_ctx_gen = True
|
|
|
|
if args.distro:
|
|
loader.build_opts.host_type.distro = args.distro
|
|
loader.build_opts.host_type.distrovers = None
|
|
rebuild_ctx_gen = True
|
|
|
|
if args.distrovers:
|
|
loader.build_opts.host_type.distrovers = args.distrovers
|
|
rebuild_ctx_gen = True
|
|
|
|
if rebuild_ctx_gen:
|
|
loader.ctx_gen = loader.build_opts.get_context_generator()
|
|
|
|
manager = loader.build_opts.host_type.get_package_manager()
|
|
|
|
all_packages = {}
|
|
for m in projects:
|
|
ctx = loader.ctx_gen.get_context(m.name)
|
|
packages = m.get_required_system_packages(ctx)
|
|
for k, v in packages.items():
|
|
merged = all_packages.get(k, [])
|
|
merged += v
|
|
all_packages[k] = merged
|
|
|
|
cmd_argss = []
|
|
if manager == "rpm":
|
|
packages = sorted(set(all_packages["rpm"]))
|
|
if packages:
|
|
cmd_argss.append(
|
|
["sudo", "dnf", "install", "-y", "--skip-broken"] + packages
|
|
)
|
|
elif manager == "deb":
|
|
packages = sorted(set(all_packages["deb"]))
|
|
if packages:
|
|
cmd_argss.append(
|
|
[
|
|
"sudo",
|
|
"--preserve-env=http_proxy",
|
|
"apt-get",
|
|
"install",
|
|
"-y",
|
|
]
|
|
+ packages
|
|
)
|
|
cmd_argss.append(["pip", "install", "pex"])
|
|
elif manager == "homebrew":
|
|
packages = sorted(set(all_packages["homebrew"]))
|
|
if packages:
|
|
cmd_argss.append(["brew", "install"] + packages)
|
|
elif manager == "pacman-package":
|
|
packages = sorted(list(set(all_packages["pacman-package"])))
|
|
if packages:
|
|
cmd_argss.append(["pacman", "-S"] + packages)
|
|
else:
|
|
host_tuple = loader.build_opts.host_type.as_tuple_string()
|
|
print(
|
|
f"I don't know how to install any packages on this system {host_tuple}"
|
|
)
|
|
return
|
|
|
|
for cmd_args in cmd_argss:
|
|
if args.dry_run:
|
|
print(" ".join(cmd_args))
|
|
else:
|
|
check_cmd(cmd_args)
|
|
else:
|
|
print("no packages to install")
|
|
|
|
|
|
@cmd("list-deps", "lists the transitive deps for a given project")
|
|
class ListDepsCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
for m in loader.manifests_in_dependency_order():
|
|
print(m.name)
|
|
return 0
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--host-type",
|
|
help=(
|
|
"Produce the list for the specified host type, "
|
|
"rather than that of the current system"
|
|
),
|
|
)
|
|
|
|
|
|
def clean_dirs(opts):
|
|
for d in ["build", "installed", "extracted", "shipit"]:
|
|
d = os.path.join(opts.scratch_dir, d)
|
|
print("Cleaning %s..." % d)
|
|
if os.path.exists(d):
|
|
shutil.rmtree(d)
|
|
|
|
|
|
@cmd("clean", "clean up the scratch dir")
|
|
class CleanCmd(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
clean_dirs(opts)
|
|
|
|
|
|
@cmd("show-scratch-dir", "show the scratch dir")
|
|
class ShowScratchDirCmd(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
print(opts.scratch_dir)
|
|
|
|
|
|
@cmd("show-build-dir", "print the build dir for a given project")
|
|
class ShowBuildDirCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.recursive:
|
|
manifests = loader.manifests_in_dependency_order()
|
|
else:
|
|
manifests = [manifest]
|
|
|
|
for m in manifests:
|
|
inst_dir = loader.get_project_build_dir(m)
|
|
print(inst_dir)
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="print the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
|
|
|
|
@cmd("show-inst-dir", "print the installation dir for a given project")
|
|
class ShowInstDirCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.recursive:
|
|
manifests = loader.manifests_in_dependency_order()
|
|
else:
|
|
manifests = [manifest]
|
|
|
|
for m in manifests:
|
|
fetcher = loader.create_fetcher(m)
|
|
if isinstance(fetcher, SystemPackageFetcher):
|
|
# We are guaranteed that if the fetcher is set to
|
|
# SystemPackageFetcher then this item is completely
|
|
# satisfied by the appropriate system packages
|
|
continue
|
|
inst_dir = loader.get_project_install_dir_respecting_install_prefix(m)
|
|
print(inst_dir)
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="print the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
|
|
|
|
@cmd("query-paths", "print the paths for tooling to use")
|
|
class QueryPathsCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.recursive:
|
|
manifests = loader.manifests_in_dependency_order()
|
|
else:
|
|
manifests = [manifest]
|
|
|
|
cache = cache_module.create_cache()
|
|
for m in manifests:
|
|
fetcher = loader.create_fetcher(m)
|
|
if isinstance(fetcher, SystemPackageFetcher):
|
|
# We are guaranteed that if the fetcher is set to
|
|
# SystemPackageFetcher then this item is completely
|
|
# satisfied by the appropriate system packages
|
|
continue
|
|
src_dir = fetcher.get_src_dir()
|
|
print(f"{m.name}_SOURCE={src_dir}")
|
|
inst_dir = loader.get_project_install_dir_respecting_install_prefix(m)
|
|
print(f"{m.name}_INSTALL={inst_dir}")
|
|
cached_project = CachedProject(cache, loader, m)
|
|
print(f"{m.name}_CACHE_KEY={cached_project.cache_key}")
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="print the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
|
|
|
|
@cmd("show-source-dir", "print the source dir for a given project")
|
|
class ShowSourceDirCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.recursive:
|
|
manifests = loader.manifests_in_dependency_order()
|
|
else:
|
|
manifests = [manifest]
|
|
|
|
for m in manifests:
|
|
fetcher = loader.create_fetcher(m)
|
|
print(fetcher.get_src_dir())
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="print the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
|
|
|
|
@cmd("build", "build a given project")
|
|
class BuildCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.clean:
|
|
clean_dirs(loader.build_opts)
|
|
|
|
print("Building on %s" % loader.ctx_gen.get_context(args.project))
|
|
projects = loader.manifests_in_dependency_order()
|
|
|
|
cache = cache_module.create_cache() if args.use_build_cache else None
|
|
|
|
dep_manifests = []
|
|
|
|
for m in projects:
|
|
dep_manifests.append(m)
|
|
|
|
fetcher = loader.create_fetcher(m)
|
|
|
|
if args.build_skip_lfs_download and hasattr(fetcher, "skip_lfs_download"):
|
|
print("skipping lfs download for %s" % m.name)
|
|
fetcher.skip_lfs_download()
|
|
|
|
if isinstance(fetcher, SystemPackageFetcher):
|
|
# We are guaranteed that if the fetcher is set to
|
|
# SystemPackageFetcher then this item is completely
|
|
# satisfied by the appropriate system packages
|
|
continue
|
|
|
|
if args.clean:
|
|
fetcher.clean()
|
|
|
|
build_dir = loader.get_project_build_dir(m)
|
|
inst_dir = loader.get_project_install_dir(m)
|
|
|
|
if (
|
|
m == manifest
|
|
and not args.only_deps
|
|
or m != manifest
|
|
and not args.no_deps
|
|
):
|
|
print("Assessing %s..." % m.name)
|
|
project_hash = loader.get_project_hash(m)
|
|
ctx = loader.ctx_gen.get_context(m.name)
|
|
built_marker = os.path.join(inst_dir, ".built-by-getdeps")
|
|
|
|
cached_project = CachedProject(cache, loader, m)
|
|
|
|
reconfigure, sources_changed = self.compute_source_change_status(
|
|
cached_project, fetcher, m, built_marker, project_hash
|
|
)
|
|
|
|
if os.path.exists(built_marker) and not cached_project.was_cached():
|
|
# We've previously built this. We may need to reconfigure if
|
|
# our deps have changed, so let's check them.
|
|
dep_reconfigure, dep_build = self.compute_dep_change_status(
|
|
m, built_marker, loader
|
|
)
|
|
if dep_reconfigure:
|
|
reconfigure = True
|
|
if dep_build:
|
|
sources_changed = True
|
|
|
|
extra_cmake_defines = (
|
|
json.loads(args.extra_cmake_defines)
|
|
if args.extra_cmake_defines
|
|
else {}
|
|
)
|
|
|
|
extra_b2_args = args.extra_b2_args or []
|
|
|
|
if sources_changed or reconfigure or not os.path.exists(built_marker):
|
|
if os.path.exists(built_marker):
|
|
os.unlink(built_marker)
|
|
src_dir = fetcher.get_src_dir()
|
|
# Prepare builders write out config before the main builder runs
|
|
prepare_builders = m.create_prepare_builders(
|
|
loader.build_opts,
|
|
ctx,
|
|
src_dir,
|
|
build_dir,
|
|
inst_dir,
|
|
loader,
|
|
dep_manifests,
|
|
)
|
|
for preparer in prepare_builders:
|
|
preparer.prepare(reconfigure=reconfigure)
|
|
|
|
builder = m.create_builder(
|
|
loader.build_opts,
|
|
src_dir,
|
|
build_dir,
|
|
inst_dir,
|
|
ctx,
|
|
loader,
|
|
dep_manifests,
|
|
final_install_prefix=loader.get_project_install_prefix(m),
|
|
extra_cmake_defines=extra_cmake_defines,
|
|
cmake_target=args.cmake_target if m == manifest else "install",
|
|
extra_b2_args=extra_b2_args,
|
|
)
|
|
builder.build(reconfigure=reconfigure)
|
|
|
|
# If we are building the project (not dependency) and a specific
|
|
# cmake_target (not 'install') has been requested, then we don't
|
|
# set the built_marker. This allows subsequent runs of getdeps.py
|
|
# for the project to run with different cmake_targets to trigger
|
|
# cmake
|
|
has_built_marker = False
|
|
if not (m == manifest and args.cmake_target != "install"):
|
|
with open(built_marker, "w") as f:
|
|
f.write(project_hash)
|
|
has_built_marker = True
|
|
|
|
# Only populate the cache from continuous build runs, and
|
|
# only if we have a built_marker.
|
|
if (
|
|
not args.skip_upload
|
|
and args.schedule_type == "continuous"
|
|
and has_built_marker
|
|
):
|
|
cached_project.upload()
|
|
elif args.verbose:
|
|
print("found good %s" % built_marker)
|
|
|
|
def compute_dep_change_status(self, m, built_marker, loader):
|
|
reconfigure = False
|
|
sources_changed = False
|
|
st = os.lstat(built_marker)
|
|
|
|
ctx = loader.ctx_gen.get_context(m.name)
|
|
dep_list = m.get_dependencies(ctx)
|
|
for dep in dep_list:
|
|
if reconfigure and sources_changed:
|
|
break
|
|
|
|
dep_manifest = loader.load_manifest(dep)
|
|
dep_root = loader.get_project_install_dir(dep_manifest)
|
|
for dep_file in list_files_under_dir_newer_than_timestamp(
|
|
dep_root, st.st_mtime
|
|
):
|
|
if os.path.basename(dep_file) == ".built-by-getdeps":
|
|
continue
|
|
if file_name_is_cmake_file(dep_file):
|
|
if not reconfigure:
|
|
reconfigure = True
|
|
print(
|
|
f"Will reconfigure cmake because {dep_file} is newer than {built_marker}"
|
|
)
|
|
else:
|
|
if not sources_changed:
|
|
sources_changed = True
|
|
print(
|
|
f"Will run build because {dep_file} is newer than {built_marker}"
|
|
)
|
|
|
|
if reconfigure and sources_changed:
|
|
break
|
|
|
|
return reconfigure, sources_changed
|
|
|
|
def compute_source_change_status(
|
|
self, cached_project, fetcher, m, built_marker, project_hash
|
|
):
|
|
reconfigure = False
|
|
sources_changed = False
|
|
if cached_project.download():
|
|
if not os.path.exists(built_marker):
|
|
fetcher.update()
|
|
else:
|
|
check_fetcher = True
|
|
if os.path.exists(built_marker):
|
|
check_fetcher = False
|
|
with open(built_marker, "r") as f:
|
|
built_hash = f.read().strip()
|
|
if built_hash == project_hash:
|
|
if cached_project.is_cacheable():
|
|
# We can blindly trust the build status
|
|
reconfigure = False
|
|
sources_changed = False
|
|
else:
|
|
# Otherwise, we may have changed the source, so let's
|
|
# check in with the fetcher layer
|
|
check_fetcher = True
|
|
else:
|
|
# Some kind of inconsistency with a prior build,
|
|
# let's run it again to be sure
|
|
os.unlink(built_marker)
|
|
reconfigure = True
|
|
sources_changed = True
|
|
# While we don't need to consult the fetcher for the
|
|
# status in this case, we may still need to have eg: shipit
|
|
# run in order to have a correct source tree.
|
|
fetcher.update()
|
|
|
|
if check_fetcher:
|
|
change_status = fetcher.update()
|
|
reconfigure = change_status.build_changed()
|
|
sources_changed = change_status.sources_changed()
|
|
|
|
return reconfigure, sources_changed
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--clean",
|
|
action="store_true",
|
|
default=False,
|
|
help=(
|
|
"Clean up the build and installation area prior to building, "
|
|
"causing the projects to be built from scratch"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--no-deps",
|
|
action="store_true",
|
|
default=False,
|
|
help=(
|
|
"Only build the named project, not its deps. "
|
|
"This is most useful after you've built all of the deps, "
|
|
"and helps to avoid waiting for relatively "
|
|
"slow up-to-date-ness checks"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--only-deps",
|
|
action="store_true",
|
|
default=False,
|
|
help=(
|
|
"Only build the named project's deps. "
|
|
"This is most useful when you want to separate out building "
|
|
"of all of the deps and your project"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--no-build-cache",
|
|
action="store_false",
|
|
default=True,
|
|
dest="use_build_cache",
|
|
help="Do not attempt to use the build cache.",
|
|
)
|
|
parser.add_argument(
|
|
"--schedule-type", help="Indicates how the build was activated"
|
|
)
|
|
parser.add_argument(
|
|
"--cmake-target",
|
|
help=("Target for cmake build."),
|
|
default="install",
|
|
)
|
|
parser.add_argument(
|
|
"--extra-b2-args",
|
|
help=(
|
|
"Repeatable argument that contains extra arguments to pass "
|
|
"to b2, which compiles boost. "
|
|
"e.g.: 'cxxflags=-fPIC' 'cflags=-fPIC'"
|
|
),
|
|
action="append",
|
|
)
|
|
parser.add_argument(
|
|
"--free-up-disk",
|
|
help="Remove unused tools and clean up intermediate files if possible to maximise space for the build",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
parser.add_argument(
|
|
"--build-type",
|
|
help="Set the build type explicitly. Cmake and cargo builders act on them. Only Debug and RelWithDebInfo widely supported.",
|
|
choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"],
|
|
action="store",
|
|
default=None,
|
|
)
|
|
|
|
|
|
@cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes")
|
|
class FixupDeps(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
projects = loader.manifests_in_dependency_order()
|
|
|
|
# Accumulate the install directories so that the build steps
|
|
# can find their dep installation
|
|
install_dirs = []
|
|
dep_manifests = []
|
|
|
|
for m in projects:
|
|
inst_dir = loader.get_project_install_dir_respecting_install_prefix(m)
|
|
install_dirs.append(inst_dir)
|
|
dep_manifests.append(m)
|
|
|
|
if m == manifest:
|
|
ctx = loader.ctx_gen.get_context(m.name)
|
|
env = loader.build_opts.compute_env_for_install_dirs(
|
|
loader, dep_manifests, ctx
|
|
)
|
|
dep_munger = create_dyn_dep_munger(
|
|
loader.build_opts, env, install_dirs, args.strip
|
|
)
|
|
if dep_munger is None:
|
|
print(f"dynamic dependency fixups not supported on {sys.platform}")
|
|
else:
|
|
dep_munger.process_deps(args.destdir, args.final_install_prefix)
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument("destdir", help="Where to copy the fixed up executables")
|
|
parser.add_argument(
|
|
"--final-install-prefix", help="specify the final installation prefix"
|
|
)
|
|
parser.add_argument(
|
|
"--strip",
|
|
action="store_true",
|
|
default=False,
|
|
help="Strip debug info while processing executables",
|
|
)
|
|
|
|
|
|
@cmd("test", "test a given project")
|
|
class TestCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if not self.check_built(loader, manifest):
|
|
print("project %s has not been built" % manifest.name)
|
|
return 1
|
|
self.create_builder(loader, manifest).run_tests(
|
|
schedule_type=args.schedule_type,
|
|
owner=args.test_owner,
|
|
test_filter=args.filter,
|
|
retry=args.retry,
|
|
no_testpilot=args.no_testpilot,
|
|
)
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--schedule-type", help="Indicates how the build was activated"
|
|
)
|
|
parser.add_argument("--test-owner", help="Owner for testpilot")
|
|
parser.add_argument("--filter", help="Only run the tests matching the regex")
|
|
parser.add_argument(
|
|
"--retry",
|
|
type=int,
|
|
default=3,
|
|
help="Number of immediate retries for failed tests "
|
|
"(noop in continuous and testwarden runs)",
|
|
)
|
|
parser.add_argument(
|
|
"--no-testpilot",
|
|
help="Do not use Test Pilot even when available",
|
|
action="store_true",
|
|
)
|
|
|
|
|
|
@cmd(
|
|
"debug",
|
|
"start a shell in the given project's build dir with the correct environment for running the build",
|
|
)
|
|
class DebugCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
self.create_builder(loader, manifest).debug(reconfigure=False)
|
|
|
|
|
|
@cmd(
|
|
"env",
|
|
"print the environment in a shell sourceable format",
|
|
)
|
|
class EnvCmd(ProjectCmdBase):
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--os-type",
|
|
help="Filter to just this OS type to run",
|
|
choices=["linux", "darwin", "windows"],
|
|
action="store",
|
|
dest="ostype",
|
|
default=None,
|
|
)
|
|
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.ostype:
|
|
loader.build_opts.host_type.ostype = args.ostype
|
|
self.create_builder(loader, manifest).printenv(reconfigure=False)
|
|
|
|
|
|
@cmd("generate-github-actions", "generate a GitHub actions configuration")
|
|
class GenerateGitHubActionsCmd(ProjectCmdBase):
|
|
RUN_ON_ALL = """ [push, pull_request]"""
|
|
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
platforms = [
|
|
HostType("linux", "ubuntu", "22"),
|
|
HostType("darwin", None, None),
|
|
HostType("windows", None, None),
|
|
]
|
|
|
|
for p in platforms:
|
|
if args.os_types and p.ostype not in args.os_types:
|
|
continue
|
|
self.write_job_for_platform(p, args)
|
|
|
|
def get_run_on(self, args):
|
|
if args.run_on_all_branches:
|
|
return self.RUN_ON_ALL
|
|
if args.cron:
|
|
if args.cron == "never":
|
|
return " {}"
|
|
elif args.cron == "workflow_dispatch":
|
|
return "\n workflow_dispatch"
|
|
else:
|
|
return f"""
|
|
schedule:
|
|
- cron: '{args.cron}'"""
|
|
|
|
return f"""
|
|
push:
|
|
branches:
|
|
- {args.main_branch}
|
|
pull_request:
|
|
branches:
|
|
- {args.main_branch}"""
|
|
|
|
# TODO: Break up complex function
|
|
def write_job_for_platform(self, platform, args): # noqa: C901
|
|
build_opts = setup_build_options(args, platform)
|
|
ctx_gen = build_opts.get_context_generator()
|
|
if args.enable_tests:
|
|
ctx_gen.set_value_for_project(args.project, "test", "on")
|
|
else:
|
|
ctx_gen.set_value_for_project(args.project, "test", "off")
|
|
loader = ManifestLoader(build_opts, ctx_gen)
|
|
self.process_project_dir_arguments(args, loader)
|
|
manifest = loader.load_manifest(args.project)
|
|
manifest_ctx = loader.ctx_gen.get_context(manifest.name)
|
|
run_tests = (
|
|
args.enable_tests
|
|
and manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off"
|
|
)
|
|
if run_tests:
|
|
manifest_ctx.set("test", "on")
|
|
run_on = self.get_run_on(args)
|
|
|
|
tests_arg = "--no-tests "
|
|
if run_tests:
|
|
tests_arg = ""
|
|
|
|
# Some projects don't do anything "useful" as a leaf project, only
|
|
# as a dep for a leaf project. Check for those here; we don't want
|
|
# to waste the effort scheduling them on CI.
|
|
# We do this by looking at the builder type in the manifest file
|
|
# rather than creating a builder and checking its type because we
|
|
# don't know enough to create the full builder instance here.
|
|
builder_name = manifest.get("build", "builder", ctx=manifest_ctx)
|
|
if builder_name == "nop":
|
|
return None
|
|
|
|
# We want to be sure that we're running things with python 3
|
|
# but python versioning is honestly a bit of a frustrating mess.
|
|
# `python` may be version 2 or version 3 depending on the system.
|
|
# python3 may not be a thing at all!
|
|
# Assume an optimistic default
|
|
py3 = "python3"
|
|
|
|
if build_opts.is_linux():
|
|
artifacts = "linux"
|
|
if args.runs_on:
|
|
runs_on = args.runs_on
|
|
else:
|
|
runs_on = f"ubuntu-{args.ubuntu_version}"
|
|
if args.cpu_cores:
|
|
runs_on = f"{args.cpu_cores}-core-ubuntu-{args.ubuntu_version}"
|
|
elif build_opts.is_windows():
|
|
artifacts = "windows"
|
|
if args.runs_on:
|
|
runs_on = args.runs_on
|
|
else:
|
|
runs_on = "windows-2022"
|
|
# The windows runners are python 3 by default; python2.exe
|
|
# is available if needed.
|
|
py3 = "python"
|
|
else:
|
|
artifacts = "mac"
|
|
if args.runs_on:
|
|
runs_on = args.runs_on
|
|
else:
|
|
runs_on = "macOS-latest"
|
|
|
|
os.makedirs(args.output_dir, exist_ok=True)
|
|
|
|
job_file_prefix = "getdeps_"
|
|
if args.job_file_prefix:
|
|
job_file_prefix = args.job_file_prefix
|
|
|
|
output_file = os.path.join(args.output_dir, f"{job_file_prefix}{artifacts}.yml")
|
|
|
|
if args.job_name_prefix:
|
|
job_name = args.job_name_prefix + artifacts.capitalize()
|
|
else:
|
|
job_name = artifacts
|
|
|
|
with open(output_file, "w") as out:
|
|
# Deliberate line break here because the @ and the generated
|
|
# symbols are meaningful to our internal tooling when they
|
|
# appear in a single token
|
|
out.write("# This file was @")
|
|
out.write("generated by getdeps.py\n")
|
|
out.write(
|
|
f"""
|
|
name: {job_name}
|
|
|
|
on:{run_on}
|
|
|
|
permissions:
|
|
contents: read # to fetch code (actions/checkout)
|
|
|
|
jobs:
|
|
"""
|
|
)
|
|
|
|
getdepscmd = f"{py3} build/fbcode_builder/getdeps.py"
|
|
|
|
out.write(" build:\n")
|
|
out.write(" runs-on: %s\n" % runs_on)
|
|
out.write(" steps:\n")
|
|
|
|
if build_opts.is_windows():
|
|
# cmake relies on BOOST_ROOT but GH deliberately don't set it in order
|
|
# to avoid versioning issues:
|
|
# https://github.com/actions/virtual-environments/issues/319
|
|
# Instead, set the version we think we need; this is effectively
|
|
# coupled with the boost manifest
|
|
# This is the unusual syntax for setting an env var for the rest of
|
|
# the steps in a workflow:
|
|
# https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/
|
|
out.write(" - name: Export boost environment\n")
|
|
out.write(
|
|
' run: "echo BOOST_ROOT=%BOOST_ROOT_1_83_0% >> %GITHUB_ENV%"\n'
|
|
)
|
|
out.write(" shell: cmd\n")
|
|
|
|
out.write(" - name: Fix Git config\n")
|
|
out.write(" run: >\n")
|
|
out.write(" git config --system core.longpaths true &&\n")
|
|
out.write(" git config --system core.autocrlf false &&\n")
|
|
# cxx crate needs symlinks enabled
|
|
out.write(" git config --system core.symlinks true\n")
|
|
# && is not supported on default windows powershell, so use cmd
|
|
out.write(" shell: cmd\n")
|
|
|
|
out.write(" - uses: actions/checkout@v4\n")
|
|
|
|
build_type_arg = ""
|
|
if args.build_type:
|
|
build_type_arg = f"--build-type {args.build_type} "
|
|
|
|
if build_opts.free_up_disk:
|
|
free_up_disk = "--free-up-disk "
|
|
if not build_opts.is_windows():
|
|
out.write(" - name: Show disk space at start\n")
|
|
out.write(" run: df -h\n")
|
|
# remove the unused github supplied android dev tools
|
|
out.write(" - name: Free up disk space\n")
|
|
out.write(" run: sudo rm -rf /usr/local/lib/android\n")
|
|
out.write(" - name: Show disk space after freeing up\n")
|
|
out.write(" run: df -h\n")
|
|
else:
|
|
free_up_disk = ""
|
|
|
|
allow_sys_arg = ""
|
|
if (
|
|
build_opts.allow_system_packages
|
|
and build_opts.host_type.get_package_manager()
|
|
):
|
|
sudo_arg = "sudo --preserve-env=http_proxy "
|
|
allow_sys_arg = " --allow-system-packages"
|
|
if build_opts.host_type.get_package_manager() == "deb":
|
|
out.write(" - name: Update system package info\n")
|
|
out.write(f" run: {sudo_arg}apt-get update\n")
|
|
|
|
out.write(" - name: Install system deps\n")
|
|
if build_opts.is_darwin():
|
|
# brew is installed as regular user
|
|
sudo_arg = ""
|
|
|
|
system_deps_cmd = f"{sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive {manifest.name}"
|
|
if build_opts.is_linux() or build_opts.is_freebsd():
|
|
system_deps_cmd += f" && {sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive patchelf"
|
|
out.write(f" run: {system_deps_cmd}\n")
|
|
|
|
required_locales = manifest.get(
|
|
"github.actions", "required_locales", ctx=manifest_ctx
|
|
)
|
|
if (
|
|
build_opts.host_type.get_package_manager() == "deb"
|
|
and required_locales
|
|
):
|
|
# ubuntu doesn't include this by default
|
|
out.write(" - name: Install locale-gen\n")
|
|
out.write(f" run: {sudo_arg}apt-get install locales\n")
|
|
for loc in required_locales.split():
|
|
out.write(f" - name: Ensure {loc} locale present\n")
|
|
out.write(f" run: {sudo_arg}locale-gen {loc}\n")
|
|
|
|
out.write(" - id: paths\n")
|
|
out.write(" name: Query paths\n")
|
|
if build_opts.is_windows():
|
|
out.write(
|
|
f" run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name} >> $env:GITHUB_OUTPUT\n"
|
|
)
|
|
out.write(" shell: pwsh\n")
|
|
else:
|
|
out.write(
|
|
f' run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name} >> "$GITHUB_OUTPUT"\n'
|
|
)
|
|
|
|
projects = loader.manifests_in_dependency_order()
|
|
|
|
main_repo_url = manifest.get_repo_url(manifest_ctx)
|
|
has_same_repo_dep = False
|
|
|
|
# Add the rust dep which doesn't have a manifest
|
|
for m in projects:
|
|
if m == manifest:
|
|
continue
|
|
mbuilder_name = m.get("build", "builder", ctx=manifest_ctx)
|
|
if (
|
|
m.name == "rust"
|
|
or builder_name == "cargo"
|
|
or mbuilder_name == "cargo"
|
|
):
|
|
out.write(" - name: Install Rust Stable\n")
|
|
out.write(" uses: dtolnay/rust-toolchain@stable\n")
|
|
break
|
|
|
|
# Normal deps that have manifests
|
|
for m in projects:
|
|
if m == manifest or m.name == "rust":
|
|
continue
|
|
ctx = loader.ctx_gen.get_context(m.name)
|
|
if m.get_repo_url(ctx) != main_repo_url:
|
|
out.write(" - name: Fetch %s\n" % m.name)
|
|
out.write(
|
|
f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n"
|
|
)
|
|
out.write(
|
|
f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
|
|
)
|
|
|
|
for m in projects:
|
|
if m == manifest or m.name == "rust":
|
|
continue
|
|
src_dir_arg = ""
|
|
ctx = loader.ctx_gen.get_context(m.name)
|
|
if main_repo_url and m.get_repo_url(ctx) == main_repo_url:
|
|
# Its in the same repo, so src-dir is also .
|
|
src_dir_arg = "--src-dir=. "
|
|
has_same_repo_dep = True
|
|
|
|
if args.use_build_cache and not src_dir_arg:
|
|
out.write(f" - name: Restore {m.name} from cache\n")
|
|
out.write(f" id: restore_{m.name}\n")
|
|
# only need to restore if would build it
|
|
out.write(
|
|
f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n"
|
|
)
|
|
out.write(" uses: actions/cache/restore@v4\n")
|
|
out.write(" with:\n")
|
|
out.write(
|
|
f" path: ${{{{ steps.paths.outputs.{m.name}_INSTALL }}}}\n"
|
|
)
|
|
out.write(
|
|
f" key: ${{{{ steps.paths.outputs.{m.name}_CACHE_KEY }}}}-install\n"
|
|
)
|
|
|
|
out.write(" - name: Build %s\n" % m.name)
|
|
if not src_dir_arg:
|
|
if args.use_build_cache:
|
|
out.write(
|
|
f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE && ! steps.restore_{m.name}.outputs.cache-hit }}}}\n"
|
|
)
|
|
else:
|
|
out.write(
|
|
f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n"
|
|
)
|
|
out.write(
|
|
f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n"
|
|
)
|
|
|
|
if args.use_build_cache and not src_dir_arg:
|
|
out.write(f" - name: Save {m.name} to cache\n")
|
|
out.write(" uses: actions/cache/save@v4\n")
|
|
out.write(
|
|
f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE && ! steps.restore_{m.name}.outputs.cache-hit }}}}\n"
|
|
)
|
|
out.write(" with:\n")
|
|
out.write(
|
|
f" path: ${{{{ steps.paths.outputs.{m.name}_INSTALL }}}}\n"
|
|
)
|
|
out.write(
|
|
f" key: ${{{{ steps.paths.outputs.{m.name}_CACHE_KEY }}}}-install\n"
|
|
)
|
|
|
|
out.write(" - name: Build %s\n" % manifest.name)
|
|
|
|
project_prefix = ""
|
|
if not build_opts.is_windows():
|
|
prefix = loader.get_project_install_prefix(manifest) or "/usr/local"
|
|
project_prefix = " --project-install-prefix %s:%s" % (
|
|
manifest.name,
|
|
prefix,
|
|
)
|
|
|
|
# If we have dep from same repo, we already built it and don't want to rebuild it again
|
|
no_deps_arg = ""
|
|
if has_same_repo_dep:
|
|
no_deps_arg = "--no-deps "
|
|
|
|
out.write(
|
|
f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{tests_arg}{no_deps_arg}--src-dir=. {manifest.name}{project_prefix}\n"
|
|
)
|
|
|
|
out.write(" - name: Copy artifacts\n")
|
|
if build_opts.is_linux():
|
|
# Strip debug info from the binaries, but only on linux.
|
|
# While the `strip` utility is also available on macOS,
|
|
# attempting to strip there results in an error.
|
|
# The `strip` utility is not available on Windows.
|
|
strip = " --strip"
|
|
else:
|
|
strip = ""
|
|
|
|
out.write(
|
|
f" run: {getdepscmd}{allow_sys_arg} fixup-dyn-deps{strip} "
|
|
f"--src-dir=. {manifest.name} _artifacts/{artifacts}{project_prefix} "
|
|
f"--final-install-prefix /usr/local\n"
|
|
)
|
|
|
|
out.write(" - uses: actions/upload-artifact@v4\n")
|
|
out.write(" with:\n")
|
|
out.write(" name: %s\n" % manifest.name)
|
|
out.write(" path: _artifacts\n")
|
|
|
|
if run_tests:
|
|
num_jobs_arg = ""
|
|
if args.num_jobs:
|
|
num_jobs_arg = f"--num-jobs {args.num_jobs} "
|
|
|
|
out.write(" - name: Test %s\n" % manifest.name)
|
|
out.write(
|
|
f" run: {getdepscmd}{allow_sys_arg} test {num_jobs_arg}--src-dir=. {manifest.name}{project_prefix}\n"
|
|
)
|
|
if build_opts.free_up_disk and not build_opts.is_windows():
|
|
out.write(" - name: Show disk space at end\n")
|
|
out.write(" if: always()\n")
|
|
out.write(" run: df -h\n")
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--disallow-system-packages",
|
|
help="Disallow satisfying third party deps from installed system packages",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
parser.add_argument(
|
|
"--output-dir", help="The directory that will contain the yml files"
|
|
)
|
|
parser.add_argument(
|
|
"--run-on-all-branches",
|
|
action="store_true",
|
|
help="Allow CI to fire on all branches - Handy for testing",
|
|
)
|
|
parser.add_argument(
|
|
"--ubuntu-version", default="22.04", help="Version of Ubuntu to use"
|
|
)
|
|
parser.add_argument(
|
|
"--cpu-cores",
|
|
help="Number of CPU cores to use (applicable for Linux OS)",
|
|
)
|
|
parser.add_argument(
|
|
"--runs-on",
|
|
help="Allow specifying explicit runs-on: for github actions",
|
|
)
|
|
parser.add_argument(
|
|
"--cron",
|
|
help="Specify that the job runs on a cron schedule instead of on pushes. Pass never to disable the action.",
|
|
)
|
|
parser.add_argument(
|
|
"--main-branch",
|
|
default="main",
|
|
help="Main branch to trigger GitHub Action on",
|
|
)
|
|
parser.add_argument(
|
|
"--os-type",
|
|
help="Filter to just this OS type to run",
|
|
choices=["linux", "darwin", "windows"],
|
|
action="append",
|
|
dest="os_types",
|
|
default=[],
|
|
)
|
|
parser.add_argument(
|
|
"--job-file-prefix",
|
|
type=str,
|
|
help="add a prefix to all job file names",
|
|
default=None,
|
|
)
|
|
parser.add_argument(
|
|
"--job-name-prefix",
|
|
type=str,
|
|
help="add a prefix to all job names",
|
|
default=None,
|
|
)
|
|
parser.add_argument(
|
|
"--free-up-disk",
|
|
help="Remove unused tools and clean up intermediate files if possible to maximise space for the build",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
parser.add_argument(
|
|
"--build-type",
|
|
help="Set the build type explicitly. Cmake and cargo builders act on them. Only Debug and RelWithDebInfo widely supported.",
|
|
choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"],
|
|
action="store",
|
|
default=None,
|
|
)
|
|
parser.add_argument(
|
|
"--no-build-cache",
|
|
action="store_false",
|
|
default=True,
|
|
dest="use_build_cache",
|
|
help="Do not attempt to use the build cache.",
|
|
)
|
|
|
|
|
|
def get_arg_var_name(args):
|
|
for arg in args:
|
|
if arg.startswith("--"):
|
|
return arg[2:].replace("-", "_")
|
|
|
|
raise Exception("unable to determine argument variable name from %r" % (args,))
|
|
|
|
|
|
def parse_args():
|
|
# We want to allow common arguments to be specified either before or after
|
|
# the subcommand name. In order to do this we add them to the main parser
|
|
# and to subcommand parsers. In order for this to work, we need to tell
|
|
# argparse that the default value is SUPPRESS, so that the default values
|
|
# from the subparser arguments won't override values set by the user from
|
|
# the main parser. We maintain our own list of desired defaults in the
|
|
# common_defaults dictionary, and manually set those if the argument wasn't
|
|
# present at all.
|
|
common_args = argparse.ArgumentParser(add_help=False)
|
|
common_defaults = {}
|
|
|
|
def add_common_arg(*args, **kwargs):
|
|
var_name = get_arg_var_name(args)
|
|
default_value = kwargs.pop("default", None)
|
|
common_defaults[var_name] = default_value
|
|
kwargs["default"] = argparse.SUPPRESS
|
|
common_args.add_argument(*args, **kwargs)
|
|
|
|
add_common_arg("--scratch-path", help="Where to maintain checkouts and build dirs")
|
|
add_common_arg(
|
|
"--vcvars-path", default=None, help="Path to the vcvarsall.bat on Windows."
|
|
)
|
|
add_common_arg(
|
|
"--install-prefix",
|
|
help=(
|
|
"Where the final build products will be installed "
|
|
"(default is [scratch-path]/installed)"
|
|
),
|
|
)
|
|
add_common_arg(
|
|
"--num-jobs",
|
|
type=int,
|
|
help=(
|
|
"Number of concurrent jobs to use while building. "
|
|
"(default=number of cpu cores)"
|
|
),
|
|
)
|
|
add_common_arg(
|
|
"--use-shipit",
|
|
help="use the real ShipIt instead of the simple shipit transformer",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
add_common_arg(
|
|
"--facebook-internal",
|
|
help="Setup the build context as an FB internal build",
|
|
action="store_true",
|
|
default=None,
|
|
)
|
|
add_common_arg(
|
|
"--no-facebook-internal",
|
|
help="Perform a non-FB internal build, even when in an fbsource repository",
|
|
action="store_false",
|
|
dest="facebook_internal",
|
|
)
|
|
add_common_arg(
|
|
"--shared-libs",
|
|
help="Build shared libraries if possible",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
add_common_arg(
|
|
"--extra-cmake-defines",
|
|
help=(
|
|
"Input json map that contains extra cmake defines to be used "
|
|
"when compiling the current project and all its deps. "
|
|
'e.g: \'{"CMAKE_CXX_FLAGS": "--bla"}\''
|
|
),
|
|
)
|
|
add_common_arg(
|
|
"--allow-system-packages",
|
|
help="Allow satisfying third party deps from installed system packages",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
add_common_arg(
|
|
"-v",
|
|
"--verbose",
|
|
help="Print more output",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
add_common_arg(
|
|
"-su",
|
|
"--skip-upload",
|
|
help="skip upload steps",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
add_common_arg(
|
|
"--lfs-path",
|
|
help="Provide a parent directory for lfs when fbsource is unavailable",
|
|
default=None,
|
|
)
|
|
add_common_arg(
|
|
"--build-skip-lfs-download",
|
|
action="store_true",
|
|
default=False,
|
|
help=(
|
|
"Download from the URL, rather than LFS. This is useful "
|
|
"in cases where the upstream project has uploaded a new "
|
|
"version of the archive with a different hash"
|
|
),
|
|
)
|
|
|
|
ap = argparse.ArgumentParser(
|
|
description="Get and build dependencies and projects", parents=[common_args]
|
|
)
|
|
sub = ap.add_subparsers(
|
|
# metavar suppresses the long and ugly default list of subcommands on a
|
|
# single line. We still render the nicer list below where we would
|
|
# have shown the nasty one.
|
|
metavar="",
|
|
title="Available commands",
|
|
help="",
|
|
)
|
|
|
|
add_subcommands(sub, common_args)
|
|
|
|
args = ap.parse_args()
|
|
for var_name, default_value in common_defaults.items():
|
|
if not hasattr(args, var_name):
|
|
setattr(args, var_name, default_value)
|
|
|
|
return ap, args
|
|
|
|
|
|
def main():
|
|
ap, args = parse_args()
|
|
if getattr(args, "func", None) is None:
|
|
ap.print_help()
|
|
return 0
|
|
try:
|
|
return args.func(args)
|
|
except UsageError as exc:
|
|
ap.error(str(exc))
|
|
return 1
|
|
except TransientFailure as exc:
|
|
print("TransientFailure: %s" % str(exc))
|
|
# This return code is treated as a retryable transient infrastructure
|
|
# error by Facebook's internal CI, rather than eg: a build or code
|
|
# related error that needs to be fixed before progress can be made.
|
|
return 128
|
|
except subprocess.CalledProcessError as exc:
|
|
print("%s" % str(exc), file=sys.stderr)
|
|
print("!! Failed", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|