From dd16065749c183176e46cdd177d563cb08abe2ec Mon Sep 17 00:00:00 2001 From: "Zeyi (Rice) Fan" Date: Thu, 19 Sep 2019 15:19:56 -0700 Subject: [PATCH] getdeps: include subdirectories when searching manifest Summary: Make getdeps to look for subdirectories for manifest files. Reviewed By: simpkins Differential Revision: D17222388 fbshipit-source-id: e13503beccd9edf6d80f78fbc3238b2a8d2053dd --- build/fbcode_builder/getdeps/buildopts.py | 4 ++ build/fbcode_builder/getdeps/errors.py | 5 ++ build/fbcode_builder/getdeps/load.py | 87 +++++++++++++++-------- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index 4c2dbf595..e730cd613 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -120,6 +120,10 @@ class BuildOptions(object): self.vcvars_path = vcvars_path + @property + def manifests_dir(self): + return os.path.join(self.fbcode_builder_dir, "manifests") + def is_darwin(self): return self.host_type.is_darwin() diff --git a/build/fbcode_builder/getdeps/errors.py b/build/fbcode_builder/getdeps/errors.py index 419a67503..6be2cfac8 100644 --- a/build/fbcode_builder/getdeps/errors.py +++ b/build/fbcode_builder/getdeps/errors.py @@ -14,3 +14,8 @@ class TransientFailure(Exception): infrastructure error """ pass + + +class ManifestNotFound(Exception): + def __init__(self, manifest_name): + super(Exception, self).__init__("Unable to find manifest '%s'" % manifest_name) diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py index 648136379..7b884d100 100644 --- a/build/fbcode_builder/getdeps/load.py +++ b/build/fbcode_builder/getdeps/load.py @@ -8,30 +8,44 @@ from __future__ import absolute_import, division, print_function, unicode_literals import base64 -import glob import hashlib import os from . import fetcher from .envfuncs import path_search +from .errors import ManifestNotFound from .manifest import ManifestParser class Loader(object): """ The loader allows our tests to patch the load operation """ + def _list_manifests(self, manifests_dir): + """ Returns a generator that iterates all the available manifests """ + for (path, _, files) in os.walk(manifests_dir): + for name in files: + # skip hidden files + if name.startswith("."): + continue + + yield os.path.join(path, name) + def load_project(self, build_opts, project_name): - manifest_path = resolve_manifest_path(build_opts, project_name) - return ManifestParser(manifest_path) + if "/" in project_name or "\\" in project_name: + # Assume this is a path already + return ManifestParser(project_name) + + for manifest in self._list_manifests(build_opts.manifests_dir): + if os.path.basename(manifest) == project_name: + return ManifestParser(manifest) + + raise ManifestNotFound(project_name) def load_all(self, build_opts): manifests_by_name = {} - manifests_dir = os.path.join(build_opts.fbcode_builder_dir, "manifests") - # We use glob rather than os.listdir because glob won't include - # eg: vim swap files that a maintainer might happen to have - # for manifests that they are editing - for name in glob.glob("%s/*" % manifests_dir): - m = ManifestParser(name) + + for manifest in self._list_manifests(build_opts.manifests_dir): + m = ManifestParser(manifest) manifests_by_name[m.name] = m return manifests_by_name @@ -41,23 +55,45 @@ class ResourceLoader(Loader): def __init__(self, namespace): self.namespace = namespace - def load_project(self, build_opts, project_name): + def _list_manifests(self): import pkg_resources - contents = pkg_resources.resource_string( - self.namespace, "manifests/%s" % project_name - ).decode("utf8") - m = ManifestParser(file_name=project_name, fp=contents) - return m + dirs = ["manifests"] + + while dirs: + current = dirs.pop(0) + for name in pkg_resources.resource_listdir(self.namespace, current): + path = "%s/%s" % (current, name) + + if pkg_resources.resource_isdir(self.namespace, path): + dirs.append(path) + else: + yield "%s/%s" % (current, name) + + def _find_manifest(self, project_name): + for name in self._list_manifests(): + if name.endswith("/%s" % project_name): + return name + + raise ManifestNotFound(project_name) + + def _load_resource_manifest(self, path): + import pkg_resources + + contents = pkg_resources.resource_string(self.namespace, path).decode("utf8") + return ManifestParser(file_name=path, fp=contents) + + def load_project(self, build_opts, project_name): + project_name = self._find_manifest(project_name) + return self._load_resource_manifest(project_name) def load_all(self, build_opts): - import pkg_resources + manifests_by_name = {} + for path in self._list_manifests(): + m = self._load_resource_manifest(path) + manifests_by_name[m.name] = m - manifest_by_name = {} - for name in pkg_resources.resource_listdir(self.namespace, "manifests"): - m = self.load_project(build_opts, name) - manifest_by_name[m.name] = m - return manifest_by_name + return manifests_by_name LOADER = Loader() @@ -68,15 +104,6 @@ def patch_loader(namespace): LOADER = ResourceLoader(namespace) -def resolve_manifest_path(build_opts, project_name): - if "/" in project_name or "\\" in project_name: - # Assume this is a path already - return project_name - - # Otherwise, resolve it relative to the manifests dir - return os.path.join(build_opts.fbcode_builder_dir, "manifests", project_name) - - def load_project(build_opts, project_name): """ given the name of a project or a path to a manifest file, load up the ManifestParser instance for it and return it """