update python to 3.8

Summary:
Update to a newer python that builds on macOS Big Sur, make eden_scm depend on it and use it from PATH

python 3.8 requires libffi which is reference via its pkg-config setup,  however python's pkg-config libffi detection is broken (https://bugs.python.org/issue34823) with the documented workaround requiring an environment variable to be passed into its ./configure step, which is why this change also adds a feature to AutoconfBuilder

With the updated python in place I was able to remove disable_env_override_pkgconfig = 1
disable_env_override_path = 1 from the eden_scm config so that it actually uses the pkg-configs, PATHs and other settings from its dependencies. This should make future python and other dependency upgrades much simpler.

Reviewed By: HarveyHunt

Differential Revision: D32231261

fbshipit-source-id: a2b6addbe22f38e3d71618c802d2c6f836fdd86c

build/fbcode_builder/getdeps/builder.py

@@ -198,11 +198,22 @@ class CMakeBootStrapBuilder(MakeBuilder):
 
 
 class AutoconfBuilder(BuilderBase):
-    def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args):
+    def __init__(
+        self,
+        build_opts,
+        ctx,
+        manifest,
+        src_dir,
+        build_dir,
+        inst_dir,
+        args,
+        conf_env_args,
+    ):
         super(AutoconfBuilder, self).__init__(
             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
         )
         self.args = args or []
+        self.conf_env_args = conf_env_args or {}
 
     def _build(self, install_dirs, reconfigure):
         configure_path = os.path.join(self.src_dir, "configure")
@@ -210,6 +221,16 @@ class AutoconfBuilder(BuilderBase):
 
         env = self._compute_env(install_dirs)
 
+        # Some configure scripts need additional env values passed derived from cmds
+        for (k, cmd_args) in self.conf_env_args.items():
+            out = (
+                subprocess.check_output(cmd_args, env=dict(env.items()))
+                .decode("utf-8")
+                .strip()
+            )
+            if out:
+                env.set(k, out)
+
         if not os.path.exists(configure_path):
             print("%s doesn't exist, so reconfiguring" % configure_path)
             # This libtoolize call is a bit gross; the issue is that

build/fbcode_builder/getdeps/buildopts.py

@@ -233,33 +233,29 @@ class BuildOptions(object):
         for d in install_dirs:
             bindir = os.path.join(d, "bin")
 
-            if not (
+            pkgconfig = os.path.join(d, "lib/pkgconfig")
-                manifest and manifest.get("build", "disable_env_override_pkgconfig")
+            if os.path.exists(pkgconfig):
-            ):
+                add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
-                pkgconfig = os.path.join(d, "lib/pkgconfig")
-                if os.path.exists(pkgconfig):
-                    add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
 
-                pkgconfig = os.path.join(d, "lib64/pkgconfig")
+            pkgconfig = os.path.join(d, "lib64/pkgconfig")
-                if os.path.exists(pkgconfig):
+            if os.path.exists(pkgconfig):
-                    add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
+                add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
 
-            if not (manifest and manifest.get("build", "disable_env_override_path")):
+            add_path_entry(env, "CMAKE_PREFIX_PATH", d)
-                add_path_entry(env, "CMAKE_PREFIX_PATH", d)
 
-                # Allow resolving shared objects built earlier (eg: zstd
+            # Allow resolving shared objects built earlier (eg: zstd
-                # doesn't include the full path to the dylib in its linkage
+            # doesn't include the full path to the dylib in its linkage
-                # so we need to give it an assist)
+            # so we need to give it an assist)
-                if lib_path:
+            if lib_path:
-                    for lib in ["lib", "lib64"]:
+                for lib in ["lib", "lib64"]:
-                        libdir = os.path.join(d, lib)
+                    libdir = os.path.join(d, lib)
-                        if os.path.exists(libdir):
+                    if os.path.exists(libdir):
-                            add_path_entry(env, lib_path, libdir)
+                        add_path_entry(env, lib_path, libdir)
 
-                # Allow resolving binaries (eg: cmake, ninja) and dlls
+            # Allow resolving binaries (eg: cmake, ninja) and dlls
-                # built by earlier steps
+            # built by earlier steps
-                if os.path.exists(bindir):
+            if os.path.exists(bindir):
-                    add_path_entry(env, "PATH", bindir, append=False)
+                add_path_entry(env, "PATH", bindir, append=False)
 
             # If rustc is present in the `bin` directory, set RUSTC to prevent
             # cargo uses the rustc installed in the system.

build/fbcode_builder/getdeps/manifest.py

@@ -69,8 +69,6 @@ SCHEMA = {
             "builder": REQUIRED,
             "subdir": OPTIONAL,
             "build_in_src_dir": OPTIONAL,
-            "disable_env_override_pkgconfig": OPTIONAL,
-            "disable_env_override_path": OPTIONAL,
         },
     },
     "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}},
@@ -84,6 +82,7 @@ SCHEMA = {
     },
     "cmake.defines": {"optional_section": True},
     "autoconf.args": {"optional_section": True},
+    "autoconf.envcmd.LDFLAGS": {"optional_section": True},
     "rpms": {"optional_section": True},
     "debs": {"optional_section": True},
     "preinstalled.env": {"optional_section": True},
@@ -101,6 +100,7 @@ SCHEMA = {
 # using the expression syntax to enable/disable sections
 ALLOWED_EXPR_SECTIONS = [
     "autoconf.args",
+    "autoconf.envcmd.LDFLAGS",
     "build",
     "cmake.defines",
     "dependencies",
@@ -469,8 +469,19 @@ class ManifestParser(object):
 
         if builder == "autoconf":
             args = self.get_section_as_args("autoconf.args", ctx)
+            conf_env_args = {}
+            ldflags_cmd = self.get_section_as_args("autoconf.envcmd.LDFLAGS", ctx)
+            if ldflags_cmd:
+                conf_env_args["LDFLAGS"] = ldflags_cmd
             return AutoconfBuilder(
-                build_options, ctx, self, src_dir, build_dir, inst_dir, args
+                build_options,
+                ctx,
+                self,
+                src_dir,
+                build_dir,
+                inst_dir,
+                args,
+                conf_env_args,
             )
 
         if builder == "boost":

build/fbcode_builder/manifests/eden_scm

@@ -10,8 +10,6 @@ repo_url = https://github.com/facebookexperimental/eden.git
 [build.os=linux]
 builder = make
 subdir = eden/scm
-disable_env_override_pkgconfig = 1
-disable_env_override_path = 1
 
 [build.os=windows]
 # For now the biggest blocker is missing "make" on windows, but there are bound
@@ -53,6 +51,7 @@ fbcode/fboss/common = common
 [dependencies]
 fb303-source
 fbthrift-source
+python
 rust-shed
 
 # We use the system openssl on linux

build/fbcode_builder/manifests/fboss

@@ -31,7 +31,7 @@ libnl
 libsai
 OpenNSA
 re2
-python
+python_3_7
 yaml-cpp
 libyaml
 CLI11

build/fbcode_builder/manifests/python

@@ -8,10 +8,27 @@ python3-devel
 [debs]
 python3-all-dev
 
-[download.os=linux]
+[download]
-url = https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tgz
+url = https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tgz
-sha256 = aeee681c235ad336af116f08ab6563361a0c81c537072c1b309d6e4050aa2114
+sha256 = 316aa33f3b7707d041e73f246efedb297a70898c4b91f127f66dc8d80c596f1a
 
-[build.os=linux]
+[build]
 builder = autoconf
-subdir = Python-3.7.6
+subdir = Python-3.8.12
+
+[autoconf.args]
+--enable-shared
+--with-ensurepip=install
+
+# python's pkg-config libffi detection is broken
+# See https://bugs.python.org/issue34823 for clearest description
+# and pending PR https://github.com/python/cpython/pull/20451
+# The documented workaround requires an environment variable derived from
+# pkg-config to be passed into its configure step
+[autoconf.envcmd.LDFLAGS]
+pkg-config
+--libs-only-L
+libffi
+
+[dependencies]
+libffi

build/fbcode_builder/manifests/python_3_7

@@ -0,0 +1,17 @@
+[manifest]
+name = python_3_7
+
+[rpms]
+python3
+python3-devel
+
+[debs]
+python3-all-dev
+
+[download.os=linux]
+url = https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tgz
+sha256 = aeee681c235ad336af116f08ab6563361a0c81c537072c1b309d6e4050aa2114
+
+[build.os=linux]
+builder = autoconf
+subdir = Python-3.7.6