From 9c3f99b0c09662d96bb3c6c99c1772f0c06fa703 Mon Sep 17 00:00:00 2001
From: Lukas Piatkowski <luk@fb.com>
Date: Tue, 13 Oct 2020 09:48:38 -0700
Subject: [PATCH] mononoke/integration tests: prepare dependencies for running
 tests by getdeps

Summary:
This diff adds all third party dependencies that are required by getdeps to be able to build and runn Mononoke's integration tests.

Also add a stub Makefile with no-op steps that will be filled in next diff.

Reviewed By: ahornby

Differential Revision: D24251894

fbshipit-source-id: 67384ecfd0ced6762dddc3c6e61feb1240b1162d
---
 build/fbcode_builder/getdeps/builder.py       | 11 +++--
 build/fbcode_builder/getdeps/manifest.py      |  6 +++
 build/fbcode_builder/manifests/gnu-bash       | 20 +++++++++
 build/fbcode_builder/manifests/gnu-coreutils  | 15 +++++++
 build/fbcode_builder/manifests/gnu-grep       | 15 +++++++
 build/fbcode_builder/manifests/gnu-sed        | 15 +++++++
 build/fbcode_builder/manifests/jq             | 24 +++++++++++
 .../manifests/mononoke_integration            | 41 +++++++++++++++++++
 build/fbcode_builder/manifests/nmap           | 25 +++++++++++
 build/fbcode_builder/manifests/python-click   |  9 ++++
 build/fbcode_builder/manifests/python-dulwich | 19 +++++++++
 build/fbcode_builder/manifests/sqlite3-bin    | 28 +++++++++++++
 build/fbcode_builder/manifests/tcl            | 20 +++++++++
 build/fbcode_builder/manifests/tree           | 34 +++++++++++++++
 14 files changed, 279 insertions(+), 3 deletions(-)
 create mode 100644 build/fbcode_builder/manifests/gnu-bash
 create mode 100644 build/fbcode_builder/manifests/gnu-coreutils
 create mode 100644 build/fbcode_builder/manifests/gnu-grep
 create mode 100644 build/fbcode_builder/manifests/gnu-sed
 create mode 100644 build/fbcode_builder/manifests/jq
 create mode 100644 build/fbcode_builder/manifests/mononoke_integration
 create mode 100644 build/fbcode_builder/manifests/nmap
 create mode 100644 build/fbcode_builder/manifests/python-click
 create mode 100644 build/fbcode_builder/manifests/python-dulwich
 create mode 100644 build/fbcode_builder/manifests/sqlite3-bin
 create mode 100644 build/fbcode_builder/manifests/tcl
 create mode 100644 build/fbcode_builder/manifests/tree

diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py
index 63a5bb788..3095d6439 100644
--- a/build/fbcode_builder/getdeps/builder.py
+++ b/build/fbcode_builder/getdeps/builder.py
@@ -156,15 +156,20 @@ class MakeBuilder(BuilderBase):
         env = self._compute_env(install_dirs)
 
         # Need to ensure that PREFIX is set prior to install because
-        # libbpf uses it when generating its pkg-config file
+        # libbpf uses it when generating its pkg-config file.
+        # The lowercase prefix is used by some projects.
         cmd = (
             ["make", "-j%s" % self.build_opts.num_jobs]
             + self.build_args
-            + ["PREFIX=" + self.inst_dir]
+            + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
         )
         self._run_cmd(cmd, env=env)
 
-        install_cmd = ["make"] + self.install_args + ["PREFIX=" + self.inst_dir]
+        install_cmd = (
+            ["make"]
+            + self.install_args
+            + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
+        )
         self._run_cmd(install_cmd, env=env)
 
 
diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py
index 35b0948cf..95e1c3b99 100644
--- a/build/fbcode_builder/getdeps/manifest.py
+++ b/build/fbcode_builder/getdeps/manifest.py
@@ -425,7 +425,13 @@ class ManifestParser(object):
             raise Exception("project %s has no builder for %r" % (self.name, ctx))
         build_in_src_dir = self.get("build", "build_in_src_dir", "false", ctx=ctx)
         if build_in_src_dir == "true":
+            # Some scripts don't work when they are configured and build in
+            # a different directory than source (or when the build directory
+            # is not a subdir of source).
             build_dir = src_dir
+            subdir = self.get("build", "subdir", None, ctx=ctx)
+            if subdir is not None:
+                build_dir = os.path.join(build_dir, subdir)
             print("build_dir is %s" % build_dir)  # just to quiet lint
 
         if builder == "make":
diff --git a/build/fbcode_builder/manifests/gnu-bash b/build/fbcode_builder/manifests/gnu-bash
new file mode 100644
index 000000000..89da77ca2
--- /dev/null
+++ b/build/fbcode_builder/manifests/gnu-bash
@@ -0,0 +1,20 @@
+[manifest]
+name = gnu-bash
+
+[download.os=darwin]
+url = https://ftp.gnu.org/gnu/bash/bash-5.1-rc1.tar.gz
+sha256 = 0b2684eb1990329d499c96decfe2459f3e150deb915b0a9d03cf1be692b1d6d3
+
+[build.os=darwin]
+# The buildin FreeBSD bash on OSX is both outdated and incompatible with the
+# modern GNU bash, so for the sake of being cross-platform friendly this
+# manifest provides GNU bash.
+# NOTE: This is the 5.1-rc1 version, which is almost the same as what Homebrew
+# uses (Homebrew installs 5.0 with the 18 patches that in fact make the 5.1-rc1
+# version).
+builder = autoconf
+subdir = bash-5.1-rc1
+build_in_src_dir = true
+
+[build.not(os=darwin)]
+builder = nop
diff --git a/build/fbcode_builder/manifests/gnu-coreutils b/build/fbcode_builder/manifests/gnu-coreutils
new file mode 100644
index 000000000..1ab4d9d4a
--- /dev/null
+++ b/build/fbcode_builder/manifests/gnu-coreutils
@@ -0,0 +1,15 @@
+[manifest]
+name = gnu-coreutils
+
+[download.os=darwin]
+url = https://ftp.gnu.org/gnu/coreutils/coreutils-8.32.tar.gz
+sha256 = d5ab07435a74058ab69a2007e838be4f6a90b5635d812c2e26671e3972fca1b8
+
+[build.os=darwin]
+# The buildin FreeBSD version incompatible with the GNU one, so for the sake of
+# being cross-platform friendly this manifest provides the GNU version.
+builder = autoconf
+subdir = coreutils-8.32
+
+[build.not(os=darwin)]
+builder = nop
diff --git a/build/fbcode_builder/manifests/gnu-grep b/build/fbcode_builder/manifests/gnu-grep
new file mode 100644
index 000000000..e6a163d37
--- /dev/null
+++ b/build/fbcode_builder/manifests/gnu-grep
@@ -0,0 +1,15 @@
+[manifest]
+name = gnu-grep
+
+[download.os=darwin]
+url = https://ftp.gnu.org/gnu/grep/grep-3.5.tar.gz
+sha256 = 9897220992a8fd38a80b70731462defa95f7ff2709b235fb54864ddd011141dd
+
+[build.os=darwin]
+# The buildin FreeBSD version incompatible with the GNU one, so for the sake of
+# being cross-platform friendly this manifest provides the GNU version.
+builder = autoconf
+subdir = grep-3.5
+
+[build.not(os=darwin)]
+builder = nop
diff --git a/build/fbcode_builder/manifests/gnu-sed b/build/fbcode_builder/manifests/gnu-sed
new file mode 100644
index 000000000..9b458df6e
--- /dev/null
+++ b/build/fbcode_builder/manifests/gnu-sed
@@ -0,0 +1,15 @@
+[manifest]
+name = gnu-sed
+
+[download.os=darwin]
+url = https://ftp.gnu.org/gnu/sed/sed-4.8.tar.gz
+sha256 = 53cf3e14c71f3a149f29d13a0da64120b3c1d3334fba39c4af3e520be053982a
+
+[build.os=darwin]
+# The buildin FreeBSD version incompatible with the GNU one, so for the sake of
+# being cross-platform friendly this manifest provides the GNU version.
+builder = autoconf
+subdir = sed-4.8
+
+[build.not(os=darwin)]
+builder = nop
diff --git a/build/fbcode_builder/manifests/jq b/build/fbcode_builder/manifests/jq
new file mode 100644
index 000000000..231818f34
--- /dev/null
+++ b/build/fbcode_builder/manifests/jq
@@ -0,0 +1,24 @@
+[manifest]
+name = jq
+
+[rpms]
+jq
+
+[debs]
+jq
+
+[download.not(os=windows)]
+url = https://github.com/stedolan/jq/releases/download/jq-1.5/jq-1.5.tar.gz
+sha256 = c4d2bfec6436341113419debf479d833692cc5cdab7eb0326b5a4d4fbe9f493c
+
+[build.not(os=windows)]
+builder = autoconf
+subdir = jq-1.5
+
+[build.os=windows]
+builder = nop
+
+[autoconf.args]
+# This argument turns off some developers tool and it is recommended in jq's
+# README
+--disable-maintainer-mode
diff --git a/build/fbcode_builder/manifests/mononoke_integration b/build/fbcode_builder/manifests/mononoke_integration
new file mode 100644
index 000000000..07a6d4a58
--- /dev/null
+++ b/build/fbcode_builder/manifests/mononoke_integration
@@ -0,0 +1,41 @@
+[manifest]
+name = mononoke_integration
+fbsource_path = fbcode/eden
+shipit_project = eden
+shipit_fbcode_builder = true
+
+[build.not(os=windows)]
+builder = make
+subdir = eden/mononoke/tests/integration
+
+[build.os=windows]
+# building Mononoke on windows is not supported
+builder = nop
+
+[make.build_args]
+build-getdeps
+
+[make.install_args]
+install-getdeps
+
+[shipit.pathmap]
+fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration
+
+[shipit.strip]
+^.*/facebook/.*$
+
+[dependencies]
+jq
+nmap
+python-click
+python-dulwich
+tree
+
+[dependencies.os=linux]
+sqlite3-bin
+
+[dependencies.os=darwin]
+gnu-bash
+gnu-coreutils
+gnu-grep
+gnu-sed
diff --git a/build/fbcode_builder/manifests/nmap b/build/fbcode_builder/manifests/nmap
new file mode 100644
index 000000000..c245e1241
--- /dev/null
+++ b/build/fbcode_builder/manifests/nmap
@@ -0,0 +1,25 @@
+[manifest]
+name = nmap
+
+[rpms]
+nmap
+
+[debs]
+nmap
+
+[download.not(os=windows)]
+url = https://api.github.com/repos/nmap/nmap/tarball/ef8213a36c2e89233c806753a57b5cd473605408
+sha256 = eda39e5a8ef4964fac7db16abf91cc11ff568eac0fa2d680b0bfa33b0ed71f4a
+
+[build.not(os=windows)]
+builder = autoconf
+subdir = nmap-nmap-ef8213a
+build_in_src_dir = true
+
+[build.os=windows]
+builder = nop
+
+[autoconf.args]
+# Without this option the build was filing to find some third party libraries
+# that we don't need
+enable_rdma=no
diff --git a/build/fbcode_builder/manifests/python-click b/build/fbcode_builder/manifests/python-click
new file mode 100644
index 000000000..ea9a9d2d3
--- /dev/null
+++ b/build/fbcode_builder/manifests/python-click
@@ -0,0 +1,9 @@
+[manifest]
+name = python-click
+
+[download]
+url = https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl
+sha256 = dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc
+
+[build]
+builder = python-wheel
diff --git a/build/fbcode_builder/manifests/python-dulwich b/build/fbcode_builder/manifests/python-dulwich
new file mode 100644
index 000000000..0d995e12f
--- /dev/null
+++ b/build/fbcode_builder/manifests/python-dulwich
@@ -0,0 +1,19 @@
+[manifest]
+name = python-dulwich
+
+# The below links point to custom github forks of project dulwich, because the
+# 0.18.6 version didn't have an official rollout of wheel packages.
+
+[download.os=linux]
+url = https://github.com/lukaspiatkowski/dulwich/releases/download/dulwich-0.18.6-wheel/dulwich-0.18.6-cp36-cp36m-linux_x86_64.whl
+sha256 = e96f545f3d003e67236785473caaba2c368e531ea85fd508a3bd016ebac3a6d8
+
+[download.os=darwin]
+url = https://github.com/lukaspiatkowski/dulwich/releases/download/dulwich-0.18.6-wheel/dulwich-0.18.6-cp37-cp37m-macosx_10_14_x86_64.whl
+sha256 = 8373652056284ad40ea5220b659b3489b0a91f25536322345a3e4b5d29069308
+
+[build.not(os=windows)]
+builder = python-wheel
+
+[build.os=windows]
+builder = nop
diff --git a/build/fbcode_builder/manifests/sqlite3-bin b/build/fbcode_builder/manifests/sqlite3-bin
new file mode 100644
index 000000000..aa138d499
--- /dev/null
+++ b/build/fbcode_builder/manifests/sqlite3-bin
@@ -0,0 +1,28 @@
+[manifest]
+name = sqlite3-bin
+
+[rpms]
+sqlite
+
+[debs]
+sqlite3
+
+[download.os=linux]
+url = https://github.com/sqlite/sqlite/archive/version-3.33.0.tar.gz
+sha256 = 48e5f989eefe9af0ac758096f82ead0f3c7b58118ac17cc5810495bd5084a331
+
+[build.os=linux]
+builder = autoconf
+subdir = sqlite-version-3.33.0
+
+[build.not(os=linux)]
+# MacOS comes with sqlite3 preinstalled and don't need Windows here
+builder = nop
+
+[dependencies.os=linux]
+tcl
+
+[autoconf.args]
+# This flag disabled tcl as a runtime library used for some functionality,
+# but tcl is still a required dependency as it is used by the build files
+--disable-tcl
diff --git a/build/fbcode_builder/manifests/tcl b/build/fbcode_builder/manifests/tcl
new file mode 100644
index 000000000..5e9892f37
--- /dev/null
+++ b/build/fbcode_builder/manifests/tcl
@@ -0,0 +1,20 @@
+[manifest]
+name = tcl
+
+[rpms]
+tcl
+
+[debs]
+tcl
+
+[download]
+url = https://github.com/tcltk/tcl/archive/core-8-7a3.tar.gz
+sha256 = 22d748f0c9652f3ecc195fed3f24a1b6eea8d449003085e6651197951528982e
+
+[build.os=linux]
+builder = autoconf
+subdir = tcl-core-8-7a3/unix
+
+[build.not(os=linux)]
+# This is for sqlite3 on Linux for now
+builder = nop
diff --git a/build/fbcode_builder/manifests/tree b/build/fbcode_builder/manifests/tree
new file mode 100644
index 000000000..0c982f35a
--- /dev/null
+++ b/build/fbcode_builder/manifests/tree
@@ -0,0 +1,34 @@
+[manifest]
+name = tree
+
+[rpms]
+tree
+
+[debs]
+tree
+
+[download.os=linux]
+url = https://salsa.debian.org/debian/tree-packaging/-/archive/debian/1.8.0-1/tree-packaging-debian-1.8.0-1.tar.gz
+sha256 = a841eee1d52bfd64a48f54caab9937b9bd92935055c48885c4ab1ae4dab7fae5
+
+[download.os=darwin]
+# The official package of tree source requires users of non-Linux platform to
+# comment/uncomment certain lines in the Makefile to build for their platform.
+# Besauce getdeps.py doesn't have that functionality we just use this custom
+# fork of tree which has proper lines uncommented for a OSX build
+url = https://github.com/lukaspiatkowski/tree-command/archive/debian/1.8.0-1-macos.tar.gz
+sha256 = 9cbe889553d95cf5a2791dd0743795d46a3c092c5bba691769c0e5c52e11229e
+
+[build.os=linux]
+builder = make
+subdir = tree-packaging-debian-1.8.0-1
+
+[build.os=darwin]
+builder = make
+subdir = tree-command-debian-1.8.0-1-macos
+
+[build.os=windows]
+builder = nop
+
+[make.install_args]
+install