From 295872b12238a4f81c3a56b18b9798b1fd92990c Mon Sep 17 00:00:00 2001
From: Alex Hornby <alex@hornby.org.uk>
Date: Mon, 7 Oct 2024 13:46:20 -0700
Subject: [PATCH] run .t tests for getdeps sapling cli build

Summary:
Connect up the sapling *.t tests so that github CI has visible cli test status.

To get them to run:
 * some needed fbpython on path. I included a shim for that in the test makefile target.
 * test-rust-hooks: Command not found message,  added a glob
 * test-identity.t: add a glob for the sapling version
 * test-eolfilename.t: output order was unstable, added sorts to make stable
 * helpers-testrepo.sh: fix assumption that system hg would be able to read test repo, check if its Sapling first.
 * added a manifest for the hexdump utility some of the tests required
 * excluded a few remaining tests (see comments in Makefile for reason)
 * fixed getdeps support for generating actions steps for test only dependencies

NB  the tests run as "hg".  The expectations would need to be updated if we were to run as "sl"

X-link: https://github.com/facebook/sapling/pull/963
X-link: https://github.com/facebookincubator/zstrong/pull/1004

Reviewed By: quark-zju

Differential Revision: D63958737

Pulled By: ahornby

fbshipit-source-id: 75c0d39258c320100d8d02b31390994bc2f3a3ce
---
 build/fbcode_builder/getdeps.py          | 31 +++++++++++++++++-------
 build/fbcode_builder/getdeps/manifest.py |  1 +
 build/fbcode_builder/manifests/hexdump   | 12 +++++++++
 build/fbcode_builder/manifests/sapling   |  6 +++++
 4 files changed, 41 insertions(+), 9 deletions(-)
 create mode 100644 build/fbcode_builder/manifests/hexdump

diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py
index 037a695d2..8a42fcd43 100755
--- a/build/fbcode_builder/getdeps.py
+++ b/build/fbcode_builder/getdeps.py
@@ -963,6 +963,12 @@ class GenerateGitHubActionsCmd(ProjectCmdBase):
         self.process_project_dir_arguments(args, loader)
         manifest = loader.load_manifest(args.project)
         manifest_ctx = loader.ctx_gen.get_context(manifest.name)
+        run_tests = (
+            args.enable_tests
+            and manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off"
+        )
+        if run_tests:
+            manifest_ctx.set("test", "on")
         run_on = self.get_run_on(args)
 
         # Some projects don't do anything "useful" as a leaf project, only
@@ -1075,10 +1081,7 @@ jobs:
                 free_up_disk = ""
 
             allow_sys_arg = ""
-            if (
-                build_opts.allow_system_packages
-                and build_opts.host_type.get_package_manager()
-            ):
+            if run_tests:
                 sudo_arg = "sudo "
                 allow_sys_arg = " --allow-system-packages"
                 if build_opts.host_type.get_package_manager() == "deb":
@@ -1097,6 +1100,19 @@ jobs:
                     out.write(
                         f"      run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf\n"
                     )
+                required_locales = manifest.get(
+                    "github.actions", "required_locales", ctx=manifest_ctx
+                )
+                if (
+                    build_opts.host_type.get_package_manager() == "deb"
+                    and required_locales
+                ):
+                    # ubuntu doesn't include this by default
+                    out.write("    - name: Install locale-gen\n")
+                    out.write(f"      run: {sudo_arg}apt-get install locales\n")
+                    for loc in required_locales.split():
+                        out.write(f"    - name: Ensure {loc} locale present\n")
+                        out.write(f"      run: {sudo_arg}locale-gen {loc}\n")
 
             projects = loader.manifests_in_dependency_order()
 
@@ -1188,11 +1204,7 @@ jobs:
             out.write("        name: %s\n" % manifest.name)
             out.write("        path: _artifacts\n")
 
-            if (
-                args.enable_tests
-                and manifest.get("github.actions", "run_tests", ctx=manifest_ctx)
-                != "off"
-            ):
+            if run_tests:
                 num_jobs_arg = ""
                 if args.num_jobs:
                     num_jobs_arg = f"--num-jobs {args.num_jobs} "
@@ -1203,6 +1215,7 @@ jobs:
                 )
             if build_opts.free_up_disk and not build_opts.is_windows():
                 out.write("    - name: Show disk space at end\n")
+                out.write("      if: always()\n")
                 out.write("      run: df -h\n")
 
     def setup_project_cmd_parser(self, parser):
diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py
index ae06f4b26..6af5e3a74 100644
--- a/build/fbcode_builder/getdeps/manifest.py
+++ b/build/fbcode_builder/getdeps/manifest.py
@@ -87,6 +87,7 @@ SCHEMA = {
         "optional_section": True,
         "fields": {
             "run_tests": OPTIONAL,
+            "required_locales": OPTIONAL,
         },
     },
     "crate.pathmap": {"optional_section": True},
diff --git a/build/fbcode_builder/manifests/hexdump b/build/fbcode_builder/manifests/hexdump
new file mode 100644
index 000000000..e80674f14
--- /dev/null
+++ b/build/fbcode_builder/manifests/hexdump
@@ -0,0 +1,12 @@
+[manifest]
+name = hexdump
+
+[rpms]
+util-linux
+
+[debs]
+bsdmainutils
+
+# only used from system packages currently
+[build]
+builder = nop
diff --git a/build/fbcode_builder/manifests/sapling b/build/fbcode_builder/manifests/sapling
index a146b1ac8..bbbfe1e40 100644
--- a/build/fbcode_builder/manifests/sapling
+++ b/build/fbcode_builder/manifests/sapling
@@ -4,6 +4,9 @@ fbsource_path = fbcode/eden
 shipit_project = eden
 shipit_fbcode_builder = true
 
+[github.actions]
+required_locales = en_US.UTF-8
+
 [git]
 repo_url = https://github.com/facebook/sapling.git
 
@@ -57,6 +60,9 @@ fb303
 fbthrift
 rust-shed
 
+[dependencies.test=on]
+hexdump
+
 [dependencies.not(os=windows)]
 python