emit a script to use for running commands from the build directory

Summary:
On Windows the build artifacts cannot be easily run directly from the build
output directory without installing them.  The `$PATH` environment variable
needs to be set correctly so that the runtime library dependencies can be
found.

This updates the builder code to emit a `run.ps1` wrapper script in the build
output directory that sets `$PATH` to support running build artifacts directly
from the build directory.

Additionally, this updates the CMake-specific builder to set properly when
running the tests with `ctest`.

Reviewed By: wez

Differential Revision: D20688290

fbshipit-source-id: 5d0f4d685692bca7e37370bd88309cf7634d87f0

build/fbcode_builder/getdeps/builder.py

@@ -12,6 +12,7 @@ import stat
 import subprocess
 import sys
 
+from .dyndeps import create_dyn_dep_munger
 from .envfuncs import Env, add_path_entry, path_search
 from .fetcher import copy_if_different
 from .runcmd import run_cmd
@@ -56,7 +57,7 @@ class BuilderBase(object):
                 return [vcvarsall, "amd64", "&&"]
         return []
 
-    def _run_cmd(self, cmd, cwd=None, env=None):
+    def _run_cmd(self, cmd, cwd=None, env=None, use_cmd_prefix=True):
         if env:
             e = self.env.copy()
             e.update(env)
@@ -64,9 +65,10 @@ class BuilderBase(object):
         else:
             env = self.env
 
-        cmd_prefix = self._get_cmd_prefix()
+        if use_cmd_prefix:
-        if cmd_prefix:
+            cmd_prefix = self._get_cmd_prefix()
-            cmd = cmd_prefix + cmd
+            if cmd_prefix:
+                cmd = cmd_prefix + cmd
 
         log_file = os.path.join(self.build_dir, "getdeps_build.log")
         run_cmd(cmd=cmd, env=env, cwd=cwd or self.build_dir, log_file=log_file)
@@ -81,6 +83,16 @@ class BuilderBase(object):
 
         self._build(install_dirs=install_dirs, reconfigure=reconfigure)
 
+        # On Windows, emit a wrapper script that can be used to run build artifacts
+        # directly from the build directory, without installing them.  On Windows $PATH
+        # needs to be updated to include all of the directories containing the runtime
+        # library dependencies in order to run the binaries.
+        if self.build_opts.is_windows():
+            script_path = self.get_dev_run_script_path()
+            dep_munger = create_dyn_dep_munger(self.build_opts, install_dirs)
+            dep_dirs = self.get_dev_run_extra_path_dirs(install_dirs, dep_munger)
+            dep_munger.emit_dev_run_script(script_path, dep_dirs)
+
     def run_tests(self, install_dirs, schedule_type, owner):
         """ Execute any tests that we know how to run.  If they fail,
         raise an exception. """
@@ -100,6 +112,16 @@ class BuilderBase(object):
         # environment, so we construct an appropriate path to pass down
         return self.build_opts.compute_env_for_install_dirs(install_dirs, env=self.env)
 
+    def get_dev_run_script_path(self):
+        assert self.build_opts.is_windows()
+        return os.path.join(self.build_dir, "run.ps1")
+
+    def get_dev_run_extra_path_dirs(self, install_dirs, dep_munger=None):
+        assert self.build_opts.is_windows()
+        if dep_munger is None:
+            dep_munger = create_dyn_dep_munger(self.build_opts, install_dirs)
+        return dep_munger.compute_dependency_paths(self.build_dir)
+
 
 class MakeBuilder(BuilderBase):
     def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args):
@@ -280,7 +302,7 @@ def main():
                 "Release",
         ] + args.cmake_args
     elif args.mode == "test":
-        full_cmd = CMD_PREFIX + [CTEST] + args.cmake_args
+        full_cmd = CMD_PREFIX + [{dev_run_script}CTEST] + args.cmake_args
     else:
         ap.error("unknown invocation mode: %s" % (args.mode,))
 
@@ -335,6 +357,13 @@ if __name__ == "__main__":
         env_lines = ["    {!r}: {!r},".format(k, v) for k, v in kwargs["env"].items()]
         kwargs["env_str"] = "\n".join(["{"] + env_lines + ["}"])
 
+        if self.build_opts.is_windows():
+            kwargs["dev_run_script"] = '"powershell.exe", {!r}, '.format(
+                self.get_dev_run_script_path()
+            )
+        else:
+            kwargs["dev_run_script"] = ""
+
         define_arg_lines = ["["]
         for arg in kwargs["define_args"]:
             # Replace the CMAKE_INSTALL_PREFIX argument to use the INSTALL_DIR
@@ -461,6 +490,23 @@ if __name__ == "__main__":
         ctest = path_search(env, "ctest")
         cmake = path_search(env, "cmake")
 
+        # On Windows, we also need to update $PATH to include the directories that
+        # contain runtime library dependencies.  This is not needed on other platforms
+        # since CMake will emit RPATH properly in the binary so they can find these
+        # dependencies.
+        if self.build_opts.is_windows():
+            path_entries = self.get_dev_run_extra_path_dirs(install_dirs)
+            path = env.get("PATH")
+            if path:
+                path_entries.insert(0, path)
+            env["PATH"] = ";".join(path_entries)
+
+        # Don't use the cmd_prefix when running tests.  This is vcvarsall.bat on
+        # Windows.  vcvarsall.bat is only needed for the build, not tests.  It
+        # unfortunately fails if invoked with a long PATH environment variable when
+        # running the tests.
+        use_cmd_prefix = False
+
         def get_property(test, propname, defval=None):
             """ extracts a named property from a cmake test info json blob.
             The properties look like:
@@ -581,11 +627,13 @@ if __name__ == "__main__":
                     testpilot_args + run,
                     cwd=self.build_opts.fbcode_builder_dir,
                     env=env,
+                    use_cmd_prefix=use_cmd_prefix,
                 )
         else:
             self._run_cmd(
                 [ctest, "--output-on-failure", "-j", str(self.build_opts.num_jobs)],
                 env=env,
+                use_cmd_prefix=use_cmd_prefix,
             )
 
 

build/fbcode_builder/getdeps/dyndeps.py

@@ -5,10 +5,12 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import errno
 import glob
 import os
 import re
 import shutil
+import stat
 import subprocess
 import sys
 from struct import unpack
@@ -16,6 +18,9 @@ from struct import unpack
 from .envfuncs import path_search
 
 
+OBJECT_SUBDIRS = ("bin", "lib", "lib64")
+
+
 def copyfile(src, dest):
     shutil.copyfile(src, dest)
     shutil.copymode(src, dest)
@@ -56,7 +61,7 @@ class DepBase(object):
         inst_dir = self.install_dirs[-1]
         print("Process deps under %s" % inst_dir, file=sys.stderr)
 
-        for dir in ["bin", "lib", "lib64"]:
+        for dir in OBJECT_SUBDIRS:
             src_dir = os.path.join(inst_dir, dir)
             if not os.path.isdir(src_dir):
                 continue
@@ -70,6 +75,23 @@ class DepBase(object):
                 copyfile(os.path.join(src_dir, objfile), dest_obj)
                 self.munge_in_place(dest_obj, final_lib_dir)
 
+    def find_all_dependencies(self, build_dir):
+        all_deps = set()
+        for objfile in self.list_objs_in_dir(
+            build_dir, recurse=True, output_prefix=build_dir
+        ):
+            for d in self.list_dynamic_deps(objfile):
+                all_deps.add(d)
+
+        interesting_deps = {d for d in all_deps if self.interesting_dep(d)}
+        dep_paths = []
+        for dep in interesting_deps:
+            dep_path = self.resolve_loader_path(dep)
+            if dep_path:
+                dep_paths.append(dep_path)
+
+        return dep_paths
+
     def munge_in_place(self, objfile, final_lib_dir):
         print("Munging %s" % objfile)
         for d in self.list_dynamic_deps(objfile):
@@ -97,19 +119,26 @@ class DepBase(object):
             return dep
         d = os.path.basename(dep)
         for inst_dir in self.install_dirs:
-            for libdir in ["bin", "lib", "lib64"]:
+            for libdir in OBJECT_SUBDIRS:
                 candidate = os.path.join(inst_dir, libdir, d)
                 if os.path.exists(candidate):
                     return candidate
         return None
 
-    def list_objs_in_dir(self, dir):
+    def list_objs_in_dir(self, dir, recurse=False, output_prefix=""):
-        objs = []
+        for entry in os.listdir(dir):
-        for d in os.listdir(dir):
+            entry_path = os.path.join(dir, entry)
-            if self.is_objfile(os.path.join(dir, d)):
+            st = os.lstat(entry_path)
-                objs.append(os.path.normcase(d))
+            if stat.S_ISREG(st.st_mode):
-
+                if self.is_objfile(entry_path):
-        return objs
+                    relative_result = os.path.join(output_prefix, entry)
+                    yield os.path.normcase(relative_result)
+            elif recurse and stat.S_ISDIR(st.st_mode):
+                child_prefix = os.path.join(output_prefix, entry)
+                for result in self.list_objs_in_dir(
+                    entry_path, recurse=recurse, output_prefix=child_prefix
+                ):
+                    yield result
 
     def is_objfile(self, objfile):
         return True
@@ -194,6 +223,89 @@ class WinDeps(DepBase):
             return True
         return False
 
+    def emit_dev_run_script(self, script_path, dep_dirs):
+        """Emit a script that can be used to run build artifacts directly from the
+        build directory, without installing them.
+
+        The dep_dirs parameter should be a list of paths that need to be added to $PATH.
+        This can be computed by calling compute_dependency_paths() or
+        compute_dependency_paths_fast().
+
+        This is only necessary on Windows, which does not have RPATH, and instead
+        requires the $PATH environment variable be updated in order to find the proper
+        library dependencies.
+        """
+        contents = self._get_dev_run_script_contents(dep_dirs)
+        with open(script_path, "w") as f:
+            f.write(contents)
+
+    def compute_dependency_paths(self, build_dir):
+        """Return a list of all directories that need to be added to $PATH to ensure
+        that library dependencies can be found correctly.  This is computed by scanning
+        binaries to determine exactly the right list of dependencies.
+
+        The compute_dependency_paths_fast() is a alternative function that runs faster
+        but may return additional extraneous paths.
+        """
+        dep_dirs = set()
+        # Find paths by scanning the binaries.
+        for dep in self.find_all_dependencies(build_dir):
+            dep_dirs.add(os.path.dirname(dep))
+
+        dep_dirs.update(self.read_custom_dep_dirs(build_dir))
+        return sorted(dep_dirs)
+
+    def compute_dependency_paths_fast(self, build_dir):
+        """Similar to compute_dependency_paths(), but rather than actually scanning
+        binaries, just add all library paths from the specified installation
+        directories.  This is much faster than scanning the binaries, but may result in
+        more paths being returned than actually necessary.
+        """
+        dep_dirs = set()
+        for inst_dir in self.install_dirs:
+            for subdir in OBJECT_SUBDIRS:
+                path = os.path.join(inst_dir, subdir)
+                if os.path.exists(path):
+                    dep_dirs.add(path)
+
+        dep_dirs.update(self.read_custom_dep_dirs(build_dir))
+        return sorted(dep_dirs)
+
+    def read_custom_dep_dirs(self, build_dir):
+        # The build system may also have included libraries from other locations that
+        # we might not be able to find normally in find_all_dependencies().
+        # To handle this situation we support reading additional library paths
+        # from a LIBRARY_DEP_DIRS.txt file that may have been generated in the build
+        # output directory.
+        dep_dirs = set()
+        try:
+            explicit_dep_dirs_path = os.path.join(build_dir, "LIBRARY_DEP_DIRS.txt")
+            with open(explicit_dep_dirs_path, "r") as f:
+                for line in f.read().splitlines():
+                    dep_dirs.add(line)
+        except OSError as ex:
+            if ex.errno != errno.ENOENT:
+                raise
+
+        return dep_dirs
+
+    def _get_dev_run_script_contents(self, path_dirs):
+        path_entries = ["$env:PATH"] + path_dirs
+        path_str = ";".join(path_entries)
+        return """\
+$orig_env = $env:PATH
+$env:PATH = "{path_str}"
+
+try {{
+    $cmd_args = $args[1..$args.length]
+    & $args[0] @cmd_args
+}} finally {{
+    $env:PATH = $orig_env
+}}
+""".format(
+            path_str=path_str
+        )
+
 
 class ElfDeps(DepBase):
     def __init__(self, buildopts, install_dirs):