speed up file copy on windows

Summary:
X-link: https://github.com/facebookincubator/fizz/pull/155

X-link: https://github.com/facebookincubator/zstrong/pull/1085

getdeps windows file copy is very slow, speed it up

Reviewed By: bigfootjon

Differential Revision: D66830544

fbshipit-source-id: 43213e258c71ae706bb059600619e276e7491a90

build/fbcode_builder/getdeps/builder.py

@@ -18,6 +18,7 @@ import typing
 from shlex import quote as shellquote
 from typing import Optional
 
+from .copytree import simple_copytree
 from .dyndeps import create_dyn_dep_munger
 from .envfuncs import add_path_entry, Env, path_search
 from .fetcher import copy_if_different
@@ -1329,7 +1330,7 @@ class NopBuilder(BuilderBase):
                     os.makedirs(dest_parent)
                 if os.path.isdir(full_src):
                     if not os.path.exists(full_dest):
-                        shutil.copytree(full_src, full_dest)
+                        simple_copytree(full_src, full_dest)
                 else:
                     shutil.copyfile(full_src, full_dest)
                     shutil.copymode(full_src, full_dest)
@@ -1341,7 +1342,7 @@ class NopBuilder(BuilderBase):
                         os.chmod(full_dest, st.st_mode | stat.S_IXUSR)
         else:
             if not os.path.exists(self.inst_dir):
-                shutil.copytree(self.src_dir, self.inst_dir)
+                simple_copytree(self.src_dir, self.inst_dir)
 
 
 class SqliteBuilder(BuilderBase):

build/fbcode_builder/getdeps/cargo.py

@@ -13,6 +13,7 @@ import sys
 import typing
 
 from .builder import BuilderBase
+from .copytree import simple_copytree
 
 if typing.TYPE_CHECKING:
     from .buildopts import BuildOptions
@@ -79,7 +80,7 @@ class CargoBuilder(BuilderBase):
                 os.remove(dst)
             else:
                 shutil.rmtree(dst)
-        shutil.copytree(src, dst)
+        simple_copytree(src, dst)
 
     def cargo_config_file(self):
         build_source_dir = self.build_dir

build/fbcode_builder/getdeps/copytree.py

@@ -10,6 +10,7 @@ import shutil
 import subprocess
 
 from .platform import is_windows
+from .runcmd import run_cmd
 
 
 PREFETCHED_DIRS = set()
@@ -65,18 +66,34 @@ def prefetch_dir_if_eden(dirpath) -> None:
     PREFETCHED_DIRS.add(dirpath)
 
 
-# pyre-fixme[9]: ignore has type `bool`; used as `None`.
+def simple_copytree(src_dir, dest_dir, symlinks=False):
-def copytree(src_dir, dest_dir, ignore: bool = None):
+    """A simple version of shutil.copytree() that can delegate to native tools if faster"""
-    """Recursively copy the src_dir to the dest_dir, filtering
+    if is_windows():
-    out entries using the ignore lambda.  The behavior of the
+        os.makedirs(dest_dir, exist_ok=True)
-    ignore lambda must match that described by `shutil.copytree`.
+        cmd = [
-    This `copytree` function knows how to prefetch data when
+            "robocopy.exe",
-    running in an eden repo.
+            src_dir,
-    TODO: I'd like to either extend this or add a variant that
+            dest_dir,
-    uses watchman to mirror src_dir into dest_dir.
+            # copy directories, including empty ones
-    """
+            "/E",
-    prefetch_dir_if_eden(src_dir)
+            # Ignore Extra files in destination
-    # pyre-fixme[6]: For 3rd param expected
+            "/XX",
-    #  `Union[typing.Callable[[Union[PathLike[str], str], List[str]], Iterable[str]],
+            # enable parallel copy
-    #  typing.Callable[[str, List[str]], Iterable[str]], None]` but got `bool`.
+            "/MT",
-    return shutil.copytree(src_dir, dest_dir, ignore=ignore)
+            # be quiet
+            "/NFL",
+            "/NDL",
+            "/NJH",
+            "/NJS",
+            "/NP",
+        ]
+        if symlinks:
+            cmd.append("/SL")
+        # robocopy exits with code 1 if it copied ok, hence allow_fail
+        # https://learn.microsoft.com/en-us/troubleshoot/windows-server/backup-and-storage/return-codes-used-robocopy-utility
+        exit_code = run_cmd(cmd, allow_fail=True)
+        if exit_code > 1:
+            raise subprocess.CalledProcessError(exit_code, cmd)
+        return dest_dir
+    else:
+        return shutil.copytree(src_dir, dest_dir, symlinks=symlinks)