From 0ad429a2614441620b3e2366fb1c7502beed529a Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Fri, 6 Oct 2023 21:20:06 -0700 Subject: [PATCH] allow getdeps github actions to free up disk Summary: X-link: https://github.com/facebookincubator/velox/pull/6927 allow getdeps github actions to free up disk Allow getdeps to free up some disk from the runner and intermediate build steps as some runs (notably the linux eden and mononoke ones) are hitting disk space limits X-link: https://github.com/facebook/sapling/pull/689 Reviewed By: sggutier Differential Revision: D49875256 Pulled By: genevievehelsel fbshipit-source-id: b85b6b2f11857670915b64f47d3c0abd4ca8ca31 --- build/fbcode_builder/getdeps.py | 30 ++++++++++++++++++++++- build/fbcode_builder/getdeps/builder.py | 10 ++++++++ build/fbcode_builder/getdeps/buildopts.py | 4 +++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 9358c425e..f7baaf8e2 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -795,6 +795,12 @@ class BuildCmd(ProjectCmdBase): action="store_true", default=False, ) + parser.add_argument( + "--free-up-disk", + help="Remove unused tools and clean up intermediate files if possible to maximise space for the build", + action="store_true", + default=False, + ) @cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes") @@ -1015,6 +1021,19 @@ jobs: out.write(" - uses: actions/checkout@v2\n") + if build_opts.free_up_disk: + free_up_disk = "--free-up-disk " + if not build_opts.is_windows(): + out.write(" - name: Show disk space at start\n") + out.write(" run: df -h\n") + # remove the unused github supplied android dev tools + out.write(" - name: Free up disk space\n") + out.write(" run: sudo rm -rf /usr/local/lib/android\n") + out.write(" - name: Show disk space after freeing up\n") + out.write(" run: df -h\n") + else: + free_up_disk = "" + allow_sys_arg = "" if ( build_opts.allow_system_packages @@ -1065,7 +1084,7 @@ jobs: has_same_repo_dep = True out.write(" - name: Build %s\n" % m.name) out.write( - f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n" + f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}{free_up_disk}--no-tests {m.name}\n" ) out.write(" - name: Build %s\n" % manifest.name) @@ -1111,6 +1130,9 @@ jobs: out.write( f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" ) + if build_opts.free_up_disk and not build_opts.is_windows(): + out.write(" - name: Show disk space at end\n") + out.write(" run: df -h\n") def setup_project_cmd_parser(self, parser): parser.add_argument( @@ -1155,6 +1177,12 @@ jobs: help="add a prefix to all job names", default=None, ) + parser.add_argument( + "--free-up-disk", + help="Remove unused tools and clean up intermediate files if possible to maximise space for the build", + action="store_true", + default=False, + ) def get_arg_var_name(args): diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index e1fae8e0f..3bcadb5d6 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -136,6 +136,16 @@ class BuilderBase(object): self._prepare(install_dirs=install_dirs, reconfigure=reconfigure) self._build(install_dirs=install_dirs, reconfigure=reconfigure) + if self.build_opts.free_up_disk: + # don't clean --src-dir=. case as user may want to build again or run tests on the build + if self.src_dir.startswith(self.build_opts.scratch_dir) and os.path.isdir( + self.build_dir + ): + if os.path.islink(self.build_dir): + os.remove(self.build_dir) + else: + shutil.rmtree(self.build_dir) + # On Windows, emit a wrapper script that can be used to run build artifacts # directly from the build directory, without installing them. On Windows $PATH # needs to be updated to include all of the directories containing the runtime diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index de76a9525..48b000f90 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -51,6 +51,7 @@ class BuildOptions(object): lfs_path=None, shared_libs: bool = False, facebook_internal=None, + free_up_disk: bool = False, ) -> None: """fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir, or for shipit-transformed repos, the build dir that @@ -65,6 +66,7 @@ class BuildOptions(object): use_shipit - use real shipit instead of the simple shipit transformer vcvars_path - Path to external VS toolchain's vsvarsall.bat shared_libs - whether to build shared libraries + free_up_disk - take extra actions to save runner disk space """ if not install_dir: @@ -103,6 +105,7 @@ class BuildOptions(object): self.allow_system_packages = allow_system_packages self.lfs_path = lfs_path self.shared_libs = shared_libs + self.free_up_disk = free_up_disk lib_path = None if self.is_darwin(): @@ -602,6 +605,7 @@ def setup_build_options(args, host_type=None) -> BuildOptions: "allow_system_packages", "lfs_path", "shared_libs", + "free_up_disk", } }