limit parallelism based on available RAM

Summary:
A long time ago, getdeps scheduled each build up to the number of hardware threads. For some builds, that was too heavy, so it got throttled back to only ever use half the hardware threads. This left parallelism on the table for CPU-bound projects that don't use much RAM per compilation.

This commit makes better use of the hardware with finer-grained logic that allows each manifest to specify a `job_weight_mib` estimate in MiB, and limit concurrency to `available_ram / job_weight`.

Reviewed By: ahornby

Differential Revision: D33754018

fbshipit-source-id: 785bed6c6cfe3c473244e0806a77cec1fc119e1f

build/fbcode_builder/getdeps/platform.py

@@ -48,6 +48,104 @@ def get_linux_type():
     return "linux", name, version_id
 
 
+# Ideally we'd use a common library like `psutil` to read system information,
+# but getdeps can't take third-party dependencies.
+
+
+def _get_available_ram_linux():
+    # TODO: Ideally, this function would inspect the current cgroup for any
+    # limits, rather than solely relying on system RAM.
+    with open("/proc/meminfo") as f:
+        for line in f:
+            try:
+                key, value = line.split(":", 1)
+            except ValueError:
+                continue
+            suffix = " kB\n"
+            if key == "MemAvailable" and value.endswith(suffix):
+                value = value[: -len(suffix)]
+                try:
+                    return int(value) // 1024
+                except ValueError:
+                    continue
+
+    raise NotImplementedError("/proc/meminfo had no valid MemAvailable")
+
+
+def _get_available_ram_macos() -> int:
+    import ctypes.util
+
+    libc = ctypes.CDLL(ctypes.util.find_library("libc"), use_errno=True)
+    sysctlbyname = libc.sysctlbyname
+    sysctlbyname.restype = ctypes.c_int
+    sysctlbyname.argtypes = [
+        ctypes.c_char_p,
+        ctypes.c_void_p,
+        ctypes.POINTER(ctypes.c_size_t),
+        ctypes.c_void_p,
+        ctypes.c_size_t,
+    ]
+    # TODO: There may be some way to approximate an availability
+    # metric, but just use total RAM for now.
+    memsize = ctypes.c_int64()
+    memsizesize = ctypes.c_size_t(8)
+    res = sysctlbyname(
+        b"hw.memsize", ctypes.byref(memsize), ctypes.byref(memsizesize), None, 0
+    )
+    if res != 0:
+        raise NotImplementedError(
+            f"failed to retrieve hw.memsize sysctl: {ctypes.get_errno()}"
+        )
+    return memsize.value // (1024 * 1024)
+
+
+def _get_available_ram_windows() -> int:
+    import ctypes
+
+    DWORD = ctypes.c_uint32
+    QWORD = ctypes.c_uint64
+
+    class MEMORYSTATUSEX(ctypes.Structure):
+        _fields_ = [
+            ("dwLength", DWORD),
+            ("dwMemoryLoad", DWORD),
+            ("ullTotalPhys", QWORD),
+            ("ullAvailPhys", QWORD),
+            ("ullTotalPageFile", QWORD),
+            ("ullAvailPageFile", QWORD),
+            ("ullTotalVirtual", QWORD),
+            ("ullAvailVirtual", QWORD),
+            ("ullExtendedVirtual", QWORD),
+        ]
+
+    ms = MEMORYSTATUSEX()
+    ms.dwLength = ctypes.sizeof(ms)
+    # pyre-ignore[16]
+    res = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(ms))
+    if res == 0:
+        raise NotImplementedError("error calling GlobalMemoryStatusEx")
+
+    # This is fuzzy, but AvailPhys is too conservative, and AvailTotal is too
+    # aggressive, so average the two. It's okay for builds to use some swap.
+    return (ms.ullAvailPhys + ms.ullTotalPhys) // (2 * 1024 * 1024)
+
+
+def get_available_ram() -> int:
+    """
+    Returns a platform-appropriate available RAM metric in MiB.
+    """
+    if sys.platform == "linux":
+        return _get_available_ram_linux()
+    elif sys.platform == "darwin":
+        return _get_available_ram_macos()
+    elif sys.platform == "win32":
+        return _get_available_ram_windows()
+    else:
+        raise NotImplementedError(
+            f"platform {sys.platform} does not have an implementation of get_available_ram"
+        )
+
+
 class HostType(object):
     def __init__(self, ostype=None, distro=None, distrovers=None):
         if ostype is None: