limit parallelism based on available RAM

Summary:
A long time ago, getdeps scheduled each build up to the number of hardware threads. For some builds, that was too heavy, so it got throttled back to only ever use half the hardware threads. This left parallelism on the table for CPU-bound projects that don't use much RAM per compilation.

This commit makes better use of the hardware with finer-grained logic that allows each manifest to specify a `job_weight_mib` estimate in MiB, and limit concurrency to `available_ram / job_weight`.

Reviewed By: ahornby

Differential Revision: D33754018

fbshipit-source-id: 785bed6c6cfe3c473244e0806a77cec1fc119e1f

build/fbcode_builder/getdeps/buildopts.py

@@ -16,7 +16,7 @@ from .copytree import containing_repo_type
 from .envfuncs import Env, add_path_entry
 from .fetcher import get_fbsource_repo_data
 from .manifest import ContextGenerator
-from .platform import HostType, is_windows
+from .platform import HostType, is_windows, get_available_ram
 
 
 def detect_project(path):
@@ -65,10 +65,6 @@ class BuildOptions(object):
         vcvars_path - Path to external VS toolchain's vsvarsall.bat
         shared_libs - whether to build shared libraries
         """
-        if not num_jobs:
-            import multiprocessing
-
-            num_jobs = max(1, multiprocessing.cpu_count() // 2)
 
         if not install_dir:
             install_dir = os.path.join(scratch_dir, "installed")
@@ -90,7 +86,7 @@ class BuildOptions(object):
         else:
             self.fbsource_dir = None
 
-        self.num_jobs = num_jobs
+        self.specified_num_jobs = num_jobs
         self.scratch_dir = scratch_dir
         self.install_dir = install_dir
         self.fbcode_builder_dir = fbcode_builder_dir
@@ -157,6 +153,17 @@ class BuildOptions(object):
     def is_linux(self):
         return self.host_type.is_linux()
 
+    def get_num_jobs(self, job_weight):
+        """Given an estimated job_weight in MiB, compute a reasonable concurrency limit."""
+        if self.specified_num_jobs:
+            return self.specified_num_jobs
+
+        available_ram = get_available_ram()
+
+        import multiprocessing
+
+        return max(1, min(multiprocessing.cpu_count(), available_ram // job_weight))
+
     def get_context_generator(self, host_tuple=None, facebook_internal=None):
         """Create a manifest ContextGenerator for the specified target platform."""
         if host_tuple is None:

build/fbcode_builder/getdeps/cargo.py

@@ -42,7 +42,7 @@ class CargoBuilder(BuilderBase):
             "cargo",
             operation,
             "--workspace",
-            "-j%s" % self.build_opts.num_jobs,
+            "-j%s" % self.num_jobs,
         ] + args
         self._run_cmd(cmd, cwd=self.workspace_dir(), env=env)
 

build/fbcode_builder/getdeps/manifest.py

@@ -63,6 +63,7 @@ SCHEMA = {
             "builder": REQUIRED,
             "subdir": OPTIONAL,
             "build_in_src_dir": OPTIONAL,
+            "job_weight_mib": OPTIONAL,
         },
     },
     "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}},

build/fbcode_builder/getdeps/platform.py

@@ -48,6 +48,104 @@ def get_linux_type():
     return "linux", name, version_id
 
 
+# Ideally we'd use a common library like `psutil` to read system information,
+# but getdeps can't take third-party dependencies.
+
+
+def _get_available_ram_linux():
+    # TODO: Ideally, this function would inspect the current cgroup for any
+    # limits, rather than solely relying on system RAM.
+    with open("/proc/meminfo") as f:
+        for line in f:
+            try:
+                key, value = line.split(":", 1)
+            except ValueError:
+                continue
+            suffix = " kB\n"
+            if key == "MemAvailable" and value.endswith(suffix):
+                value = value[: -len(suffix)]
+                try:
+                    return int(value) // 1024
+                except ValueError:
+                    continue
+
+    raise NotImplementedError("/proc/meminfo had no valid MemAvailable")
+
+
+def _get_available_ram_macos() -> int:
+    import ctypes.util
+
+    libc = ctypes.CDLL(ctypes.util.find_library("libc"), use_errno=True)
+    sysctlbyname = libc.sysctlbyname
+    sysctlbyname.restype = ctypes.c_int
+    sysctlbyname.argtypes = [
+        ctypes.c_char_p,
+        ctypes.c_void_p,
+        ctypes.POINTER(ctypes.c_size_t),
+        ctypes.c_void_p,
+        ctypes.c_size_t,
+    ]
+    # TODO: There may be some way to approximate an availability
+    # metric, but just use total RAM for now.
+    memsize = ctypes.c_int64()
+    memsizesize = ctypes.c_size_t(8)
+    res = sysctlbyname(
+        b"hw.memsize", ctypes.byref(memsize), ctypes.byref(memsizesize), None, 0
+    )
+    if res != 0:
+        raise NotImplementedError(
+            f"failed to retrieve hw.memsize sysctl: {ctypes.get_errno()}"
+        )
+    return memsize.value // (1024 * 1024)
+
+
+def _get_available_ram_windows() -> int:
+    import ctypes
+
+    DWORD = ctypes.c_uint32
+    QWORD = ctypes.c_uint64
+
+    class MEMORYSTATUSEX(ctypes.Structure):
+        _fields_ = [
+            ("dwLength", DWORD),
+            ("dwMemoryLoad", DWORD),
+            ("ullTotalPhys", QWORD),
+            ("ullAvailPhys", QWORD),
+            ("ullTotalPageFile", QWORD),
+            ("ullAvailPageFile", QWORD),
+            ("ullTotalVirtual", QWORD),
+            ("ullAvailVirtual", QWORD),
+            ("ullExtendedVirtual", QWORD),
+        ]
+
+    ms = MEMORYSTATUSEX()
+    ms.dwLength = ctypes.sizeof(ms)
+    # pyre-ignore[16]
+    res = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(ms))
+    if res == 0:
+        raise NotImplementedError("error calling GlobalMemoryStatusEx")
+
+    # This is fuzzy, but AvailPhys is too conservative, and AvailTotal is too
+    # aggressive, so average the two. It's okay for builds to use some swap.
+    return (ms.ullAvailPhys + ms.ullTotalPhys) // (2 * 1024 * 1024)
+
+
+def get_available_ram() -> int:
+    """
+    Returns a platform-appropriate available RAM metric in MiB.
+    """
+    if sys.platform == "linux":
+        return _get_available_ram_linux()
+    elif sys.platform == "darwin":
+        return _get_available_ram_macos()
+    elif sys.platform == "win32":
+        return _get_available_ram_windows()
+    else:
+        raise NotImplementedError(
+            f"platform {sys.platform} does not have an implementation of get_available_ram"
+        )
+
+
 class HostType(object):
     def __init__(self, ostype=None, distro=None, distrovers=None):
         if ostype is None:

build/fbcode_builder/manifests/boost

@@ -53,6 +53,7 @@ boost-devel
 
 [build]
 builder = boost
+job_weight_mib = 512
 
 [b2.args]
 --with-atomic

build/fbcode_builder/manifests/fboss

@@ -9,6 +9,8 @@ repo_url = https://github.com/facebook/fboss.git
 
 [build.os=linux]
 builder = cmake
+# fboss files take a lot of RAM to compile.
+job_weight_mib = 3072
 
 [build.not(os=linux)]
 builder = nop

build/fbcode_builder/manifests/fbthrift

@@ -9,6 +9,7 @@ repo_url = https://github.com/facebook/fbthrift.git
 
 [build]
 builder = cmake
+job_weight_mib = 2048
 
 [dependencies]
 bison

build/fbcode_builder/manifests/folly

@@ -9,6 +9,7 @@ repo_url = https://github.com/facebook/folly.git
 
 [build]
 builder = cmake
+job_weight_mib = 1024
 
 [dependencies]
 gflags