getdeps: Add support for brew packages and define package dependencies

Summary:
In order to speed up build times on a mac, add homebrew support to
getdeps.

Homebrew packages can be declared in a manifest using the `homebrew` header.

Futher, ahornby has added manifest entries for homebrew packages which are
included in this diff and also included a change to use the correct version of
openssl. Without this openssl change, homebrew cmake configure finds an old
openssl 1.0.2 install.

This diff provides a 2x speed up for building folly:

Timings for clean getdeps folly build on mid-2018 2.9Ghz i9 6 core intel macbook pro with 32GB RAM:

With new homebrew system deps:
```
rm -rf /Users/ahornby/.scratch/UsersZahornbyZfbsource/fbcode_builder_getdeps/
time ./opensource/fbcode_builder/getdeps.py build --allow-system-packages folly
real	17m39.329s
user	76m10.317s
sys	5m50.163s
```

Without:
```
rm -rf /Users/ahornby/.scratch/UsersZahornbyZfbsource/fbcode_builder_getdeps/
time ./opensource/fbcode_builder/getdeps.py build folly
real	32m10.344s
user	105m53.448s
sys	15m57.858s
```

Reviewed By: ahornby

Differential Revision: D33842632

fbshipit-source-id: ac785d4a8dcfa31b77292bddd9e747022ac36e3b

CMake/folly-deps.cmake

@@ -78,7 +78,7 @@ if (ZLIB_FOUND)
   list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZLIB_LIBRARIES})
 endif()
 
-find_package(OpenSSL MODULE REQUIRED)
+find_package(OpenSSL 1.1.1 MODULE REQUIRED)
 list(APPEND FOLLY_LINK_LIBRARIES ${OPENSSL_LIBRARIES})
 list(APPEND FOLLY_INCLUDE_DIRECTORIES ${OPENSSL_INCLUDE_DIR})
 list(APPEND CMAKE_REQUIRED_LIBRARIES ${OPENSSL_LIBRARIES})

README.md

@@ -88,7 +88,7 @@ The settings for folly's cmake build are held in its getdeps manifest `build/fbc
 
 ### Dependencies
 
-If on Linux you can install system dependencies to save building them:
+If on Linux or MacOS (with homebrew installed) you can install system dependencies to save building them:
 
     # Clone the repo
     git clone https://github.com/facebook/folly

build/fbcode_builder/getdeps.py

@@ -415,6 +415,11 @@ class InstallSysDepsCmd(ProjectCmdBase):
             packages = sorted(set(all_packages["deb"]))
             if packages:
                 cmd_args = ["apt", "install", "-y"] + packages
+        elif manager == "homebrew":
+            packages = sorted(set(all_packages["homebrew"]))
+            if packages:
+                cmd_args = ["brew", "install"] + packages
+
         else:
             host_tuple = loader.build_opts.host_type.as_tuple_string()
             print(

build/fbcode_builder/getdeps/buildopts.py

@@ -14,7 +14,7 @@ from typing import Optional, Mapping
 
 from .copytree import containing_repo_type
 from .envfuncs import Env, add_path_entry
-from .fetcher import get_fbsource_repo_data
+from .fetcher import get_fbsource_repo_data, homebrew_package_prefix
 from .manifest import ContextGenerator
 from .platform import HostType, is_windows, get_available_ram
 
@@ -206,6 +206,12 @@ class BuildOptions(object):
             sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"])
             env["SDKROOT"] = sdkroot.decode().strip()
 
+        # MacOS includes a version of bison so homebrew won't automatically add
+        # its own version to PATH. Find where the homebrew bison is and prepend
+        # it to PATH.
+        if self.is_darwin() and self.host_type.get_package_manager() == "homebrew":
+            add_homebrew_package_to_path(env, "bison")
+
         if self.fbsource_dir:
             env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join(
                 self.fbsource_dir, "xplat/third-party/yarn/offline-mirror"
@@ -300,6 +306,14 @@ class BuildOptions(object):
                         if os.path.isfile(cert_file):
                             env["SSL_CERT_FILE"] = cert_file
 
+        # Try extra hard to find openssl, needed with homebrew on macOS
+        if (
+            self.is_darwin()
+            and "OPENSSL_DIR" not in env
+            and "OPENSSL_ROOT_DIR" in os.environ
+        ):
+            env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"]
+
         return env
 
 
@@ -491,3 +505,9 @@ def setup_build_options(args, host_type=None):
         install_dir=args.install_prefix,
         **build_args
     )
+
+
+def add_homebrew_package_to_path(env, package):
+    prefix = homebrew_package_prefix(package)
+    if prefix and os.path.exists(os.path.join(prefix, "bin")):
+        add_path_entry(env, "PATH", os.path.join(prefix, "bin"), append=False)

build/fbcode_builder/getdeps/fetcher.py

@@ -158,6 +158,7 @@ class SystemPackageFetcher(object):
     def __init__(self, build_options, packages):
         self.manager = build_options.host_type.get_package_manager()
         self.packages = packages.get(self.manager)
+        self.host_type = build_options.host_type
         if self.packages:
             self.installed = None
         else:
@@ -172,6 +173,8 @@ class SystemPackageFetcher(object):
             cmd = ["rpm", "-q"] + sorted(self.packages)
         elif self.manager == "deb":
             cmd = ["dpkg", "-s"] + sorted(self.packages)
+        elif self.manager == "homebrew":
+            cmd = ["brew", "ls", "--versions"] + sorted(self.packages)
 
         if cmd:
             proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -181,9 +184,23 @@ class SystemPackageFetcher(object):
             else:
                 # Need all packages to be present to consider us installed
                 self.installed = False
+
         else:
             self.installed = False
 
+        # Hack to make openssl discovery with homebrew work. If openssl was
+        # built with autoconf we could use autoconf.envcmd.OPENSSL_ROOT_DIR
+        # from the manifest, but it isn't, so handle the special case here.
+        if (
+            self.installed
+            and self.host_type.is_darwin()
+            and self.manager == "homebrew"
+            and "openssl@1.1" in self.packages
+        ):
+            candidate = homebrew_package_prefix("openssl@1.1")
+            if os.path.exists(candidate):
+                os.environ["OPENSSL_ROOT_DIR"] = candidate
+
         return bool(self.installed)
 
     def update(self):
@@ -789,3 +806,14 @@ class ArchiveFetcher(Fetcher):
 
     def get_src_dir(self):
         return self.src_dir
+
+
+def homebrew_package_prefix(package):
+    cmd = ["brew", "--prefix", package]
+    try:
+        proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    except FileNotFoundError:
+        return
+
+    if proc.returncode == 0:
+        return proc.stdout.decode("utf-8").rstrip()

build/fbcode_builder/getdeps/manifest.py

@@ -81,6 +81,7 @@ SCHEMA = {
     "autoconf.envcmd.LDFLAGS": {"optional_section": True},
     "rpms": {"optional_section": True},
     "debs": {"optional_section": True},
+    "homebrew": {"optional_section": True},
     "preinstalled.env": {"optional_section": True},
     "bootstrap.args": {"optional_section": True},
     "b2.args": {"optional_section": True},
@@ -112,6 +113,7 @@ ALLOWED_EXPR_SECTIONS = [
     "debs",
     "shipit.pathmap",
     "shipit.strip",
+    "homebrew",
 ]
 
 
@@ -358,6 +360,7 @@ class ManifestParser(object):
         return {
             "rpm": self.get_section_as_args("rpms", ctx),
             "deb": self.get_section_as_args("debs", ctx),
+            "homebrew": self.get_section_as_args("homebrew", ctx),
         }
 
     def _is_satisfied_by_preinstalled_environment(self, ctx):

build/fbcode_builder/getdeps/platform.py

@@ -193,8 +193,10 @@ class HostType(object):
         )
 
     def get_package_manager(self):
-        if not self.is_linux():
+        if not self.is_linux() and not self.is_darwin():
             return None
+        if self.is_darwin():
+            return "homebrew"
         if self.distro in ("fedora", "centos", "centos_stream"):
             return "rpm"
         if self.distro.startswith(("debian", "ubuntu")):

build/fbcode_builder/manifests/autoconf

 [manifest]
 name = autoconf
 
-[rpms]
+[debs]
 autoconf
 
-[debs]
+[homebrew]
+autoconf
+
+[rpms]
 autoconf
 
 [download]

build/fbcode_builder/manifests/automake

 [manifest]
 name = automake
 
-[rpms]
+[homebrew]
 automake
 
 [debs]
 automake
 
+[rpms]
+automake
+
 [download]
 url = http://ftp.gnu.org/gnu/automake/automake-1.16.1.tar.gz
 sha256 = 608a97523f97db32f1f5d5615c98ca69326ced2054c9f82e65bade7fc4c9dea8

build/fbcode_builder/manifests/bison

 [manifest]
 name = bison
 
-[rpms]
+[debs]
 bison
 
-[debs]
+[homebrew]
+bison
+
+[rpms]
 bison
 
 [download.not(os=windows)]

build/fbcode_builder/manifests/boost

@@ -15,6 +15,9 @@ BOOST_ROOT_1_78_0
 [debs]
 libboost-all-dev
 
+[homebrew]
+boost
+
 [rpms.all(distro=centos_stream,distro_vers=8)]
 boost169
 boost169-math

build/fbcode_builder/manifests/cmake

 [manifest]
 name = cmake
 
+[homebrew]
+cmake
+
 [rpms]
 cmake
 

build/fbcode_builder/manifests/cpptoml

 [manifest]
 name = cpptoml
 
+[homebrew]
+cpptoml
+
 [download]
 url = https://github.com/chadaustin/cpptoml/archive/refs/tags/v0.1.2.tar.gz
 sha256 = beda37e94f9746874436c8090c045fd80ae6f8a51f7c668c932a2b110a4fc277

build/fbcode_builder/manifests/double-conversion

@@ -5,6 +5,9 @@ name = double-conversion
 url = https://github.com/google/double-conversion/archive/v3.1.4.tar.gz
 sha256 = 95004b65e43fefc6100f337a25da27bb99b9ef8d4071a36a33b5e83eb1f82021
 
+[homebrew]
+double-conversion
+
 [rpms]
 double-conversion
 double-conversion-devel

build/fbcode_builder/manifests/flex

 [manifest]
 name = flex
 
-[rpms]
+[debs]
 flex
 
-[debs]
+[homebrew]
+flex
+
+[rpms]
 flex
 
 [download.not(os=windows)]

build/fbcode_builder/manifests/libevent

 [manifest]
 name = libevent
 
-[rpms]
-libevent-devel
-
 [debs]
 libevent-dev
 
+[homebrew]
+libevent
+
+[rpms]
+libevent-devel
+
 # Note that the CMakeLists.txt file is present only in
 # git repo and not in the release tarball, so take care
 # to use the github generated source tarball rather than

build/fbcode_builder/manifests/libffi

 [manifest]
 name = libffi
 
+[debs]
+libffi-dev
+
+[homebrew]
+libffi
+
 [rpms]
 libffi-devel
 libffi
 
-[debs]
-libffi-dev
-
 [download]
 url = https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz
 sha256 = 540fb721619a6aba3bdeef7d940d8e9e0e6d2c193595bc243241b77ff9e93620

build/fbcode_builder/manifests/libgit2

 [manifest]
 name = libgit2
 
+[homebrew]
+libgit2
+
 [rpms]
 libgit2-devel
 

build/fbcode_builder/manifests/libsodium

 [manifest]
 name = libsodium
 
+[debs]
+libsodium-dev
+
+[homebrew]
+libsodium
+
 [rpms]
 libsodium-devel
 libsodium-static
 
-[debs]
-libsodium-dev
-
 [download.not(os=windows)]
 url = https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz
 sha256 = 0cc3dae33e642cc187b5ceb467e0ad0e1b51dcba577de1190e9ffa17766ac2b1

build/fbcode_builder/manifests/libtool

 [manifest]
 name = libtool
 
+[homebrew]
+libtool
+
 [rpms]
 libtool
 

build/fbcode_builder/manifests/libusb

 [manifest]
 name = libusb
 
+[debs]
+libusb-1.0-0-dev
+
+[homebrew]
+libusb
+
 [rpms]
 libusb-devel
 libusb
 
-[debs]
-libusb-1.0-0-dev
-
 [download]
 url = https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.tar.bz2
 sha256 = 75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157

build/fbcode_builder/manifests/libzmq

 [manifest]
 name = libzmq
 
+[debs]
+libzmq3-dev
+
+[homebrew]
+zeromq
+
 [rpms]
 zeromq-devel
 zeromq
 
-[debs]
-libzmq3-dev
-
 [download]
 url = https://github.com/zeromq/libzmq/releases/download/v4.3.1/zeromq-4.3.1.tar.gz
 sha256 = bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb

build/fbcode_builder/manifests/lz4

 [manifest]
 name = lz4
 
+[homebrew]
+lz4
+
 [rpms]
 lz4-devel
 lz4-static

build/fbcode_builder/manifests/lzo

 [manifest]
 name = lzo
 
-[rpms]
-lzo-devel
-
 [debs]
 liblzo2-dev
 
+[homebrew]
+lzo
+
+[rpms]
+lzo-devel
+
 [download]
 url = http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz 
 sha256 = c0f892943208266f9b6543b3ae308fab6284c5c90e627931446fb49b4221a072

build/fbcode_builder/manifests/ninja

 [manifest]
 name = ninja
 
-[rpms]
+[debs]
 ninja-build
 
-[debs]
+[homebrew]
+ninja
+
+[rpms]
 ninja-build
 
 [download.os=windows]

build/fbcode_builder/manifests/openssl

 [manifest]
 name = openssl
 
+[debs]
+libssl-dev
+
+[homebrew]
+openssl@1.1
+
 [rpms]
 openssl
 openssl-devel
 openssl-libs
 
-[debs]
-libssl-dev
-
 [download]
 url = https://www.openssl.org/source/openssl-1.1.1l.tar.gz
 sha256 = 0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1

build/fbcode_builder/manifests/pcre

 [manifest]
 name = pcre
 
+[homebrew]
+pcre
+
 [rpms]
 pcre-devel
 pcre-static
@@ -15,4 +18,3 @@ sha256 = 0b8e7465dc5e98c757cc3650a20a7843ee4c3edf50aaf60bb33fd879690d2c73
 [build]
 builder = cmake
 subdir = pcre-8.43
-

build/fbcode_builder/manifests/python

 [manifest]
 name = python
 
+[homebrew]
+python@3.8
+
 [rpms]
 python3
 python3-devel

build/fbcode_builder/manifests/re2

 [manifest]
 name = re2
 
-[rpms]
+[homebrew]
 re2
-re2-devel
 
 [debs]
 libre2-dev
 
+[rpms]
+re2
+re2-devel
+
 [download]
 url = https://github.com/google/re2/archive/2019-06-01.tar.gz
 sha256 = 02b7d73126bd18e9fbfe5d6375a8bb13fadaf8e99e48cbb062e4500fc18e8e2e

build/fbcode_builder/manifests/sqlite3

 [manifest]
 name = sqlite3
 
+[debs]
+libsqlite3-dev
+
+[homebrew]
+sqlite
+
 [rpms]
 sqlite-devel
 sqlite-libs
 
-[debs]
-libsqlite3-dev
-
 [download]
 url = https://sqlite.org/2019/sqlite-amalgamation-3280000.zip
 sha256 = d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327

build/fbcode_builder/manifests/tree

 [manifest]
 name = tree
 
-[rpms]
+[debs]
 tree
 
-[debs]
+[homebrew]
+tree
+
+[rpms]
 tree
 
 [download.os=linux]

build/fbcode_builder/manifests/zlib

 [manifest]
 name = zlib
 
+[debs]
+zlib1g-dev
+
+[homebrew]
+zlib
+
 [rpms]
 zlib-devel
 zlib-static
 
-[debs]
-zlib1g-dev
-
 [download]
 url = http://www.zlib.net/zlib-1.2.11.tar.gz
 sha256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1

build/fbcode_builder/manifests/zstd

 [manifest]
 name = zstd
 
+[homebrew]
+zstd
+
 [rpms]
 libzstd-devel
 libzstd