getdeps: allow satisfying deps from system packages

Summary:
From the outset, we wanted to be sure that getdeps was able
to source and build the dependencies so that we knew that we'd have
a repeatable build.  This came at the cost of build times: having
to build boost on each CI run is a bit of a chore.

This commit adds three new elements to the manifest files:

* `rpms` - a list of RPM names that are all required to be present
  in order to consider the dependency satisfied
* `debs` - like `rpms` above, but scoped to debian package names
* `preinstalled.env` - a list of environment variables that if they
  are all set and non-empty will satisfy the dependency.

A new `--allow-system-packages` option to getdeps enables the new
logic that looks for system packages; it is off by default, but
enabled in the generated GitHub Actions workflows.

A new `install-system-deps` subcommand is provided that will attempt
to install the system packages needed to satisfy the build.  This
typically needs to be run via sudo and is thus broken out separately
from the main getdeps build flow.

I made a pass over the manifest files and added package names that
satisfy the build on ubuntu-18 and fedora-31.

shri-khare: I renamed the `Python3.7.6` manifest to just `python` as
part of this change; the version of python that it pulls in through
the normal build is the same and I believe that an equal or newer
version of python3 is available in the GH actions builder.

The `preinstalled.env` is used only by the boost manifest: it references
the name of an environment variable that is set by the github
windows hosts and that points to a pre-built and pre-installed
copy of boost.  Since there is no package manager that we can
easily query for this sort of thing, probing from the environment
seems like a reasonable and fast way to check for this.  We
may need to evolve this over time to become more feature rich,
but this seems like a good starting point.

This commit has the potential to save 20 minutes of build time
from each public CI build just due to the boost dependency alone!

Refs: https://github.com/facebook/watchman/pull/797

Reviewed By: yfeldblum

Differential Revision: D20740410

fbshipit-source-id: 6c38019449c54465127656c3d18a6ff1f30adaea

build/fbcode_builder/getdeps.py

@@ -21,9 +21,11 @@ import getdeps.cache as cache_module
 from getdeps.buildopts import setup_build_options
 from getdeps.dyndeps import create_dyn_dep_munger
 from getdeps.errors import TransientFailure
+from getdeps.fetcher import SystemPackageFetcher
 from getdeps.load import ManifestLoader
 from getdeps.manifest import ManifestParser
 from getdeps.platform import HostType
+from getdeps.runcmd import run_cmd
 from getdeps.subcmd import SubCmd, add_subcommands, cmd
 
 
@@ -310,6 +312,45 @@ class FetchCmd(ProjectCmdBase):
             fetcher.update()
 
 
+@cmd("install-system-deps", "Install system packages to satisfy the deps for a project")
+class InstallSysDepsCmd(ProjectCmdBase):
+    def setup_project_cmd_parser(self, parser):
+        parser.add_argument(
+            "--recursive",
+            help="install the transitive deps also",
+            action="store_true",
+            default=False,
+        )
+
+    def run_project_cmd(self, args, loader, manifest):
+        if args.recursive:
+            projects = loader.manifests_in_dependency_order()
+        else:
+            projects = [manifest]
+
+        cache = cache_module.create_cache()
+        all_packages = {}
+        for m in projects:
+            ctx = loader.ctx_gen.get_context(m.name)
+            packages = m.get_required_system_packages(ctx)
+            for k, v in packages.items():
+                merged = all_packages.get(k, [])
+                merged += v
+                all_packages[k] = merged
+
+        manager = loader.build_opts.host_type.get_package_manager()
+        if manager == "rpm":
+            packages = sorted(list(set(all_packages["rpm"])))
+            if packages:
+                run_cmd(["dnf", "install", "-y"] + packages)
+        elif manager == "deb":
+            packages = sorted(list(set(all_packages["deb"])))
+            if packages:
+                run_cmd(["apt", "install", "-y"] + packages)
+        else:
+            print("I don't know how to install any packages on this system")
+
+
 @cmd("list-deps", "lists the transitive deps for a given project")
 class ListDepsCmd(ProjectCmdBase):
     def run_project_cmd(self, args, loader, manifest):
@@ -402,6 +443,12 @@ class BuildCmd(ProjectCmdBase):
         for m in projects:
             fetcher = loader.create_fetcher(m)
 
+            if isinstance(fetcher, SystemPackageFetcher):
+                # We are guaranteed that if the fetcher is set to
+                # SystemPackageFetcher then this item is completely
+                # satisfied by the appropriate system packages
+                continue
+
             if args.clean:
                 fetcher.clean()
 
@@ -650,7 +697,7 @@ jobs:
             job_name = "mac"
             runs_on = "macOS-latest"
 
-        getdeps = f"{py3} build/fbcode_builder/getdeps.py"
+        getdeps = f"{py3} build/fbcode_builder/getdeps.py --allow-system-packages"
 
         out.write("  %s:\n" % job_name)
         out.write("    runs-on: %s\n" % runs_on)
@@ -662,6 +709,11 @@ jobs:
             # that we want it to use them!
             out.write("    - name: Fix Git config\n")
             out.write("      run: git config --system core.longpaths true\n")
+        else:
+            out.write("    - name: Install system deps\n")
+            out.write(
+                f"      run: sudo {getdeps} install-system-deps --recursive {manifest.name}\n"
+            )
 
         projects = loader.manifests_in_dependency_order()
 
@@ -769,6 +821,12 @@ def parse_args():
         action="store_false",
         dest="facebook_internal",
     )
+    add_common_arg(
+        "--allow-system-packages",
+        help="Allow satisfying third party deps from installed system packages",
+        action="store_true",
+        default=False,
+    )
 
     ap = argparse.ArgumentParser(
         description="Get and build dependencies and projects", parents=[common_args]

build/fbcode_builder/getdeps/buildopts.py

@@ -54,6 +54,7 @@ class BuildOptions(object):
         num_jobs=0,
         use_shipit=False,
         vcvars_path=None,
+        allow_system_packages=False,
     ):
         """ fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir,
                                  or for shipit-transformed repos, the build dir that
@@ -107,6 +108,7 @@ class BuildOptions(object):
         self.fbcode_builder_dir = fbcode_builder_dir
         self.host_type = host_type
         self.use_shipit = use_shipit
+        self.allow_system_packages = allow_system_packages
         if vcvars_path is None and is_windows():
 
             # On Windows, the compiler is not available in the PATH by
@@ -386,6 +388,13 @@ def setup_build_options(args, host_type=None):
                     temp = tempfile.gettempdir()
 
                 scratch_dir = os.path.join(temp, "fbcode_builder_getdeps-%s" % munged)
+                if not is_windows() and os.geteuid() == 0:
+                    # Running as root; in the case where someone runs
+                    # sudo getdeps.py install-system-deps
+                    # and then runs as build without privs, we want to avoid creating
+                    # a scratch dir that the second stage cannot write to.
+                    # So we generate a different path if we are root.
+                    scratch_dir += "-root"
 
         if not os.path.exists(scratch_dir):
             os.makedirs(scratch_dir)
@@ -415,4 +424,5 @@ def setup_build_options(args, host_type=None):
         num_jobs=args.num_jobs,
         use_shipit=args.use_shipit,
         vcvars_path=args.vcvars_path,
+        allow_system_packages=args.allow_system_packages,
     )

build/fbcode_builder/getdeps/fetcher.py

@@ -151,6 +151,46 @@ class LocalDirFetcher(object):
         return self.path
 
 
+class SystemPackageFetcher(object):
+    def __init__(self, build_options, packages):
+        self.manager = build_options.host_type.get_package_manager()
+        self.packages = packages.get(self.manager)
+        if self.packages:
+            self.installed = None
+        else:
+            self.installed = False
+
+    def packages_are_installed(self):
+        if self.installed is not None:
+            return self.installed
+
+        if self.manager == "rpm":
+            result = run_cmd(["rpm", "-q"] + self.packages, allow_fail=True)
+            self.installed = result == 0
+        elif self.manager == "deb":
+            result = run_cmd(["dpkg", "-s"] + self.packages, allow_fail=True)
+            self.installed = result == 0
+        else:
+            self.installed = False
+
+        return self.installed
+
+    def update(self):
+        assert self.installed
+        return ChangeStatus(all_changed=False)
+
+    def hash(self):
+        return "0" * 40
+
+    def get_src_dir(self):
+        return None
+
+
+class PreinstalledNopFetcher(SystemPackageFetcher):
+    def __init__(self):
+        self.installed = True
+
+
 class GitFetcher(Fetcher):
     DEFAULT_DEPTH = 100
 

build/fbcode_builder/getdeps/manifest.py

@@ -25,8 +25,10 @@ from .expr import parse_expr
 from .fetcher import (
     ArchiveFetcher,
     GitFetcher,
+    PreinstalledNopFetcher,
     ShipitTransformerFetcher,
     SimpleShipitTransformerFetcher,
+    SystemPackageFetcher,
 )
 from .py_wheel_builder import PythonWheelBuilder
 
@@ -74,6 +76,9 @@ SCHEMA = {
     },
     "cmake.defines": {"optional_section": True},
     "autoconf.args": {"optional_section": True},
+    "rpms": {"optional_section": True},
+    "debs": {"optional_section": True},
+    "preinstalled.env": {"optional_section": True},
     "b2.args": {"optional_section": True},
     "make.args": {"optional_section": True},
     "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}},
@@ -318,6 +323,27 @@ class ManifestParser(object):
         """ returns true if this is an FB first-party project """
         return self.shipit_project is not None
 
+    def get_required_system_packages(self, ctx):
+        """ Returns dictionary of packager system -> list of packages """
+        return {
+            "rpm": self.get_section_as_args("rpms", ctx),
+            "deb": self.get_section_as_args("debs", ctx),
+        }
+
+    def _is_satisfied_by_preinstalled_environment(self, ctx):
+        envs = self.get_section_as_args("preinstalled.env", ctx)
+        if not envs:
+            return False
+        for key in envs:
+            val = os.environ.get(key, None)
+            print(f"Testing ENV[{key}]: {repr(val)}")
+            if val is None:
+                return False
+            if len(val) == 0:
+                return False
+
+        return True
+
     def create_fetcher(self, build_options, ctx):
         use_real_shipit = (
             ShipitTransformerFetcher.available() and build_options.use_shipit
@@ -339,6 +365,16 @@ class ManifestParser(object):
             # We can use the code from fbsource
             return ShipitTransformerFetcher(build_options, self.shipit_project)
 
+        # Can we satisfy this dep with system packages?
+        if build_options.allow_system_packages:
+            if self._is_satisfied_by_preinstalled_environment(ctx):
+                return PreinstalledNopFetcher()
+
+            packages = self.get_required_system_packages(ctx)
+            package_fetcher = SystemPackageFetcher(build_options, packages)
+            if package_fetcher.packages_are_installed():
+                return package_fetcher
+
         repo_url = self.get("git", "repo_url", ctx=ctx)
         if repo_url:
             rev = self.get("git", "rev")

build/fbcode_builder/getdeps/platform.py

@@ -87,6 +87,15 @@ class HostType(object):
             self.distrovers or "none",
         )
 
+    def get_package_manager(self):
+        if not self.is_linux():
+            return None
+        if self.distro in ("fedora", "centos"):
+            return "rpm"
+        if self.distro in ("debian", "ubuntu"):
+            return "deb"
+        return None
+
     @staticmethod
     def from_tuple_string(s):
         ostype, distro, distrovers = s.split("-")

build/fbcode_builder/getdeps/runcmd.py

@@ -54,9 +54,13 @@ def run_cmd(cmd, env=None, cwd=None, allow_fail=False, log_file=None):
                 log.write(msg)
                 sys.stdout.write(msg)
 
-            _run_cmd(cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_function)
+            return _run_cmd(
+                cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_function
+            )
     else:
-        _run_cmd(cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=sys.stdout.write)
+        return _run_cmd(
+            cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=sys.stdout.write
+        )
 
 
 def _run_cmd(cmd, env, cwd, allow_fail, log_fn):

build/fbcode_builder/manifests/autoconf

 [manifest]
 name = autoconf
 
+[rpms]
+autoconf
+
+[debs]
+autoconf
+
 [download]
 url = http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz
 sha256 = 954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969

build/fbcode_builder/manifests/automake

 [manifest]
 name = automake
 
+[rpms]
+automake
+
+[debs]
+automake
+
 [download]
 url = http://ftp.gnu.org/gnu/automake/automake-1.16.1.tar.gz
 sha256 = 608a97523f97db32f1f5d5615c98ca69326ced2054c9f82e65bade7fc4c9dea8

build/fbcode_builder/manifests/bison

 [manifest]
 name = bison
 
+[rpms]
+bison
+
+[debs]
+bison
+
 [download.not(os=windows)]
 url = https://mirrors.kernel.org/gnu/bison/bison-3.3.tar.gz
 sha256 = fdeafb7fffade05604a61e66b8c040af4b2b5cbb1021dcfe498ed657ac970efd

build/fbcode_builder/manifests/boost

@@ -9,6 +9,45 @@ sha256 = 8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406
 url = https://versaweb.dl.sourceforge.net/project/boost/boost/1.69.0/boost_1_69_0.zip
 sha256 = d074bcbcc0501c4917b965fc890e303ee70d8b01ff5712bae4a6c54f2b6b4e52
 
+[preinstalled.env]
+BOOST_ROOT_1_69_0
+
+[debs]
+libboost-all-dev
+
+[rpms]
+boost
+boost-math
+boost-test
+boost-fiber
+boost-graph
+boost-log
+boost-openmpi
+boost-timer
+boost-chrono
+boost-locale
+boost-thread
+boost-atomic
+boost-random
+boost-static
+boost-contract
+boost-date-time
+boost-iostreams
+boost-container
+boost-coroutine
+boost-filesystem
+boost-system
+boost-stacktrace
+boost-regex
+boost-devel
+boost-context
+boost-python3-devel
+boost-type_erasure
+boost-wave
+boost-python3
+boost-serialization
+boost-program-options
+
 [build]
 builder = boost
 

build/fbcode_builder/manifests/cmake

 [manifest]
 name = cmake
 
+[rpms]
+cmake
+
+# All current deb based distros have a cmake that is too old
+#[debs]
+#cmake
+
 [dependencies]
 ninja
 

build/fbcode_builder/manifests/fboss

@@ -32,7 +32,7 @@ libnl
 libsai
 OpenNSA
 re2
-Python-3.7.6
+python
 
 [shipit.pathmap]
 fbcode/fboss/github = .

build/fbcode_builder/manifests/flex

 [manifest]
 name = flex
 
+[rpms]
+flex
+
+[debs]
+flex
+
 [download.not(os=windows)]
 url = https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz
 sha256 = e87aae032bf07c26f85ac0ed3250998c37621d95f8bd748b31f15b33c45ee995

build/fbcode_builder/manifests/libcurl

 [manifest]
 name = libcurl
 
+[rpms]
+libcurl-devel
+libcurl
+
+[debs]
+libcurl4-openssl-dev
+
 [download]
 url = https://curl.haxx.se/download/curl-7.65.1.tar.gz
 sha256 = 821aeb78421375f70e55381c9ad2474bf279fc454b791b7e95fc83562951c690

build/fbcode_builder/manifests/libelf

 [manifest]
 name = libelf
 
+[rpms]
+elfutils-libelf-devel-static
+
+[debs]
+libelf-dev
+
 [download]
 url = https://ftp.osuosl.org/pub/blfs/conglomeration/libelf/libelf-0.8.13.tar.gz
 sha256 = 591a9b4ec81c1f2042a97aa60564e0cb79d041c52faa7416acb38bc95bd2c76d

build/fbcode_builder/manifests/libevent

 [manifest]
 name = libevent
 
+[rpms]
+libevent-devel
+
+[debs]
+libevent-dev
+
 # Note that the CMakeLists.txt file is present only in
 # git repo and not in the release tarball, so take care
 # to use the github generated source tarball rather than

build/fbcode_builder/manifests/libgit2

 [manifest]
 name = libgit2
 
+[rpms]
+libgit2-devel
+
+[debs]
+libgit2-dev
+
 [download]
 url = https://github.com/libgit2/libgit2/archive/v0.28.1.tar.gz
 sha256 = 0ca11048795b0d6338f2e57717370208c2c97ad66c6d5eac0c97a8827d13936b

build/fbcode_builder/manifests/libmnl

 [manifest]
 name = libmnl
 
+[rpms]
+libmnl-devel
+libmnl-static
+
+[debs]
+libmnl-dev
+
 [download]
 url = http://www.lg.ps.pl/mirrors/ftp.netfilter.org/libmnl/libmnl-1.0.4.tar.bz2
 sha256 = 171f89699f286a5854b72b91d06e8f8e3683064c5901fb09d954a9ab6f551f81

build/fbcode_builder/manifests/libnl

 [manifest]
 name = libnl
 
+[rpms]
+libnl3-devel
+libnl3
+
+[debs]
+libnl-3-dev
+
 [download]
 url = https://www.infradead.org/~tgr/libnl/files/libnl-3.2.25.tar.gz
 sha256 = 8beb7590674957b931de6b7f81c530b85dc7c1ad8fbda015398bc1e8d1ce8ec5

build/fbcode_builder/manifests/libsodium

 [manifest]
 name = libsodium
 
+[rpms]
+libsodium-devel
+libsodium-static
+
+[debs]
+libsodium-dev
+
 [download.not(os=windows)]
 url = https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz
 sha256 = 0cc3dae33e642cc187b5ceb467e0ad0e1b51dcba577de1190e9ffa17766ac2b1

build/fbcode_builder/manifests/libtool

 [manifest]
 name = libtool
 
+[rpms]
+libtool
+
+[debs]
+libtool
+
 [download]
 url = http://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz
 sha256 = e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3

build/fbcode_builder/manifests/libusb

 [manifest]
 name = libusb
 
+[rpms]
+libusb-devel
+libusb
+
+[debs]
+libusb-1.0-0-dev
+
 [download]
 url = https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.tar.bz2
 sha256 = 75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157

build/fbcode_builder/manifests/libzmq

 [manifest]
 name = libzmq
 
+[rpms]
+zeromq-devel
+zeromq
+
+[debs]
+libzmq-dev
+
 [download]
 url = https://github.com/zeromq/libzmq/releases/download/v4.3.1/zeromq-4.3.1.tar.gz
 sha256 = bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb

build/fbcode_builder/manifests/lz4

 [manifest]
 name = lz4
 
+[rpms]
+lz4-devel
+lz4-static
+
+[debs]
+liblz4-dev
+
 [download]
 url = https://github.com/lz4/lz4/archive/v1.8.3.tar.gz
 sha256 = 33af5936ac06536805f9745e0b6d61da606a1f8b4cc5c04dd3cbaca3b9b4fc43

build/fbcode_builder/manifests/nghttp2

 [manifest]
 name = nghttp2
 
+[rpms]
+libnghttp2-devel
+libnghttp2
+
+[debs]
+libnghttp2-dev
+
 [download]
 url = https://github.com/nghttp2/nghttp2/releases/download/v1.39.2/nghttp2-1.39.2.tar.gz
 sha256 = fc820a305e2f410fade1a3260f09229f15c0494fc089b0100312cd64a33a38c0

build/fbcode_builder/manifests/ninja

 [manifest]
 name = ninja
 
+[rpms]
+ninja-build
+
+[debs]
+ninja-build
+
 [download.os=windows]
 url = https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip
 sha256 = 2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9

build/fbcode_builder/manifests/openssl

 [manifest]
 name = openssl
 
+[rpms]
+openssl-devel
+openssl
+
+[debs]
+libssl-dev
+
 [download]
 url = https://www.openssl.org/source/openssl-1.1.1f.tar.gz
 sha256 = 186c6bfe6ecfba7a5b48c47f8a1673d0f3b0e5ba2e25602dd23b629975da3f35

build/fbcode_builder/manifests/patchelf

 [manifest]
 name = patchelf
 
+[rpms]
+patchelf
+
+[debs]
+patchelf
+
 [download]
 url = https://github.com/NixOS/patchelf/archive/0.10.tar.gz
 sha256 = b3cb6bdedcef5607ce34a350cf0b182eb979f8f7bc31eae55a93a70a3f020d13

build/fbcode_builder/manifests/pcre

 [manifest]
 name = pcre
 
+[rpms]
+pcre-devel
+pcre-static
+
+[debs]
+libpcre3-dev
+
 [download]
 url = https://ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz
 sha256 = 0b8e7465dc5e98c757cc3650a20a7843ee4c3edf50aaf60bb33fd879690d2c73

build/fbcode_builder/manifests/Python-3.7.6 → build/fbcode_builder/manifests/python

 [manifest]
-name = Python-3.7.6
+name = python
+
+[rpms]
+python3
+python3-devel
+
+[debs]
+python3-all-dev
 
 [download.os=linux]
 url = https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tgz

build/fbcode_builder/manifests/re2

 [manifest]
 name = re2
 
+[rpms]
+re2
+re2-devel
+
+[debs]
+libre2-dev
+
 [download]
 url = https://github.com/google/re2/archive/2019-06-01.tar.gz
 sha256 = 02b7d73126bd18e9fbfe5d6375a8bb13fadaf8e99e48cbb062e4500fc18e8e2e

build/fbcode_builder/manifests/snappy

 [manifest]
 name = snappy
 
+[rpms]
+snappy
+snappy-devel
+
+[debs]
+libsnappy-dev
+
 [download]
 url = https://github.com/google/snappy/archive/1.1.7.tar.gz
 sha256 = 3dfa02e873ff51a11ee02b9ca391807f0c8ea0529a4924afa645fbf97163f9d4

build/fbcode_builder/manifests/sqlite3

 [manifest]
 name = sqlite3
 
+[rpms]
+sqlite-devel
+sqlite-libs
+
+[debs]
+libsqlite3-dev
+
 [download]
 url = https://sqlite.org/2019/sqlite-amalgamation-3280000.zip
 sha256 = d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327

build/fbcode_builder/manifests/zlib

 [manifest]
 name = zlib
 
+[rpms]
+zlib-devel
+zlib-static
+
+[debs]
+zlib1g-dev
+
 [download]
 url = http://www.zlib.net/zlib-1.2.11.tar.gz
 sha256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1

build/fbcode_builder/manifests/zstd

 [manifest]
 name = zstd
 
+[rpms]
+libzstd-devel
+libzstd
+
+[debs]
+libzstd1-dev
+
 [git]
 repo_url = https://github.com/facebook/zstd.git
 rev = v1.3.8