eden/scm: provide getdeps.py way of building eden/scm on GitHub

Summary:
In order to do what the title says, this diff does:
1. Add the `eden/oss/.../third-party/rust/.../Cargo.toml` files. As mentioned in the previous diff, those are required by GitHub so that the third party dependencies that are local in fbsource are properly defined with a "git" dependency in order for Cargo to "link" crates properly.
2. Changes to `eden/scm/Makefile` to add build/install commands for getdeps to invoke. Those command knowing that they are called from withing getdeps context they link the dependencies brought by getdeps into their proper places that match their folder layout in fbsource. Those Makefile commands also pass a GETDEPS_BUILD env to the setup.py invocations so that it knows it is being called withing a getdeps build.
3. Changes to `eden/scm/setup.py` that add "thriftasset" that makes use of the getdeps.py provided "thrift" binary to build .py files out of thrift files.
4. Changes to `distutils_rust` to use the vendored crates dir provided by getdeps.
5. Changes to `getdeps/builder.py` and `getdeps/manifest.py` that enable more fine-grained configuratior of how Makefile builds are invoked.
6. Changes to `getdeps/buildopts.py` and `getdeps/manifest.py` to disable overriding PATH and pkgconfig env, so that "eden/scm" builds in getdeps using system libraries rather than getdeps-provided ones (NOTE: I've tried to use getdeps provided libraries, but the trickiest bit was that Rust links with Python, which is currently not providable by getdeps, so if you try to build everything the system provided Python libraries will collide with getdeps provided ones)
7. Added `opensource/fbcode_builder/manifests/eden_scm` for the getdeps build.

Reviewed By: quark-zju

Differential Revision: D22336485

fbshipit-source-id: 244d10c9e06ee83de61e97e62a1f2a2184d2312f

build/fbcode_builder/getdeps/builder.py

@@ -111,7 +111,9 @@ class BuilderBase(object):
     def _compute_env(self, install_dirs):
         # CMAKE_PREFIX_PATH is only respected when passed through the
         # environment, so we construct an appropriate path to pass down
-        return self.build_opts.compute_env_for_install_dirs(install_dirs, env=self.env)
+        return self.build_opts.compute_env_for_install_dirs(
+            install_dirs, env=self.env, manifest=self.manifest
+        )
 
     def get_dev_run_script_path(self):
         assert self.build_opts.is_windows()
@@ -125,11 +127,22 @@ class BuilderBase(object):
 
 
 class MakeBuilder(BuilderBase):
-    def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args):
+    def __init__(
+        self,
+        build_opts,
+        ctx,
+        manifest,
+        src_dir,
+        build_dir,
+        inst_dir,
+        build_args,
+        install_args,
+    ):
         super(MakeBuilder, self).__init__(
             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
         )
-        self.args = args or []
+        self.build_args = build_args or []
+        self.install_args = install_args or []
 
     def _build(self, install_dirs, reconfigure):
         env = self._compute_env(install_dirs)
@@ -138,12 +151,12 @@ class MakeBuilder(BuilderBase):
         # libbpf uses it when generating its pkg-config file
         cmd = (
             ["make", "-j%s" % self.build_opts.num_jobs]
-            + self.args
+            + self.build_args
             + ["PREFIX=" + self.inst_dir]
         )
         self._run_cmd(cmd, env=env)
 
-        install_cmd = ["make", "install"] + self.args + ["PREFIX=" + self.inst_dir]
+        install_cmd = ["make"] + self.install_args + ["PREFIX=" + self.inst_dir]
         self._run_cmd(install_cmd, env=env)
 
 

build/fbcode_builder/getdeps/buildopts.py

@@ -178,7 +178,7 @@ class BuildOptions(object):
             }
         )
 
-    def compute_env_for_install_dirs(self, install_dirs, env=None):
+    def compute_env_for_install_dirs(self, install_dirs, env=None, manifest=None):
         if env is not None:
             env = env.copy()
         else:
@@ -214,30 +214,35 @@ class BuildOptions(object):
             lib_path = None
 
         for d in install_dirs:
-            add_path_entry(env, "CMAKE_PREFIX_PATH", d)
-
-            pkgconfig = os.path.join(d, "lib/pkgconfig")
-            if os.path.exists(pkgconfig):
-                add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
-
-            pkgconfig = os.path.join(d, "lib64/pkgconfig")
-            if os.path.exists(pkgconfig):
-                add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
-
-            # Allow resolving shared objects built earlier (eg: zstd
-            # doesn't include the full path to the dylib in its linkage
-            # so we need to give it an assist)
-            if lib_path:
-                for lib in ["lib", "lib64"]:
-                    libdir = os.path.join(d, lib)
-                    if os.path.exists(libdir):
-                        add_path_entry(env, lib_path, libdir)
-
-            # Allow resolving binaries (eg: cmake, ninja) and dlls
-            # built by earlier steps
             bindir = os.path.join(d, "bin")
-            if os.path.exists(bindir):
-                add_path_entry(env, "PATH", bindir, append=False)
+
+            if not (
+                manifest and manifest.get("build", "disable_env_override_pkgconfig")
+            ):
+                pkgconfig = os.path.join(d, "lib/pkgconfig")
+                if os.path.exists(pkgconfig):
+                    add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
+
+                pkgconfig = os.path.join(d, "lib64/pkgconfig")
+                if os.path.exists(pkgconfig):
+                    add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
+
+            if not (manifest and manifest.get("build", "disable_env_override_path")):
+                add_path_entry(env, "CMAKE_PREFIX_PATH", d)
+
+                # Allow resolving shared objects built earlier (eg: zstd
+                # doesn't include the full path to the dylib in its linkage
+                # so we need to give it an assist)
+                if lib_path:
+                    for lib in ["lib", "lib64"]:
+                        libdir = os.path.join(d, lib)
+                        if os.path.exists(libdir):
+                            add_path_entry(env, lib_path, libdir)
+
+                # Allow resolving binaries (eg: cmake, ninja) and dlls
+                # built by earlier steps
+                if os.path.exists(bindir):
+                    add_path_entry(env, "PATH", bindir, append=False)
 
             # If rustc is present in the `bin` directory, set RUSTC to prevent
             # cargo uses the rustc installed in the system.

build/fbcode_builder/getdeps/manifest.py

@@ -67,6 +67,8 @@ SCHEMA = {
             "builder": REQUIRED,
             "subdir": OPTIONAL,
             "build_in_src_dir": OPTIONAL,
+            "disable_env_override_pkgconfig": OPTIONAL,
+            "disable_env_override_path": OPTIONAL,
         },
     },
     "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}},
@@ -80,7 +82,8 @@ SCHEMA = {
     "debs": {"optional_section": True},
     "preinstalled.env": {"optional_section": True},
     "b2.args": {"optional_section": True},
-    "make.args": {"optional_section": True},
+    "make.build_args": {"optional_section": True},
+    "make.install_args": {"optional_section": True},
     "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}},
     "shipit.pathmap": {"optional_section": True},
     "shipit.strip": {"optional_section": True},
@@ -94,7 +97,8 @@ ALLOWED_EXPR_SECTIONS = [
     "build",
     "cmake.defines",
     "dependencies",
-    "make.args",
+    "make.build_args",
+    "make.install_args",
     "b2.args",
     "download",
     "git",
@@ -234,8 +238,8 @@ class ManifestParser(object):
         return defval
 
     def get_section_as_args(self, section, ctx=None):
-        """ Intended for use with the make.args and autoconf.args
-        sections, this method collects the entries and returns an
+        """ Intended for use with the make.[build_args/install_args] and
+        autoconf.args sections, this method collects the entries and returns an
         array of strings.
         If the manifest contains conditional sections, ctx is used to
         evaluate the condition and merge in the values.
@@ -421,8 +425,18 @@ class ManifestParser(object):
             print("build_dir is %s" % build_dir)  # just to quiet lint
 
         if builder == "make":
-            args = self.get_section_as_args("make.args", ctx)
-            return MakeBuilder(build_options, ctx, self, src_dir, None, inst_dir, args)
+            build_args = self.get_section_as_args("make.build_args", ctx)
+            install_args = self.get_section_as_args("make.install_args", ctx)
+            return MakeBuilder(
+                build_options,
+                ctx,
+                self,
+                src_dir,
+                None,
+                inst_dir,
+                build_args,
+                install_args,
+            )
 
         if builder == "autoconf":
             args = self.get_section_as_args("autoconf.args", ctx)

build/fbcode_builder/manifests/eden

@@ -52,6 +52,7 @@ fbcode/tools/lfs = tools/lfs
 ^fbcode/eden/hg/.*$
 ^fbcode/eden/mononoke/(?!lfs_protocol)
 ^fbcode/eden/scm/build/.*$
+^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$
 ^fbcode/eden/.*/\.cargo/.*$
 /Cargo\.lock$
 \.pyc$

build/fbcode_builder/manifests/eden_scm

+[manifest]
+name = eden_scm
+fbsource_path = fbcode/eden
+shipit_project = eden
+shipit_fbcode_builder = true
+
+[git]
+repo_url = https://github.com/facebookexperimental/eden.git
+
+[build.not(os=windows)]
+builder = make
+subdir = eden/scm
+disable_env_override_pkgconfig = 1
+disable_env_override_path = 1
+
+[build.os=windows]
+# For now the biggest blocker is missing "make" on windows, but there are bound
+# to be more
+builder = nop
+
+[make.build_args]
+getdepsbuild
+
+[make.install_args]
+install-getdeps
+
+[shipit.pathmap]
+fbcode/eden/oss = .
+fbcode/eden = eden
+fbcode/tools/lfs = tools/lfs
+fbcode/fboss/common = common
+
+[shipit.strip]
+^fbcode/eden/fs/eden-config\.h$
+^fbcode/eden/fs/py/eden/config\.py$
+^fbcode/eden/hg/.*$
+^fbcode/eden/mononoke/(?!lfs_protocol)
+^fbcode/eden/scm/build/.*$
+^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$
+^fbcode/eden/.*/\.cargo/.*$
+^.*/fb/.*$
+/Cargo\.lock$
+\.pyc$
+
+[dependencies]
+fb303
+fbthrift
+rust-shed
+
+[dependencies.fb=on]
+rust

build/fbcode_builder/manifests/libbpf

@@ -13,7 +13,11 @@ builder = nop
 builder = make
 subdir = libbpf-0.0.4/src
 
-[make.args]
+[make.build_args]
+BUILD_STATIC_ONLY=y
+
+[make.install_args]
+install
 BUILD_STATIC_ONLY=y
 
 [dependencies]

build/fbcode_builder/manifests/mononoke

@@ -30,6 +30,7 @@ tools/rust/ossconfigs = .
 [shipit.strip]
 # strip all code unrelated to mononoke to prevent triggering unnecessary checks
 ^fbcode/eden/(?!mononoke|scm/lib/xdiff.*)/.*$
+^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$
 ^fbcode/eden/mononoke/Cargo\.toml$
 ^fbcode/eden/mononoke/(?!public_autocargo).+/Cargo\.toml$
 ^fbcode/configerator/structs/scm/mononoke/(?!public_autocargo).+/Cargo\.toml$