opensource/fbcode_builder/getdeps.py: support cargo dependencies in cargo builds

Summary:
With this change the getdeps' CargoBuilder will support depencies between cargo builds.

The way how it works is documented in the code and required few assumptions about how a cargo project has to be defined in order to support this.

This change also adds the "mononoke" manifest and few Cargo.toml files to the mononoke project to prove that this new feature works.

Reviewed By: farnz

Differential Revision: D19468912

fbshipit-source-id: f299733cbbc2ec7bca399c898ec8d710334d0fa9

build/fbcode_builder/getdeps.py

@@ -415,7 +415,7 @@ class BuildCmd(ProjectCmdBase):
                         os.unlink(built_marker)
                     src_dir = fetcher.get_src_dir()
                     builder = m.create_builder(
-                        loader.build_opts, src_dir, build_dir, inst_dir, ctx
+                        loader.build_opts, src_dir, build_dir, inst_dir, ctx, loader
                     )
                     builder.build(install_dirs, reconfigure=reconfigure)
 
@@ -544,7 +544,7 @@ class TestCmd(ProjectCmdBase):
                 ctx = loader.ctx_gen.get_context(m.name)
                 build_dir = loader.get_project_build_dir(m)
                 builder = m.create_builder(
-                    loader.build_opts, src_dir, build_dir, inst_dir, ctx
+                    loader.build_opts, src_dir, build_dir, inst_dir, ctx, loader
                 )
                 builder.run_tests(
                     install_dirs,

build/fbcode_builder/getdeps/builder.py

@@ -813,19 +813,25 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include)
 
 class CargoBuilder(BuilderBase):
     def __init__(
-        self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, build_doc
+        self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, build_doc, loader
     ):
         super(CargoBuilder, self).__init__(
             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
         )
         self.build_doc = build_doc
+        self.loader = loader
 
     def run_cargo(self, install_dirs, operation, args=None):
         args = args or []
         env = self._compute_env(install_dirs)
         # Enable using nightly features with stable compiler
         env["RUSTC_BOOTSTRAP"] = "1"
-        cmd = ["cargo", operation, "-j%s" % self.build_opts.num_jobs] + args
+        cmd = [
+            "cargo",
+            operation,
+            "--workspace",
+            "-j%s" % self.build_opts.num_jobs,
+        ] + args
         self._run_cmd(cmd, cwd=self.build_source_dir(), env=env)
 
     def build_source_dir(self):
@@ -856,6 +862,8 @@ git-fetch-with-cli = true
                 )
             )
 
+        self._patchup_workspace(build_source_dir)
+
         try:
             from getdeps.facebook.lfs import crates_io_download
 
@@ -872,3 +880,163 @@ git-fetch-with-cli = true
         self.run_cargo(install_dirs, "test")
         if self.build_doc:
             self.run_cargo(install_dirs, "doc", ["--no-deps"])
+
+    def _patchup_workspace(self, build_source_dir):
+        """
+        This method makes a lot of assumptions about the state of the project
+        and its cargo dependendies:
+        1. There is a virtual manifest with workspace in the root of this project
+        2. Crates from cargo dependencies can be extracted from Cargo.toml files
+           using _extract_crates function. It is using a heuristic so check its
+           code to understand how it is done.
+        3. The extracted cargo dependencies crates can be found in the
+           dependency's install dir using _resolve_crate_to_path function
+           which again is using a heuristic.
+
+        Notice that many things might go wrong here. E.g. if someone depends
+        on another getdeps crate by writing in their Cargo.toml file:
+
+            my-rename-of-crate = { package = "crate", git = "..." }
+
+        they can count themselves lucky because the code will raise an
+        Exception. There migh be more cases where the code will silently pass
+        producing bad results.
+        """
+        config = self._resolve_config(build_source_dir)
+        if config:
+            with open(os.path.join(build_source_dir, "Cargo.toml"), "a") as f:
+                # A fake manifest has to be crated to change the virtual
+                # manifest into a non-virtual. The virtual manifests are limited
+                # in many ways and the inability to define patches on them is
+                # one. Check https://github.com/rust-lang/cargo/issues/4934 to
+                # see if it is resolved.
+                f.write(
+                    """
+[package]
+name = "fake_manifest_of_{}"
+version = "0.0.0"
+[lib]
+path = "/dev/null"
+""".format(
+                        self.manifest.name
+                    )
+                )
+                f.write(config)
+
+    def _resolve_config(self, build_source_dir):
+        """
+        Returns a configuration to be put inside root Cargo.toml file which
+        patches the dependencies git code with local getdeps versions.
+        See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section
+        """
+        dep_to_git = self._resolve_dep_to_git()
+        dep_to_crates = CargoBuilder._resolve_dep_to_crates(
+            build_source_dir, dep_to_git
+        )
+
+        config = []
+        for name in sorted(dep_to_git.keys()):
+            git_conf = dep_to_git[name]
+            crates = sorted(dep_to_crates.get(name, []))
+            if not crates:
+                continue  # nothing to patch, move along
+            crates_patches = [
+                '{} = {{ path = "{}" }}'.format(
+                    crate,
+                    CargoBuilder._resolve_crate_to_path(crate, git_conf).replace(
+                        "\\", "\\\\"
+                    ),
+                )
+                for crate in crates
+            ]
+
+            config.append(
+                '[patch."{0}"]\n'.format(git_conf["repo_url"])
+                + "\n".join(crates_patches)
+            )
+        return "\n".join(config)
+
+    def _resolve_dep_to_git(self):
+        """
+        For each direct dependency of the currently build manifest check if it
+        is also cargo-builded and if yes then extract it's git configs and
+    install dir
+        """
+        dependencies = self.manifest.get_section_as_dict("dependencies", ctx=self.ctx)
+        if not dependencies:
+            return []
+
+        dep_to_git = {}
+        for dep in dependencies.keys():
+            dep_manifest = self.loader.load_manifest(dep)
+            if dep_manifest.get("build", "builder", ctx=self.ctx) != "cargo":
+                # This is a direct dependency, but it is not build with cargo
+                # so ignore it.
+                continue
+
+            git_conf = dep_manifest.get_section_as_dict("git", ctx=self.ctx)
+            if "repo_url" not in git_conf:
+                raise Exception(
+                    "A cargo dependency requires git.repo_url to be defined."
+                )
+            git_conf["inst_dir"] = self.loader.get_project_install_dir(dep_manifest)
+            dep_to_git[dep] = git_conf
+        return dep_to_git
+
+    @staticmethod
+    def _resolve_dep_to_crates(build_source_dir, dep_to_git):
+        """
+        This function traverse the build_source_dir in search of Cargo.toml
+        files, extracts the crate names from them using _extract_crates
+        function and returns a merged result containing crate names per
+        dependency name from all Cargo.toml files in the project.
+        """
+        if not dep_to_git:
+            return {}  # no deps, so don't waste time traversing files
+
+        dep_to_crates = {}
+        for root, _, files in os.walk(build_source_dir):
+            for f in files:
+                if f == "Cargo.toml":
+                    more_dep_to_crates = CargoBuilder._extract_crates(
+                        os.path.join(root, f), dep_to_git
+                    )
+                    for name, crates in more_dep_to_crates.items():
+                        dep_to_crates.setdefault(name, set()).update(crates)
+        return dep_to_crates
+
+    @staticmethod
+    def _extract_crates(cargo_toml_file, dep_to_git):
+        """
+        This functions reads content of provided cargo toml file and extracts
+        crate names per each dependency. The extraction is done by a heuristic
+        so it might be incorrect.
+        """
+        deps_to_crates = {}
+        with open(cargo_toml_file, "r") as f:
+            for line in f.readlines():
+                if line.startswith("#") or "git = " not in line:
+                    continue  # filter out commented lines and ones without git deps
+                for name, conf in dep_to_git.items():
+                    if 'git = "{}"'.format(conf["repo_url"]) in line:
+                        crate_name, _, _ = line.partition("=")
+                        deps_to_crates.setdefault(name, set()).add(crate_name.strip())
+        return deps_to_crates
+
+    @staticmethod
+    def _resolve_crate_to_path(crate, git_conf):
+        """
+        Tries to find <crate> in git_conf["inst_dir"] by searching a [package]
+        keyword followed by name = "<crate>".
+        """
+        source_dir = os.path.join(git_conf["inst_dir"], "source")
+        search_pattern = '[package]\nname = "{}"'.format(crate)
+
+        for root, _, files in os.walk(source_dir):
+            for fname in files:
+                if fname == "Cargo.toml":
+                    with open(os.path.join(root, fname), "r") as f:
+                        if search_pattern in f.read():
+                            return root
+
+        raise Exception("Failed to found crate {} in path {}".format(crate, source_dir))

build/fbcode_builder/getdeps/manifest.py

@@ -364,7 +364,7 @@ class ManifestParser(object):
             "project %s has no fetcher configuration matching %s" % (self.name, ctx)
         )
 
-    def create_builder(self, build_options, src_dir, build_dir, inst_dir, ctx):
+    def create_builder(self, build_options, src_dir, build_dir, inst_dir, ctx, loader):
         builder = self.get("build", "builder", ctx=ctx)
         if not builder:
             raise Exception("project %s has no builder for %r" % (self.name, ctx))
@@ -422,7 +422,14 @@ class ManifestParser(object):
         if builder == "cargo":
             build_doc = self.get("cargo", "build_doc", False, ctx)
             return CargoBuilder(
-                build_options, ctx, self, src_dir, build_dir, inst_dir, build_doc
+                build_options,
+                ctx,
+                self,
+                src_dir,
+                build_dir,
+                inst_dir,
+                build_doc,
+                loader,
             )
 
         if builder == "OpenNSA":

build/fbcode_builder/manifests/mononoke

+[manifest]
+name = mononoke
+fbsource_path = fbcode/scm/mononoke
+shipit_project = mononoke
+shipit_fbcode_builder = true
+
+[git]
+repo_url = https://github.com/facebookexperimental/mononoke.git
+
+[build.not(os=windows)]
+builder = cargo
+
+[build.os=windows]
+# building Mononoke on windows is not supported
+builder = nop
+
+[cargo]
+build_doc = true
+
+[shipit.pathmap]
+fbcode/scm/mononoke = mononoke
+fbcode/scm/mononoke/public_autocargo = mononoke
+fbcode/scm/mononoke/public_tld = .
+tools/rust/ossconfigs = .
+
+[shipit.strip]
+^fbcode/scm/mononoke/(?!public_autocargo|public_tld).+/Cargo\.toml$
+
+[dependencies]
+rust-shed
+
+[dependencies.fb=on]
+rust