Opt in opensource/fbcode_builder to pyfmt

Reviewed By: zertosh

Differential Revision: D29612107

fbshipit-source-id: ac450058134e23a3831db35d2e49c80eb8cde36a

build/fbcode_builder/CMake/fb_py_test_main.py

@@ -485,7 +485,7 @@ class Loader(object):
         return loader.suiteClass(suites)
 
 
-_COVERAGE_INI = '''\
+_COVERAGE_INI = """\
 [report]
 exclude_lines =
     pragma: no cover
@@ -495,7 +495,7 @@ exclude_lines =
     pragma:.*no${PY_IMPL}${PY_MAJOR}
     pragma:.*nopy${PY_MAJOR}
     pragma:.*nopy${PY_MAJOR}${PY_MINOR}
-'''
+"""
 
 
 class MainProgram(object):
@@ -734,7 +734,7 @@ class MainProgram(object):
         if not self.options.collect_coverage:
             return
 
-        with tempfile.NamedTemporaryFile('w', delete=False) as coverage_ini:
+        with tempfile.NamedTemporaryFile("w", delete=False) as coverage_ini:
             coverage_ini.write(_COVERAGE_INI)
             self._coverage_ini_path = coverage_ini.name
 

build/fbcode_builder/CMake/make_fbpy_archive.py

@@ -124,7 +124,7 @@ def populate_install_tree(inst_dir, path_map):
 
 
 def build_zipapp(args, path_map):
-    """ Create a self executing python binary using Python 3's built-in
+    """Create a self executing python binary using Python 3's built-in
     zipapp module.
 
     This type of Python binary is relatively simple, as zipapp is part of the
@@ -165,7 +165,7 @@ if __name__ == "__main__":
 
 
 def build_install_dir(args, path_map):
-    """ Create a directory that contains all of the sources, with a __main__
+    """Create a directory that contains all of the sources, with a __main__
     module to run the program.
     """
     # Populate a temporary directory first, then rename to the destination
@@ -188,7 +188,7 @@ def ensure_directory(path):
 
 
 def install_library(args, path_map):
-    """ Create an installation directory a python library. """
+    """Create an installation directory a python library."""
     out_dir = args.output
     out_manifest = args.output + ".manifest"
 

build/fbcode_builder/docker_builder.py

@@ -4,7 +4,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-'''
+
+"""
 
 Extends FBCodeBuilder to produce Docker context directories.
 
@@ -15,26 +16,23 @@ caching, you will want to:
    that change the least often, and
  - Put the steps that you are debugging towards the very end.
 
-'''
+"""
 import logging
 import os
 import shutil
 import tempfile
 
 from fbcode_builder import FBCodeBuilder
-from shell_quoting import (
-    raw_shell, shell_comment, shell_join, ShellQuoted, path_join
-)
+from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted, path_join
 from utils import recursively_flatten_list, run_command
 
 
 class DockerFBCodeBuilder(FBCodeBuilder):
-
     def _user(self):
-        return self.option('user', 'root')
+        return self.option("user", "root")
 
     def _change_user(self):
-        return ShellQuoted('USER {u}').format(u=self._user())
+        return ShellQuoted("USER {u}").format(u=self._user())
 
     def setup(self):
         # Please add RPM-based OSes here as appropriate.
@@ -63,17 +61,18 @@ class DockerFBCodeBuilder(FBCodeBuilder):
         # it is present when the resulting container is run add to PATH
         actions = []
         if self.option("PYTHON_VENV", "OFF") == "ON":
-            actions = ShellQuoted('ENV PATH={p}:$PATH').format(
-                p=path_join(self.option('prefix'), "venv", "bin"))
-        return(actions)
+            actions = ShellQuoted("ENV PATH={p}:$PATH").format(
+                p=path_join(self.option("prefix"), "venv", "bin")
+            )
+        return actions
 
     def step(self, name, actions):
-        assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
-        b = ShellQuoted('')
-        return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]
+        assert "\n" not in name, "Name {0} would span > 1 line".format(name)
+        b = ShellQuoted("")
+        return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b]
 
     def run(self, shell_cmd):
-        return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd)
+        return ShellQuoted("RUN {cmd}").format(cmd=shell_cmd)
 
     def set_env(self, key, value):
         return ShellQuoted("ENV {key}={val}").format(key=key, val=value)
@@ -84,12 +83,12 @@ class DockerFBCodeBuilder(FBCodeBuilder):
             # by root:root -- the explicit `mkdir` works around the bug:
             #   USER nobody
             #   WORKDIR build
-            ShellQuoted('USER root'),
-            ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format(
+            ShellQuoted("USER root"),
+            ShellQuoted("RUN mkdir -p {d} && chown {u} {d}").format(
                 d=dir, u=self._user()
             ),
             self._change_user(),
-            ShellQuoted('WORKDIR {dir}').format(dir=dir),
+            ShellQuoted("WORKDIR {dir}").format(dir=dir),
         ]
 
     def comment(self, comment):
@@ -99,60 +98,58 @@ class DockerFBCodeBuilder(FBCodeBuilder):
 
     def copy_local_repo(self, repo_dir, dest_name):
         fd, archive_path = tempfile.mkstemp(
-            prefix='local_repo_{0}_'.format(dest_name),
-            suffix='.tgz',
-            dir=os.path.abspath(self.option('docker_context_dir')),
+            prefix="local_repo_{0}_".format(dest_name),
+            suffix=".tgz",
+            dir=os.path.abspath(self.option("docker_context_dir")),
         )
         os.close(fd)
-        run_command('tar', 'czf', archive_path, '.', cwd=repo_dir)
+        run_command("tar", "czf", archive_path, ".", cwd=repo_dir)
         return [
-            ShellQuoted('ADD {archive} {dest_name}').format(
+            ShellQuoted("ADD {archive} {dest_name}").format(
                 archive=os.path.basename(archive_path), dest_name=dest_name
             ),
             # Docker permissions make very little sense... see also workdir()
-            ShellQuoted('USER root'),
-            ShellQuoted('RUN chown -R {u} {d}').format(
-                d=dest_name, u=self._user()
-            ),
+            ShellQuoted("USER root"),
+            ShellQuoted("RUN chown -R {u} {d}").format(d=dest_name, u=self._user()),
             self._change_user(),
         ]
 
     def _render_impl(self, steps):
-        return raw_shell(shell_join('\n', recursively_flatten_list(steps)))
+        return raw_shell(shell_join("\n", recursively_flatten_list(steps)))
 
     def debian_ccache_setup_steps(self):
-        source_ccache_tgz = self.option('ccache_tgz', '')
+        source_ccache_tgz = self.option("ccache_tgz", "")
         if not source_ccache_tgz:
-            logging.info('Docker ccache not enabled')
+            logging.info("Docker ccache not enabled")
             return []
 
-        dest_ccache_tgz = os.path.join(
-            self.option('docker_context_dir'), 'ccache.tgz'
-        )
+        dest_ccache_tgz = os.path.join(self.option("docker_context_dir"), "ccache.tgz")
 
         try:
             try:
                 os.link(source_ccache_tgz, dest_ccache_tgz)
             except OSError:
                 logging.exception(
-                    'Hard-linking {s} to {d} failed, falling back to copy'
-                    .format(s=source_ccache_tgz, d=dest_ccache_tgz)
+                    "Hard-linking {s} to {d} failed, falling back to copy".format(
+                        s=source_ccache_tgz, d=dest_ccache_tgz
+                    )
                 )
                 shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
         except Exception:
             logging.exception(
-                'Failed to copy or link {s} to {d}, aborting'
-                .format(s=source_ccache_tgz, d=dest_ccache_tgz)
+                "Failed to copy or link {s} to {d}, aborting".format(
+                    s=source_ccache_tgz, d=dest_ccache_tgz
+                )
             )
             raise
 
         return [
             # Separate layer so that in development we avoid re-downloads.
-            self.run(ShellQuoted('apt-get install -yq ccache')),
-            ShellQuoted('ADD ccache.tgz /'),
+            self.run(ShellQuoted("apt-get install -yq ccache")),
+            ShellQuoted("ADD ccache.tgz /"),
             ShellQuoted(
                 # Set CCACHE_DIR before the `ccache` invocations below.
-                'ENV CCACHE_DIR=/ccache '
+                "ENV CCACHE_DIR=/ccache "
                 # No clang support for now, so it's easiest to hardcode gcc.
                 'CC="ccache gcc" CXX="ccache g++" '
                 # Always log for ease of debugging. For real FB projects,
@@ -166,9 +163,10 @@ class DockerFBCodeBuilder(FBCodeBuilder):
                 #
                 #   apt-get install sharutils
                 #   bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
-                'CCACHE_LOGFILE=/tmp/ccache.log'
+                "CCACHE_LOGFILE=/tmp/ccache.log"
             ),
-            self.run(ShellQuoted(
+            self.run(
+                ShellQuoted(
                     # Future: Skipping this part made this Docker step instant,
                     # saving ~1min of build time.  It's unclear if it is the
                     # chown or the du, but probably the chown -- since a large
@@ -176,16 +174,17 @@ class DockerFBCodeBuilder(FBCodeBuilder):
                     #
                     # ccache.tgz may be empty, or may have the wrong
                     # permissions.
-                'mkdir -p /ccache && time chown -R nobody /ccache && '
-                'time du -sh /ccache && '
+                    "mkdir -p /ccache && time chown -R nobody /ccache && "
+                    "time du -sh /ccache && "
                     # Reset stats so `docker_build_with_ccache.sh` can print
                     # useful values at the end of the run.
-                'echo === Prev run stats === && ccache -s && ccache -z && '
+                    "echo === Prev run stats === && ccache -s && ccache -z && "
                     # Record the current time to let travis_build.sh figure out
                     # the number of bytes in the cache that are actually used --
                     # this is crucial for tuning the maximum cache size.
-                'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && '
+                    "date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && "
                     # The build running as `nobody` should be able to write here
-                'chown nobody /tmp/ccache.log'
-            )),
+                    "chown nobody /tmp/ccache.log"
+                )
+            ),
         ]

build/fbcode_builder/fbcode_builder_config.py

@@ -4,12 +4,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-'Demo config, so that `make_docker_context.py --help` works in this directory.'
+
+"Demo config, so that `make_docker_context.py --help` works in this directory."
 
 config = {
-    'fbcode_builder_spec': lambda _builder: {
-        'depends_on': [],
-        'steps': [],
+    "fbcode_builder_spec": lambda _builder: {
+        "depends_on": [],
+        "steps": [],
     },
-    'github_project': 'demo/project',
+    "github_project": "demo/project",
 }

build/fbcode_builder/make_docker_context.py

@@ -4,7 +4,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-'''
+
+"""
 Reads `fbcode_builder_config.py` from the current directory, and prepares a
 Docker context directory to build this project.  Prints to stdout the path
 to the context directory.
@@ -14,7 +15,7 @@ Try `.../make_docker_context.py --help` from a project's `build/` directory.
 By default, the Docker context directory will be in /tmp. It will always
 contain a Dockerfile, and might also contain copies of your local repos, and
 other data needed for the build container.
-'''
+"""
 
 import os
 import tempfile
@@ -27,7 +28,7 @@ from parse_args import parse_args_to_fbcode_builder_opts
 def make_docker_context(
     get_steps_fn, github_project, opts=None, default_context_dir=None
 ):
-    '''
+    """
     Returns a path to the Docker context directory. See parse_args.py.
 
     Helper for making a command-line utility that writes your project's
@@ -38,83 +39,97 @@ def make_docker_context(
             lambda builder: [builder.step(...), ...],
             'facebook/your_project',
         ))
-    '''
+    """
 
     if opts is None:
         opts = {}
 
     valid_versions = (
-        ('ubuntu:16.04', '5'),
-        ('ubuntu:18.04', '7'),
+        ("ubuntu:16.04", "5"),
+        ("ubuntu:18.04", "7"),
     )
 
     def add_args(parser):
         parser.add_argument(
-            '--docker-context-dir', metavar='DIR',
+            "--docker-context-dir",
+            metavar="DIR",
             default=default_context_dir,
-            help='Write the Dockerfile and its context into this directory. '
-                'If empty, make a temporary directory. Default: %(default)s.',
+            help="Write the Dockerfile and its context into this directory. "
+            "If empty, make a temporary directory. Default: %(default)s.",
         )
         parser.add_argument(
-            '--user', metavar='NAME', default=opts.get('user', 'nobody'),
-            help='Build and install as this user. Default: %(default)s.',
+            "--user",
+            metavar="NAME",
+            default=opts.get("user", "nobody"),
+            help="Build and install as this user. Default: %(default)s.",
         )
         parser.add_argument(
-            '--prefix', metavar='DIR',
-            default=opts.get('prefix', '/home/install'),
-            help='Install all libraries in this prefix. Default: %(default)s.',
+            "--prefix",
+            metavar="DIR",
+            default=opts.get("prefix", "/home/install"),
+            help="Install all libraries in this prefix. Default: %(default)s.",
         )
         parser.add_argument(
-            '--projects-dir', metavar='DIR',
-            default=opts.get('projects_dir', '/home'),
-            help='Place project code directories here. Default: %(default)s.',
+            "--projects-dir",
+            metavar="DIR",
+            default=opts.get("projects_dir", "/home"),
+            help="Place project code directories here. Default: %(default)s.",
         )
         parser.add_argument(
-            '--os-image', metavar='IMG', choices=zip(*valid_versions)[0],
-            default=opts.get('os_image', valid_versions[0][0]),
-            help='Docker OS image -- be sure to use only ones you trust (See '
-                'README.docker). Choices: %(choices)s. Default: %(default)s.',
+            "--os-image",
+            metavar="IMG",
+            choices=zip(*valid_versions)[0],
+            default=opts.get("os_image", valid_versions[0][0]),
+            help="Docker OS image -- be sure to use only ones you trust (See "
+            "README.docker). Choices: %(choices)s. Default: %(default)s.",
         )
         parser.add_argument(
-            '--gcc-version', metavar='VER',
+            "--gcc-version",
+            metavar="VER",
             choices=set(zip(*valid_versions)[1]),
-            default=opts.get('gcc_version', valid_versions[0][1]),
-            help='Choices: %(choices)s. Default: %(default)s.',
+            default=opts.get("gcc_version", valid_versions[0][1]),
+            help="Choices: %(choices)s. Default: %(default)s.",
         )
         parser.add_argument(
-            '--make-parallelism', metavar='NUM', type=int,
-            default=opts.get('make_parallelism', 1),
-            help='Use `make -j` on multi-CPU systems with lots of RAM. '
-                'Default: %(default)s.',
+            "--make-parallelism",
+            metavar="NUM",
+            type=int,
+            default=opts.get("make_parallelism", 1),
+            help="Use `make -j` on multi-CPU systems with lots of RAM. "
+            "Default: %(default)s.",
         )
         parser.add_argument(
-            '--local-repo-dir', metavar='DIR',
-            help='If set, build {0} from a local directory instead of Github.'
-                .format(github_project),
+            "--local-repo-dir",
+            metavar="DIR",
+            help="If set, build {0} from a local directory instead of Github.".format(
+                github_project
+            ),
         )
         parser.add_argument(
-            '--ccache-tgz', metavar='PATH',
-            help='If set, enable ccache for the build. To initialize the '
-                 'cache, first try to hardlink, then to copy --cache-tgz '
-                 'as ccache.tgz into the --docker-context-dir.'
+            "--ccache-tgz",
+            metavar="PATH",
+            help="If set, enable ccache for the build. To initialize the "
+            "cache, first try to hardlink, then to copy --cache-tgz "
+            "as ccache.tgz into the --docker-context-dir.",
         )
 
     opts = parse_args_to_fbcode_builder_opts(
         add_args,
         # These have add_argument() calls, others are set via --option.
         (
-            'docker_context_dir',
-            'user',
-            'prefix',
-            'projects_dir',
-            'os_image',
-            'gcc_version',
-            'make_parallelism',
-            'local_repo_dir',
-            'ccache_tgz',
+            "docker_context_dir",
+            "user",
+            "prefix",
+            "projects_dir",
+            "os_image",
+            "gcc_version",
+            "make_parallelism",
+            "local_repo_dir",
+            "ccache_tgz",
         ),
         opts,
-        help=textwrap.dedent('''
+        help=textwrap.dedent(
+            """
 
         Reads `fbcode_builder_config.py` from the current directory, and
         prepares a Docker context directory to build {github_project} and
@@ -130,47 +145,55 @@ def make_docker_context(
         Usage:
             (cd $(./make_docker_context.py) && docker build . 2>&1 | tee log)
 
-        '''.format(github_project=github_project)),
+        """.format(
+                github_project=github_project
+            )
+        ),
     )
 
     # This allows travis_docker_build.sh not to know the main Github project.
-    local_repo_dir = opts.pop('local_repo_dir', None)
+    local_repo_dir = opts.pop("local_repo_dir", None)
     if local_repo_dir is not None:
-        opts['{0}:local_repo_dir'.format(github_project)] = local_repo_dir
+        opts["{0}:local_repo_dir".format(github_project)] = local_repo_dir
 
-    if (opts.get('os_image'), opts.get('gcc_version')) not in valid_versions:
+    if (opts.get("os_image"), opts.get("gcc_version")) not in valid_versions:
         raise Exception(
-            'Due to 4/5 ABI changes (std::string), we can only use {0}'.format(
-                ' / '.join('GCC {1} on {0}'.format(*p) for p in valid_versions)
+            "Due to 4/5 ABI changes (std::string), we can only use {0}".format(
+                " / ".join("GCC {1} on {0}".format(*p) for p in valid_versions)
             )
         )
 
-    if opts.get('docker_context_dir') is None:
-        opts['docker_context_dir'] = tempfile.mkdtemp(prefix='docker-context-')
-    elif not os.path.exists(opts.get('docker_context_dir')):
-        os.makedirs(opts.get('docker_context_dir'))
+    if opts.get("docker_context_dir") is None:
+        opts["docker_context_dir"] = tempfile.mkdtemp(prefix="docker-context-")
+    elif not os.path.exists(opts.get("docker_context_dir")):
+        os.makedirs(opts.get("docker_context_dir"))
 
     builder = DockerFBCodeBuilder(**opts)
-    context_dir = builder.option('docker_context_dir')  # Mark option "in-use"
+    context_dir = builder.option("docker_context_dir")  # Mark option "in-use"
     # The renderer may also populate some files into the context_dir.
     dockerfile = builder.render(get_steps_fn(builder))
 
-    with os.fdopen(os.open(
-        os.path.join(context_dir, 'Dockerfile'),
+    with os.fdopen(
+        os.open(
+            os.path.join(context_dir, "Dockerfile"),
             os.O_RDWR | os.O_CREAT | os.O_EXCL,  # Do not overwrite existing files
             0o644,
-    ), 'w') as f:
+        ),
+        "w",
+    ) as f:
         f.write(dockerfile)
 
     return context_dir
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     from utils import read_fbcode_builder_config, build_fbcode_builder_config
 
     # Load a spec from the current directory
-    config = read_fbcode_builder_config('fbcode_builder_config.py')
-    print(make_docker_context(
+    config = read_fbcode_builder_config("fbcode_builder_config.py")
+    print(
+        make_docker_context(
             build_fbcode_builder_config(config),
-        config['github_project'],
-    ))
+            config["github_project"],
+        )
+    )

build/fbcode_builder/parse_args.py

@@ -4,7 +4,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-'Argument parsing logic shared by all fbcode_builder CLI tools.'
+
+"Argument parsing logic shared by all fbcode_builder CLI tools."
 
 import argparse
 import logging
@@ -13,7 +14,7 @@ from shell_quoting import raw_shell, ShellQuoted
 
 
 def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
-    '''
+    """
 
     Provides some standard arguments: --debug, --option, --shell-quoted-option
 
@@ -26,46 +27,52 @@ def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
 
     `help` is printed in response to the `--help` argument.
 
-    '''
+    """
     top_level_opts = set(top_level_opts)
 
     parser = argparse.ArgumentParser(
-        description=help,
-        formatter_class=argparse.RawDescriptionHelpFormatter
+        description=help, formatter_class=argparse.RawDescriptionHelpFormatter
     )
 
     add_args_fn(parser)
 
     parser.add_argument(
-        '--option', nargs=2, metavar=('KEY', 'VALUE'), action='append',
+        "--option",
+        nargs=2,
+        metavar=("KEY", "VALUE"),
+        action="append",
         default=[
-            (k, v) for k, v in opts.items()
+            (k, v)
+            for k, v in opts.items()
             if k not in top_level_opts and not isinstance(v, ShellQuoted)
         ],
-        help='Set project-specific options. These are assumed to be raw '
-            'strings, to be shell-escaped as needed. Default: %(default)s.',
+        help="Set project-specific options. These are assumed to be raw "
+        "strings, to be shell-escaped as needed. Default: %(default)s.",
     )
     parser.add_argument(
-        '--shell-quoted-option', nargs=2, metavar=('KEY', 'VALUE'),
-        action='append',
+        "--shell-quoted-option",
+        nargs=2,
+        metavar=("KEY", "VALUE"),
+        action="append",
         default=[
-            (k, raw_shell(v)) for k, v in opts.items()
+            (k, raw_shell(v))
+            for k, v in opts.items()
             if k not in top_level_opts and isinstance(v, ShellQuoted)
         ],
-        help='Set project-specific options. These are assumed to be shell-'
-            'quoted, and may be used in commands as-is. Default: %(default)s.',
+        help="Set project-specific options. These are assumed to be shell-"
+        "quoted, and may be used in commands as-is. Default: %(default)s.",
     )
 
-    parser.add_argument('--debug', action='store_true', help='Log more')
+    parser.add_argument("--debug", action="store_true", help="Log more")
     args = parser.parse_args()
 
     logging.basicConfig(
         level=logging.DEBUG if args.debug else logging.INFO,
-        format='%(levelname)s: %(message)s'
+        format="%(levelname)s: %(message)s",
     )
 
     # Map command-line args back into opts.
-    logging.debug('opts before command-line arguments: {0}'.format(opts))
+    logging.debug("opts before command-line arguments: {0}".format(opts))
 
     new_opts = {}
     for key in top_level_opts:
@@ -78,6 +85,6 @@ def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
     for key, val in args.shell_quoted_option:
         new_opts[key] = ShellQuoted(val)
 
-    logging.debug('opts after command-line arguments: {0}'.format(new_opts))
+    logging.debug("opts after command-line arguments: {0}".format(new_opts))
 
     return new_opts

build/fbcode_builder/shell_builder.py

@@ -5,7 +5,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-'''
+"""
 shell_builder.py allows running the fbcode_builder logic
 on the host rather than in a container.
 
@@ -17,50 +17,50 @@ any failing step will cause the script to exit with failure.
 cd build
 python fbcode_builder/shell_builder.py > ~/run.sh
 bash ~/run.sh
-'''
+"""
 
-import os
 import distutils.spawn
+import os
 
 from fbcode_builder import FBCodeBuilder
-from shell_quoting import (
-    raw_shell, shell_comment, shell_join, ShellQuoted
-)
+from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted
 from utils import recursively_flatten_list
 
 
 class ShellFBCodeBuilder(FBCodeBuilder):
     def _render_impl(self, steps):
-        return raw_shell(shell_join('\n', recursively_flatten_list(steps)))
+        return raw_shell(shell_join("\n", recursively_flatten_list(steps)))
 
     def set_env(self, key, value):
         return ShellQuoted("export {key}={val}").format(key=key, val=value)
 
     def workdir(self, dir):
         return [
-            ShellQuoted('mkdir -p {d} && cd {d}').format(
-                d=dir
-            ),
+            ShellQuoted("mkdir -p {d} && cd {d}").format(d=dir),
         ]
 
     def run(self, shell_cmd):
-        return ShellQuoted('{cmd}').format(cmd=shell_cmd)
+        return ShellQuoted("{cmd}").format(cmd=shell_cmd)
 
     def step(self, name, actions):
-        assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
-        b = ShellQuoted('')
-        return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]
+        assert "\n" not in name, "Name {0} would span > 1 line".format(name)
+        b = ShellQuoted("")
+        return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b]
 
     def setup(self):
-        steps = [
-            ShellQuoted('set -exo pipefail'),
-        ] + self.create_python_venv() + self.python_venv()
-        if self.has_option('ccache_dir'):
-            ccache_dir = self.option('ccache_dir')
+        steps = (
+            [
+                ShellQuoted("set -exo pipefail"),
+            ]
+            + self.create_python_venv()
+            + self.python_venv()
+        )
+        if self.has_option("ccache_dir"):
+            ccache_dir = self.option("ccache_dir")
             steps += [
                 ShellQuoted(
                     # Set CCACHE_DIR before the `ccache` invocations below.
-                    'export CCACHE_DIR={ccache_dir} '
+                    "export CCACHE_DIR={ccache_dir} "
                     'CC="ccache ${{CC:-gcc}}" CXX="ccache ${{CXX:-g++}}"'
                 ).format(ccache_dir=ccache_dir)
             ]
@@ -71,44 +71,44 @@ class ShellFBCodeBuilder(FBCodeBuilder):
 
     def copy_local_repo(self, dir, dest_name):
         return [
-            ShellQuoted('cp -r {dir} {dest_name}').format(
-                dir=dir,
-                dest_name=dest_name
-            ),
+            ShellQuoted("cp -r {dir} {dest_name}").format(dir=dir, dest_name=dest_name),
         ]
 
 
 def find_project_root():
     here = os.path.dirname(os.path.realpath(__file__))
     maybe_root = os.path.dirname(os.path.dirname(here))
-    if os.path.isdir(os.path.join(maybe_root, '.git')):
+    if os.path.isdir(os.path.join(maybe_root, ".git")):
         return maybe_root
     raise RuntimeError(
         "I expected shell_builder.py to be in the "
-        "build/fbcode_builder subdir of a git repo")
+        "build/fbcode_builder subdir of a git repo"
+    )
 
 
 def persistent_temp_dir(repo_root):
-    escaped = repo_root.replace('/', 'sZs').replace('\\', 'sZs').replace(':', '')
-    return os.path.join(os.path.expandvars("$HOME"), '.fbcode_builder-' + escaped)
+    escaped = repo_root.replace("/", "sZs").replace("\\", "sZs").replace(":", "")
+    return os.path.join(os.path.expandvars("$HOME"), ".fbcode_builder-" + escaped)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     from utils import read_fbcode_builder_config, build_fbcode_builder_config
+
     repo_root = find_project_root()
     temp = persistent_temp_dir(repo_root)
 
-    config = read_fbcode_builder_config('fbcode_builder_config.py')
+    config = read_fbcode_builder_config("fbcode_builder_config.py")
     builder = ShellFBCodeBuilder(projects_dir=temp)
 
-    if distutils.spawn.find_executable('ccache'):
-        builder.add_option('ccache_dir',
-            os.environ.get('CCACHE_DIR', os.path.join(temp, '.ccache')))
-    builder.add_option('prefix', os.path.join(temp, 'installed'))
-    builder.add_option('make_parallelism', 4)
+    if distutils.spawn.find_executable("ccache"):
+        builder.add_option(
+            "ccache_dir", os.environ.get("CCACHE_DIR", os.path.join(temp, ".ccache"))
+        )
+    builder.add_option("prefix", os.path.join(temp, "installed"))
+    builder.add_option("make_parallelism", 4)
     builder.add_option(
-        '{project}:local_repo_dir'.format(project=config['github_project']),
-        repo_root)
+        "{project}:local_repo_dir".format(project=config["github_project"]), repo_root
+    )
     make_steps = build_fbcode_builder_config(config)
     steps = make_steps(builder)
     print(builder.render(steps))

build/fbcode_builder/shell_quoting.py

@@ -4,7 +4,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-'''
+
+"""
 
 Almost every FBCodeBuilder string is ultimately passed to a shell. Escaping
 too little or too much tends to be the most common error.  The utilities in
@@ -16,15 +17,14 @@ this file give a systematic way of avoiding such bugs:
  - Use `path_join` to join path components.
  - Use `shell_join` to join already-quoted command arguments or shell lines.
 
-'''
+"""
 
 import os
-
 from collections import namedtuple
 
 
-class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))):
-    '''
+class ShellQuoted(namedtuple("ShellQuoted", ("do_not_use_raw_str",))):
+    """
 
     Wrap a string with this to make it transparent to shell_quote().  It
     will almost always suffice to use ShellQuoted.format(), path_join(),
@@ -32,27 +32,25 @@ class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))):
 
     If you really must, use raw_shell() to access the raw string.
 
-    '''
+    """
 
     def __new__(cls, s):
-        'No need to nest ShellQuoted.'
+        "No need to nest ShellQuoted."
         return super(ShellQuoted, cls).__new__(
             cls, s.do_not_use_raw_str if isinstance(s, ShellQuoted) else s
         )
 
     def __str__(self):
         raise RuntimeError(
-            'One does not simply convert {0} to a string -- use path_join() '
-            'or ShellQuoted.format() instead'.format(repr(self))
+            "One does not simply convert {0} to a string -- use path_join() "
+            "or ShellQuoted.format() instead".format(repr(self))
         )
 
     def __repr__(self):
-        return '{0}({1})'.format(
-            self.__class__.__name__, repr(self.do_not_use_raw_str)
-        )
+        return "{0}({1})".format(self.__class__.__name__, repr(self.do_not_use_raw_str))
 
     def format(self, **kwargs):
-        '''
+        """
 
         Use instead of str.format() when the arguments are either
         `ShellQuoted()` or raw strings needing to be `shell_quote()`d.
@@ -60,40 +58,46 @@ class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))):
         Positional args are deliberately not supported since they are more
         error-prone.
 
-        '''
-        return ShellQuoted(self.do_not_use_raw_str.format(**dict(
+        """
+        return ShellQuoted(
+            self.do_not_use_raw_str.format(
+                **dict(
                     (k, shell_quote(v).do_not_use_raw_str) for k, v in kwargs.items()
-        )))
+                )
+            )
+        )
 
 
 def shell_quote(s):
-    'Quotes a string if it is not already quoted'
-    return s if isinstance(s, ShellQuoted) \
+    "Quotes a string if it is not already quoted"
+    return (
+        s
+        if isinstance(s, ShellQuoted)
         else ShellQuoted("'" + str(s).replace("'", "'\\''") + "'")
+    )
 
 
 def raw_shell(s):
-    'Not a member of ShellQuoted so we get a useful error for raw strings'
+    "Not a member of ShellQuoted so we get a useful error for raw strings"
     if isinstance(s, ShellQuoted):
         return s.do_not_use_raw_str
-    raise RuntimeError('{0} should have been ShellQuoted'.format(s))
+    raise RuntimeError("{0} should have been ShellQuoted".format(s))
 
 
 def shell_join(delim, it):
-    'Joins an iterable of ShellQuoted with a delimiter between each two'
+    "Joins an iterable of ShellQuoted with a delimiter between each two"
     return ShellQuoted(delim.join(raw_shell(s) for s in it))
 
 
 def path_join(*args):
-    'Joins ShellQuoted and raw pieces of paths to make a shell-quoted path'
-    return ShellQuoted(os.path.join(*[
-        raw_shell(shell_quote(s)) for s in args
-    ]))
+    "Joins ShellQuoted and raw pieces of paths to make a shell-quoted path"
+    return ShellQuoted(os.path.join(*[raw_shell(shell_quote(s)) for s in args]))
 
 
 def shell_comment(c):
-    'Do not shell-escape raw strings in comments, but do handle line breaks.'
-    return ShellQuoted('# {c}').format(c=ShellQuoted(
-        (raw_shell(c) if isinstance(c, ShellQuoted) else c)
-            .replace('\n', '\n# ')
-    ))
+    "Do not shell-escape raw strings in comments, but do handle line breaks."
+    return ShellQuoted("# {c}").format(
+        c=ShellQuoted(
+            (raw_shell(c) if isinstance(c, ShellQuoted) else c).replace("\n", "\n# ")
+        )
+    )

build/fbcode_builder/specs/fbthrift.py

@@ -15,8 +15,8 @@ import specs.zstd as zstd
 
 def fbcode_builder_spec(builder):
     return {
-        'depends_on': [fmt, folly, fizz, sodium, wangle, zstd],
-        'steps': [
-            builder.fb_github_cmake_install('fbthrift/thrift'),
+        "depends_on": [fmt, folly, fizz, sodium, wangle, zstd],
+        "steps": [
+            builder.fb_github_cmake_install("fbthrift/thrift"),
         ],
     }

build/fbcode_builder/specs/fbzmq.py

@@ -10,31 +10,40 @@ import specs.fmt as fmt
 import specs.folly as folly
 import specs.gmock as gmock
 import specs.sodium as sodium
-
 from shell_quoting import ShellQuoted
 
 
 def fbcode_builder_spec(builder):
-    builder.add_option('zeromq/libzmq:git_hash', 'v4.2.2')
+    builder.add_option("zeromq/libzmq:git_hash", "v4.2.2")
     return {
-        'depends_on': [fmt, folly, fbthrift, gmock, sodium],
-        'steps': [
-            builder.github_project_workdir('zeromq/libzmq', '.'),
-            builder.step('Build and install zeromq/libzmq', [
-                builder.run(ShellQuoted('./autogen.sh')),
+        "depends_on": [fmt, folly, fbthrift, gmock, sodium],
+        "steps": [
+            builder.github_project_workdir("zeromq/libzmq", "."),
+            builder.step(
+                "Build and install zeromq/libzmq",
+                [
+                    builder.run(ShellQuoted("./autogen.sh")),
                     builder.configure(),
                     builder.make_and_install(),
-            ]),
-
-            builder.fb_github_project_workdir('fbzmq/_build', 'facebook'),
-            builder.step('Build and install fbzmq/', [
-                builder.cmake_configure('fbzmq/_build'),
+                ],
+            ),
+            builder.fb_github_project_workdir("fbzmq/_build", "facebook"),
+            builder.step(
+                "Build and install fbzmq/",
+                [
+                    builder.cmake_configure("fbzmq/_build"),
                     # we need the pythonpath to find the thrift compiler
-                builder.run(ShellQuoted(
+                    builder.run(
+                        ShellQuoted(
                             'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
-                    'make -j {n}'
-                ).format(p=builder.option('prefix'), n=builder.option('make_parallelism'))),
-                builder.run(ShellQuoted('make install')),
-            ]),
+                            "make -j {n}"
+                        ).format(
+                            p=builder.option("prefix"),
+                            n=builder.option("make_parallelism"),
+                        )
+                    ),
+                    builder.run(ShellQuoted("make install")),
+                ],
+            ),
         ],
     }

build/fbcode_builder/specs/fizz.py

@@ -5,9 +5,9 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import specs.gmock as gmock
 import specs.fmt as fmt
 import specs.folly as folly
+import specs.gmock as gmock
 import specs.sodium as sodium
 
 

build/fbcode_builder/specs/fmt.py

@@ -7,20 +7,20 @@ from __future__ import unicode_literals
 
 
 def fbcode_builder_spec(builder):
-    builder.add_option('fmtlib/fmt:git_hash', '6.2.1')
+    builder.add_option("fmtlib/fmt:git_hash", "6.2.1")
     builder.add_option(
-        'fmtlib/fmt:cmake_defines',
+        "fmtlib/fmt:cmake_defines",
         {
             # Avoids a bizarred failure to run tests in Bistro:
             #   test_crontab_selector: error while loading shared libraries:
             #   libfmt.so.6: cannot open shared object file:
             #   No such file or directory
-            'BUILD_SHARED_LIBS': 'OFF',
-        }
+            "BUILD_SHARED_LIBS": "OFF",
+        },
     )
     return {
-        'steps': [
-            builder.github_project_workdir('fmtlib/fmt', 'build'),
-            builder.cmake_install('fmtlib/fmt'),
+        "steps": [
+            builder.github_project_workdir("fmtlib/fmt", "build"),
+            builder.cmake_install("fmtlib/fmt"),
         ],
     }

build/fbcode_builder/specs/folly.py

@@ -11,13 +11,13 @@ import specs.fmt as fmt
 def fbcode_builder_spec(builder):
     return {
         "depends_on": [fmt],
-        'steps': [
+        "steps": [
             # on macOS the filesystem is typically case insensitive.
             # We need to ensure that the CWD is not the folly source
             # dir when we build, otherwise the system will decide
             # that `folly/String.h` is the file it wants when including
             # `string.h` and the build will fail.
-            builder.fb_github_project_workdir('folly/_build'),
-            builder.cmake_install('facebook/folly'),
+            builder.fb_github_project_workdir("folly/_build"),
+            builder.cmake_install("facebook/folly"),
         ],
     }

build/fbcode_builder/specs/gmock.py

@@ -7,18 +7,18 @@ from __future__ import unicode_literals
 
 
 def fbcode_builder_spec(builder):
-    builder.add_option('google/googletest:git_hash', 'release-1.8.1')
+    builder.add_option("google/googletest:git_hash", "release-1.8.1")
     builder.add_option(
-        'google/googletest:cmake_defines',
+        "google/googletest:cmake_defines",
         {
-            'BUILD_GTEST': 'ON',
+            "BUILD_GTEST": "ON",
             # Avoid problems with MACOSX_RPATH
-            'BUILD_SHARED_LIBS': 'OFF',
-        }
+            "BUILD_SHARED_LIBS": "OFF",
+        },
     )
     return {
-        'steps': [
-            builder.github_project_workdir('google/googletest', 'build'),
-            builder.cmake_install('google/googletest'),
+        "steps": [
+            builder.github_project_workdir("google/googletest", "build"),
+            builder.cmake_install("google/googletest"),
         ],
     }

build/fbcode_builder/specs/mvfst.py

@@ -5,9 +5,9 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import specs.gmock as gmock
-import specs.folly as folly
 import specs.fizz as fizz
+import specs.folly as folly
+import specs.gmock as gmock
 
 
 def fbcode_builder_spec(builder):

build/fbcode_builder/specs/proxygen.py

@@ -5,10 +5,10 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import specs.gmock as gmock
+import specs.fizz as fizz
 import specs.fmt as fmt
 import specs.folly as folly
-import specs.fizz as fizz
+import specs.gmock as gmock
 import specs.mvfst as mvfst
 import specs.sodium as sodium
 import specs.wangle as wangle

build/fbcode_builder/specs/proxygen_quic.py

@@ -5,10 +5,10 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import specs.gmock as gmock
+import specs.fizz as fizz
 import specs.fmt as fmt
 import specs.folly as folly
-import specs.fizz as fizz
+import specs.gmock as gmock
 import specs.mvfst as mvfst
 import specs.sodium as sodium
 import specs.wangle as wangle

build/fbcode_builder/specs/re2.py

@@ -8,8 +8,8 @@ from __future__ import unicode_literals
 
 def fbcode_builder_spec(builder):
     return {
-        'steps': [
-            builder.github_project_workdir('google/re2', 'build'),
-            builder.cmake_install('google/re2'),
+        "steps": [
+            builder.github_project_workdir("google/re2", "build"),
+            builder.cmake_install("google/re2"),
         ],
     }

build/fbcode_builder/specs/rocksdb.py

@@ -7,10 +7,13 @@ from __future__ import unicode_literals
 
 
 def fbcode_builder_spec(builder):
-    builder.add_option("rocksdb/_build:cmake_defines", {
+    builder.add_option(
+        "rocksdb/_build:cmake_defines",
+        {
             "USE_RTTI": "1",
             "PORTABLE": "ON",
-    })
+        },
+    )
     return {
         "steps": [
             builder.fb_github_cmake_install("rocksdb/_build"),

build/fbcode_builder/specs/sodium.py

@@ -9,14 +9,17 @@ from shell_quoting import ShellQuoted
 
 
 def fbcode_builder_spec(builder):
-    builder.add_option('jedisct1/libsodium:git_hash', 'stable')
+    builder.add_option("jedisct1/libsodium:git_hash", "stable")
     return {
-        'steps': [
-            builder.github_project_workdir('jedisct1/libsodium', '.'),
-            builder.step('Build and install jedisct1/libsodium', [
-                builder.run(ShellQuoted('./autogen.sh')),
+        "steps": [
+            builder.github_project_workdir("jedisct1/libsodium", "."),
+            builder.step(
+                "Build and install jedisct1/libsodium",
+                [
+                    builder.run(ShellQuoted("./autogen.sh")),
                     builder.configure(),
                     builder.make_and_install(),
-            ]),
+                ],
+            ),
         ],
     }

build/fbcode_builder/specs/wangle.py

@@ -5,10 +5,10 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import specs.gmock as gmock
+import specs.fizz as fizz
 import specs.fmt as fmt
 import specs.folly as folly
-import specs.fizz as fizz
+import specs.gmock as gmock
 import specs.sodium as sodium
 
 

build/fbcode_builder/specs/zstd.py

@@ -11,16 +11,21 @@ from shell_quoting import ShellQuoted
 def fbcode_builder_spec(builder):
     # This API should change rarely, so build the latest tag instead of master.
     builder.add_option(
-        'facebook/zstd:git_hash',
-        ShellQuoted('$(git describe --abbrev=0 --tags origin/master)')
+        "facebook/zstd:git_hash",
+        ShellQuoted("$(git describe --abbrev=0 --tags origin/master)"),
     )
     return {
-        'steps': [
-            builder.github_project_workdir('facebook/zstd', '.'),
-            builder.step('Build and install zstd', [
-                builder.make_and_install(make_vars={
-                    'PREFIX': builder.option('prefix'),
-                })
-            ]),
+        "steps": [
+            builder.github_project_workdir("facebook/zstd", "."),
+            builder.step(
+                "Build and install zstd",
+                [
+                    builder.make_and_install(
+                        make_vars={
+                            "PREFIX": builder.option("prefix"),
+                        }
+                    )
+                ],
+            ),
         ],
     }

build/fbcode_builder/utils.py

@@ -4,7 +4,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-'Miscellaneous utility functions.'
+
+"Miscellaneous utility functions."
 
 import itertools
 import logging
@@ -12,21 +13,19 @@ import os
 import shutil
 import subprocess
 import sys
-
 from contextlib import contextmanager
 
 
 def recursively_flatten_list(l):
     return itertools.chain.from_iterable(
-        (recursively_flatten_list(i) if type(i) is list else (i,))
-            for i in l
+        (recursively_flatten_list(i) if type(i) is list else (i,)) for i in l
     )
 
 
 def run_command(*cmd, **kwargs):
-    'The stdout of most fbcode_builder utilities is meant to be parsed.'
-    logging.debug('Running: {0} with {1}'.format(cmd, kwargs))
-    kwargs['stdout'] = sys.stderr
+    "The stdout of most fbcode_builder utilities is meant to be parsed."
+    logging.debug("Running: {0} with {1}".format(cmd, kwargs))
+    kwargs["stdout"] = sys.stderr
     subprocess.check_call(cmd, **kwargs)
 
 
@@ -40,13 +39,13 @@ def make_temp_dir(d):
 
 
 def _inner_read_config(path):
-    '''
+    """
     Helper to read a named config file.
     The grossness with the global is a workaround for this python bug:
     https://bugs.python.org/issue21591
     The bug prevents us from defining either a local function or a lambda
     in the scope of read_fbcode_builder_config below.
-    '''
+    """
     global _project_dir
     full_path = os.path.join(_project_dir, path)
     return read_fbcode_builder_config(full_path)
@@ -60,37 +59,37 @@ def read_fbcode_builder_config(filename):
     global _project_dir
     _project_dir = os.path.dirname(filename)
 
-    scope = {'read_fbcode_builder_config': _inner_read_config}
+    scope = {"read_fbcode_builder_config": _inner_read_config}
     with open(filename) as config_file:
-        code = compile(config_file.read(), filename, mode='exec')
+        code = compile(config_file.read(), filename, mode="exec")
     exec(code, scope)
-    return scope['config']
+    return scope["config"]
 
 
 def steps_for_spec(builder, spec, processed_modules=None):
-    '''
+    """
     Sets `builder` configuration, and returns all the builder steps
     necessary to build `spec` and its dependencies.
 
     Traverses the dependencies in depth-first order, honoring the sequencing
     in each 'depends_on' list.
-    '''
+    """
     if processed_modules is None:
         processed_modules = set()
     steps = []
-    for module in spec.get('depends_on', []):
+    for module in spec.get("depends_on", []):
         if module not in processed_modules:
             processed_modules.add(module)
-            steps.extend(steps_for_spec(
-                builder,
-                module.fbcode_builder_spec(builder),
-                processed_modules
-            ))
-    steps.extend(spec.get('steps', []))
+            steps.extend(
+                steps_for_spec(
+                    builder, module.fbcode_builder_spec(builder), processed_modules
+                )
+            )
+    steps.extend(spec.get("steps", []))
     return steps
 
 
 def build_fbcode_builder_config(config):
     return lambda builder: builder.build(
-        steps_for_spec(builder, config['fbcode_builder_spec'](builder))
+        steps_for_spec(builder, config["fbcode_builder_spec"](builder))
     )