Commit 43eb92d9 authored by Wez Furlong's avatar Wez Furlong Committed by Facebook GitHub Bot

getdeps: split monolithic GH actions CI spec up (#1360)

Summary:
Rather than have a single main.yml file containing all off the different
builds, split that up so that we have one file per build environment
(linux, mac, windows).

This has a couple of advantages:

* It is quicker to see the status of just one of the platforms
* Artifact collection for one platform is not blocked pending completion
  of the builds for all systems
* It's a little easier to understand what is happening for a single platform

To support having multiple files I've changed the output-file option to
be an output-dir.

I've included the rm of main.yml in this commit.  Once this gets
imported back to the FB internal system I'll amend in an update to
the helper script that updates all of our opensource builds and run
and amend that.
Pull Request resolved: https://github.com/facebook/folly/pull/1360

Test Plan:
the GH action status on this PR should show three different
actions running, one for each platform.

I updated and ran
`fbcode/opensource/fbcode_builder/getdeps/facebook/update-all-github-actions.sh`
to regenerate all the actions files for FB.

Reviewed By: yfeldblum

Differential Revision: D21310991

Pulled By: wez

fbshipit-source-id: 604ef652c8f746781a4b410c6b996cdee4524e0d
parent b2003abb
# This file was @generated by getdeps.py
name: linux
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v1
- name: Install system deps
run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive folly
- name: Fetch boost
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost
- name: Fetch ninja
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja
- name: Fetch cmake
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake
- name: Fetch double-conversion
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion
- name: Fetch fmt
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt
- name: Fetch gflags
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags
- name: Fetch glog
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog
- name: Fetch googletest
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest
- name: Fetch libevent
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent
- name: Fetch snappy
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy
- name: Fetch zstd
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd
- name: Build boost
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost
- name: Build ninja
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja
- name: Build cmake
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake
- name: Build double-conversion
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion
- name: Build fmt
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt
- name: Build gflags
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags
- name: Build glog
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog
- name: Build googletest
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest
- name: Build libevent
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent
- name: Build snappy
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy
- name: Build zstd
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd
- name: Build folly
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. folly --project-install-prefix folly:/usr/local
- name: Copy artifacts
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. folly _artifacts/linux --final-install-prefix /usr/local
- uses: actions/upload-artifact@master
with:
name: folly
path: _artifacts
- name: Test folly
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. folly --project-install-prefix folly:/usr/local
# This file was @generated by getdeps.py
name: mac
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:
runs-on: macOS-latest
steps:
- uses: actions/checkout@v1
- name: Install system deps
run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive folly
- name: Fetch boost
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost
- name: Fetch openssl
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl
- name: Fetch ninja
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja
- name: Fetch cmake
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake
- name: Fetch double-conversion
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion
- name: Fetch fmt
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt
- name: Fetch gflags
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags
- name: Fetch glog
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog
- name: Fetch googletest
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest
- name: Fetch libevent
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent
- name: Fetch snappy
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy
- name: Fetch zstd
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd
- name: Build boost
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost
- name: Build openssl
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests openssl
- name: Build ninja
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja
- name: Build cmake
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake
- name: Build double-conversion
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion
- name: Build fmt
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt
- name: Build gflags
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags
- name: Build glog
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog
- name: Build googletest
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest
- name: Build libevent
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent
- name: Build snappy
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy
- name: Build zstd
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd
- name: Build folly
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. folly --project-install-prefix folly:/usr/local
- name: Copy artifacts
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. folly _artifacts/mac --final-install-prefix /usr/local
- uses: actions/upload-artifact@master
with:
name: folly
path: _artifacts
- name: Test folly
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. folly --project-install-prefix folly:/usr/local
# This file was @generated by getdeps.py # This file was @generated by getdeps.py
name: CI name: windows
on: on:
push: push:
...@@ -11,131 +11,7 @@ on: ...@@ -11,131 +11,7 @@ on:
- master - master
jobs: jobs:
linux: build:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v1
- name: Install system deps
run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive folly
- name: Fetch boost
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost
- name: Fetch ninja
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja
- name: Fetch cmake
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake
- name: Fetch double-conversion
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion
- name: Fetch fmt
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt
- name: Fetch gflags
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags
- name: Fetch glog
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog
- name: Fetch googletest
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest
- name: Fetch libevent
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent
- name: Fetch snappy
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy
- name: Fetch zstd
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd
- name: Build boost
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost
- name: Build ninja
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja
- name: Build cmake
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake
- name: Build double-conversion
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion
- name: Build fmt
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt
- name: Build gflags
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags
- name: Build glog
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog
- name: Build googletest
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest
- name: Build libevent
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent
- name: Build snappy
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy
- name: Build zstd
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd
- name: Build folly
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. folly --project-install-prefix folly:/usr/local
- name: Copy artifacts
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. folly _artifacts/linux --final-install-prefix /usr/local
- uses: actions/upload-artifact@master
with:
name: folly
path: _artifacts
- name: Test folly
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. folly --project-install-prefix folly:/usr/local
mac:
runs-on: macOS-latest
steps:
- uses: actions/checkout@v1
- name: Install system deps
run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive folly
- name: Fetch boost
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost
- name: Fetch openssl
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl
- name: Fetch ninja
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja
- name: Fetch cmake
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake
- name: Fetch double-conversion
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion
- name: Fetch fmt
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt
- name: Fetch gflags
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags
- name: Fetch glog
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog
- name: Fetch googletest
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest
- name: Fetch libevent
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent
- name: Fetch snappy
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy
- name: Fetch zstd
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd
- name: Build boost
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost
- name: Build openssl
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests openssl
- name: Build ninja
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja
- name: Build cmake
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake
- name: Build double-conversion
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion
- name: Build fmt
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt
- name: Build gflags
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags
- name: Build glog
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog
- name: Build googletest
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest
- name: Build libevent
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent
- name: Build snappy
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy
- name: Build zstd
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd
- name: Build folly
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. folly --project-install-prefix folly:/usr/local
- name: Copy artifacts
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. folly _artifacts/mac --final-install-prefix /usr/local
- uses: actions/upload-artifact@master
with:
name: folly
path: _artifacts
- name: Test folly
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. folly --project-install-prefix folly:/usr/local
windows:
runs-on: windows-2016 runs-on: windows-2016
steps: steps:
- uses: actions/checkout@v1 - uses: actions/checkout@v1
......
...@@ -633,32 +633,11 @@ class GenerateGitHubActionsCmd(ProjectCmdBase): ...@@ -633,32 +633,11 @@ class GenerateGitHubActionsCmd(ProjectCmdBase):
HostType("windows", None, None), HostType("windows", None, None),
] ]
with open(args.output_file, "w") as out: for p in platforms:
# Deliberate line break here because the @ and the generated self.write_job_for_platform(p, args)
# symbols are meaningful to our internal tooling when they
# appear in a single token
out.write("# This file was @")
out.write("generated by getdeps.py\n")
out.write(
"""
name: CI
on: def write_job_for_platform(self, platform, args):
push: build_opts = setup_build_options(args, platform)
branches:
- master
pull_request:
branches:
- master
jobs:
"""
)
for p in platforms:
build_opts = setup_build_options(args, p)
self.write_job_for_platform(out, args, build_opts)
def write_job_for_platform(self, out, args, build_opts):
ctx_gen = build_opts.get_context_generator() ctx_gen = build_opts.get_context_generator()
loader = ManifestLoader(build_opts, ctx_gen) loader = ManifestLoader(build_opts, ctx_gen)
manifest = loader.load_manifest(args.project) manifest = loader.load_manifest(args.project)
...@@ -697,78 +676,106 @@ jobs: ...@@ -697,78 +676,106 @@ jobs:
job_name = "mac" job_name = "mac"
runs_on = "macOS-latest" runs_on = "macOS-latest"
getdeps = f"{py3} build/fbcode_builder/getdeps.py --allow-system-packages" os.makedirs(args.output_dir, exist_ok=True)
output_file = os.path.join(args.output_dir, f"getdeps_{job_name}.yml")
out.write(" %s:\n" % job_name) with open(output_file, "w") as out:
out.write(" runs-on: %s\n" % runs_on) # Deliberate line break here because the @ and the generated
out.write(" steps:\n") # symbols are meaningful to our internal tooling when they
out.write(" - uses: actions/checkout@v1\n") # appear in a single token
out.write("# This file was @")
if build_opts.is_windows(): out.write("generated by getdeps.py\n")
# cmake relies on BOOST_ROOT but GH deliberately don't set it in order
# to avoid versioning issues:
# https://github.com/actions/virtual-environments/issues/319
# Instead, set the version we think we need; this is effectively
# coupled with the boost manifest
# This is the unusual syntax for setting an env var for the rest of
# the steps in a workflow:
# https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
out.write(" - name: Export boost environment\n")
out.write( out.write(
' run: "echo ::set-env name=BOOST_ROOT::%BOOST_ROOT_1_69_0%"\n' f"""
) name: {job_name}
out.write(" shell: cmd\n")
# The git installation may not like long filenames, so tell it on:
# that we want it to use them! push:
out.write(" - name: Fix Git config\n") branches:
out.write(" run: git config --system core.longpaths true\n") - master
else: pull_request:
out.write(" - name: Install system deps\n") branches:
out.write( - master
f" run: sudo {getdeps} install-system-deps --recursive {manifest.name}\n"
jobs:
"""
) )
projects = loader.manifests_in_dependency_order() getdeps = f"{py3} build/fbcode_builder/getdeps.py --allow-system-packages"
out.write(" build:\n")
out.write(" runs-on: %s\n" % runs_on)
out.write(" steps:\n")
out.write(" - uses: actions/checkout@v1\n")
if build_opts.is_windows():
# cmake relies on BOOST_ROOT but GH deliberately don't set it in order
# to avoid versioning issues:
# https://github.com/actions/virtual-environments/issues/319
# Instead, set the version we think we need; this is effectively
# coupled with the boost manifest
# This is the unusual syntax for setting an env var for the rest of
# the steps in a workflow:
# https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
out.write(" - name: Export boost environment\n")
out.write(
' run: "echo ::set-env name=BOOST_ROOT::%BOOST_ROOT_1_69_0%"\n'
)
out.write(" shell: cmd\n")
for m in projects: # The git installation may not like long filenames, so tell it
if m != manifest: # that we want it to use them!
out.write(" - name: Fetch %s\n" % m.name) out.write(" - name: Fix Git config\n")
out.write(f" run: {getdeps} fetch --no-tests {m.name}\n") out.write(" run: git config --system core.longpaths true\n")
else:
out.write(" - name: Install system deps\n")
out.write(
f" run: sudo {getdeps} install-system-deps --recursive {manifest.name}\n"
)
for m in projects: projects = loader.manifests_in_dependency_order()
if m != manifest:
out.write(" - name: Build %s\n" % m.name)
out.write(f" run: {getdeps} build --no-tests {m.name}\n")
out.write(" - name: Build %s\n" % manifest.name) for m in projects:
if m != manifest:
out.write(" - name: Fetch %s\n" % m.name)
out.write(f" run: {getdeps} fetch --no-tests {m.name}\n")
project_prefix = "" for m in projects:
if not build_opts.is_windows(): if m != manifest:
project_prefix = " --project-install-prefix %s:/usr/local" % manifest.name out.write(" - name: Build %s\n" % m.name)
out.write(f" run: {getdeps} build --no-tests {m.name}\n")
out.write( out.write(" - name: Build %s\n" % manifest.name)
f" run: {getdeps} build --src-dir=. {manifest.name} {project_prefix}\n"
)
out.write(" - name: Copy artifacts\n") project_prefix = ""
out.write( if not build_opts.is_windows():
f" run: {getdeps} fixup-dyn-deps " project_prefix = (
f"--src-dir=. {manifest.name} _artifacts/{job_name} --final-install-prefix /usr/local\n" " --project-install-prefix %s:/usr/local" % manifest.name
) )
out.write(" - uses: actions/upload-artifact@master\n") out.write(
out.write(" with:\n") f" run: {getdeps} build --src-dir=. {manifest.name} {project_prefix}\n"
out.write(" name: %s\n" % manifest.name) )
out.write(" path: _artifacts\n")
out.write(" - name: Test %s\n" % manifest.name) out.write(" - name: Copy artifacts\n")
out.write( out.write(
f" run: {getdeps} test --src-dir=. {manifest.name} {project_prefix}\n" f" run: {getdeps} fixup-dyn-deps "
) f"--src-dir=. {manifest.name} _artifacts/{job_name} --final-install-prefix /usr/local\n"
)
out.write(" - uses: actions/upload-artifact@master\n")
out.write(" with:\n")
out.write(" name: %s\n" % manifest.name)
out.write(" path: _artifacts\n")
out.write(" - name: Test %s\n" % manifest.name)
out.write(
f" run: {getdeps} test --src-dir=. {manifest.name} {project_prefix}\n"
)
def setup_project_cmd_parser(self, parser): def setup_project_cmd_parser(self, parser):
parser.add_argument("--output-file", help="The name of the yaml file") parser.add_argument(
"--output-dir", help="The directory that will contain the yml files"
)
def get_arg_var_name(args): def get_arg_var_name(args):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment