Bug 1808605 - part 14: Port fenix build jobs
(cherry picked from commit e82203fa85690b877326090bf89c8767d1e31813)pull/600/head
Geoff Brown
2 years ago
committed by
mergify[bot]
parent
d3d4593fc4
commit
a99cc5aa3f
@ -1,222 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: taskgraph.loader.transform:loader
|
||||
|
||||
transforms:
|
||||
- fenix_taskgraph.transforms.build:transforms
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
- toolchain
|
||||
|
||||
|
||||
task-defaults:
|
||||
# Builds generate multiple APKs with different ABIs. For each APK described
|
||||
# by `gradlew printVariant`, an artifact will be generated. `variant` and
|
||||
# the per-apk config from `printVariant` can be used as substitutions in
|
||||
# `name` and `path`.
|
||||
apk-artifact-template:
|
||||
type: file
|
||||
name: public/build/{abi}/target.apk
|
||||
github-name: 'fenix-{version}-{abi}.apk'
|
||||
path: '/builds/worker/checkouts/vcs/app/build/outputs/apk/{gradle_build_type}/{fileName}'
|
||||
description: Build Fenix from source code.
|
||||
fetches:
|
||||
toolchain:
|
||||
- android-sdk-linux
|
||||
- android-gradle-dependencies
|
||||
run:
|
||||
using: gradlew
|
||||
use-caches: false
|
||||
run-on-tasks-for: []
|
||||
treeherder:
|
||||
kind: build
|
||||
symbol: B
|
||||
platform: android-all/opt
|
||||
tier: 1
|
||||
worker-type: b-android
|
||||
worker:
|
||||
docker-image: {in-tree: base}
|
||||
max-run-time: 7200
|
||||
chain-of-trust: true
|
||||
|
||||
|
||||
tasks:
|
||||
debug:
|
||||
attributes:
|
||||
code-review: true
|
||||
run-on-tasks-for: [github-pull-request, github-push]
|
||||
run:
|
||||
gradle-build-type: debug
|
||||
track-apk-size: true
|
||||
treeherder:
|
||||
symbol: debug(B)
|
||||
|
||||
beta-firebase:
|
||||
disable-optimization: true
|
||||
run-on-tasks-for: [github-push] # We want this on push so that we detect problem before triggering a new beta
|
||||
run:
|
||||
gradle-build-type: beta
|
||||
test-build-type: beta
|
||||
treeherder:
|
||||
symbol: beta(Bf)
|
||||
|
||||
nightly-firebase:
|
||||
disable-optimization: true
|
||||
run-on-tasks-for: [github-push] # We want this on push so that we detect problem before triggering a new nightly
|
||||
run:
|
||||
gradle-build-type: nightly
|
||||
test-build-type: nightly
|
||||
treeherder:
|
||||
symbol: nightly(Bf)
|
||||
|
||||
android-test-debug:
|
||||
attributes:
|
||||
code-review: true
|
||||
run-on-tasks-for: [github-pull-request, github-push]
|
||||
run:
|
||||
gradle-build-type: androidTest
|
||||
apk-artifact-template:
|
||||
# 2 differences here: "androidTest/" is added and "{gradle_build_type}" is forced to "debug"
|
||||
path: '/builds/worker/checkouts/vcs/app/build/outputs/apk/androidTest/debug/{fileName}'
|
||||
treeherder:
|
||||
symbol: debug(Bat)
|
||||
|
||||
# android-test-nightly and android-test-beta, while still being debug builds, are meant to be signed
|
||||
# with the nightly/beta key. The Firebase testing infrastructure requires both the androidTest APK
|
||||
# and the APK under test to be signed with the same key. Thus, the nightly APK being signed with
|
||||
# nightly means we need an androidTest APK with the same signature.
|
||||
#
|
||||
# TODO: See if we can tweak the signing kind to make 2 signing jobs out of a single `android-test`
|
||||
# job.
|
||||
android-test-nightly:
|
||||
attributes:
|
||||
nightly: true
|
||||
apk-artifact-template:
|
||||
# 2 differences here:
|
||||
# * "androidTest/" is added
|
||||
# * "{gradle_build_type}" is forced to "debug"
|
||||
path: '/builds/worker/checkouts/vcs/app/build/outputs/apk/androidTest/nightly/app-nightly-androidTest.apk'
|
||||
disable-optimization: true
|
||||
run:
|
||||
gradle-build-type: androidTest
|
||||
test-build-type: nightly
|
||||
run-on-tasks-for: [github-push]
|
||||
treeherder:
|
||||
symbol: nightly(Bat)
|
||||
|
||||
android-test-beta:
|
||||
apk-artifact-template:
|
||||
# 3 differences here:
|
||||
# * "androidTest/" is added
|
||||
# * "{gradle_build_type}" is forced to "beta"
|
||||
# * "{fileName}" is forced to "app-beta-androidTest.apk"
|
||||
path: '/builds/worker/checkouts/vcs/app/build/outputs/apk/androidTest/beta/app-beta-androidTest.apk'
|
||||
disable-optimization: true
|
||||
run:
|
||||
gradle-build-type: androidTest
|
||||
test-build-type: beta
|
||||
run-on-tasks-for: [github-push] # We want this on push so that we detect problem before triggering a new beta
|
||||
treeherder:
|
||||
symbol: beta(Bat)
|
||||
|
||||
android-test-mozillaonline:
|
||||
apk-artifact-template:
|
||||
# 3 differences here:
|
||||
# * "androidTest/" is added
|
||||
# * "{gradle_build_type}" is forced to "beta"
|
||||
# * "{fileName}" is forced to "app-beta-androidTest.apk"
|
||||
path: '/builds/worker/checkouts/vcs/app/build/outputs/apk/androidTest/beta/app-beta-androidTest.apk'
|
||||
disable-optimization: true
|
||||
run:
|
||||
gradle-build-type: androidTest
|
||||
gradle-extra-options:
|
||||
- -PmozillaOnline
|
||||
test-build-type: beta
|
||||
run-on-tasks-for: [github-push] # We want this on push so that we detect problem before triggering a new beta
|
||||
treeherder:
|
||||
symbol: beta(Bat-mo)
|
||||
|
||||
nightly-simulation:
|
||||
attributes:
|
||||
nightly: false
|
||||
run-on-tasks-for: [github-push]
|
||||
include-nightly-version: true
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
gradle-build-type: nightly
|
||||
treeherder:
|
||||
symbol: nightlySim(B)
|
||||
|
||||
nightly:
|
||||
attributes:
|
||||
nightly: true
|
||||
include-nightly-version: true
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
gradle-build-type: nightly
|
||||
track-apk-size: true
|
||||
run-on-tasks-for: []
|
||||
treeherder:
|
||||
symbol: nightly(B)
|
||||
|
||||
beta:
|
||||
attributes:
|
||||
release-type: beta
|
||||
shipping_phase: promote
|
||||
include-release-version: true
|
||||
include-shippable-secrets: true
|
||||
filter-incomplete-translations: true
|
||||
run:
|
||||
gradle-build-type: beta
|
||||
track-apk-size: true
|
||||
treeherder:
|
||||
symbol: beta(B)
|
||||
|
||||
beta-mozillaonline:
|
||||
apk-artifact-template:
|
||||
github-name: 'fenix-mozillaonline-{version}-{abi}.apk'
|
||||
attributes:
|
||||
release-type: beta
|
||||
shipping_phase: promote
|
||||
include-release-version: true
|
||||
include-shippable-secrets: true
|
||||
filter-incomplete-translations: true
|
||||
run:
|
||||
gradle-build-type: beta
|
||||
gradle-extra-options:
|
||||
- -PmozillaOnline
|
||||
treeherder:
|
||||
symbol: beta(Bmo)
|
||||
|
||||
release:
|
||||
attributes:
|
||||
release-type: release
|
||||
shipping_phase: promote
|
||||
include-release-version: true
|
||||
include-shippable-secrets: true
|
||||
filter-incomplete-translations: true
|
||||
run:
|
||||
gradle-build-type: release
|
||||
track-apk-size: true
|
||||
treeherder:
|
||||
symbol: release(B)
|
||||
|
||||
release-mozillaonline:
|
||||
apk-artifact-template:
|
||||
github-name: 'fenix-mozillaonline-{version}-{abi}.apk'
|
||||
attributes:
|
||||
release-type: release
|
||||
shipping_phase: promote
|
||||
include-release-version: true
|
||||
include-shippable-secrets: true
|
||||
filter-incomplete-translations: true
|
||||
run:
|
||||
gradle-build-type: release
|
||||
gradle-extra-options:
|
||||
- -PmozillaOnline
|
||||
treeherder:
|
||||
symbol: release(Bmo)
|
||||
@ -1,58 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from taskgraph.util.memoize import memoize
|
||||
|
||||
|
||||
def get_variant(build_type):
|
||||
all_variants = _fetch_all_variants()
|
||||
matching_variants = [
|
||||
variant for variant in all_variants if variant["build_type"] == build_type
|
||||
]
|
||||
number_of_matching_variants = len(matching_variants)
|
||||
if number_of_matching_variants == 0:
|
||||
raise ValueError('No variant found for build type "{}"'.format(build_type))
|
||||
elif number_of_matching_variants > 1:
|
||||
raise ValueError(
|
||||
'Too many variants found for build type "{}"": {}'.format(
|
||||
build_type, matching_variants
|
||||
)
|
||||
)
|
||||
|
||||
return matching_variants.pop()
|
||||
|
||||
|
||||
@memoize
|
||||
def _fetch_all_variants():
|
||||
output = _run_gradle_process("printVariants")
|
||||
content = _extract_content_from_command_output(output, prefix="variants: ")
|
||||
return json.loads(content)
|
||||
|
||||
|
||||
def _run_gradle_process(gradle_command, **kwargs):
|
||||
gradle_properties = [
|
||||
"-P{property_name}={value}".format(property_name=property_name, value=value)
|
||||
for property_name, value in kwargs.items()
|
||||
]
|
||||
|
||||
process = subprocess.Popen(
|
||||
["./gradlew", "--no-daemon", "--quiet", gradle_command] + gradle_properties,
|
||||
stdout=subprocess.PIPE,
|
||||
universal_newlines=True,
|
||||
)
|
||||
output, err = process.communicate()
|
||||
exit_code = process.wait()
|
||||
|
||||
if exit_code != 0:
|
||||
raise RuntimeError("Gradle command returned error: {}".format(exit_code))
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def _extract_content_from_command_output(output, prefix):
|
||||
variants_line = [line for line in output.split("\n") if line.startswith(prefix)][0]
|
||||
return variants_line.split(" ", 1)[1]
|
||||
@ -1,199 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from fenix_taskgraph.gradle import get_variant
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_variant_config(config, tasks):
|
||||
for task in tasks:
|
||||
attributes = task.setdefault("attributes", {})
|
||||
if not attributes.get("build-type"):
|
||||
attributes["build-type"] = task["name"]
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_shippable_secrets(config, tasks):
|
||||
for task in tasks:
|
||||
secrets = task["run"].setdefault("secrets", [])
|
||||
dummy_secrets = task["run"].setdefault("dummy-secrets", [])
|
||||
|
||||
if (
|
||||
task.pop("include-shippable-secrets", False)
|
||||
and config.params["level"] == "3"
|
||||
):
|
||||
build_type = task["attributes"]["build-type"]
|
||||
gradle_build_type = task["run"]["gradle-build-type"]
|
||||
secret_index = "project/mobile/fenix/{}".format(build_type)
|
||||
secrets.extend(
|
||||
[
|
||||
{
|
||||
"key": key,
|
||||
"name": secret_index,
|
||||
"path": target_file,
|
||||
}
|
||||
for key, target_file in (
|
||||
("adjust", ".adjust_token"),
|
||||
(
|
||||
"firebase",
|
||||
"app/src/{}/res/values/firebase.xml".format(
|
||||
gradle_build_type
|
||||
),
|
||||
),
|
||||
("sentry_dsn", ".sentry_token"),
|
||||
("mls", ".mls_token"),
|
||||
("nimbus_url", ".nimbus"),
|
||||
("wallpaper_url", ".wallpaper_url"),
|
||||
("pocket_consumer_key", ".pocket_consumer_key"),
|
||||
)
|
||||
]
|
||||
)
|
||||
else:
|
||||
dummy_secrets.extend(
|
||||
[
|
||||
{
|
||||
"content": fake_value,
|
||||
"path": target_file,
|
||||
}
|
||||
for fake_value, target_file in (
|
||||
("faketoken", ".adjust_token"),
|
||||
("faketoken", ".mls_token"),
|
||||
("https://fake@sentry.prod.mozaws.net/368", ".sentry_token"),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_gradle_command(config, tasks):
|
||||
for task in tasks:
|
||||
gradle_build_type = task["run"]["gradle-build-type"]
|
||||
variant_config = get_variant(gradle_build_type)
|
||||
|
||||
task["run"]["gradlew"] = [
|
||||
"clean",
|
||||
"assemble{}".format(variant_config["name"].capitalize()),
|
||||
]
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def track_apk_size(config, tasks):
|
||||
for task in tasks:
|
||||
gradle_build_type = task["run"]["gradle-build-type"]
|
||||
variant_config = get_variant(gradle_build_type)
|
||||
|
||||
should_track_apk_size = task["run"].pop("track-apk-size", False)
|
||||
if should_track_apk_size:
|
||||
task["run"]["gradlew"].append(
|
||||
"apkSize{}".format(variant_config["name"].capitalize())
|
||||
)
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def extra_gradle_options(config, tasks):
|
||||
for task in tasks:
|
||||
for extra in task["run"].pop("gradle-extra-options", []):
|
||||
task["run"]["gradlew"].append(extra)
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_test_build_type(config, tasks):
|
||||
for task in tasks:
|
||||
test_build_type = task["run"].pop("test-build-type", "")
|
||||
if test_build_type:
|
||||
task["run"]["gradlew"].append("-PtestBuildType={}".format(test_build_type))
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_disable_optimization(config, tasks):
|
||||
for task in tasks:
|
||||
if task.pop("disable-optimization", False):
|
||||
task["run"]["gradlew"].append("-PdisableOptimization")
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_nightly_version(config, tasks):
|
||||
for task in tasks:
|
||||
if task.pop("include-nightly-version", False):
|
||||
task["run"]["gradlew"].extend(
|
||||
[
|
||||
# We only set the `official` flag here. The actual version name will be determined
|
||||
# by Gradle (depending on the Gecko/A-C version being used)
|
||||
"-Pofficial"
|
||||
]
|
||||
)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_release_version(config, tasks):
|
||||
for task in tasks:
|
||||
if task.pop("include-release-version", False):
|
||||
task["run"]["gradlew"].extend(
|
||||
["-PversionName={}".format(config.params["version"]), "-Pofficial"]
|
||||
)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_artifacts(config, tasks):
|
||||
for task in tasks:
|
||||
gradle_build_type = task["run"].pop("gradle-build-type")
|
||||
variant_config = get_variant(gradle_build_type)
|
||||
artifacts = task.setdefault("worker", {}).setdefault("artifacts", [])
|
||||
task["attributes"]["apks"] = apks = {}
|
||||
|
||||
if "apk-artifact-template" in task:
|
||||
artifact_template = task.pop("apk-artifact-template")
|
||||
for apk in variant_config["apks"]:
|
||||
apk_name = artifact_template["name"].format(**apk)
|
||||
artifacts.append(
|
||||
{
|
||||
"type": artifact_template["type"],
|
||||
"name": apk_name,
|
||||
"path": artifact_template["path"].format(
|
||||
gradle_build_type=gradle_build_type, **apk
|
||||
),
|
||||
}
|
||||
)
|
||||
apks[apk["abi"]] = {
|
||||
"name": apk_name,
|
||||
"github-name": artifact_template["github-name"].format(
|
||||
version=config.params["version"], **apk
|
||||
),
|
||||
}
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def filter_incomplete_translation(config, tasks):
|
||||
for task in tasks:
|
||||
if task.pop("filter-incomplete-translations", False):
|
||||
# filter-release-translations modifies source, which could cause problems if we ever start caching source
|
||||
pre_gradlew = task["run"].setdefault("pre-gradlew", [])
|
||||
pre_gradlew.append(
|
||||
["python", "automation/taskcluster/l10n/filter-release-translations.py"]
|
||||
)
|
||||
yield task
|
||||
Loading…
Reference in New Issue