Bug 1580778 - Migrate build tasks to taskgraph and retire the old decision task (#5488)
parent
7c55aff1be
commit
6a8f7224dc
@ -1,87 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
"""
|
||||
Decision task for nightly releases.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
|
||||
import taskcluster
|
||||
|
||||
from lib.gradle import get_variant
|
||||
from lib.tasks import (
|
||||
fetch_mozharness_task_id,
|
||||
schedule_task_graph,
|
||||
TaskBuilder,
|
||||
)
|
||||
from lib.chain_of_trust import (
|
||||
populate_chain_of_trust_task_graph,
|
||||
populate_chain_of_trust_required_but_unused_files
|
||||
)
|
||||
|
||||
def pr(builder):
|
||||
tasks = []
|
||||
|
||||
variant = get_variant('debug', 'geckoNightly')
|
||||
tasks.append(builder.craft_assemble_pr_task(variant))
|
||||
tasks.append(builder.craft_test_pr_task(variant))
|
||||
|
||||
for task in tasks:
|
||||
task['attributes']['code-review'] = True
|
||||
|
||||
return tasks
|
||||
|
||||
def push(builder):
|
||||
# We want the same tasks on pushes than on PRs, for now.
|
||||
return pr(builder)
|
||||
|
||||
def raptor(builder, is_staging):
|
||||
mozharness_task_id = fetch_mozharness_task_id()
|
||||
gecko_revision = taskcluster.Queue({
|
||||
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
|
||||
}).task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
|
||||
|
||||
variant = get_variant('forPerformanceTest', 'geckoNightly')
|
||||
build_task = builder.craft_assemble_raptor_task(variant)
|
||||
|
||||
# Signing and Raptor tasks are generated in taskgraph
|
||||
return [build_task]
|
||||
|
||||
|
||||
def release(builder, channel, engine, is_staging, version_name):
|
||||
variant = get_variant('fenix' + channel.capitalize(), engine)
|
||||
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||
|
||||
# The signing push-apk tasks are generated by taskgraph
|
||||
return [build_task]
|
||||
|
||||
|
||||
def release_as_fennec(builder, is_staging, version_name):
|
||||
variant = get_variant('fennecProduction', 'geckoBeta')
|
||||
channel = 'fennec-production'
|
||||
|
||||
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||
|
||||
# The signing task is generated by taskgraph
|
||||
return [build_task]
|
||||
|
||||
|
||||
def nightly_to_production_app(builder, is_staging, version_name):
|
||||
# Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the
|
||||
# "nightly" track. We're moving towards having each channel be published to its own app, but we need to
|
||||
# keep updating this "backwards-compatible" nightly for a while yet
|
||||
variant = get_variant('fenixNightlyLegacy', 'geckoNightly')
|
||||
taskcluster_apk_paths = variant.upstream_artifacts()
|
||||
|
||||
build_task = builder.craft_assemble_release_task(
|
||||
variant, 'nightly-legacy', is_staging, version_name)
|
||||
|
||||
# Nimbledroid, signing and push-apk tasks are generated by taskgraph
|
||||
return [build_task]
|
@ -1,20 +0,0 @@
|
||||
import json
|
||||
|
||||
|
||||
def populate_chain_of_trust_required_but_unused_files():
|
||||
# These files are needed to keep chainOfTrust happy. However, they have no
|
||||
# need for android-components, at the moment. For more details, see:
|
||||
# https://github.com/mozilla-releng/scriptworker/pull/209/files#r184180585
|
||||
|
||||
for file_names in ('actions.json', 'parameters.yml'):
|
||||
with open(file_names, 'w') as f:
|
||||
json.dump({}, f) # Yaml is a super-set of JSON.
|
||||
|
||||
|
||||
def populate_chain_of_trust_task_graph(full_task_graph):
|
||||
# full_task_graph must follow the format:
|
||||
# {
|
||||
# task_id: full_task_definition
|
||||
# }
|
||||
with open('task-graph.json', 'w') as f:
|
||||
json.dump(full_task_graph, f)
|
@ -1,15 +0,0 @@
|
||||
import re
|
||||
|
||||
|
||||
def convert_camel_case_into_kebab_case(string):
|
||||
# Inspired from https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case # noqa: E501
|
||||
first_pass = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', string)
|
||||
return re.sub('([a-z0-9])([A-Z])', r'\1-\2', first_pass).lower()
|
||||
|
||||
|
||||
def lower_case_first_letter(string):
|
||||
return '{}{}'.format(string[0].lower(), string[1:])
|
||||
|
||||
|
||||
def upper_case_first_letter(string):
|
||||
return string[0].upper() + string[1:]
|
@ -1,34 +0,0 @@
|
||||
import taskcluster
|
||||
|
||||
|
||||
class VariantApk:
|
||||
def __init__(self, build_type, abi, engine, file_name):
|
||||
self.abi = abi
|
||||
self.taskcluster_path = u'public/build/{}/{}/target.apk'.format(abi, engine)
|
||||
self.absolute_path = '/opt/fenix/app/build/outputs/apk/{}/{}/{}'.format(engine, build_type, file_name)
|
||||
|
||||
|
||||
class Variant:
|
||||
def __init__(self, name, build_type, apks):
|
||||
self.name = name
|
||||
self.build_type = build_type
|
||||
self._apks = apks
|
||||
|
||||
def get_apk(self, abi):
|
||||
return [apk for apk in self._apks if apk.abi == abi][0]
|
||||
|
||||
def artifacts(self):
|
||||
return {
|
||||
apk.taskcluster_path: {
|
||||
'type': 'file',
|
||||
'path': apk.absolute_path,
|
||||
'expires': taskcluster.stringDate(taskcluster.fromNow('1 year')),
|
||||
} for apk in self._apks
|
||||
}
|
||||
|
||||
def upstream_artifacts(self):
|
||||
return [apk.taskcluster_path for apk in self._apks]
|
||||
|
||||
@property
|
||||
def upstream_artifacts_per_abi(self):
|
||||
return {apk.abi: apk.taskcluster_path for apk in self._apks}
|
@ -1,105 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
import argparse
|
||||
import datetime
|
||||
import jsone
|
||||
import os
|
||||
import slugid
|
||||
import taskcluster
|
||||
import yaml
|
||||
|
||||
from git import Repo
|
||||
from lib.tasks import schedule_task
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(__file__), '../..')
|
||||
|
||||
|
||||
class InvalidGithubRepositoryError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def calculate_git_references(root):
|
||||
repo = Repo(root)
|
||||
remote = repo.remote()
|
||||
branch = repo.head.reference
|
||||
|
||||
if not remote.url.startswith('https://github.com'):
|
||||
raise InvalidGithubRepositoryError('expected remote to be a GitHub repository (accessed via HTTPs)')
|
||||
|
||||
html_url = remote.url[:-4] if remote.url.endswith('.git') else remote.url
|
||||
return html_url, str(branch), str(branch.commit)
|
||||
|
||||
|
||||
def make_decision_task(params):
|
||||
"""Generate a basic decision task, based on the root .taskcluster.yml"""
|
||||
with open(os.path.join(ROOT, '.taskcluster.yml'), 'rb') as f:
|
||||
taskcluster_yml = yaml.safe_load(f)
|
||||
|
||||
slugids = {}
|
||||
|
||||
def as_slugid(name):
|
||||
if name not in slugids:
|
||||
slugids[name] = slugid.nice()
|
||||
return slugids[name]
|
||||
|
||||
repository_parts = params['html_url'].split('/')
|
||||
repository_full_name = '/'.join((repository_parts[-2], repository_parts[-1]))
|
||||
|
||||
# provide a similar JSON-e context to what taskcluster-github provides
|
||||
context = {
|
||||
'tasks_for': 'cron',
|
||||
'cron': {
|
||||
'task_id': params['cron_task_id'],
|
||||
'name': params['name'],
|
||||
},
|
||||
'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z',
|
||||
'as_slugid': as_slugid,
|
||||
'event': {
|
||||
'repository': {
|
||||
'html_url': params['html_url'],
|
||||
'full_name': repository_full_name,
|
||||
},
|
||||
'release': {
|
||||
'tag_name': params['head_rev'],
|
||||
'target_commitish': params['branch'],
|
||||
},
|
||||
'sender': {
|
||||
'login': 'TaskclusterHook',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rendered = jsone.render(taskcluster_yml, context)
|
||||
if len(rendered['tasks']) != 1:
|
||||
raise Exception('Expected .taskcluster.yml to only produce one cron task')
|
||||
task = rendered['tasks'][0]
|
||||
|
||||
task_id = task.pop('taskId')
|
||||
return task_id, task
|
||||
|
||||
|
||||
def schedule():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Creates and submit a graph of tasks on Taskcluster.'
|
||||
)
|
||||
|
||||
parser.add_argument('name', choices=['nightly', 'raptor'])
|
||||
result = parser.parse_args()
|
||||
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
|
||||
|
||||
html_url, branch, head_rev = calculate_git_references(ROOT)
|
||||
params = {
|
||||
'html_url': html_url,
|
||||
'head_rev': head_rev,
|
||||
'branch': branch,
|
||||
'cron_task_id': os.environ.get('CRON_TASK_ID', '<cron_task_id>'),
|
||||
'name': result.name,
|
||||
}
|
||||
decision_task_id, decision_task = make_decision_task(params)
|
||||
schedule_task(queue, decision_task_id, decision_task)
|
||||
print('All scheduled!')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
schedule()
|
@ -0,0 +1,108 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: taskgraph.loader.transform:loader
|
||||
|
||||
transforms:
|
||||
- fenix_taskgraph.transforms.build:transforms
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
job-defaults:
|
||||
description: Build Fenix from source code.
|
||||
treeherder:
|
||||
kind: build
|
||||
symbol: B
|
||||
platform: android-all/opt
|
||||
tier: 1
|
||||
worker-type: b-android
|
||||
worker:
|
||||
docker-image: {in-tree: base}
|
||||
max-run-time: 7200
|
||||
chain-of-trust: true
|
||||
run:
|
||||
using: gradlew
|
||||
use-caches: false
|
||||
run-on-tasks-for: []
|
||||
# Builds generate multiple APKs with different ABIs. For each APK described
|
||||
# by `gradlew printVariant`, an artifact will be generated. `variant` and
|
||||
# the per-apk config from `printVariant` can be used as substitutions in
|
||||
# `name` and `path`.
|
||||
apk-artifact-template:
|
||||
type: file
|
||||
name: public/build/{abi}/{geckoview_engine}/target.apk
|
||||
path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}'
|
||||
|
||||
jobs:
|
||||
debug:
|
||||
attributes:
|
||||
code-review: true
|
||||
run-on-tasks-for: [github-pull-request, github-push]
|
||||
run:
|
||||
geckoview-engine: geckoNightly
|
||||
gradle-build-type: debug
|
||||
treeherder:
|
||||
symbol: debug(B)
|
||||
|
||||
performance-test:
|
||||
run:
|
||||
geckoview-engine: geckoNightly
|
||||
gradle-build-type: forPerformanceTest
|
||||
treeherder:
|
||||
symbol: forPerformanceTest(B)
|
||||
|
||||
nightly:
|
||||
attributes:
|
||||
nightly: true
|
||||
include-nightly-version: true
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
geckoview-engine: geckoNightly
|
||||
gradle-build-type: fenixNightly
|
||||
treeherder:
|
||||
symbol: nightly(B)
|
||||
|
||||
nightly-legacy:
|
||||
attributes:
|
||||
nightly: true
|
||||
include-nightly-version: true
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
geckoview-engine: geckoNightly
|
||||
gradle-build-type: fenixNightlyLegacy
|
||||
treeherder:
|
||||
symbol: nightlyLegacy(B)
|
||||
|
||||
fennec-production:
|
||||
attributes:
|
||||
nightly: true
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
geckoview-engine: geckoBeta
|
||||
gradle-build-type: fennecProduction
|
||||
treeherder:
|
||||
symbol: nightlyFennec(B)
|
||||
|
||||
beta:
|
||||
attributes:
|
||||
build-type: production
|
||||
release-type: beta
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
geckoview-engine: geckoBeta
|
||||
gradle-build-type: fenixBeta
|
||||
run-on-tasks-for: [github-release]
|
||||
treeherder:
|
||||
symbol: beta(B)
|
||||
|
||||
production:
|
||||
attributes:
|
||||
release-type: production
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
geckoview-engine: geckoBeta
|
||||
gradle-build-type: fenixProduction
|
||||
run-on-tasks-for: [github-release]
|
||||
treeherder:
|
||||
symbol: production(B)
|
@ -1,81 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
current_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..'))
|
||||
sys.path.append(project_dir)
|
||||
|
||||
from automation.taskcluster.decision_task import (
|
||||
pr,
|
||||
push,
|
||||
raptor,
|
||||
nightly_to_production_app,
|
||||
release,
|
||||
release_as_fennec,
|
||||
)
|
||||
from automation.taskcluster.lib.tasks import TaskBuilder
|
||||
|
||||
|
||||
def loader(kind, path, config, params, loaded_tasks):
|
||||
repo_url = params['head_repository']
|
||||
commit = params['head_rev']
|
||||
trust_level = int(params['level'])
|
||||
|
||||
builder = TaskBuilder(
|
||||
task_id=os.environ.get('TASK_ID'),
|
||||
repo_url=repo_url,
|
||||
git_ref=params['head_ref'],
|
||||
short_head_branch=params['head_ref'],
|
||||
commit=commit,
|
||||
owner=params['owner'],
|
||||
source='{}/raw/{}/.taskcluster.yml'.format(repo_url, commit),
|
||||
scheduler_id='mobile-level-{}'.format(trust_level),
|
||||
tasks_priority='highest', # TODO parametrize
|
||||
date_string=params['moz_build_date'],
|
||||
trust_level=trust_level,
|
||||
)
|
||||
|
||||
is_staging = trust_level != 3
|
||||
|
||||
tasks_for = params['tasks_for']
|
||||
if tasks_for == 'github-pull-request':
|
||||
ordered_groups_of_tasks = pr(builder)
|
||||
elif tasks_for == 'github-push':
|
||||
ordered_groups_of_tasks = push(builder)
|
||||
elif tasks_for == 'github-release':
|
||||
git_tag = os.environ['GIT_TAG']
|
||||
version = git_tag[1:] # remove prefixed "v"
|
||||
beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
|
||||
production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
|
||||
if beta_semver.match(git_tag):
|
||||
ordered_groups_of_tasks = release(builder, 'beta', 'geckoBeta', is_staging, version)
|
||||
elif production_semver.match(git_tag):
|
||||
ordered_groups_of_tasks = release(builder, 'production', 'geckoBeta', is_staging, version)
|
||||
else:
|
||||
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
|
||||
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
|
||||
elif tasks_for == 'cron':
|
||||
target_tasks_method = params['target_tasks_method']
|
||||
if target_tasks_method == 'raptor':
|
||||
ordered_groups_of_tasks = raptor(builder, is_staging)
|
||||
elif target_tasks_method == 'nightly':
|
||||
now = datetime.datetime.now().strftime('%y%m%d %H:%M')
|
||||
nightly_version = 'Nightly {}'.format(now)
|
||||
ordered_groups_of_tasks = release(builder, 'nightly', 'geckoNightly', is_staging, nightly_version) \
|
||||
+ nightly_to_production_app(builder, is_staging, nightly_version)
|
||||
ordered_groups_of_tasks += release_as_fennec(builder, is_staging, 'Signed-as-Fennec Nightly {}'.format(now))
|
||||
else:
|
||||
raise NotImplementedError('Unsupported task_name "{}"'.format(params))
|
||||
else:
|
||||
raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for))
|
||||
|
||||
for task in ordered_groups_of_tasks:
|
||||
yield task
|
@ -1,27 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the nimbledroid
|
||||
kind. This transform won't be necessary anymore once the full tasgraph migration is
|
||||
done
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.treeherder import inherit_treeherder_from_dep
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def filter_out_non_nightly_legacy(config, tasks):
|
||||
for dep_task in config.kind_dependencies_tasks:
|
||||
if dep_task.label == 'Build FenixNightlyLegacy task':
|
||||
for task in tasks:
|
||||
yield task
|
@ -1,75 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
_CHANNEL_PER_TASK_NAME = {
|
||||
'nightly': 'nightly',
|
||||
'nightly-legacy': 'production',
|
||||
'production': 'production',
|
||||
}
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_name_and_attributes(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task["primary-dependency"]
|
||||
task["dependencies"] = {"signing": dep.label}
|
||||
task.setdefault("attributes", {}).update(dep.attributes.copy())
|
||||
task["name"] = _get_dependent_job_name_without_its_kind(dep)
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
def _get_dependent_job_name_without_its_kind(dependent_job):
|
||||
return dependent_job.label[len(dependent_job.kind) + 1:]
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_treeherder_definition(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task["primary-dependency"]
|
||||
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
|
||||
treeherder_group = dep.task["extra"]["treeherder"]["groupSymbol"]
|
||||
treeherder_symbol = join_symbol(treeherder_group, 'gp')
|
||||
task["treeherder"]["symbol"] = treeherder_symbol
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_worker_definition(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task.pop("primary-dependency")
|
||||
task_name = task["name"]
|
||||
|
||||
worker_definition = {}
|
||||
worker_definition["upstream-artifacts"] = [{
|
||||
"taskId": {"task-reference": "<signing>"},
|
||||
"taskType": "signing",
|
||||
"paths": dep.attributes["apks"].values(),
|
||||
}]
|
||||
worker_definition["dep"] = config.params["level"] != "3"
|
||||
worker_definition["channel"] = _CHANNEL_PER_TASK_NAME[task_name]
|
||||
|
||||
# Fenix production doesn't follow the rule {product}-{channel}
|
||||
worker_definition["certificate-alias"] = 'fenix' if task_name == 'production' else \
|
||||
"{}-{}".format(task["worker"]["product"], worker_definition["channel"])
|
||||
|
||||
if task_name == 'nightly-legacy':
|
||||
worker_definition["google-play-track"] = 'nightly'
|
||||
|
||||
task["worker"].update(worker_definition)
|
||||
yield task
|
@ -1,67 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
# TODO remove this transform once build tasks are migrated to taskgraph
|
||||
@transforms.add
|
||||
def fetch_old_decision_dependency(config, tasks):
|
||||
for task in tasks:
|
||||
for dep_task in config.kind_dependencies_tasks:
|
||||
expected_signing_dep_label = task['dependencies']['build']
|
||||
if dep_task.label != expected_signing_dep_label:
|
||||
continue
|
||||
|
||||
task['primary-dependency'] = dep_task
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def define_signing_attributes(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task["primary-dependency"]
|
||||
attributes = task.setdefault("attributes", {})
|
||||
upstream_attributes = dep.attributes.copy()
|
||||
upstream_attributes.update(attributes)
|
||||
task["attributes"] = upstream_attributes
|
||||
|
||||
task["attributes"]["signed"] = True
|
||||
if "run_on_tasks_for" in task["attributes"]:
|
||||
task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
|
||||
|
||||
for key in ("worker-type", "worker.signing-type"):
|
||||
resolve_keyed_by(
|
||||
task,
|
||||
key,
|
||||
item_name=task["name"],
|
||||
variant=task["attributes"]["build-type"],
|
||||
level=config.params["level"],
|
||||
)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_signing_task(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task.pop("primary-dependency")
|
||||
task["dependencies"] = {"build": dep.label}
|
||||
task["worker"]["upstream-artifacts"] = [
|
||||
{
|
||||
"taskId": {"task-reference": "<build>"},
|
||||
"taskType": "build",
|
||||
"paths": dep.attributes["apks"].values(),
|
||||
"formats": ["autograph_apk"],
|
||||
}
|
||||
]
|
||||
yield task
|
@ -0,0 +1,128 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import datetime
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from fenix_taskgraph.gradle import get_variant
|
||||
from fenix_taskgraph.util import upper_case_first_letter
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_variant_config(config, tasks):
|
||||
for task in tasks:
|
||||
attributes = task.setdefault("attributes", {})
|
||||
if not attributes.get("build-type"):
|
||||
attributes["build-type"] = task["name"]
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_shippable_secrets(config, tasks):
|
||||
for task in tasks:
|
||||
secrets = task["run"].setdefault("secrets", [])
|
||||
|
||||
if task.pop("include-shippable-secrets", False) and config.params["level"] == "3":
|
||||
build_type = task["attributes"]["build-type"]
|
||||
secret_index = 'project/mobile/fenix/{}'.format(build_type)
|
||||
secrets.extend([{
|
||||
"key": key,
|
||||
"name": secret_index,
|
||||
"path": target_file,
|
||||
} for key, target_file in (
|
||||
('adjust', '.adjust_token'),
|
||||
('firebase', 'app/src/{}/res/values/firebase.xml'.format(build_type)),
|
||||
('digital_asset_links', '.digital_asset_links_token'),
|
||||
('leanplum', '.leanplum_token'),
|
||||
('sentry_dsn', '.sentry_token'),
|
||||
)])
|
||||
else:
|
||||
task["run"]["pre-gradlew"] = [[
|
||||
"echo", '"{}"'.format(fake_value), ">", target_file
|
||||
] for fake_value, target_file in (
|
||||
("--", ".adjust_token"),
|
||||
("", ".digital_asset_links_token"),
|
||||
("-:-", ".leanplum_token"),
|
||||
("https://fake@sentry.prod.mozaws.net/368", ".sentry_token"),
|
||||
)]
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_gradle_command(config, tasks):
|
||||
for task in tasks:
|
||||
gradle_build_type = task["run"]["gradle-build-type"]
|
||||
geckoview_engine = task["run"]["geckoview-engine"]
|
||||
variant_config = get_variant(gradle_build_type, geckoview_engine)
|
||||
|
||||
task["run"]["gradlew"] = [
|
||||
"clean",
|
||||
"assemble{}".format(upper_case_first_letter(variant_config["name"]))
|
||||
]
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_nightly_version(config, tasks):
|
||||
push_date_string = config.params["moz_build_date"]
|
||||
push_date_time = datetime.datetime.strptime(push_date_string, "%Y%m%d%H%M%S")
|
||||
formated_date_time = 'Nightly {}'.format(push_date_time.strftime('%y%m%d %H:%M'))
|
||||
|
||||
for task in tasks:
|
||||
if task.pop("include-nightly-version", False):
|
||||
task["run"]["gradlew"].append('-PversionName="{}"'.format(formated_date_time))
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_release_version(config, tasks):
|
||||
for task in tasks:
|
||||
if task.pop("include-release-version", False):
|
||||
# TODO Move GIT_TAG to a parameter
|
||||
git_tag = os.environ['GIT_TAG']
|
||||
version = git_tag[1:] # remove prefixed "v"
|
||||
|
||||
task["run"]["gradlew"].append('-PversionName="{}"'.format(version))
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_artifacts(config, tasks):
|
||||
for task in tasks:
|
||||
gradle_build_type = task["run"].pop("gradle-build-type")
|
||||
geckoview_engine = task["run"].pop("geckoview-engine")
|
||||
variant_config = get_variant(gradle_build_type, geckoview_engine)
|
||||
artifacts = task.setdefault("worker", {}).setdefault("artifacts", [])
|
||||
task["attributes"]["apks"] = apks = {}
|
||||
|
||||
if "apk-artifact-template" in task:
|
||||
artifact_template = task.pop("apk-artifact-template")
|
||||
for apk in variant_config["apks"]:
|
||||
apk_name = artifact_template["name"].format(
|
||||
geckoview_engine=geckoview_engine, **apk
|
||||
)
|
||||
artifacts.append({
|
||||
"type": artifact_template["type"],
|
||||
"name": apk_name,
|
||||
"path": artifact_template["path"].format(
|
||||
geckoview_engine=geckoview_engine,
|
||||
gradle_build_type=gradle_build_type,
|
||||
**apk
|
||||
),
|
||||
})
|
||||
apks[apk["abi"]] = apk_name
|
||||
|
||||
yield task
|
@ -0,0 +1,46 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def resolve_keys(config, tasks):
|
||||
for task in tasks:
|
||||
for key in ("worker.channel", "worker.dep", "worker.google-play-track"):
|
||||
resolve_keyed_by(
|
||||
task,
|
||||
key,
|
||||
item_name=task["name"],
|
||||
**{
|
||||
'build-type': task["attributes"]["build-type"],
|
||||
'level': config.params["level"],
|
||||
}
|
||||
)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_worker_definition(config, tasks):
|
||||
for task in tasks:
|
||||
worker_definition = {}
|
||||
worker_definition["certificate-alias"] = "{}-{}".format(
|
||||
task["worker"]["product"], task["worker"]["channel"]
|
||||
)
|
||||
# Fenix production doesn't follow the rule {product}-{channel}
|
||||
if task["attributes"]["build-type"] == "production":
|
||||
worker_definition["certificate-alias"] = "fenix"
|
||||
|
||||
task["worker"].update(worker_definition)
|
||||
yield task
|
@ -0,0 +1,46 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def resolve_keys(config, tasks):
|
||||
for task in tasks:
|
||||
for key in ("index", "worker-type", "worker.signing-type"):
|
||||
resolve_keyed_by(
|
||||
task,
|
||||
key,
|
||||
item_name=task["name"],
|
||||
**{
|
||||
'build-type': task["attributes"]["build-type"],
|
||||
'level': config.params["level"],
|
||||
}
|
||||
)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def set_signing_attributes(config, tasks):
|
||||
for task in tasks:
|
||||
task["attributes"]["signed"] = True
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def set_signing_format(config, tasks):
|
||||
for task in tasks:
|
||||
for upstream_artifact in task["worker"]["upstream-artifacts"]:
|
||||
upstream_artifact["formats"] = ["autograph_apk"]
|
||||
yield task
|
@ -0,0 +1,62 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the single_dep jobs.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_name_and_attributes(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task["primary-dependency"]
|
||||
task["dependencies"] = {dep.kind: dep.label}
|
||||
copy_of_attributes = dep.attributes.copy()
|
||||
task.setdefault("attributes", copy_of_attributes)
|
||||
# run_on_tasks_for is set as an attribute later in the pipeline
|
||||
task.setdefault("run-on-tasks-for", copy_of_attributes['run_on_tasks_for'])
|
||||
task["name"] = _get_dependent_job_name_without_its_kind(dep)
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
def _get_dependent_job_name_without_its_kind(dependent_job):
|
||||
return dependent_job.label[len(dependent_job.kind) + 1:]
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_upstream_artifacts(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task["primary-dependency"]
|
||||
|
||||
worker_definition = {}
|
||||
worker_definition["upstream-artifacts"] = [{
|
||||
"taskId": {"task-reference": "<{}>".format(dep.kind)},
|
||||
"taskType": dep.kind,
|
||||
"paths": sorted(dep.attributes["apks"].values()),
|
||||
}]
|
||||
|
||||
task["worker"].update(worker_definition)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_treeherder_definition(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task.pop("primary-dependency")
|
||||
|
||||
task.setdefault("treeherder", {}).update(inherit_treeherder_from_dep(task, dep))
|
||||
job_group = dep.task["extra"]["treeherder"].get("groupSymbol", "?")
|
||||
job_symbol = task["treeherder"].pop("job-symbol")
|
||||
full_symbol = join_symbol(job_group, job_symbol)
|
||||
task["treeherder"]["symbol"] = full_symbol
|
||||
|
||||
yield task
|
@ -1,9 +1,11 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: fenix_taskgraph.loader.old_decision:loader
|
||||
|
||||
transforms: []
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
# XXX Everything is done in the loader until everything is migrated
|
||||
import re
|
||||
|
||||
|
||||
def upper_case_first_letter(string):
|
||||
return string[0].upper() + string[1:]
|
Loading…
Reference in New Issue