mirror of
https://github.com/fork-maintainers/iceraven-browser
synced 2024-11-05 21:20:45 +00:00
Bug 1808606 - part 2: Make multi_dep merge attribute dictionaries
(cherry picked from commit 2559085104611f9b135e3f8c8db6b901b1ed363d)
This commit is contained in:
Johan Lorenzo
mergify[bot]
parent
c9c91f3599
commit
0203ae73f1
@ -1,104 +0,0 @@
|
|||||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
||||||
|
|
||||||
import copy
|
|
||||||
|
|
||||||
|
|
||||||
# Define a collection of group_by functions
|
|
||||||
GROUP_BY_MAP = {}
|
|
||||||
|
|
||||||
|
|
||||||
def group_by(name):
|
|
||||||
def wrapper(func):
|
|
||||||
GROUP_BY_MAP[name] = func
|
|
||||||
return func
|
|
||||||
|
|
||||||
return wrapper
|
|
||||||
|
|
||||||
|
|
||||||
def group_tasks(config, tasks):
|
|
||||||
group_by_fn = GROUP_BY_MAP[config["group-by"]]
|
|
||||||
|
|
||||||
groups = group_by_fn(config, tasks)
|
|
||||||
|
|
||||||
for combinations in groups.values():
|
|
||||||
dependencies = [copy.deepcopy(t) for t in combinations]
|
|
||||||
yield dependencies
|
|
||||||
|
|
||||||
|
|
||||||
@group_by("build-type")
|
|
||||||
def build_type_grouping(config, tasks):
|
|
||||||
groups = {}
|
|
||||||
kind_dependencies = config.get("kind-dependencies")
|
|
||||||
only_build_type = config.get("only-for-build-types")
|
|
||||||
|
|
||||||
for task in tasks:
|
|
||||||
if task.kind not in kind_dependencies:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if only_build_type:
|
|
||||||
build_type = task.attributes.get("build-type")
|
|
||||||
if build_type not in only_build_type:
|
|
||||||
continue
|
|
||||||
|
|
||||||
build_type = task.attributes.get("build-type")
|
|
||||||
|
|
||||||
groups.setdefault(build_type, []).append(task)
|
|
||||||
|
|
||||||
return groups
|
|
||||||
|
|
||||||
|
|
||||||
@group_by("attributes")
|
|
||||||
def attributes_grouping(config, tasks):
|
|
||||||
groups = {}
|
|
||||||
kind_dependencies = config.get("kind-dependencies")
|
|
||||||
only_attributes = config.get("only-for-attributes")
|
|
||||||
|
|
||||||
for task in tasks:
|
|
||||||
if task.kind not in kind_dependencies:
|
|
||||||
continue
|
|
||||||
|
|
||||||
group_attr = None
|
|
||||||
if only_attributes:
|
|
||||||
if not any(attr in task.attributes for attr in only_attributes):
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
continue
|
|
||||||
|
|
||||||
groups.setdefault(task.label, []).append(task)
|
|
||||||
|
|
||||||
return groups
|
|
||||||
|
|
||||||
|
|
||||||
@group_by("single-locale")
|
|
||||||
def single_locale_grouping(config, tasks):
|
|
||||||
"""Split by a single locale (but also by platform, build-type, product)
|
|
||||||
|
|
||||||
The locale can be `None` (en-US build/signing/repackage), a single locale,
|
|
||||||
or multiple locales per task, e.g. for l10n chunking. In the case of a task
|
|
||||||
with, say, five locales, the task will show up in all five locale groupings.
|
|
||||||
|
|
||||||
This grouping is written for non-partner-repack beetmover, but might also
|
|
||||||
be useful elsewhere.
|
|
||||||
|
|
||||||
"""
|
|
||||||
groups = {}
|
|
||||||
|
|
||||||
for task in tasks:
|
|
||||||
if task.kind not in config.get("kind-dependencies", []):
|
|
||||||
continue
|
|
||||||
|
|
||||||
platform = task.attributes.get("build_platform")
|
|
||||||
build_type = task.attributes.get("build_type")
|
|
||||||
task_locale = task.attributes.get("locale")
|
|
||||||
chunk_locales = task.attributes.get("chunk_locales")
|
|
||||||
locales = chunk_locales or [task_locale]
|
|
||||||
|
|
||||||
for locale in locales:
|
|
||||||
locale_key = (platform, build_type, locale)
|
|
||||||
groups.setdefault(locale_key, [])
|
|
||||||
if task not in groups[locale_key]:
|
|
||||||
groups[locale_key].append(task)
|
|
||||||
|
|
||||||
return groups
|
|
||||||
@ -1,88 +0,0 @@
|
|||||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
||||||
|
|
||||||
import copy
|
|
||||||
|
|
||||||
from voluptuous import Required
|
|
||||||
|
|
||||||
from taskgraph.task import Task
|
|
||||||
from taskgraph.util.schema import Schema
|
|
||||||
|
|
||||||
from . import group_tasks
|
|
||||||
|
|
||||||
|
|
||||||
schema = Schema(
|
|
||||||
{
|
|
||||||
Required("primary-dependency", "primary dependency task"): Task,
|
|
||||||
Required(
|
|
||||||
"dependent-tasks",
|
|
||||||
"dictionary of dependent tasks, keyed by kind",
|
|
||||||
): {str: Task},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def loader(kind, path, config, params, loaded_tasks):
|
|
||||||
"""
|
|
||||||
Load tasks based on the jobs dependant kinds, designed for use as
|
|
||||||
multiple-dependent needs.
|
|
||||||
Required ``group-by-fn`` is used to define how we coalesce the
|
|
||||||
multiple deps together to pass to transforms, e.g. all kinds specified get
|
|
||||||
collapsed by platform with `platform`
|
|
||||||
Optional ``primary-dependency`` (ordered list or string) is used to determine
|
|
||||||
which upstream kind to inherit attrs from. See ``get_primary_dep``.
|
|
||||||
The `only-for-build-type` kind configuration, if specified, will limit
|
|
||||||
the build types for which a job will be created.
|
|
||||||
Optional ``job-template`` kind configuration value, if specified, will be used to
|
|
||||||
pass configuration down to the specified transforms used.
|
|
||||||
"""
|
|
||||||
job_template = config.get("job-template")
|
|
||||||
|
|
||||||
for dep_tasks in group_tasks(config, loaded_tasks):
|
|
||||||
kinds = [dep.kind for dep in dep_tasks]
|
|
||||||
kinds_occurrences = {kind: kinds.count(kind) for kind in kinds}
|
|
||||||
|
|
||||||
dep_tasks_per_unique_key = {
|
|
||||||
dep.kind if kinds_occurrences[dep.kind] == 1 else dep.label: dep
|
|
||||||
for dep in dep_tasks
|
|
||||||
}
|
|
||||||
|
|
||||||
job = {"dependent-tasks": dep_tasks_per_unique_key}
|
|
||||||
job["primary-dependency"] = get_primary_dep(config, dep_tasks_per_unique_key)
|
|
||||||
if job_template:
|
|
||||||
job.update(copy.deepcopy(job_template))
|
|
||||||
|
|
||||||
yield job
|
|
||||||
|
|
||||||
|
|
||||||
def get_primary_dep(config, dep_tasks):
|
|
||||||
"""Find the dependent task to inherit attributes from.
|
|
||||||
If ``primary-dependency`` is defined in ``kind.yml`` and is a string,
|
|
||||||
then find the first dep with that task kind and return it. If it is
|
|
||||||
defined and is a list, the first kind in that list with a matching dep
|
|
||||||
is the primary dependency. If it's undefined, return the first dep.
|
|
||||||
"""
|
|
||||||
primary_dependencies = config.get("primary-dependency")
|
|
||||||
if isinstance(primary_dependencies, str):
|
|
||||||
primary_dependencies = [primary_dependencies]
|
|
||||||
if not primary_dependencies:
|
|
||||||
assert len(dep_tasks) == 1, "Must define a primary-dependency!"
|
|
||||||
return dep_tasks.values()[0]
|
|
||||||
primary_dep = None
|
|
||||||
for primary_kind in primary_dependencies:
|
|
||||||
for dep_kind in dep_tasks:
|
|
||||||
if dep_kind == primary_kind:
|
|
||||||
assert (
|
|
||||||
primary_dep is None
|
|
||||||
), "Too many primary dependent tasks in dep_tasks: {}!".format(
|
|
||||||
[t.label for t in dep_tasks]
|
|
||||||
)
|
|
||||||
primary_dep = dep_tasks[dep_kind]
|
|
||||||
if primary_dep is None:
|
|
||||||
raise Exception(
|
|
||||||
"Can't find dependency of {}: {}".format(
|
|
||||||
config["primary-dependency"], config
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return primary_dep
|
|
||||||
@ -1,118 +0,0 @@
|
|||||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
||||||
"""
|
|
||||||
Apply some defaults and minor modifications to the single_dep jobs.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from taskgraph.transforms.base import TransformSequence
|
|
||||||
from taskgraph.util.schema import resolve_keyed_by
|
|
||||||
from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol
|
|
||||||
|
|
||||||
from fenix_taskgraph.util.scriptworker import generate_beetmover_upstream_artifacts
|
|
||||||
|
|
||||||
|
|
||||||
transforms = TransformSequence()
|
|
||||||
|
|
||||||
|
|
||||||
@transforms.add
|
|
||||||
def build_name_and_attributes(config, tasks):
|
|
||||||
for task in tasks:
|
|
||||||
task["dependencies"] = {
|
|
||||||
dep_key: dep.label for dep_key, dep in _get_all_deps(task).items()
|
|
||||||
}
|
|
||||||
primary_dep = task["primary-dependency"]
|
|
||||||
attributes = primary_dep.attributes.copy()
|
|
||||||
attributes.update(task.get("attributes", {}))
|
|
||||||
task["attributes"] = attributes
|
|
||||||
# run_on_tasks_for is set as an attribute later in the pipeline
|
|
||||||
task.setdefault("run-on-tasks-for", attributes["run_on_tasks_for"])
|
|
||||||
task["name"] = _get_dependent_job_name_without_its_kind(primary_dep)
|
|
||||||
|
|
||||||
yield task
|
|
||||||
|
|
||||||
|
|
||||||
def _get_dependent_job_name_without_its_kind(dependent_job):
|
|
||||||
return dependent_job.label[len(dependent_job.kind) + 1 :]
|
|
||||||
|
|
||||||
|
|
||||||
def _get_all_deps(task):
|
|
||||||
if task.get("dependent-tasks"):
|
|
||||||
return task["dependent-tasks"]
|
|
||||||
|
|
||||||
return {task["primary-dependency"].kind: task["primary-dependency"]}
|
|
||||||
|
|
||||||
|
|
||||||
@transforms.add
|
|
||||||
def resolve_keys(config, tasks):
|
|
||||||
for task in tasks:
|
|
||||||
resolve_keyed_by(
|
|
||||||
task,
|
|
||||||
"treeherder.job-symbol",
|
|
||||||
item_name=task["name"],
|
|
||||||
**{
|
|
||||||
"build-type": task["attributes"]["build-type"],
|
|
||||||
"level": config.params["level"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
yield task
|
|
||||||
|
|
||||||
|
|
||||||
@transforms.add
|
|
||||||
def build_upstream_artifacts(config, tasks):
|
|
||||||
for task in tasks:
|
|
||||||
worker_definition = {
|
|
||||||
"upstream-artifacts": [],
|
|
||||||
}
|
|
||||||
|
|
||||||
if "artifact_map" in task["attributes"]:
|
|
||||||
# Beetmover tasks use declarative artifacts.
|
|
||||||
locale = task["attributes"].get("locale")
|
|
||||||
build_type = task["attributes"]["build-type"]
|
|
||||||
worker_definition[
|
|
||||||
"upstream-artifacts"
|
|
||||||
] = generate_beetmover_upstream_artifacts(config, task, build_type, locale)
|
|
||||||
else:
|
|
||||||
for dep in _get_all_deps(task).values():
|
|
||||||
paths = sorted(
|
|
||||||
[
|
|
||||||
apk_metadata["name"]
|
|
||||||
for apk_metadata in dep.attributes.get("apks", {}).values()
|
|
||||||
]
|
|
||||||
)
|
|
||||||
if paths:
|
|
||||||
worker_definition["upstream-artifacts"].append(
|
|
||||||
{
|
|
||||||
"taskId": {"task-reference": "<{}>".format(dep.kind)},
|
|
||||||
"taskType": dep.kind,
|
|
||||||
"paths": paths,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
task.setdefault("worker", {}).update(worker_definition)
|
|
||||||
yield task
|
|
||||||
|
|
||||||
|
|
||||||
@transforms.add
|
|
||||||
def build_treeherder_definition(config, tasks):
|
|
||||||
for task in tasks:
|
|
||||||
dep = task.pop("primary-dependency")
|
|
||||||
|
|
||||||
task.setdefault("treeherder", {}).update(inherit_treeherder_from_dep(task, dep))
|
|
||||||
job_group = dep.task["extra"]["treeherder"].get("groupSymbol", "?")
|
|
||||||
job_symbol = task["treeherder"].pop("job-symbol")
|
|
||||||
full_symbol = join_symbol(job_group, job_symbol)
|
|
||||||
task["treeherder"]["symbol"] = full_symbol
|
|
||||||
|
|
||||||
yield task
|
|
||||||
|
|
||||||
|
|
||||||
@transforms.add
|
|
||||||
def remove_dependent_tasks(config, tasks):
|
|
||||||
for task in tasks:
|
|
||||||
try:
|
|
||||||
del task["dependent-tasks"]
|
|
||||||
except KeyError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
yield task
|
|
||||||
Loading…
Reference in New Issue
Block a user