diff --git a/taskcluster/fenix_taskgraph/loader/__init__.py b/taskcluster/fenix_taskgraph/loader/__init__.py deleted file mode 100644 index c07732f691..0000000000 --- a/taskcluster/fenix_taskgraph/loader/__init__.py +++ /dev/null @@ -1,104 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import copy - - -# Define a collection of group_by functions -GROUP_BY_MAP = {} - - -def group_by(name): - def wrapper(func): - GROUP_BY_MAP[name] = func - return func - - return wrapper - - -def group_tasks(config, tasks): - group_by_fn = GROUP_BY_MAP[config["group-by"]] - - groups = group_by_fn(config, tasks) - - for combinations in groups.values(): - dependencies = [copy.deepcopy(t) for t in combinations] - yield dependencies - - -@group_by("build-type") -def build_type_grouping(config, tasks): - groups = {} - kind_dependencies = config.get("kind-dependencies") - only_build_type = config.get("only-for-build-types") - - for task in tasks: - if task.kind not in kind_dependencies: - continue - - if only_build_type: - build_type = task.attributes.get("build-type") - if build_type not in only_build_type: - continue - - build_type = task.attributes.get("build-type") - - groups.setdefault(build_type, []).append(task) - - return groups - - -@group_by("attributes") -def attributes_grouping(config, tasks): - groups = {} - kind_dependencies = config.get("kind-dependencies") - only_attributes = config.get("only-for-attributes") - - for task in tasks: - if task.kind not in kind_dependencies: - continue - - group_attr = None - if only_attributes: - if not any(attr in task.attributes for attr in only_attributes): - continue - else: - continue - - groups.setdefault(task.label, []).append(task) - - return groups - - -@group_by("single-locale") -def single_locale_grouping(config, tasks): - """Split by a single locale (but also by platform, build-type, product) - - The locale can be `None` (en-US build/signing/repackage), a single locale, - or multiple locales per task, e.g. for l10n chunking. In the case of a task - with, say, five locales, the task will show up in all five locale groupings. - - This grouping is written for non-partner-repack beetmover, but might also - be useful elsewhere. - - """ - groups = {} - - for task in tasks: - if task.kind not in config.get("kind-dependencies", []): - continue - - platform = task.attributes.get("build_platform") - build_type = task.attributes.get("build_type") - task_locale = task.attributes.get("locale") - chunk_locales = task.attributes.get("chunk_locales") - locales = chunk_locales or [task_locale] - - for locale in locales: - locale_key = (platform, build_type, locale) - groups.setdefault(locale_key, []) - if task not in groups[locale_key]: - groups[locale_key].append(task) - - return groups diff --git a/taskcluster/fenix_taskgraph/loader/multi_dep.py b/taskcluster/fenix_taskgraph/loader/multi_dep.py deleted file mode 100644 index 3309f85bf8..0000000000 --- a/taskcluster/fenix_taskgraph/loader/multi_dep.py +++ /dev/null @@ -1,88 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import copy - -from voluptuous import Required - -from taskgraph.task import Task -from taskgraph.util.schema import Schema - -from . import group_tasks - - -schema = Schema( - { - Required("primary-dependency", "primary dependency task"): Task, - Required( - "dependent-tasks", - "dictionary of dependent tasks, keyed by kind", - ): {str: Task}, - } -) - - -def loader(kind, path, config, params, loaded_tasks): - """ - Load tasks based on the jobs dependant kinds, designed for use as - multiple-dependent needs. - Required ``group-by-fn`` is used to define how we coalesce the - multiple deps together to pass to transforms, e.g. all kinds specified get - collapsed by platform with `platform` - Optional ``primary-dependency`` (ordered list or string) is used to determine - which upstream kind to inherit attrs from. See ``get_primary_dep``. - The `only-for-build-type` kind configuration, if specified, will limit - the build types for which a job will be created. - Optional ``job-template`` kind configuration value, if specified, will be used to - pass configuration down to the specified transforms used. - """ - job_template = config.get("job-template") - - for dep_tasks in group_tasks(config, loaded_tasks): - kinds = [dep.kind for dep in dep_tasks] - kinds_occurrences = {kind: kinds.count(kind) for kind in kinds} - - dep_tasks_per_unique_key = { - dep.kind if kinds_occurrences[dep.kind] == 1 else dep.label: dep - for dep in dep_tasks - } - - job = {"dependent-tasks": dep_tasks_per_unique_key} - job["primary-dependency"] = get_primary_dep(config, dep_tasks_per_unique_key) - if job_template: - job.update(copy.deepcopy(job_template)) - - yield job - - -def get_primary_dep(config, dep_tasks): - """Find the dependent task to inherit attributes from. - If ``primary-dependency`` is defined in ``kind.yml`` and is a string, - then find the first dep with that task kind and return it. If it is - defined and is a list, the first kind in that list with a matching dep - is the primary dependency. If it's undefined, return the first dep. - """ - primary_dependencies = config.get("primary-dependency") - if isinstance(primary_dependencies, str): - primary_dependencies = [primary_dependencies] - if not primary_dependencies: - assert len(dep_tasks) == 1, "Must define a primary-dependency!" - return dep_tasks.values()[0] - primary_dep = None - for primary_kind in primary_dependencies: - for dep_kind in dep_tasks: - if dep_kind == primary_kind: - assert ( - primary_dep is None - ), "Too many primary dependent tasks in dep_tasks: {}!".format( - [t.label for t in dep_tasks] - ) - primary_dep = dep_tasks[dep_kind] - if primary_dep is None: - raise Exception( - "Can't find dependency of {}: {}".format( - config["primary-dependency"], config - ) - ) - return primary_dep diff --git a/taskcluster/fenix_taskgraph/transforms/multi_dep.py b/taskcluster/fenix_taskgraph/transforms/multi_dep.py deleted file mode 100644 index 7bd2828a87..0000000000 --- a/taskcluster/fenix_taskgraph/transforms/multi_dep.py +++ /dev/null @@ -1,118 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Apply some defaults and minor modifications to the single_dep jobs. -""" - -from taskgraph.transforms.base import TransformSequence -from taskgraph.util.schema import resolve_keyed_by -from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol - -from fenix_taskgraph.util.scriptworker import generate_beetmover_upstream_artifacts - - -transforms = TransformSequence() - - -@transforms.add -def build_name_and_attributes(config, tasks): - for task in tasks: - task["dependencies"] = { - dep_key: dep.label for dep_key, dep in _get_all_deps(task).items() - } - primary_dep = task["primary-dependency"] - attributes = primary_dep.attributes.copy() - attributes.update(task.get("attributes", {})) - task["attributes"] = attributes - # run_on_tasks_for is set as an attribute later in the pipeline - task.setdefault("run-on-tasks-for", attributes["run_on_tasks_for"]) - task["name"] = _get_dependent_job_name_without_its_kind(primary_dep) - - yield task - - -def _get_dependent_job_name_without_its_kind(dependent_job): - return dependent_job.label[len(dependent_job.kind) + 1 :] - - -def _get_all_deps(task): - if task.get("dependent-tasks"): - return task["dependent-tasks"] - - return {task["primary-dependency"].kind: task["primary-dependency"]} - - -@transforms.add -def resolve_keys(config, tasks): - for task in tasks: - resolve_keyed_by( - task, - "treeherder.job-symbol", - item_name=task["name"], - **{ - "build-type": task["attributes"]["build-type"], - "level": config.params["level"], - } - ) - yield task - - -@transforms.add -def build_upstream_artifacts(config, tasks): - for task in tasks: - worker_definition = { - "upstream-artifacts": [], - } - - if "artifact_map" in task["attributes"]: - # Beetmover tasks use declarative artifacts. - locale = task["attributes"].get("locale") - build_type = task["attributes"]["build-type"] - worker_definition[ - "upstream-artifacts" - ] = generate_beetmover_upstream_artifacts(config, task, build_type, locale) - else: - for dep in _get_all_deps(task).values(): - paths = sorted( - [ - apk_metadata["name"] - for apk_metadata in dep.attributes.get("apks", {}).values() - ] - ) - if paths: - worker_definition["upstream-artifacts"].append( - { - "taskId": {"task-reference": "<{}>".format(dep.kind)}, - "taskType": dep.kind, - "paths": paths, - } - ) - - task.setdefault("worker", {}).update(worker_definition) - yield task - - -@transforms.add -def build_treeherder_definition(config, tasks): - for task in tasks: - dep = task.pop("primary-dependency") - - task.setdefault("treeherder", {}).update(inherit_treeherder_from_dep(task, dep)) - job_group = dep.task["extra"]["treeherder"].get("groupSymbol", "?") - job_symbol = task["treeherder"].pop("job-symbol") - full_symbol = join_symbol(job_group, job_symbol) - task["treeherder"]["symbol"] = full_symbol - - yield task - - -@transforms.add -def remove_dependent_tasks(config, tasks): - for task in tasks: - try: - del task["dependent-tasks"] - except KeyError: - pass - - yield task