From e42c8ba7a079e0f8b2d8ecb64d40e869b491712d Mon Sep 17 00:00:00 2001 From: Geoff Brown Date: Wed, 18 Jan 2023 16:03:34 -0700 Subject: [PATCH] Bug 1808606 - part 3: Migrate beetmover tasks (cherry picked from commit 64067aedd1054d6fb35f50c51591279d62ff922d) --- taskcluster/ci/beetmover/kind.yml | 37 --- .../manifests/fenix_releases.yml | 87 ------ .../fenix_taskgraph/transforms/beetmover.py | 111 ------- taskcluster/fenix_taskgraph/util/__init__.py | 0 .../fenix_taskgraph/util/scriptworker.py | 286 ------------------ 5 files changed, 521 deletions(-) delete mode 100644 taskcluster/ci/beetmover/kind.yml delete mode 100644 taskcluster/fenix_taskgraph/manifests/fenix_releases.yml delete mode 100644 taskcluster/fenix_taskgraph/transforms/beetmover.py delete mode 100644 taskcluster/fenix_taskgraph/util/__init__.py delete mode 100644 taskcluster/fenix_taskgraph/util/scriptworker.py diff --git a/taskcluster/ci/beetmover/kind.yml b/taskcluster/ci/beetmover/kind.yml deleted file mode 100644 index bb7103b3e0..0000000000 --- a/taskcluster/ci/beetmover/kind.yml +++ /dev/null @@ -1,37 +0,0 @@ -# This source code form is subject to the terms of the mozilla public -# license, v. 2.0. if a copy of the mpl was not distributed with this -# file, you can obtain one at http://mozilla.org/mpl/2.0/. ---- -loader: fenix_taskgraph.loader.multi_dep:loader - -group-by: build-type - -transforms: - - fenix_taskgraph.transforms.multi_dep:transforms - - fenix_taskgraph.transforms.beetmover:transforms - - taskgraph.transforms.task:transforms - -kind-dependencies: - - signing - -primary-dependency: signing - -only-for-build-types: - - release - - beta - - nightly - -job-template: - attributes: - artifact_map: taskcluster/fenix_taskgraph/manifests/fenix_releases.yml - shipping_phase: ship - treeherder: - job-symbol: BM - bucket-scope: - by-level: - '3': - by-build-type: - 'nightly': "project:mobile:fenix:releng:beetmover:bucket:nightly" - (release|beta): "project:mobile:fenix:releng:beetmover:bucket:release" - default: "project:mobile:fenix:releng:beetmover:bucket:dep" - default: "project:mobile:fenix:releng:beetmover:bucket:dep" diff --git a/taskcluster/fenix_taskgraph/manifests/fenix_releases.yml b/taskcluster/fenix_taskgraph/manifests/fenix_releases.yml deleted file mode 100644 index 9a2719e7cb..0000000000 --- a/taskcluster/fenix_taskgraph/manifests/fenix_releases.yml +++ /dev/null @@ -1,87 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. ---- -# This file contains exhaustive information about all the release artifacs that -# are needed within a type of release. -# -# Structure -# -------- -# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets -# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of -# destinations. If given an empty locale, it uses these locales instead. -# `tasktype_map` -- mapping between task reference and task type, particularly usefule when -# composing the upstreamArtifacts for scriptworker. -# `platform_names` -- various platform mappings used in reckoning artifacts or other paths -# `default` -- a default entry, which the mappings extend and override in such a way that -# final path full-destinations will be a concatenation of the following: -# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` -# `from` -- specifies the dependency(ies) from which to expect the particular artifact -# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected -# for all locales or just the default one -# `description` -- brief summary of what that artifact is -# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not -# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location -# for example `public/build` vs `public/build/ach/`. -# `destinations` -- final list of directories where to push the artifacts in S3 -# `pretty_name` -- the final name the artifact will have at destination -# `checksums_path` -- the name to identify one artifact within the checksums file -# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform -# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform -# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present -# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest -# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest - -s3_bucket_paths: - by-build-type: - nightly: - - pub/fenix/nightly - default: - - pub/fenix/releases -default_locales: - - multi -tasktype_map: - signing: signing -platform_names: - path_platform: android - tools_platform: android - filename_platform: android - -default: &default - from: - - signing - all_locales: true - description: "TO_BE_OVERRIDDEN" - # Hard coded 'multi' locale - locale_prefix: '${locale}' - source_path_modifier: - by-locale: - default: '${locale}' - multi: '' - checksums_path: "TODO" - -mapping: - arm64-v8a/target.apk: - <<: *default - description: "Android package for arm64-v8a" - pretty_name: fenix-${version}.${locale}.android-arm64-v8a.apk - destinations: - - ${folder_prefix}fenix-${version}-android-arm64-v8a - armeabi-v7a/target.apk: - <<: *default - description: "Android package for armeabi-v7a" - pretty_name: fenix-${version}.${locale}.android-armeabi-v7a.apk - destinations: - - ${folder_prefix}fenix-${version}-android-armeabi-v7a - x86/target.apk: - <<: *default - description: "Android package for x86" - pretty_name: fenix-${version}.${locale}.android-x86.apk - destinations: - - ${folder_prefix}fenix-${version}-android-x86 - x86_64/target.apk: - <<: *default - description: "Android package for x86_64" - pretty_name: fenix-${version}.${locale}.android-x86_64.apk - destinations: - - ${folder_prefix}fenix-${version}-android-x86_64 diff --git a/taskcluster/fenix_taskgraph/transforms/beetmover.py b/taskcluster/fenix_taskgraph/transforms/beetmover.py deleted file mode 100644 index 8eaaf91fcd..0000000000 --- a/taskcluster/fenix_taskgraph/transforms/beetmover.py +++ /dev/null @@ -1,111 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Transform the beetmover task into an actual task description. -""" - -import logging - -from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by -from taskgraph.transforms.base import TransformSequence -from taskgraph.transforms.task import task_description_schema -from voluptuous import Optional, Required, Schema - -from fenix_taskgraph.util.scriptworker import generate_beetmover_artifact_map - -logger = logging.getLogger(__name__) - -beetmover_description_schema = Schema( - { - # unique name to describe this beetmover task, defaults to {dep.label}-beetmover - Required("name"): str, - Required("worker"): {"upstream-artifacts": [dict]}, - # treeherder is allowed here to override any defaults we use for beetmover. - Optional("treeherder"): task_description_schema["treeherder"], - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("run-on-tasks-for"): [str], - Optional("bucket-scope"): optionally_keyed_by("level", "build-type", str), - } -) - -transforms = TransformSequence() -transforms.add_validate(beetmover_description_schema) - - -@transforms.add -def make_task_description(config, tasks): - for task in tasks: - attributes = task["attributes"] - - label = "beetmover-{}".format(task["name"]) - description = "Beetmover submission for build type '{build_type}'".format( - build_type=attributes.get("build-type"), - ) - - if task.get("locale"): - attributes["locale"] = task["locale"] - - resolve_keyed_by( - task, - "bucket-scope", - item_name=task["name"], - **{ - "build-type": task["attributes"]["build-type"], - "level": config.params["level"], - } - ) - bucket_scope = task.pop("bucket-scope") - - task = { - "label": label, - "description": description, - "worker-type": "beetmover", - "worker": task["worker"], - "scopes": [ - bucket_scope, - "project:mobile:fenix:releng:beetmover:action:direct-push-to-bucket", - ], - "dependencies": task["dependencies"], - "attributes": attributes, - "run-on-projects": attributes.get("run_on_projects"), - "run-on-tasks-for": attributes.get("run_on_tasks_for"), - "treeherder": task["treeherder"], - } - - yield task - - -def craft_release_properties(config, task): - params = config.params - return { - "app-name": str(params["project"]), - "app-version": str(params["version"]), - "branch": str(params["project"]), - "build-id": str(params["moz_build_date"]), - "hash-type": "sha512", - "platform": "android", - } - - -@transforms.add -def make_task_worker(config, tasks): - for task in tasks: - locale = task["attributes"].get("locale") - build_type = task["attributes"]["build-type"] - - task["worker"].update( - { - "implementation": "beetmover", - "release-properties": craft_release_properties(config, task), - "artifact-map": generate_beetmover_artifact_map( - config, task, platform=build_type, locale=locale - ), - } - ) - - if locale: - task["worker"]["locale"] = locale - - yield task diff --git a/taskcluster/fenix_taskgraph/util/__init__.py b/taskcluster/fenix_taskgraph/util/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/taskcluster/fenix_taskgraph/util/scriptworker.py b/taskcluster/fenix_taskgraph/util/scriptworker.py deleted file mode 100644 index 95aa99aab6..0000000000 --- a/taskcluster/fenix_taskgraph/util/scriptworker.py +++ /dev/null @@ -1,286 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import itertools -import os -from copy import deepcopy -from datetime import datetime - -import jsone - -from ..release_promotion import read_version_file -from taskgraph.util.memoize import memoize -from taskgraph.util.schema import resolve_keyed_by -from taskgraph.util.taskcluster import get_artifact_prefix -from taskgraph.util.yaml import load_yaml - -cached_load_yaml = memoize(load_yaml) - - -def generate_beetmover_upstream_artifacts( - config, job, platform, locale=None, dependencies=None, **kwargs -): - """Generate the upstream artifacts for beetmover, using the artifact map. - - Currently only applies to beetmover tasks. - - Args: - job (dict): The current job being generated - dependencies (list): A list of the job's dependency labels. - platform (str): The current build platform - locale (str): The current locale being beetmoved. - - Returns: - list: A list of dictionaries conforming to the upstream_artifacts spec. - """ - base_artifact_prefix = get_artifact_prefix(job) - resolve_keyed_by( - job, - "attributes.artifact_map", - "artifact map", - **{ - "release-type": config.params["release_type"], - "platform": platform, - }, - ) - map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) - upstream_artifacts = list() - - if not locale: - locales = map_config["default_locales"] - elif isinstance(locale, list): - locales = locale - else: - locales = [locale] - - if not dependencies: - if job.get("dependencies"): - dependencies = job["dependencies"].keys() - elif job.get("primary-dependency"): - dependencies = [job["primary-dependency"].kind] - else: - raise Exception("Unsupported type of dependency. Got job: {}".format(job)) - - for locale, dep in itertools.product(locales, dependencies): - paths = list() - - for filename in map_config["mapping"]: - if dep not in map_config["mapping"][filename]["from"]: - continue - if locale != "multi" and not map_config["mapping"][filename]["all_locales"]: - continue - if ( - "only_for_platforms" in map_config["mapping"][filename] - and platform - not in map_config["mapping"][filename]["only_for_platforms"] - ): - continue - if ( - "not_for_platforms" in map_config["mapping"][filename] - and platform in map_config["mapping"][filename]["not_for_platforms"] - ): - continue - if "partials_only" in map_config["mapping"][filename]: - continue - # The next time we look at this file it might be a different locale. - file_config = deepcopy(map_config["mapping"][filename]) - resolve_keyed_by( - file_config, - "source_path_modifier", - "source path modifier", - locale=locale, - ) - - kwargs["locale"] = locale - - paths.append( - os.path.join( - base_artifact_prefix, - jsone.render(file_config["source_path_modifier"], kwargs), - jsone.render(filename, kwargs), - ) - ) - - if job.get("dependencies") and getattr( - job["dependencies"][dep], "release_artifacts", None - ): - paths = [ - path - for path in paths - if path in job["dependencies"][dep].release_artifacts - ] - - if not paths: - continue - - upstream_artifacts.append( - { - "taskId": {"task-reference": "<{}>".format(dep)}, - "taskType": map_config["tasktype_map"].get(dep), - "paths": sorted(paths), - "locale": locale, - } - ) - - upstream_artifacts.sort(key=lambda u: u["paths"]) - return upstream_artifacts - - -def generate_beetmover_artifact_map(config, job, **kwargs): - """Generate the beetmover artifact map. - - Currently only applies to beetmover tasks. - - Args: - config (): Current taskgraph configuration. - job (dict): The current job being generated - Common kwargs: - platform (str): The current build platform - locale (str): The current locale being beetmoved. - - Returns: - list: A list of dictionaries containing source->destination - maps for beetmover. - """ - platform = kwargs.get("platform", "") - resolve_keyed_by( - job, - "attributes.artifact_map", - job["label"], - **{ - "release-type": config.params["release_type"], - "platform": platform, - }, - ) - map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) - base_artifact_prefix = map_config.get( - "base_artifact_prefix", get_artifact_prefix(job) - ) - - artifacts = list() - - dependencies = job["dependencies"].keys() - - if kwargs.get("locale"): - if isinstance(kwargs["locale"], list): - locales = kwargs["locale"] - else: - locales = [kwargs["locale"]] - else: - locales = map_config["default_locales"] - - resolve_keyed_by( - map_config, - "s3_bucket_paths", - job["label"], - **{"build-type": job["attributes"]["build-type"]}, - ) - - for locale, dep in sorted(itertools.product(locales, dependencies)): - paths = dict() - for filename in map_config["mapping"]: - # Relevancy checks - if dep not in map_config["mapping"][filename]["from"]: - # We don't get this file from this dependency. - continue - if locale != "multi" and not map_config["mapping"][filename]["all_locales"]: - # This locale either doesn't produce or shouldn't upload this file. - continue - if ( - "only_for_platforms" in map_config["mapping"][filename] - and platform - not in map_config["mapping"][filename]["only_for_platforms"] - ): - # This platform either doesn't produce or shouldn't upload this file. - continue - if ( - "not_for_platforms" in map_config["mapping"][filename] - and platform in map_config["mapping"][filename]["not_for_platforms"] - ): - # This platform either doesn't produce or shouldn't upload this file. - continue - if "partials_only" in map_config["mapping"][filename]: - continue - - # deepcopy because the next time we look at this file the locale will differ. - file_config = deepcopy(map_config["mapping"][filename]) - - for field in [ - "destinations", - "locale_prefix", - "source_path_modifier", - "update_balrog_manifest", - "pretty_name", - "checksums_path", - ]: - resolve_keyed_by(file_config, field, job["label"], locale=locale) - - # This format string should ideally be in the configuration file, - # but this would mean keeping variable names in sync between code + config. - destinations = [ - "{s3_bucket_path}/{dest_path}/{filename}".format( - s3_bucket_path=bucket_path, - dest_path=dest_path, - locale_prefix=file_config["locale_prefix"], - filename=file_config.get("pretty_name", filename), - ) - for dest_path, bucket_path in itertools.product( - file_config["destinations"], map_config["s3_bucket_paths"] - ) - ] - # Creating map entries - # Key must be artifact path, to avoid trampling duplicates, such - # as public/build/target.apk and public/build/multi/target.apk - key = os.path.join( - base_artifact_prefix, - file_config["source_path_modifier"], - filename, - ) - - paths[key] = { - "destinations": destinations, - } - if file_config.get("checksums_path"): - paths[key]["checksums_path"] = file_config["checksums_path"] - - # optional flag: balrog manifest - if file_config.get("update_balrog_manifest"): - paths[key]["update_balrog_manifest"] = True - if file_config.get("balrog_format"): - paths[key]["balrog_format"] = file_config["balrog_format"] - - if not paths: - # No files for this dependency/locale combination. - continue - - # Render all variables for the artifact map - platforms = deepcopy(map_config.get("platform_names", {})) - if platform: - for key in platforms.keys(): - resolve_keyed_by(platforms, key, job["label"], platform=platform) - - version = read_version_file() - upload_date = datetime.fromtimestamp(config.params["build_date"]) - - if job["attributes"]["build-type"] == "nightly": - folder_prefix = upload_date.strftime("%Y/%m/%Y-%m-%d-%H-%M-%S-") - # TODO: Remove this when version.txt has versioning fixed - version = version.split("-")[0] - else: - folder_prefix = f"{version}/android/" - - kwargs.update( - {"locale": locale, "version": version, "folder_prefix": folder_prefix} - ) - kwargs.update(**platforms) - paths = jsone.render(paths, kwargs) - artifacts.append( - { - "taskId": {"task-reference": "<{}>".format(dep)}, - "locale": locale, - "paths": paths, - } - ) - - return artifacts