2
0
mirror of https://github.com/fork-maintainers/iceraven-browser synced 2024-11-05 21:20:45 +00:00

Bug 1808606 - part 3: Migrate beetmover tasks

(cherry picked from commit 64067aedd1054d6fb35f50c51591279d62ff922d)
This commit is contained in:
Geoff Brown 2023-01-18 16:03:34 -07:00 committed by mergify[bot]
parent 0203ae73f1
commit e42c8ba7a0
5 changed files with 0 additions and 521 deletions

View File

@ -1,37 +0,0 @@
# This source code form is subject to the terms of the mozilla public
# license, v. 2.0. if a copy of the mpl was not distributed with this
# file, you can obtain one at http://mozilla.org/mpl/2.0/.
---
loader: fenix_taskgraph.loader.multi_dep:loader
group-by: build-type
transforms:
- fenix_taskgraph.transforms.multi_dep:transforms
- fenix_taskgraph.transforms.beetmover:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- signing
primary-dependency: signing
only-for-build-types:
- release
- beta
- nightly
job-template:
attributes:
artifact_map: taskcluster/fenix_taskgraph/manifests/fenix_releases.yml
shipping_phase: ship
treeherder:
job-symbol: BM
bucket-scope:
by-level:
'3':
by-build-type:
'nightly': "project:mobile:fenix:releng:beetmover:bucket:nightly"
(release|beta): "project:mobile:fenix:releng:beetmover:bucket:release"
default: "project:mobile:fenix:releng:beetmover:bucket:dep"
default: "project:mobile:fenix:releng:beetmover:bucket:dep"

View File

@ -1,87 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
# This file contains exhaustive information about all the release artifacs that
# are needed within a type of release.
#
# Structure
# --------
# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets
# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of
# destinations. If given an empty locale, it uses these locales instead.
# `tasktype_map` -- mapping between task reference and task type, particularly usefule when
# composing the upstreamArtifacts for scriptworker.
# `platform_names` -- various platform mappings used in reckoning artifacts or other paths
# `default` -- a default entry, which the mappings extend and override in such a way that
# final path full-destinations will be a concatenation of the following:
# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name`
# `from` -- specifies the dependency(ies) from which to expect the particular artifact
# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected
# for all locales or just the default one
# `description` -- brief summary of what that artifact is
# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not
# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location
# for example `public/build` vs `public/build/ach/`.
# `destinations` -- final list of directories where to push the artifacts in S3
# `pretty_name` -- the final name the artifact will have at destination
# `checksums_path` -- the name to identify one artifact within the checksums file
# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform
# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform
# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present
# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest
# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest
s3_bucket_paths:
by-build-type:
nightly:
- pub/fenix/nightly
default:
- pub/fenix/releases
default_locales:
- multi
tasktype_map:
signing: signing
platform_names:
path_platform: android
tools_platform: android
filename_platform: android
default: &default
from:
- signing
all_locales: true
description: "TO_BE_OVERRIDDEN"
# Hard coded 'multi' locale
locale_prefix: '${locale}'
source_path_modifier:
by-locale:
default: '${locale}'
multi: ''
checksums_path: "TODO"
mapping:
arm64-v8a/target.apk:
<<: *default
description: "Android package for arm64-v8a"
pretty_name: fenix-${version}.${locale}.android-arm64-v8a.apk
destinations:
- ${folder_prefix}fenix-${version}-android-arm64-v8a
armeabi-v7a/target.apk:
<<: *default
description: "Android package for armeabi-v7a"
pretty_name: fenix-${version}.${locale}.android-armeabi-v7a.apk
destinations:
- ${folder_prefix}fenix-${version}-android-armeabi-v7a
x86/target.apk:
<<: *default
description: "Android package for x86"
pretty_name: fenix-${version}.${locale}.android-x86.apk
destinations:
- ${folder_prefix}fenix-${version}-android-x86
x86_64/target.apk:
<<: *default
description: "Android package for x86_64"
pretty_name: fenix-${version}.${locale}.android-x86_64.apk
destinations:
- ${folder_prefix}fenix-${version}-android-x86_64

View File

@ -1,111 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""
import logging
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import task_description_schema
from voluptuous import Optional, Required, Schema
from fenix_taskgraph.util.scriptworker import generate_beetmover_artifact_map
logger = logging.getLogger(__name__)
beetmover_description_schema = Schema(
{
# unique name to describe this beetmover task, defaults to {dep.label}-beetmover
Required("name"): str,
Required("worker"): {"upstream-artifacts": [dict]},
# treeherder is allowed here to override any defaults we use for beetmover.
Optional("treeherder"): task_description_schema["treeherder"],
Optional("attributes"): task_description_schema["attributes"],
Optional("dependencies"): task_description_schema["dependencies"],
Optional("run-on-tasks-for"): [str],
Optional("bucket-scope"): optionally_keyed_by("level", "build-type", str),
}
)
transforms = TransformSequence()
transforms.add_validate(beetmover_description_schema)
@transforms.add
def make_task_description(config, tasks):
for task in tasks:
attributes = task["attributes"]
label = "beetmover-{}".format(task["name"])
description = "Beetmover submission for build type '{build_type}'".format(
build_type=attributes.get("build-type"),
)
if task.get("locale"):
attributes["locale"] = task["locale"]
resolve_keyed_by(
task,
"bucket-scope",
item_name=task["name"],
**{
"build-type": task["attributes"]["build-type"],
"level": config.params["level"],
}
)
bucket_scope = task.pop("bucket-scope")
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"worker": task["worker"],
"scopes": [
bucket_scope,
"project:mobile:fenix:releng:beetmover:action:direct-push-to-bucket",
],
"dependencies": task["dependencies"],
"attributes": attributes,
"run-on-projects": attributes.get("run_on_projects"),
"run-on-tasks-for": attributes.get("run_on_tasks_for"),
"treeherder": task["treeherder"],
}
yield task
def craft_release_properties(config, task):
params = config.params
return {
"app-name": str(params["project"]),
"app-version": str(params["version"]),
"branch": str(params["project"]),
"build-id": str(params["moz_build_date"]),
"hash-type": "sha512",
"platform": "android",
}
@transforms.add
def make_task_worker(config, tasks):
for task in tasks:
locale = task["attributes"].get("locale")
build_type = task["attributes"]["build-type"]
task["worker"].update(
{
"implementation": "beetmover",
"release-properties": craft_release_properties(config, task),
"artifact-map": generate_beetmover_artifact_map(
config, task, platform=build_type, locale=locale
),
}
)
if locale:
task["worker"]["locale"] = locale
yield task

View File

@ -1,286 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import itertools
import os
from copy import deepcopy
from datetime import datetime
import jsone
from ..release_promotion import read_version_file
from taskgraph.util.memoize import memoize
from taskgraph.util.schema import resolve_keyed_by
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.util.yaml import load_yaml
cached_load_yaml = memoize(load_yaml)
def generate_beetmover_upstream_artifacts(
config, job, platform, locale=None, dependencies=None, **kwargs
):
"""Generate the upstream artifacts for beetmover, using the artifact map.
Currently only applies to beetmover tasks.
Args:
job (dict): The current job being generated
dependencies (list): A list of the job's dependency labels.
platform (str): The current build platform
locale (str): The current locale being beetmoved.
Returns:
list: A list of dictionaries conforming to the upstream_artifacts spec.
"""
base_artifact_prefix = get_artifact_prefix(job)
resolve_keyed_by(
job,
"attributes.artifact_map",
"artifact map",
**{
"release-type": config.params["release_type"],
"platform": platform,
},
)
map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"]))
upstream_artifacts = list()
if not locale:
locales = map_config["default_locales"]
elif isinstance(locale, list):
locales = locale
else:
locales = [locale]
if not dependencies:
if job.get("dependencies"):
dependencies = job["dependencies"].keys()
elif job.get("primary-dependency"):
dependencies = [job["primary-dependency"].kind]
else:
raise Exception("Unsupported type of dependency. Got job: {}".format(job))
for locale, dep in itertools.product(locales, dependencies):
paths = list()
for filename in map_config["mapping"]:
if dep not in map_config["mapping"][filename]["from"]:
continue
if locale != "multi" and not map_config["mapping"][filename]["all_locales"]:
continue
if (
"only_for_platforms" in map_config["mapping"][filename]
and platform
not in map_config["mapping"][filename]["only_for_platforms"]
):
continue
if (
"not_for_platforms" in map_config["mapping"][filename]
and platform in map_config["mapping"][filename]["not_for_platforms"]
):
continue
if "partials_only" in map_config["mapping"][filename]:
continue
# The next time we look at this file it might be a different locale.
file_config = deepcopy(map_config["mapping"][filename])
resolve_keyed_by(
file_config,
"source_path_modifier",
"source path modifier",
locale=locale,
)
kwargs["locale"] = locale
paths.append(
os.path.join(
base_artifact_prefix,
jsone.render(file_config["source_path_modifier"], kwargs),
jsone.render(filename, kwargs),
)
)
if job.get("dependencies") and getattr(
job["dependencies"][dep], "release_artifacts", None
):
paths = [
path
for path in paths
if path in job["dependencies"][dep].release_artifacts
]
if not paths:
continue
upstream_artifacts.append(
{
"taskId": {"task-reference": "<{}>".format(dep)},
"taskType": map_config["tasktype_map"].get(dep),
"paths": sorted(paths),
"locale": locale,
}
)
upstream_artifacts.sort(key=lambda u: u["paths"])
return upstream_artifacts
def generate_beetmover_artifact_map(config, job, **kwargs):
"""Generate the beetmover artifact map.
Currently only applies to beetmover tasks.
Args:
config (): Current taskgraph configuration.
job (dict): The current job being generated
Common kwargs:
platform (str): The current build platform
locale (str): The current locale being beetmoved.
Returns:
list: A list of dictionaries containing source->destination
maps for beetmover.
"""
platform = kwargs.get("platform", "")
resolve_keyed_by(
job,
"attributes.artifact_map",
job["label"],
**{
"release-type": config.params["release_type"],
"platform": platform,
},
)
map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"]))
base_artifact_prefix = map_config.get(
"base_artifact_prefix", get_artifact_prefix(job)
)
artifacts = list()
dependencies = job["dependencies"].keys()
if kwargs.get("locale"):
if isinstance(kwargs["locale"], list):
locales = kwargs["locale"]
else:
locales = [kwargs["locale"]]
else:
locales = map_config["default_locales"]
resolve_keyed_by(
map_config,
"s3_bucket_paths",
job["label"],
**{"build-type": job["attributes"]["build-type"]},
)
for locale, dep in sorted(itertools.product(locales, dependencies)):
paths = dict()
for filename in map_config["mapping"]:
# Relevancy checks
if dep not in map_config["mapping"][filename]["from"]:
# We don't get this file from this dependency.
continue
if locale != "multi" and not map_config["mapping"][filename]["all_locales"]:
# This locale either doesn't produce or shouldn't upload this file.
continue
if (
"only_for_platforms" in map_config["mapping"][filename]
and platform
not in map_config["mapping"][filename]["only_for_platforms"]
):
# This platform either doesn't produce or shouldn't upload this file.
continue
if (
"not_for_platforms" in map_config["mapping"][filename]
and platform in map_config["mapping"][filename]["not_for_platforms"]
):
# This platform either doesn't produce or shouldn't upload this file.
continue
if "partials_only" in map_config["mapping"][filename]:
continue
# deepcopy because the next time we look at this file the locale will differ.
file_config = deepcopy(map_config["mapping"][filename])
for field in [
"destinations",
"locale_prefix",
"source_path_modifier",
"update_balrog_manifest",
"pretty_name",
"checksums_path",
]:
resolve_keyed_by(file_config, field, job["label"], locale=locale)
# This format string should ideally be in the configuration file,
# but this would mean keeping variable names in sync between code + config.
destinations = [
"{s3_bucket_path}/{dest_path}/{filename}".format(
s3_bucket_path=bucket_path,
dest_path=dest_path,
locale_prefix=file_config["locale_prefix"],
filename=file_config.get("pretty_name", filename),
)
for dest_path, bucket_path in itertools.product(
file_config["destinations"], map_config["s3_bucket_paths"]
)
]
# Creating map entries
# Key must be artifact path, to avoid trampling duplicates, such
# as public/build/target.apk and public/build/multi/target.apk
key = os.path.join(
base_artifact_prefix,
file_config["source_path_modifier"],
filename,
)
paths[key] = {
"destinations": destinations,
}
if file_config.get("checksums_path"):
paths[key]["checksums_path"] = file_config["checksums_path"]
# optional flag: balrog manifest
if file_config.get("update_balrog_manifest"):
paths[key]["update_balrog_manifest"] = True
if file_config.get("balrog_format"):
paths[key]["balrog_format"] = file_config["balrog_format"]
if not paths:
# No files for this dependency/locale combination.
continue
# Render all variables for the artifact map
platforms = deepcopy(map_config.get("platform_names", {}))
if platform:
for key in platforms.keys():
resolve_keyed_by(platforms, key, job["label"], platform=platform)
version = read_version_file()
upload_date = datetime.fromtimestamp(config.params["build_date"])
if job["attributes"]["build-type"] == "nightly":
folder_prefix = upload_date.strftime("%Y/%m/%Y-%m-%d-%H-%M-%S-")
# TODO: Remove this when version.txt has versioning fixed
version = version.split("-")[0]
else:
folder_prefix = f"{version}/android/"
kwargs.update(
{"locale": locale, "version": version, "folder_prefix": folder_prefix}
)
kwargs.update(**platforms)
paths = jsone.render(paths, kwargs)
artifacts.append(
{
"taskId": {"task-reference": "<{}>".format(dep)},
"locale": locale,
"paths": paths,
}
)
return artifacts