[fenix] Bug 1580778 - Migrate signing tasks to taskgraph (https://github.com/mozilla-mobile/fenix/pull/5409)

automation/taskcluster/decision_task.py

@@ -59,25 +59,17 @@ def raptor(builder, is_staging):
 
     variant = get_variant('forPerformanceTest', 'geckoNightly')
     build_task = builder.craft_assemble_raptor_task(variant)
-    signing_task = builder.craft_raptor_signing_task(build_task['label'], variant, is_staging)
 
-    # Raptor tests are generated in taskgraph
-    return [build_task, signing_task]
+    # Signing and Raptor tasks are generated in taskgraph
+    return [build_task]
 
 
 def release(builder, channel, engine, is_staging, version_name):
     variant = get_variant('fenix' + channel.capitalize(), engine)
     build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
 
-    signing_task = builder.craft_release_signing_task(
-        build_task['label'],
-        variant,
-        channel=channel,
-        is_staging=is_staging,
-    )
-
-    # The push-apk task is generated by taskgraph
-    return [build_task, signing_task]
+    # The signing push-apk tasks are generated by taskgraph
+    return [build_task]
 
 
 def release_as_fennec(builder, is_staging, version_name):
@@ -85,15 +77,9 @@ def release_as_fennec(builder, is_staging, version_name):
     channel = 'fennec-production'
 
     build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
-    signing_task = builder.craft_release_signing_task(
-        build_task['label'],
-        variant,
-        channel,
-        variant,
-        is_staging,
-    )
 
-    return [build_task, signing_task]
+    # The signing task is generated by taskgraph
+    return [build_task]
 
 
 def nightly_to_production_app(builder, is_staging, version_name):
@@ -106,16 +92,8 @@ def nightly_to_production_app(builder, is_staging, version_name):
     build_task = builder.craft_assemble_release_task(
         variant, 'nightly-legacy', is_staging, version_name)
 
-    signing_task = builder.craft_release_signing_task(
-        build_task['label'],
-        variant,
-        channel='production',  # Since we're publishing to the "production" app, we need to sign for production
-        is_staging=is_staging,
-        publish_to_index=False,
-    )
-
-    # The push-apk task is generated by taskgraph
-    tasks = [build_task, signing_task]
+    # The signing and push-apk tasks are generated by taskgraph
+    tasks = [build_task]
     if not is_staging:
         tasks.append(builder.craft_upload_apk_nimbledroid_task(build_task['label']))
 

automation/taskcluster/lib/tasks.py

@@ -97,9 +97,15 @@ class TaskBuilder(object):
                 'machine': {
                     'platform': 'android-all',
                 },
+                'collection': {
+                    'opt': True,
+                },
                 'symbol': '{}-A'.format(variant.build_type),
                 'tier': 1,
             },
+            attributes={
+                'apks': variant.upstream_artifacts_per_abi,
+            }
         )
 
     def craft_assemble_raptor_task(self, variant):
@@ -125,7 +131,10 @@ class TaskBuilder(object):
                 'symbol': 'A',
                 'tier': 1,
             },
-            attributes={'build-type': 'raptor'},
+            attributes={
+                'build-type': 'raptor',
+                'apks': variant.upstream_artifacts_per_abi,
+            },
         )
 
     def craft_assemble_pr_task(self, variant):
@@ -142,7 +151,7 @@ class TaskBuilder(object):
                 },
                 'symbol': 'A',
                 'tier': 1,
-            }
+            },
         )
 
     def craft_test_pr_task(self, variant):
@@ -367,33 +376,6 @@ class TaskBuilder(object):
             attributes=attributes,
         )
 
-    def _craft_signing_task(self, name, description, signing_type, assemble_task_label, apk_paths, routes, treeherder, attributes=None):
-        signing_format = "autograph_apk"
-        payload = {
-            'upstreamArtifacts': [{
-                'paths': apk_paths,
-                'formats': [signing_format],
-                'taskId': {'task-reference': '<build>'},
-                'taskType': 'build'
-            }]
-        }
-
-        return self._craft_default_task_definition(
-            worker_type='mobile-signing-dep-v1' if signing_type == 'dep' else 'mobile-signing-v1',
-            provisioner_id='scriptworker-prov-v1',
-            dependencies={'build': assemble_task_label},
-            routes=routes,
-            scopes=[
-                "project:mobile:fenix:releng:signing:format:{}".format(signing_format),
-                "project:mobile:fenix:releng:signing:cert:{}-signing".format(signing_type),
-            ],
-            name=name,
-            description=description,
-            payload=payload,
-            treeherder=treeherder,
-            attributes=attributes,
-        )
-
     def _craft_default_task_definition(
         self,
         worker_type,
@@ -457,86 +439,6 @@ class TaskBuilder(object):
             },
         }
 
-    def craft_raptor_signing_task(
-        self, assemble_task_label, variant, is_staging,
-    ):
-        staging_prefix = '.staging' if is_staging else ''
-        routes = [
-            "index.project.mobile.fenix.v2{}.performance-test.{}.{}.{}.latest".format(
-                staging_prefix, self.date.year, self.date.month, self.date.day
-            ),
-            "index.project.mobile.fenix.v2{}.performance-test.{}.{}.{}.revision.{}".format(
-                staging_prefix, self.date.year, self.date.month, self.date.day, self.commit
-            ),
-            "index.project.mobile.fenix.v2{}.performance-test.latest".format(staging_prefix),
-        ]
-
-        return self._craft_signing_task(
-            name='sign: {}'.format('forPerformanceTest'),
-            description='Dep-signing variant {}'.format('forPerformanceTest'),
-            signing_type="dep",
-            assemble_task_label=assemble_task_label,
-            apk_paths=variant.upstream_artifacts(),
-            routes=routes,
-            treeherder={
-                'groupSymbol': 'forPerformanceTest',
-                'jobKind': 'other',
-                'machine': {
-                    'platform': 'android-all',
-                },
-                'collection': {
-                    'opt': True,
-                },
-                'symbol': 'As',
-                'tier': 1,
-            },
-            attributes={
-                'build-type': 'raptor',
-                'apks': variant.upstream_artifacts_per_abi,
-            },
-        )
-
-    def craft_release_signing_task(
-        self, build_task_id, variant, channel, is_staging, publish_to_index=True
-    ):
-        if publish_to_index:
-            staging_prefix = '.staging' if is_staging else ''
-            routes = [
-                "index.project.mobile.fenix.v2{}.{}.{}.{}.{}.latest".format(
-                    staging_prefix, channel, self.date.year, self.date.month, self.date.day
-                ),
-                "index.project.mobile.fenix.v2{}.{}.{}.{}.{}.revision.{}".format(
-                    staging_prefix, channel, self.date.year, self.date.month, self.date.day, self.commit
-                ),
-                "index.project.mobile.fenix.v2{}.{}.latest".format(staging_prefix, channel),
-            ]
-        else:
-            routes = []
-
-        capitalized_channel = upper_case_first_letter(channel)
-        return self._craft_signing_task(
-            name="Signing {} task".format(capitalized_channel),
-            description="Sign {} builds of Fenix".format(capitalized_channel),
-            signing_type="dep" if is_staging else channel,
-            assemble_task_label=build_task_id,
-            apk_paths=variant.upstream_artifacts(),
-            routes=routes,
-            treeherder={
-                'jobKind': 'other',
-                'machine': {
-                  'platform': 'android-all',
-                },
-                'collection': {
-                    'opt': True,
-                },
-                'symbol': '{}-s'.format(channel),
-                'tier': 1,
-            },
-            attributes={
-                'apks': variant.upstream_artifacts_per_abi,
-            },
-        )
-
 
 def schedule_task(queue, taskId, task):
     print("TASK", taskId)

taskcluster/ci/config.yml

@@ -2,7 +2,11 @@
 trust-domain: mobile
 treeherder:
     group-names:
+        'forPerformanceTest': 'Builds made for Raptor and other performance tests'
         'I': 'Docker Image Builds'
+        'nightly': 'Nightly-related tasks'
+        'nightlyLegacy': 'Nightly-related tasks that ship to https://play.google.com/store/apps/details?id=org.mozilla.fenix'
+        'production': 'Release-related tasks'
         'Rap': 'Raptor tests'
         'Rap-P': 'Raptor power tests'
 

taskcluster/ci/push-apk/kind.yml

@@ -2,55 +2,25 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
-loader: taskgraph.loader.transform:loader
+loader: fenix_taskgraph.loader.single_dep:loader
 
 transforms:
     - fenix_taskgraph.pushapk:transforms
     - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-    - old-decision
+    - signing
 
-job-defaults:
+only-for-build-types:
+    - nightly
+    - nightly-legacy
+    - production
+
+job-template:
     description: Publish Fenix
     worker-type: push-apk
     worker:
         commit: true
         product: fenix
-    run-on-tasks-for: []
     treeherder:
         kind: build
-        tier: 1
-
-jobs:
-    release:
-        dependencies:
-            signing: 'Signing Production task'  # comes from the old-decision task
-        worker:
-            channel: production
-            certificate-alias: fenix
-        run-on-tasks-for: [github-release]
-        treeherder:
-            symbol: release-gp
-
-    nightly:
-        attributes:
-            nightly: true
-        dependencies:
-            signing: 'Signing Nightly task' # comes from the old-decision task
-        worker:
-            channel: nightly
-            google-play-track: internal
-        treeherder:
-            symbol: nightly-gp
-
-    nightly-legacy:
-        attributes:
-            nightly: true
-        dependencies:
-            signing: 'Signing Production task'  # comes from the old-decision task
-        worker:
-            channel: production
-            google-play-track: nightly
-        treeherder:
-            symbol: production-gp

taskcluster/ci/raptor/kind.yml

@@ -6,7 +6,7 @@ transforms:
     - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-    - old-decision
+    - signing
 only-for-build-types:
     - raptor
 only-for-abis:
@@ -28,7 +28,6 @@ job-defaults:
                 armeabi-v7a: android-hw-g5-7-0-arm7-api-16/opt
     dependencies:
         geckoview-nightly: geckoview-nightly
-        signing: 'sign: forPerformanceTest'     # comes from the old-decision task
     worker:
         max-run-time: 3600
         env:

taskcluster/ci/signing/kind.yml

@@ -0,0 +1,74 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - fenix_taskgraph.signing:transforms
+    - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+    - old-decision
+
+job-defaults:
+    description: Sign Fenix
+    index:
+        type: signing
+    worker-type:
+        by-variant:
+            (nightly|nightly-legacy|production):
+                by-level:
+                    '3': signing
+                    default: dep-signing
+            default: dep-signing
+    worker:
+        signing-type:
+            by-variant:
+                (nightly|nightly-legacy|production):
+                    by-level:
+                        '3': release-signing
+                        default: dep-signing
+                default: dep-signing
+    run-on-tasks-for: []
+    treeherder:
+        kind: build
+        tier: 2
+        platform: android-all/opt
+
+jobs:
+    performance-test:
+        attributes:
+            build-type: performance-test
+        worker-type: dep-signing
+        dependencies:
+            build: 'assemble: geckoNightlyForPerformanceTest'   # comes from the old-decision task
+        treeherder:
+            symbol: forPerformanceTest(s)
+
+    production:
+        attributes:
+            build-type: production
+        dependencies:
+            build: 'Build FenixProduction task'  # comes from the old-decision task
+        run-on-tasks-for: [github-release]
+        treeherder:
+            symbol: production(s)
+
+    nightly:
+        attributes:
+            build-type: nightly
+            nightly: true
+        dependencies:
+            build: 'Build FenixNightly task' # comes from the old-decision task
+        treeherder:
+            symbol: nightly(s)
+
+    nightly-legacy:
+        attributes:
+            build-type: nightly-legacy
+            nightly: true
+        dependencies:
+            build: 'Build FenixNightlyLegacy task'  # comes from the old-decision task
+        treeherder:
+            symbol: nightly-legacy(s)

taskcluster/fenix_taskgraph/loader/single_dep.py

@@ -0,0 +1,52 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+
+from voluptuous import Required
+
+from taskgraph.task import Task
+from taskgraph.util.schema import Schema
+
+schema = Schema({Required("primary-dependency", "primary dependency task"): Task})
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    """
+    Load tasks based on the jobs dependant kinds.
+
+    Optional `only-for-attributes` kind configuration, if specified, will limit
+    the jobs chosen to ones which have the specified attribute, with the specified
+    value.
+
+    Optional `job-template` kind configuration value, if specified, will be used to
+    pass configuration down to the specified transforms used.
+    """
+    only_attributes = config.get("only-for-attributes")
+    only_build_type = config.get('only-for-build-types')
+    job_template = config.get("job-template")
+
+    for task in loaded_tasks:
+        if task.kind not in config.get("kind-dependencies", []):
+            continue
+
+        if only_attributes:
+            config_attrs = set(only_attributes)
+            if not config_attrs & set(task.attributes):
+                # make sure any attribute exists
+                continue
+
+        if only_build_type:
+            build_type = task.attributes.get('build-type')
+            if build_type not in only_build_type:
+                continue
+
+        job = {"primary-dependency": task}
+
+        if job_template:
+            job.update(copy.deepcopy(job_template))
+
+        yield job

taskcluster/fenix_taskgraph/pushapk.py

@@ -9,43 +9,72 @@ kind.
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.treeherder import inherit_treeherder_from_dep
+from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol
 
 
 transforms = TransformSequence()
 
 
-# TODO remove this transform once signing task are migrated to taskgraph
+_CHANNEL_PER_TASK_NAME = {
+    'nightly': 'nightly',
+    'nightly-legacy': 'production',
+    'production': 'production',
+}
+
+_GOOGLE_PLAY_TRACK_PER_TASK_NAME = {
+    'nightly': 'internal',
+    'nightly-legacy': 'nightly',
+}
+
+
 @transforms.add
-def fetch_old_decision_dependency(config, tasks):
+def build_name_and_attributes(config, tasks):
     for task in tasks:
-        for dep_task in config.kind_dependencies_tasks:
-            expected_signing_dep_label = task['dependencies']['signing']
-            if dep_task.label != expected_signing_dep_label:
-                continue
+        dep = task["primary-dependency"]
+        task["dependencies"] = {"signing": dep.label}
+        task.setdefault("attributes", {}).update(dep.attributes.copy())
+        task["name"] = _get_dependent_job_name_without_its_kind(dep)
 
-            task['primary-dependency'] = dep_task
-            yield task
+        yield task
+
+
+def _get_dependent_job_name_without_its_kind(dependent_job):
+    return dependent_job.label[len(dependent_job.kind) + 1:]
 
 
 @transforms.add
-def build_pushapk_task(config, tasks):
+def build_treeherder_definition(config, tasks):
+    for task in tasks:
+        dep = task["primary-dependency"]
+        task["treeherder"] = inherit_treeherder_from_dep(task, dep)
+        treeherder_group = dep.task["extra"]["treeherder"]["groupSymbol"]
+        treeherder_symbol = join_symbol(treeherder_group, 'gp')
+        task["treeherder"]["symbol"] = treeherder_symbol
+
+        yield task
+
+
+@transforms.add
+def build_worker_definition(config, tasks):
     for task in tasks:
         dep = task.pop("primary-dependency")
-        task.setdefault("attributes", {}).update(dep.attributes.copy())
-        if "run_on_tasks_for" in task["attributes"]:
-            task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
+        task_name = task["name"]
 
-        task["treeherder"] = inherit_treeherder_from_dep(task, dep)
-        task["worker"]["upstream-artifacts"] = [{
+        worker_definition = {}
+        worker_definition["upstream-artifacts"] = [{
             "taskId": {"task-reference": "<signing>"},
             "taskType": "signing",
             "paths": dep.attributes["apks"].values(),
         }]
-        task["worker"]["dep"] = config.params["level"] != "3"
-        if not task["worker"].get("certificate-alias"):
-            task["worker"]["certificate-alias"] = "{}-{}".format(
-                task["worker"]["product"], task["worker"]["channel"]
-            )
+        worker_definition["dep"] = config.params["level"] != "3"
+        worker_definition["channel"] = _CHANNEL_PER_TASK_NAME[task_name]
 
+        # Fenix production doesn't follow the rule {product}-{channel}
+        worker_definition["certificate-alias"] = 'fenix' if task_name == 'production' else \
+            "{}-{}".format(task["worker"]["product"], worker_definition["channel"])
+
+        if _GOOGLE_PLAY_TRACK_PER_TASK_NAME.get(task_name):
+            worker_definition["google-play-track"] = _GOOGLE_PLAY_TRACK_PER_TASK_NAME[task_name]
+
+        task["worker"].update(worker_definition)
         yield task

taskcluster/fenix_taskgraph/raptor.py

@@ -30,10 +30,6 @@ def add_variants(config, tasks):
         if build_type not in only_types:
             continue
 
-        # TODO remove this if statement once signing tasks are fully created in taskgraph
-        if not dep_task.label.startswith('sign: '):
-            continue
-
         for abi, apk_path in dep_task.attributes["apks"].items():
             if abi not in only_abis:
                 continue
@@ -52,6 +48,7 @@ def add_variants(config, tasks):
 def build_raptor_task(config, tasks):
     for task in tasks:
         signing = task.pop("primary-dependency")
+        task["dependencies"]["signing"] = signing.label
         build_type = task["attributes"]["build-type"]
         abi = task["attributes"]["abi"]
         apk = task["attributes"]["apk"]

taskcluster/fenix_taskgraph/routes.py

@@ -9,9 +9,9 @@ import time
 from taskgraph.transforms.task import index_builder
 
 SIGNING_ROUTE_TEMPLATES = [
-    "index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.revision.{head_rev}",
-    "index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.latest",
-    "index.project.{trust-domain}.{project}.v3.{variant}.latest",
+    "index.project.{trust-domain}.{project}.v2.{variant}.{build_date}.revision.{head_rev}",
+    "index.project.{trust-domain}.{project}.v2.{variant}.{build_date}.latest",
+    "index.project.{trust-domain}.{project}.v2.{variant}.latest",
 ]
 
 

taskcluster/fenix_taskgraph/signing.py

@@ -0,0 +1,64 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Apply some defaults and minor modifications to the jobs defined in the build
+kind.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+
+
+transforms = TransformSequence()
+
+# TODO remove this transform once build tasks are migrated to taskgraph
+@transforms.add
+def fetch_old_decision_dependency(config, tasks):
+    for task in tasks:
+        for dep_task in config.kind_dependencies_tasks:
+            expected_signing_dep_label = task['dependencies']['build']
+            if dep_task.label != expected_signing_dep_label:
+                continue
+
+            task['primary-dependency'] = dep_task
+            yield task
+
+
+@transforms.add
+def define_signing_flags(config, tasks):
+    for task in tasks:
+        dep = task["primary-dependency"]
+        # Current kind will be prepended later in the transform chain.
+        task.setdefault("attributes", {}).update(dep.attributes.copy())
+        task["attributes"]["signed"] = True
+        if "run_on_tasks_for" in task["attributes"]:
+            task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
+
+        for key in ("worker-type", "worker.signing-type"):
+            resolve_keyed_by(
+                task,
+                key,
+                item_name=task["name"],
+                variant=task["attributes"]["build-type"],
+                level=config.params["level"],
+            )
+        yield task
+
+
+@transforms.add
+def build_signing_task(config, tasks):
+    for task in tasks:
+        dep = task.pop("primary-dependency")
+        task["dependencies"] = {"build": dep.label}
+        task["worker"]["upstream-artifacts"] = [
+            {
+                "taskId": {"task-reference": "<build>"},
+                "taskType": "build",
+                "paths": dep.attributes["apks"].values(),
+                "formats": ["autograph_apk"],
+            }
+        ]
+        yield task