[fenix] Adds raptor hook ability to schedule_nightly_graph (https://github.com/mozilla-mobile/fenix/pull/2341)

.taskcluster.yml

@@ -123,99 +123,128 @@ tasks:
             owner: ${user}@users.noreply.github.com
             source: ${repository}/raw/${head_rev}/.taskcluster.yml
       in:
-        - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
+        $flatten:
-          then:
+          - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
-            $let:
+            then:
-              pull_request_title: ${event.pull_request.title}
+              $let:
-              pull_request_number: ${event.pull_request.number}
+                pull_request_title: ${event.pull_request.title}
-              pull_request_url: ${event.pull_request.html_url}
+                pull_request_number: ${event.pull_request.number}
-            in:
+                pull_request_url: ${event.pull_request.html_url}
+              in:
+                $mergeDeep:
+                  - {$eval: 'default_task_definition'}
+                  - scopes:
+                      - ${assume_scope_prefix}:pull-request
+                    payload:
+                      command:
+                        - >-
+                          git fetch ${repository} ${head_branch}
+                          && git config advice.detachedHead false
+                          && git checkout FETCH_HEAD
+                          && python automation/taskcluster/decision_task.py pull-request
+                      env:
+                        GITHUB_PULL_TITLE: ${pull_request_title}
+                    extra:
+                      treeherder:
+                        symbol: D-PR
+                    metadata:
+                      name: 'Fenix - Decision task (Pull Request #${pull_request_number})'
+                      description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})'
+          - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
+            then:
               $mergeDeep:
                 - {$eval: 'default_task_definition'}
                 - scopes:
-                    - ${assume_scope_prefix}:pull-request
+                    - ${assume_scope_prefix}:branch:${short_head_branch}
                   payload:
                     command:
                       - >-
                         git fetch ${repository} ${head_branch}
                         && git config advice.detachedHead false
                         && git checkout FETCH_HEAD
-                        && python automation/taskcluster/decision_task.py pull-request
+                        && python automation/taskcluster/decision_task.py push
-                    env:
-                      GITHUB_PULL_TITLE: ${pull_request_title}
                   extra:
                     treeherder:
-                      symbol: D-PR
+                      symbol: D
                   metadata:
-                    name: 'Fenix - Decision task (Pull Request #${pull_request_number})'
+                    name: Fenix VCS-Push Decision task
-                    description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})'
+                    description: Schedules the build and test tasks for Fenix.
-        - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
+          - $if: 'tasks_for == "github-release" && event["action"] == "published"'
-          then:
+            then:
-            $mergeDeep:
+              $mergeDeep:
-              - {$eval: 'default_task_definition'}
+                - {$eval: 'default_task_definition'}
-              - scopes:
+                - scopes:
-                  - ${assume_scope_prefix}:branch:${short_head_branch}
+                  - ${assume_scope_prefix}:release
-                payload:
+                  payload:
-                  command:
-                    - >-
-                      git fetch ${repository} ${head_branch}
-                      && git config advice.detachedHead false
-                      && git checkout FETCH_HEAD
-                      && python automation/taskcluster/decision_task.py push
-                extra:
-                  treeherder:
-                    symbol: D
-                metadata:
-                  name: Fenix - Decision task
-                  description: Schedules the build and test tasks for Fenix.
-        - $if: 'tasks_for == "github-release" && event["action"] == "published"'
-          then:
-            $mergeDeep:
-              - {$eval: 'default_task_definition'}
-              - scopes:
-                - ${assume_scope_prefix}:release
-                payload:
-                  command:
-                    - >-
-                      git fetch ${repository} refs/tags/${head_rev}
-                      && git config advice.detachedHead false
-                      && git checkout FETCH_HEAD
-                      && python automation/taskcluster/decision_task.py beta ${event.release.tag_name}
-                extra:
-                  treeherder:
-                    symbol: beta-D
-                metadata:
-                  name: Fenix Beta Decision Task
-                  description: Building and releasing Fenix to the beta channel - triggered by release ${event.release.tag_name}
-        - $if: 'tasks_for == "cron"'
-          then:
-            $mergeDeep:
-              - {$eval: 'default_task_definition'}
-              - scopes:
-                  - $if: 'trust_level == 3'
-                    then: assume:hook-id:project-mobile/fenix-nightly
-                    else: assume:hook-id:project-mobile/fenix-nightly-staging
-                routes:
-                  $if: 'trust_level == 3'
-                  then:
-                    - notify.email.fenix-eng-notifications@mozilla.com.on-failed
-                payload:
-                  $let:
-                    staging_flag:
-                      $if: 'trust_level == 3'
-                      then: ''
-                      else: '--staging'
-                  in:
                     command:
                       - >-
-                        git fetch ${repository} ${head_branch}
+                        git fetch ${repository} refs/tags/${head_rev}
                         && git config advice.detachedHead false
                         && git checkout FETCH_HEAD
-                        && python automation/taskcluster/decision_task.py nightly ${staging_flag}
+                        && python automation/taskcluster/decision_task.py beta ${event.release.tag_name}
-                extra:
+                  extra:
-                  cron: {$json: {$eval: 'cron'}}
+                    treeherder:
-                  treeherder:
+                      symbol: beta-D
-                    symbol: nightly-D
+                  metadata:
-                metadata:
+                    name: Fenix Beta Decision Task
-                  name: Fenix Nightly Decision Task
+                    description: Building and releasing Fenix to the beta channel - triggered by release ${event.release.tag_name}
-                  description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
+          - $if: 'tasks_for == "cron"'
+            then:
+              $let:
+                staging_flag:
+                  $if: 'trust_level == 3'
+                  then: ''
+                  else: '--staging'
+              in:
+              - $if: 'cron.name == "nightly"'
+                then:
+                  $mergeDeep:
+                    - {$eval: 'default_task_definition'}
+                    - scopes:
+                        - $if: 'trust_level == 3'
+                          then: assume:hook-id:project-mobile/fenix-nightly
+                          else: assume:hook-id:project-mobile/fenix-nightly-staging
+                      routes:
+                        $if: 'trust_level == 3'
+                        then:
+                          - notify.email.fenix-eng-notifications@mozilla.com.on-failed
+                      payload:
+                        command:
+                          - >-
+                            git fetch ${repository} ${head_branch}
+                            && git config advice.detachedHead false
+                            && git checkout FETCH_HEAD
+                            && python automation/taskcluster/decision_task.py nightly ${staging_flag}
+                      extra:
+                        cron: {$json: {$eval: 'cron'}}
+                        treeherder:
+                          symbol: nightly-D
+                      metadata:
+                        name: Fenix Nightly Decision Task
+                        description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
+              - $if: 'cron.name == "raptor"'
+                then:
+                  $mergeDeep:
+                    - {$eval: 'default_task_definition'}
+                    - scopes:
+                        - $if: 'trust_level == 3'
+                          then: assume:hook-id:project-mobile/fenix-raptor
+                          else: assume:hook-id:project-mobile/fenix-raptor-staging
+                      routes:
+                        $if: 'trust_level == 3'
+                        then:
+                          - notify.email.fenix-eng-notifications@mozilla.com.on-failed
+                      payload:
+                        command:
+                          - >-
+                            git fetch ${repository} ${head_branch}
+                            && git config advice.detachedHead false
+                            && git checkout FETCH_HEAD
+                            && python automation/taskcluster/decision_task.py raptor ${staging_flag}
+                      extra:
+                        cron: {$json: {$eval: 'cron'}}
+                        treeherder:
+                          symbol: raptor-D
+                      metadata:
+                        name: Fenix Raptor Decision Task
+                        description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})

automation/taskcluster/decision_task.py

@@ -57,10 +57,6 @@ def pr_or_push(is_push):
         print("Exit")
         return {}
 
-    geckoview_nightly_version = get_geckoview_versions()['nightly']
-    mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version)
-    gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
-
     build_tasks = {}
     signing_tasks = {}
     other_tasks = {}
@@ -70,23 +66,6 @@ def pr_or_push(is_push):
         build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
         build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant)
 
-    if is_push and SHORT_HEAD_BRANCH == 'master':
-        other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task()
-
-        for variant in [Variant.from_values(abi, False, 'raptor') for abi in ('aarch64', 'arm')]:
-            assemble_task_id = taskcluster.slugId()
-            build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
-            signing_task_id = taskcluster.slugId()
-            signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant)
-
-            ALL_RAPTOR_CRAFT_FUNCTIONS = [
-                BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
-                for i in range(1, 15)
-            ]
-            for craft_function in ALL_RAPTOR_CRAFT_FUNCTIONS:
-                args = (signing_task_id, mozharness_task_id, variant, gecko_revision)
-                other_tasks[taskcluster.slugId()] = craft_function(*args)
-
     for craft_function in (
         BUILDER.craft_detekt_task,
         BUILDER.craft_ktlint_task,
@@ -95,6 +74,35 @@ def pr_or_push(is_push):
     ):
         other_tasks[taskcluster.slugId()] = craft_function()
 
+    if is_push and SHORT_HEAD_BRANCH == 'master':
+        other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task()
+
+    return (build_tasks, signing_tasks, other_tasks)
+
+
+def raptor(is_staging):
+    build_tasks = {}
+    signing_tasks = {}
+    other_tasks = {}
+
+    geckoview_nightly_version = get_geckoview_versions()['nightly']
+    mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version)
+    gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
+
+    for variant in [Variant.from_values(abi, False, 'raptor') for abi in ('aarch64', 'arm')]:
+        assemble_task_id = taskcluster.slugId()
+        build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
+        signing_task_id = taskcluster.slugId()
+        signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
+
+        all_raptor_craft_functions = [
+            BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
+            for i in range(1, 15)
+        ]
+        for craft_function in all_raptor_craft_functions:
+            args = (signing_task_id, mozharness_task_id, variant, gecko_revision)
+            other_tasks[taskcluster.slugId()] = craft_function(*args)
+
     return (build_tasks, signing_tasks, other_tasks)
 
 
@@ -138,6 +146,9 @@ if __name__ == "__main__":
     subparsers.add_parser('pull-request')
     subparsers.add_parser('push')
 
+    raptor_parser = subparsers.add_parser('raptor')
+    raptor_parser.add_argument('--staging', action='store_true')
+
     nightly_parser = subparsers.add_parser('nightly')
     nightly_parser.add_argument('--staging', action='store_true')
 
@@ -152,6 +163,8 @@ if __name__ == "__main__":
         ordered_groups_of_tasks = pr_or_push(False)
     elif command == 'push':
         ordered_groups_of_tasks = pr_or_push(True)
+    elif command == 'raptor':
+        ordered_groups_of_tasks = raptor(result.staging)
     elif command == 'nightly':
         formatted_date = datetime.datetime.now().strftime('%y%V')
         ordered_groups_of_tasks = release('nightly', result.staging, '1.0.{}'.format(formatted_date))

automation/taskcluster/lib/tasks.py

@@ -375,24 +375,18 @@ class TaskBuilder(object):
         }
 
     def craft_raptor_signing_task(
-        self, assemble_task_id, variant
+        self, assemble_task_id, variant, is_staging,
     ):
-        routes = []
+        staging_prefix = '.staging' if is_staging else ''
-        if self.repo_url == _OFFICIAL_REPO_URL:
+        routes = [
-            routes = [
+            "index.project.mobile.fenix.v2{}.raptor.{}.{}.{}.latest.{}".format(
-                'index.project.mobile.fenix.v2.branch.master.revision.{}.raptor.{}'.format(
+                staging_prefix, self.date.year, self.date.month, self.date.day, variant.abi
-                    self.commit, variant.abi
+            ),
-                ),
+            "index.project.mobile.fenix.v2{}.raptor.{}.{}.{}.revision.{}.{}".format(
-                'index.project.mobile.fenix.v2.branch.master.latest.raptor.{}'.format(
+                staging_prefix, self.date.year, self.date.month, self.date.day, self.commit, variant.abi
-                    variant.abi
+            ),
-                ),
+            "index.project.mobile.fenix.v2{}.raptor.latest.{}".format(staging_prefix, variant.abi),
-                'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.revision.{}.raptor.{}'.format(
+        ]
-                    self.date.year, self.date.month, self.date.day, self.commit, variant.abi
-                ),
-                'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.latest.raptor.{}'.format(
-                    self.date.year, self.date.month, self.date.day, variant.abi
-                ),
-            ]
 
         return self._craft_signing_task(
             name='sign: {}'.format(variant.raw),
@@ -413,19 +407,19 @@ class TaskBuilder(object):
         )
 
     def craft_release_signing_task(
-        self, build_task_id, apk_paths, track, is_staging=False,
+        self, build_task_id, apk_paths, track, is_staging,
     ):
         capitalized_track = upper_case_first_letter(track)
-        index_release = 'staging.{}'.format(track) if is_staging else track
+        staging_prefix = '.staging' if is_staging else ''
 
         routes = [
-            "index.project.mobile.fenix.v2.{}.{}.{}.{}.latest".format(
+            "index.project.mobile.fenix.v2{}.{}.{}.{}.{}.latest".format(
-                index_release, self.date.year, self.date.month, self.date.day
+                staging_prefix, track, self.date.year, self.date.month, self.date.day
             ),
-            "index.project.mobile.fenix.v2.{}.{}.{}.{}.revision.{}".format(
+            "index.project.mobile.fenix.v2{}.{}.{}.{}.{}.revision.{}".format(
-                index_release, self.date.year, self.date.month, self.date.day, self.commit
+                staging_prefix, track, self.date.year, self.date.month, self.date.day, self.commit
             ),
-            "index.project.mobile.fenix.v2.{}.latest".format(index_release),
+            "index.project.mobile.fenix.v2{}.{}.latest".format(staging_prefix, track),
         ]
 
         return self._craft_signing_task(

automation/taskcluster/schedule_nightly_graph.py

@@ -1,7 +1,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
+import argparse
 import datetime
 import jsone
 import os
@@ -50,7 +50,8 @@ def make_decision_task(params):
     context = {
         'tasks_for': 'cron',
         'cron': {
-            'task_id': params['cron_task_id']
+            'task_id': params['cron_task_id'],
+            'name': params['name'],
         },
         'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z',
         'as_slugid': as_slugid,
@@ -79,6 +80,12 @@ def make_decision_task(params):
 
 
 def schedule():
+    parser = argparse.ArgumentParser(
+        description='Creates and submit a graph of tasks on Taskcluster.'
+    )
+
+    parser.add_argument('name', choices=['nightly', 'raptor'])
+    result = parser.parse_args()
     queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
 
     html_url, branch, head_rev = calculate_git_references(ROOT)
@@ -86,7 +93,8 @@ def schedule():
         'html_url': html_url,
         'head_rev': head_rev,
         'branch': branch,
-        'cron_task_id': os.environ.get('CRON_TASK_ID', '<cron_task_id>')
+        'cron_task_id': os.environ.get('CRON_TASK_ID', '<cron_task_id>'),
+        'name': result.name,
     }
     decision_task_id, decision_task = make_decision_task(params)
     schedule_task(queue, decision_task_id, decision_task)