[fenix] Adds raptor hook ability to schedule_nightly_graph (https://github.com/mozilla-mobile/fenix/pull/2341)

.taskcluster.yml

@@ -123,6 +123,7 @@ tasks:
             owner: ${user}@users.noreply.github.com
             source: ${repository}/raw/${head_rev}/.taskcluster.yml
       in:
+        $flatten:
           - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
             then:
               $let:
@@ -166,7 +167,7 @@ tasks:
                     treeherder:
                       symbol: D
                   metadata:
-                  name: Fenix - Decision task
+                    name: Fenix VCS-Push Decision task
                     description: Schedules the build and test tasks for Fenix.
           - $if: 'tasks_for == "github-release" && event["action"] == "published"'
             then:
@@ -188,6 +189,14 @@ tasks:
                     name: Fenix Beta Decision Task
                     description: Building and releasing Fenix to the beta channel - triggered by release ${event.release.tag_name}
           - $if: 'tasks_for == "cron"'
+            then:
+              $let:
+                staging_flag:
+                  $if: 'trust_level == 3'
+                  then: ''
+                  else: '--staging'
+              in:
+              - $if: 'cron.name == "nightly"'
                 then:
                   $mergeDeep:
                     - {$eval: 'default_task_definition'}
@@ -200,12 +209,6 @@ tasks:
                         then:
                           - notify.email.fenix-eng-notifications@mozilla.com.on-failed
                       payload:
-                  $let:
-                    staging_flag:
-                      $if: 'trust_level == 3'
-                      then: ''
-                      else: '--staging'
-                  in:
                         command:
                           - >-
                             git fetch ${repository} ${head_branch}
@@ -219,3 +222,29 @@ tasks:
                       metadata:
                         name: Fenix Nightly Decision Task
                         description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
+              - $if: 'cron.name == "raptor"'
+                then:
+                  $mergeDeep:
+                    - {$eval: 'default_task_definition'}
+                    - scopes:
+                        - $if: 'trust_level == 3'
+                          then: assume:hook-id:project-mobile/fenix-raptor
+                          else: assume:hook-id:project-mobile/fenix-raptor-staging
+                      routes:
+                        $if: 'trust_level == 3'
+                        then:
+                          - notify.email.fenix-eng-notifications@mozilla.com.on-failed
+                      payload:
+                        command:
+                          - >-
+                            git fetch ${repository} ${head_branch}
+                            && git config advice.detachedHead false
+                            && git checkout FETCH_HEAD
+                            && python automation/taskcluster/decision_task.py raptor ${staging_flag}
+                      extra:
+                        cron: {$json: {$eval: 'cron'}}
+                        treeherder:
+                          symbol: raptor-D
+                      metadata:
+                        name: Fenix Raptor Decision Task
+                        description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})

automation/taskcluster/decision_task.py

@@ -57,10 +57,6 @@ def pr_or_push(is_push):
         print("Exit")
         return {}
 
-    geckoview_nightly_version = get_geckoview_versions()['nightly']
-    mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version)
-    gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
-
     build_tasks = {}
     signing_tasks = {}
     other_tasks = {}
@@ -70,23 +66,6 @@ def pr_or_push(is_push):
         build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
         build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant)
 
-    if is_push and SHORT_HEAD_BRANCH == 'master':
-        other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task()
-
-        for variant in [Variant.from_values(abi, False, 'raptor') for abi in ('aarch64', 'arm')]:
-            assemble_task_id = taskcluster.slugId()
-            build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
-            signing_task_id = taskcluster.slugId()
-            signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant)
-
-            ALL_RAPTOR_CRAFT_FUNCTIONS = [
-                BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
-                for i in range(1, 15)
-            ]
-            for craft_function in ALL_RAPTOR_CRAFT_FUNCTIONS:
-                args = (signing_task_id, mozharness_task_id, variant, gecko_revision)
-                other_tasks[taskcluster.slugId()] = craft_function(*args)
-
     for craft_function in (
         BUILDER.craft_detekt_task,
         BUILDER.craft_ktlint_task,
@@ -95,6 +74,35 @@ def pr_or_push(is_push):
     ):
         other_tasks[taskcluster.slugId()] = craft_function()
 
+    if is_push and SHORT_HEAD_BRANCH == 'master':
+        other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task()
+
+    return (build_tasks, signing_tasks, other_tasks)
+
+
+def raptor(is_staging):
+    build_tasks = {}
+    signing_tasks = {}
+    other_tasks = {}
+
+    geckoview_nightly_version = get_geckoview_versions()['nightly']
+    mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version)
+    gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
+
+    for variant in [Variant.from_values(abi, False, 'raptor') for abi in ('aarch64', 'arm')]:
+        assemble_task_id = taskcluster.slugId()
+        build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
+        signing_task_id = taskcluster.slugId()
+        signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
+
+        all_raptor_craft_functions = [
+            BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
+            for i in range(1, 15)
+        ]
+        for craft_function in all_raptor_craft_functions:
+            args = (signing_task_id, mozharness_task_id, variant, gecko_revision)
+            other_tasks[taskcluster.slugId()] = craft_function(*args)
+
     return (build_tasks, signing_tasks, other_tasks)
 
 
@@ -138,6 +146,9 @@ if __name__ == "__main__":
     subparsers.add_parser('pull-request')
     subparsers.add_parser('push')
 
+    raptor_parser = subparsers.add_parser('raptor')
+    raptor_parser.add_argument('--staging', action='store_true')
+
     nightly_parser = subparsers.add_parser('nightly')
     nightly_parser.add_argument('--staging', action='store_true')
 
@@ -152,6 +163,8 @@ if __name__ == "__main__":
         ordered_groups_of_tasks = pr_or_push(False)
     elif command == 'push':
         ordered_groups_of_tasks = pr_or_push(True)
+    elif command == 'raptor':
+        ordered_groups_of_tasks = raptor(result.staging)
     elif command == 'nightly':
         formatted_date = datetime.datetime.now().strftime('%y%V')
         ordered_groups_of_tasks = release('nightly', result.staging, '1.0.{}'.format(formatted_date))

automation/taskcluster/lib/tasks.py

@@ -375,23 +375,17 @@ class TaskBuilder(object):
         }
 
     def craft_raptor_signing_task(
-        self, assemble_task_id, variant
+        self, assemble_task_id, variant, is_staging,
     ):
-        routes = []
-        if self.repo_url == _OFFICIAL_REPO_URL:
+        staging_prefix = '.staging' if is_staging else ''
         routes = [
-                'index.project.mobile.fenix.v2.branch.master.revision.{}.raptor.{}'.format(
-                    self.commit, variant.abi
+            "index.project.mobile.fenix.v2{}.raptor.{}.{}.{}.latest.{}".format(
+                staging_prefix, self.date.year, self.date.month, self.date.day, variant.abi
             ),
-                'index.project.mobile.fenix.v2.branch.master.latest.raptor.{}'.format(
-                    variant.abi
-                ),
-                'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.revision.{}.raptor.{}'.format(
-                    self.date.year, self.date.month, self.date.day, self.commit, variant.abi
-                ),
-                'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.latest.raptor.{}'.format(
-                    self.date.year, self.date.month, self.date.day, variant.abi
+            "index.project.mobile.fenix.v2{}.raptor.{}.{}.{}.revision.{}.{}".format(
+                staging_prefix, self.date.year, self.date.month, self.date.day, self.commit, variant.abi
             ),
+            "index.project.mobile.fenix.v2{}.raptor.latest.{}".format(staging_prefix, variant.abi),
         ]
 
         return self._craft_signing_task(
@@ -413,19 +407,19 @@ class TaskBuilder(object):
         )
 
     def craft_release_signing_task(
-        self, build_task_id, apk_paths, track, is_staging=False,
+        self, build_task_id, apk_paths, track, is_staging,
     ):
         capitalized_track = upper_case_first_letter(track)
-        index_release = 'staging.{}'.format(track) if is_staging else track
+        staging_prefix = '.staging' if is_staging else ''
 
         routes = [
-            "index.project.mobile.fenix.v2.{}.{}.{}.{}.latest".format(
-                index_release, self.date.year, self.date.month, self.date.day
+            "index.project.mobile.fenix.v2{}.{}.{}.{}.{}.latest".format(
+                staging_prefix, track, self.date.year, self.date.month, self.date.day
             ),
-            "index.project.mobile.fenix.v2.{}.{}.{}.{}.revision.{}".format(
-                index_release, self.date.year, self.date.month, self.date.day, self.commit
+            "index.project.mobile.fenix.v2{}.{}.{}.{}.{}.revision.{}".format(
+                staging_prefix, track, self.date.year, self.date.month, self.date.day, self.commit
             ),
-            "index.project.mobile.fenix.v2.{}.latest".format(index_release),
+            "index.project.mobile.fenix.v2{}.{}.latest".format(staging_prefix, track),
         ]
 
         return self._craft_signing_task(

automation/taskcluster/schedule_nightly_graph.py

@@ -1,7 +1,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
+import argparse
 import datetime
 import jsone
 import os
@@ -50,7 +50,8 @@ def make_decision_task(params):
     context = {
         'tasks_for': 'cron',
         'cron': {
-            'task_id': params['cron_task_id']
+            'task_id': params['cron_task_id'],
+            'name': params['name'],
         },
         'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z',
         'as_slugid': as_slugid,
@@ -79,6 +80,12 @@ def make_decision_task(params):
 
 
 def schedule():
+    parser = argparse.ArgumentParser(
+        description='Creates and submit a graph of tasks on Taskcluster.'
+    )
+
+    parser.add_argument('name', choices=['nightly', 'raptor'])
+    result = parser.parse_args()
     queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
 
     html_url, branch, head_rev = calculate_git_references(ROOT)
@@ -86,7 +93,8 @@ def schedule():
         'html_url': html_url,
         'head_rev': head_rev,
         'branch': branch,
-        'cron_task_id': os.environ.get('CRON_TASK_ID', '<cron_task_id>')
+        'cron_task_id': os.environ.get('CRON_TASK_ID', '<cron_task_id>'),
+        'name': result.name,
     }
     decision_task_id, decision_task = make_decision_task(params)
     schedule_task(queue, decision_task_id, decision_task)