Refactor decision_task.py to match android-components'
parent
7269f47acc
commit
64edb1a35b
@ -0,0 +1,116 @@
|
|||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Decision task for nightly releases.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import taskcluster
|
||||||
|
|
||||||
|
from lib.tasks import TaskBuilder, schedule_task_graph
|
||||||
|
from lib.util import (
|
||||||
|
populate_chain_of_trust_task_graph,
|
||||||
|
populate_chain_of_trust_required_but_unused_files
|
||||||
|
)
|
||||||
|
|
||||||
|
REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY')
|
||||||
|
COMMIT = os.environ.get('MOBILE_HEAD_REV')
|
||||||
|
PR_TITLE = os.environ.get('GITHUB_PULL_TITLE', '')
|
||||||
|
|
||||||
|
# If we see this text inside a pull request title then we will not execute any tasks for this PR.
|
||||||
|
SKIP_TASKS_TRIGGER = '[ci skip]'
|
||||||
|
|
||||||
|
|
||||||
|
BUILDER = TaskBuilder(
|
||||||
|
task_id=os.environ.get('TASK_ID'),
|
||||||
|
repo_url=os.environ.get('MOBILE_HEAD_REPOSITORY'),
|
||||||
|
branch=os.environ.get('MOBILE_HEAD_BRANCH'),
|
||||||
|
commit=COMMIT,
|
||||||
|
owner="fenix-eng-notifications@mozilla.com",
|
||||||
|
source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT),
|
||||||
|
scheduler_id=os.environ.get('SCHEDULER_ID', 'taskcluster-github'),
|
||||||
|
tasks_priority=os.environ.get('TASKS_PRIORITY'),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def nightly(apks, track, commit, date_string):
|
||||||
|
is_staging = track == 'staging-nightly'
|
||||||
|
|
||||||
|
build_tasks = {}
|
||||||
|
signing_tasks = {}
|
||||||
|
push_tasks = {}
|
||||||
|
artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks]
|
||||||
|
|
||||||
|
build_task_id = taskcluster.slugId()
|
||||||
|
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(apks, is_staging)
|
||||||
|
|
||||||
|
signing_task_id = taskcluster.slugId()
|
||||||
|
signing_tasks[signing_task_id] = BUILDER.craft_signing_task(
|
||||||
|
build_task_id,
|
||||||
|
apks=artifacts,
|
||||||
|
date_string=date_string,
|
||||||
|
is_staging=is_staging,
|
||||||
|
)
|
||||||
|
|
||||||
|
push_task_id = taskcluster.slugId()
|
||||||
|
push_tasks[push_task_id] = BUILDER.craft_push_task(
|
||||||
|
signing_task_id,
|
||||||
|
apks=artifacts,
|
||||||
|
commit=commit,
|
||||||
|
is_staging=is_staging
|
||||||
|
)
|
||||||
|
|
||||||
|
return (build_tasks, signing_tasks, push_tasks)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description='Creates and submit a graph of tasks on Taskcluster.'
|
||||||
|
)
|
||||||
|
|
||||||
|
subparsers = parser.add_subparsers(dest='command')
|
||||||
|
|
||||||
|
subparsers.add_parser('pr-or-push')
|
||||||
|
release_parser = subparsers.add_parser('release')
|
||||||
|
|
||||||
|
release_parser.add_argument('--nightly', action="store_true", default=False)
|
||||||
|
release_parser.add_argument(
|
||||||
|
'--track', action="store", choices=['nightly', 'staging-nightly'], required=True
|
||||||
|
)
|
||||||
|
release_parser.add_argument(
|
||||||
|
'--commit', action="store_true", help="commit the google play transaction"
|
||||||
|
)
|
||||||
|
release_parser.add_argument(
|
||||||
|
'--apk', dest="apks", metavar="path", action="append",
|
||||||
|
help="Path to APKs to sign and upload", required=True
|
||||||
|
)
|
||||||
|
release_parser.add_argument(
|
||||||
|
'--output', metavar="path", action="store", help="Path to the build output", required=True
|
||||||
|
)
|
||||||
|
release_parser.add_argument('--date', action="store", help="ISO8601 timestamp for build")
|
||||||
|
|
||||||
|
result = parser.parse_args()
|
||||||
|
|
||||||
|
command = result.command
|
||||||
|
|
||||||
|
if command == 'pr-or-push':
|
||||||
|
# TODO
|
||||||
|
ordered_groups_of_tasks = {}
|
||||||
|
elif command == 'release':
|
||||||
|
apks = ["{}/{}".format(result.output, apk) for apk in result.apks]
|
||||||
|
# nightly(apks, result.track, result.commit, result.date)
|
||||||
|
ordered_groups_of_tasks = nightly(
|
||||||
|
apks, result.track, result.commit, result.date
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise Exception('Unsupported command "{}"'.format(command))
|
||||||
|
|
||||||
|
full_task_graph = schedule_task_graph(ordered_groups_of_tasks)
|
||||||
|
|
||||||
|
populate_chain_of_trust_task_graph(full_task_graph)
|
||||||
|
populate_chain_of_trust_required_but_unused_files()
|
@ -1,172 +0,0 @@
|
|||||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
||||||
|
|
||||||
"""
|
|
||||||
Decision task for nightly releases.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import print_function
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import arrow
|
|
||||||
import json
|
|
||||||
import lib.tasks
|
|
||||||
import os
|
|
||||||
import taskcluster
|
|
||||||
|
|
||||||
TASK_ID = os.environ.get('TASK_ID')
|
|
||||||
SCHEDULER_ID = os.environ.get('SCHEDULER_ID')
|
|
||||||
GITHUB_HTTP_REPOSITORY = os.environ.get('MOBILE_HEAD_REPOSITORY')
|
|
||||||
HEAD_REV = os.environ.get('MOBILE_HEAD_REV')
|
|
||||||
HEAD_BRANCH = os.environ.get('MOBILE_HEAD_BRANCH')
|
|
||||||
|
|
||||||
BUILDER = lib.tasks.TaskBuilder(
|
|
||||||
task_id=TASK_ID,
|
|
||||||
owner="fenix-eng-notifications@mozilla.com",
|
|
||||||
source='{}/raw/{}/.taskcluster.yml'.format(GITHUB_HTTP_REPOSITORY, HEAD_REV),
|
|
||||||
scheduler_id=SCHEDULER_ID,
|
|
||||||
build_worker_type=os.environ.get('BUILD_WORKER_TYPE'),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def generate_build_task(apks, is_staging):
|
|
||||||
artifacts = {'public/{}'.format(os.path.basename(apk)): {
|
|
||||||
"type": 'file',
|
|
||||||
"path": apk,
|
|
||||||
"expires": taskcluster.stringDate(taskcluster.fromNow('1 year')),
|
|
||||||
} for apk in apks}
|
|
||||||
|
|
||||||
checkout = (
|
|
||||||
"export TERM=dumb && git fetch {} {} --tags && "
|
|
||||||
"git config advice.detachedHead false && "
|
|
||||||
"git checkout {}".format(
|
|
||||||
GITHUB_HTTP_REPOSITORY, HEAD_BRANCH, HEAD_REV
|
|
||||||
)
|
|
||||||
)
|
|
||||||
sentry_secret = '{}project/mobile/fenix/sentry'.format('garbage/staging/' if is_staging else '')
|
|
||||||
leanplum_secret = '{}project/mobile/fenix/leanplum'.format('garbage/staging/' if is_staging else '')
|
|
||||||
|
|
||||||
return taskcluster.slugId(), BUILDER.build_task(
|
|
||||||
name="(Fenix) Build task",
|
|
||||||
description="Build Fenix from source code.",
|
|
||||||
command=(
|
|
||||||
checkout +
|
|
||||||
' && python automation/taskcluster/helper/get-secret.py'
|
|
||||||
' -s {} -k dsn -f .sentry_token'.format(sentry_secret) +
|
|
||||||
' && python automation/taskcluster/helper/get-secret.py'
|
|
||||||
' -s {} -k production -f .leanplum_token'.format(leanplum_secret) +
|
|
||||||
' && ./gradlew --no-daemon -PcrashReports=true clean test assembleRelease'),
|
|
||||||
features={
|
|
||||||
"chainOfTrust": True,
|
|
||||||
"taskclusterProxy": True
|
|
||||||
},
|
|
||||||
artifacts=artifacts,
|
|
||||||
scopes=[
|
|
||||||
"secrets:get:{}".format(sentry_secret),
|
|
||||||
"secrets:get:{}".format(leanplum_secret)
|
|
||||||
],
|
|
||||||
routes=["notify.email.fenix-eng-notifications@mozilla.com.on-failed"]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def generate_signing_task(build_task_id, apks, date, is_staging):
|
|
||||||
artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks]
|
|
||||||
|
|
||||||
signing_format = 'autograph_apk'
|
|
||||||
index_release = 'staging-signed-nightly' if is_staging else 'signed-nightly'
|
|
||||||
routes = [
|
|
||||||
"index.project.mobile.fenix.{}.nightly.{}.{}.{}.latest".format(index_release, date.year, date.month, date.day),
|
|
||||||
"index.project.mobile.fenix.{}.nightly.{}.{}.{}.revision.{}".format(index_release, date.year, date.month, date.day, HEAD_REV),
|
|
||||||
"index.project.mobile.fenix.{}.nightly.latest".format(index_release),
|
|
||||||
]
|
|
||||||
scopes = [
|
|
||||||
"project:mobile:fenix:releng:signing:format:{}".format(signing_format),
|
|
||||||
"project:mobile:fenix:releng:signing:cert:{}".format('dep-signing' if is_staging else 'release-signing')
|
|
||||||
]
|
|
||||||
|
|
||||||
return taskcluster.slugId(), BUILDER.craft_signing_task(
|
|
||||||
build_task_id,
|
|
||||||
name="(Fenix) Signing task",
|
|
||||||
description="Sign release builds of Fenix",
|
|
||||||
apks=artifacts,
|
|
||||||
scopes=scopes,
|
|
||||||
routes=routes,
|
|
||||||
signing_format=signing_format,
|
|
||||||
is_staging=is_staging
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def generate_push_task(signing_task_id, apks, commit, is_staging):
|
|
||||||
artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks]
|
|
||||||
|
|
||||||
return taskcluster.slugId(), BUILDER.craft_push_task(
|
|
||||||
signing_task_id,
|
|
||||||
name="(Fenix) Push task",
|
|
||||||
description="Upload signed release builds of Fenix to Google Play",
|
|
||||||
apks=artifacts,
|
|
||||||
scopes=[
|
|
||||||
"project:mobile:fenix:releng:googleplay:product:fenix{}".format(':dep' if is_staging else '')
|
|
||||||
],
|
|
||||||
commit=commit,
|
|
||||||
is_staging=is_staging
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def populate_chain_of_trust_required_but_unused_files():
|
|
||||||
# These files are needed to keep chainOfTrust happy. However, they have no need for Fenix
|
|
||||||
# at the moment. For more details, see: https://github.com/mozilla-releng/scriptworker/pull/209/files#r184180585
|
|
||||||
|
|
||||||
for file_name in ('actions.json', 'parameters.yml'):
|
|
||||||
with open(file_name, 'w') as f:
|
|
||||||
json.dump({}, f)
|
|
||||||
|
|
||||||
|
|
||||||
def nightly(apks, track, commit, date_string):
|
|
||||||
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
|
|
||||||
date = arrow.get(date_string)
|
|
||||||
is_staging = track == 'staging-nightly'
|
|
||||||
|
|
||||||
task_graph = {}
|
|
||||||
|
|
||||||
build_task_id, build_task = generate_build_task(apks, is_staging)
|
|
||||||
lib.tasks.schedule_task(queue, build_task_id, build_task)
|
|
||||||
|
|
||||||
task_graph[build_task_id] = {}
|
|
||||||
task_graph[build_task_id]['task'] = queue.task(build_task_id)
|
|
||||||
|
|
||||||
sign_task_id, sign_task = generate_signing_task(build_task_id, apks, date, is_staging)
|
|
||||||
lib.tasks.schedule_task(queue, sign_task_id, sign_task)
|
|
||||||
|
|
||||||
task_graph[sign_task_id] = {}
|
|
||||||
task_graph[sign_task_id]['task'] = queue.task(sign_task_id)
|
|
||||||
|
|
||||||
push_task_id, push_task = generate_push_task(sign_task_id, apks, commit, is_staging)
|
|
||||||
lib.tasks.schedule_task(queue, push_task_id, push_task)
|
|
||||||
|
|
||||||
task_graph[push_task_id] = {}
|
|
||||||
task_graph[push_task_id]['task'] = queue.task(push_task_id)
|
|
||||||
|
|
||||||
print(json.dumps(task_graph, indent=4, separators=(',', ': ')))
|
|
||||||
|
|
||||||
with open('task-graph.json', 'w') as f:
|
|
||||||
json.dump(task_graph, f)
|
|
||||||
|
|
||||||
populate_chain_of_trust_required_but_unused_files()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description='Create a release pipeline (build, sign, publish) on taskcluster.')
|
|
||||||
|
|
||||||
parser.add_argument('--track', dest="track", action="store", choices=['nightly', 'staging-nightly'], required=True)
|
|
||||||
parser.add_argument('--commit', action="store_true", help="commit the google play transaction")
|
|
||||||
parser.add_argument('--apk', dest="apks", metavar="path", action="append", help="Path to APKs to sign and upload",
|
|
||||||
required=True)
|
|
||||||
parser.add_argument('--output', metavar="path", action="store", help="Path to the build output",
|
|
||||||
required=True)
|
|
||||||
parser.add_argument('--date', action="store", help="ISO8601 timestamp for build")
|
|
||||||
|
|
||||||
result = parser.parse_args()
|
|
||||||
apks = ["{}/{}".format(result.output, apk) for apk in result.apks]
|
|
||||||
nightly(apks, result.track, result.commit, result.date)
|
|
@ -0,0 +1,20 @@
|
|||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
|
def populate_chain_of_trust_required_but_unused_files():
|
||||||
|
# Thoses files are needed to keep chainOfTrust happy. However, they have no
|
||||||
|
# need for android-components, at the moment. For more details, see:
|
||||||
|
# https://github.com/mozilla-releng/scriptworker/pull/209/files#r184180585
|
||||||
|
|
||||||
|
for file_names in ('actions.json', 'parameters.yml'):
|
||||||
|
with open(file_names, 'w') as f:
|
||||||
|
json.dump({}, f) # Yaml is a super-set of JSON.
|
||||||
|
|
||||||
|
|
||||||
|
def populate_chain_of_trust_task_graph(full_task_graph):
|
||||||
|
# taskgraph must follow the format:
|
||||||
|
# {
|
||||||
|
# task_id: full_task_definition
|
||||||
|
# }
|
||||||
|
with open('task-graph.json', 'w') as f:
|
||||||
|
json.dump(full_task_graph, f)
|
Loading…
Reference in New Issue